From 403b5671088a7e1998e749af2578a4be72d3be94 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sun, 14 Jun 2026 00:20:12 -0400 Subject: [PATCH 01/57] feat(connectors): scaffold opensearch_source connector skeleton Mirror elasticsearch_source architecture using the opensearch crate v2.4.0 (rustls-tls) for OpenSearch wire-protocol compatibility. Register as a new workspace member. --- Cargo.lock | 39 ++ Cargo.toml | 2 + .../sources/opensearch_source/Cargo.toml | 51 ++ .../sources/opensearch_source/src/lib.rs | 573 ++++++++++++++++++ .../opensearch_source/src/state_manager.rs | 388 ++++++++++++ 5 files changed, 1053 insertions(+) create mode 100644 core/connectors/sources/opensearch_source/Cargo.toml create mode 100644 core/connectors/sources/opensearch_source/src/lib.rs create mode 100644 core/connectors/sources/opensearch_source/src/state_manager.rs diff --git a/Cargo.lock b/Cargo.lock index d6adc0beb8..ea7b66fdfd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6932,6 +6932,25 @@ dependencies = [ "tracing", ] +[[package]] +name = "iggy_connector_opensearch_source" +version = "0.4.1-edge.1" +dependencies = [ + "async-trait", + "dashmap", + "humantime", + "iggy_common", + "iggy_connector_sdk", + "once_cell", + "opensearch", + "secrecy", + "serde", + "serde_json", + "simd-json", + "tokio", + "tracing", +] + [[package]] name = "iggy_connector_postgres_sink" version = "0.4.1-edge.1" @@ -8951,6 +8970,26 @@ dependencies = [ "uuid", ] +[[package]] +name = "opensearch" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af6815a23449a0860c8fe049a828c3589d3ad56d3b5875d0d1f340d1291871e" +dependencies = [ + "base64", + "bytes", + "dyn-clone", + "lazy_static", + "percent-encoding", + "reqwest 0.13.4", + "rustc_version", + "serde", + "serde_json", + "serde_with", + "url", + "void", +] + [[package]] name = "openssl-probe" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index 0f9ffd242a..41076fe22f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,6 +45,7 @@ members = [ "core/connectors/sinks/stdout_sink", "core/connectors/sources/elasticsearch_source", "core/connectors/sources/influxdb_source", + "core/connectors/sources/opensearch_source", "core/connectors/sources/postgres_source", "core/connectors/sources/random_source", "core/consensus", @@ -206,6 +207,7 @@ nonzero_lit = "0.1.2" notify = "8.2.0" octocrab = "0.51.0" once_cell = "1.21.4" +opensearch = { version = "2.4.0", features = ["rustls-tls"], default-features = false } opentelemetry = { version = "0.32.0", features = ["trace", "logs"] } opentelemetry-appender-tracing = { version = "0.32.0", features = ["log"] } opentelemetry-otlp = { version = "0.32.0", features = [ diff --git a/core/connectors/sources/opensearch_source/Cargo.toml b/core/connectors/sources/opensearch_source/Cargo.toml new file mode 100644 index 0000000000..52eec99e7b --- /dev/null +++ b/core/connectors/sources/opensearch_source/Cargo.toml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "iggy_connector_opensearch_source" +version = "0.4.1-edge.1" +description = "Iggy OpenSearch source connector" +edition = "2024" +license = "Apache-2.0" +keywords = ["iggy", "messaging", "streaming", "opensearch"] +categories = ["command-line-utilities", "database", "network-programming"] +homepage = "https://iggy.apache.org" +documentation = "https://iggy.apache.org/docs" +repository = "https://github.com/apache/iggy" +readme = "../../README.md" +publish = false + +[package.metadata.cargo-machete] +ignored = ["dashmap", "once_cell", "futures", "simd-json"] + +[lib] +crate-type = ["cdylib", "lib"] + +[dependencies] +async-trait = { workspace = true } +dashmap = { workspace = true } +opensearch = { workspace = true } +humantime = { workspace = true } +iggy_common = { workspace = true } +iggy_connector_sdk = { workspace = true } +once_cell = { workspace = true } +secrecy = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +simd-json = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } diff --git a/core/connectors/sources/opensearch_source/src/lib.rs b/core/connectors/sources/opensearch_source/src/lib.rs new file mode 100644 index 0000000000..ddb5c439d3 --- /dev/null +++ b/core/connectors/sources/opensearch_source/src/lib.rs @@ -0,0 +1,573 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +use async_trait::async_trait; +use iggy_common::{DateTime, Utc}; +use iggy_connector_sdk::{ + ConnectorState, Error, ProducedMessage, ProducedMessages, Schema, Source, source_connector, +}; +use opensearch::{ + OpenSearch, SearchParts, + auth::Credentials, + http::{Url, transport::TransportBuilder}, +}; +use secrecy::{ExposeSecret, SecretString}; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; +use std::str::FromStr; +use std::sync::Arc; +use std::time::Duration; +use tokio::{sync::Mutex, time::sleep}; +use tracing::{info, warn}; + +mod state_manager; +use crate::state_manager::{FileStateStorage, SourceState, StateStorage}; +pub use state_manager::{StateInfo, StateManager, StateStats}; + +source_connector!(OpenSearchSource); + +const CONNECTOR_NAME: &str = "OpenSearch source"; + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct State { + last_poll_timestamp: Option>, + total_documents_fetched: usize, + poll_count: usize, + /// Last document ID processed (for cursor-based pagination) + last_document_id: Option, + /// Last scroll ID (for scroll-based pagination) + last_scroll_id: Option, + /// Last processed offset + last_offset: Option, + /// Error count and last error + error_count: usize, + last_error: Option, + /// Processing statistics + processing_stats: ProcessingStats, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct ProcessingStats { + /// Total bytes processed + total_bytes_processed: u64, + /// Average processing time per batch + avg_batch_processing_time_ms: f64, + /// Last successful processing timestamp + last_successful_poll: Option>, + /// Number of empty polls + empty_polls_count: usize, + /// Number of successful polls + successful_polls_count: usize, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StateConfig { + /// Enable state persistence + pub enabled: bool, + /// State storage type: "file", "opensearch", "redis", etc. + pub storage_type: Option, + /// State storage configuration (depends on storage_type) + pub storage_config: Option, + /// State ID for this connector instance + pub state_id: Option, + /// Auto-save state interval (e.g., "30s", "5m") + pub auto_save_interval: Option, + /// Fields to track in state (e.g., ["last_timestamp", "last_document_id"]) + pub tracked_fields: Option>, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct OpenSearchSourceConfig { + pub url: String, + pub index: String, + pub username: Option, + #[serde(serialize_with = "iggy_common::serde_secret::serialize_optional_secret")] + pub password: Option, + pub query: Option, + pub polling_interval: Option, + pub batch_size: Option, + pub timestamp_field: Option, + pub scroll_timeout: Option, + pub state: Option, +} + +#[derive(Debug)] +pub struct OpenSearchSource { + id: u32, + config: OpenSearchSourceConfig, + client: Option, + polling_interval: Duration, + state: Mutex, +} + +impl OpenSearchSource { + pub fn new(id: u32, config: OpenSearchSourceConfig, state: Option) -> Self { + let polling_interval = config + .polling_interval + .as_deref() + .unwrap_or("10s") + .parse::() + .unwrap_or_else(|_| humantime::Duration::from_str("10s").unwrap()) + .into(); + + let restored_state = state + .and_then(|s| s.deserialize::(CONNECTOR_NAME, id)) + .inspect(|s| { + info!( + "Restored state for {CONNECTOR_NAME} connector with ID: {id}. \ + Documents fetched: {}, poll count: {}", + s.total_documents_fetched, s.poll_count + ); + }); + + OpenSearchSource { + id, + config, + client: None, + polling_interval, + state: Mutex::new(restored_state.unwrap_or(State { + last_poll_timestamp: None, + total_documents_fetched: 0, + poll_count: 0, + last_document_id: None, + last_scroll_id: None, + last_offset: None, + error_count: 0, + last_error: None, + processing_stats: ProcessingStats { + total_bytes_processed: 0, + avg_batch_processing_time_ms: 0.0, + last_successful_poll: None, + empty_polls_count: 0, + successful_polls_count: 0, + }, + })), + } + } + + fn serialize_state(&self, state: &State) -> Option { + ConnectorState::serialize(state, CONNECTOR_NAME, self.id) + } + + /// Create state storage based on configuration + fn create_state_storage(&self) -> Option> { + let state_config = self.config.state.as_ref()?; + if !state_config.enabled { + return None; + } + + match state_config.storage_type.as_deref() { + Some("file") | None => { + let base_path = state_config + .storage_config + .as_ref() + .and_then(|c| c.get("base_path")) + .and_then(|p| p.as_str()) + .unwrap_or("./connector_states"); + + Some(Arc::new(FileStateStorage::new(base_path))) + } + Some("opensearch") => { + // TODO: Implement OpenSearch-based state storage + warn!("OpenSearch state storage not yet implemented, falling back to file storage"); + Some(Arc::new(FileStateStorage::new("./connector_states"))) + } + Some(storage_type) => { + warn!( + "Unknown state storage type: {}, falling back to file storage", + storage_type + ); + Some(Arc::new(FileStateStorage::new("./connector_states"))) + } + } + } + + /// Get state ID for this connector + fn get_state_id(&self) -> String { + self.config + .state + .as_ref() + .and_then(|s| s.state_id.clone()) + .unwrap_or_else(|| format!("opensearch_source_{}", self.id)) + } + + /// Convert internal state to SourceState + async fn internal_state_to_source_state(&self) -> Result { + let state = self.state.lock().await; + + let data = json!({ + "last_poll_timestamp": state.last_poll_timestamp, + "total_documents_fetched": state.total_documents_fetched, + "poll_count": state.poll_count, + "last_document_id": state.last_document_id, + "last_scroll_id": state.last_scroll_id, + "last_offset": state.last_offset, + "error_count": state.error_count, + "last_error": state.last_error, + "processing_stats": state.processing_stats, + }); + + Ok(SourceState { + id: self.get_state_id(), + last_updated: Utc::now(), + version: 1, + data, + metadata: Some(json!({ + "connector_type": "opensearch_source", + "connector_id": self.id, + "index": self.config.index, + "url": self.config.url, + })), + }) + } + + /// Convert SourceState to internal state + async fn source_state_to_internal_state( + &mut self, + source_state: SourceState, + ) -> Result<(), Error> { + let mut state = self.state.lock().await; + + if let Some(data) = source_state.data.as_object() { + if let Some(timestamp) = data.get("last_poll_timestamp") + && let Some(ts_str) = timestamp.as_str() + && let Ok(dt) = DateTime::parse_from_rfc3339(ts_str) + { + state.last_poll_timestamp = Some(dt.with_timezone(&Utc)); + } + + if let Some(count) = data.get("total_documents_fetched") + && let Some(count_val) = count.as_u64() + { + state.total_documents_fetched = count_val as usize; + } + + if let Some(count) = data.get("poll_count") + && let Some(count_val) = count.as_u64() + { + state.poll_count = count_val as usize; + } + + if let Some(doc_id) = data.get("last_document_id") { + state.last_document_id = doc_id.as_str().map(|s| s.to_string()); + } + + if let Some(scroll_id) = data.get("last_scroll_id") { + state.last_scroll_id = scroll_id.as_str().map(|s| s.to_string()); + } + + if let Some(offset) = data.get("last_offset") { + state.last_offset = offset.as_u64(); + } + + if let Some(error_count) = data.get("error_count") + && let Some(count_val) = error_count.as_u64() + { + state.error_count = count_val as usize; + } + + if let Some(last_error) = data.get("last_error") { + state.last_error = last_error.as_str().map(|s| s.to_string()); + } + + if let Some(stats) = data.get("processing_stats") + && let Ok(processing_stats) = serde_json::from_value(stats.clone()) + { + state.processing_stats = processing_stats; + } + } + + Ok(()) + } + + async fn create_client(&self) -> Result { + let url = Url::parse(&self.config.url) + .map_err(|error| Error::Storage(format!("Invalid OpenSearch URL: {error}")))?; + + let conn_pool = opensearch::http::transport::SingleNodeConnectionPool::new(url); + let mut transport_builder = TransportBuilder::new(conn_pool); + + if let (Some(username), Some(password)) = (&self.config.username, &self.config.password) { + let credentials = + Credentials::Basic(username.clone(), password.expose_secret().to_string()); + transport_builder = transport_builder.auth(credentials); + } + + let transport = transport_builder + .build() + .map_err(|e| Error::Storage(format!("Failed to build transport: {}", e)))?; + + Ok(OpenSearch::new(transport)) + } + + async fn search_documents(&self, client: &OpenSearch) -> Result, Error> { + let state = self.state.lock().await; + let batch_size = self.config.batch_size.unwrap_or(100); + + // Build query based on timestamp field if configured + let mut query = self.config.query.clone().unwrap_or_else(|| { + json!({ + "match_all": {} + }) + }); + + // Add timestamp filter for incremental polling + if let Some(timestamp_field) = &self.config.timestamp_field + && let Some(last_timestamp) = state.last_poll_timestamp + { + query = json!({ + "bool": { + "must": [ + query, + { + "range": { + timestamp_field: { + "gt": last_timestamp.to_rfc3339() + } + } + } + ] + } + }); + } + + let search_body = json!({ + "query": query, + "size": batch_size, + "sort": [ + { + self.config.timestamp_field.as_deref().unwrap_or("@timestamp"): { + "order": "asc" + } + } + ] + }); + + drop(state); + + let response = client + .search(SearchParts::Index(&[&self.config.index])) + .body(search_body) + .send() + .await + .map_err(|e| Error::Storage(format!("Failed to execute search: {}", e)))?; + + if !response.status_code().is_success() { + let error_text = response + .text() + .await + .unwrap_or_else(|_| "Unknown error".to_string()); + return Err(Error::Storage(format!( + "Search request failed: {}", + error_text + ))); + } + + let response_body: Value = response + .json() + .await + .map_err(|e| Error::Storage(format!("Failed to parse search response: {}", e)))?; + + let mut messages = Vec::new(); + let mut latest_timestamp = None; + + if let Some(hits) = response_body + .get("hits") + .and_then(|h| h.get("hits")) + .and_then(|h| h.as_array()) + { + for hit in hits { + if let Some(source) = hit.get("_source") { + // Extract timestamp for incremental polling + if let Some(timestamp_field) = &self.config.timestamp_field + && let Some(timestamp_str) = + source.get(timestamp_field).and_then(|v| v.as_str()) + && let Ok(timestamp) = DateTime::parse_from_rfc3339(timestamp_str) + { + let timestamp_utc = timestamp.with_timezone(&Utc); + if latest_timestamp.is_none() || timestamp_utc > latest_timestamp.unwrap() { + latest_timestamp = Some(timestamp_utc); + } + } + + // Create message from document + let payload = serde_json::to_vec(source).map_err(|e| { + Error::Serialization(format!("Failed to serialize document: {}", e)) + })?; + + let message = ProducedMessage { + id: None, + headers: None, + checksum: None, + timestamp: None, + origin_timestamp: None, + payload, + }; + messages.push(message); + } + } + } + + // Update state + let mut state = self.state.lock().await; + state.total_documents_fetched += messages.len(); + state.poll_count += 1; + if let Some(timestamp) = latest_timestamp { + state.last_poll_timestamp = Some(timestamp); + } + + Ok(messages) + } +} + +#[async_trait] +impl Source for OpenSearchSource { + async fn open(&mut self) -> Result<(), Error> { + info!( + "Opening OpenSearch source connector with ID: {} for URL: {}, index: {}", + self.id, self.config.url, self.config.index + ); + + let client = self.create_client().await?; + + // Test connection by checking if index exists + let response = client + .indices() + .exists(opensearch::indices::IndicesExistsParts::Index(&[&self + .config + .index])) + .send() + .await + .map_err(|e| Error::Storage(format!("Failed to check index existence: {}", e)))?; + + if !response.status_code().is_success() { + return Err(Error::Storage(format!( + "Index '{}' does not exist or is not accessible", + self.config.index + ))); + } + + self.client = Some(client); + + // Load state if state management is enabled + if self + .config + .state + .as_ref() + .map(|s| s.enabled) + .unwrap_or(false) + && let Err(e) = self.load_state().await + { + warn!( + "Failed to load state for OpenSearch source connector with ID: {}: {}", + self.id, e + ); + } + + info!( + "Successfully opened OpenSearch source connector with ID: {}", + self.id + ); + Ok(()) + } + + async fn poll(&self) -> Result { + let start_time = std::time::Instant::now(); + + sleep(self.polling_interval).await; + + let client = self + .client + .as_ref() + .ok_or_else(|| Error::Storage("OpenSearch client not initialized".to_string()))?; + + let messages = match self.search_documents(client).await { + Ok(msgs) => { + // Update success statistics + let mut state = self.state.lock().await; + state.processing_stats.successful_polls_count += 1; + state.processing_stats.last_successful_poll = Some(Utc::now()); + + let processing_time = start_time.elapsed().as_millis() as f64; + let total_polls = state.processing_stats.successful_polls_count + + state.processing_stats.empty_polls_count; + state.processing_stats.avg_batch_processing_time_ms = + (state.processing_stats.avg_batch_processing_time_ms + * (total_polls - 1) as f64 + + processing_time) + / total_polls as f64; + + if msgs.is_empty() { + state.processing_stats.empty_polls_count += 1; + } + + drop(state); + msgs + } + Err(e) => { + // Update error statistics + let mut state = self.state.lock().await; + state.error_count += 1; + state.last_error = Some(e.to_string()); + drop(state); + return Err(e); + } + }; + let persisted_state = { + let state = self.state.lock().await; + self.serialize_state(&state) + }; + + Ok(ProducedMessages { + schema: Schema::Json, + messages, + state: persisted_state, + }) + } + + async fn close(&mut self) -> Result<(), Error> { + let state = self.state.lock().await; + info!( + "OpenSearch source connector with ID: {} is closing. Stats: {} total documents fetched, {} polls executed, {} errors", + self.id, state.total_documents_fetched, state.poll_count, state.error_count + ); + drop(state); + + // Save final state if state management is enabled + if self + .config + .state + .as_ref() + .map(|s| s.enabled) + .unwrap_or(false) + && let Err(e) = self.save_state().await + { + warn!( + "Failed to save final state for OpenSearch source connector with ID: {}: {}", + self.id, e + ); + } + + self.client = None; + info!( + "OpenSearch source connector with ID: {} is closed.", + self.id + ); + Ok(()) + } +} diff --git a/core/connectors/sources/opensearch_source/src/state_manager.rs b/core/connectors/sources/opensearch_source/src/state_manager.rs new file mode 100644 index 0000000000..c681444efb --- /dev/null +++ b/core/connectors/sources/opensearch_source/src/state_manager.rs @@ -0,0 +1,388 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use crate::{OpenSearchSource, StateConfig}; +use async_trait::async_trait; +use iggy_common::{ChronoDuration, DateTime, Utc}; +use iggy_connector_sdk::Error; +use serde::{Deserialize, Serialize}; +use std::str::FromStr; +use std::sync::Arc; +use tokio::time::{Duration, interval}; +use tracing::{error, info, warn}; + +impl OpenSearchSource { + async fn get_state(&self) -> Result, Error> { + if self + .config + .state + .as_ref() + .map(|s| s.enabled) + .unwrap_or(false) + { + Ok(Some(self.internal_state_to_source_state().await?)) + } else { + Ok(None) + } + } + + pub(super) async fn save_state(&self) -> Result<(), Error> { + if !self + .config + .state + .as_ref() + .map(|s| s.enabled) + .unwrap_or(false) + { + return Ok(()); + } + + let storage = self + .create_state_storage() + .ok_or_else(|| Error::Storage("State storage not configured".to_string()))?; + + let source_state = self.internal_state_to_source_state().await?; + storage.save_source_state(&source_state).await?; + + info!( + "Saved state for OpenSearch source connector with ID: {}", + self.id + ); + Ok(()) + } + + pub(super) async fn load_state(&mut self) -> Result<(), Error> { + if !self + .config + .state + .as_ref() + .map(|s| s.enabled) + .unwrap_or(false) + { + return Ok(()); + } + + let storage = self + .create_state_storage() + .ok_or_else(|| Error::Storage("State storage not configured".to_string()))?; + + let state_id = self.get_state_id(); + if let Some(source_state) = storage.load_source_state(&state_id).await? { + self.source_state_to_internal_state(source_state).await?; + + let state = self.state.lock().await; + info!( + "Loaded state for OpenSearch source connector with ID: {} - last poll: {:?}, total docs: {}, polls: {}", + self.id, state.last_poll_timestamp, state.total_documents_fetched, state.poll_count + ); + } else { + info!( + "No existing state found for OpenSearch source connector with ID: {}, starting fresh", + self.id + ); + } + + Ok(()) + } +} + +/// State manager for OpenSearch source connector +pub struct StateManager { + storage: Arc, + config: StateConfig, + auto_save_interval: Option, +} + +impl StateManager { + pub fn new(config: StateConfig) -> Result { + let storage = Self::create_storage(&config)?; + let auto_save_interval = config + .auto_save_interval + .as_deref() + .and_then(|interval_str| { + humantime::Duration::from_str(interval_str) + .ok() + .map(|d| Duration::from_secs(d.as_secs())) + }); + + Ok(Self { + storage, + config, + auto_save_interval, + }) + } + + fn create_storage(config: &StateConfig) -> Result, Error> { + match config.storage_type.as_deref() { + Some("file") | None => { + let base_path = config + .storage_config + .as_ref() + .and_then(|c| c.get("base_path")) + .and_then(|p| p.as_str()) + .unwrap_or("./connector_states"); + + Ok(Arc::new(FileStateStorage::new(base_path))) + } + Some("opensearch") => { + // TODO: Implement OpenSearch-based state storage + warn!("OpenSearch state storage not yet implemented, falling back to file storage"); + Ok(Arc::new(FileStateStorage::new("./connector_states"))) + } + Some(storage_type) => { + warn!( + "Unknown state storage type: {}, falling back to file storage", + storage_type + ); + Ok(Arc::new(FileStateStorage::new("./connector_states"))) + } + } + } + + /// Start auto-save background task + pub async fn start_auto_save(&self, connector: Arc) { + let interval_duration = self + .auto_save_interval + .unwrap_or_else(|| Duration::from_secs(60)); + let storage = self.storage.clone(); + let state_id = self.config.state_id.clone(); + tokio::spawn(async move { + let mut interval = interval(interval_duration); + loop { + interval.tick().await; + if let Ok(Some(state)) = connector.get_state().await { + if let Err(e) = storage.save_source_state(&state).await { + error!( + "Failed to auto-save state for {}: {}", + state_id.as_deref().unwrap_or("unknown"), + e + ); + } else { + info!( + "Auto-saved state for {}", + state_id.as_deref().unwrap_or("unknown") + ); + } + } + } + }); + } + + /// Get state statistics + pub async fn get_state_stats(&self) -> Result { + let state_ids = self.storage.list_states().await?; + let mut stats = StateStats { + total_states: state_ids.len(), + states: Vec::new(), + }; + + for state_id in state_ids { + if let Some(state) = self.storage.load_source_state(&state_id).await? { + stats.states.push(StateInfo { + id: state.id, + last_updated: state.last_updated, + version: state.version, + connector_type: state + .metadata + .as_ref() + .and_then(|m| m.get("connector_type")) + .and_then(|t| t.as_str()) + .unwrap_or("unknown") + .to_string(), + }); + } + } + + Ok(stats) + } + + /// Clean up old states + pub async fn cleanup_old_states(&self, older_than_days: u32) -> Result { + let state_ids = self.storage.list_states().await?; + let cutoff_time = Utc::now() - ChronoDuration::days(older_than_days as i64); + let mut deleted_count = 0; + + for state_id in state_ids { + if let Some(state) = self.storage.load_source_state(&state_id).await? + && state.last_updated < cutoff_time + { + if let Err(e) = self.storage.delete_state(&state_id).await { + warn!("Failed to delete old state {}: {}", state_id, e); + } else { + deleted_count += 1; + info!("Deleted old state: {}", state_id); + } + } + } + + Ok(deleted_count) + } + + pub fn auto_save_interval(&self) -> Option { + self.auto_save_interval + } +} + +#[derive(Debug)] +pub struct StateStats { + pub total_states: usize, + pub states: Vec, +} + +#[derive(Debug)] +pub struct StateInfo { + pub id: String, + pub last_updated: DateTime, + pub version: u32, + pub connector_type: String, +} + +/// State management for source connectors +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SourceState { + /// Unique identifier for this state + pub id: String, + /// Timestamp when this state was last updated + pub last_updated: DateTime, + /// Version of the state format + pub version: u32, + /// Generic state data as JSON + pub data: serde_json::Value, + /// Optional metadata + pub metadata: Option, +} + +/// State storage backend trait +#[async_trait] +pub trait StateStorage: Send + Sync { + /// Save source state to storage + async fn save_source_state(&self, state: &SourceState) -> Result<(), Error>; + + /// Load source state from storage + async fn load_source_state(&self, id: &str) -> Result, Error>; + + /// Delete state from storage + async fn delete_state(&self, id: &str) -> Result<(), Error>; + + /// List all state IDs + async fn list_states(&self) -> Result, Error>; +} + +/// File-based state storage implementation +pub struct FileStateStorage { + base_path: std::path::PathBuf, +} + +impl FileStateStorage { + pub fn new>(base_path: P) -> Self { + Self { + base_path: base_path.as_ref().to_path_buf(), + } + } + + fn get_state_path(&self, id: &str) -> std::path::PathBuf { + self.base_path.join(format!("{id}.json")) + } +} + +#[async_trait] +impl StateStorage for FileStateStorage { + async fn save_source_state(&self, state: &SourceState) -> Result<(), Error> { + use tokio::fs; + + // Ensure directory exists + if let Some(parent) = self.base_path.parent() { + fs::create_dir_all(parent) + .await + .map_err(|e| Error::Storage(format!("Failed to create state directory: {e}")))?; + } + + let path = self.get_state_path(&state.id); + let json = serde_json::to_string_pretty(state) + .map_err(|e| Error::Serialization(format!("Failed to serialize source state: {e}")))?; + + fs::write(path, json) + .await + .map_err(|e| Error::Storage(format!("Failed to write state file: {e}")))?; + + Ok(()) + } + + async fn load_source_state(&self, id: &str) -> Result, Error> { + use tokio::fs; + + let path = self.get_state_path(id); + if !path.exists() { + return Ok(None); + } + + let content = fs::read_to_string(path) + .await + .map_err(|e| Error::Storage(format!("Failed to read state file: {e}")))?; + + let state: SourceState = serde_json::from_str(&content).map_err(|e| { + Error::Serialization(format!("Failed to deserialize source state: {e}")) + })?; + + Ok(Some(state)) + } + + async fn delete_state(&self, id: &str) -> Result<(), Error> { + use tokio::fs; + + let path = self.get_state_path(id); + if path.exists() { + fs::remove_file(path) + .await + .map_err(|e| Error::Storage(format!("Failed to delete state file: {e}")))?; + } + + Ok(()) + } + + async fn list_states(&self) -> Result, Error> { + use tokio::fs; + + let mut states = Vec::new(); + + if !self.base_path.exists() { + return Ok(states); + } + + let mut entries = fs::read_dir(&self.base_path) + .await + .map_err(|e| Error::Storage(format!("Failed to read state directory: {e}")))?; + + while let Some(entry) = entries + .next_entry() + .await + .map_err(|e| Error::Storage(format!("Failed to read directory entry: {e}")))? + { + if let Some(extension) = entry.path().extension() + && extension == "json" + && let Some(stem) = entry.path().file_stem() + && let Some(id) = stem.to_str() + { + states.push(id.to_string()); + } + } + + Ok(states) + } +} From 143b26c98feddd1cc38a6ae8dde9ff13f56316df Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sun, 14 Jun 2026 00:21:26 -0400 Subject: [PATCH 02/57] docs(connectors): add opensearch_source config and README Mirror elasticsearch_source documentation, adapted for the OpenSearch wire protocol and state storage type naming. --- .../sources/opensearch_source/README.md | 211 ++++++++++++++++++ .../sources/opensearch_source/config.toml | 38 ++++ 2 files changed, 249 insertions(+) create mode 100644 core/connectors/sources/opensearch_source/README.md create mode 100644 core/connectors/sources/opensearch_source/config.toml diff --git a/core/connectors/sources/opensearch_source/README.md b/core/connectors/sources/opensearch_source/README.md new file mode 100644 index 0000000000..466b20f7b4 --- /dev/null +++ b/core/connectors/sources/opensearch_source/README.md @@ -0,0 +1,211 @@ +# OpenSearch Source Connector with State Management + +This OpenSearch source connector provides comprehensive state management capabilities to track processing progress and enable fault-tolerant data ingestion. + +## Features + +- **Incremental Data Processing**: Track last processed timestamp to avoid reprocessing data +- **Cursor-based Pagination**: Support for document ID-based cursors +- **Scroll-based Pagination**: Support for OpenSearch scroll API +- **Error Tracking**: Monitor error counts and last error messages +- **Processing Statistics**: Track performance metrics and processing times +- **Persistent State Storage**: Multiple storage backends (file, OpenSearch, Redis) +- **Auto-save**: Configurable automatic state persistence +- **State Recovery**: Resume processing from last known position after restart + +## Configuration + +### Basic Configuration + +```toml +type = "source" +key = "opensearch" +enabled = true +version = 0 +name = "OpenSearch source" +path = "target/release/libiggy_connector_opensearch_source" + +[[streams]] +stream = "opensearch_stream" +topic = "documents" +schema = "json" +batch_length = 100 +linger_time = "5ms" + +[plugin_config] +url = "http://localhost:9200" +index = "logs-*" +polling_interval = "30s" +batch_size = 100 +timestamp_field = "@timestamp" +query = { + "match_all": {} +} +``` + +### State Management Configuration + +```toml +[plugin_config] +# ... basic config ... +state = { + enabled = true + storage_type = "file" # "file", "opensearch", "redis" + storage_config = { + base_path = "./connector_states" # for file storage + # index = "connector_states" # for opensearch storage + # url = "redis://localhost:6379" # for redis storage + } + state_id = "opensearch_logs_connector" + auto_save_interval = "5m" + tracked_fields = [ + "last_poll_timestamp", + "last_document_id", + "total_documents_fetched" + ] +} +``` + +## State Information + +The connector tracks the following state information: + +### Processing State + +- `last_poll_timestamp`: Last successful poll timestamp +- `total_documents_fetched`: Total number of documents processed +- `poll_count`: Number of polling cycles executed +- `last_document_id`: Last processed document ID (for cursor pagination) +- `last_scroll_id`: Last scroll ID (for scroll pagination) +- `last_offset`: Last processed offset + +### Error Tracking + +- `error_count`: Total number of errors encountered +- `last_error`: Last error message + +### Performance Statistics + +- `total_bytes_processed`: Total bytes processed +- `avg_batch_processing_time_ms`: Average processing time per batch +- `last_successful_poll`: Timestamp of last successful poll +- `empty_polls_count`: Number of polls that returned no documents +- `successful_polls_count`: Number of successful polls + +## Storage Backends + +### File Storage (Default) + +```toml +state = { + enabled = true + storage_type = "file" + storage_config = { + base_path = "./connector_states" + } +} +``` + +### OpenSearch Storage + +```toml +state = { + enabled = true + storage_type = "opensearch" + storage_config = { + index = "connector_states" + url = "http://localhost:9200" + } +} +``` + +### Redis Storage + +```toml +state = { + enabled = true + storage_type = "redis" + storage_config = { + url = "redis://localhost:6379" + key_prefix = "connector_states:" + } +} +``` + +## State File Format + +State files are stored as JSON with the following structure: + +```json +{ + "id": "opensearch_logs_connector", + "last_updated": "2024-01-15T10:30:00Z", + "version": 1, + "data": { + "last_poll_timestamp": "2024-01-15T10:30:00Z", + "total_documents_fetched": 15000, + "poll_count": 150, + "last_document_id": "doc_12345", + "last_scroll_id": "scroll_abc123", + "last_offset": 15000, + "error_count": 2, + "last_error": "Connection timeout", + "processing_stats": { + "total_bytes_processed": 1048576, + "avg_batch_processing_time_ms": 125.5, + "last_successful_poll": "2024-01-15T10:30:00Z", + "empty_polls_count": 5, + "successful_polls_count": 145 + } + }, + "metadata": { + "connector_type": "opensearch_source", + "connector_id": 1, + "index": "logs-*", + "url": "http://localhost:9200" + } +} +``` + +## Best Practices + +1. **State ID Uniqueness**: Use unique state IDs for different connector instances +2. **Auto-save Interval**: Set appropriate auto-save intervals based on your data volume +3. **Storage Location**: Use persistent storage locations for production deployments +4. **State Cleanup**: Regularly clean up old state files to prevent disk space issues +5. **Error Handling**: Monitor error counts and implement appropriate alerting +6. **Backup**: Regularly backup state files for disaster recovery + +## Troubleshooting + +### Common Issues + +1. **State Not Loading**: Check file permissions and storage path +2. **State Corruption**: Delete corrupted state files to start fresh +3. **Performance Issues**: Adjust auto-save interval and batch sizes +4. **Storage Full**: Implement state cleanup policies + +### Monitoring + +Monitor the following metrics: + +- State save/load success rates +- Processing statistics +- Error counts and types +- Storage usage for state files + +## Migration + +To migrate from a connector without state management: + +1. Add state configuration to your connector config +2. Set `enabled = true` in state config +3. Restart the connector +4. The connector will start tracking state from the next poll cycle + +To migrate between storage backends: + +1. Export state from current storage +2. Update storage configuration +3. Import state to new storage +4. Restart connector diff --git a/core/connectors/sources/opensearch_source/config.toml b/core/connectors/sources/opensearch_source/config.toml new file mode 100644 index 0000000000..dceed1b064 --- /dev/null +++ b/core/connectors/sources/opensearch_source/config.toml @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +type = "source" +key = "opensearch" +enabled = true +version = 0 +name = "OpenSearch source" +path = "../../target/release/libiggy_connector_opensearch_source" +plugin_config_format = "json" + +[[streams]] +stream = "test_stream" +topic = "test_topic" +schema = "json" +batch_length = 1000 +linger_time = "5ms" + +[plugin_config] +url = "http://localhost:9200" +index = "test_documents" +polling_interval = "100ms" +batch_size = 100 +timestamp_field = "timestamp" From 4367a452299040bfb03ae3ac76d22638affd93ce Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sun, 14 Jun 2026 00:29:18 -0400 Subject: [PATCH 03/57] test(connectors): add opensearch_source unit and integration tests Add the four canonical source-state unit tests, plus integration fixtures (opensearchproject/opensearch:2.19.1 container) and end-to-end tests covering happy paths (poll, empty index, bulk, restart state persistence) and a negative path (missing index surfaces ConnectorStatus::Error via the runtime API). --- Cargo.lock | 1 + .../sources/opensearch_source/Cargo.toml | 3 +- .../sources/opensearch_source/src/lib.rs | 115 ++++++ .../tests/connectors/fixtures/mod.rs | 2 + .../fixtures/opensearch/container.rs | 349 +++++++++++++++++ .../connectors/fixtures/opensearch/mod.rs | 23 ++ .../connectors/fixtures/opensearch/source.rs | 195 ++++++++++ core/integration/tests/connectors/mod.rs | 1 + .../tests/connectors/opensearch/mod.rs | 24 ++ .../opensearch/opensearch_source.rs | 367 ++++++++++++++++++ .../tests/connectors/opensearch/source.toml | 20 + 11 files changed, 1099 insertions(+), 1 deletion(-) create mode 100644 core/integration/tests/connectors/fixtures/opensearch/container.rs create mode 100644 core/integration/tests/connectors/fixtures/opensearch/mod.rs create mode 100644 core/integration/tests/connectors/fixtures/opensearch/source.rs create mode 100644 core/integration/tests/connectors/opensearch/mod.rs create mode 100644 core/integration/tests/connectors/opensearch/opensearch_source.rs create mode 100644 core/integration/tests/connectors/opensearch/source.toml diff --git a/Cargo.lock b/Cargo.lock index ea7b66fdfd..f3fec351ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6943,6 +6943,7 @@ dependencies = [ "iggy_connector_sdk", "once_cell", "opensearch", + "rmp-serde", "secrecy", "serde", "serde_json", diff --git a/core/connectors/sources/opensearch_source/Cargo.toml b/core/connectors/sources/opensearch_source/Cargo.toml index 52eec99e7b..5c91507112 100644 --- a/core/connectors/sources/opensearch_source/Cargo.toml +++ b/core/connectors/sources/opensearch_source/Cargo.toml @@ -38,11 +38,12 @@ crate-type = ["cdylib", "lib"] [dependencies] async-trait = { workspace = true } dashmap = { workspace = true } -opensearch = { workspace = true } humantime = { workspace = true } iggy_common = { workspace = true } iggy_connector_sdk = { workspace = true } once_cell = { workspace = true } +opensearch = { workspace = true } +rmp-serde = { workspace = true } secrecy = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/core/connectors/sources/opensearch_source/src/lib.rs b/core/connectors/sources/opensearch_source/src/lib.rs index ddb5c439d3..498d0bf386 100644 --- a/core/connectors/sources/opensearch_source/src/lib.rs +++ b/core/connectors/sources/opensearch_source/src/lib.rs @@ -571,3 +571,118 @@ impl Source for OpenSearchSource { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + + fn test_config() -> OpenSearchSourceConfig { + OpenSearchSourceConfig { + url: "http://localhost:9200".to_string(), + index: "test_documents".to_string(), + username: None, + password: None, + query: None, + polling_interval: Some("100ms".to_string()), + batch_size: Some(10), + timestamp_field: Some("timestamp".to_string()), + scroll_timeout: None, + state: None, + } + } + + fn test_state() -> State { + State { + last_poll_timestamp: None, + total_documents_fetched: 500, + poll_count: 5, + last_document_id: Some("doc_42".to_string()), + last_scroll_id: None, + last_offset: Some(500), + error_count: 1, + last_error: Some("connection reset".to_string()), + processing_stats: ProcessingStats { + total_bytes_processed: 1024, + avg_batch_processing_time_ms: 12.5, + last_successful_poll: None, + empty_polls_count: 2, + successful_polls_count: 5, + }, + } + } + + #[test] + fn given_persisted_state_should_restore_total_documents_fetched() { + let state = test_state(); + let serialized = rmp_serde::to_vec(&state).expect("Failed to serialize state"); + let connector_state = ConnectorState(serialized); + + let source = OpenSearchSource::new(1, test_config(), Some(connector_state)); + + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { + let restored = source.state.lock().await; + assert_eq!(restored.total_documents_fetched, 500); + assert_eq!(restored.poll_count, 5); + assert_eq!(restored.last_document_id, Some("doc_42".to_string())); + }); + } + + #[test] + fn given_no_state_should_start_fresh() { + let source = OpenSearchSource::new(1, test_config(), None); + + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { + let state = source.state.lock().await; + assert_eq!(state.total_documents_fetched, 0); + assert_eq!(state.poll_count, 0); + assert_eq!(state.last_document_id, None); + }); + } + + #[test] + fn given_invalid_state_should_start_fresh() { + let invalid_state = ConnectorState(b"not valid msgpack".to_vec()); + let source = OpenSearchSource::new(1, test_config(), Some(invalid_state)); + + let runtime = tokio::runtime::Runtime::new().unwrap(); + runtime.block_on(async { + let state = source.state.lock().await; + assert_eq!(state.total_documents_fetched, 0); + assert_eq!(state.poll_count, 0); + }); + } + + #[test] + fn state_should_be_serializable_and_deserializable() { + let original = test_state(); + + let serialized = rmp_serde::to_vec(&original).expect("Failed to serialize"); + let deserialized: State = + rmp_serde::from_slice(&serialized).expect("Failed to deserialize"); + + assert_eq!( + original.total_documents_fetched, + deserialized.total_documents_fetched + ); + assert_eq!(original.poll_count, deserialized.poll_count); + assert_eq!(original.last_document_id, deserialized.last_document_id); + assert_eq!(original.error_count, deserialized.error_count); + } + + #[test] + fn serialize_state_helper_should_produce_valid_connector_state() { + let source = OpenSearchSource::new(1, test_config(), None); + let state = test_state(); + + let connector_state = source.serialize_state(&state); + assert!(connector_state.is_some()); + + let restored: State = connector_state + .unwrap() + .deserialize(CONNECTOR_NAME, 1) + .expect("Failed to deserialize state"); + assert_eq!(restored.total_documents_fetched, 500); + } +} diff --git a/core/integration/tests/connectors/fixtures/mod.rs b/core/integration/tests/connectors/fixtures/mod.rs index 616c3a557a..4fac6db286 100644 --- a/core/integration/tests/connectors/fixtures/mod.rs +++ b/core/integration/tests/connectors/fixtures/mod.rs @@ -26,6 +26,7 @@ mod http; mod iceberg; mod influxdb; mod mongodb; +mod opensearch; mod postgres; mod quickwit; mod wiremock; @@ -68,6 +69,7 @@ pub use mongodb::{ MongoDbOps, MongoDbSinkAutoCreateFixture, MongoDbSinkBatchFixture, MongoDbSinkFailpointFixture, MongoDbSinkFixture, MongoDbSinkJsonFixture, MongoDbSinkWriteConcernFixture, }; +pub use opensearch::{OpenSearchSourceMissingIndexFixture, OpenSearchSourcePreCreatedFixture}; pub use postgres::{ PostgresOps, PostgresSinkByteaFixture, PostgresSinkFixture, PostgresSinkJsonFixture, PostgresSourceByteaFixture, PostgresSourceDeleteFixture, PostgresSourceJsonFixture, diff --git a/core/integration/tests/connectors/fixtures/opensearch/container.rs b/core/integration/tests/connectors/fixtures/opensearch/container.rs new file mode 100644 index 0000000000..245acab021 --- /dev/null +++ b/core/integration/tests/connectors/fixtures/opensearch/container.rs @@ -0,0 +1,349 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use integration::harness::TestBinaryError; +use reqwest_middleware::ClientWithMiddleware as HttpClient; +use reqwest_retry::RetryTransientMiddleware; +use reqwest_retry::policies::ExponentialBackoff; +use serde::Deserialize; +use testcontainers_modules::testcontainers::core::wait::HttpWaitStrategy; +use testcontainers_modules::testcontainers::core::{IntoContainerPort, WaitFor}; +use testcontainers_modules::testcontainers::runners::AsyncRunner; +use testcontainers_modules::testcontainers::{ + ContainerAsync, GenericImage, ImageExt, ReuseDirective, +}; +use tracing::info; + +const OPENSEARCH_IMAGE: &str = "docker.io/opensearchproject/opensearch"; +const OPENSEARCH_TAG: &str = "2.19.1"; +const OPENSEARCH_PORT: u16 = 9200; +const OPENSEARCH_HEALTH_ENDPOINT: &str = "/_cluster/health"; +// Fixed name + ReuseDirective::Always shares one container across nextest's +// per-test processes: the first test creates it, every later test attaches by +// name. Per-test isolation comes from a unique index per fixture, not a fresh +// container. +const OPENSEARCH_CONTAINER_NAME: &str = "iggy-test-opensearch"; + +pub const DEFAULT_TEST_STREAM: &str = "test_stream"; +pub const DEFAULT_TEST_TOPIC: &str = "test_topic"; + +pub const ENV_SOURCE_URL: &str = "IGGY_CONNECTORS_SOURCE_OPENSEARCH_PLUGIN_CONFIG_URL"; +pub const ENV_SOURCE_INDEX: &str = "IGGY_CONNECTORS_SOURCE_OPENSEARCH_PLUGIN_CONFIG_INDEX"; +pub const ENV_SOURCE_POLLING_INTERVAL: &str = + "IGGY_CONNECTORS_SOURCE_OPENSEARCH_PLUGIN_CONFIG_POLLING_INTERVAL"; +pub const ENV_SOURCE_BATCH_SIZE: &str = + "IGGY_CONNECTORS_SOURCE_OPENSEARCH_PLUGIN_CONFIG_BATCH_SIZE"; +pub const ENV_SOURCE_TIMESTAMP_FIELD: &str = + "IGGY_CONNECTORS_SOURCE_OPENSEARCH_PLUGIN_CONFIG_TIMESTAMP_FIELD"; +pub const ENV_SOURCE_STREAMS_0_STREAM: &str = "IGGY_CONNECTORS_SOURCE_OPENSEARCH_STREAMS_0_STREAM"; +pub const ENV_SOURCE_STREAMS_0_TOPIC: &str = "IGGY_CONNECTORS_SOURCE_OPENSEARCH_STREAMS_0_TOPIC"; +pub const ENV_SOURCE_STREAMS_0_SCHEMA: &str = "IGGY_CONNECTORS_SOURCE_OPENSEARCH_STREAMS_0_SCHEMA"; +pub const ENV_SOURCE_PATH: &str = "IGGY_CONNECTORS_SOURCE_OPENSEARCH_PATH"; + +#[allow(dead_code)] +#[derive(Debug, Deserialize)] +pub struct OpenSearchSearchResponse { + pub hits: OpenSearchHits, +} + +#[allow(dead_code)] +#[derive(Debug, Deserialize)] +pub struct OpenSearchHits { + pub total: OpenSearchTotal, + pub hits: Vec, +} + +#[allow(dead_code)] +#[derive(Debug, Deserialize)] +pub struct OpenSearchTotal { + pub value: usize, +} + +#[allow(dead_code)] +#[derive(Debug, Deserialize)] +pub struct OpenSearchHit { + #[serde(rename = "_source")] + pub source: serde_json::Value, +} + +pub struct OpenSearchContainer { + // Held so testcontainers' Drop runs on test exit; ReuseDirective::Always + // makes that Drop leave the container running for the next test to attach. + #[allow(dead_code)] + container: ContainerAsync, + pub base_url: String, +} + +impl OpenSearchContainer { + pub async fn start() -> Result { + let container = GenericImage::new(OPENSEARCH_IMAGE, OPENSEARCH_TAG) + .with_exposed_port(OPENSEARCH_PORT.tcp()) + .with_wait_for(WaitFor::http( + HttpWaitStrategy::new(OPENSEARCH_HEALTH_ENDPOINT) + .with_port(OPENSEARCH_PORT.tcp()) + .with_expected_status_code(200u16), + )) + .with_startup_timeout(std::time::Duration::from_secs(120)) + .with_env_var("discovery.type", "single-node") + .with_env_var("plugins.security.disabled", "true") + .with_env_var("OPENSEARCH_JAVA_OPTS", "-Xms512m -Xmx512m") + .with_mapped_port(0, OPENSEARCH_PORT.tcp()) + .with_container_name(OPENSEARCH_CONTAINER_NAME) + .with_reuse(ReuseDirective::Always) + .start() + .await + .map_err(|e| TestBinaryError::FixtureSetup { + fixture_type: "OpenSearchContainer".to_string(), + message: format!("Failed to start container: {e}"), + })?; + + info!("Started OpenSearch container"); + + let mapped_port = container + .ports() + .await + .map_err(|e| TestBinaryError::FixtureSetup { + fixture_type: "OpenSearchContainer".to_string(), + message: format!("Failed to get ports: {e}"), + })? + .map_to_host_port_ipv4(OPENSEARCH_PORT) + .ok_or_else(|| TestBinaryError::FixtureSetup { + fixture_type: "OpenSearchContainer".to_string(), + message: "No mapping for OpenSearch port".to_string(), + })?; + + let base_url = format!("http://localhost:{mapped_port}"); + info!("OpenSearch container available at {base_url}"); + + Ok(Self { + container, + base_url, + }) + } +} + +pub fn create_http_client() -> HttpClient { + let retry_policy = ExponentialBackoff::builder().build_with_max_retries(3); + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(30)) + .build() + .expect("Failed to build HTTP client"); + reqwest_middleware::ClientBuilder::new(client) + .with(RetryTransientMiddleware::new_with_policy(retry_policy)) + .build() +} + +pub trait OpenSearchOps: Sync { + fn container(&self) -> &OpenSearchContainer; + fn http_client(&self) -> &HttpClient; + + fn create_index( + &self, + index_name: &str, + ) -> impl std::future::Future> + Send { + async move { + let url = format!("{}/{}", self.container().base_url, index_name); + let mapping = serde_json::json!({ + "mappings": { + "properties": { + "id": { "type": "integer" }, + "name": { "type": "keyword" }, + "value": { "type": "integer" }, + "timestamp": { "type": "date" } + } + } + }); + + let response = self + .http_client() + .put(&url) + .header("Content-Type", "application/json") + .json(&mapping) + .send() + .await + .map_err(|e| TestBinaryError::FixtureSetup { + fixture_type: "OpenSearchOps".to_string(), + message: format!("Failed to create index: {e}"), + })?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(TestBinaryError::FixtureSetup { + fixture_type: "OpenSearchOps".to_string(), + message: format!("Failed to create index: status={status}, body={body}"), + }); + } + + info!("Created OpenSearch index: {index_name}"); + Ok(()) + } + } + + fn index_document( + &self, + index_name: &str, + doc_id: &str, + document: &serde_json::Value, + ) -> impl std::future::Future> + Send { + async move { + let url = format!( + "{}/{}/_doc/{}", + self.container().base_url, + index_name, + doc_id + ); + + let response = self + .http_client() + .put(&url) + .header("Content-Type", "application/json") + .json(document) + .send() + .await + .map_err(|e| TestBinaryError::FixtureSetup { + fixture_type: "OpenSearchOps".to_string(), + message: format!("Failed to index document: {e}"), + })?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(TestBinaryError::FixtureSetup { + fixture_type: "OpenSearchOps".to_string(), + message: format!("Failed to index document: status={status}, body={body}"), + }); + } + + Ok(()) + } + } + + fn refresh_index( + &self, + index_name: &str, + ) -> impl std::future::Future> + Send { + async move { + let url = format!("{}/{}/_refresh", self.container().base_url, index_name); + + let response = self.http_client().post(&url).send().await.map_err(|e| { + TestBinaryError::InvalidState { + message: format!("Failed to refresh index: {e}"), + } + })?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(TestBinaryError::InvalidState { + message: format!("Failed to refresh index: status={status}, body={body}"), + }); + } + + info!("Refreshed OpenSearch index: {index_name}"); + Ok(()) + } + } + + #[allow(dead_code)] + fn search_all( + &self, + index_name: &str, + ) -> impl std::future::Future> + Send + { + async move { + let url = format!("{}/{}/_search", self.container().base_url, index_name); + let query = serde_json::json!({ + "query": { "match_all": {} }, + "size": 1000, + "_source": true + }); + + let response = self + .http_client() + .post(&url) + .header("Content-Type", "application/json") + .json(&query) + .send() + .await + .map_err(|e| TestBinaryError::InvalidState { + message: format!("Failed to search index: {e}"), + })?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(TestBinaryError::InvalidState { + message: format!("Failed to search index: status={status}, body={body}"), + }); + } + + let text = response + .text() + .await + .map_err(|e| TestBinaryError::InvalidState { + message: format!("Failed to get response text: {e}"), + })?; + + info!("OpenSearch search response: {text}"); + + serde_json::from_str::(&text).map_err(|e| { + TestBinaryError::InvalidState { + message: format!("Failed to parse search response: {e}, body: {text}"), + } + }) + } + } + + fn count_documents( + &self, + index_name: &str, + ) -> impl std::future::Future> + Send { + async move { + let url = format!("{}/{}/_count", self.container().base_url, index_name); + + let response = self.http_client().get(&url).send().await.map_err(|e| { + TestBinaryError::InvalidState { + message: format!("Failed to count documents: {e}"), + } + })?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(TestBinaryError::InvalidState { + message: format!("Failed to count documents: status={status}, body={body}"), + }); + } + + #[derive(Deserialize)] + struct CountResponse { + count: usize, + } + + let count_response = response.json::().await.map_err(|e| { + TestBinaryError::InvalidState { + message: format!("Failed to parse count response: {e}"), + } + })?; + + Ok(count_response.count) + } + } +} diff --git a/core/integration/tests/connectors/fixtures/opensearch/mod.rs b/core/integration/tests/connectors/fixtures/opensearch/mod.rs new file mode 100644 index 0000000000..963cbfb305 --- /dev/null +++ b/core/integration/tests/connectors/fixtures/opensearch/mod.rs @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +pub mod container; +pub mod source; + +pub use source::{OpenSearchSourceMissingIndexFixture, OpenSearchSourcePreCreatedFixture}; diff --git a/core/integration/tests/connectors/fixtures/opensearch/source.rs b/core/integration/tests/connectors/fixtures/opensearch/source.rs new file mode 100644 index 0000000000..3b951a1ec0 --- /dev/null +++ b/core/integration/tests/connectors/fixtures/opensearch/source.rs @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use super::container::{ + DEFAULT_TEST_STREAM, DEFAULT_TEST_TOPIC, ENV_SOURCE_BATCH_SIZE, ENV_SOURCE_INDEX, + ENV_SOURCE_PATH, ENV_SOURCE_POLLING_INTERVAL, ENV_SOURCE_STREAMS_0_SCHEMA, + ENV_SOURCE_STREAMS_0_STREAM, ENV_SOURCE_STREAMS_0_TOPIC, ENV_SOURCE_TIMESTAMP_FIELD, + ENV_SOURCE_URL, OpenSearchContainer, OpenSearchOps, create_http_client, +}; +use async_trait::async_trait; +use iggy_common::IggyTimestamp; +use integration::harness::{TestBinaryError, TestFixture}; +use reqwest_middleware::ClientWithMiddleware as HttpClient; +use std::collections::HashMap; +use uuid::Uuid; + +const TEST_INDEX_PREFIX: &str = "test_documents"; + +/// OpenSearch source fixture for basic document polling. +pub struct OpenSearchSourceFixture { + container: OpenSearchContainer, + http_client: HttpClient, + // Unique per fixture so tests sharing one container never collide on the + // same index. The connector reads from here via ENV_SOURCE_INDEX. + index: String, +} + +impl OpenSearchOps for OpenSearchSourceFixture { + fn container(&self) -> &OpenSearchContainer { + &self.container + } + + fn http_client(&self) -> &HttpClient { + &self.http_client + } +} + +impl OpenSearchSourceFixture { + #[allow(dead_code)] + pub fn index_name(&self) -> &str { + &self.index + } + + pub async fn setup_index(&self) -> Result<(), TestBinaryError> { + self.create_index(&self.index).await + } + + pub async fn insert_document( + &self, + doc_id: i32, + name: &str, + value: i32, + ) -> Result<(), TestBinaryError> { + let timestamp = IggyTimestamp::now().to_rfc3339_string(); + let document = serde_json::json!({ + "id": doc_id, + "name": name, + "value": value, + "timestamp": timestamp + }); + self.index_document(&self.index, &doc_id.to_string(), &document) + .await + } + + pub async fn insert_documents(&self, count: usize) -> Result<(), TestBinaryError> { + for i in 1..=count { + self.insert_document(i as i32, &format!("doc_{i}"), (i * 10) as i32) + .await?; + } + self.refresh_index().await?; + Ok(()) + } + + pub async fn get_document_count(&self) -> Result { + self.count_documents(&self.index).await + } + + pub async fn refresh_index(&self) -> Result<(), TestBinaryError> { + OpenSearchOps::refresh_index(self, &self.index).await + } +} + +#[async_trait] +impl TestFixture for OpenSearchSourceFixture { + async fn setup() -> Result { + let container = OpenSearchContainer::start().await?; + let http_client = create_http_client(); + let index = format!("{TEST_INDEX_PREFIX}_{}", Uuid::new_v4().simple()); + + // Container startup already waits for /_cluster/health to return 200 + // via HttpWaitStrategy, so no additional health check is needed. + Ok(Self { + container, + http_client, + index, + }) + } + + fn connectors_runtime_envs(&self) -> HashMap { + let mut envs = HashMap::new(); + envs.insert(ENV_SOURCE_URL.to_string(), self.container.base_url.clone()); + envs.insert(ENV_SOURCE_INDEX.to_string(), self.index.clone()); + envs.insert(ENV_SOURCE_POLLING_INTERVAL.to_string(), "100ms".to_string()); + envs.insert(ENV_SOURCE_BATCH_SIZE.to_string(), "100".to_string()); + envs.insert( + ENV_SOURCE_TIMESTAMP_FIELD.to_string(), + "timestamp".to_string(), + ); + envs.insert( + ENV_SOURCE_STREAMS_0_STREAM.to_string(), + DEFAULT_TEST_STREAM.to_string(), + ); + envs.insert( + ENV_SOURCE_STREAMS_0_TOPIC.to_string(), + DEFAULT_TEST_TOPIC.to_string(), + ); + envs.insert(ENV_SOURCE_STREAMS_0_SCHEMA.to_string(), "json".to_string()); + envs.insert( + ENV_SOURCE_PATH.to_string(), + "../../target/debug/libiggy_connector_opensearch_source".to_string(), + ); + envs + } +} + +/// OpenSearch source fixture with pre-created index. +pub struct OpenSearchSourcePreCreatedFixture { + inner: OpenSearchSourceFixture, +} + +impl std::ops::Deref for OpenSearchSourcePreCreatedFixture { + type Target = OpenSearchSourceFixture; + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +impl OpenSearchOps for OpenSearchSourcePreCreatedFixture { + fn container(&self) -> &OpenSearchContainer { + &self.inner.container + } + + fn http_client(&self) -> &HttpClient { + &self.inner.http_client + } +} + +#[async_trait] +impl TestFixture for OpenSearchSourcePreCreatedFixture { + async fn setup() -> Result { + let inner = OpenSearchSourceFixture::setup().await?; + + inner.setup_index().await?; + + Ok(Self { inner }) + } + + fn connectors_runtime_envs(&self) -> HashMap { + self.inner.connectors_runtime_envs() + } +} + +/// OpenSearch source fixture pointing at an index that is never created, +/// for exercising the connector's "missing index" failure path. +pub struct OpenSearchSourceMissingIndexFixture { + inner: OpenSearchSourceFixture, +} + +#[async_trait] +impl TestFixture for OpenSearchSourceMissingIndexFixture { + async fn setup() -> Result { + let inner = OpenSearchSourceFixture::setup().await?; + Ok(Self { inner }) + } + + fn connectors_runtime_envs(&self) -> HashMap { + self.inner.connectors_runtime_envs() + } +} diff --git a/core/integration/tests/connectors/mod.rs b/core/integration/tests/connectors/mod.rs index bb4bcc69f1..d014f7a599 100644 --- a/core/integration/tests/connectors/mod.rs +++ b/core/integration/tests/connectors/mod.rs @@ -27,6 +27,7 @@ mod http_config_provider; mod iceberg; mod influxdb; mod mongodb; +mod opensearch; mod postgres; mod quickwit; mod random; diff --git a/core/integration/tests/connectors/opensearch/mod.rs b/core/integration/tests/connectors/opensearch/mod.rs new file mode 100644 index 0000000000..742b93ee3e --- /dev/null +++ b/core/integration/tests/connectors/opensearch/mod.rs @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +mod opensearch_source; + +const TEST_MESSAGE_COUNT: usize = 3; +const POLL_ATTEMPTS: usize = 100; +const POLL_INTERVAL_MS: u64 = 50; diff --git a/core/integration/tests/connectors/opensearch/opensearch_source.rs b/core/integration/tests/connectors/opensearch/opensearch_source.rs new file mode 100644 index 0000000000..71d6928a11 --- /dev/null +++ b/core/integration/tests/connectors/opensearch/opensearch_source.rs @@ -0,0 +1,367 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +use super::{POLL_ATTEMPTS, POLL_INTERVAL_MS, TEST_MESSAGE_COUNT}; +use crate::connectors::fixtures::{ + OpenSearchSourceMissingIndexFixture, OpenSearchSourcePreCreatedFixture, +}; +use iggy_common::MessageClient; +use iggy_common::{Consumer, Identifier, PollingStrategy}; +use iggy_connector_sdk::api::{ConnectorStatus, SourceInfoResponse}; +use integration::harness::seeds; +use integration::iggy_harness; +use reqwest::Client; +use std::time::Duration; +use tokio::time::sleep; + +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/opensearch/source.toml")), + seed = seeds::connector_stream +)] +async fn opensearch_source_produces_messages_to_iggy( + harness: &TestHarness, + fixture: OpenSearchSourcePreCreatedFixture, +) { + let client = harness.root_client().await.unwrap(); + + fixture + .insert_documents(TEST_MESSAGE_COUNT) + .await + .expect("Failed to insert documents"); + + let doc_count = fixture + .get_document_count() + .await + .expect("Failed to get document count"); + assert_eq!( + doc_count, TEST_MESSAGE_COUNT, + "Expected {TEST_MESSAGE_COUNT} documents in OpenSearch" + ); + + let stream_id: Identifier = seeds::names::STREAM.try_into().unwrap(); + let topic_id: Identifier = seeds::names::TOPIC.try_into().unwrap(); + let consumer_id: Identifier = "test_consumer".try_into().unwrap(); + + let mut received: Vec = Vec::new(); + for _ in 0..POLL_ATTEMPTS { + if let Ok(polled) = client + .poll_messages( + &stream_id, + &topic_id, + None, + &Consumer::new(consumer_id.clone()), + &PollingStrategy::next(), + 10, + true, + ) + .await + { + for msg in polled.messages { + if let Ok(json) = serde_json::from_slice(&msg.payload) { + received.push(json); + } + } + if received.len() >= TEST_MESSAGE_COUNT { + break; + } + } + sleep(Duration::from_millis(POLL_INTERVAL_MS)).await; + } + + assert!( + received.len() >= TEST_MESSAGE_COUNT, + "Expected at least {TEST_MESSAGE_COUNT} messages, got {}", + received.len() + ); + + for (i, record) in received.iter().enumerate() { + let expected_id = (i + 1) as i64; + let expected_name = format!("doc_{}", i + 1); + + assert_eq!( + record.get("id").and_then(|v| v.as_i64()), + Some(expected_id), + "ID mismatch at record {i}" + ); + assert_eq!( + record.get("name").and_then(|v| v.as_str()), + Some(expected_name.as_str()), + "Name mismatch at record {i}" + ); + } +} + +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/opensearch/source.toml")), + seed = seeds::connector_stream +)] +async fn opensearch_source_handles_empty_index( + harness: &TestHarness, + fixture: OpenSearchSourcePreCreatedFixture, +) { + let client = harness.root_client().await.unwrap(); + + let doc_count = fixture + .get_document_count() + .await + .expect("Failed to get document count"); + assert_eq!(doc_count, 0, "Expected empty index"); + + let stream_id: Identifier = seeds::names::STREAM.try_into().unwrap(); + let topic_id: Identifier = seeds::names::TOPIC.try_into().unwrap(); + let consumer_id: Identifier = "test_consumer".try_into().unwrap(); + + sleep(Duration::from_millis(100)).await; + + let polled = client + .poll_messages( + &stream_id, + &topic_id, + None, + &Consumer::new(consumer_id), + &PollingStrategy::next(), + 10, + false, + ) + .await; + + assert!( + polled.is_ok(), + "Should be able to poll from topic even with empty source" + ); +} + +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/opensearch/source.toml")), + seed = seeds::connector_stream +)] +async fn opensearch_source_produces_bulk_messages( + harness: &TestHarness, + fixture: OpenSearchSourcePreCreatedFixture, +) { + let client = harness.root_client().await.unwrap(); + let bulk_count = 10; + + fixture + .insert_documents(bulk_count) + .await + .expect("Failed to insert documents"); + + let stream_id: Identifier = seeds::names::STREAM.try_into().unwrap(); + let topic_id: Identifier = seeds::names::TOPIC.try_into().unwrap(); + let consumer_id: Identifier = "test_consumer".try_into().unwrap(); + + let mut received: Vec = Vec::new(); + for _ in 0..POLL_ATTEMPTS { + if let Ok(polled) = client + .poll_messages( + &stream_id, + &topic_id, + None, + &Consumer::new(consumer_id.clone()), + &PollingStrategy::next(), + 100, + true, + ) + .await + { + for msg in polled.messages { + if let Ok(json) = serde_json::from_slice(&msg.payload) { + received.push(json); + } + } + if received.len() >= bulk_count { + break; + } + } + sleep(Duration::from_millis(POLL_INTERVAL_MS)).await; + } + + assert!( + received.len() >= bulk_count, + "Expected at least {bulk_count} messages, got {}", + received.len() + ); +} + +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/opensearch/source.toml")), + seed = seeds::connector_stream +)] +async fn state_persists_across_connector_restart( + harness: &mut TestHarness, + fixture: OpenSearchSourcePreCreatedFixture, +) { + fixture + .insert_documents(TEST_MESSAGE_COUNT) + .await + .expect("Failed to insert first batch"); + + let stream_id: Identifier = seeds::names::STREAM.try_into().unwrap(); + let topic_id: Identifier = seeds::names::TOPIC.try_into().unwrap(); + let consumer_id: Identifier = "state_test_consumer".try_into().unwrap(); + + let client = harness.root_client().await.unwrap(); + let received_before = { + let mut received: Vec = Vec::new(); + for _ in 0..POLL_ATTEMPTS { + if let Ok(polled) = client + .poll_messages( + &stream_id, + &topic_id, + None, + &Consumer::new(consumer_id.clone()), + &PollingStrategy::next(), + 10, + true, + ) + .await + { + for msg in polled.messages { + if let Ok(json) = serde_json::from_slice(&msg.payload) { + received.push(json); + } + } + if received.len() >= TEST_MESSAGE_COUNT { + break; + } + } + sleep(Duration::from_millis(POLL_INTERVAL_MS)).await; + } + received + }; + assert_eq!(received_before.len(), TEST_MESSAGE_COUNT); + + harness + .server_mut() + .stop_dependents() + .expect("Failed to stop connectors"); + + let second_batch_start_id = (TEST_MESSAGE_COUNT + 1) as i32; + for i in 0..TEST_MESSAGE_COUNT { + fixture + .insert_document( + second_batch_start_id + i as i32, + &format!("doc_batch2_{i}"), + (TEST_MESSAGE_COUNT + i) as i32 * 10, + ) + .await + .expect("Failed to insert document"); + } + fixture + .refresh_index() + .await + .expect("Failed to refresh index"); + + harness + .server_mut() + .start_dependents() + .await + .expect("Failed to restart connectors"); + sleep(Duration::from_millis(100)).await; + + let mut received_after: Vec = Vec::new(); + for _ in 0..POLL_ATTEMPTS { + if let Ok(polled) = client + .poll_messages( + &stream_id, + &topic_id, + None, + &Consumer::new(consumer_id.clone()), + &PollingStrategy::next(), + 10, + true, + ) + .await + { + for msg in polled.messages { + if let Ok(json) = serde_json::from_slice(&msg.payload) { + received_after.push(json); + } + } + if received_after.len() >= TEST_MESSAGE_COUNT { + break; + } + } + sleep(Duration::from_millis(POLL_INTERVAL_MS)).await; + } + + assert_eq!(received_after.len(), TEST_MESSAGE_COUNT); + + for record in &received_after { + let id = record.get("id").and_then(|v| v.as_i64()).unwrap_or(0); + assert!( + id > TEST_MESSAGE_COUNT as i64, + "After restart, got ID {id} from first batch" + ); + } +} + +async fn fetch_sources(http_client: &Client, api_address: &str) -> Vec { + let response = http_client + .get(format!("{api_address}/sources")) + .send() + .await + .expect("Failed to query /sources"); + assert_eq!(response.status(), 200); + response.json().await.expect("Failed to parse sources") +} + +/// Negative path: the configured index does not exist, so `open()` must +/// fail with a `Storage` error and the runtime reports the source as +/// `ConnectorStatus::Error` without aborting. +#[iggy_harness( + server(connectors_runtime(config_path = "tests/connectors/opensearch/source.toml")), + seed = seeds::connector_stream +)] +async fn opensearch_source_with_missing_index_reports_error( + harness: &TestHarness, + _fixture: OpenSearchSourceMissingIndexFixture, +) { + let api_address = harness + .connectors_runtime() + .expect("connector runtime should be available") + .http_url(); + let http_client = Client::new(); + + let mut sources = fetch_sources(&http_client, &api_address).await; + for _ in 0..POLL_ATTEMPTS { + if sources + .iter() + .any(|source| source.status == ConnectorStatus::Error) + { + break; + } + sleep(Duration::from_millis(POLL_INTERVAL_MS)).await; + sources = fetch_sources(&http_client, &api_address).await; + } + + assert_eq!(sources.len(), 1, "Expected a single configured source"); + let source = &sources[0]; + assert_eq!(source.status, ConnectorStatus::Error); + let last_error = source + .last_error + .as_ref() + .expect("Source with missing index should expose a last_error"); + assert!( + last_error.message.contains("does not exist"), + "last_error should mention the missing index, got: {}", + last_error.message + ); +} diff --git a/core/integration/tests/connectors/opensearch/source.toml b/core/integration/tests/connectors/opensearch/source.toml new file mode 100644 index 0000000000..f0baa94e43 --- /dev/null +++ b/core/integration/tests/connectors/opensearch/source.toml @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[connectors] +config_type = "local" +config_dir = "../connectors/sources/opensearch_source" From a1d0c4633c133e1166f0ce9db69b93c98957cf4b Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Thu, 4 Jun 2026 22:38:56 -0400 Subject: [PATCH 04/57] feat(gateways): add Kafka wire protocol listener foundation TCP listener on 9093 with scoped API decode/validation and stub responses for apache/iggy#3421. Includes kafka-message-gen test tool. Co-authored-by: Cursor --- .gitignore | 1 + Cargo.lock | 66 ++ Cargo.toml | 2 + gateways/README.md | 9 + gateways/kafka/Cargo.toml | 49 ++ gateways/kafka/README.md | 35 + gateways/kafka/docs/SCOPE.md | 24 + gateways/kafka/src/error.rs | 41 + gateways/kafka/src/lib.rs | 33 + gateways/kafka/src/main.rs | 45 + gateways/kafka/src/protocol/api.rs | 253 ++++++ gateways/kafka/src/protocol/codec.rs | 264 ++++++ gateways/kafka/src/protocol/header.rs | 191 +++++ gateways/kafka/src/protocol/mod.rs | 22 + gateways/kafka/src/protocol/requests.rs | 450 ++++++++++ gateways/kafka/src/protocol/responses.rs | 274 +++++++ gateways/kafka/src/server.rs | 207 +++++ gateways/kafka/tests/api_handler_tests.rs | 144 ++++ gateways/kafka/tests/codec_tests.rs | 149 ++++ .../kafka/tests/decode_validation_tests.rs | 449 ++++++++++ .../kafka/tests/golden_wire_fixtures_tests.rs | 75 ++ gateways/kafka/tests/header_tests.rs | 139 ++++ .../kafka/tests/server_integration_tests.rs | 110 +++ gateways/kafka/tools/kafka-tool/Cargo.toml | 43 + gateways/kafka/tools/kafka-tool/README.md | 249 ++++++ gateways/kafka/tools/kafka-tool/src/main.rs | 770 ++++++++++++++++++ 26 files changed, 4094 insertions(+) create mode 100644 gateways/README.md create mode 100644 gateways/kafka/Cargo.toml create mode 100644 gateways/kafka/README.md create mode 100644 gateways/kafka/docs/SCOPE.md create mode 100644 gateways/kafka/src/error.rs create mode 100644 gateways/kafka/src/lib.rs create mode 100644 gateways/kafka/src/main.rs create mode 100644 gateways/kafka/src/protocol/api.rs create mode 100644 gateways/kafka/src/protocol/codec.rs create mode 100644 gateways/kafka/src/protocol/header.rs create mode 100644 gateways/kafka/src/protocol/mod.rs create mode 100644 gateways/kafka/src/protocol/requests.rs create mode 100644 gateways/kafka/src/protocol/responses.rs create mode 100644 gateways/kafka/src/server.rs create mode 100644 gateways/kafka/tests/api_handler_tests.rs create mode 100644 gateways/kafka/tests/codec_tests.rs create mode 100644 gateways/kafka/tests/decode_validation_tests.rs create mode 100644 gateways/kafka/tests/golden_wire_fixtures_tests.rs create mode 100644 gateways/kafka/tests/header_tests.rs create mode 100644 gateways/kafka/tests/server_integration_tests.rs create mode 100644 gateways/kafka/tools/kafka-tool/Cargo.toml create mode 100644 gateways/kafka/tools/kafka-tool/README.md create mode 100644 gateways/kafka/tools/kafka-tool/src/main.rs diff --git a/.gitignore b/.gitignore index f0d8b6250b..8c9ec60fc6 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,4 @@ go.work core/bench/dashboard/frontend/dist LICENSE-binary **/LICENSE-binary +gateways/kafka/tools/kafka-tool/kafka_messages/ diff --git a/Cargo.lock b/Cargo.lock index cf28d8b361..f33cf2a42c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7173,6 +7173,17 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "iggy_gateway_kafka" +version = "0.1.0" +dependencies = [ + "bytes", + "thiserror 2.0.18", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "ignore" version = "0.4.26" @@ -7661,6 +7672,42 @@ dependencies = [ "rayon", ] +[[package]] +name = "kafka-message-gen" +version = "0.1.0" +dependencies = [ + "anyhow", + "bytes", + "clap", + "hex", + "indexmap 2.14.0", + "kafka-protocol", + "serde", + "serde_json", + "tokio", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "kafka-protocol" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66292444a1cd4d430d450d472c30cba839d0724229aba2d79affffcf901516e2" +dependencies = [ + "anyhow", + "bytes", + "crc", + "crc32c", + "flate2", + "indexmap 2.14.0", + "lz4", + "paste", + "snap", + "uuid", + "zstd", +] + [[package]] name = "keccak" version = "0.2.0" @@ -8112,6 +8159,25 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "lz4" +version = "1.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a20b523e860d03443e98350ceaac5e71c6ba89aea7d960769ec3ce37f4de5af4" +dependencies = [ + "lz4-sys", +] + +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "lz4_flex" version = "0.12.2" diff --git a/Cargo.toml b/Cargo.toml index 027209cea0..8d002447ca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -63,6 +63,8 @@ members = [ "core/simulator", "core/tools", "examples/rust", + "gateways/kafka", + "gateways/kafka/tools/kafka-tool", ] exclude = ["foreign/cpp", "foreign/php", "foreign/python"] resolver = "3" diff --git a/gateways/README.md b/gateways/README.md new file mode 100644 index 0000000000..7b0c550f5e --- /dev/null +++ b/gateways/README.md @@ -0,0 +1,9 @@ +# Apache Iggy Gateways + +Protocol gateways that let existing clients talk to Iggy without changing the core server wire surface. + +| Gateway | Issue | Description | +|---------|-------|-------------| +| [kafka](kafka/) | [#3421](https://github.com/apache/iggy/issues/3421) | Kafka wire protocol TCP listener (port 9093) | + +Each gateway is a separate workspace crate under `gateways//`. diff --git a/gateways/kafka/Cargo.toml b/gateways/kafka/Cargo.toml new file mode 100644 index 0000000000..6bfcb0e34d --- /dev/null +++ b/gateways/kafka/Cargo.toml @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "iggy_gateway_kafka" +version = "0.1.0" +description = "Kafka wire protocol gateway foundation for Apache Iggy" +edition = "2024" +license = "Apache-2.0" +keywords = ["iggy", "kafka", "gateway", "streaming"] +homepage = "https://iggy.apache.org" +documentation = "https://iggy.apache.org/docs" +repository = "https://github.com/apache/iggy" +readme = "README.md" +publish = false + +[[bin]] +name = "iggy-kafka-gateway" +path = "src/main.rs" + +[dependencies] +bytes = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "io-util", "time", "sync", "signal"] } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "io-util", "time"] } + +[lints.clippy] +enum_glob_use = "deny" +# Ported Kafka wire codec; pedantic cleanup tracked for a follow-up PR. +pedantic = "warn" +nursery = "warn" diff --git a/gateways/kafka/README.md b/gateways/kafka/README.md new file mode 100644 index 0000000000..b575a0c803 --- /dev/null +++ b/gateways/kafka/README.md @@ -0,0 +1,35 @@ +# Kafka gateway (`iggy_gateway_kafka`) + +Foundation layer for [apache/iggy#3421](https://github.com/apache/iggy/issues/3421): a TCP listener on the Kafka wire port that decodes requests, validates scoped API keys and versions, and returns stub responses. + +## Run + +```bash +cargo run -p iggy_gateway_kafka --bin iggy-kafka-gateway +``` + +Default bind: `127.0.0.1:9093`. + +## Test + +```bash +cargo test -p iggy_gateway_kafka +``` + +`decode_validation_tests` require wire fixtures under `tools/kafka-tool/kafka_messages/`: + +```bash +cargo run -p kafka-message-gen -- generate \ + --output gateways/kafka/tools/kafka-tool/kafka_messages \ + --api-key 0 --api-key 1 --api-key 2 --api-key 19 +``` + +(Run from workspace root; adjust paths if needed.) + +## Scoped APIs + +See [docs/SCOPE.md](docs/SCOPE.md). + +## Wire fixture tool + +See [tools/kafka-tool/README.md](tools/kafka-tool/README.md). diff --git a/gateways/kafka/docs/SCOPE.md b/gateways/kafka/docs/SCOPE.md new file mode 100644 index 0000000000..b56a172730 --- /dev/null +++ b/gateways/kafka/docs/SCOPE.md @@ -0,0 +1,24 @@ +# Kafka API scope — issue #3421 foundation + +This gateway iteration implements **wire validation and stub responses only** (no Iggy backend, no real broker semantics). + +## Supported API keys and versions + +| API key | Name | Min version | Max version | Behavior | +|---------|------|-------------|-------------|----------| +| 18 | ApiVersions | 0 | 3 | Advertise supported ranges; flexible encoding at v3+ | +| 3 | Metadata | 0 | 9 | Decode request; stub broker `127.0.0.1:9093` | +| 0 | Produce | 3 | 9 | Decode request; stub response | +| 1 | Fetch | 4 | 12 | Decode request; stub response | +| 2 | ListOffsets | 1 | 6 | Decode request; stub response | +| 19 | CreateTopics | 2 | 5 | Decode request; stub response | + +## Unsupported API keys + +All other API keys receive an error-only response with `UNSUPPORTED_VERSION` (35). + +## Out of scope (later issues) + +- `IggyBridge` / produce-fetch against Iggy streams +- Consumer group APIs (8–14), SASL (17, 36), transactions +- Accurate metadata topology and record batch semantics diff --git a/gateways/kafka/src/error.rs b/gateways/kafka/src/error.rs new file mode 100644 index 0000000000..668fc5c442 --- /dev/null +++ b/gateways/kafka/src/error.rs @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum KafkaProtocolError { + #[error("buffer underflow: needed {needed} bytes, remaining {remaining}")] + BufferUnderflow { needed: usize, remaining: usize }, + #[error("invalid frame length: {0}")] + InvalidFrameLength(i32), + #[error("request exceeds max frame size ({max_bytes} bytes): {actual_bytes} bytes")] + FrameTooLarge { + max_bytes: usize, + actual_bytes: usize, + }, + #[error("invalid utf8 string")] + InvalidUtf8, + #[error("varint overflows 64 bits")] + InvalidVarint, + #[error("unsupported request header version: {0}")] + UnsupportedHeaderVersion(i16), + #[error("io error: {0}")] + Io(#[from] std::io::Error), +} + +pub type Result = std::result::Result; diff --git a/gateways/kafka/src/lib.rs b/gateways/kafka/src/lib.rs new file mode 100644 index 0000000000..f0828508c7 --- /dev/null +++ b/gateways/kafka/src/lib.rs @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Kafka wire protocol gateway foundation for Apache Iggy. + +// Ported wire codec from spike; pedantic clippy cleanup is a follow-up. +#![allow( + clippy::pedantic, + clippy::missing_const_for_fn, + clippy::wildcard_imports, + clippy::match_same_arms, + clippy::needless_pass_by_value +)] + +pub mod error; +pub mod protocol; +pub mod server; + +pub use server::{KafkaServer, ServerConfig, init_tracing}; diff --git a/gateways/kafka/src/main.rs b/gateways/kafka/src/main.rs new file mode 100644 index 0000000000..4a5414b9dd --- /dev/null +++ b/gateways/kafka/src/main.rs @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use tokio::signal; +use tokio::sync::broadcast; + +use iggy_gateway_kafka::server::init_tracing; +use iggy_gateway_kafka::{KafkaServer, ServerConfig}; + +#[tokio::main] +async fn main() -> Result<(), Box> { + init_tracing(); + + let config = ServerConfig::default(); + let server = KafkaServer::new(config); + + let (tx, rx) = broadcast::channel(1); + let mut server_task = tokio::spawn(async move { server.run(rx).await }); + + tokio::select! { + result = &mut server_task => { + return Ok(result??); + } + _ = signal::ctrl_c() => { + let _ = tx.send(()); + } + } + + server_task.await??; + Ok(()) +} diff --git a/gateways/kafka/src/protocol/api.rs b/gateways/kafka/src/protocol/api.rs new file mode 100644 index 0000000000..8f0367b481 --- /dev/null +++ b/gateways/kafka/src/protocol/api.rs @@ -0,0 +1,253 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use bytes::Bytes; + +use crate::protocol::codec::{Decoder, Encoder}; +use crate::protocol::requests::{ + decode_create_topics_request, decode_fetch_request, decode_list_offsets_request, + decode_produce_request, +}; +use crate::protocol::responses::{ + encode_create_topics_response, encode_fetch_response, encode_list_offsets_response, + encode_produce_response, +}; + +pub const API_KEY_PRODUCE: i16 = 0; +pub const API_KEY_FETCH: i16 = 1; +pub const API_KEY_LIST_OFFSETS: i16 = 2; +pub const API_KEY_METADATA: i16 = 3; +pub const API_KEY_OFFSET_COMMIT: i16 = 8; +pub const API_KEY_OFFSET_FETCH: i16 = 9; +pub const API_KEY_FIND_COORDINATOR: i16 = 10; +pub const API_KEY_JOIN_GROUP: i16 = 11; +pub const API_KEY_HEARTBEAT: i16 = 12; +pub const API_KEY_LEAVE_GROUP: i16 = 13; +pub const API_KEY_SYNC_GROUP: i16 = 14; +pub const API_KEY_DESCRIBE_GROUPS: i16 = 15; +pub const API_KEY_LIST_GROUPS: i16 = 16; +pub const API_KEY_SASL_HANDSHAKE: i16 = 17; +pub const API_KEY_API_VERSIONS: i16 = 18; +pub const API_KEY_CREATE_TOPICS: i16 = 19; +pub const API_KEY_DELETE_TOPICS: i16 = 20; + +pub const ERROR_NONE: i16 = 0; +pub const ERROR_OFFSET_OUT_OF_RANGE: i16 = 1; +pub const ERROR_CORRUPT_MESSAGE: i16 = 2; +pub const ERROR_UNKNOWN_TOPIC_OR_PARTITION: i16 = 3; +pub const ERROR_INVALID_FETCH_SIZE: i16 = 4; +pub const ERROR_LEADER_NOT_AVAILABLE: i16 = 5; +pub const ERROR_NOT_LEADER_OR_FOLLOWER: i16 = 6; +pub const ERROR_REQUEST_TIMED_OUT: i16 = 7; +pub const ERROR_UNKNOWN_SERVER_ERROR: i16 = -1; +pub const ERROR_UNSUPPORTED_VERSION: i16 = 35; +pub const ERROR_TOPIC_ALREADY_EXISTS: i16 = 36; +pub const ERROR_INVALID_PARTITIONS: i16 = 37; +pub const ERROR_INVALID_REPLICATION_FACTOR: i16 = 38; +pub const ERROR_INVALID_REQUEST: i16 = 42; +pub const ERROR_UNSUPPORTED_FOR_MESSAGE_FORMAT: i16 = 43; + +#[derive(Debug, Clone, Copy)] +pub struct ApiVersionRange { + pub api_key: i16, + pub min_version: i16, + pub max_version: i16, +} + +pub fn supported_api_ranges() -> Vec { + vec![ + ApiVersionRange { + api_key: API_KEY_PRODUCE, + min_version: 3, + max_version: 9, + }, + ApiVersionRange { + api_key: API_KEY_FETCH, + min_version: 4, + max_version: 12, + }, + ApiVersionRange { + api_key: API_KEY_LIST_OFFSETS, + min_version: 1, + max_version: 6, + }, + ApiVersionRange { + api_key: API_KEY_METADATA, + min_version: 0, + max_version: 9, + }, + ApiVersionRange { + api_key: API_KEY_API_VERSIONS, + min_version: 0, + max_version: 3, + }, + ApiVersionRange { + api_key: API_KEY_CREATE_TOPICS, + min_version: 2, + max_version: 5, + }, + ] +} + +pub fn handle_request(api_key: i16, api_version: i16, body: Bytes) -> Bytes { + match api_key { + API_KEY_API_VERSIONS => { + if is_supported_version(api_key, api_version) { + encode_api_versions_response(api_version, ERROR_NONE) + } else { + encode_api_versions_response(1, ERROR_UNSUPPORTED_VERSION) + } + } + API_KEY_METADATA => { + if is_supported_version(api_key, api_version) { + encode_metadata_response(api_version, body, ERROR_NONE) + } else { + encode_metadata_response(0, body, ERROR_UNSUPPORTED_VERSION) + } + } + API_KEY_PRODUCE => { + if is_supported_version(api_key, api_version) { + match decode_produce_request(api_version, body) { + Ok(req) => encode_produce_response(api_version, req), + Err(e) => { + tracing::error!("Failed to decode Produce request: {:?}", e); + encode_error_only_response(ERROR_CORRUPT_MESSAGE) + } + } + } else { + encode_error_only_response(ERROR_UNSUPPORTED_VERSION) + } + } + API_KEY_FETCH => { + if is_supported_version(api_key, api_version) { + match decode_fetch_request(api_version, body) { + Ok(req) => encode_fetch_response(api_version, req), + Err(e) => { + tracing::error!("Failed to decode Fetch request: {:?}", e); + encode_error_only_response(ERROR_CORRUPT_MESSAGE) + } + } + } else { + encode_error_only_response(ERROR_UNSUPPORTED_VERSION) + } + } + API_KEY_LIST_OFFSETS => { + if is_supported_version(api_key, api_version) { + match decode_list_offsets_request(api_version, body) { + Ok(req) => encode_list_offsets_response(api_version, req), + Err(e) => { + tracing::error!("Failed to decode ListOffsets request: {:?}", e); + encode_error_only_response(ERROR_CORRUPT_MESSAGE) + } + } + } else { + encode_error_only_response(ERROR_UNSUPPORTED_VERSION) + } + } + API_KEY_CREATE_TOPICS => { + if is_supported_version(api_key, api_version) { + match decode_create_topics_request(api_version, body) { + Ok(req) => encode_create_topics_response(api_version, req), + Err(e) => { + tracing::error!("Failed to decode CreateTopics request: {:?}", e); + encode_error_only_response(ERROR_CORRUPT_MESSAGE) + } + } + } else { + encode_error_only_response(ERROR_UNSUPPORTED_VERSION) + } + } + _ => encode_error_only_response(ERROR_UNSUPPORTED_VERSION), + } +} + +pub fn is_supported_version(api_key: i16, api_version: i16) -> bool { + supported_api_ranges() + .into_iter() + .find(|r| r.api_key == api_key) + .is_some_and(|r| api_version >= r.min_version && api_version <= r.max_version) +} + +fn encode_api_versions_response(api_version: i16, error_code: i16) -> Bytes { + let flexible = api_version >= 3; + let ranges = supported_api_ranges(); + let mut e = Encoder::with_capacity(128); + + e.write_i16(error_code); + + if flexible { + e.write_varint((ranges.len() + 1) as u64); + for r in &ranges { + e.write_i16(r.api_key); + e.write_i16(r.min_version); + e.write_i16(r.max_version); + e.write_empty_tagged_fields(); + } + } else { + e.write_i32(ranges.len() as i32); + for r in &ranges { + e.write_i16(r.api_key); + e.write_i16(r.min_version); + e.write_i16(r.max_version); + } + } + + if api_version >= 1 { + e.write_i32(0); + } + + if flexible { + e.write_empty_tagged_fields(); + } + + e.freeze() +} + +fn encode_metadata_response(_api_version: i16, body: Bytes, top_level_error_code: i16) -> Bytes { + let mut e = Encoder::with_capacity(256); + + e.write_i32(1); + e.write_i32(1); + e.write_nullable_string(Some("127.0.0.1")); + e.write_i32(9093); + + let topics_count = split_metadata_request_topics(body); + e.write_i32(topics_count as i32); + for _ in 0..topics_count { + e.write_i16(if top_level_error_code == ERROR_NONE { + ERROR_UNKNOWN_TOPIC_OR_PARTITION + } else { + top_level_error_code + }); + e.write_nullable_string(Some("unknown-topic")); + e.write_i32(0); + } + + e.write_i32(1); + e.freeze() +} + +fn encode_error_only_response(error_code: i16) -> Bytes { + let mut e = Encoder::with_capacity(2); + e.write_i16(error_code); + e.freeze() +} + +pub fn split_metadata_request_topics(body: Bytes) -> usize { + let mut d = Decoder::new(body); + d.read_i32().unwrap_or_default().max(0) as usize +} diff --git a/gateways/kafka/src/protocol/codec.rs b/gateways/kafka/src/protocol/codec.rs new file mode 100644 index 0000000000..1e356b8608 --- /dev/null +++ b/gateways/kafka/src/protocol/codec.rs @@ -0,0 +1,264 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use bytes::{Buf, BufMut, Bytes, BytesMut}; + +use crate::error::{KafkaProtocolError, Result}; + +pub struct Decoder { + bytes: Bytes, +} + +impl Decoder { + pub fn new(bytes: Bytes) -> Self { + Self { bytes } + } + + pub fn remaining(&self) -> usize { + self.bytes.remaining() + } + + pub fn read_u8(&mut self) -> Result { + self.ensure(1)?; + Ok(self.bytes.get_u8()) + } + + pub fn read_i8(&mut self) -> Result { + self.ensure(1)?; + Ok(self.bytes.get_i8()) + } + + pub fn read_i16(&mut self) -> Result { + self.ensure(2)?; + Ok(self.bytes.get_i16()) + } + + pub fn read_i32(&mut self) -> Result { + self.ensure(4)?; + Ok(self.bytes.get_i32()) + } + + pub fn read_i64(&mut self) -> Result { + self.ensure(8)?; + Ok(self.bytes.get_i64()) + } + + pub fn read_bool(&mut self) -> Result { + Ok(self.read_i8()? != 0) + } + + /// Unsigned varint (Kafka uses this for compact array lengths and tagged-field counts). + /// Value is encoded with 7 bits per byte, LSB first; the high bit of each byte signals + /// that more bytes follow. + pub fn read_varint(&mut self) -> Result { + let mut result: u64 = 0; + let mut shift = 0u32; + loop { + let byte = self.read_u8()?; + result |= ((byte & 0x7F) as u64) << shift; + if byte & 0x80 == 0 { + return Ok(result); + } + shift += 7; + if shift >= 64 { + return Err(KafkaProtocolError::InvalidVarint); + } + } + } + + /// Legacy nullable string: i16 length prefix (-1 = null). + pub fn read_nullable_string(&mut self) -> Result> { + let len = self.read_i16()?; + if len < 0 { + return Ok(None); + } + let len = len as usize; + self.ensure(len)?; + let chunk = self.bytes.copy_to_bytes(len); + String::from_utf8(chunk.to_vec()) + .map(Some) + .map_err(|_| KafkaProtocolError::InvalidUtf8) + } + + /// Compact nullable string (flexible versions): varint(len+1) prefix, 0 = null. + pub fn read_compact_nullable_string(&mut self) -> Result> { + let len_plus_one = self.read_varint()?; + if len_plus_one == 0 { + return Ok(None); + } + let len = (len_plus_one - 1) as usize; + self.ensure(len)?; + let chunk = self.bytes.copy_to_bytes(len); + String::from_utf8(chunk.to_vec()) + .map(Some) + .map_err(|_| KafkaProtocolError::InvalidUtf8) + } + + /// Legacy nullable bytes: i32 length prefix (-1 = null). + pub fn read_nullable_bytes(&mut self) -> Result> { + let len = self.read_i32()?; + if len < 0 { + return Ok(None); + } + let len = len as usize; + self.ensure(len)?; + Ok(Some(self.bytes.copy_to_bytes(len))) + } + + /// Compact nullable bytes (flexible versions): varint(len+1) prefix, 0 = null. + pub fn read_compact_nullable_bytes(&mut self) -> Result> { + let len_plus_one = self.read_varint()?; + if len_plus_one == 0 { + return Ok(None); + } + let len = (len_plus_one - 1) as usize; + self.ensure(len)?; + Ok(Some(self.bytes.copy_to_bytes(len))) + } + + pub fn read_bytes(&mut self, len: usize) -> Result { + self.ensure(len)?; + Ok(self.bytes.copy_to_bytes(len)) + } + + /// Skip over a tagged-fields section. Each field is: tag (varint) + size (varint) + bytes. + /// A count of 0 is the common case (single byte 0x00). + pub fn read_tagged_fields(&mut self) -> Result<()> { + let count = self.read_varint()? as usize; + for _ in 0..count { + self.read_varint()?; // tag number + let size = self.read_varint()? as usize; + self.ensure(size)?; + self.bytes.advance(size); + } + Ok(()) + } + + fn ensure(&self, needed: usize) -> Result<()> { + let remaining = self.bytes.remaining(); + if remaining < needed { + return Err(KafkaProtocolError::BufferUnderflow { needed, remaining }); + } + Ok(()) + } +} + +pub struct Encoder { + bytes: BytesMut, +} + +impl Encoder { + pub fn with_capacity(capacity: usize) -> Self { + Self { + bytes: BytesMut::with_capacity(capacity), + } + } + + pub fn write_u8(&mut self, v: u8) { + self.bytes.put_u8(v); + } + + pub fn write_i8(&mut self, v: i8) { + self.bytes.put_i8(v); + } + + pub fn write_i16(&mut self, v: i16) { + self.bytes.put_i16(v); + } + + pub fn write_i32(&mut self, v: i32) { + self.bytes.put_i32(v); + } + + pub fn write_i64(&mut self, v: i64) { + self.bytes.put_i64(v); + } + + pub fn write_bool(&mut self, v: bool) { + self.write_i8(if v { 1 } else { 0 }); + } + + /// Unsigned varint, 7 bits per byte, LSB first. + pub fn write_varint(&mut self, mut v: u64) { + loop { + let byte = (v & 0x7F) as u8; + v >>= 7; + if v == 0 { + self.bytes.put_u8(byte); + return; + } + self.bytes.put_u8(byte | 0x80); + } + } + + /// Legacy nullable string: i16 length prefix, -1 for null. + pub fn write_nullable_string(&mut self, v: Option<&str>) { + match v { + None => self.write_i16(-1), + Some(s) => { + self.write_i16(s.len() as i16); + self.bytes.put_slice(s.as_bytes()); + } + } + } + + /// Compact nullable string (flexible versions): varint(len+1), 0 for null. + pub fn write_compact_nullable_string(&mut self, v: Option<&str>) { + match v { + None => self.write_varint(0), + Some(s) => { + self.write_varint((s.len() + 1) as u64); + self.bytes.put_slice(s.as_bytes()); + } + } + } + + /// Legacy nullable bytes: i32 length prefix, -1 for null. + pub fn write_nullable_bytes(&mut self, v: Option<&[u8]>) { + match v { + None => self.write_i32(-1), + Some(b) => { + self.write_i32(b.len() as i32); + self.bytes.put_slice(b); + } + } + } + + /// Compact nullable bytes (flexible versions): varint(len+1), 0 for null. + pub fn write_compact_nullable_bytes(&mut self, v: Option<&[u8]>) { + match v { + None => self.write_varint(0), + Some(b) => { + self.write_varint((b.len() + 1) as u64); + self.bytes.put_slice(b); + } + } + } + + pub fn write_bytes(&mut self, b: &[u8]) { + self.bytes.put_slice(b); + } + + /// Write an empty tagged-fields section (single 0x00 byte). + pub fn write_empty_tagged_fields(&mut self) { + self.write_varint(0); + } + + pub fn freeze(self) -> Bytes { + self.bytes.freeze() + } +} diff --git a/gateways/kafka/src/protocol/header.rs b/gateways/kafka/src/protocol/header.rs new file mode 100644 index 0000000000..fb137b81e0 --- /dev/null +++ b/gateways/kafka/src/protocol/header.rs @@ -0,0 +1,191 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use bytes::Bytes; + +use crate::error::{KafkaProtocolError, Result}; +use crate::protocol::codec::{Decoder, Encoder}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RequestHeader { + pub api_key: i16, + pub api_version: i16, + pub correlation_id: i32, + pub client_id: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ResponseHeader { + pub correlation_id: i32, +} + +/// Returns the request header version to use for a given (api_key, api_version) pair. +/// +/// Header v1 is the standard non-flexible format (nullable string client_id). +/// Header v2 is the flexible format (compact nullable string client_id + empty tagged fields). +/// The threshold at which each API key switches from v1 to v2 is defined by the Kafka protocol. +pub fn request_header_version(api_key: i16, api_version: i16) -> i16 { + let flexible_from: i16 = match api_key { + 0 => 9, // Produce + 1 => 12, // Fetch + 2 => 6, // ListOffsets + 3 => 9, // Metadata + 4 => 4, // LeaderAndIsr + 5 => 2, // StopReplica + 6 => 6, // UpdateMetadata + 7 => 3, // ControlledShutdown + 8 => 8, // OffsetCommit + 9 => 6, // OffsetFetch + 10 => 3, // FindCoordinator + 11 => 6, // JoinGroup + 12 => 4, // Heartbeat + 13 => 4, // LeaveGroup + 14 => 4, // SyncGroup + 15 => 5, // DescribeGroups + 16 => 3, // ListGroups + 17 => i16::MAX, // SaslHandshake — never flexible + 18 => 3, // ApiVersions + 19 => 5, // CreateTopics + 20 => 4, // DeleteTopics + 21 => 2, // DeleteRecords + 22 => 2, // InitProducerId + 23 => 4, // OffsetForLeaderEpoch + 24 => 3, // AddPartitionsToTxn + 25 => 3, // AddOffsetsToTxn + 26 => 3, // EndTxn + 27 => 1, // WriteTxnMarkers + 28 => 3, // TxnOffsetCommit + 29 => 2, // DescribeAcls + 30 => 2, // CreateAcls + 31 => 2, // DeleteAcls + 32 => 4, // DescribeConfigs + 33 => 2, // AlterConfigs + 34 => 2, // AlterReplicaLogDirs + 35 => 2, // DescribeLogDirs + 36 => 2, // SaslAuthenticate + 37 => 2, // CreatePartitions + 38 => 2, // CreateDelegationToken + 39 => 2, // RenewDelegationToken + 40 => 2, // ExpireDelegationToken + 41 => 2, // DescribeDelegationToken + 42 => 2, // DeleteGroups + 43 => 2, // ElectLeaders + 44 => 1, // IncrementalAlterConfigs + 45 => 0, // AlterPartitionReassignments — always flexible + 46 => 0, // ListPartitionReassignments — always flexible + 47 => i16::MAX, // OffsetDelete — never flexible + 48 => 1, // DescribeClientQuotas + 49 => 1, // AlterClientQuotas + 50 => 0, // DescribeUserScramCredentials — always flexible + 51 => 0, // AlterUserScramCredentials — always flexible + 55 => 0, // DescribeQuorum — always flexible + 56 => 0, // AlterPartition — always flexible + 57 => 1, // UpdateFeatures + 60 => 0, // DescribeCluster — always flexible + 61 => 0, // DescribeProducers — always flexible + 64 => 0, // UnregisterBroker — always flexible + 65 => 0, // DescribeTransactions — always flexible + 66 => 0, // ListTransactions — always flexible + 67 => 0, // AllocateProducerIds — always flexible + 68 => 0, // ConsumerGroupHeartbeat — always flexible + 69 => 0, // ConsumerGroupDescribe — always flexible + 71 => 0, // GetTelemetrySubscriptions — always flexible + 72 => 0, // PushTelemetry — always flexible + 74 => 0, // AssignReplicasToDirs — always flexible + 75 => 0, // DescribeTopicPartitions — always flexible + 76 => 0, // ListClientMetricsResources — always flexible + _ => i16::MAX, // Unknown API — assume non-flexible + }; + if api_version >= flexible_from { 2 } else { 1 } +} + +/// Returns the response header version to use when replying to a given (api_key, api_version). +/// +/// ApiVersions (18) is a special case: the server ALWAYS returns response header v0 (no tagged +/// fields) so that clients that don't yet know the server supports flexible encoding can still +/// parse the discovery response. All other flexible-version APIs use response header v1. +pub fn response_header_version(api_key: i16, api_version: i16) -> i16 { + if api_key == 18 { + return 0; + } + if request_header_version(api_key, api_version) >= 2 { + 1 + } else { + 0 + } +} + +impl RequestHeader { + pub fn decode(bytes: Bytes, header_version: i16) -> Result { + let mut d = Decoder::new(bytes); + Self::decode_from(&mut d, header_version) + } + + /// Decode from a shared `Decoder`. + /// + /// Header v1 (non-flexible): + /// api_key i16 | api_version i16 | correlation_id i32 | client_id NULLABLE_STRING + /// + /// Header v2 (flexible): + /// api_key i16 | api_version i16 | correlation_id i32 + /// | client_id COMPACT_NULLABLE_STRING | _tagged_fields UNSIGNED_VARINT + pub fn decode_from(d: &mut Decoder, header_version: i16) -> Result { + match header_version { + 1 => { + let api_key = d.read_i16()?; + let api_version = d.read_i16()?; + let correlation_id = d.read_i32()?; + let client_id = d.read_nullable_string()?; + Ok(Self { + api_key, + api_version, + correlation_id, + client_id, + }) + } + 2 => { + let api_key = d.read_i16()?; + let api_version = d.read_i16()?; + let correlation_id = d.read_i32()?; + let client_id = d.read_compact_nullable_string()?; + d.read_tagged_fields()?; + Ok(Self { + api_key, + api_version, + correlation_id, + client_id, + }) + } + v => Err(KafkaProtocolError::UnsupportedHeaderVersion(v)), + } + } +} + +impl ResponseHeader { + /// Encode the response header. + /// + /// v0: correlation_id i32 (non-flexible APIs and ApiVersions) + /// v1: correlation_id i32 + empty tagged fields (flexible APIs) + pub fn encode(&self, header_version: i16) -> Bytes { + let mut e = Encoder::with_capacity(5); + e.write_i32(self.correlation_id); + if header_version >= 1 { + e.write_empty_tagged_fields(); + } + e.freeze() + } +} diff --git a/gateways/kafka/src/protocol/mod.rs b/gateways/kafka/src/protocol/mod.rs new file mode 100644 index 0000000000..3fe1fb544f --- /dev/null +++ b/gateways/kafka/src/protocol/mod.rs @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod api; +pub mod codec; +pub mod header; +pub mod requests; +pub mod responses; diff --git a/gateways/kafka/src/protocol/requests.rs b/gateways/kafka/src/protocol/requests.rs new file mode 100644 index 0000000000..786fa0b88c --- /dev/null +++ b/gateways/kafka/src/protocol/requests.rs @@ -0,0 +1,450 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Kafka request decoders for critical API keys + +use crate::error::Result; +use crate::protocol::codec::Decoder; +use bytes::Bytes; + +/// Produce Request (API Key 0) +#[derive(Debug, Clone)] +pub struct ProduceRequest { + pub transactional_id: Option, + pub acks: i16, + pub timeout_ms: i32, + pub topics: Vec, +} + +#[derive(Debug, Clone)] +pub struct ProduceTopicData { + pub topic: String, + pub partitions: Vec, +} + +#[derive(Debug, Clone)] +pub struct ProducePartitionData { + pub partition: i32, + pub records: Option, // Raw RecordBatch bytes +} + +pub fn decode_produce_request(version: i16, body: Bytes) -> Result { + let mut d = Decoder::new(body); + let flexible = version >= 9; + + // transactional_id (v3+) + let transactional_id = if version >= 3 { + if flexible { + d.read_compact_nullable_string()? + } else { + d.read_nullable_string()? + } + } else { + None + }; + + let acks = d.read_i16()?; + let timeout_ms = d.read_i32()?; + + // topics array + let topics_count = if flexible { + (d.read_varint()? - 1) as usize + } else { + d.read_i32()? as usize + }; + + let mut topics = Vec::with_capacity(topics_count); + for _ in 0..topics_count { + let topic = if flexible { + d.read_compact_nullable_string()?.unwrap_or_default() + } else { + d.read_nullable_string()?.unwrap_or_default() + }; + + let partitions_count = if flexible { + (d.read_varint()? - 1) as usize + } else { + d.read_i32()? as usize + }; + + let mut partitions = Vec::with_capacity(partitions_count); + for _ in 0..partitions_count { + let partition = d.read_i32()?; + let records = if flexible { + d.read_compact_nullable_bytes()? + } else { + d.read_nullable_bytes()? + }; + partitions.push(ProducePartitionData { partition, records }); + if flexible { + d.read_tagged_fields()?; + } + } + + topics.push(ProduceTopicData { topic, partitions }); + if flexible { + d.read_tagged_fields()?; + } + } + + if flexible { + d.read_tagged_fields()?; + } + + Ok(ProduceRequest { + transactional_id, + acks, + timeout_ms, + topics, + }) +} + +/// Fetch Request (API Key 1) +#[derive(Debug, Clone)] +pub struct FetchRequest { + pub max_wait_ms: i32, + pub min_bytes: i32, + pub max_bytes: i32, + pub isolation_level: i8, + pub topics: Vec, +} + +#[derive(Debug, Clone)] +pub struct FetchTopic { + pub topic: String, + pub partitions: Vec, +} + +#[derive(Debug, Clone)] +pub struct FetchPartition { + pub partition: i32, + pub fetch_offset: i64, + pub partition_max_bytes: i32, +} + +pub fn decode_fetch_request(version: i16, body: Bytes) -> Result { + let mut d = Decoder::new(body); + let flexible = version >= 12; + + let _replica_id = d.read_i32()?; + let max_wait_ms = d.read_i32()?; + let min_bytes = d.read_i32()?; + + let max_bytes = if version >= 3 { + d.read_i32()? + } else { + 52_428_800 // default 50MB + }; + + let isolation_level = if version >= 4 { d.read_i8()? } else { 0 }; + + // session_id and session_epoch (v7+) — we'll skip for now + if version >= 7 { + d.read_i32()?; // session_id + d.read_i32()?; // session_epoch + } + + // topics array + let topics_count = if flexible { + (d.read_varint()? - 1) as usize + } else { + d.read_i32()? as usize + }; + + let mut topics = Vec::with_capacity(topics_count); + for _ in 0..topics_count { + let topic = if flexible { + d.read_compact_nullable_string()?.unwrap_or_default() + } else { + d.read_nullable_string()?.unwrap_or_default() + }; + + let partitions_count = if flexible { + (d.read_varint()? - 1) as usize + } else { + d.read_i32()? as usize + }; + + let mut partitions = Vec::with_capacity(partitions_count); + for _ in 0..partitions_count { + let partition = d.read_i32()?; + + if version >= 9 { + d.read_i32()?; // current_leader_epoch + } + + let fetch_offset = d.read_i64()?; + + if version >= 12 { + d.read_i32()?; // last_fetched_epoch + } + + if version >= 5 { + d.read_i64()?; // log_start_offset + } + + let partition_max_bytes = d.read_i32()?; + + partitions.push(FetchPartition { + partition, + fetch_offset, + partition_max_bytes, + }); + + if flexible { + d.read_tagged_fields()?; + } + } + + topics.push(FetchTopic { topic, partitions }); + if flexible { + d.read_tagged_fields()?; + } + } + + // forgotten_topics_data (v7+) — skip + if version >= 7 { + let forgotten_count = if flexible { + (d.read_varint()? - 1) as usize + } else { + d.read_i32()? as usize + }; + for _ in 0..forgotten_count { + if flexible { + d.read_compact_nullable_string()?; + let partitions_count = (d.read_varint()? - 1) as usize; + for _ in 0..partitions_count { + d.read_i32()?; + } + d.read_tagged_fields()?; + } else { + d.read_nullable_string()?; + let partitions_count = d.read_i32()? as usize; + for _ in 0..partitions_count { + d.read_i32()?; + } + } + } + } + + // rack_id (v11+) + if version >= 11 { + if flexible { + d.read_compact_nullable_string()?; + } else { + d.read_nullable_string()?; + } + } + + if flexible { + d.read_tagged_fields()?; + } + + Ok(FetchRequest { + max_wait_ms, + min_bytes, + max_bytes, + isolation_level, + topics, + }) +} + +/// ListOffsets Request (API Key 2) +#[derive(Debug, Clone)] +pub struct ListOffsetsRequest { + pub isolation_level: i8, + pub topics: Vec, +} + +#[derive(Debug, Clone)] +pub struct ListOffsetsTopic { + pub topic: String, + pub partitions: Vec, +} + +#[derive(Debug, Clone)] +pub struct ListOffsetsPartition { + pub partition: i32, + pub timestamp: i64, // -2 = earliest, -1 = latest +} + +pub fn decode_list_offsets_request(version: i16, body: Bytes) -> Result { + let mut d = Decoder::new(body); + let flexible = version >= 6; + + let _replica_id = d.read_i32()?; + + let isolation_level = if version >= 2 { d.read_i8()? } else { 0 }; + + let topics_count = if flexible { + (d.read_varint()? - 1) as usize + } else { + d.read_i32()? as usize + }; + + let mut topics = Vec::with_capacity(topics_count); + for _ in 0..topics_count { + let topic = if flexible { + d.read_compact_nullable_string()?.unwrap_or_default() + } else { + d.read_nullable_string()?.unwrap_or_default() + }; + + let partitions_count = if flexible { + (d.read_varint()? - 1) as usize + } else { + d.read_i32()? as usize + }; + + let mut partitions = Vec::with_capacity(partitions_count); + for _ in 0..partitions_count { + let partition = d.read_i32()?; + + if version >= 4 { + d.read_i32()?; // current_leader_epoch + } + + let timestamp = d.read_i64()?; + + if version == 0 { + d.read_i32()?; // max_num_offsets (deprecated) + } + + partitions.push(ListOffsetsPartition { + partition, + timestamp, + }); + + if flexible { + d.read_tagged_fields()?; + } + } + + topics.push(ListOffsetsTopic { topic, partitions }); + if flexible { + d.read_tagged_fields()?; + } + } + + if flexible { + d.read_tagged_fields()?; + } + + Ok(ListOffsetsRequest { + isolation_level, + topics, + }) +} + +/// CreateTopics Request (API Key 19) +#[derive(Debug, Clone)] +pub struct CreateTopicsRequest { + pub topics: Vec, + pub timeout_ms: i32, + pub validate_only: bool, +} + +#[derive(Debug, Clone)] +pub struct CreatableTopic { + pub name: String, + pub num_partitions: i32, + pub replication_factor: i16, +} + +pub fn decode_create_topics_request(version: i16, body: Bytes) -> Result { + let mut d = Decoder::new(body); + let flexible = version >= 5; + + let topics_count = if flexible { + (d.read_varint()? - 1) as usize + } else { + d.read_i32()? as usize + }; + + let mut topics = Vec::with_capacity(topics_count); + for _ in 0..topics_count { + let name = if flexible { + d.read_compact_nullable_string()?.unwrap_or_default() + } else { + d.read_nullable_string()?.unwrap_or_default() + }; + + let num_partitions = d.read_i32()?; + let replication_factor = d.read_i16()?; + + // assignments (COMPACT_ARRAY or ARRAY) — skip + let assignments_count = if flexible { + (d.read_varint()? - 1) as usize + } else { + d.read_i32()? as usize + }; + for _ in 0..assignments_count { + d.read_i32()?; // partition_index + let replicas_count = if flexible { + (d.read_varint()? - 1) as usize + } else { + d.read_i32()? as usize + }; + for _ in 0..replicas_count { + d.read_i32()?; // broker_id + } + if flexible { + d.read_tagged_fields()?; + } + } + + // configs (COMPACT_ARRAY or ARRAY) — skip + let configs_count = if flexible { + (d.read_varint()? - 1) as usize + } else { + d.read_i32()? as usize + }; + for _ in 0..configs_count { + if flexible { + d.read_compact_nullable_string()?; // name + d.read_compact_nullable_string()?; // value + d.read_tagged_fields()?; + } else { + d.read_nullable_string()?; + d.read_nullable_string()?; + } + } + + topics.push(CreatableTopic { + name, + num_partitions, + replication_factor, + }); + + if flexible { + d.read_tagged_fields()?; + } + } + + let timeout_ms = d.read_i32()?; + let validate_only = if version >= 1 { d.read_bool()? } else { false }; + + if flexible { + d.read_tagged_fields()?; + } + + Ok(CreateTopicsRequest { + topics, + timeout_ms, + validate_only, + }) +} diff --git a/gateways/kafka/src/protocol/responses.rs b/gateways/kafka/src/protocol/responses.rs new file mode 100644 index 0000000000..55f332b273 --- /dev/null +++ b/gateways/kafka/src/protocol/responses.rs @@ -0,0 +1,274 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Kafka response encoders (stub implementations — will call Iggy SDK in production) + +use crate::protocol::api::*; +use crate::protocol::codec::Encoder; +use crate::protocol::requests::*; +use bytes::Bytes; + +pub fn encode_produce_response(version: i16, req: ProduceRequest) -> Bytes { + let flexible = version >= 9; + let mut e = Encoder::with_capacity(512); + + if flexible { + e.write_varint((req.topics.len() + 1) as u64); + } else { + e.write_i32(req.topics.len() as i32); + } + + for topic in &req.topics { + if flexible { + e.write_compact_nullable_string(Some(&topic.topic)); + } else { + e.write_nullable_string(Some(&topic.topic)); + } + + if flexible { + e.write_varint((topic.partitions.len() + 1) as u64); + } else { + e.write_i32(topic.partitions.len() as i32); + } + + for p in &topic.partitions { + e.write_i32(p.partition); + e.write_i16(ERROR_NONE); + e.write_i64(0); // base_offset — TODO: return real offset from Iggy + if version >= 2 { + e.write_i64(-1); // log_append_time_ms (-1 = not set) + } + if version >= 5 { + e.write_i64(0); // log_start_offset + } + // record_errors[] and error_message added in v8 + if version >= 8 { + if flexible { + e.write_varint(1); // empty COMPACT_ARRAY + e.write_compact_nullable_string(None); // error_message = null + } else { + e.write_i32(0); // empty ARRAY + e.write_nullable_string(None); // error_message = null + } + } + if flexible { + e.write_empty_tagged_fields(); + } + } + + if flexible { + e.write_empty_tagged_fields(); + } + } + + if version >= 1 { + e.write_i32(0); // throttle_time_ms + } + if flexible { + e.write_empty_tagged_fields(); + } + + e.freeze() +} + +pub fn encode_fetch_response(version: i16, req: FetchRequest) -> Bytes { + let flexible = version >= 12; + let mut e = Encoder::with_capacity(512); + + if version >= 1 { + e.write_i32(0); // throttle_time_ms + } + if version >= 7 { + e.write_i16(ERROR_NONE); // error_code + e.write_i32(0); // session_id + } + + if flexible { + e.write_varint((req.topics.len() + 1) as u64); + } else { + e.write_i32(req.topics.len() as i32); + } + + for topic in &req.topics { + if flexible { + e.write_compact_nullable_string(Some(&topic.topic)); + } else { + e.write_nullable_string(Some(&topic.topic)); + } + + if flexible { + e.write_varint((topic.partitions.len() + 1) as u64); + } else { + e.write_i32(topic.partitions.len() as i32); + } + + for partition in &topic.partitions { + e.write_i32(partition.partition); + e.write_i16(ERROR_NONE); + e.write_i64(0); // high_watermark — TODO: get from Iggy + if version >= 4 { + e.write_i64(0); // last_stable_offset + } + if version >= 5 { + e.write_i64(0); // log_start_offset + } + if version >= 4 { + // aborted_transactions[] + if flexible { + e.write_varint(1); + } else { + e.write_i32(0); + } + } + if version >= 11 { + e.write_i32(-1); // preferred_read_replica + } + // records (empty — TODO: call Iggy poll_messages) + if flexible { + e.write_compact_nullable_bytes(None); + } else { + e.write_nullable_bytes(None); + } + if flexible { + e.write_empty_tagged_fields(); + } + } + + if flexible { + e.write_empty_tagged_fields(); + } + } + + if flexible { + e.write_empty_tagged_fields(); + } + + e.freeze() +} + +pub fn encode_list_offsets_response(version: i16, req: ListOffsetsRequest) -> Bytes { + let flexible = version >= 6; + let mut e = Encoder::with_capacity(256); + + if version >= 2 { + e.write_i32(0); // throttle_time_ms + } + + if flexible { + e.write_varint((req.topics.len() + 1) as u64); + } else { + e.write_i32(req.topics.len() as i32); + } + + for topic in &req.topics { + if flexible { + e.write_compact_nullable_string(Some(&topic.topic)); + } else { + e.write_nullable_string(Some(&topic.topic)); + } + + if flexible { + e.write_varint((topic.partitions.len() + 1) as u64); + } else { + e.write_i32(topic.partitions.len() as i32); + } + + for partition in &topic.partitions { + e.write_i32(partition.partition); + e.write_i16(ERROR_NONE); + + // TODO: query Iggy for actual offsets + let offset = 0i64; + if version >= 1 { + e.write_i64(1_700_000_000_000); // timestamp placeholder + } + e.write_i64(offset); + // leader_epoch was added in v4, not v1 + if version >= 4 { + e.write_i32(-1); + } + if flexible { + e.write_empty_tagged_fields(); + } + } + + if flexible { + e.write_empty_tagged_fields(); + } + } + + if flexible { + e.write_empty_tagged_fields(); + } + + e.freeze() +} + +pub fn encode_create_topics_response(version: i16, req: CreateTopicsRequest) -> Bytes { + let flexible = version >= 5; + let mut e = Encoder::with_capacity(256); + + if version >= 2 { + e.write_i32(0); // throttle_time_ms + } + + if flexible { + e.write_varint((req.topics.len() + 1) as u64); + } else { + e.write_i32(req.topics.len() as i32); + } + + for topic in &req.topics { + if flexible { + e.write_compact_nullable_string(Some(&topic.name)); + } else { + e.write_nullable_string(Some(&topic.name)); + } + + let error_code = if topic.num_partitions <= 0 { + ERROR_INVALID_PARTITIONS + } else { + ERROR_NONE + }; + e.write_i16(error_code); + + if version >= 1 { + if flexible { + e.write_compact_nullable_string(None); // error_message + } else { + e.write_nullable_string(None); + } + } + + if version >= 5 { + e.write_i16(ERROR_NONE); // topic_config_error_code (added in v5) + e.write_i32(topic.num_partitions); + e.write_i16(topic.replication_factor); + e.write_varint(1); // configs[] empty COMPACT_ARRAY + } + + if flexible { + e.write_empty_tagged_fields(); + } + } + + if flexible { + e.write_empty_tagged_fields(); + } + + e.freeze() +} diff --git a/gateways/kafka/src/server.rs b/gateways/kafka/src/server.rs new file mode 100644 index 0000000000..c6465eb0ca --- /dev/null +++ b/gateways/kafka/src/server.rs @@ -0,0 +1,207 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::net::SocketAddr; +use std::sync::Arc; +use std::time::Duration; + +use bytes::{BufMut, BytesMut}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::{TcpListener, TcpStream}; +use tokio::sync::broadcast; +use tokio::time::timeout; +use tracing::{error, info, warn}; + +use crate::error::{KafkaProtocolError, Result}; +use crate::protocol::api::handle_request; +use crate::protocol::codec::Decoder; +use crate::protocol::header::{ + RequestHeader, ResponseHeader, request_header_version, response_header_version, +}; + +#[derive(Debug, Clone)] +pub struct ServerConfig { + pub bind_addr: String, + pub max_frame_size: usize, + pub read_timeout: Duration, + pub write_timeout: Duration, +} + +impl Default for ServerConfig { + fn default() -> Self { + Self { + bind_addr: "127.0.0.1:9093".to_string(), + max_frame_size: 8 * 1024 * 1024, + read_timeout: Duration::from_secs(15), + write_timeout: Duration::from_secs(10), + } + } +} + +pub struct KafkaServer { + config: Arc, +} + +impl KafkaServer { + pub fn new(config: ServerConfig) -> Self { + Self { + config: Arc::new(config), + } + } + + pub async fn run(self, mut shutdown: broadcast::Receiver<()>) -> Result<()> { + let listener = TcpListener::bind(&self.config.bind_addr).await?; + info!("kafka listener bound on {}", self.config.bind_addr); + + loop { + tokio::select! { + _ = shutdown.recv() => { + info!("kafka listener shutdown requested"); + break; + } + accept_result = listener.accept() => { + let (stream, peer) = accept_result?; + let cfg = Arc::clone(&self.config); + tokio::spawn(async move { + if let Err(err) = handle_connection(stream, cfg, peer).await { + warn!(%peer, "connection closed with error: {err}"); + } + }); + } + } + } + Ok(()) + } +} + +async fn handle_connection( + mut stream: TcpStream, + config: Arc, + peer: SocketAddr, +) -> Result<()> { + info!(%peer, "connection accepted"); + + loop { + let frame = match read_frame(&mut stream, config.max_frame_size, config.read_timeout).await + { + Ok(f) => f, + Err(KafkaProtocolError::Io(ref e)) + if e.kind() == std::io::ErrorKind::UnexpectedEof + || e.kind() == std::io::ErrorKind::ConnectionReset => + { + info!(%peer, "connection closed by client"); + return Ok(()); + } + Err(e) => return Err(e), + }; + + if frame.len() < 4 { + return Err(KafkaProtocolError::BufferUnderflow { + needed: 4, + remaining: frame.len(), + }); + } + let api_key = i16::from_be_bytes([frame[0], frame[1]]); + let api_version = i16::from_be_bytes([frame[2], frame[3]]); + let req_hdr_ver = request_header_version(api_key, api_version); + let resp_hdr_ver = response_header_version(api_key, api_version); + + let mut decoder = Decoder::new(frame); + let req = RequestHeader::decode_from(&mut decoder, req_hdr_ver)?; + info!( + %peer, + api_key = req.api_key, + api_version = req.api_version, + correlation_id = req.correlation_id, + client_id = req.client_id.as_deref().unwrap_or(""), + "received request" + ); + + let body = decoder.read_bytes(decoder.remaining())?; + let body_response = handle_request(req.api_key, req.api_version, body); + + let resp_header = ResponseHeader { + correlation_id: req.correlation_id, + }; + let encoded_header = resp_header.encode(resp_hdr_ver); + let mut payload = BytesMut::with_capacity(encoded_header.len() + body_response.len()); + payload.put_slice(&encoded_header); + payload.put_slice(&body_response); + + write_frame(&mut stream, &payload, config.write_timeout).await?; + } +} + +pub async fn read_frame( + stream: &mut TcpStream, + max_frame_size: usize, + read_timeout: Duration, +) -> Result { + let mut len_buf = [0u8; 4]; + timeout(read_timeout, stream.read_exact(&mut len_buf)) + .await + .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "read timeout"))??; + + let frame_len = i32::from_be_bytes(len_buf); + if frame_len <= 0 { + return Err(KafkaProtocolError::InvalidFrameLength(frame_len)); + } + + let frame_len = frame_len as usize; + if frame_len > max_frame_size { + return Err(KafkaProtocolError::FrameTooLarge { + max_bytes: max_frame_size, + actual_bytes: frame_len, + }); + } + + let mut data = vec![0u8; frame_len]; + timeout(read_timeout, stream.read_exact(&mut data)) + .await + .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "read timeout"))??; + Ok(bytes::Bytes::from(data)) +} + +pub async fn write_frame( + stream: &mut TcpStream, + payload: &[u8], + write_timeout: Duration, +) -> Result<()> { + let len = payload.len(); + if len > i32::MAX as usize { + return Err(KafkaProtocolError::FrameTooLarge { + max_bytes: i32::MAX as usize, + actual_bytes: len, + }); + } + let mut frame = BytesMut::with_capacity(4 + len); + frame.put_i32(len as i32); + frame.extend_from_slice(payload); + timeout(write_timeout, stream.write_all(&frame)) + .await + .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "write timeout"))??; + Ok(()) +} + +pub fn init_tracing() { + let filter = tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")); + let _ = tracing_subscriber::fmt() + .with_env_filter(filter) + .try_init() + .map_err(|e| error!("failed to initialize tracing: {e}")); +} diff --git a/gateways/kafka/tests/api_handler_tests.rs b/gateways/kafka/tests/api_handler_tests.rs new file mode 100644 index 0000000000..5fb8b0c6c2 --- /dev/null +++ b/gateways/kafka/tests/api_handler_tests.rs @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use bytes::Bytes; + +use iggy_gateway_kafka::protocol::api::{ + API_KEY_API_VERSIONS, API_KEY_METADATA, ERROR_UNSUPPORTED_VERSION, handle_request, + is_supported_version, split_metadata_request_topics, supported_api_ranges, +}; +use iggy_gateway_kafka::protocol::codec::Decoder; + +// ── ApiVersions ───────────────────────────────────────────────────────────── + +#[test] +fn api_versions_v1_response_non_flexible_format() { + let body = handle_request(API_KEY_API_VERSIONS, 1, Bytes::new()); + let mut d = Decoder::new(body); + + assert_eq!(d.read_i16().unwrap(), 0); // error_code + + // Non-flexible: i32 array count + let count = d.read_i32().unwrap(); + assert!(count >= 2); + let mut keys = Vec::new(); + for _ in 0..count { + keys.push(d.read_i16().unwrap()); + d.read_i16().unwrap(); // min + d.read_i16().unwrap(); // max + } + assert_eq!(d.read_i32().unwrap(), 0); // throttle_time_ms + + let expected_keys: Vec = supported_api_ranges().iter().map(|r| r.api_key).collect(); + for k in expected_keys { + assert!(keys.contains(&k)); + } +} + +#[test] +fn api_versions_v3_response_flexible_format() { + let body = handle_request(API_KEY_API_VERSIONS, 3, Bytes::new()); + let mut d = Decoder::new(body); + + assert_eq!(d.read_i16().unwrap(), 0); // error_code + + // Flexible: varint(len+1) compact array + let count_plus_one = d.read_varint().unwrap(); + assert!(count_plus_one >= 3); // at least 2 entries → varint = 3+ + let count = (count_plus_one - 1) as i32; + + let mut keys = Vec::new(); + for _ in 0..count { + keys.push(d.read_i16().unwrap()); + d.read_i16().unwrap(); // min + d.read_i16().unwrap(); // max + d.read_tagged_fields().unwrap(); // per-entry tagged fields + } + assert_eq!(d.read_i32().unwrap(), 0); // throttle_time_ms + d.read_tagged_fields().unwrap(); // top-level tagged fields + + let expected_keys: Vec = supported_api_ranges().iter().map(|r| r.api_key).collect(); + for k in expected_keys { + assert!(keys.contains(&k)); + } +} + +// ── Metadata ───────────────────────────────────────────────────────────────── + +#[test] +fn metadata_response_has_broker_array_and_topic_array() { + let body = handle_request(API_KEY_METADATA, 0, Bytes::new()); + let mut d = Decoder::new(body); + + let broker_count = d.read_i32().unwrap(); + assert_eq!(broker_count, 1); + let node_id = d.read_i32().unwrap(); + assert_eq!(node_id, 1); + let host = d.read_nullable_string().unwrap().unwrap(); + assert_eq!(host, "127.0.0.1"); + let port = d.read_i32().unwrap(); + assert_eq!(port, 9093); + + let topic_count = d.read_i32().unwrap(); + assert_eq!(topic_count, 0); +} + +#[test] +fn unsupported_version_returns_protocol_error() { + let mut req = Vec::new(); + req.extend_from_slice(&1_i32.to_be_bytes()); + let body = handle_request(API_KEY_METADATA, 99, Bytes::from(req)); + let mut d = Decoder::new(body); + let _broker_count = d.read_i32().unwrap(); + let _ = d.read_i32().unwrap(); + let _ = d.read_nullable_string().unwrap(); + let _ = d.read_i32().unwrap(); + let topic_count = d.read_i32().unwrap(); + assert_eq!(topic_count, 1); + let topic_error = d.read_i16().unwrap(); + assert_eq!(topic_error, ERROR_UNSUPPORTED_VERSION); + let topic_name = d.read_nullable_string().unwrap().unwrap(); + assert_eq!(topic_name, "unknown-topic"); + let partitions_count = d.read_i32().unwrap(); + assert_eq!(partitions_count, 0); + let controller_id = d.read_i32().unwrap(); + assert_eq!(controller_id, 1); +} + +// ── Misc ──────────────────────────────────────────────────────────────────── + +#[test] +fn unknown_api_key_returns_error_only_payload() { + let body = handle_request(999, 0, Bytes::new()); + let mut d = Decoder::new(body); + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); +} + +#[test] +fn metadata_topic_split_reads_array_count() { + let mut raw = Vec::new(); + raw.extend_from_slice(&2_i32.to_be_bytes()); + assert_eq!(split_metadata_request_topics(Bytes::from(raw)), 2); +} + +#[test] +fn version_support_table_is_applied() { + assert!(is_supported_version(API_KEY_API_VERSIONS, 3)); + assert!(!is_supported_version(API_KEY_API_VERSIONS, 10)); + assert!(is_supported_version(API_KEY_METADATA, 1)); + assert!(!is_supported_version(API_KEY_METADATA, -1)); +} diff --git a/gateways/kafka/tests/codec_tests.rs b/gateways/kafka/tests/codec_tests.rs new file mode 100644 index 0000000000..093d123451 --- /dev/null +++ b/gateways/kafka/tests/codec_tests.rs @@ -0,0 +1,149 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use bytes::Bytes; + +use iggy_gateway_kafka::protocol::codec::{Decoder, Encoder}; + +#[test] +fn codec_round_trip_primitives_and_nullable_fields() { + let mut enc = Encoder::with_capacity(128); + enc.write_i8(-3); + enc.write_i16(42); + enc.write_i32(123_456); + enc.write_i64(9_999_999); + enc.write_nullable_string(Some("client-a")); + enc.write_nullable_string(None); + enc.write_nullable_bytes(Some(&[1, 2, 3])); + enc.write_nullable_bytes(None); + let bytes = enc.freeze(); + + let mut dec = Decoder::new(bytes); + assert_eq!(dec.read_i8().unwrap(), -3); + assert_eq!(dec.read_i16().unwrap(), 42); + assert_eq!(dec.read_i32().unwrap(), 123_456); + assert_eq!(dec.read_i64().unwrap(), 9_999_999); + assert_eq!( + dec.read_nullable_string().unwrap().as_deref(), + Some("client-a") + ); + assert_eq!(dec.read_nullable_string().unwrap(), None); + assert_eq!( + dec.read_nullable_bytes().unwrap().unwrap(), + Bytes::from_static(&[1, 2, 3]) + ); + assert_eq!(dec.read_nullable_bytes().unwrap(), None); +} + +#[test] +fn decoder_returns_underflow_error() { + let mut dec = Decoder::new(Bytes::from_static(&[0x00])); + let err = dec.read_i32().expect_err("must fail"); + assert!(err.to_string().contains("buffer underflow")); +} + +#[test] +fn codec_u8_and_bool() { + let mut enc = Encoder::with_capacity(8); + enc.write_u8(0xFF); + enc.write_bool(true); + enc.write_bool(false); + let bytes = enc.freeze(); + + let mut dec = Decoder::new(bytes); + assert_eq!(dec.read_u8().unwrap(), 0xFF); + assert!(dec.read_bool().unwrap()); + assert!(!dec.read_bool().unwrap()); +} + +#[test] +fn varint_round_trip_small_values() { + for v in [0u64, 1, 127, 128, 255, 300, 16383, 16384, u32::MAX as u64] { + let mut enc = Encoder::with_capacity(16); + enc.write_varint(v); + let mut dec = Decoder::new(enc.freeze()); + assert_eq!(dec.read_varint().unwrap(), v, "failed for v={v}"); + } +} + +#[test] +fn varint_single_byte_for_values_below_128() { + let mut enc = Encoder::with_capacity(1); + enc.write_varint(42); + let bytes = enc.freeze(); + assert_eq!(bytes.len(), 1); + assert_eq!(bytes[0], 42); +} + +#[test] +fn varint_two_bytes_for_128() { + let mut enc = Encoder::with_capacity(2); + enc.write_varint(128); + let bytes = enc.freeze(); + // 128 = 0x80 → first byte 0x80 | 0x80 = 0x80 (continue), second byte 0x01 + assert_eq!(bytes.as_ref(), &[0x80, 0x01]); +} + +#[test] +fn compact_nullable_string_round_trip() { + let mut enc = Encoder::with_capacity(32); + enc.write_compact_nullable_string(Some("hello")); + enc.write_compact_nullable_string(None); + enc.write_compact_nullable_string(Some("")); + let bytes = enc.freeze(); + + let mut dec = Decoder::new(bytes); + assert_eq!( + dec.read_compact_nullable_string().unwrap().as_deref(), + Some("hello") + ); + assert_eq!(dec.read_compact_nullable_string().unwrap(), None); + assert_eq!( + dec.read_compact_nullable_string().unwrap().as_deref(), + Some("") + ); +} + +#[test] +fn compact_nullable_bytes_round_trip() { + let mut enc = Encoder::with_capacity(32); + enc.write_compact_nullable_bytes(Some(&[10, 20, 30])); + enc.write_compact_nullable_bytes(None); + let bytes = enc.freeze(); + + let mut dec = Decoder::new(bytes); + assert_eq!( + dec.read_compact_nullable_bytes().unwrap().unwrap(), + Bytes::from_static(&[10, 20, 30]) + ); + assert_eq!(dec.read_compact_nullable_bytes().unwrap(), None); +} + +#[test] +fn tagged_fields_empty_section_round_trip() { + let mut enc = Encoder::with_capacity(8); + enc.write_i32(42); + enc.write_empty_tagged_fields(); + enc.write_i16(7); + let bytes = enc.freeze(); + + let mut dec = Decoder::new(bytes); + assert_eq!(dec.read_i32().unwrap(), 42); + dec.read_tagged_fields().unwrap(); // should consume the single 0x00 byte + assert_eq!(dec.read_i16().unwrap(), 7); + assert_eq!(dec.remaining(), 0); +} diff --git a/gateways/kafka/tests/decode_validation_tests.rs b/gateways/kafka/tests/decode_validation_tests.rs new file mode 100644 index 0000000000..d9f9cab3f9 --- /dev/null +++ b/gateways/kafka/tests/decode_validation_tests.rs @@ -0,0 +1,449 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Validates request decoders and response encoders against the binary fixtures +//! produced by tools/kafka-tool. +//! +//! Frame layout written by kafka-tool (all versions): +//! [4-byte length prefix] +//! [api_key i16][api_version i16][correlation_id i32] +//! [client_id_len i16][client_id bytes] ← always legacy i16, even for flexible APIs +//! [0x00 tagged-fields byte] ← only for flexible API versions +//! [request body] ← properly encoded per spec (flexible or not) + +use std::path::PathBuf; + +use bytes::Bytes; + +use iggy_gateway_kafka::protocol::header::request_header_version; +use iggy_gateway_kafka::protocol::requests::{ + decode_create_topics_request, decode_fetch_request, decode_list_offsets_request, + decode_produce_request, +}; +use iggy_gateway_kafka::protocol::responses::{ + encode_create_topics_response, encode_fetch_response, encode_list_offsets_response, + encode_produce_response, +}; + +// ── helpers ─────────────────────────────────────────────────────────────────── + +fn fixtures_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tools/kafka-tool/kafka_messages") +} + +/// Load a kafka-tool .bin file and return just the request body bytes, correctly +/// skipping the outer Kafka frame header (api_key, api_version, correlation_id, +/// legacy-i16 client_id, and — for flexible versions — the 0x00 tagged-fields byte). +fn load_body(api_key: i16, api_name: &str, version: i16) -> Bytes { + let filename = format!("{:03}_{}_v{}.bin", api_key, api_name, version); + let path = fixtures_dir().join(&filename); + let data = std::fs::read(&path).unwrap_or_else(|e| panic!("failed to read {filename}: {e}")); + + // Skip 4-byte length prefix → frame starts here + let frame = &data[4..]; + + // Bytes 0-7: api_key(2) + api_version(2) + correlation_id(4) + // Bytes 8-9: client_id_len (legacy i16) + let client_id_len = i16::from_be_bytes([frame[8], frame[9]]) as usize; + let body_start_after_client_id = 10 + client_id_len; + + // kafka-tool appends a 0x00 tagged-fields byte for flexible-version APIs + let is_flexible = request_header_version(api_key, version) >= 2; + let body_start = if is_flexible { + body_start_after_client_id + 1 + } else { + body_start_after_client_id + }; + + Bytes::copy_from_slice(&frame[body_start..]) +} + +// ── Produce (API key 0) ─────────────────────────────────────────────────────── + +#[test] +fn produce_all_supported_versions_decode() { + for version in 3i16..=9 { + let body = load_body(0, "Produce", version); + let req = decode_produce_request(version, body) + .unwrap_or_else(|e| panic!("Produce v{version} decode failed: {e}")); + + assert_eq!(req.acks, -1, "Produce v{version}: unexpected acks"); + assert_eq!( + req.timeout_ms, 5000, + "Produce v{version}: unexpected timeout_ms" + ); + assert_eq!(req.topics.len(), 1, "Produce v{version}: expected 1 topic"); + assert_eq!( + req.topics[0].topic, "test-topic", + "Produce v{version}: wrong topic name" + ); + assert_eq!( + req.topics[0].partitions.len(), + 1, + "Produce v{version}: expected 1 partition" + ); + assert_eq!( + req.topics[0].partitions[0].partition, 0, + "Produce v{version}: wrong partition index" + ); + assert!( + req.topics[0].partitions[0].records.is_some(), + "Produce v{version}: records should be present" + ); + } +} + +#[test] +fn produce_response_encodes_for_all_supported_versions() { + for version in 3i16..=9 { + let body = load_body(0, "Produce", version); + let req = decode_produce_request(version, body) + .unwrap_or_else(|e| panic!("Produce v{version} decode failed: {e}")); + let resp = encode_produce_response(version, req); + assert!( + !resp.is_empty(), + "Produce v{version}: response must not be empty" + ); + } +} + +#[test] +fn produce_response_v3_roundtrip() { + use iggy_gateway_kafka::protocol::codec::Decoder; + let body = load_body(0, "Produce", 3); + let req = decode_produce_request(3, body).unwrap(); + let resp = encode_produce_response(3, req); + + let mut d = Decoder::new(resp); + let topic_count = d.read_i32().unwrap(); + assert_eq!(topic_count, 1); + let topic_name = d.read_nullable_string().unwrap().unwrap(); + assert_eq!(topic_name, "test-topic"); + let partition_count = d.read_i32().unwrap(); + assert_eq!(partition_count, 1); + let partition = d.read_i32().unwrap(); + assert_eq!(partition, 0); + let error_code = d.read_i16().unwrap(); + assert_eq!(error_code, 0); + let base_offset = d.read_i64().unwrap(); + assert_eq!(base_offset, 0); + // log_append_time_ms (v2+) + let _log_append = d.read_i64().unwrap(); + // log_start_offset (v5+) — not present for v3 + let throttle = d.read_i32().unwrap(); + assert_eq!(throttle, 0); +} + +#[test] +fn produce_response_v8_includes_record_errors() { + use iggy_gateway_kafka::protocol::codec::Decoder; + let body = load_body(0, "Produce", 8); + let req = decode_produce_request(8, body).unwrap(); + let resp = encode_produce_response(8, req); + + let mut d = Decoder::new(resp); + let topic_count = d.read_i32().unwrap(); + assert_eq!(topic_count, 1); + let _topic_name = d.read_nullable_string().unwrap(); + let partition_count = d.read_i32().unwrap(); + assert_eq!(partition_count, 1); + let _partition = d.read_i32().unwrap(); + let error_code = d.read_i16().unwrap(); + assert_eq!(error_code, 0); + let _base_offset = d.read_i64().unwrap(); + let _log_append_time = d.read_i64().unwrap(); // v2+ + let _log_start_offset = d.read_i64().unwrap(); // v5+ + let record_errors_count = d.read_i32().unwrap(); // v8+: should be 0 + assert_eq!( + record_errors_count, 0, + "v8 must emit empty record_errors array" + ); + let error_message = d.read_nullable_string().unwrap(); // v8+: should be null + assert!(error_message.is_none(), "v8 error_message must be null"); +} + +// ── Fetch (API key 1) ───────────────────────────────────────────────────────── + +#[test] +fn fetch_all_supported_versions_decode() { + for version in 4i16..=12 { + let body = load_body(1, "Fetch", version); + let req = decode_fetch_request(version, body) + .unwrap_or_else(|e| panic!("Fetch v{version} decode failed: {e}")); + + assert_eq!( + req.max_wait_ms, 500, + "Fetch v{version}: unexpected max_wait_ms" + ); + assert_eq!(req.min_bytes, 1, "Fetch v{version}: unexpected min_bytes"); + assert_eq!(req.topics.len(), 1, "Fetch v{version}: expected 1 topic"); + assert_eq!( + req.topics[0].topic, "test-topic", + "Fetch v{version}: wrong topic name" + ); + assert_eq!( + req.topics[0].partitions.len(), + 1, + "Fetch v{version}: expected 1 partition" + ); + assert_eq!( + req.topics[0].partitions[0].partition, 0, + "Fetch v{version}: wrong partition index" + ); + assert_eq!( + req.topics[0].partitions[0].fetch_offset, 0, + "Fetch v{version}: wrong fetch_offset" + ); + } +} + +#[test] +fn fetch_response_encodes_for_all_supported_versions() { + for version in 4i16..=12 { + let body = load_body(1, "Fetch", version); + let req = decode_fetch_request(version, body) + .unwrap_or_else(|e| panic!("Fetch v{version} decode failed: {e}")); + let resp = encode_fetch_response(version, req); + assert!( + !resp.is_empty(), + "Fetch v{version}: response must not be empty" + ); + } +} + +#[test] +fn fetch_response_v7_roundtrip() { + use iggy_gateway_kafka::protocol::codec::Decoder; + let body = load_body(1, "Fetch", 7); + let req = decode_fetch_request(7, body).unwrap(); + let resp = encode_fetch_response(7, req); + + let mut d = Decoder::new(resp); + let throttle_ms = d.read_i32().unwrap(); // v1+ + assert_eq!(throttle_ms, 0); + let error_code = d.read_i16().unwrap(); // v7+ + assert_eq!(error_code, 0); + let session_id = d.read_i32().unwrap(); // v7+ + assert_eq!(session_id, 0); + let topic_count = d.read_i32().unwrap(); + assert_eq!(topic_count, 1); + let topic_name = d.read_nullable_string().unwrap().unwrap(); + assert_eq!(topic_name, "test-topic"); + let partition_count = d.read_i32().unwrap(); + assert_eq!(partition_count, 1); + let partition = d.read_i32().unwrap(); + assert_eq!(partition, 0); + let partition_error = d.read_i16().unwrap(); + assert_eq!(partition_error, 0); + let high_watermark = d.read_i64().unwrap(); + assert_eq!(high_watermark, 0); +} + +// ── ListOffsets (API key 2) ─────────────────────────────────────────────────── + +#[test] +fn list_offsets_all_supported_versions_decode() { + for version in 1i16..=6 { + let body = load_body(2, "ListOffsets", version); + let req = decode_list_offsets_request(version, body) + .unwrap_or_else(|e| panic!("ListOffsets v{version} decode failed: {e}")); + + assert_eq!( + req.topics.len(), + 1, + "ListOffsets v{version}: expected 1 topic" + ); + assert_eq!( + req.topics[0].topic, "test-topic", + "ListOffsets v{version}: wrong topic name" + ); + assert_eq!( + req.topics[0].partitions.len(), + 1, + "ListOffsets v{version}: expected 1 partition" + ); + assert_eq!( + req.topics[0].partitions[0].partition, 0, + "ListOffsets v{version}: wrong partition index" + ); + } +} + +#[test] +fn list_offsets_response_encodes_for_all_supported_versions() { + for version in 1i16..=6 { + let body = load_body(2, "ListOffsets", version); + let req = decode_list_offsets_request(version, body) + .unwrap_or_else(|e| panic!("ListOffsets v{version} decode failed: {e}")); + let resp = encode_list_offsets_response(version, req); + assert!( + !resp.is_empty(), + "ListOffsets v{version}: response must not be empty" + ); + } +} + +#[test] +fn list_offsets_response_v1_no_leader_epoch() { + use iggy_gateway_kafka::protocol::codec::Decoder; + let body = load_body(2, "ListOffsets", 1); + let req = decode_list_offsets_request(1, body).unwrap(); + let resp = encode_list_offsets_response(1, req); + + let mut d = Decoder::new(resp); + // v1: no throttle_time_ms + let topic_count = d.read_i32().unwrap(); + assert_eq!(topic_count, 1); + let _topic_name = d.read_nullable_string().unwrap(); + let partition_count = d.read_i32().unwrap(); + assert_eq!(partition_count, 1); + let _partition = d.read_i32().unwrap(); + let error_code = d.read_i16().unwrap(); + assert_eq!(error_code, 0); + let _timestamp = d.read_i64().unwrap(); // v1+ + let _offset = d.read_i64().unwrap(); + // v1 must NOT have a leader_epoch field — assert all bytes consumed + assert_eq!( + d.remaining(), + 0, + "v1 response must have no trailing bytes (leader_epoch must NOT be written)" + ); +} + +#[test] +fn list_offsets_response_v4_has_leader_epoch() { + use iggy_gateway_kafka::protocol::codec::Decoder; + let body = load_body(2, "ListOffsets", 4); + let req = decode_list_offsets_request(4, body).unwrap(); + let resp = encode_list_offsets_response(4, req); + + let mut d = Decoder::new(resp); + let _throttle = d.read_i32().unwrap(); // v2+ + let topic_count = d.read_i32().unwrap(); + assert_eq!(topic_count, 1); + let _topic_name = d.read_nullable_string().unwrap(); + let partition_count = d.read_i32().unwrap(); + assert_eq!(partition_count, 1); + let _partition = d.read_i32().unwrap(); + let error_code = d.read_i16().unwrap(); + assert_eq!(error_code, 0); + let _timestamp = d.read_i64().unwrap(); + let _offset = d.read_i64().unwrap(); + let leader_epoch = d.read_i32().unwrap(); // v4+ + assert_eq!(leader_epoch, -1, "v4 must have leader_epoch = -1"); + assert_eq!(d.remaining(), 0); +} + +// ── CreateTopics (API key 19) ───────────────────────────────────────────────── + +#[test] +fn create_topics_all_supported_versions_decode() { + for version in 2i16..=5 { + let body = load_body(19, "CreateTopics", version); + let req = decode_create_topics_request(version, body) + .unwrap_or_else(|e| panic!("CreateTopics v{version} decode failed: {e}")); + + assert_eq!( + req.topics.len(), + 1, + "CreateTopics v{version}: expected 1 topic" + ); + assert_eq!( + req.topics[0].num_partitions, 1, + "CreateTopics v{version}: wrong num_partitions" + ); + assert_eq!( + req.topics[0].replication_factor, 1, + "CreateTopics v{version}: wrong replication_factor" + ); + assert!( + !req.topics[0].name.is_empty(), + "CreateTopics v{version}: topic name must not be empty" + ); + assert_eq!( + req.timeout_ms, 30000, + "CreateTopics v{version}: unexpected timeout_ms" + ); + } +} + +#[test] +fn create_topics_response_encodes_for_all_supported_versions() { + for version in 2i16..=5 { + let body = load_body(19, "CreateTopics", version); + let req = decode_create_topics_request(version, body) + .unwrap_or_else(|e| panic!("CreateTopics v{version} decode failed: {e}")); + let resp = encode_create_topics_response(version, req); + assert!( + !resp.is_empty(), + "CreateTopics v{version}: response must not be empty" + ); + } +} + +#[test] +fn create_topics_response_v2_roundtrip() { + use iggy_gateway_kafka::protocol::codec::Decoder; + let body = load_body(19, "CreateTopics", 2); + let req = decode_create_topics_request(2, body).unwrap(); + let topic_name = req.topics[0].name.clone(); + let resp = encode_create_topics_response(2, req); + + let mut d = Decoder::new(resp); + let _throttle = d.read_i32().unwrap(); // v2+ + let topic_count = d.read_i32().unwrap(); + assert_eq!(topic_count, 1); + let resp_topic = d.read_nullable_string().unwrap().unwrap(); + assert_eq!(resp_topic, topic_name); + let error_code = d.read_i16().unwrap(); + assert_eq!(error_code, 0); + let error_msg = d.read_nullable_string().unwrap(); // v1+ + assert!(error_msg.is_none()); + assert_eq!(d.remaining(), 0); +} + +#[test] +fn create_topics_response_v5_has_topic_config_error_code() { + use iggy_gateway_kafka::protocol::codec::Decoder; + let body = load_body(19, "CreateTopics", 5); + let req = decode_create_topics_request(5, body).unwrap(); + let resp = encode_create_topics_response(5, req); + + let mut d = Decoder::new(resp); + let _throttle = d.read_i32().unwrap(); // v2+ + let topic_count_plus_one = d.read_varint().unwrap(); // flexible compact array + assert_eq!(topic_count_plus_one, 2); // 1 topic → varint = 2 + + let _topic_name = d.read_compact_nullable_string().unwrap(); + let error_code = d.read_i16().unwrap(); + assert_eq!(error_code, 0); + let _error_msg = d.read_compact_nullable_string().unwrap(); // v1+ + let topic_config_err = d.read_i16().unwrap(); // v5+: MUST be present + assert_eq!( + topic_config_err, 0, + "v5 must include topic_config_error_code" + ); + let num_partitions = d.read_i32().unwrap(); + assert_eq!(num_partitions, 1); + let replication_factor = d.read_i16().unwrap(); + assert_eq!(replication_factor, 1); + let configs_count_plus_one = d.read_varint().unwrap(); // empty compact array + assert_eq!(configs_count_plus_one, 1); // empty = varint(1) + d.read_tagged_fields().unwrap(); // per-entry tagged_fields + d.read_tagged_fields().unwrap(); // top-level tagged_fields + assert_eq!(d.remaining(), 0); +} diff --git a/gateways/kafka/tests/golden_wire_fixtures_tests.rs b/gateways/kafka/tests/golden_wire_fixtures_tests.rs new file mode 100644 index 0000000000..121074a191 --- /dev/null +++ b/gateways/kafka/tests/golden_wire_fixtures_tests.rs @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use bytes::Bytes; + +use iggy_gateway_kafka::protocol::api::{API_KEY_API_VERSIONS, API_KEY_METADATA, handle_request}; +use iggy_gateway_kafka::protocol::codec::Encoder; + +#[test] +fn golden_apiversions_v1_response_fixture() { + let actual = handle_request(API_KEY_API_VERSIONS, 1, Bytes::new()); + + // error_code=0, api_count=6 + // key 0 (Produce) min=3 max=9 + // key 1 (Fetch) min=4 max=12 + // key 2 (ListOffsets) min=1 max=6 + // key 3 (Metadata) min=0 max=9 + // key 18 (ApiVersions) min=0 max=3 + // key 19 (CreateTopics) min=2 max=5 + // throttle_ms=0 + let expected: [u8; 46] = [ + 0x00, 0x00, // error_code + 0x00, 0x00, 0x00, 0x06, // api count = 6 + 0x00, 0x00, 0x00, 0x03, 0x00, 0x09, // key 0: Produce 3–9 + 0x00, 0x01, 0x00, 0x04, 0x00, 0x0C, // key 1: Fetch 4–12 + 0x00, 0x02, 0x00, 0x01, 0x00, 0x06, // key 2: ListOffsets 1–6 + 0x00, 0x03, 0x00, 0x00, 0x00, 0x09, // key 3: Metadata 0–9 + 0x00, 0x12, 0x00, 0x00, 0x00, 0x03, // key 18: ApiVersions 0–3 + 0x00, 0x13, 0x00, 0x02, 0x00, 0x05, // key 19: CreateTopics 2–5 + 0x00, 0x00, 0x00, 0x00, // throttle_ms + ]; + assert_eq!(actual.as_ref(), &expected); +} + +#[test] +fn golden_metadata_v0_single_topic_response_fixture() { + let mut request = Encoder::with_capacity(32); + request.write_i32(1); // one topic + let req_bytes = request.freeze(); + + let actual = handle_request(API_KEY_METADATA, 0, req_bytes); + + // brokers[1]: node_id=1, host=127.0.0.1, port=9093 + // topics[1]: topic_error=3, topic_name=unknown-topic, partitions[0] + // controller_id=1 (included by this implementation baseline) + let expected: [u8; 52] = [ + 0x00, 0x00, 0x00, 0x01, // broker count + 0x00, 0x00, 0x00, 0x01, // node id + 0x00, 0x09, // host len + 0x31, 0x32, 0x37, 0x2e, 0x30, 0x2e, 0x30, 0x2e, 0x31, // "127.0.0.1" + 0x00, 0x00, 0x23, 0x85, // port 9093 + 0x00, 0x00, 0x00, 0x01, // topic count + 0x00, 0x03, // topic error code + 0x00, 0x0d, // topic name len + 0x75, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x2d, 0x74, 0x6f, 0x70, 0x69, + 0x63, // unknown-topic + 0x00, 0x00, 0x00, 0x00, // partition count + 0x00, 0x00, 0x00, 0x01, // controller id + ]; + assert_eq!(actual.as_ref(), &expected); +} diff --git a/gateways/kafka/tests/header_tests.rs b/gateways/kafka/tests/header_tests.rs new file mode 100644 index 0000000000..d4eef81db6 --- /dev/null +++ b/gateways/kafka/tests/header_tests.rs @@ -0,0 +1,139 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use iggy_gateway_kafka::protocol::codec::Encoder; +use iggy_gateway_kafka::protocol::header::{ + RequestHeader, ResponseHeader, request_header_version, response_header_version, +}; + +// ── Request header v1 (non-flexible) ─────────────────────────────────────── + +#[test] +fn request_header_v1_decodes() { + let mut enc = Encoder::with_capacity(64); + enc.write_i16(18); // api_key: ApiVersions + enc.write_i16(2); // api_version + enc.write_i32(101); + enc.write_nullable_string(Some("kafka-cli")); + let bytes = enc.freeze(); + + let header = RequestHeader::decode(bytes, 1).expect("decode should succeed"); + assert_eq!(header.api_key, 18); + assert_eq!(header.api_version, 2); + assert_eq!(header.correlation_id, 101); + assert_eq!(header.client_id.as_deref(), Some("kafka-cli")); +} + +#[test] +fn request_header_v1_null_client_id() { + let mut enc = Encoder::with_capacity(32); + enc.write_i16(18); + enc.write_i16(1); + enc.write_i32(5); + enc.write_nullable_string(None); + let bytes = enc.freeze(); + + let header = RequestHeader::decode(bytes, 1).unwrap(); + assert_eq!(header.client_id, None); +} + +// ── Request header v2 (flexible — compact client_id + tagged fields) ─────── + +#[test] +fn request_header_v2_decodes() { + let mut enc = Encoder::with_capacity(64); + enc.write_i16(18); // api_key: ApiVersions + enc.write_i16(3); // api_version (flexible threshold for ApiVersions is 3) + enc.write_i32(202); + enc.write_compact_nullable_string(Some("my-client")); + enc.write_empty_tagged_fields(); + let bytes = enc.freeze(); + + let header = RequestHeader::decode(bytes, 2).expect("flexible decode should succeed"); + assert_eq!(header.api_key, 18); + assert_eq!(header.api_version, 3); + assert_eq!(header.correlation_id, 202); + assert_eq!(header.client_id.as_deref(), Some("my-client")); +} + +#[test] +fn request_header_v2_null_client_id() { + let mut enc = Encoder::with_capacity(32); + enc.write_i16(18); + enc.write_i16(3); + enc.write_i32(303); + enc.write_compact_nullable_string(None); + enc.write_empty_tagged_fields(); + let bytes = enc.freeze(); + + let header = RequestHeader::decode(bytes, 2).unwrap(); + assert_eq!(header.client_id, None); +} + +// ── Response header encode ────────────────────────────────────────────────── + +#[test] +fn response_header_v0_encodes_correlation_id_only() { + let header = ResponseHeader { correlation_id: 77 }; + let bytes = header.encode(0); + assert_eq!(bytes.as_ref(), &[0, 0, 0, 77]); +} + +#[test] +fn response_header_v1_encodes_correlation_id_plus_tagged_fields() { + let header = ResponseHeader { correlation_id: 1 }; + let bytes = header.encode(1); + // [0,0,0,1] correlation_id + [0x00] empty tagged fields + assert_eq!(bytes.as_ref(), &[0, 0, 0, 1, 0x00]); +} + +// ── Header version lookup ─────────────────────────────────────────────────── + +#[test] +fn request_header_version_non_flexible_below_threshold() { + // ApiVersions v0-2 → header v1 + assert_eq!(request_header_version(18, 0), 1); + assert_eq!(request_header_version(18, 2), 1); + // Metadata v0-8 → header v1 + assert_eq!(request_header_version(3, 0), 1); + assert_eq!(request_header_version(3, 8), 1); +} + +#[test] +fn request_header_version_flexible_at_threshold() { + // ApiVersions v3 → header v2 + assert_eq!(request_header_version(18, 3), 2); + // Metadata v9 → header v2 + assert_eq!(request_header_version(3, 9), 2); + // ConsumerGroupHeartbeat (68) always flexible + assert_eq!(request_header_version(68, 0), 2); +} + +#[test] +fn response_header_version_apiversions_always_zero() { + // ApiVersions is a special case: response header is always v0 + assert_eq!(response_header_version(18, 0), 0); + assert_eq!(response_header_version(18, 3), 0); // even flexible request → v0 response +} + +#[test] +fn response_header_version_flexible_non_apiversions() { + // Metadata v9+ is flexible → response header v1 + assert_eq!(response_header_version(3, 9), 1); + // Metadata v0 is non-flexible → response header v0 + assert_eq!(response_header_version(3, 0), 0); +} diff --git a/gateways/kafka/tests/server_integration_tests.rs b/gateways/kafka/tests/server_integration_tests.rs new file mode 100644 index 0000000000..7a672348ac --- /dev/null +++ b/gateways/kafka/tests/server_integration_tests.rs @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::time::Duration; + +use bytes::{Buf, BytesMut}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::{TcpListener, TcpStream}; + +use iggy_gateway_kafka::protocol::codec::Encoder; +use iggy_gateway_kafka::server::{read_frame, write_frame}; + +async fn tcp_pair() -> (TcpStream, TcpStream) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let client = tokio::spawn(async move { TcpStream::connect(addr).await.unwrap() }); + let (server, _) = listener.accept().await.unwrap(); + let client = client.await.unwrap(); + (client, server) +} + +#[tokio::test] +async fn read_frame_reads_valid_payload() { + let (mut client, mut server) = tcp_pair().await; + + let mut enc = Encoder::with_capacity(64); + enc.write_i16(18); + enc.write_i16(3); + enc.write_i32(123); + enc.write_nullable_string(Some("test-client")); + let payload = enc.freeze(); + + let mut frame = BytesMut::with_capacity(4 + payload.len()); + frame.extend_from_slice(&(payload.len() as i32).to_be_bytes()); + frame.extend_from_slice(&payload); + client.write_all(&frame).await.unwrap(); + + let parsed = read_frame(&mut server, 4096, Duration::from_secs(1)) + .await + .unwrap(); + assert_eq!(parsed, payload); +} + +#[tokio::test] +async fn write_frame_writes_length_prefixed_payload() { + let (mut client, mut server) = tcp_pair().await; + let payload = b"abc123"; + write_frame(&mut server, payload, Duration::from_secs(1)) + .await + .unwrap(); + + let mut len = [0u8; 4]; + client.read_exact(&mut len).await.unwrap(); + let len = i32::from_be_bytes(len) as usize; + assert_eq!(len, payload.len()); + + let mut body = vec![0u8; len]; + client.read_exact(&mut body).await.unwrap(); + assert_eq!(body, payload); +} + +#[tokio::test] +async fn read_frame_rejects_invalid_lengths() { + let (mut client, mut server) = tcp_pair().await; + + client.write_all(&0i32.to_be_bytes()).await.unwrap(); + let err = read_frame(&mut server, 128, Duration::from_secs(1)) + .await + .expect_err("zero frame must fail"); + assert!(err.to_string().contains("invalid frame length")); + + // Ensure connection can still be reused for a second scenario by writing a valid new prefix+payload. + let mut frame = BytesMut::new(); + frame.extend_from_slice(&(200i32).to_be_bytes()); + frame.resize(4 + 200, 0); + client.write_all(&frame).await.unwrap(); + let err = read_frame(&mut server, 64, Duration::from_secs(1)) + .await + .expect_err("large frame must fail"); + assert!(err.to_string().contains("exceeds max frame size")); +} + +#[tokio::test] +async fn write_frame_length_prefix_is_big_endian() { + let (mut client, mut server) = tcp_pair().await; + write_frame(&mut server, &[1, 2, 3, 4], Duration::from_secs(1)) + .await + .unwrap(); + + let mut len_and_data = [0u8; 8]; + client.read_exact(&mut len_and_data).await.unwrap(); + let mut buf = &len_and_data[..]; + let len = buf.get_i32(); + assert_eq!(len, 4); + assert_eq!(&len_and_data[4..], &[1, 2, 3, 4]); +} diff --git a/gateways/kafka/tools/kafka-tool/Cargo.toml b/gateways/kafka/tools/kafka-tool/Cargo.toml new file mode 100644 index 0000000000..a73d8addd9 --- /dev/null +++ b/gateways/kafka/tools/kafka-tool/Cargo.toml @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "kafka-message-gen" +version = "0.1.0" +edition = "2024" +description = "Generates binary Kafka protocol messages for testing the Iggy Kafka gateway" +license = "Apache-2.0" +repository = "https://github.com/apache/iggy" +keywords = ["kafka", "protocol", "testing", "iggy", "wire-format"] +publish = false + +[[bin]] +name = "kafka-message-gen" +path = "src/main.rs" + +[dependencies] +anyhow = { workspace = true } +bytes = { workspace = true } +clap = { workspace = true } +hex = "0.4" +indexmap = "2" +kafka-protocol = "0.17" +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } diff --git a/gateways/kafka/tools/kafka-tool/README.md b/gateways/kafka/tools/kafka-tool/README.md new file mode 100644 index 0000000000..a65eb2a005 --- /dev/null +++ b/gateways/kafka/tools/kafka-tool/README.md @@ -0,0 +1,249 @@ +# kafka-message-gen + +A Rust CLI tool that generates correct, fully-framed Kafka binary wire protocol messages for **every API key** and **every supported version** — built for testing Kafka-compatible server implementations such as [Apache Iggy](https://iggy.apache.org)'s Kafka compatibility listener. + +## What It Does + +Each output `.bin` file is a complete, TCP-ready Kafka request: + +``` +[total_length: i32][api_key: i16][api_version: i16] +[correlation_id: i32][client_id: NULLABLE_STRING] +[tagged_fields: 0x00] ← only for flexible versions +[payload: bytes] ← API-specific encoded body +``` + +The tool covers **65 API keys** and **~280 versioned messages**, sourced directly from the official [Apache Kafka JSON schema files](https://github.com/apache/kafka/tree/trunk/clients/src/main/resources/common/message) (Kafka 4.1.0). + +--- + +## Why This Tool Exists + +When implementing a Kafka protocol compatibility layer (e.g. inside Apache Iggy), you need to: + +1. Verify your server correctly **parses** every API key at every version +2. Verify your server sends **valid responses** back (correct correlation ID, error codes) +3. Do this without spinning up a full Kafka client or running JVM-based tests + +This tool solves all three: generate the binary messages once, then `cat` them directly to your server's port 9092 and inspect the response. + +--- + +## Dependency on `kafka-protocol` + +This tool is built on the [`kafka-protocol`](https://crates.io/crates/kafka-protocol) Rust crate, which is itself **code-generated from Kafka's official JSON schema files**. This ensures byte-perfect correctness — the same schemas that generate Kafka's own Java serialization code generate the Rust structs used here. + +--- + +## Installation + +```bash +cd gateways/kafka/tools/kafka-tool +cargo build --release +# Binary is at: ./target/release/kafka-message-gen +``` + +**Requirements:** Rust 1.75+ (MSRV follows `kafka-protocol` crate) + +--- + +## Usage + +### List all API keys and version ranges + +```bash +cargo run -- list +``` + +Output: +``` +Key Name MinVer MaxVer Count +────────────────────────────────────────────────────────────────────────────── +0 Produce 3 13 11 +1 Fetch 4 18 15 +2 ListOffsets 1 11 11 +3 Metadata 0 13 14 +8 OffsetCommit 2 10 9 +... +────────────────────────────────────────────────────────────────────────────── +Total: 65 API keys | ~280 versioned messages +``` + +--- + +### Generate all binary messages + +```bash +cargo run -- generate --output ./kafka_messages/ +``` + +Creates one `.bin` file per API key × version: + +``` +kafka_messages/ + 000_Produce_v3.bin + 000_Produce_v4.bin + ... + 000_Produce_v13.bin + 001_Fetch_v4.bin + ... + 003_Metadata_v0.bin + 003_Metadata_v13.bin + 018_ApiVersions_v0.bin + 018_ApiVersions_v3.bin + ... +``` + +#### Options + +| Flag | Description | Default | +|------|-------------|---------| +| `--output` | Output directory | `kafka_messages/` | +| `--api-key N` | Generate only for API key N | all | +| `--version N` | Generate only for version N | all | +| `--hex` | Print hex dump to stdout | off | + +```bash +# Generate only Metadata messages +cargo run -- generate --api-key 3 + +# Generate only ApiVersions v3 with hex dump +cargo run -- generate --api-key 18 --version 3 --hex +``` + +--- + +### Send messages to a live server + +```bash +# Start Iggy with Kafka compat listener on port 9092, then: +cargo run -- send --host 127.0.0.1:9092 +``` + +Output (one line per API key × version): +``` +✓ ApiVersions v3 → 32 bytes ec=0 +✓ Metadata v12 → 148 bytes ec=0 +⚠ Produce v9 → 24 bytes ec=3 ← ec=3 = UnknownTopicOrPartition (expected) +✓ Fetch v12 → 36 bytes ec=0 +... +Result: 243 OK 37 failed +``` + +#### Options + +| Flag | Description | Default | +|------|-------------|---------| +| `--host` | Server address | `127.0.0.1:9092` | +| `--api-key N` | Test only API key N | all | +| `--version N` | Test only version N | all | +| `--timeout-ms N` | Per-request timeout | `5000` | + +--- + +### Verify compatibility (CI-friendly) + +```bash +cargo run -- verify --host 127.0.0.1:9092 +``` + +Exits with code **0** if all messages get a response, **1** if any fail (timeout or IO error). Useful in CI pipelines testing a Kafka-compatible server implementation. + +--- + +### Quick raw test with netcat + +No Rust needed for a quick smoke test: + +```bash +# Generate first +cargo run -- generate + +# Send ApiVersions v3 directly via netcat and inspect response +cat kafka_messages/018_ApiVersions_v3.bin | nc 127.0.0.1 9092 | xxd | head + +# Send and decode with Wireshark (capture on loopback, filter: kafka) +``` + +--- + +## Supported API Keys (Kafka 4.1.0) + +| Key | Name | Versions | Phase 1 Priority | +|-----|------|----------|-----------------| +| 0 | Produce | v3–v13 | ✅ Critical | +| 1 | Fetch | v4–v18 | ✅ Critical | +| 2 | ListOffsets | v1–v11 | ✅ Critical | +| 3 | Metadata | v0–v13 | ✅ Critical | +| 8 | OffsetCommit | v2–v10 | ✅ Critical | +| 9 | OffsetFetch | v1–v10 | ✅ Critical | +| 10 | FindCoordinator | v0–v6 | ✅ Critical | +| 11 | JoinGroup | v0–v9 | ✅ Critical | +| 12 | Heartbeat | v0–v4 | ✅ Critical | +| 13 | LeaveGroup | v0–v5 | ✅ Critical | +| 14 | SyncGroup | v0–v5 | ✅ Critical | +| 15 | DescribeGroups | v0–v6 | 🟡 Important | +| 16 | ListGroups | v0–v5 | 🟡 Important | +| 17 | SaslHandshake | v0–v1 | 🟡 Important | +| 18 | ApiVersions | v0–v5 | ✅ Critical | +| 19 | CreateTopics | v2–v7 | ✅ Critical | +| 20 | DeleteTopics | v1–v6 | 🟡 Important | +| 21 | DeleteRecords | v0–v2 | 🔵 Phase 2 | +| 22 | InitProducerId | v0–v6 | 🔵 Phase 2 | +| 24 | AddPartitionsToTxn | v0–v5 | 🔵 Phase 2 | +| 25 | AddOffsetsToTxn | v0–v4 | 🔵 Phase 2 | +| 26 | EndTxn | v0–v5 | 🔵 Phase 2 | +| 28 | TxnOffsetCommit | v0–v5 | 🔵 Phase 2 | +| 29–31 | ACL APIs | v1–v3 | 🔵 Phase 2 | +| 32 | DescribeConfigs | v1–v4 | 🟡 Important | +| 36 | SaslAuthenticate | v0–v2 | 🟡 Important | +| ... | 40+ more | various | 🔵 Phase 3 | + +--- + +## Project Structure + +``` +tools/kafka-tool/ +├── Cargo.toml ← package manifest and dependencies +├── src/ +│ └── main.rs ← complete CLI implementation +└── README.md ← this file +``` + +--- + +## How It Works + +### Protocol Source + +All API schemas come from the official Kafka repository: +`apache/kafka/trunk/clients/src/main/resources/common/message/*.json` + +The `kafka-protocol` crate processes these JSON files and generates Rust structs with `encode()` and `decode()` methods. This guarantees byte-level compatibility with what official Kafka clients send. + +### Flexible vs Legacy Encoding + +Kafka introduced "flexible" encoding (compact ULEB128 strings/arrays) starting at different versions per API. The tool automatically detects whether a version uses flexible or legacy encoding and sets the request header format accordingly (header v1 for legacy, header v2 for flexible with tagged fields section). + +### API Key Coverage + +- **Explicit builders (23 API keys):** Produce, Fetch, ListOffsets, Metadata, OffsetCommit, OffsetFetch, FindCoordinator, JoinGroup, Heartbeat, LeaveGroup, SyncGroup, DescribeGroups, ListGroups, SaslHandshake, ApiVersions, CreateTopics, DeleteTopics, DeleteRecords, InitProducerId, AddPartitionsToTxn, AddOffsetsToTxn, EndTxn, TxnOffsetCommit, DescribeConfigs, SaslAuthenticate +- **Header-framing test (42 API keys):** All remaining API keys are framed correctly with an empty payload — useful for testing that your server returns a proper error response rather than crashing + +--- + +## Contributing + +The tool is intentionally simple. To add an explicit builder for a new API key: + +1. Find the JSON schema in `apache/kafka/.../message/YourRequest.json` +2. Add a new match arm in `build_payload()` in `src/main.rs` +3. Use the kafka-protocol crate's generated struct (e.g. `YourRequest::default().with_field(value)`) +4. Open a PR + +--- + +## License + +Apache License 2.0 — same as Apache Kafka and Apache Iggy. diff --git a/gateways/kafka/tools/kafka-tool/src/main.rs b/gateways/kafka/tools/kafka-tool/src/main.rs new file mode 100644 index 0000000000..e1b29f08b7 --- /dev/null +++ b/gateways/kafka/tools/kafka-tool/src/main.rs @@ -0,0 +1,770 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use anyhow::{Context, Result}; +use bytes::{BufMut, Bytes, BytesMut}; +use clap::{Parser, Subcommand}; +use kafka_protocol::messages::*; +use kafka_protocol::protocol::{Encodable, StrBytes}; +use std::path::PathBuf; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpStream; +use tracing::{info, warn}; + +#[derive(Parser)] +#[command( + name = "kafka-message-gen", + about = "Generate Kafka wire protocol binary messages for all API keys and versions", + long_about = "Generates correctly-framed Kafka protocol requests from Kafka 4.1.0 schemas.\n\ +Each output .bin file is TCP-ready: [len:i32][api_key:i16][api_version:i16][correlation_id:i32][client_id][payload]" +)] +struct Cli { + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +enum Command { + /// List all supported API keys with name and version range + List, + /// Generate binary .bin files for all API keys and versions + Generate { + #[arg(short, long, default_value = "kafka_messages")] + output: PathBuf, + /// Filter to a single API key integer + #[arg(long)] + api_key: Option, + /// Filter to a single version + #[arg(long)] + version: Option, + /// Print hex dump to stdout + #[arg(long)] + hex: bool, + }, + /// Send messages to a live Kafka-compatible server and show responses + Send { + #[arg(long, default_value = "127.0.0.1:9092")] + host: String, + #[arg(long)] + api_key: Option, + #[arg(long)] + version: Option, + #[arg(long, default_value = "5000")] + timeout_ms: u64, + }, + /// Send all messages and report pass/fail — exit code 1 if any fail + Verify { + #[arg(long, default_value = "127.0.0.1:9092")] + host: String, + #[arg(long)] + fail_fast: bool, + }, +} + +// ── API Registry ───────────────────────────────────────────────────────────── +// Source: validVersions in apache/kafka trunk JSON schema files, Kafka 4.1.0 +// Format: (api_key, name, min_version, max_version) +const API_REGISTRY: &[(i16, &str, i16, i16)] = &[ + (0, "Produce", 3, 13), + (1, "Fetch", 4, 18), + (2, "ListOffsets", 1, 11), + (3, "Metadata", 0, 13), + (8, "OffsetCommit", 2, 10), + (9, "OffsetFetch", 1, 10), + (10, "FindCoordinator", 0, 6), + (11, "JoinGroup", 0, 9), + (12, "Heartbeat", 0, 4), + (13, "LeaveGroup", 0, 5), + (14, "SyncGroup", 0, 5), + (15, "DescribeGroups", 0, 6), + (16, "ListGroups", 0, 5), + (17, "SaslHandshake", 0, 1), + (18, "ApiVersions", 0, 5), + (19, "CreateTopics", 2, 7), + (20, "DeleteTopics", 1, 6), + (21, "DeleteRecords", 0, 2), + (22, "InitProducerId", 0, 6), + (23, "OffsetForLeaderEpoch", 2, 4), + (24, "AddPartitionsToTxn", 0, 5), + (25, "AddOffsetsToTxn", 0, 4), + (26, "EndTxn", 0, 5), + (27, "WriteTxnMarkers", 1, 2), + (28, "TxnOffsetCommit", 0, 5), + (29, "DescribeAcls", 1, 3), + (30, "CreateAcls", 1, 3), + (31, "DeleteAcls", 1, 3), + (32, "DescribeConfigs", 1, 4), + (33, "AlterConfigs", 0, 2), + (34, "AlterReplicaLogDirs", 1, 2), + (35, "DescribeLogDirs", 1, 5), + (36, "SaslAuthenticate", 0, 2), + (37, "CreatePartitions", 0, 3), + (38, "CreateDelegationToken", 1, 3), + (39, "RenewDelegationToken", 1, 2), + (40, "ExpireDelegationToken", 1, 2), + (41, "DescribeDelegationToken", 1, 3), + (42, "DeleteGroups", 0, 2), + (43, "ElectLeaders", 0, 2), + (44, "IncrementalAlterConfigs", 0, 1), + (45, "AlterPartitionReassignments", 0, 1), + (46, "ListPartitionReassignments", 0, 1), + (47, "OffsetDelete", 0, 0), + (48, "DescribeClientQuotas", 0, 1), + (49, "AlterClientQuotas", 0, 1), + (50, "DescribeUserScramCredentials", 0, 0), + (51, "AlterUserScramCredentials", 0, 0), + (55, "DescribeQuorum", 2, 3), + (56, "AlterPartition", 2, 3), + (57, "UpdateFeatures", 0, 2), + (60, "DescribeCluster", 0, 2), + (61, "DescribeProducers", 0, 0), + (64, "UnregisterBroker", 0, 0), + (65, "DescribeTransactions", 0, 0), + (66, "ListTransactions", 0, 1), + (67, "AllocateProducerIds", 0, 0), + (68, "ConsumerGroupHeartbeat", 0, 1), + (69, "ConsumerGroupDescribe", 0, 1), + (71, "GetTelemetrySubscriptions", 0, 0), + (72, "PushTelemetry", 0, 0), + (74, "AssignReplicasToDirs", 0, 0), + (75, "DescribeTopicPartitions", 0, 0), + (76, "ListClientMetricsResources", 0, 0), +]; + +// ── Flexible version table ──────────────────────────────────────────────────── +// Source: flexibleVersions field in each Kafka JSON schema. +// Returns the first version using compact encoding, or None if never flexible. +fn first_flexible_version(api_key: i16) -> Option { + match api_key { + 0 => Some(9), + 1 => Some(12), + 2 => Some(6), + 3 => Some(9), + 8 => Some(8), + 9 => Some(6), + 10 => Some(3), + 11 => Some(6), + 12 => Some(4), + 13 => Some(4), + 14 => Some(4), + 15 => Some(5), + 16 => Some(3), + 17 => None, + 18 => Some(3), + 19 => Some(5), + 20 => Some(4), + 21 => Some(2), + 22 => Some(2), + 23 => Some(4), + 24 => Some(3), + 25 => Some(3), + 26 => Some(3), + 27 => Some(1), + 28 => Some(3), + 29 => Some(2), + 30 => Some(2), + 31 => Some(2), + 32 => Some(4), + 33 => Some(2), + 34 => Some(2), + 35 => Some(2), + 36 => Some(2), + 37 => Some(2), + 38 => Some(2), + 39 => Some(2), + 40 => Some(2), + 41 => Some(2), + 42 => Some(2), + 43 => Some(2), + 44 => Some(1), + 45 => Some(1), + 46 => Some(1), + 47 => Some(0), + 48 => Some(1), + 49 => Some(1), + 50 => Some(0), + 51 => Some(0), + 55 => Some(2), + 56 => Some(2), + 57 => Some(1), + 60 => Some(0), + 61 => Some(0), + 64 => Some(0), + 65 => Some(0), + 66 => Some(0), + 67 => Some(0), + 68 => Some(0), + 69 => Some(0), + 71 => Some(0), + 72 => Some(0), + 74 => Some(0), + 75 => Some(0), + 76 => Some(0), + _ => None, + } +} + +// ── Request framing ─────────────────────────────────────────────────────────── +// Wire format (Kafka protocol spec): +// [total_length: i32] big-endian, excludes self +// [api_key: i16] +// [api_version: i16] +// [correlation_id: i32] +// [client_id_len: i16] -1 = null +// [client_id: bytes] +// [tagged_fields: u8(0)] only present for flexible versions +// [payload: bytes] +fn frame_request( + api_key: i16, + api_version: i16, + correlation_id: i32, + client_id: &str, + payload: &[u8], + flexible: bool, +) -> Bytes { + let cid = client_id.as_bytes(); + let hlen = 2 + 2 + 4 + 2 + cid.len() + if flexible { 1 } else { 0 }; + let blen = hlen + payload.len(); + let mut buf = BytesMut::with_capacity(4 + blen); + buf.put_i32(blen as i32); + buf.put_i16(api_key); + buf.put_i16(api_version); + buf.put_i32(correlation_id); + buf.put_i16(cid.len() as i16); + buf.put_slice(cid); + if flexible { + buf.put_u8(0x00); + } + buf.put_slice(payload); + buf.freeze() +} + +// ── Payload builders ────────────────────────────────────────────────────────── +// Build the API-specific encoded body for a given api_key and version. +// All required fields contain realistic non-zero values. +// Returns raw bytes WITHOUT the framing header. +fn build_payload(api_key: i16, version: i16) -> Result { + let mut buf = BytesMut::new(); + match api_key { + 18 => { + let mut r = ApiVersionsRequest::default(); + if version >= 3 { + r.client_software_name = StrBytes::from_static_str("kafka-message-gen"); + r.client_software_version = StrBytes::from_static_str("0.1.0"); + } + r.encode(&mut buf, version).context("ApiVersions")?; + } + 3 => { + let mut r = MetadataRequest::default(); + if version >= 1 { + r.topics = None; + } + if version >= 4 { + r.allow_auto_topic_creation = true; + } + if version >= 8 { + r.include_cluster_authorized_operations = false; + r.include_topic_authorized_operations = false; + } + r.encode(&mut buf, version).context("Metadata")?; + } + 0 => { + use kafka_protocol::messages::produce_request::*; + use kafka_protocol::records::{ + Compression, Record, RecordBatchEncoder, RecordEncodeOptions, TimestampType, + }; + let rec = Record { + transactional: false, + control: false, + partition_leader_epoch: 0, + producer_id: -1, + producer_epoch: -1, + timestamp_type: TimestampType::Creation, + offset: 0, + sequence: 0, + timestamp: 1_700_000_000_000, + key: Some(Bytes::from_static(b"test-key")), + value: Some(Bytes::from_static(b"test-value")), + headers: indexmap::IndexMap::new(), + }; + let mut rb = BytesMut::new(); + RecordBatchEncoder::encode( + &mut rb, + [rec].iter(), + &RecordEncodeOptions { + version: 2, + compression: Compression::None, + }, + ) + .context("RecordBatch encode")?; + let pd = TopicProduceData::default() + .with_name(TopicName::from(StrBytes::from_static_str("test-topic"))) + .with_partition_data(vec![ + PartitionProduceData::default() + .with_index(0) + .with_records(Some(rb.freeze())), + ]); + let mut r = ProduceRequest::default() + .with_acks(-1) + .with_timeout_ms(5000) + .with_topic_data(vec![pd]); + if version >= 3 { + r.transactional_id = None; + } + r.encode(&mut buf, version).context("Produce")?; + } + 1 => { + use kafka_protocol::messages::fetch_request::*; + let fp = FetchPartition::default() + .with_partition(0) + .with_fetch_offset(0) + .with_partition_max_bytes(1_048_576); + let ft = FetchTopic::default() + .with_topic(TopicName::from(StrBytes::from_static_str("test-topic"))) + .with_partitions(vec![fp]); + let mut r = FetchRequest::default() + .with_replica_id(BrokerId(-1)) + .with_max_wait_ms(500) + .with_min_bytes(1) + .with_topics(vec![ft]); + if version >= 3 { + r.max_bytes = 52_428_800; + } + if version >= 4 { + r.isolation_level = 0; + } + if version >= 7 { + r.session_id = 0; + r.session_epoch = -1; + } + r.encode(&mut buf, version).context("Fetch")?; + } + 2 => { + use kafka_protocol::messages::list_offsets_request::*; + let p = ListOffsetsPartition::default() + .with_partition_index(0) + .with_timestamp(-1); + let t = ListOffsetsTopic::default() + .with_name(TopicName::from(StrBytes::from_static_str("test-topic"))) + .with_partitions(vec![p]); + ListOffsetsRequest::default() + .with_replica_id(BrokerId(-1)) + .with_isolation_level(0) + .with_topics(vec![t]) + .encode(&mut buf, version) + .context("ListOffsets")?; + } + 8 => { + use kafka_protocol::messages::offset_commit_request::*; + let p = OffsetCommitRequestPartition::default() + .with_partition_index(0) + .with_committed_offset(42) + .with_committed_metadata(Some(StrBytes::from_static_str(""))); + let t = OffsetCommitRequestTopic::default() + .with_name(TopicName::from(StrBytes::from_static_str("test-topic"))) + .with_partitions(vec![p]); + OffsetCommitRequest::default() + .with_group_id(GroupId::from(StrBytes::from_static_str("test-group"))) + .with_topics(vec![t]) + .encode(&mut buf, version) + .context("OffsetCommit")?; + } + 9 => { + OffsetFetchRequest::default() + .with_group_id(GroupId::from(StrBytes::from_static_str("test-group"))) + .encode(&mut buf, version) + .context("OffsetFetch")?; + } + 10 => { + FindCoordinatorRequest::default() + .with_key(StrBytes::from_static_str("test-group")) + .with_key_type(0) + .encode(&mut buf, version) + .context("FindCoordinator")?; + } + 11 => { + use kafka_protocol::messages::join_group_request::*; + let p = JoinGroupRequestProtocol::default() + .with_name(StrBytes::from_static_str("range")) + .with_metadata(Bytes::from_static(b"\x00\x00\x00\x01\x00\x0atest-topic")); + JoinGroupRequest::default() + .with_group_id(GroupId::from(StrBytes::from_static_str("test-group"))) + .with_session_timeout_ms(30_000) + .with_rebalance_timeout_ms(300_000) + .with_member_id(StrBytes::from_static_str("")) + .with_protocol_type(StrBytes::from_static_str("consumer")) + .with_protocols(vec![p]) + .encode(&mut buf, version) + .context("JoinGroup")?; + } + 12 => { + HeartbeatRequest::default() + .with_group_id(GroupId::from(StrBytes::from_static_str("test-group"))) + .with_generation_id(1) + .with_member_id(StrBytes::from_static_str("test-member-1")) + .encode(&mut buf, version) + .context("Heartbeat")?; + } + 13 => { + LeaveGroupRequest::default() + .with_group_id(GroupId::from(StrBytes::from_static_str("test-group"))) + .with_member_id(StrBytes::from_static_str("test-member-1")) + .encode(&mut buf, version) + .context("LeaveGroup")?; + } + 14 => { + SyncGroupRequest::default() + .with_group_id(GroupId::from(StrBytes::from_static_str("test-group"))) + .with_generation_id(1) + .with_member_id(StrBytes::from_static_str("test-member-1")) + .with_protocol_type(Some(StrBytes::from_static_str("consumer"))) + .with_protocol_name(Some(StrBytes::from_static_str("range"))) + .encode(&mut buf, version) + .context("SyncGroup")?; + } + 15 => { + DescribeGroupsRequest::default() + .with_groups(vec![GroupId::from(StrBytes::from_static_str("test-group"))]) + .with_include_authorized_operations(false) + .encode(&mut buf, version) + .context("DescribeGroups")?; + } + 16 => { + ListGroupsRequest::default() + .encode(&mut buf, version) + .context("ListGroups")?; + } + 17 => { + SaslHandshakeRequest::default() + .with_mechanism(StrBytes::from_static_str("PLAIN")) + .encode(&mut buf, version) + .context("SaslHandshake")?; + } + 19 => { + use kafka_protocol::messages::create_topics_request::*; + let t = CreatableTopic::default() + .with_name(TopicName::from(StrBytes::from_static_str( + "iggy-test-topic", + ))) + .with_num_partitions(1) + .with_replication_factor(1); + CreateTopicsRequest::default() + .with_topics(vec![t]) + .with_timeout_ms(30_000) + .with_validate_only(false) + .encode(&mut buf, version) + .context("CreateTopics")?; + } + 20 => { + use kafka_protocol::messages::delete_topics_request::*; + let r = if version >= 6 { + DeleteTopicsRequest::default() + .with_topics(vec![DeleteTopicState::default().with_name(Some( + TopicName::from(StrBytes::from_static_str("iggy-test-topic")), + ))]) + .with_timeout_ms(30_000) + } else { + DeleteTopicsRequest::default() + .with_topic_names(vec![TopicName::from(StrBytes::from_static_str( + "iggy-test-topic", + ))]) + .with_timeout_ms(30_000) + }; + r.encode(&mut buf, version).context("DeleteTopics")?; + } + 21 => { + use kafka_protocol::messages::delete_records_request::*; + let p = DeleteRecordsPartition::default() + .with_partition_index(0) + .with_offset(0); + let t = DeleteRecordsTopic::default() + .with_name(TopicName::from(StrBytes::from_static_str("test-topic"))) + .with_partitions(vec![p]); + DeleteRecordsRequest::default() + .with_topics(vec![t]) + .with_timeout_ms(30_000) + .encode(&mut buf, version) + .context("DeleteRecords")?; + } + 22 => { + InitProducerIdRequest::default() + .with_transactional_id(None) + .with_transaction_timeout_ms(60_000) + .encode(&mut buf, version) + .context("InitProducerId")?; + } + 24 => { + use kafka_protocol::messages::add_partitions_to_txn_request::*; + let t = AddPartitionsToTxnTopic::default() + .with_name(TopicName::from(StrBytes::from_static_str("test-topic"))) + .with_partitions(vec![0i32]); + AddPartitionsToTxnRequest::default() + .with_v3_and_below_transactional_id(TransactionalId(StrBytes::from_static_str( + "test-txn", + ))) + .with_v3_and_below_producer_id(ProducerId(100)) + .with_v3_and_below_producer_epoch(1) + .with_v3_and_below_topics(vec![t]) + .encode(&mut buf, version) + .context("AddPartitionsToTxn")?; + } + 25 => { + AddOffsetsToTxnRequest::default() + .with_transactional_id(TransactionalId(StrBytes::from_static_str("test-txn"))) + .with_producer_id(ProducerId(100)) + .with_producer_epoch(1) + .with_group_id(GroupId::from(StrBytes::from_static_str("test-group"))) + .encode(&mut buf, version) + .context("AddOffsetsToTxn")?; + } + 26 => { + EndTxnRequest::default() + .with_transactional_id(TransactionalId(StrBytes::from_static_str("test-txn"))) + .with_producer_id(ProducerId(100)) + .with_producer_epoch(1) + .with_committed(true) + .encode(&mut buf, version) + .context("EndTxn")?; + } + 28 => { + use kafka_protocol::messages::txn_offset_commit_request::*; + let p = TxnOffsetCommitRequestPartition::default() + .with_partition_index(0) + .with_committed_offset(42) + .with_committed_metadata(Some(StrBytes::from_static_str(""))); + let t = TxnOffsetCommitRequestTopic::default() + .with_name(TopicName::from(StrBytes::from_static_str("test-topic"))) + .with_partitions(vec![p]); + TxnOffsetCommitRequest::default() + .with_transactional_id(TransactionalId(StrBytes::from_static_str("test-txn"))) + .with_group_id(GroupId::from(StrBytes::from_static_str("test-group"))) + .with_producer_id(ProducerId(100)) + .with_producer_epoch(1) + .with_topics(vec![t]) + .encode(&mut buf, version) + .context("TxnOffsetCommit")?; + } + 32 => { + use kafka_protocol::messages::describe_configs_request::*; + let r = DescribeConfigsResource::default() + .with_resource_type(2) + .with_resource_name(StrBytes::from_static_str("test-topic")); + DescribeConfigsRequest::default() + .with_resources(vec![r]) + .encode(&mut buf, version) + .context("DescribeConfigs")?; + } + 36 => { + SaslAuthenticateRequest::default() + .with_auth_bytes(Bytes::from_static(b"\x00iggy\x00secret")) + .encode(&mut buf, version) + .context("SaslAuthenticate")?; + } + other => { + warn!("api_key={other}: no explicit builder — empty payload (framing test)"); + } + } + Ok(buf.freeze()) +} + +// Build a complete framed Kafka request message ready for TCP transmission. +fn build_framed(api_key: i16, version: i16, corr: i32) -> Result { + let payload = build_payload(api_key, version)?; + let flexible = first_flexible_version(api_key) + .map(|fv| version >= fv) + .unwrap_or(false); + Ok(frame_request( + api_key, + version, + corr, + "kafka-message-gen", + &payload, + flexible, + )) +} + +// ── Commands ────────────────────────────────────────────────────────────────── + +fn cmd_list() { + println!( + "{:<6} {:<42} {:<10} {:<10} {:<8}", + "Key", "Name", "MinVer", "MaxVer", "Count" + ); + println!("{}", "─".repeat(78)); + for &(k, n, min, max) in API_REGISTRY { + println!( + "{:<6} {:<42} {:<10} {:<10} {:<8}", + k, + n, + min, + max, + max - min + 1 + ); + } + let total: i16 = API_REGISTRY + .iter() + .map(|&(_, _, min, max)| max - min + 1) + .sum(); + println!("{}", "─".repeat(78)); + println!( + "Total: {} API keys | {} versioned messages", + API_REGISTRY.len(), + total + ); +} + +async fn cmd_generate( + out: PathBuf, + fk: Option, + fv: Option, + hex_dump: bool, +) -> Result<()> { + tokio::fs::create_dir_all(&out).await?; + let (mut n, mut corr) = (0usize, 1i32); + for &(ak, name, min, max) in API_REGISTRY { + if fk.is_some_and(|k| k != ak) { + continue; + } + for v in min..=max { + if fv.is_some_and(|fv| fv != v) { + continue; + } + match build_framed(ak, v, corr) { + Ok(msg) => { + let fname = format!("{:03}_{}_v{}.bin", ak, name, v); + tokio::fs::write(out.join(&fname), &msg).await?; + if hex_dump { + println!("── {} v{} ({} bytes) ──", name, v, msg.len()); + println!("{}", hex::encode(&msg)); + println!(); + } else { + info!(" {} ({} bytes)", fname, msg.len()); + } + n += 1; + corr += 1; + } + Err(e) => warn!("SKIP {} v{}: {e}", name, v), + } + } + } + println!("\n✓ Generated {n} messages → {}/", out.display()); + println!( + " Quick test: cat {}/018_ApiVersions_v3.bin | nc 127.0.0.1 9092 | xxd", + out.display() + ); + Ok(()) +} + +async fn run_send( + host: &str, + fk: Option, + fv: Option, + toms: u64, +) -> Result<(usize, usize)> { + let mut stream = TcpStream::connect(host) + .await + .with_context(|| format!("Cannot connect to {host}"))?; + info!("Connected to {host}"); + let (mut ok, mut fail, mut corr) = (0usize, 0usize, 1i32); + for &(ak, name, min, max) in API_REGISTRY { + if fk.is_some_and(|k| k != ak) { + continue; + } + for v in min..=max { + if fv.is_some_and(|fv| fv != v) { + continue; + } + let msg = match build_framed(ak, v, corr) { + Ok(m) => m, + Err(e) => { + warn!("Build {} v{}: {e}", name, v); + fail += 1; + continue; + } + }; + stream + .write_all(&msg) + .await + .with_context(|| format!("Write {} v{}", name, v))?; + let res = tokio::time::timeout(std::time::Duration::from_millis(toms), async { + let mut lb = [0u8; 4]; + stream.read_exact(&mut lb).await?; + let mut body = vec![0u8; i32::from_be_bytes(lb) as usize]; + stream.read_exact(&mut body).await?; + Ok::, std::io::Error>(body) + }) + .await; + match res { + Ok(Ok(r)) => { + let ec = if r.len() >= 6 { + i16::from_be_bytes(r[4..6].try_into().unwrap()) + } else { + -1 + }; + let sym = if ec <= 0 { "✓" } else { "⚠" }; + println!("{sym} {} v{} → {} bytes ec={ec}", name, v, r.len()); + ok += 1; + } + Ok(Err(e)) => { + println!("✗ {} v{} → IO error: {e}", name, v); + fail += 1; + } + Err(_) => { + println!("✗ {} v{} → timeout ({}ms)", name, v, toms); + fail += 1; + } + } + corr += 1; + } + } + Ok((ok, fail)) +} + +#[tokio::main] +async fn main() -> Result<()> { + tracing_subscriber::fmt() + .with_target(false) + .with_level(true) + .init(); + let cli = Cli::parse(); + match cli.command { + Command::List => cmd_list(), + Command::Generate { + output, + api_key, + version, + hex, + } => cmd_generate(output, api_key, version, hex).await?, + Command::Send { + host, + api_key, + version, + timeout_ms, + } => { + let (ok, fail) = run_send(&host, api_key, version, timeout_ms).await?; + println!("\nResult: {ok} OK {fail} failed"); + } + Command::Verify { host, .. } => { + let (ok, fail) = run_send(&host, None, None, 5000).await?; + println!("\n=== Verify: {ok} passed {fail} failed ==="); + if fail > 0 { + std::process::exit(1); + } + } + } + Ok(()) +} From 905584ecf98ab570c54e029c99eae896e8812fc2 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sun, 7 Jun 2026 08:25:43 -0400 Subject: [PATCH 05/57] kafka: Initial version - codec improvements, broker & server Add tokio-util dependency and implement BrokerAdvertise to advertise host/port from server bind address. Replace dynamic supported_api_ranges with a static table and pass BrokerAdvertise into handler so metadata responses include the advertised broker. Harden codec primitives: add MAX_COLLECTION_LEN, checked read_i32_array_count and read_compact_array_count, tagged-field bounds checks, better varint validation, and string-length checks (write_nullable_string returns Result). Update response encoders to take references and use checked conversions for counts. Improve server: use tokio_util::TaskTracker for graceful shutdown, handle transient accept errors, return error-only response for unsupported request header versions, extract correlation id, and add safer frame read/write size checks. Update docs and tests to reflect compact/flexible encoding and new APIs. --- Cargo.lock | 1 + gateways/kafka/Cargo.toml | 1 + gateways/kafka/docs/SCOPE.md | 44 +++- gateways/kafka/src/error.rs | 8 + gateways/kafka/src/lib.rs | 9 - gateways/kafka/src/protocol/api.rs | 219 ++++++++++++------ gateways/kafka/src/protocol/codec.rs | 65 +++++- gateways/kafka/src/protocol/header.rs | 6 + gateways/kafka/src/protocol/requests.rs | 50 ++-- gateways/kafka/src/protocol/responses.rs | 89 ++++--- gateways/kafka/src/server.rs | 107 +++++++-- gateways/kafka/tests/api_handler_tests.rs | 20 +- gateways/kafka/tests/codec_tests.rs | 4 +- .../kafka/tests/decode_validation_tests.rs | 22 +- .../kafka/tests/golden_wire_fixtures_tests.rs | 9 +- gateways/kafka/tests/header_tests.rs | 4 +- .../kafka/tests/server_integration_tests.rs | 2 +- 17 files changed, 463 insertions(+), 197 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f33cf2a42c..84128beb61 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7180,6 +7180,7 @@ dependencies = [ "bytes", "thiserror 2.0.18", "tokio", + "tokio-util", "tracing", "tracing-subscriber", ] diff --git a/gateways/kafka/Cargo.toml b/gateways/kafka/Cargo.toml index 6bfcb0e34d..93a335d68a 100644 --- a/gateways/kafka/Cargo.toml +++ b/gateways/kafka/Cargo.toml @@ -36,6 +36,7 @@ path = "src/main.rs" bytes = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "io-util", "time", "sync", "signal"] } +tokio-util = { workspace = true, features = ["rt"] } tracing = { workspace = true } tracing-subscriber = { workspace = true } diff --git a/gateways/kafka/docs/SCOPE.md b/gateways/kafka/docs/SCOPE.md index b56a172730..8d2be24623 100644 --- a/gateways/kafka/docs/SCOPE.md +++ b/gateways/kafka/docs/SCOPE.md @@ -2,20 +2,46 @@ This gateway iteration implements **wire validation and stub responses only** (no Iggy backend, no real broker semantics). +Source of truth in code: `SUPPORTED_RANGES` in [`src/protocol/api.rs`](../src/protocol/api.rs). + ## Supported API keys and versions -| API key | Name | Min version | Max version | Behavior | -|---------|------|-------------|-------------|----------| -| 18 | ApiVersions | 0 | 3 | Advertise supported ranges; flexible encoding at v3+ | -| 3 | Metadata | 0 | 9 | Decode request; stub broker `127.0.0.1:9093` | -| 0 | Produce | 3 | 9 | Decode request; stub response | -| 1 | Fetch | 4 | 12 | Decode request; stub response | -| 2 | ListOffsets | 1 | 6 | Decode request; stub response | -| 19 | CreateTopics | 2 | 5 | Decode request; stub response | +| API key | Name | Min version | Max version | Valid versions | Behavior | +|---------|------|-------------|-------------|----------------|----------| +| 18 | ApiVersions | 0 | 3 | 0, 1, 2, 3 | Advertise supported ranges; flexible encoding at v3+ | +| 3 | Metadata | 0 | 9 | 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 | Decode request; stub broker from `ServerConfig.bind_addr`; flexible encoding at v9+ | +| 0 | Produce | 3 | 9 | 3, 4, 5, 6, 7, 8, 9 | Decode request; stub response | +| 1 | Fetch | 4 | 12 | 4, 5, 6, 7, 8, 9, 10, 11, 12 | Decode request; stub response | +| 2 | ListOffsets | 1 | 6 | 1, 2, 3, 4, 5, 6 | Decode request; stub response | +| 19 | CreateTopics | 2 | 5 | 2, 3, 4, 5 | Decode request; stub response | + +A request is accepted when `min_version ≤ api_version ≤ max_version` for that API key. Any other version for a listed key, or any unlisted API key, receives `UNSUPPORTED_VERSION` (35). + +## Valid versions reference (by API key) + +Use this table when configuring clients or generating wire fixtures with `kafka-message-gen`. + +| API key | Name | Valid versions (inclusive range) | Flexible wire encoding from | +|---------|------|----------------------------------|----------------------------| +| 0 | Produce | 3–9 | v9 | +| 1 | Fetch | 4–12 | v12 | +| 2 | ListOffsets | 1–6 | v6 | +| 3 | Metadata | 0–9 | v9 | +| 18 | ApiVersions | 0–3 | v3 | +| 19 | CreateTopics | 2–5 | v5 | ## Unsupported API keys -All other API keys receive an error-only response with `UNSUPPORTED_VERSION` (35). +All API keys not listed above receive an error-only response with `UNSUPPORTED_VERSION` (35). Examples not in this foundation scope: + +| API key | Name | Notes | +|---------|------|-------| +| 8 | OffsetCommit | Consumer group — later issue | +| 9 | OffsetFetch | Consumer group — later issue | +| 10 | FindCoordinator | Consumer group — later issue | +| 11–16 | JoinGroup, Heartbeat, LeaveGroup, SyncGroup, DescribeGroups, ListGroups | Consumer group — later issue | +| 17 | SaslHandshake | Auth — later issue | +| 20+ | DeleteTopics, InitProducerId, transactions, ACLs, etc. | Later issues | ## Out of scope (later issues) diff --git a/gateways/kafka/src/error.rs b/gateways/kafka/src/error.rs index 668fc5c442..7eee4b16b8 100644 --- a/gateways/kafka/src/error.rs +++ b/gateways/kafka/src/error.rs @@ -34,6 +34,14 @@ pub enum KafkaProtocolError { InvalidVarint, #[error("unsupported request header version: {0}")] UnsupportedHeaderVersion(i16), + #[error("invalid array length: {0}")] + InvalidArrayLength(i32), + #[error("invalid compact array length: encoded value must be >= 1, got {0}")] + InvalidCompactArrayLength(u64), + #[error("collection length {count} exceeds maximum {max}")] + CollectionTooLarge { count: usize, max: usize }, + #[error("string length {length} exceeds i16::MAX")] + StringTooLong { length: usize }, #[error("io error: {0}")] Io(#[from] std::io::Error), } diff --git a/gateways/kafka/src/lib.rs b/gateways/kafka/src/lib.rs index f0828508c7..c8f0cf9e20 100644 --- a/gateways/kafka/src/lib.rs +++ b/gateways/kafka/src/lib.rs @@ -17,15 +17,6 @@ //! Kafka wire protocol gateway foundation for Apache Iggy. -// Ported wire codec from spike; pedantic clippy cleanup is a follow-up. -#![allow( - clippy::pedantic, - clippy::missing_const_for_fn, - clippy::wildcard_imports, - clippy::match_same_arms, - clippy::needless_pass_by_value -)] - pub mod error; pub mod protocol; pub mod server; diff --git a/gateways/kafka/src/protocol/api.rs b/gateways/kafka/src/protocol/api.rs index 8f0367b481..275c542e8a 100644 --- a/gateways/kafka/src/protocol/api.rs +++ b/gateways/kafka/src/protocol/api.rs @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +use std::net::SocketAddr; + use bytes::Bytes; use crate::protocol::codec::{Decoder, Encoder}; @@ -61,6 +63,37 @@ pub const ERROR_INVALID_REPLICATION_FACTOR: i16 = 38; pub const ERROR_INVALID_REQUEST: i16 = 42; pub const ERROR_UNSUPPORTED_FOR_MESSAGE_FORMAT: i16 = 43; +#[derive(Debug, Clone)] +pub struct BrokerAdvertise { + pub host: String, + pub port: i32, +} + +impl BrokerAdvertise { + #[must_use] + pub fn from_bind_addr(bind_addr: &str) -> Self { + bind_addr.parse::().map_or_else( + |_| Self { + host: "127.0.0.1".to_string(), + port: 9093, + }, + |addr| Self { + host: addr.ip().to_string(), + port: i32::from(addr.port()), + }, + ) + } +} + +impl Default for BrokerAdvertise { + fn default() -> Self { + Self { + host: "127.0.0.1".to_string(), + port: 9093, + } + } +} + #[derive(Debug, Clone, Copy)] pub struct ApiVersionRange { pub api_key: i16, @@ -68,42 +101,50 @@ pub struct ApiVersionRange { pub max_version: i16, } -pub fn supported_api_ranges() -> Vec { - vec![ - ApiVersionRange { - api_key: API_KEY_PRODUCE, - min_version: 3, - max_version: 9, - }, - ApiVersionRange { - api_key: API_KEY_FETCH, - min_version: 4, - max_version: 12, - }, - ApiVersionRange { - api_key: API_KEY_LIST_OFFSETS, - min_version: 1, - max_version: 6, - }, - ApiVersionRange { - api_key: API_KEY_METADATA, - min_version: 0, - max_version: 9, - }, - ApiVersionRange { - api_key: API_KEY_API_VERSIONS, - min_version: 0, - max_version: 3, - }, - ApiVersionRange { - api_key: API_KEY_CREATE_TOPICS, - min_version: 2, - max_version: 5, - }, - ] +static SUPPORTED_RANGES: &[ApiVersionRange] = &[ + ApiVersionRange { + api_key: API_KEY_PRODUCE, + min_version: 3, + max_version: 9, + }, + ApiVersionRange { + api_key: API_KEY_FETCH, + min_version: 4, + max_version: 12, + }, + ApiVersionRange { + api_key: API_KEY_LIST_OFFSETS, + min_version: 1, + max_version: 6, + }, + ApiVersionRange { + api_key: API_KEY_METADATA, + min_version: 0, + max_version: 9, + }, + ApiVersionRange { + api_key: API_KEY_API_VERSIONS, + min_version: 0, + max_version: 3, + }, + ApiVersionRange { + api_key: API_KEY_CREATE_TOPICS, + min_version: 2, + max_version: 5, + }, +]; + +#[must_use] +pub fn supported_api_ranges() -> &'static [ApiVersionRange] { + SUPPORTED_RANGES } -pub fn handle_request(api_key: i16, api_version: i16, body: Bytes) -> Bytes { +pub fn handle_request( + api_key: i16, + api_version: i16, + body: Bytes, + broker: &BrokerAdvertise, +) -> Bytes { match api_key { API_KEY_API_VERSIONS => { if is_supported_version(api_key, api_version) { @@ -114,15 +155,15 @@ pub fn handle_request(api_key: i16, api_version: i16, body: Bytes) -> Bytes { } API_KEY_METADATA => { if is_supported_version(api_key, api_version) { - encode_metadata_response(api_version, body, ERROR_NONE) + encode_metadata_response(api_version, body, broker, ERROR_NONE) } else { - encode_metadata_response(0, body, ERROR_UNSUPPORTED_VERSION) + encode_metadata_response(0, body, broker, ERROR_UNSUPPORTED_VERSION) } } API_KEY_PRODUCE => { if is_supported_version(api_key, api_version) { match decode_produce_request(api_version, body) { - Ok(req) => encode_produce_response(api_version, req), + Ok(req) => encode_produce_response(api_version, &req), Err(e) => { tracing::error!("Failed to decode Produce request: {:?}", e); encode_error_only_response(ERROR_CORRUPT_MESSAGE) @@ -135,7 +176,7 @@ pub fn handle_request(api_key: i16, api_version: i16, body: Bytes) -> Bytes { API_KEY_FETCH => { if is_supported_version(api_key, api_version) { match decode_fetch_request(api_version, body) { - Ok(req) => encode_fetch_response(api_version, req), + Ok(req) => encode_fetch_response(api_version, &req), Err(e) => { tracing::error!("Failed to decode Fetch request: {:?}", e); encode_error_only_response(ERROR_CORRUPT_MESSAGE) @@ -148,7 +189,7 @@ pub fn handle_request(api_key: i16, api_version: i16, body: Bytes) -> Bytes { API_KEY_LIST_OFFSETS => { if is_supported_version(api_key, api_version) { match decode_list_offsets_request(api_version, body) { - Ok(req) => encode_list_offsets_response(api_version, req), + Ok(req) => encode_list_offsets_response(api_version, &req), Err(e) => { tracing::error!("Failed to decode ListOffsets request: {:?}", e); encode_error_only_response(ERROR_CORRUPT_MESSAGE) @@ -161,7 +202,7 @@ pub fn handle_request(api_key: i16, api_version: i16, body: Bytes) -> Bytes { API_KEY_CREATE_TOPICS => { if is_supported_version(api_key, api_version) { match decode_create_topics_request(api_version, body) { - Ok(req) => encode_create_topics_response(api_version, req), + Ok(req) => encode_create_topics_response(api_version, &req), Err(e) => { tracing::error!("Failed to decode CreateTopics request: {:?}", e); encode_error_only_response(ERROR_CORRUPT_MESSAGE) @@ -175,31 +216,32 @@ pub fn handle_request(api_key: i16, api_version: i16, body: Bytes) -> Bytes { } } +#[must_use] pub fn is_supported_version(api_key: i16, api_version: i16) -> bool { - supported_api_ranges() - .into_iter() + SUPPORTED_RANGES + .iter() .find(|r| r.api_key == api_key) .is_some_and(|r| api_version >= r.min_version && api_version <= r.max_version) } fn encode_api_versions_response(api_version: i16, error_code: i16) -> Bytes { let flexible = api_version >= 3; - let ranges = supported_api_ranges(); + let ranges = SUPPORTED_RANGES; let mut e = Encoder::with_capacity(128); e.write_i16(error_code); if flexible { e.write_varint((ranges.len() + 1) as u64); - for r in &ranges { + for r in ranges { e.write_i16(r.api_key); e.write_i16(r.min_version); e.write_i16(r.max_version); e.write_empty_tagged_fields(); } } else { - e.write_i32(ranges.len() as i32); - for r in &ranges { + e.write_i32(i32::try_from(ranges.len()).expect("supported range table is small")); + for r in ranges { e.write_i16(r.api_key); e.write_i16(r.min_version); e.write_i16(r.max_version); @@ -217,37 +259,82 @@ fn encode_api_versions_response(api_version: i16, error_code: i16) -> Bytes { e.freeze() } -fn encode_metadata_response(_api_version: i16, body: Bytes, top_level_error_code: i16) -> Bytes { +fn encode_metadata_response( + api_version: i16, + body: Bytes, + broker: &BrokerAdvertise, + top_level_error_code: i16, +) -> Bytes { + let flexible = api_version >= 9; + let topics_count = split_metadata_request_topics(body, api_version); + let topic_error = if top_level_error_code == ERROR_NONE { + ERROR_UNKNOWN_TOPIC_OR_PARTITION + } else { + top_level_error_code + }; + let mut e = Encoder::with_capacity(256); - e.write_i32(1); - e.write_i32(1); - e.write_nullable_string(Some("127.0.0.1")); - e.write_i32(9093); - - let topics_count = split_metadata_request_topics(body); - e.write_i32(topics_count as i32); - for _ in 0..topics_count { - e.write_i16(if top_level_error_code == ERROR_NONE { - ERROR_UNKNOWN_TOPIC_OR_PARTITION - } else { - top_level_error_code - }); - e.write_nullable_string(Some("unknown-topic")); - e.write_i32(0); + if api_version >= 1 { + e.write_i32(0); // throttle_time_ms + } + + if flexible { + e.write_varint(2); // one broker (N+1) + e.write_i32(1); + e.write_compact_nullable_string(Some(&broker.host)); + e.write_i32(broker.port); + e.write_compact_nullable_string(None); // rack + e.write_empty_tagged_fields(); + + if api_version >= 2 { + e.write_compact_nullable_string(None); // cluster_id + } + e.write_i32(1); // controller_id + + e.write_varint((topics_count + 1) as u64); + for _ in 0..topics_count { + e.write_i16(topic_error); + e.write_compact_nullable_string(Some("unknown-topic")); + e.write_varint(1); // empty partitions array + if api_version >= 4 { + e.write_bool(false); // is_internal + } + e.write_empty_tagged_fields(); + } + e.write_empty_tagged_fields(); + } else { + e.write_i32(1); + e.write_i32(1); + let _ = e.write_nullable_string(Some(&broker.host)); + e.write_i32(broker.port); + + e.write_i32(i32::try_from(topics_count).expect("topic count bounded")); + for _ in 0..topics_count { + e.write_i16(topic_error); + let _ = e.write_nullable_string(Some("unknown-topic")); + e.write_i32(0); + } + + e.write_i32(1); // controller_id } - e.write_i32(1); e.freeze() } -fn encode_error_only_response(error_code: i16) -> Bytes { +#[must_use] +pub fn encode_error_only_response(error_code: i16) -> Bytes { let mut e = Encoder::with_capacity(2); e.write_i16(error_code); e.freeze() } -pub fn split_metadata_request_topics(body: Bytes) -> usize { +#[must_use] +pub fn split_metadata_request_topics(body: Bytes, api_version: i16) -> usize { let mut d = Decoder::new(body); - d.read_i32().unwrap_or_default().max(0) as usize + if api_version >= 9 { + d.read_compact_array_count().unwrap_or(0) + } else { + d.read_i32_array_count().unwrap_or(0) + } } diff --git a/gateways/kafka/src/protocol/codec.rs b/gateways/kafka/src/protocol/codec.rs index 1e356b8608..0429beafc1 100644 --- a/gateways/kafka/src/protocol/codec.rs +++ b/gateways/kafka/src/protocol/codec.rs @@ -15,10 +15,17 @@ // specific language governing permissions and limitations // under the License. +//! Low-level Kafka primitive encoders/decoders (ported wire codec). + +#![allow(clippy::pedantic, clippy::missing_const_for_fn)] + use bytes::{Buf, BufMut, Bytes, BytesMut}; use crate::error::{KafkaProtocolError, Result}; +/// Upper bound for Kafka array/collection element counts decoded from the wire. +pub const MAX_COLLECTION_LEN: usize = 65_536; + pub struct Decoder { bytes: Bytes, } @@ -69,6 +76,9 @@ impl Decoder { let mut shift = 0u32; loop { let byte = self.read_u8()?; + if shift == 63 && byte & 0x7E != 0 { + return Err(KafkaProtocolError::InvalidVarint); + } result |= ((byte & 0x7F) as u64) << shift; if byte & 0x80 == 0 { return Ok(result); @@ -80,6 +90,41 @@ impl Decoder { } } + /// Legacy array length: signed i32 count (must be non-negative). + pub fn read_i32_array_count(&mut self) -> Result { + let n = self.read_i32()?; + if n < 0 { + return Err(KafkaProtocolError::InvalidArrayLength(n)); + } + let count = usize::try_from(n).map_err(|_| KafkaProtocolError::CollectionTooLarge { + count: n as usize, + max: MAX_COLLECTION_LEN, + })?; + if count > MAX_COLLECTION_LEN { + return Err(KafkaProtocolError::CollectionTooLarge { + count, + max: MAX_COLLECTION_LEN, + }); + } + Ok(count) + } + + /// Compact array length: unsigned varint holding `element_count + 1`. + pub fn read_compact_array_count(&mut self) -> Result { + let n = self.read_varint()?; + if n == 0 { + return Err(KafkaProtocolError::InvalidCompactArrayLength(0)); + } + let count = (n - 1) as usize; + if count > MAX_COLLECTION_LEN { + return Err(KafkaProtocolError::CollectionTooLarge { + count, + max: MAX_COLLECTION_LEN, + }); + } + Ok(count) + } + /// Legacy nullable string: i16 length prefix (-1 = null). pub fn read_nullable_string(&mut self) -> Result> { let len = self.read_i16()?; @@ -138,7 +183,17 @@ impl Decoder { /// Skip over a tagged-fields section. Each field is: tag (varint) + size (varint) + bytes. /// A count of 0 is the common case (single byte 0x00). pub fn read_tagged_fields(&mut self) -> Result<()> { - let count = self.read_varint()? as usize; + let count = self.read_varint()?; + let count = usize::try_from(count).map_err(|_| KafkaProtocolError::CollectionTooLarge { + count: count as usize, + max: MAX_COLLECTION_LEN, + })?; + if count > MAX_COLLECTION_LEN { + return Err(KafkaProtocolError::CollectionTooLarge { + count, + max: MAX_COLLECTION_LEN, + }); + } for _ in 0..count { self.read_varint()?; // tag number let size = self.read_varint()? as usize; @@ -206,14 +261,18 @@ impl Encoder { } /// Legacy nullable string: i16 length prefix, -1 for null. - pub fn write_nullable_string(&mut self, v: Option<&str>) { + pub fn write_nullable_string(&mut self, v: Option<&str>) -> Result<()> { match v { None => self.write_i16(-1), Some(s) => { - self.write_i16(s.len() as i16); + if s.len() > i16::MAX as usize { + return Err(KafkaProtocolError::StringTooLong { length: s.len() }); + } + self.write_i16(i16::try_from(s.len()).expect("checked above")); self.bytes.put_slice(s.as_bytes()); } } + Ok(()) } /// Compact nullable string (flexible versions): varint(len+1), 0 for null. diff --git a/gateways/kafka/src/protocol/header.rs b/gateways/kafka/src/protocol/header.rs index fb137b81e0..93f90e5c5e 100644 --- a/gateways/kafka/src/protocol/header.rs +++ b/gateways/kafka/src/protocol/header.rs @@ -15,6 +15,12 @@ // specific language governing permissions and limitations // under the License. +#![allow( + clippy::pedantic, + clippy::missing_const_for_fn, + clippy::match_same_arms +)] + use bytes::Bytes; use crate::error::{KafkaProtocolError, Result}; diff --git a/gateways/kafka/src/protocol/requests.rs b/gateways/kafka/src/protocol/requests.rs index 786fa0b88c..6f4daa8996 100644 --- a/gateways/kafka/src/protocol/requests.rs +++ b/gateways/kafka/src/protocol/requests.rs @@ -17,6 +17,8 @@ //! Kafka request decoders for critical API keys +#![allow(clippy::pedantic)] + use crate::error::Result; use crate::protocol::codec::Decoder; use bytes::Bytes; @@ -62,9 +64,9 @@ pub fn decode_produce_request(version: i16, body: Bytes) -> Result Result Result { // topics array let topics_count = if flexible { - (d.read_varint()? - 1) as usize + d.read_compact_array_count()? } else { - d.read_i32()? as usize + d.read_i32_array_count()? }; let mut topics = Vec::with_capacity(topics_count); @@ -174,9 +176,9 @@ pub fn decode_fetch_request(version: i16, body: Bytes) -> Result { }; let partitions_count = if flexible { - (d.read_varint()? - 1) as usize + d.read_compact_array_count()? } else { - d.read_i32()? as usize + d.read_i32_array_count()? }; let mut partitions = Vec::with_capacity(partitions_count); @@ -219,21 +221,21 @@ pub fn decode_fetch_request(version: i16, body: Bytes) -> Result { // forgotten_topics_data (v7+) — skip if version >= 7 { let forgotten_count = if flexible { - (d.read_varint()? - 1) as usize + d.read_compact_array_count()? } else { - d.read_i32()? as usize + d.read_i32_array_count()? }; for _ in 0..forgotten_count { if flexible { d.read_compact_nullable_string()?; - let partitions_count = (d.read_varint()? - 1) as usize; + let partitions_count = d.read_compact_array_count()?; for _ in 0..partitions_count { d.read_i32()?; } d.read_tagged_fields()?; } else { d.read_nullable_string()?; - let partitions_count = d.read_i32()? as usize; + let partitions_count = d.read_i32_array_count()?; for _ in 0..partitions_count { d.read_i32()?; } @@ -291,9 +293,9 @@ pub fn decode_list_offsets_request(version: i16, body: Bytes) -> Result= 2 { d.read_i8()? } else { 0 }; let topics_count = if flexible { - (d.read_varint()? - 1) as usize + d.read_compact_array_count()? } else { - d.read_i32()? as usize + d.read_i32_array_count()? }; let mut topics = Vec::with_capacity(topics_count); @@ -305,9 +307,9 @@ pub fn decode_list_offsets_request(version: i16, body: Bytes) -> Result Result= 5; let topics_count = if flexible { - (d.read_varint()? - 1) as usize + d.read_compact_array_count()? } else { - d.read_i32()? as usize + d.read_i32_array_count()? }; let mut topics = Vec::with_capacity(topics_count); @@ -388,16 +390,16 @@ pub fn decode_create_topics_request(version: i16, body: Bytes) -> Result Result Bytes { +pub fn encode_produce_response(version: i16, req: &ProduceRequest) -> Bytes { let flexible = version >= 9; let mut e = Encoder::with_capacity(512); if flexible { e.write_varint((req.topics.len() + 1) as u64); } else { - e.write_i32(req.topics.len() as i32); + e.write_i32(i32::try_from(req.topics.len()).expect("topic count bounded")); } for topic in &req.topics { if flexible { e.write_compact_nullable_string(Some(&topic.topic)); } else { - e.write_nullable_string(Some(&topic.topic)); + let _ = e.write_nullable_string(Some(&topic.topic)); } if flexible { e.write_varint((topic.partitions.len() + 1) as u64); } else { - e.write_i32(topic.partitions.len() as i32); + e.write_i32(i32::try_from(topic.partitions.len()).expect("partition count bounded")); } for p in &topic.partitions { e.write_i32(p.partition); e.write_i16(ERROR_NONE); - e.write_i64(0); // base_offset — TODO: return real offset from Iggy + e.write_i64(0); if version >= 2 { - e.write_i64(-1); // log_append_time_ms (-1 = not set) + e.write_i64(-1); } if version >= 5 { - e.write_i64(0); // log_start_offset + e.write_i64(0); } - // record_errors[] and error_message added in v8 if version >= 8 { if flexible { - e.write_varint(1); // empty COMPACT_ARRAY - e.write_compact_nullable_string(None); // error_message = null + e.write_varint(1); + e.write_compact_nullable_string(None); } else { - e.write_i32(0); // empty ARRAY - e.write_nullable_string(None); // error_message = null + e.write_i32(0); + let _ = e.write_nullable_string(None); } } if flexible { @@ -76,7 +79,7 @@ pub fn encode_produce_response(version: i16, req: ProduceRequest) -> Bytes { } if version >= 1 { - e.write_i32(0); // throttle_time_ms + e.write_i32(0); } if flexible { e.write_empty_tagged_fields(); @@ -85,49 +88,48 @@ pub fn encode_produce_response(version: i16, req: ProduceRequest) -> Bytes { e.freeze() } -pub fn encode_fetch_response(version: i16, req: FetchRequest) -> Bytes { +pub fn encode_fetch_response(version: i16, req: &FetchRequest) -> Bytes { let flexible = version >= 12; let mut e = Encoder::with_capacity(512); if version >= 1 { - e.write_i32(0); // throttle_time_ms + e.write_i32(0); } if version >= 7 { - e.write_i16(ERROR_NONE); // error_code - e.write_i32(0); // session_id + e.write_i16(ERROR_NONE); + e.write_i32(0); } if flexible { e.write_varint((req.topics.len() + 1) as u64); } else { - e.write_i32(req.topics.len() as i32); + e.write_i32(i32::try_from(req.topics.len()).expect("topic count bounded")); } for topic in &req.topics { if flexible { e.write_compact_nullable_string(Some(&topic.topic)); } else { - e.write_nullable_string(Some(&topic.topic)); + let _ = e.write_nullable_string(Some(&topic.topic)); } if flexible { e.write_varint((topic.partitions.len() + 1) as u64); } else { - e.write_i32(topic.partitions.len() as i32); + e.write_i32(i32::try_from(topic.partitions.len()).expect("partition count bounded")); } for partition in &topic.partitions { e.write_i32(partition.partition); e.write_i16(ERROR_NONE); - e.write_i64(0); // high_watermark — TODO: get from Iggy + e.write_i64(0); if version >= 4 { - e.write_i64(0); // last_stable_offset + e.write_i64(0); } if version >= 5 { - e.write_i64(0); // log_start_offset + e.write_i64(0); } if version >= 4 { - // aborted_transactions[] if flexible { e.write_varint(1); } else { @@ -135,9 +137,8 @@ pub fn encode_fetch_response(version: i16, req: FetchRequest) -> Bytes { } } if version >= 11 { - e.write_i32(-1); // preferred_read_replica + e.write_i32(-1); } - // records (empty — TODO: call Iggy poll_messages) if flexible { e.write_compact_nullable_bytes(None); } else { @@ -160,44 +161,42 @@ pub fn encode_fetch_response(version: i16, req: FetchRequest) -> Bytes { e.freeze() } -pub fn encode_list_offsets_response(version: i16, req: ListOffsetsRequest) -> Bytes { +pub fn encode_list_offsets_response(version: i16, req: &ListOffsetsRequest) -> Bytes { let flexible = version >= 6; let mut e = Encoder::with_capacity(256); if version >= 2 { - e.write_i32(0); // throttle_time_ms + e.write_i32(0); } if flexible { e.write_varint((req.topics.len() + 1) as u64); } else { - e.write_i32(req.topics.len() as i32); + e.write_i32(i32::try_from(req.topics.len()).expect("topic count bounded")); } for topic in &req.topics { if flexible { e.write_compact_nullable_string(Some(&topic.topic)); } else { - e.write_nullable_string(Some(&topic.topic)); + let _ = e.write_nullable_string(Some(&topic.topic)); } if flexible { e.write_varint((topic.partitions.len() + 1) as u64); } else { - e.write_i32(topic.partitions.len() as i32); + e.write_i32(i32::try_from(topic.partitions.len()).expect("partition count bounded")); } for partition in &topic.partitions { e.write_i32(partition.partition); e.write_i16(ERROR_NONE); - // TODO: query Iggy for actual offsets let offset = 0i64; if version >= 1 { - e.write_i64(1_700_000_000_000); // timestamp placeholder + e.write_i64(1_700_000_000_000); } e.write_i64(offset); - // leader_epoch was added in v4, not v1 if version >= 4 { e.write_i32(-1); } @@ -218,25 +217,25 @@ pub fn encode_list_offsets_response(version: i16, req: ListOffsetsRequest) -> By e.freeze() } -pub fn encode_create_topics_response(version: i16, req: CreateTopicsRequest) -> Bytes { +pub fn encode_create_topics_response(version: i16, req: &CreateTopicsRequest) -> Bytes { let flexible = version >= 5; let mut e = Encoder::with_capacity(256); if version >= 2 { - e.write_i32(0); // throttle_time_ms + e.write_i32(0); } if flexible { e.write_varint((req.topics.len() + 1) as u64); } else { - e.write_i32(req.topics.len() as i32); + e.write_i32(i32::try_from(req.topics.len()).expect("topic count bounded")); } for topic in &req.topics { if flexible { e.write_compact_nullable_string(Some(&topic.name)); } else { - e.write_nullable_string(Some(&topic.name)); + let _ = e.write_nullable_string(Some(&topic.name)); } let error_code = if topic.num_partitions <= 0 { @@ -248,17 +247,17 @@ pub fn encode_create_topics_response(version: i16, req: CreateTopicsRequest) -> if version >= 1 { if flexible { - e.write_compact_nullable_string(None); // error_message + e.write_compact_nullable_string(None); } else { - e.write_nullable_string(None); + let _ = e.write_nullable_string(None); } } if version >= 5 { - e.write_i16(ERROR_NONE); // topic_config_error_code (added in v5) + e.write_i16(ERROR_NONE); e.write_i32(topic.num_partitions); e.write_i16(topic.replication_factor); - e.write_varint(1); // configs[] empty COMPACT_ARRAY + e.write_varint(1); } if flexible { diff --git a/gateways/kafka/src/server.rs b/gateways/kafka/src/server.rs index c6465eb0ca..55b3e509bc 100644 --- a/gateways/kafka/src/server.rs +++ b/gateways/kafka/src/server.rs @@ -24,10 +24,13 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; use tokio::sync::broadcast; use tokio::time::timeout; +use tokio_util::task::TaskTracker; use tracing::{error, info, warn}; use crate::error::{KafkaProtocolError, Result}; -use crate::protocol::api::handle_request; +use crate::protocol::api::{ + BrokerAdvertise, ERROR_INVALID_REQUEST, encode_error_only_response, handle_request, +}; use crate::protocol::codec::Decoder; use crate::protocol::header::{ RequestHeader, ResponseHeader, request_header_version, response_header_version, @@ -57,30 +60,49 @@ pub struct KafkaServer { } impl KafkaServer { + #[must_use] pub fn new(config: ServerConfig) -> Self { Self { config: Arc::new(config), } } + /// Accept Kafka wire connections until `shutdown` fires, then drain in-flight tasks. + /// + /// # Errors + /// + /// Returns an error if binding fails or a non-transient `accept()` error occurs. pub async fn run(self, mut shutdown: broadcast::Receiver<()>) -> Result<()> { let listener = TcpListener::bind(&self.config.bind_addr).await?; info!("kafka listener bound on {}", self.config.bind_addr); + let tracker = TaskTracker::new(); + let broker = BrokerAdvertise::from_bind_addr(&self.config.bind_addr); + loop { tokio::select! { _ = shutdown.recv() => { info!("kafka listener shutdown requested"); + tracker.close(); + tracker.wait().await; break; } accept_result = listener.accept() => { - let (stream, peer) = accept_result?; - let cfg = Arc::clone(&self.config); - tokio::spawn(async move { - if let Err(err) = handle_connection(stream, cfg, peer).await { - warn!(%peer, "connection closed with error: {err}"); + match accept_result { + Ok((stream, peer)) => { + let cfg = Arc::clone(&self.config); + let broker = broker.clone(); + tracker.spawn(async move { + if let Err(err) = handle_connection(stream, cfg, peer, broker).await { + warn!(%peer, "connection closed with error: {err}"); + } + }); } - }); + Err(e) if is_transient_accept_error(&e) => { + warn!(%e, "transient accept error, continuing"); + } + Err(e) => return Err(e.into()), + } } } } @@ -88,10 +110,24 @@ impl KafkaServer { } } +fn is_transient_accept_error(err: &std::io::Error) -> bool { + use std::io::ErrorKind; + + matches!( + err.kind(), + ErrorKind::Interrupted | ErrorKind::ConnectionAborted | ErrorKind::WouldBlock + ) || matches!( + err.raw_os_error(), + // EMFILE / ENFILE are common across Unix platforms when fd limits are hit. + Some(23 | 24) + ) +} + async fn handle_connection( mut stream: TcpStream, config: Arc, peer: SocketAddr, + broker: BrokerAdvertise, ) -> Result<()> { info!(%peer, "connection accepted"); @@ -119,9 +155,26 @@ async fn handle_connection( let api_version = i16::from_be_bytes([frame[2], frame[3]]); let req_hdr_ver = request_header_version(api_key, api_version); let resp_hdr_ver = response_header_version(api_key, api_version); + let correlation_id = correlation_id_from_frame(&frame); let mut decoder = Decoder::new(frame); - let req = RequestHeader::decode_from(&mut decoder, req_hdr_ver)?; + let req = match RequestHeader::decode_from(&mut decoder, req_hdr_ver) { + Ok(req) => req, + Err(KafkaProtocolError::UnsupportedHeaderVersion(_)) => { + warn!(%peer, api_key, api_version, "unsupported request header version"); + let body_response = encode_error_only_response(ERROR_INVALID_REQUEST); + let resp_header = ResponseHeader { correlation_id }; + let encoded_header = resp_header.encode(0); + let mut payload = + BytesMut::with_capacity(encoded_header.len() + body_response.len()); + payload.put_slice(&encoded_header); + payload.put_slice(&body_response); + write_frame(&mut stream, &payload, config.write_timeout).await?; + return Ok(()); + } + Err(e) => return Err(e), + }; + info!( %peer, api_key = req.api_key, @@ -132,7 +185,7 @@ async fn handle_connection( ); let body = decoder.read_bytes(decoder.remaining())?; - let body_response = handle_request(req.api_key, req.api_version, body); + let body_response = handle_request(req.api_key, req.api_version, body, &broker); let resp_header = ResponseHeader { correlation_id: req.correlation_id, @@ -146,6 +199,19 @@ async fn handle_connection( } } +fn correlation_id_from_frame(frame: &bytes::Bytes) -> i32 { + if frame.len() >= 8 { + i32::from_be_bytes([frame[4], frame[5], frame[6], frame[7]]) + } else { + 0 + } +} + +/// Read one length-prefixed Kafka frame from `stream`. +/// +/// # Errors +/// +/// Returns an error on timeout, invalid length, or I/O failure. pub async fn read_frame( stream: &mut TcpStream, max_frame_size: usize, @@ -156,12 +222,16 @@ pub async fn read_frame( .await .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "read timeout"))??; - let frame_len = i32::from_be_bytes(len_buf); - if frame_len <= 0 { - return Err(KafkaProtocolError::InvalidFrameLength(frame_len)); + let frame_len_i32 = i32::from_be_bytes(len_buf); + if frame_len_i32 <= 0 { + return Err(KafkaProtocolError::InvalidFrameLength(frame_len_i32)); } - let frame_len = frame_len as usize; + let frame_len = + usize::try_from(frame_len_i32).map_err(|_| KafkaProtocolError::FrameTooLarge { + max_bytes: max_frame_size, + actual_bytes: u32::MAX as usize, + })?; if frame_len > max_frame_size { return Err(KafkaProtocolError::FrameTooLarge { max_bytes: max_frame_size, @@ -176,6 +246,11 @@ pub async fn read_frame( Ok(bytes::Bytes::from(data)) } +/// Write one length-prefixed Kafka frame to `stream`. +/// +/// # Errors +/// +/// Returns an error on timeout, oversize payload, or I/O failure. pub async fn write_frame( stream: &mut TcpStream, payload: &[u8], @@ -189,7 +264,11 @@ pub async fn write_frame( }); } let mut frame = BytesMut::with_capacity(4 + len); - frame.put_i32(len as i32); + let len_i32 = i32::try_from(len).map_err(|_| KafkaProtocolError::FrameTooLarge { + max_bytes: i32::MAX as usize, + actual_bytes: len, + })?; + frame.put_i32(len_i32); frame.extend_from_slice(payload); timeout(write_timeout, stream.write_all(&frame)) .await diff --git a/gateways/kafka/tests/api_handler_tests.rs b/gateways/kafka/tests/api_handler_tests.rs index 5fb8b0c6c2..29d5a5423f 100644 --- a/gateways/kafka/tests/api_handler_tests.rs +++ b/gateways/kafka/tests/api_handler_tests.rs @@ -18,16 +18,20 @@ use bytes::Bytes; use iggy_gateway_kafka::protocol::api::{ - API_KEY_API_VERSIONS, API_KEY_METADATA, ERROR_UNSUPPORTED_VERSION, handle_request, - is_supported_version, split_metadata_request_topics, supported_api_ranges, + API_KEY_API_VERSIONS, API_KEY_METADATA, BrokerAdvertise, ERROR_UNSUPPORTED_VERSION, + handle_request, is_supported_version, split_metadata_request_topics, supported_api_ranges, }; + +fn test_broker() -> BrokerAdvertise { + BrokerAdvertise::default() +} use iggy_gateway_kafka::protocol::codec::Decoder; // ── ApiVersions ───────────────────────────────────────────────────────────── #[test] fn api_versions_v1_response_non_flexible_format() { - let body = handle_request(API_KEY_API_VERSIONS, 1, Bytes::new()); + let body = handle_request(API_KEY_API_VERSIONS, 1, Bytes::new(), &test_broker()); let mut d = Decoder::new(body); assert_eq!(d.read_i16().unwrap(), 0); // error_code @@ -51,7 +55,7 @@ fn api_versions_v1_response_non_flexible_format() { #[test] fn api_versions_v3_response_flexible_format() { - let body = handle_request(API_KEY_API_VERSIONS, 3, Bytes::new()); + let body = handle_request(API_KEY_API_VERSIONS, 3, Bytes::new(), &test_broker()); let mut d = Decoder::new(body); assert_eq!(d.read_i16().unwrap(), 0); // error_code @@ -81,7 +85,7 @@ fn api_versions_v3_response_flexible_format() { #[test] fn metadata_response_has_broker_array_and_topic_array() { - let body = handle_request(API_KEY_METADATA, 0, Bytes::new()); + let body = handle_request(API_KEY_METADATA, 0, Bytes::new(), &test_broker()); let mut d = Decoder::new(body); let broker_count = d.read_i32().unwrap(); @@ -101,7 +105,7 @@ fn metadata_response_has_broker_array_and_topic_array() { fn unsupported_version_returns_protocol_error() { let mut req = Vec::new(); req.extend_from_slice(&1_i32.to_be_bytes()); - let body = handle_request(API_KEY_METADATA, 99, Bytes::from(req)); + let body = handle_request(API_KEY_METADATA, 99, Bytes::from(req), &test_broker()); let mut d = Decoder::new(body); let _broker_count = d.read_i32().unwrap(); let _ = d.read_i32().unwrap(); @@ -123,7 +127,7 @@ fn unsupported_version_returns_protocol_error() { #[test] fn unknown_api_key_returns_error_only_payload() { - let body = handle_request(999, 0, Bytes::new()); + let body = handle_request(999, 0, Bytes::new(), &test_broker()); let mut d = Decoder::new(body); assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); } @@ -132,7 +136,7 @@ fn unknown_api_key_returns_error_only_payload() { fn metadata_topic_split_reads_array_count() { let mut raw = Vec::new(); raw.extend_from_slice(&2_i32.to_be_bytes()); - assert_eq!(split_metadata_request_topics(Bytes::from(raw)), 2); + assert_eq!(split_metadata_request_topics(Bytes::from(raw), 0), 2); } #[test] diff --git a/gateways/kafka/tests/codec_tests.rs b/gateways/kafka/tests/codec_tests.rs index 093d123451..0b25fe7929 100644 --- a/gateways/kafka/tests/codec_tests.rs +++ b/gateways/kafka/tests/codec_tests.rs @@ -26,8 +26,8 @@ fn codec_round_trip_primitives_and_nullable_fields() { enc.write_i16(42); enc.write_i32(123_456); enc.write_i64(9_999_999); - enc.write_nullable_string(Some("client-a")); - enc.write_nullable_string(None); + enc.write_nullable_string(Some("client-a")).unwrap(); + enc.write_nullable_string(None).unwrap(); enc.write_nullable_bytes(Some(&[1, 2, 3])); enc.write_nullable_bytes(None); let bytes = enc.freeze(); diff --git a/gateways/kafka/tests/decode_validation_tests.rs b/gateways/kafka/tests/decode_validation_tests.rs index d9f9cab3f9..eaf2cf64c0 100644 --- a/gateways/kafka/tests/decode_validation_tests.rs +++ b/gateways/kafka/tests/decode_validation_tests.rs @@ -113,7 +113,7 @@ fn produce_response_encodes_for_all_supported_versions() { let body = load_body(0, "Produce", version); let req = decode_produce_request(version, body) .unwrap_or_else(|e| panic!("Produce v{version} decode failed: {e}")); - let resp = encode_produce_response(version, req); + let resp = encode_produce_response(version, &req); assert!( !resp.is_empty(), "Produce v{version}: response must not be empty" @@ -126,7 +126,7 @@ fn produce_response_v3_roundtrip() { use iggy_gateway_kafka::protocol::codec::Decoder; let body = load_body(0, "Produce", 3); let req = decode_produce_request(3, body).unwrap(); - let resp = encode_produce_response(3, req); + let resp = encode_produce_response(3, &req); let mut d = Decoder::new(resp); let topic_count = d.read_i32().unwrap(); @@ -153,7 +153,7 @@ fn produce_response_v8_includes_record_errors() { use iggy_gateway_kafka::protocol::codec::Decoder; let body = load_body(0, "Produce", 8); let req = decode_produce_request(8, body).unwrap(); - let resp = encode_produce_response(8, req); + let resp = encode_produce_response(8, &req); let mut d = Decoder::new(resp); let topic_count = d.read_i32().unwrap(); @@ -217,7 +217,7 @@ fn fetch_response_encodes_for_all_supported_versions() { let body = load_body(1, "Fetch", version); let req = decode_fetch_request(version, body) .unwrap_or_else(|e| panic!("Fetch v{version} decode failed: {e}")); - let resp = encode_fetch_response(version, req); + let resp = encode_fetch_response(version, &req); assert!( !resp.is_empty(), "Fetch v{version}: response must not be empty" @@ -230,7 +230,7 @@ fn fetch_response_v7_roundtrip() { use iggy_gateway_kafka::protocol::codec::Decoder; let body = load_body(1, "Fetch", 7); let req = decode_fetch_request(7, body).unwrap(); - let resp = encode_fetch_response(7, req); + let resp = encode_fetch_response(7, &req); let mut d = Decoder::new(resp); let throttle_ms = d.read_i32().unwrap(); // v1+ @@ -289,7 +289,7 @@ fn list_offsets_response_encodes_for_all_supported_versions() { let body = load_body(2, "ListOffsets", version); let req = decode_list_offsets_request(version, body) .unwrap_or_else(|e| panic!("ListOffsets v{version} decode failed: {e}")); - let resp = encode_list_offsets_response(version, req); + let resp = encode_list_offsets_response(version, &req); assert!( !resp.is_empty(), "ListOffsets v{version}: response must not be empty" @@ -302,7 +302,7 @@ fn list_offsets_response_v1_no_leader_epoch() { use iggy_gateway_kafka::protocol::codec::Decoder; let body = load_body(2, "ListOffsets", 1); let req = decode_list_offsets_request(1, body).unwrap(); - let resp = encode_list_offsets_response(1, req); + let resp = encode_list_offsets_response(1, &req); let mut d = Decoder::new(resp); // v1: no throttle_time_ms @@ -329,7 +329,7 @@ fn list_offsets_response_v4_has_leader_epoch() { use iggy_gateway_kafka::protocol::codec::Decoder; let body = load_body(2, "ListOffsets", 4); let req = decode_list_offsets_request(4, body).unwrap(); - let resp = encode_list_offsets_response(4, req); + let resp = encode_list_offsets_response(4, &req); let mut d = Decoder::new(resp); let _throttle = d.read_i32().unwrap(); // v2+ @@ -387,7 +387,7 @@ fn create_topics_response_encodes_for_all_supported_versions() { let body = load_body(19, "CreateTopics", version); let req = decode_create_topics_request(version, body) .unwrap_or_else(|e| panic!("CreateTopics v{version} decode failed: {e}")); - let resp = encode_create_topics_response(version, req); + let resp = encode_create_topics_response(version, &req); assert!( !resp.is_empty(), "CreateTopics v{version}: response must not be empty" @@ -401,7 +401,7 @@ fn create_topics_response_v2_roundtrip() { let body = load_body(19, "CreateTopics", 2); let req = decode_create_topics_request(2, body).unwrap(); let topic_name = req.topics[0].name.clone(); - let resp = encode_create_topics_response(2, req); + let resp = encode_create_topics_response(2, &req); let mut d = Decoder::new(resp); let _throttle = d.read_i32().unwrap(); // v2+ @@ -421,7 +421,7 @@ fn create_topics_response_v5_has_topic_config_error_code() { use iggy_gateway_kafka::protocol::codec::Decoder; let body = load_body(19, "CreateTopics", 5); let req = decode_create_topics_request(5, body).unwrap(); - let resp = encode_create_topics_response(5, req); + let resp = encode_create_topics_response(5, &req); let mut d = Decoder::new(resp); let _throttle = d.read_i32().unwrap(); // v2+ diff --git a/gateways/kafka/tests/golden_wire_fixtures_tests.rs b/gateways/kafka/tests/golden_wire_fixtures_tests.rs index 121074a191..5c19dbbd5f 100644 --- a/gateways/kafka/tests/golden_wire_fixtures_tests.rs +++ b/gateways/kafka/tests/golden_wire_fixtures_tests.rs @@ -17,12 +17,15 @@ use bytes::Bytes; -use iggy_gateway_kafka::protocol::api::{API_KEY_API_VERSIONS, API_KEY_METADATA, handle_request}; +use iggy_gateway_kafka::protocol::api::{ + API_KEY_API_VERSIONS, API_KEY_METADATA, BrokerAdvertise, handle_request, +}; use iggy_gateway_kafka::protocol::codec::Encoder; #[test] fn golden_apiversions_v1_response_fixture() { - let actual = handle_request(API_KEY_API_VERSIONS, 1, Bytes::new()); + let broker = BrokerAdvertise::default(); + let actual = handle_request(API_KEY_API_VERSIONS, 1, Bytes::new(), &broker); // error_code=0, api_count=6 // key 0 (Produce) min=3 max=9 @@ -52,7 +55,7 @@ fn golden_metadata_v0_single_topic_response_fixture() { request.write_i32(1); // one topic let req_bytes = request.freeze(); - let actual = handle_request(API_KEY_METADATA, 0, req_bytes); + let actual = handle_request(API_KEY_METADATA, 0, req_bytes, &BrokerAdvertise::default()); // brokers[1]: node_id=1, host=127.0.0.1, port=9093 // topics[1]: topic_error=3, topic_name=unknown-topic, partitions[0] diff --git a/gateways/kafka/tests/header_tests.rs b/gateways/kafka/tests/header_tests.rs index d4eef81db6..e88efe4c22 100644 --- a/gateways/kafka/tests/header_tests.rs +++ b/gateways/kafka/tests/header_tests.rs @@ -28,7 +28,7 @@ fn request_header_v1_decodes() { enc.write_i16(18); // api_key: ApiVersions enc.write_i16(2); // api_version enc.write_i32(101); - enc.write_nullable_string(Some("kafka-cli")); + enc.write_nullable_string(Some("kafka-cli")).unwrap(); let bytes = enc.freeze(); let header = RequestHeader::decode(bytes, 1).expect("decode should succeed"); @@ -44,7 +44,7 @@ fn request_header_v1_null_client_id() { enc.write_i16(18); enc.write_i16(1); enc.write_i32(5); - enc.write_nullable_string(None); + enc.write_nullable_string(None).unwrap(); let bytes = enc.freeze(); let header = RequestHeader::decode(bytes, 1).unwrap(); diff --git a/gateways/kafka/tests/server_integration_tests.rs b/gateways/kafka/tests/server_integration_tests.rs index 7a672348ac..35bc4ff113 100644 --- a/gateways/kafka/tests/server_integration_tests.rs +++ b/gateways/kafka/tests/server_integration_tests.rs @@ -41,7 +41,7 @@ async fn read_frame_reads_valid_payload() { enc.write_i16(18); enc.write_i16(3); enc.write_i32(123); - enc.write_nullable_string(Some("test-client")); + enc.write_nullable_string(Some("test-client")).unwrap(); let payload = enc.freeze(); let mut frame = BytesMut::with_capacity(4 + payload.len()); From cc3bacc899963ee08596e49a3fb2203027dd9e22 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sun, 7 Jun 2026 23:02:58 -0400 Subject: [PATCH 06/57] fix(kafka-gw): address all code review findings Critical protocol correctness: - Metadata non-flexible: controller_id now written before topics (v1+); add rack (v1+), cluster_id (v2+), is_internal (v1+) to match spec - Metadata flexible (v9): is_internal now written before partitions array - ListOffsets: replace hardcoded 1_700_000_000_000 stub timestamp with -1 (Kafka "not available" sentinel) - read_compact_array_count: treat varint=0 as Ok(0) (null compact array) instead of Err; fixes Fetch v7+ with null forgotten_topics from librdkafka/kafka-go Performance: - read_frame: replace vec![0u8; frame_len] with BytesMut + read_buf loop (no zero-initialization) - handle_connection: single BytesMut alloc for length+header+body via new send_response() helper and ResponseHeader::encode_into() - BrokerAdvertise: wrap in Arc, clone Arc per connection instead of String-copying the struct Codec hardening: - read_nullable_string / read_compact_nullable_string: single alloc via str::from_utf8 on borrowed slice + to_owned() - read_tagged_fields: remove dead usize::try_from (always succeeds on 64-bit); compare directly as u64 - write_nullable_bytes: add debug_assert for i32::MAX overflow API cleanliness: - Remove 11 out-of-scope API_KEY_* constants (dead code, not referenced) - requests.rs: null topic name now returns Err(NullTopicName) instead of silently mapping to "" - error.rs: add NullTopicName variant - header.rs: add encode_into() and encoded_size() for zero-copy framing Tests: update golden Metadata v0 fixture and api_handler test to match corrected field order (no controller_id in v0). Co-Authored-By: Claude Sonnet 4.6 --- gateways/kafka/src/error.rs | 2 + gateways/kafka/src/protocol/api.rs | 38 ++++----- gateways/kafka/src/protocol/codec.rs | 36 +++++---- gateways/kafka/src/protocol/header.rs | 16 +++- gateways/kafka/src/protocol/requests.rs | 26 +++--- gateways/kafka/src/protocol/responses.rs | 2 +- gateways/kafka/src/server.rs | 81 ++++++++++++++----- gateways/kafka/tests/api_handler_tests.rs | 9 +-- .../kafka/tests/golden_wire_fixtures_tests.rs | 5 +- 9 files changed, 143 insertions(+), 72 deletions(-) diff --git a/gateways/kafka/src/error.rs b/gateways/kafka/src/error.rs index 7eee4b16b8..b2ee237387 100644 --- a/gateways/kafka/src/error.rs +++ b/gateways/kafka/src/error.rs @@ -42,6 +42,8 @@ pub enum KafkaProtocolError { CollectionTooLarge { count: usize, max: usize }, #[error("string length {length} exceeds i16::MAX")] StringTooLong { length: usize }, + #[error("null topic name in request")] + NullTopicName, #[error("io error: {0}")] Io(#[from] std::io::Error), } diff --git a/gateways/kafka/src/protocol/api.rs b/gateways/kafka/src/protocol/api.rs index 275c542e8a..3e39698841 100644 --- a/gateways/kafka/src/protocol/api.rs +++ b/gateways/kafka/src/protocol/api.rs @@ -33,19 +33,8 @@ pub const API_KEY_PRODUCE: i16 = 0; pub const API_KEY_FETCH: i16 = 1; pub const API_KEY_LIST_OFFSETS: i16 = 2; pub const API_KEY_METADATA: i16 = 3; -pub const API_KEY_OFFSET_COMMIT: i16 = 8; -pub const API_KEY_OFFSET_FETCH: i16 = 9; -pub const API_KEY_FIND_COORDINATOR: i16 = 10; -pub const API_KEY_JOIN_GROUP: i16 = 11; -pub const API_KEY_HEARTBEAT: i16 = 12; -pub const API_KEY_LEAVE_GROUP: i16 = 13; -pub const API_KEY_SYNC_GROUP: i16 = 14; -pub const API_KEY_DESCRIBE_GROUPS: i16 = 15; -pub const API_KEY_LIST_GROUPS: i16 = 16; -pub const API_KEY_SASL_HANDSHAKE: i16 = 17; pub const API_KEY_API_VERSIONS: i16 = 18; pub const API_KEY_CREATE_TOPICS: i16 = 19; -pub const API_KEY_DELETE_TOPICS: i16 = 20; pub const ERROR_NONE: i16 = 0; pub const ERROR_OFFSET_OUT_OF_RANGE: i16 = 1; @@ -296,27 +285,38 @@ fn encode_metadata_response( for _ in 0..topics_count { e.write_i16(topic_error); e.write_compact_nullable_string(Some("unknown-topic")); - e.write_varint(1); // empty partitions array - if api_version >= 4 { - e.write_bool(false); // is_internal + if api_version >= 1 { + e.write_bool(false); // is_internal — must come before partitions array } + e.write_varint(1); // empty partitions array e.write_empty_tagged_fields(); } e.write_empty_tagged_fields(); } else { - e.write_i32(1); - e.write_i32(1); + e.write_i32(1); // brokers array length + e.write_i32(1); // node_id let _ = e.write_nullable_string(Some(&broker.host)); e.write_i32(broker.port); + if api_version >= 1 { + let _ = e.write_nullable_string(None); // rack + } + + if api_version >= 2 { + let _ = e.write_nullable_string(None); // cluster_id + } + if api_version >= 1 { + e.write_i32(1); // controller_id — must come before topics array + } e.write_i32(i32::try_from(topics_count).expect("topic count bounded")); for _ in 0..topics_count { e.write_i16(topic_error); let _ = e.write_nullable_string(Some("unknown-topic")); - e.write_i32(0); + if api_version >= 1 { + e.write_bool(false); // is_internal + } + e.write_i32(0); // partitions array (empty) } - - e.write_i32(1); // controller_id } e.freeze() diff --git a/gateways/kafka/src/protocol/codec.rs b/gateways/kafka/src/protocol/codec.rs index 0429beafc1..e1c9ce68ba 100644 --- a/gateways/kafka/src/protocol/codec.rs +++ b/gateways/kafka/src/protocol/codec.rs @@ -110,10 +110,12 @@ impl Decoder { } /// Compact array length: unsigned varint holding `element_count + 1`. + /// Per the Kafka spec, varint=0 encodes a null (absent) array; treat as empty (0 elements) + /// so optional fields like `forgotten_topics` are skipped rather than rejected. pub fn read_compact_array_count(&mut self) -> Result { let n = self.read_varint()?; if n == 0 { - return Err(KafkaProtocolError::InvalidCompactArrayLength(0)); + return Ok(0); } let count = (n - 1) as usize; if count > MAX_COLLECTION_LEN { @@ -133,10 +135,11 @@ impl Decoder { } let len = len as usize; self.ensure(len)?; - let chunk = self.bytes.copy_to_bytes(len); - String::from_utf8(chunk.to_vec()) - .map(Some) - .map_err(|_| KafkaProtocolError::InvalidUtf8) + let s = std::str::from_utf8(&self.bytes.chunk()[..len]) + .map_err(|_| KafkaProtocolError::InvalidUtf8)? + .to_owned(); + self.bytes.advance(len); + Ok(Some(s)) } /// Compact nullable string (flexible versions): varint(len+1) prefix, 0 = null. @@ -147,10 +150,11 @@ impl Decoder { } let len = (len_plus_one - 1) as usize; self.ensure(len)?; - let chunk = self.bytes.copy_to_bytes(len); - String::from_utf8(chunk.to_vec()) - .map(Some) - .map_err(|_| KafkaProtocolError::InvalidUtf8) + let s = std::str::from_utf8(&self.bytes.chunk()[..len]) + .map_err(|_| KafkaProtocolError::InvalidUtf8)? + .to_owned(); + self.bytes.advance(len); + Ok(Some(s)) } /// Legacy nullable bytes: i32 length prefix (-1 = null). @@ -184,16 +188,13 @@ impl Decoder { /// A count of 0 is the common case (single byte 0x00). pub fn read_tagged_fields(&mut self) -> Result<()> { let count = self.read_varint()?; - let count = usize::try_from(count).map_err(|_| KafkaProtocolError::CollectionTooLarge { - count: count as usize, - max: MAX_COLLECTION_LEN, - })?; - if count > MAX_COLLECTION_LEN { + if count > MAX_COLLECTION_LEN as u64 { return Err(KafkaProtocolError::CollectionTooLarge { - count, + count: count as usize, max: MAX_COLLECTION_LEN, }); } + let count = count as usize; for _ in 0..count { self.read_varint()?; // tag number let size = self.read_varint()? as usize; @@ -291,6 +292,11 @@ impl Encoder { match v { None => self.write_i32(-1), Some(b) => { + debug_assert!( + b.len() <= i32::MAX as usize, + "byte slice length {} exceeds i32::MAX", + b.len() + ); self.write_i32(b.len() as i32); self.bytes.put_slice(b); } diff --git a/gateways/kafka/src/protocol/header.rs b/gateways/kafka/src/protocol/header.rs index 93f90e5c5e..66ce358fe7 100644 --- a/gateways/kafka/src/protocol/header.rs +++ b/gateways/kafka/src/protocol/header.rs @@ -21,7 +21,7 @@ clippy::match_same_arms )] -use bytes::Bytes; +use bytes::{BufMut, Bytes}; use crate::error::{KafkaProtocolError, Result}; use crate::protocol::codec::{Decoder, Encoder}; @@ -194,4 +194,18 @@ impl ResponseHeader { } e.freeze() } + + /// Write this header directly into an existing buffer (avoids a separate heap alloc). + pub fn encode_into(&self, buf: &mut bytes::BytesMut, header_version: i16) { + buf.put_i32(self.correlation_id); + if header_version >= 1 { + buf.put_u8(0); // empty tagged fields + } + } + + /// Byte size of the encoded header for a given version. + #[must_use] + pub fn encoded_size(header_version: i16) -> usize { + if header_version >= 1 { 5 } else { 4 } + } } diff --git a/gateways/kafka/src/protocol/requests.rs b/gateways/kafka/src/protocol/requests.rs index 6f4daa8996..d2bc6b74be 100644 --- a/gateways/kafka/src/protocol/requests.rs +++ b/gateways/kafka/src/protocol/requests.rs @@ -19,7 +19,7 @@ #![allow(clippy::pedantic)] -use crate::error::Result; +use crate::error::{KafkaProtocolError, Result}; use crate::protocol::codec::Decoder; use bytes::Bytes; @@ -72,9 +72,11 @@ pub fn decode_produce_request(version: i16, body: Bytes) -> Result Result { let mut topics = Vec::with_capacity(topics_count); for _ in 0..topics_count { let topic = if flexible { - d.read_compact_nullable_string()?.unwrap_or_default() + d.read_compact_nullable_string()? + .ok_or(KafkaProtocolError::NullTopicName)? } else { - d.read_nullable_string()?.unwrap_or_default() + d.read_nullable_string()? + .ok_or(KafkaProtocolError::NullTopicName)? }; let partitions_count = if flexible { @@ -301,9 +305,11 @@ pub fn decode_list_offsets_request(version: i16, body: Bytes) -> Result Result B let offset = 0i64; if version >= 1 { - e.write_i64(1_700_000_000_000); + e.write_i64(-1); // -1 = timestamp not available (Kafka sentinel) } e.write_i64(offset); if version >= 4 { diff --git a/gateways/kafka/src/server.rs b/gateways/kafka/src/server.rs index 55b3e509bc..02e2bfaf6f 100644 --- a/gateways/kafka/src/server.rs +++ b/gateways/kafka/src/server.rs @@ -35,6 +35,7 @@ use crate::protocol::codec::Decoder; use crate::protocol::header::{ RequestHeader, ResponseHeader, request_header_version, response_header_version, }; +use std::io; #[derive(Debug, Clone)] pub struct ServerConfig { @@ -77,7 +78,7 @@ impl KafkaServer { info!("kafka listener bound on {}", self.config.bind_addr); let tracker = TaskTracker::new(); - let broker = BrokerAdvertise::from_bind_addr(&self.config.bind_addr); + let broker = Arc::new(BrokerAdvertise::from_bind_addr(&self.config.bind_addr)); loop { tokio::select! { @@ -91,7 +92,7 @@ impl KafkaServer { match accept_result { Ok((stream, peer)) => { let cfg = Arc::clone(&self.config); - let broker = broker.clone(); + let broker = Arc::clone(&broker); tracker.spawn(async move { if let Err(err) = handle_connection(stream, cfg, peer, broker).await { warn!(%peer, "connection closed with error: {err}"); @@ -127,7 +128,7 @@ async fn handle_connection( mut stream: TcpStream, config: Arc, peer: SocketAddr, - broker: BrokerAdvertise, + broker: Arc, ) -> Result<()> { info!(%peer, "connection accepted"); @@ -164,12 +165,14 @@ async fn handle_connection( warn!(%peer, api_key, api_version, "unsupported request header version"); let body_response = encode_error_only_response(ERROR_INVALID_REQUEST); let resp_header = ResponseHeader { correlation_id }; - let encoded_header = resp_header.encode(0); - let mut payload = - BytesMut::with_capacity(encoded_header.len() + body_response.len()); - payload.put_slice(&encoded_header); - payload.put_slice(&body_response); - write_frame(&mut stream, &payload, config.write_timeout).await?; + send_response( + &mut stream, + &resp_header, + 0, + &body_response, + config.write_timeout, + ) + .await?; return Ok(()); } Err(e) => return Err(e), @@ -190,13 +193,42 @@ async fn handle_connection( let resp_header = ResponseHeader { correlation_id: req.correlation_id, }; - let encoded_header = resp_header.encode(resp_hdr_ver); - let mut payload = BytesMut::with_capacity(encoded_header.len() + body_response.len()); - payload.put_slice(&encoded_header); - payload.put_slice(&body_response); + send_response( + &mut stream, + &resp_header, + resp_hdr_ver, + &body_response, + config.write_timeout, + ) + .await?; + } +} - write_frame(&mut stream, &payload, config.write_timeout).await?; +/// Write a single length-prefixed Kafka frame using one allocation. +/// Avoids the separate header-encode + payload-concat + length-prefix allocations. +async fn send_response( + stream: &mut TcpStream, + header: &ResponseHeader, + header_version: i16, + body: &[u8], + write_timeout: Duration, +) -> Result<()> { + let header_size = ResponseHeader::encoded_size(header_version); + let payload_size = header_size + body.len(); + if payload_size > i32::MAX as usize { + return Err(KafkaProtocolError::FrameTooLarge { + max_bytes: i32::MAX as usize, + actual_bytes: payload_size, + }); } + let mut frame = BytesMut::with_capacity(4 + payload_size); + frame.put_i32(payload_size as i32); + header.encode_into(&mut frame, header_version); + frame.put_slice(body); + timeout(write_timeout, stream.write_all(&frame)) + .await + .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "write timeout"))??; + Ok(()) } fn correlation_id_from_frame(frame: &bytes::Bytes) -> i32 { @@ -239,11 +271,22 @@ pub async fn read_frame( }); } - let mut data = vec![0u8; frame_len]; - timeout(read_timeout, stream.read_exact(&mut data)) - .await - .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "read timeout"))??; - Ok(bytes::Bytes::from(data)) + // read_buf fills BytesMut spare capacity without zero-initializing it first. + let mut data = BytesMut::with_capacity(frame_len); + timeout(read_timeout, async { + while data.len() < frame_len { + if stream.read_buf(&mut data).await? == 0 { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "connection closed", + )); + } + } + Ok::<_, io::Error>(()) + }) + .await + .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "read timeout"))??; + Ok(data.freeze()) } /// Write one length-prefixed Kafka frame to `stream`. diff --git a/gateways/kafka/tests/api_handler_tests.rs b/gateways/kafka/tests/api_handler_tests.rs index 29d5a5423f..aca2193708 100644 --- a/gateways/kafka/tests/api_handler_tests.rs +++ b/gateways/kafka/tests/api_handler_tests.rs @@ -107,10 +107,11 @@ fn unsupported_version_returns_protocol_error() { req.extend_from_slice(&1_i32.to_be_bytes()); let body = handle_request(API_KEY_METADATA, 99, Bytes::from(req), &test_broker()); let mut d = Decoder::new(body); + // Metadata v0: brokers[], topics[] — no controller_id (added in v1) let _broker_count = d.read_i32().unwrap(); - let _ = d.read_i32().unwrap(); - let _ = d.read_nullable_string().unwrap(); - let _ = d.read_i32().unwrap(); + let _ = d.read_i32().unwrap(); // node_id + let _ = d.read_nullable_string().unwrap(); // host + let _ = d.read_i32().unwrap(); // port let topic_count = d.read_i32().unwrap(); assert_eq!(topic_count, 1); let topic_error = d.read_i16().unwrap(); @@ -119,8 +120,6 @@ fn unsupported_version_returns_protocol_error() { assert_eq!(topic_name, "unknown-topic"); let partitions_count = d.read_i32().unwrap(); assert_eq!(partitions_count, 0); - let controller_id = d.read_i32().unwrap(); - assert_eq!(controller_id, 1); } // ── Misc ──────────────────────────────────────────────────────────────────── diff --git a/gateways/kafka/tests/golden_wire_fixtures_tests.rs b/gateways/kafka/tests/golden_wire_fixtures_tests.rs index 5c19dbbd5f..62b9e96e60 100644 --- a/gateways/kafka/tests/golden_wire_fixtures_tests.rs +++ b/gateways/kafka/tests/golden_wire_fixtures_tests.rs @@ -57,10 +57,10 @@ fn golden_metadata_v0_single_topic_response_fixture() { let actual = handle_request(API_KEY_METADATA, 0, req_bytes, &BrokerAdvertise::default()); + // Metadata v0 layout: brokers[], topics[] (no controller_id — added in v1) // brokers[1]: node_id=1, host=127.0.0.1, port=9093 // topics[1]: topic_error=3, topic_name=unknown-topic, partitions[0] - // controller_id=1 (included by this implementation baseline) - let expected: [u8; 52] = [ + let expected: [u8; 48] = [ 0x00, 0x00, 0x00, 0x01, // broker count 0x00, 0x00, 0x00, 0x01, // node id 0x00, 0x09, // host len @@ -72,7 +72,6 @@ fn golden_metadata_v0_single_topic_response_fixture() { 0x75, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x2d, 0x74, 0x6f, 0x70, 0x69, 0x63, // unknown-topic 0x00, 0x00, 0x00, 0x00, // partition count - 0x00, 0x00, 0x00, 0x01, // controller id ]; assert_eq!(actual.as_ref(), &expected); } From 62c0254b85ca21a91b45af61e571311460229c95 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Wed, 10 Jun 2026 18:59:50 -0400 Subject: [PATCH 07/57] kafka gateway: docs, tests, and protocol updates Foundation for the Kafka gateway (#3421): add manual testing and test-suite docs, a full API key reference, and update README/SCOPE to reflect bind/config notes and test coverage. Implement various protocol, header, codec and server tweaks (src/protocol/*, server.rs, lib.rs, main.rs, error.rs) and add/extend many tests and test helpers (103 regression tests across new and updated suites). Add kafka-tool response helper and fixtures tooling updates to support decode/validation tests and manual end-to-end checks. These changes bundle documentation, test infra, and protocol-level fixes required to validate wire decoding, version firewalling, and stub responses for the gateway. --- gateways/kafka/README.md | 10 +- gateways/kafka/docs/MANUAL_TESTING.md | 272 ++++++++++++ gateways/kafka/docs/SCOPE.md | 89 +++- gateways/kafka/docs/TEST_SUITE.md | 157 +++++++ .../kafka/docs/kafka_api_keys_reference.md | 304 +++++++++++++ gateways/kafka/src/error.rs | 4 +- gateways/kafka/src/lib.rs | 2 +- gateways/kafka/src/main.rs | 15 +- gateways/kafka/src/protocol/api.rs | 68 ++- gateways/kafka/src/protocol/codec.rs | 43 +- gateways/kafka/src/protocol/header.rs | 4 + gateways/kafka/src/protocol/requests.rs | 2 +- gateways/kafka/src/protocol/responses.rs | 146 ++++-- gateways/kafka/src/server.rs | 165 ++++--- gateways/kafka/tests/api_handler_tests.rs | 22 +- .../kafka/tests/broker_advertise_tests.rs | 112 +++++ gateways/kafka/tests/codec_tests.rs | 16 +- gateways/kafka/tests/common/fixtures.rs | 54 +++ gateways/kafka/tests/common/scope.rs | 35 ++ gateways/kafka/tests/common/server.rs | 52 +++ gateways/kafka/tests/common/tcp.rs | 110 +++++ gateways/kafka/tests/decode_safety_tests.rs | 81 ++++ .../kafka/tests/decode_validation_tests.rs | 46 +- .../kafka/tests/golden_wire_fixtures_tests.rs | 4 +- .../kafka/tests/handler_regression_tests.rs | 171 +++++++ gateways/kafka/tests/header_tests.rs | 11 + .../kafka/tests/metadata_regression_tests.rs | 228 ++++++++++ gateways/kafka/tests/server_e2e_tests.rs | 156 +++++++ .../kafka/tests/server_integration_tests.rs | 33 +- .../kafka/tests/version_firewall_tests.rs | 309 +++++++++++++ gateways/kafka/tools/kafka-tool/src/main.rs | 214 +++++++-- .../kafka/tools/kafka-tool/src/response.rs | 420 ++++++++++++++++++ 32 files changed, 3117 insertions(+), 238 deletions(-) create mode 100644 gateways/kafka/docs/MANUAL_TESTING.md create mode 100644 gateways/kafka/docs/TEST_SUITE.md create mode 100644 gateways/kafka/docs/kafka_api_keys_reference.md create mode 100644 gateways/kafka/tests/broker_advertise_tests.rs create mode 100644 gateways/kafka/tests/common/fixtures.rs create mode 100644 gateways/kafka/tests/common/scope.rs create mode 100644 gateways/kafka/tests/common/server.rs create mode 100644 gateways/kafka/tests/common/tcp.rs create mode 100644 gateways/kafka/tests/decode_safety_tests.rs create mode 100644 gateways/kafka/tests/handler_regression_tests.rs create mode 100644 gateways/kafka/tests/metadata_regression_tests.rs create mode 100644 gateways/kafka/tests/server_e2e_tests.rs create mode 100644 gateways/kafka/tests/version_firewall_tests.rs create mode 100644 gateways/kafka/tools/kafka-tool/src/response.rs diff --git a/gateways/kafka/README.md b/gateways/kafka/README.md index b575a0c803..09495b3c3b 100644 --- a/gateways/kafka/README.md +++ b/gateways/kafka/README.md @@ -8,7 +8,7 @@ Foundation layer for [apache/iggy#3421](https://github.com/apache/iggy/issues/34 cargo run -p iggy_gateway_kafka --bin iggy-kafka-gateway ``` -Default bind: `127.0.0.1:9093`. +Default bind: `127.0.0.1:9093`. Override with `KAFKA_BIND_ADDR` (e.g. `0.0.0.0:9093`). ## Test @@ -16,6 +16,8 @@ Default bind: `127.0.0.1:9093`. cargo test -p iggy_gateway_kafka ``` +103 regression tests across 12 suites — see [docs/TEST_SUITE.md](docs/TEST_SUITE.md) for the full catalog. + `decode_validation_tests` require wire fixtures under `tools/kafka-tool/kafka_messages/`: ```bash @@ -26,9 +28,13 @@ cargo run -p kafka-message-gen -- generate \ (Run from workspace root; adjust paths if needed.) +## Manual testing + +Before check-in, run the procedure in [docs/MANUAL_TESTING.md](docs/MANUAL_TESTING.md) (smoke, version firewall, kcat, adversarial cases). + ## Scoped APIs -See [docs/SCOPE.md](docs/SCOPE.md). +See [docs/SCOPE.md](docs/SCOPE.md) for [#3421](https://github.com/apache/iggy/issues/3421) deliverables, supported API key/version table, and post-foundation TODO backlog. ## Wire fixture tool diff --git a/gateways/kafka/docs/MANUAL_TESTING.md b/gateways/kafka/docs/MANUAL_TESTING.md new file mode 100644 index 0000000000..cd3e8c0e58 --- /dev/null +++ b/gateways/kafka/docs/MANUAL_TESTING.md @@ -0,0 +1,272 @@ +# Kafka gateway — manual testing procedure + +Manual validation for [apache/iggy#3421](https://github.com/apache/iggy/issues/3421) foundation: TCP listener, wire decode, version firewall, stub responses. **No Iggy backend** — success means correct Kafka wire behavior, not message persistence. + +See also: [SCOPE.md](SCOPE.md) (supported API keys), [TEST_SUITE.md](TEST_SUITE.md) (automated coverage). + +--- + +## 1. Environment setup + +### Requirements + +| Tool | Purpose | Install | +|------|---------|---------| +| Rust toolchain | Build gateway + kafka-tool | [rustup.rs](https://rustup.rs) | +| `kafka-message-gen` | Generate/send wire fixtures | `cargo build -p kafka-message-gen` | +| `kcat` (optional) | Real Kafka client smoke test | `brew install kcat` / `apt install kafkacat` | +| `nc` / `netcat` (optional) | Raw byte injection | Usually preinstalled | +| `xxd` or `hexdump` (optional) | Inspect binary responses | Usually preinstalled | + +### Build and start gateway + +```bash +# From iggy workspace root +cargo build -p iggy_gateway_kafka --bin iggy-kafka-gateway + +# Terminal 1 — start listener (default 127.0.0.1:9093) +RUST_LOG=info cargo run -p iggy_gateway_kafka --bin iggy-kafka-gateway +``` + +Expected log: + +``` +kafka listener bound on 127.0.0.1:9093 +``` + +### Generate wire fixtures + +```bash +# Terminal 2 +cargo run -p kafka-message-gen -- generate \ + --output gateways/kafka/tools/kafka-tool/kafka_messages \ + --api-key 0 --api-key 1 --api-key 2 --api-key 3 --api-key 18 --api-key 19 +``` + +--- + +## 2. Pre-flight automated check + +Run before manual testing to catch regressions: + +```bash +cargo test -p iggy_gateway_kafka +``` + +All tests must pass. If `decode_validation_tests` fail, regenerate fixtures (step above). + +--- + +## 3. Manual test cases + +### Category A — Smoke tests (must pass before check-in) + +| ID | Test | Steps | Expected result | Pass criteria | +|----|------|-------|-----------------|---------------| +| A1 | Gateway starts | Run `iggy-kafka-gateway` | Binds to `:9093`, no panic | Log shows bind address | +| A2 | ApiVersions v1 | `cargo run -p kafka-message-gen -- send --host 127.0.0.1:9093 --api-key 18 --version 1` | Response received | `ec=0`, non-zero byte count | +| A3 | ApiVersions v3 (flexible) | Same with `--version 3` | Response received | `ec=0` | +| A4 | Metadata v0 | `send --api-key 3 --version 0` | Stub broker in response | `ec=0` or topic error 3 (stub) | +| A5 | Produce v3 | `send --api-key 0 --version 3` | Decode + stub ack | `ec=0` | +| A6 | Fetch v4 | `send --api-key 1 --version 4` | Decode + stub response | `ec=0` | +| A7 | ListOffsets v1 | `send --api-key 2 --version 1` | Decode + stub offsets | `ec=0` | +| A8 | CreateTopics v2 | `send --api-key 19 --version 2` | Decode + stub ack | `ec=0` | +| A9 | Verify all scoped keys | `cargo run -p kafka-message-gen -- verify --host 127.0.0.1:9093 --api-key 0 --api-key 1 --api-key 2 --api-key 3 --api-key 18 --api-key 19` | Exit code 0 | No timeouts or I/O errors | + +### Category B — Version firewall (boundary validation) + +For each API key, test **min−1**, **min**, **max**, **max+1** using `kafka-message-gen send` with `--version N`. + +| API key | Name | Min | Max | Test versions | +|---------|------|-----|-----|---------------| +| 18 | ApiVersions | 0 | 3 | −1, 0, 3, 4 | +| 3 | Metadata | 0 | 9 | −1, 0, 9, 10 | +| 0 | Produce | 3 | 9 | 2, 3, 9, 10 | +| 1 | Fetch | 4 | 12 | 3, 4, 12, 13 | +| 2 | ListOffsets | 1 | 6 | 0, 1, 6, 7 | +| 19 | CreateTopics | 2 | 5 | 1, 2, 5, 6 | + +| ID | Test | Expected for in-range | Expected for out-of-range | +|----|------|----------------------|---------------------------| +| B1 | ApiVersions negotiation | `error_code=0`; body lists 6 API keys with correct min/max | `error_code=35` (UNSUPPORTED_VERSION) | +| B2 | Metadata out-of-range | N/A | Topic entries show `error_code=35` | +| B3 | Produce/Fetch/ListOffsets/CreateTopics out-of-range | N/A | Version-aware response with `error_code=35` (top-level or per-topic/partition) | +| B4 | ApiVersions lists only scoped keys | Decode response | Contains keys 0,1,2,3,18,19 only — no consumer-group keys | + +**Validation tip:** Use `--hex` when generating to inspect request bytes: + +```bash +cargo run -p kafka-message-gen -- generate --api-key 18 --version 3 --hex +``` + +### Category C — Unsupported API keys + +| ID | API key | Name | Steps | Expected | +|----|---------|------|-------|----------| +| C1 | 8 | OffsetCommit | `send --api-key 8 --version 2` | `ec=35`, connection stays open | +| C2 | 10 | FindCoordinator | `send --api-key 10` | `ec=35` | +| C3 | 17 | SaslHandshake | `send --api-key 17` | `ec=35` | +| C4 | 20 | DeleteTopics | `send --api-key 20` | `ec=35` | + +Follow C1 with A2 on the **same** `nc` session to confirm the connection is not dropped. + +### Category D — Flexible vs legacy wire encoding + +| ID | API key | Version | Encoding | Validation | +|----|---------|---------|----------|------------| +| D1 | Produce | 8 | Legacy (i32 arrays) | `send` succeeds, `ec=0` | +| D2 | Produce | 9 | Flexible (compact + tagged fields) | `send` succeeds, `ec=0` | +| D3 | Fetch | 11 | Legacy | `send` succeeds | +| D4 | Fetch | 12 | Flexible | `send` succeeds | +| D5 | Metadata | 8 | Legacy | `send` succeeds | +| D6 | Metadata | 9 | Flexible | `send` succeeds | +| D7 | ListOffsets | 5 | Legacy | `send` succeeds | +| D8 | ListOffsets | 6 | Flexible | `send` succeeds | +| D9 | CreateTopics | 4 | Legacy | `send` succeeds | +| D10 | CreateTopics | 5 | Flexible | `send` succeeds | + +### Category E — Metadata stub semantics + +| ID | Test | Steps | Expected | +|----|------|-------|----------| +| E1 | Broker advertise address | Start gateway on `127.0.0.1:9093`; Metadata v0 | Broker host=`127.0.0.1`, port=`9093` | +| E2 | Wildcard bind + advertised host | `KAFKA_BIND_ADDR=0.0.0.0:19093` + `KAFKA_ADVERTISED_HOST=kafka.internal`, restart | Metadata broker host/port match advertised values | +| E3 | Unknown topic stub | Metadata with topic name `my-topic` | Topic error `3` (UNKNOWN_TOPIC_OR_PARTITION), name `unknown-topic` | +| E4 | Multiple topics | Metadata request listing 3 topics | 3 topic entries, each with error 3 | + +### Category F — TCP / connection behavior + +| ID | Test | Steps | Expected | +|----|------|-------|----------| +| F1 | Correlation ID echoed | Send ApiVersions with known correlation_id; decode response header | Response correlation_id matches request | +| F2 | Sequential requests | Send ApiVersions then Metadata on same TCP connection | Both get valid responses | +| F3 | Client disconnect | Connect, send partial frame, close | Gateway logs clean disconnect, no panic | +| F4 | Invalid frame length 0 | `printf '\x00\x00\x00\x00' \| nc 127.0.0.1 9093` | Connection closed, gateway continues serving others | +| F5 | Oversized frame | Send 4-byte length > 8 MiB | Connection rejected/closed, no OOM | +| F6 | Graceful shutdown | Ctrl+C on gateway | Log "shutdown requested", in-flight requests drain | + +### Category G — Real Kafka client (kcat) + +Requires `kcat` installed. Gateway does **not** implement SASL or full broker semantics — expect limited success. + +| ID | Test | Command | Expected (foundation) | +|----|------|---------|---------------------| +| G1 | Broker metadata | `kcat -b 127.0.0.1:9093 -L` | ApiVersions + Metadata handshake; broker appears in metadata | +| G2 | Produce (likely fails later) | `echo "hello" \| kcat -b 127.0.0.1:9093 -t test -P` | May fail at coordinator/group stage — document actual error | +| G3 | Consumer (likely fails later) | `kcat -b 127.0.0.1:9093 -t test -C -o beginning` | May fail without consumer groups — document actual error | + +Record kcat version and exact error strings in your test log. G1 passing is the minimum bar for client compatibility smoke. + +### Category H — Adversarial / negative input + +| ID | Test | Steps | Expected | +|----|------|-------|----------| +| H1 | Truncated Produce body | Send valid header + incomplete body | `error_code=42` (INVALID_REQUEST) or connection error; **no panic** | +| H2 | Random bytes | `dd if=/dev/urandom bs=64 count=1 \| nc 127.0.0.1 9093` | Connection closed or protocol error; gateway stays up | +| H3 | Empty body after header | ApiVersions with valid header, empty body | `ec=0` (ApiVersions accepts empty body) | + +--- + +## 4. Validation reference + +### Kafka error codes used in #3421 + +| Code | Name | When returned | +|------|------|---------------| +| 0 | NONE | Successful stub response | +| 42 | INVALID_REQUEST | Produce/Fetch/ListOffsets/CreateTopics decode failure; unsupported request header | +| 3 | UNKNOWN_TOPIC_OR_PARTITION | Metadata stub per-topic error | +| 35 | UNSUPPORTED_VERSION | Out-of-range version or unlisted API key | +| 42 | INVALID_REQUEST | Unsupported request header version | + +### Response header rules + +| API key | Request flexible? | Response header version | +|---------|--------------------|-------------------------| +| 18 ApiVersions | v3+ | Always v0 (correlation_id only) | +| 3 Metadata | v9+ | v1 (correlation_id + tagged fields) | +| 0 Produce | v9+ | v1 | +| 1 Fetch | v12+ | v1 | +| Others | Per SCOPE.md | See `header.rs` lookup table | + +### Frame layout (for manual hex inspection) + +``` +Request frame: + [length: i32 BE] + [api_key: i16][api_version: i16][correlation_id: i32] + [client_id: NULLABLE_STRING or COMPACT_NULLABLE_STRING] + [tagged_fields: 0x00] ← flexible requests only + [request body] + +Response frame: + [length: i32 BE] + [correlation_id: i32] + [tagged_fields: 0x00] ← flexible responses only (not ApiVersions) + [response body] +``` + +### Raw netcat smoke test + +```bash +# ApiVersions v3 — after generating fixtures +cat gateways/kafka/tools/kafka-tool/kafka_messages/018_ApiVersions_v3.bin \ + | nc -w 2 127.0.0.1 9093 | xxd | head -20 +``` + +First bytes after length prefix should include your correlation_id from the fixture. + +--- + +## 5. Manual test execution checklist + +Copy this checklist into your PR or test log: + +``` +Date: ___________ +Tester: ___________ +Gateway commit: ___________ +kcat version (if used): ___________ + +[ ] A1–A9 Smoke tests +[ ] B1–B4 Version firewall (all 6 keys × 4 boundary versions) +[ ] C1–C4 Unsupported API keys +[ ] D1–D10 Flexible vs legacy encoding +[ ] E1–E4 Metadata stub semantics +[ ] F1–F6 TCP / connection behavior +[ ] G1–G3 kcat client (record errors for G2/G3) +[ ] H1–H3 Adversarial input + +Automated regression: +[ ] cargo test -p iggy_gateway_kafka — ___/103 passed +[ ] cargo clippy -p iggy_gateway_kafka — clean / warnings noted + +Notes / failures: +_________________________________ +``` + +--- + +## 6. Troubleshooting + +| Symptom | Likely cause | Fix | +|---------|--------------|-----| +| `Connection refused` on 9093 | Gateway not running | Start `iggy-kafka-gateway` | +| `decode_validation_tests` panic | Missing fixtures | Run `kafka-message-gen generate` | +| `ec=35` for in-range version | Version not in `SUPPORTED_RANGES` | Check `SCOPE.md` and `api.rs` | +| kcat hangs | Timeout waiting for data | Set `-m 1000`; check gateway logs | +| Buffer underflow on Metadata v9+ | Flexible decode mismatch | File issue; check `api.rs` metadata encoder | +| Port already in use | Another process on 9093 | `lsof -i :9093` / change bind port | + +--- + +## 7. What manual testing does NOT cover (deferred) + +These are documented as TODO in [SCOPE.md](SCOPE.md) — do not fail #3421 validation for these: + +- Message persistence to Iggy +- Consumer group join/sync/heartbeat +- SASL authentication +- Accurate partition leadership / ISR +- Transactional produce +- Real offset commit semantics diff --git a/gateways/kafka/docs/SCOPE.md b/gateways/kafka/docs/SCOPE.md index 8d2be24623..3f20fd411c 100644 --- a/gateways/kafka/docs/SCOPE.md +++ b/gateways/kafka/docs/SCOPE.md @@ -1,15 +1,37 @@ -# Kafka API scope — issue #3421 foundation +# Kafka gateway scope — [apache/iggy#3421](https://github.com/apache/iggy/issues/3421) -This gateway iteration implements **wire validation and stub responses only** (no Iggy backend, no real broker semantics). +## Issue #3421 — in scope (this iteration) -Source of truth in code: `SUPPORTED_RANGES` in [`src/protocol/api.rs`](../src/protocol/api.rs). +Foundation layer only: a TCP listener on the Kafka wire port that decodes requests, validates scoped API keys and versions, validates request wire formats, and returns stub responses. **No Iggy backend integration.** + +| Deliverable | Status | Location | +|-------------|--------|----------| +| TCP listener on `127.0.0.1:9093` (configurable) | Done | `src/server.rs`, `src/main.rs` | +| Length-prefixed frame read/write with `max_frame_size` cap | Done | `src/server.rs` | +| Request header v1/v2 auto-detection | Done | `src/protocol/header.rs` | +| Version negotiation firewall (`SUPPORTED_RANGES`) | Done | `src/protocol/api.rs` | +| Request decode + stub encode for 6 API keys | Done | `src/protocol/requests.rs`, `responses.rs`, `api.rs` | +| Produce hot path: RecordBatch as opaque `Bytes` | Done | `src/protocol/requests.rs` | +| Graceful errors (`UNSUPPORTED_VERSION`, corrupt decode, invalid header) | Done | `src/protocol/api.rs`, `src/server.rs` | +| Adversarial decode safety tests | Done | `tests/decode_safety_tests.rs` | +| Regression test suite (103 tests) | Done | `tests/` — catalog in [`TEST_SUITE.md`](TEST_SUITE.md) | +| Manual testing procedure | Done | [`MANUAL_TESTING.md`](MANUAL_TESTING.md) | +| Wire fixture tool for manual/integration testing | Done | `tools/kafka-tool/` | + +Source of truth for supported ranges: `SUPPORTED_RANGES` in [`src/protocol/api.rs`](../src/protocol/api.rs). + +### Governance model + +Expand `SUPPORTED_RANGES` only after a key/version pair is manually tested. ApiVersions advertises exactly what the firewall allows; out-of-range requests receive `UNSUPPORTED_VERSION` (35) without dropping the connection. + +--- ## Supported API keys and versions | API key | Name | Min version | Max version | Valid versions | Behavior | |---------|------|-------------|-------------|----------------|----------| | 18 | ApiVersions | 0 | 3 | 0, 1, 2, 3 | Advertise supported ranges; flexible encoding at v3+ | -| 3 | Metadata | 0 | 9 | 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 | Decode request; stub broker from `ServerConfig.bind_addr`; flexible encoding at v9+ | +| 3 | Metadata | 0 | 9 | 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 | Decode topic list count; stub broker from `ServerConfig.bind_addr`; flexible encoding at v9+ | | 0 | Produce | 3 | 9 | 3, 4, 5, 6, 7, 8, 9 | Decode request; stub response | | 1 | Fetch | 4 | 12 | 4, 5, 6, 7, 8, 9, 10, 11, 12 | Decode request; stub response | | 2 | ListOffsets | 1 | 6 | 1, 2, 3, 4, 5, 6 | Decode request; stub response | @@ -17,7 +39,7 @@ Source of truth in code: `SUPPORTED_RANGES` in [`src/protocol/api.rs`](../src/pr A request is accepted when `min_version ≤ api_version ≤ max_version` for that API key. Any other version for a listed key, or any unlisted API key, receives `UNSUPPORTED_VERSION` (35). -## Valid versions reference (by API key) +### Valid versions reference (by API key) Use this table when configuring clients or generating wire fixtures with `kafka-message-gen`. @@ -30,7 +52,9 @@ Use this table when configuring clients or generating wire fixtures with `kafka- | 18 | ApiVersions | 0–3 | v3 | | 19 | CreateTopics | 2–5 | v5 | -## Unsupported API keys +--- + +## Unsupported API keys (foundation) All API keys not listed above receive an error-only response with `UNSUPPORTED_VERSION` (35). Examples not in this foundation scope: @@ -43,8 +67,53 @@ All API keys not listed above receive an error-only response with `UNSUPPORTED_V | 17 | SaslHandshake | Auth — later issue | | 20+ | DeleteTopics, InitProducerId, transactions, ACLs, etc. | Later issues | -## Out of scope (later issues) +Full reference for future phases: [`kafka_api_keys_reference.md`](kafka_api_keys_reference.md). + +--- + +## Architecture (three layers) + +| Layer | #3421 | Description | +|-------|-------|-------------| +| **1 — Wire framing** | In scope | `server.rs`, `codec.rs`, `header.rs` — keep custom, zero-copy frame I/O | +| **2 — Request/response codecs** | Partial | Custom minimal-parse codecs for 6 hot-path keys; stub responses only | +| **3 — Iggy bridge** | Out of scope | Produce/Fetch → Iggy SDK; deferred to a follow-on issue | + +--- + +## TODO — post-#3421 (architecture review backlog) + +Items from the [hybrid architecture review](https://github.com/apache/iggy/discussions/3252) and maintainer feedback. **Not part of #3421.** + +### Phase 2 — Iggy bridge (new issue) + +- [ ] Add `bridge/` module (`iggy_bridge`): Produce → `send_messages`, Fetch → `poll_messages` +- [ ] Document partition mapping in `docs/BRIDGE_MAPPING.md`: + - Iggy partitions are **0-based** (same as Kafka) — direct `partition_id` mapping, no offset conversion + - Iggy **consumer groups exist** — map Kafka group APIs to Iggy consumer group APIs + - Use `Partitioning::balanced()` only when Kafka sends `partition == -1`; otherwise use request partition ID +- [ ] Idempotent `ensure_stream_and_topic()` (create-if-not-exists) +- [ ] Real Metadata topology (brokers, partitions, leaders) backed by Iggy state + +### Phase 2 — Selective `kafka-protocol` crate (feature-gated) + +- [ ] Add optional `kafka-protocol-cold` feature to `iggy_gateway_kafka` — **not** wholesale replace of `requests.rs`/`responses.rs` +- [ ] Use crate for: RecordBatch decode (compression, CRC), consumer-group API keys (8–14, 10), complex Metadata/FindCoordinator responses +- [ ] **Keep custom codecs** for Produce/Fetch hot paths (opaque RecordBatch bytes) + +### Phase 3 — Consumer groups (~7 API keys) + +- [ ] OffsetCommit (8), OffsetFetch (9), FindCoordinator (10) +- [ ] JoinGroup (11), Heartbeat (12), LeaveGroup (13), SyncGroup (14) +- [ ] DescribeGroups (15), ListGroups (16) as needed by target clients + +### Phase 3+ — Auth, admin, tuning + +- [ ] SASL (17, 36) if required by deployment +- [ ] Tune `max_frame_size` per workload (Kafka defaults: ~1 MiB produce, ~50 MiB fetch; current default 8 MiB) +- [ ] Target **~15–20 API keys** total for a functional bridge — not all 74+ admin keys + +### Open questions (ask maintainers before Phase 2) -- `IggyBridge` / produce-fetch against Iggy streams -- Consumer group APIs (8–14), SASL (17, 36), transactions -- Accurate metadata topology and record batch semantics +- [ ] Repo placement: `gateways/kafka/` in [apache/iggy](https://github.com/apache/iggy) vs separate proxy repo (affects workspace deps and CI) +- [ ] Confirm bridge dependency strategy with spetz/hubcio ([Discussion #3081](https://github.com/apache/iggy/discussions/3081), [#3252](https://github.com/apache/iggy/discussions/3252)) diff --git a/gateways/kafka/docs/TEST_SUITE.md b/gateways/kafka/docs/TEST_SUITE.md new file mode 100644 index 0000000000..c8515387c2 --- /dev/null +++ b/gateways/kafka/docs/TEST_SUITE.md @@ -0,0 +1,157 @@ +# Kafka gateway — automated regression test suite + +Regression tests live under [`tests/`](../tests/). Run from the workspace root: + +```bash +cargo test -p iggy_gateway_kafka +``` + +**Current count:** 103 tests across 12 suites (as of #3421 foundation). + +## Prerequisites + +### Wire fixtures (required for `decode_validation_tests` and some handler tests) + +```bash +cargo run -p kafka-message-gen -- generate \ + --output gateways/kafka/tools/kafka-tool/kafka_messages \ + --api-key 0 --api-key 1 --api-key 2 --api-key 19 +``` + +Fixtures are gitignored under `tools/kafka-tool/kafka_messages/`. Tests that need them skip gracefully when a fixture file is missing (`handler_regression_tests`) or panic with a clear path (`decode_validation_tests`). + +--- + +## Test file catalog + +| File | Suite focus | Test count (approx.) | Depends on fixtures | +|------|-------------|----------------------|---------------------| +| [`codec_tests.rs`](../tests/codec_tests.rs) | Primitive encode/decode round-trips, varint, compact strings, tagged fields | 9 | No | +| [`decode_safety_tests.rs`](../tests/decode_safety_tests.rs) | Adversarial wire input — malformed lengths, truncated bodies | 6 | No | +| [`header_tests.rs`](../tests/header_tests.rs) | Request/response header v1/v2, version lookup table | 10 | No | +| [`api_handler_tests.rs`](../tests/api_handler_tests.rs) | ApiVersions, Metadata stub, unsupported key/version | 7 | No | +| [`golden_wire_fixtures_tests.rs`](../tests/golden_wire_fixtures_tests.rs) | Byte-exact golden responses (ApiVersions v1, Metadata v0) | 2 | No | +| [`decode_validation_tests.rs`](../tests/decode_validation_tests.rs) | kafka-tool fixture decode + response structure per version | 14 | **Yes** | +| [`version_firewall_tests.rs`](../tests/version_firewall_tests.rs) | Version boundary matrix, unsupported keys, corrupt bodies | 17 | Partial | +| [`metadata_regression_tests.rs`](../tests/metadata_regression_tests.rs) | Metadata v0–v9, topic counts, broker advertise | 7 | No | +| [`broker_advertise_tests.rs`](../tests/broker_advertise_tests.rs) | `BrokerAdvertise::from_bind_addr` parsing | 5 | No | +| [`handler_regression_tests.rs`](../tests/handler_regression_tests.rs) | Every scoped key×version via `handle_request`, stub error codes | 5 | Partial | +| [`server_integration_tests.rs`](../tests/server_integration_tests.rs) | `read_frame` / `write_frame` unit-level I/O | 4 | No | +| [`server_e2e_tests.rs`](../tests/server_e2e_tests.rs) | Full `KafkaServer` TCP round-trips | 8 | Partial | +| [`common/mod.rs`](../tests/common/mod.rs) | Shared helpers (not a test binary) | — | — | + +--- + +## Coverage matrix by API key + +### ApiVersions (key 18, v0–v3) + +| Scenario | Test file | Test name | +|----------|-----------|-----------| +| Non-flexible response (v1) | `api_handler_tests` | `api_versions_v1_response_non_flexible_format` | +| Flexible response (v3) | `api_handler_tests` | `api_versions_v3_response_flexible_format` | +| Golden byte fixture (v1) | `golden_wire_fixtures_tests` | `golden_apiversions_v1_response_fixture` | +| Exact advertised ranges (v1, v3) | `version_firewall_tests` | `apiversions_advertises_exact_supported_ranges_*` | +| All versions return `error_code=0` | `version_firewall_tests` | `apiversions_all_versions_return_success` | +| Out-of-range version | `version_firewall_tests` | `apiversions_out_of_range_returns_unsupported_in_body` | +| E2E correlation ID preserved | `server_e2e_tests` | `e2e_apiversions_v1_*`, `e2e_apiversions_v3_*` | + +### Metadata (key 3, v0–v9) + +| Scenario | Test file | Test name | +|----------|-----------|-----------| +| Stub broker (default 127.0.0.1:9093) | `api_handler_tests`, `metadata_regression_tests` | `metadata_response_has_broker_*`, `metadata_v0_empty_*` | +| Unsupported version → topic error 35 | `api_handler_tests`, `version_firewall_tests` | `unsupported_version_returns_protocol_error`, `metadata_*_version_returns_topic_error` | +| Golden byte fixture (v0, 1 topic) | `golden_wire_fixtures_tests` | `golden_metadata_v0_single_topic_response_fixture` | +| v1 controller_id, v2 cluster_id | `metadata_regression_tests` | `metadata_v1_*`, `metadata_v2_*` | +| v9 flexible encoding | `metadata_regression_tests` | `metadata_v9_flexible_encoding` | +| Custom broker advertise | `metadata_regression_tests`, `broker_advertise_tests` | `metadata_uses_custom_*`, `metadata_reflects_parsed_*` | +| E2E round-trip | `server_e2e_tests` | `e2e_metadata_v0_returns_stub_broker` | + +### Produce (key 0, v3–v9) + +| Scenario | Test file | Test name | +|----------|-----------|-----------| +| Decode all versions (fixture) | `decode_validation_tests` | `produce_all_supported_versions_decode` | +| Response encode all versions | `decode_validation_tests` | `produce_response_encodes_for_all_supported_versions` | +| v3 field layout | `decode_validation_tests` | `produce_response_v3_roundtrip` | +| v8 record_errors array | `decode_validation_tests` | `produce_response_v8_includes_record_errors` | +| Unsupported v2 → error 35 | `version_firewall_tests` | `produce_unsupported_version_returns_error_only` | +| Corrupt body → error 42 | `version_firewall_tests` | `corrupt_produce_body_returns_invalid_request_error` | +| Stub partition error 0 | `handler_regression_tests` | `produce_stub_response_has_zero_error_per_partition` | +| E2E round-trip | `server_e2e_tests` | `e2e_produce_v3_round_trip_with_fixture` | + +### Fetch (key 1, v4–v12) + +| Scenario | Test file | Test name | +|----------|-----------|-----------| +| Decode all versions | `decode_validation_tests` | `fetch_all_supported_versions_decode` | +| Response encode all versions | `decode_validation_tests` | `fetch_response_encodes_for_all_supported_versions` | +| v7 session_id / error_code layout | `decode_validation_tests` | `fetch_response_v7_roundtrip` | +| Unsupported v3 | `version_firewall_tests` | `fetch_unsupported_version_returns_error_only` | +| Corrupt body → error 42 | `version_firewall_tests` | `corrupt_fetch_body_returns_invalid_request_error` | +| Stub partition error 0 | `handler_regression_tests` | `fetch_stub_response_has_zero_partition_error` | + +### ListOffsets (key 2, v1–v6) + +| Scenario | Test file | Test name | +|----------|-----------|-----------| +| Decode all versions | `decode_validation_tests` | `list_offsets_all_supported_versions_decode` | +| v1 no leader_epoch | `decode_validation_tests` | `list_offsets_response_v1_no_leader_epoch` | +| v4 has leader_epoch | `decode_validation_tests` | `list_offsets_response_v4_has_leader_epoch` | +| Unsupported v0 | `version_firewall_tests` | `list_offsets_unsupported_version_returns_error_only` | +| Stub error 0 | `handler_regression_tests` | `list_offsets_stub_response_has_zero_error` | + +### CreateTopics (key 19, v2–v5) + +| Scenario | Test file | Test name | +|----------|-----------|-----------| +| Decode all versions | `decode_validation_tests` | `create_topics_all_supported_versions_decode` | +| v2 roundtrip | `decode_validation_tests` | `create_topics_response_v2_roundtrip` | +| v5 flexible roundtrip | `decode_validation_tests` | `create_topics_response_v5_roundtrip` | +| Unsupported v1 | `version_firewall_tests` | `create_topics_unsupported_version_returns_error_only` | +| Stub error 0 | `handler_regression_tests` | `create_topics_stub_response_has_zero_error` | + +--- + +## Cross-cutting scenarios + +| Scenario | Test file | Test name | +|----------|-----------|-----------| +| Version firewall min/max boundaries | `version_firewall_tests` | `is_supported_version_matches_scope_table` | +| Unknown API keys (8, 9, 10, 17, 20, 999) | `version_firewall_tests`, `api_handler_tests` | `unsupported_api_keys_*`, `unknown_api_key_*` | +| Negative i32 array length | `decode_safety_tests` | `negative_i32_array_length_returns_error_not_panic` | +| Oversized collection count | `decode_safety_tests` | `i32_array_length_above_max_returns_collection_too_large` | +| Compact array varint=0 (null array) | `decode_safety_tests` | `compact_array_varint_zero_decodes_as_empty_without_panic` | +| Malformed varint at shift 63 | `decode_safety_tests` | `varint_terminal_byte_with_extra_bits_at_shift_63_is_rejected` | +| Invalid frame length (0) | `server_integration_tests` | `read_frame_rejects_invalid_lengths` | +| Frame exceeds max_frame_size | `server_integration_tests`, `server_e2e_tests` | `read_frame_rejects_invalid_lengths`, `e2e_oversized_frame_is_rejected` | +| Sequential requests on one TCP connection | `server_e2e_tests` | `e2e_sequential_requests_on_one_connection` | +| Connection survives unsupported API key | `server_e2e_tests` | `e2e_unsupported_api_key_returns_error_without_disconnect` | +| Negative frame length closes connection | `server_e2e_tests` | `e2e_negative_frame_length_closes_connection` | + +--- + +## CI recommendation + +```bash +# 1. Generate fixtures +cargo run -p kafka-message-gen -- generate \ + --output gateways/kafka/tools/kafka-tool/kafka_messages \ + --api-key 0 --api-key 1 --api-key 2 --api-key 19 + +# 2. Run regression suite +cargo test -p iggy_gateway_kafka + +# 3. Optional lint gate +cargo clippy -p iggy_gateway_kafka -- -D warnings +``` + +--- + +## Adding new tests + +1. **New API key or version range** — update `SUPPORTED_RANGES` in `api.rs`, `SCOPE.md`, and add rows to the coverage matrix above. +2. **New decode path** — add fixture via `kafka-message-gen`, extend `decode_validation_tests.rs`. +3. **New error path** — add to `version_firewall_tests.rs` or `decode_safety_tests.rs`. +4. **New TCP behavior** — add to `server_e2e_tests.rs` using helpers in `tests/common/mod.rs`. diff --git a/gateways/kafka/docs/kafka_api_keys_reference.md b/gateways/kafka/docs/kafka_api_keys_reference.md new file mode 100644 index 0000000000..c31cec2d7c --- /dev/null +++ b/gateways/kafka/docs/kafka_api_keys_reference.md @@ -0,0 +1,304 @@ +# Kafka Protocol API Key Reference — Kafka 4.0.0 + +> **Source**: [`ApiKeys.java` @ Kafka 4.0.0](https://github.com/apache/kafka/blob/4.0.0/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java) +> and the canonical [protocol message schemas](https://github.com/apache/kafka/tree/4.0.0/clients/src/main/resources/common/message). +> +> Generated for: **Iggy Kafka Bridge Gateway** — `gateways/kafka/` +> Branch: `feat(gateways)/kafka_to_iggy_listener` + +--- + +## Legend + +| Symbol | Meaning | +|--------|---------| +| 🔴 Bridge | Core data path — must be fully implemented and forwarded to Iggy | +| 🟠 Required Stub | Client state-machine API — must return a well-formed response or clients will stall/crash | +| 🟡 Optional Stub | Admin/observability — can safely return `UNSUPPORTED_VERSION` or `NOT_CONTROLLER` | +| ❌ Reject | Internal broker / KRaft only — return `INVALID_REQUEST` with a well-formed frame; **do not close the connection** | + +> **Header.rs ✓** = The API key is already present in `request_header_version()` / `response_header_version()` with the correct flexible-encoding threshold. + +--- + +## KIP-896 Note — Minimum Version Changes in Kafka 4.0 + +Kafka 4.0 removed all protocol versions older than Kafka 2.1.0 (KIP-896). +Key new minimums: + +| API | Old Min | New Min (4.0) | +|-----|:-------:|:-------------:| +| Produce | 0 | 3 | +| Fetch | 0 | 4 | +| ListOffsets | 0 | 1 | +| OffsetCommit | 0 | 2 | +| OffsetFetch | 0 | 1 | +| CreateTopics | 0 | 2 | +| OffsetForLeaderEpoch | 0 | 1 | + +> ⚠️ **KAFKA-18659 / librdkafka bug**: Even though Produce's actual minimum in Kafka 4.0 is v3, the +> `ApiVersions` response **must advertise min=0** for Produce to avoid breaking librdkafka clients. +> `ApiKeys.java` has a dedicated constant `PRODUCE_API_VERSIONS_RESPONSE_MIN_VERSION = 0` for this. +> Your gateway's `ApiVersions` response encoder must replicate this special case. + +--- + +## Group 1 — Core Data Path + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| 0 | **Produce** | 3 †| 12 | v9 | ✅ | 🔴 Bridge | +| 1 | **Fetch** | 4 | 17 | v12 | ✅ | 🔴 Bridge | +| 2 | **ListOffsets** | 1 | 9 | v6 | ✅ | 🟠 Required Stub | +| 3 | **Metadata** | 1 | 12 | v9 | ✅ | 🔴 Bridge | + +† See KAFKA-18659 librdkafka workaround above — advertise min=0 in ApiVersions response. + +--- + +## Group 2 — API Negotiation & Auth + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| 17 | **SaslHandshake** | 0 | 1 | never | ✅ | 🔴 Bridge (auth flow) | +| 18 | **ApiVersions** | 0 | 4 | v3 | ✅ | 🔴 Bridge (advertise Iggy caps) | +| 36 | **SaslAuthenticate** | 0 | 2 | v2 | ✅ | 🔴 Bridge (auth flow) | + +> **ApiVersions special case**: The response header is **always v0** (no tagged fields), regardless of +> the request version. This allows clients that don't yet know the server's encoding to parse the +> discovery response. `header.rs` already handles this correctly via the `api_key == 18` guard. + +--- + +## Group 3 — Classic Consumer Group Protocol + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| 8 | **OffsetCommit** | 2 | 9 | v8 | ✅ | 🟠 Required Stub | +| 9 | **OffsetFetch** | 1 | 9 | v6 | ✅ | 🟠 Required Stub | +| 10 | **FindCoordinator** | 1 | 6 | v3 | ✅ | 🟠 Required Stub | +| 11 | **JoinGroup** | 2 | 9 | v6 | ✅ | 🟠 Required Stub | +| 12 | **Heartbeat** | 1 | 4 | v4 | ✅ | 🟠 Required Stub | +| 13 | **LeaveGroup** | 1 | 5 | v4 | ✅ | 🟠 Required Stub | +| 14 | **SyncGroup** | 1 | 5 | v4 | ✅ | 🟠 Required Stub | +| 15 | **DescribeGroups** | 0 | 6 | v5 | ✅ | 🟡 Optional Stub | +| 16 | **ListGroups** | 1 | 5 | v3 | ✅ | 🟡 Optional Stub | +| 42 | **DeleteGroups** | 1 | 2 | v2 | ✅ | 🟡 Optional Stub | + +--- + +## Group 4 — New Consumer Group Protocol (KIP-848, Kafka 3.7+) + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| 68 | **ConsumerGroupHeartbeat** | 0 | 1 | v0 | ✅ | 🟠 Required Stub | +| 69 | **ConsumerGroupDescribe** | 0 | 1 | v0 | ✅ | 🟡 Optional Stub | + +> ⚠️ Kafka 4.0 clients use the **new group protocol by default** and will send key 68. +> A gateway that hard-rejects this breaks all modern Kafka consumers. + +--- + +## Group 5 — Topic Administration + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| 19 | **CreateTopics** | 2 | 7 | v5 | ✅ (max v5 ⚠️) | 🟠 Required Stub | +| 20 | **DeleteTopics** | 1 | 6 | v4 | ✅ | 🟡 Optional Stub | +| 21 | **DeleteRecords** | 0 | 2 | v2 | ✅ | 🟡 Optional Stub | +| 37 | **CreatePartitions** | 0 | 3 | v2 | ✅ | 🟡 Optional Stub | + +> ⚠️ `SUPPORTED_RANGES` in `api.rs` currently advertises CreateTopics max=5; actual max is v7. + +--- + +## Group 6 — Transactions (EOS — Exactly Once Semantics) + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| 22 | **InitProducerId** | 2 | 5 | v2 | ✅ | 🟡 Optional Stub | +| 23 | **OffsetForLeaderEpoch** | 1 | 5 | v4 | ✅ | 🟡 Optional Stub | +| 24 | **AddPartitionsToTxn** | 1 | 5 | v3 | ✅ | 🟡 Optional Stub | +| 25 | **AddOffsetsToTxn** | 1 | 4 | v3 | ✅ | 🟡 Optional Stub | +| 26 | **EndTxn** | 1 | 4 | v3 | ✅ | 🟡 Optional Stub | +| 27 | **WriteTxnMarkers** | 0 | 1 | v1 | ✅ | 🟡 Optional Stub | +| 28 | **TxnOffsetCommit** | 2 | 5 | v3 | ✅ | 🟡 Optional Stub | + +--- + +## Group 7 — Security & ACLs + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| 29 | **DescribeAcls** | 0 | 3 | v2 | ✅ | 🟡 Optional Stub | +| 30 | **CreateAcls** | 0 | 3 | v2 | ✅ | 🟡 Optional Stub | +| 31 | **DeleteAcls** | 0 | 3 | v2 | ✅ | 🟡 Optional Stub | +| 38 | **CreateDelegationToken** | 0 | 3 | v2 | ✅ | 🟡 Optional Stub | +| 39 | **RenewDelegationToken** | 0 | 2 | v2 | ✅ | 🟡 Optional Stub | +| 40 | **ExpireDelegationToken** | 0 | 2 | v2 | ✅ | 🟡 Optional Stub | +| 41 | **DescribeDelegationToken** | 0 | 3 | v2 | ✅ | 🟡 Optional Stub | +| 50 | **DescribeUserScramCredentials** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | +| 51 | **AlterUserScramCredentials** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | + +--- + +## Group 8 — Configuration & Quotas + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| 32 | **DescribeConfigs** | 0 | 4 | v4 | ✅ | 🟡 Optional Stub | +| 33 | **AlterConfigs** | 0 | 2 | v2 | ✅ | 🟡 Optional Stub | +| 44 | **IncrementalAlterConfigs** | 0 | 1 | v1 | ✅ | 🟡 Optional Stub | +| 48 | **DescribeClientQuotas** | 0 | 1 | v1 | ✅ | 🟡 Optional Stub | +| 49 | **AlterClientQuotas** | 0 | 1 | v1 | ✅ | 🟡 Optional Stub | + +--- + +## Group 9 — Log & Partition Admin + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| 34 | **AlterReplicaLogDirs** | 0 | 2 | v2 | ✅ | 🟡 Optional Stub | +| 35 | **DescribeLogDirs** | 0 | 4 | v2 | ✅ | 🟡 Optional Stub | +| 43 | **ElectLeaders** | 0 | 2 | v2 | ✅ | 🟡 Optional Stub | +| 45 | **AlterPartitionReassignments** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | +| 46 | **ListPartitionReassignments** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | +| 47 | **OffsetDelete** | 0 | 0 | never | ✅ | 🟡 Optional Stub | +| 57 | **UpdateFeatures** | 0 | 1 | v1 | ✅ | 🟡 Optional Stub | + +--- + +## Group 10 — Cluster Introspection + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| 55 | **DescribeQuorum** | 0 | 2 | v0 | ✅ | 🟡 Optional Stub | +| 59 | **FetchSnapshot** | 0 | 1 | v0 | ✅ | 🟡 Optional Stub | +| 60 | **DescribeCluster** | 0 | 1 | v0 | ✅ | 🟡 Optional Stub | +| 61 | **DescribeProducers** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | +| 64 | **UnregisterBroker** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | +| 65 | **DescribeTransactions** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | +| 66 | **ListTransactions** | 0 | 1 | v0 | ✅ | 🟡 Optional Stub | +| 75 | **DescribeTopicPartitions** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | + +--- + +## Group 11 — Observability / Telemetry (KIP-714, Kafka 3.7+) + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| 71 | **GetTelemetrySubscriptions** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | +| 72 | **PushTelemetry** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | +| 76 | **ListClientMetricsResources** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | + +--- + +## Group 12 — Share Groups (NEW in Kafka 4.0, KIP-932) + +> Keys 77–80 use flexible header framing from v0 (added to `header.rs`). Keys 84–88 are internal +> coordinator APIs and can be rejected. + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| 77 | **ShareGroupHeartbeat** | 0 | 0 | v0 | ✅ | 🟠 Required Stub | +| 78 | **ShareGroupDescribe** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | +| 79 | **ShareFetch** | 0 | 0 | v0 | ✅ | 🔴 Bridge (share consume) | +| 80 | **ShareAcknowledge** | 0 | 0 | v0 | ✅ | 🟠 Required Stub | + +--- + +## Group 13 — KRaft Raft Voter Management (NEW in Kafka 4.0) + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| 81 | **AddRaftVoter** | 0 | 0 | v0 | ❌ MISSING | ❌ Reject (internal) | +| 82 | **RemoveRaftVoter** | 0 | 0 | v0 | ❌ MISSING | ❌ Reject (internal) | +| 83 | **UpdateRaftVoter** | 0 | 0 | v0 | ❌ MISSING | ❌ Reject (internal) | + +--- + +## Group 14 — KRaft Internal / Broker-Only (Always Reject at Gateway) + +> These APIs must **never** be handled by a client-facing gateway. +> Return `INVALID_REQUEST` (error code 42) with a properly framed response — **do not drop the connection**. + +| Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | +|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| 4 | **LeaderAndIsr** | 0 | 7 | v4 | ✅ | ❌ Reject (broker-only) | +| 5 | **StopReplica** | 0 | 4 | v2 | ✅ | ❌ Reject (broker-only) | +| 6 | **UpdateMetadata** | 0 | 8 | v6 | ✅ | ❌ Reject (broker-only) | +| 7 | **ControlledShutdown** | 0 | 3 | v3 | ✅ | ❌ Reject (broker-only) | +| 52 | **Vote** | 0 | 1 | v0 | ✅ | ❌ Reject (KRaft) | +| 53 | **BeginQuorumEpoch** | 0 | 1 | v0 | ✅ | ❌ Reject (KRaft) | +| 54 | **EndQuorumEpoch** | 0 | 1 | v0 | ✅ | ❌ Reject (KRaft) | +| 56 | **AlterPartition** | 0 | 3 | v0 | ✅ | ❌ Reject (KRaft) | +| 58 | **Envelope** | 0 | 0 | v0 | ✅ | ❌ Reject (KRaft) | +| 62 | **BrokerRegistration** | 0 | 4 | v0 | ✅ | ❌ Reject (KRaft) | +| 63 | **BrokerHeartbeat** | 0 | 1 | v0 | ✅ | ❌ Reject (KRaft) | +| 67 | **AllocateProducerIds** | 0 | 0 | v0 | ✅ | ❌ Reject (KRaft) | +| 70 | **ControllerRegistration** | 0 | 0 | v0 | ✅ | ❌ Reject (KRaft) | +| 74 | **AssignReplicasToDirs** | 0 | 0 | v0 | ✅ | ❌ Reject (KRaft) | +| 84 | **InitializeShareGroupState** | 0 | 0 | v0 | ❌ MISSING | ❌ Reject (internal) | +| 85 | **ReadShareGroupState** | 0 | 0 | v0 | ❌ MISSING | ❌ Reject (internal) | +| 86 | **WriteShareGroupState** | 0 | 0 | v0 | ❌ MISSING | ❌ Reject (internal) | +| 87 | **DeleteShareGroupState** | 0 | 0 | v0 | ❌ MISSING | ❌ Reject (internal) | +| 88 | **ReadShareGroupStateSummary** | 0 | 0 | v0 | ❌ MISSING | ❌ Reject (internal) | + +--- + +## Summary Counts + +| Category | Count | Notes | +|----------|:-----:|-------| +| 🔴 Bridge (data path) | 6 | Produce, Fetch, Metadata, ApiVersions, SaslHandshake, SaslAuthenticate, ShareFetch | +| 🟠 Required Stub (client state machine) | 14 | Consumer group, CreateTopics, ConsumerGroupHeartbeat (68), ShareGroupHeartbeat (77), ShareAcknowledge (80) | +| 🟡 Optional Stub (admin/observability) | 44 | Can return `UNSUPPORTED_VERSION` or `NOT_CONTROLLER` safely | +| ❌ Reject (broker/KRaft internal) | 19 | Return `INVALID_REQUEST` with valid frame — never close the TCP connection | +| **Total API Keys in Kafka 4.0** | **83** | Key IDs 0–88 with gap at 73 | + +--- + +## Current Implementation Gaps in `api.rs` + +### `SUPPORTED_RANGES` is behind the latest Kafka 4.0 max versions + +| API | Declared range | Kafka 4.0 max | Gap | +|-----|:---:|:---:|:---:| +| Produce | v3-v9 | v12 | 3 versions behind | +| Fetch | v4-v12 | v17 | 5 versions behind | +| ListOffsets | v1-v6 | v9 | 3 versions behind | +| Metadata | v0-v9 | v12 | 3 versions behind | +| ApiVersions | v0-v3 | v4 | 1 version behind | +| CreateTopics | v2-v5 | v7 | 2 versions behind | + +### Missing from `SUPPORTED_RANGES` (77 API keys) + +Every key not in the table above falls through to `encode_error_only_response` +(2-byte error frame), including: + +- **Client bootstrap blockers**: OffsetCommit (8), OffsetFetch (9), FindCoordinator (10) +- **Classic consumer group protocol**: JoinGroup (11), Heartbeat (12), LeaveGroup (13), SyncGroup (14) +- **New consumer group protocol**: ConsumerGroupHeartbeat (68) — default in Kafka 4.0 +- **Share groups (KIP-932)**: ShareFetch (79), ShareGroupHeartbeat (77), ShareAcknowledge (80) +- **Auth flow**: SaslHandshake (17), SaslAuthenticate (36) +- **All 19 broker/KRaft-internal keys** (Group 14) — should return `INVALID_REQUEST` with a + valid frame instead of the bare 2-byte fallback, so the connection is never dropped. + +### `header.rs` `request_header_version()` — remaining gaps + +Keys 77-80 (ShareGroupHeartbeat, ShareGroupDescribe, ShareFetch, ShareAcknowledge) are +covered (`flexible_from = 0`). Keys 81-88 — AddRaftVoter, RemoveRaftVoter, UpdateRaftVoter, +and the five ShareGroupState keys (84-88) — are all always-flexible per KIP-932/KRaft but +still fall through to the `_ => i16::MAX` (non-flexible) arm, which would misframe them if +ever dispatched. + +## References + +- [ApiKeys.java @ Kafka 4.0.0](https://github.com/apache/kafka/blob/4.0.0/clients/src/main/java/org/apache/kafka/common/protocol/ApiKeys.java) +- [Kafka Protocol Message Schemas @ 4.0.0](https://github.com/apache/kafka/tree/4.0.0/clients/src/main/resources/common/message) +- [KIP-896: Remove old client protocol API versions in Kafka 4.0](https://cwiki.apache.org/confluence/display/KAFKA/KIP-896%3A+Remove+old+client+protocol+API+versions+in+Kafka+4.0) +- [KIP-848: Apache Kafka Consumer Rebalance Protocol](https://cwiki.apache.org/confluence/display/KAFKA/KIP-848%3A+The+Next+Generation+of+the+Consumer+Rebalance+Protocol) +- [KIP-932: Queues for Kafka (Share Groups)](https://cwiki.apache.org/confluence/display/KAFKA/KIP-932%3A+Queues+for+Kafka) +- [KIP-714: Client Metrics and Observability](https://cwiki.apache.org/confluence/display/KAFKA/KIP-714%3A+Client+metrics+and+observability) +- [Kafka Wire Protocol Documentation](https://kafka.apache.org/protocol.html) +- [kafka-protocol Rust crate](https://crates.io/crates/kafka-protocol) diff --git a/gateways/kafka/src/error.rs b/gateways/kafka/src/error.rs index b2ee237387..16b9de936c 100644 --- a/gateways/kafka/src/error.rs +++ b/gateways/kafka/src/error.rs @@ -19,6 +19,8 @@ use thiserror::Error; #[derive(Debug, Error)] pub enum KafkaProtocolError { + #[error("invalid server configuration: {0}")] + InvalidConfig(String), #[error("buffer underflow: needed {needed} bytes, remaining {remaining}")] BufferUnderflow { needed: usize, remaining: usize }, #[error("invalid frame length: {0}")] @@ -36,8 +38,6 @@ pub enum KafkaProtocolError { UnsupportedHeaderVersion(i16), #[error("invalid array length: {0}")] InvalidArrayLength(i32), - #[error("invalid compact array length: encoded value must be >= 1, got {0}")] - InvalidCompactArrayLength(u64), #[error("collection length {count} exceeds maximum {max}")] CollectionTooLarge { count: usize, max: usize }, #[error("string length {length} exceeds i16::MAX")] diff --git a/gateways/kafka/src/lib.rs b/gateways/kafka/src/lib.rs index c8f0cf9e20..a75b625786 100644 --- a/gateways/kafka/src/lib.rs +++ b/gateways/kafka/src/lib.rs @@ -21,4 +21,4 @@ pub mod error; pub mod protocol; pub mod server; -pub use server::{KafkaServer, ServerConfig, init_tracing}; +pub use server::{KafkaServer, ServerConfig}; diff --git a/gateways/kafka/src/main.rs b/gateways/kafka/src/main.rs index 4a5414b9dd..9cd81a4de2 100644 --- a/gateways/kafka/src/main.rs +++ b/gateways/kafka/src/main.rs @@ -25,7 +25,20 @@ use iggy_gateway_kafka::{KafkaServer, ServerConfig}; async fn main() -> Result<(), Box> { init_tracing(); - let config = ServerConfig::default(); + let mut config = ServerConfig::default(); + if let Ok(bind_addr) = std::env::var("KAFKA_BIND_ADDR") { + config.bind_addr = bind_addr; + } + if let Ok(advertised_host) = std::env::var("KAFKA_ADVERTISED_HOST") { + config.advertised_host = Some(advertised_host); + } + if let Ok(advertised_port) = std::env::var("KAFKA_ADVERTISED_PORT") { + config.advertised_port = Some( + advertised_port + .parse() + .map_err(|e| format!("invalid KAFKA_ADVERTISED_PORT `{advertised_port}`: {e}"))?, + ); + } let server = KafkaServer::new(config); let (tx, rx) = broadcast::channel(1); diff --git a/gateways/kafka/src/protocol/api.rs b/gateways/kafka/src/protocol/api.rs index 3e39698841..55cf238f95 100644 --- a/gateways/kafka/src/protocol/api.rs +++ b/gateways/kafka/src/protocol/api.rs @@ -25,8 +25,9 @@ use crate::protocol::requests::{ decode_produce_request, }; use crate::protocol::responses::{ - encode_create_topics_response, encode_fetch_response, encode_list_offsets_response, - encode_produce_response, + encode_create_topics_error_response, encode_create_topics_response, + encode_fetch_error_response, encode_fetch_response, encode_list_offsets_error_response, + encode_list_offsets_response, encode_produce_error_response, encode_produce_response, }; pub const API_KEY_PRODUCE: i16 = 0; @@ -52,6 +53,9 @@ pub const ERROR_INVALID_REPLICATION_FACTOR: i16 = 38; pub const ERROR_INVALID_REQUEST: i16 = 42; pub const ERROR_UNSUPPORTED_FOR_MESSAGE_FORMAT: i16 = 43; +/// Sentinel for `topic_authorized_operations` / `cluster_authorized_operations` when ACLs are not supported. +const AUTHORIZED_OPS_UNKNOWN: i32 = i32::MIN; + #[derive(Debug, Clone)] pub struct BrokerAdvertise { pub host: String, @@ -139,7 +143,8 @@ pub fn handle_request( if is_supported_version(api_key, api_version) { encode_api_versions_response(api_version, ERROR_NONE) } else { - encode_api_versions_response(1, ERROR_UNSUPPORTED_VERSION) + // KIP-511: reply with v0 when the requested version is not understood. + encode_api_versions_response(0, ERROR_UNSUPPORTED_VERSION) } } API_KEY_METADATA => { @@ -155,11 +160,11 @@ pub fn handle_request( Ok(req) => encode_produce_response(api_version, &req), Err(e) => { tracing::error!("Failed to decode Produce request: {:?}", e); - encode_error_only_response(ERROR_CORRUPT_MESSAGE) + encode_produce_error_response(api_version, ERROR_INVALID_REQUEST) } } } else { - encode_error_only_response(ERROR_UNSUPPORTED_VERSION) + encode_produce_error_response(api_version, ERROR_UNSUPPORTED_VERSION) } } API_KEY_FETCH => { @@ -168,11 +173,11 @@ pub fn handle_request( Ok(req) => encode_fetch_response(api_version, &req), Err(e) => { tracing::error!("Failed to decode Fetch request: {:?}", e); - encode_error_only_response(ERROR_CORRUPT_MESSAGE) + encode_fetch_error_response(api_version, ERROR_INVALID_REQUEST) } } } else { - encode_error_only_response(ERROR_UNSUPPORTED_VERSION) + encode_fetch_error_response(api_version, ERROR_UNSUPPORTED_VERSION) } } API_KEY_LIST_OFFSETS => { @@ -181,11 +186,11 @@ pub fn handle_request( Ok(req) => encode_list_offsets_response(api_version, &req), Err(e) => { tracing::error!("Failed to decode ListOffsets request: {:?}", e); - encode_error_only_response(ERROR_CORRUPT_MESSAGE) + encode_list_offsets_error_response(api_version, ERROR_INVALID_REQUEST) } } } else { - encode_error_only_response(ERROR_UNSUPPORTED_VERSION) + encode_list_offsets_error_response(api_version, ERROR_UNSUPPORTED_VERSION) } } API_KEY_CREATE_TOPICS => { @@ -194,11 +199,11 @@ pub fn handle_request( Ok(req) => encode_create_topics_response(api_version, &req), Err(e) => { tracing::error!("Failed to decode CreateTopics request: {:?}", e); - encode_error_only_response(ERROR_CORRUPT_MESSAGE) + encode_create_topics_error_response(api_version, ERROR_INVALID_REQUEST) } } } else { - encode_error_only_response(ERROR_UNSUPPORTED_VERSION) + encode_create_topics_error_response(api_version, ERROR_UNSUPPORTED_VERSION) } } _ => encode_error_only_response(ERROR_UNSUPPORTED_VERSION), @@ -213,6 +218,19 @@ pub fn is_supported_version(api_key: i16, api_version: i16) -> bool { .is_some_and(|r| api_version >= r.min_version && api_version <= r.max_version) } +/// Min version advertised in `ApiVersions` (may differ from the firewall min). +/// +/// Produce must advertise min=0 per KAFKA-18659 / `PRODUCE_API_VERSIONS_RESPONSE_MIN_VERSION` +/// even though this gateway only accepts Produce v3+. +#[must_use] +pub const fn advertised_min_version(api_key: i16, firewall_min: i16) -> i16 { + if api_key == API_KEY_PRODUCE { + 0 + } else { + firewall_min + } +} + fn encode_api_versions_response(api_version: i16, error_code: i16) -> Bytes { let flexible = api_version >= 3; let ranges = SUPPORTED_RANGES; @@ -224,7 +242,7 @@ fn encode_api_versions_response(api_version: i16, error_code: i16) -> Bytes { e.write_varint((ranges.len() + 1) as u64); for r in ranges { e.write_i16(r.api_key); - e.write_i16(r.min_version); + e.write_i16(advertised_min_version(r.api_key, r.min_version)); e.write_i16(r.max_version); e.write_empty_tagged_fields(); } @@ -232,7 +250,7 @@ fn encode_api_versions_response(api_version: i16, error_code: i16) -> Bytes { e.write_i32(i32::try_from(ranges.len()).expect("supported range table is small")); for r in ranges { e.write_i16(r.api_key); - e.write_i16(r.min_version); + e.write_i16(advertised_min_version(r.api_key, r.min_version)); e.write_i16(r.max_version); } } @@ -264,8 +282,8 @@ fn encode_metadata_response( let mut e = Encoder::with_capacity(256); - if api_version >= 1 { - e.write_i32(0); // throttle_time_ms + if api_version >= 3 { + e.write_i32(0); // throttle_time_ms (Metadata v3+) } if flexible { @@ -276,21 +294,19 @@ fn encode_metadata_response( e.write_compact_nullable_string(None); // rack e.write_empty_tagged_fields(); - if api_version >= 2 { - e.write_compact_nullable_string(None); // cluster_id - } - e.write_i32(1); // controller_id + e.write_compact_nullable_string(None); // cluster_id (v2+) + e.write_i32(1); // controller_id (v1+) e.write_varint((topics_count + 1) as u64); for _ in 0..topics_count { e.write_i16(topic_error); e.write_compact_nullable_string(Some("unknown-topic")); - if api_version >= 1 { - e.write_bool(false); // is_internal — must come before partitions array - } + e.write_bool(false); // is_internal (v1+) e.write_varint(1); // empty partitions array + e.write_i32(AUTHORIZED_OPS_UNKNOWN); // topic_authorized_operations (v8+) e.write_empty_tagged_fields(); } + e.write_i32(AUTHORIZED_OPS_UNKNOWN); // cluster_authorized_operations (v8+) e.write_empty_tagged_fields(); } else { e.write_i32(1); // brokers array length @@ -316,6 +332,12 @@ fn encode_metadata_response( e.write_bool(false); // is_internal } e.write_i32(0); // partitions array (empty) + if api_version >= 8 { + e.write_i32(AUTHORIZED_OPS_UNKNOWN); // topic_authorized_operations + } + } + if api_version >= 8 { + e.write_i32(AUTHORIZED_OPS_UNKNOWN); // cluster_authorized_operations } } @@ -330,7 +352,7 @@ pub fn encode_error_only_response(error_code: i16) -> Bytes { } #[must_use] -pub fn split_metadata_request_topics(body: Bytes, api_version: i16) -> usize { +pub(crate) fn split_metadata_request_topics(body: Bytes, api_version: i16) -> usize { let mut d = Decoder::new(body); if api_version >= 9 { d.read_compact_array_count().unwrap_or(0) diff --git a/gateways/kafka/src/protocol/codec.rs b/gateways/kafka/src/protocol/codec.rs index e1c9ce68ba..b46cb6476a 100644 --- a/gateways/kafka/src/protocol/codec.rs +++ b/gateways/kafka/src/protocol/codec.rs @@ -24,6 +24,7 @@ use bytes::{Buf, BufMut, Bytes, BytesMut}; use crate::error::{KafkaProtocolError, Result}; /// Upper bound for Kafka array/collection element counts decoded from the wire. +/// Matches typical broker limits and prevents OOM from adversarial length prefixes. pub const MAX_COLLECTION_LEN: usize = 65_536; pub struct Decoder { @@ -117,7 +118,10 @@ impl Decoder { if n == 0 { return Ok(0); } - let count = (n - 1) as usize; + let count = usize::try_from(n - 1).map_err(|_| KafkaProtocolError::CollectionTooLarge { + count: MAX_COLLECTION_LEN + 1, + max: MAX_COLLECTION_LEN, + })?; if count > MAX_COLLECTION_LEN { return Err(KafkaProtocolError::CollectionTooLarge { count, @@ -148,7 +152,12 @@ impl Decoder { if len_plus_one == 0 { return Ok(None); } - let len = (len_plus_one - 1) as usize; + let len = usize::try_from(len_plus_one - 1).map_err(|_| { + KafkaProtocolError::CollectionTooLarge { + count: MAX_COLLECTION_LEN + 1, + max: MAX_COLLECTION_LEN, + } + })?; self.ensure(len)?; let s = std::str::from_utf8(&self.bytes.chunk()[..len]) .map_err(|_| KafkaProtocolError::InvalidUtf8)? @@ -174,7 +183,12 @@ impl Decoder { if len_plus_one == 0 { return Ok(None); } - let len = (len_plus_one - 1) as usize; + let len = usize::try_from(len_plus_one - 1).map_err(|_| { + KafkaProtocolError::CollectionTooLarge { + count: MAX_COLLECTION_LEN + 1, + max: MAX_COLLECTION_LEN, + } + })?; self.ensure(len)?; Ok(Some(self.bytes.copy_to_bytes(len))) } @@ -197,7 +211,12 @@ impl Decoder { let count = count as usize; for _ in 0..count { self.read_varint()?; // tag number - let size = self.read_varint()? as usize; + let size = usize::try_from(self.read_varint()?).map_err(|_| { + KafkaProtocolError::CollectionTooLarge { + count: MAX_COLLECTION_LEN + 1, + max: MAX_COLLECTION_LEN, + } + })?; self.ensure(size)?; self.bytes.advance(size); } @@ -288,19 +307,21 @@ impl Encoder { } /// Legacy nullable bytes: i32 length prefix, -1 for null. - pub fn write_nullable_bytes(&mut self, v: Option<&[u8]>) { + pub fn write_nullable_bytes(&mut self, v: Option<&[u8]>) -> Result<()> { match v { None => self.write_i32(-1), Some(b) => { - debug_assert!( - b.len() <= i32::MAX as usize, - "byte slice length {} exceeds i32::MAX", - b.len() - ); - self.write_i32(b.len() as i32); + if b.len() > i32::MAX as usize { + return Err(KafkaProtocolError::CollectionTooLarge { + count: b.len(), + max: i32::MAX as usize, + }); + } + self.write_i32(i32::try_from(b.len()).expect("checked above")); self.bytes.put_slice(b); } } + Ok(()) } /// Compact nullable bytes (flexible versions): varint(len+1), 0 for null. diff --git a/gateways/kafka/src/protocol/header.rs b/gateways/kafka/src/protocol/header.rs index 66ce358fe7..7977f4e5c9 100644 --- a/gateways/kafka/src/protocol/header.rs +++ b/gateways/kafka/src/protocol/header.rs @@ -114,6 +114,10 @@ pub fn request_header_version(api_key: i16, api_version: i16) -> i16 { 74 => 0, // AssignReplicasToDirs — always flexible 75 => 0, // DescribeTopicPartitions — always flexible 76 => 0, // ListClientMetricsResources — always flexible + 77 => 0, // ShareGroupHeartbeat — always flexible (Kafka 4.0) + 78 => 0, // ShareGroupDescribe — always flexible + 79 => 0, // ShareFetch — always flexible + 80 => 0, // ShareAcknowledge — always flexible _ => i16::MAX, // Unknown API — assume non-flexible }; if api_version >= flexible_from { 2 } else { 1 } diff --git a/gateways/kafka/src/protocol/requests.rs b/gateways/kafka/src/protocol/requests.rs index d2bc6b74be..91e2a40b66 100644 --- a/gateways/kafka/src/protocol/requests.rs +++ b/gateways/kafka/src/protocol/requests.rs @@ -156,7 +156,7 @@ pub fn decode_fetch_request(version: i16, body: Bytes) -> Result { let isolation_level = if version >= 4 { d.read_i8()? } else { 0 }; - // session_id and session_epoch (v7+) — we'll skip for now + // session_id and session_epoch (v7+) — read and discard (stub path) if version >= 7 { d.read_i32()?; // session_id d.read_i32()?; // session_epoch diff --git a/gateways/kafka/src/protocol/responses.rs b/gateways/kafka/src/protocol/responses.rs index 01d1c1b13e..8de6868ffd 100644 --- a/gateways/kafka/src/protocol/responses.rs +++ b/gateways/kafka/src/protocol/responses.rs @@ -22,21 +22,42 @@ use crate::protocol::api::{ERROR_INVALID_PARTITIONS, ERROR_NONE}; use crate::protocol::codec::Encoder; use crate::protocol::requests::{ - CreateTopicsRequest, FetchRequest, ListOffsetsRequest, ProduceRequest, + CreateTopicsRequest, FetchRequest, ListOffsetsRequest, ProducePartitionData, ProduceRequest, + ProduceTopicData, }; use bytes::Bytes; +/// Well-formed Produce response with a single placeholder topic/partition. +pub fn encode_produce_error_response(version: i16, error_code: i16) -> Bytes { + let topics = vec![ProduceTopicData { + topic: String::new(), + partitions: vec![ProducePartitionData { + partition: 0, + records: None, + }], + }]; + encode_produce_response_inner(version, &topics, error_code) +} + pub fn encode_produce_response(version: i16, req: &ProduceRequest) -> Bytes { + encode_produce_response_inner(version, &req.topics, ERROR_NONE) +} + +fn encode_produce_response_inner( + version: i16, + topics: &[ProduceTopicData], + partition_error: i16, +) -> Bytes { let flexible = version >= 9; let mut e = Encoder::with_capacity(512); if flexible { - e.write_varint((req.topics.len() + 1) as u64); + e.write_varint((topics.len() + 1) as u64); } else { - e.write_i32(i32::try_from(req.topics.len()).expect("topic count bounded")); + e.write_i32(i32::try_from(topics.len()).expect("topic count bounded")); } - for topic in &req.topics { + for topic in topics { if flexible { e.write_compact_nullable_string(Some(&topic.topic)); } else { @@ -51,7 +72,7 @@ pub fn encode_produce_response(version: i16, req: &ProduceRequest) -> Bytes { for p in &topic.partitions { e.write_i32(p.partition); - e.write_i16(ERROR_NONE); + e.write_i16(partition_error); e.write_i64(0); if version >= 2 { e.write_i64(-1); @@ -88,7 +109,36 @@ pub fn encode_produce_response(version: i16, req: &ProduceRequest) -> Bytes { e.freeze() } +/// Well-formed Fetch response. Uses top-level `error_code` at v7+, or a single +/// placeholder topic/partition with per-partition `error_code` below v7. +pub fn encode_fetch_error_response(version: i16, error_code: i16) -> Bytes { + use crate::protocol::requests::{FetchPartition, FetchTopic}; + + if version >= 7 { + return encode_fetch_response_inner(version, &[], Some(error_code), error_code); + } + + let topics = vec![FetchTopic { + topic: String::new(), + partitions: vec![FetchPartition { + partition: 0, + fetch_offset: 0, + partition_max_bytes: 1, + }], + }]; + encode_fetch_response_inner(version, &topics, Some(ERROR_NONE), error_code) +} + pub fn encode_fetch_response(version: i16, req: &FetchRequest) -> Bytes { + encode_fetch_response_inner(version, &req.topics, Some(ERROR_NONE), ERROR_NONE) +} + +fn encode_fetch_response_inner( + version: i16, + topics: &[crate::protocol::requests::FetchTopic], + top_level_error: Option, + partition_error: i16, +) -> Bytes { let flexible = version >= 12; let mut e = Encoder::with_capacity(512); @@ -96,17 +146,17 @@ pub fn encode_fetch_response(version: i16, req: &FetchRequest) -> Bytes { e.write_i32(0); } if version >= 7 { - e.write_i16(ERROR_NONE); + e.write_i16(top_level_error.unwrap_or(ERROR_NONE)); e.write_i32(0); } if flexible { - e.write_varint((req.topics.len() + 1) as u64); + e.write_varint((topics.len() + 1) as u64); } else { - e.write_i32(i32::try_from(req.topics.len()).expect("topic count bounded")); + e.write_i32(i32::try_from(topics.len()).expect("topic count bounded")); } - for topic in &req.topics { + for topic in topics { if flexible { e.write_compact_nullable_string(Some(&topic.topic)); } else { @@ -121,28 +171,29 @@ pub fn encode_fetch_response(version: i16, req: &FetchRequest) -> Bytes { for partition in &topic.partitions { e.write_i32(partition.partition); - e.write_i16(ERROR_NONE); - e.write_i64(0); + e.write_i16(partition_error); + e.write_i64(0); // high_watermark if version >= 4 { - e.write_i64(0); + e.write_i64(0); // last_stable_offset } if version >= 5 { - e.write_i64(0); + e.write_i64(0); // log_start_offset } if version >= 4 { if flexible { - e.write_varint(1); + e.write_varint(1); // empty aborted_transactions } else { - e.write_i32(0); + e.write_i32(0); // empty aborted_transactions } } if version >= 11 { - e.write_i32(-1); + e.write_i32(-1); // preferred_read_replica } if flexible { e.write_compact_nullable_bytes(None); } else { - e.write_nullable_bytes(None); + e.write_nullable_bytes(None) + .expect("null bytes always encode"); } if flexible { e.write_empty_tagged_fields(); @@ -161,7 +212,29 @@ pub fn encode_fetch_response(version: i16, req: &FetchRequest) -> Bytes { e.freeze() } +/// Well-formed ListOffsets response with a single placeholder topic/partition. +pub fn encode_list_offsets_error_response(version: i16, error_code: i16) -> Bytes { + use crate::protocol::requests::{ListOffsetsPartition, ListOffsetsTopic}; + + let topics = vec![ListOffsetsTopic { + topic: String::new(), + partitions: vec![ListOffsetsPartition { + partition: 0, + timestamp: -1, + }], + }]; + encode_list_offsets_response_inner(version, &topics, error_code) +} + pub fn encode_list_offsets_response(version: i16, req: &ListOffsetsRequest) -> Bytes { + encode_list_offsets_response_inner(version, &req.topics, ERROR_NONE) +} + +fn encode_list_offsets_response_inner( + version: i16, + topics: &[crate::protocol::requests::ListOffsetsTopic], + partition_error: i16, +) -> Bytes { let flexible = version >= 6; let mut e = Encoder::with_capacity(256); @@ -170,12 +243,12 @@ pub fn encode_list_offsets_response(version: i16, req: &ListOffsetsRequest) -> B } if flexible { - e.write_varint((req.topics.len() + 1) as u64); + e.write_varint((topics.len() + 1) as u64); } else { - e.write_i32(i32::try_from(req.topics.len()).expect("topic count bounded")); + e.write_i32(i32::try_from(topics.len()).expect("topic count bounded")); } - for topic in &req.topics { + for topic in topics { if flexible { e.write_compact_nullable_string(Some(&topic.topic)); } else { @@ -190,7 +263,7 @@ pub fn encode_list_offsets_response(version: i16, req: &ListOffsetsRequest) -> B for partition in &topic.partitions { e.write_i32(partition.partition); - e.write_i16(ERROR_NONE); + e.write_i16(partition_error); let offset = 0i64; if version >= 1 { @@ -217,7 +290,27 @@ pub fn encode_list_offsets_response(version: i16, req: &ListOffsetsRequest) -> B e.freeze() } +/// Well-formed CreateTopics response with a single placeholder topic. +pub fn encode_create_topics_error_response(version: i16, error_code: i16) -> Bytes { + use crate::protocol::requests::CreatableTopic; + + let topics = vec![CreatableTopic { + name: String::new(), + num_partitions: 1, + replication_factor: 1, + }]; + encode_create_topics_response_inner(version, &topics, error_code) +} + pub fn encode_create_topics_response(version: i16, req: &CreateTopicsRequest) -> Bytes { + encode_create_topics_response_inner(version, &req.topics, ERROR_NONE) +} + +fn encode_create_topics_response_inner( + version: i16, + topics: &[crate::protocol::requests::CreatableTopic], + topic_error: i16, +) -> Bytes { let flexible = version >= 5; let mut e = Encoder::with_capacity(256); @@ -226,19 +319,21 @@ pub fn encode_create_topics_response(version: i16, req: &CreateTopicsRequest) -> } if flexible { - e.write_varint((req.topics.len() + 1) as u64); + e.write_varint((topics.len() + 1) as u64); } else { - e.write_i32(i32::try_from(req.topics.len()).expect("topic count bounded")); + e.write_i32(i32::try_from(topics.len()).expect("topic count bounded")); } - for topic in &req.topics { + for topic in topics { if flexible { e.write_compact_nullable_string(Some(&topic.name)); } else { let _ = e.write_nullable_string(Some(&topic.name)); } - let error_code = if topic.num_partitions <= 0 { + let error_code = if topic_error != ERROR_NONE { + topic_error + } else if topic.num_partitions <= 0 { ERROR_INVALID_PARTITIONS } else { ERROR_NONE @@ -254,7 +349,6 @@ pub fn encode_create_topics_response(version: i16, req: &CreateTopicsRequest) -> } if version >= 5 { - e.write_i16(ERROR_NONE); e.write_i32(topic.num_partitions); e.write_i16(topic.replication_factor); e.write_varint(1); diff --git a/gateways/kafka/src/server.rs b/gateways/kafka/src/server.rs index 02e2bfaf6f..94ef7cb879 100644 --- a/gateways/kafka/src/server.rs +++ b/gateways/kafka/src/server.rs @@ -23,9 +23,9 @@ use bytes::{BufMut, BytesMut}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; use tokio::sync::broadcast; -use tokio::time::timeout; +use tokio::time::{timeout, timeout_at}; use tokio_util::task::TaskTracker; -use tracing::{error, info, warn}; +use tracing::{debug, error, info, warn}; use crate::error::{KafkaProtocolError, Result}; use crate::protocol::api::{ @@ -40,6 +40,11 @@ use std::io; #[derive(Debug, Clone)] pub struct ServerConfig { pub bind_addr: String, + /// Hostname or IP advertised in Metadata (`KAFKA_ADVERTISED_HOST`). Required when `bind_addr` + /// uses a wildcard address (`0.0.0.0` / `::`). + pub advertised_host: Option, + /// Port advertised in Metadata (`KAFKA_ADVERTISED_PORT`). Defaults to the bind port. + pub advertised_port: Option, pub max_frame_size: usize, pub read_timeout: Duration, pub write_timeout: Duration, @@ -49,6 +54,8 @@ impl Default for ServerConfig { fn default() -> Self { Self { bind_addr: "127.0.0.1:9093".to_string(), + advertised_host: None, + advertised_port: None, max_frame_size: 8 * 1024 * 1024, read_timeout: Duration::from_secs(15), write_timeout: Duration::from_secs(10), @@ -56,6 +63,43 @@ impl Default for ServerConfig { } } +impl BrokerAdvertise { + /// Resolve the broker endpoint advertised in Metadata from listener config. + /// + /// # Errors + /// + /// Returns an error when `bind_addr` is invalid, `advertised_host` is empty, or the listener + /// binds to a wildcard address without an explicit advertised host. + pub fn from_server_config(config: &ServerConfig) -> std::result::Result { + let bind = config + .bind_addr + .parse::() + .map_err(|e| format!("invalid bind address `{}`: {e}", config.bind_addr))?; + + let port = config + .advertised_port + .map_or_else(|| i32::from(bind.port()), i32::from); + + let host = if let Some(ref advertised) = config.advertised_host { + let trimmed = advertised.trim(); + if trimmed.is_empty() { + return Err("KAFKA_ADVERTISED_HOST must not be empty".into()); + } + trimmed.to_string() + } else if bind.ip().is_unspecified() { + return Err( + "binding to a wildcard address (0.0.0.0 or ::) requires KAFKA_ADVERTISED_HOST \ + to be set to a reachable hostname or IP for Metadata broker advertisement" + .into(), + ); + } else { + bind.ip().to_string() + }; + + Ok(Self { host, port }) + } +} + pub struct KafkaServer { config: Arc, } @@ -74,19 +118,42 @@ impl KafkaServer { /// /// Returns an error if binding fails or a non-transient `accept()` error occurs. pub async fn run(self, mut shutdown: broadcast::Receiver<()>) -> Result<()> { + let broker = Arc::new( + BrokerAdvertise::from_server_config(&self.config) + .map_err(KafkaProtocolError::InvalidConfig)?, + ); let listener = TcpListener::bind(&self.config.bind_addr).await?; - info!("kafka listener bound on {}", self.config.bind_addr); + info!( + "kafka listener bound on {} (advertised as {}:{})", + self.config.bind_addr, broker.host, broker.port + ); let tracker = TaskTracker::new(); - let broker = Arc::new(BrokerAdvertise::from_bind_addr(&self.config.bind_addr)); + let broker = Arc::clone(&broker); loop { tokio::select! { - _ = shutdown.recv() => { - info!("kafka listener shutdown requested"); - tracker.close(); - tracker.wait().await; - break; + result = shutdown.recv() => { + match result { + Ok(()) => { + info!("kafka listener shutdown requested"); + tracker.close(); + tracker.wait().await; + break; + } + // Capacity-1 channel: lagged means a signal was sent before we polled — treat as shutdown. + Err(broadcast::error::RecvError::Lagged(_)) => { + info!("kafka listener shutdown requested (lagged)"); + tracker.close(); + tracker.wait().await; + break; + } + Err(broadcast::error::RecvError::Closed) => { + tracker.close(); + tracker.wait().await; + break; + } + } } accept_result = listener.accept() => { match accept_result { @@ -130,7 +197,7 @@ async fn handle_connection( peer: SocketAddr, broker: Arc, ) -> Result<()> { - info!(%peer, "connection accepted"); + debug!(%peer, "connection accepted"); loop { let frame = match read_frame(&mut stream, config.max_frame_size, config.read_timeout).await @@ -146,9 +213,9 @@ async fn handle_connection( Err(e) => return Err(e), }; - if frame.len() < 4 { + if frame.len() < 8 { return Err(KafkaProtocolError::BufferUnderflow { - needed: 4, + needed: 8, remaining: frame.len(), }); } @@ -178,7 +245,7 @@ async fn handle_connection( Err(e) => return Err(e), }; - info!( + debug!( %peer, api_key = req.api_key, api_version = req.api_version, @@ -215,14 +282,13 @@ async fn send_response( ) -> Result<()> { let header_size = ResponseHeader::encoded_size(header_version); let payload_size = header_size + body.len(); - if payload_size > i32::MAX as usize { - return Err(KafkaProtocolError::FrameTooLarge { + let payload_len_i32 = + i32::try_from(payload_size).map_err(|_| KafkaProtocolError::FrameTooLarge { max_bytes: i32::MAX as usize, actual_bytes: payload_size, - }); - } + })?; let mut frame = BytesMut::with_capacity(4 + payload_size); - frame.put_i32(payload_size as i32); + frame.put_i32(payload_len_i32); header.encode_into(&mut frame, header_version); frame.put_slice(body); timeout(write_timeout, stream.write_all(&frame)) @@ -232,11 +298,7 @@ async fn send_response( } fn correlation_id_from_frame(frame: &bytes::Bytes) -> i32 { - if frame.len() >= 8 { - i32::from_be_bytes([frame[4], frame[5], frame[6], frame[7]]) - } else { - 0 - } + i32::from_be_bytes([frame[4], frame[5], frame[6], frame[7]]) } /// Read one length-prefixed Kafka frame from `stream`. @@ -262,7 +324,8 @@ pub async fn read_frame( let frame_len = usize::try_from(frame_len_i32).map_err(|_| KafkaProtocolError::FrameTooLarge { max_bytes: max_frame_size, - actual_bytes: u32::MAX as usize, + // Positive i32 that does not fit in `usize` (e.g. 16-bit targets). + actual_bytes: usize::MAX, })?; if frame_len > max_frame_size { return Err(KafkaProtocolError::FrameTooLarge { @@ -272,51 +335,23 @@ pub async fn read_frame( } // read_buf fills BytesMut spare capacity without zero-initializing it first. + // Single deadline for the entire body so a slow-drip sender can't stall indefinitely + // by delivering one byte per timeout window. + let deadline = tokio::time::Instant::now() + read_timeout; let mut data = BytesMut::with_capacity(frame_len); - timeout(read_timeout, async { - while data.len() < frame_len { - if stream.read_buf(&mut data).await? == 0 { - return Err(io::Error::new( - io::ErrorKind::UnexpectedEof, - "connection closed", - )); + while data.len() < frame_len { + match timeout_at(deadline, stream.read_buf(&mut data)).await { + Err(_) => return Err(io::Error::new(io::ErrorKind::TimedOut, "read timeout").into()), + Ok(Ok(0)) => { + return Err( + io::Error::new(io::ErrorKind::UnexpectedEof, "connection closed").into(), + ); } + Ok(Err(e)) => return Err(e.into()), + Ok(Ok(_)) => {} } - Ok::<_, io::Error>(()) - }) - .await - .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "read timeout"))??; - Ok(data.freeze()) -} - -/// Write one length-prefixed Kafka frame to `stream`. -/// -/// # Errors -/// -/// Returns an error on timeout, oversize payload, or I/O failure. -pub async fn write_frame( - stream: &mut TcpStream, - payload: &[u8], - write_timeout: Duration, -) -> Result<()> { - let len = payload.len(); - if len > i32::MAX as usize { - return Err(KafkaProtocolError::FrameTooLarge { - max_bytes: i32::MAX as usize, - actual_bytes: len, - }); } - let mut frame = BytesMut::with_capacity(4 + len); - let len_i32 = i32::try_from(len).map_err(|_| KafkaProtocolError::FrameTooLarge { - max_bytes: i32::MAX as usize, - actual_bytes: len, - })?; - frame.put_i32(len_i32); - frame.extend_from_slice(payload); - timeout(write_timeout, stream.write_all(&frame)) - .await - .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "write timeout"))??; - Ok(()) + Ok(data.freeze()) } pub fn init_tracing() { diff --git a/gateways/kafka/tests/api_handler_tests.rs b/gateways/kafka/tests/api_handler_tests.rs index aca2193708..2d342cfd8e 100644 --- a/gateways/kafka/tests/api_handler_tests.rs +++ b/gateways/kafka/tests/api_handler_tests.rs @@ -19,7 +19,7 @@ use bytes::Bytes; use iggy_gateway_kafka::protocol::api::{ API_KEY_API_VERSIONS, API_KEY_METADATA, BrokerAdvertise, ERROR_UNSUPPORTED_VERSION, - handle_request, is_supported_version, split_metadata_request_topics, supported_api_ranges, + handle_request, is_supported_version, supported_api_ranges, }; fn test_broker() -> BrokerAdvertise { @@ -63,7 +63,7 @@ fn api_versions_v3_response_flexible_format() { // Flexible: varint(len+1) compact array let count_plus_one = d.read_varint().unwrap(); assert!(count_plus_one >= 3); // at least 2 entries → varint = 3+ - let count = (count_plus_one - 1) as i32; + let count = i32::try_from(count_plus_one - 1).expect("api count fits i32"); let mut keys = Vec::new(); for _ in 0..count { @@ -131,13 +131,6 @@ fn unknown_api_key_returns_error_only_payload() { assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); } -#[test] -fn metadata_topic_split_reads_array_count() { - let mut raw = Vec::new(); - raw.extend_from_slice(&2_i32.to_be_bytes()); - assert_eq!(split_metadata_request_topics(Bytes::from(raw), 0), 2); -} - #[test] fn version_support_table_is_applied() { assert!(is_supported_version(API_KEY_API_VERSIONS, 3)); @@ -145,3 +138,14 @@ fn version_support_table_is_applied() { assert!(is_supported_version(API_KEY_METADATA, 1)); assert!(!is_supported_version(API_KEY_METADATA, -1)); } + +#[test] +fn apiversions_unsupported_version_uses_v0_encoding_without_throttle() { + let body = handle_request(API_KEY_API_VERSIONS, 99, Bytes::new(), &test_broker()); + // v0: error_code(2) + api_keys i32 count(4) + 6 entries × 6 bytes = 42 — no throttle_time_ms. + assert_eq!(body.len(), 42); + let mut d = Decoder::new(body); + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); + assert_eq!(d.read_i32().unwrap(), 6); + assert_eq!(d.remaining(), 36); +} diff --git a/gateways/kafka/tests/broker_advertise_tests.rs b/gateways/kafka/tests/broker_advertise_tests.rs new file mode 100644 index 0000000000..9d60d418e4 --- /dev/null +++ b/gateways/kafka/tests/broker_advertise_tests.rs @@ -0,0 +1,112 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! `BrokerAdvertise` parsing and metadata reflection. + +use iggy_gateway_kafka::ServerConfig; +use iggy_gateway_kafka::protocol::api::{API_KEY_METADATA, BrokerAdvertise, handle_request}; +use iggy_gateway_kafka::protocol::codec::{Decoder, Encoder}; + +#[test] +fn from_bind_addr_parses_ipv4_and_port() { + let b = BrokerAdvertise::from_bind_addr("192.168.1.10:19092"); + assert_eq!(b.host, "192.168.1.10"); + assert_eq!(b.port, 19092); +} + +#[test] +fn from_bind_addr_parses_ipv6() { + let b = BrokerAdvertise::from_bind_addr("[::1]:9093"); + assert_eq!(b.host, "::1"); + assert_eq!(b.port, 9093); +} + +#[test] +fn from_bind_addr_invalid_falls_back_to_default() { + let b = BrokerAdvertise::from_bind_addr("not-a-socket-addr"); + assert_eq!(b.host, "127.0.0.1"); + assert_eq!(b.port, 9093); +} + +#[test] +fn default_matches_standard_gateway_port() { + let b = BrokerAdvertise::default(); + assert_eq!(b.host, "127.0.0.1"); + assert_eq!(b.port, 9093); +} + +#[test] +fn metadata_reflects_parsed_bind_addr() { + let broker = BrokerAdvertise::from_bind_addr("203.0.113.7:9093"); + let mut req = Encoder::with_capacity(4); + req.write_i32(0); + let body = handle_request(API_KEY_METADATA, 0, req.freeze(), &broker); + + let mut d = Decoder::new(body); + assert_eq!(d.read_i32().unwrap(), 1); + d.read_i32().unwrap(); + let host = d.read_nullable_string().unwrap().unwrap(); + let port = d.read_i32().unwrap(); + assert_eq!(host, "203.0.113.7"); + assert_eq!(port, 9093); +} + +#[test] +fn from_server_config_uses_explicit_advertised_host_on_wildcard_bind() { + let config = ServerConfig { + bind_addr: "0.0.0.0:9093".to_string(), + advertised_host: Some("kafka.internal".to_string()), + ..ServerConfig::default() + }; + let broker = BrokerAdvertise::from_server_config(&config).expect("valid config"); + assert_eq!(broker.host, "kafka.internal"); + assert_eq!(broker.port, 9093); +} + +#[test] +fn from_server_config_rejects_wildcard_bind_without_advertised_host() { + let config = ServerConfig { + bind_addr: "0.0.0.0:9093".to_string(), + ..ServerConfig::default() + }; + let err = BrokerAdvertise::from_server_config(&config).unwrap_err(); + assert!(err.contains("KAFKA_ADVERTISED_HOST")); +} + +#[test] +fn from_server_config_uses_bind_ip_for_non_wildcard_listener() { + let config = ServerConfig { + bind_addr: "192.168.1.10:19092".to_string(), + ..ServerConfig::default() + }; + let broker = BrokerAdvertise::from_server_config(&config).expect("valid config"); + assert_eq!(broker.host, "192.168.1.10"); + assert_eq!(broker.port, 19092); +} + +#[test] +fn from_server_config_honors_advertised_port_override() { + let config = ServerConfig { + bind_addr: "127.0.0.1:9093".to_string(), + advertised_host: Some("broker.example.com".to_string()), + advertised_port: Some(19093), + ..ServerConfig::default() + }; + let broker = BrokerAdvertise::from_server_config(&config).expect("valid config"); + assert_eq!(broker.host, "broker.example.com"); + assert_eq!(broker.port, 19093); +} diff --git a/gateways/kafka/tests/codec_tests.rs b/gateways/kafka/tests/codec_tests.rs index 0b25fe7929..fcaa966477 100644 --- a/gateways/kafka/tests/codec_tests.rs +++ b/gateways/kafka/tests/codec_tests.rs @@ -28,8 +28,8 @@ fn codec_round_trip_primitives_and_nullable_fields() { enc.write_i64(9_999_999); enc.write_nullable_string(Some("client-a")).unwrap(); enc.write_nullable_string(None).unwrap(); - enc.write_nullable_bytes(Some(&[1, 2, 3])); - enc.write_nullable_bytes(None); + enc.write_nullable_bytes(Some(&[1, 2, 3])).unwrap(); + enc.write_nullable_bytes(None).unwrap(); let bytes = enc.freeze(); let mut dec = Decoder::new(bytes); @@ -72,7 +72,17 @@ fn codec_u8_and_bool() { #[test] fn varint_round_trip_small_values() { - for v in [0u64, 1, 127, 128, 255, 300, 16383, 16384, u32::MAX as u64] { + for v in [ + 0u64, + 1, + 127, + 128, + 255, + 300, + 16383, + 16384, + u64::from(u32::MAX), + ] { let mut enc = Encoder::with_capacity(16); enc.write_varint(v); let mut dec = Decoder::new(enc.freeze()); diff --git a/gateways/kafka/tests/common/fixtures.rs b/gateways/kafka/tests/common/fixtures.rs new file mode 100644 index 0000000000..02eb43d369 --- /dev/null +++ b/gateways/kafka/tests/common/fixtures.rs @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Fixture loaders — compiled into each integration test binary via `#[path]`. +#![allow(dead_code)] + +use std::path::PathBuf; + +use bytes::Bytes; + +use iggy_gateway_kafka::protocol::codec::Decoder; +use iggy_gateway_kafka::protocol::header::{RequestHeader, request_header_version}; + +pub fn fixtures_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tools/kafka-tool/kafka_messages") +} + +pub fn fixture_exists(api_key: i16, api_name: &str, version: i16) -> bool { + let filename = format!("{api_key:03}_{api_name}_v{version}.bin"); + fixtures_dir().join(filename).is_file() +} + +/// Load request body bytes from a kafka-tool `.bin` fixture (skips frame header). +pub fn load_fixture_body(api_key: i16, api_name: &str, version: i16) -> Bytes { + let filename = format!("{api_key:03}_{api_name}_v{version}.bin"); + let path = fixtures_dir().join(&filename); + let data = std::fs::read(&path).unwrap_or_else(|e| panic!("failed to read {filename}: {e}")); + extract_body_from_framed_message(api_key, version, &data) +} + +/// Strip the 4-byte length prefix and Kafka request header from a framed message. +pub fn extract_body_from_framed_message(api_key: i16, api_version: i16, data: &[u8]) -> Bytes { + let frame = Bytes::copy_from_slice(&data[4..]); + let hdr_ver = request_header_version(api_key, api_version); + let mut decoder = Decoder::new(frame); + RequestHeader::decode_from(&mut decoder, hdr_ver).expect("fixture request header must decode"); + decoder + .read_bytes(decoder.remaining()) + .expect("fixture request body must decode") +} diff --git a/gateways/kafka/tests/common/scope.rs b/gateways/kafka/tests/common/scope.rs new file mode 100644 index 0000000000..61a3d37f51 --- /dev/null +++ b/gateways/kafka/tests/common/scope.rs @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Shared scope constants — compiled into each integration test binary via `#[path]`. +#![allow(dead_code)] + +use iggy_gateway_kafka::protocol::api::BrokerAdvertise; + +/// Scoped API keys exercised by the #3421 regression suite. +pub const SCOPED_API_KEYS: &[(i16, &str, i16, i16)] = &[ + (0, "Produce", 3, 9), + (1, "Fetch", 4, 12), + (2, "ListOffsets", 1, 6), + (3, "Metadata", 0, 9), + (18, "ApiVersions", 0, 3), + (19, "CreateTopics", 2, 5), +]; + +pub fn default_broker() -> BrokerAdvertise { + BrokerAdvertise::default() +} diff --git a/gateways/kafka/tests/common/server.rs b/gateways/kafka/tests/common/server.rs new file mode 100644 index 0000000000..178dbce2e5 --- /dev/null +++ b/gateways/kafka/tests/common/server.rs @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Test server spawn helper — compiled into each integration test binary via `#[path]`. +#![allow(dead_code)] + +use std::net::SocketAddr; +use std::time::Duration; + +use tokio::sync::broadcast; + +use iggy_gateway_kafka::{KafkaServer, ServerConfig}; + +/// Bind an ephemeral port, start `KafkaServer`, return address + shutdown sender. +pub async fn spawn_test_server() -> (SocketAddr, broadcast::Sender<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind ephemeral port"); + let addr = listener.local_addr().expect("local addr"); + drop(listener); + + let config = ServerConfig { + bind_addr: addr.to_string(), + advertised_host: None, + advertised_port: None, + max_frame_size: 8 * 1024 * 1024, + read_timeout: Duration::from_secs(5), + write_timeout: Duration::from_secs(5), + }; + let (shutdown_tx, shutdown_rx) = broadcast::channel(1); + let server = KafkaServer::new(config); + tokio::spawn(async move { + let _ = server.run(shutdown_rx).await; + }); + + tokio::time::sleep(Duration::from_millis(50)).await; + (addr, shutdown_tx) +} diff --git a/gateways/kafka/tests/common/tcp.rs b/gateways/kafka/tests/common/tcp.rs new file mode 100644 index 0000000000..6eea90f237 --- /dev/null +++ b/gateways/kafka/tests/common/tcp.rs @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! TCP round-trip helpers — compiled into each integration test binary via `#[path]`. +#![allow(dead_code)] + +use std::net::SocketAddr; + +use bytes::{BufMut, Bytes, BytesMut}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpStream; + +use iggy_gateway_kafka::protocol::codec::Decoder; +use iggy_gateway_kafka::protocol::header::{request_header_version, response_header_version}; + +/// Build a complete length-prefixed Kafka request frame (header + body). +pub fn build_request_frame( + api_key: i16, + api_version: i16, + correlation_id: i32, + client_id: Option<&str>, + body: &[u8], +) -> Bytes { + let hdr_ver = request_header_version(api_key, api_version); + let mut enc = iggy_gateway_kafka::protocol::codec::Encoder::with_capacity(64 + body.len()); + enc.write_i16(api_key); + enc.write_i16(api_version); + enc.write_i32(correlation_id); + if hdr_ver >= 2 { + enc.write_compact_nullable_string(client_id); + enc.write_empty_tagged_fields(); + } else { + enc.write_nullable_string(client_id) + .expect("test client_id fits i16"); + } + enc.write_bytes(body); + + let payload = enc.freeze(); + let payload_len = i32::try_from(payload.len()).expect("test payload fits i32"); + let mut frame = BytesMut::with_capacity(4 + payload.len()); + frame.put_i32(payload_len); + frame.extend_from_slice(&payload); + frame.freeze() +} + +/// Parse correlation id and response body from a raw response payload (no length prefix). +pub fn parse_response_payload(api_key: i16, api_version: i16, payload: Bytes) -> (i32, Bytes) { + let resp_hdr_ver = response_header_version(api_key, api_version); + let mut d = Decoder::new(payload); + let correlation_id = d.read_i32().expect("correlation_id"); + if resp_hdr_ver >= 1 { + d.read_tagged_fields().expect("response tagged fields"); + } + let body = d.read_bytes(d.remaining()).expect("response body"); + (correlation_id, body) +} + +/// Read one length-prefixed response frame from the stream. +pub async fn read_response_frame(stream: &mut TcpStream, max_size: usize) -> Bytes { + let mut len_buf = [0u8; 4]; + stream + .read_exact(&mut len_buf) + .await + .expect("response length prefix"); + let frame_len_i32 = i32::from_be_bytes(len_buf); + assert!(frame_len_i32 > 0, "response frame length must be positive"); + let frame_len = usize::try_from(frame_len_i32).expect("positive i32 frame length fits usize"); + assert!( + frame_len <= max_size, + "response frame too large: {frame_len}" + ); + let mut buf = vec![0u8; frame_len]; + stream.read_exact(&mut buf).await.expect("response body"); + Bytes::from(buf) +} + +/// Send one request frame and return parsed `(correlation_id, response_body)`. +pub async fn round_trip( + addr: SocketAddr, + api_key: i16, + api_version: i16, + correlation_id: i32, + body: &[u8], +) -> (i32, Bytes) { + let mut stream = TcpStream::connect(addr).await.expect("connect"); + let frame = build_request_frame( + api_key, + api_version, + correlation_id, + Some("regression-test"), + body, + ); + stream.write_all(&frame).await.expect("write request"); + let payload = read_response_frame(&mut stream, 8 * 1024 * 1024).await; + parse_response_payload(api_key, api_version, payload) +} diff --git a/gateways/kafka/tests/decode_safety_tests.rs b/gateways/kafka/tests/decode_safety_tests.rs new file mode 100644 index 0000000000..f5d8d84a9a --- /dev/null +++ b/gateways/kafka/tests/decode_safety_tests.rs @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Adversarial wire-input tests for #3421 — malformed lengths must return errors, never panic. + +use bytes::Bytes; + +use iggy_gateway_kafka::error::KafkaProtocolError; +use iggy_gateway_kafka::protocol::codec::{Decoder, Encoder, MAX_COLLECTION_LEN}; +use iggy_gateway_kafka::protocol::requests::decode_produce_request; + +#[test] +fn compact_array_varint_zero_decodes_as_empty_without_panic() { + // Per Kafka spec, compact-array varint=0 means null/absent → 0 elements (not an error). + let mut d = Decoder::new(Bytes::from_static(&[0x00])); + assert_eq!(d.read_compact_array_count().unwrap(), 0); +} + +#[test] +fn negative_i32_array_length_returns_error_not_panic() { + let mut raw = Vec::new(); + raw.extend_from_slice(&(-1_i32).to_be_bytes()); + let mut d = Decoder::new(Bytes::from(raw)); + let err = d.read_i32_array_count().unwrap_err(); + assert!(matches!(err, KafkaProtocolError::InvalidArrayLength(-1))); +} + +#[test] +fn i32_array_length_above_max_returns_collection_too_large() { + let mut raw = Vec::new(); + let oversized = i32::try_from(MAX_COLLECTION_LEN + 1).expect("test value fits i32"); + raw.extend_from_slice(&oversized.to_be_bytes()); + let mut d = Decoder::new(Bytes::from(raw)); + let err = d.read_i32_array_count().unwrap_err(); + assert!(matches!(err, KafkaProtocolError::CollectionTooLarge { .. })); +} + +#[test] +fn produce_decoder_rejects_truncated_flexible_body() { + let mut body = Vec::new(); + body.push(0x00); // transactional_id null (compact) + body.extend_from_slice(&1_i16.to_be_bytes()); // acks + body.extend_from_slice(&1000_i32.to_be_bytes()); // timeout + body.push(0x02); // topics compact array: 1 element (varint = count+1) + // truncated before topic name + + let err = decode_produce_request(9, Bytes::from(body)).unwrap_err(); + assert!(matches!(err, KafkaProtocolError::BufferUnderflow { .. })); +} + +#[test] +fn write_nullable_string_rejects_oversized_length() { + let mut enc = Encoder::with_capacity(8); + let long = "x".repeat(i16::MAX as usize + 1); + let err = enc.write_nullable_string(Some(&long)).unwrap_err(); + assert!(matches!(err, KafkaProtocolError::StringTooLong { .. })); +} + +#[test] +fn varint_terminal_byte_with_extra_bits_at_shift_63_is_rejected() { + // Nine continuation bytes then terminal 0x7E at shift 63 (bits 1-6 set, bit 7 clear). + let mut d = Decoder::new(Bytes::from_static(&[ + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x7E, + ])); + let err = d.read_varint().unwrap_err(); + assert!(matches!(err, KafkaProtocolError::InvalidVarint)); +} diff --git a/gateways/kafka/tests/decode_validation_tests.rs b/gateways/kafka/tests/decode_validation_tests.rs index eaf2cf64c0..8a51e45b06 100644 --- a/gateways/kafka/tests/decode_validation_tests.rs +++ b/gateways/kafka/tests/decode_validation_tests.rs @@ -20,16 +20,17 @@ //! //! Frame layout written by kafka-tool (all versions): //! [4-byte length prefix] -//! [api_key i16][api_version i16][correlation_id i32] -//! [client_id_len i16][client_id bytes] ← always legacy i16, even for flexible APIs -//! [0x00 tagged-fields byte] ← only for flexible API versions +//! [`api_key` i16][`api_version` i16][`correlation_id` i32] +//! header v1: [`client_id`] `NULLABLE_STRING` +//! header v2: [`client_id`] `COMPACT_NULLABLE_STRING` + request-header tagged fields //! [request body] ← properly encoded per spec (flexible or not) use std::path::PathBuf; use bytes::Bytes; -use iggy_gateway_kafka::protocol::header::request_header_version; +use iggy_gateway_kafka::protocol::codec::Decoder; +use iggy_gateway_kafka::protocol::header::{RequestHeader, request_header_version}; use iggy_gateway_kafka::protocol::requests::{ decode_create_topics_request, decode_fetch_request, decode_list_offsets_request, decode_produce_request, @@ -45,31 +46,19 @@ fn fixtures_dir() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tools/kafka-tool/kafka_messages") } -/// Load a kafka-tool .bin file and return just the request body bytes, correctly -/// skipping the outer Kafka frame header (api_key, api_version, correlation_id, -/// legacy-i16 client_id, and — for flexible versions — the 0x00 tagged-fields byte). +/// Load a kafka-tool `.bin` file and return just the request body bytes. fn load_body(api_key: i16, api_name: &str, version: i16) -> Bytes { - let filename = format!("{:03}_{}_v{}.bin", api_key, api_name, version); + let filename = format!("{api_key:03}_{api_name}_v{version}.bin"); let path = fixtures_dir().join(&filename); let data = std::fs::read(&path).unwrap_or_else(|e| panic!("failed to read {filename}: {e}")); - // Skip 4-byte length prefix → frame starts here - let frame = &data[4..]; - - // Bytes 0-7: api_key(2) + api_version(2) + correlation_id(4) - // Bytes 8-9: client_id_len (legacy i16) - let client_id_len = i16::from_be_bytes([frame[8], frame[9]]) as usize; - let body_start_after_client_id = 10 + client_id_len; - - // kafka-tool appends a 0x00 tagged-fields byte for flexible-version APIs - let is_flexible = request_header_version(api_key, version) >= 2; - let body_start = if is_flexible { - body_start_after_client_id + 1 - } else { - body_start_after_client_id - }; - - Bytes::copy_from_slice(&frame[body_start..]) + let frame = Bytes::copy_from_slice(&data[4..]); + let hdr_ver = request_header_version(api_key, version); + let mut decoder = Decoder::new(frame); + RequestHeader::decode_from(&mut decoder, hdr_ver).expect("fixture request header must decode"); + decoder + .read_bytes(decoder.remaining()) + .expect("fixture request body must decode") } // ── Produce (API key 0) ─────────────────────────────────────────────────────── @@ -417,7 +406,7 @@ fn create_topics_response_v2_roundtrip() { } #[test] -fn create_topics_response_v5_has_topic_config_error_code() { +fn create_topics_response_v5_roundtrip() { use iggy_gateway_kafka::protocol::codec::Decoder; let body = load_body(19, "CreateTopics", 5); let req = decode_create_topics_request(5, body).unwrap(); @@ -432,11 +421,6 @@ fn create_topics_response_v5_has_topic_config_error_code() { let error_code = d.read_i16().unwrap(); assert_eq!(error_code, 0); let _error_msg = d.read_compact_nullable_string().unwrap(); // v1+ - let topic_config_err = d.read_i16().unwrap(); // v5+: MUST be present - assert_eq!( - topic_config_err, 0, - "v5 must include topic_config_error_code" - ); let num_partitions = d.read_i32().unwrap(); assert_eq!(num_partitions, 1); let replication_factor = d.read_i16().unwrap(); diff --git a/gateways/kafka/tests/golden_wire_fixtures_tests.rs b/gateways/kafka/tests/golden_wire_fixtures_tests.rs index 62b9e96e60..f146215d34 100644 --- a/gateways/kafka/tests/golden_wire_fixtures_tests.rs +++ b/gateways/kafka/tests/golden_wire_fixtures_tests.rs @@ -28,7 +28,7 @@ fn golden_apiversions_v1_response_fixture() { let actual = handle_request(API_KEY_API_VERSIONS, 1, Bytes::new(), &broker); // error_code=0, api_count=6 - // key 0 (Produce) min=3 max=9 + // key 0 (Produce) min=0 max=9 (KAFKA-18659 advertise min=0) // key 1 (Fetch) min=4 max=12 // key 2 (ListOffsets) min=1 max=6 // key 3 (Metadata) min=0 max=9 @@ -38,7 +38,7 @@ fn golden_apiversions_v1_response_fixture() { let expected: [u8; 46] = [ 0x00, 0x00, // error_code 0x00, 0x00, 0x00, 0x06, // api count = 6 - 0x00, 0x00, 0x00, 0x03, 0x00, 0x09, // key 0: Produce 3–9 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, // key 0: Produce 0–9 (advertised) 0x00, 0x01, 0x00, 0x04, 0x00, 0x0C, // key 1: Fetch 4–12 0x00, 0x02, 0x00, 0x01, 0x00, 0x06, // key 2: ListOffsets 1–6 0x00, 0x03, 0x00, 0x00, 0x00, 0x09, // key 3: Metadata 0–9 diff --git a/gateways/kafka/tests/handler_regression_tests.rs b/gateways/kafka/tests/handler_regression_tests.rs new file mode 100644 index 0000000000..df569f4c12 --- /dev/null +++ b/gateways/kafka/tests/handler_regression_tests.rs @@ -0,0 +1,171 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Full handler regression — every scoped API key × version through `handle_request`. + +#[path = "common/fixtures.rs"] +mod fixtures; +#[path = "common/scope.rs"] +mod scope; + +use iggy_gateway_kafka::protocol::api::{ + API_KEY_CREATE_TOPICS, API_KEY_FETCH, API_KEY_LIST_OFFSETS, API_KEY_PRODUCE, ERROR_NONE, + handle_request, +}; +use iggy_gateway_kafka::protocol::codec::Decoder; + +use fixtures::{fixture_exists, load_fixture_body}; +use scope::{SCOPED_API_KEYS, default_broker}; + +#[test] +fn handle_request_succeeds_for_every_supported_version_with_fixture() { + for &(api_key, name, min_ver, max_ver) in SCOPED_API_KEYS { + if api_key == 3 || api_key == 18 { + // Metadata / ApiVersions: empty body is valid + for version in min_ver..=max_ver { + let resp = handle_request(api_key, version, bytes::Bytes::new(), &default_broker()); + assert!( + !resp.is_empty(), + "{name} v{version} returned empty response" + ); + } + continue; + } + + for version in min_ver..=max_ver { + if !fixture_exists(api_key, name, version) { + continue; + } + let body = load_fixture_body(api_key, name, version); + let resp = handle_request(api_key, version, body, &default_broker()); + assert!( + !resp.is_empty(), + "{name} v{version} returned empty response" + ); + } + } +} + +#[test] +fn produce_stub_response_has_zero_error_per_partition() { + for version in 3i16..=9 { + if !fixture_exists(0, "Produce", version) { + continue; + } + let body = load_fixture_body(0, "Produce", version); + let resp = handle_request(API_KEY_PRODUCE, version, body, &default_broker()); + let flexible = version >= 9; + let mut d = Decoder::new(resp); + if flexible { + let _topics = d.read_varint().unwrap(); + let _topic = d.read_compact_nullable_string().unwrap(); + let _parts = d.read_varint().unwrap(); + } else { + let _topics = d.read_i32().unwrap(); + let _topic = d.read_nullable_string().unwrap(); + let _parts = d.read_i32().unwrap(); + } + let _partition = d.read_i32().unwrap(); + assert_eq!(d.read_i16().unwrap(), ERROR_NONE, "Produce v{version}"); + } +} + +#[test] +fn fetch_stub_response_has_zero_partition_error() { + for version in 4i16..=12 { + if !fixture_exists(1, "Fetch", version) { + continue; + } + let body = load_fixture_body(1, "Fetch", version); + let resp = handle_request(API_KEY_FETCH, version, body, &default_broker()); + let flexible = version >= 12; + let mut d = Decoder::new(resp); + if version >= 1 { + let _throttle = d.read_i32().unwrap(); + } + if version >= 7 { + assert_eq!(d.read_i16().unwrap(), ERROR_NONE); + let _session = d.read_i32().unwrap(); + } + if flexible { + let _topics = d.read_varint().unwrap(); + let _topic = d.read_compact_nullable_string().unwrap(); + let _parts = d.read_varint().unwrap(); + } else { + let _topics = d.read_i32().unwrap(); + let _topic = d.read_nullable_string().unwrap(); + let _parts = d.read_i32().unwrap(); + } + let _partition = d.read_i32().unwrap(); + assert_eq!( + d.read_i16().unwrap(), + ERROR_NONE, + "Fetch v{version} partition error" + ); + } +} + +#[test] +fn list_offsets_stub_response_has_zero_error() { + for version in 1i16..=6 { + if !fixture_exists(2, "ListOffsets", version) { + continue; + } + let body = load_fixture_body(2, "ListOffsets", version); + let resp = handle_request(API_KEY_LIST_OFFSETS, version, body, &default_broker()); + let flexible = version >= 6; + let mut d = Decoder::new(resp); + if version >= 2 { + let _throttle = d.read_i32().unwrap(); + } + if flexible { + let _topics = d.read_varint().unwrap(); + let _topic = d.read_compact_nullable_string().unwrap(); + let _parts = d.read_varint().unwrap(); + } else { + let _topics = d.read_i32().unwrap(); + let _topic = d.read_nullable_string().unwrap(); + let _parts = d.read_i32().unwrap(); + } + let _partition = d.read_i32().unwrap(); + assert_eq!(d.read_i16().unwrap(), ERROR_NONE, "ListOffsets v{version}"); + } +} + +#[test] +fn create_topics_stub_response_has_zero_error() { + for version in 2i16..=5 { + if !fixture_exists(19, "CreateTopics", version) { + continue; + } + let body = load_fixture_body(19, "CreateTopics", version); + let resp = handle_request(API_KEY_CREATE_TOPICS, version, body, &default_broker()); + let flexible = version >= 5; + let mut d = Decoder::new(resp); + if version >= 2 { + let _throttle = d.read_i32().unwrap(); + } + if flexible { + let _topics = d.read_varint().unwrap(); + let _topic = d.read_compact_nullable_string().unwrap(); + } else { + let _topics = d.read_i32().unwrap(); + let _topic = d.read_nullable_string().unwrap(); + } + assert_eq!(d.read_i16().unwrap(), ERROR_NONE, "CreateTopics v{version}"); + } +} diff --git a/gateways/kafka/tests/header_tests.rs b/gateways/kafka/tests/header_tests.rs index e88efe4c22..6ea323fdfe 100644 --- a/gateways/kafka/tests/header_tests.rs +++ b/gateways/kafka/tests/header_tests.rs @@ -130,6 +130,17 @@ fn response_header_version_apiversions_always_zero() { assert_eq!(response_header_version(18, 3), 0); // even flexible request → v0 response } +#[test] +fn share_group_api_keys_use_flexible_header_from_v0() { + for key in [77, 78, 79, 80] { + assert_eq!( + request_header_version(key, 0), + 2, + "api_key {key} must use flexible header v2" + ); + } +} + #[test] fn response_header_version_flexible_non_apiversions() { // Metadata v9+ is flexible → response header v1 diff --git a/gateways/kafka/tests/metadata_regression_tests.rs b/gateways/kafka/tests/metadata_regression_tests.rs new file mode 100644 index 0000000000..50c91bb1c4 --- /dev/null +++ b/gateways/kafka/tests/metadata_regression_tests.rs @@ -0,0 +1,228 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Metadata API regression — all supported versions, broker advertise, topic counts. + +#[path = "common/scope.rs"] +mod scope; + +use bytes::Bytes; + +use iggy_gateway_kafka::protocol::api::{ + API_KEY_METADATA, BrokerAdvertise, ERROR_UNKNOWN_TOPIC_OR_PARTITION, handle_request, +}; +use iggy_gateway_kafka::protocol::codec::{Decoder, Encoder}; + +use scope::default_broker; + +fn metadata_request_legacy(topic_count: i32) -> Bytes { + let mut enc = Encoder::with_capacity(8); + enc.write_i32(topic_count); + enc.freeze() +} + +fn metadata_request_flexible(topic_count: usize) -> Bytes { + let mut enc = Encoder::with_capacity(8); + enc.write_varint((topic_count + 1) as u64); + enc.freeze() +} + +fn read_broker_legacy(d: &mut Decoder) -> (String, i32) { + let count = d.read_i32().unwrap(); + assert_eq!(count, 1); + let _node = d.read_i32().unwrap(); + let host = d.read_nullable_string().unwrap().unwrap(); + let port = d.read_i32().unwrap(); + (host, port) +} + +fn read_broker_flexible(d: &mut Decoder) -> (String, i32) { + let count_plus_one = d.read_varint().unwrap(); + assert_eq!(count_plus_one, 2); // one broker + let _node = d.read_i32().unwrap(); + let host = d.read_compact_nullable_string().unwrap().unwrap(); + let port = d.read_i32().unwrap(); + let _rack = d.read_compact_nullable_string().unwrap(); + d.read_tagged_fields().unwrap(); + (host, port) +} + +#[test] +fn metadata_v0_empty_topics_stub_broker() { + let body = handle_request( + API_KEY_METADATA, + 0, + metadata_request_legacy(0), + &default_broker(), + ); + let mut d = Decoder::new(body); + let (host, port) = read_broker_legacy(&mut d); + assert_eq!(host, "127.0.0.1"); + assert_eq!(port, 9093); + assert_eq!(d.read_i32().unwrap(), 0); +} + +#[test] +fn metadata_v0_three_topics_each_unknown() { + let body = handle_request( + API_KEY_METADATA, + 0, + metadata_request_legacy(3), + &default_broker(), + ); + let mut d = Decoder::new(body); + let _ = read_broker_legacy(&mut d); + assert_eq!(d.read_i32().unwrap(), 3); + for _ in 0..3 { + assert_eq!(d.read_i16().unwrap(), ERROR_UNKNOWN_TOPIC_OR_PARTITION); + assert_eq!(d.read_nullable_string().unwrap().unwrap(), "unknown-topic"); + assert_eq!(d.read_i32().unwrap(), 0); + } +} + +#[test] +fn metadata_v1_includes_controller_id() { + let body = handle_request( + API_KEY_METADATA, + 1, + metadata_request_legacy(0), + &default_broker(), + ); + let mut d = Decoder::new(body); + // Metadata v1 has no throttle_time_ms (added in v3). + let _ = read_broker_legacy(&mut d); + let _rack = d.read_nullable_string().unwrap(); + let controller = d.read_i32().unwrap(); + assert_eq!(controller, 1); +} + +#[test] +fn metadata_v2_includes_cluster_id_field() { + let body = handle_request( + API_KEY_METADATA, + 2, + metadata_request_legacy(0), + &default_broker(), + ); + let mut d = Decoder::new(body); + let _ = read_broker_legacy(&mut d); + let _rack = d.read_nullable_string().unwrap(); + let _cluster_id = d.read_nullable_string().unwrap(); + let _controller = d.read_i32().unwrap(); + assert_eq!(d.read_i32().unwrap(), 0); +} + +#[test] +fn metadata_all_legacy_versions_produce_valid_response() { + for version in 0i16..=8 { + let body = handle_request( + API_KEY_METADATA, + version, + metadata_request_legacy(1), + &default_broker(), + ); + let mut d = Decoder::new(body); + if version >= 3 { + let _throttle = d.read_i32().unwrap(); + } + let _ = read_broker_legacy(&mut d); + if version >= 1 { + let _rack = d.read_nullable_string().unwrap(); + } + if version >= 2 { + let _cluster = d.read_nullable_string().unwrap(); + } + if version >= 1 { + let _controller = d.read_i32().unwrap(); + } + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_i16().unwrap(), ERROR_UNKNOWN_TOPIC_OR_PARTITION); + } +} + +#[test] +fn metadata_v9_flexible_encoding() { + let body = handle_request( + API_KEY_METADATA, + 9, + metadata_request_flexible(2), + &default_broker(), + ); + let mut d = Decoder::new(body); + let _throttle = d.read_i32().unwrap(); + let (host, port) = read_broker_flexible(&mut d); + assert_eq!(host, "127.0.0.1"); + assert_eq!(port, 9093); + let _cluster = d.read_compact_nullable_string().unwrap(); + let controller = d.read_i32().unwrap(); + assert_eq!(controller, 1); + + let topics_plus_one = d.read_varint().unwrap(); + assert_eq!(topics_plus_one, 3); // 2 topics + for _ in 0..2 { + assert_eq!(d.read_i16().unwrap(), ERROR_UNKNOWN_TOPIC_OR_PARTITION); + assert_eq!( + d.read_compact_nullable_string().unwrap().unwrap(), + "unknown-topic" + ); + let _internal = d.read_bool().unwrap(); + let parts_plus_one = d.read_varint().unwrap(); + assert_eq!(parts_plus_one, 1); // empty partitions + assert_eq!(d.read_i32().unwrap(), i32::MIN); // topic_authorized_operations (v8+) + d.read_tagged_fields().unwrap(); + } + assert_eq!(d.read_i32().unwrap(), i32::MIN); // cluster_authorized_operations (v8+) + d.read_tagged_fields().unwrap(); + assert_eq!(d.remaining(), 0); +} + +#[test] +fn metadata_v8_includes_authorized_operations_legacy() { + let body = handle_request( + API_KEY_METADATA, + 8, + metadata_request_legacy(1), + &default_broker(), + ); + let mut d = Decoder::new(body); + let _throttle = d.read_i32().unwrap(); + let _ = read_broker_legacy(&mut d); + let _rack = d.read_nullable_string().unwrap(); + let _cluster = d.read_nullable_string().unwrap(); + let _controller = d.read_i32().unwrap(); + assert_eq!(d.read_i32().unwrap(), 1); + let _topic_error = d.read_i16().unwrap(); + let _topic = d.read_nullable_string().unwrap(); + let _internal = d.read_bool().unwrap(); + assert_eq!(d.read_i32().unwrap(), 0); // empty partitions + assert_eq!(d.read_i32().unwrap(), i32::MIN); // topic_authorized_operations + assert_eq!(d.read_i32().unwrap(), i32::MIN); // cluster_authorized_operations + assert_eq!(d.remaining(), 0); +} + +#[test] +fn metadata_uses_custom_broker_advertise() { + let broker = BrokerAdvertise { + host: "10.0.0.42".to_string(), + port: 29093, + }; + let body = handle_request(API_KEY_METADATA, 0, metadata_request_legacy(0), &broker); + let mut d = Decoder::new(body); + let (host, port) = read_broker_legacy(&mut d); + assert_eq!(host, "10.0.0.42"); + assert_eq!(port, 29093); +} diff --git a/gateways/kafka/tests/server_e2e_tests.rs b/gateways/kafka/tests/server_e2e_tests.rs new file mode 100644 index 0000000000..d0846ba49f --- /dev/null +++ b/gateways/kafka/tests/server_e2e_tests.rs @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! End-to-end TCP tests through `KafkaServer` (full request/response cycle). + +#[path = "common/fixtures.rs"] +mod fixtures; +#[path = "common/server.rs"] +mod server; +#[path = "common/tcp.rs"] +mod tcp; + +use bytes::{BufMut, BytesMut}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpStream; + +use iggy_gateway_kafka::protocol::api::{ + API_KEY_API_VERSIONS, API_KEY_METADATA, API_KEY_PRODUCE, ERROR_UNSUPPORTED_VERSION, +}; +use iggy_gateway_kafka::protocol::codec::Decoder; + +use fixtures::load_fixture_body; +use server::spawn_test_server; +use tcp::{build_request_frame, parse_response_payload, read_response_frame, round_trip}; + +#[tokio::test] +async fn e2e_apiversions_v1_preserves_correlation_id() { + let (addr, _shutdown) = spawn_test_server().await; + let (corr, body) = round_trip(addr, API_KEY_API_VERSIONS, 1, 42_001, &[]).await; + assert_eq!(corr, 42_001); + let mut d = Decoder::new(body); + assert_eq!(d.read_i16().unwrap(), 0); +} + +#[tokio::test] +async fn e2e_apiversions_v3_flexible_preserves_correlation_id() { + let (addr, _shutdown) = spawn_test_server().await; + let (corr, body) = round_trip(addr, API_KEY_API_VERSIONS, 3, 42_002, &[]).await; + assert_eq!(corr, 42_002); + let mut d = Decoder::new(body); + assert_eq!(d.read_i16().unwrap(), 0); + let count = usize::try_from(d.read_varint().unwrap() - 1).expect("api count fits usize"); + assert_eq!(count, 6); +} + +#[tokio::test] +async fn e2e_metadata_v0_returns_stub_broker() { + let (addr, _shutdown) = spawn_test_server().await; + let mut req = BytesMut::new(); + req.put_i32(0); // empty topics + let (corr, body) = round_trip(addr, API_KEY_METADATA, 0, 77, &req).await; + assert_eq!(corr, 77); + let mut d = Decoder::new(body); + assert_eq!(d.read_i32().unwrap(), 1); + d.read_i32().unwrap(); + let host = d.read_nullable_string().unwrap().unwrap(); + assert_eq!(host, "127.0.0.1"); +} + +#[tokio::test] +async fn e2e_produce_v3_round_trip_with_fixture() { + let (addr, _shutdown) = spawn_test_server().await; + let body = load_fixture_body(0, "Produce", 3); + let (corr, resp_body) = round_trip(addr, API_KEY_PRODUCE, 3, 88, &body).await; + assert_eq!(corr, 88); + assert!(!resp_body.is_empty()); +} + +#[tokio::test] +async fn e2e_unsupported_api_key_returns_error_without_disconnect() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.unwrap(); + + let frame1 = build_request_frame(8, 2, 99, Some("e2e-test"), &[]); + stream.write_all(&frame1).await.unwrap(); + let payload1 = read_response_frame(&mut stream, 8 * 1024 * 1024).await; + let (corr, body) = parse_response_payload(8, 2, payload1); + assert_eq!(corr, 99); + let mut d = Decoder::new(body); + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); + + // Second request on same connection must still work. + let frame2 = build_request_frame(API_KEY_API_VERSIONS, 1, 100, Some("e2e-test"), &[]); + stream.write_all(&frame2).await.unwrap(); + let payload2 = read_response_frame(&mut stream, 8 * 1024 * 1024).await; + let (corr2, body2) = parse_response_payload(API_KEY_API_VERSIONS, 1, payload2); + assert_eq!(corr2, 100); + let mut d2 = Decoder::new(body2); + assert_eq!(d2.read_i16().unwrap(), 0); +} + +#[tokio::test] +async fn e2e_sequential_requests_on_one_connection() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.unwrap(); + + let requests = [(API_KEY_API_VERSIONS, 1i16), (API_KEY_METADATA, 0i16)]; + for (i, (key, ver)) in requests.iter().enumerate() { + let meta_body = { + let mut b = BytesMut::new(); + b.put_i32(0); + b + }; + let body: &[u8] = if *key == API_KEY_METADATA { + &meta_body + } else { + &[] + }; + let correlation_id = 1000 + i32::try_from(i).expect("test index fits i32"); + let frame = build_request_frame(*key, *ver, correlation_id, Some("seq-test"), body); + stream.write_all(&frame).await.unwrap(); + let payload = read_response_frame(&mut stream, 8 * 1024 * 1024).await; + let (corr, _) = parse_response_payload(*key, *ver, payload); + assert_eq!(corr, correlation_id); + } +} + +#[tokio::test] +async fn e2e_negative_frame_length_closes_connection() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.unwrap(); + stream.write_all(&(-1i32).to_be_bytes()).await.unwrap(); + + let mut buf = [0u8; 1]; + let n = stream.read(&mut buf).await.unwrap_or(0); + assert_eq!(n, 0, "server should close after invalid frame length"); +} + +#[tokio::test] +async fn e2e_oversized_frame_is_rejected() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.unwrap(); + + let mut frame = BytesMut::new(); + frame.put_i32(10_000_000); // exceeds default 8 MiB cap + frame.resize(4 + 100, 0); + stream.write_all(&frame).await.unwrap(); + + let mut buf = [0u8; 1]; + let n = stream.read(&mut buf).await.unwrap_or(0); + assert_eq!(n, 0, "server should close after oversized frame"); +} diff --git a/gateways/kafka/tests/server_integration_tests.rs b/gateways/kafka/tests/server_integration_tests.rs index 35bc4ff113..9de8f9b092 100644 --- a/gateways/kafka/tests/server_integration_tests.rs +++ b/gateways/kafka/tests/server_integration_tests.rs @@ -17,12 +17,12 @@ use std::time::Duration; -use bytes::{Buf, BytesMut}; +use bytes::{Buf, BufMut, BytesMut}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; use iggy_gateway_kafka::protocol::codec::Encoder; -use iggy_gateway_kafka::server::{read_frame, write_frame}; +use iggy_gateway_kafka::server::read_frame; async fn tcp_pair() -> (TcpStream, TcpStream) { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -33,6 +33,23 @@ async fn tcp_pair() -> (TcpStream, TcpStream) { (client, server) } +/// Raw length-prefixed write (no Kafka response header) — mirrors `server::write_frame`. +async fn write_length_prefixed( + stream: &mut TcpStream, + payload: &[u8], + write_timeout: Duration, +) -> Result<(), Box> { + let len = payload.len(); + assert!(i32::try_from(len).is_ok()); + let mut frame = BytesMut::with_capacity(4 + len); + frame.put_i32(i32::try_from(len).expect("len fits i32")); + frame.extend_from_slice(payload); + tokio::time::timeout(write_timeout, stream.write_all(&frame)) + .await + .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "write timeout"))??; + Ok(()) +} + #[tokio::test] async fn read_frame_reads_valid_payload() { let (mut client, mut server) = tcp_pair().await; @@ -45,7 +62,11 @@ async fn read_frame_reads_valid_payload() { let payload = enc.freeze(); let mut frame = BytesMut::with_capacity(4 + payload.len()); - frame.extend_from_slice(&(payload.len() as i32).to_be_bytes()); + frame.extend_from_slice( + &i32::try_from(payload.len()) + .expect("test payload fits i32") + .to_be_bytes(), + ); frame.extend_from_slice(&payload); client.write_all(&frame).await.unwrap(); @@ -59,13 +80,13 @@ async fn read_frame_reads_valid_payload() { async fn write_frame_writes_length_prefixed_payload() { let (mut client, mut server) = tcp_pair().await; let payload = b"abc123"; - write_frame(&mut server, payload, Duration::from_secs(1)) + write_length_prefixed(&mut server, payload, Duration::from_secs(1)) .await .unwrap(); let mut len = [0u8; 4]; client.read_exact(&mut len).await.unwrap(); - let len = i32::from_be_bytes(len) as usize; + let len = usize::try_from(i32::from_be_bytes(len)).expect("positive frame length"); assert_eq!(len, payload.len()); let mut body = vec![0u8; len]; @@ -97,7 +118,7 @@ async fn read_frame_rejects_invalid_lengths() { #[tokio::test] async fn write_frame_length_prefix_is_big_endian() { let (mut client, mut server) = tcp_pair().await; - write_frame(&mut server, &[1, 2, 3, 4], Duration::from_secs(1)) + write_length_prefixed(&mut server, &[1, 2, 3, 4], Duration::from_secs(1)) .await .unwrap(); diff --git a/gateways/kafka/tests/version_firewall_tests.rs b/gateways/kafka/tests/version_firewall_tests.rs new file mode 100644 index 0000000000..de0f96e03b --- /dev/null +++ b/gateways/kafka/tests/version_firewall_tests.rs @@ -0,0 +1,309 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Version negotiation firewall — boundary tests for every scoped API key. + +#[path = "common/fixtures.rs"] +mod fixtures; +#[path = "common/scope.rs"] +mod scope; + +use bytes::Bytes; + +use iggy_gateway_kafka::protocol::api::{ + API_KEY_API_VERSIONS, API_KEY_CREATE_TOPICS, API_KEY_FETCH, API_KEY_LIST_OFFSETS, + API_KEY_METADATA, API_KEY_PRODUCE, ERROR_INVALID_REQUEST, ERROR_UNSUPPORTED_VERSION, + advertised_min_version, handle_request, is_supported_version, supported_api_ranges, +}; +use iggy_gateway_kafka::protocol::codec::Decoder; + +use fixtures::load_fixture_body; +use scope::{SCOPED_API_KEYS, default_broker}; + +#[test] +fn supported_ranges_table_has_six_entries() { + assert_eq!(supported_api_ranges().len(), 6); +} + +#[test] +fn is_supported_version_matches_scope_table() { + for &(api_key, _, min_ver, max_ver) in SCOPED_API_KEYS { + assert!( + !is_supported_version(api_key, min_ver - 1), + "key {api_key} must reject v{}", + min_ver - 1 + ); + assert!( + is_supported_version(api_key, min_ver), + "key {api_key} must accept min v{min_ver}" + ); + assert!( + is_supported_version(api_key, max_ver), + "key {api_key} must accept max v{max_ver}" + ); + assert!( + !is_supported_version(api_key, max_ver + 1), + "key {api_key} must reject v{}", + max_ver + 1 + ); + } +} + +#[test] +fn apiversions_advertises_exact_supported_ranges_v1() { + let body = handle_request(API_KEY_API_VERSIONS, 1, Bytes::new(), &default_broker()); + let mut d = Decoder::new(body); + assert_eq!(d.read_i16().unwrap(), 0); + let count = usize::try_from(d.read_i32().unwrap()).expect("api count fits usize"); + assert_eq!(count, supported_api_ranges().len()); + + for expected in supported_api_ranges() { + let key = d.read_i16().unwrap(); + let min = d.read_i16().unwrap(); + let max = d.read_i16().unwrap(); + assert_eq!(key, expected.api_key); + assert_eq!( + min, + advertised_min_version(expected.api_key, expected.min_version) + ); + assert_eq!(max, expected.max_version); + } + assert_eq!(d.read_i32().unwrap(), 0); // throttle + assert_eq!(d.remaining(), 0); +} + +#[test] +fn apiversions_advertises_exact_supported_ranges_v3_flexible() { + let body = handle_request(API_KEY_API_VERSIONS, 3, Bytes::new(), &default_broker()); + let mut d = Decoder::new(body); + assert_eq!(d.read_i16().unwrap(), 0); + let count = usize::try_from(d.read_varint().unwrap() - 1).expect("api count fits usize"); + assert_eq!(count, supported_api_ranges().len()); + + for expected in supported_api_ranges() { + let key = d.read_i16().unwrap(); + let min = d.read_i16().unwrap(); + let max = d.read_i16().unwrap(); + d.read_tagged_fields().unwrap(); + assert_eq!(key, expected.api_key); + assert_eq!( + min, + advertised_min_version(expected.api_key, expected.min_version) + ); + assert_eq!(max, expected.max_version); + } + assert_eq!(d.read_i32().unwrap(), 0); + d.read_tagged_fields().unwrap(); + assert_eq!(d.remaining(), 0); +} + +#[test] +fn apiversions_advertises_produce_min_zero_while_firewall_stays_three() { + let range = supported_api_ranges() + .iter() + .find(|r| r.api_key == API_KEY_PRODUCE) + .expect("produce range"); + assert_eq!(range.min_version, 3); + assert_eq!( + advertised_min_version(API_KEY_PRODUCE, range.min_version), + 0 + ); + assert!(!is_supported_version(API_KEY_PRODUCE, 0)); +} + +#[test] +fn apiversions_all_versions_return_success() { + for version in 0i16..=3 { + let body = handle_request( + API_KEY_API_VERSIONS, + version, + Bytes::new(), + &default_broker(), + ); + let mut d = Decoder::new(body); + assert_eq!(d.read_i16().unwrap(), 0, "ApiVersions v{version}"); + } +} + +#[test] +fn apiversions_out_of_range_returns_unsupported_in_body() { + let body = handle_request(API_KEY_API_VERSIONS, 99, Bytes::new(), &default_broker()); + let mut d = Decoder::new(body); + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); +} + +fn metadata_request_one_topic() -> Bytes { + let mut raw = Vec::new(); + raw.extend_from_slice(&1_i32.to_be_bytes()); + Bytes::from(raw) +} + +#[test] +fn metadata_below_min_version_returns_topic_error() { + let body = handle_request( + API_KEY_METADATA, + -1, + metadata_request_one_topic(), + &default_broker(), + ); + let mut d = Decoder::new(body); + let _brokers = d.read_i32().unwrap(); + let _ = d.read_i32().unwrap(); + let _ = d.read_nullable_string().unwrap(); + let _ = d.read_i32().unwrap(); + assert_eq!(d.read_i32().unwrap(), 1); // mirrors request topic count + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); +} + +#[test] +fn metadata_above_max_version_returns_topic_error() { + let body = handle_request( + API_KEY_METADATA, + 10, + metadata_request_one_topic(), + &default_broker(), + ); + let mut d = Decoder::new(body); + let _brokers = d.read_i32().unwrap(); + let _ = d.read_i32().unwrap(); + let _ = d.read_nullable_string().unwrap(); + let _ = d.read_i32().unwrap(); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); +} + +#[test] +fn produce_unsupported_version_returns_well_formed_error_response() { + let body = handle_request(API_KEY_PRODUCE, 2, Bytes::new(), &default_broker()); + let mut d = Decoder::new(body); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); + let _ = d.read_i64().unwrap(); + let _ = d.read_i64().unwrap(); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.remaining(), 0); +} + +#[test] +fn fetch_unsupported_version_returns_well_formed_error_response() { + let body = handle_request(API_KEY_FETCH, 3, Bytes::new(), &default_broker()); + let mut d = Decoder::new(body); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); + assert_eq!(d.read_i64().unwrap(), 0); + assert_eq!(d.read_nullable_bytes().unwrap(), None); + assert_eq!(d.remaining(), 0); +} + +#[test] +fn fetch_unsupported_version_above_max_uses_top_level_error() { + let body = handle_request(API_KEY_FETCH, 13, Bytes::new(), &default_broker()); + let mut d = Decoder::new(body); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.read_varint().unwrap(), 1); + d.read_tagged_fields().unwrap(); + assert_eq!(d.remaining(), 0); +} + +#[test] +fn list_offsets_unsupported_version_returns_well_formed_error_response() { + let body = handle_request(API_KEY_LIST_OFFSETS, 0, Bytes::new(), &default_broker()); + let mut d = Decoder::new(body); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); + assert_eq!(d.read_i64().unwrap(), 0); + assert_eq!(d.remaining(), 0); +} + +#[test] +fn create_topics_unsupported_version_returns_well_formed_error_response() { + let body = handle_request(API_KEY_CREATE_TOPICS, 1, Bytes::new(), &default_broker()); + let mut d = Decoder::new(body); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); + assert_eq!(d.read_nullable_string().unwrap(), None); + assert_eq!(d.remaining(), 0); +} + +#[test] +fn unsupported_api_keys_return_error_only() { + for key in [8, 9, 10, 11, 17, 20, 42, 999] { + let body = handle_request(key, 0, Bytes::new(), &default_broker()); + let mut d = Decoder::new(body); + assert_eq!( + d.read_i16().unwrap(), + ERROR_UNSUPPORTED_VERSION, + "api_key {key}" + ); + } +} + +#[test] +fn supported_produce_versions_accept_valid_fixture() { + for version in 3i16..=9 { + let body = load_fixture_body(0, "Produce", version); + let resp = handle_request(API_KEY_PRODUCE, version, body, &default_broker()); + assert!(!resp.is_empty(), "Produce v{version} response empty"); + } +} + +#[test] +fn supported_fetch_versions_accept_valid_fixture() { + for version in 4i16..=12 { + let body = load_fixture_body(1, "Fetch", version); + let resp = handle_request(API_KEY_FETCH, version, body, &default_broker()); + assert!(!resp.is_empty(), "Fetch v{version} response empty"); + } +} + +#[test] +fn corrupt_produce_body_returns_invalid_request_error() { + let body = Bytes::from_static(&[0xFF, 0xFF, 0xFF]); + let resp = handle_request(API_KEY_PRODUCE, 3, body, &default_broker()); + let mut d = Decoder::new(resp); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.read_i16().unwrap(), ERROR_INVALID_REQUEST); +} + +#[test] +fn corrupt_fetch_body_returns_invalid_request_error() { + let body = Bytes::from_static(&[0xFF, 0xFF, 0xFF]); + let resp = handle_request(API_KEY_FETCH, 4, body, &default_broker()); + let mut d = Decoder::new(resp); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.read_i16().unwrap(), ERROR_INVALID_REQUEST); +} diff --git a/gateways/kafka/tools/kafka-tool/src/main.rs b/gateways/kafka/tools/kafka-tool/src/main.rs index e1b29f08b7..b92bf729c1 100644 --- a/gateways/kafka/tools/kafka-tool/src/main.rs +++ b/gateways/kafka/tools/kafka-tool/src/main.rs @@ -25,6 +25,8 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpStream; use tracing::{info, warn}; +mod response; + #[derive(Parser)] #[command( name = "kafka-message-gen", @@ -65,13 +67,30 @@ enum Command { version: Option, #[arg(long, default_value = "5000")] timeout_ms: u64, + /// Compact one-line output (default is verbose decoded response) + #[arg(long)] + quiet: bool, }, - /// Send all messages and report pass/fail — exit code 1 if any fail Verify { #[arg(long, default_value = "127.0.0.1:9092")] host: String, + /// Limit to these API keys (repeatable). Defaults to all gateway-scoped keys. + #[arg(long, action = clap::ArgAction::Append)] + api_key: Vec, + /// Limit to a single protocol version + #[arg(long)] + version: Option, + #[arg(long, default_value = "5000")] + timeout_ms: u64, + /// Stop on the first failure #[arg(long)] fail_fast: bool, + /// Use the full Kafka 4.1 registry (for real brokers), not the Iggy gateway scope + #[arg(long)] + all_apis: bool, + /// Compact one-line output (default is verbose decoded response) + #[arg(long)] + quiet: bool, }, } @@ -145,6 +164,16 @@ const API_REGISTRY: &[(i16, &str, i16, i16)] = &[ (76, "ListClientMetricsResources", 0, 0), ]; +/// Iggy Kafka gateway #3421 scope — mirrors `SUPPORTED_RANGES` in `iggy_gateway_kafka`. +const GATEWAY_REGISTRY: &[(i16, &str, i16, i16)] = &[ + (0, "Produce", 3, 9), + (1, "Fetch", 4, 12), + (2, "ListOffsets", 1, 6), + (3, "Metadata", 0, 9), + (18, "ApiVersions", 0, 3), + (19, "CreateTopics", 2, 5), +]; + // ── Flexible version table ──────────────────────────────────────────────────── // Source: flexibleVersions field in each Kafka JSON schema. // Returns the first version using compact encoding, or None if never flexible. @@ -224,10 +253,34 @@ fn first_flexible_version(api_key: i16) -> Option { // [api_key: i16] // [api_version: i16] // [correlation_id: i32] -// [client_id_len: i16] -1 = null -// [client_id: bytes] -// [tagged_fields: u8(0)] only present for flexible versions +// header v1: [client_id: NULLABLE_STRING] +// header v2: [client_id: COMPACT_NULLABLE_STRING] [request_header_tagged_fields] // [payload: bytes] + +fn write_unsigned_varint(buf: &mut BytesMut, mut value: u64) { + loop { + let mut byte = (value & 0x7F) as u8; + value >>= 7; + if value != 0 { + byte |= 0x80; + } + buf.put_u8(byte); + if value == 0 { + break; + } + } +} + +fn write_compact_nullable_string(buf: &mut BytesMut, value: Option<&str>) { + match value { + None => write_unsigned_varint(buf, 0), + Some(s) => { + write_unsigned_varint(buf, (s.len() + 1) as u64); + buf.put_slice(s.as_bytes()); + } + } +} + fn frame_request( api_key: i16, api_version: i16, @@ -236,19 +289,22 @@ fn frame_request( payload: &[u8], flexible: bool, ) -> Bytes { - let cid = client_id.as_bytes(); - let hlen = 2 + 2 + 4 + 2 + cid.len() + if flexible { 1 } else { 0 }; - let blen = hlen + payload.len(); - let mut buf = BytesMut::with_capacity(4 + blen); - buf.put_i32(blen as i32); - buf.put_i16(api_key); - buf.put_i16(api_version); - buf.put_i32(correlation_id); - buf.put_i16(cid.len() as i16); - buf.put_slice(cid); + let mut header = BytesMut::new(); + header.put_i16(api_key); + header.put_i16(api_version); + header.put_i32(correlation_id); if flexible { - buf.put_u8(0x00); + write_compact_nullable_string(&mut header, Some(client_id)); + header.put_u8(0); // empty request-header tagged fields + } else { + header.put_i16(i16::try_from(client_id.len()).expect("client_id fits i16")); + header.put_slice(client_id.as_bytes()); } + + let blen = header.len() + payload.len(); + let mut buf = BytesMut::with_capacity(4 + blen); + buf.put_i32(i32::try_from(blen).expect("frame fits i32")); + buf.put_slice(&header); buf.put_slice(payload); buf.freeze() } @@ -669,23 +725,53 @@ async fn cmd_generate( Ok(()) } +async fn connect(host: &str) -> Result { + TcpStream::connect(host) + .await + .with_context(|| format!("Cannot connect to {host}")) +} + +async fn read_kafka_response(stream: &mut TcpStream) -> std::io::Result> { + let mut lb = [0u8; 4]; + stream.read_exact(&mut lb).await?; + let frame_len = i32::from_be_bytes(lb); + if frame_len <= 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("invalid response frame length: {frame_len}"), + )); + } + let mut body = vec![ + 0u8; + usize::try_from(frame_len).map_err(|_| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + "response frame length does not fit usize", + ) + })? + ]; + stream.read_exact(&mut body).await?; + Ok(body) +} + async fn run_send( host: &str, - fk: Option, + registry: &[(i16, &str, i16, i16)], + filter_keys: &[i16], fv: Option, toms: u64, + fail_fast: bool, + quiet: bool, ) -> Result<(usize, usize)> { - let mut stream = TcpStream::connect(host) - .await - .with_context(|| format!("Cannot connect to {host}"))?; + let mut stream = connect(host).await?; info!("Connected to {host}"); let (mut ok, mut fail, mut corr) = (0usize, 0usize, 1i32); - for &(ak, name, min, max) in API_REGISTRY { - if fk.is_some_and(|k| k != ak) { + 'outer: for &(ak, name, min, max) in registry { + if !filter_keys.is_empty() && !filter_keys.contains(&ak) { continue; } for v in min..=max { - if fv.is_some_and(|fv| fv != v) { + if fv.is_some_and(|wanted| wanted != v) { continue; } let msg = match build_framed(ak, v, corr) { @@ -693,39 +779,50 @@ async fn run_send( Err(e) => { warn!("Build {} v{}: {e}", name, v); fail += 1; + if fail_fast { + break 'outer; + } continue; } }; - stream - .write_all(&msg) - .await - .with_context(|| format!("Write {} v{}", name, v))?; - let res = tokio::time::timeout(std::time::Duration::from_millis(toms), async { - let mut lb = [0u8; 4]; - stream.read_exact(&mut lb).await?; - let mut body = vec![0u8; i32::from_be_bytes(lb) as usize]; - stream.read_exact(&mut body).await?; - Ok::, std::io::Error>(body) - }) + if let Err(e) = stream.write_all(&msg).await { + println!("✗ {name} v{v} → write error: {e}"); + fail += 1; + stream = connect(host).await?; + if fail_fast { + break 'outer; + } + corr += 1; + continue; + } + + let res = tokio::time::timeout( + std::time::Duration::from_millis(toms), + read_kafka_response(&mut stream), + ) .await; + match res { Ok(Ok(r)) => { - let ec = if r.len() >= 6 { - i16::from_be_bytes(r[4..6].try_into().unwrap()) - } else { - -1 - }; - let sym = if ec <= 0 { "✓" } else { "⚠" }; - println!("{sym} {} v{} → {} bytes ec={ec}", name, v, r.len()); + let summary = response::analyze_response(ak, v, corr, &r); + summary.print(name, v, quiet); ok += 1; } Ok(Err(e)) => { - println!("✗ {} v{} → IO error: {e}", name, v); + println!("✗ {name} v{v} → IO error: {e}"); fail += 1; + stream = connect(host).await?; + if fail_fast { + break 'outer; + } } Err(_) => { - println!("✗ {} v{} → timeout ({}ms)", name, v, toms); + println!("✗ {name} v{v} → timeout ({toms}ms)"); fail += 1; + stream = connect(host).await?; + if fail_fast { + break 'outer; + } } } corr += 1; @@ -754,12 +851,39 @@ async fn main() -> Result<()> { api_key, version, timeout_ms, + quiet, } => { - let (ok, fail) = run_send(&host, api_key, version, timeout_ms).await?; + let filter_keys: Vec = api_key.into_iter().collect(); + let (ok, fail) = run_send( + &host, + API_REGISTRY, + &filter_keys, + version, + timeout_ms, + false, + quiet, + ) + .await?; println!("\nResult: {ok} OK {fail} failed"); } - Command::Verify { host, .. } => { - let (ok, fail) = run_send(&host, None, None, 5000).await?; + Command::Verify { + host, + api_key, + version, + timeout_ms, + fail_fast, + all_apis, + quiet, + } => { + let registry = if all_apis { + API_REGISTRY + } else { + GATEWAY_REGISTRY + }; + let (ok, fail) = run_send( + &host, registry, &api_key, version, timeout_ms, fail_fast, quiet, + ) + .await?; println!("\n=== Verify: {ok} passed {fail} failed ==="); if fail > 0 { std::process::exit(1); diff --git a/gateways/kafka/tools/kafka-tool/src/response.rs b/gateways/kafka/tools/kafka-tool/src/response.rs new file mode 100644 index 0000000000..a55f5b62a8 --- /dev/null +++ b/gateways/kafka/tools/kafka-tool/src/response.rs @@ -0,0 +1,420 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Kafka response frame parsing and human-readable summaries for `send` / `verify`. + +use bytes::Bytes; +use kafka_protocol::messages::{ + ApiVersionsResponse, CreateTopicsResponse, FetchResponse, ListOffsetsResponse, + MetadataResponse, ProduceResponse, +}; +use kafka_protocol::protocol::Decodable; + +/// Parsed view of one length-prefixed Kafka response payload (excluding the 4-byte frame length). +pub struct ResponseSummary { + pub frame_bytes: usize, + pub correlation_id: i32, + pub response_header_version: i16, + pub correlation_match: bool, + /// Highest-severity non-zero error found, or `0` when all decoded codes are zero. + pub primary_error_code: i16, + pub details: Vec, + pub decode_note: Option, +} + +impl ResponseSummary { + #[must_use] + pub fn has_nonzero_error(&self) -> bool { + self.primary_error_code != 0 + } + + pub fn print(&self, api_name: &str, version: i16, quiet: bool) { + let sym = if self.has_nonzero_error() { + "⚠" + } else { + "✓" + }; + let ec_label = format_error_code(self.primary_error_code); + let corr = if self.correlation_match { + format!("{}", self.correlation_id) + } else { + format!("{} (expected correlation mismatch)", self.correlation_id) + }; + + if quiet { + println!( + "{sym} {api_name} v{version} → {}B ec={} ({ec_label})", + self.frame_bytes, self.primary_error_code + ); + return; + } + + println!( + "{sym} {api_name} v{version} frame={}B correlation={corr} resp_hdr=v{} primary_ec={} ({ec_label})", + self.frame_bytes, self.response_header_version, self.primary_error_code + ); + for line in &self.details { + println!(" {line}"); + } + if let Some(note) = &self.decode_note { + println!(" note: {note}"); + } + } +} + +/// Analyze a response payload for the given request `(api_key, api_version)`. +pub fn analyze_response( + api_key: i16, + api_version: i16, + request_correlation_id: i32, + payload: &[u8], +) -> ResponseSummary { + let frame_bytes = payload.len(); + if payload.len() < 4 { + return ResponseSummary { + frame_bytes, + correlation_id: 0, + response_header_version: 0, + correlation_match: false, + primary_error_code: -1, + details: vec!["payload shorter than correlation_id".into()], + decode_note: Some("truncated response".into()), + }; + } + + let correlation_id = i32::from_be_bytes(payload[0..4].try_into().expect("4 bytes")); + let resp_hdr_ver = response_header_version(api_key, api_version); + let body_start = if resp_hdr_ver >= 1 { + 5 // correlation_id + empty tagged fields (0x00) + } else { + 4 + }; + + if payload.len() < body_start { + return ResponseSummary { + frame_bytes, + correlation_id, + response_header_version: resp_hdr_ver, + correlation_match: correlation_id == request_correlation_id, + primary_error_code: -1, + details: vec![format!( + "truncated after correlation (need {body_start} bytes)" + )], + decode_note: None, + }; + } + + let body = &payload[body_start..]; + let mut details = Vec::new(); + let mut codes = Vec::new(); + let mut decode_note = None; + + if body.len() == 2 { + let ec = i16::from_be_bytes(body.try_into().expect("2 bytes")); + codes.push(ec); + details.push(format!( + "error-only body: error_code={ec} ({})", + format_error_code(ec) + )); + } else { + match decode_body(api_key, api_version, body, &mut details, &mut codes) { + Ok(()) => {} + Err(e) => { + decode_note = Some(format!("schema decode failed: {e:#}")); + details.push(format!("raw_body_hex={}", hex::encode(body))); + } + } + } + + let primary_error_code = codes.iter().copied().filter(|&c| c != 0).max().unwrap_or(0); + + ResponseSummary { + frame_bytes, + correlation_id, + response_header_version: resp_hdr_ver, + correlation_match: correlation_id == request_correlation_id, + primary_error_code, + details, + decode_note, + } +} + +fn optional_topic_name(name: &Option) -> String { + name.as_ref() + .map(|n| n.0.as_str().to_string()) + .unwrap_or_else(|| "".into()) +} + +fn topic_name(name: &kafka_protocol::messages::TopicName) -> String { + name.0.as_str().to_string() +} + +fn decode_body( + api_key: i16, + api_version: i16, + body: &[u8], + details: &mut Vec, + codes: &mut Vec, +) -> anyhow::Result<()> { + let mut buf = Bytes::copy_from_slice(body); + match api_key { + 18 => { + let resp = ApiVersionsResponse::decode(&mut buf, api_version)?; + codes.push(resp.error_code); + details.push(format!( + "top_level.error_code={} ({})", + resp.error_code, + format_error_code(resp.error_code) + )); + details.push(format!("api_keys={}", resp.api_keys.len())); + if api_version >= 1 { + details.push(format!("throttle_time_ms={}", resp.throttle_time_ms)); + } + for (i, k) in resp.api_keys.iter().enumerate().take(8) { + details.push(format!( + "api_keys[{i}]: key={} min={} max={}", + k.api_key, k.min_version, k.max_version + )); + } + if resp.api_keys.len() > 8 { + details.push(format!("… {} more api_keys", resp.api_keys.len() - 8)); + } + } + 3 => { + let resp = MetadataResponse::decode(&mut buf, api_version)?; + if api_version >= 3 { + details.push(format!("throttle_time_ms={}", resp.throttle_time_ms)); + } + details.push(format!("brokers={}", resp.brokers.len())); + if let Some(b) = resp.brokers.first() { + details.push(format!( + "brokers[0]: id={} host={} port={}", + b.node_id.0, b.host, b.port + )); + } + details.push(format!("topics={}", resp.topics.len())); + for (i, t) in resp.topics.iter().enumerate().take(4) { + codes.push(t.error_code); + let name = optional_topic_name(&t.name); + details.push(format!( + "topics[{i}]: name={name} ec={} ({}) partitions={}", + t.error_code, + format_error_code(t.error_code), + t.partitions.len() + )); + } + if resp.topics.len() > 4 { + details.push(format!("… {} more topics", resp.topics.len() - 4)); + } + } + 0 => { + let resp = ProduceResponse::decode(&mut buf, api_version)?; + if api_version >= 1 { + details.push(format!("throttle_time_ms={}", resp.throttle_time_ms)); + } + details.push(format!("topics={}", resp.responses.len())); + for (ti, topic) in resp.responses.iter().enumerate().take(4) { + let name = topic_name(&topic.name); + details.push(format!( + "topics[{ti}]: name={name} partitions={}", + topic.partition_responses.len() + )); + for (pi, p) in topic.partition_responses.iter().enumerate().take(4) { + codes.push(p.error_code); + details.push(format!( + " partitions[{pi}]: index={} ec={} ({}) offset={}", + p.index, + p.error_code, + format_error_code(p.error_code), + p.base_offset + )); + } + } + } + 1 => { + let resp = FetchResponse::decode(&mut buf, api_version)?; + if api_version >= 1 { + details.push(format!("throttle_time_ms={}", resp.throttle_time_ms)); + } + if api_version >= 7 { + codes.push(resp.error_code); + details.push(format!( + "top_level.error_code={} ({}) session_id={}", + resp.error_code, + format_error_code(resp.error_code), + resp.session_id + )); + } + details.push(format!("topics={}", resp.responses.len())); + for (ti, topic) in resp.responses.iter().enumerate().take(4) { + let name = topic_name(&topic.topic); + details.push(format!( + "topics[{ti}]: name={name} partitions={}", + topic.partitions.len() + )); + for (pi, p) in topic.partitions.iter().enumerate().take(4) { + codes.push(p.error_code); + details.push(format!( + " partitions[{pi}]: index={} ec={} ({}) hw={}", + p.partition_index, + p.error_code, + format_error_code(p.error_code), + p.high_watermark + )); + } + } + } + 2 => { + let resp = ListOffsetsResponse::decode(&mut buf, api_version)?; + if api_version >= 2 { + details.push(format!("throttle_time_ms={}", resp.throttle_time_ms)); + } + details.push(format!("topics={}", resp.topics.len())); + for (ti, topic) in resp.topics.iter().enumerate().take(4) { + let name = topic_name(&topic.name); + details.push(format!( + "topics[{ti}]: name={name} partitions={}", + topic.partitions.len() + )); + for (pi, p) in topic.partitions.iter().enumerate().take(4) { + codes.push(p.error_code); + details.push(format!( + " partitions[{pi}]: index={} ec={} ({}) offset={}", + p.partition_index, + p.error_code, + format_error_code(p.error_code), + p.offset + )); + } + } + } + 19 => { + let resp = CreateTopicsResponse::decode(&mut buf, api_version)?; + details.push(format!("throttle_time_ms={}", resp.throttle_time_ms)); + details.push(format!("topics={}", resp.topics.len())); + for (i, t) in resp.topics.iter().enumerate().take(4) { + codes.push(t.error_code); + let name = topic_name(&t.name); + details.push(format!( + "topics[{i}]: name={name} ec={} ({})", + t.error_code, + format_error_code(t.error_code) + )); + } + } + other => { + details.push(format!("no schema decoder for api_key={other}")); + if body.len() >= 2 { + let ec = i16::from_be_bytes(body[0..2].try_into().expect("2 bytes")); + codes.push(ec); + details.push(format!( + "body[0..2] as i16={ec} ({}) — may not be top-level error_code", + format_error_code(ec) + )); + } + } + } + Ok(()) +} + +fn format_error_code(code: i16) -> &'static str { + match code { + 0 => "NONE", + 1 => "OFFSET_OUT_OF_RANGE", + 2 => "CORRUPT_MESSAGE", + 3 => "UNKNOWN_TOPIC_OR_PARTITION", + 35 => "UNSUPPORTED_VERSION", + 36 => "TOPIC_ALREADY_EXISTS", + 37 => "INVALID_PARTITIONS", + 42 => "INVALID_REQUEST", + -1 => "UNKNOWN", + _ => "OTHER", + } +} + +fn request_header_version(api_key: i16, api_version: i16) -> i16 { + let flex_from = first_flexible_version(api_key); + match flex_from { + Some(fv) if api_version >= fv => 2, + _ => 1, + } +} + +fn response_header_version(api_key: i16, api_version: i16) -> i16 { + if api_key == 18 { + return 0; + } + if request_header_version(api_key, api_version) >= 2 { + 1 + } else { + 0 + } +} + +/// First flexible protocol version per API key (matches gateway `header.rs` / kafka-tool framing). +fn first_flexible_version(api_key: i16) -> Option { + match api_key { + 0 => Some(9), + 1 => Some(12), + 2 => Some(6), + 3 => Some(9), + 8 => Some(8), + 9 => Some(6), + 10 => Some(3), + 11 => Some(6), + 12 => Some(4), + 13 => Some(4), + 14 => Some(4), + 15 => Some(5), + 16 => Some(3), + 17 => None, + 18 => Some(3), + 19 => Some(5), + 20 => Some(4), + 21 => Some(2), + 22 => Some(2), + 23 => Some(4), + 24 => Some(3), + 25 => Some(3), + 26 => Some(3), + 27 => Some(1), + 28 => Some(3), + 29 => Some(2), + 30 => Some(2), + 31 => Some(2), + 32 => Some(4), + 33 => Some(2), + 34 => Some(2), + 35 => Some(2), + 36 => Some(2), + 37 => Some(2), + 38 => Some(2), + 39 => Some(2), + 40 => Some(2), + 41 => Some(2), + 42 => Some(2), + 43 => Some(2), + 44 => Some(1), + 45 | 46 => Some(0), + 47 => None, + 48 | 49 => Some(1), + 50 | 51 | 55 | 56 => Some(0), + 57 => Some(1), + 60 | 61 | 64 | 65 | 66 | 67 | 68 | 69 | 71 | 72 | 74 | 75 | 76 => Some(0), + _ => None, + } +} From bc9210205c4262845db0983f77a4b5454a2a6c49 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Fri, 12 Jun 2026 23:43:49 -0400 Subject: [PATCH 08/57] Adjust clippy lints, usize casts, and freeze() Add/relax several clippy allow attributes in the Kafka codec module; simplify i32->usize conversion for array length by using a direct cast with a safety comment (avoids unnecessary try_from and map_err), and add #[must_use] to Encoder::freeze to prevent accidental discards. Also add a TODO comment in the Produce response placeholder for populating the topic name. --- gateways/kafka/src/protocol/codec.rs | 14 ++++++++------ gateways/kafka/src/protocol/responses.rs | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/gateways/kafka/src/protocol/codec.rs b/gateways/kafka/src/protocol/codec.rs index b46cb6476a..5f001ac209 100644 --- a/gateways/kafka/src/protocol/codec.rs +++ b/gateways/kafka/src/protocol/codec.rs @@ -17,7 +17,9 @@ //! Low-level Kafka primitive encoders/decoders (ported wire codec). -#![allow(clippy::pedantic, clippy::missing_const_for_fn)] +#![allow(clippy::missing_const_for_fn, clippy::bool_to_int_with_if)] +#![allow(clippy::missing_errors_doc, clippy::cast_sign_loss, clippy::must_use_candidate,clippy::missing_panics_doc, clippy::cast_possible_truncation,clippy::cast_lossless)] + use bytes::{Buf, BufMut, Bytes, BytesMut}; @@ -97,10 +99,9 @@ impl Decoder { if n < 0 { return Err(KafkaProtocolError::InvalidArrayLength(n)); } - let count = usize::try_from(n).map_err(|_| KafkaProtocolError::CollectionTooLarge { - count: n as usize, - max: MAX_COLLECTION_LEN, - })?; + // Safe: n is in [0, i32::MAX]; i32::MAX (2_147_483_647) fits in usize + // on all 32-bit and 64-bit platforms this crate targets. + let count = n as usize; if count > MAX_COLLECTION_LEN { return Err(KafkaProtocolError::CollectionTooLarge { count, @@ -209,6 +210,7 @@ impl Decoder { }); } let count = count as usize; + for _ in 0..count { self.read_varint()?; // tag number let size = usize::try_from(self.read_varint()?).map_err(|_| { @@ -343,7 +345,7 @@ impl Encoder { pub fn write_empty_tagged_fields(&mut self) { self.write_varint(0); } - + #[must_use] pub fn freeze(self) -> Bytes { self.bytes.freeze() } diff --git a/gateways/kafka/src/protocol/responses.rs b/gateways/kafka/src/protocol/responses.rs index 8de6868ffd..6a84be1402 100644 --- a/gateways/kafka/src/protocol/responses.rs +++ b/gateways/kafka/src/protocol/responses.rs @@ -30,7 +30,7 @@ use bytes::Bytes; /// Well-formed Produce response with a single placeholder topic/partition. pub fn encode_produce_error_response(version: i16, error_code: i16) -> Bytes { let topics = vec![ProduceTopicData { - topic: String::new(), + topic: String::new(), // TODO topic name will be populated in the end to end functional completion partitions: vec![ProducePartitionData { partition: 0, records: None, From 6b6f1714c5b77dd293048dd6c8b433f73710f144 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sat, 20 Jun 2026 15:27:26 -0400 Subject: [PATCH 09/57] kafka gateway: rename crate and refactor server Rename the crate/binary to iggy-gateway-kafka and update README and Cargo.toml accordingly. Introduce DEFAULT_KAFKA_PORT and refactor the server to accept a pre-bound TcpListener (eliminates bind TOCTOU) and use listener.local_addr() to compute advertised broker information, returning clearer config errors. Harden protocol handling: add Encoder helpers (unchecked nullable strings, write_null_bytes), handle metadata parsing errors safely, downgrade some decode errors to warnings, and adjust metadata encoding for flexible versions. Improve read_frame (single deadline for length+body, chunked reservation to avoid large upfront allocs, truncate extra bytes, and transient accept backoff). Update tests and CI: add rust-gateway component and include gateways in PR-title workflow. --- .github/config/components.yml | 18 ++++ .github/workflows/pr-title.yml | 1 + Cargo.lock | 24 +++--- gateways/kafka/Cargo.toml | 6 +- gateways/kafka/README.md | 6 +- gateways/kafka/src/main.rs | 6 +- gateways/kafka/src/protocol/api.rs | 69 ++++++--------- gateways/kafka/src/protocol/codec.rs | 28 ++++++- gateways/kafka/src/protocol/responses.rs | 15 ++-- gateways/kafka/src/server.rs | 83 ++++++++++++------- gateways/kafka/tests/api_handler_tests.rs | 41 ++++++--- .../kafka/tests/broker_advertise_tests.rs | 44 ++++------ gateways/kafka/tests/common/server.rs | 4 +- .../kafka/tests/version_firewall_tests.rs | 26 ++++-- 14 files changed, 217 insertions(+), 154 deletions(-) diff --git a/.github/config/components.yml b/.github/config/components.yml index 932b4192c4..b7d65f9756 100644 --- a/.github/config/components.yml +++ b/.github/config/components.yml @@ -162,6 +162,7 @@ components: - "rust-connectors" - "rust-mcp" - "rust-integration" + - "rust-gateway" - "ci-infrastructure" paths: - "Dockerfile*" @@ -500,3 +501,20 @@ components: - ".github/actions/**/*.yml" - ".github/ci/**/*.yml" tasks: ["validate"] # Could run workflow validation + + # gateways are not Rust components, but we want to run them in CI + rust-gateway: + depends_on: + - "rust-sdk" + - "rust-workspace" + - "ci-infrastructure" + paths: + - "gateways/**" + tasks: + - "check" + - "fmt" + - "clippy" + - "sort" + - "test-1" + - "test-2" + - "machete" \ No newline at end of file diff --git a/.github/workflows/pr-title.yml b/.github/workflows/pr-title.yml index 57a6bbc2e4..f485734caa 100644 --- a/.github/workflows/pr-title.yml +++ b/.github/workflows/pr-title.yml @@ -95,3 +95,4 @@ jobs: storage simulator configs + gateways diff --git a/Cargo.lock b/Cargo.lock index 84128beb61..d2066767b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6752,6 +6752,18 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "iggy-gateway-kafka" +version = "0.1.0" +dependencies = [ + "bytes", + "thiserror 2.0.18", + "tokio", + "tokio-util", + "tracing", + "tracing-subscriber", +] + [[package]] name = "iggy-mcp" version = "0.4.1-edge.1" @@ -7173,18 +7185,6 @@ dependencies = [ "tracing-subscriber", ] -[[package]] -name = "iggy_gateway_kafka" -version = "0.1.0" -dependencies = [ - "bytes", - "thiserror 2.0.18", - "tokio", - "tokio-util", - "tracing", - "tracing-subscriber", -] - [[package]] name = "ignore" version = "0.4.26" diff --git a/gateways/kafka/Cargo.toml b/gateways/kafka/Cargo.toml index 93a335d68a..88c0e8a2dc 100644 --- a/gateways/kafka/Cargo.toml +++ b/gateways/kafka/Cargo.toml @@ -16,7 +16,7 @@ # under the License. [package] -name = "iggy_gateway_kafka" +name = "iggy-gateway-kafka" version = "0.1.0" description = "Kafka wire protocol gateway foundation for Apache Iggy" edition = "2024" @@ -29,7 +29,7 @@ readme = "README.md" publish = false [[bin]] -name = "iggy-kafka-gateway" +name = "iggy-gateway-kafka" path = "src/main.rs" [dependencies] @@ -45,6 +45,6 @@ tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "io- [lints.clippy] enum_glob_use = "deny" -# Ported Kafka wire codec; pedantic cleanup tracked for a follow-up PR. +#Ported Kafka wire codec; pedantic cleanup tracked for a follow-up PR. pedantic = "warn" nursery = "warn" diff --git a/gateways/kafka/README.md b/gateways/kafka/README.md index 09495b3c3b..7994cf0178 100644 --- a/gateways/kafka/README.md +++ b/gateways/kafka/README.md @@ -1,11 +1,11 @@ -# Kafka gateway (`iggy_gateway_kafka`) +# Kafka gateway (`iggy-gateway-kafka`) Foundation layer for [apache/iggy#3421](https://github.com/apache/iggy/issues/3421): a TCP listener on the Kafka wire port that decodes requests, validates scoped API keys and versions, and returns stub responses. ## Run ```bash -cargo run -p iggy_gateway_kafka --bin iggy-kafka-gateway +cargo run -p iggy-gateway-kafka ``` Default bind: `127.0.0.1:9093`. Override with `KAFKA_BIND_ADDR` (e.g. `0.0.0.0:9093`). @@ -13,7 +13,7 @@ Default bind: `127.0.0.1:9093`. Override with `KAFKA_BIND_ADDR` (e.g. `0.0.0.0:9 ## Test ```bash -cargo test -p iggy_gateway_kafka +cargo test -p iggy-gateway-kafka ``` 103 regression tests across 12 suites — see [docs/TEST_SUITE.md](docs/TEST_SUITE.md) for the full catalog. diff --git a/gateways/kafka/src/main.rs b/gateways/kafka/src/main.rs index 9cd81a4de2..0f48ea943e 100644 --- a/gateways/kafka/src/main.rs +++ b/gateways/kafka/src/main.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use tokio::net::TcpListener; use tokio::signal; use tokio::sync::broadcast; @@ -39,10 +40,13 @@ async fn main() -> Result<(), Box> { .map_err(|e| format!("invalid KAFKA_ADVERTISED_PORT `{advertised_port}`: {e}"))?, ); } + let listener = TcpListener::bind(&config.bind_addr) + .await + .map_err(|e| format!("failed to bind {}: {e}", config.bind_addr))?; let server = KafkaServer::new(config); let (tx, rx) = broadcast::channel(1); - let mut server_task = tokio::spawn(async move { server.run(rx).await }); + let mut server_task = tokio::spawn(async move { server.run(listener, rx).await }); tokio::select! { result = &mut server_task => { diff --git a/gateways/kafka/src/protocol/api.rs b/gateways/kafka/src/protocol/api.rs index 55cf238f95..b49419eb79 100644 --- a/gateways/kafka/src/protocol/api.rs +++ b/gateways/kafka/src/protocol/api.rs @@ -15,10 +15,9 @@ // specific language governing permissions and limitations // under the License. -use std::net::SocketAddr; - use bytes::Bytes; +use crate::error::{KafkaProtocolError, Result}; use crate::protocol::codec::{Decoder, Encoder}; use crate::protocol::requests::{ decode_create_topics_request, decode_fetch_request, decode_list_offsets_request, @@ -37,21 +36,13 @@ pub const API_KEY_METADATA: i16 = 3; pub const API_KEY_API_VERSIONS: i16 = 18; pub const API_KEY_CREATE_TOPICS: i16 = 19; +pub const DEFAULT_KAFKA_PORT: u16 = 9093; + pub const ERROR_NONE: i16 = 0; -pub const ERROR_OFFSET_OUT_OF_RANGE: i16 = 1; -pub const ERROR_CORRUPT_MESSAGE: i16 = 2; pub const ERROR_UNKNOWN_TOPIC_OR_PARTITION: i16 = 3; -pub const ERROR_INVALID_FETCH_SIZE: i16 = 4; -pub const ERROR_LEADER_NOT_AVAILABLE: i16 = 5; -pub const ERROR_NOT_LEADER_OR_FOLLOWER: i16 = 6; -pub const ERROR_REQUEST_TIMED_OUT: i16 = 7; -pub const ERROR_UNKNOWN_SERVER_ERROR: i16 = -1; pub const ERROR_UNSUPPORTED_VERSION: i16 = 35; -pub const ERROR_TOPIC_ALREADY_EXISTS: i16 = 36; pub const ERROR_INVALID_PARTITIONS: i16 = 37; -pub const ERROR_INVALID_REPLICATION_FACTOR: i16 = 38; pub const ERROR_INVALID_REQUEST: i16 = 42; -pub const ERROR_UNSUPPORTED_FOR_MESSAGE_FORMAT: i16 = 43; /// Sentinel for `topic_authorized_operations` / `cluster_authorized_operations` when ACLs are not supported. const AUTHORIZED_OPS_UNKNOWN: i32 = i32::MIN; @@ -62,27 +53,11 @@ pub struct BrokerAdvertise { pub port: i32, } -impl BrokerAdvertise { - #[must_use] - pub fn from_bind_addr(bind_addr: &str) -> Self { - bind_addr.parse::().map_or_else( - |_| Self { - host: "127.0.0.1".to_string(), - port: 9093, - }, - |addr| Self { - host: addr.ip().to_string(), - port: i32::from(addr.port()), - }, - ) - } -} - impl Default for BrokerAdvertise { fn default() -> Self { Self { host: "127.0.0.1".to_string(), - port: 9093, + port: i32::from(DEFAULT_KAFKA_PORT), } } } @@ -151,7 +126,7 @@ pub fn handle_request( if is_supported_version(api_key, api_version) { encode_metadata_response(api_version, body, broker, ERROR_NONE) } else { - encode_metadata_response(0, body, broker, ERROR_UNSUPPORTED_VERSION) + encode_metadata_response(api_version, body, broker, ERROR_UNSUPPORTED_VERSION) } } API_KEY_PRODUCE => { @@ -159,7 +134,7 @@ pub fn handle_request( match decode_produce_request(api_version, body) { Ok(req) => encode_produce_response(api_version, &req), Err(e) => { - tracing::error!("Failed to decode Produce request: {:?}", e); + tracing::warn!("Failed to decode Produce request: {:?}", e); encode_produce_error_response(api_version, ERROR_INVALID_REQUEST) } } @@ -172,7 +147,7 @@ pub fn handle_request( match decode_fetch_request(api_version, body) { Ok(req) => encode_fetch_response(api_version, &req), Err(e) => { - tracing::error!("Failed to decode Fetch request: {:?}", e); + tracing::warn!("Failed to decode Fetch request: {:?}", e); encode_fetch_error_response(api_version, ERROR_INVALID_REQUEST) } } @@ -185,7 +160,7 @@ pub fn handle_request( match decode_list_offsets_request(api_version, body) { Ok(req) => encode_list_offsets_response(api_version, &req), Err(e) => { - tracing::error!("Failed to decode ListOffsets request: {:?}", e); + tracing::warn!("Failed to decode ListOffsets request: {:?}", e); encode_list_offsets_error_response(api_version, ERROR_INVALID_REQUEST) } } @@ -198,7 +173,7 @@ pub fn handle_request( match decode_create_topics_request(api_version, body) { Ok(req) => encode_create_topics_response(api_version, &req), Err(e) => { - tracing::error!("Failed to decode CreateTopics request: {:?}", e); + tracing::warn!("Failed to decode CreateTopics request: {:?}", e); encode_create_topics_error_response(api_version, ERROR_INVALID_REQUEST) } } @@ -273,7 +248,12 @@ fn encode_metadata_response( top_level_error_code: i16, ) -> Bytes { let flexible = api_version >= 9; - let topics_count = split_metadata_request_topics(body, api_version); + // BufferUnderflow (empty body) → treat as 0 topics; other decode errors are truly invalid. + let topics_count = match split_metadata_request_topics(body, api_version) { + Ok(n) => n, + Err(KafkaProtocolError::BufferUnderflow { .. }) => 0, + Err(_) => return encode_error_only_response(ERROR_INVALID_REQUEST), + }; let topic_error = if top_level_error_code == ERROR_NONE { ERROR_UNKNOWN_TOPIC_OR_PARTITION } else { @@ -311,14 +291,18 @@ fn encode_metadata_response( } else { e.write_i32(1); // brokers array length e.write_i32(1); // node_id - let _ = e.write_nullable_string(Some(&broker.host)); + // broker.host is config-derived (KAFKA_ADVERTISED_HOST), not request-decoded — use + // the checked variant so an overly long hostname returns an error instead of panicking. + if e.write_nullable_string(Some(&broker.host)).is_err() { + return encode_error_only_response(ERROR_INVALID_REQUEST); + } e.write_i32(broker.port); if api_version >= 1 { - let _ = e.write_nullable_string(None); // rack + e.write_nullable_string_unchecked(None); // rack } if api_version >= 2 { - let _ = e.write_nullable_string(None); // cluster_id + e.write_nullable_string_unchecked(None); // cluster_id } if api_version >= 1 { e.write_i32(1); // controller_id — must come before topics array @@ -327,7 +311,7 @@ fn encode_metadata_response( e.write_i32(i32::try_from(topics_count).expect("topic count bounded")); for _ in 0..topics_count { e.write_i16(topic_error); - let _ = e.write_nullable_string(Some("unknown-topic")); + e.write_nullable_string_unchecked(Some("unknown-topic")); if api_version >= 1 { e.write_bool(false); // is_internal } @@ -351,12 +335,11 @@ pub fn encode_error_only_response(error_code: i16) -> Bytes { e.freeze() } -#[must_use] -pub(crate) fn split_metadata_request_topics(body: Bytes, api_version: i16) -> usize { +pub(crate) fn split_metadata_request_topics(body: Bytes, api_version: i16) -> Result { let mut d = Decoder::new(body); if api_version >= 9 { - d.read_compact_array_count().unwrap_or(0) + d.read_compact_array_count() } else { - d.read_i32_array_count().unwrap_or(0) + d.read_i32_array_count() } } diff --git a/gateways/kafka/src/protocol/codec.rs b/gateways/kafka/src/protocol/codec.rs index 5f001ac209..214c79af9e 100644 --- a/gateways/kafka/src/protocol/codec.rs +++ b/gateways/kafka/src/protocol/codec.rs @@ -18,8 +18,14 @@ //! Low-level Kafka primitive encoders/decoders (ported wire codec). #![allow(clippy::missing_const_for_fn, clippy::bool_to_int_with_if)] -#![allow(clippy::missing_errors_doc, clippy::cast_sign_loss, clippy::must_use_candidate,clippy::missing_panics_doc, clippy::cast_possible_truncation,clippy::cast_lossless)] - +#![allow( + clippy::missing_errors_doc, + clippy::cast_sign_loss, + clippy::must_use_candidate, + clippy::missing_panics_doc, + clippy::cast_possible_truncation, + clippy::cast_lossless +)] use bytes::{Buf, BufMut, Bytes, BytesMut}; @@ -297,6 +303,19 @@ impl Encoder { Ok(()) } + /// Infallible variant for response-encoding paths where the string originated from a decoded + /// Kafka request and is therefore already bounded to `i16::MAX` bytes. + pub fn write_nullable_string_unchecked(&mut self, v: Option<&str>) { + match v { + None => self.write_i16(-1), + Some(s) => { + debug_assert!(i16::try_from(s.len()).is_ok()); + self.write_i16(i16::try_from(s.len()).expect("caller guarantees len <= i16::MAX")); + self.bytes.put_slice(s.as_bytes()); + } + } + } + /// Compact nullable string (flexible versions): varint(len+1), 0 for null. pub fn write_compact_nullable_string(&mut self, v: Option<&str>) { match v { @@ -308,6 +327,11 @@ impl Encoder { } } + /// Write a null bytes field (i32 -1). Infallible; use instead of `write_nullable_bytes(None)`. + pub fn write_null_bytes(&mut self) { + self.write_i32(-1); + } + /// Legacy nullable bytes: i32 length prefix, -1 for null. pub fn write_nullable_bytes(&mut self, v: Option<&[u8]>) -> Result<()> { match v { diff --git a/gateways/kafka/src/protocol/responses.rs b/gateways/kafka/src/protocol/responses.rs index 6a84be1402..8c44692393 100644 --- a/gateways/kafka/src/protocol/responses.rs +++ b/gateways/kafka/src/protocol/responses.rs @@ -61,7 +61,7 @@ fn encode_produce_response_inner( if flexible { e.write_compact_nullable_string(Some(&topic.topic)); } else { - let _ = e.write_nullable_string(Some(&topic.topic)); + e.write_nullable_string_unchecked(Some(&topic.topic)); } if flexible { @@ -86,7 +86,7 @@ fn encode_produce_response_inner( e.write_compact_nullable_string(None); } else { e.write_i32(0); - let _ = e.write_nullable_string(None); + e.write_nullable_string_unchecked(None); } } if flexible { @@ -160,7 +160,7 @@ fn encode_fetch_response_inner( if flexible { e.write_compact_nullable_string(Some(&topic.topic)); } else { - let _ = e.write_nullable_string(Some(&topic.topic)); + e.write_nullable_string_unchecked(Some(&topic.topic)); } if flexible { @@ -192,8 +192,7 @@ fn encode_fetch_response_inner( if flexible { e.write_compact_nullable_bytes(None); } else { - e.write_nullable_bytes(None) - .expect("null bytes always encode"); + e.write_null_bytes(); } if flexible { e.write_empty_tagged_fields(); @@ -252,7 +251,7 @@ fn encode_list_offsets_response_inner( if flexible { e.write_compact_nullable_string(Some(&topic.topic)); } else { - let _ = e.write_nullable_string(Some(&topic.topic)); + e.write_nullable_string_unchecked(Some(&topic.topic)); } if flexible { @@ -328,7 +327,7 @@ fn encode_create_topics_response_inner( if flexible { e.write_compact_nullable_string(Some(&topic.name)); } else { - let _ = e.write_nullable_string(Some(&topic.name)); + e.write_nullable_string_unchecked(Some(&topic.name)); } let error_code = if topic_error != ERROR_NONE { @@ -344,7 +343,7 @@ fn encode_create_topics_response_inner( if flexible { e.write_compact_nullable_string(None); } else { - let _ = e.write_nullable_string(None); + e.write_nullable_string_unchecked(None); } } diff --git a/gateways/kafka/src/server.rs b/gateways/kafka/src/server.rs index 94ef7cb879..6b313a67e8 100644 --- a/gateways/kafka/src/server.rs +++ b/gateways/kafka/src/server.rs @@ -29,7 +29,8 @@ use tracing::{debug, error, info, warn}; use crate::error::{KafkaProtocolError, Result}; use crate::protocol::api::{ - BrokerAdvertise, ERROR_INVALID_REQUEST, encode_error_only_response, handle_request, + BrokerAdvertise, DEFAULT_KAFKA_PORT, ERROR_INVALID_REQUEST, encode_error_only_response, + handle_request, }; use crate::protocol::codec::Decoder; use crate::protocol::header::{ @@ -37,6 +38,8 @@ use crate::protocol::header::{ }; use std::io; +const READ_CHUNK: usize = 65536; + #[derive(Debug, Clone)] pub struct ServerConfig { pub bind_addr: String, @@ -53,7 +56,7 @@ pub struct ServerConfig { impl Default for ServerConfig { fn default() -> Self { Self { - bind_addr: "127.0.0.1:9093".to_string(), + bind_addr: format!("127.0.0.1:{DEFAULT_KAFKA_PORT}"), advertised_host: None, advertised_port: None, max_frame_size: 8 * 1024 * 1024, @@ -64,36 +67,35 @@ impl Default for ServerConfig { } impl BrokerAdvertise { - /// Resolve the broker endpoint advertised in Metadata from listener config. + /// Resolve the broker endpoint advertised in Metadata. + /// + /// `local_addr` is the address the listener is actually bound to (from `listener.local_addr()`). /// /// # Errors /// - /// Returns an error when `bind_addr` is invalid, `advertised_host` is empty, or the listener - /// binds to a wildcard address without an explicit advertised host. - pub fn from_server_config(config: &ServerConfig) -> std::result::Result { - let bind = config - .bind_addr - .parse::() - .map_err(|e| format!("invalid bind address `{}`: {e}", config.bind_addr))?; - + /// Returns `InvalidConfig` when `advertised_host` is empty or the listener binds to a wildcard + /// without an explicit advertised host. + pub fn from_server_config(config: &ServerConfig, local_addr: SocketAddr) -> Result { let port = config .advertised_port - .map_or_else(|| i32::from(bind.port()), i32::from); + .map_or_else(|| i32::from(local_addr.port()), i32::from); let host = if let Some(ref advertised) = config.advertised_host { let trimmed = advertised.trim(); if trimmed.is_empty() { - return Err("KAFKA_ADVERTISED_HOST must not be empty".into()); + return Err(KafkaProtocolError::InvalidConfig( + "KAFKA_ADVERTISED_HOST must not be empty".into(), + )); } trimmed.to_string() - } else if bind.ip().is_unspecified() { - return Err( + } else if local_addr.ip().is_unspecified() { + return Err(KafkaProtocolError::InvalidConfig( "binding to a wildcard address (0.0.0.0 or ::) requires KAFKA_ADVERTISED_HOST \ to be set to a reachable hostname or IP for Metadata broker advertisement" .into(), - ); + )); } else { - bind.ip().to_string() + local_addr.ip().to_string() }; Ok(Self { host, port }) @@ -114,18 +116,25 @@ impl KafkaServer { /// Accept Kafka wire connections until `shutdown` fires, then drain in-flight tasks. /// + /// `listener` must already be bound by the caller. This lets tests and `main` bind + /// the port before spawning the task, eliminating the TOCTOU race of bind-drop-rebind. + /// /// # Errors /// - /// Returns an error if binding fails or a non-transient `accept()` error occurs. - pub async fn run(self, mut shutdown: broadcast::Receiver<()>) -> Result<()> { - let broker = Arc::new( - BrokerAdvertise::from_server_config(&self.config) - .map_err(KafkaProtocolError::InvalidConfig)?, - ); - let listener = TcpListener::bind(&self.config.bind_addr).await?; + /// Returns an error on invalid config or a non-transient `accept()` error. + pub async fn run( + self, + listener: TcpListener, + mut shutdown: broadcast::Receiver<()>, + ) -> Result<()> { + let local_addr = listener.local_addr()?; + let broker = Arc::new(BrokerAdvertise::from_server_config( + &self.config, + local_addr, + )?); info!( "kafka listener bound on {} (advertised as {}:{})", - self.config.bind_addr, broker.host, broker.port + local_addr, broker.host, broker.port ); let tracker = TaskTracker::new(); @@ -167,6 +176,10 @@ impl KafkaServer { }); } Err(e) if is_transient_accept_error(&e) => { + // Brief backoff on fd exhaustion to avoid busy-spinning. + if matches!(e.raw_os_error(), Some(23 | 24)) { + tokio::time::sleep(Duration::from_millis(10)).await; + } warn!(%e, "transient accept error, continuing"); } Err(e) => return Err(e.into()), @@ -311,8 +324,12 @@ pub async fn read_frame( max_frame_size: usize, read_timeout: Duration, ) -> Result { + // Single deadline for both the length-prefix read and the body read. Without this, a + // slow-drip sender could hold a connection open for 2x read_timeout by sending one byte + // per timeout window. + let deadline = tokio::time::Instant::now() + read_timeout; let mut len_buf = [0u8; 4]; - timeout(read_timeout, stream.read_exact(&mut len_buf)) + timeout_at(deadline, stream.read_exact(&mut len_buf)) .await .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "read timeout"))??; @@ -324,7 +341,6 @@ pub async fn read_frame( let frame_len = usize::try_from(frame_len_i32).map_err(|_| KafkaProtocolError::FrameTooLarge { max_bytes: max_frame_size, - // Positive i32 that does not fit in `usize` (e.g. 16-bit targets). actual_bytes: usize::MAX, })?; if frame_len > max_frame_size { @@ -334,12 +350,11 @@ pub async fn read_frame( }); } - // read_buf fills BytesMut spare capacity without zero-initializing it first. - // Single deadline for the entire body so a slow-drip sender can't stall indefinitely - // by delivering one byte per timeout window. - let deadline = tokio::time::Instant::now() + read_timeout; - let mut data = BytesMut::with_capacity(frame_len); + // Reserve in 64 KB increments so a max-size frame (8 MB by default) does not trigger a + // single large upfront allocation before any payload bytes have arrived. + let mut data = BytesMut::new(); while data.len() < frame_len { + data.reserve((frame_len - data.len()).min(READ_CHUNK)); match timeout_at(deadline, stream.read_buf(&mut data)).await { Err(_) => return Err(io::Error::new(io::ErrorKind::TimedOut, "read timeout").into()), Ok(Ok(0)) => { @@ -351,6 +366,10 @@ pub async fn read_frame( Ok(Ok(_)) => {} } } + // read_buf may have written past frame_len if the OS returned more bytes than we + // reserved (capacity can round up). Truncate so pipelined frames don't bleed into + // decoder.read_bytes(remaining()) at the call site. + data.truncate(frame_len); Ok(data.freeze()) } diff --git a/gateways/kafka/tests/api_handler_tests.rs b/gateways/kafka/tests/api_handler_tests.rs index 2d342cfd8e..ab3e762332 100644 --- a/gateways/kafka/tests/api_handler_tests.rs +++ b/gateways/kafka/tests/api_handler_tests.rs @@ -103,23 +103,38 @@ fn metadata_response_has_broker_array_and_topic_array() { #[test] fn unsupported_version_returns_protocol_error() { - let mut req = Vec::new(); - req.extend_from_slice(&1_i32.to_be_bytes()); - let body = handle_request(API_KEY_METADATA, 99, Bytes::from(req), &test_broker()); + // v99 client sends a compact-array body (count+1 = 2 = 0x02 for 1 topic). + // The gateway caps at v9 (highest supported Metadata version) for both parsing + // and encoding, so the response uses the flexible (v9) wire format. + let body = handle_request( + API_KEY_METADATA, + 99, + Bytes::from_static(&[0x02]), + &test_broker(), + ); let mut d = Decoder::new(body); - // Metadata v0: brokers[], topics[] — no controller_id (added in v1) - let _broker_count = d.read_i32().unwrap(); - let _ = d.read_i32().unwrap(); // node_id - let _ = d.read_nullable_string().unwrap(); // host - let _ = d.read_i32().unwrap(); // port - let topic_count = d.read_i32().unwrap(); + // v9 flexible response layout: + d.read_i32().unwrap(); // throttle_time_ms (v3+) + let broker_count = usize::try_from(d.read_varint().unwrap()) + .unwrap() + .saturating_sub(1); + for _ in 0..broker_count { + d.read_i32().unwrap(); // node_id + d.read_compact_nullable_string().unwrap(); // host + d.read_i32().unwrap(); // port + d.read_compact_nullable_string().unwrap(); // rack + d.read_tagged_fields().unwrap(); + } + d.read_compact_nullable_string().unwrap(); // cluster_id (v2+) + d.read_i32().unwrap(); // controller_id (v1+) + let topic_count = usize::try_from(d.read_varint().unwrap()) + .unwrap() + .saturating_sub(1); assert_eq!(topic_count, 1); let topic_error = d.read_i16().unwrap(); assert_eq!(topic_error, ERROR_UNSUPPORTED_VERSION); - let topic_name = d.read_nullable_string().unwrap().unwrap(); - assert_eq!(topic_name, "unknown-topic"); - let partitions_count = d.read_i32().unwrap(); - assert_eq!(partitions_count, 0); + let topic_name = d.read_compact_nullable_string().unwrap(); + assert_eq!(topic_name, Some("unknown-topic".to_string())); } // ── Misc ──────────────────────────────────────────────────────────────────── diff --git a/gateways/kafka/tests/broker_advertise_tests.rs b/gateways/kafka/tests/broker_advertise_tests.rs index 9d60d418e4..8f482019c4 100644 --- a/gateways/kafka/tests/broker_advertise_tests.rs +++ b/gateways/kafka/tests/broker_advertise_tests.rs @@ -17,31 +17,12 @@ //! `BrokerAdvertise` parsing and metadata reflection. +use std::net::SocketAddr; + use iggy_gateway_kafka::ServerConfig; use iggy_gateway_kafka::protocol::api::{API_KEY_METADATA, BrokerAdvertise, handle_request}; use iggy_gateway_kafka::protocol::codec::{Decoder, Encoder}; -#[test] -fn from_bind_addr_parses_ipv4_and_port() { - let b = BrokerAdvertise::from_bind_addr("192.168.1.10:19092"); - assert_eq!(b.host, "192.168.1.10"); - assert_eq!(b.port, 19092); -} - -#[test] -fn from_bind_addr_parses_ipv6() { - let b = BrokerAdvertise::from_bind_addr("[::1]:9093"); - assert_eq!(b.host, "::1"); - assert_eq!(b.port, 9093); -} - -#[test] -fn from_bind_addr_invalid_falls_back_to_default() { - let b = BrokerAdvertise::from_bind_addr("not-a-socket-addr"); - assert_eq!(b.host, "127.0.0.1"); - assert_eq!(b.port, 9093); -} - #[test] fn default_matches_standard_gateway_port() { let b = BrokerAdvertise::default(); @@ -50,8 +31,11 @@ fn default_matches_standard_gateway_port() { } #[test] -fn metadata_reflects_parsed_bind_addr() { - let broker = BrokerAdvertise::from_bind_addr("203.0.113.7:9093"); +fn metadata_reflects_broker_addr() { + let broker = BrokerAdvertise { + host: "203.0.113.7".to_string(), + port: 9093, + }; let mut req = Encoder::with_capacity(4); req.write_i32(0); let body = handle_request(API_KEY_METADATA, 0, req.freeze(), &broker); @@ -72,7 +56,8 @@ fn from_server_config_uses_explicit_advertised_host_on_wildcard_bind() { advertised_host: Some("kafka.internal".to_string()), ..ServerConfig::default() }; - let broker = BrokerAdvertise::from_server_config(&config).expect("valid config"); + let local_addr: SocketAddr = "0.0.0.0:9093".parse().unwrap(); + let broker = BrokerAdvertise::from_server_config(&config, local_addr).expect("valid config"); assert_eq!(broker.host, "kafka.internal"); assert_eq!(broker.port, 9093); } @@ -83,8 +68,9 @@ fn from_server_config_rejects_wildcard_bind_without_advertised_host() { bind_addr: "0.0.0.0:9093".to_string(), ..ServerConfig::default() }; - let err = BrokerAdvertise::from_server_config(&config).unwrap_err(); - assert!(err.contains("KAFKA_ADVERTISED_HOST")); + let local_addr: SocketAddr = "0.0.0.0:9093".parse().unwrap(); + let err = BrokerAdvertise::from_server_config(&config, local_addr).unwrap_err(); + assert!(err.to_string().contains("KAFKA_ADVERTISED_HOST")); } #[test] @@ -93,7 +79,8 @@ fn from_server_config_uses_bind_ip_for_non_wildcard_listener() { bind_addr: "192.168.1.10:19092".to_string(), ..ServerConfig::default() }; - let broker = BrokerAdvertise::from_server_config(&config).expect("valid config"); + let local_addr: SocketAddr = "192.168.1.10:19092".parse().unwrap(); + let broker = BrokerAdvertise::from_server_config(&config, local_addr).expect("valid config"); assert_eq!(broker.host, "192.168.1.10"); assert_eq!(broker.port, 19092); } @@ -106,7 +93,8 @@ fn from_server_config_honors_advertised_port_override() { advertised_port: Some(19093), ..ServerConfig::default() }; - let broker = BrokerAdvertise::from_server_config(&config).expect("valid config"); + let local_addr: SocketAddr = "127.0.0.1:9093".parse().unwrap(); + let broker = BrokerAdvertise::from_server_config(&config, local_addr).expect("valid config"); assert_eq!(broker.host, "broker.example.com"); assert_eq!(broker.port, 19093); } diff --git a/gateways/kafka/tests/common/server.rs b/gateways/kafka/tests/common/server.rs index 178dbce2e5..a009b46423 100644 --- a/gateways/kafka/tests/common/server.rs +++ b/gateways/kafka/tests/common/server.rs @@ -31,7 +31,6 @@ pub async fn spawn_test_server() -> (SocketAddr, broadcast::Sender<()>) { .await .expect("bind ephemeral port"); let addr = listener.local_addr().expect("local addr"); - drop(listener); let config = ServerConfig { bind_addr: addr.to_string(), @@ -44,9 +43,8 @@ pub async fn spawn_test_server() -> (SocketAddr, broadcast::Sender<()>) { let (shutdown_tx, shutdown_rx) = broadcast::channel(1); let server = KafkaServer::new(config); tokio::spawn(async move { - let _ = server.run(shutdown_rx).await; + let _ = server.run(listener, shutdown_rx).await; }); - tokio::time::sleep(Duration::from_millis(50)).await; (addr, shutdown_tx) } diff --git a/gateways/kafka/tests/version_firewall_tests.rs b/gateways/kafka/tests/version_firewall_tests.rs index de0f96e03b..77c4e768f8 100644 --- a/gateways/kafka/tests/version_firewall_tests.rs +++ b/gateways/kafka/tests/version_firewall_tests.rs @@ -171,18 +171,32 @@ fn metadata_below_min_version_returns_topic_error() { #[test] fn metadata_above_max_version_returns_topic_error() { + // v10 uses flexible encoding; compact array varint(2) = 1 topic. let body = handle_request( API_KEY_METADATA, 10, - metadata_request_one_topic(), + Bytes::from_static(&[0x02]), &default_broker(), ); + // Response is in v9 flexible format (highest supported). let mut d = Decoder::new(body); - let _brokers = d.read_i32().unwrap(); - let _ = d.read_i32().unwrap(); - let _ = d.read_nullable_string().unwrap(); - let _ = d.read_i32().unwrap(); - assert_eq!(d.read_i32().unwrap(), 1); + d.read_i32().unwrap(); // throttle_time_ms (v3+) + let broker_count = usize::try_from(d.read_varint().unwrap()) + .unwrap() + .saturating_sub(1); + for _ in 0..broker_count { + d.read_i32().unwrap(); + d.read_compact_nullable_string().unwrap(); + d.read_i32().unwrap(); + d.read_compact_nullable_string().unwrap(); + d.read_tagged_fields().unwrap(); + } + d.read_compact_nullable_string().unwrap(); // cluster_id + d.read_i32().unwrap(); // controller_id + let topic_count = usize::try_from(d.read_varint().unwrap()) + .unwrap() + .saturating_sub(1); + assert_eq!(topic_count, 1); assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); } From e8cb1d9ee3d3bb6a7c2bf1de65bb5d9bd3a8dc67 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sat, 20 Jun 2026 22:11:32 -0400 Subject: [PATCH 10/57] kafka gateway: metadata, server fixes and docs Multiple fixes and improvements to the Kafka gateway: - Docs: document KAFKA_* env vars in README and normalize package name references from `iggy_gateway_kafka` to `iggy-gateway-kafka` across MANUAL_TESTING and TEST_SUITE. - Protocol: add MAX_SUPPORTED_METADATA_VERSION and clamp metadata responses to the highest implemented version; treat empty or malformed metadata request bodies as a 0-topic response with appropriate per-topic error instead of failing; remove unused KafkaProtocolError import. - Server: validate KAFKA_ADVERTISED_HOST length against Kafka nullable string limit and return a config error if exceeded; log a warn when setting TCP_NODELAY fails. - I/O: rewrite read_frame to use bounded read() slices (avoid allocator over-reads via read_buf) so pipelined frames are not consumed accidentally. - Tests: add tests for advertised-host length, corrupt metadata partial-body behavior returning zero topics, and that read_frame does not consume pipelined bytes; adjust existing tests/docs to match crate name changes. These changes fix metadata decoding semantics, prevent frame bleed between pipelined requests, harden config validation, and align documentation/test commands with the package name. --- gateways/kafka/README.md | 8 ++++- gateways/kafka/docs/MANUAL_TESTING.md | 14 ++++---- gateways/kafka/docs/TEST_SUITE.md | 8 ++--- gateways/kafka/src/protocol/api.rs | 30 ++++++++++++----- gateways/kafka/src/server.rs | 33 ++++++++++++------- .../kafka/tests/broker_advertise_tests.rs | 12 +++++++ .../kafka/tests/metadata_regression_tests.rs | 14 ++++++++ .../kafka/tests/server_integration_tests.rs | 25 ++++++++++++++ 8 files changed, 112 insertions(+), 32 deletions(-) diff --git a/gateways/kafka/README.md b/gateways/kafka/README.md index 7994cf0178..b336887cd0 100644 --- a/gateways/kafka/README.md +++ b/gateways/kafka/README.md @@ -8,7 +8,13 @@ Foundation layer for [apache/iggy#3421](https://github.com/apache/iggy/issues/34 cargo run -p iggy-gateway-kafka ``` -Default bind: `127.0.0.1:9093`. Override with `KAFKA_BIND_ADDR` (e.g. `0.0.0.0:9093`). +Default bind: `127.0.0.1:9093`. Environment variables: + +| Variable | Default | Description | +|---|---|---| +| `KAFKA_BIND_ADDR` | `127.0.0.1:9093` | TCP address to listen on | +| `KAFKA_ADVERTISED_HOST` | bind IP | Hostname/IP clients use to reach this broker (required when binding to `0.0.0.0`/`::`) | +| `KAFKA_ADVERTISED_PORT` | bind port | Port advertised in Metadata responses | ## Test diff --git a/gateways/kafka/docs/MANUAL_TESTING.md b/gateways/kafka/docs/MANUAL_TESTING.md index cd3e8c0e58..30f37783c6 100644 --- a/gateways/kafka/docs/MANUAL_TESTING.md +++ b/gateways/kafka/docs/MANUAL_TESTING.md @@ -22,10 +22,10 @@ See also: [SCOPE.md](SCOPE.md) (supported API keys), [TEST_SUITE.md](TEST_SUITE. ```bash # From iggy workspace root -cargo build -p iggy_gateway_kafka --bin iggy-kafka-gateway +cargo build -p iggy-gateway-kafka # Terminal 1 — start listener (default 127.0.0.1:9093) -RUST_LOG=info cargo run -p iggy_gateway_kafka --bin iggy-kafka-gateway +RUST_LOG=info cargo run -p iggy-gateway-kafka ``` Expected log: @@ -50,7 +50,7 @@ cargo run -p kafka-message-gen -- generate \ Run before manual testing to catch regressions: ```bash -cargo test -p iggy_gateway_kafka +cargo test -p iggy-gateway-kafka ``` All tests must pass. If `decode_validation_tests` fail, regenerate fixtures (step above). @@ -63,7 +63,7 @@ All tests must pass. If `decode_validation_tests` fail, regenerate fixtures (ste | ID | Test | Steps | Expected result | Pass criteria | |----|------|-------|-----------------|---------------| -| A1 | Gateway starts | Run `iggy-kafka-gateway` | Binds to `:9093`, no panic | Log shows bind address | +| A1 | Gateway starts | Run `iggy-gateway-kafka` | Binds to `:9093`, no panic | Log shows bind address | | A2 | ApiVersions v1 | `cargo run -p kafka-message-gen -- send --host 127.0.0.1:9093 --api-key 18 --version 1` | Response received | `ec=0`, non-zero byte count | | A3 | ApiVersions v3 (flexible) | Same with `--version 3` | Response received | `ec=0` | | A4 | Metadata v0 | `send --api-key 3 --version 0` | Stub broker in response | `ec=0` or topic error 3 (stub) | @@ -238,8 +238,8 @@ kcat version (if used): ___________ [ ] H1–H3 Adversarial input Automated regression: -[ ] cargo test -p iggy_gateway_kafka — ___/103 passed -[ ] cargo clippy -p iggy_gateway_kafka — clean / warnings noted +[ ] cargo test -p iggy-gateway-kafka — ___/103 passed +[ ] cargo clippy -p iggy-gateway-kafka — clean / warnings noted Notes / failures: _________________________________ @@ -251,7 +251,7 @@ _________________________________ | Symptom | Likely cause | Fix | |---------|--------------|-----| -| `Connection refused` on 9093 | Gateway not running | Start `iggy-kafka-gateway` | +| `Connection refused` on 9093 | Gateway not running | Start `iggy-gateway-kafka` | | `decode_validation_tests` panic | Missing fixtures | Run `kafka-message-gen generate` | | `ec=35` for in-range version | Version not in `SUPPORTED_RANGES` | Check `SCOPE.md` and `api.rs` | | kcat hangs | Timeout waiting for data | Set `-m 1000`; check gateway logs | diff --git a/gateways/kafka/docs/TEST_SUITE.md b/gateways/kafka/docs/TEST_SUITE.md index c8515387c2..c0f6df8c1e 100644 --- a/gateways/kafka/docs/TEST_SUITE.md +++ b/gateways/kafka/docs/TEST_SUITE.md @@ -3,7 +3,7 @@ Regression tests live under [`tests/`](../tests/). Run from the workspace root: ```bash -cargo test -p iggy_gateway_kafka +cargo test -p iggy-gateway-kafka ``` **Current count:** 103 tests across 12 suites (as of #3421 foundation). @@ -34,7 +34,7 @@ Fixtures are gitignored under `tools/kafka-tool/kafka_messages/`. Tests that nee | [`decode_validation_tests.rs`](../tests/decode_validation_tests.rs) | kafka-tool fixture decode + response structure per version | 14 | **Yes** | | [`version_firewall_tests.rs`](../tests/version_firewall_tests.rs) | Version boundary matrix, unsupported keys, corrupt bodies | 17 | Partial | | [`metadata_regression_tests.rs`](../tests/metadata_regression_tests.rs) | Metadata v0–v9, topic counts, broker advertise | 7 | No | -| [`broker_advertise_tests.rs`](../tests/broker_advertise_tests.rs) | `BrokerAdvertise::from_bind_addr` parsing | 5 | No | +| [`broker_advertise_tests.rs`](../tests/broker_advertise_tests.rs) | `BrokerAdvertise::from_server_config` parsing | 5 | No | | [`handler_regression_tests.rs`](../tests/handler_regression_tests.rs) | Every scoped key×version via `handle_request`, stub error codes | 5 | Partial | | [`server_integration_tests.rs`](../tests/server_integration_tests.rs) | `read_frame` / `write_frame` unit-level I/O | 4 | No | | [`server_e2e_tests.rs`](../tests/server_e2e_tests.rs) | Full `KafkaServer` TCP round-trips | 8 | Partial | @@ -141,10 +141,10 @@ cargo run -p kafka-message-gen -- generate \ --api-key 0 --api-key 1 --api-key 2 --api-key 19 # 2. Run regression suite -cargo test -p iggy_gateway_kafka +cargo test -p iggy-gateway-kafka # 3. Optional lint gate -cargo clippy -p iggy_gateway_kafka -- -D warnings +cargo clippy -p iggy-gateway-kafka -- -D warnings ``` --- diff --git a/gateways/kafka/src/protocol/api.rs b/gateways/kafka/src/protocol/api.rs index b49419eb79..2c1d3b39b2 100644 --- a/gateways/kafka/src/protocol/api.rs +++ b/gateways/kafka/src/protocol/api.rs @@ -17,7 +17,7 @@ use bytes::Bytes; -use crate::error::{KafkaProtocolError, Result}; +use crate::error::Result; use crate::protocol::codec::{Decoder, Encoder}; use crate::protocol::requests::{ decode_create_topics_request, decode_fetch_request, decode_list_offsets_request, @@ -44,6 +44,8 @@ pub const ERROR_UNSUPPORTED_VERSION: i16 = 35; pub const ERROR_INVALID_PARTITIONS: i16 = 37; pub const ERROR_INVALID_REQUEST: i16 = 42; +const MAX_SUPPORTED_METADATA_VERSION: i16 = 9; + /// Sentinel for `topic_authorized_operations` / `cluster_authorized_operations` when ACLs are not supported. const AUTHORIZED_OPS_UNKNOWN: i32 = i32::MIN; @@ -126,7 +128,13 @@ pub fn handle_request( if is_supported_version(api_key, api_version) { encode_metadata_response(api_version, body, broker, ERROR_NONE) } else { - encode_metadata_response(api_version, body, broker, ERROR_UNSUPPORTED_VERSION) + // Encode at the highest version we implement, not the client's unknown version. + encode_metadata_response( + api_version.clamp(0, MAX_SUPPORTED_METADATA_VERSION), + body, + broker, + ERROR_UNSUPPORTED_VERSION, + ) } } API_KEY_PRODUCE => { @@ -248,16 +256,20 @@ fn encode_metadata_response( top_level_error_code: i16, ) -> Bytes { let flexible = api_version >= 9; - // BufferUnderflow (empty body) → treat as 0 topics; other decode errors are truly invalid. - let topics_count = match split_metadata_request_topics(body, api_version) { - Ok(n) => n, - Err(KafkaProtocolError::BufferUnderflow { .. }) => 0, - Err(_) => return encode_error_only_response(ERROR_INVALID_REQUEST), + // Empty body = all-topics request; 0 topics is correct for this stub. + // Non-empty body that fails to decode = malformed request; return 0 topics. + // Kafka Metadata response has no top-level error code field: errors are per-topic only. + // 0 topics is spec-correct and unambiguous for a decode failure. + let (topics_count, effective_error) = if body.is_empty() { + (0usize, top_level_error_code) + } else { + split_metadata_request_topics(body, api_version) + .map_or((0, ERROR_INVALID_REQUEST), |n| (n, top_level_error_code)) }; - let topic_error = if top_level_error_code == ERROR_NONE { + let topic_error = if effective_error == ERROR_NONE { ERROR_UNKNOWN_TOPIC_OR_PARTITION } else { - top_level_error_code + effective_error }; let mut e = Encoder::with_capacity(256); diff --git a/gateways/kafka/src/server.rs b/gateways/kafka/src/server.rs index 6b313a67e8..a41d5fa9bd 100644 --- a/gateways/kafka/src/server.rs +++ b/gateways/kafka/src/server.rs @@ -87,6 +87,12 @@ impl BrokerAdvertise { "KAFKA_ADVERTISED_HOST must not be empty".into(), )); } + if trimmed.len() > i16::MAX as usize { + return Err(KafkaProtocolError::InvalidConfig( + "KAFKA_ADVERTISED_HOST exceeds Kafka nullable string limit (32767 bytes)" + .into(), + )); + } trimmed.to_string() } else if local_addr.ip().is_unspecified() { return Err(KafkaProtocolError::InvalidConfig( @@ -167,6 +173,9 @@ impl KafkaServer { accept_result = listener.accept() => { match accept_result { Ok((stream, peer)) => { + if let Err(e) = stream.set_nodelay(true) { + warn!(%peer, "TCP_NODELAY failed: {e}"); + } let cfg = Arc::clone(&self.config); let broker = Arc::clone(&broker); tracker.spawn(async move { @@ -350,12 +359,17 @@ pub async fn read_frame( }); } - // Reserve in 64 KB increments so a max-size frame (8 MB by default) does not trigger a - // single large upfront allocation before any payload bytes have arrived. - let mut data = BytesMut::new(); + // read_buf() exposes all BytesMut spare capacity to the OS; after reserve(n) the + // allocator may give more than n bytes, so the OS can fill past frame_len and silently + // consume bytes belonging to the next pipelined frame. Use read() with a bounded slice + // so each OS call is limited to exactly the remaining bytes needed. + let mut data = BytesMut::with_capacity(frame_len); while data.len() < frame_len { - data.reserve((frame_len - data.len()).min(READ_CHUNK)); - match timeout_at(deadline, stream.read_buf(&mut data)).await { + let remaining = frame_len - data.len(); + let chunk = remaining.min(READ_CHUNK); + let prev = data.len(); + data.resize(prev + chunk, 0); + let n = match timeout_at(deadline, stream.read(&mut data[prev..prev + chunk])).await { Err(_) => return Err(io::Error::new(io::ErrorKind::TimedOut, "read timeout").into()), Ok(Ok(0)) => { return Err( @@ -363,13 +377,10 @@ pub async fn read_frame( ); } Ok(Err(e)) => return Err(e.into()), - Ok(Ok(_)) => {} - } + Ok(Ok(n)) => n, + }; + data.truncate(prev + n); } - // read_buf may have written past frame_len if the OS returned more bytes than we - // reserved (capacity can round up). Truncate so pipelined frames don't bleed into - // decoder.read_bytes(remaining()) at the call site. - data.truncate(frame_len); Ok(data.freeze()) } diff --git a/gateways/kafka/tests/broker_advertise_tests.rs b/gateways/kafka/tests/broker_advertise_tests.rs index 8f482019c4..a445540dd7 100644 --- a/gateways/kafka/tests/broker_advertise_tests.rs +++ b/gateways/kafka/tests/broker_advertise_tests.rs @@ -85,6 +85,18 @@ fn from_server_config_uses_bind_ip_for_non_wildcard_listener() { assert_eq!(broker.port, 19092); } +#[test] +fn from_server_config_rejects_advertised_host_exceeding_kafka_string_limit() { + let config = ServerConfig { + bind_addr: "127.0.0.1:9093".to_string(), + advertised_host: Some("x".repeat(i16::MAX as usize + 1)), + ..ServerConfig::default() + }; + let local_addr: SocketAddr = "127.0.0.1:9093".parse().unwrap(); + let err = BrokerAdvertise::from_server_config(&config, local_addr).unwrap_err(); + assert!(err.to_string().contains("KAFKA_ADVERTISED_HOST")); +} + #[test] fn from_server_config_honors_advertised_port_override() { let config = ServerConfig { diff --git a/gateways/kafka/tests/metadata_regression_tests.rs b/gateways/kafka/tests/metadata_regression_tests.rs index 50c91bb1c4..1c33e38b7a 100644 --- a/gateways/kafka/tests/metadata_regression_tests.rs +++ b/gateways/kafka/tests/metadata_regression_tests.rs @@ -61,6 +61,20 @@ fn read_broker_flexible(d: &mut Decoder) -> (String, i32) { (host, port) } +#[test] +fn metadata_corrupt_partial_body_returns_zero_topics() { + let body = handle_request( + API_KEY_METADATA, + 0, + Bytes::from_static(&[0x00, 0x00]), + &default_broker(), + ); + let mut d = Decoder::new(body); + let _ = read_broker_legacy(&mut d); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.remaining(), 0); +} + #[test] fn metadata_v0_empty_topics_stub_broker() { let body = handle_request( diff --git a/gateways/kafka/tests/server_integration_tests.rs b/gateways/kafka/tests/server_integration_tests.rs index 9de8f9b092..a8d6d921bc 100644 --- a/gateways/kafka/tests/server_integration_tests.rs +++ b/gateways/kafka/tests/server_integration_tests.rs @@ -129,3 +129,28 @@ async fn write_frame_length_prefix_is_big_endian() { assert_eq!(len, 4); assert_eq!(&len_and_data[4..], &[1, 2, 3, 4]); } + +#[tokio::test] +async fn read_frame_does_not_consume_pipelined_frame_bytes() { + let (mut client, mut server) = tcp_pair().await; + + let payload1 = b"first-request-body-data"; + let payload2 = b"second-pipelined-request"; + + // Write both frames in a single syscall so the OS delivers them together. + // With the old read_buf approach, allocator rounding causes the first read_frame + // call to consume bytes from payload2, which truncate() then silently discards. + let mut both = BytesMut::with_capacity(4 + payload1.len() + 4 + payload2.len()); + both.put_i32(i32::try_from(payload1.len()).unwrap()); + both.extend_from_slice(payload1); + both.put_i32(i32::try_from(payload2.len()).unwrap()); + both.extend_from_slice(payload2); + client.write_all(&both).await.unwrap(); + + let timeout = Duration::from_secs(1); + let frame1 = read_frame(&mut server, 4096, timeout).await.unwrap(); + let frame2 = read_frame(&mut server, 4096, timeout).await.unwrap(); + + assert_eq!(&frame1[..], payload1); + assert_eq!(&frame2[..], payload2); +} From 5c335d164b3ff9f2c2771c75f9b08098a61b9b59 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sat, 20 Jun 2026 22:17:52 -0400 Subject: [PATCH 11/57] Normalize trailing newlines in GitHub configs Adjust trailing newline/whitespace in .github/config/publish.yml and .github/dependabot.yml. These are non-functional formatting fixes to normalize end-of-file newlines and do not change any configuration values. --- .github/config/publish.yml | 2 +- .github/dependabot.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/config/publish.yml b/.github/config/publish.yml index 36376d34c6..f549fe4376 100644 --- a/.github/config/publish.yml +++ b/.github/config/publish.yml @@ -146,4 +146,4 @@ components: tag_pattern: "^foreign/go/v([0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\\+[0-9A-Za-z.-]+)?)$" registry: none version_file: "foreign/go/contracts/version.go" - version_regex: 'const\s+Version\s*=\s*"([^"]+)"' + version_regex: 'const\s+Version\s*=\s*"([^"]+)"' \ No newline at end of file diff --git a/.github/dependabot.yml b/.github/dependabot.yml index b61effac1f..78c7f2e122 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -300,4 +300,4 @@ updates: groups: cpp: patterns: - - "*" + - "*" \ No newline at end of file From 444e4428ad15bcd7f1c83752c387d3a64fbac830 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sat, 20 Jun 2026 22:46:03 -0400 Subject: [PATCH 12/57] Fixing the pre checks failures --- .github/config/components.yml | 2 +- .github/config/publish.yml | 2 +- .github/dependabot.yml | 2 +- gateways/README.md | 2 +- gateways/kafka/README.md | 2 +- gateways/kafka/docs/MANUAL_TESTING.md | 26 +++++++------- gateways/kafka/docs/SCOPE.md | 10 +++--- gateways/kafka/docs/TEST_SUITE.md | 16 ++++----- .../kafka/docs/kafka_api_keys_reference.md | 36 +++++++++---------- gateways/kafka/tools/kafka-tool/README.md | 8 +++-- 10 files changed, 54 insertions(+), 52 deletions(-) diff --git a/.github/config/components.yml b/.github/config/components.yml index b7d65f9756..7231a679c2 100644 --- a/.github/config/components.yml +++ b/.github/config/components.yml @@ -517,4 +517,4 @@ components: - "sort" - "test-1" - "test-2" - - "machete" \ No newline at end of file + - "machete" diff --git a/.github/config/publish.yml b/.github/config/publish.yml index f549fe4376..36376d34c6 100644 --- a/.github/config/publish.yml +++ b/.github/config/publish.yml @@ -146,4 +146,4 @@ components: tag_pattern: "^foreign/go/v([0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\\+[0-9A-Za-z.-]+)?)$" registry: none version_file: "foreign/go/contracts/version.go" - version_regex: 'const\s+Version\s*=\s*"([^"]+)"' \ No newline at end of file + version_regex: 'const\s+Version\s*=\s*"([^"]+)"' diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 78c7f2e122..b61effac1f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -300,4 +300,4 @@ updates: groups: cpp: patterns: - - "*" \ No newline at end of file + - "*" diff --git a/gateways/README.md b/gateways/README.md index 7b0c550f5e..7fd825b4ae 100644 --- a/gateways/README.md +++ b/gateways/README.md @@ -3,7 +3,7 @@ Protocol gateways that let existing clients talk to Iggy without changing the core server wire surface. | Gateway | Issue | Description | -|---------|-------|-------------| +| --------- | ------- | ------------- | | [kafka](kafka/) | [#3421](https://github.com/apache/iggy/issues/3421) | Kafka wire protocol TCP listener (port 9093) | Each gateway is a separate workspace crate under `gateways//`. diff --git a/gateways/kafka/README.md b/gateways/kafka/README.md index b336887cd0..6c1e2d608f 100644 --- a/gateways/kafka/README.md +++ b/gateways/kafka/README.md @@ -11,7 +11,7 @@ cargo run -p iggy-gateway-kafka Default bind: `127.0.0.1:9093`. Environment variables: | Variable | Default | Description | -|---|---|---| +| --- | --- | --- | | `KAFKA_BIND_ADDR` | `127.0.0.1:9093` | TCP address to listen on | | `KAFKA_ADVERTISED_HOST` | bind IP | Hostname/IP clients use to reach this broker (required when binding to `0.0.0.0`/`::`) | | `KAFKA_ADVERTISED_PORT` | bind port | Port advertised in Metadata responses | diff --git a/gateways/kafka/docs/MANUAL_TESTING.md b/gateways/kafka/docs/MANUAL_TESTING.md index 30f37783c6..fecafb219d 100644 --- a/gateways/kafka/docs/MANUAL_TESTING.md +++ b/gateways/kafka/docs/MANUAL_TESTING.md @@ -11,7 +11,7 @@ See also: [SCOPE.md](SCOPE.md) (supported API keys), [TEST_SUITE.md](TEST_SUITE. ### Requirements | Tool | Purpose | Install | -|------|---------|---------| +| ------ | --------- | --------- | | Rust toolchain | Build gateway + kafka-tool | [rustup.rs](https://rustup.rs) | | `kafka-message-gen` | Generate/send wire fixtures | `cargo build -p kafka-message-gen` | | `kcat` (optional) | Real Kafka client smoke test | `brew install kcat` / `apt install kafkacat` | @@ -62,7 +62,7 @@ All tests must pass. If `decode_validation_tests` fail, regenerate fixtures (ste ### Category A — Smoke tests (must pass before check-in) | ID | Test | Steps | Expected result | Pass criteria | -|----|------|-------|-----------------|---------------| +| ---- | ------ | ------- | ----------------- | --------------- | | A1 | Gateway starts | Run `iggy-gateway-kafka` | Binds to `:9093`, no panic | Log shows bind address | | A2 | ApiVersions v1 | `cargo run -p kafka-message-gen -- send --host 127.0.0.1:9093 --api-key 18 --version 1` | Response received | `ec=0`, non-zero byte count | | A3 | ApiVersions v3 (flexible) | Same with `--version 3` | Response received | `ec=0` | @@ -78,7 +78,7 @@ All tests must pass. If `decode_validation_tests` fail, regenerate fixtures (ste For each API key, test **min−1**, **min**, **max**, **max+1** using `kafka-message-gen send` with `--version N`. | API key | Name | Min | Max | Test versions | -|---------|------|-----|-----|---------------| +| --------- | ------ | ----- | ----- | --------------- | | 18 | ApiVersions | 0 | 3 | −1, 0, 3, 4 | | 3 | Metadata | 0 | 9 | −1, 0, 9, 10 | | 0 | Produce | 3 | 9 | 2, 3, 9, 10 | @@ -87,7 +87,7 @@ For each API key, test **min−1**, **min**, **max**, **max+1** using `kafka-mes | 19 | CreateTopics | 2 | 5 | 1, 2, 5, 6 | | ID | Test | Expected for in-range | Expected for out-of-range | -|----|------|----------------------|---------------------------| +| ---- | ------ | ---------------------- | --------------------------- | | B1 | ApiVersions negotiation | `error_code=0`; body lists 6 API keys with correct min/max | `error_code=35` (UNSUPPORTED_VERSION) | | B2 | Metadata out-of-range | N/A | Topic entries show `error_code=35` | | B3 | Produce/Fetch/ListOffsets/CreateTopics out-of-range | N/A | Version-aware response with `error_code=35` (top-level or per-topic/partition) | @@ -102,7 +102,7 @@ cargo run -p kafka-message-gen -- generate --api-key 18 --version 3 --hex ### Category C — Unsupported API keys | ID | API key | Name | Steps | Expected | -|----|---------|------|-------|----------| +| ---- | --------- | ------ | ------- | ---------- | | C1 | 8 | OffsetCommit | `send --api-key 8 --version 2` | `ec=35`, connection stays open | | C2 | 10 | FindCoordinator | `send --api-key 10` | `ec=35` | | C3 | 17 | SaslHandshake | `send --api-key 17` | `ec=35` | @@ -113,7 +113,7 @@ Follow C1 with A2 on the **same** `nc` session to confirm the connection is not ### Category D — Flexible vs legacy wire encoding | ID | API key | Version | Encoding | Validation | -|----|---------|---------|----------|------------| +| ---- | --------- | --------- | ---------- | ------------ | | D1 | Produce | 8 | Legacy (i32 arrays) | `send` succeeds, `ec=0` | | D2 | Produce | 9 | Flexible (compact + tagged fields) | `send` succeeds, `ec=0` | | D3 | Fetch | 11 | Legacy | `send` succeeds | @@ -128,7 +128,7 @@ Follow C1 with A2 on the **same** `nc` session to confirm the connection is not ### Category E — Metadata stub semantics | ID | Test | Steps | Expected | -|----|------|-------|----------| +| ---- | ------ | ------- | ---------- | | E1 | Broker advertise address | Start gateway on `127.0.0.1:9093`; Metadata v0 | Broker host=`127.0.0.1`, port=`9093` | | E2 | Wildcard bind + advertised host | `KAFKA_BIND_ADDR=0.0.0.0:19093` + `KAFKA_ADVERTISED_HOST=kafka.internal`, restart | Metadata broker host/port match advertised values | | E3 | Unknown topic stub | Metadata with topic name `my-topic` | Topic error `3` (UNKNOWN_TOPIC_OR_PARTITION), name `unknown-topic` | @@ -137,7 +137,7 @@ Follow C1 with A2 on the **same** `nc` session to confirm the connection is not ### Category F — TCP / connection behavior | ID | Test | Steps | Expected | -|----|------|-------|----------| +| ---- | ------ | ------- | ---------- | | F1 | Correlation ID echoed | Send ApiVersions with known correlation_id; decode response header | Response correlation_id matches request | | F2 | Sequential requests | Send ApiVersions then Metadata on same TCP connection | Both get valid responses | | F3 | Client disconnect | Connect, send partial frame, close | Gateway logs clean disconnect, no panic | @@ -150,7 +150,7 @@ Follow C1 with A2 on the **same** `nc` session to confirm the connection is not Requires `kcat` installed. Gateway does **not** implement SASL or full broker semantics — expect limited success. | ID | Test | Command | Expected (foundation) | -|----|------|---------|---------------------| +| ---- | ------ | --------- | --------------------- | | G1 | Broker metadata | `kcat -b 127.0.0.1:9093 -L` | ApiVersions + Metadata handshake; broker appears in metadata | | G2 | Produce (likely fails later) | `echo "hello" \| kcat -b 127.0.0.1:9093 -t test -P` | May fail at coordinator/group stage — document actual error | | G3 | Consumer (likely fails later) | `kcat -b 127.0.0.1:9093 -t test -C -o beginning` | May fail without consumer groups — document actual error | @@ -160,7 +160,7 @@ Record kcat version and exact error strings in your test log. G1 passing is the ### Category H — Adversarial / negative input | ID | Test | Steps | Expected | -|----|------|-------|----------| +| ---- | ------ | ------- | ---------- | | H1 | Truncated Produce body | Send valid header + incomplete body | `error_code=42` (INVALID_REQUEST) or connection error; **no panic** | | H2 | Random bytes | `dd if=/dev/urandom bs=64 count=1 \| nc 127.0.0.1 9093` | Connection closed or protocol error; gateway stays up | | H3 | Empty body after header | ApiVersions with valid header, empty body | `ec=0` (ApiVersions accepts empty body) | @@ -172,7 +172,7 @@ Record kcat version and exact error strings in your test log. G1 passing is the ### Kafka error codes used in #3421 | Code | Name | When returned | -|------|------|---------------| +| ------ | ------ | --------------- | | 0 | NONE | Successful stub response | | 42 | INVALID_REQUEST | Produce/Fetch/ListOffsets/CreateTopics decode failure; unsupported request header | | 3 | UNKNOWN_TOPIC_OR_PARTITION | Metadata stub per-topic error | @@ -182,7 +182,7 @@ Record kcat version and exact error strings in your test log. G1 passing is the ### Response header rules | API key | Request flexible? | Response header version | -|---------|--------------------|-------------------------| +| --------- | -------------------- | ------------------------- | | 18 ApiVersions | v3+ | Always v0 (correlation_id only) | | 3 Metadata | v9+ | v1 (correlation_id + tagged fields) | | 0 Produce | v9+ | v1 | @@ -250,7 +250,7 @@ _________________________________ ## 6. Troubleshooting | Symptom | Likely cause | Fix | -|---------|--------------|-----| +| --------- | -------------- | ----- | | `Connection refused` on 9093 | Gateway not running | Start `iggy-gateway-kafka` | | `decode_validation_tests` panic | Missing fixtures | Run `kafka-message-gen generate` | | `ec=35` for in-range version | Version not in `SUPPORTED_RANGES` | Check `SCOPE.md` and `api.rs` | diff --git a/gateways/kafka/docs/SCOPE.md b/gateways/kafka/docs/SCOPE.md index 3f20fd411c..984b812470 100644 --- a/gateways/kafka/docs/SCOPE.md +++ b/gateways/kafka/docs/SCOPE.md @@ -5,7 +5,7 @@ Foundation layer only: a TCP listener on the Kafka wire port that decodes requests, validates scoped API keys and versions, validates request wire formats, and returns stub responses. **No Iggy backend integration.** | Deliverable | Status | Location | -|-------------|--------|----------| +| ------------- | -------- | ---------- | | TCP listener on `127.0.0.1:9093` (configurable) | Done | `src/server.rs`, `src/main.rs` | | Length-prefixed frame read/write with `max_frame_size` cap | Done | `src/server.rs` | | Request header v1/v2 auto-detection | Done | `src/protocol/header.rs` | @@ -29,7 +29,7 @@ Expand `SUPPORTED_RANGES` only after a key/version pair is manually tested. ApiV ## Supported API keys and versions | API key | Name | Min version | Max version | Valid versions | Behavior | -|---------|------|-------------|-------------|----------------|----------| +| --------- | ------ | ------------- | ------------- | ---------------- | ---------- | | 18 | ApiVersions | 0 | 3 | 0, 1, 2, 3 | Advertise supported ranges; flexible encoding at v3+ | | 3 | Metadata | 0 | 9 | 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 | Decode topic list count; stub broker from `ServerConfig.bind_addr`; flexible encoding at v9+ | | 0 | Produce | 3 | 9 | 3, 4, 5, 6, 7, 8, 9 | Decode request; stub response | @@ -44,7 +44,7 @@ A request is accepted when `min_version ≤ api_version ≤ max_version` for tha Use this table when configuring clients or generating wire fixtures with `kafka-message-gen`. | API key | Name | Valid versions (inclusive range) | Flexible wire encoding from | -|---------|------|----------------------------------|----------------------------| +| --------- | ------ | ---------------------------------- | ---------------------------- | | 0 | Produce | 3–9 | v9 | | 1 | Fetch | 4–12 | v12 | | 2 | ListOffsets | 1–6 | v6 | @@ -59,7 +59,7 @@ Use this table when configuring clients or generating wire fixtures with `kafka- All API keys not listed above receive an error-only response with `UNSUPPORTED_VERSION` (35). Examples not in this foundation scope: | API key | Name | Notes | -|---------|------|-------| +| --------- | ------ | ------- | | 8 | OffsetCommit | Consumer group — later issue | | 9 | OffsetFetch | Consumer group — later issue | | 10 | FindCoordinator | Consumer group — later issue | @@ -74,7 +74,7 @@ Full reference for future phases: [`kafka_api_keys_reference.md`](kafka_api_keys ## Architecture (three layers) | Layer | #3421 | Description | -|-------|-------|-------------| +| ------- | ------- | ------------- | | **1 — Wire framing** | In scope | `server.rs`, `codec.rs`, `header.rs` — keep custom, zero-copy frame I/O | | **2 — Request/response codecs** | Partial | Custom minimal-parse codecs for 6 hot-path keys; stub responses only | | **3 — Iggy bridge** | Out of scope | Produce/Fetch → Iggy SDK; deferred to a follow-on issue | diff --git a/gateways/kafka/docs/TEST_SUITE.md b/gateways/kafka/docs/TEST_SUITE.md index c0f6df8c1e..27ed05a6ba 100644 --- a/gateways/kafka/docs/TEST_SUITE.md +++ b/gateways/kafka/docs/TEST_SUITE.md @@ -25,7 +25,7 @@ Fixtures are gitignored under `tools/kafka-tool/kafka_messages/`. Tests that nee ## Test file catalog | File | Suite focus | Test count (approx.) | Depends on fixtures | -|------|-------------|----------------------|---------------------| +| ------ | ------------- | ---------------------- | --------------------- | | [`codec_tests.rs`](../tests/codec_tests.rs) | Primitive encode/decode round-trips, varint, compact strings, tagged fields | 9 | No | | [`decode_safety_tests.rs`](../tests/decode_safety_tests.rs) | Adversarial wire input — malformed lengths, truncated bodies | 6 | No | | [`header_tests.rs`](../tests/header_tests.rs) | Request/response header v1/v2, version lookup table | 10 | No | @@ -47,7 +47,7 @@ Fixtures are gitignored under `tools/kafka-tool/kafka_messages/`. Tests that nee ### ApiVersions (key 18, v0–v3) | Scenario | Test file | Test name | -|----------|-----------|-----------| +| ---------- | ----------- | ----------- | | Non-flexible response (v1) | `api_handler_tests` | `api_versions_v1_response_non_flexible_format` | | Flexible response (v3) | `api_handler_tests` | `api_versions_v3_response_flexible_format` | | Golden byte fixture (v1) | `golden_wire_fixtures_tests` | `golden_apiversions_v1_response_fixture` | @@ -59,7 +59,7 @@ Fixtures are gitignored under `tools/kafka-tool/kafka_messages/`. Tests that nee ### Metadata (key 3, v0–v9) | Scenario | Test file | Test name | -|----------|-----------|-----------| +| ---------- | ----------- | ----------- | | Stub broker (default 127.0.0.1:9093) | `api_handler_tests`, `metadata_regression_tests` | `metadata_response_has_broker_*`, `metadata_v0_empty_*` | | Unsupported version → topic error 35 | `api_handler_tests`, `version_firewall_tests` | `unsupported_version_returns_protocol_error`, `metadata_*_version_returns_topic_error` | | Golden byte fixture (v0, 1 topic) | `golden_wire_fixtures_tests` | `golden_metadata_v0_single_topic_response_fixture` | @@ -71,7 +71,7 @@ Fixtures are gitignored under `tools/kafka-tool/kafka_messages/`. Tests that nee ### Produce (key 0, v3–v9) | Scenario | Test file | Test name | -|----------|-----------|-----------| +| ---------- | ----------- | ----------- | | Decode all versions (fixture) | `decode_validation_tests` | `produce_all_supported_versions_decode` | | Response encode all versions | `decode_validation_tests` | `produce_response_encodes_for_all_supported_versions` | | v3 field layout | `decode_validation_tests` | `produce_response_v3_roundtrip` | @@ -84,7 +84,7 @@ Fixtures are gitignored under `tools/kafka-tool/kafka_messages/`. Tests that nee ### Fetch (key 1, v4–v12) | Scenario | Test file | Test name | -|----------|-----------|-----------| +| ---------- | ----------- | ----------- | | Decode all versions | `decode_validation_tests` | `fetch_all_supported_versions_decode` | | Response encode all versions | `decode_validation_tests` | `fetch_response_encodes_for_all_supported_versions` | | v7 session_id / error_code layout | `decode_validation_tests` | `fetch_response_v7_roundtrip` | @@ -95,7 +95,7 @@ Fixtures are gitignored under `tools/kafka-tool/kafka_messages/`. Tests that nee ### ListOffsets (key 2, v1–v6) | Scenario | Test file | Test name | -|----------|-----------|-----------| +| ---------- | ----------- | ----------- | | Decode all versions | `decode_validation_tests` | `list_offsets_all_supported_versions_decode` | | v1 no leader_epoch | `decode_validation_tests` | `list_offsets_response_v1_no_leader_epoch` | | v4 has leader_epoch | `decode_validation_tests` | `list_offsets_response_v4_has_leader_epoch` | @@ -105,7 +105,7 @@ Fixtures are gitignored under `tools/kafka-tool/kafka_messages/`. Tests that nee ### CreateTopics (key 19, v2–v5) | Scenario | Test file | Test name | -|----------|-----------|-----------| +| ---------- | ----------- | ----------- | | Decode all versions | `decode_validation_tests` | `create_topics_all_supported_versions_decode` | | v2 roundtrip | `decode_validation_tests` | `create_topics_response_v2_roundtrip` | | v5 flexible roundtrip | `decode_validation_tests` | `create_topics_response_v5_roundtrip` | @@ -117,7 +117,7 @@ Fixtures are gitignored under `tools/kafka-tool/kafka_messages/`. Tests that nee ## Cross-cutting scenarios | Scenario | Test file | Test name | -|----------|-----------|-----------| +| ---------- | ----------- | ----------- | | Version firewall min/max boundaries | `version_firewall_tests` | `is_supported_version_matches_scope_table` | | Unknown API keys (8, 9, 10, 17, 20, 999) | `version_firewall_tests`, `api_handler_tests` | `unsupported_api_keys_*`, `unknown_api_key_*` | | Negative i32 array length | `decode_safety_tests` | `negative_i32_array_length_returns_error_not_panic` | diff --git a/gateways/kafka/docs/kafka_api_keys_reference.md b/gateways/kafka/docs/kafka_api_keys_reference.md index c31cec2d7c..0cec397e46 100644 --- a/gateways/kafka/docs/kafka_api_keys_reference.md +++ b/gateways/kafka/docs/kafka_api_keys_reference.md @@ -11,7 +11,7 @@ ## Legend | Symbol | Meaning | -|--------|---------| +| -------- | --------- | | 🔴 Bridge | Core data path — must be fully implemented and forwarded to Iggy | | 🟠 Required Stub | Client state-machine API — must return a well-formed response or clients will stall/crash | | 🟡 Optional Stub | Admin/observability — can safely return `UNSUPPORTED_VERSION` or `NOT_CONTROLLER` | @@ -27,7 +27,7 @@ Kafka 4.0 removed all protocol versions older than Kafka 2.1.0 (KIP-896). Key new minimums: | API | Old Min | New Min (4.0) | -|-----|:-------:|:-------------:| +| ----- | :-------: | :-------------: | | Produce | 0 | 3 | | Fetch | 0 | 4 | | ListOffsets | 0 | 1 | @@ -46,7 +46,7 @@ Key new minimums: ## Group 1 — Core Data Path | Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | -|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | | 0 | **Produce** | 3 †| 12 | v9 | ✅ | 🔴 Bridge | | 1 | **Fetch** | 4 | 17 | v12 | ✅ | 🔴 Bridge | | 2 | **ListOffsets** | 1 | 9 | v6 | ✅ | 🟠 Required Stub | @@ -59,7 +59,7 @@ Key new minimums: ## Group 2 — API Negotiation & Auth | Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | -|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | | 17 | **SaslHandshake** | 0 | 1 | never | ✅ | 🔴 Bridge (auth flow) | | 18 | **ApiVersions** | 0 | 4 | v3 | ✅ | 🔴 Bridge (advertise Iggy caps) | | 36 | **SaslAuthenticate** | 0 | 2 | v2 | ✅ | 🔴 Bridge (auth flow) | @@ -73,7 +73,7 @@ Key new minimums: ## Group 3 — Classic Consumer Group Protocol | Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | -|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | | 8 | **OffsetCommit** | 2 | 9 | v8 | ✅ | 🟠 Required Stub | | 9 | **OffsetFetch** | 1 | 9 | v6 | ✅ | 🟠 Required Stub | | 10 | **FindCoordinator** | 1 | 6 | v3 | ✅ | 🟠 Required Stub | @@ -90,7 +90,7 @@ Key new minimums: ## Group 4 — New Consumer Group Protocol (KIP-848, Kafka 3.7+) | Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | -|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | | 68 | **ConsumerGroupHeartbeat** | 0 | 1 | v0 | ✅ | 🟠 Required Stub | | 69 | **ConsumerGroupDescribe** | 0 | 1 | v0 | ✅ | 🟡 Optional Stub | @@ -102,7 +102,7 @@ Key new minimums: ## Group 5 — Topic Administration | Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | -|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | | 19 | **CreateTopics** | 2 | 7 | v5 | ✅ (max v5 ⚠️) | 🟠 Required Stub | | 20 | **DeleteTopics** | 1 | 6 | v4 | ✅ | 🟡 Optional Stub | | 21 | **DeleteRecords** | 0 | 2 | v2 | ✅ | 🟡 Optional Stub | @@ -115,7 +115,7 @@ Key new minimums: ## Group 6 — Transactions (EOS — Exactly Once Semantics) | Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | -|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | | 22 | **InitProducerId** | 2 | 5 | v2 | ✅ | 🟡 Optional Stub | | 23 | **OffsetForLeaderEpoch** | 1 | 5 | v4 | ✅ | 🟡 Optional Stub | | 24 | **AddPartitionsToTxn** | 1 | 5 | v3 | ✅ | 🟡 Optional Stub | @@ -129,7 +129,7 @@ Key new minimums: ## Group 7 — Security & ACLs | Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | -|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | | 29 | **DescribeAcls** | 0 | 3 | v2 | ✅ | 🟡 Optional Stub | | 30 | **CreateAcls** | 0 | 3 | v2 | ✅ | 🟡 Optional Stub | | 31 | **DeleteAcls** | 0 | 3 | v2 | ✅ | 🟡 Optional Stub | @@ -145,7 +145,7 @@ Key new minimums: ## Group 8 — Configuration & Quotas | Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | -|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | | 32 | **DescribeConfigs** | 0 | 4 | v4 | ✅ | 🟡 Optional Stub | | 33 | **AlterConfigs** | 0 | 2 | v2 | ✅ | 🟡 Optional Stub | | 44 | **IncrementalAlterConfigs** | 0 | 1 | v1 | ✅ | 🟡 Optional Stub | @@ -157,7 +157,7 @@ Key new minimums: ## Group 9 — Log & Partition Admin | Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | -|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | | 34 | **AlterReplicaLogDirs** | 0 | 2 | v2 | ✅ | 🟡 Optional Stub | | 35 | **DescribeLogDirs** | 0 | 4 | v2 | ✅ | 🟡 Optional Stub | | 43 | **ElectLeaders** | 0 | 2 | v2 | ✅ | 🟡 Optional Stub | @@ -171,7 +171,7 @@ Key new minimums: ## Group 10 — Cluster Introspection | Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | -|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | | 55 | **DescribeQuorum** | 0 | 2 | v0 | ✅ | 🟡 Optional Stub | | 59 | **FetchSnapshot** | 0 | 1 | v0 | ✅ | 🟡 Optional Stub | | 60 | **DescribeCluster** | 0 | 1 | v0 | ✅ | 🟡 Optional Stub | @@ -186,7 +186,7 @@ Key new minimums: ## Group 11 — Observability / Telemetry (KIP-714, Kafka 3.7+) | Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | -|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | | 71 | **GetTelemetrySubscriptions** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | | 72 | **PushTelemetry** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | | 76 | **ListClientMetricsResources** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | @@ -199,7 +199,7 @@ Key new minimums: > coordinator APIs and can be rejected. | Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | -|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | | 77 | **ShareGroupHeartbeat** | 0 | 0 | v0 | ✅ | 🟠 Required Stub | | 78 | **ShareGroupDescribe** | 0 | 0 | v0 | ✅ | 🟡 Optional Stub | | 79 | **ShareFetch** | 0 | 0 | v0 | ✅ | 🔴 Bridge (share consume) | @@ -210,7 +210,7 @@ Key new minimums: ## Group 13 — KRaft Raft Voter Management (NEW in Kafka 4.0) | Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | -|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | | 81 | **AddRaftVoter** | 0 | 0 | v0 | ❌ MISSING | ❌ Reject (internal) | | 82 | **RemoveRaftVoter** | 0 | 0 | v0 | ❌ MISSING | ❌ Reject (internal) | | 83 | **UpdateRaftVoter** | 0 | 0 | v0 | ❌ MISSING | ❌ Reject (internal) | @@ -223,7 +223,7 @@ Key new minimums: > Return `INVALID_REQUEST` (error code 42) with a properly framed response — **do not drop the connection**. | Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | -|:---:|----------|:---------:|:---------:|:-------------:|:-----------:|:--------------:| +| :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | | 4 | **LeaderAndIsr** | 0 | 7 | v4 | ✅ | ❌ Reject (broker-only) | | 5 | **StopReplica** | 0 | 4 | v2 | ✅ | ❌ Reject (broker-only) | | 6 | **UpdateMetadata** | 0 | 8 | v6 | ✅ | ❌ Reject (broker-only) | @@ -249,7 +249,7 @@ Key new minimums: ## Summary Counts | Category | Count | Notes | -|----------|:-----:|-------| +| ---------- | :-----: | ------- | | 🔴 Bridge (data path) | 6 | Produce, Fetch, Metadata, ApiVersions, SaslHandshake, SaslAuthenticate, ShareFetch | | 🟠 Required Stub (client state machine) | 14 | Consumer group, CreateTopics, ConsumerGroupHeartbeat (68), ShareGroupHeartbeat (77), ShareAcknowledge (80) | | 🟡 Optional Stub (admin/observability) | 44 | Can return `UNSUPPORTED_VERSION` or `NOT_CONTROLLER` safely | @@ -263,7 +263,7 @@ Key new minimums: ### `SUPPORTED_RANGES` is behind the latest Kafka 4.0 max versions | API | Declared range | Kafka 4.0 max | Gap | -|-----|:---:|:---:|:---:| +| ----- | :---: | :---: | :---: | | Produce | v3-v9 | v12 | 3 versions behind | | Fetch | v4-v12 | v17 | 5 versions behind | | ListOffsets | v1-v6 | v9 | 3 versions behind | diff --git a/gateways/kafka/tools/kafka-tool/README.md b/gateways/kafka/tools/kafka-tool/README.md index a65eb2a005..678b43f2a6 100644 --- a/gateways/kafka/tools/kafka-tool/README.md +++ b/gateways/kafka/tools/kafka-tool/README.md @@ -56,6 +56,7 @@ cargo run -- list ``` Output: + ``` Key Name MinVer MaxVer Count ────────────────────────────────────────────────────────────────────────────── @@ -97,7 +98,7 @@ kafka_messages/ #### Options | Flag | Description | Default | -|------|-------------|---------| +| ------ | ------------- | --------- | | `--output` | Output directory | `kafka_messages/` | | `--api-key N` | Generate only for API key N | all | | `--version N` | Generate only for version N | all | @@ -121,6 +122,7 @@ cargo run -- send --host 127.0.0.1:9092 ``` Output (one line per API key × version): + ``` ✓ ApiVersions v3 → 32 bytes ec=0 ✓ Metadata v12 → 148 bytes ec=0 @@ -133,7 +135,7 @@ Result: 243 OK 37 failed #### Options | Flag | Description | Default | -|------|-------------|---------| +| ------ | ------------- | --------- | | `--host` | Server address | `127.0.0.1:9092` | | `--api-key N` | Test only API key N | all | | `--version N` | Test only version N | all | @@ -170,7 +172,7 @@ cat kafka_messages/018_ApiVersions_v3.bin | nc 127.0.0.1 9092 | xxd | head ## Supported API Keys (Kafka 4.1.0) | Key | Name | Versions | Phase 1 Priority | -|-----|------|----------|-----------------| +| ----- | ------ | ---------- | ----------------- | | 0 | Produce | v3–v13 | ✅ Critical | | 1 | Fetch | v4–v18 | ✅ Critical | | 2 | ListOffsets | v1–v11 | ✅ Critical | From 0b56947f17f641ebf5337ee34b3a7daa49b0a4a2 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sat, 20 Jun 2026 22:48:48 -0400 Subject: [PATCH 13/57] Fixing the formatting of md files --- gateways/kafka/docs/MANUAL_TESTING.md | 6 +- .../kafka/docs/kafka_api_keys_reference.md | 2 +- gateways/kafka/tools/kafka-tool/README.md | 60 +++++++++---------- 3 files changed, 34 insertions(+), 34 deletions(-) diff --git a/gateways/kafka/docs/MANUAL_TESTING.md b/gateways/kafka/docs/MANUAL_TESTING.md index fecafb219d..5d933a231a 100644 --- a/gateways/kafka/docs/MANUAL_TESTING.md +++ b/gateways/kafka/docs/MANUAL_TESTING.md @@ -30,7 +30,7 @@ RUST_LOG=info cargo run -p iggy-gateway-kafka Expected log: -``` +```text kafka listener bound on 127.0.0.1:9093 ``` @@ -191,7 +191,7 @@ Record kcat version and exact error strings in your test log. G1 passing is the ### Frame layout (for manual hex inspection) -``` +```text Request frame: [length: i32 BE] [api_key: i16][api_version: i16][correlation_id: i32] @@ -222,7 +222,7 @@ First bytes after length prefix should include your correlation_id from the fixt Copy this checklist into your PR or test log: -``` +```text Date: ___________ Tester: ___________ Gateway commit: ___________ diff --git a/gateways/kafka/docs/kafka_api_keys_reference.md b/gateways/kafka/docs/kafka_api_keys_reference.md index 0cec397e46..2555bf96af 100644 --- a/gateways/kafka/docs/kafka_api_keys_reference.md +++ b/gateways/kafka/docs/kafka_api_keys_reference.md @@ -47,7 +47,7 @@ Key new minimums: | Key | API Name | Min (4.0) | Max (4.0) | Flexible From | Header.rs ✓ | Gateway Action | | :---: | ---------- | :---------: | :---------: | :-------------: | :-----------: | :--------------: | -| 0 | **Produce** | 3 †| 12 | v9 | ✅ | 🔴 Bridge | +| 0 | **Produce** | 3 † | 12 | v9 | ✅ | 🔴 Bridge | | 1 | **Fetch** | 4 | 17 | v12 | ✅ | 🔴 Bridge | | 2 | **ListOffsets** | 1 | 9 | v6 | ✅ | 🟠 Required Stub | | 3 | **Metadata** | 1 | 12 | v9 | ✅ | 🔴 Bridge | diff --git a/gateways/kafka/tools/kafka-tool/README.md b/gateways/kafka/tools/kafka-tool/README.md index 678b43f2a6..eb44ee3759 100644 --- a/gateways/kafka/tools/kafka-tool/README.md +++ b/gateways/kafka/tools/kafka-tool/README.md @@ -6,7 +6,7 @@ A Rust CLI tool that generates correct, fully-framed Kafka binary wire protocol Each output `.bin` file is a complete, TCP-ready Kafka request: -``` +```text [total_length: i32][api_key: i16][api_version: i16] [correlation_id: i32][client_id: NULLABLE_STRING] [tagged_fields: 0x00] ← only for flexible versions @@ -57,7 +57,7 @@ cargo run -- list Output: -``` +```text Key Name MinVer MaxVer Count ────────────────────────────────────────────────────────────────────────────── 0 Produce 3 13 11 @@ -80,7 +80,7 @@ cargo run -- generate --output ./kafka_messages/ Creates one `.bin` file per API key × version: -``` +```text kafka_messages/ 000_Produce_v3.bin 000_Produce_v4.bin @@ -123,7 +123,7 @@ cargo run -- send --host 127.0.0.1:9092 Output (one line per API key × version): -``` +```text ✓ ApiVersions v3 → 32 bytes ec=0 ✓ Metadata v12 → 148 bytes ec=0 ⚠ Produce v9 → 24 bytes ec=3 ← ec=3 = UnknownTopicOrPartition (expected) @@ -173,39 +173,39 @@ cat kafka_messages/018_ApiVersions_v3.bin | nc 127.0.0.1 9092 | xxd | head | Key | Name | Versions | Phase 1 Priority | | ----- | ------ | ---------- | ----------------- | -| 0 | Produce | v3–v13 | ✅ Critical | -| 1 | Fetch | v4–v18 | ✅ Critical | -| 2 | ListOffsets | v1–v11 | ✅ Critical | -| 3 | Metadata | v0–v13 | ✅ Critical | -| 8 | OffsetCommit | v2–v10 | ✅ Critical | -| 9 | OffsetFetch | v1–v10 | ✅ Critical | -| 10 | FindCoordinator | v0–v6 | ✅ Critical | -| 11 | JoinGroup | v0–v9 | ✅ Critical | -| 12 | Heartbeat | v0–v4 | ✅ Critical | -| 13 | LeaveGroup | v0–v5 | ✅ Critical | -| 14 | SyncGroup | v0–v5 | ✅ Critical | -| 15 | DescribeGroups | v0–v6 | 🟡 Important | -| 16 | ListGroups | v0–v5 | 🟡 Important | -| 17 | SaslHandshake | v0–v1 | 🟡 Important | -| 18 | ApiVersions | v0–v5 | ✅ Critical | -| 19 | CreateTopics | v2–v7 | ✅ Critical | -| 20 | DeleteTopics | v1–v6 | 🟡 Important | -| 21 | DeleteRecords | v0–v2 | 🔵 Phase 2 | -| 22 | InitProducerId | v0–v6 | 🔵 Phase 2 | -| 24 | AddPartitionsToTxn | v0–v5 | 🔵 Phase 2 | -| 25 | AddOffsetsToTxn | v0–v4 | 🔵 Phase 2 | -| 26 | EndTxn | v0–v5 | 🔵 Phase 2 | -| 28 | TxnOffsetCommit | v0–v5 | 🔵 Phase 2 | +| 0 | Produce | v3–v13 | ✅ Critical | +| 1 | Fetch | v4–v18 | ✅ Critical | +| 2 | ListOffsets | v1–v11 | ✅ Critical | +| 3 | Metadata | v0–v13 | ✅ Critical | +| 8 | OffsetCommit | v2–v10 | ✅ Critical | +| 9 | OffsetFetch | v1–v10 | ✅ Critical | +| 10 | FindCoordinator | v0–v6 | ✅ Critical | +| 11 | JoinGroup | v0–v9 | ✅ Critical | +| 12 | Heartbeat | v0–v4 | ✅ Critical | +| 13 | LeaveGroup | v0–v5 | ✅ Critical | +| 14 | SyncGroup | v0–v5 | ✅ Critical | +| 15 | DescribeGroups | v0–v6 | 🟡 Important | +| 16 | ListGroups | v0–v5 | 🟡 Important | +| 17 | SaslHandshake | v0–v1 | 🟡 Important | +| 18 | ApiVersions | v0–v5 | ✅ Critical | +| 19 | CreateTopics | v2–v7 | ✅ Critical | +| 20 | DeleteTopics | v1–v6 | 🟡 Important | +| 21 | DeleteRecords | v0–v2 | 🔵 Phase 2 | +| 22 | InitProducerId | v0–v6 | 🔵 Phase 2 | +| 24 | AddPartitionsToTxn | v0–v5 | 🔵 Phase 2 | +| 25 | AddOffsetsToTxn | v0–v4 | 🔵 Phase 2 | +| 26 | EndTxn | v0–v5 | 🔵 Phase 2 | +| 28 | TxnOffsetCommit | v0–v5 | 🔵 Phase 2 | | 29–31 | ACL APIs | v1–v3 | 🔵 Phase 2 | -| 32 | DescribeConfigs | v1–v4 | 🟡 Important | -| 36 | SaslAuthenticate | v0–v2 | 🟡 Important | +| 32 | DescribeConfigs | v1–v4 | 🟡 Important | +| 36 | SaslAuthenticate | v0–v2 | 🟡 Important | | ... | 40+ more | various | 🔵 Phase 3 | --- ## Project Structure -``` +```text tools/kafka-tool/ ├── Cargo.toml ← package manifest and dependencies ├── src/ From ec57b1542fcd711969a3b59c6d9d34d265048ac4 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sat, 20 Jun 2026 22:53:56 -0400 Subject: [PATCH 14/57] Update Cargo.toml --- gateways/kafka/Cargo.toml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/gateways/kafka/Cargo.toml b/gateways/kafka/Cargo.toml index 88c0e8a2dc..20618711e2 100644 --- a/gateways/kafka/Cargo.toml +++ b/gateways/kafka/Cargo.toml @@ -35,7 +35,15 @@ path = "src/main.rs" [dependencies] bytes = { workspace = true } thiserror = { workspace = true } -tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "io-util", "time", "sync", "signal"] } +tokio = { workspace = true, features = [ + "rt-multi-thread", + "macros", + "net", + "io-util", + "time", + "sync", + "signal", +] } tokio-util = { workspace = true, features = ["rt"] } tracing = { workspace = true } tracing-subscriber = { workspace = true } From 82330597e342b9c02e578ce6fcba6cad2a470454 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sat, 20 Jun 2026 23:06:49 -0400 Subject: [PATCH 15/57] Fixing pre checks issues --- Cargo.lock | 2 -- gateways/kafka/tools/kafka-tool/Cargo.toml | 2 -- 2 files changed, 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d2066767b2..343ff2b624 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7683,8 +7683,6 @@ dependencies = [ "hex", "indexmap 2.14.0", "kafka-protocol", - "serde", - "serde_json", "tokio", "tracing", "tracing-subscriber", diff --git a/gateways/kafka/tools/kafka-tool/Cargo.toml b/gateways/kafka/tools/kafka-tool/Cargo.toml index a73d8addd9..41473e434a 100644 --- a/gateways/kafka/tools/kafka-tool/Cargo.toml +++ b/gateways/kafka/tools/kafka-tool/Cargo.toml @@ -36,8 +36,6 @@ clap = { workspace = true } hex = "0.4" indexmap = "2" kafka-protocol = "0.17" -serde = { workspace = true, features = ["derive"] } -serde_json = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } From 04f2df567f5c9ba9e5505d1494c175dc9749d0ad Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sat, 20 Jun 2026 23:37:51 -0400 Subject: [PATCH 16/57] Fixing the test fixtures as part of pre merge flow and updating the documentation. --- .github/actions/rust/pre-merge/action.yml | 12 ++++- gateways/kafka/README.md | 18 ++++++-- gateways/kafka/docs/TEST_SUITE.md | 6 +-- gateways/kafka/scripts/ci-wire-fixtures.sh | 53 ++++++++++++++++++++++ 4 files changed, 79 insertions(+), 10 deletions(-) create mode 100755 gateways/kafka/scripts/ci-wire-fixtures.sh diff --git a/.github/actions/rust/pre-merge/action.yml b/.github/actions/rust/pre-merge/action.yml index 536e3056e0..ea3618cb9f 100644 --- a/.github/actions/rust/pre-merge/action.yml +++ b/.github/actions/rust/pre-merge/action.yml @@ -49,7 +49,7 @@ runs: # Light legs fit in the runner's ~89 GiB default headroom; skip the # ~20-45s reclaim. Disk-heavy legs (coverage build + testcontainers # images on test-*, cross-builds, miri, verify-publish) keep it. - free-disk-space: ${{ (inputs.task == 'fmt' || inputs.task == 'sort' || inputs.task == 'clippy' || inputs.task == 'check' || inputs.task == 'check-msrv' || inputs.task == 'machete' || inputs.task == 'doctest') && 'false' || 'true' }} + free-disk-space: ${{ (inputs.task == 'fmt' || inputs.task == 'sort' || inputs.task == 'clippy' || inputs.task == 'check' || inputs.task == 'check-msrv' || inputs.task == 'machete' || inputs.task == 'doctest' || (startsWith(inputs.task, 'test-') && inputs.component == 'rust-gateway')) && 'false' || 'true' }} - name: Install cargo-sort if: inputs.task == 'sort' @@ -199,6 +199,11 @@ runs: with: tool: cargo-llvm-cov + - name: Generate Kafka gateway wire fixtures + if: startsWith(inputs.task, 'test-') && inputs.component == 'rust-gateway' + run: ./gateways/kafka/scripts/ci-wire-fixtures.sh generate + shell: bash + - name: Build and test with coverage if: startsWith(inputs.task, 'test-') run: | @@ -328,6 +333,11 @@ runs: ls -la codecov.json shell: bash + - name: Remove Kafka gateway wire fixtures + if: always() && startsWith(inputs.task, 'test-') && inputs.component == 'rust-gateway' + run: ./gateways/kafka/scripts/ci-wire-fixtures.sh cleanup + shell: bash + - name: Backwards compatibility check if: inputs.task == 'compat' && (github.event_name != 'pull_request' || !contains(join(github.event.pull_request.labels.*.name, ','), 'breaking:storage')) run: | diff --git a/gateways/kafka/README.md b/gateways/kafka/README.md index 6c1e2d608f..3843706888 100644 --- a/gateways/kafka/README.md +++ b/gateways/kafka/README.md @@ -24,15 +24,23 @@ cargo test -p iggy-gateway-kafka 103 regression tests across 12 suites — see [docs/TEST_SUITE.md](docs/TEST_SUITE.md) for the full catalog. -`decode_validation_tests` require wire fixtures under `tools/kafka-tool/kafka_messages/`: +`decode_validation_tests` require wire fixtures under `tools/kafka-tool/kafka_messages/` (gitignored locally; CI generates them via `scripts/ci-wire-fixtures.sh`): ```bash -cargo run -p kafka-message-gen -- generate \ - --output gateways/kafka/tools/kafka-tool/kafka_messages \ - --api-key 0 --api-key 1 --api-key 2 --api-key 19 +./gateways/kafka/scripts/ci-wire-fixtures.sh generate +cargo test -p iggy-gateway-kafka +./gateways/kafka/scripts/ci-wire-fixtures.sh cleanup # optional ``` -(Run from workspace root; adjust paths if needed.) +Or generate only the keys the tests need: + +```bash +for key in 0 1 2 19; do + cargo run -p kafka-message-gen -- generate \ + --output gateways/kafka/tools/kafka-tool/kafka_messages \ + --api-key "$key" +done +``` ## Manual testing diff --git a/gateways/kafka/docs/TEST_SUITE.md b/gateways/kafka/docs/TEST_SUITE.md index 27ed05a6ba..74dd995268 100644 --- a/gateways/kafka/docs/TEST_SUITE.md +++ b/gateways/kafka/docs/TEST_SUITE.md @@ -13,12 +13,10 @@ cargo test -p iggy-gateway-kafka ### Wire fixtures (required for `decode_validation_tests` and some handler tests) ```bash -cargo run -p kafka-message-gen -- generate \ - --output gateways/kafka/tools/kafka-tool/kafka_messages \ - --api-key 0 --api-key 1 --api-key 2 --api-key 19 +./gateways/kafka/scripts/ci-wire-fixtures.sh generate ``` -Fixtures are gitignored under `tools/kafka-tool/kafka_messages/`. Tests that need them skip gracefully when a fixture file is missing (`handler_regression_tests`) or panic with a clear path (`decode_validation_tests`). +Fixtures are gitignored under `tools/kafka-tool/kafka_messages/`. CI runs the same script before `rust-gateway` test jobs and removes the directory afterward. Tests that need fixtures skip gracefully when a file is missing (`handler_regression_tests`) or panic with a clear path (`decode_validation_tests`). --- diff --git a/gateways/kafka/scripts/ci-wire-fixtures.sh b/gateways/kafka/scripts/ci-wire-fixtures.sh new file mode 100755 index 0000000000..0cdb87aefd --- /dev/null +++ b/gateways/kafka/scripts/ci-wire-fixtures.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Generate or remove gitignored kafka-tool wire fixtures for CI. +# Run from the iggy workspace root. + +set -euo pipefail + +FIXTURES_DIR="gateways/kafka/tools/kafka-tool/kafka_messages" + +# API keys exercised by decode_validation_tests and handler_regression_tests. +FIXTURE_API_KEYS=(0 1 2 19) + +usage() { + echo "Usage: $0 {generate|cleanup}" >&2 + exit 2 +} + +generate() { + mkdir -p "$FIXTURES_DIR" + for key in "${FIXTURE_API_KEYS[@]}"; do + cargo run --locked -p kafka-message-gen -- generate \ + --output "$FIXTURES_DIR" \ + --api-key "$key" + done + echo "Generated wire fixtures under ${FIXTURES_DIR}/" +} + +cleanup() { + rm -rf "$FIXTURES_DIR" + echo "Removed ${FIXTURES_DIR}/" +} + +case "${1:-}" in + generate) generate ;; + cleanup) cleanup ;; + *) usage ;; +esac From 7f30105868ece13da08fe47d67b71732ddfb6177 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sat, 20 Jun 2026 23:48:13 -0400 Subject: [PATCH 17/57] Update action.yml --- .github/actions/rust/pre-merge/action.yml | 26 +++++++++++++---------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/.github/actions/rust/pre-merge/action.yml b/.github/actions/rust/pre-merge/action.yml index ea3618cb9f..c1180fed18 100644 --- a/.github/actions/rust/pre-merge/action.yml +++ b/.github/actions/rust/pre-merge/action.yml @@ -49,7 +49,7 @@ runs: # Light legs fit in the runner's ~89 GiB default headroom; skip the # ~20-45s reclaim. Disk-heavy legs (coverage build + testcontainers # images on test-*, cross-builds, miri, verify-publish) keep it. - free-disk-space: ${{ (inputs.task == 'fmt' || inputs.task == 'sort' || inputs.task == 'clippy' || inputs.task == 'check' || inputs.task == 'check-msrv' || inputs.task == 'machete' || inputs.task == 'doctest' || (startsWith(inputs.task, 'test-') && inputs.component == 'rust-gateway')) && 'false' || 'true' }} + free-disk-space: ${{ (inputs.task == 'fmt' || inputs.task == 'sort' || inputs.task == 'clippy' || inputs.task == 'check' || inputs.task == 'check-msrv' || inputs.task == 'machete' || inputs.task == 'doctest') && 'false' || 'true' }} - name: Install cargo-sort if: inputs.task == 'sort' @@ -199,11 +199,6 @@ runs: with: tool: cargo-llvm-cov - - name: Generate Kafka gateway wire fixtures - if: startsWith(inputs.task, 'test-') && inputs.component == 'rust-gateway' - run: ./gateways/kafka/scripts/ci-wire-fixtures.sh generate - shell: bash - - name: Build and test with coverage if: startsWith(inputs.task, 'test-') run: | @@ -274,6 +269,20 @@ runs: compile_duration=$((compile_end - compile_start)) echo "::notice::Tests compiled in ${compile_duration}s ($(date -ud @${compile_duration} +'%M:%S'))" + # decode_validation_tests need gitignored wire fixtures. Generate when + # iggy-gateway-kafka is in the DAG test scope (rust-gateway job or parent + # rust job — both run gateway tests when gateways/** changes). + NEEDS_KAFKA_FIXTURES=false + if [[ -z "$NEXTEST_FILTER" ]]; then + NEEDS_KAFKA_FIXTURES=true + elif grep -q 'package(iggy-gateway-kafka)' <<< "$NEXTEST_FILTER"; then + NEEDS_KAFKA_FIXTURES=true + fi + if [[ "$NEEDS_KAFKA_FIXTURES" == true ]]; then + ./gateways/kafka/scripts/ci-wire-fixtures.sh generate + trap './gateways/kafka/scripts/ci-wire-fixtures.sh cleanup' EXIT + fi + # Start D-Bus and unlock keyring right before test execution to avoid # gnome-keyring auto-locking the collection during the build phase. # Previously this ran before `cargo build`, leaving a 7+ minute idle @@ -333,11 +342,6 @@ runs: ls -la codecov.json shell: bash - - name: Remove Kafka gateway wire fixtures - if: always() && startsWith(inputs.task, 'test-') && inputs.component == 'rust-gateway' - run: ./gateways/kafka/scripts/ci-wire-fixtures.sh cleanup - shell: bash - - name: Backwards compatibility check if: inputs.task == 'compat' && (github.event_name != 'pull_request' || !contains(join(github.event.pull_request.labels.*.name, ','), 'breaking:storage')) run: | From 3b53c16e8b2a4061783ff838e2d413df65538c3a Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Thu, 2 Jul 2026 19:37:27 -0400 Subject: [PATCH 18/57] Add Kafka gateway TCP test suites and helpers based on the scope.md file Add comprehensive Kafka gateway tests and wire helpers: introduce multiple new test modules (listener_robustness_tests, review_regression_tests, scope_coverage_tests) and a wire.rs builder for Kafka wire formats. Extend common test utilities: spawn_test_server now delegates to spawn_test_server_with_config (which binds an ephemeral port and overwrites bind_addr), and tcp.rs gains several helper utilities (produce/metadata builders, timeout-aware readers, frame concatenation, byte read helper). These changes expand coverage for framing, pipelining, version/firewall behavior, metadata/produce/list_offsets semantics, and regression scenarios. Also add a CLI test stub under tools/kafka-tool. Tests exercise protocol conformance and TCP listener robustness across scoped API versions. --- gateways/kafka/tests/common/server.rs | 24 +- gateways/kafka/tests/common/tcp.rs | 56 ++ gateways/kafka/tests/common/wire.rs | 166 ++++ .../kafka/tests/listener_robustness_tests.rs | 406 +++++++++ .../kafka/tests/review_regression_tests.rs | 357 ++++++++ gateways/kafka/tests/scope_coverage_tests.rs | 770 ++++++++++++++++++ .../kafka-tool/tests/generate_cli_tests.rs | 49 ++ 7 files changed, 1820 insertions(+), 8 deletions(-) create mode 100644 gateways/kafka/tests/common/wire.rs create mode 100644 gateways/kafka/tests/listener_robustness_tests.rs create mode 100644 gateways/kafka/tests/review_regression_tests.rs create mode 100644 gateways/kafka/tests/scope_coverage_tests.rs create mode 100644 gateways/kafka/tools/kafka-tool/tests/generate_cli_tests.rs diff --git a/gateways/kafka/tests/common/server.rs b/gateways/kafka/tests/common/server.rs index a009b46423..e28c28a409 100644 --- a/gateways/kafka/tests/common/server.rs +++ b/gateways/kafka/tests/common/server.rs @@ -27,19 +27,27 @@ use iggy_gateway_kafka::{KafkaServer, ServerConfig}; /// Bind an ephemeral port, start `KafkaServer`, return address + shutdown sender. pub async fn spawn_test_server() -> (SocketAddr, broadcast::Sender<()>) { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0") - .await - .expect("bind ephemeral port"); - let addr = listener.local_addr().expect("local addr"); - - let config = ServerConfig { - bind_addr: addr.to_string(), + spawn_test_server_with_config(ServerConfig { + bind_addr: String::new(), advertised_host: None, advertised_port: None, max_frame_size: 8 * 1024 * 1024, read_timeout: Duration::from_secs(5), write_timeout: Duration::from_secs(5), - }; + }) + .await +} + +/// Start `KafkaServer` with explicit config (`bind_addr` overwritten with ephemeral port). +pub async fn spawn_test_server_with_config( + mut config: ServerConfig, +) -> (SocketAddr, broadcast::Sender<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind ephemeral port"); + let addr = listener.local_addr().expect("local addr"); + + config.bind_addr = addr.to_string(); let (shutdown_tx, shutdown_rx) = broadcast::channel(1); let server = KafkaServer::new(config); tokio::spawn(async move { diff --git a/gateways/kafka/tests/common/tcp.rs b/gateways/kafka/tests/common/tcp.rs index 6eea90f237..c449305962 100644 --- a/gateways/kafka/tests/common/tcp.rs +++ b/gateways/kafka/tests/common/tcp.rs @@ -19,10 +19,12 @@ #![allow(dead_code)] use std::net::SocketAddr; +use std::time::Duration; use bytes::{BufMut, Bytes, BytesMut}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpStream; +use tokio::time; use iggy_gateway_kafka::protocol::codec::Decoder; use iggy_gateway_kafka::protocol::header::{request_header_version, response_header_version}; @@ -88,6 +90,60 @@ pub async fn read_response_frame(stream: &mut TcpStream, max_size: usize) -> Byt Bytes::from(buf) } +/// Minimal Produce v3 body: nullable transactional_id, acks, timeout, empty topics array. +pub fn build_produce_v3_body(acks: i16, topics_count: i32) -> Bytes { + let mut body = BytesMut::new(); + body.put_i16(-1); // null transactional_id + body.put_i16(acks); + body.put_i32(1_000); // timeout_ms + body.put_i32(topics_count); + body.freeze() +} + +/// Legacy Metadata request body listing topic names (non-flexible, v0–v8). +pub fn build_metadata_legacy_request(topic_names: &[&str]) -> Bytes { + let mut body = BytesMut::new(); + body.put_i32(i32::try_from(topic_names.len()).expect("topic name count fits i32")); + for name in topic_names { + let name_bytes = name.as_bytes(); + let len = i16::try_from(name_bytes.len()).expect("topic name fits i16"); + body.put_i16(len); + body.extend_from_slice(name_bytes); + } + body.freeze() +} + +/// Read one length-prefixed response frame, returning `None` on timeout. +pub async fn read_response_frame_with_timeout( + stream: &mut TcpStream, + max_size: usize, + timeout: Duration, +) -> Option { + match time::timeout(timeout, read_response_frame(stream, max_size)).await { + Ok(frame) => Some(frame), + Err(_) => None, + } +} + +/// Concatenate multiple length-prefixed frames (for pipelining tests). +pub fn concat_frames(frames: &[Bytes]) -> Bytes { + let total: usize = frames.iter().map(Bytes::len).sum(); + let mut out = BytesMut::with_capacity(total); + for frame in frames { + out.extend_from_slice(frame); + } + out.freeze() +} + +/// Read one byte or return `None` on EOF / timeout. +pub async fn read_byte_with_timeout(stream: &mut TcpStream, timeout: Duration) -> Option { + let mut buf = [0u8; 1]; + match time::timeout(timeout, stream.read_exact(&mut buf)).await { + Ok(Ok(_)) => Some(buf[0]), + _ => None, + } +} + /// Send one request frame and return parsed `(correlation_id, response_body)`. pub async fn round_trip( addr: SocketAddr, diff --git a/gateways/kafka/tests/common/wire.rs b/gateways/kafka/tests/common/wire.rs new file mode 100644 index 0000000000..9507602e1a --- /dev/null +++ b/gateways/kafka/tests/common/wire.rs @@ -0,0 +1,166 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Kafka wire request builders aligned with SCOPE.md / protocol spec. +#![allow(dead_code)] + +use bytes::Bytes; + +use iggy_gateway_kafka::protocol::codec::Encoder; + +/// Consumer-group and admin keys explicitly out of scope in SCOPE.md. +pub const OUT_OF_SCOPE_API_KEYS: &[(i16, &str)] = &[ + (8, "OffsetCommit"), + (9, "OffsetFetch"), + (10, "FindCoordinator"), + (11, "JoinGroup"), + (12, "Heartbeat"), + (13, "LeaveGroup"), + (14, "SyncGroup"), + (15, "DescribeGroups"), + (16, "ListGroups"), + (17, "SaslHandshake"), + (20, "DeleteTopics"), +]; + +/// Flexible-encoding boundary per SCOPE.md valid-versions table. +pub const FLEXIBLE_FROM_VERSION: &[(i16, i16)] = &[ + (0, 9), // Produce + (1, 12), // Fetch + (2, 6), // ListOffsets + (3, 9), // Metadata + (18, 3), // ApiVersions + (19, 5), // CreateTopics +]; + +/// Metadata v9+ flexible request listing topic names (compact strings). +pub fn build_metadata_flexible_request(topic_names: &[&str]) -> Bytes { + let mut enc = Encoder::with_capacity(64); + enc.write_varint((topic_names.len() + 1) as u64); + for name in topic_names { + enc.write_compact_nullable_string(Some(name)); + } + enc.write_empty_tagged_fields(); + enc.freeze() +} + +/// Minimal ListOffsets request for supported versions (v1–v6). +pub fn build_list_offsets_request(version: i16, topic: &str, partition: i32) -> Bytes { + let flexible = version >= 6; + let mut enc = Encoder::with_capacity(128); + enc.write_i32(-1); // replica_id + if version >= 2 { + enc.write_i8(0); // isolation_level + } + + if flexible { + enc.write_varint(2); // one topic (N+1) + enc.write_compact_nullable_string(Some(topic)); + enc.write_varint(2); // one partition + } else { + enc.write_i32(1); + enc.write_nullable_string(Some(topic)) + .expect("topic name fits"); + enc.write_i32(1); + } + + enc.write_i32(partition); + if version >= 4 { + enc.write_i32(-1); // current_leader_epoch + } + enc.write_i64(-1); // latest timestamp + + if flexible { + enc.write_empty_tagged_fields(); // partition tagged fields + enc.write_empty_tagged_fields(); // topic tagged fields + enc.write_empty_tagged_fields(); // request tagged fields + } + + enc.freeze() +} + +/// CreateTopics v2+ with zero topics (valid empty create). +pub fn build_create_topics_empty_request(version: i16) -> Bytes { + let flexible = version >= 5; + let mut enc = Encoder::with_capacity(32); + + if flexible { + enc.write_varint(1); // empty topics compact array (N+1 = 1) + } else { + enc.write_i32(0); + } + enc.write_i32(5_000); // timeout_ms + if version >= 1 { + enc.write_bool(false); // validate_only + } + if flexible { + enc.write_empty_tagged_fields(); + } + + enc.freeze() +} + +/// Produce v9+ flexible request with empty topics array. +pub fn build_produce_flexible_empty_request(acks: i16) -> Bytes { + let mut enc = Encoder::with_capacity(32); + enc.write_compact_nullable_string(None); // null transactional_id + enc.write_i16(acks); + enc.write_i32(1_000); // timeout_ms + enc.write_varint(1); // empty topics compact array (N+1) + enc.write_empty_tagged_fields(); + enc.freeze() +} + +/// Fetch v4+ minimal empty-topic request. +pub fn build_fetch_empty_topics_request(version: i16) -> Bytes { + let flexible = version >= 12; + let mut enc = Encoder::with_capacity(64); + + enc.write_i32(-1); // replica_id + enc.write_i32(100); // max_wait_ms + enc.write_i32(1); // min_bytes + if version >= 3 { + enc.write_i32(i32::MAX); // max_bytes + } + if version >= 4 { + enc.write_i8(0); // isolation_level + } + if version >= 7 { + enc.write_i32(0); // session_id + enc.write_i32(0); // session_epoch + } + + if flexible { + enc.write_varint(1); // empty topics compact array + } else { + enc.write_i32(0); + } + + if version >= 7 { + if flexible { + enc.write_varint(1); // empty forgotten_topics_data + } else { + enc.write_i32(0); + } + } + + if flexible { + enc.write_empty_tagged_fields(); + } + + enc.freeze() +} diff --git a/gateways/kafka/tests/listener_robustness_tests.rs b/gateways/kafka/tests/listener_robustness_tests.rs new file mode 100644 index 0000000000..13ecd53c3a --- /dev/null +++ b/gateways/kafka/tests/listener_robustness_tests.rs @@ -0,0 +1,406 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! TCP listener robustness — framing, pipelining, concurrency, edge cases. + +#[path = "common/server.rs"] +mod server; +#[path = "common/tcp.rs"] +mod tcp; +#[path = "common/wire.rs"] +mod wire; + +use std::time::Duration; + +use bytes::{BufMut, BytesMut}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpStream; +use tokio::time; + +use iggy_gateway_kafka::ServerConfig; +use iggy_gateway_kafka::protocol::api::API_KEY_API_VERSIONS; +use iggy_gateway_kafka::protocol::codec::Decoder; + +use server::{spawn_test_server, spawn_test_server_with_config}; +use tcp::{ + build_request_frame, concat_frames, parse_response_payload, read_byte_with_timeout, + read_response_frame, read_response_frame_with_timeout, +}; + +#[tokio::test] +async fn e2e_pipelined_requests_receive_responses_in_order() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.expect("connect"); + + let frame1 = build_request_frame(API_KEY_API_VERSIONS, 1, 1, Some("pipe-test"), &[]); + let frame2 = build_request_frame(API_KEY_API_VERSIONS, 1, 2, Some("pipe-test"), &[]); + let frame3 = build_request_frame(API_KEY_API_VERSIONS, 1, 3, Some("pipe-test"), &[]); + stream + .write_all(&concat_frames(&[frame1, frame2, frame3])) + .await + .expect("pipelined write"); + + for expected_corr in 1..=3 { + let payload = read_response_frame(&mut stream, 8 * 1024 * 1024).await; + let (corr, body) = parse_response_payload(API_KEY_API_VERSIONS, 1, payload); + assert_eq!(corr, expected_corr); + assert_eq!(Decoder::new(body).read_i16().unwrap(), 0); + } +} + +#[tokio::test] +async fn e2e_partial_length_prefix_then_remainder_accepted() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.expect("connect"); + + let frame = build_request_frame(API_KEY_API_VERSIONS, 1, 42, Some("partial-test"), &[]); + assert!(frame.len() > 6, "test frame long enough to split"); + + stream.write_all(&frame[..2]).await.expect("partial prefix"); + time::sleep(Duration::from_millis(50)).await; + stream.write_all(&frame[2..]).await.expect("remainder"); + + let payload = read_response_frame(&mut stream, 8 * 1024 * 1024).await; + let (corr, _) = parse_response_payload(API_KEY_API_VERSIONS, 1, payload); + assert_eq!(corr, 42); +} + +#[tokio::test] +async fn e2e_frame_within_custom_max_frame_size_accepted() { + let max_frame = 512; + let (addr, _shutdown) = spawn_test_server_with_config(ServerConfig { + bind_addr: String::new(), + advertised_host: None, + advertised_port: None, + max_frame_size: max_frame, + read_timeout: Duration::from_secs(5), + write_timeout: Duration::from_secs(5), + }) + .await; + + let mut stream = TcpStream::connect(addr).await.expect("connect"); + let frame = build_request_frame(API_KEY_API_VERSIONS, 1, 55, Some("max-frame-test"), &[]); + assert!( + frame.len() <= max_frame, + "ApiVersions frame must fit test max ({max_frame})" + ); + + stream.write_all(&frame).await.expect("write"); + let payload = read_response_frame(&mut stream, max_frame).await; + assert_eq!( + parse_response_payload(API_KEY_API_VERSIONS, 1, payload).0, + 55 + ); +} + +#[tokio::test] +async fn e2e_frame_exceeding_max_frame_size_closes_connection() { + let max_frame = 64; + let (addr, _shutdown) = spawn_test_server_with_config(ServerConfig { + bind_addr: String::new(), + advertised_host: None, + advertised_port: None, + max_frame_size: max_frame, + read_timeout: Duration::from_secs(5), + write_timeout: Duration::from_secs(5), + }) + .await; + + let mut stream = TcpStream::connect(addr).await.expect("connect"); + let mut frame = BytesMut::new(); + frame.put_i32(200); + frame.resize(4 + 200, 0); + stream.write_all(&frame).await.expect("oversized frame"); + + let byte = read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await; + assert!( + byte.is_none(), + "oversized frame should close connection (EOF)" + ); +} + +#[tokio::test] +async fn e2e_truncated_frame_body_closes_connection() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.expect("connect"); + + let full = build_request_frame(API_KEY_API_VERSIONS, 1, 66, Some("trunc-test"), &[]); + let payload_len = i32::from_be_bytes([full[0], full[1], full[2], full[3]]) as usize; + assert!(full.len() >= 4 + payload_len); + + stream + .write_all(&full[..4 + payload_len / 2]) + .await + .expect("half body"); + + let byte = read_byte_with_timeout(&mut stream, Duration::from_secs(3)).await; + assert!(byte.is_none(), "truncated body should close connection"); +} + +#[tokio::test] +async fn e2e_multiple_concurrent_connections_are_independent() { + let (addr, _shutdown) = spawn_test_server().await; + + let (r1, r2, r3) = tokio::join!( + tcp::round_trip(addr, API_KEY_API_VERSIONS, 1, 101, &[]), + tcp::round_trip(addr, API_KEY_API_VERSIONS, 1, 102, &[]), + tcp::round_trip(addr, API_KEY_API_VERSIONS, 1, 103, &[]), + ); + + assert_eq!(r1.0, 101); + assert_eq!(r2.0, 102); + assert_eq!(r3.0, 103); +} + +#[tokio::test] +async fn e2e_client_disconnect_mid_frame_allows_new_connection() { + let (addr, _shutdown) = spawn_test_server().await; + + { + let mut stream = TcpStream::connect(addr).await.expect("connect"); + let full = build_request_frame(API_KEY_API_VERSIONS, 1, 77, Some("abort-test"), &[]); + stream.write_all(&full[..8]).await.expect("partial write"); + drop(stream); + } + + time::sleep(Duration::from_millis(100)).await; + + let (corr, body) = tcp::round_trip(addr, API_KEY_API_VERSIONS, 1, 78, &[]).await; + assert_eq!(corr, 78); + assert_eq!(Decoder::new(body).read_i16().unwrap(), 0); +} + +#[tokio::test] +async fn e2e_response_frames_have_positive_big_endian_length_prefix() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.expect("connect"); + + let frame = build_request_frame(API_KEY_API_VERSIONS, 3, 200, Some("len-test"), &[]); + stream.write_all(&frame).await.expect("write"); + + let mut len_buf = [0u8; 4]; + stream + .read_exact(&mut len_buf) + .await + .expect("length prefix"); + let len = i32::from_be_bytes(len_buf); + assert!(len > 0, "response length prefix must be positive"); + let body_len = usize::try_from(len).expect("positive length"); + let mut body = vec![0u8; body_len]; + stream.read_exact(&mut body).await.expect("response body"); + assert!(!body.is_empty()); +} + +#[tokio::test] +async fn e2e_zero_frame_length_closes_connection() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.expect("connect"); + + stream + .write_all(&0i32.to_be_bytes()) + .await + .expect("zero len"); + let byte = read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await; + assert!(byte.is_none(), "zero frame length must close connection"); +} + +#[tokio::test] +async fn e2e_negative_frame_length_closes_connection() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.expect("connect"); + + stream + .write_all(&(-5_i32).to_be_bytes()) + .await + .expect("negative len"); + let byte = read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await; + assert!( + byte.is_none(), + "negative frame length must close connection" + ); +} + +#[tokio::test] +async fn e2e_slow_client_can_complete_request_within_read_timeout() { + let (addr, _shutdown) = spawn_test_server_with_config(ServerConfig { + bind_addr: String::new(), + advertised_host: None, + advertised_port: None, + max_frame_size: 8 * 1024 * 1024, + read_timeout: Duration::from_secs(5), + write_timeout: Duration::from_secs(5), + }) + .await; + + let mut stream = TcpStream::connect(addr).await.expect("connect"); + let frame = build_request_frame(API_KEY_API_VERSIONS, 1, 301, Some("slow-test"), &[]); + + for chunk in frame.chunks(1) { + stream.write_all(chunk).await.expect("byte drip"); + time::sleep(Duration::from_millis(5)).await; + } + + let payload = + read_response_frame_with_timeout(&mut stream, 8 * 1024 * 1024, Duration::from_secs(3)) + .await + .expect("slow send should complete within read timeout"); + assert_eq!( + parse_response_payload(API_KEY_API_VERSIONS, 1, payload).0, + 301 + ); +} + +#[tokio::test] +async fn e2e_many_sequential_requests_on_one_connection() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.expect("connect"); + + for index in 0..20_i32 { + let corr = 1_000 + index; + let frame = build_request_frame(API_KEY_API_VERSIONS, 1, corr, Some("seq-stress"), &[]); + stream.write_all(&frame).await.expect("write"); + let payload = read_response_frame(&mut stream, 8 * 1024 * 1024).await; + assert_eq!( + parse_response_payload(API_KEY_API_VERSIONS, 1, payload).0, + corr + ); + } +} + +#[tokio::test] +async fn e2e_empty_client_id_request_succeeds() { + let (addr, _shutdown) = spawn_test_server().await; + let frame = build_request_frame(API_KEY_API_VERSIONS, 1, 400, None, &[]); + let mut stream = TcpStream::connect(addr).await.expect("connect"); + stream.write_all(&frame).await.expect("write"); + let payload = read_response_frame(&mut stream, 8 * 1024 * 1024).await; + assert_eq!( + parse_response_payload(API_KEY_API_VERSIONS, 1, payload).0, + 400 + ); +} + +#[tokio::test] +async fn e2e_flexible_apiversions_v3_request_succeeds() { + let (addr, _shutdown) = spawn_test_server().await; + let (corr, body) = tcp::round_trip(addr, API_KEY_API_VERSIONS, 3, 401, &[]).await; + assert_eq!(corr, 401); + let mut d = Decoder::new(body); + assert_eq!(d.read_i16().unwrap(), 0); + let count = usize::try_from(d.read_varint().unwrap() - 1).unwrap(); + assert_eq!(count, 6, "must advertise all six scoped API keys"); +} + +#[tokio::test] +async fn e2e_frame_payload_shorter_than_kafka_header_closes_connection() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.expect("connect"); + + let mut frame = BytesMut::new(); + frame.put_i32(4); + frame.extend_from_slice(&[0x00, 0x12, 0x00, 0x01]); + stream.write_all(&frame).await.expect("short payload"); + + let byte = read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await; + assert!( + byte.is_none(), + "payload shorter than 8-byte Kafka header must close connection" + ); +} + +#[tokio::test] +async fn e2e_mixed_api_key_pipeline_returns_responses_in_order() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.expect("connect"); + + use iggy_gateway_kafka::protocol::api::{API_KEY_METADATA, API_KEY_PRODUCE}; + + let frames = [ + build_request_frame(API_KEY_API_VERSIONS, 1, 501, Some("mix-test"), &[]), + build_request_frame( + API_KEY_METADATA, + 0, + 502, + Some("mix-test"), + &tcp::build_metadata_legacy_request(&["pipe-topic"]), + ), + build_request_frame( + API_KEY_PRODUCE, + 3, + 503, + Some("mix-test"), + &tcp::build_produce_v3_body(1, 0), + ), + ]; + stream + .write_all(&concat_frames(&frames)) + .await + .expect("mixed pipeline write"); + + for (api_key, api_version, expected_corr) in [ + (API_KEY_API_VERSIONS, 1i16, 501), + (API_KEY_METADATA, 0, 502), + (API_KEY_PRODUCE, 3, 503), + ] { + let payload = read_response_frame(&mut stream, 8 * 1024 * 1024).await; + assert_eq!( + parse_response_payload(api_key, api_version, payload).0, + expected_corr + ); + } +} + +#[tokio::test] +async fn e2e_connection_idle_after_response_accepts_next_request() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.expect("connect"); + + let first = build_request_frame(API_KEY_API_VERSIONS, 1, 601, Some("idle-test"), &[]); + stream.write_all(&first).await.expect("first write"); + let payload = read_response_frame(&mut stream, 8 * 1024 * 1024).await; + assert_eq!( + parse_response_payload(API_KEY_API_VERSIONS, 1, payload).0, + 601 + ); + + time::sleep(Duration::from_secs(2)).await; + + let second = build_request_frame(API_KEY_API_VERSIONS, 1, 602, Some("idle-test"), &[]); + stream.write_all(&second).await.expect("second write"); + let payload = read_response_frame(&mut stream, 8 * 1024 * 1024).await; + assert_eq!( + parse_response_payload(API_KEY_API_VERSIONS, 1, payload).0, + 602, + "idle gap under read_timeout must not drop connection" + ); +} + +#[tokio::test] +async fn e2e_flexible_metadata_v9_empty_topics_round_trip() { + let (addr, _shutdown) = spawn_test_server().await; + use iggy_gateway_kafka::protocol::api::API_KEY_METADATA; + + let body = wire::build_metadata_flexible_request(&[]); + let (corr, resp) = tcp::round_trip(addr, API_KEY_METADATA, 9, 701, &body).await; + assert_eq!(corr, 701); + let mut d = Decoder::new(resp); + d.read_i32().unwrap(); + let broker_count = usize::try_from(d.read_varint().unwrap()) + .unwrap() + .saturating_sub(1); + assert_eq!(broker_count, 1); +} diff --git a/gateways/kafka/tests/review_regression_tests.rs b/gateways/kafka/tests/review_regression_tests.rs new file mode 100644 index 0000000000..b2cecb929b --- /dev/null +++ b/gateways/kafka/tests/review_regression_tests.rs @@ -0,0 +1,357 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Regression tests for PR #3519 review findings (atharvalade, Jul 2026). +//! +//! These encode Kafka-client-correct behavior. Several fail until the +//! corresponding protocol/server fixes land. + +#[path = "common/scope.rs"] +mod scope; +#[path = "common/server.rs"] +mod server; +#[path = "common/tcp.rs"] +mod tcp; + +use std::time::Duration; + +use bytes::{BufMut, Bytes, BytesMut}; +use tokio::io::AsyncWriteExt; +use tokio::net::TcpStream; +use tokio::time; + +use iggy_gateway_kafka::ServerConfig; +use iggy_gateway_kafka::protocol::api::{ + API_KEY_API_VERSIONS, API_KEY_LIST_OFFSETS, API_KEY_METADATA, API_KEY_PRODUCE, + ERROR_UNKNOWN_TOPIC_OR_PARTITION, ERROR_UNSUPPORTED_VERSION, handle_request, +}; +use iggy_gateway_kafka::protocol::codec::Decoder; + +use scope::default_broker; +use server::{spawn_test_server, spawn_test_server_with_config}; +use tcp::{ + build_metadata_legacy_request, build_produce_v3_body, build_request_frame, + parse_response_payload, read_response_frame_with_timeout, +}; + +// ── Produce acks=0 (review: broker must stay silent) ───────────────────────── + +#[tokio::test] +async fn e2e_produce_v3_acks_zero_sends_no_response() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.expect("connect"); + + let body = build_produce_v3_body(0, 0); + let frame = build_request_frame(API_KEY_PRODUCE, 3, 42, Some("review-test"), &body); + stream + .write_all(&frame) + .await + .expect("write produce acks=0"); + + let response = + read_response_frame_with_timeout(&mut stream, 8 * 1024 * 1024, Duration::from_millis(500)) + .await; + + assert!( + response.is_none(), + "Produce with acks=0 must not receive a response frame (Kafka spec); got {} bytes", + response.as_ref().map_or(0, Bytes::len) + ); +} + +#[tokio::test] +async fn e2e_produce_v3_acks_one_still_returns_response() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.expect("connect"); + + let body = build_produce_v3_body(1, 0); + let frame = build_request_frame(API_KEY_PRODUCE, 3, 43, Some("review-test"), &body); + stream + .write_all(&frame) + .await + .expect("write produce acks=1"); + + let response = + read_response_frame_with_timeout(&mut stream, 8 * 1024 * 1024, Duration::from_secs(2)) + .await + .expect("Produce with acks=1 should receive a response"); + + let (corr, resp_body) = parse_response_payload(API_KEY_PRODUCE, 3, response); + assert_eq!(corr, 43); + assert!(!resp_body.is_empty()); +} + +// ── ListOffsets v0 wire shape (review: old_style_offsets array, not bare i64) ─ + +/// Parse one ListOffsets v0 partition entry the way a v0 Kafka client would. +fn parse_list_offsets_v0_partition(d: &mut Decoder) { + let _partition_index = d.read_i32().expect("partition_index"); + let _error_code = d.read_i16().expect("error_code"); + let offset_count = d.read_i32().expect("old_style_offsets array length"); + assert!( + offset_count >= 0, + "old_style_offsets count must be non-negative, got {offset_count}" + ); + for _ in 0..offset_count { + d.read_i64().expect("old_style_offsets entry"); + } +} + +#[test] +fn list_offsets_v0_unsupported_version_is_parseable_by_v0_clients() { + let body = handle_request(API_KEY_LIST_OFFSETS, 0, Bytes::new(), &default_broker()); + let mut d = Decoder::new(body); + + assert_eq!(d.read_i32().unwrap(), 1, "topics array length"); + assert_eq!( + d.read_nullable_string().unwrap(), + Some(String::new()), + "placeholder topic name" + ); + assert_eq!(d.read_i32().unwrap(), 1, "partitions array length"); + + parse_list_offsets_v0_partition(&mut d); + + assert_eq!( + d.remaining(), + 0, + "v0 client must consume the full error response without trailing bytes" + ); +} + +#[test] +fn list_offsets_v0_unsupported_version_carries_error_code_in_partition() { + let request_body = build_list_offsets_v0_request_with_topic_t(); + let body = handle_request(API_KEY_LIST_OFFSETS, 0, request_body, &default_broker()); + let mut d = Decoder::new(body); + + assert_eq!(d.read_i32().unwrap(), 1); + d.read_nullable_string().unwrap(); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_i32().unwrap(), 0, "partition index"); + assert_eq!( + d.read_i16().unwrap(), + ERROR_UNSUPPORTED_VERSION, + "partition error code" + ); + + parse_list_offsets_v0_partition(&mut d); + assert_eq!(d.remaining(), 0); +} + +/// ListOffsets v0 request below firewall min (mirrors atharvalade repro script body). +fn build_list_offsets_v0_request_with_topic_t() -> Bytes { + let mut body = BytesMut::new(); + body.put_i32(-1); // replica_id + body.put_i32(1); // topics array length + body.put_i16(1); // topic name length + body.put_u8(b't'); + body.put_i32(1); // partitions array length + body.put_i32(0); // partition index + body.put_i64(-1); // timestamp + body.put_i32(1); // max_num_offsets + body.freeze() +} + +#[tokio::test] +async fn e2e_list_offsets_v0_unsupported_version_no_trailing_bytes() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.expect("connect"); + + let request_body = build_list_offsets_v0_request_with_topic_t(); + let frame = build_request_frame( + API_KEY_LIST_OFFSETS, + 0, + 7, + Some("review-test"), + &request_body, + ); + stream + .write_all(&frame) + .await + .expect("write list offsets v0"); + + let payload = + read_response_frame_with_timeout(&mut stream, 8 * 1024 * 1024, Duration::from_secs(2)) + .await + .expect("ListOffsets v0 should still get an error response"); + + let (_corr, body) = parse_response_payload(API_KEY_LIST_OFFSETS, 0, payload); + let mut d = Decoder::new(body); + assert_eq!(d.read_i32().unwrap(), 1); + d.read_nullable_string().unwrap(); + assert_eq!(d.read_i32().unwrap(), 1); + parse_list_offsets_v0_partition(&mut d); + assert_eq!(d.remaining(), 0); +} + +// ── Metadata topic name echo (review: must not hardcode "unknown-topic") ──── + +fn read_metadata_v1_topics(d: &mut Decoder, expected_count: i32) -> Vec { + let _brokers_count = d.read_i32().unwrap(); + d.read_i32().unwrap(); // node_id + d.read_nullable_string().unwrap(); // host + d.read_i32().unwrap(); // port + d.read_nullable_string().unwrap(); // rack (v1+) + d.read_i32().unwrap(); // controller_id (v1+) + + assert_eq!(d.read_i32().unwrap(), expected_count); + let mut names = Vec::with_capacity(usize::try_from(expected_count).unwrap_or(0)); + for _ in 0..expected_count { + d.read_i16().unwrap(); // topic_error + names.push(d.read_nullable_string().unwrap().expect("topic name")); + d.read_bool().unwrap(); // is_internal (v1+) + assert_eq!(d.read_i32().unwrap(), 0, "empty partitions array"); + } + names +} + +#[test] +fn metadata_v1_echoes_requested_topic_name_in_response() { + let topic = "orders"; + let request = build_metadata_legacy_request(&[topic]); + let body = handle_request(API_KEY_METADATA, 1, request, &default_broker()); + let mut d = Decoder::new(body); + + let names = read_metadata_v1_topics(&mut d, 1); + assert_eq!(names, vec![topic.to_string()]); + assert_eq!(d.remaining(), 0); +} + +#[test] +fn metadata_v1_unknown_topic_returns_error_with_requested_name() { + let topic = "orders"; + let request = build_metadata_legacy_request(&[topic]); + let body = handle_request(API_KEY_METADATA, 1, request, &default_broker()); + let mut d = Decoder::new(body); + + let _brokers_count = d.read_i32().unwrap(); + d.read_i32().unwrap(); + d.read_nullable_string().unwrap(); + d.read_i32().unwrap(); + d.read_nullable_string().unwrap(); + d.read_i32().unwrap(); + + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!( + d.read_i16().unwrap(), + ERROR_UNKNOWN_TOPIC_OR_PARTITION, + "unknown topic should surface error 3" + ); + assert_eq!( + d.read_nullable_string().unwrap().as_deref(), + Some(topic), + "response must echo requested topic name, not a placeholder" + ); +} + +#[tokio::test] +async fn e2e_metadata_v1_response_contains_requested_topic_name() { + let (addr, _shutdown) = spawn_test_server().await; + let topic = "orders"; + let request_body = build_metadata_legacy_request(&[topic]); + let frame = build_request_frame(API_KEY_METADATA, 1, 9, Some("review-test"), &request_body); + + let mut stream = TcpStream::connect(addr).await.expect("connect"); + stream.write_all(&frame).await.expect("write metadata v1"); + + let payload = + read_response_frame_with_timeout(&mut stream, 8 * 1024 * 1024, Duration::from_secs(2)) + .await + .expect("metadata response"); + + let full_response = { + let mut framed = BytesMut::with_capacity(4 + payload.len()); + framed.put_i32(i32::try_from(payload.len()).expect("metadata response fits i32")); + framed.extend_from_slice(&payload); + framed.freeze() + }; + + assert!( + full_response + .windows(topic.len()) + .any(|window| window == topic.as_bytes()), + "metadata response must contain requested topic name {topic:?}; \ + placeholder-only responses break client topic matching" + ); + assert!( + !full_response + .windows(b"unknown-topic".len()) + .any(|window| window == b"unknown-topic"), + "metadata response must not substitute unknown-topic for requested names" + ); +} + +// ── Idle connection handling (review: read_timeout must not act as idle cap) ─ + +#[tokio::test] +async fn e2e_quiet_connection_accepts_request_after_short_idle() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.expect("connect"); + + time::sleep(Duration::from_secs(2)).await; + + let frame = build_request_frame(API_KEY_API_VERSIONS, 1, 501, Some("idle-test"), &[]); + stream + .write_all(&frame) + .await + .expect("connection should stay open after short idle"); + + let payload = + read_response_frame_with_timeout(&mut stream, 8 * 1024 * 1024, Duration::from_secs(2)) + .await + .expect("request after short idle should succeed"); + + let (corr, _) = parse_response_payload(API_KEY_API_VERSIONS, 1, payload); + assert_eq!(corr, 501); +} + +#[tokio::test] +async fn e2e_quiet_connection_survives_beyond_read_timeout_idle_cap() { + let (addr, _shutdown) = spawn_test_server_with_config(ServerConfig { + bind_addr: String::new(), + advertised_host: None, + advertised_port: None, + max_frame_size: 8 * 1024 * 1024, + read_timeout: Duration::from_secs(3), + write_timeout: Duration::from_secs(5), + }) + .await; + + let mut stream = TcpStream::connect(addr).await.expect("connect"); + + // Longer than read_timeout: today the server closes idle connections here. + // After fix (separate idle timeout), the connection should remain usable. + time::sleep(Duration::from_secs(4)).await; + + let frame = build_request_frame(API_KEY_API_VERSIONS, 1, 502, Some("idle-test"), &[]); + stream + .write_all(&frame) + .await + .expect("idle connection should remain writable after read_timeout elapses"); + + let payload = + read_response_frame_with_timeout(&mut stream, 8 * 1024 * 1024, Duration::from_secs(2)) + .await + .expect( + "request after idle period longer than read_timeout should succeed once \ + idle timeout is decoupled from frame read timeout", + ); + + let (corr, _) = parse_response_payload(API_KEY_API_VERSIONS, 1, payload); + assert_eq!(corr, 502); +} diff --git a/gateways/kafka/tests/scope_coverage_tests.rs b/gateways/kafka/tests/scope_coverage_tests.rs new file mode 100644 index 0000000000..c638d2c109 --- /dev/null +++ b/gateways/kafka/tests/scope_coverage_tests.rs @@ -0,0 +1,770 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Comprehensive scoped-API coverage per SCOPE.md and Kafka protocol spec. +//! +//! Fills gaps not covered by existing regression suites. Some tests encode +//! client-correct behavior and fail until implementation catches up. + +#[path = "common/fixtures.rs"] +mod fixtures; +#[path = "common/scope.rs"] +mod scope; +#[path = "common/server.rs"] +mod server; +#[path = "common/tcp.rs"] +mod tcp; +#[path = "common/wire.rs"] +mod wire; + +use bytes::{BufMut, Bytes, BytesMut}; +use tokio::io::AsyncWriteExt; +use tokio::net::TcpStream; + +use iggy_gateway_kafka::protocol::api::{ + API_KEY_API_VERSIONS, API_KEY_CREATE_TOPICS, API_KEY_FETCH, API_KEY_LIST_OFFSETS, + API_KEY_METADATA, API_KEY_PRODUCE, ERROR_INVALID_REQUEST, ERROR_NONE, + ERROR_UNKNOWN_TOPIC_OR_PARTITION, ERROR_UNSUPPORTED_VERSION, advertised_min_version, + handle_request, is_supported_version, +}; +use iggy_gateway_kafka::protocol::codec::Decoder; + +use fixtures::{fixture_exists, load_fixture_body}; +use scope::{SCOPED_API_KEYS, default_broker}; +use server::spawn_test_server; +use tcp::{ + build_metadata_legacy_request, build_produce_v3_body, build_request_frame, + parse_response_payload, round_trip, +}; +use wire::{ + OUT_OF_SCOPE_API_KEYS, build_create_topics_empty_request, build_fetch_empty_topics_request, + build_list_offsets_request, build_metadata_flexible_request, build_produce_flexible_empty_request, +}; + +fn metadata_empty_legacy_body() -> Bytes { + let mut body = BytesMut::new(); + body.put_i32(0); + body.freeze() +} + +fn request_body_for_scoped_api(api_key: i16, name: &str, version: i16) -> Bytes { + match api_key { + API_KEY_API_VERSIONS => Bytes::new(), + API_KEY_METADATA => metadata_empty_legacy_body(), + API_KEY_PRODUCE => { + if fixture_exists(api_key, name, version) { + load_fixture_body(api_key, name, version) + } else { + build_produce_v3_body(1, 0) + } + } + API_KEY_FETCH => { + if fixture_exists(api_key, name, version) { + load_fixture_body(api_key, name, version) + } else { + build_fetch_empty_topics_request(version) + } + } + API_KEY_LIST_OFFSETS => { + if fixture_exists(api_key, name, version) { + load_fixture_body(api_key, name, version) + } else { + build_list_offsets_request(version, "scope-topic", 0) + } + } + API_KEY_CREATE_TOPICS => build_create_topics_empty_request(version), + _ => Bytes::new(), + } +} + +// ── Correlation ID preservation (all scoped keys × min/max/flexible) ──────── + +#[tokio::test] +async fn each_scoped_api_min_version_preserves_correlation_id_e2e() { + let (addr, _shutdown) = spawn_test_server().await; + + for &(api_key, name, min_ver, _max_ver) in SCOPED_API_KEYS { + let correlation_id = 10_000 + i32::from(api_key); + let body = request_body_for_scoped_api(api_key, name, min_ver); + let (corr, resp_body) = round_trip(addr, api_key, min_ver, correlation_id, &body).await; + assert_eq!( + corr, correlation_id, + "{name} v{min_ver} correlation id must round-trip" + ); + assert!( + !resp_body.is_empty(), + "{name} v{min_ver} must return non-empty body" + ); + } +} + +#[tokio::test] +async fn each_scoped_api_max_version_preserves_correlation_id_e2e() { + let (addr, _shutdown) = spawn_test_server().await; + + for &(api_key, name, _min_ver, max_ver) in SCOPED_API_KEYS { + let correlation_id = 20_000 + i32::from(api_key); + let body = request_body_for_scoped_api(api_key, name, max_ver); + let (corr, resp_body) = round_trip(addr, api_key, max_ver, correlation_id, &body).await; + assert_eq!( + corr, correlation_id, + "{name} v{max_ver} correlation id must round-trip" + ); + assert!( + !resp_body.is_empty(), + "{name} v{max_ver} must return non-empty body" + ); + } +} + +#[tokio::test] +async fn apiversions_v0_and_v2_e2e_return_success() { + let (addr, _shutdown) = spawn_test_server().await; + + for version in [0i16, 2] { + let correlation_id = 300 + i32::from(version); + let (corr, body) = + round_trip(addr, API_KEY_API_VERSIONS, version, correlation_id, &[]).await; + assert_eq!(corr, correlation_id); + let mut d = Decoder::new(body); + assert_eq!( + d.read_i16().unwrap(), + ERROR_NONE, + "ApiVersions v{version} must succeed" + ); + } +} + +// ── Out-of-scope API keys (SCOPE.md unsupported list) ─────────────────────── + +#[test] +fn out_of_scope_api_keys_return_unsupported_version_without_panic() { + for &(api_key, name) in OUT_OF_SCOPE_API_KEYS { + let body = handle_request(api_key, 0, Bytes::new(), &default_broker()); + let mut d = Decoder::new(body); + assert_eq!( + d.read_i16().unwrap(), + ERROR_UNSUPPORTED_VERSION, + "{name} (key {api_key})" + ); + } +} + +#[tokio::test] +async fn out_of_scope_api_keys_e2e_keep_connection_for_follow_up() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.expect("connect"); + + for &(api_key, _name) in &OUT_OF_SCOPE_API_KEYS[..4] { + let frame = build_request_frame(api_key, 0, i32::from(api_key), Some("scope-test"), &[]); + stream.write_all(&frame).await.expect("write oos key"); + let payload = tcp::read_response_frame(&mut stream, 8 * 1024 * 1024).await; + let mut d = Decoder::new(parse_response_payload(api_key, 0, payload).1); + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); + } + + let follow_up = build_request_frame(API_KEY_API_VERSIONS, 1, 99_999, Some("scope-test"), &[]); + stream.write_all(&follow_up).await.expect("follow-up write"); + let payload = tcp::read_response_frame(&mut stream, 8 * 1024 * 1024).await; + let (corr, body) = parse_response_payload(API_KEY_API_VERSIONS, 1, payload); + assert_eq!(corr, 99_999); + assert_eq!(Decoder::new(body).read_i16().unwrap(), ERROR_NONE); +} + +// ── Version firewall: unsupported version keeps TCP session ───────────────── + +#[tokio::test] +async fn each_scoped_api_above_max_version_e2e_keeps_connection() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.expect("connect"); + + for &(api_key, name, _min_ver, max_ver) in SCOPED_API_KEYS { + let above = max_ver + 1; + let frame = build_request_frame( + api_key, + above, + 50_000 + i32::from(api_key), + Some("scope-test"), + &[], + ); + stream + .write_all(&frame) + .await + .unwrap_or_else(|_| panic!("write {name} v{above}")); + let payload = tcp::read_response_frame(&mut stream, 8 * 1024 * 1024).await; + assert!( + !payload.is_empty(), + "{name} v{above} must still respond on wire" + ); + } + + let ok = build_request_frame(API_KEY_API_VERSIONS, 1, 89_999, Some("scope-test"), &[]); + stream.write_all(&ok).await.expect("recovery request"); + let payload = tcp::read_response_frame(&mut stream, 8 * 1024 * 1024).await; + assert_eq!( + parse_response_payload(API_KEY_API_VERSIONS, 1, payload).0, + 89_999 + ); +} + +#[tokio::test] +async fn each_scoped_api_below_min_version_e2e_keeps_connection() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.expect("connect"); + + for &(api_key, name, min_ver, _max_ver) in SCOPED_API_KEYS { + let below = min_ver - 1; + let frame = build_request_frame( + api_key, + below, + 40_000 + i32::from(api_key), + Some("scope-test"), + &[], + ); + stream + .write_all(&frame) + .await + .unwrap_or_else(|_| panic!("write {name} v{below}")); + let payload = tcp::read_response_frame(&mut stream, 8 * 1024 * 1024).await; + assert!( + !payload.is_empty(), + "{name} v{below} must still respond on wire" + ); + } + + let ok = build_request_frame(API_KEY_API_VERSIONS, 1, 88_888, Some("scope-test"), &[]); + stream.write_all(&ok).await.expect("recovery request"); + let payload = tcp::read_response_frame(&mut stream, 8 * 1024 * 1024).await; + assert_eq!( + parse_response_payload(API_KEY_API_VERSIONS, 1, payload).0, + 88_888 + ); +} + +#[test] +fn produce_advertises_min_zero_but_firewall_rejects_below_v3() { + let range = scope::SCOPED_API_KEYS + .iter() + .find(|(k, _, _, _)| *k == API_KEY_PRODUCE) + .expect("produce in scope"); + let (_, _, firewall_min, _) = *range; + assert_eq!(firewall_min, 3); + assert_eq!(advertised_min_version(API_KEY_PRODUCE, firewall_min), 0); + assert!(!is_supported_version(API_KEY_PRODUCE, 0)); + assert!(!is_supported_version(API_KEY_PRODUCE, 2)); + + let body = handle_request(API_KEY_PRODUCE, 2, Bytes::new(), &default_broker()); + let mut d = Decoder::new(body); + let _topics = d.read_i32().unwrap(); + let _name = d.read_nullable_string().unwrap(); + let _parts = d.read_i32().unwrap(); + assert_eq!(d.read_i32().unwrap(), 0, "partition index"); + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); +} + +// ── Metadata (spec + SCOPE) ───────────────────────────────────────────────── + +#[test] +fn metadata_v0_empty_topics_returns_zero_length_topic_array() { + let body = handle_request( + API_KEY_METADATA, + 0, + metadata_empty_legacy_body(), + &default_broker(), + ); + let mut d = Decoder::new(body); + let _brokers = d.read_i32().unwrap(); + d.read_i32().unwrap(); + d.read_nullable_string().unwrap(); + d.read_i32().unwrap(); + assert_eq!(d.read_i32().unwrap(), 0, "empty request → zero topics"); + assert_eq!(d.remaining(), 0); +} + +#[test] +fn metadata_v3_includes_throttle_time_ms_before_brokers() { + let body = handle_request( + API_KEY_METADATA, + 3, + metadata_empty_legacy_body(), + &default_broker(), + ); + let mut d = Decoder::new(body); + assert_eq!(d.read_i32().unwrap(), 0, "throttle_time_ms"); +} + +#[test] +fn metadata_v9_flexible_empty_topics_returns_zero_topics() { + let body = handle_request( + API_KEY_METADATA, + 9, + build_metadata_flexible_request(&[]), + &default_broker(), + ); + let mut d = Decoder::new(body); + d.read_i32().unwrap(); // throttle + let broker_count = usize::try_from(d.read_varint().unwrap()) + .unwrap() + .saturating_sub(1); + for _ in 0..broker_count { + d.read_i32().unwrap(); + d.read_compact_nullable_string().unwrap(); + d.read_i32().unwrap(); + d.read_compact_nullable_string().unwrap(); + d.read_tagged_fields().unwrap(); + } + d.read_compact_nullable_string().unwrap(); // cluster_id + d.read_i32().unwrap(); // controller_id + let topic_count = usize::try_from(d.read_varint().unwrap()) + .unwrap() + .saturating_sub(1); + assert_eq!(topic_count, 0); +} + +#[test] +fn metadata_v9_flexible_echoes_each_requested_topic_name() { + let topics = ["orders", "payments", "inventory"]; + let body = handle_request( + API_KEY_METADATA, + 9, + build_metadata_flexible_request(&topics), + &default_broker(), + ); + + let mut d = Decoder::new(body); + d.read_i32().unwrap(); + let broker_count = usize::try_from(d.read_varint().unwrap()) + .unwrap() + .saturating_sub(1); + for _ in 0..broker_count { + d.read_i32().unwrap(); + d.read_compact_nullable_string().unwrap(); + d.read_i32().unwrap(); + d.read_compact_nullable_string().unwrap(); + d.read_tagged_fields().unwrap(); + } + d.read_compact_nullable_string().unwrap(); + d.read_i32().unwrap(); + + let topic_count = usize::try_from(d.read_varint().unwrap()) + .unwrap() + .saturating_sub(1); + assert_eq!(topic_count, topics.len()); + + let mut names = Vec::new(); + for _ in 0..topic_count { + assert_eq!( + d.read_i16().unwrap(), + ERROR_UNKNOWN_TOPIC_OR_PARTITION, + "stub gateway returns unknown topic error per topic" + ); + names.push( + d.read_compact_nullable_string() + .unwrap() + .expect("topic name"), + ); + d.read_bool().unwrap(); + assert_eq!( + usize::try_from(d.read_varint().unwrap()) + .unwrap() + .saturating_sub(1), + 0, + "empty partitions array" + ); + d.read_tagged_fields().unwrap(); + } + + assert_eq!( + names, + topics + .iter() + .map(|topic| (*topic).to_string()) + .collect::>(), + "metadata must echo requested topic names for client matching" + ); +} + +#[test] +fn metadata_v1_legacy_multiple_topics_echo_names() { + let topics = ["alpha", "beta"]; + let body = handle_request( + API_KEY_METADATA, + 1, + build_metadata_legacy_request(&topics), + &default_broker(), + ); + + let mut d = Decoder::new(body); + d.read_i32().unwrap(); + d.read_i32().unwrap(); + d.read_nullable_string().unwrap(); + d.read_i32().unwrap(); + d.read_nullable_string().unwrap(); + d.read_i32().unwrap(); + assert_eq!(d.read_i32().unwrap(), 2); + + for expected in topics { + assert_eq!(d.read_i16().unwrap(), ERROR_UNKNOWN_TOPIC_OR_PARTITION); + assert_eq!( + d.read_nullable_string().unwrap().as_deref(), + Some(expected), + "metadata v1 must echo {expected}" + ); + d.read_bool().unwrap(); + assert_eq!(d.read_i32().unwrap(), 0); + } +} + +// ── Produce (Kafka spec acks semantics) ───────────────────────────────────── + +#[tokio::test] +async fn e2e_produce_v3_acks_all_minus_one_returns_response() { + let (addr, _shutdown) = spawn_test_server().await; + let body = build_produce_v3_body(-1, 0); + let (corr, resp) = round_trip(addr, API_KEY_PRODUCE, 3, 501, &body).await; + assert_eq!(corr, 501); + assert!(!resp.is_empty()); +} + +#[tokio::test] +async fn e2e_produce_v9_flexible_header_with_empty_topics() { + let (addr, _shutdown) = spawn_test_server().await; + let body = build_produce_flexible_empty_request(1); + let (corr, resp) = round_trip(addr, API_KEY_PRODUCE, 9, 502, &body).await; + assert_eq!(corr, 502); + let mut d = Decoder::new(resp); + assert!( + d.read_varint().unwrap() >= 1, + "flexible topics array header" + ); +} + +#[tokio::test] +async fn produce_v3_through_v9_e2e_preserve_correlation_id() { + let (addr, _shutdown) = spawn_test_server().await; + + for version in 3i16..=9 { + let body = if version >= 9 { + build_produce_flexible_empty_request(1) + } else { + build_produce_v3_body(1, 0) + }; + let correlation_id = 510 + i32::from(version); + let (corr, resp) = + round_trip(addr, API_KEY_PRODUCE, version, correlation_id, &body).await; + assert_eq!(corr, correlation_id, "Produce v{version} correlation"); + assert!(!resp.is_empty(), "Produce v{version} response"); + } +} + +// ── ListOffsets supported versions ────────────────────────────────────────── + +#[tokio::test] +async fn list_offsets_v1_through_v6_e2e_return_partition_error_zero() { + let (addr, _shutdown) = spawn_test_server().await; + + for version in 1i16..=6 { + let body = build_list_offsets_request(version, "offsets-topic", 0); + let correlation_id = 600 + i32::from(version); + let (corr, resp) = + round_trip(addr, API_KEY_LIST_OFFSETS, version, correlation_id, &body).await; + assert_eq!(corr, correlation_id); + + let flexible = version >= 6; + let mut d = Decoder::new(resp); + if version >= 2 { + d.read_i32().unwrap(); + } + if flexible { + d.read_varint().unwrap(); + d.read_compact_nullable_string().unwrap(); + d.read_varint().unwrap(); + } else { + d.read_i32().unwrap(); + d.read_nullable_string().unwrap(); + d.read_i32().unwrap(); + } + d.read_i32().unwrap(); + assert_eq!( + d.read_i16().unwrap(), + ERROR_NONE, + "ListOffsets v{version} stub partition error" + ); + } +} + +// ── CreateTopics empty create ─────────────────────────────────────────────── + +#[tokio::test] +async fn create_topics_v2_through_v5_empty_request_e2e_succeeds() { + let (addr, _shutdown) = spawn_test_server().await; + + for version in 2i16..=5 { + let body = build_create_topics_empty_request(version); + let correlation_id = 700 + i32::from(version); + let (corr, resp) = + round_trip(addr, API_KEY_CREATE_TOPICS, version, correlation_id, &body).await; + assert_eq!(corr, correlation_id); + let mut d = Decoder::new(resp); + if version >= 2 { + d.read_i32().unwrap(); + } + if version >= 5 { + assert!(d.read_varint().unwrap() >= 1); + } else { + assert_eq!(d.read_i32().unwrap(), 0); + } + } +} + +// ── Fetch flexible boundary ───────────────────────────────────────────────── + +#[tokio::test] +async fn fetch_v4_through_v12_e2e_preserve_correlation_id() { + let (addr, _shutdown) = spawn_test_server().await; + + for version in 4i16..=12 { + let body = request_body_for_scoped_api(API_KEY_FETCH, "Fetch", version); + let correlation_id = 800 + i32::from(version); + let (corr, resp) = round_trip(addr, API_KEY_FETCH, version, correlation_id, &body).await; + assert_eq!(corr, correlation_id, "Fetch v{version} correlation"); + assert!(!resp.is_empty(), "Fetch v{version} response"); + } +} + +#[tokio::test] +async fn apiversions_v4_out_of_range_e2e_returns_unsupported() { + let (addr, _shutdown) = spawn_test_server().await; + let (corr, body) = round_trip(addr, API_KEY_API_VERSIONS, 4, 350, &[]).await; + assert_eq!(corr, 350); + let mut d = Decoder::new(body); + assert_eq!( + d.read_i16().unwrap(), + ERROR_UNSUPPORTED_VERSION, + "ApiVersions v4 must return UNSUPPORTED_VERSION per KIP-511" + ); +} + +#[tokio::test] +async fn metadata_empty_body_e2e_all_topics_request_returns_broker() { + let (addr, _shutdown) = spawn_test_server().await; + let (corr, body) = round_trip(addr, API_KEY_METADATA, 0, 360, &[]).await; + assert_eq!(corr, 360); + let mut d = Decoder::new(body); + assert_eq!(d.read_i32().unwrap(), 1, "one stub broker"); + d.read_i32().unwrap(); // node_id + let host = d.read_nullable_string().unwrap().expect("broker host"); + assert!(!host.is_empty()); + let port = d.read_i32().unwrap(); + assert!(port > 0); +} + +#[tokio::test] +async fn list_offsets_v7_unsupported_e2e_returns_error() { + let (addr, _shutdown) = spawn_test_server().await; + let (corr, body) = + round_trip(addr, API_KEY_LIST_OFFSETS, 7, 370, &[0x00, 0x00, 0x00, 0x00]).await; + assert_eq!(corr, 370); + assert!( + scan_for_error_code(&body, ERROR_UNSUPPORTED_VERSION), + "ListOffsets v7 must be rejected" + ); +} + +#[tokio::test] +async fn create_topics_v1_unsupported_e2e_returns_error() { + let (addr, _shutdown) = spawn_test_server().await; + let (corr, body) = round_trip(addr, API_KEY_CREATE_TOPICS, 1, 380, &[]).await; + assert_eq!(corr, 380); + assert!( + scan_for_error_code(&body, ERROR_UNSUPPORTED_VERSION), + "CreateTopics v1 must be rejected" + ); +} + +#[tokio::test] +async fn corrupt_produce_body_e2e_returns_error_without_disconnect() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.expect("connect"); + + let bad = build_request_frame( + API_KEY_PRODUCE, + 3, + 391, + Some("scope-test"), + &[0xFF, 0xFF, 0xFF], + ); + stream.write_all(&bad).await.expect("corrupt produce"); + let payload = tcp::read_response_frame(&mut stream, 8 * 1024 * 1024).await; + assert!( + scan_for_error_code(&parse_response_payload(API_KEY_PRODUCE, 3, payload).1, ERROR_INVALID_REQUEST), + "corrupt Produce must surface INVALID_REQUEST" + ); + + let ok = build_request_frame(API_KEY_API_VERSIONS, 1, 392, Some("scope-test"), &[]); + stream.write_all(&ok).await.expect("follow-up"); + let payload = tcp::read_response_frame(&mut stream, 8 * 1024 * 1024).await; + assert_eq!( + parse_response_payload(API_KEY_API_VERSIONS, 1, payload).0, + 392 + ); +} + +#[tokio::test] +async fn corrupt_fetch_body_e2e_returns_error_without_disconnect() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.expect("connect"); + + let bad = build_request_frame( + API_KEY_FETCH, + 4, + 393, + Some("scope-test"), + &[0xFF, 0xFF, 0xFF], + ); + stream.write_all(&bad).await.expect("corrupt fetch"); + let payload = tcp::read_response_frame(&mut stream, 8 * 1024 * 1024).await; + assert!( + scan_for_error_code(&parse_response_payload(API_KEY_FETCH, 4, payload).1, ERROR_INVALID_REQUEST), + "corrupt Fetch must surface INVALID_REQUEST" + ); + + let ok = build_request_frame(API_KEY_API_VERSIONS, 1, 394, Some("scope-test"), &[]); + stream.write_all(&ok).await.expect("follow-up"); + let payload = tcp::read_response_frame(&mut stream, 8 * 1024 * 1024).await; + assert_eq!( + parse_response_payload(API_KEY_API_VERSIONS, 1, payload).0, + 394 + ); +} + +// ── Corrupt decode paths for remaining scoped APIs ────────────────────────── + +#[test] +fn corrupt_list_offsets_body_returns_invalid_request_error() { + let body = Bytes::from_static(&[0xFF, 0xFF, 0xFF]); + let resp = handle_request(API_KEY_LIST_OFFSETS, 1, body, &default_broker()); + assert!(!resp.is_empty()); + assert!( + scan_for_error_code(&resp, ERROR_INVALID_REQUEST) + || scan_for_error_code(&resp, ERROR_UNSUPPORTED_VERSION), + "corrupt ListOffsets must surface protocol error" + ); +} + +#[test] +fn corrupt_create_topics_body_returns_invalid_request_error() { + let body = Bytes::from_static(&[0xFF, 0xFF, 0xFF]); + let resp = handle_request(API_KEY_CREATE_TOPICS, 2, body, &default_broker()); + assert!(!resp.is_empty()); + assert!( + scan_for_error_code(&resp, ERROR_INVALID_REQUEST) + || scan_for_error_code(&resp, ERROR_UNSUPPORTED_VERSION) + ); +} + +#[test] +fn corrupt_metadata_body_returns_zero_topics_not_panic() { + let body = Bytes::from_static(&[0x00, 0x00]); + let resp = handle_request(API_KEY_METADATA, 0, body, &default_broker()); + assert!(!resp.is_empty()); + let mut d = Decoder::new(resp); + d.read_i32().unwrap(); + d.read_i32().unwrap(); + d.read_nullable_string().unwrap(); + d.read_i32().unwrap(); + assert_eq!( + d.read_i32().unwrap(), + 0, + "malformed topic list → zero topics" + ); +} + +fn scan_for_error_code(body: &Bytes, code: i16) -> bool { + body.windows(2) + .any(|w| i16::from_be_bytes([w[0], w[1]]) == code) +} + +// ── Flexible encoding boundaries (SCOPE.md) ─────────────────────────────────── + +#[test] +fn request_header_version_switches_at_scope_flexible_boundaries() { + use iggy_gateway_kafka::protocol::header::request_header_version; + + assert_eq!(request_header_version(API_KEY_PRODUCE, 8), 1); + assert_eq!(request_header_version(API_KEY_PRODUCE, 9), 2); + assert_eq!(request_header_version(API_KEY_FETCH, 11), 1); + assert_eq!(request_header_version(API_KEY_FETCH, 12), 2); + assert_eq!(request_header_version(API_KEY_LIST_OFFSETS, 5), 1); + assert_eq!(request_header_version(API_KEY_LIST_OFFSETS, 6), 2); + assert_eq!(request_header_version(API_KEY_METADATA, 8), 1); + assert_eq!(request_header_version(API_KEY_METADATA, 9), 2); + assert_eq!(request_header_version(API_KEY_API_VERSIONS, 2), 1); + assert_eq!(request_header_version(API_KEY_API_VERSIONS, 3), 2); + assert_eq!(request_header_version(API_KEY_CREATE_TOPICS, 4), 1); + assert_eq!(request_header_version(API_KEY_CREATE_TOPICS, 5), 2); +} + +#[test] +fn metadata_v9_request_with_three_topics_yields_three_response_slots() { + let topics = ["a", "b", "c"]; + let body = handle_request( + API_KEY_METADATA, + 9, + build_metadata_flexible_request(&topics), + &default_broker(), + ); + let mut d = Decoder::new(body); + d.read_i32().unwrap(); + skip_metadata_v9_prefix(&mut d); + let topic_count = usize::try_from(d.read_varint().unwrap()) + .unwrap() + .saturating_sub(1); + assert_eq!( + topic_count, + topics.len(), + "response topic count must mirror request topic count" + ); +} + +fn skip_metadata_v9_prefix(d: &mut Decoder) { + let broker_count = usize::try_from(d.read_varint().unwrap()) + .unwrap() + .saturating_sub(1); + for _ in 0..broker_count { + d.read_i32().unwrap(); + d.read_compact_nullable_string().unwrap(); + d.read_i32().unwrap(); + d.read_compact_nullable_string().unwrap(); + d.read_tagged_fields().unwrap(); + } + d.read_compact_nullable_string().unwrap(); + d.read_i32().unwrap(); +} + +// ── Handler-level: every in-range version returns bytes ───────────────────── + +#[test] +fn every_in_range_version_returns_non_empty_handler_response() { + for &(api_key, name, min_ver, max_ver) in SCOPED_API_KEYS { + for version in min_ver..=max_ver { + let body = request_body_for_scoped_api(api_key, name, version); + let resp = handle_request(api_key, version, body, &default_broker()); + assert!(!resp.is_empty(), "{name} v{version} handler returned empty"); + } + } +} diff --git a/gateways/kafka/tools/kafka-tool/tests/generate_cli_tests.rs b/gateways/kafka/tools/kafka-tool/tests/generate_cli_tests.rs new file mode 100644 index 0000000000..fd3dccd74b --- /dev/null +++ b/gateways/kafka/tools/kafka-tool/tests/generate_cli_tests.rs @@ -0,0 +1,49 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! CLI regression for PR #3519 review: `generate` must accept repeated `--api-key`. + +use std::process::Command; +use std::time::{SystemTime, UNIX_EPOCH}; + +#[test] +fn generate_accepts_repeated_api_key_flags() { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock") + .as_nanos(); + let output = std::env::temp_dir().join(format!("kafka-gen-test-{nanos}")); + + let status = Command::new(env!("CARGO_BIN_EXE_kafka-message-gen")) + .arg("generate") + .arg("--output") + .arg(&output) + .arg("--api-key") + .arg("0") + .arg("--api-key") + .arg("1") + .status() + .expect("run kafka-message-gen generate"); + + assert!( + status.success(), + "generate should accept multiple --api-key flags (PR description / Verify parity); \ + got exit status {status:?}" + ); + + let _ = std::fs::remove_dir_all(output); +} From 4bd65367cedc7c40ac5b67543b46277aea73b354 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Fri, 3 Jul 2026 07:19:44 -0400 Subject: [PATCH 19/57] fix(gateways/kafka): resolve clippy and fmt failures in test suite - Move inline `use` imports to file level (items_after_statements) - Replace `match { Ok(x) => Some(x), Err(_) => None }` with `.ok()` - Parse frame length as `u32` to avoid cast_sign_loss on `i32 as usize` - Drop explicit `API_KEY_API_VERSIONS` arm shadowed by wildcard (match_same_arms) - Wrap identifiers in backticks in four doc comments (doc_markdown) - Re-run `cargo fmt --all` to fix line-length wraps in scope_coverage_tests Co-Authored-By: Claude Sonnet 4.6 --- gateways/kafka/docs/TEST_SUITE.md | 5 +++- gateways/kafka/tests/common/tcp.rs | 9 +++---- gateways/kafka/tests/common/wire.rs | 4 +-- .../kafka/tests/listener_robustness_tests.rs | 7 ++--- .../kafka/tests/review_regression_tests.rs | 4 +-- gateways/kafka/tests/scope_coverage_tests.rs | 27 +++++++++++++------ 6 files changed, 33 insertions(+), 23 deletions(-) diff --git a/gateways/kafka/docs/TEST_SUITE.md b/gateways/kafka/docs/TEST_SUITE.md index 74dd995268..08860c18cb 100644 --- a/gateways/kafka/docs/TEST_SUITE.md +++ b/gateways/kafka/docs/TEST_SUITE.md @@ -136,7 +136,10 @@ Fixtures are gitignored under `tools/kafka-tool/kafka_messages/`. CI runs the sa # 1. Generate fixtures cargo run -p kafka-message-gen -- generate \ --output gateways/kafka/tools/kafka-tool/kafka_messages \ - --api-key 0 --api-key 1 --api-key 2 --api-key 19 + --api-key 0 + ## use one api-key in each command. Bulk API kys needs to be implemented + allowed values are + --api-key 1 --api-key 2 --api-key 19 # 2. Run regression suite cargo test -p iggy-gateway-kafka diff --git a/gateways/kafka/tests/common/tcp.rs b/gateways/kafka/tests/common/tcp.rs index c449305962..aaeea56b29 100644 --- a/gateways/kafka/tests/common/tcp.rs +++ b/gateways/kafka/tests/common/tcp.rs @@ -90,7 +90,7 @@ pub async fn read_response_frame(stream: &mut TcpStream, max_size: usize) -> Byt Bytes::from(buf) } -/// Minimal Produce v3 body: nullable transactional_id, acks, timeout, empty topics array. +/// Minimal Produce v3 body: nullable `transactional_id`, acks, timeout, empty topics array. pub fn build_produce_v3_body(acks: i16, topics_count: i32) -> Bytes { let mut body = BytesMut::new(); body.put_i16(-1); // null transactional_id @@ -119,10 +119,9 @@ pub async fn read_response_frame_with_timeout( max_size: usize, timeout: Duration, ) -> Option { - match time::timeout(timeout, read_response_frame(stream, max_size)).await { - Ok(frame) => Some(frame), - Err(_) => None, - } + time::timeout(timeout, read_response_frame(stream, max_size)) + .await + .ok() } /// Concatenate multiple length-prefixed frames (for pipelining tests). diff --git a/gateways/kafka/tests/common/wire.rs b/gateways/kafka/tests/common/wire.rs index 9507602e1a..3f3277ce85 100644 --- a/gateways/kafka/tests/common/wire.rs +++ b/gateways/kafka/tests/common/wire.rs @@ -58,7 +58,7 @@ pub fn build_metadata_flexible_request(topic_names: &[&str]) -> Bytes { enc.freeze() } -/// Minimal ListOffsets request for supported versions (v1–v6). +/// Minimal `ListOffsets` request for supported versions (v1–v6). pub fn build_list_offsets_request(version: i16, topic: &str, partition: i32) -> Bytes { let flexible = version >= 6; let mut enc = Encoder::with_capacity(128); @@ -93,7 +93,7 @@ pub fn build_list_offsets_request(version: i16, topic: &str, partition: i32) -> enc.freeze() } -/// CreateTopics v2+ with zero topics (valid empty create). +/// `CreateTopics` v2+ with zero topics (valid empty create). pub fn build_create_topics_empty_request(version: i16) -> Bytes { let flexible = version >= 5; let mut enc = Encoder::with_capacity(32); diff --git a/gateways/kafka/tests/listener_robustness_tests.rs b/gateways/kafka/tests/listener_robustness_tests.rs index 13ecd53c3a..8568214b0f 100644 --- a/gateways/kafka/tests/listener_robustness_tests.rs +++ b/gateways/kafka/tests/listener_robustness_tests.rs @@ -32,7 +32,7 @@ use tokio::net::TcpStream; use tokio::time; use iggy_gateway_kafka::ServerConfig; -use iggy_gateway_kafka::protocol::api::API_KEY_API_VERSIONS; +use iggy_gateway_kafka::protocol::api::{API_KEY_API_VERSIONS, API_KEY_METADATA, API_KEY_PRODUCE}; use iggy_gateway_kafka::protocol::codec::Decoder; use server::{spawn_test_server, spawn_test_server_with_config}; @@ -139,7 +139,7 @@ async fn e2e_truncated_frame_body_closes_connection() { let mut stream = TcpStream::connect(addr).await.expect("connect"); let full = build_request_frame(API_KEY_API_VERSIONS, 1, 66, Some("trunc-test"), &[]); - let payload_len = i32::from_be_bytes([full[0], full[1], full[2], full[3]]) as usize; + let payload_len = u32::from_be_bytes([full[0], full[1], full[2], full[3]]) as usize; assert!(full.len() >= 4 + payload_len); stream @@ -327,8 +327,6 @@ async fn e2e_mixed_api_key_pipeline_returns_responses_in_order() { let (addr, _shutdown) = spawn_test_server().await; let mut stream = TcpStream::connect(addr).await.expect("connect"); - use iggy_gateway_kafka::protocol::api::{API_KEY_METADATA, API_KEY_PRODUCE}; - let frames = [ build_request_frame(API_KEY_API_VERSIONS, 1, 501, Some("mix-test"), &[]), build_request_frame( @@ -392,7 +390,6 @@ async fn e2e_connection_idle_after_response_accepts_next_request() { #[tokio::test] async fn e2e_flexible_metadata_v9_empty_topics_round_trip() { let (addr, _shutdown) = spawn_test_server().await; - use iggy_gateway_kafka::protocol::api::API_KEY_METADATA; let body = wire::build_metadata_flexible_request(&[]); let (corr, resp) = tcp::round_trip(addr, API_KEY_METADATA, 9, 701, &body).await; diff --git a/gateways/kafka/tests/review_regression_tests.rs b/gateways/kafka/tests/review_regression_tests.rs index b2cecb929b..a1cebf1de5 100644 --- a/gateways/kafka/tests/review_regression_tests.rs +++ b/gateways/kafka/tests/review_regression_tests.rs @@ -97,7 +97,7 @@ async fn e2e_produce_v3_acks_one_still_returns_response() { // ── ListOffsets v0 wire shape (review: old_style_offsets array, not bare i64) ─ -/// Parse one ListOffsets v0 partition entry the way a v0 Kafka client would. +/// Parse one `ListOffsets` v0 partition entry the way a v0 Kafka client would. fn parse_list_offsets_v0_partition(d: &mut Decoder) { let _partition_index = d.read_i32().expect("partition_index"); let _error_code = d.read_i16().expect("error_code"); @@ -153,7 +153,7 @@ fn list_offsets_v0_unsupported_version_carries_error_code_in_partition() { assert_eq!(d.remaining(), 0); } -/// ListOffsets v0 request below firewall min (mirrors atharvalade repro script body). +/// `ListOffsets` v0 request below firewall min (mirrors atharvalade repro script body). fn build_list_offsets_v0_request_with_topic_t() -> Bytes { let mut body = BytesMut::new(); body.put_i32(-1); // replica_id diff --git a/gateways/kafka/tests/scope_coverage_tests.rs b/gateways/kafka/tests/scope_coverage_tests.rs index c638d2c109..ad922463c3 100644 --- a/gateways/kafka/tests/scope_coverage_tests.rs +++ b/gateways/kafka/tests/scope_coverage_tests.rs @@ -52,7 +52,8 @@ use tcp::{ }; use wire::{ OUT_OF_SCOPE_API_KEYS, build_create_topics_empty_request, build_fetch_empty_topics_request, - build_list_offsets_request, build_metadata_flexible_request, build_produce_flexible_empty_request, + build_list_offsets_request, build_metadata_flexible_request, + build_produce_flexible_empty_request, }; fn metadata_empty_legacy_body() -> Bytes { @@ -63,7 +64,6 @@ fn metadata_empty_legacy_body() -> Bytes { fn request_body_for_scoped_api(api_key: i16, name: &str, version: i16) -> Bytes { match api_key { - API_KEY_API_VERSIONS => Bytes::new(), API_KEY_METADATA => metadata_empty_legacy_body(), API_KEY_PRODUCE => { if fixture_exists(api_key, name, version) { @@ -464,8 +464,7 @@ async fn produce_v3_through_v9_e2e_preserve_correlation_id() { build_produce_v3_body(1, 0) }; let correlation_id = 510 + i32::from(version); - let (corr, resp) = - round_trip(addr, API_KEY_PRODUCE, version, correlation_id, &body).await; + let (corr, resp) = round_trip(addr, API_KEY_PRODUCE, version, correlation_id, &body).await; assert_eq!(corr, correlation_id, "Produce v{version} correlation"); assert!(!resp.is_empty(), "Produce v{version} response"); } @@ -576,8 +575,14 @@ async fn metadata_empty_body_e2e_all_topics_request_returns_broker() { #[tokio::test] async fn list_offsets_v7_unsupported_e2e_returns_error() { let (addr, _shutdown) = spawn_test_server().await; - let (corr, body) = - round_trip(addr, API_KEY_LIST_OFFSETS, 7, 370, &[0x00, 0x00, 0x00, 0x00]).await; + let (corr, body) = round_trip( + addr, + API_KEY_LIST_OFFSETS, + 7, + 370, + &[0x00, 0x00, 0x00, 0x00], + ) + .await; assert_eq!(corr, 370); assert!( scan_for_error_code(&body, ERROR_UNSUPPORTED_VERSION), @@ -611,7 +616,10 @@ async fn corrupt_produce_body_e2e_returns_error_without_disconnect() { stream.write_all(&bad).await.expect("corrupt produce"); let payload = tcp::read_response_frame(&mut stream, 8 * 1024 * 1024).await; assert!( - scan_for_error_code(&parse_response_payload(API_KEY_PRODUCE, 3, payload).1, ERROR_INVALID_REQUEST), + scan_for_error_code( + &parse_response_payload(API_KEY_PRODUCE, 3, payload).1, + ERROR_INVALID_REQUEST + ), "corrupt Produce must surface INVALID_REQUEST" ); @@ -639,7 +647,10 @@ async fn corrupt_fetch_body_e2e_returns_error_without_disconnect() { stream.write_all(&bad).await.expect("corrupt fetch"); let payload = tcp::read_response_frame(&mut stream, 8 * 1024 * 1024).await; assert!( - scan_for_error_code(&parse_response_payload(API_KEY_FETCH, 4, payload).1, ERROR_INVALID_REQUEST), + scan_for_error_code( + &parse_response_payload(API_KEY_FETCH, 4, payload).1, + ERROR_INVALID_REQUEST + ), "corrupt Fetch must surface INVALID_REQUEST" ); From 56f9f28a6612e25136ad688044cc8dd18c267aa1 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Fri, 3 Jul 2026 07:28:01 -0400 Subject: [PATCH 20/57] Update TEST_SUITE.md --- gateways/kafka/docs/TEST_SUITE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gateways/kafka/docs/TEST_SUITE.md b/gateways/kafka/docs/TEST_SUITE.md index 08860c18cb..bb33691ff5 100644 --- a/gateways/kafka/docs/TEST_SUITE.md +++ b/gateways/kafka/docs/TEST_SUITE.md @@ -136,9 +136,9 @@ Fixtures are gitignored under `tools/kafka-tool/kafka_messages/`. CI runs the sa # 1. Generate fixtures cargo run -p kafka-message-gen -- generate \ --output gateways/kafka/tools/kafka-tool/kafka_messages \ - --api-key 0 + --api-key 0 ## use one api-key in each command. Bulk API kys needs to be implemented - allowed values are + allowed values are --api-key 1 --api-key 2 --api-key 19 # 2. Run regression suite From c0d3e43f6996fca1e1c594e1ce80bd38323fdd57 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Mon, 6 Jul 2026 07:06:13 -0400 Subject: [PATCH 21/57] Skip Kafka reply for Produce acks=0 Update Kafka request handling to model no-response operations explicitly: `handle_request` now returns `Option` and returns `None` for Produce requests with `acks=0` (fire-and-forget), preventing protocol desync from sending an unexpected response. Connection handling now skips writing a response when `None` is returned, and tests were updated to unwrap expected response paths explicitly. --- gateways/kafka/src/protocol/api.rs | 64 +++++++++++++------ gateways/kafka/src/server.rs | 6 +- gateways/kafka/tests/api_handler_tests.rs | 12 ++-- .../kafka/tests/broker_advertise_tests.rs | 2 +- .../kafka/tests/golden_wire_fixtures_tests.rs | 4 +- .../kafka/tests/handler_regression_tests.rs | 12 ++-- .../kafka/tests/metadata_regression_tests.rs | 18 +++--- .../kafka/tests/review_regression_tests.rs | 8 +-- gateways/kafka/tests/scope_coverage_tests.rs | 24 +++---- .../kafka/tests/version_firewall_tests.rs | 32 +++++----- 10 files changed, 106 insertions(+), 76 deletions(-) diff --git a/gateways/kafka/src/protocol/api.rs b/gateways/kafka/src/protocol/api.rs index 2c1d3b39b2..414d5bea90 100644 --- a/gateways/kafka/src/protocol/api.rs +++ b/gateways/kafka/src/protocol/api.rs @@ -109,87 +109,113 @@ pub fn supported_api_ranges() -> &'static [ApiVersionRange] { SUPPORTED_RANGES } +/// Handles one decoded request frame and returns the response to write back, +/// or `None` when the wire protocol forbids a response (Produce with `acks=0`). pub fn handle_request( api_key: i16, api_version: i16, body: Bytes, broker: &BrokerAdvertise, -) -> Bytes { +) -> Option { match api_key { API_KEY_API_VERSIONS => { if is_supported_version(api_key, api_version) { - encode_api_versions_response(api_version, ERROR_NONE) + Some(encode_api_versions_response(api_version, ERROR_NONE)) } else { // KIP-511: reply with v0 when the requested version is not understood. - encode_api_versions_response(0, ERROR_UNSUPPORTED_VERSION) + Some(encode_api_versions_response(0, ERROR_UNSUPPORTED_VERSION)) } } API_KEY_METADATA => { if is_supported_version(api_key, api_version) { - encode_metadata_response(api_version, body, broker, ERROR_NONE) + Some(encode_metadata_response(api_version, body, broker, ERROR_NONE)) } else { // Encode at the highest version we implement, not the client's unknown version. - encode_metadata_response( + Some(encode_metadata_response( api_version.clamp(0, MAX_SUPPORTED_METADATA_VERSION), body, broker, ERROR_UNSUPPORTED_VERSION, - ) + )) } } API_KEY_PRODUCE => { if is_supported_version(api_key, api_version) { match decode_produce_request(api_version, body) { - Ok(req) => encode_produce_response(api_version, &req), + // acks=0 is fire-and-forget: the client isn't reading a response, so + // sending one desyncs the next correlation id it expects. + Ok(req) if req.acks == 0 => None, + Ok(req) => Some(encode_produce_response(api_version, &req)), Err(e) => { tracing::warn!("Failed to decode Produce request: {:?}", e); - encode_produce_error_response(api_version, ERROR_INVALID_REQUEST) + Some(encode_produce_error_response( + api_version, + ERROR_INVALID_REQUEST, + )) } } } else { - encode_produce_error_response(api_version, ERROR_UNSUPPORTED_VERSION) + Some(encode_produce_error_response( + api_version, + ERROR_UNSUPPORTED_VERSION, + )) } } API_KEY_FETCH => { if is_supported_version(api_key, api_version) { match decode_fetch_request(api_version, body) { - Ok(req) => encode_fetch_response(api_version, &req), + Ok(req) => Some(encode_fetch_response(api_version, &req)), Err(e) => { tracing::warn!("Failed to decode Fetch request: {:?}", e); - encode_fetch_error_response(api_version, ERROR_INVALID_REQUEST) + Some(encode_fetch_error_response(api_version, ERROR_INVALID_REQUEST)) } } } else { - encode_fetch_error_response(api_version, ERROR_UNSUPPORTED_VERSION) + Some(encode_fetch_error_response( + api_version, + ERROR_UNSUPPORTED_VERSION, + )) } } API_KEY_LIST_OFFSETS => { if is_supported_version(api_key, api_version) { match decode_list_offsets_request(api_version, body) { - Ok(req) => encode_list_offsets_response(api_version, &req), + Ok(req) => Some(encode_list_offsets_response(api_version, &req)), Err(e) => { tracing::warn!("Failed to decode ListOffsets request: {:?}", e); - encode_list_offsets_error_response(api_version, ERROR_INVALID_REQUEST) + Some(encode_list_offsets_error_response( + api_version, + ERROR_INVALID_REQUEST, + )) } } } else { - encode_list_offsets_error_response(api_version, ERROR_UNSUPPORTED_VERSION) + Some(encode_list_offsets_error_response( + api_version, + ERROR_UNSUPPORTED_VERSION, + )) } } API_KEY_CREATE_TOPICS => { if is_supported_version(api_key, api_version) { match decode_create_topics_request(api_version, body) { - Ok(req) => encode_create_topics_response(api_version, &req), + Ok(req) => Some(encode_create_topics_response(api_version, &req)), Err(e) => { tracing::warn!("Failed to decode CreateTopics request: {:?}", e); - encode_create_topics_error_response(api_version, ERROR_INVALID_REQUEST) + Some(encode_create_topics_error_response( + api_version, + ERROR_INVALID_REQUEST, + )) } } } else { - encode_create_topics_error_response(api_version, ERROR_UNSUPPORTED_VERSION) + Some(encode_create_topics_error_response( + api_version, + ERROR_UNSUPPORTED_VERSION, + )) } } - _ => encode_error_only_response(ERROR_UNSUPPORTED_VERSION), + _ => Some(encode_error_only_response(ERROR_UNSUPPORTED_VERSION)), } } diff --git a/gateways/kafka/src/server.rs b/gateways/kafka/src/server.rs index a41d5fa9bd..c28e6daecb 100644 --- a/gateways/kafka/src/server.rs +++ b/gateways/kafka/src/server.rs @@ -277,7 +277,11 @@ async fn handle_connection( ); let body = decoder.read_bytes(decoder.remaining())?; - let body_response = handle_request(req.api_key, req.api_version, body, &broker); + let Some(body_response) = handle_request(req.api_key, req.api_version, body, &broker) + else { + // Produce with acks=0: the wire protocol forbids a response. + continue; + }; let resp_header = ResponseHeader { correlation_id: req.correlation_id, diff --git a/gateways/kafka/tests/api_handler_tests.rs b/gateways/kafka/tests/api_handler_tests.rs index ab3e762332..b51f882c12 100644 --- a/gateways/kafka/tests/api_handler_tests.rs +++ b/gateways/kafka/tests/api_handler_tests.rs @@ -31,7 +31,7 @@ use iggy_gateway_kafka::protocol::codec::Decoder; #[test] fn api_versions_v1_response_non_flexible_format() { - let body = handle_request(API_KEY_API_VERSIONS, 1, Bytes::new(), &test_broker()); + let body = handle_request(API_KEY_API_VERSIONS, 1, Bytes::new(), &test_broker()).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i16().unwrap(), 0); // error_code @@ -55,7 +55,7 @@ fn api_versions_v1_response_non_flexible_format() { #[test] fn api_versions_v3_response_flexible_format() { - let body = handle_request(API_KEY_API_VERSIONS, 3, Bytes::new(), &test_broker()); + let body = handle_request(API_KEY_API_VERSIONS, 3, Bytes::new(), &test_broker()).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i16().unwrap(), 0); // error_code @@ -85,7 +85,7 @@ fn api_versions_v3_response_flexible_format() { #[test] fn metadata_response_has_broker_array_and_topic_array() { - let body = handle_request(API_KEY_METADATA, 0, Bytes::new(), &test_broker()); + let body = handle_request(API_KEY_METADATA, 0, Bytes::new(), &test_broker()).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let broker_count = d.read_i32().unwrap(); @@ -111,7 +111,7 @@ fn unsupported_version_returns_protocol_error() { 99, Bytes::from_static(&[0x02]), &test_broker(), - ); + ).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); // v9 flexible response layout: d.read_i32().unwrap(); // throttle_time_ms (v3+) @@ -141,7 +141,7 @@ fn unsupported_version_returns_protocol_error() { #[test] fn unknown_api_key_returns_error_only_payload() { - let body = handle_request(999, 0, Bytes::new(), &test_broker()); + let body = handle_request(999, 0, Bytes::new(), &test_broker()).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); } @@ -156,7 +156,7 @@ fn version_support_table_is_applied() { #[test] fn apiversions_unsupported_version_uses_v0_encoding_without_throttle() { - let body = handle_request(API_KEY_API_VERSIONS, 99, Bytes::new(), &test_broker()); + let body = handle_request(API_KEY_API_VERSIONS, 99, Bytes::new(), &test_broker()).expect("test request has acks != 0 and expects a response"); // v0: error_code(2) + api_keys i32 count(4) + 6 entries × 6 bytes = 42 — no throttle_time_ms. assert_eq!(body.len(), 42); let mut d = Decoder::new(body); diff --git a/gateways/kafka/tests/broker_advertise_tests.rs b/gateways/kafka/tests/broker_advertise_tests.rs index a445540dd7..3e6af07ee2 100644 --- a/gateways/kafka/tests/broker_advertise_tests.rs +++ b/gateways/kafka/tests/broker_advertise_tests.rs @@ -38,7 +38,7 @@ fn metadata_reflects_broker_addr() { }; let mut req = Encoder::with_capacity(4); req.write_i32(0); - let body = handle_request(API_KEY_METADATA, 0, req.freeze(), &broker); + let body = handle_request(API_KEY_METADATA, 0, req.freeze(), &broker).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 1); diff --git a/gateways/kafka/tests/golden_wire_fixtures_tests.rs b/gateways/kafka/tests/golden_wire_fixtures_tests.rs index f146215d34..bdf202ce67 100644 --- a/gateways/kafka/tests/golden_wire_fixtures_tests.rs +++ b/gateways/kafka/tests/golden_wire_fixtures_tests.rs @@ -25,7 +25,7 @@ use iggy_gateway_kafka::protocol::codec::Encoder; #[test] fn golden_apiversions_v1_response_fixture() { let broker = BrokerAdvertise::default(); - let actual = handle_request(API_KEY_API_VERSIONS, 1, Bytes::new(), &broker); + let actual = handle_request(API_KEY_API_VERSIONS, 1, Bytes::new(), &broker).expect("test request has acks != 0 and expects a response"); // error_code=0, api_count=6 // key 0 (Produce) min=0 max=9 (KAFKA-18659 advertise min=0) @@ -55,7 +55,7 @@ fn golden_metadata_v0_single_topic_response_fixture() { request.write_i32(1); // one topic let req_bytes = request.freeze(); - let actual = handle_request(API_KEY_METADATA, 0, req_bytes, &BrokerAdvertise::default()); + let actual = handle_request(API_KEY_METADATA, 0, req_bytes, &BrokerAdvertise::default()).expect("test request has acks != 0 and expects a response"); // Metadata v0 layout: brokers[], topics[] (no controller_id — added in v1) // brokers[1]: node_id=1, host=127.0.0.1, port=9093 diff --git a/gateways/kafka/tests/handler_regression_tests.rs b/gateways/kafka/tests/handler_regression_tests.rs index df569f4c12..9f69e28e4c 100644 --- a/gateways/kafka/tests/handler_regression_tests.rs +++ b/gateways/kafka/tests/handler_regression_tests.rs @@ -37,7 +37,7 @@ fn handle_request_succeeds_for_every_supported_version_with_fixture() { if api_key == 3 || api_key == 18 { // Metadata / ApiVersions: empty body is valid for version in min_ver..=max_ver { - let resp = handle_request(api_key, version, bytes::Bytes::new(), &default_broker()); + let resp = handle_request(api_key, version, bytes::Bytes::new(), &default_broker()).expect("test request has acks != 0 and expects a response"); assert!( !resp.is_empty(), "{name} v{version} returned empty response" @@ -51,7 +51,7 @@ fn handle_request_succeeds_for_every_supported_version_with_fixture() { continue; } let body = load_fixture_body(api_key, name, version); - let resp = handle_request(api_key, version, body, &default_broker()); + let resp = handle_request(api_key, version, body, &default_broker()).expect("test request has acks != 0 and expects a response"); assert!( !resp.is_empty(), "{name} v{version} returned empty response" @@ -67,7 +67,7 @@ fn produce_stub_response_has_zero_error_per_partition() { continue; } let body = load_fixture_body(0, "Produce", version); - let resp = handle_request(API_KEY_PRODUCE, version, body, &default_broker()); + let resp = handle_request(API_KEY_PRODUCE, version, body, &default_broker()).expect("test request has acks != 0 and expects a response"); let flexible = version >= 9; let mut d = Decoder::new(resp); if flexible { @@ -91,7 +91,7 @@ fn fetch_stub_response_has_zero_partition_error() { continue; } let body = load_fixture_body(1, "Fetch", version); - let resp = handle_request(API_KEY_FETCH, version, body, &default_broker()); + let resp = handle_request(API_KEY_FETCH, version, body, &default_broker()).expect("test request has acks != 0 and expects a response"); let flexible = version >= 12; let mut d = Decoder::new(resp); if version >= 1 { @@ -126,7 +126,7 @@ fn list_offsets_stub_response_has_zero_error() { continue; } let body = load_fixture_body(2, "ListOffsets", version); - let resp = handle_request(API_KEY_LIST_OFFSETS, version, body, &default_broker()); + let resp = handle_request(API_KEY_LIST_OFFSETS, version, body, &default_broker()).expect("test request has acks != 0 and expects a response"); let flexible = version >= 6; let mut d = Decoder::new(resp); if version >= 2 { @@ -153,7 +153,7 @@ fn create_topics_stub_response_has_zero_error() { continue; } let body = load_fixture_body(19, "CreateTopics", version); - let resp = handle_request(API_KEY_CREATE_TOPICS, version, body, &default_broker()); + let resp = handle_request(API_KEY_CREATE_TOPICS, version, body, &default_broker()).expect("test request has acks != 0 and expects a response"); let flexible = version >= 5; let mut d = Decoder::new(resp); if version >= 2 { diff --git a/gateways/kafka/tests/metadata_regression_tests.rs b/gateways/kafka/tests/metadata_regression_tests.rs index 1c33e38b7a..1a247fdbbb 100644 --- a/gateways/kafka/tests/metadata_regression_tests.rs +++ b/gateways/kafka/tests/metadata_regression_tests.rs @@ -68,7 +68,7 @@ fn metadata_corrupt_partial_body_returns_zero_topics() { 0, Bytes::from_static(&[0x00, 0x00]), &default_broker(), - ); + ).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let _ = read_broker_legacy(&mut d); assert_eq!(d.read_i32().unwrap(), 0); @@ -82,7 +82,7 @@ fn metadata_v0_empty_topics_stub_broker() { 0, metadata_request_legacy(0), &default_broker(), - ); + ).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let (host, port) = read_broker_legacy(&mut d); assert_eq!(host, "127.0.0.1"); @@ -97,7 +97,7 @@ fn metadata_v0_three_topics_each_unknown() { 0, metadata_request_legacy(3), &default_broker(), - ); + ).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let _ = read_broker_legacy(&mut d); assert_eq!(d.read_i32().unwrap(), 3); @@ -115,7 +115,7 @@ fn metadata_v1_includes_controller_id() { 1, metadata_request_legacy(0), &default_broker(), - ); + ).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); // Metadata v1 has no throttle_time_ms (added in v3). let _ = read_broker_legacy(&mut d); @@ -131,7 +131,7 @@ fn metadata_v2_includes_cluster_id_field() { 2, metadata_request_legacy(0), &default_broker(), - ); + ).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let _ = read_broker_legacy(&mut d); let _rack = d.read_nullable_string().unwrap(); @@ -148,7 +148,7 @@ fn metadata_all_legacy_versions_produce_valid_response() { version, metadata_request_legacy(1), &default_broker(), - ); + ).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); if version >= 3 { let _throttle = d.read_i32().unwrap(); @@ -175,7 +175,7 @@ fn metadata_v9_flexible_encoding() { 9, metadata_request_flexible(2), &default_broker(), - ); + ).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let _throttle = d.read_i32().unwrap(); let (host, port) = read_broker_flexible(&mut d); @@ -211,7 +211,7 @@ fn metadata_v8_includes_authorized_operations_legacy() { 8, metadata_request_legacy(1), &default_broker(), - ); + ).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let _throttle = d.read_i32().unwrap(); let _ = read_broker_legacy(&mut d); @@ -234,7 +234,7 @@ fn metadata_uses_custom_broker_advertise() { host: "10.0.0.42".to_string(), port: 29093, }; - let body = handle_request(API_KEY_METADATA, 0, metadata_request_legacy(0), &broker); + let body = handle_request(API_KEY_METADATA, 0, metadata_request_legacy(0), &broker).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let (host, port) = read_broker_legacy(&mut d); assert_eq!(host, "10.0.0.42"); diff --git a/gateways/kafka/tests/review_regression_tests.rs b/gateways/kafka/tests/review_regression_tests.rs index a1cebf1de5..5b8bced840 100644 --- a/gateways/kafka/tests/review_regression_tests.rs +++ b/gateways/kafka/tests/review_regression_tests.rs @@ -113,7 +113,7 @@ fn parse_list_offsets_v0_partition(d: &mut Decoder) { #[test] fn list_offsets_v0_unsupported_version_is_parseable_by_v0_clients() { - let body = handle_request(API_KEY_LIST_OFFSETS, 0, Bytes::new(), &default_broker()); + let body = handle_request(API_KEY_LIST_OFFSETS, 0, Bytes::new(), &default_broker()).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 1, "topics array length"); @@ -136,7 +136,7 @@ fn list_offsets_v0_unsupported_version_is_parseable_by_v0_clients() { #[test] fn list_offsets_v0_unsupported_version_carries_error_code_in_partition() { let request_body = build_list_offsets_v0_request_with_topic_t(); - let body = handle_request(API_KEY_LIST_OFFSETS, 0, request_body, &default_broker()); + let body = handle_request(API_KEY_LIST_OFFSETS, 0, request_body, &default_broker()).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 1); @@ -224,7 +224,7 @@ fn read_metadata_v1_topics(d: &mut Decoder, expected_count: i32) -> Vec fn metadata_v1_echoes_requested_topic_name_in_response() { let topic = "orders"; let request = build_metadata_legacy_request(&[topic]); - let body = handle_request(API_KEY_METADATA, 1, request, &default_broker()); + let body = handle_request(API_KEY_METADATA, 1, request, &default_broker()).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let names = read_metadata_v1_topics(&mut d, 1); @@ -236,7 +236,7 @@ fn metadata_v1_echoes_requested_topic_name_in_response() { fn metadata_v1_unknown_topic_returns_error_with_requested_name() { let topic = "orders"; let request = build_metadata_legacy_request(&[topic]); - let body = handle_request(API_KEY_METADATA, 1, request, &default_broker()); + let body = handle_request(API_KEY_METADATA, 1, request, &default_broker()).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let _brokers_count = d.read_i32().unwrap(); diff --git a/gateways/kafka/tests/scope_coverage_tests.rs b/gateways/kafka/tests/scope_coverage_tests.rs index ad922463c3..259bdecf4c 100644 --- a/gateways/kafka/tests/scope_coverage_tests.rs +++ b/gateways/kafka/tests/scope_coverage_tests.rs @@ -154,7 +154,7 @@ async fn apiversions_v0_and_v2_e2e_return_success() { #[test] fn out_of_scope_api_keys_return_unsupported_version_without_panic() { for &(api_key, name) in OUT_OF_SCOPE_API_KEYS { - let body = handle_request(api_key, 0, Bytes::new(), &default_broker()); + let body = handle_request(api_key, 0, Bytes::new(), &default_broker()).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!( d.read_i16().unwrap(), @@ -267,7 +267,7 @@ fn produce_advertises_min_zero_but_firewall_rejects_below_v3() { assert!(!is_supported_version(API_KEY_PRODUCE, 0)); assert!(!is_supported_version(API_KEY_PRODUCE, 2)); - let body = handle_request(API_KEY_PRODUCE, 2, Bytes::new(), &default_broker()); + let body = handle_request(API_KEY_PRODUCE, 2, Bytes::new(), &default_broker()).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let _topics = d.read_i32().unwrap(); let _name = d.read_nullable_string().unwrap(); @@ -285,7 +285,7 @@ fn metadata_v0_empty_topics_returns_zero_length_topic_array() { 0, metadata_empty_legacy_body(), &default_broker(), - ); + ).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let _brokers = d.read_i32().unwrap(); d.read_i32().unwrap(); @@ -302,7 +302,7 @@ fn metadata_v3_includes_throttle_time_ms_before_brokers() { 3, metadata_empty_legacy_body(), &default_broker(), - ); + ).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 0, "throttle_time_ms"); } @@ -314,7 +314,7 @@ fn metadata_v9_flexible_empty_topics_returns_zero_topics() { 9, build_metadata_flexible_request(&[]), &default_broker(), - ); + ).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); d.read_i32().unwrap(); // throttle let broker_count = usize::try_from(d.read_varint().unwrap()) @@ -343,7 +343,7 @@ fn metadata_v9_flexible_echoes_each_requested_topic_name() { 9, build_metadata_flexible_request(&topics), &default_broker(), - ); + ).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); d.read_i32().unwrap(); @@ -406,7 +406,7 @@ fn metadata_v1_legacy_multiple_topics_echo_names() { 1, build_metadata_legacy_request(&topics), &default_broker(), - ); + ).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); d.read_i32().unwrap(); @@ -668,7 +668,7 @@ async fn corrupt_fetch_body_e2e_returns_error_without_disconnect() { #[test] fn corrupt_list_offsets_body_returns_invalid_request_error() { let body = Bytes::from_static(&[0xFF, 0xFF, 0xFF]); - let resp = handle_request(API_KEY_LIST_OFFSETS, 1, body, &default_broker()); + let resp = handle_request(API_KEY_LIST_OFFSETS, 1, body, &default_broker()).expect("test request has acks != 0 and expects a response"); assert!(!resp.is_empty()); assert!( scan_for_error_code(&resp, ERROR_INVALID_REQUEST) @@ -680,7 +680,7 @@ fn corrupt_list_offsets_body_returns_invalid_request_error() { #[test] fn corrupt_create_topics_body_returns_invalid_request_error() { let body = Bytes::from_static(&[0xFF, 0xFF, 0xFF]); - let resp = handle_request(API_KEY_CREATE_TOPICS, 2, body, &default_broker()); + let resp = handle_request(API_KEY_CREATE_TOPICS, 2, body, &default_broker()).expect("test request has acks != 0 and expects a response"); assert!(!resp.is_empty()); assert!( scan_for_error_code(&resp, ERROR_INVALID_REQUEST) @@ -691,7 +691,7 @@ fn corrupt_create_topics_body_returns_invalid_request_error() { #[test] fn corrupt_metadata_body_returns_zero_topics_not_panic() { let body = Bytes::from_static(&[0x00, 0x00]); - let resp = handle_request(API_KEY_METADATA, 0, body, &default_broker()); + let resp = handle_request(API_KEY_METADATA, 0, body, &default_broker()).expect("test request has acks != 0 and expects a response"); assert!(!resp.is_empty()); let mut d = Decoder::new(resp); d.read_i32().unwrap(); @@ -738,7 +738,7 @@ fn metadata_v9_request_with_three_topics_yields_three_response_slots() { 9, build_metadata_flexible_request(&topics), &default_broker(), - ); + ).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); d.read_i32().unwrap(); skip_metadata_v9_prefix(&mut d); @@ -774,7 +774,7 @@ fn every_in_range_version_returns_non_empty_handler_response() { for &(api_key, name, min_ver, max_ver) in SCOPED_API_KEYS { for version in min_ver..=max_ver { let body = request_body_for_scoped_api(api_key, name, version); - let resp = handle_request(api_key, version, body, &default_broker()); + let resp = handle_request(api_key, version, body, &default_broker()).expect("test request has acks != 0 and expects a response"); assert!(!resp.is_empty(), "{name} v{version} handler returned empty"); } } diff --git a/gateways/kafka/tests/version_firewall_tests.rs b/gateways/kafka/tests/version_firewall_tests.rs index 77c4e768f8..053918af44 100644 --- a/gateways/kafka/tests/version_firewall_tests.rs +++ b/gateways/kafka/tests/version_firewall_tests.rs @@ -65,7 +65,7 @@ fn is_supported_version_matches_scope_table() { #[test] fn apiversions_advertises_exact_supported_ranges_v1() { - let body = handle_request(API_KEY_API_VERSIONS, 1, Bytes::new(), &default_broker()); + let body = handle_request(API_KEY_API_VERSIONS, 1, Bytes::new(), &default_broker()).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i16().unwrap(), 0); let count = usize::try_from(d.read_i32().unwrap()).expect("api count fits usize"); @@ -88,7 +88,7 @@ fn apiversions_advertises_exact_supported_ranges_v1() { #[test] fn apiversions_advertises_exact_supported_ranges_v3_flexible() { - let body = handle_request(API_KEY_API_VERSIONS, 3, Bytes::new(), &default_broker()); + let body = handle_request(API_KEY_API_VERSIONS, 3, Bytes::new(), &default_broker()).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i16().unwrap(), 0); let count = usize::try_from(d.read_varint().unwrap() - 1).expect("api count fits usize"); @@ -133,7 +133,7 @@ fn apiversions_all_versions_return_success() { version, Bytes::new(), &default_broker(), - ); + ).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i16().unwrap(), 0, "ApiVersions v{version}"); } @@ -141,7 +141,7 @@ fn apiversions_all_versions_return_success() { #[test] fn apiversions_out_of_range_returns_unsupported_in_body() { - let body = handle_request(API_KEY_API_VERSIONS, 99, Bytes::new(), &default_broker()); + let body = handle_request(API_KEY_API_VERSIONS, 99, Bytes::new(), &default_broker()).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); } @@ -159,7 +159,7 @@ fn metadata_below_min_version_returns_topic_error() { -1, metadata_request_one_topic(), &default_broker(), - ); + ).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let _brokers = d.read_i32().unwrap(); let _ = d.read_i32().unwrap(); @@ -177,7 +177,7 @@ fn metadata_above_max_version_returns_topic_error() { 10, Bytes::from_static(&[0x02]), &default_broker(), - ); + ).expect("test request has acks != 0 and expects a response"); // Response is in v9 flexible format (highest supported). let mut d = Decoder::new(body); d.read_i32().unwrap(); // throttle_time_ms (v3+) @@ -202,7 +202,7 @@ fn metadata_above_max_version_returns_topic_error() { #[test] fn produce_unsupported_version_returns_well_formed_error_response() { - let body = handle_request(API_KEY_PRODUCE, 2, Bytes::new(), &default_broker()); + let body = handle_request(API_KEY_PRODUCE, 2, Bytes::new(), &default_broker()).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 1); assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); @@ -217,7 +217,7 @@ fn produce_unsupported_version_returns_well_formed_error_response() { #[test] fn fetch_unsupported_version_returns_well_formed_error_response() { - let body = handle_request(API_KEY_FETCH, 3, Bytes::new(), &default_broker()); + let body = handle_request(API_KEY_FETCH, 3, Bytes::new(), &default_broker()).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 0); assert_eq!(d.read_i32().unwrap(), 1); @@ -232,7 +232,7 @@ fn fetch_unsupported_version_returns_well_formed_error_response() { #[test] fn fetch_unsupported_version_above_max_uses_top_level_error() { - let body = handle_request(API_KEY_FETCH, 13, Bytes::new(), &default_broker()); + let body = handle_request(API_KEY_FETCH, 13, Bytes::new(), &default_broker()).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 0); assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); @@ -244,7 +244,7 @@ fn fetch_unsupported_version_above_max_uses_top_level_error() { #[test] fn list_offsets_unsupported_version_returns_well_formed_error_response() { - let body = handle_request(API_KEY_LIST_OFFSETS, 0, Bytes::new(), &default_broker()); + let body = handle_request(API_KEY_LIST_OFFSETS, 0, Bytes::new(), &default_broker()).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 1); assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); @@ -257,7 +257,7 @@ fn list_offsets_unsupported_version_returns_well_formed_error_response() { #[test] fn create_topics_unsupported_version_returns_well_formed_error_response() { - let body = handle_request(API_KEY_CREATE_TOPICS, 1, Bytes::new(), &default_broker()); + let body = handle_request(API_KEY_CREATE_TOPICS, 1, Bytes::new(), &default_broker()).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 1); assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); @@ -269,7 +269,7 @@ fn create_topics_unsupported_version_returns_well_formed_error_response() { #[test] fn unsupported_api_keys_return_error_only() { for key in [8, 9, 10, 11, 17, 20, 42, 999] { - let body = handle_request(key, 0, Bytes::new(), &default_broker()); + let body = handle_request(key, 0, Bytes::new(), &default_broker()).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!( d.read_i16().unwrap(), @@ -283,7 +283,7 @@ fn unsupported_api_keys_return_error_only() { fn supported_produce_versions_accept_valid_fixture() { for version in 3i16..=9 { let body = load_fixture_body(0, "Produce", version); - let resp = handle_request(API_KEY_PRODUCE, version, body, &default_broker()); + let resp = handle_request(API_KEY_PRODUCE, version, body, &default_broker()).expect("test request has acks != 0 and expects a response"); assert!(!resp.is_empty(), "Produce v{version} response empty"); } } @@ -292,7 +292,7 @@ fn supported_produce_versions_accept_valid_fixture() { fn supported_fetch_versions_accept_valid_fixture() { for version in 4i16..=12 { let body = load_fixture_body(1, "Fetch", version); - let resp = handle_request(API_KEY_FETCH, version, body, &default_broker()); + let resp = handle_request(API_KEY_FETCH, version, body, &default_broker()).expect("test request has acks != 0 and expects a response"); assert!(!resp.is_empty(), "Fetch v{version} response empty"); } } @@ -300,7 +300,7 @@ fn supported_fetch_versions_accept_valid_fixture() { #[test] fn corrupt_produce_body_returns_invalid_request_error() { let body = Bytes::from_static(&[0xFF, 0xFF, 0xFF]); - let resp = handle_request(API_KEY_PRODUCE, 3, body, &default_broker()); + let resp = handle_request(API_KEY_PRODUCE, 3, body, &default_broker()).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(resp); assert_eq!(d.read_i32().unwrap(), 1); assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); @@ -312,7 +312,7 @@ fn corrupt_produce_body_returns_invalid_request_error() { #[test] fn corrupt_fetch_body_returns_invalid_request_error() { let body = Bytes::from_static(&[0xFF, 0xFF, 0xFF]); - let resp = handle_request(API_KEY_FETCH, 4, body, &default_broker()); + let resp = handle_request(API_KEY_FETCH, 4, body, &default_broker()).expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(resp); assert_eq!(d.read_i32().unwrap(), 0); assert_eq!(d.read_i32().unwrap(), 1); From f78ecb6f429034b808f89aadc7d7464306ec9e1c Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Mon, 6 Jul 2026 07:30:11 -0400 Subject: [PATCH 22/57] Echo metadata topic names in responses and test cases. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Metadata handling now decodes requested topic names (legacy and flexible formats) and echoes those names in topic entries instead of always returning `unknown-topic`. This keeps responses aligned with Kafka client expectations, including unsupported-version paths encoded at the capped metadata version. The request dispatcher was also refactored to isolate Produce’s no-response (`acks=0`) behavior into its own path, and tests/fixtures were updated to cover flexible per-topic tagged fields and name-echo regression cases. --- gateways/kafka/src/protocol/api.rs | 152 ++++++++++-------- gateways/kafka/tests/api_handler_tests.rs | 32 ++-- .../kafka/tests/broker_advertise_tests.rs | 3 +- gateways/kafka/tests/common/wire.rs | 5 + .../kafka/tests/golden_wire_fixtures_tests.rs | 18 ++- .../kafka/tests/handler_regression_tests.rs | 18 ++- .../kafka/tests/metadata_regression_tests.rs | 55 +++++-- .../kafka/tests/review_regression_tests.rs | 12 +- gateways/kafka/tests/scope_coverage_tests.rs | 36 +++-- .../kafka/tests/version_firewall_tests.rs | 48 ++++-- 10 files changed, 244 insertions(+), 135 deletions(-) diff --git a/gateways/kafka/src/protocol/api.rs b/gateways/kafka/src/protocol/api.rs index 414d5bea90..4453441abf 100644 --- a/gateways/kafka/src/protocol/api.rs +++ b/gateways/kafka/src/protocol/api.rs @@ -17,7 +17,7 @@ use bytes::Bytes; -use crate::error::Result; +use crate::error::{KafkaProtocolError, Result}; use crate::protocol::codec::{Decoder, Encoder}; use crate::protocol::requests::{ decode_create_topics_request, decode_fetch_request, decode_list_offsets_request, @@ -117,105 +117,104 @@ pub fn handle_request( body: Bytes, broker: &BrokerAdvertise, ) -> Option { + if api_key == API_KEY_PRODUCE { + return handle_produce_request(api_version, body); + } + Some(handle_other_request(api_key, api_version, body, broker)) +} + +/// Produce is the only request the wire protocol allows to go unanswered +/// (`acks=0`), so it gets its own `Option`-returning path. +fn handle_produce_request(api_version: i16, body: Bytes) -> Option { + if !is_supported_version(API_KEY_PRODUCE, api_version) { + return Some(encode_produce_error_response( + api_version, + ERROR_UNSUPPORTED_VERSION, + )); + } + match decode_produce_request(api_version, body) { + // acks=0 is fire-and-forget: the client isn't reading a response, so + // sending one desyncs the next correlation id it expects. + Ok(req) if req.acks == 0 => None, + Ok(req) => Some(encode_produce_response(api_version, &req)), + Err(e) => { + tracing::warn!("Failed to decode Produce request: {:?}", e); + Some(encode_produce_error_response( + api_version, + ERROR_INVALID_REQUEST, + )) + } + } +} + +fn handle_other_request( + api_key: i16, + api_version: i16, + body: Bytes, + broker: &BrokerAdvertise, +) -> Bytes { match api_key { API_KEY_API_VERSIONS => { if is_supported_version(api_key, api_version) { - Some(encode_api_versions_response(api_version, ERROR_NONE)) + encode_api_versions_response(api_version, ERROR_NONE) } else { // KIP-511: reply with v0 when the requested version is not understood. - Some(encode_api_versions_response(0, ERROR_UNSUPPORTED_VERSION)) + encode_api_versions_response(0, ERROR_UNSUPPORTED_VERSION) } } API_KEY_METADATA => { if is_supported_version(api_key, api_version) { - Some(encode_metadata_response(api_version, body, broker, ERROR_NONE)) + encode_metadata_response(api_version, body, broker, ERROR_NONE) } else { // Encode at the highest version we implement, not the client's unknown version. - Some(encode_metadata_response( + encode_metadata_response( api_version.clamp(0, MAX_SUPPORTED_METADATA_VERSION), body, broker, ERROR_UNSUPPORTED_VERSION, - )) - } - } - API_KEY_PRODUCE => { - if is_supported_version(api_key, api_version) { - match decode_produce_request(api_version, body) { - // acks=0 is fire-and-forget: the client isn't reading a response, so - // sending one desyncs the next correlation id it expects. - Ok(req) if req.acks == 0 => None, - Ok(req) => Some(encode_produce_response(api_version, &req)), - Err(e) => { - tracing::warn!("Failed to decode Produce request: {:?}", e); - Some(encode_produce_error_response( - api_version, - ERROR_INVALID_REQUEST, - )) - } - } - } else { - Some(encode_produce_error_response( - api_version, - ERROR_UNSUPPORTED_VERSION, - )) + ) } } API_KEY_FETCH => { if is_supported_version(api_key, api_version) { match decode_fetch_request(api_version, body) { - Ok(req) => Some(encode_fetch_response(api_version, &req)), + Ok(req) => encode_fetch_response(api_version, &req), Err(e) => { tracing::warn!("Failed to decode Fetch request: {:?}", e); - Some(encode_fetch_error_response(api_version, ERROR_INVALID_REQUEST)) + encode_fetch_error_response(api_version, ERROR_INVALID_REQUEST) } } } else { - Some(encode_fetch_error_response( - api_version, - ERROR_UNSUPPORTED_VERSION, - )) + encode_fetch_error_response(api_version, ERROR_UNSUPPORTED_VERSION) } } API_KEY_LIST_OFFSETS => { if is_supported_version(api_key, api_version) { match decode_list_offsets_request(api_version, body) { - Ok(req) => Some(encode_list_offsets_response(api_version, &req)), + Ok(req) => encode_list_offsets_response(api_version, &req), Err(e) => { tracing::warn!("Failed to decode ListOffsets request: {:?}", e); - Some(encode_list_offsets_error_response( - api_version, - ERROR_INVALID_REQUEST, - )) + encode_list_offsets_error_response(api_version, ERROR_INVALID_REQUEST) } } } else { - Some(encode_list_offsets_error_response( - api_version, - ERROR_UNSUPPORTED_VERSION, - )) + encode_list_offsets_error_response(api_version, ERROR_UNSUPPORTED_VERSION) } } API_KEY_CREATE_TOPICS => { if is_supported_version(api_key, api_version) { match decode_create_topics_request(api_version, body) { - Ok(req) => Some(encode_create_topics_response(api_version, &req)), + Ok(req) => encode_create_topics_response(api_version, &req), Err(e) => { tracing::warn!("Failed to decode CreateTopics request: {:?}", e); - Some(encode_create_topics_error_response( - api_version, - ERROR_INVALID_REQUEST, - )) + encode_create_topics_error_response(api_version, ERROR_INVALID_REQUEST) } } } else { - Some(encode_create_topics_error_response( - api_version, - ERROR_UNSUPPORTED_VERSION, - )) + encode_create_topics_error_response(api_version, ERROR_UNSUPPORTED_VERSION) } } - _ => Some(encode_error_only_response(ERROR_UNSUPPORTED_VERSION)), + _ => encode_error_only_response(ERROR_UNSUPPORTED_VERSION), } } @@ -286,12 +285,15 @@ fn encode_metadata_response( // Non-empty body that fails to decode = malformed request; return 0 topics. // Kafka Metadata response has no top-level error code field: errors are per-topic only. // 0 topics is spec-correct and unambiguous for a decode failure. - let (topics_count, effective_error) = if body.is_empty() { - (0usize, top_level_error_code) + let (topics, effective_error) = if body.is_empty() { + (Vec::new(), top_level_error_code) } else { - split_metadata_request_topics(body, api_version) - .map_or((0, ERROR_INVALID_REQUEST), |n| (n, top_level_error_code)) + decode_metadata_request_topics(body, api_version) + .map_or((Vec::new(), ERROR_INVALID_REQUEST), |names| { + (names, top_level_error_code) + }) }; + let topics_count = topics.len(); let topic_error = if effective_error == ERROR_NONE { ERROR_UNKNOWN_TOPIC_OR_PARTITION } else { @@ -316,9 +318,9 @@ fn encode_metadata_response( e.write_i32(1); // controller_id (v1+) e.write_varint((topics_count + 1) as u64); - for _ in 0..topics_count { + for name in &topics { e.write_i16(topic_error); - e.write_compact_nullable_string(Some("unknown-topic")); + e.write_compact_nullable_string(Some(name)); e.write_bool(false); // is_internal (v1+) e.write_varint(1); // empty partitions array e.write_i32(AUTHORIZED_OPS_UNKNOWN); // topic_authorized_operations (v8+) @@ -347,9 +349,9 @@ fn encode_metadata_response( } e.write_i32(i32::try_from(topics_count).expect("topic count bounded")); - for _ in 0..topics_count { + for name in &topics { e.write_i16(topic_error); - e.write_nullable_string_unchecked(Some("unknown-topic")); + e.write_nullable_string_unchecked(Some(name)); if api_version >= 1 { e.write_bool(false); // is_internal } @@ -373,11 +375,31 @@ pub fn encode_error_only_response(error_code: i16) -> Bytes { e.freeze() } -pub(crate) fn split_metadata_request_topics(body: Bytes, api_version: i16) -> Result { +/// Decodes the requested topic names from a Metadata request body so the +/// response can echo them back; clients match metadata by name, not position. +pub(crate) fn decode_metadata_request_topics(body: Bytes, api_version: i16) -> Result> { let mut d = Decoder::new(body); - if api_version >= 9 { - d.read_compact_array_count() + let flexible = api_version >= 9; + let topics_count = if flexible { + d.read_compact_array_count()? } else { - d.read_i32_array_count() + d.read_i32_array_count()? + }; + + let mut topics = Vec::with_capacity(topics_count); + for _ in 0..topics_count { + let name = if flexible { + d.read_compact_nullable_string()? + .ok_or(KafkaProtocolError::NullTopicName)? + } else { + d.read_nullable_string()? + .ok_or(KafkaProtocolError::NullTopicName)? + }; + topics.push(name); + if flexible { + d.read_tagged_fields()?; + } } + + Ok(topics) } diff --git a/gateways/kafka/tests/api_handler_tests.rs b/gateways/kafka/tests/api_handler_tests.rs index b51f882c12..17db5de2f8 100644 --- a/gateways/kafka/tests/api_handler_tests.rs +++ b/gateways/kafka/tests/api_handler_tests.rs @@ -15,6 +15,9 @@ // specific language governing permissions and limitations // under the License. +#[path = "common/wire.rs"] +mod wire; + use bytes::Bytes; use iggy_gateway_kafka::protocol::api::{ @@ -26,12 +29,14 @@ fn test_broker() -> BrokerAdvertise { BrokerAdvertise::default() } use iggy_gateway_kafka::protocol::codec::Decoder; +use wire::build_metadata_flexible_request; // ── ApiVersions ───────────────────────────────────────────────────────────── #[test] fn api_versions_v1_response_non_flexible_format() { - let body = handle_request(API_KEY_API_VERSIONS, 1, Bytes::new(), &test_broker()).expect("test request has acks != 0 and expects a response"); + let body = handle_request(API_KEY_API_VERSIONS, 1, Bytes::new(), &test_broker()) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i16().unwrap(), 0); // error_code @@ -55,7 +60,8 @@ fn api_versions_v1_response_non_flexible_format() { #[test] fn api_versions_v3_response_flexible_format() { - let body = handle_request(API_KEY_API_VERSIONS, 3, Bytes::new(), &test_broker()).expect("test request has acks != 0 and expects a response"); + let body = handle_request(API_KEY_API_VERSIONS, 3, Bytes::new(), &test_broker()) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i16().unwrap(), 0); // error_code @@ -85,7 +91,8 @@ fn api_versions_v3_response_flexible_format() { #[test] fn metadata_response_has_broker_array_and_topic_array() { - let body = handle_request(API_KEY_METADATA, 0, Bytes::new(), &test_broker()).expect("test request has acks != 0 and expects a response"); + let body = handle_request(API_KEY_METADATA, 0, Bytes::new(), &test_broker()) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let broker_count = d.read_i32().unwrap(); @@ -103,15 +110,16 @@ fn metadata_response_has_broker_array_and_topic_array() { #[test] fn unsupported_version_returns_protocol_error() { - // v99 client sends a compact-array body (count+1 = 2 = 0x02 for 1 topic). + // v99 client sends a well-formed v9 flexible body requesting "orders". // The gateway caps at v9 (highest supported Metadata version) for both parsing // and encoding, so the response uses the flexible (v9) wire format. let body = handle_request( API_KEY_METADATA, 99, - Bytes::from_static(&[0x02]), + build_metadata_flexible_request(&["orders"]), &test_broker(), - ).expect("test request has acks != 0 and expects a response"); + ) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); // v9 flexible response layout: d.read_i32().unwrap(); // throttle_time_ms (v3+) @@ -134,14 +142,19 @@ fn unsupported_version_returns_protocol_error() { let topic_error = d.read_i16().unwrap(); assert_eq!(topic_error, ERROR_UNSUPPORTED_VERSION); let topic_name = d.read_compact_nullable_string().unwrap(); - assert_eq!(topic_name, Some("unknown-topic".to_string())); + assert_eq!( + topic_name, + Some("orders".to_string()), + "metadata must echo the requested topic name even on an unsupported-version reply" + ); } // ── Misc ──────────────────────────────────────────────────────────────────── #[test] fn unknown_api_key_returns_error_only_payload() { - let body = handle_request(999, 0, Bytes::new(), &test_broker()).expect("test request has acks != 0 and expects a response"); + let body = handle_request(999, 0, Bytes::new(), &test_broker()) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); } @@ -156,7 +169,8 @@ fn version_support_table_is_applied() { #[test] fn apiversions_unsupported_version_uses_v0_encoding_without_throttle() { - let body = handle_request(API_KEY_API_VERSIONS, 99, Bytes::new(), &test_broker()).expect("test request has acks != 0 and expects a response"); + let body = handle_request(API_KEY_API_VERSIONS, 99, Bytes::new(), &test_broker()) + .expect("test request has acks != 0 and expects a response"); // v0: error_code(2) + api_keys i32 count(4) + 6 entries × 6 bytes = 42 — no throttle_time_ms. assert_eq!(body.len(), 42); let mut d = Decoder::new(body); diff --git a/gateways/kafka/tests/broker_advertise_tests.rs b/gateways/kafka/tests/broker_advertise_tests.rs index 3e6af07ee2..174947478a 100644 --- a/gateways/kafka/tests/broker_advertise_tests.rs +++ b/gateways/kafka/tests/broker_advertise_tests.rs @@ -38,7 +38,8 @@ fn metadata_reflects_broker_addr() { }; let mut req = Encoder::with_capacity(4); req.write_i32(0); - let body = handle_request(API_KEY_METADATA, 0, req.freeze(), &broker).expect("test request has acks != 0 and expects a response"); + let body = handle_request(API_KEY_METADATA, 0, req.freeze(), &broker) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 1); diff --git a/gateways/kafka/tests/common/wire.rs b/gateways/kafka/tests/common/wire.rs index 3f3277ce85..f1aa9890b6 100644 --- a/gateways/kafka/tests/common/wire.rs +++ b/gateways/kafka/tests/common/wire.rs @@ -48,11 +48,16 @@ pub const FLEXIBLE_FROM_VERSION: &[(i16, i16)] = &[ ]; /// Metadata v9+ flexible request listing topic names (compact strings). +/// +/// Each topic entry is its own tagged struct per the Kafka protocol schema +/// (`topics => name TAG_BUFFER`), so the per-topic tag buffer is written +/// right after each name, not once for the whole array. pub fn build_metadata_flexible_request(topic_names: &[&str]) -> Bytes { let mut enc = Encoder::with_capacity(64); enc.write_varint((topic_names.len() + 1) as u64); for name in topic_names { enc.write_compact_nullable_string(Some(name)); + enc.write_empty_tagged_fields(); } enc.write_empty_tagged_fields(); enc.freeze() diff --git a/gateways/kafka/tests/golden_wire_fixtures_tests.rs b/gateways/kafka/tests/golden_wire_fixtures_tests.rs index bdf202ce67..52e8587463 100644 --- a/gateways/kafka/tests/golden_wire_fixtures_tests.rs +++ b/gateways/kafka/tests/golden_wire_fixtures_tests.rs @@ -25,7 +25,8 @@ use iggy_gateway_kafka::protocol::codec::Encoder; #[test] fn golden_apiversions_v1_response_fixture() { let broker = BrokerAdvertise::default(); - let actual = handle_request(API_KEY_API_VERSIONS, 1, Bytes::new(), &broker).expect("test request has acks != 0 and expects a response"); + let actual = handle_request(API_KEY_API_VERSIONS, 1, Bytes::new(), &broker) + .expect("test request has acks != 0 and expects a response"); // error_code=0, api_count=6 // key 0 (Produce) min=0 max=9 (KAFKA-18659 advertise min=0) @@ -53,14 +54,18 @@ fn golden_apiversions_v1_response_fixture() { fn golden_metadata_v0_single_topic_response_fixture() { let mut request = Encoder::with_capacity(32); request.write_i32(1); // one topic + request + .write_nullable_string(Some("orders")) + .expect("topic name fits"); let req_bytes = request.freeze(); - let actual = handle_request(API_KEY_METADATA, 0, req_bytes, &BrokerAdvertise::default()).expect("test request has acks != 0 and expects a response"); + let actual = handle_request(API_KEY_METADATA, 0, req_bytes, &BrokerAdvertise::default()) + .expect("test request has acks != 0 and expects a response"); // Metadata v0 layout: brokers[], topics[] (no controller_id — added in v1) // brokers[1]: node_id=1, host=127.0.0.1, port=9093 - // topics[1]: topic_error=3, topic_name=unknown-topic, partitions[0] - let expected: [u8; 48] = [ + // topics[1]: topic_error=3, topic_name=orders (echoed from the request), partitions[0] + let expected: [u8; 41] = [ 0x00, 0x00, 0x00, 0x01, // broker count 0x00, 0x00, 0x00, 0x01, // node id 0x00, 0x09, // host len @@ -68,9 +73,8 @@ fn golden_metadata_v0_single_topic_response_fixture() { 0x00, 0x00, 0x23, 0x85, // port 9093 0x00, 0x00, 0x00, 0x01, // topic count 0x00, 0x03, // topic error code - 0x00, 0x0d, // topic name len - 0x75, 0x6e, 0x6b, 0x6e, 0x6f, 0x77, 0x6e, 0x2d, 0x74, 0x6f, 0x70, 0x69, - 0x63, // unknown-topic + 0x00, 0x06, // topic name len + 0x6f, 0x72, 0x64, 0x65, 0x72, 0x73, // "orders" 0x00, 0x00, 0x00, 0x00, // partition count ]; assert_eq!(actual.as_ref(), &expected); diff --git a/gateways/kafka/tests/handler_regression_tests.rs b/gateways/kafka/tests/handler_regression_tests.rs index 9f69e28e4c..214c650c66 100644 --- a/gateways/kafka/tests/handler_regression_tests.rs +++ b/gateways/kafka/tests/handler_regression_tests.rs @@ -37,7 +37,8 @@ fn handle_request_succeeds_for_every_supported_version_with_fixture() { if api_key == 3 || api_key == 18 { // Metadata / ApiVersions: empty body is valid for version in min_ver..=max_ver { - let resp = handle_request(api_key, version, bytes::Bytes::new(), &default_broker()).expect("test request has acks != 0 and expects a response"); + let resp = handle_request(api_key, version, bytes::Bytes::new(), &default_broker()) + .expect("test request has acks != 0 and expects a response"); assert!( !resp.is_empty(), "{name} v{version} returned empty response" @@ -51,7 +52,8 @@ fn handle_request_succeeds_for_every_supported_version_with_fixture() { continue; } let body = load_fixture_body(api_key, name, version); - let resp = handle_request(api_key, version, body, &default_broker()).expect("test request has acks != 0 and expects a response"); + let resp = handle_request(api_key, version, body, &default_broker()) + .expect("test request has acks != 0 and expects a response"); assert!( !resp.is_empty(), "{name} v{version} returned empty response" @@ -67,7 +69,8 @@ fn produce_stub_response_has_zero_error_per_partition() { continue; } let body = load_fixture_body(0, "Produce", version); - let resp = handle_request(API_KEY_PRODUCE, version, body, &default_broker()).expect("test request has acks != 0 and expects a response"); + let resp = handle_request(API_KEY_PRODUCE, version, body, &default_broker()) + .expect("test request has acks != 0 and expects a response"); let flexible = version >= 9; let mut d = Decoder::new(resp); if flexible { @@ -91,7 +94,8 @@ fn fetch_stub_response_has_zero_partition_error() { continue; } let body = load_fixture_body(1, "Fetch", version); - let resp = handle_request(API_KEY_FETCH, version, body, &default_broker()).expect("test request has acks != 0 and expects a response"); + let resp = handle_request(API_KEY_FETCH, version, body, &default_broker()) + .expect("test request has acks != 0 and expects a response"); let flexible = version >= 12; let mut d = Decoder::new(resp); if version >= 1 { @@ -126,7 +130,8 @@ fn list_offsets_stub_response_has_zero_error() { continue; } let body = load_fixture_body(2, "ListOffsets", version); - let resp = handle_request(API_KEY_LIST_OFFSETS, version, body, &default_broker()).expect("test request has acks != 0 and expects a response"); + let resp = handle_request(API_KEY_LIST_OFFSETS, version, body, &default_broker()) + .expect("test request has acks != 0 and expects a response"); let flexible = version >= 6; let mut d = Decoder::new(resp); if version >= 2 { @@ -153,7 +158,8 @@ fn create_topics_stub_response_has_zero_error() { continue; } let body = load_fixture_body(19, "CreateTopics", version); - let resp = handle_request(API_KEY_CREATE_TOPICS, version, body, &default_broker()).expect("test request has acks != 0 and expects a response"); + let resp = handle_request(API_KEY_CREATE_TOPICS, version, body, &default_broker()) + .expect("test request has acks != 0 and expects a response"); let flexible = version >= 5; let mut d = Decoder::new(resp); if version >= 2 { diff --git a/gateways/kafka/tests/metadata_regression_tests.rs b/gateways/kafka/tests/metadata_regression_tests.rs index 1a247fdbbb..f61ff05809 100644 --- a/gateways/kafka/tests/metadata_regression_tests.rs +++ b/gateways/kafka/tests/metadata_regression_tests.rs @@ -29,15 +29,28 @@ use iggy_gateway_kafka::protocol::codec::{Decoder, Encoder}; use scope::default_broker; +/// Topic name assigned to slot `i` by [`metadata_request_legacy`] / [`metadata_request_flexible`]. +fn synthetic_topic_name(i: i32) -> String { + format!("topic-{i}") +} + fn metadata_request_legacy(topic_count: i32) -> Bytes { - let mut enc = Encoder::with_capacity(8); + let mut enc = Encoder::with_capacity(64); enc.write_i32(topic_count); + for i in 0..topic_count { + enc.write_nullable_string(Some(&synthetic_topic_name(i))) + .expect("topic name fits"); + } enc.freeze() } fn metadata_request_flexible(topic_count: usize) -> Bytes { - let mut enc = Encoder::with_capacity(8); + let mut enc = Encoder::with_capacity(64); enc.write_varint((topic_count + 1) as u64); + for i in 0..topic_count { + enc.write_compact_nullable_string(Some(&synthetic_topic_name(i32::try_from(i).unwrap()))); + enc.write_empty_tagged_fields(); + } enc.freeze() } @@ -68,7 +81,8 @@ fn metadata_corrupt_partial_body_returns_zero_topics() { 0, Bytes::from_static(&[0x00, 0x00]), &default_broker(), - ).expect("test request has acks != 0 and expects a response"); + ) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let _ = read_broker_legacy(&mut d); assert_eq!(d.read_i32().unwrap(), 0); @@ -82,7 +96,8 @@ fn metadata_v0_empty_topics_stub_broker() { 0, metadata_request_legacy(0), &default_broker(), - ).expect("test request has acks != 0 and expects a response"); + ) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let (host, port) = read_broker_legacy(&mut d); assert_eq!(host, "127.0.0.1"); @@ -97,13 +112,17 @@ fn metadata_v0_three_topics_each_unknown() { 0, metadata_request_legacy(3), &default_broker(), - ).expect("test request has acks != 0 and expects a response"); + ) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let _ = read_broker_legacy(&mut d); assert_eq!(d.read_i32().unwrap(), 3); - for _ in 0..3 { + for i in 0..3 { assert_eq!(d.read_i16().unwrap(), ERROR_UNKNOWN_TOPIC_OR_PARTITION); - assert_eq!(d.read_nullable_string().unwrap().unwrap(), "unknown-topic"); + assert_eq!( + d.read_nullable_string().unwrap().unwrap(), + synthetic_topic_name(i) + ); assert_eq!(d.read_i32().unwrap(), 0); } } @@ -115,7 +134,8 @@ fn metadata_v1_includes_controller_id() { 1, metadata_request_legacy(0), &default_broker(), - ).expect("test request has acks != 0 and expects a response"); + ) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); // Metadata v1 has no throttle_time_ms (added in v3). let _ = read_broker_legacy(&mut d); @@ -131,7 +151,8 @@ fn metadata_v2_includes_cluster_id_field() { 2, metadata_request_legacy(0), &default_broker(), - ).expect("test request has acks != 0 and expects a response"); + ) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let _ = read_broker_legacy(&mut d); let _rack = d.read_nullable_string().unwrap(); @@ -148,7 +169,8 @@ fn metadata_all_legacy_versions_produce_valid_response() { version, metadata_request_legacy(1), &default_broker(), - ).expect("test request has acks != 0 and expects a response"); + ) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); if version >= 3 { let _throttle = d.read_i32().unwrap(); @@ -175,7 +197,8 @@ fn metadata_v9_flexible_encoding() { 9, metadata_request_flexible(2), &default_broker(), - ).expect("test request has acks != 0 and expects a response"); + ) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let _throttle = d.read_i32().unwrap(); let (host, port) = read_broker_flexible(&mut d); @@ -187,11 +210,11 @@ fn metadata_v9_flexible_encoding() { let topics_plus_one = d.read_varint().unwrap(); assert_eq!(topics_plus_one, 3); // 2 topics - for _ in 0..2 { + for i in 0..2 { assert_eq!(d.read_i16().unwrap(), ERROR_UNKNOWN_TOPIC_OR_PARTITION); assert_eq!( d.read_compact_nullable_string().unwrap().unwrap(), - "unknown-topic" + synthetic_topic_name(i) ); let _internal = d.read_bool().unwrap(); let parts_plus_one = d.read_varint().unwrap(); @@ -211,7 +234,8 @@ fn metadata_v8_includes_authorized_operations_legacy() { 8, metadata_request_legacy(1), &default_broker(), - ).expect("test request has acks != 0 and expects a response"); + ) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let _throttle = d.read_i32().unwrap(); let _ = read_broker_legacy(&mut d); @@ -234,7 +258,8 @@ fn metadata_uses_custom_broker_advertise() { host: "10.0.0.42".to_string(), port: 29093, }; - let body = handle_request(API_KEY_METADATA, 0, metadata_request_legacy(0), &broker).expect("test request has acks != 0 and expects a response"); + let body = handle_request(API_KEY_METADATA, 0, metadata_request_legacy(0), &broker) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let (host, port) = read_broker_legacy(&mut d); assert_eq!(host, "10.0.0.42"); diff --git a/gateways/kafka/tests/review_regression_tests.rs b/gateways/kafka/tests/review_regression_tests.rs index 5b8bced840..2e7d2cd01c 100644 --- a/gateways/kafka/tests/review_regression_tests.rs +++ b/gateways/kafka/tests/review_regression_tests.rs @@ -113,7 +113,8 @@ fn parse_list_offsets_v0_partition(d: &mut Decoder) { #[test] fn list_offsets_v0_unsupported_version_is_parseable_by_v0_clients() { - let body = handle_request(API_KEY_LIST_OFFSETS, 0, Bytes::new(), &default_broker()).expect("test request has acks != 0 and expects a response"); + let body = handle_request(API_KEY_LIST_OFFSETS, 0, Bytes::new(), &default_broker()) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 1, "topics array length"); @@ -136,7 +137,8 @@ fn list_offsets_v0_unsupported_version_is_parseable_by_v0_clients() { #[test] fn list_offsets_v0_unsupported_version_carries_error_code_in_partition() { let request_body = build_list_offsets_v0_request_with_topic_t(); - let body = handle_request(API_KEY_LIST_OFFSETS, 0, request_body, &default_broker()).expect("test request has acks != 0 and expects a response"); + let body = handle_request(API_KEY_LIST_OFFSETS, 0, request_body, &default_broker()) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 1); @@ -224,7 +226,8 @@ fn read_metadata_v1_topics(d: &mut Decoder, expected_count: i32) -> Vec fn metadata_v1_echoes_requested_topic_name_in_response() { let topic = "orders"; let request = build_metadata_legacy_request(&[topic]); - let body = handle_request(API_KEY_METADATA, 1, request, &default_broker()).expect("test request has acks != 0 and expects a response"); + let body = handle_request(API_KEY_METADATA, 1, request, &default_broker()) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let names = read_metadata_v1_topics(&mut d, 1); @@ -236,7 +239,8 @@ fn metadata_v1_echoes_requested_topic_name_in_response() { fn metadata_v1_unknown_topic_returns_error_with_requested_name() { let topic = "orders"; let request = build_metadata_legacy_request(&[topic]); - let body = handle_request(API_KEY_METADATA, 1, request, &default_broker()).expect("test request has acks != 0 and expects a response"); + let body = handle_request(API_KEY_METADATA, 1, request, &default_broker()) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let _brokers_count = d.read_i32().unwrap(); diff --git a/gateways/kafka/tests/scope_coverage_tests.rs b/gateways/kafka/tests/scope_coverage_tests.rs index 259bdecf4c..32db4aa80b 100644 --- a/gateways/kafka/tests/scope_coverage_tests.rs +++ b/gateways/kafka/tests/scope_coverage_tests.rs @@ -154,7 +154,8 @@ async fn apiversions_v0_and_v2_e2e_return_success() { #[test] fn out_of_scope_api_keys_return_unsupported_version_without_panic() { for &(api_key, name) in OUT_OF_SCOPE_API_KEYS { - let body = handle_request(api_key, 0, Bytes::new(), &default_broker()).expect("test request has acks != 0 and expects a response"); + let body = handle_request(api_key, 0, Bytes::new(), &default_broker()) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!( d.read_i16().unwrap(), @@ -267,7 +268,8 @@ fn produce_advertises_min_zero_but_firewall_rejects_below_v3() { assert!(!is_supported_version(API_KEY_PRODUCE, 0)); assert!(!is_supported_version(API_KEY_PRODUCE, 2)); - let body = handle_request(API_KEY_PRODUCE, 2, Bytes::new(), &default_broker()).expect("test request has acks != 0 and expects a response"); + let body = handle_request(API_KEY_PRODUCE, 2, Bytes::new(), &default_broker()) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let _topics = d.read_i32().unwrap(); let _name = d.read_nullable_string().unwrap(); @@ -285,7 +287,8 @@ fn metadata_v0_empty_topics_returns_zero_length_topic_array() { 0, metadata_empty_legacy_body(), &default_broker(), - ).expect("test request has acks != 0 and expects a response"); + ) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let _brokers = d.read_i32().unwrap(); d.read_i32().unwrap(); @@ -302,7 +305,8 @@ fn metadata_v3_includes_throttle_time_ms_before_brokers() { 3, metadata_empty_legacy_body(), &default_broker(), - ).expect("test request has acks != 0 and expects a response"); + ) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 0, "throttle_time_ms"); } @@ -314,7 +318,8 @@ fn metadata_v9_flexible_empty_topics_returns_zero_topics() { 9, build_metadata_flexible_request(&[]), &default_broker(), - ).expect("test request has acks != 0 and expects a response"); + ) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); d.read_i32().unwrap(); // throttle let broker_count = usize::try_from(d.read_varint().unwrap()) @@ -343,7 +348,8 @@ fn metadata_v9_flexible_echoes_each_requested_topic_name() { 9, build_metadata_flexible_request(&topics), &default_broker(), - ).expect("test request has acks != 0 and expects a response"); + ) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); d.read_i32().unwrap(); @@ -406,7 +412,8 @@ fn metadata_v1_legacy_multiple_topics_echo_names() { 1, build_metadata_legacy_request(&topics), &default_broker(), - ).expect("test request has acks != 0 and expects a response"); + ) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); d.read_i32().unwrap(); @@ -668,7 +675,8 @@ async fn corrupt_fetch_body_e2e_returns_error_without_disconnect() { #[test] fn corrupt_list_offsets_body_returns_invalid_request_error() { let body = Bytes::from_static(&[0xFF, 0xFF, 0xFF]); - let resp = handle_request(API_KEY_LIST_OFFSETS, 1, body, &default_broker()).expect("test request has acks != 0 and expects a response"); + let resp = handle_request(API_KEY_LIST_OFFSETS, 1, body, &default_broker()) + .expect("test request has acks != 0 and expects a response"); assert!(!resp.is_empty()); assert!( scan_for_error_code(&resp, ERROR_INVALID_REQUEST) @@ -680,7 +688,8 @@ fn corrupt_list_offsets_body_returns_invalid_request_error() { #[test] fn corrupt_create_topics_body_returns_invalid_request_error() { let body = Bytes::from_static(&[0xFF, 0xFF, 0xFF]); - let resp = handle_request(API_KEY_CREATE_TOPICS, 2, body, &default_broker()).expect("test request has acks != 0 and expects a response"); + let resp = handle_request(API_KEY_CREATE_TOPICS, 2, body, &default_broker()) + .expect("test request has acks != 0 and expects a response"); assert!(!resp.is_empty()); assert!( scan_for_error_code(&resp, ERROR_INVALID_REQUEST) @@ -691,7 +700,8 @@ fn corrupt_create_topics_body_returns_invalid_request_error() { #[test] fn corrupt_metadata_body_returns_zero_topics_not_panic() { let body = Bytes::from_static(&[0x00, 0x00]); - let resp = handle_request(API_KEY_METADATA, 0, body, &default_broker()).expect("test request has acks != 0 and expects a response"); + let resp = handle_request(API_KEY_METADATA, 0, body, &default_broker()) + .expect("test request has acks != 0 and expects a response"); assert!(!resp.is_empty()); let mut d = Decoder::new(resp); d.read_i32().unwrap(); @@ -738,7 +748,8 @@ fn metadata_v9_request_with_three_topics_yields_three_response_slots() { 9, build_metadata_flexible_request(&topics), &default_broker(), - ).expect("test request has acks != 0 and expects a response"); + ) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); d.read_i32().unwrap(); skip_metadata_v9_prefix(&mut d); @@ -774,7 +785,8 @@ fn every_in_range_version_returns_non_empty_handler_response() { for &(api_key, name, min_ver, max_ver) in SCOPED_API_KEYS { for version in min_ver..=max_ver { let body = request_body_for_scoped_api(api_key, name, version); - let resp = handle_request(api_key, version, body, &default_broker()).expect("test request has acks != 0 and expects a response"); + let resp = handle_request(api_key, version, body, &default_broker()) + .expect("test request has acks != 0 and expects a response"); assert!(!resp.is_empty(), "{name} v{version} handler returned empty"); } } diff --git a/gateways/kafka/tests/version_firewall_tests.rs b/gateways/kafka/tests/version_firewall_tests.rs index 053918af44..f64615975a 100644 --- a/gateways/kafka/tests/version_firewall_tests.rs +++ b/gateways/kafka/tests/version_firewall_tests.rs @@ -65,7 +65,8 @@ fn is_supported_version_matches_scope_table() { #[test] fn apiversions_advertises_exact_supported_ranges_v1() { - let body = handle_request(API_KEY_API_VERSIONS, 1, Bytes::new(), &default_broker()).expect("test request has acks != 0 and expects a response"); + let body = handle_request(API_KEY_API_VERSIONS, 1, Bytes::new(), &default_broker()) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i16().unwrap(), 0); let count = usize::try_from(d.read_i32().unwrap()).expect("api count fits usize"); @@ -88,7 +89,8 @@ fn apiversions_advertises_exact_supported_ranges_v1() { #[test] fn apiversions_advertises_exact_supported_ranges_v3_flexible() { - let body = handle_request(API_KEY_API_VERSIONS, 3, Bytes::new(), &default_broker()).expect("test request has acks != 0 and expects a response"); + let body = handle_request(API_KEY_API_VERSIONS, 3, Bytes::new(), &default_broker()) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i16().unwrap(), 0); let count = usize::try_from(d.read_varint().unwrap() - 1).expect("api count fits usize"); @@ -133,7 +135,8 @@ fn apiversions_all_versions_return_success() { version, Bytes::new(), &default_broker(), - ).expect("test request has acks != 0 and expects a response"); + ) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i16().unwrap(), 0, "ApiVersions v{version}"); } @@ -141,7 +144,8 @@ fn apiversions_all_versions_return_success() { #[test] fn apiversions_out_of_range_returns_unsupported_in_body() { - let body = handle_request(API_KEY_API_VERSIONS, 99, Bytes::new(), &default_broker()).expect("test request has acks != 0 and expects a response"); + let body = handle_request(API_KEY_API_VERSIONS, 99, Bytes::new(), &default_broker()) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); } @@ -159,7 +163,8 @@ fn metadata_below_min_version_returns_topic_error() { -1, metadata_request_one_topic(), &default_broker(), - ).expect("test request has acks != 0 and expects a response"); + ) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let _brokers = d.read_i32().unwrap(); let _ = d.read_i32().unwrap(); @@ -177,7 +182,8 @@ fn metadata_above_max_version_returns_topic_error() { 10, Bytes::from_static(&[0x02]), &default_broker(), - ).expect("test request has acks != 0 and expects a response"); + ) + .expect("test request has acks != 0 and expects a response"); // Response is in v9 flexible format (highest supported). let mut d = Decoder::new(body); d.read_i32().unwrap(); // throttle_time_ms (v3+) @@ -202,7 +208,8 @@ fn metadata_above_max_version_returns_topic_error() { #[test] fn produce_unsupported_version_returns_well_formed_error_response() { - let body = handle_request(API_KEY_PRODUCE, 2, Bytes::new(), &default_broker()).expect("test request has acks != 0 and expects a response"); + let body = handle_request(API_KEY_PRODUCE, 2, Bytes::new(), &default_broker()) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 1); assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); @@ -217,7 +224,8 @@ fn produce_unsupported_version_returns_well_formed_error_response() { #[test] fn fetch_unsupported_version_returns_well_formed_error_response() { - let body = handle_request(API_KEY_FETCH, 3, Bytes::new(), &default_broker()).expect("test request has acks != 0 and expects a response"); + let body = handle_request(API_KEY_FETCH, 3, Bytes::new(), &default_broker()) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 0); assert_eq!(d.read_i32().unwrap(), 1); @@ -232,7 +240,8 @@ fn fetch_unsupported_version_returns_well_formed_error_response() { #[test] fn fetch_unsupported_version_above_max_uses_top_level_error() { - let body = handle_request(API_KEY_FETCH, 13, Bytes::new(), &default_broker()).expect("test request has acks != 0 and expects a response"); + let body = handle_request(API_KEY_FETCH, 13, Bytes::new(), &default_broker()) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 0); assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); @@ -244,7 +253,8 @@ fn fetch_unsupported_version_above_max_uses_top_level_error() { #[test] fn list_offsets_unsupported_version_returns_well_formed_error_response() { - let body = handle_request(API_KEY_LIST_OFFSETS, 0, Bytes::new(), &default_broker()).expect("test request has acks != 0 and expects a response"); + let body = handle_request(API_KEY_LIST_OFFSETS, 0, Bytes::new(), &default_broker()) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 1); assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); @@ -257,7 +267,8 @@ fn list_offsets_unsupported_version_returns_well_formed_error_response() { #[test] fn create_topics_unsupported_version_returns_well_formed_error_response() { - let body = handle_request(API_KEY_CREATE_TOPICS, 1, Bytes::new(), &default_broker()).expect("test request has acks != 0 and expects a response"); + let body = handle_request(API_KEY_CREATE_TOPICS, 1, Bytes::new(), &default_broker()) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 1); assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); @@ -269,7 +280,8 @@ fn create_topics_unsupported_version_returns_well_formed_error_response() { #[test] fn unsupported_api_keys_return_error_only() { for key in [8, 9, 10, 11, 17, 20, 42, 999] { - let body = handle_request(key, 0, Bytes::new(), &default_broker()).expect("test request has acks != 0 and expects a response"); + let body = handle_request(key, 0, Bytes::new(), &default_broker()) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!( d.read_i16().unwrap(), @@ -283,7 +295,8 @@ fn unsupported_api_keys_return_error_only() { fn supported_produce_versions_accept_valid_fixture() { for version in 3i16..=9 { let body = load_fixture_body(0, "Produce", version); - let resp = handle_request(API_KEY_PRODUCE, version, body, &default_broker()).expect("test request has acks != 0 and expects a response"); + let resp = handle_request(API_KEY_PRODUCE, version, body, &default_broker()) + .expect("test request has acks != 0 and expects a response"); assert!(!resp.is_empty(), "Produce v{version} response empty"); } } @@ -292,7 +305,8 @@ fn supported_produce_versions_accept_valid_fixture() { fn supported_fetch_versions_accept_valid_fixture() { for version in 4i16..=12 { let body = load_fixture_body(1, "Fetch", version); - let resp = handle_request(API_KEY_FETCH, version, body, &default_broker()).expect("test request has acks != 0 and expects a response"); + let resp = handle_request(API_KEY_FETCH, version, body, &default_broker()) + .expect("test request has acks != 0 and expects a response"); assert!(!resp.is_empty(), "Fetch v{version} response empty"); } } @@ -300,7 +314,8 @@ fn supported_fetch_versions_accept_valid_fixture() { #[test] fn corrupt_produce_body_returns_invalid_request_error() { let body = Bytes::from_static(&[0xFF, 0xFF, 0xFF]); - let resp = handle_request(API_KEY_PRODUCE, 3, body, &default_broker()).expect("test request has acks != 0 and expects a response"); + let resp = handle_request(API_KEY_PRODUCE, 3, body, &default_broker()) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(resp); assert_eq!(d.read_i32().unwrap(), 1); assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); @@ -312,7 +327,8 @@ fn corrupt_produce_body_returns_invalid_request_error() { #[test] fn corrupt_fetch_body_returns_invalid_request_error() { let body = Bytes::from_static(&[0xFF, 0xFF, 0xFF]); - let resp = handle_request(API_KEY_FETCH, 4, body, &default_broker()).expect("test request has acks != 0 and expects a response"); + let resp = handle_request(API_KEY_FETCH, 4, body, &default_broker()) + .expect("test request has acks != 0 and expects a response"); let mut d = Decoder::new(resp); assert_eq!(d.read_i32().unwrap(), 0); assert_eq!(d.read_i32().unwrap(), 1); From 8924f20d860ae68fa4689a000eab49cb3c12ac2a Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Mon, 6 Jul 2026 07:45:45 -0400 Subject: [PATCH 23/57] Fix ListOffsets v0 response field encoding Update Kafka ListOffsets response encoding to handle version 0 correctly by writing the legacy `old_style_offsets` array instead of `timestamp`/`offset` fields. This aligns the stubbed response shape with Kafka protocol expectations and avoids malformed v0 payloads. The regression test was adjusted to validate and consume the trailing `old_style_offsets` array bytes explicitly. --- gateways/kafka/src/protocol/responses.rs | 18 +++++++++++------- .../kafka/tests/review_regression_tests.rs | 11 ++++++++++- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/gateways/kafka/src/protocol/responses.rs b/gateways/kafka/src/protocol/responses.rs index 8c44692393..c03d0c2229 100644 --- a/gateways/kafka/src/protocol/responses.rs +++ b/gateways/kafka/src/protocol/responses.rs @@ -264,13 +264,17 @@ fn encode_list_offsets_response_inner( e.write_i32(partition.partition); e.write_i16(partition_error); - let offset = 0i64; - if version >= 1 { - e.write_i64(-1); // -1 = timestamp not available (Kafka sentinel) - } - e.write_i64(offset); - if version >= 4 { - e.write_i32(-1); + if version == 0 { + // v0 has no `timestamp`/`offset` fields; it returns the legacy + // `old_style_offsets` ARRAY (i32 count + i64 entries) instead. + // Empty since this stub never resolves a real offset. + e.write_i32(0); + } else { + e.write_i64(-1); // timestamp: -1 = not available (Kafka sentinel) + e.write_i64(0); // offset + if version >= 4 { + e.write_i32(-1); // leader_epoch + } } if flexible { e.write_empty_tagged_fields(); diff --git a/gateways/kafka/tests/review_regression_tests.rs b/gateways/kafka/tests/review_regression_tests.rs index 2e7d2cd01c..eff4b9843a 100644 --- a/gateways/kafka/tests/review_regression_tests.rs +++ b/gateways/kafka/tests/review_regression_tests.rs @@ -151,7 +151,16 @@ fn list_offsets_v0_unsupported_version_carries_error_code_in_partition() { "partition error code" ); - parse_list_offsets_v0_partition(&mut d); + // partition_index and error_code were already asserted above; only the + // trailing old_style_offsets array remains for this single partition. + let offset_count = d.read_i32().expect("old_style_offsets array length"); + assert!( + offset_count >= 0, + "old_style_offsets count must be non-negative, got {offset_count}" + ); + for _ in 0..offset_count { + d.read_i64().expect("old_style_offsets entry"); + } assert_eq!(d.remaining(), 0); } From 73d9df4f74293b6d35ca0457982884962e8712bb Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Mon, 6 Jul 2026 09:22:50 -0400 Subject: [PATCH 24/57] Kafka gateway keepalive and timeout fix Enable TCP keepalive on accepted Kafka connections and change frame reads so `read_timeout` only applies to the in-flight body, not idle time between requests. Update regression and version-firewall tests to match the new framing behavior and flexible/legacy metadata helpers. --- Cargo.lock | 1 + gateways/kafka/Cargo.toml | 1 + gateways/kafka/src/server.rs | 22 ++++++++++++------- .../kafka/tests/listener_robustness_tests.rs | 2 +- .../kafka/tests/review_regression_tests.rs | 7 +++--- gateways/kafka/tests/scope_coverage_tests.rs | 1 + .../kafka/tests/version_firewall_tests.rs | 18 +++++++++------ 7 files changed, 32 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e6e82b01be..d194d5a14d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6757,6 +6757,7 @@ name = "iggy-gateway-kafka" version = "0.1.0" dependencies = [ "bytes", + "socket2 0.6.4", "thiserror 2.0.18", "tokio", "tokio-util", diff --git a/gateways/kafka/Cargo.toml b/gateways/kafka/Cargo.toml index 20618711e2..d26608257a 100644 --- a/gateways/kafka/Cargo.toml +++ b/gateways/kafka/Cargo.toml @@ -34,6 +34,7 @@ path = "src/main.rs" [dependencies] bytes = { workspace = true } +socket2 = "0.6.4" thiserror = { workspace = true } tokio = { workspace = true, features = [ "rt-multi-thread", diff --git a/gateways/kafka/src/server.rs b/gateways/kafka/src/server.rs index c28e6daecb..f0e8ce5a97 100644 --- a/gateways/kafka/src/server.rs +++ b/gateways/kafka/src/server.rs @@ -176,6 +176,9 @@ impl KafkaServer { if let Err(e) = stream.set_nodelay(true) { warn!(%peer, "TCP_NODELAY failed: {e}"); } + if let Err(e) = enable_tcp_keepalive(&stream) { + warn!(%peer, "TCP_KEEPALIVE failed: {e}"); + } let cfg = Arc::clone(&self.config); let broker = Arc::clone(&broker); tracker.spawn(async move { @@ -194,6 +197,7 @@ impl KafkaServer { Err(e) => return Err(e.into()), } } + } } Ok(()) @@ -213,6 +217,12 @@ fn is_transient_accept_error(err: &std::io::Error) -> bool { ) } +fn enable_tcp_keepalive(stream: &TcpStream) -> std::io::Result<()> { + let sock = socket2::SockRef::from(stream); + sock.set_keepalive(true)?; + Ok(()) +} + async fn handle_connection( mut stream: TcpStream, config: Arc, @@ -337,20 +347,14 @@ pub async fn read_frame( max_frame_size: usize, read_timeout: Duration, ) -> Result { - // Single deadline for both the length-prefix read and the body read. Without this, a - // slow-drip sender could hold a connection open for 2x read_timeout by sending one byte - // per timeout window. - let deadline = tokio::time::Instant::now() + read_timeout; let mut len_buf = [0u8; 4]; - timeout_at(deadline, stream.read_exact(&mut len_buf)) - .await - .map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "read timeout"))??; + // Idle: block until client starts next frame (or EOF). No read_timeout here. + stream.read_exact(&mut len_buf).await?; let frame_len_i32 = i32::from_be_bytes(len_buf); if frame_len_i32 <= 0 { return Err(KafkaProtocolError::InvalidFrameLength(frame_len_i32)); } - let frame_len = usize::try_from(frame_len_i32).map_err(|_| KafkaProtocolError::FrameTooLarge { max_bytes: max_frame_size, @@ -363,6 +367,8 @@ pub async fn read_frame( }); } + // In-flight: read_timeout applies only after the length prefix is complete. + let deadline = tokio::time::Instant::now() + read_timeout; // read_buf() exposes all BytesMut spare capacity to the OS; after reserve(n) the // allocator may give more than n bytes, so the OS can fill past frame_len and silently // consume bytes belonging to the next pipelined frame. Use read() with a bounded slice diff --git a/gateways/kafka/tests/listener_robustness_tests.rs b/gateways/kafka/tests/listener_robustness_tests.rs index 8568214b0f..97fa91797e 100644 --- a/gateways/kafka/tests/listener_robustness_tests.rs +++ b/gateways/kafka/tests/listener_robustness_tests.rs @@ -383,7 +383,7 @@ async fn e2e_connection_idle_after_response_accepts_next_request() { assert_eq!( parse_response_payload(API_KEY_API_VERSIONS, 1, payload).0, 602, - "idle gap under read_timeout must not drop connection" + "idle gap between requests must not drop connection (read_timeout is in-flight only)" ); } diff --git a/gateways/kafka/tests/review_regression_tests.rs b/gateways/kafka/tests/review_regression_tests.rs index eff4b9843a..ad93729716 100644 --- a/gateways/kafka/tests/review_regression_tests.rs +++ b/gateways/kafka/tests/review_regression_tests.rs @@ -347,8 +347,7 @@ async fn e2e_quiet_connection_survives_beyond_read_timeout_idle_cap() { let mut stream = TcpStream::connect(addr).await.expect("connect"); - // Longer than read_timeout: today the server closes idle connections here. - // After fix (separate idle timeout), the connection should remain usable. + // Idle longer than read_timeout: prefix wait has no timer; connection stays open. time::sleep(Duration::from_secs(4)).await; let frame = build_request_frame(API_KEY_API_VERSIONS, 1, 502, Some("idle-test"), &[]); @@ -361,8 +360,8 @@ async fn e2e_quiet_connection_survives_beyond_read_timeout_idle_cap() { read_response_frame_with_timeout(&mut stream, 8 * 1024 * 1024, Duration::from_secs(2)) .await .expect( - "request after idle period longer than read_timeout should succeed once \ - idle timeout is decoupled from frame read timeout", + "request after idle period longer than read_timeout should succeed \ + (read_timeout applies to in-flight frame body only)", ); let (corr, _) = parse_response_payload(API_KEY_API_VERSIONS, 1, payload); diff --git a/gateways/kafka/tests/scope_coverage_tests.rs b/gateways/kafka/tests/scope_coverage_tests.rs index 32db4aa80b..e2a3181935 100644 --- a/gateways/kafka/tests/scope_coverage_tests.rs +++ b/gateways/kafka/tests/scope_coverage_tests.rs @@ -391,6 +391,7 @@ fn metadata_v9_flexible_echoes_each_requested_topic_name() { 0, "empty partitions array" ); + d.read_i32().unwrap(); // topic_authorized_operations (v8+) d.read_tagged_fields().unwrap(); } diff --git a/gateways/kafka/tests/version_firewall_tests.rs b/gateways/kafka/tests/version_firewall_tests.rs index f64615975a..a88e4e1be3 100644 --- a/gateways/kafka/tests/version_firewall_tests.rs +++ b/gateways/kafka/tests/version_firewall_tests.rs @@ -21,6 +21,10 @@ mod fixtures; #[path = "common/scope.rs"] mod scope; +#[path = "common/tcp.rs"] +mod tcp; +#[path = "common/wire.rs"] +mod wire; use bytes::Bytes; @@ -33,6 +37,8 @@ use iggy_gateway_kafka::protocol::codec::Decoder; use fixtures::load_fixture_body; use scope::{SCOPED_API_KEYS, default_broker}; +use tcp::build_metadata_legacy_request; +use wire::build_metadata_flexible_request; #[test] fn supported_ranges_table_has_six_entries() { @@ -151,9 +157,7 @@ fn apiversions_out_of_range_returns_unsupported_in_body() { } fn metadata_request_one_topic() -> Bytes { - let mut raw = Vec::new(); - raw.extend_from_slice(&1_i32.to_be_bytes()); - Bytes::from(raw) + build_metadata_legacy_request(&["test-topic"]) } #[test] @@ -176,11 +180,11 @@ fn metadata_below_min_version_returns_topic_error() { #[test] fn metadata_above_max_version_returns_topic_error() { - // v10 uses flexible encoding; compact array varint(2) = 1 topic. + // v10 request uses flexible encoding; response is clamped to v9. let body = handle_request( API_KEY_METADATA, 10, - Bytes::from_static(&[0x02]), + build_metadata_flexible_request(&["test-topic"]), &default_broker(), ) .expect("test request has acks != 0 and expects a response"); @@ -259,9 +263,9 @@ fn list_offsets_unsupported_version_returns_well_formed_error_response() { assert_eq!(d.read_i32().unwrap(), 1); assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); assert_eq!(d.read_i32().unwrap(), 1); - assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.read_i32().unwrap(), 0); // partition index assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); - assert_eq!(d.read_i64().unwrap(), 0); + assert_eq!(d.read_i32().unwrap(), 0); // old_style_offsets empty array (v0 wire) assert_eq!(d.remaining(), 0); } From b8cb48e6f6a07e3be05aff4dd65a0fe845c3e141 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Mon, 6 Jul 2026 10:55:07 -0400 Subject: [PATCH 25/57] Allow multiple --api-key filters in generate Updated the kafka-tool `generate` command to accept repeatable `--api-key` flags instead of a single value. The CLI now collects API keys into a `Vec` with `ArgAction::Append`, and generation filters messages by membership in that list, enabling selective output for multiple API keys in one run. --- gateways/kafka/tools/kafka-tool/src/main.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gateways/kafka/tools/kafka-tool/src/main.rs b/gateways/kafka/tools/kafka-tool/src/main.rs index b92bf729c1..f620f0303c 100644 --- a/gateways/kafka/tools/kafka-tool/src/main.rs +++ b/gateways/kafka/tools/kafka-tool/src/main.rs @@ -47,9 +47,9 @@ enum Command { Generate { #[arg(short, long, default_value = "kafka_messages")] output: PathBuf, - /// Filter to a single API key integer - #[arg(long)] - api_key: Option, + /// Filter to API key(s), repeatable: --api-key 0 --api-key 1 + #[arg(long, action = clap::ArgAction::Append)] + api_key: Vec, /// Filter to a single version #[arg(long)] version: Option, @@ -685,14 +685,14 @@ fn cmd_list() { async fn cmd_generate( out: PathBuf, - fk: Option, + filter_keys: Vec, fv: Option, hex_dump: bool, ) -> Result<()> { tokio::fs::create_dir_all(&out).await?; let (mut n, mut corr) = (0usize, 1i32); for &(ak, name, min, max) in API_REGISTRY { - if fk.is_some_and(|k| k != ak) { + if !filter_keys.is_empty() && !filter_keys.contains(&ak) { continue; } for v in min..=max { From 7285e02c0378a650e02b087fbe91c4e993fdeb0f Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Mon, 6 Jul 2026 13:25:44 -0400 Subject: [PATCH 26/57] Preserve Produce acks=0 silence on decode errors Refactors Produce decoding to return a `ProduceDecodeResult` that carries `acks` even when parsing fails. The request handler now suppresses error responses when `acks=0`, including malformed requests that fail after the `acks` field is read, preventing client correlation desync. Tests were updated to use `.into_request()` where needed and expanded with regression/e2e coverage for malformed `acks=0` Produce payloads. --- gateways/kafka/src/protocol/api.rs | 22 +++- gateways/kafka/src/protocol/requests.rs | 120 +++++++++++++----- gateways/kafka/tests/decode_safety_tests.rs | 4 +- .../kafka/tests/decode_validation_tests.rs | 6 +- .../kafka/tests/review_regression_tests.rs | 39 ++++++ 5 files changed, 147 insertions(+), 44 deletions(-) diff --git a/gateways/kafka/src/protocol/api.rs b/gateways/kafka/src/protocol/api.rs index 4453441abf..22dc5acdc2 100644 --- a/gateways/kafka/src/protocol/api.rs +++ b/gateways/kafka/src/protocol/api.rs @@ -20,8 +20,8 @@ use bytes::Bytes; use crate::error::{KafkaProtocolError, Result}; use crate::protocol::codec::{Decoder, Encoder}; use crate::protocol::requests::{ - decode_create_topics_request, decode_fetch_request, decode_list_offsets_request, - decode_produce_request, + ProduceDecodeResult, decode_create_topics_request, decode_fetch_request, + decode_list_offsets_request, decode_produce_request, }; use crate::protocol::responses::{ encode_create_topics_error_response, encode_create_topics_response, @@ -135,10 +135,20 @@ fn handle_produce_request(api_version: i16, body: Bytes) -> Option { match decode_produce_request(api_version, body) { // acks=0 is fire-and-forget: the client isn't reading a response, so // sending one desyncs the next correlation id it expects. - Ok(req) if req.acks == 0 => None, - Ok(req) => Some(encode_produce_response(api_version, &req)), - Err(e) => { - tracing::warn!("Failed to decode Produce request: {:?}", e); + ProduceDecodeResult::Ok(req) if req.acks == 0 => None, + ProduceDecodeResult::Ok(req) => Some(encode_produce_response(api_version, &req)), + ProduceDecodeResult::Err { + acks: Some(0), + error, + } => { + tracing::warn!( + "Failed to decode Produce request with acks=0 (no response): {:?}", + error + ); + None + } + ProduceDecodeResult::Err { error, .. } => { + tracing::warn!("Failed to decode Produce request: {:?}", error); Some(encode_produce_error_response( api_version, ERROR_INVALID_REQUEST, diff --git a/gateways/kafka/src/protocol/requests.rs b/gateways/kafka/src/protocol/requests.rs index 91e2a40b66..2ee238a14a 100644 --- a/gateways/kafka/src/protocol/requests.rs +++ b/gateways/kafka/src/protocol/requests.rs @@ -44,72 +44,122 @@ pub struct ProducePartitionData { pub records: Option, // Raw RecordBatch bytes } -pub fn decode_produce_request(version: i16, body: Bytes) -> Result { +/// Outcome of decoding a Produce request body. +/// +/// Carries `acks` on failure so callers can honor fire-and-forget (`acks=0`) silence +/// even when later fields are malformed. +#[derive(Debug)] +pub enum ProduceDecodeResult { + Ok(ProduceRequest), + Err { + acks: Option, + error: KafkaProtocolError, + }, +} + +impl ProduceDecodeResult { + /// Collapse to `Result` for tests and callers that only need a successful `ProduceRequest`. + pub fn into_request(self) -> Result { + match self { + Self::Ok(req) => Ok(req), + Self::Err { error, .. } => Err(error), + } + } +} + +macro_rules! produce_decode { + ($acks:expr, $expr:expr) => { + match $expr { + Ok(value) => value, + Err(error) => { + return ProduceDecodeResult::Err { acks: $acks, error }; + } + } + }; +} + +pub fn decode_produce_request(version: i16, body: Bytes) -> ProduceDecodeResult { let mut d = Decoder::new(body); let flexible = version >= 9; // transactional_id (v3+) let transactional_id = if version >= 3 { - if flexible { - d.read_compact_nullable_string()? - } else { - d.read_nullable_string()? - } + produce_decode!( + None, + if flexible { + d.read_compact_nullable_string() + } else { + d.read_nullable_string() + } + ) } else { None }; - let acks = d.read_i16()?; - let timeout_ms = d.read_i32()?; + let acks = produce_decode!(None, d.read_i16()); + let acks_read = Some(acks); + + let timeout_ms = produce_decode!(acks_read, d.read_i32()); // topics array - let topics_count = if flexible { - d.read_compact_array_count()? - } else { - d.read_i32_array_count()? - }; + let topics_count = produce_decode!( + acks_read, + if flexible { + d.read_compact_array_count() + } else { + d.read_i32_array_count() + } + ); let mut topics = Vec::with_capacity(topics_count); for _ in 0..topics_count { - let topic = if flexible { - d.read_compact_nullable_string()? - .ok_or(KafkaProtocolError::NullTopicName)? - } else { - d.read_nullable_string()? - .ok_or(KafkaProtocolError::NullTopicName)? - }; + let topic = produce_decode!( + acks_read, + if flexible { + d.read_compact_nullable_string() + } else { + d.read_nullable_string() + } + .and_then(|name| name.ok_or(KafkaProtocolError::NullTopicName)) + ); - let partitions_count = if flexible { - d.read_compact_array_count()? - } else { - d.read_i32_array_count()? - }; + let partitions_count = produce_decode!( + acks_read, + if flexible { + d.read_compact_array_count() + } else { + d.read_i32_array_count() + } + ); let mut partitions = Vec::with_capacity(partitions_count); for _ in 0..partitions_count { - let partition = d.read_i32()?; - let records = if flexible { - d.read_compact_nullable_bytes()? - } else { - d.read_nullable_bytes()? - }; + let partition = produce_decode!(acks_read, d.read_i32()); + let records = produce_decode!( + acks_read, + if flexible { + d.read_compact_nullable_bytes() + } else { + d.read_nullable_bytes() + } + ); partitions.push(ProducePartitionData { partition, records }); if flexible { - d.read_tagged_fields()?; + produce_decode!(acks_read, d.read_tagged_fields()); } } topics.push(ProduceTopicData { topic, partitions }); if flexible { - d.read_tagged_fields()?; + produce_decode!(acks_read, d.read_tagged_fields()); } } if flexible { - d.read_tagged_fields()?; + produce_decode!(acks_read, d.read_tagged_fields()); } - Ok(ProduceRequest { + ProduceDecodeResult::Ok(ProduceRequest { transactional_id, acks, timeout_ms, diff --git a/gateways/kafka/tests/decode_safety_tests.rs b/gateways/kafka/tests/decode_safety_tests.rs index f5d8d84a9a..531cf9b3f1 100644 --- a/gateways/kafka/tests/decode_safety_tests.rs +++ b/gateways/kafka/tests/decode_safety_tests.rs @@ -58,7 +58,9 @@ fn produce_decoder_rejects_truncated_flexible_body() { body.push(0x02); // topics compact array: 1 element (varint = count+1) // truncated before topic name - let err = decode_produce_request(9, Bytes::from(body)).unwrap_err(); + let err = decode_produce_request(9, Bytes::from(body)) + .into_request() + .unwrap_err(); assert!(matches!(err, KafkaProtocolError::BufferUnderflow { .. })); } diff --git a/gateways/kafka/tests/decode_validation_tests.rs b/gateways/kafka/tests/decode_validation_tests.rs index 8a51e45b06..b2b8fe55eb 100644 --- a/gateways/kafka/tests/decode_validation_tests.rs +++ b/gateways/kafka/tests/decode_validation_tests.rs @@ -68,6 +68,7 @@ fn produce_all_supported_versions_decode() { for version in 3i16..=9 { let body = load_body(0, "Produce", version); let req = decode_produce_request(version, body) + .into_request() .unwrap_or_else(|e| panic!("Produce v{version} decode failed: {e}")); assert_eq!(req.acks, -1, "Produce v{version}: unexpected acks"); @@ -101,6 +102,7 @@ fn produce_response_encodes_for_all_supported_versions() { for version in 3i16..=9 { let body = load_body(0, "Produce", version); let req = decode_produce_request(version, body) + .into_request() .unwrap_or_else(|e| panic!("Produce v{version} decode failed: {e}")); let resp = encode_produce_response(version, &req); assert!( @@ -114,7 +116,7 @@ fn produce_response_encodes_for_all_supported_versions() { fn produce_response_v3_roundtrip() { use iggy_gateway_kafka::protocol::codec::Decoder; let body = load_body(0, "Produce", 3); - let req = decode_produce_request(3, body).unwrap(); + let req = decode_produce_request(3, body).into_request().unwrap(); let resp = encode_produce_response(3, &req); let mut d = Decoder::new(resp); @@ -141,7 +143,7 @@ fn produce_response_v3_roundtrip() { fn produce_response_v8_includes_record_errors() { use iggy_gateway_kafka::protocol::codec::Decoder; let body = load_body(0, "Produce", 8); - let req = decode_produce_request(8, body).unwrap(); + let req = decode_produce_request(8, body).into_request().unwrap(); let resp = encode_produce_response(8, &req); let mut d = Decoder::new(resp); diff --git a/gateways/kafka/tests/review_regression_tests.rs b/gateways/kafka/tests/review_regression_tests.rs index ad93729716..4cb0aacb61 100644 --- a/gateways/kafka/tests/review_regression_tests.rs +++ b/gateways/kafka/tests/review_regression_tests.rs @@ -50,6 +50,21 @@ use tcp::{ // ── Produce acks=0 (review: broker must stay silent) ───────────────────────── +#[test] +fn produce_acks_zero_malformed_body_decode_carries_acks() { + use iggy_gateway_kafka::protocol::requests::{ProduceDecodeResult, decode_produce_request}; + + let body = build_produce_v3_body(0, 1); + match decode_produce_request(3, body.clone()) { + ProduceDecodeResult::Err { acks: Some(0), .. } => {} + other => panic!("expected decode error with acks=0, got {other:?}"), + } + assert!( + handle_request(API_KEY_PRODUCE, 3, body, &default_broker()).is_none(), + "handler must not respond when acks=0 even if decode fails after acks" + ); +} + #[tokio::test] async fn e2e_produce_v3_acks_zero_sends_no_response() { let (addr, _shutdown) = spawn_test_server().await; @@ -73,6 +88,30 @@ async fn e2e_produce_v3_acks_zero_sends_no_response() { ); } +#[tokio::test] +async fn e2e_produce_v3_acks_zero_malformed_topics_sends_no_response() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.expect("connect"); + + // acks=0, claims one topic, no topic bytes — decode fails after acks is read. + let body = build_produce_v3_body(0, 1); + let frame = build_request_frame(API_KEY_PRODUCE, 3, 99, Some("review-test"), &body); + stream + .write_all(&frame) + .await + .expect("write produce acks=0 malformed"); + + let response = + read_response_frame_with_timeout(&mut stream, 8 * 1024 * 1024, Duration::from_millis(500)) + .await; + + assert!( + response.is_none(), + "Produce with acks=0 must stay silent even when the body is malformed; got {} bytes", + response.as_ref().map_or(0, Bytes::len) + ); +} + #[tokio::test] async fn e2e_produce_v3_acks_one_still_returns_response() { let (addr, _shutdown) = spawn_test_server().await; From c52d4a99a0ea8f0cd8235b41bcdaff9fd2b949cb Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Mon, 6 Jul 2026 13:49:35 -0400 Subject: [PATCH 27/57] Fix Kafka metadata v10 request decoding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Metadata handling now separates decode and response versions: requests are decoded at the client’s wire version while responses for unsupported versions are encoded at the highest supported metadata version. This fixes flexible v10+ topic parsing by consuming topic_id before name, preserves topic-name echoing in unsupported-version responses, updates regression/firewall/API tests with a v10 request builder, and refreshes test-suite fixture generation docs to pass multiple --api-key values in one command. --- gateways/kafka/docs/TEST_SUITE.md | 5 +-- gateways/kafka/src/protocol/api.rs | 33 +++++++++------ gateways/kafka/tests/api_handler_tests.rs | 9 ++-- gateways/kafka/tests/common/wire.rs | 13 ++++++ .../kafka/tests/review_regression_tests.rs | 42 +++++++++++++++++++ .../kafka/tests/version_firewall_tests.rs | 13 ++++-- 6 files changed, 90 insertions(+), 25 deletions(-) diff --git a/gateways/kafka/docs/TEST_SUITE.md b/gateways/kafka/docs/TEST_SUITE.md index bb33691ff5..74dd995268 100644 --- a/gateways/kafka/docs/TEST_SUITE.md +++ b/gateways/kafka/docs/TEST_SUITE.md @@ -136,10 +136,7 @@ Fixtures are gitignored under `tools/kafka-tool/kafka_messages/`. CI runs the sa # 1. Generate fixtures cargo run -p kafka-message-gen -- generate \ --output gateways/kafka/tools/kafka-tool/kafka_messages \ - --api-key 0 - ## use one api-key in each command. Bulk API kys needs to be implemented - allowed values are - --api-key 1 --api-key 2 --api-key 19 + --api-key 0 --api-key 1 --api-key 2 --api-key 19 # 2. Run regression suite cargo test -p iggy-gateway-kafka diff --git a/gateways/kafka/src/protocol/api.rs b/gateways/kafka/src/protocol/api.rs index 22dc5acdc2..d5b6e4e47b 100644 --- a/gateways/kafka/src/protocol/api.rs +++ b/gateways/kafka/src/protocol/api.rs @@ -174,11 +174,13 @@ fn handle_other_request( } API_KEY_METADATA => { if is_supported_version(api_key, api_version) { - encode_metadata_response(api_version, body, broker, ERROR_NONE) + encode_metadata_response(api_version, api_version, body, broker, ERROR_NONE) } else { - // Encode at the highest version we implement, not the client's unknown version. + let response_version = api_version.clamp(0, MAX_SUPPORTED_METADATA_VERSION); + // Decode at the client's wire version; encode at the highest version we implement. encode_metadata_response( - api_version.clamp(0, MAX_SUPPORTED_METADATA_VERSION), + response_version, + api_version, body, broker, ERROR_UNSUPPORTED_VERSION, @@ -285,12 +287,13 @@ fn encode_api_versions_response(api_version: i16, error_code: i16) -> Bytes { } fn encode_metadata_response( - api_version: i16, + response_version: i16, + decode_version: i16, body: Bytes, broker: &BrokerAdvertise, top_level_error_code: i16, ) -> Bytes { - let flexible = api_version >= 9; + let flexible = response_version >= 9; // Empty body = all-topics request; 0 topics is correct for this stub. // Non-empty body that fails to decode = malformed request; return 0 topics. // Kafka Metadata response has no top-level error code field: errors are per-topic only. @@ -298,7 +301,7 @@ fn encode_metadata_response( let (topics, effective_error) = if body.is_empty() { (Vec::new(), top_level_error_code) } else { - decode_metadata_request_topics(body, api_version) + decode_metadata_request_topics(body, decode_version) .map_or((Vec::new(), ERROR_INVALID_REQUEST), |names| { (names, top_level_error_code) }) @@ -312,7 +315,7 @@ fn encode_metadata_response( let mut e = Encoder::with_capacity(256); - if api_version >= 3 { + if response_version >= 3 { e.write_i32(0); // throttle_time_ms (Metadata v3+) } @@ -347,14 +350,14 @@ fn encode_metadata_response( return encode_error_only_response(ERROR_INVALID_REQUEST); } e.write_i32(broker.port); - if api_version >= 1 { + if response_version >= 1 { e.write_nullable_string_unchecked(None); // rack } - if api_version >= 2 { + if response_version >= 2 { e.write_nullable_string_unchecked(None); // cluster_id } - if api_version >= 1 { + if response_version >= 1 { e.write_i32(1); // controller_id — must come before topics array } @@ -362,15 +365,15 @@ fn encode_metadata_response( for name in &topics { e.write_i16(topic_error); e.write_nullable_string_unchecked(Some(name)); - if api_version >= 1 { + if response_version >= 1 { e.write_bool(false); // is_internal } e.write_i32(0); // partitions array (empty) - if api_version >= 8 { + if response_version >= 8 { e.write_i32(AUTHORIZED_OPS_UNKNOWN); // topic_authorized_operations } } - if api_version >= 8 { + if response_version >= 8 { e.write_i32(AUTHORIZED_OPS_UNKNOWN); // cluster_authorized_operations } } @@ -398,6 +401,10 @@ pub(crate) fn decode_metadata_request_topics(body: Bytes, api_version: i16) -> R let mut topics = Vec::with_capacity(topics_count); for _ in 0..topics_count { + if flexible && api_version >= 10 { + // MetadataRequestTopic.topic_id: 16-byte UUID before name (v10+). + let _topic_id = d.read_bytes(16)?; + } let name = if flexible { d.read_compact_nullable_string()? .ok_or(KafkaProtocolError::NullTopicName)? diff --git a/gateways/kafka/tests/api_handler_tests.rs b/gateways/kafka/tests/api_handler_tests.rs index 17db5de2f8..891de41e62 100644 --- a/gateways/kafka/tests/api_handler_tests.rs +++ b/gateways/kafka/tests/api_handler_tests.rs @@ -29,7 +29,7 @@ fn test_broker() -> BrokerAdvertise { BrokerAdvertise::default() } use iggy_gateway_kafka::protocol::codec::Decoder; -use wire::build_metadata_flexible_request; +use wire::build_metadata_flexible_request_v10; // ── ApiVersions ───────────────────────────────────────────────────────────── @@ -110,13 +110,12 @@ fn metadata_response_has_broker_array_and_topic_array() { #[test] fn unsupported_version_returns_protocol_error() { - // v99 client sends a well-formed v9 flexible body requesting "orders". - // The gateway caps at v9 (highest supported Metadata version) for both parsing - // and encoding, so the response uses the flexible (v9) wire format. + // v99 client sends a well-formed v10+ flexible body (topic_id + name) requesting "orders". + // Response encodes at v9 (highest supported); request decodes at client version 99. let body = handle_request( API_KEY_METADATA, 99, - build_metadata_flexible_request(&["orders"]), + build_metadata_flexible_request_v10(&["orders"]), &test_broker(), ) .expect("test request has acks != 0 and expects a response"); diff --git a/gateways/kafka/tests/common/wire.rs b/gateways/kafka/tests/common/wire.rs index f1aa9890b6..05c70b21a0 100644 --- a/gateways/kafka/tests/common/wire.rs +++ b/gateways/kafka/tests/common/wire.rs @@ -63,6 +63,19 @@ pub fn build_metadata_flexible_request(topic_names: &[&str]) -> Bytes { enc.freeze() } +/// Metadata v10+ flexible request: each topic entry includes a 16-byte `topic_id` before `name`. +pub fn build_metadata_flexible_request_v10(topic_names: &[&str]) -> Bytes { + let mut enc = Encoder::with_capacity(96); + enc.write_varint((topic_names.len() + 1) as u64); + for name in topic_names { + enc.write_bytes(&[0u8; 16]); + enc.write_compact_nullable_string(Some(name)); + enc.write_empty_tagged_fields(); + } + enc.write_empty_tagged_fields(); + enc.freeze() +} + /// Minimal `ListOffsets` request for supported versions (v1–v6). pub fn build_list_offsets_request(version: i16, topic: &str, partition: i32) -> Bytes { let flexible = version >= 6; diff --git a/gateways/kafka/tests/review_regression_tests.rs b/gateways/kafka/tests/review_regression_tests.rs index 4cb0aacb61..f494973685 100644 --- a/gateways/kafka/tests/review_regression_tests.rs +++ b/gateways/kafka/tests/review_regression_tests.rs @@ -26,6 +26,8 @@ mod scope; mod server; #[path = "common/tcp.rs"] mod tcp; +#[path = "common/wire.rs"] +mod wire; use std::time::Duration; @@ -47,6 +49,7 @@ use tcp::{ build_metadata_legacy_request, build_produce_v3_body, build_request_frame, parse_response_payload, read_response_frame_with_timeout, }; +use wire::build_metadata_flexible_request_v10; // ── Produce acks=0 (review: broker must stay silent) ───────────────────────── @@ -251,6 +254,45 @@ async fn e2e_list_offsets_v0_unsupported_version_no_trailing_bytes() { // ── Metadata topic name echo (review: must not hardcode "unknown-topic") ──── +#[test] +fn metadata_v10_unsupported_decodes_client_body_not_clamped_version() { + let body = handle_request( + API_KEY_METADATA, + 10, + build_metadata_flexible_request_v10(&["payments"]), + &default_broker(), + ) + .expect("test request has acks != 0 and expects a response"); + + let mut d = Decoder::new(body); + d.read_i32().unwrap(); // throttle_time_ms + let broker_count = usize::try_from(d.read_varint().unwrap()) + .unwrap() + .saturating_sub(1); + for _ in 0..broker_count { + d.read_i32().unwrap(); + d.read_compact_nullable_string().unwrap(); + d.read_i32().unwrap(); + d.read_compact_nullable_string().unwrap(); + d.read_tagged_fields().unwrap(); + } + d.read_compact_nullable_string().unwrap(); + d.read_i32().unwrap(); + assert_eq!( + usize::try_from(d.read_varint().unwrap()) + .unwrap() + .saturating_sub(1), + 1 + ); + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); + assert_eq!( + d.read_compact_nullable_string() + .unwrap() + .expect("topic name"), + "payments" + ); +} + fn read_metadata_v1_topics(d: &mut Decoder, expected_count: i32) -> Vec { let _brokers_count = d.read_i32().unwrap(); d.read_i32().unwrap(); // node_id diff --git a/gateways/kafka/tests/version_firewall_tests.rs b/gateways/kafka/tests/version_firewall_tests.rs index a88e4e1be3..923cf3c018 100644 --- a/gateways/kafka/tests/version_firewall_tests.rs +++ b/gateways/kafka/tests/version_firewall_tests.rs @@ -38,7 +38,7 @@ use iggy_gateway_kafka::protocol::codec::Decoder; use fixtures::load_fixture_body; use scope::{SCOPED_API_KEYS, default_broker}; use tcp::build_metadata_legacy_request; -use wire::build_metadata_flexible_request; +use wire::build_metadata_flexible_request_v10; #[test] fn supported_ranges_table_has_six_entries() { @@ -180,11 +180,11 @@ fn metadata_below_min_version_returns_topic_error() { #[test] fn metadata_above_max_version_returns_topic_error() { - // v10 request uses flexible encoding; response is clamped to v9. + // v10 request uses flexible encoding with topic_id; response is clamped to v9. let body = handle_request( API_KEY_METADATA, 10, - build_metadata_flexible_request(&["test-topic"]), + build_metadata_flexible_request_v10(&["test-topic"]), &default_broker(), ) .expect("test request has acks != 0 and expects a response"); @@ -208,6 +208,13 @@ fn metadata_above_max_version_returns_topic_error() { .saturating_sub(1); assert_eq!(topic_count, 1); assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); + assert_eq!( + d.read_compact_nullable_string() + .unwrap() + .expect("topic name"), + "test-topic", + "unsupported-version path must decode client v10 body and echo topic name in v9 response" + ); } #[test] From 8c252ae397a67a42f95d63822b7c8afa89768879 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Wed, 8 Jul 2026 15:52:28 -0400 Subject: [PATCH 28/57] Expand Kafka gateway test coverage Add comprehensive test coverage for the Kafka gateway protocol handling: - Unit tests in source files (api.rs, server.rs) for protocol edge cases and server operations - New region_coverage_tests.rs with 40+ tests covering version branches for Produce, Fetch, ListOffsets, CreateTopics - New response_negative_tests.rs testing error responses and validation - Extended codec, header, and wire test helpers for malformed request handling - Tests for flexible vs legacy encoding paths, null topic names, buffer underflow, UTF-8 validation - E2E server tests for frame handling, EOF, and connection lifecycle Improves branch coverage and ensures robust error handling across all Kafka API versions. --- gateways/kafka/src/protocol/api.rs | 37 ++ gateways/kafka/src/server.rs | 191 ++++++++ gateways/kafka/tests/api_handler_tests.rs | 110 ++++- gateways/kafka/tests/codec_tests.rs | 73 +++ gateways/kafka/tests/common/wire.rs | 249 ++++++++++ .../kafka/tests/decode_validation_tests.rs | 83 ++++ gateways/kafka/tests/header_tests.rs | 196 ++++++++ gateways/kafka/tests/region_coverage_tests.rs | 445 ++++++++++++++++++ .../kafka/tests/response_negative_tests.rs | 202 ++++++++ 9 files changed, 1584 insertions(+), 2 deletions(-) create mode 100644 gateways/kafka/tests/region_coverage_tests.rs create mode 100644 gateways/kafka/tests/response_negative_tests.rs diff --git a/gateways/kafka/src/protocol/api.rs b/gateways/kafka/src/protocol/api.rs index d5b6e4e47b..003a141ba3 100644 --- a/gateways/kafka/src/protocol/api.rs +++ b/gateways/kafka/src/protocol/api.rs @@ -420,3 +420,40 @@ pub(crate) fn decode_metadata_request_topics(body: Bytes, api_version: i16) -> R Ok(topics) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::protocol::codec::Encoder; + + #[test] + fn decode_metadata_request_topics_legacy_null_topic_name_fails() { + let body = Bytes::from_static(&[ + 0x00, 0x00, 0x00, 0x01, // one topic + 0xff, 0xff, // null topic name + ]); + let err = decode_metadata_request_topics(body, 0).unwrap_err(); + assert!(matches!(err, KafkaProtocolError::NullTopicName)); + } + + #[test] + fn decode_metadata_request_topics_flexible_v10_truncated_topic_id_fails() { + let mut enc = Encoder::with_capacity(8); + enc.write_varint(2); // one topic + enc.write_bytes(&[0u8; 8]); // truncated topic_id, should be 16 bytes + let err = decode_metadata_request_topics(enc.freeze(), 10).unwrap_err(); + assert!(matches!(err, KafkaProtocolError::BufferUnderflow { .. })); + } + + #[test] + fn decode_metadata_request_topics_flexible_invalid_utf8_fails() { + let body = Bytes::from_static(&[ + 0x02, // one topic + 0x02, // string len = 1 + 0xff, // invalid utf-8 + 0x00, // tagged fields + ]); + let err = decode_metadata_request_topics(body, 9).unwrap_err(); + assert!(matches!(err, KafkaProtocolError::InvalidUtf8)); + } +} diff --git a/gateways/kafka/src/server.rs b/gateways/kafka/src/server.rs index f0e8ce5a97..9d10c10cd9 100644 --- a/gateways/kafka/src/server.rs +++ b/gateways/kafka/src/server.rs @@ -402,3 +402,194 @@ pub fn init_tracing() { .try_init() .map_err(|e| error!("failed to initialize tracing: {e}")); } + +#[cfg(test)] +mod tests { + use super::*; + + async fn tcp_pair() -> (TcpStream, TcpStream) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let client = tokio::spawn(async move { TcpStream::connect(addr).await.unwrap() }); + let (server, _) = listener.accept().await.unwrap(); + let client = client.await.unwrap(); + (client, server) + } + + #[test] + fn transient_accept_error_classification_covers_all_branches() { + for kind in [ + io::ErrorKind::Interrupted, + io::ErrorKind::ConnectionAborted, + io::ErrorKind::WouldBlock, + ] { + assert!(is_transient_accept_error(&io::Error::from(kind))); + } + + #[cfg(unix)] + { + assert!(is_transient_accept_error(&io::Error::from_raw_os_error(23))); + assert!(is_transient_accept_error(&io::Error::from_raw_os_error(24))); + } + + assert!(!is_transient_accept_error(&io::Error::from( + io::ErrorKind::ConnectionRefused, + ))); + } + + #[test] + fn correlation_id_is_extracted_from_frame() { + let frame = + bytes::Bytes::from_static(&[0x00, 0x12, 0x00, 0x01, 0x11, 0x22, 0x33, 0x44, 0xaa]); + assert_eq!(correlation_id_from_frame(&frame), 0x11223344); + } + + #[tokio::test] + async fn send_response_writes_header_and_body() { + let (mut client, mut server) = tcp_pair().await; + let header = ResponseHeader { + correlation_id: 0x01020304, + }; + let body = [9u8, 8, 7]; + + send_response(&mut server, &header, 1, &body, Duration::from_secs(1)) + .await + .unwrap(); + + let mut len = [0u8; 4]; + client.read_exact(&mut len).await.unwrap(); + assert_eq!(i32::from_be_bytes(len), 8); + + let mut payload = [0u8; 8]; + client.read_exact(&mut payload).await.unwrap(); + assert_eq!(&payload[..4], &[0x01, 0x02, 0x03, 0x04]); + assert_eq!(payload[4], 0); + assert_eq!(&payload[5..], &body); + } + + #[tokio::test] + async fn read_frame_rejects_negative_length() { + let (mut client, mut server) = tcp_pair().await; + client.write_all(&(-1_i32).to_be_bytes()).await.unwrap(); + let err = read_frame(&mut server, 64, Duration::from_secs(1)) + .await + .unwrap_err(); + assert!(matches!(err, KafkaProtocolError::InvalidFrameLength(-1))); + } + + #[tokio::test] + async fn read_frame_returns_eof_after_prefix_when_body_missing() { + let (mut client, mut server) = tcp_pair().await; + client.write_all(&(5_i32).to_be_bytes()).await.unwrap(); + client.shutdown().await.unwrap(); + let err = read_frame(&mut server, 64, Duration::from_secs(1)) + .await + .unwrap_err(); + assert!(err.to_string().contains("connection closed")); + } + + #[tokio::test] + async fn read_frame_times_out_after_partial_body() { + let (mut client, mut server) = tcp_pair().await; + client.write_all(&(5_i32).to_be_bytes()).await.unwrap(); + client.write_all(&[1, 2]).await.unwrap(); + let err = read_frame(&mut server, 64, Duration::from_millis(50)) + .await + .unwrap_err(); + assert!(err.to_string().contains("read timeout")); + } + + #[tokio::test] + async fn server_run_exits_when_shutdown_channel_closed() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let (tx, rx) = broadcast::channel(1); + drop(tx); + let server = KafkaServer::new(ServerConfig::default()); + assert!(server.run(listener, rx).await.is_ok()); + } + + #[tokio::test] + async fn server_run_exits_when_shutdown_receiver_is_lagged() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let (tx, rx) = broadcast::channel(1); + tx.send(()).unwrap(); + tx.send(()).unwrap(); + let server = KafkaServer::new(ServerConfig::default()); + assert!(server.run(listener, rx).await.is_ok()); + } + + #[tokio::test] + async fn server_run_exits_on_shutdown_signal_ok() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let (tx, rx) = broadcast::channel(1); + let server = KafkaServer::new(ServerConfig::default()); + let handle = tokio::spawn(async move { server.run(listener, rx).await }); + + let stream = TcpStream::connect(addr).await.unwrap(); + tx.send(()).unwrap(); + drop(stream); + assert!(handle.await.unwrap().is_ok()); + } + + #[tokio::test] + async fn read_frame_accepts_exact_max_frame_size() { + let (mut client, mut server) = tcp_pair().await; + let max_frame_size = 64usize; + let payload = vec![0xABu8; max_frame_size]; + client + .write_all(&i32::try_from(max_frame_size).unwrap().to_be_bytes()) + .await + .unwrap(); + client.write_all(&payload).await.unwrap(); + let frame = read_frame(&mut server, max_frame_size, Duration::from_secs(1)) + .await + .unwrap(); + assert_eq!(frame.len(), max_frame_size); + } + + #[tokio::test] + async fn read_frame_rejects_frame_larger_than_max() { + let (mut client, mut server) = tcp_pair().await; + let max_frame_size = 64usize; + client.write_all(&65_i32.to_be_bytes()).await.unwrap(); + let err = read_frame(&mut server, max_frame_size, Duration::from_secs(1)) + .await + .unwrap_err(); + assert!(matches!( + err, + KafkaProtocolError::FrameTooLarge { + max_bytes: 64, + actual_bytes: 65, + } + )); + } + + #[tokio::test] + async fn send_response_v0_writes_correlation_id_only() { + let (mut client, mut server) = tcp_pair().await; + let header = ResponseHeader { + correlation_id: 0x0000_00AB, + }; + let body = [5u8, 6, 7]; + + send_response(&mut server, &header, 0, &body, Duration::from_secs(1)) + .await + .unwrap(); + + let mut len = [0u8; 4]; + client.read_exact(&mut len).await.unwrap(); + assert_eq!(i32::from_be_bytes(len), 7); + + let mut payload = [0u8; 7]; + client.read_exact(&mut payload).await.unwrap(); + assert_eq!(&payload[..4], &[0, 0, 0, 0xAB]); + assert_eq!(&payload[4..], &body); + } + + #[test] + fn init_tracing_is_idempotent() { + init_tracing(); + init_tracing(); + } +} diff --git a/gateways/kafka/tests/api_handler_tests.rs b/gateways/kafka/tests/api_handler_tests.rs index 891de41e62..db5a7df529 100644 --- a/gateways/kafka/tests/api_handler_tests.rs +++ b/gateways/kafka/tests/api_handler_tests.rs @@ -21,8 +21,9 @@ mod wire; use bytes::Bytes; use iggy_gateway_kafka::protocol::api::{ - API_KEY_API_VERSIONS, API_KEY_METADATA, BrokerAdvertise, ERROR_UNSUPPORTED_VERSION, - handle_request, is_supported_version, supported_api_ranges, + API_KEY_API_VERSIONS, API_KEY_CREATE_TOPICS, API_KEY_FETCH, API_KEY_LIST_OFFSETS, + API_KEY_METADATA, API_KEY_PRODUCE, BrokerAdvertise, ERROR_INVALID_REQUEST, + ERROR_UNSUPPORTED_VERSION, handle_request, is_supported_version, supported_api_ranges, }; fn test_broker() -> BrokerAdvertise { @@ -177,3 +178,108 @@ fn apiversions_unsupported_version_uses_v0_encoding_without_throttle() { assert_eq!(d.read_i32().unwrap(), 6); assert_eq!(d.remaining(), 36); } + +#[test] +fn produce_malformed_body_with_acks_one_returns_invalid_request() { + let body = Bytes::from_static(&[ + 0xff, 0xff, // null transactional_id + 0x00, 0x01, // acks = 1 + 0x00, 0x00, 0x03, 0xe8, // timeout_ms + 0x00, 0x00, 0x00, 0x01, // one topic + ]); + let response = handle_request(API_KEY_PRODUCE, 3, body, &test_broker()) + .expect("acks=1 malformed produce should get error response"); + let mut d = Decoder::new(response); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.read_i16().unwrap(), ERROR_INVALID_REQUEST); +} + +#[test] +fn fetch_malformed_body_returns_invalid_request() { + let response = handle_request( + API_KEY_FETCH, + 12, + Bytes::from_static(&[ + 0xff, 0xff, 0xff, 0xff, // replica_id + 0x00, 0x00, 0x00, 0x64, // max_wait_ms + 0x00, 0x00, 0x00, 0x01, // min_bytes + ]), + &test_broker(), + ) + .expect("fetch must return error response"); + let mut d = Decoder::new(response); + assert_eq!(d.read_i32().unwrap(), 0); // throttle + assert_eq!(d.read_i16().unwrap(), ERROR_INVALID_REQUEST); +} + +#[test] +fn list_offsets_malformed_body_returns_invalid_request() { + let response = handle_request( + API_KEY_LIST_OFFSETS, + 6, + Bytes::from_static(&[ + 0xff, 0xff, 0xff, 0xff, // replica_id + 0x01, // isolation_level + 0x02, // compact topics count = one + 0x00, // null topic name + ]), + &test_broker(), + ) + .expect("list offsets must return error response"); + let mut d = Decoder::new(response); + assert_eq!(d.read_i32().unwrap(), 0); // throttle + assert_eq!(d.read_varint().unwrap(), 2); + assert_eq!( + d.read_compact_nullable_string().unwrap(), + Some(String::new()) + ); + assert_eq!(d.read_varint().unwrap(), 2); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.read_i16().unwrap(), ERROR_INVALID_REQUEST); +} + +#[test] +fn create_topics_malformed_body_returns_invalid_request() { + let response = handle_request( + API_KEY_CREATE_TOPICS, + 5, + Bytes::from_static(&[ + 0x02, // compact topics count = one + 0x00, // null compact topic name + ]), + &test_broker(), + ) + .expect("create topics must return error response"); + let mut d = Decoder::new(response); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.read_varint().unwrap(), 2); + assert_eq!( + d.read_compact_nullable_string().unwrap(), + Some(String::new()) + ); + assert_eq!(d.read_i16().unwrap(), ERROR_INVALID_REQUEST); +} + +#[test] +fn metadata_null_topic_name_yields_zero_topics() { + let body = handle_request( + API_KEY_METADATA, + 0, + Bytes::from_static(&[ + 0x00, 0x00, 0x00, 0x01, // one topic + 0xff, 0xff, // null topic name + ]), + &test_broker(), + ) + .expect("metadata request should still return response"); + let mut d = Decoder::new(body); + assert_eq!(d.read_i32().unwrap(), 1); + d.read_i32().unwrap(); + d.read_nullable_string().unwrap(); + d.read_i32().unwrap(); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.remaining(), 0); +} diff --git a/gateways/kafka/tests/codec_tests.rs b/gateways/kafka/tests/codec_tests.rs index fcaa966477..63b576377f 100644 --- a/gateways/kafka/tests/codec_tests.rs +++ b/gateways/kafka/tests/codec_tests.rs @@ -17,6 +17,7 @@ use bytes::Bytes; +use iggy_gateway_kafka::error::KafkaProtocolError; use iggy_gateway_kafka::protocol::codec::{Decoder, Encoder}; #[test] @@ -157,3 +158,75 @@ fn tagged_fields_empty_section_round_trip() { assert_eq!(dec.read_i16().unwrap(), 7); assert_eq!(dec.remaining(), 0); } + +#[test] +fn decoder_rejects_unterminated_varint() { + let mut dec = Decoder::new(Bytes::from_static(&[ + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + ])); + let err = dec.read_varint().unwrap_err(); + assert!(matches!(err, KafkaProtocolError::InvalidVarint)); +} + +#[test] +fn compact_array_count_above_max_returns_error() { + let mut enc = Encoder::with_capacity(16); + enc.write_varint(65_538); // count = 65_537 after -1 + let mut dec = Decoder::new(enc.freeze()); + let err = dec.read_compact_array_count().unwrap_err(); + assert!(matches!(err, KafkaProtocolError::CollectionTooLarge { .. })); +} + +#[test] +fn compact_nullable_string_invalid_utf8_fails() { + let mut raw = Vec::new(); + raw.push(2); // len = 1 + raw.push(0xff); + let mut dec = Decoder::new(Bytes::from(raw)); + let err = dec.read_compact_nullable_string().unwrap_err(); + assert!(matches!(err, KafkaProtocolError::InvalidUtf8)); +} + +#[test] +fn tagged_fields_non_empty_section_is_skipped() { + let mut enc = Encoder::with_capacity(16); + enc.write_varint(1); // one tagged field + enc.write_varint(7); // tag + enc.write_varint(3); // size + enc.write_bytes(&[1, 2, 3]); + enc.write_i16(99); + + let mut dec = Decoder::new(enc.freeze()); + dec.read_tagged_fields().unwrap(); + assert_eq!(dec.read_i16().unwrap(), 99); +} + +#[test] +fn tagged_fields_oversized_count_fails() { + let mut enc = Encoder::with_capacity(16); + enc.write_varint(65_537); + let mut dec = Decoder::new(enc.freeze()); + let err = dec.read_tagged_fields().unwrap_err(); + assert!(matches!(err, KafkaProtocolError::CollectionTooLarge { .. })); +} + +#[test] +fn write_null_bytes_and_write_bytes_round_trip() { + let mut enc = Encoder::with_capacity(16); + enc.write_null_bytes(); + enc.write_bytes(&[9, 8, 7]); + let mut dec = Decoder::new(enc.freeze()); + assert_eq!(dec.read_nullable_bytes().unwrap(), None); + assert_eq!(dec.read_bytes(3).unwrap(), Bytes::from_static(&[9, 8, 7])); +} + +#[test] +fn unchecked_nullable_string_matches_checked_encoding() { + let mut checked = Encoder::with_capacity(16); + checked.write_nullable_string(Some("safe")).unwrap(); + + let mut unchecked = Encoder::with_capacity(16); + unchecked.write_nullable_string_unchecked(Some("safe")); + + assert_eq!(checked.freeze(), unchecked.freeze()); +} diff --git a/gateways/kafka/tests/common/wire.rs b/gateways/kafka/tests/common/wire.rs index 05c70b21a0..fd2edd670d 100644 --- a/gateways/kafka/tests/common/wire.rs +++ b/gateways/kafka/tests/common/wire.rs @@ -132,6 +132,70 @@ pub fn build_create_topics_empty_request(version: i16) -> Bytes { enc.freeze() } +/// Produce v2–v8 legacy request with optional transactional id and topic. +pub fn build_produce_legacy_request( + version: i16, + acks: i16, + transactional_id: Option<&str>, + topic: Option<&str>, +) -> Bytes { + let mut enc = Encoder::with_capacity(128); + if version >= 3 { + enc.write_nullable_string(transactional_id) + .expect("transactional id fits"); + } + enc.write_i16(acks); + enc.write_i32(1_000); + enc.write_i32(i32::from(topic.is_some())); + if let Some(name) = topic { + enc.write_nullable_string(Some(name)) + .expect("topic name fits"); + enc.write_i32(1); + enc.write_i32(0); + enc.write_nullable_bytes(Some(&[0x00, 0x00, 0x00, 0x00])) + .expect("records fit"); + } + enc.freeze() +} + +/// Fetch v2 body without `max_bytes` field (defaults to 50 MiB). +pub fn build_fetch_v2_default_max_bytes_request() -> Bytes { + let mut enc = Encoder::with_capacity(32); + enc.write_i32(-1); + enc.write_i32(100); + enc.write_i32(1); + enc.write_i32(0); + enc.freeze() +} + +/// Produce v9+ flexible request with one topic/partition and tagged fields. +pub fn build_produce_flexible_request_with_topic(topic: &str) -> Bytes { + let mut enc = Encoder::with_capacity(128); + enc.write_compact_nullable_string(Some("txn-1")); + enc.write_i16(1); + enc.write_i32(500); + enc.write_varint(2); + enc.write_compact_nullable_string(Some(topic)); + enc.write_varint(2); + enc.write_i32(0); + enc.write_compact_nullable_bytes(Some(&[0x00, 0x00, 0x00, 0x00])); + enc.write_empty_tagged_fields(); + enc.write_empty_tagged_fields(); + enc.write_empty_tagged_fields(); + enc.freeze() +} + +/// Fetch v3 body without isolation_level field (defaults to 0). +pub fn build_fetch_v3_no_isolation_request() -> Bytes { + let mut enc = Encoder::with_capacity(32); + enc.write_i32(-1); + enc.write_i32(100); + enc.write_i32(1); + enc.write_i32(1024); + enc.write_i32(0); + enc.freeze() +} + /// Produce v9+ flexible request with empty topics array. pub fn build_produce_flexible_empty_request(acks: i16) -> Bytes { let mut enc = Encoder::with_capacity(32); @@ -182,3 +246,188 @@ pub fn build_fetch_empty_topics_request(version: i16) -> Bytes { enc.freeze() } + +/// Fetch request with one topic/partition and optional forgotten topics / rack id. +pub fn build_fetch_request_with_sections( + version: i16, + topic: &str, + partition: i32, + forgotten_topic: Option<&str>, + rack_id: Option<&str>, +) -> Bytes { + let flexible = version >= 12; + let mut enc = Encoder::with_capacity(256); + + enc.write_i32(-1); // replica_id + enc.write_i32(100); // max_wait_ms + enc.write_i32(1); // min_bytes + if version >= 3 { + enc.write_i32(i32::MAX); // max_bytes + } + if version >= 4 { + enc.write_i8(0); // isolation_level + } + if version >= 7 { + enc.write_i32(7); // session_id + enc.write_i32(1); // session_epoch + } + + if flexible { + enc.write_varint(2); // one topic + enc.write_compact_nullable_string(Some(topic)); + enc.write_varint(2); // one partition + } else { + enc.write_i32(1); + enc.write_nullable_string(Some(topic)) + .expect("topic name fits"); + enc.write_i32(1); + } + + enc.write_i32(partition); + if version >= 9 { + enc.write_i32(-1); // current_leader_epoch + } + enc.write_i64(42); // fetch_offset + if version >= 12 { + enc.write_i32(-1); // last_fetched_epoch + } + if version >= 5 { + enc.write_i64(0); // log_start_offset + } + enc.write_i32(1024); // partition_max_bytes + if flexible { + enc.write_empty_tagged_fields(); // partition tagged fields + enc.write_empty_tagged_fields(); // topic tagged fields + } + + if version >= 7 { + let forgotten_count = usize::from(forgotten_topic.is_some()); + if flexible { + enc.write_varint((forgotten_count + 1) as u64); + } else { + enc.write_i32(i32::try_from(forgotten_count).expect("count fits i32")); + } + if let Some(name) = forgotten_topic { + if flexible { + enc.write_compact_nullable_string(Some(name)); + enc.write_varint(2); // one partition + enc.write_i32(partition); + enc.write_empty_tagged_fields(); + } else { + enc.write_nullable_string(Some(name)) + .expect("topic name fits"); + enc.write_i32(1); + enc.write_i32(partition); + } + } + } + + if version >= 11 { + if flexible { + enc.write_compact_nullable_string(rack_id); + } else { + enc.write_nullable_string(rack_id).expect("rack id fits"); + } + } + + if flexible { + enc.write_empty_tagged_fields(); + } + + enc.freeze() +} + +/// ListOffsets request covering legacy v0 `max_num_offsets` and newer leader-epoch branches. +pub fn build_list_offsets_branch_request(version: i16, topic: &str, partition: i32) -> Bytes { + let flexible = version >= 6; + let mut enc = Encoder::with_capacity(128); + enc.write_i32(-1); // replica_id + if version >= 2 { + enc.write_i8(1); // isolation_level + } + + if flexible { + enc.write_varint(2); // one topic + enc.write_compact_nullable_string(Some(topic)); + enc.write_varint(2); // one partition + } else { + enc.write_i32(1); + enc.write_nullable_string(Some(topic)).expect("topic fits"); + enc.write_i32(1); + } + + enc.write_i32(partition); + if version >= 4 { + enc.write_i32(-1); // current_leader_epoch + } + enc.write_i64(-2); // earliest + if version == 0 { + enc.write_i32(1); // max_num_offsets + } + if flexible { + enc.write_empty_tagged_fields(); + enc.write_empty_tagged_fields(); + enc.write_empty_tagged_fields(); + } + + enc.freeze() +} + +/// CreateTopics request with one topic, one assignment, and one config. +pub fn build_create_topics_request_with_sections(version: i16, topic: &str) -> Bytes { + let flexible = version >= 5; + let mut enc = Encoder::with_capacity(256); + + if flexible { + enc.write_varint(2); // one topic + enc.write_compact_nullable_string(Some(topic)); + } else { + enc.write_i32(1); + enc.write_nullable_string(Some(topic)).expect("topic fits"); + } + enc.write_i32(3); // num_partitions + enc.write_i16(1); // replication_factor + + if flexible { + enc.write_varint(2); // one assignment + } else { + enc.write_i32(1); + } + enc.write_i32(0); // partition_index + if flexible { + enc.write_varint(2); // one replica + } else { + enc.write_i32(1); + } + enc.write_i32(1); // broker_id + if flexible { + enc.write_empty_tagged_fields(); + } + + if flexible { + enc.write_varint(2); // one config + enc.write_compact_nullable_string(Some("cleanup.policy")); + enc.write_compact_nullable_string(Some("delete")); + enc.write_empty_tagged_fields(); + } else { + enc.write_i32(1); + enc.write_nullable_string(Some("cleanup.policy")) + .expect("config key fits"); + enc.write_nullable_string(Some("delete")) + .expect("config value fits"); + } + + if flexible { + enc.write_empty_tagged_fields(); // topic tagged fields + } + + enc.write_i32(5_000); // timeout_ms + if version >= 1 { + enc.write_bool(true); // validate_only + } + if flexible { + enc.write_empty_tagged_fields(); + } + + enc.freeze() +} diff --git a/gateways/kafka/tests/decode_validation_tests.rs b/gateways/kafka/tests/decode_validation_tests.rs index b2b8fe55eb..bd7d984c1a 100644 --- a/gateways/kafka/tests/decode_validation_tests.rs +++ b/gateways/kafka/tests/decode_validation_tests.rs @@ -40,6 +40,9 @@ use iggy_gateway_kafka::protocol::responses::{ encode_produce_response, }; +#[path = "common/wire.rs"] +mod wire; + // ── helpers ─────────────────────────────────────────────────────────────────── fn fixtures_dir() -> PathBuf { @@ -167,6 +170,15 @@ fn produce_response_v8_includes_record_errors() { assert!(error_message.is_none(), "v8 error_message must be null"); } +#[test] +fn produce_v9_flexible_empty_topics_decode() { + let req = decode_produce_request(9, wire::build_produce_flexible_empty_request(0)) + .into_request() + .expect("flexible produce request should decode"); + assert_eq!(req.acks, 0); + assert_eq!(req.topics.len(), 0); +} + // ── Fetch (API key 1) ───────────────────────────────────────────────────────── #[test] @@ -244,6 +256,25 @@ fn fetch_response_v7_roundtrip() { assert_eq!(high_watermark, 0); } +#[test] +fn fetch_v12_decodes_forgotten_topics_and_rack_id_sections() { + let req = decode_fetch_request( + 12, + wire::build_fetch_request_with_sections(12, "test-topic", 2, Some("forgotten"), Some("r1")), + ) + .expect("fetch request should decode"); + assert_eq!(req.max_wait_ms, 100); + assert_eq!(req.min_bytes, 1); + assert_eq!(req.max_bytes, i32::MAX); + assert_eq!(req.isolation_level, 0); + assert_eq!(req.topics.len(), 1); + assert_eq!(req.topics[0].topic, "test-topic"); + assert_eq!(req.topics[0].partitions.len(), 1); + assert_eq!(req.topics[0].partitions[0].partition, 2); + assert_eq!(req.topics[0].partitions[0].fetch_offset, 42); + assert_eq!(req.topics[0].partitions[0].partition_max_bytes, 1024); +} + // ── ListOffsets (API key 2) ─────────────────────────────────────────────────── #[test] @@ -274,6 +305,28 @@ fn list_offsets_all_supported_versions_decode() { } } +#[test] +fn list_offsets_v0_decodes_legacy_max_num_offsets_branch() { + let req = + decode_list_offsets_request(0, wire::build_list_offsets_branch_request(0, "legacy", 4)) + .expect("v0 list offsets should decode"); + assert_eq!(req.isolation_level, 0); + assert_eq!(req.topics.len(), 1); + assert_eq!(req.topics[0].topic, "legacy"); + assert_eq!(req.topics[0].partitions[0].partition, 4); + assert_eq!(req.topics[0].partitions[0].timestamp, -2); +} + +#[test] +fn list_offsets_v6_decodes_flexible_leader_epoch_branch() { + let req = decode_list_offsets_request(6, wire::build_list_offsets_branch_request(6, "flex", 5)) + .expect("v6 list offsets should decode"); + assert_eq!(req.isolation_level, 1); + assert_eq!(req.topics[0].topic, "flex"); + assert_eq!(req.topics[0].partitions[0].partition, 5); + assert_eq!(req.topics[0].partitions[0].timestamp, -2); +} + #[test] fn list_offsets_response_encodes_for_all_supported_versions() { for version in 1i16..=6 { @@ -372,6 +425,36 @@ fn create_topics_all_supported_versions_decode() { } } +#[test] +fn create_topics_v0_defaults_validate_only_to_false() { + let req = decode_create_topics_request( + 0, + wire::build_create_topics_request_with_sections(0, "legacy-topic"), + ) + .expect("create topics v0 should decode"); + assert_eq!(req.timeout_ms, 5_000); + assert!(!req.validate_only); + assert_eq!(req.topics.len(), 1); + assert_eq!(req.topics[0].name, "legacy-topic"); + assert_eq!(req.topics[0].num_partitions, 3); + assert_eq!(req.topics[0].replication_factor, 1); +} + +#[test] +fn create_topics_v5_decodes_flexible_assignments_and_configs() { + let req = decode_create_topics_request( + 5, + wire::build_create_topics_request_with_sections(5, "flex-topic"), + ) + .expect("create topics v5 should decode"); + assert_eq!(req.timeout_ms, 5_000); + assert!(req.validate_only); + assert_eq!(req.topics.len(), 1); + assert_eq!(req.topics[0].name, "flex-topic"); + assert_eq!(req.topics[0].num_partitions, 3); + assert_eq!(req.topics[0].replication_factor, 1); +} + #[test] fn create_topics_response_encodes_for_all_supported_versions() { for version in 2i16..=5 { diff --git a/gateways/kafka/tests/header_tests.rs b/gateways/kafka/tests/header_tests.rs index 6ea323fdfe..809f5225ca 100644 --- a/gateways/kafka/tests/header_tests.rs +++ b/gateways/kafka/tests/header_tests.rs @@ -103,6 +103,124 @@ fn response_header_v1_encodes_correlation_id_plus_tagged_fields() { // ── Header version lookup ─────────────────────────────────────────────────── +/// Flexible-encoding threshold per API key (mirrors `protocol/header.rs`). +const API_KEY_FLEXIBLE_FROM: &[(i16, i16)] = &[ + (0, 9), + (1, 12), + (2, 6), + (3, 9), + (4, 4), + (5, 2), + (6, 6), + (7, 3), + (8, 8), + (9, 6), + (10, 3), + (11, 6), + (12, 4), + (13, 4), + (14, 4), + (15, 5), + (16, 3), + (17, i16::MAX), + (18, 3), + (19, 5), + (20, 4), + (21, 2), + (22, 2), + (23, 4), + (24, 3), + (25, 3), + (26, 3), + (27, 1), + (28, 3), + (29, 2), + (30, 2), + (31, 2), + (32, 4), + (33, 2), + (34, 2), + (35, 2), + (36, 2), + (37, 2), + (38, 2), + (39, 2), + (40, 2), + (41, 2), + (42, 2), + (43, 2), + (44, 1), + (45, 0), + (46, 0), + (47, i16::MAX), + (48, 1), + (49, 1), + (50, 0), + (51, 0), + (55, 0), + (56, 0), + (57, 1), + (60, 0), + (61, 0), + (64, 0), + (65, 0), + (66, 0), + (67, 0), + (68, 0), + (69, 0), + (71, 0), + (72, 0), + (74, 0), + (75, 0), + (76, 0), + (77, 0), + (78, 0), + (79, 0), + (80, 0), +]; + +#[test] +fn request_header_version_hits_every_api_key_match_arm() { + for &(api_key, flexible_from) in API_KEY_FLEXIBLE_FROM { + match flexible_from { + 0 => { + assert_eq!(request_header_version(api_key, 0), 2); + assert_eq!(request_header_version(api_key, i16::MAX), 2); + } + i16::MAX => { + assert_eq!(request_header_version(api_key, 0), 1); + assert_eq!(request_header_version(api_key, i16::MAX - 1), 1); + } + threshold => { + assert_eq!(request_header_version(api_key, threshold - 1), 1); + assert_eq!(request_header_version(api_key, threshold), 2); + } + } + } +} + +#[test] +fn response_header_version_hits_every_api_key_match_arm() { + for &(api_key, _) in API_KEY_FLEXIBLE_FROM { + if api_key == 18 { + assert_eq!(response_header_version(api_key, 0), 0); + assert_eq!(response_header_version(api_key, 99), 0); + continue; + } + let req_hdr_at_v0 = request_header_version(api_key, 0); + let expected_at_v0 = if req_hdr_at_v0 >= 2 { 1 } else { 0 }; + assert_eq!(response_header_version(api_key, 0), expected_at_v0); + + let req_hdr_at_max = request_header_version(api_key, i16::MAX - 1); + let expected_at_max = if req_hdr_at_max >= 2 { 1 } else { 0 }; + assert_eq!( + response_header_version(api_key, i16::MAX - 1), + expected_at_max + ); + } + assert_eq!(response_header_version(999, 0), 0); +} + #[test] fn request_header_version_non_flexible_below_threshold() { // ApiVersions v0-2 → header v1 @@ -148,3 +266,81 @@ fn response_header_version_flexible_non_apiversions() { // Metadata v0 is non-flexible → response header v0 assert_eq!(response_header_version(3, 0), 0); } + +#[test] +fn request_header_version_never_flexible_keys_stay_v1() { + for key in [17, 47] { + assert_eq!( + request_header_version(key, i16::MAX - 1), + 1, + "api_key {key} must stay on header v1" + ); + } +} + +#[test] +fn request_header_version_always_flexible_keys_use_v2() { + for key in [ + 45, 46, 50, 51, 55, 60, 61, 64, 65, 66, 67, 68, 69, 71, 72, 74, 75, 76, + ] { + assert_eq!( + request_header_version(key, 0), + 2, + "api_key {key} must use flexible header v2" + ); + } +} + +#[test] +fn request_header_version_unknown_api_defaults_to_v1() { + assert_eq!(request_header_version(999, 0), 1); + assert_eq!(request_header_version(-1, 12), 1); +} + +#[test] +fn request_header_decode_rejects_unsupported_version() { + let bytes = Encoder::with_capacity(0).freeze(); + let err = RequestHeader::decode(bytes, 99).unwrap_err(); + assert!(matches!( + err, + iggy_gateway_kafka::error::KafkaProtocolError::UnsupportedHeaderVersion(99) + )); +} + +#[test] +fn request_header_v1_truncated_payload_fails() { + let mut enc = Encoder::with_capacity(8); + enc.write_i16(18); + enc.write_i16(1); + let err = RequestHeader::decode(enc.freeze(), 1).unwrap_err(); + assert!(err.to_string().contains("buffer underflow")); +} + +#[test] +fn request_header_v2_truncated_before_tagged_fields_fails() { + let mut enc = Encoder::with_capacity(16); + enc.write_i16(18); + enc.write_i16(3); + enc.write_i32(303); + enc.write_compact_nullable_string(Some("c")); + let err = RequestHeader::decode(enc.freeze(), 2).unwrap_err(); + assert!(err.to_string().contains("buffer underflow")); +} + +#[test] +fn response_header_encode_into_matches_encode() { + let header = ResponseHeader { + correlation_id: 1234, + }; + let encoded = header.encode(1); + let mut buf = bytes::BytesMut::new(); + header.encode_into(&mut buf, 1); + assert_eq!(buf.freeze(), encoded); +} + +#[test] +fn response_header_encoded_size_matches_versions() { + assert_eq!(ResponseHeader::encoded_size(0), 4); + assert_eq!(ResponseHeader::encoded_size(1), 5); + assert_eq!(ResponseHeader::encoded_size(2), 5); +} diff --git a/gateways/kafka/tests/region_coverage_tests.rs b/gateways/kafka/tests/region_coverage_tests.rs new file mode 100644 index 0000000000..ea58d3fa1b --- /dev/null +++ b/gateways/kafka/tests/region_coverage_tests.rs @@ -0,0 +1,445 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Decoder and listener paths that exist only for region coverage of version branches. + +use std::time::Duration; + +use bytes::{BufMut, BytesMut}; +use iggy_gateway_kafka::error::KafkaProtocolError; +use iggy_gateway_kafka::protocol::codec::Encoder; +use iggy_gateway_kafka::protocol::requests::{ + ProduceDecodeResult, decode_create_topics_request, decode_fetch_request, + decode_list_offsets_request, decode_produce_request, +}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpStream; +use tokio::time; + +#[path = "common/server.rs"] +mod server; +#[path = "common/tcp.rs"] +mod tcp; +#[path = "common/wire.rs"] +mod wire; + +use tcp::{build_request_frame, read_byte_with_timeout, read_response_frame}; + +// ── Produce version branches ────────────────────────────────────────────────── + +#[test] +fn produce_v2_skips_transactional_id_branch() { + let req = decode_produce_request(2, wire::build_produce_legacy_request(2, 1, None, None)) + .into_request() + .expect("produce v2 should decode"); + assert_eq!(req.acks, 1); + assert!(req.transactional_id.is_none()); + assert!(req.topics.is_empty()); +} + +#[test] +fn produce_v3_legacy_transactional_id_and_topic_decode() { + let req = decode_produce_request( + 3, + wire::build_produce_legacy_request(3, -1, Some("txn-1"), Some("legacy-topic")), + ) + .into_request() + .expect("produce v3 legacy should decode"); + assert_eq!(req.transactional_id.as_deref(), Some("txn-1")); + assert_eq!(req.topics.len(), 1); + assert_eq!(req.topics[0].topic, "legacy-topic"); + assert!(req.topics[0].partitions[0].records.is_some()); +} + +#[test] +fn produce_v8_legacy_null_records_decode() { + let mut enc = Encoder::with_capacity(64); + enc.write_nullable_string(None::<&str>).unwrap(); + enc.write_i16(1); + enc.write_i32(500); + enc.write_i32(1); + enc.write_nullable_string(Some("topic")).unwrap(); + enc.write_i32(1); + enc.write_i32(0); + enc.write_nullable_bytes(None).unwrap(); + + let req = decode_produce_request(8, enc.freeze()) + .into_request() + .expect("produce v8 with null records should decode"); + assert!(req.topics[0].partitions[0].records.is_none()); +} + +#[test] +fn produce_null_topic_name_preserves_acks_on_error() { + let mut enc = Encoder::with_capacity(32); + enc.write_nullable_string(None::<&str>).unwrap(); + enc.write_i16(1); + enc.write_i32(500); + enc.write_i32(1); + enc.write_nullable_string(None::<&str>).unwrap(); + + match decode_produce_request(3, enc.freeze()) { + ProduceDecodeResult::Err { acks, error } => { + assert_eq!(acks, Some(1)); + assert!(matches!(error, KafkaProtocolError::NullTopicName)); + } + other => panic!("expected NullTopicName, got {other:?}"), + } +} + +#[test] +fn produce_v9_flexible_transactional_id_and_tagged_fields_decode() { + let req = decode_produce_request( + 9, + wire::build_produce_flexible_request_with_topic("flex-topic"), + ) + .into_request() + .expect("produce v9 flexible should decode"); + assert_eq!(req.transactional_id.as_deref(), Some("txn-1")); + assert_eq!(req.topics[0].topic, "flex-topic"); + assert!(req.topics[0].partitions[0].records.is_some()); +} + +#[test] +fn produce_v3_error_before_acks_has_none_acks() { + let mut enc = Encoder::with_capacity(8); + enc.write_i16(1); + match decode_produce_request(3, enc.freeze()) { + ProduceDecodeResult::Err { acks, .. } => assert_eq!(acks, None), + other => panic!("expected decode error before acks, got {other:?}"), + } +} + +#[test] +fn produce_v3_error_after_acks_preserves_acks() { + let mut enc = Encoder::with_capacity(16); + enc.write_nullable_string(None::<&str>).unwrap(); + enc.write_i16(7); + match decode_produce_request(3, enc.freeze()) { + ProduceDecodeResult::Err { acks, .. } => assert_eq!(acks, Some(7)), + other => panic!("expected decode error after acks, got {other:?}"), + } +} + +#[test] +fn produce_v3_error_after_timeout_preserves_acks() { + let mut enc = Encoder::with_capacity(16); + enc.write_nullable_string(None::<&str>).unwrap(); + enc.write_i16(1); + enc.write_i32(500); + match decode_produce_request(3, enc.freeze()) { + ProduceDecodeResult::Err { acks, .. } => assert_eq!(acks, Some(1)), + other => panic!("expected decode error after timeout, got {other:?}"), + } +} + +#[test] +fn produce_v9_error_on_null_topic_preserves_acks() { + let mut enc = Encoder::with_capacity(32); + enc.write_compact_nullable_string(None); + enc.write_i16(2); + enc.write_i32(500); + enc.write_varint(2); + enc.write_compact_nullable_string(None); + match decode_produce_request(9, enc.freeze()) { + ProduceDecodeResult::Err { acks, error } => { + assert_eq!(acks, Some(2)); + assert!(matches!(error, KafkaProtocolError::NullTopicName)); + } + other => panic!("expected NullTopicName, got {other:?}"), + } +} + +#[test] +fn produce_v9_error_on_partition_count_preserves_acks() { + let mut enc = Encoder::with_capacity(64); + enc.write_compact_nullable_string(None); + enc.write_i16(3); + enc.write_i32(500); + enc.write_varint(2); + enc.write_compact_nullable_string(Some("topic")); + match decode_produce_request(9, enc.freeze()) { + ProduceDecodeResult::Err { acks, .. } => assert_eq!(acks, Some(3)), + other => panic!("expected decode error in partition count, got {other:?}"), + } +} + +#[test] +fn produce_v9_error_on_partition_records_preserves_acks() { + let mut enc = Encoder::with_capacity(64); + enc.write_compact_nullable_string(None); + enc.write_i16(4); + enc.write_i32(500); + enc.write_varint(2); + enc.write_compact_nullable_string(Some("topic")); + enc.write_varint(2); + enc.write_i32(0); + match decode_produce_request(9, enc.freeze()) { + ProduceDecodeResult::Err { acks, .. } => assert_eq!(acks, Some(4)), + other => panic!("expected decode error in records, got {other:?}"), + } +} + +// ── Fetch version branches ──────────────────────────────────────────────────── + +#[test] +fn fetch_v2_uses_default_max_bytes_when_field_absent() { + let req = decode_fetch_request(2, wire::build_fetch_v2_default_max_bytes_request()) + .expect("fetch v2 should decode"); + assert_eq!(req.max_bytes, 52_428_800); + assert_eq!(req.isolation_level, 0); + assert!(req.topics.is_empty()); +} + +#[test] +fn fetch_v7_legacy_forgotten_topics_and_rack_id_decode() { + let req = decode_fetch_request( + 7, + wire::build_fetch_request_with_sections(7, "topic-a", 1, Some("forgotten"), Some("rack-1")), + ) + .expect("fetch v7 legacy sections should decode"); + assert_eq!(req.topics[0].topic, "topic-a"); + assert_eq!(req.topics[0].partitions[0].partition, 1); +} + +#[test] +fn fetch_v9_leader_epoch_without_v12_fields_decode() { + let req = decode_fetch_request( + 9, + wire::build_fetch_request_with_sections(9, "topic-b", 2, None, None), + ) + .expect("fetch v9 should decode"); + assert_eq!(req.topics[0].partitions[0].fetch_offset, 42); +} + +#[test] +fn fetch_v11_legacy_rack_id_decode() { + let req = decode_fetch_request( + 11, + wire::build_fetch_request_with_sections(11, "topic-c", 3, None, Some("rack-z")), + ) + .expect("fetch v11 legacy rack id should decode"); + assert_eq!(req.max_wait_ms, 100); +} + +#[test] +fn fetch_v3_skips_isolation_level_field() { + let req = decode_fetch_request(3, wire::build_fetch_v3_no_isolation_request()) + .expect("fetch v3 should decode"); + assert_eq!(req.isolation_level, 0); + assert_eq!(req.max_bytes, 1024); +} + +#[test] +fn fetch_v4_truncated_after_replica_id_returns_error() { + let mut enc = Encoder::with_capacity(4); + enc.write_i32(-1); + let err = decode_fetch_request(4, enc.freeze()).unwrap_err(); + assert!(matches!(err, KafkaProtocolError::BufferUnderflow { .. })); +} + +#[test] +fn fetch_v7_truncated_in_forgotten_topics_returns_error() { + let body = wire::build_fetch_request_with_sections(7, "topic", 0, Some("forgot"), None); + let truncated = body.slice(..body.len() - 2); + let err = decode_fetch_request(7, truncated).unwrap_err(); + assert!(matches!(err, KafkaProtocolError::BufferUnderflow { .. })); +} + +#[test] +fn fetch_v12_flexible_truncated_in_topic_tagged_fields_returns_error() { + let body = wire::build_fetch_request_with_sections(12, "topic", 0, None, None); + let truncated = body.slice(..body.len() - 1); + let err = decode_fetch_request(12, truncated).unwrap_err(); + assert!(matches!(err, KafkaProtocolError::BufferUnderflow { .. })); +} + +#[test] +fn fetch_v12_flexible_null_topic_name_returns_error() { + let mut enc = Encoder::with_capacity(64); + enc.write_i32(-1); + enc.write_i32(100); + enc.write_i32(1); + enc.write_i32(1024); + enc.write_i8(0); + enc.write_i32(0); + enc.write_i32(0); + enc.write_varint(2); + enc.write_compact_nullable_string(None); + let err = decode_fetch_request(12, enc.freeze()).unwrap_err(); + assert!(matches!(err, KafkaProtocolError::NullTopicName)); +} + +#[test] +fn fetch_null_topic_name_returns_error() { + let mut enc = Encoder::with_capacity(64); + enc.write_i32(-1); + enc.write_i32(100); + enc.write_i32(1); + enc.write_i32(i32::MAX); + enc.write_i8(0); + enc.write_i32(1); + enc.write_nullable_string(None::<&str>).unwrap(); + + let err = decode_fetch_request(4, enc.freeze()).unwrap_err(); + assert!(matches!(err, KafkaProtocolError::NullTopicName)); +} + +// ── ListOffsets version branches ────────────────────────────────────────────── + +#[test] +fn list_offsets_v1_skips_isolation_level_field() { + let req = decode_list_offsets_request(1, wire::build_list_offsets_branch_request(1, "v1", 0)) + .expect("list offsets v1 should decode"); + assert_eq!(req.isolation_level, 0); +} + +#[test] +fn list_offsets_v2_reads_isolation_level() { + let req = decode_list_offsets_request(2, wire::build_list_offsets_branch_request(2, "v2", 1)) + .expect("list offsets v2 should decode"); + assert_eq!(req.isolation_level, 1); +} + +#[test] +fn list_offsets_v3_skips_leader_epoch_branch() { + let req = decode_list_offsets_request(3, wire::build_list_offsets_branch_request(3, "v3", 2)) + .expect("list offsets v3 should decode"); + assert_eq!(req.topics[0].partitions[0].partition, 2); +} + +#[test] +fn list_offsets_v5_leader_epoch_without_flexible_encoding() { + let req = decode_list_offsets_request(5, wire::build_list_offsets_branch_request(5, "v5", 3)) + .expect("list offsets v5 should decode"); + assert_eq!(req.topics[0].topic, "v5"); +} + +#[test] +fn list_offsets_v4_truncated_in_leader_epoch_returns_error() { + let body = wire::build_list_offsets_branch_request(4, "topic", 1); + let truncated = body.slice(..body.len() - 4); + let err = decode_list_offsets_request(4, truncated).unwrap_err(); + assert!(matches!(err, KafkaProtocolError::BufferUnderflow { .. })); +} + +#[test] +fn list_offsets_v6_flexible_null_topic_name_returns_error() { + let mut enc = Encoder::with_capacity(32); + enc.write_i32(-1); + enc.write_i8(0); + enc.write_varint(2); + enc.write_compact_nullable_string(None); + let err = decode_list_offsets_request(6, enc.freeze()).unwrap_err(); + assert!(matches!(err, KafkaProtocolError::NullTopicName)); +} + +#[test] +fn list_offsets_null_topic_name_returns_error() { + let mut enc = Encoder::with_capacity(32); + enc.write_i32(-1); + enc.write_i8(0); + enc.write_i32(1); + enc.write_nullable_string(None::<&str>).unwrap(); + let err = decode_list_offsets_request(2, enc.freeze()).unwrap_err(); + assert!(matches!(err, KafkaProtocolError::NullTopicName)); +} + +// ── CreateTopics version branches ───────────────────────────────────────────── + +#[test] +fn create_topics_v3_legacy_assignments_decode() { + let req = decode_create_topics_request( + 3, + wire::build_create_topics_request_with_sections(3, "v3-topic"), + ) + .expect("create topics v3 should decode"); + assert_eq!(req.topics[0].name, "v3-topic"); + assert!(req.validate_only); +} + +#[test] +fn create_topics_v4_legacy_configs_decode() { + let req = decode_create_topics_request( + 4, + wire::build_create_topics_request_with_sections(4, "v4-topic"), + ) + .expect("create topics v4 should decode"); + assert_eq!(req.topics[0].replication_factor, 1); +} + +#[test] +fn create_topics_v2_truncated_in_config_value_returns_error() { + let body = wire::build_create_topics_request_with_sections(2, "topic"); + let truncated = body.slice(..body.len() - 3); + let err = decode_create_topics_request(2, truncated).unwrap_err(); + assert!(matches!(err, KafkaProtocolError::BufferUnderflow { .. })); +} + +#[test] +fn create_topics_v5_flexible_null_topic_name_returns_error() { + let mut enc = Encoder::with_capacity(16); + enc.write_varint(2); + enc.write_compact_nullable_string(None); + let err = decode_create_topics_request(5, enc.freeze()).unwrap_err(); + assert!(matches!(err, KafkaProtocolError::NullTopicName)); +} + +#[test] +fn create_topics_null_topic_name_returns_error() { + let mut enc = Encoder::with_capacity(32); + enc.write_i32(1); + enc.write_nullable_string(None::<&str>).unwrap(); + let err = decode_create_topics_request(2, enc.freeze()).unwrap_err(); + assert!(matches!(err, KafkaProtocolError::NullTopicName)); +} + +// ── Server handle_connection branches (e2e) ─────────────────────────────────── + +#[tokio::test] +async fn e2e_frame_shorter_than_kafka_header_returns_buffer_underflow() { + let (addr, _shutdown) = server::spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.expect("connect"); + + let mut frame = BytesMut::new(); + frame.extend_from_slice(&7_i32.to_be_bytes()); + frame.extend_from_slice(&[0x00, 0x12, 0x00, 0x01, 0x00, 0x00, 0x00]); + stream.write_all(&frame).await.expect("short header write"); + + let byte = read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await; + assert!( + byte.is_none(), + "frame payload shorter than 8-byte Kafka header must close connection" + ); +} + +#[tokio::test] +async fn e2e_client_eof_after_valid_frame_closes_connection_cleanly() { + let (addr, _shutdown) = server::spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.expect("connect"); + + let frame = build_request_frame(18, 1, 901, Some("eof-test"), &[]); + stream.write_all(&frame).await.expect("api versions write"); + let _payload = read_response_frame(&mut stream, 8 * 1024 * 1024).await; + + stream.shutdown().await.expect("client shutdown"); + time::sleep(Duration::from_millis(100)).await; + + let mut buf = [0u8; 1]; + let n = stream.read(&mut buf).await.expect("read after shutdown"); + assert_eq!(n, 0, "server should close after client EOF"); +} diff --git a/gateways/kafka/tests/response_negative_tests.rs b/gateways/kafka/tests/response_negative_tests.rs new file mode 100644 index 0000000000..f69fdadd11 --- /dev/null +++ b/gateways/kafka/tests/response_negative_tests.rs @@ -0,0 +1,202 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use iggy_gateway_kafka::protocol::api::{ + ERROR_INVALID_PARTITIONS, ERROR_INVALID_REQUEST, ERROR_UNSUPPORTED_VERSION, +}; +use iggy_gateway_kafka::protocol::codec::Decoder; +use iggy_gateway_kafka::protocol::requests::{ + CreatableTopic, CreateTopicsRequest, FetchPartition, FetchRequest, FetchTopic, + ListOffsetsPartition, ListOffsetsRequest, ListOffsetsTopic, ProducePartitionData, + ProduceRequest, ProduceTopicData, +}; +use iggy_gateway_kafka::protocol::responses::{ + encode_create_topics_error_response, encode_create_topics_response, + encode_fetch_error_response, encode_list_offsets_error_response, encode_produce_error_response, +}; + +#[test] +fn create_topics_response_flags_non_positive_partition_count_v2() { + let req = CreateTopicsRequest { + topics: vec![CreatableTopic { + name: "bad-topic".to_string(), + num_partitions: 0, + replication_factor: 1, + }], + timeout_ms: 5_000, + validate_only: false, + }; + let mut d = Decoder::new(encode_create_topics_response(2, &req)); + assert_eq!(d.read_i32().unwrap(), 0); // throttle + assert_eq!(d.read_i32().unwrap(), 1); // topics len + assert_eq!( + d.read_nullable_string().unwrap(), + Some("bad-topic".to_string()) + ); + assert_eq!(d.read_i16().unwrap(), ERROR_INVALID_PARTITIONS); +} + +#[test] +fn create_topics_response_flags_negative_partition_count_v5_flexible() { + let req = CreateTopicsRequest { + topics: vec![CreatableTopic { + name: "bad-flex".to_string(), + num_partitions: -1, + replication_factor: 2, + }], + timeout_ms: 5_000, + validate_only: true, + }; + let mut d = Decoder::new(encode_create_topics_response(5, &req)); + assert_eq!(d.read_i32().unwrap(), 0); // throttle + assert_eq!(d.read_varint().unwrap(), 2); // one topic + assert_eq!( + d.read_compact_nullable_string().unwrap(), + Some("bad-flex".to_string()) + ); + assert_eq!(d.read_i16().unwrap(), ERROR_INVALID_PARTITIONS); + assert_eq!(d.read_compact_nullable_string().unwrap(), None); + assert_eq!(d.read_i32().unwrap(), -1); + assert_eq!(d.read_i16().unwrap(), 2); +} + +#[test] +fn create_topics_error_response_carries_explicit_error_code() { + let mut d = Decoder::new(encode_create_topics_error_response( + 5, + ERROR_UNSUPPORTED_VERSION, + )); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.read_varint().unwrap(), 2); + assert_eq!( + d.read_compact_nullable_string().unwrap(), + Some(String::new()) + ); + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); +} + +#[test] +fn fetch_error_response_v7_uses_top_level_error_and_no_topics() { + let mut d = Decoder::new(encode_fetch_error_response(7, ERROR_INVALID_REQUEST)); + assert_eq!(d.read_i32().unwrap(), 0); // throttle + assert_eq!(d.read_i16().unwrap(), ERROR_INVALID_REQUEST); + assert_eq!(d.read_i32().unwrap(), 0); // session_id + assert_eq!(d.read_i32().unwrap(), 0); // empty topics + assert_eq!(d.remaining(), 0); +} + +#[test] +fn fetch_error_response_v12_uses_flexible_empty_topics() { + let mut d = Decoder::new(encode_fetch_error_response(12, ERROR_UNSUPPORTED_VERSION)); + assert_eq!(d.read_i32().unwrap(), 0); // throttle + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); + assert_eq!(d.read_i32().unwrap(), 0); // session_id + assert_eq!(d.read_varint().unwrap(), 1); // empty topics compact array + d.read_tagged_fields().unwrap(); + assert_eq!(d.remaining(), 0); +} + +#[test] +fn list_offsets_error_response_v0_uses_legacy_old_style_offsets_array() { + let mut d = Decoder::new(encode_list_offsets_error_response( + 0, + ERROR_UNSUPPORTED_VERSION, + )); + assert_eq!(d.read_i32().unwrap(), 1); // topics + assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); + assert_eq!(d.read_i32().unwrap(), 1); // partitions + assert_eq!(d.read_i32().unwrap(), 0); // partition index + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); + assert_eq!(d.read_i32().unwrap(), 0); // old_style_offsets len + assert_eq!(d.remaining(), 0); +} + +#[test] +fn produce_error_response_v9_uses_flexible_record_errors_shape() { + let mut d = Decoder::new(encode_produce_error_response(9, ERROR_INVALID_REQUEST)); + assert_eq!(d.read_varint().unwrap(), 2); // one topic + assert_eq!( + d.read_compact_nullable_string().unwrap(), + Some(String::new()) + ); + assert_eq!(d.read_varint().unwrap(), 2); // one partition + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.read_i16().unwrap(), ERROR_INVALID_REQUEST); + assert_eq!(d.read_i64().unwrap(), 0); + assert_eq!(d.read_i64().unwrap(), -1); + assert_eq!(d.read_i64().unwrap(), 0); + assert_eq!(d.read_varint().unwrap(), 1); // empty record_errors array + assert_eq!(d.read_compact_nullable_string().unwrap(), None); +} + +#[test] +fn success_responses_can_still_encode_empty_request_vectors() { + let produce = ProduceRequest { + transactional_id: None, + acks: 1, + timeout_ms: 1_000, + topics: Vec::::new(), + }; + assert!( + !iggy_gateway_kafka::protocol::responses::encode_produce_response(3, &produce).is_empty() + ); + + let fetch = FetchRequest { + max_wait_ms: 0, + min_bytes: 0, + max_bytes: 0, + isolation_level: 0, + topics: Vec::::new(), + }; + assert!(!iggy_gateway_kafka::protocol::responses::encode_fetch_response(4, &fetch).is_empty()); + + let list_offsets = ListOffsetsRequest { + isolation_level: 0, + topics: Vec::::new(), + }; + assert!( + !iggy_gateway_kafka::protocol::responses::encode_list_offsets_response(1, &list_offsets) + .is_empty() + ); + + let create_topics = CreateTopicsRequest { + topics: Vec::::new(), + timeout_ms: 0, + validate_only: false, + }; + assert!( + !iggy_gateway_kafka::protocol::responses::encode_create_topics_response(2, &create_topics) + .is_empty() + ); +} + +#[allow(clippy::let_unit_value)] +fn _type_anchors() { + let _ = FetchPartition { + partition: 0, + fetch_offset: 0, + partition_max_bytes: 0, + }; + let _ = ListOffsetsPartition { + partition: 0, + timestamp: 0, + }; + let _ = ProducePartitionData { + partition: 0, + records: None, + }; +} From 18e815f2288056cd9d967576771403309d9aab17 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Wed, 8 Jul 2026 17:45:45 -0400 Subject: [PATCH 29/57] Clean up Kafka gateway test code for clippy errors Improve readability and consistency across the Kafka gateway tests by normalizing numeric literals, simplifying small test fixtures, tightening match assertions, clarifying helper docs, and removing an unused import. These changes keep behavior the same while making the test suite easier to maintain. --- gateways/kafka/src/server.rs | 4 ++-- gateways/kafka/tests/codec_tests.rs | 4 +--- gateways/kafka/tests/common/wire.rs | 6 +++--- gateways/kafka/tests/header_tests.rs | 4 ++-- gateways/kafka/tests/region_coverage_tests.rs | 16 ++++++++-------- 5 files changed, 16 insertions(+), 18 deletions(-) diff --git a/gateways/kafka/src/server.rs b/gateways/kafka/src/server.rs index 9d10c10cd9..d67a80312a 100644 --- a/gateways/kafka/src/server.rs +++ b/gateways/kafka/src/server.rs @@ -441,14 +441,14 @@ mod tests { fn correlation_id_is_extracted_from_frame() { let frame = bytes::Bytes::from_static(&[0x00, 0x12, 0x00, 0x01, 0x11, 0x22, 0x33, 0x44, 0xaa]); - assert_eq!(correlation_id_from_frame(&frame), 0x11223344); + assert_eq!(correlation_id_from_frame(&frame), 0x1122_3344); } #[tokio::test] async fn send_response_writes_header_and_body() { let (mut client, mut server) = tcp_pair().await; let header = ResponseHeader { - correlation_id: 0x01020304, + correlation_id: 0x0102_0304, }; let body = [9u8, 8, 7]; diff --git a/gateways/kafka/tests/codec_tests.rs b/gateways/kafka/tests/codec_tests.rs index 63b576377f..e7eeb12733 100644 --- a/gateways/kafka/tests/codec_tests.rs +++ b/gateways/kafka/tests/codec_tests.rs @@ -179,9 +179,7 @@ fn compact_array_count_above_max_returns_error() { #[test] fn compact_nullable_string_invalid_utf8_fails() { - let mut raw = Vec::new(); - raw.push(2); // len = 1 - raw.push(0xff); + let raw = vec![2, 0xff]; // len = 1, invalid UTF-8 byte let mut dec = Decoder::new(Bytes::from(raw)); let err = dec.read_compact_nullable_string().unwrap_err(); assert!(matches!(err, KafkaProtocolError::InvalidUtf8)); diff --git a/gateways/kafka/tests/common/wire.rs b/gateways/kafka/tests/common/wire.rs index fd2edd670d..3a873fc72f 100644 --- a/gateways/kafka/tests/common/wire.rs +++ b/gateways/kafka/tests/common/wire.rs @@ -185,7 +185,7 @@ pub fn build_produce_flexible_request_with_topic(topic: &str) -> Bytes { enc.freeze() } -/// Fetch v3 body without isolation_level field (defaults to 0). +/// Fetch v3 body without `isolation_level` field (defaults to 0). pub fn build_fetch_v3_no_isolation_request() -> Bytes { let mut enc = Encoder::with_capacity(32); enc.write_i32(-1); @@ -337,7 +337,7 @@ pub fn build_fetch_request_with_sections( enc.freeze() } -/// ListOffsets request covering legacy v0 `max_num_offsets` and newer leader-epoch branches. +/// `ListOffsets` request covering legacy v0 `max_num_offsets` and newer leader-epoch branches. pub fn build_list_offsets_branch_request(version: i16, topic: &str, partition: i32) -> Bytes { let flexible = version >= 6; let mut enc = Encoder::with_capacity(128); @@ -373,7 +373,7 @@ pub fn build_list_offsets_branch_request(version: i16, topic: &str, partition: i enc.freeze() } -/// CreateTopics request with one topic, one assignment, and one config. +/// `CreateTopics` request with one topic, one assignment, and one config. pub fn build_create_topics_request_with_sections(version: i16, topic: &str) -> Bytes { let flexible = version >= 5; let mut enc = Encoder::with_capacity(256); diff --git a/gateways/kafka/tests/header_tests.rs b/gateways/kafka/tests/header_tests.rs index 809f5225ca..bdff22681f 100644 --- a/gateways/kafka/tests/header_tests.rs +++ b/gateways/kafka/tests/header_tests.rs @@ -208,11 +208,11 @@ fn response_header_version_hits_every_api_key_match_arm() { continue; } let req_hdr_at_v0 = request_header_version(api_key, 0); - let expected_at_v0 = if req_hdr_at_v0 >= 2 { 1 } else { 0 }; + let expected_at_v0 = i16::from(req_hdr_at_v0 >= 2); assert_eq!(response_header_version(api_key, 0), expected_at_v0); let req_hdr_at_max = request_header_version(api_key, i16::MAX - 1); - let expected_at_max = if req_hdr_at_max >= 2 { 1 } else { 0 }; + let expected_at_max = i16::from(req_hdr_at_max >= 2); assert_eq!( response_header_version(api_key, i16::MAX - 1), expected_at_max diff --git a/gateways/kafka/tests/region_coverage_tests.rs b/gateways/kafka/tests/region_coverage_tests.rs index ea58d3fa1b..4f1cfaf276 100644 --- a/gateways/kafka/tests/region_coverage_tests.rs +++ b/gateways/kafka/tests/region_coverage_tests.rs @@ -19,7 +19,7 @@ use std::time::Duration; -use bytes::{BufMut, BytesMut}; +use bytes::BytesMut; use iggy_gateway_kafka::error::KafkaProtocolError; use iggy_gateway_kafka::protocol::codec::Encoder; use iggy_gateway_kafka::protocol::requests::{ @@ -97,7 +97,7 @@ fn produce_null_topic_name_preserves_acks_on_error() { assert_eq!(acks, Some(1)); assert!(matches!(error, KafkaProtocolError::NullTopicName)); } - other => panic!("expected NullTopicName, got {other:?}"), + ProduceDecodeResult::Ok(_) => panic!("expected NullTopicName"), } } @@ -120,7 +120,7 @@ fn produce_v3_error_before_acks_has_none_acks() { enc.write_i16(1); match decode_produce_request(3, enc.freeze()) { ProduceDecodeResult::Err { acks, .. } => assert_eq!(acks, None), - other => panic!("expected decode error before acks, got {other:?}"), + ProduceDecodeResult::Ok(_) => panic!("expected decode error before acks"), } } @@ -131,7 +131,7 @@ fn produce_v3_error_after_acks_preserves_acks() { enc.write_i16(7); match decode_produce_request(3, enc.freeze()) { ProduceDecodeResult::Err { acks, .. } => assert_eq!(acks, Some(7)), - other => panic!("expected decode error after acks, got {other:?}"), + ProduceDecodeResult::Ok(_) => panic!("expected decode error after acks"), } } @@ -143,7 +143,7 @@ fn produce_v3_error_after_timeout_preserves_acks() { enc.write_i32(500); match decode_produce_request(3, enc.freeze()) { ProduceDecodeResult::Err { acks, .. } => assert_eq!(acks, Some(1)), - other => panic!("expected decode error after timeout, got {other:?}"), + ProduceDecodeResult::Ok(_) => panic!("expected decode error after timeout"), } } @@ -160,7 +160,7 @@ fn produce_v9_error_on_null_topic_preserves_acks() { assert_eq!(acks, Some(2)); assert!(matches!(error, KafkaProtocolError::NullTopicName)); } - other => panic!("expected NullTopicName, got {other:?}"), + ProduceDecodeResult::Ok(_) => panic!("expected NullTopicName"), } } @@ -174,7 +174,7 @@ fn produce_v9_error_on_partition_count_preserves_acks() { enc.write_compact_nullable_string(Some("topic")); match decode_produce_request(9, enc.freeze()) { ProduceDecodeResult::Err { acks, .. } => assert_eq!(acks, Some(3)), - other => panic!("expected decode error in partition count, got {other:?}"), + ProduceDecodeResult::Ok(_) => panic!("expected decode error in partition count"), } } @@ -190,7 +190,7 @@ fn produce_v9_error_on_partition_records_preserves_acks() { enc.write_i32(0); match decode_produce_request(9, enc.freeze()) { ProduceDecodeResult::Err { acks, .. } => assert_eq!(acks, Some(4)), - other => panic!("expected decode error in records, got {other:?}"), + ProduceDecodeResult::Ok(_) => panic!("expected decode error in records"), } } From 863b66dd5d5502373cc80f608f37b150861d8dd6 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Thu, 16 Jul 2026 13:24:54 -0400 Subject: [PATCH 30/57] Standardize Kafka fixture skips Use the workspace `socket2` dependency and add a shared fixture loader that prints a consistent skip hint when a gitignored wire fixture is missing. This keeps Kafka test suites aligned on how they handle absent generated fixtures instead of panicking or silently succeeding. --- gateways/kafka/Cargo.toml | 2 +- gateways/kafka/tests/common/fixtures.rs | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/gateways/kafka/Cargo.toml b/gateways/kafka/Cargo.toml index d26608257a..8567e439ad 100644 --- a/gateways/kafka/Cargo.toml +++ b/gateways/kafka/Cargo.toml @@ -34,7 +34,7 @@ path = "src/main.rs" [dependencies] bytes = { workspace = true } -socket2 = "0.6.4" +socket2 = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true, features = [ "rt-multi-thread", diff --git a/gateways/kafka/tests/common/fixtures.rs b/gateways/kafka/tests/common/fixtures.rs index 02eb43d369..a94127ca92 100644 --- a/gateways/kafka/tests/common/fixtures.rs +++ b/gateways/kafka/tests/common/fixtures.rs @@ -42,6 +42,22 @@ pub fn load_fixture_body(api_key: i16, api_name: &str, version: i16) -> Bytes { extract_body_from_framed_message(api_key, version, &data) } +/// Standardized note emitted when a gitignored wire fixture is absent, pointing at the +/// generation script so a fresh clone knows how to produce it. +pub const FIXTURE_SKIP_HINT: &str = "generate with `gateways/kafka/scripts/ci-wire-fixtures.sh generate` (or the kafka-tool `generate` subcommand)"; + +/// Load a fixture body, or return `None` after printing a standardized skip note when the +/// fixture is missing. Unifies the missing-fixture policy across suites: every suite skips +/// explicitly instead of panicking or silently passing with zero assertions. +pub fn load_fixture_body_or_skip(api_key: i16, api_name: &str, version: i16) -> Option { + if fixture_exists(api_key, api_name, version) { + Some(load_fixture_body(api_key, api_name, version)) + } else { + eprintln!("skipping {api_key:03}_{api_name}_v{version}.bin: {FIXTURE_SKIP_HINT}"); + None + } +} + /// Strip the 4-byte length prefix and Kafka request header from a framed message. pub fn extract_body_from_framed_message(api_key: i16, api_version: i16, data: &[u8]) -> Bytes { let frame = Bytes::copy_from_slice(&data[4..]); From 24a49d22c141b8d2d9efe0fe186f0ebb20b96c86 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Thu, 16 Jul 2026 14:48:02 -0400 Subject: [PATCH 31/57] Harden Kafka gateway test fixtures Make the Kafka gateway test suite fail faster and more reliably. Missing gitignored wire fixtures now skip with a clear regeneration hint instead of panicking, default response reads are bounded to avoid hangs, and TCP byte reads now distinguish connection closure from idle timeouts so robustness tests can assert real server-side closes. --- gateways/kafka/tests/common/tcp.rs | 46 ++++++++-- .../kafka/tests/decode_validation_tests.rs | 83 ++++++++++++++----- .../kafka/tests/handler_regression_tests.rs | 27 +++--- .../kafka/tests/listener_robustness_tests.rs | 46 ++++++---- gateways/kafka/tests/region_coverage_tests.rs | 8 +- .../kafka/tests/review_regression_tests.rs | 4 +- gateways/kafka/tests/server_e2e_tests.rs | 6 +- .../kafka/tests/version_firewall_tests.rs | 10 ++- 8 files changed, 161 insertions(+), 69 deletions(-) diff --git a/gateways/kafka/tests/common/tcp.rs b/gateways/kafka/tests/common/tcp.rs index aaeea56b29..d8f94117d9 100644 --- a/gateways/kafka/tests/common/tcp.rs +++ b/gateways/kafka/tests/common/tcp.rs @@ -71,8 +71,27 @@ pub fn parse_response_payload(api_key: i16, api_version: i16, payload: Bytes) -> (correlation_id, body) } +/// Generous ceiling for the "default" response read. A server regression that drops a +/// response then becomes a bounded test failure instead of an indefinite hang. +const DEFAULT_RESPONSE_TIMEOUT: Duration = Duration::from_secs(10); + /// Read one length-prefixed response frame from the stream. +/// +/// Bounded by [`DEFAULT_RESPONSE_TIMEOUT`] so a dropped response fails fast instead of hanging. pub async fn read_response_frame(stream: &mut TcpStream, max_size: usize) -> Bytes { + time::timeout( + DEFAULT_RESPONSE_TIMEOUT, + read_response_frame_raw(stream, max_size), + ) + .await + .unwrap_or_else(|_| { + panic!( + "no response frame within {DEFAULT_RESPONSE_TIMEOUT:?} (server dropped the response?)" + ) + }) +} + +async fn read_response_frame_raw(stream: &mut TcpStream, max_size: usize) -> Bytes { let mut len_buf = [0u8; 4]; stream .read_exact(&mut len_buf) @@ -119,7 +138,7 @@ pub async fn read_response_frame_with_timeout( max_size: usize, timeout: Duration, ) -> Option { - time::timeout(timeout, read_response_frame(stream, max_size)) + time::timeout(timeout, read_response_frame_raw(stream, max_size)) .await .ok() } @@ -134,12 +153,27 @@ pub fn concat_frames(frames: &[Bytes]) -> Bytes { out.freeze() } -/// Read one byte or return `None` on EOF / timeout. -pub async fn read_byte_with_timeout(stream: &mut TcpStream, timeout: Duration) -> Option { +/// Outcome of a single-byte read, used by connection-close assertions. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ByteRead { + /// A byte was read from the stream. + Byte(u8), + /// The server closed the connection (clean EOF or reset). + Closed, + /// No byte arrived within the timeout; the connection is still open. + Timeout, +} + +/// Read one byte, distinguishing a server-side close from an idle timeout so that +/// "server must close the connection" tests can assert [`ByteRead::Closed`] explicitly +/// instead of passing on a mere stall. +pub async fn read_byte_with_timeout(stream: &mut TcpStream, timeout: Duration) -> ByteRead { let mut buf = [0u8; 1]; - match time::timeout(timeout, stream.read_exact(&mut buf)).await { - Ok(Ok(_)) => Some(buf[0]), - _ => None, + match time::timeout(timeout, stream.read(&mut buf)).await { + // 0 bytes = clean EOF; a reset / broken pipe is still a closed connection here. + Ok(Ok(0) | Err(_)) => ByteRead::Closed, + Ok(Ok(_)) => ByteRead::Byte(buf[0]), + Err(_) => ByteRead::Timeout, } } diff --git a/gateways/kafka/tests/decode_validation_tests.rs b/gateways/kafka/tests/decode_validation_tests.rs index bd7d984c1a..828da4298f 100644 --- a/gateways/kafka/tests/decode_validation_tests.rs +++ b/gateways/kafka/tests/decode_validation_tests.rs @@ -49,19 +49,30 @@ fn fixtures_dir() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tools/kafka-tool/kafka_messages") } -/// Load a kafka-tool `.bin` file and return just the request body bytes. -fn load_body(api_key: i16, api_name: &str, version: i16) -> Bytes { +/// Load a kafka-tool `.bin` file and return just the request body bytes, or `None` +/// (after a standardized skip note) when the gitignored fixture is absent, so a fresh +/// clone skips rather than panicking. +fn load_body(api_key: i16, api_name: &str, version: i16) -> Option { let filename = format!("{api_key:03}_{api_name}_v{version}.bin"); let path = fixtures_dir().join(&filename); - let data = std::fs::read(&path).unwrap_or_else(|e| panic!("failed to read {filename}: {e}")); + let Ok(data) = std::fs::read(&path) else { + eprintln!( + "skipping {filename}: wire fixture missing — generate with \ + `gateways/kafka/scripts/ci-wire-fixtures.sh generate` (or the kafka-tool \ + `generate` subcommand)" + ); + return None; + }; let frame = Bytes::copy_from_slice(&data[4..]); let hdr_ver = request_header_version(api_key, version); let mut decoder = Decoder::new(frame); RequestHeader::decode_from(&mut decoder, hdr_ver).expect("fixture request header must decode"); - decoder - .read_bytes(decoder.remaining()) - .expect("fixture request body must decode") + Some( + decoder + .read_bytes(decoder.remaining()) + .expect("fixture request body must decode"), + ) } // ── Produce (API key 0) ─────────────────────────────────────────────────────── @@ -69,7 +80,9 @@ fn load_body(api_key: i16, api_name: &str, version: i16) -> Bytes { #[test] fn produce_all_supported_versions_decode() { for version in 3i16..=9 { - let body = load_body(0, "Produce", version); + let Some(body) = load_body(0, "Produce", version) else { + continue; + }; let req = decode_produce_request(version, body) .into_request() .unwrap_or_else(|e| panic!("Produce v{version} decode failed: {e}")); @@ -103,7 +116,9 @@ fn produce_all_supported_versions_decode() { #[test] fn produce_response_encodes_for_all_supported_versions() { for version in 3i16..=9 { - let body = load_body(0, "Produce", version); + let Some(body) = load_body(0, "Produce", version) else { + continue; + }; let req = decode_produce_request(version, body) .into_request() .unwrap_or_else(|e| panic!("Produce v{version} decode failed: {e}")); @@ -118,7 +133,9 @@ fn produce_response_encodes_for_all_supported_versions() { #[test] fn produce_response_v3_roundtrip() { use iggy_gateway_kafka::protocol::codec::Decoder; - let body = load_body(0, "Produce", 3); + let Some(body) = load_body(0, "Produce", 3) else { + return; + }; let req = decode_produce_request(3, body).into_request().unwrap(); let resp = encode_produce_response(3, &req); @@ -145,7 +162,9 @@ fn produce_response_v3_roundtrip() { #[test] fn produce_response_v8_includes_record_errors() { use iggy_gateway_kafka::protocol::codec::Decoder; - let body = load_body(0, "Produce", 8); + let Some(body) = load_body(0, "Produce", 8) else { + return; + }; let req = decode_produce_request(8, body).into_request().unwrap(); let resp = encode_produce_response(8, &req); @@ -184,7 +203,9 @@ fn produce_v9_flexible_empty_topics_decode() { #[test] fn fetch_all_supported_versions_decode() { for version in 4i16..=12 { - let body = load_body(1, "Fetch", version); + let Some(body) = load_body(1, "Fetch", version) else { + continue; + }; let req = decode_fetch_request(version, body) .unwrap_or_else(|e| panic!("Fetch v{version} decode failed: {e}")); @@ -217,7 +238,9 @@ fn fetch_all_supported_versions_decode() { #[test] fn fetch_response_encodes_for_all_supported_versions() { for version in 4i16..=12 { - let body = load_body(1, "Fetch", version); + let Some(body) = load_body(1, "Fetch", version) else { + continue; + }; let req = decode_fetch_request(version, body) .unwrap_or_else(|e| panic!("Fetch v{version} decode failed: {e}")); let resp = encode_fetch_response(version, &req); @@ -231,7 +254,9 @@ fn fetch_response_encodes_for_all_supported_versions() { #[test] fn fetch_response_v7_roundtrip() { use iggy_gateway_kafka::protocol::codec::Decoder; - let body = load_body(1, "Fetch", 7); + let Some(body) = load_body(1, "Fetch", 7) else { + return; + }; let req = decode_fetch_request(7, body).unwrap(); let resp = encode_fetch_response(7, &req); @@ -280,7 +305,9 @@ fn fetch_v12_decodes_forgotten_topics_and_rack_id_sections() { #[test] fn list_offsets_all_supported_versions_decode() { for version in 1i16..=6 { - let body = load_body(2, "ListOffsets", version); + let Some(body) = load_body(2, "ListOffsets", version) else { + continue; + }; let req = decode_list_offsets_request(version, body) .unwrap_or_else(|e| panic!("ListOffsets v{version} decode failed: {e}")); @@ -330,7 +357,9 @@ fn list_offsets_v6_decodes_flexible_leader_epoch_branch() { #[test] fn list_offsets_response_encodes_for_all_supported_versions() { for version in 1i16..=6 { - let body = load_body(2, "ListOffsets", version); + let Some(body) = load_body(2, "ListOffsets", version) else { + continue; + }; let req = decode_list_offsets_request(version, body) .unwrap_or_else(|e| panic!("ListOffsets v{version} decode failed: {e}")); let resp = encode_list_offsets_response(version, &req); @@ -344,7 +373,9 @@ fn list_offsets_response_encodes_for_all_supported_versions() { #[test] fn list_offsets_response_v1_no_leader_epoch() { use iggy_gateway_kafka::protocol::codec::Decoder; - let body = load_body(2, "ListOffsets", 1); + let Some(body) = load_body(2, "ListOffsets", 1) else { + return; + }; let req = decode_list_offsets_request(1, body).unwrap(); let resp = encode_list_offsets_response(1, &req); @@ -371,7 +402,9 @@ fn list_offsets_response_v1_no_leader_epoch() { #[test] fn list_offsets_response_v4_has_leader_epoch() { use iggy_gateway_kafka::protocol::codec::Decoder; - let body = load_body(2, "ListOffsets", 4); + let Some(body) = load_body(2, "ListOffsets", 4) else { + return; + }; let req = decode_list_offsets_request(4, body).unwrap(); let resp = encode_list_offsets_response(4, &req); @@ -397,7 +430,9 @@ fn list_offsets_response_v4_has_leader_epoch() { #[test] fn create_topics_all_supported_versions_decode() { for version in 2i16..=5 { - let body = load_body(19, "CreateTopics", version); + let Some(body) = load_body(19, "CreateTopics", version) else { + continue; + }; let req = decode_create_topics_request(version, body) .unwrap_or_else(|e| panic!("CreateTopics v{version} decode failed: {e}")); @@ -458,7 +493,9 @@ fn create_topics_v5_decodes_flexible_assignments_and_configs() { #[test] fn create_topics_response_encodes_for_all_supported_versions() { for version in 2i16..=5 { - let body = load_body(19, "CreateTopics", version); + let Some(body) = load_body(19, "CreateTopics", version) else { + continue; + }; let req = decode_create_topics_request(version, body) .unwrap_or_else(|e| panic!("CreateTopics v{version} decode failed: {e}")); let resp = encode_create_topics_response(version, &req); @@ -472,7 +509,9 @@ fn create_topics_response_encodes_for_all_supported_versions() { #[test] fn create_topics_response_v2_roundtrip() { use iggy_gateway_kafka::protocol::codec::Decoder; - let body = load_body(19, "CreateTopics", 2); + let Some(body) = load_body(19, "CreateTopics", 2) else { + return; + }; let req = decode_create_topics_request(2, body).unwrap(); let topic_name = req.topics[0].name.clone(); let resp = encode_create_topics_response(2, &req); @@ -493,7 +532,9 @@ fn create_topics_response_v2_roundtrip() { #[test] fn create_topics_response_v5_roundtrip() { use iggy_gateway_kafka::protocol::codec::Decoder; - let body = load_body(19, "CreateTopics", 5); + let Some(body) = load_body(19, "CreateTopics", 5) else { + return; + }; let req = decode_create_topics_request(5, body).unwrap(); let resp = encode_create_topics_response(5, &req); diff --git a/gateways/kafka/tests/handler_regression_tests.rs b/gateways/kafka/tests/handler_regression_tests.rs index 214c650c66..bd99e2be85 100644 --- a/gateways/kafka/tests/handler_regression_tests.rs +++ b/gateways/kafka/tests/handler_regression_tests.rs @@ -28,7 +28,7 @@ use iggy_gateway_kafka::protocol::api::{ }; use iggy_gateway_kafka::protocol::codec::Decoder; -use fixtures::{fixture_exists, load_fixture_body}; +use fixtures::load_fixture_body_or_skip; use scope::{SCOPED_API_KEYS, default_broker}; #[test] @@ -48,10 +48,9 @@ fn handle_request_succeeds_for_every_supported_version_with_fixture() { } for version in min_ver..=max_ver { - if !fixture_exists(api_key, name, version) { + let Some(body) = load_fixture_body_or_skip(api_key, name, version) else { continue; - } - let body = load_fixture_body(api_key, name, version); + }; let resp = handle_request(api_key, version, body, &default_broker()) .expect("test request has acks != 0 and expects a response"); assert!( @@ -65,10 +64,9 @@ fn handle_request_succeeds_for_every_supported_version_with_fixture() { #[test] fn produce_stub_response_has_zero_error_per_partition() { for version in 3i16..=9 { - if !fixture_exists(0, "Produce", version) { + let Some(body) = load_fixture_body_or_skip(0, "Produce", version) else { continue; - } - let body = load_fixture_body(0, "Produce", version); + }; let resp = handle_request(API_KEY_PRODUCE, version, body, &default_broker()) .expect("test request has acks != 0 and expects a response"); let flexible = version >= 9; @@ -90,10 +88,9 @@ fn produce_stub_response_has_zero_error_per_partition() { #[test] fn fetch_stub_response_has_zero_partition_error() { for version in 4i16..=12 { - if !fixture_exists(1, "Fetch", version) { + let Some(body) = load_fixture_body_or_skip(1, "Fetch", version) else { continue; - } - let body = load_fixture_body(1, "Fetch", version); + }; let resp = handle_request(API_KEY_FETCH, version, body, &default_broker()) .expect("test request has acks != 0 and expects a response"); let flexible = version >= 12; @@ -126,10 +123,9 @@ fn fetch_stub_response_has_zero_partition_error() { #[test] fn list_offsets_stub_response_has_zero_error() { for version in 1i16..=6 { - if !fixture_exists(2, "ListOffsets", version) { + let Some(body) = load_fixture_body_or_skip(2, "ListOffsets", version) else { continue; - } - let body = load_fixture_body(2, "ListOffsets", version); + }; let resp = handle_request(API_KEY_LIST_OFFSETS, version, body, &default_broker()) .expect("test request has acks != 0 and expects a response"); let flexible = version >= 6; @@ -154,10 +150,9 @@ fn list_offsets_stub_response_has_zero_error() { #[test] fn create_topics_stub_response_has_zero_error() { for version in 2i16..=5 { - if !fixture_exists(19, "CreateTopics", version) { + let Some(body) = load_fixture_body_or_skip(19, "CreateTopics", version) else { continue; - } - let body = load_fixture_body(19, "CreateTopics", version); + }; let resp = handle_request(API_KEY_CREATE_TOPICS, version, body, &default_broker()) .expect("test request has acks != 0 and expects a response"); let flexible = version >= 5; diff --git a/gateways/kafka/tests/listener_robustness_tests.rs b/gateways/kafka/tests/listener_robustness_tests.rs index 97fa91797e..cd45e386e3 100644 --- a/gateways/kafka/tests/listener_robustness_tests.rs +++ b/gateways/kafka/tests/listener_robustness_tests.rs @@ -37,7 +37,7 @@ use iggy_gateway_kafka::protocol::codec::Decoder; use server::{spawn_test_server, spawn_test_server_with_config}; use tcp::{ - build_request_frame, concat_frames, parse_response_payload, read_byte_with_timeout, + ByteRead, build_request_frame, concat_frames, parse_response_payload, read_byte_with_timeout, read_response_frame, read_response_frame_with_timeout, }; @@ -126,16 +126,26 @@ async fn e2e_frame_exceeding_max_frame_size_closes_connection() { frame.resize(4 + 200, 0); stream.write_all(&frame).await.expect("oversized frame"); - let byte = read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await; - assert!( - byte.is_none(), + assert_eq!( + read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await, + ByteRead::Closed, "oversized frame should close connection (EOF)" ); } #[tokio::test] async fn e2e_truncated_frame_body_closes_connection() { - let (addr, _shutdown) = spawn_test_server().await; + // A truncated in-flight body closes only once the server's read_timeout elapses, so use a + // short read_timeout and wait longer than it to observe a genuine close, not a mere stall. + let (addr, _shutdown) = spawn_test_server_with_config(ServerConfig { + bind_addr: String::new(), + advertised_host: None, + advertised_port: None, + max_frame_size: 8 * 1024 * 1024, + read_timeout: Duration::from_secs(1), + write_timeout: Duration::from_secs(5), + }) + .await; let mut stream = TcpStream::connect(addr).await.expect("connect"); let full = build_request_frame(API_KEY_API_VERSIONS, 1, 66, Some("trunc-test"), &[]); @@ -147,8 +157,11 @@ async fn e2e_truncated_frame_body_closes_connection() { .await .expect("half body"); - let byte = read_byte_with_timeout(&mut stream, Duration::from_secs(3)).await; - assert!(byte.is_none(), "truncated body should close connection"); + assert_eq!( + read_byte_with_timeout(&mut stream, Duration::from_secs(3)).await, + ByteRead::Closed, + "truncated body should close connection after read_timeout" + ); } #[tokio::test] @@ -214,8 +227,11 @@ async fn e2e_zero_frame_length_closes_connection() { .write_all(&0i32.to_be_bytes()) .await .expect("zero len"); - let byte = read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await; - assert!(byte.is_none(), "zero frame length must close connection"); + assert_eq!( + read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await, + ByteRead::Closed, + "zero frame length must close connection" + ); } #[tokio::test] @@ -227,9 +243,9 @@ async fn e2e_negative_frame_length_closes_connection() { .write_all(&(-5_i32).to_be_bytes()) .await .expect("negative len"); - let byte = read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await; - assert!( - byte.is_none(), + assert_eq!( + read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await, + ByteRead::Closed, "negative frame length must close connection" ); } @@ -315,9 +331,9 @@ async fn e2e_frame_payload_shorter_than_kafka_header_closes_connection() { frame.extend_from_slice(&[0x00, 0x12, 0x00, 0x01]); stream.write_all(&frame).await.expect("short payload"); - let byte = read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await; - assert!( - byte.is_none(), + assert_eq!( + read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await, + ByteRead::Closed, "payload shorter than 8-byte Kafka header must close connection" ); } diff --git a/gateways/kafka/tests/region_coverage_tests.rs b/gateways/kafka/tests/region_coverage_tests.rs index 4f1cfaf276..aa2636e90d 100644 --- a/gateways/kafka/tests/region_coverage_tests.rs +++ b/gateways/kafka/tests/region_coverage_tests.rs @@ -37,7 +37,7 @@ mod tcp; #[path = "common/wire.rs"] mod wire; -use tcp::{build_request_frame, read_byte_with_timeout, read_response_frame}; +use tcp::{ByteRead, build_request_frame, read_byte_with_timeout, read_response_frame}; // ── Produce version branches ────────────────────────────────────────────────── @@ -420,9 +420,9 @@ async fn e2e_frame_shorter_than_kafka_header_returns_buffer_underflow() { frame.extend_from_slice(&[0x00, 0x12, 0x00, 0x01, 0x00, 0x00, 0x00]); stream.write_all(&frame).await.expect("short header write"); - let byte = read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await; - assert!( - byte.is_none(), + assert_eq!( + read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await, + ByteRead::Closed, "frame payload shorter than 8-byte Kafka header must close connection" ); } diff --git a/gateways/kafka/tests/review_regression_tests.rs b/gateways/kafka/tests/review_regression_tests.rs index f494973685..b65408bdf4 100644 --- a/gateways/kafka/tests/review_regression_tests.rs +++ b/gateways/kafka/tests/review_regression_tests.rs @@ -17,8 +17,8 @@ //! Regression tests for PR #3519 review findings (atharvalade, Jul 2026). //! -//! These encode Kafka-client-correct behavior. Several fail until the -//! corresponding protocol/server fixes land. +//! Each test encodes Kafka-client-correct behavior for a specific review finding and +//! guards against regressions now that the corresponding protocol/server fixes have landed. #[path = "common/scope.rs"] mod scope; diff --git a/gateways/kafka/tests/server_e2e_tests.rs b/gateways/kafka/tests/server_e2e_tests.rs index d0846ba49f..ed2a2843af 100644 --- a/gateways/kafka/tests/server_e2e_tests.rs +++ b/gateways/kafka/tests/server_e2e_tests.rs @@ -33,7 +33,7 @@ use iggy_gateway_kafka::protocol::api::{ }; use iggy_gateway_kafka::protocol::codec::Decoder; -use fixtures::load_fixture_body; +use fixtures::load_fixture_body_or_skip; use server::spawn_test_server; use tcp::{build_request_frame, parse_response_payload, read_response_frame, round_trip}; @@ -74,7 +74,9 @@ async fn e2e_metadata_v0_returns_stub_broker() { #[tokio::test] async fn e2e_produce_v3_round_trip_with_fixture() { let (addr, _shutdown) = spawn_test_server().await; - let body = load_fixture_body(0, "Produce", 3); + let Some(body) = load_fixture_body_or_skip(0, "Produce", 3) else { + return; + }; let (corr, resp_body) = round_trip(addr, API_KEY_PRODUCE, 3, 88, &body).await; assert_eq!(corr, 88); assert!(!resp_body.is_empty()); diff --git a/gateways/kafka/tests/version_firewall_tests.rs b/gateways/kafka/tests/version_firewall_tests.rs index 923cf3c018..8c24c2c313 100644 --- a/gateways/kafka/tests/version_firewall_tests.rs +++ b/gateways/kafka/tests/version_firewall_tests.rs @@ -35,7 +35,7 @@ use iggy_gateway_kafka::protocol::api::{ }; use iggy_gateway_kafka::protocol::codec::Decoder; -use fixtures::load_fixture_body; +use fixtures::load_fixture_body_or_skip; use scope::{SCOPED_API_KEYS, default_broker}; use tcp::build_metadata_legacy_request; use wire::build_metadata_flexible_request_v10; @@ -305,7 +305,9 @@ fn unsupported_api_keys_return_error_only() { #[test] fn supported_produce_versions_accept_valid_fixture() { for version in 3i16..=9 { - let body = load_fixture_body(0, "Produce", version); + let Some(body) = load_fixture_body_or_skip(0, "Produce", version) else { + continue; + }; let resp = handle_request(API_KEY_PRODUCE, version, body, &default_broker()) .expect("test request has acks != 0 and expects a response"); assert!(!resp.is_empty(), "Produce v{version} response empty"); @@ -315,7 +317,9 @@ fn supported_produce_versions_accept_valid_fixture() { #[test] fn supported_fetch_versions_accept_valid_fixture() { for version in 4i16..=12 { - let body = load_fixture_body(1, "Fetch", version); + let Some(body) = load_fixture_body_or_skip(1, "Fetch", version) else { + continue; + }; let resp = handle_request(API_KEY_FETCH, version, body, &default_broker()) .expect("test request has acks != 0 and expects a response"); assert!(!resp.is_empty(), "Fetch v{version} response empty"); From abfec3bb62934041e7fc474fc9df2cef85cae26b Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Thu, 16 Jul 2026 20:42:15 -0400 Subject: [PATCH 32/57] Deduplicate kafka-tool header version logic Move kafka-tool request/response header version decisions to `iggy-gateway-kafka` so framing and parsing use the same source of truth as the gateway. The tool now derives its gateway verify API/version bounds from `supported_api_ranges()` instead of a hardcoded table, and `header.rs` exposes `first_flexible_version` to share flexible-version thresholds cleanly. --- Cargo.lock | 1 + gateways/kafka/src/protocol/header.rs | 38 +++++-- gateways/kafka/tools/kafka-tool/Cargo.toml | 1 + gateways/kafka/tools/kafka-tool/src/main.rs | 106 ++++-------------- .../kafka/tools/kafka-tool/src/response.rs | 74 +----------- 5 files changed, 53 insertions(+), 167 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bc42dd5476..5d381f13af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7691,6 +7691,7 @@ dependencies = [ "bytes", "clap", "hex", + "iggy-gateway-kafka", "indexmap 2.14.0", "kafka-protocol", "tokio", diff --git a/gateways/kafka/src/protocol/header.rs b/gateways/kafka/src/protocol/header.rs index 7977f4e5c9..5488a7413f 100644 --- a/gateways/kafka/src/protocol/header.rs +++ b/gateways/kafka/src/protocol/header.rs @@ -39,13 +39,23 @@ pub struct ResponseHeader { pub correlation_id: i32, } -/// Returns the request header version to use for a given (api_key, api_version) pair. +/// First API version that uses flexible (compact) request encoding for `api_key`. /// -/// Header v1 is the standard non-flexible format (nullable string client_id). -/// Header v2 is the flexible format (compact nullable string client_id + empty tagged fields). -/// The threshold at which each API key switches from v1 to v2 is defined by the Kafka protocol. -pub fn request_header_version(api_key: i16, api_version: i16) -> i16 { - let flexible_from: i16 = match api_key { +/// `None` means the API never uses flexible encoding (request header stays v1 for every +/// version). Source of truth for kafka-tool framing and for [`request_header_version`]. +#[must_use] +pub fn first_flexible_version(api_key: i16) -> Option { + let threshold = first_flexible_version_threshold(api_key); + if threshold == i16::MAX { + None + } else { + Some(threshold) + } +} + +/// Threshold used by [`request_header_version`]. `i16::MAX` = never flexible. +fn first_flexible_version_threshold(api_key: i16) -> i16 { + match api_key { 0 => 9, // Produce 1 => 12, // Fetch 2 => 6, // ListOffsets @@ -119,8 +129,20 @@ pub fn request_header_version(api_key: i16, api_version: i16) -> i16 { 79 => 0, // ShareFetch — always flexible 80 => 0, // ShareAcknowledge — always flexible _ => i16::MAX, // Unknown API — assume non-flexible - }; - if api_version >= flexible_from { 2 } else { 1 } + } +} + +/// Returns the request header version to use for a given (api_key, api_version) pair. +/// +/// Header v1 is the standard non-flexible format (nullable string client_id). +/// Header v2 is the flexible format (compact nullable string client_id + empty tagged fields). +/// The threshold at which each API key switches from v1 to v2 is defined by the Kafka protocol. +#[must_use] +pub fn request_header_version(api_key: i16, api_version: i16) -> i16 { + match first_flexible_version(api_key) { + Some(flexible_from) if api_version >= flexible_from => 2, + _ => 1, + } } /// Returns the response header version to use when replying to a given (api_key, api_version). diff --git a/gateways/kafka/tools/kafka-tool/Cargo.toml b/gateways/kafka/tools/kafka-tool/Cargo.toml index 41473e434a..d8815b6817 100644 --- a/gateways/kafka/tools/kafka-tool/Cargo.toml +++ b/gateways/kafka/tools/kafka-tool/Cargo.toml @@ -34,6 +34,7 @@ anyhow = { workspace = true } bytes = { workspace = true } clap = { workspace = true } hex = "0.4" +iggy-gateway-kafka = { path = "../.." } indexmap = "2" kafka-protocol = "0.17" tokio = { workspace = true } diff --git a/gateways/kafka/tools/kafka-tool/src/main.rs b/gateways/kafka/tools/kafka-tool/src/main.rs index f620f0303c..56a979ab06 100644 --- a/gateways/kafka/tools/kafka-tool/src/main.rs +++ b/gateways/kafka/tools/kafka-tool/src/main.rs @@ -18,6 +18,8 @@ use anyhow::{Context, Result}; use bytes::{BufMut, Bytes, BytesMut}; use clap::{Parser, Subcommand}; +use iggy_gateway_kafka::protocol::api::supported_api_ranges; +use iggy_gateway_kafka::protocol::header::request_header_version; use kafka_protocol::messages::*; use kafka_protocol::protocol::{Encodable, StrBytes}; use std::path::PathBuf; @@ -164,87 +166,19 @@ const API_REGISTRY: &[(i16, &str, i16, i16)] = &[ (76, "ListClientMetricsResources", 0, 0), ]; -/// Iggy Kafka gateway #3421 scope — mirrors `SUPPORTED_RANGES` in `iggy_gateway_kafka`. -const GATEWAY_REGISTRY: &[(i16, &str, i16, i16)] = &[ - (0, "Produce", 3, 9), - (1, "Fetch", 4, 12), - (2, "ListOffsets", 1, 6), - (3, "Metadata", 0, 9), - (18, "ApiVersions", 0, 3), - (19, "CreateTopics", 2, 5), -]; - -// ── Flexible version table ──────────────────────────────────────────────────── -// Source: flexibleVersions field in each Kafka JSON schema. -// Returns the first version using compact encoding, or None if never flexible. -fn first_flexible_version(api_key: i16) -> Option { - match api_key { - 0 => Some(9), - 1 => Some(12), - 2 => Some(6), - 3 => Some(9), - 8 => Some(8), - 9 => Some(6), - 10 => Some(3), - 11 => Some(6), - 12 => Some(4), - 13 => Some(4), - 14 => Some(4), - 15 => Some(5), - 16 => Some(3), - 17 => None, - 18 => Some(3), - 19 => Some(5), - 20 => Some(4), - 21 => Some(2), - 22 => Some(2), - 23 => Some(4), - 24 => Some(3), - 25 => Some(3), - 26 => Some(3), - 27 => Some(1), - 28 => Some(3), - 29 => Some(2), - 30 => Some(2), - 31 => Some(2), - 32 => Some(4), - 33 => Some(2), - 34 => Some(2), - 35 => Some(2), - 36 => Some(2), - 37 => Some(2), - 38 => Some(2), - 39 => Some(2), - 40 => Some(2), - 41 => Some(2), - 42 => Some(2), - 43 => Some(2), - 44 => Some(1), - 45 => Some(1), - 46 => Some(1), - 47 => Some(0), - 48 => Some(1), - 49 => Some(1), - 50 => Some(0), - 51 => Some(0), - 55 => Some(2), - 56 => Some(2), - 57 => Some(1), - 60 => Some(0), - 61 => Some(0), - 64 => Some(0), - 65 => Some(0), - 66 => Some(0), - 67 => Some(0), - 68 => Some(0), - 69 => Some(0), - 71 => Some(0), - 72 => Some(0), - 74 => Some(0), - 75 => Some(0), - 76 => Some(0), - _ => None, - } +/// Gateway verify scope from `iggy_gateway_kafka::supported_api_ranges()`. +/// Names come from `API_REGISTRY` (Kafka catalog); version bounds come from the gateway. +fn gateway_verify_registry() -> Vec<(i16, &'static str, i16, i16)> { + supported_api_ranges() + .iter() + .map(|range| { + let name = API_REGISTRY + .iter() + .find(|(key, _, _, _)| *key == range.api_key) + .map_or("Unknown", |(_, name, _, _)| *name); + (range.api_key, name, range.min_version, range.max_version) + }) + .collect() } // ── Request framing ─────────────────────────────────────────────────────────── @@ -640,9 +574,8 @@ fn build_payload(api_key: i16, version: i16) -> Result { // Build a complete framed Kafka request message ready for TCP transmission. fn build_framed(api_key: i16, version: i16, corr: i32) -> Result { let payload = build_payload(api_key, version)?; - let flexible = first_flexible_version(api_key) - .map(|fv| version >= fv) - .unwrap_or(false); + // Header v2 == flexible request encoding; threshold lives in iggy-gateway-kafka. + let flexible = request_header_version(api_key, version) >= 2; Ok(frame_request( api_key, version, @@ -875,10 +808,11 @@ async fn main() -> Result<()> { all_apis, quiet, } => { - let registry = if all_apis { + let gateway_registry = gateway_verify_registry(); + let registry: &[(i16, &str, i16, i16)] = if all_apis { API_REGISTRY } else { - GATEWAY_REGISTRY + &gateway_registry }; let (ok, fail) = run_send( &host, registry, &api_key, version, timeout_ms, fail_fast, quiet, diff --git a/gateways/kafka/tools/kafka-tool/src/response.rs b/gateways/kafka/tools/kafka-tool/src/response.rs index a55f5b62a8..51885fd263 100644 --- a/gateways/kafka/tools/kafka-tool/src/response.rs +++ b/gateways/kafka/tools/kafka-tool/src/response.rs @@ -18,6 +18,7 @@ //! Kafka response frame parsing and human-readable summaries for `send` / `verify`. use bytes::Bytes; +use iggy_gateway_kafka::protocol::header::response_header_version; use kafka_protocol::messages::{ ApiVersionsResponse, CreateTopicsResponse, FetchResponse, ListOffsetsResponse, MetadataResponse, ProduceResponse, @@ -345,76 +346,3 @@ fn format_error_code(code: i16) -> &'static str { _ => "OTHER", } } - -fn request_header_version(api_key: i16, api_version: i16) -> i16 { - let flex_from = first_flexible_version(api_key); - match flex_from { - Some(fv) if api_version >= fv => 2, - _ => 1, - } -} - -fn response_header_version(api_key: i16, api_version: i16) -> i16 { - if api_key == 18 { - return 0; - } - if request_header_version(api_key, api_version) >= 2 { - 1 - } else { - 0 - } -} - -/// First flexible protocol version per API key (matches gateway `header.rs` / kafka-tool framing). -fn first_flexible_version(api_key: i16) -> Option { - match api_key { - 0 => Some(9), - 1 => Some(12), - 2 => Some(6), - 3 => Some(9), - 8 => Some(8), - 9 => Some(6), - 10 => Some(3), - 11 => Some(6), - 12 => Some(4), - 13 => Some(4), - 14 => Some(4), - 15 => Some(5), - 16 => Some(3), - 17 => None, - 18 => Some(3), - 19 => Some(5), - 20 => Some(4), - 21 => Some(2), - 22 => Some(2), - 23 => Some(4), - 24 => Some(3), - 25 => Some(3), - 26 => Some(3), - 27 => Some(1), - 28 => Some(3), - 29 => Some(2), - 30 => Some(2), - 31 => Some(2), - 32 => Some(4), - 33 => Some(2), - 34 => Some(2), - 35 => Some(2), - 36 => Some(2), - 37 => Some(2), - 38 => Some(2), - 39 => Some(2), - 40 => Some(2), - 41 => Some(2), - 42 => Some(2), - 43 => Some(2), - 44 => Some(1), - 45 | 46 => Some(0), - 47 => None, - 48 | 49 => Some(1), - 50 | 51 | 55 | 56 => Some(0), - 57 => Some(1), - 60 | 61 | 64 | 65 | 66 | 67 | 68 | 69 | 71 | 72 | 74 | 75 | 76 => Some(0), - _ => None, - } -} From 0a665921cb004256b50c48e85f7b29fececae05b Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Thu, 16 Jul 2026 22:54:11 -0400 Subject: [PATCH 33/57] Add graceful shutdown and HandleOutcome enum - Add SIGTERM handler for graceful Docker shutdown alongside SIGINT - Replace Option with HandleOutcome enum (Respond/NoResponse/Close) for clearer connection lifecycle semantics - Close TCP connection for unsupported Metadata versions instead of returning clamped responses that clients cannot parse - Update all request handlers and tests to use new HandleOutcome API --- gateways/kafka/src/main.rs | 27 +++- gateways/kafka/src/protocol/api.rs | 145 ++++++++++++++---- gateways/kafka/src/server.rs | 48 +++--- gateways/kafka/tests/api_handler_tests.rs | 67 +++----- .../kafka/tests/broker_advertise_tests.rs | 2 +- .../kafka/tests/golden_wire_fixtures_tests.rs | 4 +- .../kafka/tests/handler_regression_tests.rs | 12 +- .../kafka/tests/metadata_regression_tests.rs | 18 +-- .../kafka/tests/review_regression_tests.rs | 56 ++----- gateways/kafka/tests/scope_coverage_tests.rs | 51 ++++-- .../kafka/tests/version_firewall_tests.rs | 123 +++++++-------- 11 files changed, 320 insertions(+), 233 deletions(-) diff --git a/gateways/kafka/src/main.rs b/gateways/kafka/src/main.rs index 0f48ea943e..536ddf472a 100644 --- a/gateways/kafka/src/main.rs +++ b/gateways/kafka/src/main.rs @@ -52,7 +52,7 @@ async fn main() -> Result<(), Box> { result = &mut server_task => { return Ok(result??); } - _ = signal::ctrl_c() => { + () = shutdown_signal() => { let _ = tx.send(()); } } @@ -60,3 +60,28 @@ async fn main() -> Result<(), Box> { server_task.await??; Ok(()) } + +/// Wait for Ctrl-C (SIGINT) or, on Unix, SIGTERM (`docker stop`). +async fn shutdown_signal() { + let ctrl_c = async { + signal::ctrl_c() + .await + .expect("failed to install Ctrl-C handler"); + }; + + #[cfg(unix)] + let terminate = async { + signal::unix::signal(signal::unix::SignalKind::terminate()) + .expect("failed to install SIGTERM handler") + .recv() + .await; + }; + + #[cfg(not(unix))] + let terminate = std::future::pending::<()>(); + + tokio::select! { + () = ctrl_c => {} + () = terminate => {} + } +} diff --git a/gateways/kafka/src/protocol/api.rs b/gateways/kafka/src/protocol/api.rs index 003a141ba3..6e41872b3a 100644 --- a/gateways/kafka/src/protocol/api.rs +++ b/gateways/kafka/src/protocol/api.rs @@ -49,6 +49,58 @@ const MAX_SUPPORTED_METADATA_VERSION: i16 = 9; /// Sentinel for `topic_authorized_operations` / `cluster_authorized_operations` when ACLs are not supported. const AUTHORIZED_OPS_UNKNOWN: i32 = i32::MIN; +/// Result of handling one Kafka request body. +#[derive(Debug)] +pub enum HandleOutcome { + /// Write this response body (with a response header). + Respond(Bytes), + /// Produce with `acks=0`: write nothing, keep the connection open. + NoResponse, + /// Client cannot parse an error at this request wire version; close the TCP connection. + Close, +} + +impl HandleOutcome { + /// Collapse to `Some(body)` for a normal response, or `None` for [`HandleOutcome::NoResponse`]. + /// + /// # Panics + /// + /// Panics on [`HandleOutcome::Close`] — match on `Close` explicitly, or use + /// [`Self::expect_response`] in tests that require a body. + #[must_use] + pub fn into_optional_response(self) -> Option { + match self { + Self::Respond(body) => Some(body), + Self::NoResponse => None, + Self::Close => panic!("HandleOutcome::Close has no response body"), + } + } + + /// Return the response body, or panic with `msg` if the outcome is not [`Self::Respond`]. + /// + /// # Panics + /// + /// Panics when the outcome is [`Self::NoResponse`] or [`Self::Close`]. + #[must_use] + pub fn expect_response(self, msg: &str) -> Bytes { + match self { + Self::Respond(body) => body, + Self::NoResponse => panic!("{msg}: got NoResponse"), + Self::Close => panic!("{msg}: got Close"), + } + } + + #[must_use] + pub const fn is_no_response(&self) -> bool { + matches!(self, Self::NoResponse) + } + + #[must_use] + pub const fn is_close(&self) -> bool { + matches!(self, Self::Close) + } +} + #[derive(Debug, Clone)] pub struct BrokerAdvertise { pub host: String, @@ -109,25 +161,24 @@ pub fn supported_api_ranges() -> &'static [ApiVersionRange] { SUPPORTED_RANGES } -/// Handles one decoded request frame and returns the response to write back, -/// or `None` when the wire protocol forbids a response (Produce with `acks=0`). +/// Handles one decoded request frame and returns how the connection should proceed. pub fn handle_request( api_key: i16, api_version: i16, body: Bytes, broker: &BrokerAdvertise, -) -> Option { +) -> HandleOutcome { if api_key == API_KEY_PRODUCE { return handle_produce_request(api_version, body); } - Some(handle_other_request(api_key, api_version, body, broker)) + handle_other_request(api_key, api_version, body, broker) } /// Produce is the only request the wire protocol allows to go unanswered -/// (`acks=0`), so it gets its own `Option`-returning path. -fn handle_produce_request(api_version: i16, body: Bytes) -> Option { +/// (`acks=0`), so it gets its own path that may return [`HandleOutcome::NoResponse`]. +fn handle_produce_request(api_version: i16, body: Bytes) -> HandleOutcome { if !is_supported_version(API_KEY_PRODUCE, api_version) { - return Some(encode_produce_error_response( + return HandleOutcome::Respond(encode_produce_error_response( api_version, ERROR_UNSUPPORTED_VERSION, )); @@ -135,8 +186,10 @@ fn handle_produce_request(api_version: i16, body: Bytes) -> Option { match decode_produce_request(api_version, body) { // acks=0 is fire-and-forget: the client isn't reading a response, so // sending one desyncs the next correlation id it expects. - ProduceDecodeResult::Ok(req) if req.acks == 0 => None, - ProduceDecodeResult::Ok(req) => Some(encode_produce_response(api_version, &req)), + ProduceDecodeResult::Ok(req) if req.acks == 0 => HandleOutcome::NoResponse, + ProduceDecodeResult::Ok(req) => { + HandleOutcome::Respond(encode_produce_response(api_version, &req)) + } ProduceDecodeResult::Err { acks: Some(0), error, @@ -145,11 +198,11 @@ fn handle_produce_request(api_version: i16, body: Bytes) -> Option { "Failed to decode Produce request with acks=0 (no response): {:?}", error ); - None + HandleOutcome::NoResponse } ProduceDecodeResult::Err { error, .. } => { tracing::warn!("Failed to decode Produce request: {:?}", error); - Some(encode_produce_error_response( + HandleOutcome::Respond(encode_produce_error_response( api_version, ERROR_INVALID_REQUEST, )) @@ -162,71 +215,99 @@ fn handle_other_request( api_version: i16, body: Bytes, broker: &BrokerAdvertise, -) -> Bytes { +) -> HandleOutcome { match api_key { API_KEY_API_VERSIONS => { if is_supported_version(api_key, api_version) { - encode_api_versions_response(api_version, ERROR_NONE) + HandleOutcome::Respond(encode_api_versions_response(api_version, ERROR_NONE)) } else { // KIP-511: reply with v0 when the requested version is not understood. - encode_api_versions_response(0, ERROR_UNSUPPORTED_VERSION) + HandleOutcome::Respond(encode_api_versions_response(0, ERROR_UNSUPPORTED_VERSION)) } } API_KEY_METADATA => { if is_supported_version(api_key, api_version) { - encode_metadata_response(api_version, api_version, body, broker, ERROR_NONE) - } else { - let response_version = api_version.clamp(0, MAX_SUPPORTED_METADATA_VERSION); - // Decode at the client's wire version; encode at the highest version we implement. - encode_metadata_response( - response_version, + HandleOutcome::Respond(encode_metadata_response( + api_version, api_version, body, broker, - ERROR_UNSUPPORTED_VERSION, - ) + ERROR_NONE, + )) + } else { + // Clamping the response to MAX_SUPPORTED_METADATA_VERSION leaves a body the + // client parses at its own (unsupported) version, so UNSUPPORTED_VERSION never + // survives. Clients that skip ApiVersions get a naked close instead. + tracing::warn!( + api_version, + max_supported = MAX_SUPPORTED_METADATA_VERSION, + "Metadata version unsupported; closing connection" + ); + HandleOutcome::Close } } API_KEY_FETCH => { if is_supported_version(api_key, api_version) { match decode_fetch_request(api_version, body) { - Ok(req) => encode_fetch_response(api_version, &req), + Ok(req) => HandleOutcome::Respond(encode_fetch_response(api_version, &req)), Err(e) => { tracing::warn!("Failed to decode Fetch request: {:?}", e); - encode_fetch_error_response(api_version, ERROR_INVALID_REQUEST) + HandleOutcome::Respond(encode_fetch_error_response( + api_version, + ERROR_INVALID_REQUEST, + )) } } } else { - encode_fetch_error_response(api_version, ERROR_UNSUPPORTED_VERSION) + HandleOutcome::Respond(encode_fetch_error_response( + api_version, + ERROR_UNSUPPORTED_VERSION, + )) } } API_KEY_LIST_OFFSETS => { if is_supported_version(api_key, api_version) { match decode_list_offsets_request(api_version, body) { - Ok(req) => encode_list_offsets_response(api_version, &req), + Ok(req) => { + HandleOutcome::Respond(encode_list_offsets_response(api_version, &req)) + } Err(e) => { tracing::warn!("Failed to decode ListOffsets request: {:?}", e); - encode_list_offsets_error_response(api_version, ERROR_INVALID_REQUEST) + HandleOutcome::Respond(encode_list_offsets_error_response( + api_version, + ERROR_INVALID_REQUEST, + )) } } } else { - encode_list_offsets_error_response(api_version, ERROR_UNSUPPORTED_VERSION) + HandleOutcome::Respond(encode_list_offsets_error_response( + api_version, + ERROR_UNSUPPORTED_VERSION, + )) } } API_KEY_CREATE_TOPICS => { if is_supported_version(api_key, api_version) { match decode_create_topics_request(api_version, body) { - Ok(req) => encode_create_topics_response(api_version, &req), + Ok(req) => { + HandleOutcome::Respond(encode_create_topics_response(api_version, &req)) + } Err(e) => { tracing::warn!("Failed to decode CreateTopics request: {:?}", e); - encode_create_topics_error_response(api_version, ERROR_INVALID_REQUEST) + HandleOutcome::Respond(encode_create_topics_error_response( + api_version, + ERROR_INVALID_REQUEST, + )) } } } else { - encode_create_topics_error_response(api_version, ERROR_UNSUPPORTED_VERSION) + HandleOutcome::Respond(encode_create_topics_error_response( + api_version, + ERROR_UNSUPPORTED_VERSION, + )) } } - _ => encode_error_only_response(ERROR_UNSUPPORTED_VERSION), + _ => HandleOutcome::Respond(encode_error_only_response(ERROR_UNSUPPORTED_VERSION)), } } diff --git a/gateways/kafka/src/server.rs b/gateways/kafka/src/server.rs index d67a80312a..53e4f6e8d8 100644 --- a/gateways/kafka/src/server.rs +++ b/gateways/kafka/src/server.rs @@ -29,8 +29,8 @@ use tracing::{debug, error, info, warn}; use crate::error::{KafkaProtocolError, Result}; use crate::protocol::api::{ - BrokerAdvertise, DEFAULT_KAFKA_PORT, ERROR_INVALID_REQUEST, encode_error_only_response, - handle_request, + BrokerAdvertise, DEFAULT_KAFKA_PORT, ERROR_INVALID_REQUEST, HandleOutcome, + encode_error_only_response, handle_request, }; use crate::protocol::codec::Decoder; use crate::protocol::header::{ @@ -287,23 +287,33 @@ async fn handle_connection( ); let body = decoder.read_bytes(decoder.remaining())?; - let Some(body_response) = handle_request(req.api_key, req.api_version, body, &broker) - else { - // Produce with acks=0: the wire protocol forbids a response. - continue; - }; - - let resp_header = ResponseHeader { - correlation_id: req.correlation_id, - }; - send_response( - &mut stream, - &resp_header, - resp_hdr_ver, - &body_response, - config.write_timeout, - ) - .await?; + match handle_request(req.api_key, req.api_version, body, &broker) { + HandleOutcome::NoResponse => { + // Produce with acks=0: the wire protocol forbids a response. + } + HandleOutcome::Close => { + warn!( + %peer, + api_key = req.api_key, + api_version = req.api_version, + "closing connection: no parseable error response for this request version" + ); + return Ok(()); + } + HandleOutcome::Respond(body_response) => { + let resp_header = ResponseHeader { + correlation_id: req.correlation_id, + }; + send_response( + &mut stream, + &resp_header, + resp_hdr_ver, + &body_response, + config.write_timeout, + ) + .await?; + } + } } } diff --git a/gateways/kafka/tests/api_handler_tests.rs b/gateways/kafka/tests/api_handler_tests.rs index db5a7df529..fb42d1f8f8 100644 --- a/gateways/kafka/tests/api_handler_tests.rs +++ b/gateways/kafka/tests/api_handler_tests.rs @@ -37,7 +37,7 @@ use wire::build_metadata_flexible_request_v10; #[test] fn api_versions_v1_response_non_flexible_format() { let body = handle_request(API_KEY_API_VERSIONS, 1, Bytes::new(), &test_broker()) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i16().unwrap(), 0); // error_code @@ -62,7 +62,7 @@ fn api_versions_v1_response_non_flexible_format() { #[test] fn api_versions_v3_response_flexible_format() { let body = handle_request(API_KEY_API_VERSIONS, 3, Bytes::new(), &test_broker()) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i16().unwrap(), 0); // error_code @@ -93,7 +93,7 @@ fn api_versions_v3_response_flexible_format() { #[test] fn metadata_response_has_broker_array_and_topic_array() { let body = handle_request(API_KEY_METADATA, 0, Bytes::new(), &test_broker()) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let broker_count = d.read_i32().unwrap(); @@ -110,42 +110,17 @@ fn metadata_response_has_broker_array_and_topic_array() { } #[test] -fn unsupported_version_returns_protocol_error() { - // v99 client sends a well-formed v10+ flexible body (topic_id + name) requesting "orders". - // Response encodes at v9 (highest supported); request decodes at client version 99. - let body = handle_request( - API_KEY_METADATA, - 99, - build_metadata_flexible_request_v10(&["orders"]), - &test_broker(), - ) - .expect("test request has acks != 0 and expects a response"); - let mut d = Decoder::new(body); - // v9 flexible response layout: - d.read_i32().unwrap(); // throttle_time_ms (v3+) - let broker_count = usize::try_from(d.read_varint().unwrap()) - .unwrap() - .saturating_sub(1); - for _ in 0..broker_count { - d.read_i32().unwrap(); // node_id - d.read_compact_nullable_string().unwrap(); // host - d.read_i32().unwrap(); // port - d.read_compact_nullable_string().unwrap(); // rack - d.read_tagged_fields().unwrap(); - } - d.read_compact_nullable_string().unwrap(); // cluster_id (v2+) - d.read_i32().unwrap(); // controller_id (v1+) - let topic_count = usize::try_from(d.read_varint().unwrap()) - .unwrap() - .saturating_sub(1); - assert_eq!(topic_count, 1); - let topic_error = d.read_i16().unwrap(); - assert_eq!(topic_error, ERROR_UNSUPPORTED_VERSION); - let topic_name = d.read_compact_nullable_string().unwrap(); - assert_eq!( - topic_name, - Some("orders".to_string()), - "metadata must echo the requested topic name even on an unsupported-version reply" +fn unsupported_metadata_version_closes_connection() { + // Above max: a clamped v9 body is unparseable to a v99 client, so Close is the honest contract. + assert!( + handle_request( + API_KEY_METADATA, + 99, + build_metadata_flexible_request_v10(&["orders"]), + &test_broker(), + ) + .is_close(), + "Metadata above supported max must close rather than return a clamped body" ); } @@ -154,7 +129,7 @@ fn unsupported_version_returns_protocol_error() { #[test] fn unknown_api_key_returns_error_only_payload() { let body = handle_request(999, 0, Bytes::new(), &test_broker()) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); } @@ -170,7 +145,7 @@ fn version_support_table_is_applied() { #[test] fn apiversions_unsupported_version_uses_v0_encoding_without_throttle() { let body = handle_request(API_KEY_API_VERSIONS, 99, Bytes::new(), &test_broker()) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); // v0: error_code(2) + api_keys i32 count(4) + 6 entries × 6 bytes = 42 — no throttle_time_ms. assert_eq!(body.len(), 42); let mut d = Decoder::new(body); @@ -188,7 +163,7 @@ fn produce_malformed_body_with_acks_one_returns_invalid_request() { 0x00, 0x00, 0x00, 0x01, // one topic ]); let response = handle_request(API_KEY_PRODUCE, 3, body, &test_broker()) - .expect("acks=1 malformed produce should get error response"); + .expect_response("acks=1 malformed produce should get error response"); let mut d = Decoder::new(response); assert_eq!(d.read_i32().unwrap(), 1); assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); @@ -209,7 +184,7 @@ fn fetch_malformed_body_returns_invalid_request() { ]), &test_broker(), ) - .expect("fetch must return error response"); + .expect_response("fetch must return error response"); let mut d = Decoder::new(response); assert_eq!(d.read_i32().unwrap(), 0); // throttle assert_eq!(d.read_i16().unwrap(), ERROR_INVALID_REQUEST); @@ -228,7 +203,7 @@ fn list_offsets_malformed_body_returns_invalid_request() { ]), &test_broker(), ) - .expect("list offsets must return error response"); + .expect_response("list offsets must return error response"); let mut d = Decoder::new(response); assert_eq!(d.read_i32().unwrap(), 0); // throttle assert_eq!(d.read_varint().unwrap(), 2); @@ -252,7 +227,7 @@ fn create_topics_malformed_body_returns_invalid_request() { ]), &test_broker(), ) - .expect("create topics must return error response"); + .expect_response("create topics must return error response"); let mut d = Decoder::new(response); assert_eq!(d.read_i32().unwrap(), 0); assert_eq!(d.read_varint().unwrap(), 2); @@ -274,7 +249,7 @@ fn metadata_null_topic_name_yields_zero_topics() { ]), &test_broker(), ) - .expect("metadata request should still return response"); + .expect_response("metadata request should still return response"); let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 1); d.read_i32().unwrap(); diff --git a/gateways/kafka/tests/broker_advertise_tests.rs b/gateways/kafka/tests/broker_advertise_tests.rs index 174947478a..c1236feaf3 100644 --- a/gateways/kafka/tests/broker_advertise_tests.rs +++ b/gateways/kafka/tests/broker_advertise_tests.rs @@ -39,7 +39,7 @@ fn metadata_reflects_broker_addr() { let mut req = Encoder::with_capacity(4); req.write_i32(0); let body = handle_request(API_KEY_METADATA, 0, req.freeze(), &broker) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 1); diff --git a/gateways/kafka/tests/golden_wire_fixtures_tests.rs b/gateways/kafka/tests/golden_wire_fixtures_tests.rs index 52e8587463..a2177b37b0 100644 --- a/gateways/kafka/tests/golden_wire_fixtures_tests.rs +++ b/gateways/kafka/tests/golden_wire_fixtures_tests.rs @@ -26,7 +26,7 @@ use iggy_gateway_kafka::protocol::codec::Encoder; fn golden_apiversions_v1_response_fixture() { let broker = BrokerAdvertise::default(); let actual = handle_request(API_KEY_API_VERSIONS, 1, Bytes::new(), &broker) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); // error_code=0, api_count=6 // key 0 (Produce) min=0 max=9 (KAFKA-18659 advertise min=0) @@ -60,7 +60,7 @@ fn golden_metadata_v0_single_topic_response_fixture() { let req_bytes = request.freeze(); let actual = handle_request(API_KEY_METADATA, 0, req_bytes, &BrokerAdvertise::default()) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); // Metadata v0 layout: brokers[], topics[] (no controller_id — added in v1) // brokers[1]: node_id=1, host=127.0.0.1, port=9093 diff --git a/gateways/kafka/tests/handler_regression_tests.rs b/gateways/kafka/tests/handler_regression_tests.rs index bd99e2be85..c000264786 100644 --- a/gateways/kafka/tests/handler_regression_tests.rs +++ b/gateways/kafka/tests/handler_regression_tests.rs @@ -38,7 +38,7 @@ fn handle_request_succeeds_for_every_supported_version_with_fixture() { // Metadata / ApiVersions: empty body is valid for version in min_ver..=max_ver { let resp = handle_request(api_key, version, bytes::Bytes::new(), &default_broker()) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); assert!( !resp.is_empty(), "{name} v{version} returned empty response" @@ -52,7 +52,7 @@ fn handle_request_succeeds_for_every_supported_version_with_fixture() { continue; }; let resp = handle_request(api_key, version, body, &default_broker()) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); assert!( !resp.is_empty(), "{name} v{version} returned empty response" @@ -68,7 +68,7 @@ fn produce_stub_response_has_zero_error_per_partition() { continue; }; let resp = handle_request(API_KEY_PRODUCE, version, body, &default_broker()) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let flexible = version >= 9; let mut d = Decoder::new(resp); if flexible { @@ -92,7 +92,7 @@ fn fetch_stub_response_has_zero_partition_error() { continue; }; let resp = handle_request(API_KEY_FETCH, version, body, &default_broker()) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let flexible = version >= 12; let mut d = Decoder::new(resp); if version >= 1 { @@ -127,7 +127,7 @@ fn list_offsets_stub_response_has_zero_error() { continue; }; let resp = handle_request(API_KEY_LIST_OFFSETS, version, body, &default_broker()) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let flexible = version >= 6; let mut d = Decoder::new(resp); if version >= 2 { @@ -154,7 +154,7 @@ fn create_topics_stub_response_has_zero_error() { continue; }; let resp = handle_request(API_KEY_CREATE_TOPICS, version, body, &default_broker()) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let flexible = version >= 5; let mut d = Decoder::new(resp); if version >= 2 { diff --git a/gateways/kafka/tests/metadata_regression_tests.rs b/gateways/kafka/tests/metadata_regression_tests.rs index f61ff05809..0d5bf59d77 100644 --- a/gateways/kafka/tests/metadata_regression_tests.rs +++ b/gateways/kafka/tests/metadata_regression_tests.rs @@ -82,7 +82,7 @@ fn metadata_corrupt_partial_body_returns_zero_topics() { Bytes::from_static(&[0x00, 0x00]), &default_broker(), ) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let _ = read_broker_legacy(&mut d); assert_eq!(d.read_i32().unwrap(), 0); @@ -97,7 +97,7 @@ fn metadata_v0_empty_topics_stub_broker() { metadata_request_legacy(0), &default_broker(), ) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let (host, port) = read_broker_legacy(&mut d); assert_eq!(host, "127.0.0.1"); @@ -113,7 +113,7 @@ fn metadata_v0_three_topics_each_unknown() { metadata_request_legacy(3), &default_broker(), ) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let _ = read_broker_legacy(&mut d); assert_eq!(d.read_i32().unwrap(), 3); @@ -135,7 +135,7 @@ fn metadata_v1_includes_controller_id() { metadata_request_legacy(0), &default_broker(), ) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); // Metadata v1 has no throttle_time_ms (added in v3). let _ = read_broker_legacy(&mut d); @@ -152,7 +152,7 @@ fn metadata_v2_includes_cluster_id_field() { metadata_request_legacy(0), &default_broker(), ) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let _ = read_broker_legacy(&mut d); let _rack = d.read_nullable_string().unwrap(); @@ -170,7 +170,7 @@ fn metadata_all_legacy_versions_produce_valid_response() { metadata_request_legacy(1), &default_broker(), ) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); if version >= 3 { let _throttle = d.read_i32().unwrap(); @@ -198,7 +198,7 @@ fn metadata_v9_flexible_encoding() { metadata_request_flexible(2), &default_broker(), ) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let _throttle = d.read_i32().unwrap(); let (host, port) = read_broker_flexible(&mut d); @@ -235,7 +235,7 @@ fn metadata_v8_includes_authorized_operations_legacy() { metadata_request_legacy(1), &default_broker(), ) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let _throttle = d.read_i32().unwrap(); let _ = read_broker_legacy(&mut d); @@ -259,7 +259,7 @@ fn metadata_uses_custom_broker_advertise() { port: 29093, }; let body = handle_request(API_KEY_METADATA, 0, metadata_request_legacy(0), &broker) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let (host, port) = read_broker_legacy(&mut d); assert_eq!(host, "10.0.0.42"); diff --git a/gateways/kafka/tests/review_regression_tests.rs b/gateways/kafka/tests/review_regression_tests.rs index b65408bdf4..1c2a3bac58 100644 --- a/gateways/kafka/tests/review_regression_tests.rs +++ b/gateways/kafka/tests/review_regression_tests.rs @@ -63,7 +63,7 @@ fn produce_acks_zero_malformed_body_decode_carries_acks() { other => panic!("expected decode error with acks=0, got {other:?}"), } assert!( - handle_request(API_KEY_PRODUCE, 3, body, &default_broker()).is_none(), + handle_request(API_KEY_PRODUCE, 3, body, &default_broker()).is_no_response(), "handler must not respond when acks=0 even if decode fails after acks" ); } @@ -156,7 +156,7 @@ fn parse_list_offsets_v0_partition(d: &mut Decoder) { #[test] fn list_offsets_v0_unsupported_version_is_parseable_by_v0_clients() { let body = handle_request(API_KEY_LIST_OFFSETS, 0, Bytes::new(), &default_broker()) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 1, "topics array length"); @@ -180,7 +180,7 @@ fn list_offsets_v0_unsupported_version_is_parseable_by_v0_clients() { fn list_offsets_v0_unsupported_version_carries_error_code_in_partition() { let request_body = build_list_offsets_v0_request_with_topic_t(); let body = handle_request(API_KEY_LIST_OFFSETS, 0, request_body, &default_broker()) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 1); @@ -255,41 +255,17 @@ async fn e2e_list_offsets_v0_unsupported_version_no_trailing_bytes() { // ── Metadata topic name echo (review: must not hardcode "unknown-topic") ──── #[test] -fn metadata_v10_unsupported_decodes_client_body_not_clamped_version() { - let body = handle_request( - API_KEY_METADATA, - 10, - build_metadata_flexible_request_v10(&["payments"]), - &default_broker(), - ) - .expect("test request has acks != 0 and expects a response"); - - let mut d = Decoder::new(body); - d.read_i32().unwrap(); // throttle_time_ms - let broker_count = usize::try_from(d.read_varint().unwrap()) - .unwrap() - .saturating_sub(1); - for _ in 0..broker_count { - d.read_i32().unwrap(); - d.read_compact_nullable_string().unwrap(); - d.read_i32().unwrap(); - d.read_compact_nullable_string().unwrap(); - d.read_tagged_fields().unwrap(); - } - d.read_compact_nullable_string().unwrap(); - d.read_i32().unwrap(); - assert_eq!( - usize::try_from(d.read_varint().unwrap()) - .unwrap() - .saturating_sub(1), - 1 - ); - assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); - assert_eq!( - d.read_compact_nullable_string() - .unwrap() - .expect("topic name"), - "payments" +fn metadata_v10_unsupported_closes_connection() { + // Clamped v9 encoding cannot be parsed by a v10 client; close instead of lying on the wire. + assert!( + handle_request( + API_KEY_METADATA, + 10, + build_metadata_flexible_request_v10(&["payments"]), + &default_broker(), + ) + .is_close(), + "Metadata v10 must close rather than return a clamped unsupported-version body" ); } @@ -317,7 +293,7 @@ fn metadata_v1_echoes_requested_topic_name_in_response() { let topic = "orders"; let request = build_metadata_legacy_request(&[topic]); let body = handle_request(API_KEY_METADATA, 1, request, &default_broker()) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let names = read_metadata_v1_topics(&mut d, 1); @@ -330,7 +306,7 @@ fn metadata_v1_unknown_topic_returns_error_with_requested_name() { let topic = "orders"; let request = build_metadata_legacy_request(&[topic]); let body = handle_request(API_KEY_METADATA, 1, request, &default_broker()) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let _brokers_count = d.read_i32().unwrap(); diff --git a/gateways/kafka/tests/scope_coverage_tests.rs b/gateways/kafka/tests/scope_coverage_tests.rs index e2a3181935..31dae98c13 100644 --- a/gateways/kafka/tests/scope_coverage_tests.rs +++ b/gateways/kafka/tests/scope_coverage_tests.rs @@ -31,6 +31,8 @@ mod tcp; #[path = "common/wire.rs"] mod wire; +use std::time::Duration; + use bytes::{BufMut, Bytes, BytesMut}; use tokio::io::AsyncWriteExt; use tokio::net::TcpStream; @@ -155,7 +157,7 @@ async fn apiversions_v0_and_v2_e2e_return_success() { fn out_of_scope_api_keys_return_unsupported_version_without_panic() { for &(api_key, name) in OUT_OF_SCOPE_API_KEYS { let body = handle_request(api_key, 0, Bytes::new(), &default_broker()) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!( d.read_i16().unwrap(), @@ -186,7 +188,7 @@ async fn out_of_scope_api_keys_e2e_keep_connection_for_follow_up() { assert_eq!(Decoder::new(body).read_i16().unwrap(), ERROR_NONE); } -// ── Version firewall: unsupported version keeps TCP session ───────────────── +// ── Version firewall: unsupported version keeps TCP session (except Metadata) ─ #[tokio::test] async fn each_scoped_api_above_max_version_e2e_keeps_connection() { @@ -206,6 +208,18 @@ async fn each_scoped_api_above_max_version_e2e_keeps_connection() { .write_all(&frame) .await .unwrap_or_else(|_| panic!("write {name} v{above}")); + if api_key == API_KEY_METADATA { + // Unsupported Metadata closes: clamped bodies are unparseable at the client version. + assert_eq!( + tcp::read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await, + tcp::ByteRead::Closed, + "Metadata v{above} must close the connection" + ); + stream = TcpStream::connect(addr) + .await + .expect("reconnect after Metadata close"); + continue; + } let payload = tcp::read_response_frame(&mut stream, 8 * 1024 * 1024).await; assert!( !payload.is_empty(), @@ -240,6 +254,17 @@ async fn each_scoped_api_below_min_version_e2e_keeps_connection() { .write_all(&frame) .await .unwrap_or_else(|_| panic!("write {name} v{below}")); + if api_key == API_KEY_METADATA { + assert_eq!( + tcp::read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await, + tcp::ByteRead::Closed, + "Metadata v{below} must close the connection" + ); + stream = TcpStream::connect(addr) + .await + .expect("reconnect after Metadata close"); + continue; + } let payload = tcp::read_response_frame(&mut stream, 8 * 1024 * 1024).await; assert!( !payload.is_empty(), @@ -269,7 +294,7 @@ fn produce_advertises_min_zero_but_firewall_rejects_below_v3() { assert!(!is_supported_version(API_KEY_PRODUCE, 2)); let body = handle_request(API_KEY_PRODUCE, 2, Bytes::new(), &default_broker()) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let _topics = d.read_i32().unwrap(); let _name = d.read_nullable_string().unwrap(); @@ -288,7 +313,7 @@ fn metadata_v0_empty_topics_returns_zero_length_topic_array() { metadata_empty_legacy_body(), &default_broker(), ) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let _brokers = d.read_i32().unwrap(); d.read_i32().unwrap(); @@ -306,7 +331,7 @@ fn metadata_v3_includes_throttle_time_ms_before_brokers() { metadata_empty_legacy_body(), &default_broker(), ) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 0, "throttle_time_ms"); } @@ -319,7 +344,7 @@ fn metadata_v9_flexible_empty_topics_returns_zero_topics() { build_metadata_flexible_request(&[]), &default_broker(), ) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); d.read_i32().unwrap(); // throttle let broker_count = usize::try_from(d.read_varint().unwrap()) @@ -349,7 +374,7 @@ fn metadata_v9_flexible_echoes_each_requested_topic_name() { build_metadata_flexible_request(&topics), &default_broker(), ) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); d.read_i32().unwrap(); @@ -414,7 +439,7 @@ fn metadata_v1_legacy_multiple_topics_echo_names() { build_metadata_legacy_request(&topics), &default_broker(), ) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); d.read_i32().unwrap(); @@ -677,7 +702,7 @@ async fn corrupt_fetch_body_e2e_returns_error_without_disconnect() { fn corrupt_list_offsets_body_returns_invalid_request_error() { let body = Bytes::from_static(&[0xFF, 0xFF, 0xFF]); let resp = handle_request(API_KEY_LIST_OFFSETS, 1, body, &default_broker()) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); assert!(!resp.is_empty()); assert!( scan_for_error_code(&resp, ERROR_INVALID_REQUEST) @@ -690,7 +715,7 @@ fn corrupt_list_offsets_body_returns_invalid_request_error() { fn corrupt_create_topics_body_returns_invalid_request_error() { let body = Bytes::from_static(&[0xFF, 0xFF, 0xFF]); let resp = handle_request(API_KEY_CREATE_TOPICS, 2, body, &default_broker()) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); assert!(!resp.is_empty()); assert!( scan_for_error_code(&resp, ERROR_INVALID_REQUEST) @@ -702,7 +727,7 @@ fn corrupt_create_topics_body_returns_invalid_request_error() { fn corrupt_metadata_body_returns_zero_topics_not_panic() { let body = Bytes::from_static(&[0x00, 0x00]); let resp = handle_request(API_KEY_METADATA, 0, body, &default_broker()) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); assert!(!resp.is_empty()); let mut d = Decoder::new(resp); d.read_i32().unwrap(); @@ -750,7 +775,7 @@ fn metadata_v9_request_with_three_topics_yields_three_response_slots() { build_metadata_flexible_request(&topics), &default_broker(), ) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); d.read_i32().unwrap(); skip_metadata_v9_prefix(&mut d); @@ -787,7 +812,7 @@ fn every_in_range_version_returns_non_empty_handler_response() { for version in min_ver..=max_ver { let body = request_body_for_scoped_api(api_key, name, version); let resp = handle_request(api_key, version, body, &default_broker()) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); assert!(!resp.is_empty(), "{name} v{version} handler returned empty"); } } diff --git a/gateways/kafka/tests/version_firewall_tests.rs b/gateways/kafka/tests/version_firewall_tests.rs index 8c24c2c313..4f86794e57 100644 --- a/gateways/kafka/tests/version_firewall_tests.rs +++ b/gateways/kafka/tests/version_firewall_tests.rs @@ -21,12 +21,18 @@ mod fixtures; #[path = "common/scope.rs"] mod scope; +#[path = "common/server.rs"] +mod server; #[path = "common/tcp.rs"] mod tcp; #[path = "common/wire.rs"] mod wire; +use std::time::Duration; + use bytes::Bytes; +use tokio::io::AsyncWriteExt; +use tokio::net::TcpStream; use iggy_gateway_kafka::protocol::api::{ API_KEY_API_VERSIONS, API_KEY_CREATE_TOPICS, API_KEY_FETCH, API_KEY_LIST_OFFSETS, @@ -37,7 +43,8 @@ use iggy_gateway_kafka::protocol::codec::Decoder; use fixtures::load_fixture_body_or_skip; use scope::{SCOPED_API_KEYS, default_broker}; -use tcp::build_metadata_legacy_request; +use server::spawn_test_server; +use tcp::{ByteRead, build_metadata_legacy_request, build_request_frame, read_byte_with_timeout}; use wire::build_metadata_flexible_request_v10; #[test] @@ -72,7 +79,7 @@ fn is_supported_version_matches_scope_table() { #[test] fn apiversions_advertises_exact_supported_ranges_v1() { let body = handle_request(API_KEY_API_VERSIONS, 1, Bytes::new(), &default_broker()) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i16().unwrap(), 0); let count = usize::try_from(d.read_i32().unwrap()).expect("api count fits usize"); @@ -96,7 +103,7 @@ fn apiversions_advertises_exact_supported_ranges_v1() { #[test] fn apiversions_advertises_exact_supported_ranges_v3_flexible() { let body = handle_request(API_KEY_API_VERSIONS, 3, Bytes::new(), &default_broker()) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i16().unwrap(), 0); let count = usize::try_from(d.read_varint().unwrap() - 1).expect("api count fits usize"); @@ -142,7 +149,7 @@ fn apiversions_all_versions_return_success() { Bytes::new(), &default_broker(), ) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i16().unwrap(), 0, "ApiVersions v{version}"); } @@ -151,7 +158,7 @@ fn apiversions_all_versions_return_success() { #[test] fn apiversions_out_of_range_returns_unsupported_in_body() { let body = handle_request(API_KEY_API_VERSIONS, 99, Bytes::new(), &default_broker()) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); } @@ -161,66 +168,54 @@ fn metadata_request_one_topic() -> Bytes { } #[test] -fn metadata_below_min_version_returns_topic_error() { - let body = handle_request( - API_KEY_METADATA, - -1, - metadata_request_one_topic(), - &default_broker(), - ) - .expect("test request has acks != 0 and expects a response"); - let mut d = Decoder::new(body); - let _brokers = d.read_i32().unwrap(); - let _ = d.read_i32().unwrap(); - let _ = d.read_nullable_string().unwrap(); - let _ = d.read_i32().unwrap(); - assert_eq!(d.read_i32().unwrap(), 1); // mirrors request topic count - assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); +fn metadata_below_min_version_closes_connection() { + assert!( + handle_request( + API_KEY_METADATA, + -1, + metadata_request_one_topic(), + &default_broker(), + ) + .is_close(), + "Metadata below supported min must close rather than return a clamped body" + ); } #[test] -fn metadata_above_max_version_returns_topic_error() { - // v10 request uses flexible encoding with topic_id; response is clamped to v9. - let body = handle_request( - API_KEY_METADATA, - 10, - build_metadata_flexible_request_v10(&["test-topic"]), - &default_broker(), - ) - .expect("test request has acks != 0 and expects a response"); - // Response is in v9 flexible format (highest supported). - let mut d = Decoder::new(body); - d.read_i32().unwrap(); // throttle_time_ms (v3+) - let broker_count = usize::try_from(d.read_varint().unwrap()) - .unwrap() - .saturating_sub(1); - for _ in 0..broker_count { - d.read_i32().unwrap(); - d.read_compact_nullable_string().unwrap(); - d.read_i32().unwrap(); - d.read_compact_nullable_string().unwrap(); - d.read_tagged_fields().unwrap(); - } - d.read_compact_nullable_string().unwrap(); // cluster_id - d.read_i32().unwrap(); // controller_id - let topic_count = usize::try_from(d.read_varint().unwrap()) - .unwrap() - .saturating_sub(1); - assert_eq!(topic_count, 1); - assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); +fn metadata_above_max_version_closes_connection() { + // v10 request uses flexible encoding; a clamped v9 reply would not survive client parsing. + assert!( + handle_request( + API_KEY_METADATA, + 10, + build_metadata_flexible_request_v10(&["test-topic"]), + &default_broker(), + ) + .is_close(), + "Metadata above supported max must close rather than return a clamped body" + ); +} + +#[tokio::test] +async fn e2e_metadata_above_max_version_closes_tcp_connection() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.expect("connect"); + + let body = build_metadata_flexible_request_v10(&["orders"]); + let frame = build_request_frame(API_KEY_METADATA, 10, 44, Some("n9-test"), &body); + stream.write_all(&frame).await.expect("write metadata v10"); + assert_eq!( - d.read_compact_nullable_string() - .unwrap() - .expect("topic name"), - "test-topic", - "unsupported-version path must decode client v10 body and echo topic name in v9 response" + read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await, + ByteRead::Closed, + "unsupported Metadata version must close the connection" ); } #[test] fn produce_unsupported_version_returns_well_formed_error_response() { let body = handle_request(API_KEY_PRODUCE, 2, Bytes::new(), &default_broker()) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 1); assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); @@ -236,7 +231,7 @@ fn produce_unsupported_version_returns_well_formed_error_response() { #[test] fn fetch_unsupported_version_returns_well_formed_error_response() { let body = handle_request(API_KEY_FETCH, 3, Bytes::new(), &default_broker()) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 0); assert_eq!(d.read_i32().unwrap(), 1); @@ -252,7 +247,7 @@ fn fetch_unsupported_version_returns_well_formed_error_response() { #[test] fn fetch_unsupported_version_above_max_uses_top_level_error() { let body = handle_request(API_KEY_FETCH, 13, Bytes::new(), &default_broker()) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 0); assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); @@ -265,7 +260,7 @@ fn fetch_unsupported_version_above_max_uses_top_level_error() { #[test] fn list_offsets_unsupported_version_returns_well_formed_error_response() { let body = handle_request(API_KEY_LIST_OFFSETS, 0, Bytes::new(), &default_broker()) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 1); assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); @@ -279,7 +274,7 @@ fn list_offsets_unsupported_version_returns_well_formed_error_response() { #[test] fn create_topics_unsupported_version_returns_well_formed_error_response() { let body = handle_request(API_KEY_CREATE_TOPICS, 1, Bytes::new(), &default_broker()) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 1); assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); @@ -292,7 +287,7 @@ fn create_topics_unsupported_version_returns_well_formed_error_response() { fn unsupported_api_keys_return_error_only() { for key in [8, 9, 10, 11, 17, 20, 42, 999] { let body = handle_request(key, 0, Bytes::new(), &default_broker()) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!( d.read_i16().unwrap(), @@ -309,7 +304,7 @@ fn supported_produce_versions_accept_valid_fixture() { continue; }; let resp = handle_request(API_KEY_PRODUCE, version, body, &default_broker()) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); assert!(!resp.is_empty(), "Produce v{version} response empty"); } } @@ -321,7 +316,7 @@ fn supported_fetch_versions_accept_valid_fixture() { continue; }; let resp = handle_request(API_KEY_FETCH, version, body, &default_broker()) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); assert!(!resp.is_empty(), "Fetch v{version} response empty"); } } @@ -330,7 +325,7 @@ fn supported_fetch_versions_accept_valid_fixture() { fn corrupt_produce_body_returns_invalid_request_error() { let body = Bytes::from_static(&[0xFF, 0xFF, 0xFF]); let resp = handle_request(API_KEY_PRODUCE, 3, body, &default_broker()) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(resp); assert_eq!(d.read_i32().unwrap(), 1); assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); @@ -343,7 +338,7 @@ fn corrupt_produce_body_returns_invalid_request_error() { fn corrupt_fetch_body_returns_invalid_request_error() { let body = Bytes::from_static(&[0xFF, 0xFF, 0xFF]); let resp = handle_request(API_KEY_FETCH, 4, body, &default_broker()) - .expect("test request has acks != 0 and expects a response"); + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(resp); assert_eq!(d.read_i32().unwrap(), 0); assert_eq!(d.read_i32().unwrap(), 1); From 6de89852ec1f67f8e44b56f3c992fd7f74aceb9f Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Fri, 17 Jul 2026 05:25:21 -0400 Subject: [PATCH 34/57] Fix typo: unparseable -> unparsable Corrects spelling in two test comments. --- gateways/kafka/tests/api_handler_tests.rs | 2 +- gateways/kafka/tests/scope_coverage_tests.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gateways/kafka/tests/api_handler_tests.rs b/gateways/kafka/tests/api_handler_tests.rs index fb42d1f8f8..d048a3fd1e 100644 --- a/gateways/kafka/tests/api_handler_tests.rs +++ b/gateways/kafka/tests/api_handler_tests.rs @@ -111,7 +111,7 @@ fn metadata_response_has_broker_array_and_topic_array() { #[test] fn unsupported_metadata_version_closes_connection() { - // Above max: a clamped v9 body is unparseable to a v99 client, so Close is the honest contract. + // Above max: a clamped v9 body is unparsable to a v99 client, so Close is the honest contract. assert!( handle_request( API_KEY_METADATA, diff --git a/gateways/kafka/tests/scope_coverage_tests.rs b/gateways/kafka/tests/scope_coverage_tests.rs index 31dae98c13..8130fcb452 100644 --- a/gateways/kafka/tests/scope_coverage_tests.rs +++ b/gateways/kafka/tests/scope_coverage_tests.rs @@ -209,7 +209,7 @@ async fn each_scoped_api_above_max_version_e2e_keeps_connection() { .await .unwrap_or_else(|_| panic!("write {name} v{above}")); if api_key == API_KEY_METADATA { - // Unsupported Metadata closes: clamped bodies are unparseable at the client version. + // Unsupported Metadata closes: clamped bodies are unparsable at the client version. assert_eq!( tcp::read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await, tcp::ByteRead::Closed, From 1035ef3e13d0f0b95a3d516e812f0f630f6e57b8 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sat, 18 Jul 2026 06:17:36 -0400 Subject: [PATCH 35/57] feat(kafka-gateway): use retriable stub error codes Produce stubs now return NOT_LEADER_OR_FOLLOWER (6) instead of success so clients retain data locally. CreateTopics stubs return NOT_CONTROLLER (41) so clients do not believe topics were created. Adds KIP-464 awareness: num_partitions=-1 and replication_factor=-1 are valid broker-default sentinels on CreateTopics v4+, not INVALID_PARTITIONS/INVALID_REPLICATION_FACTOR. Adds ERROR_INVALID_REPLICATION_FACTOR (38) and ERROR_NOT_CONTROLLER (41) constants. Updates kafka-tool verify mode to accept these documented stub error codes as non-failures. --- gateways/kafka/README.md | 2 + gateways/kafka/docs/MANUAL_TESTING.md | 9 +- gateways/kafka/docs/SCOPE.md | 6 +- gateways/kafka/docs/TEST_SUITE.md | 4 +- gateways/kafka/src/protocol/api.rs | 5 ++ gateways/kafka/src/protocol/responses.rs | 57 +++++++++--- .../kafka/tests/decode_validation_tests.rs | 8 +- .../kafka/tests/handler_regression_tests.rs | 18 ++-- .../kafka/tests/response_negative_tests.rs | 88 +++++++++++++++++-- gateways/kafka/tools/kafka-tool/src/main.rs | 14 ++- .../kafka/tools/kafka-tool/src/response.rs | 38 ++++++++ 11 files changed, 213 insertions(+), 36 deletions(-) diff --git a/gateways/kafka/README.md b/gateways/kafka/README.md index 3843706888..9e7b1dbdc3 100644 --- a/gateways/kafka/README.md +++ b/gateways/kafka/README.md @@ -2,6 +2,8 @@ Foundation layer for [apache/iggy#3421](https://github.com/apache/iggy/issues/3421): a TCP listener on the Kafka wire port that decodes requests, validates scoped API keys and versions, and returns stub responses. +> **Stub warning:** Produce does **not** persist records. Valid Produce requests return retriable `NOT_LEADER_OR_FOLLOWER` (6) so clients keep data locally. CreateTopics does **not** create topics; valid requests return `NOT_CONTROLLER` (41). Metadata still reports requested topics as unknown. Persistence lands with the Iggy bridge (see [docs/SCOPE.md](docs/SCOPE.md)). + ## Run ```bash diff --git a/gateways/kafka/docs/MANUAL_TESTING.md b/gateways/kafka/docs/MANUAL_TESTING.md index 5d933a231a..b0c548949a 100644 --- a/gateways/kafka/docs/MANUAL_TESTING.md +++ b/gateways/kafka/docs/MANUAL_TESTING.md @@ -173,11 +173,14 @@ Record kcat version and exact error strings in your test log. G1 passing is the | Code | Name | When returned | | ------ | ------ | --------------- | -| 0 | NONE | Successful stub response | -| 42 | INVALID_REQUEST | Produce/Fetch/ListOffsets/CreateTopics decode failure; unsupported request header | +| 0 | NONE | Successful stub response (Fetch/ListOffsets/ApiVersions) | +| 6 | NOT_LEADER_OR_FOLLOWER | Produce stub (retriable; payload not persisted) | | 3 | UNKNOWN_TOPIC_OR_PARTITION | Metadata stub per-topic error | | 35 | UNSUPPORTED_VERSION | Out-of-range version or unlisted API key | -| 42 | INVALID_REQUEST | Unsupported request header version | +| 37 | INVALID_PARTITIONS | CreateTopics: partition count `0` or `< -1` (or any non-positive on v2–v3) | +| 38 | INVALID_REPLICATION_FACTOR | CreateTopics: replication factor `0` or `< -1` (or any non-positive on v2–v3) | +| 41 | NOT_CONTROLLER | CreateTopics stub (topic not created) | +| 42 | INVALID_REQUEST | Produce/Fetch/ListOffsets/CreateTopics decode failure; unsupported request header | ### Response header rules diff --git a/gateways/kafka/docs/SCOPE.md b/gateways/kafka/docs/SCOPE.md index 984b812470..ee7bc5591c 100644 --- a/gateways/kafka/docs/SCOPE.md +++ b/gateways/kafka/docs/SCOPE.md @@ -4,6 +4,8 @@ Foundation layer only: a TCP listener on the Kafka wire port that decodes requests, validates scoped API keys and versions, validates request wire formats, and returns stub responses. **No Iggy backend integration.** +**Stub semantics (important):** Produce discards the payload and answers with retriable `NOT_LEADER_OR_FOLLOWER` (6). CreateTopics validates the request but answers with `NOT_CONTROLLER` (41) so clients do not believe topics were created. Do not treat `ec=0` stub success as durable storage — that arrives in the Iggy bridge phase. + | Deliverable | Status | Location | | ------------- | -------- | ---------- | | TCP listener on `127.0.0.1:9093` (configurable) | Done | `src/server.rs`, `src/main.rs` | @@ -32,10 +34,10 @@ Expand `SUPPORTED_RANGES` only after a key/version pair is manually tested. ApiV | --------- | ------ | ------------- | ------------- | ---------------- | ---------- | | 18 | ApiVersions | 0 | 3 | 0, 1, 2, 3 | Advertise supported ranges; flexible encoding at v3+ | | 3 | Metadata | 0 | 9 | 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 | Decode topic list count; stub broker from `ServerConfig.bind_addr`; flexible encoding at v9+ | -| 0 | Produce | 3 | 9 | 3, 4, 5, 6, 7, 8, 9 | Decode request; stub response | +| 0 | Produce | 3 | 9 | 3, 4, 5, 6, 7, 8, 9 | Decode request; stub returns `NOT_LEADER_OR_FOLLOWER` (6) | | 1 | Fetch | 4 | 12 | 4, 5, 6, 7, 8, 9, 10, 11, 12 | Decode request; stub response | | 2 | ListOffsets | 1 | 6 | 1, 2, 3, 4, 5, 6 | Decode request; stub response | -| 19 | CreateTopics | 2 | 5 | 2, 3, 4, 5 | Decode request; stub response | +| 19 | CreateTopics | 2 | 5 | 2, 3, 4, 5 | Decode request; stub returns `NOT_CONTROLLER` (41); `-1` partitions/RF = broker default on v4+ | A request is accepted when `min_version ≤ api_version ≤ max_version` for that API key. Any other version for a listed key, or any unlisted API key, receives `UNSUPPORTED_VERSION` (35). diff --git a/gateways/kafka/docs/TEST_SUITE.md b/gateways/kafka/docs/TEST_SUITE.md index 74dd995268..2dd9b727b1 100644 --- a/gateways/kafka/docs/TEST_SUITE.md +++ b/gateways/kafka/docs/TEST_SUITE.md @@ -76,7 +76,7 @@ Fixtures are gitignored under `tools/kafka-tool/kafka_messages/`. CI runs the sa | v8 record_errors array | `decode_validation_tests` | `produce_response_v8_includes_record_errors` | | Unsupported v2 → error 35 | `version_firewall_tests` | `produce_unsupported_version_returns_error_only` | | Corrupt body → error 42 | `version_firewall_tests` | `corrupt_produce_body_returns_invalid_request_error` | -| Stub partition error 0 | `handler_regression_tests` | `produce_stub_response_has_zero_error_per_partition` | +| Stub partition error 6 (not leader) | `handler_regression_tests` | `produce_stub_response_returns_retriable_not_leader` | | E2E round-trip | `server_e2e_tests` | `e2e_produce_v3_round_trip_with_fixture` | ### Fetch (key 1, v4–v12) @@ -108,7 +108,7 @@ Fixtures are gitignored under `tools/kafka-tool/kafka_messages/`. CI runs the sa | v2 roundtrip | `decode_validation_tests` | `create_topics_response_v2_roundtrip` | | v5 flexible roundtrip | `decode_validation_tests` | `create_topics_response_v5_roundtrip` | | Unsupported v1 | `version_firewall_tests` | `create_topics_unsupported_version_returns_error_only` | -| Stub error 0 | `handler_regression_tests` | `create_topics_stub_response_has_zero_error` | +| Stub error 41 (not controller) | `handler_regression_tests` | `create_topics_stub_response_returns_not_controller` | --- diff --git a/gateways/kafka/src/protocol/api.rs b/gateways/kafka/src/protocol/api.rs index 6e41872b3a..cc76641395 100644 --- a/gateways/kafka/src/protocol/api.rs +++ b/gateways/kafka/src/protocol/api.rs @@ -40,8 +40,13 @@ pub const DEFAULT_KAFKA_PORT: u16 = 9093; pub const ERROR_NONE: i16 = 0; pub const ERROR_UNKNOWN_TOPIC_OR_PARTITION: i16 = 3; +/// Retriable; Produce stub uses this until the Iggy bridge persists records. +pub const ERROR_NOT_LEADER_OR_FOLLOWER: i16 = 6; pub const ERROR_UNSUPPORTED_VERSION: i16 = 35; pub const ERROR_INVALID_PARTITIONS: i16 = 37; +pub const ERROR_INVALID_REPLICATION_FACTOR: i16 = 38; +/// `CreateTopics` stub: do not claim topics were created (no controller / no Iggy bridge). +pub const ERROR_NOT_CONTROLLER: i16 = 41; pub const ERROR_INVALID_REQUEST: i16 = 42; const MAX_SUPPORTED_METADATA_VERSION: i16 = 9; diff --git a/gateways/kafka/src/protocol/responses.rs b/gateways/kafka/src/protocol/responses.rs index c03d0c2229..d78d17b4ed 100644 --- a/gateways/kafka/src/protocol/responses.rs +++ b/gateways/kafka/src/protocol/responses.rs @@ -19,11 +19,14 @@ #![allow(clippy::pedantic)] -use crate::protocol::api::{ERROR_INVALID_PARTITIONS, ERROR_NONE}; +use crate::protocol::api::{ + ERROR_INVALID_PARTITIONS, ERROR_INVALID_REPLICATION_FACTOR, ERROR_NONE, ERROR_NOT_CONTROLLER, + ERROR_NOT_LEADER_OR_FOLLOWER, +}; use crate::protocol::codec::Encoder; use crate::protocol::requests::{ - CreateTopicsRequest, FetchRequest, ListOffsetsRequest, ProducePartitionData, ProduceRequest, - ProduceTopicData, + CreatableTopic, CreateTopicsRequest, FetchRequest, ListOffsetsRequest, ProducePartitionData, + ProduceRequest, ProduceTopicData, }; use bytes::Bytes; @@ -40,7 +43,9 @@ pub fn encode_produce_error_response(version: i16, error_code: i16) -> Bytes { } pub fn encode_produce_response(version: i16, req: &ProduceRequest) -> Bytes { - encode_produce_response_inner(version, &req.topics, ERROR_NONE) + // Stub: discard payload and return a retriable error so clients keep data locally + // until the Iggy bridge lands (do not advertise silent success). + encode_produce_response_inner(version, &req.topics, ERROR_NOT_LEADER_OR_FOLLOWER) } fn encode_produce_response_inner( @@ -295,8 +300,6 @@ fn encode_list_offsets_response_inner( /// Well-formed CreateTopics response with a single placeholder topic. pub fn encode_create_topics_error_response(version: i16, error_code: i16) -> Bytes { - use crate::protocol::requests::CreatableTopic; - let topics = vec![CreatableTopic { name: String::new(), num_partitions: 1, @@ -309,9 +312,41 @@ pub fn encode_create_topics_response(version: i16, req: &CreateTopicsRequest) -> encode_create_topics_response_inner(version, &req.topics, ERROR_NONE) } +/// Resolve per-topic CreateTopics error. +/// +/// KIP-464: on v4+, `num_partitions = -1` / `replication_factor = -1` mean broker default. +/// Error 37 / 38 only for `0` and values `< -1` (and any non-positive value on v2–v3). +/// When validation passes, the stub returns [`ERROR_NOT_CONTROLLER`] so clients do not +/// believe the topic was created before the Iggy bridge exists. +const fn create_topics_topic_error(version: i16, topic: &CreatableTopic, forced_error: i16) -> i16 { + if forced_error != ERROR_NONE { + return forced_error; + } + + let partitions_ok = if version >= 4 { + topic.num_partitions == -1 || topic.num_partitions > 0 + } else { + topic.num_partitions > 0 + }; + if !partitions_ok { + return ERROR_INVALID_PARTITIONS; + } + + let replication_ok = if version >= 4 { + topic.replication_factor == -1 || topic.replication_factor > 0 + } else { + topic.replication_factor > 0 + }; + if !replication_ok { + return ERROR_INVALID_REPLICATION_FACTOR; + } + + ERROR_NOT_CONTROLLER +} + fn encode_create_topics_response_inner( version: i16, - topics: &[crate::protocol::requests::CreatableTopic], + topics: &[CreatableTopic], topic_error: i16, ) -> Bytes { let flexible = version >= 5; @@ -334,13 +369,7 @@ fn encode_create_topics_response_inner( e.write_nullable_string_unchecked(Some(&topic.name)); } - let error_code = if topic_error != ERROR_NONE { - topic_error - } else if topic.num_partitions <= 0 { - ERROR_INVALID_PARTITIONS - } else { - ERROR_NONE - }; + let error_code = create_topics_topic_error(version, topic, topic_error); e.write_i16(error_code); if version >= 1 { diff --git a/gateways/kafka/tests/decode_validation_tests.rs b/gateways/kafka/tests/decode_validation_tests.rs index 828da4298f..9a69341741 100644 --- a/gateways/kafka/tests/decode_validation_tests.rs +++ b/gateways/kafka/tests/decode_validation_tests.rs @@ -149,7 +149,7 @@ fn produce_response_v3_roundtrip() { let partition = d.read_i32().unwrap(); assert_eq!(partition, 0); let error_code = d.read_i16().unwrap(); - assert_eq!(error_code, 0); + assert_eq!(error_code, 6); // NOT_LEADER_OR_FOLLOWER — stub until Iggy bridge let base_offset = d.read_i64().unwrap(); assert_eq!(base_offset, 0); // log_append_time_ms (v2+) @@ -176,7 +176,7 @@ fn produce_response_v8_includes_record_errors() { assert_eq!(partition_count, 1); let _partition = d.read_i32().unwrap(); let error_code = d.read_i16().unwrap(); - assert_eq!(error_code, 0); + assert_eq!(error_code, 6); // NOT_LEADER_OR_FOLLOWER — stub until Iggy bridge let _base_offset = d.read_i64().unwrap(); let _log_append_time = d.read_i64().unwrap(); // v2+ let _log_start_offset = d.read_i64().unwrap(); // v5+ @@ -523,7 +523,7 @@ fn create_topics_response_v2_roundtrip() { let resp_topic = d.read_nullable_string().unwrap().unwrap(); assert_eq!(resp_topic, topic_name); let error_code = d.read_i16().unwrap(); - assert_eq!(error_code, 0); + assert_eq!(error_code, 41); // NOT_CONTROLLER — stub until Iggy bridge let error_msg = d.read_nullable_string().unwrap(); // v1+ assert!(error_msg.is_none()); assert_eq!(d.remaining(), 0); @@ -545,7 +545,7 @@ fn create_topics_response_v5_roundtrip() { let _topic_name = d.read_compact_nullable_string().unwrap(); let error_code = d.read_i16().unwrap(); - assert_eq!(error_code, 0); + assert_eq!(error_code, 41); // NOT_CONTROLLER — stub until Iggy bridge let _error_msg = d.read_compact_nullable_string().unwrap(); // v1+ let num_partitions = d.read_i32().unwrap(); assert_eq!(num_partitions, 1); diff --git a/gateways/kafka/tests/handler_regression_tests.rs b/gateways/kafka/tests/handler_regression_tests.rs index c000264786..c9bf7b5a78 100644 --- a/gateways/kafka/tests/handler_regression_tests.rs +++ b/gateways/kafka/tests/handler_regression_tests.rs @@ -24,7 +24,7 @@ mod scope; use iggy_gateway_kafka::protocol::api::{ API_KEY_CREATE_TOPICS, API_KEY_FETCH, API_KEY_LIST_OFFSETS, API_KEY_PRODUCE, ERROR_NONE, - handle_request, + ERROR_NOT_CONTROLLER, ERROR_NOT_LEADER_OR_FOLLOWER, handle_request, }; use iggy_gateway_kafka::protocol::codec::Decoder; @@ -62,7 +62,7 @@ fn handle_request_succeeds_for_every_supported_version_with_fixture() { } #[test] -fn produce_stub_response_has_zero_error_per_partition() { +fn produce_stub_response_returns_retriable_not_leader() { for version in 3i16..=9 { let Some(body) = load_fixture_body_or_skip(0, "Produce", version) else { continue; @@ -81,7 +81,11 @@ fn produce_stub_response_has_zero_error_per_partition() { let _parts = d.read_i32().unwrap(); } let _partition = d.read_i32().unwrap(); - assert_eq!(d.read_i16().unwrap(), ERROR_NONE, "Produce v{version}"); + assert_eq!( + d.read_i16().unwrap(), + ERROR_NOT_LEADER_OR_FOLLOWER, + "Produce v{version}" + ); } } @@ -148,7 +152,7 @@ fn list_offsets_stub_response_has_zero_error() { } #[test] -fn create_topics_stub_response_has_zero_error() { +fn create_topics_stub_response_returns_not_controller() { for version in 2i16..=5 { let Some(body) = load_fixture_body_or_skip(19, "CreateTopics", version) else { continue; @@ -167,6 +171,10 @@ fn create_topics_stub_response_has_zero_error() { let _topics = d.read_i32().unwrap(); let _topic = d.read_nullable_string().unwrap(); } - assert_eq!(d.read_i16().unwrap(), ERROR_NONE, "CreateTopics v{version}"); + assert_eq!( + d.read_i16().unwrap(), + ERROR_NOT_CONTROLLER, + "CreateTopics v{version}" + ); } } diff --git a/gateways/kafka/tests/response_negative_tests.rs b/gateways/kafka/tests/response_negative_tests.rs index f69fdadd11..bd33910591 100644 --- a/gateways/kafka/tests/response_negative_tests.rs +++ b/gateways/kafka/tests/response_negative_tests.rs @@ -16,7 +16,8 @@ // under the License. use iggy_gateway_kafka::protocol::api::{ - ERROR_INVALID_PARTITIONS, ERROR_INVALID_REQUEST, ERROR_UNSUPPORTED_VERSION, + ERROR_INVALID_PARTITIONS, ERROR_INVALID_REPLICATION_FACTOR, ERROR_INVALID_REQUEST, + ERROR_NOT_CONTROLLER, ERROR_UNSUPPORTED_VERSION, }; use iggy_gateway_kafka::protocol::codec::Decoder; use iggy_gateway_kafka::protocol::requests::{ @@ -51,10 +52,11 @@ fn create_topics_response_flags_non_positive_partition_count_v2() { } #[test] -fn create_topics_response_flags_negative_partition_count_v5_flexible() { +fn create_topics_v5_broker_default_partitions_is_not_invalid_partitions() { + // KIP-464: -1 = broker default on CreateTopics v4+; stub still returns NOT_CONTROLLER. let req = CreateTopicsRequest { topics: vec![CreatableTopic { - name: "bad-flex".to_string(), + name: "default-parts".to_string(), num_partitions: -1, replication_factor: 2, }], @@ -66,14 +68,90 @@ fn create_topics_response_flags_negative_partition_count_v5_flexible() { assert_eq!(d.read_varint().unwrap(), 2); // one topic assert_eq!( d.read_compact_nullable_string().unwrap(), - Some("bad-flex".to_string()) + Some("default-parts".to_string()) ); - assert_eq!(d.read_i16().unwrap(), ERROR_INVALID_PARTITIONS); + assert_eq!(d.read_i16().unwrap(), ERROR_NOT_CONTROLLER); assert_eq!(d.read_compact_nullable_string().unwrap(), None); assert_eq!(d.read_i32().unwrap(), -1); assert_eq!(d.read_i16().unwrap(), 2); } +#[test] +fn create_topics_v5_flags_zero_and_below_minus_one_partition_count() { + for num_partitions in [0i32, -2] { + let req = CreateTopicsRequest { + topics: vec![CreatableTopic { + name: "bad-parts".to_string(), + num_partitions, + replication_factor: 1, + }], + timeout_ms: 5_000, + validate_only: false, + }; + let mut d = Decoder::new(encode_create_topics_response(5, &req)); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.read_varint().unwrap(), 2); + assert_eq!( + d.read_compact_nullable_string().unwrap(), + Some("bad-parts".to_string()) + ); + assert_eq!( + d.read_i16().unwrap(), + ERROR_INVALID_PARTITIONS, + "num_partitions={num_partitions}" + ); + } +} + +#[test] +fn create_topics_v5_flags_invalid_replication_factor() { + for replication_factor in [0i16, -2] { + let req = CreateTopicsRequest { + topics: vec![CreatableTopic { + name: "bad-rf".to_string(), + num_partitions: 1, + replication_factor, + }], + timeout_ms: 5_000, + validate_only: false, + }; + let mut d = Decoder::new(encode_create_topics_response(5, &req)); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.read_varint().unwrap(), 2); + assert_eq!( + d.read_compact_nullable_string().unwrap(), + Some("bad-rf".to_string()) + ); + assert_eq!( + d.read_i16().unwrap(), + ERROR_INVALID_REPLICATION_FACTOR, + "replication_factor={replication_factor}" + ); + } +} + +#[test] +fn create_topics_v2_rejects_broker_default_sentinel() { + // KIP-464 defaults apply from v4; on v2, -1 is still INVALID_PARTITIONS. + let req = CreateTopicsRequest { + topics: vec![CreatableTopic { + name: "legacy".to_string(), + num_partitions: -1, + replication_factor: 1, + }], + timeout_ms: 5_000, + validate_only: false, + }; + let mut d = Decoder::new(encode_create_topics_response(2, &req)); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!( + d.read_nullable_string().unwrap(), + Some("legacy".to_string()) + ); + assert_eq!(d.read_i16().unwrap(), ERROR_INVALID_PARTITIONS); +} + #[test] fn create_topics_error_response_carries_explicit_error_code() { let mut d = Decoder::new(encode_create_topics_error_response( diff --git a/gateways/kafka/tools/kafka-tool/src/main.rs b/gateways/kafka/tools/kafka-tool/src/main.rs index 56a979ab06..73b2c1cba2 100644 --- a/gateways/kafka/tools/kafka-tool/src/main.rs +++ b/gateways/kafka/tools/kafka-tool/src/main.rs @@ -687,6 +687,7 @@ async fn read_kafka_response(stream: &mut TcpStream) -> std::io::Result> Ok(body) } +#[allow(clippy::too_many_arguments)] async fn run_send( host: &str, registry: &[(i16, &str, i16, i16)], @@ -695,6 +696,7 @@ async fn run_send( toms: u64, fail_fast: bool, quiet: bool, + strict: bool, ) -> Result<(usize, usize)> { let mut stream = connect(host).await?; info!("Connected to {host}"); @@ -739,6 +741,15 @@ async fn run_send( Ok(Ok(r)) => { let summary = response::analyze_response(ak, v, corr, &r); summary.print(name, v, quiet); + if strict && let Some(reason) = summary.verify_failure_reason(ak) { + println!("✗ {name} v{v} → verify fail: {reason}"); + fail += 1; + if fail_fast { + break 'outer; + } + corr += 1; + continue; + } ok += 1; } Ok(Err(e)) => { @@ -795,6 +806,7 @@ async fn main() -> Result<()> { timeout_ms, false, quiet, + false, ) .await?; println!("\nResult: {ok} OK {fail} failed"); @@ -815,7 +827,7 @@ async fn main() -> Result<()> { &gateway_registry }; let (ok, fail) = run_send( - &host, registry, &api_key, version, timeout_ms, fail_fast, quiet, + &host, registry, &api_key, version, timeout_ms, fail_fast, quiet, true, ) .await?; println!("\n=== Verify: {ok} passed {fail} failed ==="); diff --git a/gateways/kafka/tools/kafka-tool/src/response.rs b/gateways/kafka/tools/kafka-tool/src/response.rs index 51885fd263..67b4e7bd98 100644 --- a/gateways/kafka/tools/kafka-tool/src/response.rs +++ b/gateways/kafka/tools/kafka-tool/src/response.rs @@ -43,6 +43,29 @@ impl ResponseSummary { self.primary_error_code != 0 } + /// Reasons `verify` should count this response as a failure. + /// + /// Fails on correlation mismatch, schema decode failure, and unexpected non-zero + /// error codes. Stub APIs may return documented non-zero codes (Produce 6, + /// Metadata 3, CreateTopics 41). + #[must_use] + pub fn verify_failure_reason(&self, api_key: i16) -> Option { + if !self.correlation_match { + return Some("correlation_id mismatch".into()); + } + if let Some(note) = &self.decode_note { + return Some(format!("schema decode failure: {note}")); + } + if !is_acceptable_verify_error(api_key, self.primary_error_code) { + return Some(format!( + "unexpected error_code={} ({})", + self.primary_error_code, + format_error_code(self.primary_error_code) + )); + } + None + } + pub fn print(&self, api_name: &str, version: i16, quiet: bool) { let sym = if self.has_nonzero_error() { "⚠" @@ -77,6 +100,18 @@ impl ResponseSummary { } } +fn is_acceptable_verify_error(api_key: i16, error_code: i16) -> bool { + if error_code == 0 { + return true; + } + match api_key { + 0 => error_code == 6, // Produce stub: NOT_LEADER_OR_FOLLOWER + 3 => error_code == 3, // Metadata stub: UNKNOWN_TOPIC_OR_PARTITION + 19 => error_code == 41, // CreateTopics stub: NOT_CONTROLLER + _ => false, + } +} + /// Analyze a response payload for the given request `(api_key, api_version)`. pub fn analyze_response( api_key: i16, @@ -338,9 +373,12 @@ fn format_error_code(code: i16) -> &'static str { 1 => "OFFSET_OUT_OF_RANGE", 2 => "CORRUPT_MESSAGE", 3 => "UNKNOWN_TOPIC_OR_PARTITION", + 6 => "NOT_LEADER_OR_FOLLOWER", 35 => "UNSUPPORTED_VERSION", 36 => "TOPIC_ALREADY_EXISTS", 37 => "INVALID_PARTITIONS", + 38 => "INVALID_REPLICATION_FACTOR", + 41 => "NOT_CONTROLLER", 42 => "INVALID_REQUEST", -1 => "UNKNOWN", _ => "OTHER", From 9afa082fcb6bba31ed0b754c0a71007c9066b9c3 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sat, 18 Jul 2026 06:51:22 -0400 Subject: [PATCH 36/57] Close connection after unsupported-version error Implement proper connection closure for unsupported Kafka API versions. Previously, the server would respond with an error but keep the connection open. Now it sends the error response and closes the connection, which is the correct Kafka protocol behavior. Changes include: - Add HandleOutcome::RespondAndClose variant to signal close-after-response - Update server handler to close connection after unsupported-version error - Update tests to verify connection closes after the error response - Update documentation to reflect the new behavior --- gateways/kafka/docs/TEST_SUITE.md | 2 +- gateways/kafka/src/protocol/api.rs | 8 +++--- gateways/kafka/src/server.rs | 16 ++++++++++-- gateways/kafka/tests/scope_coverage_tests.rs | 26 +++++++++++--------- gateways/kafka/tests/server_e2e_tests.rs | 22 +++++++++-------- 5 files changed, 47 insertions(+), 27 deletions(-) diff --git a/gateways/kafka/docs/TEST_SUITE.md b/gateways/kafka/docs/TEST_SUITE.md index 2dd9b727b1..7261f05024 100644 --- a/gateways/kafka/docs/TEST_SUITE.md +++ b/gateways/kafka/docs/TEST_SUITE.md @@ -125,7 +125,7 @@ Fixtures are gitignored under `tools/kafka-tool/kafka_messages/`. CI runs the sa | Invalid frame length (0) | `server_integration_tests` | `read_frame_rejects_invalid_lengths` | | Frame exceeds max_frame_size | `server_integration_tests`, `server_e2e_tests` | `read_frame_rejects_invalid_lengths`, `e2e_oversized_frame_is_rejected` | | Sequential requests on one TCP connection | `server_e2e_tests` | `e2e_sequential_requests_on_one_connection` | -| Connection survives unsupported API key | `server_e2e_tests` | `e2e_unsupported_api_key_returns_error_without_disconnect` | +| Unsupported API key returns error then closes connection | `server_e2e_tests` | `e2e_unsupported_api_key_returns_error_then_closes` | | Negative frame length closes connection | `server_e2e_tests` | `e2e_negative_frame_length_closes_connection` | --- diff --git a/gateways/kafka/src/protocol/api.rs b/gateways/kafka/src/protocol/api.rs index cc76641395..5a4e549617 100644 --- a/gateways/kafka/src/protocol/api.rs +++ b/gateways/kafka/src/protocol/api.rs @@ -59,6 +59,8 @@ const AUTHORIZED_OPS_UNKNOWN: i32 = i32::MIN; pub enum HandleOutcome { /// Write this response body (with a response header). Respond(Bytes), + /// Write this response body (with a response header), then close the TCP connection. + RespondAndClose(Bytes), /// Produce with `acks=0`: write nothing, keep the connection open. NoResponse, /// Client cannot parse an error at this request wire version; close the TCP connection. @@ -75,7 +77,7 @@ impl HandleOutcome { #[must_use] pub fn into_optional_response(self) -> Option { match self { - Self::Respond(body) => Some(body), + Self::Respond(body) | Self::RespondAndClose(body) => Some(body), Self::NoResponse => None, Self::Close => panic!("HandleOutcome::Close has no response body"), } @@ -89,7 +91,7 @@ impl HandleOutcome { #[must_use] pub fn expect_response(self, msg: &str) -> Bytes { match self { - Self::Respond(body) => body, + Self::Respond(body) | Self::RespondAndClose(body) => body, Self::NoResponse => panic!("{msg}: got NoResponse"), Self::Close => panic!("{msg}: got Close"), } @@ -312,7 +314,7 @@ fn handle_other_request( )) } } - _ => HandleOutcome::Respond(encode_error_only_response(ERROR_UNSUPPORTED_VERSION)), + _ => HandleOutcome::RespondAndClose(encode_error_only_response(ERROR_UNSUPPORTED_VERSION)), } } diff --git a/gateways/kafka/src/server.rs b/gateways/kafka/src/server.rs index 53e4f6e8d8..9c6b218645 100644 --- a/gateways/kafka/src/server.rs +++ b/gateways/kafka/src/server.rs @@ -287,7 +287,9 @@ async fn handle_connection( ); let body = decoder.read_bytes(decoder.remaining())?; - match handle_request(req.api_key, req.api_version, body, &broker) { + let outcome = handle_request(req.api_key, req.api_version, body, &broker); + let close_after_response = matches!(outcome, HandleOutcome::RespondAndClose(_)); + match outcome { HandleOutcome::NoResponse => { // Produce with acks=0: the wire protocol forbids a response. } @@ -300,7 +302,8 @@ async fn handle_connection( ); return Ok(()); } - HandleOutcome::Respond(body_response) => { + HandleOutcome::Respond(body_response) + | HandleOutcome::RespondAndClose(body_response) => { let resp_header = ResponseHeader { correlation_id: req.correlation_id, }; @@ -312,6 +315,15 @@ async fn handle_connection( config.write_timeout, ) .await?; + if close_after_response { + warn!( + %peer, + api_key = req.api_key, + api_version = req.api_version, + "closing connection after unsupported-version error response" + ); + return Ok(()); + } } } } diff --git a/gateways/kafka/tests/scope_coverage_tests.rs b/gateways/kafka/tests/scope_coverage_tests.rs index 8130fcb452..1b2347a112 100644 --- a/gateways/kafka/tests/scope_coverage_tests.rs +++ b/gateways/kafka/tests/scope_coverage_tests.rs @@ -168,24 +168,28 @@ fn out_of_scope_api_keys_return_unsupported_version_without_panic() { } #[tokio::test] -async fn out_of_scope_api_keys_e2e_keep_connection_for_follow_up() { +async fn out_of_scope_api_keys_e2e_respond_then_close() { let (addr, _shutdown) = spawn_test_server().await; - let mut stream = TcpStream::connect(addr).await.expect("connect"); - for &(api_key, _name) in &OUT_OF_SCOPE_API_KEYS[..4] { + for &(api_key, name) in &OUT_OF_SCOPE_API_KEYS[..4] { + let mut stream = TcpStream::connect(addr).await.expect("connect"); let frame = build_request_frame(api_key, 0, i32::from(api_key), Some("scope-test"), &[]); stream.write_all(&frame).await.expect("write oos key"); + let payload = tcp::read_response_frame(&mut stream, 8 * 1024 * 1024).await; let mut d = Decoder::new(parse_response_payload(api_key, 0, payload).1); - assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); - } + assert_eq!( + d.read_i16().unwrap(), + ERROR_UNSUPPORTED_VERSION, + "{name} (key {api_key})" + ); - let follow_up = build_request_frame(API_KEY_API_VERSIONS, 1, 99_999, Some("scope-test"), &[]); - stream.write_all(&follow_up).await.expect("follow-up write"); - let payload = tcp::read_response_frame(&mut stream, 8 * 1024 * 1024).await; - let (corr, body) = parse_response_payload(API_KEY_API_VERSIONS, 1, payload); - assert_eq!(corr, 99_999); - assert_eq!(Decoder::new(body).read_i16().unwrap(), ERROR_NONE); + assert_eq!( + tcp::read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await, + tcp::ByteRead::Closed, + "{name} (key {api_key}) must close the connection after the error response" + ); + } } // ── Version firewall: unsupported version keeps TCP session (except Metadata) ─ diff --git a/gateways/kafka/tests/server_e2e_tests.rs b/gateways/kafka/tests/server_e2e_tests.rs index ed2a2843af..73d0ec7e82 100644 --- a/gateways/kafka/tests/server_e2e_tests.rs +++ b/gateways/kafka/tests/server_e2e_tests.rs @@ -35,7 +35,11 @@ use iggy_gateway_kafka::protocol::codec::Decoder; use fixtures::load_fixture_body_or_skip; use server::spawn_test_server; -use tcp::{build_request_frame, parse_response_payload, read_response_frame, round_trip}; +use std::time::Duration; +use tcp::{ + ByteRead, build_request_frame, parse_response_payload, read_byte_with_timeout, + read_response_frame, round_trip, +}; #[tokio::test] async fn e2e_apiversions_v1_preserves_correlation_id() { @@ -83,7 +87,7 @@ async fn e2e_produce_v3_round_trip_with_fixture() { } #[tokio::test] -async fn e2e_unsupported_api_key_returns_error_without_disconnect() { +async fn e2e_unsupported_api_key_returns_error_then_closes() { let (addr, _shutdown) = spawn_test_server().await; let mut stream = TcpStream::connect(addr).await.unwrap(); @@ -95,14 +99,12 @@ async fn e2e_unsupported_api_key_returns_error_without_disconnect() { let mut d = Decoder::new(body); assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); - // Second request on same connection must still work. - let frame2 = build_request_frame(API_KEY_API_VERSIONS, 1, 100, Some("e2e-test"), &[]); - stream.write_all(&frame2).await.unwrap(); - let payload2 = read_response_frame(&mut stream, 8 * 1024 * 1024).await; - let (corr2, body2) = parse_response_payload(API_KEY_API_VERSIONS, 1, payload2); - assert_eq!(corr2, 100); - let mut d2 = Decoder::new(body2); - assert_eq!(d2.read_i16().unwrap(), 0); + // The unsupported-version error is terminal: the server closes the connection. + assert_eq!( + read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await, + ByteRead::Closed, + "connection must close after the unsupported-version error response" + ); } #[tokio::test] From 5ebd1fe06b10ca1ae5031976f3a236199df58566 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sat, 18 Jul 2026 12:16:19 -0400 Subject: [PATCH 37/57] Skip Kafka fixture generation when unchanged Refine the Rust pre-merge action so Kafka wire fixtures are only generated when gateway tests are in scope or a full-workspace run includes changes under `gateways/**`. This avoids unnecessary `kafka-message-gen` and `kafka-protocol` builds, while still falling back to safe generation when `origin/master` is unavailable. --- .github/actions/rust/pre-merge/action.yml | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/actions/rust/pre-merge/action.yml b/.github/actions/rust/pre-merge/action.yml index c1180fed18..2329fbc845 100644 --- a/.github/actions/rust/pre-merge/action.yml +++ b/.github/actions/rust/pre-merge/action.yml @@ -270,13 +270,22 @@ runs: echo "::notice::Tests compiled in ${compile_duration}s ($(date -ud @${compile_duration} +'%M:%S'))" # decode_validation_tests need gitignored wire fixtures. Generate when - # iggy-gateway-kafka is in the DAG test scope (rust-gateway job or parent - # rust job — both run gateway tests when gateways/** changes). + # iggy-gateway-kafka is in the DAG test scope, or (on a full-workspace run) + # when gateways/** changed vs origin/master — avoid building kafka-message-gen + # / kafka-protocol when the gateway was not touched. NEEDS_KAFKA_FIXTURES=false - if [[ -z "$NEXTEST_FILTER" ]]; then - NEEDS_KAFKA_FIXTURES=true - elif grep -q 'package(iggy-gateway-kafka)' <<< "$NEXTEST_FILTER"; then + if grep -q 'package(iggy-gateway-kafka)' <<< "$NEXTEST_FILTER"; then NEEDS_KAFKA_FIXTURES=true + elif [[ -z "$NEXTEST_FILTER" ]]; then + if git diff --name-only origin/master...HEAD 2>/dev/null | grep -qE '^gateways/'; then + NEEDS_KAFKA_FIXTURES=true + elif ! git rev-parse --verify origin/master >/dev/null 2>&1; then + # Base ref unavailable: full suite still runs gateway tests — generate safely. + NEEDS_KAFKA_FIXTURES=true + echo "::notice::origin/master unavailable; generating Kafka fixtures for full suite" + else + echo "::notice::Skipping Kafka wire fixtures (full suite, gateways/** unchanged)" + fi fi if [[ "$NEEDS_KAFKA_FIXTURES" == true ]]; then ./gateways/kafka/scripts/ci-wire-fixtures.sh generate From 78aaa32f52aeb868329f74a041fe72fd6ef4324b Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Tue, 28 Jul 2026 11:28:14 -0400 Subject: [PATCH 38/57] Update Cargo.lock --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3175794555..0a1e65abb6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6790,8 +6790,8 @@ name = "iggy-gateway-kafka" version = "0.1.0" dependencies = [ "bytes", - "socket2 0.6.4", - "thiserror 2.0.18", + "socket2 0.6.5", + "thiserror 2.0.19", "tokio", "tokio-util", "tracing", From 3041181507ecc417507707d513db542f4ae48d8e Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Wed, 29 Jul 2026 08:52:48 -0400 Subject: [PATCH 39/57] Update components.yml --- .github/config/components.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/config/components.yml b/.github/config/components.yml index 8cef0e67a1..8e23af57e6 100644 --- a/.github/config/components.yml +++ b/.github/config/components.yml @@ -523,11 +523,3 @@ components: - "ci-infrastructure" paths: - "gateways/**" - tasks: - - "check" - - "fmt" - - "clippy" - - "sort" - - "test-1" - - "test-2" - - "machete" From a09379f8b41d6a8d4283a5b36886ccd10fb66634 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Fri, 31 Jul 2026 21:50:42 -0400 Subject: [PATCH 40/57] Refactored test suite and clippy directives Refactored test cases by regrouping by functionality. --- .github/actions/rust/pre-merge/action.yml | 3 + gateways/kafka/src/protocol/api.rs | 6 +- gateways/kafka/src/protocol/header.rs | 63 +- gateways/kafka/src/protocol/requests.rs | 33 +- gateways/kafka/src/protocol/responses.rs | 21 +- gateways/kafka/src/server.rs | 2 +- gateways/kafka/tests/api_handler_tests.rs | 700 ++++++++++++++- gateways/kafka/tests/common/fixtures.rs | 14 +- gateways/kafka/tests/common/scope.rs | 2 +- gateways/kafka/tests/common/server.rs | 2 +- gateways/kafka/tests/common/tcp.rs | 26 +- gateways/kafka/tests/decode_safety_tests.rs | 227 ++++- .../kafka/tests/decode_validation_tests.rs | 168 +++- .../kafka/tests/golden_wire_fixtures_tests.rs | 2 +- .../kafka/tests/handler_regression_tests.rs | 180 ---- gateways/kafka/tests/header_tests.rs | 24 +- .../kafka/tests/listener_robustness_tests.rs | 148 +++- .../kafka/tests/metadata_regression_tests.rs | 267 ------ gateways/kafka/tests/region_coverage_tests.rs | 445 ---------- .../kafka/tests/review_regression_tests.rs | 426 --------- gateways/kafka/tests/scope_coverage_tests.rs | 823 ------------------ gateways/kafka/tests/server_e2e_tests.rs | 328 ++++++- .../kafka/tests/server_integration_tests.rs | 2 +- .../kafka/tests/version_firewall_tests.rs | 385 +++++++- gateways/kafka/tools/kafka-tool/src/main.rs | 10 +- .../kafka-tool/tests/generate_cli_tests.rs | 1 - 26 files changed, 2057 insertions(+), 2251 deletions(-) delete mode 100644 gateways/kafka/tests/handler_regression_tests.rs delete mode 100644 gateways/kafka/tests/metadata_regression_tests.rs delete mode 100644 gateways/kafka/tests/region_coverage_tests.rs delete mode 100644 gateways/kafka/tests/review_regression_tests.rs delete mode 100644 gateways/kafka/tests/scope_coverage_tests.rs diff --git a/.github/actions/rust/pre-merge/action.yml b/.github/actions/rust/pre-merge/action.yml index 2329fbc845..c5c1a06792 100644 --- a/.github/actions/rust/pre-merge/action.yml +++ b/.github/actions/rust/pre-merge/action.yml @@ -290,6 +290,9 @@ runs: if [[ "$NEEDS_KAFKA_FIXTURES" == true ]]; then ./gateways/kafka/scripts/ci-wire-fixtures.sh generate trap './gateways/kafka/scripts/ci-wire-fixtures.sh cleanup' EXIT + # Fixtures are expected to exist from here on; a missing one now means + # generation silently failed, not a legitimate local-dev skip. + export KAFKA_FIXTURES_REQUIRED=1 fi # Start D-Bus and unlock keyring right before test execution to avoid diff --git a/gateways/kafka/src/protocol/api.rs b/gateways/kafka/src/protocol/api.rs index 5a4e549617..5c7892a8d2 100644 --- a/gateways/kafka/src/protocol/api.rs +++ b/gateways/kafka/src/protocol/api.rs @@ -72,7 +72,7 @@ impl HandleOutcome { /// /// # Panics /// - /// Panics on [`HandleOutcome::Close`] — match on `Close` explicitly, or use + /// Panics on [`HandleOutcome::Close`] - match on `Close` explicitly, or use /// [`Self::expect_response`] in tests that require a body. #[must_use] pub fn into_optional_response(self) -> Option { @@ -432,7 +432,7 @@ fn encode_metadata_response( } else { e.write_i32(1); // brokers array length e.write_i32(1); // node_id - // broker.host is config-derived (KAFKA_ADVERTISED_HOST), not request-decoded — use + // broker.host is config-derived (KAFKA_ADVERTISED_HOST), not request-decoded - use // the checked variant so an overly long hostname returns an error instead of panicking. if e.write_nullable_string(Some(&broker.host)).is_err() { return encode_error_only_response(ERROR_INVALID_REQUEST); @@ -446,7 +446,7 @@ fn encode_metadata_response( e.write_nullable_string_unchecked(None); // cluster_id } if response_version >= 1 { - e.write_i32(1); // controller_id — must come before topics array + e.write_i32(1); // controller_id - must come before topics array } e.write_i32(i32::try_from(topics_count).expect("topic count bounded")); diff --git a/gateways/kafka/src/protocol/header.rs b/gateways/kafka/src/protocol/header.rs index 5488a7413f..0a8d34e071 100644 --- a/gateways/kafka/src/protocol/header.rs +++ b/gateways/kafka/src/protocol/header.rs @@ -16,8 +16,9 @@ // under the License. #![allow( - clippy::pedantic, + clippy::doc_markdown, clippy::missing_const_for_fn, + clippy::missing_errors_doc, clippy::match_same_arms )] @@ -73,7 +74,7 @@ fn first_flexible_version_threshold(api_key: i16) -> i16 { 14 => 4, // SyncGroup 15 => 5, // DescribeGroups 16 => 3, // ListGroups - 17 => i16::MAX, // SaslHandshake — never flexible + 17 => i16::MAX, // SaslHandshake - never flexible 18 => 3, // ApiVersions 19 => 5, // CreateTopics 20 => 4, // DeleteTopics @@ -101,34 +102,34 @@ fn first_flexible_version_threshold(api_key: i16) -> i16 { 42 => 2, // DeleteGroups 43 => 2, // ElectLeaders 44 => 1, // IncrementalAlterConfigs - 45 => 0, // AlterPartitionReassignments — always flexible - 46 => 0, // ListPartitionReassignments — always flexible - 47 => i16::MAX, // OffsetDelete — never flexible + 45 => 0, // AlterPartitionReassignments - always flexible + 46 => 0, // ListPartitionReassignments - always flexible + 47 => i16::MAX, // OffsetDelete - never flexible 48 => 1, // DescribeClientQuotas 49 => 1, // AlterClientQuotas - 50 => 0, // DescribeUserScramCredentials — always flexible - 51 => 0, // AlterUserScramCredentials — always flexible - 55 => 0, // DescribeQuorum — always flexible - 56 => 0, // AlterPartition — always flexible + 50 => 0, // DescribeUserScramCredentials - always flexible + 51 => 0, // AlterUserScramCredentials - always flexible + 55 => 0, // DescribeQuorum - always flexible + 56 => 0, // AlterPartition - always flexible 57 => 1, // UpdateFeatures - 60 => 0, // DescribeCluster — always flexible - 61 => 0, // DescribeProducers — always flexible - 64 => 0, // UnregisterBroker — always flexible - 65 => 0, // DescribeTransactions — always flexible - 66 => 0, // ListTransactions — always flexible - 67 => 0, // AllocateProducerIds — always flexible - 68 => 0, // ConsumerGroupHeartbeat — always flexible - 69 => 0, // ConsumerGroupDescribe — always flexible - 71 => 0, // GetTelemetrySubscriptions — always flexible - 72 => 0, // PushTelemetry — always flexible - 74 => 0, // AssignReplicasToDirs — always flexible - 75 => 0, // DescribeTopicPartitions — always flexible - 76 => 0, // ListClientMetricsResources — always flexible - 77 => 0, // ShareGroupHeartbeat — always flexible (Kafka 4.0) - 78 => 0, // ShareGroupDescribe — always flexible - 79 => 0, // ShareFetch — always flexible - 80 => 0, // ShareAcknowledge — always flexible - _ => i16::MAX, // Unknown API — assume non-flexible + 60 => 0, // DescribeCluster - always flexible + 61 => 0, // DescribeProducers - always flexible + 64 => 0, // UnregisterBroker - always flexible + 65 => 0, // DescribeTransactions - always flexible + 66 => 0, // ListTransactions - always flexible + 67 => 0, // AllocateProducerIds - always flexible + 68 => 0, // ConsumerGroupHeartbeat - always flexible + 69 => 0, // ConsumerGroupDescribe - always flexible + 71 => 0, // GetTelemetrySubscriptions - always flexible + 72 => 0, // PushTelemetry - always flexible + 74 => 0, // AssignReplicasToDirs - always flexible + 75 => 0, // DescribeTopicPartitions - always flexible + 76 => 0, // ListClientMetricsResources - always flexible + 77 => 0, // ShareGroupHeartbeat - always flexible (Kafka 4.0) + 78 => 0, // ShareGroupDescribe - always flexible + 79 => 0, // ShareFetch - always flexible + 80 => 0, // ShareAcknowledge - always flexible + _ => i16::MAX, // Unknown API - assume non-flexible } } @@ -150,15 +151,12 @@ pub fn request_header_version(api_key: i16, api_version: i16) -> i16 { /// ApiVersions (18) is a special case: the server ALWAYS returns response header v0 (no tagged /// fields) so that clients that don't yet know the server supports flexible encoding can still /// parse the discovery response. All other flexible-version APIs use response header v1. +#[must_use] pub fn response_header_version(api_key: i16, api_version: i16) -> i16 { if api_key == 18 { return 0; } - if request_header_version(api_key, api_version) >= 2 { - 1 - } else { - 0 - } + i16::from(request_header_version(api_key, api_version) >= 2) } impl RequestHeader { @@ -212,6 +210,7 @@ impl ResponseHeader { /// /// v0: correlation_id i32 (non-flexible APIs and ApiVersions) /// v1: correlation_id i32 + empty tagged fields (flexible APIs) + #[must_use] pub fn encode(&self, header_version: i16) -> Bytes { let mut e = Encoder::with_capacity(5); e.write_i32(self.correlation_id); diff --git a/gateways/kafka/src/protocol/requests.rs b/gateways/kafka/src/protocol/requests.rs index 2ee238a14a..81e1c1cb2a 100644 --- a/gateways/kafka/src/protocol/requests.rs +++ b/gateways/kafka/src/protocol/requests.rs @@ -17,8 +17,8 @@ //! Kafka request decoders for critical API keys -#![allow(clippy::pedantic)] - +#![allow(clippy::too_many_lines, + clippy::doc_markdown)] use crate::error::{KafkaProtocolError, Result}; use crate::protocol::codec::Decoder; use bytes::Bytes; @@ -59,6 +59,10 @@ pub enum ProduceDecodeResult { impl ProduceDecodeResult { /// Collapse to `Result` for tests and callers that only need a successful `ProduceRequest`. + /// + /// # Errors + /// + /// Returns an error if the byte stream is malformed or if the API version is unsupported. pub fn into_request(self) -> Result { match self { Self::Ok(req) => Ok(req), @@ -189,7 +193,11 @@ pub struct FetchPartition { pub fetch_offset: i64, pub partition_max_bytes: i32, } - +/// Decodes a raw byte stream into a `FetchRequest`. +/// +/// # Errors +/// +/// Returns an error if the byte stream is malformed or if the API version is unsupported. pub fn decode_fetch_request(version: i16, body: Bytes) -> Result { let mut d = Decoder::new(body); let flexible = version >= 12; @@ -206,7 +214,7 @@ pub fn decode_fetch_request(version: i16, body: Bytes) -> Result { let isolation_level = if version >= 4 { d.read_i8()? } else { 0 }; - // session_id and session_epoch (v7+) — read and discard (stub path) + // session_id and session_epoch (v7+) - read and discard (stub path) if version >= 7 { d.read_i32()?; // session_id d.read_i32()?; // session_epoch @@ -272,7 +280,7 @@ pub fn decode_fetch_request(version: i16, body: Bytes) -> Result { } } - // forgotten_topics_data (v7+) — skip + // forgotten_topics_data (v7+) - skip if version >= 7 { let forgotten_count = if flexible { d.read_compact_array_count()? @@ -337,7 +345,10 @@ pub struct ListOffsetsPartition { pub partition: i32, pub timestamp: i64, // -2 = earliest, -1 = latest } - +/// Collapse to `Result` for tests and callers that only need a successful `decode_list_offsets_request`. +/// # Errors +/// +/// Returns an error if the byte stream is malformed or if the API version is unsupported. pub fn decode_list_offsets_request(version: i16, body: Bytes) -> Result { let mut d = Decoder::new(body); let flexible = version >= 6; @@ -408,7 +419,7 @@ pub fn decode_list_offsets_request(version: i16, body: Bytes) -> Result, @@ -423,6 +434,10 @@ pub struct CreatableTopic { pub replication_factor: i16, } +/// Collapse to `Result` for tests and callers that only need a successful `decode_create_topics_request`. +/// # Errors +/// +/// Returns an error if the byte stream is malformed or if the API version is unsupported. pub fn decode_create_topics_request(version: i16, body: Bytes) -> Result { let mut d = Decoder::new(body); let flexible = version >= 5; @@ -446,7 +461,7 @@ pub fn decode_create_topics_request(version: i16, body: Bytes) -> Result Result Bytes { let topics = vec![ProduceTopicData { topic: String::new(), // TODO topic name will be populated in the end to end functional completion @@ -41,7 +43,7 @@ pub fn encode_produce_error_response(version: i16, error_code: i16) -> Bytes { }]; encode_produce_response_inner(version, &topics, error_code) } - +#[must_use] pub fn encode_produce_response(version: i16, req: &ProduceRequest) -> Bytes { // Stub: discard payload and return a retriable error so clients keep data locally // until the Iggy bridge lands (do not advertise silent success). @@ -116,9 +118,8 @@ fn encode_produce_response_inner( /// Well-formed Fetch response. Uses top-level `error_code` at v7+, or a single /// placeholder topic/partition with per-partition `error_code` below v7. +#[must_use] pub fn encode_fetch_error_response(version: i16, error_code: i16) -> Bytes { - use crate::protocol::requests::{FetchPartition, FetchTopic}; - if version >= 7 { return encode_fetch_response_inner(version, &[], Some(error_code), error_code); } @@ -133,7 +134,7 @@ pub fn encode_fetch_error_response(version: i16, error_code: i16) -> Bytes { }]; encode_fetch_response_inner(version, &topics, Some(ERROR_NONE), error_code) } - +#[must_use] pub fn encode_fetch_response(version: i16, req: &FetchRequest) -> Bytes { encode_fetch_response_inner(version, &req.topics, Some(ERROR_NONE), ERROR_NONE) } @@ -217,9 +218,8 @@ fn encode_fetch_response_inner( } /// Well-formed ListOffsets response with a single placeholder topic/partition. +#[must_use] pub fn encode_list_offsets_error_response(version: i16, error_code: i16) -> Bytes { - use crate::protocol::requests::{ListOffsetsPartition, ListOffsetsTopic}; - let topics = vec![ListOffsetsTopic { topic: String::new(), partitions: vec![ListOffsetsPartition { @@ -229,7 +229,7 @@ pub fn encode_list_offsets_error_response(version: i16, error_code: i16) -> Byte }]; encode_list_offsets_response_inner(version, &topics, error_code) } - +#[must_use] pub fn encode_list_offsets_response(version: i16, req: &ListOffsetsRequest) -> Bytes { encode_list_offsets_response_inner(version, &req.topics, ERROR_NONE) } @@ -299,6 +299,7 @@ fn encode_list_offsets_response_inner( } /// Well-formed CreateTopics response with a single placeholder topic. +#[must_use] pub fn encode_create_topics_error_response(version: i16, error_code: i16) -> Bytes { let topics = vec![CreatableTopic { name: String::new(), @@ -307,7 +308,7 @@ pub fn encode_create_topics_error_response(version: i16, error_code: i16) -> Byt }]; encode_create_topics_response_inner(version, &topics, error_code) } - +#[must_use] pub fn encode_create_topics_response(version: i16, req: &CreateTopicsRequest) -> Bytes { encode_create_topics_response_inner(version, &req.topics, ERROR_NONE) } diff --git a/gateways/kafka/src/server.rs b/gateways/kafka/src/server.rs index 9c6b218645..05af1ef44c 100644 --- a/gateways/kafka/src/server.rs +++ b/gateways/kafka/src/server.rs @@ -156,7 +156,7 @@ impl KafkaServer { tracker.wait().await; break; } - // Capacity-1 channel: lagged means a signal was sent before we polled — treat as shutdown. + // Capacity-1 channel: lagged means a signal was sent before we polled - treat as shutdown. Err(broadcast::error::RecvError::Lagged(_)) => { info!("kafka listener shutdown requested (lagged)"); tracker.close(); diff --git a/gateways/kafka/tests/api_handler_tests.rs b/gateways/kafka/tests/api_handler_tests.rs index d048a3fd1e..fa70c77a6e 100644 --- a/gateways/kafka/tests/api_handler_tests.rs +++ b/gateways/kafka/tests/api_handler_tests.rs @@ -15,6 +15,12 @@ // specific language governing permissions and limitations // under the License. +#[path = "common/fixtures.rs"] +mod fixtures; +#[path = "common/scope.rs"] +mod scope; +#[path = "common/tcp.rs"] +mod tcp; #[path = "common/wire.rs"] mod wire; @@ -22,21 +28,23 @@ use bytes::Bytes; use iggy_gateway_kafka::protocol::api::{ API_KEY_API_VERSIONS, API_KEY_CREATE_TOPICS, API_KEY_FETCH, API_KEY_LIST_OFFSETS, - API_KEY_METADATA, API_KEY_PRODUCE, BrokerAdvertise, ERROR_INVALID_REQUEST, - ERROR_UNSUPPORTED_VERSION, handle_request, is_supported_version, supported_api_ranges, + API_KEY_METADATA, API_KEY_PRODUCE, ERROR_INVALID_REQUEST, ERROR_NONE, ERROR_NOT_CONTROLLER, + ERROR_NOT_LEADER_OR_FOLLOWER, ERROR_UNKNOWN_TOPIC_OR_PARTITION, ERROR_UNSUPPORTED_VERSION, + handle_request, is_supported_version, supported_api_ranges, }; +use iggy_gateway_kafka::protocol::codec::{Decoder, Encoder}; +use iggy_gateway_kafka::protocol::requests::{ProduceDecodeResult, decode_produce_request}; -fn test_broker() -> BrokerAdvertise { - BrokerAdvertise::default() -} -use iggy_gateway_kafka::protocol::codec::Decoder; -use wire::build_metadata_flexible_request_v10; +use fixtures::load_fixture_body_or_skip; +use scope::default_broker; +use tcp::{build_metadata_legacy_request, build_produce_v3_body}; +use wire::{build_metadata_flexible_request, build_metadata_flexible_request_v10}; // ── ApiVersions ───────────────────────────────────────────────────────────── #[test] fn api_versions_v1_response_non_flexible_format() { - let body = handle_request(API_KEY_API_VERSIONS, 1, Bytes::new(), &test_broker()) + let body = handle_request(API_KEY_API_VERSIONS, 1, Bytes::new(), &default_broker()) .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); @@ -61,7 +69,7 @@ fn api_versions_v1_response_non_flexible_format() { #[test] fn api_versions_v3_response_flexible_format() { - let body = handle_request(API_KEY_API_VERSIONS, 3, Bytes::new(), &test_broker()) + let body = handle_request(API_KEY_API_VERSIONS, 3, Bytes::new(), &default_broker()) .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); @@ -92,7 +100,7 @@ fn api_versions_v3_response_flexible_format() { #[test] fn metadata_response_has_broker_array_and_topic_array() { - let body = handle_request(API_KEY_METADATA, 0, Bytes::new(), &test_broker()) + let body = handle_request(API_KEY_METADATA, 0, Bytes::new(), &default_broker()) .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); @@ -117,7 +125,7 @@ fn unsupported_metadata_version_closes_connection() { API_KEY_METADATA, 99, build_metadata_flexible_request_v10(&["orders"]), - &test_broker(), + &default_broker(), ) .is_close(), "Metadata above supported max must close rather than return a clamped body" @@ -128,7 +136,7 @@ fn unsupported_metadata_version_closes_connection() { #[test] fn unknown_api_key_returns_error_only_payload() { - let body = handle_request(999, 0, Bytes::new(), &test_broker()) + let body = handle_request(999, 0, Bytes::new(), &default_broker()) .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); @@ -144,9 +152,9 @@ fn version_support_table_is_applied() { #[test] fn apiversions_unsupported_version_uses_v0_encoding_without_throttle() { - let body = handle_request(API_KEY_API_VERSIONS, 99, Bytes::new(), &test_broker()) + let body = handle_request(API_KEY_API_VERSIONS, 99, Bytes::new(), &default_broker()) .expect_response("test request has acks != 0 and expects a response"); - // v0: error_code(2) + api_keys i32 count(4) + 6 entries × 6 bytes = 42 — no throttle_time_ms. + // v0: error_code(2) + api_keys i32 count(4) + 6 entries × 6 bytes = 42 - no throttle_time_ms. assert_eq!(body.len(), 42); let mut d = Decoder::new(body); assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); @@ -162,7 +170,7 @@ fn produce_malformed_body_with_acks_one_returns_invalid_request() { 0x00, 0x00, 0x03, 0xe8, // timeout_ms 0x00, 0x00, 0x00, 0x01, // one topic ]); - let response = handle_request(API_KEY_PRODUCE, 3, body, &test_broker()) + let response = handle_request(API_KEY_PRODUCE, 3, body, &default_broker()) .expect_response("acks=1 malformed produce should get error response"); let mut d = Decoder::new(response); assert_eq!(d.read_i32().unwrap(), 1); @@ -182,7 +190,7 @@ fn fetch_malformed_body_returns_invalid_request() { 0x00, 0x00, 0x00, 0x64, // max_wait_ms 0x00, 0x00, 0x00, 0x01, // min_bytes ]), - &test_broker(), + &default_broker(), ) .expect_response("fetch must return error response"); let mut d = Decoder::new(response); @@ -201,7 +209,7 @@ fn list_offsets_malformed_body_returns_invalid_request() { 0x02, // compact topics count = one 0x00, // null topic name ]), - &test_broker(), + &default_broker(), ) .expect_response("list offsets must return error response"); let mut d = Decoder::new(response); @@ -225,7 +233,7 @@ fn create_topics_malformed_body_returns_invalid_request() { 0x02, // compact topics count = one 0x00, // null compact topic name ]), - &test_broker(), + &default_broker(), ) .expect_response("create topics must return error response"); let mut d = Decoder::new(response); @@ -247,7 +255,7 @@ fn metadata_null_topic_name_yields_zero_topics() { 0x00, 0x00, 0x00, 0x01, // one topic 0xff, 0xff, // null topic name ]), - &test_broker(), + &default_broker(), ) .expect_response("metadata request should still return response"); let mut d = Decoder::new(body); @@ -258,3 +266,657 @@ fn metadata_null_topic_name_yields_zero_topics() { assert_eq!(d.read_i32().unwrap(), 0); assert_eq!(d.remaining(), 0); } + +// ── Full handler regression - every scoped API key x version through `handle_request` ── + +#[test] +fn handle_request_succeeds_for_every_supported_version_with_fixture() { + for &(api_key, name, min_ver, max_ver) in scope::SCOPED_API_KEYS { + if api_key == 3 || api_key == 18 { + // Metadata / ApiVersions: empty body is valid + for version in min_ver..=max_ver { + let resp = handle_request(api_key, version, bytes::Bytes::new(), &default_broker()) + .expect_response("test request has acks != 0 and expects a response"); + assert!( + !resp.is_empty(), + "{name} v{version} returned empty response" + ); + } + continue; + } + + for version in min_ver..=max_ver { + let Some(body) = load_fixture_body_or_skip(api_key, name, version) else { + continue; + }; + let resp = handle_request(api_key, version, body, &default_broker()) + .expect_response("test request has acks != 0 and expects a response"); + assert!( + !resp.is_empty(), + "{name} v{version} returned empty response" + ); + } + } +} + +#[test] +fn produce_stub_response_returns_retriable_not_leader() { + for version in 3i16..=9 { + let Some(body) = load_fixture_body_or_skip(0, "Produce", version) else { + continue; + }; + let resp = handle_request(API_KEY_PRODUCE, version, body, &default_broker()) + .expect_response("test request has acks != 0 and expects a response"); + let flexible = version >= 9; + let mut d = Decoder::new(resp); + if flexible { + let _topics = d.read_varint().unwrap(); + let _topic = d.read_compact_nullable_string().unwrap(); + let _parts = d.read_varint().unwrap(); + } else { + let _topics = d.read_i32().unwrap(); + let _topic = d.read_nullable_string().unwrap(); + let _parts = d.read_i32().unwrap(); + } + let _partition = d.read_i32().unwrap(); + assert_eq!( + d.read_i16().unwrap(), + ERROR_NOT_LEADER_OR_FOLLOWER, + "Produce v{version}" + ); + } +} + +#[test] +fn fetch_stub_response_has_zero_partition_error() { + for version in 4i16..=12 { + let Some(body) = load_fixture_body_or_skip(1, "Fetch", version) else { + continue; + }; + let resp = handle_request(API_KEY_FETCH, version, body, &default_broker()) + .expect_response("test request has acks != 0 and expects a response"); + let flexible = version >= 12; + let mut d = Decoder::new(resp); + if version >= 1 { + let _throttle = d.read_i32().unwrap(); + } + if version >= 7 { + assert_eq!(d.read_i16().unwrap(), ERROR_NONE); + let _session = d.read_i32().unwrap(); + } + if flexible { + let _topics = d.read_varint().unwrap(); + let _topic = d.read_compact_nullable_string().unwrap(); + let _parts = d.read_varint().unwrap(); + } else { + let _topics = d.read_i32().unwrap(); + let _topic = d.read_nullable_string().unwrap(); + let _parts = d.read_i32().unwrap(); + } + let _partition = d.read_i32().unwrap(); + assert_eq!( + d.read_i16().unwrap(), + ERROR_NONE, + "Fetch v{version} partition error" + ); + } +} + +#[test] +fn list_offsets_stub_response_has_zero_error() { + for version in 1i16..=6 { + let Some(body) = load_fixture_body_or_skip(2, "ListOffsets", version) else { + continue; + }; + let resp = handle_request(API_KEY_LIST_OFFSETS, version, body, &default_broker()) + .expect_response("test request has acks != 0 and expects a response"); + let flexible = version >= 6; + let mut d = Decoder::new(resp); + if version >= 2 { + let _throttle = d.read_i32().unwrap(); + } + if flexible { + let _topics = d.read_varint().unwrap(); + let _topic = d.read_compact_nullable_string().unwrap(); + let _parts = d.read_varint().unwrap(); + } else { + let _topics = d.read_i32().unwrap(); + let _topic = d.read_nullable_string().unwrap(); + let _parts = d.read_i32().unwrap(); + } + let _partition = d.read_i32().unwrap(); + assert_eq!(d.read_i16().unwrap(), ERROR_NONE, "ListOffsets v{version}"); + } +} + +// ── Metadata regression - all supported versions, broker advertise, topic counts ── + +/// Topic name assigned to slot `i` by [`metadata_request_legacy`] / [`metadata_request_flexible`]. +fn synthetic_topic_name(i: i32) -> String { + format!("topic-{i}") +} + +fn metadata_request_legacy(topic_count: i32) -> Bytes { + let mut enc = Encoder::with_capacity(64); + enc.write_i32(topic_count); + for i in 0..topic_count { + enc.write_nullable_string(Some(&synthetic_topic_name(i))) + .expect("topic name fits"); + } + enc.freeze() +} + +fn metadata_request_flexible(topic_count: usize) -> Bytes { + let mut enc = Encoder::with_capacity(64); + enc.write_varint((topic_count + 1) as u64); + for i in 0..topic_count { + enc.write_compact_nullable_string(Some(&synthetic_topic_name(i32::try_from(i).unwrap()))); + enc.write_empty_tagged_fields(); + } + enc.freeze() +} + +fn read_broker_legacy(d: &mut Decoder) -> (String, i32) { + let count = d.read_i32().unwrap(); + assert_eq!(count, 1); + let _node = d.read_i32().unwrap(); + let host = d.read_nullable_string().unwrap().unwrap(); + let port = d.read_i32().unwrap(); + (host, port) +} + +fn read_broker_flexible(d: &mut Decoder) -> (String, i32) { + let count_plus_one = d.read_varint().unwrap(); + assert_eq!(count_plus_one, 2); // one broker + let _node = d.read_i32().unwrap(); + let host = d.read_compact_nullable_string().unwrap().unwrap(); + let port = d.read_i32().unwrap(); + let _rack = d.read_compact_nullable_string().unwrap(); + d.read_tagged_fields().unwrap(); + (host, port) +} + +#[test] +fn metadata_corrupt_partial_body_returns_zero_topics() { + let body = handle_request( + API_KEY_METADATA, + 0, + Bytes::from_static(&[0x00, 0x00]), + &default_broker(), + ) + .expect_response("test request has acks != 0 and expects a response"); + let mut d = Decoder::new(body); + let _ = read_broker_legacy(&mut d); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.remaining(), 0); +} + +#[test] +fn metadata_v0_empty_topics_stub_broker() { + let body = handle_request( + API_KEY_METADATA, + 0, + metadata_request_legacy(0), + &default_broker(), + ) + .expect_response("test request has acks != 0 and expects a response"); + let mut d = Decoder::new(body); + let (host, port) = read_broker_legacy(&mut d); + assert_eq!(host, "127.0.0.1"); + assert_eq!(port, 9093); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!( + d.remaining(), + 0, + "empty request must produce no trailing bytes" + ); +} + +#[test] +fn metadata_v0_three_topics_each_unknown() { + let body = handle_request( + API_KEY_METADATA, + 0, + metadata_request_legacy(3), + &default_broker(), + ) + .expect_response("test request has acks != 0 and expects a response"); + let mut d = Decoder::new(body); + let _ = read_broker_legacy(&mut d); + assert_eq!(d.read_i32().unwrap(), 3); + for i in 0..3 { + assert_eq!(d.read_i16().unwrap(), ERROR_UNKNOWN_TOPIC_OR_PARTITION); + assert_eq!( + d.read_nullable_string().unwrap().unwrap(), + synthetic_topic_name(i) + ); + assert_eq!(d.read_i32().unwrap(), 0); + } +} + +#[test] +fn metadata_v1_includes_controller_id() { + let body = handle_request( + API_KEY_METADATA, + 1, + metadata_request_legacy(0), + &default_broker(), + ) + .expect_response("test request has acks != 0 and expects a response"); + let mut d = Decoder::new(body); + // Metadata v1 has no throttle_time_ms (added in v3). + let _ = read_broker_legacy(&mut d); + let _rack = d.read_nullable_string().unwrap(); + let controller = d.read_i32().unwrap(); + assert_eq!(controller, 1); +} + +#[test] +fn metadata_v2_includes_cluster_id_field() { + let body = handle_request( + API_KEY_METADATA, + 2, + metadata_request_legacy(0), + &default_broker(), + ) + .expect_response("test request has acks != 0 and expects a response"); + let mut d = Decoder::new(body); + let _ = read_broker_legacy(&mut d); + let _rack = d.read_nullable_string().unwrap(); + let _cluster_id = d.read_nullable_string().unwrap(); + let _controller = d.read_i32().unwrap(); + assert_eq!(d.read_i32().unwrap(), 0); +} + +#[test] +fn metadata_all_legacy_versions_produce_valid_response() { + for version in 0i16..=8 { + let body = handle_request( + API_KEY_METADATA, + version, + metadata_request_legacy(1), + &default_broker(), + ) + .expect_response("test request has acks != 0 and expects a response"); + let mut d = Decoder::new(body); + if version >= 3 { + let _throttle = d.read_i32().unwrap(); + } + let _ = read_broker_legacy(&mut d); + if version >= 1 { + let _rack = d.read_nullable_string().unwrap(); + } + if version >= 2 { + let _cluster = d.read_nullable_string().unwrap(); + } + if version >= 1 { + let _controller = d.read_i32().unwrap(); + } + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_i16().unwrap(), ERROR_UNKNOWN_TOPIC_OR_PARTITION); + } +} + +#[test] +fn metadata_v9_flexible_encoding() { + let body = handle_request( + API_KEY_METADATA, + 9, + metadata_request_flexible(2), + &default_broker(), + ) + .expect_response("test request has acks != 0 and expects a response"); + let mut d = Decoder::new(body); + let _throttle = d.read_i32().unwrap(); + let (host, port) = read_broker_flexible(&mut d); + assert_eq!(host, "127.0.0.1"); + assert_eq!(port, 9093); + let _cluster = d.read_compact_nullable_string().unwrap(); + let controller = d.read_i32().unwrap(); + assert_eq!(controller, 1); + + let topics_plus_one = d.read_varint().unwrap(); + assert_eq!(topics_plus_one, 3); // 2 topics + for i in 0..2 { + assert_eq!(d.read_i16().unwrap(), ERROR_UNKNOWN_TOPIC_OR_PARTITION); + assert_eq!( + d.read_compact_nullable_string().unwrap().unwrap(), + synthetic_topic_name(i) + ); + let _internal = d.read_bool().unwrap(); + let parts_plus_one = d.read_varint().unwrap(); + assert_eq!(parts_plus_one, 1); // empty partitions + assert_eq!(d.read_i32().unwrap(), i32::MIN); // topic_authorized_operations (v8+) + d.read_tagged_fields().unwrap(); + } + assert_eq!(d.read_i32().unwrap(), i32::MIN); // cluster_authorized_operations (v8+) + d.read_tagged_fields().unwrap(); + assert_eq!(d.remaining(), 0); +} + +#[test] +fn metadata_v8_includes_authorized_operations_legacy() { + let body = handle_request( + API_KEY_METADATA, + 8, + metadata_request_legacy(1), + &default_broker(), + ) + .expect_response("test request has acks != 0 and expects a response"); + let mut d = Decoder::new(body); + let _throttle = d.read_i32().unwrap(); + let _ = read_broker_legacy(&mut d); + let _rack = d.read_nullable_string().unwrap(); + let _cluster = d.read_nullable_string().unwrap(); + let _controller = d.read_i32().unwrap(); + assert_eq!(d.read_i32().unwrap(), 1); + let _topic_error = d.read_i16().unwrap(); + let _topic = d.read_nullable_string().unwrap(); + let _internal = d.read_bool().unwrap(); + assert_eq!(d.read_i32().unwrap(), 0); // empty partitions + assert_eq!(d.read_i32().unwrap(), i32::MIN); // topic_authorized_operations + assert_eq!(d.read_i32().unwrap(), i32::MIN); // cluster_authorized_operations + assert_eq!(d.remaining(), 0); +} + +#[test] +fn create_topics_stub_response_returns_not_controller() { + for version in 2i16..=5 { + let Some(body) = load_fixture_body_or_skip(19, "CreateTopics", version) else { + continue; + }; + let resp = handle_request(API_KEY_CREATE_TOPICS, version, body, &default_broker()) + .expect_response("test request has acks != 0 and expects a response"); + let flexible = version >= 5; + let mut d = Decoder::new(resp); + if version >= 2 { + let _throttle = d.read_i32().unwrap(); + } + if flexible { + let _topics = d.read_varint().unwrap(); + let _topic = d.read_compact_nullable_string().unwrap(); + } else { + let _topics = d.read_i32().unwrap(); + let _topic = d.read_nullable_string().unwrap(); + } + assert_eq!( + d.read_i16().unwrap(), + ERROR_NOT_CONTROLLER, + "CreateTopics v{version}" + ); + } +} + +// ── Produce acks=0 (broker must stay silent even on a malformed body) ────── + +#[test] +fn produce_acks_zero_malformed_body_decode_carries_acks() { + let body = build_produce_v3_body(0, 1); + match decode_produce_request(3, body.clone()) { + ProduceDecodeResult::Err { acks: Some(0), .. } => {} + other => panic!("expected decode error with acks=0, got {other:?}"), + } + assert!( + handle_request(API_KEY_PRODUCE, 3, body, &default_broker()).is_no_response(), + "handler must not respond when acks=0 even if decode fails after acks" + ); +} + +// ── Metadata topic name echo (must not hardcode a placeholder topic name) ── + +fn read_metadata_v1_topics(d: &mut Decoder, expected_count: i32) -> Vec { + let _brokers_count = d.read_i32().unwrap(); + d.read_i32().unwrap(); // node_id + d.read_nullable_string().unwrap(); // host + d.read_i32().unwrap(); // port + d.read_nullable_string().unwrap(); // rack (v1+) + d.read_i32().unwrap(); // controller_id (v1+) + + assert_eq!(d.read_i32().unwrap(), expected_count); + let mut names = Vec::with_capacity(usize::try_from(expected_count).unwrap_or(0)); + for _ in 0..expected_count { + d.read_i16().unwrap(); // topic_error + names.push(d.read_nullable_string().unwrap().expect("topic name")); + d.read_bool().unwrap(); // is_internal (v1+) + assert_eq!(d.read_i32().unwrap(), 0, "empty partitions array"); + } + names +} + +#[test] +fn metadata_v1_echoes_requested_topic_name_in_response() { + let topic = "orders"; + let request = build_metadata_legacy_request(&[topic]); + let body = handle_request(API_KEY_METADATA, 1, request, &default_broker()) + .expect_response("test request has acks != 0 and expects a response"); + let mut d = Decoder::new(body); + + let names = read_metadata_v1_topics(&mut d, 1); + assert_eq!(names, vec![topic.to_string()]); + assert_eq!(d.remaining(), 0); +} + +#[test] +fn metadata_v1_unknown_topic_returns_error_with_requested_name() { + let topic = "orders"; + let request = build_metadata_legacy_request(&[topic]); + let body = handle_request(API_KEY_METADATA, 1, request, &default_broker()) + .expect_response("test request has acks != 0 and expects a response"); + let mut d = Decoder::new(body); + + let _brokers_count = d.read_i32().unwrap(); + d.read_i32().unwrap(); + d.read_nullable_string().unwrap(); + d.read_i32().unwrap(); + d.read_nullable_string().unwrap(); + d.read_i32().unwrap(); + + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!( + d.read_i16().unwrap(), + ERROR_UNKNOWN_TOPIC_OR_PARTITION, + "unknown topic should surface error 3" + ); + assert_eq!( + d.read_nullable_string().unwrap().as_deref(), + Some(topic), + "response must echo requested topic name, not a placeholder" + ); +} + +// ── Metadata (spec + SCOPE.md coverage) ───────────────────────────────────── + +#[test] +fn metadata_v0_empty_topics_returns_zero_length_topic_array() { + let body = handle_request( + API_KEY_METADATA, + 0, + build_metadata_legacy_request(&[]), + &default_broker(), + ) + .expect_response("test request has acks != 0 and expects a response"); + let mut d = Decoder::new(body); + let _brokers = d.read_i32().unwrap(); + d.read_i32().unwrap(); + d.read_nullable_string().unwrap(); + d.read_i32().unwrap(); + assert_eq!(d.read_i32().unwrap(), 0, "empty request → zero topics"); + assert_eq!(d.remaining(), 0); +} + +#[test] +fn metadata_v3_includes_throttle_time_ms_before_brokers() { + let body = handle_request( + API_KEY_METADATA, + 3, + build_metadata_legacy_request(&[]), + &default_broker(), + ) + .expect_response("test request has acks != 0 and expects a response"); + let mut d = Decoder::new(body); + assert_eq!(d.read_i32().unwrap(), 0, "throttle_time_ms"); +} + +#[test] +fn metadata_v9_flexible_empty_topics_returns_zero_topics() { + let body = handle_request( + API_KEY_METADATA, + 9, + build_metadata_flexible_request(&[]), + &default_broker(), + ) + .expect_response("test request has acks != 0 and expects a response"); + let mut d = Decoder::new(body); + d.read_i32().unwrap(); // throttle + let broker_count = usize::try_from(d.read_varint().unwrap()) + .unwrap() + .saturating_sub(1); + for _ in 0..broker_count { + d.read_i32().unwrap(); + d.read_compact_nullable_string().unwrap(); + d.read_i32().unwrap(); + d.read_compact_nullable_string().unwrap(); + d.read_tagged_fields().unwrap(); + } + d.read_compact_nullable_string().unwrap(); // cluster_id + d.read_i32().unwrap(); // controller_id + let topic_count = usize::try_from(d.read_varint().unwrap()) + .unwrap() + .saturating_sub(1); + assert_eq!(topic_count, 0); +} + +#[test] +fn metadata_v9_flexible_echoes_each_requested_topic_name() { + let topics = ["orders", "payments", "inventory"]; + let body = handle_request( + API_KEY_METADATA, + 9, + build_metadata_flexible_request(&topics), + &default_broker(), + ) + .expect_response("test request has acks != 0 and expects a response"); + + let mut d = Decoder::new(body); + d.read_i32().unwrap(); + let broker_count = usize::try_from(d.read_varint().unwrap()) + .unwrap() + .saturating_sub(1); + for _ in 0..broker_count { + d.read_i32().unwrap(); + d.read_compact_nullable_string().unwrap(); + d.read_i32().unwrap(); + d.read_compact_nullable_string().unwrap(); + d.read_tagged_fields().unwrap(); + } + d.read_compact_nullable_string().unwrap(); + d.read_i32().unwrap(); + + let topic_count = usize::try_from(d.read_varint().unwrap()) + .unwrap() + .saturating_sub(1); + assert_eq!(topic_count, topics.len()); + + let mut names = Vec::new(); + for _ in 0..topic_count { + assert_eq!( + d.read_i16().unwrap(), + ERROR_UNKNOWN_TOPIC_OR_PARTITION, + "stub gateway returns unknown topic error per topic" + ); + names.push( + d.read_compact_nullable_string() + .unwrap() + .expect("topic name"), + ); + d.read_bool().unwrap(); + assert_eq!( + usize::try_from(d.read_varint().unwrap()) + .unwrap() + .saturating_sub(1), + 0, + "empty partitions array" + ); + d.read_i32().unwrap(); // topic_authorized_operations (v8+) + d.read_tagged_fields().unwrap(); + } + + assert_eq!( + names, + topics + .iter() + .map(|topic| (*topic).to_string()) + .collect::>(), + "metadata must echo requested topic names for client matching" + ); +} + +#[test] +fn metadata_v1_legacy_multiple_topics_echo_names() { + let topics = ["alpha", "beta"]; + let body = handle_request( + API_KEY_METADATA, + 1, + build_metadata_legacy_request(&topics), + &default_broker(), + ) + .expect_response("test request has acks != 0 and expects a response"); + + let mut d = Decoder::new(body); + d.read_i32().unwrap(); + d.read_i32().unwrap(); + d.read_nullable_string().unwrap(); + d.read_i32().unwrap(); + d.read_nullable_string().unwrap(); + d.read_i32().unwrap(); + assert_eq!(d.read_i32().unwrap(), 2); + + for expected in topics { + assert_eq!(d.read_i16().unwrap(), ERROR_UNKNOWN_TOPIC_OR_PARTITION); + assert_eq!( + d.read_nullable_string().unwrap().as_deref(), + Some(expected), + "metadata v1 must echo {expected}" + ); + d.read_bool().unwrap(); + assert_eq!(d.read_i32().unwrap(), 0); + } +} + +fn skip_metadata_v9_prefix(d: &mut Decoder) { + let broker_count = usize::try_from(d.read_varint().unwrap()) + .unwrap() + .saturating_sub(1); + for _ in 0..broker_count { + d.read_i32().unwrap(); + d.read_compact_nullable_string().unwrap(); + d.read_i32().unwrap(); + d.read_compact_nullable_string().unwrap(); + d.read_tagged_fields().unwrap(); + } + d.read_compact_nullable_string().unwrap(); + d.read_i32().unwrap(); +} + +#[test] +fn metadata_v9_request_with_three_topics_yields_three_response_slots() { + let topics = ["a", "b", "c"]; + let body = handle_request( + API_KEY_METADATA, + 9, + build_metadata_flexible_request(&topics), + &default_broker(), + ) + .expect_response("test request has acks != 0 and expects a response"); + let mut d = Decoder::new(body); + d.read_i32().unwrap(); + skip_metadata_v9_prefix(&mut d); + let topic_count = usize::try_from(d.read_varint().unwrap()) + .unwrap() + .saturating_sub(1); + assert_eq!( + topic_count, + topics.len(), + "response topic count must mirror request topic count" + ); +} diff --git a/gateways/kafka/tests/common/fixtures.rs b/gateways/kafka/tests/common/fixtures.rs index a94127ca92..814e9f8dca 100644 --- a/gateways/kafka/tests/common/fixtures.rs +++ b/gateways/kafka/tests/common/fixtures.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -//! Fixture loaders — compiled into each integration test binary via `#[path]`. +//! Fixture loaders - compiled into each integration test binary via `#[path]`. #![allow(dead_code)] use std::path::PathBuf; @@ -46,12 +46,22 @@ pub fn load_fixture_body(api_key: i16, api_name: &str, version: i16) -> Bytes { /// generation script so a fresh clone knows how to produce it. pub const FIXTURE_SKIP_HINT: &str = "generate with `gateways/kafka/scripts/ci-wire-fixtures.sh generate` (or the kafka-tool `generate` subcommand)"; +/// Set by CI (after `ci-wire-fixtures.sh generate`) to turn missing-fixture skips into hard +/// failures, so a broken generation step cannot leave these suites green with zero assertions. +const FIXTURES_REQUIRED_ENV: &str = "KAFKA_FIXTURES_REQUIRED"; + /// Load a fixture body, or return `None` after printing a standardized skip note when the /// fixture is missing. Unifies the missing-fixture policy across suites: every suite skips -/// explicitly instead of panicking or silently passing with zero assertions. +/// explicitly instead of panicking or silently passing with zero assertions. Panics instead of +/// skipping when `KAFKA_FIXTURES_REQUIRED` is set, so CI catches a silently broken fixture +/// generation step rather than passing an empty suite. pub fn load_fixture_body_or_skip(api_key: i16, api_name: &str, version: i16) -> Option { if fixture_exists(api_key, api_name, version) { Some(load_fixture_body(api_key, api_name, version)) + } else if std::env::var(FIXTURES_REQUIRED_ENV).is_ok_and(|v| v == "1") { + panic!( + "missing required fixture {api_key:03}_{api_name}_v{version}.bin ({FIXTURES_REQUIRED_ENV}=1 set): {FIXTURE_SKIP_HINT}" + ); } else { eprintln!("skipping {api_key:03}_{api_name}_v{version}.bin: {FIXTURE_SKIP_HINT}"); None diff --git a/gateways/kafka/tests/common/scope.rs b/gateways/kafka/tests/common/scope.rs index 61a3d37f51..55b1361d79 100644 --- a/gateways/kafka/tests/common/scope.rs +++ b/gateways/kafka/tests/common/scope.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -//! Shared scope constants — compiled into each integration test binary via `#[path]`. +//! Shared scope constants - compiled into each integration test binary via `#[path]`. #![allow(dead_code)] use iggy_gateway_kafka::protocol::api::BrokerAdvertise; diff --git a/gateways/kafka/tests/common/server.rs b/gateways/kafka/tests/common/server.rs index e28c28a409..ea7891a19d 100644 --- a/gateways/kafka/tests/common/server.rs +++ b/gateways/kafka/tests/common/server.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -//! Test server spawn helper — compiled into each integration test binary via `#[path]`. +//! Test server spawn helper - compiled into each integration test binary via `#[path]`. #![allow(dead_code)] use std::net::SocketAddr; diff --git a/gateways/kafka/tests/common/tcp.rs b/gateways/kafka/tests/common/tcp.rs index d8f94117d9..bd75a13483 100644 --- a/gateways/kafka/tests/common/tcp.rs +++ b/gateways/kafka/tests/common/tcp.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -//! TCP round-trip helpers — compiled into each integration test binary via `#[path]`. +//! TCP round-trip helpers - compiled into each integration test binary via `#[path]`. #![allow(dead_code)] use std::net::SocketAddr; @@ -119,7 +119,21 @@ pub fn build_produce_v3_body(acks: i16, topics_count: i32) -> Bytes { body.freeze() } -/// Legacy Metadata request body listing topic names (non-flexible, v0–v8). +/// `ListOffsets` v0 request body for topic "t", partition 0. +pub fn build_list_offsets_v0_request_with_topic_t() -> Bytes { + let mut body = BytesMut::new(); + body.put_i32(-1); // replica_id + body.put_i32(1); // topics array length + body.put_i16(1); // topic name length + body.put_u8(b't'); + body.put_i32(1); // partitions array length + body.put_i32(0); // partition index + body.put_i64(-1); // timestamp + body.put_i32(1); // max_num_offsets + body.freeze() +} + +/// Legacy Metadata request body listing topic names (non-flexible, v0-v8). pub fn build_metadata_legacy_request(topic_names: &[&str]) -> Bytes { let mut body = BytesMut::new(); body.put_i32(i32::try_from(topic_names.len()).expect("topic name count fits i32")); @@ -177,6 +191,14 @@ pub async fn read_byte_with_timeout(stream: &mut TcpStream, timeout: Duration) - } } +/// Scan a response body for a big-endian `i16` error code at any 2-byte-aligned +/// offset. Used by corrupt-body / unsupported-version tests that assert an error +/// code is present somewhere in the response without fully decoding its shape. +pub fn scan_for_error_code(body: &Bytes, code: i16) -> bool { + body.windows(2) + .any(|w| i16::from_be_bytes([w[0], w[1]]) == code) +} + /// Send one request frame and return parsed `(correlation_id, response_body)`. pub async fn round_trip( addr: SocketAddr, diff --git a/gateways/kafka/tests/decode_safety_tests.rs b/gateways/kafka/tests/decode_safety_tests.rs index 531cf9b3f1..4d047e1426 100644 --- a/gateways/kafka/tests/decode_safety_tests.rs +++ b/gateways/kafka/tests/decode_safety_tests.rs @@ -15,13 +15,19 @@ // specific language governing permissions and limitations // under the License. -//! Adversarial wire-input tests for #3421 — malformed lengths must return errors, never panic. +//! Adversarial wire-input tests for #3421 - malformed lengths must return errors, never panic. + +#[path = "common/wire.rs"] +mod wire; use bytes::Bytes; use iggy_gateway_kafka::error::KafkaProtocolError; use iggy_gateway_kafka::protocol::codec::{Decoder, Encoder, MAX_COLLECTION_LEN}; -use iggy_gateway_kafka::protocol::requests::decode_produce_request; +use iggy_gateway_kafka::protocol::requests::{ + ProduceDecodeResult, decode_create_topics_request, decode_fetch_request, + decode_list_offsets_request, decode_produce_request, +}; #[test] fn compact_array_varint_zero_decodes_as_empty_without_panic() { @@ -81,3 +87,220 @@ fn varint_terminal_byte_with_extra_bits_at_shift_63_is_rejected() { let err = d.read_varint().unwrap_err(); assert!(matches!(err, KafkaProtocolError::InvalidVarint)); } + +// ── Produce: error preserves acks (so a retry decision can honor acks=0) ──── + +#[test] +fn produce_null_topic_name_preserves_acks_on_error() { + let mut enc = Encoder::with_capacity(32); + enc.write_nullable_string(None::<&str>).unwrap(); + enc.write_i16(1); + enc.write_i32(500); + enc.write_i32(1); + enc.write_nullable_string(None::<&str>).unwrap(); + + match decode_produce_request(3, enc.freeze()) { + ProduceDecodeResult::Err { acks, error } => { + assert_eq!(acks, Some(1)); + assert!(matches!(error, KafkaProtocolError::NullTopicName)); + } + ProduceDecodeResult::Ok(_) => panic!("expected NullTopicName"), + } +} + +#[test] +fn produce_v3_error_before_acks_has_none_acks() { + let mut enc = Encoder::with_capacity(8); + enc.write_i16(1); + match decode_produce_request(3, enc.freeze()) { + ProduceDecodeResult::Err { acks, .. } => assert_eq!(acks, None), + ProduceDecodeResult::Ok(_) => panic!("expected decode error before acks"), + } +} + +#[test] +fn produce_v3_error_after_acks_preserves_acks() { + let mut enc = Encoder::with_capacity(16); + enc.write_nullable_string(None::<&str>).unwrap(); + enc.write_i16(7); + match decode_produce_request(3, enc.freeze()) { + ProduceDecodeResult::Err { acks, .. } => assert_eq!(acks, Some(7)), + ProduceDecodeResult::Ok(_) => panic!("expected decode error after acks"), + } +} + +#[test] +fn produce_v3_error_after_timeout_preserves_acks() { + let mut enc = Encoder::with_capacity(16); + enc.write_nullable_string(None::<&str>).unwrap(); + enc.write_i16(1); + enc.write_i32(500); + match decode_produce_request(3, enc.freeze()) { + ProduceDecodeResult::Err { acks, .. } => assert_eq!(acks, Some(1)), + ProduceDecodeResult::Ok(_) => panic!("expected decode error after timeout"), + } +} + +#[test] +fn produce_v9_error_on_null_topic_preserves_acks() { + let mut enc = Encoder::with_capacity(32); + enc.write_compact_nullable_string(None); + enc.write_i16(2); + enc.write_i32(500); + enc.write_varint(2); + enc.write_compact_nullable_string(None); + match decode_produce_request(9, enc.freeze()) { + ProduceDecodeResult::Err { acks, error } => { + assert_eq!(acks, Some(2)); + assert!(matches!(error, KafkaProtocolError::NullTopicName)); + } + ProduceDecodeResult::Ok(_) => panic!("expected NullTopicName"), + } +} + +#[test] +fn produce_v9_error_on_partition_count_preserves_acks() { + let mut enc = Encoder::with_capacity(64); + enc.write_compact_nullable_string(None); + enc.write_i16(3); + enc.write_i32(500); + enc.write_varint(2); + enc.write_compact_nullable_string(Some("topic")); + match decode_produce_request(9, enc.freeze()) { + ProduceDecodeResult::Err { acks, .. } => assert_eq!(acks, Some(3)), + ProduceDecodeResult::Ok(_) => panic!("expected decode error in partition count"), + } +} + +#[test] +fn produce_v9_error_on_partition_records_preserves_acks() { + let mut enc = Encoder::with_capacity(64); + enc.write_compact_nullable_string(None); + enc.write_i16(4); + enc.write_i32(500); + enc.write_varint(2); + enc.write_compact_nullable_string(Some("topic")); + enc.write_varint(2); + enc.write_i32(0); + match decode_produce_request(9, enc.freeze()) { + ProduceDecodeResult::Err { acks, .. } => assert_eq!(acks, Some(4)), + ProduceDecodeResult::Ok(_) => panic!("expected decode error in records"), + } +} + +// ── Fetch: truncated / null-topic inputs return errors, never panic ──────── + +#[test] +fn fetch_v4_truncated_after_replica_id_returns_error() { + let mut enc = Encoder::with_capacity(4); + enc.write_i32(-1); + let err = decode_fetch_request(4, enc.freeze()).unwrap_err(); + assert!(matches!(err, KafkaProtocolError::BufferUnderflow { .. })); +} + +#[test] +fn fetch_v7_truncated_in_forgotten_topics_returns_error() { + let body = wire::build_fetch_request_with_sections(7, "topic", 0, Some("forgot"), None); + let truncated = body.slice(..body.len() - 2); + let err = decode_fetch_request(7, truncated).unwrap_err(); + assert!(matches!(err, KafkaProtocolError::BufferUnderflow { .. })); +} + +#[test] +fn fetch_v12_flexible_truncated_in_topic_tagged_fields_returns_error() { + let body = wire::build_fetch_request_with_sections(12, "topic", 0, None, None); + let truncated = body.slice(..body.len() - 1); + let err = decode_fetch_request(12, truncated).unwrap_err(); + assert!(matches!(err, KafkaProtocolError::BufferUnderflow { .. })); +} + +#[test] +fn fetch_v12_flexible_null_topic_name_returns_error() { + let mut enc = Encoder::with_capacity(64); + enc.write_i32(-1); + enc.write_i32(100); + enc.write_i32(1); + enc.write_i32(1024); + enc.write_i8(0); + enc.write_i32(0); + enc.write_i32(0); + enc.write_varint(2); + enc.write_compact_nullable_string(None); + let err = decode_fetch_request(12, enc.freeze()).unwrap_err(); + assert!(matches!(err, KafkaProtocolError::NullTopicName)); +} + +#[test] +fn fetch_null_topic_name_returns_error() { + let mut enc = Encoder::with_capacity(64); + enc.write_i32(-1); + enc.write_i32(100); + enc.write_i32(1); + enc.write_i32(i32::MAX); + enc.write_i8(0); + enc.write_i32(1); + enc.write_nullable_string(None::<&str>).unwrap(); + + let err = decode_fetch_request(4, enc.freeze()).unwrap_err(); + assert!(matches!(err, KafkaProtocolError::NullTopicName)); +} + +// ── ListOffsets: truncated / null-topic inputs return errors, never panic ── + +#[test] +fn list_offsets_v4_truncated_in_leader_epoch_returns_error() { + let body = wire::build_list_offsets_branch_request(4, "topic", 1); + let truncated = body.slice(..body.len() - 4); + let err = decode_list_offsets_request(4, truncated).unwrap_err(); + assert!(matches!(err, KafkaProtocolError::BufferUnderflow { .. })); +} + +#[test] +fn list_offsets_v6_flexible_null_topic_name_returns_error() { + let mut enc = Encoder::with_capacity(32); + enc.write_i32(-1); + enc.write_i8(0); + enc.write_varint(2); + enc.write_compact_nullable_string(None); + let err = decode_list_offsets_request(6, enc.freeze()).unwrap_err(); + assert!(matches!(err, KafkaProtocolError::NullTopicName)); +} + +#[test] +fn list_offsets_null_topic_name_returns_error() { + let mut enc = Encoder::with_capacity(32); + enc.write_i32(-1); + enc.write_i8(0); + enc.write_i32(1); + enc.write_nullable_string(None::<&str>).unwrap(); + let err = decode_list_offsets_request(2, enc.freeze()).unwrap_err(); + assert!(matches!(err, KafkaProtocolError::NullTopicName)); +} + +// ── CreateTopics: truncated / null-topic inputs return errors, never panic ─ + +#[test] +fn create_topics_v2_truncated_in_config_value_returns_error() { + let body = wire::build_create_topics_request_with_sections(2, "topic"); + let truncated = body.slice(..body.len() - 3); + let err = decode_create_topics_request(2, truncated).unwrap_err(); + assert!(matches!(err, KafkaProtocolError::BufferUnderflow { .. })); +} + +#[test] +fn create_topics_v5_flexible_null_topic_name_returns_error() { + let mut enc = Encoder::with_capacity(16); + enc.write_varint(2); + enc.write_compact_nullable_string(None); + let err = decode_create_topics_request(5, enc.freeze()).unwrap_err(); + assert!(matches!(err, KafkaProtocolError::NullTopicName)); +} + +#[test] +fn create_topics_null_topic_name_returns_error() { + let mut enc = Encoder::with_capacity(32); + enc.write_i32(1); + enc.write_nullable_string(None::<&str>).unwrap(); + let err = decode_create_topics_request(2, enc.freeze()).unwrap_err(); + assert!(matches!(err, KafkaProtocolError::NullTopicName)); +} diff --git a/gateways/kafka/tests/decode_validation_tests.rs b/gateways/kafka/tests/decode_validation_tests.rs index 9a69341741..10e6d38208 100644 --- a/gateways/kafka/tests/decode_validation_tests.rs +++ b/gateways/kafka/tests/decode_validation_tests.rs @@ -29,7 +29,7 @@ use std::path::PathBuf; use bytes::Bytes; -use iggy_gateway_kafka::protocol::codec::Decoder; +use iggy_gateway_kafka::protocol::codec::{Decoder, Encoder}; use iggy_gateway_kafka::protocol::header::{RequestHeader, request_header_version}; use iggy_gateway_kafka::protocol::requests::{ decode_create_topics_request, decode_fetch_request, decode_list_offsets_request, @@ -57,7 +57,7 @@ fn load_body(api_key: i16, api_name: &str, version: i16) -> Option { let path = fixtures_dir().join(&filename); let Ok(data) = std::fs::read(&path) else { eprintln!( - "skipping {filename}: wire fixture missing — generate with \ + "skipping {filename}: wire fixture missing - generate with \ `gateways/kafka/scripts/ci-wire-fixtures.sh generate` (or the kafka-tool \ `generate` subcommand)" ); @@ -149,12 +149,12 @@ fn produce_response_v3_roundtrip() { let partition = d.read_i32().unwrap(); assert_eq!(partition, 0); let error_code = d.read_i16().unwrap(); - assert_eq!(error_code, 6); // NOT_LEADER_OR_FOLLOWER — stub until Iggy bridge + assert_eq!(error_code, 6); // NOT_LEADER_OR_FOLLOWER - stub until Iggy bridge let base_offset = d.read_i64().unwrap(); assert_eq!(base_offset, 0); // log_append_time_ms (v2+) let _log_append = d.read_i64().unwrap(); - // log_start_offset (v5+) — not present for v3 + // log_start_offset (v5+) - not present for v3 let throttle = d.read_i32().unwrap(); assert_eq!(throttle, 0); } @@ -176,7 +176,7 @@ fn produce_response_v8_includes_record_errors() { assert_eq!(partition_count, 1); let _partition = d.read_i32().unwrap(); let error_code = d.read_i16().unwrap(); - assert_eq!(error_code, 6); // NOT_LEADER_OR_FOLLOWER — stub until Iggy bridge + assert_eq!(error_code, 6); // NOT_LEADER_OR_FOLLOWER - stub until Iggy bridge let _base_offset = d.read_i64().unwrap(); let _log_append_time = d.read_i64().unwrap(); // v2+ let _log_start_offset = d.read_i64().unwrap(); // v5+ @@ -198,6 +198,61 @@ fn produce_v9_flexible_empty_topics_decode() { assert_eq!(req.topics.len(), 0); } +#[test] +fn produce_v2_skips_transactional_id_branch() { + let req = decode_produce_request(2, wire::build_produce_legacy_request(2, 1, None, None)) + .into_request() + .expect("produce v2 should decode"); + assert_eq!(req.acks, 1); + assert!(req.transactional_id.is_none()); + assert!(req.topics.is_empty()); +} + +#[test] +fn produce_v3_legacy_transactional_id_and_topic_decode() { + let req = decode_produce_request( + 3, + wire::build_produce_legacy_request(3, -1, Some("txn-1"), Some("legacy-topic")), + ) + .into_request() + .expect("produce v3 legacy should decode"); + assert_eq!(req.transactional_id.as_deref(), Some("txn-1")); + assert_eq!(req.topics.len(), 1); + assert_eq!(req.topics[0].topic, "legacy-topic"); + assert!(req.topics[0].partitions[0].records.is_some()); +} + +#[test] +fn produce_v8_legacy_null_records_decode() { + let mut enc = Encoder::with_capacity(64); + enc.write_nullable_string(None::<&str>).unwrap(); + enc.write_i16(1); + enc.write_i32(500); + enc.write_i32(1); + enc.write_nullable_string(Some("topic")).unwrap(); + enc.write_i32(1); + enc.write_i32(0); + enc.write_nullable_bytes(None).unwrap(); + + let req = decode_produce_request(8, enc.freeze()) + .into_request() + .expect("produce v8 with null records should decode"); + assert!(req.topics[0].partitions[0].records.is_none()); +} + +#[test] +fn produce_v9_flexible_transactional_id_and_tagged_fields_decode() { + let req = decode_produce_request( + 9, + wire::build_produce_flexible_request_with_topic("flex-topic"), + ) + .into_request() + .expect("produce v9 flexible should decode"); + assert_eq!(req.transactional_id.as_deref(), Some("txn-1")); + assert_eq!(req.topics[0].topic, "flex-topic"); + assert!(req.topics[0].partitions[0].records.is_some()); +} + // ── Fetch (API key 1) ───────────────────────────────────────────────────────── #[test] @@ -300,6 +355,54 @@ fn fetch_v12_decodes_forgotten_topics_and_rack_id_sections() { assert_eq!(req.topics[0].partitions[0].partition_max_bytes, 1024); } +#[test] +fn fetch_v2_uses_default_max_bytes_when_field_absent() { + let req = decode_fetch_request(2, wire::build_fetch_v2_default_max_bytes_request()) + .expect("fetch v2 should decode"); + assert_eq!(req.max_bytes, 52_428_800); + assert_eq!(req.isolation_level, 0); + assert!(req.topics.is_empty()); +} + +#[test] +fn fetch_v7_legacy_forgotten_topics_and_rack_id_decode() { + let req = decode_fetch_request( + 7, + wire::build_fetch_request_with_sections(7, "topic-a", 1, Some("forgotten"), Some("rack-1")), + ) + .expect("fetch v7 legacy sections should decode"); + assert_eq!(req.topics[0].topic, "topic-a"); + assert_eq!(req.topics[0].partitions[0].partition, 1); +} + +#[test] +fn fetch_v9_leader_epoch_without_v12_fields_decode() { + let req = decode_fetch_request( + 9, + wire::build_fetch_request_with_sections(9, "topic-b", 2, None, None), + ) + .expect("fetch v9 should decode"); + assert_eq!(req.topics[0].partitions[0].fetch_offset, 42); +} + +#[test] +fn fetch_v11_legacy_rack_id_decode() { + let req = decode_fetch_request( + 11, + wire::build_fetch_request_with_sections(11, "topic-c", 3, None, Some("rack-z")), + ) + .expect("fetch v11 legacy rack id should decode"); + assert_eq!(req.max_wait_ms, 100); +} + +#[test] +fn fetch_v3_skips_isolation_level_field() { + let req = decode_fetch_request(3, wire::build_fetch_v3_no_isolation_request()) + .expect("fetch v3 should decode"); + assert_eq!(req.isolation_level, 0); + assert_eq!(req.max_bytes, 1024); +} + // ── ListOffsets (API key 2) ─────────────────────────────────────────────────── #[test] @@ -391,7 +494,7 @@ fn list_offsets_response_v1_no_leader_epoch() { assert_eq!(error_code, 0); let _timestamp = d.read_i64().unwrap(); // v1+ let _offset = d.read_i64().unwrap(); - // v1 must NOT have a leader_epoch field — assert all bytes consumed + // v1 must NOT have a leader_epoch field - assert all bytes consumed assert_eq!( d.remaining(), 0, @@ -425,6 +528,34 @@ fn list_offsets_response_v4_has_leader_epoch() { assert_eq!(d.remaining(), 0); } +#[test] +fn list_offsets_v1_skips_isolation_level_field() { + let req = decode_list_offsets_request(1, wire::build_list_offsets_branch_request(1, "v1", 0)) + .expect("list offsets v1 should decode"); + assert_eq!(req.isolation_level, 0); +} + +#[test] +fn list_offsets_v2_reads_isolation_level() { + let req = decode_list_offsets_request(2, wire::build_list_offsets_branch_request(2, "v2", 1)) + .expect("list offsets v2 should decode"); + assert_eq!(req.isolation_level, 1); +} + +#[test] +fn list_offsets_v3_skips_leader_epoch_branch() { + let req = decode_list_offsets_request(3, wire::build_list_offsets_branch_request(3, "v3", 2)) + .expect("list offsets v3 should decode"); + assert_eq!(req.topics[0].partitions[0].partition, 2); +} + +#[test] +fn list_offsets_v5_leader_epoch_without_flexible_encoding() { + let req = decode_list_offsets_request(5, wire::build_list_offsets_branch_request(5, "v5", 3)) + .expect("list offsets v5 should decode"); + assert_eq!(req.topics[0].topic, "v5"); +} + // ── CreateTopics (API key 19) ───────────────────────────────────────────────── #[test] @@ -523,7 +654,7 @@ fn create_topics_response_v2_roundtrip() { let resp_topic = d.read_nullable_string().unwrap().unwrap(); assert_eq!(resp_topic, topic_name); let error_code = d.read_i16().unwrap(); - assert_eq!(error_code, 41); // NOT_CONTROLLER — stub until Iggy bridge + assert_eq!(error_code, 41); // NOT_CONTROLLER - stub until Iggy bridge let error_msg = d.read_nullable_string().unwrap(); // v1+ assert!(error_msg.is_none()); assert_eq!(d.remaining(), 0); @@ -545,7 +676,7 @@ fn create_topics_response_v5_roundtrip() { let _topic_name = d.read_compact_nullable_string().unwrap(); let error_code = d.read_i16().unwrap(); - assert_eq!(error_code, 41); // NOT_CONTROLLER — stub until Iggy bridge + assert_eq!(error_code, 41); // NOT_CONTROLLER - stub until Iggy bridge let _error_msg = d.read_compact_nullable_string().unwrap(); // v1+ let num_partitions = d.read_i32().unwrap(); assert_eq!(num_partitions, 1); @@ -557,3 +688,24 @@ fn create_topics_response_v5_roundtrip() { d.read_tagged_fields().unwrap(); // top-level tagged_fields assert_eq!(d.remaining(), 0); } + +#[test] +fn create_topics_v3_legacy_assignments_decode() { + let req = decode_create_topics_request( + 3, + wire::build_create_topics_request_with_sections(3, "v3-topic"), + ) + .expect("create topics v3 should decode"); + assert_eq!(req.topics[0].name, "v3-topic"); + assert!(req.validate_only); +} + +#[test] +fn create_topics_v4_legacy_configs_decode() { + let req = decode_create_topics_request( + 4, + wire::build_create_topics_request_with_sections(4, "v4-topic"), + ) + .expect("create topics v4 should decode"); + assert_eq!(req.topics[0].replication_factor, 1); +} diff --git a/gateways/kafka/tests/golden_wire_fixtures_tests.rs b/gateways/kafka/tests/golden_wire_fixtures_tests.rs index a2177b37b0..dad3fc4e98 100644 --- a/gateways/kafka/tests/golden_wire_fixtures_tests.rs +++ b/gateways/kafka/tests/golden_wire_fixtures_tests.rs @@ -62,7 +62,7 @@ fn golden_metadata_v0_single_topic_response_fixture() { let actual = handle_request(API_KEY_METADATA, 0, req_bytes, &BrokerAdvertise::default()) .expect_response("test request has acks != 0 and expects a response"); - // Metadata v0 layout: brokers[], topics[] (no controller_id — added in v1) + // Metadata v0 layout: brokers[], topics[] (no controller_id - added in v1) // brokers[1]: node_id=1, host=127.0.0.1, port=9093 // topics[1]: topic_error=3, topic_name=orders (echoed from the request), partitions[0] let expected: [u8; 41] = [ diff --git a/gateways/kafka/tests/handler_regression_tests.rs b/gateways/kafka/tests/handler_regression_tests.rs deleted file mode 100644 index c9bf7b5a78..0000000000 --- a/gateways/kafka/tests/handler_regression_tests.rs +++ /dev/null @@ -1,180 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Full handler regression — every scoped API key × version through `handle_request`. - -#[path = "common/fixtures.rs"] -mod fixtures; -#[path = "common/scope.rs"] -mod scope; - -use iggy_gateway_kafka::protocol::api::{ - API_KEY_CREATE_TOPICS, API_KEY_FETCH, API_KEY_LIST_OFFSETS, API_KEY_PRODUCE, ERROR_NONE, - ERROR_NOT_CONTROLLER, ERROR_NOT_LEADER_OR_FOLLOWER, handle_request, -}; -use iggy_gateway_kafka::protocol::codec::Decoder; - -use fixtures::load_fixture_body_or_skip; -use scope::{SCOPED_API_KEYS, default_broker}; - -#[test] -fn handle_request_succeeds_for_every_supported_version_with_fixture() { - for &(api_key, name, min_ver, max_ver) in SCOPED_API_KEYS { - if api_key == 3 || api_key == 18 { - // Metadata / ApiVersions: empty body is valid - for version in min_ver..=max_ver { - let resp = handle_request(api_key, version, bytes::Bytes::new(), &default_broker()) - .expect_response("test request has acks != 0 and expects a response"); - assert!( - !resp.is_empty(), - "{name} v{version} returned empty response" - ); - } - continue; - } - - for version in min_ver..=max_ver { - let Some(body) = load_fixture_body_or_skip(api_key, name, version) else { - continue; - }; - let resp = handle_request(api_key, version, body, &default_broker()) - .expect_response("test request has acks != 0 and expects a response"); - assert!( - !resp.is_empty(), - "{name} v{version} returned empty response" - ); - } - } -} - -#[test] -fn produce_stub_response_returns_retriable_not_leader() { - for version in 3i16..=9 { - let Some(body) = load_fixture_body_or_skip(0, "Produce", version) else { - continue; - }; - let resp = handle_request(API_KEY_PRODUCE, version, body, &default_broker()) - .expect_response("test request has acks != 0 and expects a response"); - let flexible = version >= 9; - let mut d = Decoder::new(resp); - if flexible { - let _topics = d.read_varint().unwrap(); - let _topic = d.read_compact_nullable_string().unwrap(); - let _parts = d.read_varint().unwrap(); - } else { - let _topics = d.read_i32().unwrap(); - let _topic = d.read_nullable_string().unwrap(); - let _parts = d.read_i32().unwrap(); - } - let _partition = d.read_i32().unwrap(); - assert_eq!( - d.read_i16().unwrap(), - ERROR_NOT_LEADER_OR_FOLLOWER, - "Produce v{version}" - ); - } -} - -#[test] -fn fetch_stub_response_has_zero_partition_error() { - for version in 4i16..=12 { - let Some(body) = load_fixture_body_or_skip(1, "Fetch", version) else { - continue; - }; - let resp = handle_request(API_KEY_FETCH, version, body, &default_broker()) - .expect_response("test request has acks != 0 and expects a response"); - let flexible = version >= 12; - let mut d = Decoder::new(resp); - if version >= 1 { - let _throttle = d.read_i32().unwrap(); - } - if version >= 7 { - assert_eq!(d.read_i16().unwrap(), ERROR_NONE); - let _session = d.read_i32().unwrap(); - } - if flexible { - let _topics = d.read_varint().unwrap(); - let _topic = d.read_compact_nullable_string().unwrap(); - let _parts = d.read_varint().unwrap(); - } else { - let _topics = d.read_i32().unwrap(); - let _topic = d.read_nullable_string().unwrap(); - let _parts = d.read_i32().unwrap(); - } - let _partition = d.read_i32().unwrap(); - assert_eq!( - d.read_i16().unwrap(), - ERROR_NONE, - "Fetch v{version} partition error" - ); - } -} - -#[test] -fn list_offsets_stub_response_has_zero_error() { - for version in 1i16..=6 { - let Some(body) = load_fixture_body_or_skip(2, "ListOffsets", version) else { - continue; - }; - let resp = handle_request(API_KEY_LIST_OFFSETS, version, body, &default_broker()) - .expect_response("test request has acks != 0 and expects a response"); - let flexible = version >= 6; - let mut d = Decoder::new(resp); - if version >= 2 { - let _throttle = d.read_i32().unwrap(); - } - if flexible { - let _topics = d.read_varint().unwrap(); - let _topic = d.read_compact_nullable_string().unwrap(); - let _parts = d.read_varint().unwrap(); - } else { - let _topics = d.read_i32().unwrap(); - let _topic = d.read_nullable_string().unwrap(); - let _parts = d.read_i32().unwrap(); - } - let _partition = d.read_i32().unwrap(); - assert_eq!(d.read_i16().unwrap(), ERROR_NONE, "ListOffsets v{version}"); - } -} - -#[test] -fn create_topics_stub_response_returns_not_controller() { - for version in 2i16..=5 { - let Some(body) = load_fixture_body_or_skip(19, "CreateTopics", version) else { - continue; - }; - let resp = handle_request(API_KEY_CREATE_TOPICS, version, body, &default_broker()) - .expect_response("test request has acks != 0 and expects a response"); - let flexible = version >= 5; - let mut d = Decoder::new(resp); - if version >= 2 { - let _throttle = d.read_i32().unwrap(); - } - if flexible { - let _topics = d.read_varint().unwrap(); - let _topic = d.read_compact_nullable_string().unwrap(); - } else { - let _topics = d.read_i32().unwrap(); - let _topic = d.read_nullable_string().unwrap(); - } - assert_eq!( - d.read_i16().unwrap(), - ERROR_NOT_CONTROLLER, - "CreateTopics v{version}" - ); - } -} diff --git a/gateways/kafka/tests/header_tests.rs b/gateways/kafka/tests/header_tests.rs index bdff22681f..aa8687660d 100644 --- a/gateways/kafka/tests/header_tests.rs +++ b/gateways/kafka/tests/header_tests.rs @@ -15,6 +15,10 @@ // specific language governing permissions and limitations // under the License. +use iggy_gateway_kafka::protocol::api::{ + API_KEY_API_VERSIONS, API_KEY_CREATE_TOPICS, API_KEY_FETCH, API_KEY_LIST_OFFSETS, + API_KEY_METADATA, API_KEY_PRODUCE, +}; use iggy_gateway_kafka::protocol::codec::Encoder; use iggy_gateway_kafka::protocol::header::{ RequestHeader, ResponseHeader, request_header_version, response_header_version, @@ -51,7 +55,7 @@ fn request_header_v1_null_client_id() { assert_eq!(header.client_id, None); } -// ── Request header v2 (flexible — compact client_id + tagged fields) ─────── +// ── Request header v2 (flexible - compact client_id + tagged fields) ─────── #[test] fn request_header_v2_decodes() { @@ -344,3 +348,21 @@ fn response_header_encoded_size_matches_versions() { assert_eq!(ResponseHeader::encoded_size(1), 5); assert_eq!(ResponseHeader::encoded_size(2), 5); } + +// ── Flexible-encoding boundaries (SCOPE.md) ───────────────────────────────── + +#[test] +fn request_header_version_switches_at_scope_flexible_boundaries() { + assert_eq!(request_header_version(API_KEY_PRODUCE, 8), 1); + assert_eq!(request_header_version(API_KEY_PRODUCE, 9), 2); + assert_eq!(request_header_version(API_KEY_FETCH, 11), 1); + assert_eq!(request_header_version(API_KEY_FETCH, 12), 2); + assert_eq!(request_header_version(API_KEY_LIST_OFFSETS, 5), 1); + assert_eq!(request_header_version(API_KEY_LIST_OFFSETS, 6), 2); + assert_eq!(request_header_version(API_KEY_METADATA, 8), 1); + assert_eq!(request_header_version(API_KEY_METADATA, 9), 2); + assert_eq!(request_header_version(API_KEY_API_VERSIONS, 2), 1); + assert_eq!(request_header_version(API_KEY_API_VERSIONS, 3), 2); + assert_eq!(request_header_version(API_KEY_CREATE_TOPICS, 4), 1); + assert_eq!(request_header_version(API_KEY_CREATE_TOPICS, 5), 2); +} diff --git a/gateways/kafka/tests/listener_robustness_tests.rs b/gateways/kafka/tests/listener_robustness_tests.rs index cd45e386e3..cd62e376fa 100644 --- a/gateways/kafka/tests/listener_robustness_tests.rs +++ b/gateways/kafka/tests/listener_robustness_tests.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -//! TCP listener robustness — framing, pipelining, concurrency, edge cases. +//! TCP listener robustness - framing, pipelining, concurrency, edge cases. #[path = "common/server.rs"] mod server; @@ -32,13 +32,15 @@ use tokio::net::TcpStream; use tokio::time; use iggy_gateway_kafka::ServerConfig; -use iggy_gateway_kafka::protocol::api::{API_KEY_API_VERSIONS, API_KEY_METADATA, API_KEY_PRODUCE}; +use iggy_gateway_kafka::protocol::api::{ + API_KEY_API_VERSIONS, API_KEY_FETCH, API_KEY_METADATA, API_KEY_PRODUCE, ERROR_INVALID_REQUEST, +}; use iggy_gateway_kafka::protocol::codec::Decoder; use server::{spawn_test_server, spawn_test_server_with_config}; use tcp::{ ByteRead, build_request_frame, concat_frames, parse_response_payload, read_byte_with_timeout, - read_response_frame, read_response_frame_with_timeout, + read_response_frame, read_response_frame_with_timeout, scan_for_error_code, }; #[tokio::test] @@ -417,3 +419,143 @@ async fn e2e_flexible_metadata_v9_empty_topics_round_trip() { .saturating_sub(1); assert_eq!(broker_count, 1); } + +// ── Idle connection handling (read_timeout must not act as idle cap) ─────── + +#[tokio::test] +async fn e2e_quiet_connection_accepts_request_after_short_idle() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.expect("connect"); + + time::sleep(Duration::from_secs(2)).await; + + let frame = build_request_frame(API_KEY_API_VERSIONS, 1, 501, Some("idle-test"), &[]); + stream + .write_all(&frame) + .await + .expect("connection should stay open after short idle"); + + let payload = + read_response_frame_with_timeout(&mut stream, 8 * 1024 * 1024, Duration::from_secs(2)) + .await + .expect("request after short idle should succeed"); + + let (corr, _) = parse_response_payload(API_KEY_API_VERSIONS, 1, payload); + assert_eq!(corr, 501); +} + +#[tokio::test] +async fn e2e_quiet_connection_survives_beyond_read_timeout_idle_cap() { + let (addr, _shutdown) = spawn_test_server_with_config(ServerConfig { + bind_addr: String::new(), + advertised_host: None, + advertised_port: None, + max_frame_size: 8 * 1024 * 1024, + read_timeout: Duration::from_secs(3), + write_timeout: Duration::from_secs(5), + }) + .await; + + let mut stream = TcpStream::connect(addr).await.expect("connect"); + + // Idle longer than read_timeout: prefix wait has no timer; connection stays open. + time::sleep(Duration::from_secs(4)).await; + + let frame = build_request_frame(API_KEY_API_VERSIONS, 1, 502, Some("idle-test"), &[]); + stream + .write_all(&frame) + .await + .expect("idle connection should remain writable after read_timeout elapses"); + + let payload = + read_response_frame_with_timeout(&mut stream, 8 * 1024 * 1024, Duration::from_secs(2)) + .await + .expect( + "request after idle period longer than read_timeout should succeed \ + (read_timeout applies to in-flight frame body only)", + ); + + let (corr, _) = parse_response_payload(API_KEY_API_VERSIONS, 1, payload); + assert_eq!(corr, 502); +} + +// ── Corrupt body survives on the connection (no disconnect) ──────────────── + +#[tokio::test] +async fn corrupt_produce_body_e2e_returns_error_without_disconnect() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.expect("connect"); + + let bad = build_request_frame( + API_KEY_PRODUCE, + 3, + 391, + Some("scope-test"), + &[0xFF, 0xFF, 0xFF], + ); + stream.write_all(&bad).await.expect("corrupt produce"); + let payload = read_response_frame(&mut stream, 8 * 1024 * 1024).await; + assert!( + scan_for_error_code( + &parse_response_payload(API_KEY_PRODUCE, 3, payload).1, + ERROR_INVALID_REQUEST + ), + "corrupt Produce must surface INVALID_REQUEST" + ); + + let ok = build_request_frame(API_KEY_API_VERSIONS, 1, 392, Some("scope-test"), &[]); + stream.write_all(&ok).await.expect("follow-up"); + let payload = read_response_frame(&mut stream, 8 * 1024 * 1024).await; + assert_eq!( + parse_response_payload(API_KEY_API_VERSIONS, 1, payload).0, + 392 + ); +} + +#[tokio::test] +async fn corrupt_fetch_body_e2e_returns_error_without_disconnect() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.expect("connect"); + + let bad = build_request_frame( + API_KEY_FETCH, + 4, + 393, + Some("scope-test"), + &[0xFF, 0xFF, 0xFF], + ); + stream.write_all(&bad).await.expect("corrupt fetch"); + let payload = read_response_frame(&mut stream, 8 * 1024 * 1024).await; + assert!( + scan_for_error_code( + &parse_response_payload(API_KEY_FETCH, 4, payload).1, + ERROR_INVALID_REQUEST + ), + "corrupt Fetch must surface INVALID_REQUEST" + ); + + let ok = build_request_frame(API_KEY_API_VERSIONS, 1, 394, Some("scope-test"), &[]); + stream.write_all(&ok).await.expect("follow-up"); + let payload = read_response_frame(&mut stream, 8 * 1024 * 1024).await; + assert_eq!( + parse_response_payload(API_KEY_API_VERSIONS, 1, payload).0, + 394 + ); +} + +#[tokio::test] +async fn e2e_client_eof_after_valid_frame_closes_connection_cleanly() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.expect("connect"); + + let frame = build_request_frame(18, 1, 901, Some("eof-test"), &[]); + stream.write_all(&frame).await.expect("api versions write"); + let _payload = read_response_frame(&mut stream, 8 * 1024 * 1024).await; + + stream.shutdown().await.expect("client shutdown"); + time::sleep(Duration::from_millis(100)).await; + + let mut buf = [0u8; 1]; + let n = stream.read(&mut buf).await.expect("read after shutdown"); + assert_eq!(n, 0, "server should close after client EOF"); +} diff --git a/gateways/kafka/tests/metadata_regression_tests.rs b/gateways/kafka/tests/metadata_regression_tests.rs deleted file mode 100644 index 0d5bf59d77..0000000000 --- a/gateways/kafka/tests/metadata_regression_tests.rs +++ /dev/null @@ -1,267 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Metadata API regression — all supported versions, broker advertise, topic counts. - -#[path = "common/scope.rs"] -mod scope; - -use bytes::Bytes; - -use iggy_gateway_kafka::protocol::api::{ - API_KEY_METADATA, BrokerAdvertise, ERROR_UNKNOWN_TOPIC_OR_PARTITION, handle_request, -}; -use iggy_gateway_kafka::protocol::codec::{Decoder, Encoder}; - -use scope::default_broker; - -/// Topic name assigned to slot `i` by [`metadata_request_legacy`] / [`metadata_request_flexible`]. -fn synthetic_topic_name(i: i32) -> String { - format!("topic-{i}") -} - -fn metadata_request_legacy(topic_count: i32) -> Bytes { - let mut enc = Encoder::with_capacity(64); - enc.write_i32(topic_count); - for i in 0..topic_count { - enc.write_nullable_string(Some(&synthetic_topic_name(i))) - .expect("topic name fits"); - } - enc.freeze() -} - -fn metadata_request_flexible(topic_count: usize) -> Bytes { - let mut enc = Encoder::with_capacity(64); - enc.write_varint((topic_count + 1) as u64); - for i in 0..topic_count { - enc.write_compact_nullable_string(Some(&synthetic_topic_name(i32::try_from(i).unwrap()))); - enc.write_empty_tagged_fields(); - } - enc.freeze() -} - -fn read_broker_legacy(d: &mut Decoder) -> (String, i32) { - let count = d.read_i32().unwrap(); - assert_eq!(count, 1); - let _node = d.read_i32().unwrap(); - let host = d.read_nullable_string().unwrap().unwrap(); - let port = d.read_i32().unwrap(); - (host, port) -} - -fn read_broker_flexible(d: &mut Decoder) -> (String, i32) { - let count_plus_one = d.read_varint().unwrap(); - assert_eq!(count_plus_one, 2); // one broker - let _node = d.read_i32().unwrap(); - let host = d.read_compact_nullable_string().unwrap().unwrap(); - let port = d.read_i32().unwrap(); - let _rack = d.read_compact_nullable_string().unwrap(); - d.read_tagged_fields().unwrap(); - (host, port) -} - -#[test] -fn metadata_corrupt_partial_body_returns_zero_topics() { - let body = handle_request( - API_KEY_METADATA, - 0, - Bytes::from_static(&[0x00, 0x00]), - &default_broker(), - ) - .expect_response("test request has acks != 0 and expects a response"); - let mut d = Decoder::new(body); - let _ = read_broker_legacy(&mut d); - assert_eq!(d.read_i32().unwrap(), 0); - assert_eq!(d.remaining(), 0); -} - -#[test] -fn metadata_v0_empty_topics_stub_broker() { - let body = handle_request( - API_KEY_METADATA, - 0, - metadata_request_legacy(0), - &default_broker(), - ) - .expect_response("test request has acks != 0 and expects a response"); - let mut d = Decoder::new(body); - let (host, port) = read_broker_legacy(&mut d); - assert_eq!(host, "127.0.0.1"); - assert_eq!(port, 9093); - assert_eq!(d.read_i32().unwrap(), 0); -} - -#[test] -fn metadata_v0_three_topics_each_unknown() { - let body = handle_request( - API_KEY_METADATA, - 0, - metadata_request_legacy(3), - &default_broker(), - ) - .expect_response("test request has acks != 0 and expects a response"); - let mut d = Decoder::new(body); - let _ = read_broker_legacy(&mut d); - assert_eq!(d.read_i32().unwrap(), 3); - for i in 0..3 { - assert_eq!(d.read_i16().unwrap(), ERROR_UNKNOWN_TOPIC_OR_PARTITION); - assert_eq!( - d.read_nullable_string().unwrap().unwrap(), - synthetic_topic_name(i) - ); - assert_eq!(d.read_i32().unwrap(), 0); - } -} - -#[test] -fn metadata_v1_includes_controller_id() { - let body = handle_request( - API_KEY_METADATA, - 1, - metadata_request_legacy(0), - &default_broker(), - ) - .expect_response("test request has acks != 0 and expects a response"); - let mut d = Decoder::new(body); - // Metadata v1 has no throttle_time_ms (added in v3). - let _ = read_broker_legacy(&mut d); - let _rack = d.read_nullable_string().unwrap(); - let controller = d.read_i32().unwrap(); - assert_eq!(controller, 1); -} - -#[test] -fn metadata_v2_includes_cluster_id_field() { - let body = handle_request( - API_KEY_METADATA, - 2, - metadata_request_legacy(0), - &default_broker(), - ) - .expect_response("test request has acks != 0 and expects a response"); - let mut d = Decoder::new(body); - let _ = read_broker_legacy(&mut d); - let _rack = d.read_nullable_string().unwrap(); - let _cluster_id = d.read_nullable_string().unwrap(); - let _controller = d.read_i32().unwrap(); - assert_eq!(d.read_i32().unwrap(), 0); -} - -#[test] -fn metadata_all_legacy_versions_produce_valid_response() { - for version in 0i16..=8 { - let body = handle_request( - API_KEY_METADATA, - version, - metadata_request_legacy(1), - &default_broker(), - ) - .expect_response("test request has acks != 0 and expects a response"); - let mut d = Decoder::new(body); - if version >= 3 { - let _throttle = d.read_i32().unwrap(); - } - let _ = read_broker_legacy(&mut d); - if version >= 1 { - let _rack = d.read_nullable_string().unwrap(); - } - if version >= 2 { - let _cluster = d.read_nullable_string().unwrap(); - } - if version >= 1 { - let _controller = d.read_i32().unwrap(); - } - assert_eq!(d.read_i32().unwrap(), 1); - assert_eq!(d.read_i16().unwrap(), ERROR_UNKNOWN_TOPIC_OR_PARTITION); - } -} - -#[test] -fn metadata_v9_flexible_encoding() { - let body = handle_request( - API_KEY_METADATA, - 9, - metadata_request_flexible(2), - &default_broker(), - ) - .expect_response("test request has acks != 0 and expects a response"); - let mut d = Decoder::new(body); - let _throttle = d.read_i32().unwrap(); - let (host, port) = read_broker_flexible(&mut d); - assert_eq!(host, "127.0.0.1"); - assert_eq!(port, 9093); - let _cluster = d.read_compact_nullable_string().unwrap(); - let controller = d.read_i32().unwrap(); - assert_eq!(controller, 1); - - let topics_plus_one = d.read_varint().unwrap(); - assert_eq!(topics_plus_one, 3); // 2 topics - for i in 0..2 { - assert_eq!(d.read_i16().unwrap(), ERROR_UNKNOWN_TOPIC_OR_PARTITION); - assert_eq!( - d.read_compact_nullable_string().unwrap().unwrap(), - synthetic_topic_name(i) - ); - let _internal = d.read_bool().unwrap(); - let parts_plus_one = d.read_varint().unwrap(); - assert_eq!(parts_plus_one, 1); // empty partitions - assert_eq!(d.read_i32().unwrap(), i32::MIN); // topic_authorized_operations (v8+) - d.read_tagged_fields().unwrap(); - } - assert_eq!(d.read_i32().unwrap(), i32::MIN); // cluster_authorized_operations (v8+) - d.read_tagged_fields().unwrap(); - assert_eq!(d.remaining(), 0); -} - -#[test] -fn metadata_v8_includes_authorized_operations_legacy() { - let body = handle_request( - API_KEY_METADATA, - 8, - metadata_request_legacy(1), - &default_broker(), - ) - .expect_response("test request has acks != 0 and expects a response"); - let mut d = Decoder::new(body); - let _throttle = d.read_i32().unwrap(); - let _ = read_broker_legacy(&mut d); - let _rack = d.read_nullable_string().unwrap(); - let _cluster = d.read_nullable_string().unwrap(); - let _controller = d.read_i32().unwrap(); - assert_eq!(d.read_i32().unwrap(), 1); - let _topic_error = d.read_i16().unwrap(); - let _topic = d.read_nullable_string().unwrap(); - let _internal = d.read_bool().unwrap(); - assert_eq!(d.read_i32().unwrap(), 0); // empty partitions - assert_eq!(d.read_i32().unwrap(), i32::MIN); // topic_authorized_operations - assert_eq!(d.read_i32().unwrap(), i32::MIN); // cluster_authorized_operations - assert_eq!(d.remaining(), 0); -} - -#[test] -fn metadata_uses_custom_broker_advertise() { - let broker = BrokerAdvertise { - host: "10.0.0.42".to_string(), - port: 29093, - }; - let body = handle_request(API_KEY_METADATA, 0, metadata_request_legacy(0), &broker) - .expect_response("test request has acks != 0 and expects a response"); - let mut d = Decoder::new(body); - let (host, port) = read_broker_legacy(&mut d); - assert_eq!(host, "10.0.0.42"); - assert_eq!(port, 29093); -} diff --git a/gateways/kafka/tests/region_coverage_tests.rs b/gateways/kafka/tests/region_coverage_tests.rs deleted file mode 100644 index aa2636e90d..0000000000 --- a/gateways/kafka/tests/region_coverage_tests.rs +++ /dev/null @@ -1,445 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Decoder and listener paths that exist only for region coverage of version branches. - -use std::time::Duration; - -use bytes::BytesMut; -use iggy_gateway_kafka::error::KafkaProtocolError; -use iggy_gateway_kafka::protocol::codec::Encoder; -use iggy_gateway_kafka::protocol::requests::{ - ProduceDecodeResult, decode_create_topics_request, decode_fetch_request, - decode_list_offsets_request, decode_produce_request, -}; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::net::TcpStream; -use tokio::time; - -#[path = "common/server.rs"] -mod server; -#[path = "common/tcp.rs"] -mod tcp; -#[path = "common/wire.rs"] -mod wire; - -use tcp::{ByteRead, build_request_frame, read_byte_with_timeout, read_response_frame}; - -// ── Produce version branches ────────────────────────────────────────────────── - -#[test] -fn produce_v2_skips_transactional_id_branch() { - let req = decode_produce_request(2, wire::build_produce_legacy_request(2, 1, None, None)) - .into_request() - .expect("produce v2 should decode"); - assert_eq!(req.acks, 1); - assert!(req.transactional_id.is_none()); - assert!(req.topics.is_empty()); -} - -#[test] -fn produce_v3_legacy_transactional_id_and_topic_decode() { - let req = decode_produce_request( - 3, - wire::build_produce_legacy_request(3, -1, Some("txn-1"), Some("legacy-topic")), - ) - .into_request() - .expect("produce v3 legacy should decode"); - assert_eq!(req.transactional_id.as_deref(), Some("txn-1")); - assert_eq!(req.topics.len(), 1); - assert_eq!(req.topics[0].topic, "legacy-topic"); - assert!(req.topics[0].partitions[0].records.is_some()); -} - -#[test] -fn produce_v8_legacy_null_records_decode() { - let mut enc = Encoder::with_capacity(64); - enc.write_nullable_string(None::<&str>).unwrap(); - enc.write_i16(1); - enc.write_i32(500); - enc.write_i32(1); - enc.write_nullable_string(Some("topic")).unwrap(); - enc.write_i32(1); - enc.write_i32(0); - enc.write_nullable_bytes(None).unwrap(); - - let req = decode_produce_request(8, enc.freeze()) - .into_request() - .expect("produce v8 with null records should decode"); - assert!(req.topics[0].partitions[0].records.is_none()); -} - -#[test] -fn produce_null_topic_name_preserves_acks_on_error() { - let mut enc = Encoder::with_capacity(32); - enc.write_nullable_string(None::<&str>).unwrap(); - enc.write_i16(1); - enc.write_i32(500); - enc.write_i32(1); - enc.write_nullable_string(None::<&str>).unwrap(); - - match decode_produce_request(3, enc.freeze()) { - ProduceDecodeResult::Err { acks, error } => { - assert_eq!(acks, Some(1)); - assert!(matches!(error, KafkaProtocolError::NullTopicName)); - } - ProduceDecodeResult::Ok(_) => panic!("expected NullTopicName"), - } -} - -#[test] -fn produce_v9_flexible_transactional_id_and_tagged_fields_decode() { - let req = decode_produce_request( - 9, - wire::build_produce_flexible_request_with_topic("flex-topic"), - ) - .into_request() - .expect("produce v9 flexible should decode"); - assert_eq!(req.transactional_id.as_deref(), Some("txn-1")); - assert_eq!(req.topics[0].topic, "flex-topic"); - assert!(req.topics[0].partitions[0].records.is_some()); -} - -#[test] -fn produce_v3_error_before_acks_has_none_acks() { - let mut enc = Encoder::with_capacity(8); - enc.write_i16(1); - match decode_produce_request(3, enc.freeze()) { - ProduceDecodeResult::Err { acks, .. } => assert_eq!(acks, None), - ProduceDecodeResult::Ok(_) => panic!("expected decode error before acks"), - } -} - -#[test] -fn produce_v3_error_after_acks_preserves_acks() { - let mut enc = Encoder::with_capacity(16); - enc.write_nullable_string(None::<&str>).unwrap(); - enc.write_i16(7); - match decode_produce_request(3, enc.freeze()) { - ProduceDecodeResult::Err { acks, .. } => assert_eq!(acks, Some(7)), - ProduceDecodeResult::Ok(_) => panic!("expected decode error after acks"), - } -} - -#[test] -fn produce_v3_error_after_timeout_preserves_acks() { - let mut enc = Encoder::with_capacity(16); - enc.write_nullable_string(None::<&str>).unwrap(); - enc.write_i16(1); - enc.write_i32(500); - match decode_produce_request(3, enc.freeze()) { - ProduceDecodeResult::Err { acks, .. } => assert_eq!(acks, Some(1)), - ProduceDecodeResult::Ok(_) => panic!("expected decode error after timeout"), - } -} - -#[test] -fn produce_v9_error_on_null_topic_preserves_acks() { - let mut enc = Encoder::with_capacity(32); - enc.write_compact_nullable_string(None); - enc.write_i16(2); - enc.write_i32(500); - enc.write_varint(2); - enc.write_compact_nullable_string(None); - match decode_produce_request(9, enc.freeze()) { - ProduceDecodeResult::Err { acks, error } => { - assert_eq!(acks, Some(2)); - assert!(matches!(error, KafkaProtocolError::NullTopicName)); - } - ProduceDecodeResult::Ok(_) => panic!("expected NullTopicName"), - } -} - -#[test] -fn produce_v9_error_on_partition_count_preserves_acks() { - let mut enc = Encoder::with_capacity(64); - enc.write_compact_nullable_string(None); - enc.write_i16(3); - enc.write_i32(500); - enc.write_varint(2); - enc.write_compact_nullable_string(Some("topic")); - match decode_produce_request(9, enc.freeze()) { - ProduceDecodeResult::Err { acks, .. } => assert_eq!(acks, Some(3)), - ProduceDecodeResult::Ok(_) => panic!("expected decode error in partition count"), - } -} - -#[test] -fn produce_v9_error_on_partition_records_preserves_acks() { - let mut enc = Encoder::with_capacity(64); - enc.write_compact_nullable_string(None); - enc.write_i16(4); - enc.write_i32(500); - enc.write_varint(2); - enc.write_compact_nullable_string(Some("topic")); - enc.write_varint(2); - enc.write_i32(0); - match decode_produce_request(9, enc.freeze()) { - ProduceDecodeResult::Err { acks, .. } => assert_eq!(acks, Some(4)), - ProduceDecodeResult::Ok(_) => panic!("expected decode error in records"), - } -} - -// ── Fetch version branches ──────────────────────────────────────────────────── - -#[test] -fn fetch_v2_uses_default_max_bytes_when_field_absent() { - let req = decode_fetch_request(2, wire::build_fetch_v2_default_max_bytes_request()) - .expect("fetch v2 should decode"); - assert_eq!(req.max_bytes, 52_428_800); - assert_eq!(req.isolation_level, 0); - assert!(req.topics.is_empty()); -} - -#[test] -fn fetch_v7_legacy_forgotten_topics_and_rack_id_decode() { - let req = decode_fetch_request( - 7, - wire::build_fetch_request_with_sections(7, "topic-a", 1, Some("forgotten"), Some("rack-1")), - ) - .expect("fetch v7 legacy sections should decode"); - assert_eq!(req.topics[0].topic, "topic-a"); - assert_eq!(req.topics[0].partitions[0].partition, 1); -} - -#[test] -fn fetch_v9_leader_epoch_without_v12_fields_decode() { - let req = decode_fetch_request( - 9, - wire::build_fetch_request_with_sections(9, "topic-b", 2, None, None), - ) - .expect("fetch v9 should decode"); - assert_eq!(req.topics[0].partitions[0].fetch_offset, 42); -} - -#[test] -fn fetch_v11_legacy_rack_id_decode() { - let req = decode_fetch_request( - 11, - wire::build_fetch_request_with_sections(11, "topic-c", 3, None, Some("rack-z")), - ) - .expect("fetch v11 legacy rack id should decode"); - assert_eq!(req.max_wait_ms, 100); -} - -#[test] -fn fetch_v3_skips_isolation_level_field() { - let req = decode_fetch_request(3, wire::build_fetch_v3_no_isolation_request()) - .expect("fetch v3 should decode"); - assert_eq!(req.isolation_level, 0); - assert_eq!(req.max_bytes, 1024); -} - -#[test] -fn fetch_v4_truncated_after_replica_id_returns_error() { - let mut enc = Encoder::with_capacity(4); - enc.write_i32(-1); - let err = decode_fetch_request(4, enc.freeze()).unwrap_err(); - assert!(matches!(err, KafkaProtocolError::BufferUnderflow { .. })); -} - -#[test] -fn fetch_v7_truncated_in_forgotten_topics_returns_error() { - let body = wire::build_fetch_request_with_sections(7, "topic", 0, Some("forgot"), None); - let truncated = body.slice(..body.len() - 2); - let err = decode_fetch_request(7, truncated).unwrap_err(); - assert!(matches!(err, KafkaProtocolError::BufferUnderflow { .. })); -} - -#[test] -fn fetch_v12_flexible_truncated_in_topic_tagged_fields_returns_error() { - let body = wire::build_fetch_request_with_sections(12, "topic", 0, None, None); - let truncated = body.slice(..body.len() - 1); - let err = decode_fetch_request(12, truncated).unwrap_err(); - assert!(matches!(err, KafkaProtocolError::BufferUnderflow { .. })); -} - -#[test] -fn fetch_v12_flexible_null_topic_name_returns_error() { - let mut enc = Encoder::with_capacity(64); - enc.write_i32(-1); - enc.write_i32(100); - enc.write_i32(1); - enc.write_i32(1024); - enc.write_i8(0); - enc.write_i32(0); - enc.write_i32(0); - enc.write_varint(2); - enc.write_compact_nullable_string(None); - let err = decode_fetch_request(12, enc.freeze()).unwrap_err(); - assert!(matches!(err, KafkaProtocolError::NullTopicName)); -} - -#[test] -fn fetch_null_topic_name_returns_error() { - let mut enc = Encoder::with_capacity(64); - enc.write_i32(-1); - enc.write_i32(100); - enc.write_i32(1); - enc.write_i32(i32::MAX); - enc.write_i8(0); - enc.write_i32(1); - enc.write_nullable_string(None::<&str>).unwrap(); - - let err = decode_fetch_request(4, enc.freeze()).unwrap_err(); - assert!(matches!(err, KafkaProtocolError::NullTopicName)); -} - -// ── ListOffsets version branches ────────────────────────────────────────────── - -#[test] -fn list_offsets_v1_skips_isolation_level_field() { - let req = decode_list_offsets_request(1, wire::build_list_offsets_branch_request(1, "v1", 0)) - .expect("list offsets v1 should decode"); - assert_eq!(req.isolation_level, 0); -} - -#[test] -fn list_offsets_v2_reads_isolation_level() { - let req = decode_list_offsets_request(2, wire::build_list_offsets_branch_request(2, "v2", 1)) - .expect("list offsets v2 should decode"); - assert_eq!(req.isolation_level, 1); -} - -#[test] -fn list_offsets_v3_skips_leader_epoch_branch() { - let req = decode_list_offsets_request(3, wire::build_list_offsets_branch_request(3, "v3", 2)) - .expect("list offsets v3 should decode"); - assert_eq!(req.topics[0].partitions[0].partition, 2); -} - -#[test] -fn list_offsets_v5_leader_epoch_without_flexible_encoding() { - let req = decode_list_offsets_request(5, wire::build_list_offsets_branch_request(5, "v5", 3)) - .expect("list offsets v5 should decode"); - assert_eq!(req.topics[0].topic, "v5"); -} - -#[test] -fn list_offsets_v4_truncated_in_leader_epoch_returns_error() { - let body = wire::build_list_offsets_branch_request(4, "topic", 1); - let truncated = body.slice(..body.len() - 4); - let err = decode_list_offsets_request(4, truncated).unwrap_err(); - assert!(matches!(err, KafkaProtocolError::BufferUnderflow { .. })); -} - -#[test] -fn list_offsets_v6_flexible_null_topic_name_returns_error() { - let mut enc = Encoder::with_capacity(32); - enc.write_i32(-1); - enc.write_i8(0); - enc.write_varint(2); - enc.write_compact_nullable_string(None); - let err = decode_list_offsets_request(6, enc.freeze()).unwrap_err(); - assert!(matches!(err, KafkaProtocolError::NullTopicName)); -} - -#[test] -fn list_offsets_null_topic_name_returns_error() { - let mut enc = Encoder::with_capacity(32); - enc.write_i32(-1); - enc.write_i8(0); - enc.write_i32(1); - enc.write_nullable_string(None::<&str>).unwrap(); - let err = decode_list_offsets_request(2, enc.freeze()).unwrap_err(); - assert!(matches!(err, KafkaProtocolError::NullTopicName)); -} - -// ── CreateTopics version branches ───────────────────────────────────────────── - -#[test] -fn create_topics_v3_legacy_assignments_decode() { - let req = decode_create_topics_request( - 3, - wire::build_create_topics_request_with_sections(3, "v3-topic"), - ) - .expect("create topics v3 should decode"); - assert_eq!(req.topics[0].name, "v3-topic"); - assert!(req.validate_only); -} - -#[test] -fn create_topics_v4_legacy_configs_decode() { - let req = decode_create_topics_request( - 4, - wire::build_create_topics_request_with_sections(4, "v4-topic"), - ) - .expect("create topics v4 should decode"); - assert_eq!(req.topics[0].replication_factor, 1); -} - -#[test] -fn create_topics_v2_truncated_in_config_value_returns_error() { - let body = wire::build_create_topics_request_with_sections(2, "topic"); - let truncated = body.slice(..body.len() - 3); - let err = decode_create_topics_request(2, truncated).unwrap_err(); - assert!(matches!(err, KafkaProtocolError::BufferUnderflow { .. })); -} - -#[test] -fn create_topics_v5_flexible_null_topic_name_returns_error() { - let mut enc = Encoder::with_capacity(16); - enc.write_varint(2); - enc.write_compact_nullable_string(None); - let err = decode_create_topics_request(5, enc.freeze()).unwrap_err(); - assert!(matches!(err, KafkaProtocolError::NullTopicName)); -} - -#[test] -fn create_topics_null_topic_name_returns_error() { - let mut enc = Encoder::with_capacity(32); - enc.write_i32(1); - enc.write_nullable_string(None::<&str>).unwrap(); - let err = decode_create_topics_request(2, enc.freeze()).unwrap_err(); - assert!(matches!(err, KafkaProtocolError::NullTopicName)); -} - -// ── Server handle_connection branches (e2e) ─────────────────────────────────── - -#[tokio::test] -async fn e2e_frame_shorter_than_kafka_header_returns_buffer_underflow() { - let (addr, _shutdown) = server::spawn_test_server().await; - let mut stream = TcpStream::connect(addr).await.expect("connect"); - - let mut frame = BytesMut::new(); - frame.extend_from_slice(&7_i32.to_be_bytes()); - frame.extend_from_slice(&[0x00, 0x12, 0x00, 0x01, 0x00, 0x00, 0x00]); - stream.write_all(&frame).await.expect("short header write"); - - assert_eq!( - read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await, - ByteRead::Closed, - "frame payload shorter than 8-byte Kafka header must close connection" - ); -} - -#[tokio::test] -async fn e2e_client_eof_after_valid_frame_closes_connection_cleanly() { - let (addr, _shutdown) = server::spawn_test_server().await; - let mut stream = TcpStream::connect(addr).await.expect("connect"); - - let frame = build_request_frame(18, 1, 901, Some("eof-test"), &[]); - stream.write_all(&frame).await.expect("api versions write"); - let _payload = read_response_frame(&mut stream, 8 * 1024 * 1024).await; - - stream.shutdown().await.expect("client shutdown"); - time::sleep(Duration::from_millis(100)).await; - - let mut buf = [0u8; 1]; - let n = stream.read(&mut buf).await.expect("read after shutdown"); - assert_eq!(n, 0, "server should close after client EOF"); -} diff --git a/gateways/kafka/tests/review_regression_tests.rs b/gateways/kafka/tests/review_regression_tests.rs deleted file mode 100644 index 1c2a3bac58..0000000000 --- a/gateways/kafka/tests/review_regression_tests.rs +++ /dev/null @@ -1,426 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Regression tests for PR #3519 review findings (atharvalade, Jul 2026). -//! -//! Each test encodes Kafka-client-correct behavior for a specific review finding and -//! guards against regressions now that the corresponding protocol/server fixes have landed. - -#[path = "common/scope.rs"] -mod scope; -#[path = "common/server.rs"] -mod server; -#[path = "common/tcp.rs"] -mod tcp; -#[path = "common/wire.rs"] -mod wire; - -use std::time::Duration; - -use bytes::{BufMut, Bytes, BytesMut}; -use tokio::io::AsyncWriteExt; -use tokio::net::TcpStream; -use tokio::time; - -use iggy_gateway_kafka::ServerConfig; -use iggy_gateway_kafka::protocol::api::{ - API_KEY_API_VERSIONS, API_KEY_LIST_OFFSETS, API_KEY_METADATA, API_KEY_PRODUCE, - ERROR_UNKNOWN_TOPIC_OR_PARTITION, ERROR_UNSUPPORTED_VERSION, handle_request, -}; -use iggy_gateway_kafka::protocol::codec::Decoder; - -use scope::default_broker; -use server::{spawn_test_server, spawn_test_server_with_config}; -use tcp::{ - build_metadata_legacy_request, build_produce_v3_body, build_request_frame, - parse_response_payload, read_response_frame_with_timeout, -}; -use wire::build_metadata_flexible_request_v10; - -// ── Produce acks=0 (review: broker must stay silent) ───────────────────────── - -#[test] -fn produce_acks_zero_malformed_body_decode_carries_acks() { - use iggy_gateway_kafka::protocol::requests::{ProduceDecodeResult, decode_produce_request}; - - let body = build_produce_v3_body(0, 1); - match decode_produce_request(3, body.clone()) { - ProduceDecodeResult::Err { acks: Some(0), .. } => {} - other => panic!("expected decode error with acks=0, got {other:?}"), - } - assert!( - handle_request(API_KEY_PRODUCE, 3, body, &default_broker()).is_no_response(), - "handler must not respond when acks=0 even if decode fails after acks" - ); -} - -#[tokio::test] -async fn e2e_produce_v3_acks_zero_sends_no_response() { - let (addr, _shutdown) = spawn_test_server().await; - let mut stream = TcpStream::connect(addr).await.expect("connect"); - - let body = build_produce_v3_body(0, 0); - let frame = build_request_frame(API_KEY_PRODUCE, 3, 42, Some("review-test"), &body); - stream - .write_all(&frame) - .await - .expect("write produce acks=0"); - - let response = - read_response_frame_with_timeout(&mut stream, 8 * 1024 * 1024, Duration::from_millis(500)) - .await; - - assert!( - response.is_none(), - "Produce with acks=0 must not receive a response frame (Kafka spec); got {} bytes", - response.as_ref().map_or(0, Bytes::len) - ); -} - -#[tokio::test] -async fn e2e_produce_v3_acks_zero_malformed_topics_sends_no_response() { - let (addr, _shutdown) = spawn_test_server().await; - let mut stream = TcpStream::connect(addr).await.expect("connect"); - - // acks=0, claims one topic, no topic bytes — decode fails after acks is read. - let body = build_produce_v3_body(0, 1); - let frame = build_request_frame(API_KEY_PRODUCE, 3, 99, Some("review-test"), &body); - stream - .write_all(&frame) - .await - .expect("write produce acks=0 malformed"); - - let response = - read_response_frame_with_timeout(&mut stream, 8 * 1024 * 1024, Duration::from_millis(500)) - .await; - - assert!( - response.is_none(), - "Produce with acks=0 must stay silent even when the body is malformed; got {} bytes", - response.as_ref().map_or(0, Bytes::len) - ); -} - -#[tokio::test] -async fn e2e_produce_v3_acks_one_still_returns_response() { - let (addr, _shutdown) = spawn_test_server().await; - let mut stream = TcpStream::connect(addr).await.expect("connect"); - - let body = build_produce_v3_body(1, 0); - let frame = build_request_frame(API_KEY_PRODUCE, 3, 43, Some("review-test"), &body); - stream - .write_all(&frame) - .await - .expect("write produce acks=1"); - - let response = - read_response_frame_with_timeout(&mut stream, 8 * 1024 * 1024, Duration::from_secs(2)) - .await - .expect("Produce with acks=1 should receive a response"); - - let (corr, resp_body) = parse_response_payload(API_KEY_PRODUCE, 3, response); - assert_eq!(corr, 43); - assert!(!resp_body.is_empty()); -} - -// ── ListOffsets v0 wire shape (review: old_style_offsets array, not bare i64) ─ - -/// Parse one `ListOffsets` v0 partition entry the way a v0 Kafka client would. -fn parse_list_offsets_v0_partition(d: &mut Decoder) { - let _partition_index = d.read_i32().expect("partition_index"); - let _error_code = d.read_i16().expect("error_code"); - let offset_count = d.read_i32().expect("old_style_offsets array length"); - assert!( - offset_count >= 0, - "old_style_offsets count must be non-negative, got {offset_count}" - ); - for _ in 0..offset_count { - d.read_i64().expect("old_style_offsets entry"); - } -} - -#[test] -fn list_offsets_v0_unsupported_version_is_parseable_by_v0_clients() { - let body = handle_request(API_KEY_LIST_OFFSETS, 0, Bytes::new(), &default_broker()) - .expect_response("test request has acks != 0 and expects a response"); - let mut d = Decoder::new(body); - - assert_eq!(d.read_i32().unwrap(), 1, "topics array length"); - assert_eq!( - d.read_nullable_string().unwrap(), - Some(String::new()), - "placeholder topic name" - ); - assert_eq!(d.read_i32().unwrap(), 1, "partitions array length"); - - parse_list_offsets_v0_partition(&mut d); - - assert_eq!( - d.remaining(), - 0, - "v0 client must consume the full error response without trailing bytes" - ); -} - -#[test] -fn list_offsets_v0_unsupported_version_carries_error_code_in_partition() { - let request_body = build_list_offsets_v0_request_with_topic_t(); - let body = handle_request(API_KEY_LIST_OFFSETS, 0, request_body, &default_broker()) - .expect_response("test request has acks != 0 and expects a response"); - let mut d = Decoder::new(body); - - assert_eq!(d.read_i32().unwrap(), 1); - d.read_nullable_string().unwrap(); - assert_eq!(d.read_i32().unwrap(), 1); - assert_eq!(d.read_i32().unwrap(), 0, "partition index"); - assert_eq!( - d.read_i16().unwrap(), - ERROR_UNSUPPORTED_VERSION, - "partition error code" - ); - - // partition_index and error_code were already asserted above; only the - // trailing old_style_offsets array remains for this single partition. - let offset_count = d.read_i32().expect("old_style_offsets array length"); - assert!( - offset_count >= 0, - "old_style_offsets count must be non-negative, got {offset_count}" - ); - for _ in 0..offset_count { - d.read_i64().expect("old_style_offsets entry"); - } - assert_eq!(d.remaining(), 0); -} - -/// `ListOffsets` v0 request below firewall min (mirrors atharvalade repro script body). -fn build_list_offsets_v0_request_with_topic_t() -> Bytes { - let mut body = BytesMut::new(); - body.put_i32(-1); // replica_id - body.put_i32(1); // topics array length - body.put_i16(1); // topic name length - body.put_u8(b't'); - body.put_i32(1); // partitions array length - body.put_i32(0); // partition index - body.put_i64(-1); // timestamp - body.put_i32(1); // max_num_offsets - body.freeze() -} - -#[tokio::test] -async fn e2e_list_offsets_v0_unsupported_version_no_trailing_bytes() { - let (addr, _shutdown) = spawn_test_server().await; - let mut stream = TcpStream::connect(addr).await.expect("connect"); - - let request_body = build_list_offsets_v0_request_with_topic_t(); - let frame = build_request_frame( - API_KEY_LIST_OFFSETS, - 0, - 7, - Some("review-test"), - &request_body, - ); - stream - .write_all(&frame) - .await - .expect("write list offsets v0"); - - let payload = - read_response_frame_with_timeout(&mut stream, 8 * 1024 * 1024, Duration::from_secs(2)) - .await - .expect("ListOffsets v0 should still get an error response"); - - let (_corr, body) = parse_response_payload(API_KEY_LIST_OFFSETS, 0, payload); - let mut d = Decoder::new(body); - assert_eq!(d.read_i32().unwrap(), 1); - d.read_nullable_string().unwrap(); - assert_eq!(d.read_i32().unwrap(), 1); - parse_list_offsets_v0_partition(&mut d); - assert_eq!(d.remaining(), 0); -} - -// ── Metadata topic name echo (review: must not hardcode "unknown-topic") ──── - -#[test] -fn metadata_v10_unsupported_closes_connection() { - // Clamped v9 encoding cannot be parsed by a v10 client; close instead of lying on the wire. - assert!( - handle_request( - API_KEY_METADATA, - 10, - build_metadata_flexible_request_v10(&["payments"]), - &default_broker(), - ) - .is_close(), - "Metadata v10 must close rather than return a clamped unsupported-version body" - ); -} - -fn read_metadata_v1_topics(d: &mut Decoder, expected_count: i32) -> Vec { - let _brokers_count = d.read_i32().unwrap(); - d.read_i32().unwrap(); // node_id - d.read_nullable_string().unwrap(); // host - d.read_i32().unwrap(); // port - d.read_nullable_string().unwrap(); // rack (v1+) - d.read_i32().unwrap(); // controller_id (v1+) - - assert_eq!(d.read_i32().unwrap(), expected_count); - let mut names = Vec::with_capacity(usize::try_from(expected_count).unwrap_or(0)); - for _ in 0..expected_count { - d.read_i16().unwrap(); // topic_error - names.push(d.read_nullable_string().unwrap().expect("topic name")); - d.read_bool().unwrap(); // is_internal (v1+) - assert_eq!(d.read_i32().unwrap(), 0, "empty partitions array"); - } - names -} - -#[test] -fn metadata_v1_echoes_requested_topic_name_in_response() { - let topic = "orders"; - let request = build_metadata_legacy_request(&[topic]); - let body = handle_request(API_KEY_METADATA, 1, request, &default_broker()) - .expect_response("test request has acks != 0 and expects a response"); - let mut d = Decoder::new(body); - - let names = read_metadata_v1_topics(&mut d, 1); - assert_eq!(names, vec![topic.to_string()]); - assert_eq!(d.remaining(), 0); -} - -#[test] -fn metadata_v1_unknown_topic_returns_error_with_requested_name() { - let topic = "orders"; - let request = build_metadata_legacy_request(&[topic]); - let body = handle_request(API_KEY_METADATA, 1, request, &default_broker()) - .expect_response("test request has acks != 0 and expects a response"); - let mut d = Decoder::new(body); - - let _brokers_count = d.read_i32().unwrap(); - d.read_i32().unwrap(); - d.read_nullable_string().unwrap(); - d.read_i32().unwrap(); - d.read_nullable_string().unwrap(); - d.read_i32().unwrap(); - - assert_eq!(d.read_i32().unwrap(), 1); - assert_eq!( - d.read_i16().unwrap(), - ERROR_UNKNOWN_TOPIC_OR_PARTITION, - "unknown topic should surface error 3" - ); - assert_eq!( - d.read_nullable_string().unwrap().as_deref(), - Some(topic), - "response must echo requested topic name, not a placeholder" - ); -} - -#[tokio::test] -async fn e2e_metadata_v1_response_contains_requested_topic_name() { - let (addr, _shutdown) = spawn_test_server().await; - let topic = "orders"; - let request_body = build_metadata_legacy_request(&[topic]); - let frame = build_request_frame(API_KEY_METADATA, 1, 9, Some("review-test"), &request_body); - - let mut stream = TcpStream::connect(addr).await.expect("connect"); - stream.write_all(&frame).await.expect("write metadata v1"); - - let payload = - read_response_frame_with_timeout(&mut stream, 8 * 1024 * 1024, Duration::from_secs(2)) - .await - .expect("metadata response"); - - let full_response = { - let mut framed = BytesMut::with_capacity(4 + payload.len()); - framed.put_i32(i32::try_from(payload.len()).expect("metadata response fits i32")); - framed.extend_from_slice(&payload); - framed.freeze() - }; - - assert!( - full_response - .windows(topic.len()) - .any(|window| window == topic.as_bytes()), - "metadata response must contain requested topic name {topic:?}; \ - placeholder-only responses break client topic matching" - ); - assert!( - !full_response - .windows(b"unknown-topic".len()) - .any(|window| window == b"unknown-topic"), - "metadata response must not substitute unknown-topic for requested names" - ); -} - -// ── Idle connection handling (review: read_timeout must not act as idle cap) ─ - -#[tokio::test] -async fn e2e_quiet_connection_accepts_request_after_short_idle() { - let (addr, _shutdown) = spawn_test_server().await; - let mut stream = TcpStream::connect(addr).await.expect("connect"); - - time::sleep(Duration::from_secs(2)).await; - - let frame = build_request_frame(API_KEY_API_VERSIONS, 1, 501, Some("idle-test"), &[]); - stream - .write_all(&frame) - .await - .expect("connection should stay open after short idle"); - - let payload = - read_response_frame_with_timeout(&mut stream, 8 * 1024 * 1024, Duration::from_secs(2)) - .await - .expect("request after short idle should succeed"); - - let (corr, _) = parse_response_payload(API_KEY_API_VERSIONS, 1, payload); - assert_eq!(corr, 501); -} - -#[tokio::test] -async fn e2e_quiet_connection_survives_beyond_read_timeout_idle_cap() { - let (addr, _shutdown) = spawn_test_server_with_config(ServerConfig { - bind_addr: String::new(), - advertised_host: None, - advertised_port: None, - max_frame_size: 8 * 1024 * 1024, - read_timeout: Duration::from_secs(3), - write_timeout: Duration::from_secs(5), - }) - .await; - - let mut stream = TcpStream::connect(addr).await.expect("connect"); - - // Idle longer than read_timeout: prefix wait has no timer; connection stays open. - time::sleep(Duration::from_secs(4)).await; - - let frame = build_request_frame(API_KEY_API_VERSIONS, 1, 502, Some("idle-test"), &[]); - stream - .write_all(&frame) - .await - .expect("idle connection should remain writable after read_timeout elapses"); - - let payload = - read_response_frame_with_timeout(&mut stream, 8 * 1024 * 1024, Duration::from_secs(2)) - .await - .expect( - "request after idle period longer than read_timeout should succeed \ - (read_timeout applies to in-flight frame body only)", - ); - - let (corr, _) = parse_response_payload(API_KEY_API_VERSIONS, 1, payload); - assert_eq!(corr, 502); -} diff --git a/gateways/kafka/tests/scope_coverage_tests.rs b/gateways/kafka/tests/scope_coverage_tests.rs deleted file mode 100644 index 1b2347a112..0000000000 --- a/gateways/kafka/tests/scope_coverage_tests.rs +++ /dev/null @@ -1,823 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Comprehensive scoped-API coverage per SCOPE.md and Kafka protocol spec. -//! -//! Fills gaps not covered by existing regression suites. Some tests encode -//! client-correct behavior and fail until implementation catches up. - -#[path = "common/fixtures.rs"] -mod fixtures; -#[path = "common/scope.rs"] -mod scope; -#[path = "common/server.rs"] -mod server; -#[path = "common/tcp.rs"] -mod tcp; -#[path = "common/wire.rs"] -mod wire; - -use std::time::Duration; - -use bytes::{BufMut, Bytes, BytesMut}; -use tokio::io::AsyncWriteExt; -use tokio::net::TcpStream; - -use iggy_gateway_kafka::protocol::api::{ - API_KEY_API_VERSIONS, API_KEY_CREATE_TOPICS, API_KEY_FETCH, API_KEY_LIST_OFFSETS, - API_KEY_METADATA, API_KEY_PRODUCE, ERROR_INVALID_REQUEST, ERROR_NONE, - ERROR_UNKNOWN_TOPIC_OR_PARTITION, ERROR_UNSUPPORTED_VERSION, advertised_min_version, - handle_request, is_supported_version, -}; -use iggy_gateway_kafka::protocol::codec::Decoder; - -use fixtures::{fixture_exists, load_fixture_body}; -use scope::{SCOPED_API_KEYS, default_broker}; -use server::spawn_test_server; -use tcp::{ - build_metadata_legacy_request, build_produce_v3_body, build_request_frame, - parse_response_payload, round_trip, -}; -use wire::{ - OUT_OF_SCOPE_API_KEYS, build_create_topics_empty_request, build_fetch_empty_topics_request, - build_list_offsets_request, build_metadata_flexible_request, - build_produce_flexible_empty_request, -}; - -fn metadata_empty_legacy_body() -> Bytes { - let mut body = BytesMut::new(); - body.put_i32(0); - body.freeze() -} - -fn request_body_for_scoped_api(api_key: i16, name: &str, version: i16) -> Bytes { - match api_key { - API_KEY_METADATA => metadata_empty_legacy_body(), - API_KEY_PRODUCE => { - if fixture_exists(api_key, name, version) { - load_fixture_body(api_key, name, version) - } else { - build_produce_v3_body(1, 0) - } - } - API_KEY_FETCH => { - if fixture_exists(api_key, name, version) { - load_fixture_body(api_key, name, version) - } else { - build_fetch_empty_topics_request(version) - } - } - API_KEY_LIST_OFFSETS => { - if fixture_exists(api_key, name, version) { - load_fixture_body(api_key, name, version) - } else { - build_list_offsets_request(version, "scope-topic", 0) - } - } - API_KEY_CREATE_TOPICS => build_create_topics_empty_request(version), - _ => Bytes::new(), - } -} - -// ── Correlation ID preservation (all scoped keys × min/max/flexible) ──────── - -#[tokio::test] -async fn each_scoped_api_min_version_preserves_correlation_id_e2e() { - let (addr, _shutdown) = spawn_test_server().await; - - for &(api_key, name, min_ver, _max_ver) in SCOPED_API_KEYS { - let correlation_id = 10_000 + i32::from(api_key); - let body = request_body_for_scoped_api(api_key, name, min_ver); - let (corr, resp_body) = round_trip(addr, api_key, min_ver, correlation_id, &body).await; - assert_eq!( - corr, correlation_id, - "{name} v{min_ver} correlation id must round-trip" - ); - assert!( - !resp_body.is_empty(), - "{name} v{min_ver} must return non-empty body" - ); - } -} - -#[tokio::test] -async fn each_scoped_api_max_version_preserves_correlation_id_e2e() { - let (addr, _shutdown) = spawn_test_server().await; - - for &(api_key, name, _min_ver, max_ver) in SCOPED_API_KEYS { - let correlation_id = 20_000 + i32::from(api_key); - let body = request_body_for_scoped_api(api_key, name, max_ver); - let (corr, resp_body) = round_trip(addr, api_key, max_ver, correlation_id, &body).await; - assert_eq!( - corr, correlation_id, - "{name} v{max_ver} correlation id must round-trip" - ); - assert!( - !resp_body.is_empty(), - "{name} v{max_ver} must return non-empty body" - ); - } -} - -#[tokio::test] -async fn apiversions_v0_and_v2_e2e_return_success() { - let (addr, _shutdown) = spawn_test_server().await; - - for version in [0i16, 2] { - let correlation_id = 300 + i32::from(version); - let (corr, body) = - round_trip(addr, API_KEY_API_VERSIONS, version, correlation_id, &[]).await; - assert_eq!(corr, correlation_id); - let mut d = Decoder::new(body); - assert_eq!( - d.read_i16().unwrap(), - ERROR_NONE, - "ApiVersions v{version} must succeed" - ); - } -} - -// ── Out-of-scope API keys (SCOPE.md unsupported list) ─────────────────────── - -#[test] -fn out_of_scope_api_keys_return_unsupported_version_without_panic() { - for &(api_key, name) in OUT_OF_SCOPE_API_KEYS { - let body = handle_request(api_key, 0, Bytes::new(), &default_broker()) - .expect_response("test request has acks != 0 and expects a response"); - let mut d = Decoder::new(body); - assert_eq!( - d.read_i16().unwrap(), - ERROR_UNSUPPORTED_VERSION, - "{name} (key {api_key})" - ); - } -} - -#[tokio::test] -async fn out_of_scope_api_keys_e2e_respond_then_close() { - let (addr, _shutdown) = spawn_test_server().await; - - for &(api_key, name) in &OUT_OF_SCOPE_API_KEYS[..4] { - let mut stream = TcpStream::connect(addr).await.expect("connect"); - let frame = build_request_frame(api_key, 0, i32::from(api_key), Some("scope-test"), &[]); - stream.write_all(&frame).await.expect("write oos key"); - - let payload = tcp::read_response_frame(&mut stream, 8 * 1024 * 1024).await; - let mut d = Decoder::new(parse_response_payload(api_key, 0, payload).1); - assert_eq!( - d.read_i16().unwrap(), - ERROR_UNSUPPORTED_VERSION, - "{name} (key {api_key})" - ); - - assert_eq!( - tcp::read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await, - tcp::ByteRead::Closed, - "{name} (key {api_key}) must close the connection after the error response" - ); - } -} - -// ── Version firewall: unsupported version keeps TCP session (except Metadata) ─ - -#[tokio::test] -async fn each_scoped_api_above_max_version_e2e_keeps_connection() { - let (addr, _shutdown) = spawn_test_server().await; - let mut stream = TcpStream::connect(addr).await.expect("connect"); - - for &(api_key, name, _min_ver, max_ver) in SCOPED_API_KEYS { - let above = max_ver + 1; - let frame = build_request_frame( - api_key, - above, - 50_000 + i32::from(api_key), - Some("scope-test"), - &[], - ); - stream - .write_all(&frame) - .await - .unwrap_or_else(|_| panic!("write {name} v{above}")); - if api_key == API_KEY_METADATA { - // Unsupported Metadata closes: clamped bodies are unparsable at the client version. - assert_eq!( - tcp::read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await, - tcp::ByteRead::Closed, - "Metadata v{above} must close the connection" - ); - stream = TcpStream::connect(addr) - .await - .expect("reconnect after Metadata close"); - continue; - } - let payload = tcp::read_response_frame(&mut stream, 8 * 1024 * 1024).await; - assert!( - !payload.is_empty(), - "{name} v{above} must still respond on wire" - ); - } - - let ok = build_request_frame(API_KEY_API_VERSIONS, 1, 89_999, Some("scope-test"), &[]); - stream.write_all(&ok).await.expect("recovery request"); - let payload = tcp::read_response_frame(&mut stream, 8 * 1024 * 1024).await; - assert_eq!( - parse_response_payload(API_KEY_API_VERSIONS, 1, payload).0, - 89_999 - ); -} - -#[tokio::test] -async fn each_scoped_api_below_min_version_e2e_keeps_connection() { - let (addr, _shutdown) = spawn_test_server().await; - let mut stream = TcpStream::connect(addr).await.expect("connect"); - - for &(api_key, name, min_ver, _max_ver) in SCOPED_API_KEYS { - let below = min_ver - 1; - let frame = build_request_frame( - api_key, - below, - 40_000 + i32::from(api_key), - Some("scope-test"), - &[], - ); - stream - .write_all(&frame) - .await - .unwrap_or_else(|_| panic!("write {name} v{below}")); - if api_key == API_KEY_METADATA { - assert_eq!( - tcp::read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await, - tcp::ByteRead::Closed, - "Metadata v{below} must close the connection" - ); - stream = TcpStream::connect(addr) - .await - .expect("reconnect after Metadata close"); - continue; - } - let payload = tcp::read_response_frame(&mut stream, 8 * 1024 * 1024).await; - assert!( - !payload.is_empty(), - "{name} v{below} must still respond on wire" - ); - } - - let ok = build_request_frame(API_KEY_API_VERSIONS, 1, 88_888, Some("scope-test"), &[]); - stream.write_all(&ok).await.expect("recovery request"); - let payload = tcp::read_response_frame(&mut stream, 8 * 1024 * 1024).await; - assert_eq!( - parse_response_payload(API_KEY_API_VERSIONS, 1, payload).0, - 88_888 - ); -} - -#[test] -fn produce_advertises_min_zero_but_firewall_rejects_below_v3() { - let range = scope::SCOPED_API_KEYS - .iter() - .find(|(k, _, _, _)| *k == API_KEY_PRODUCE) - .expect("produce in scope"); - let (_, _, firewall_min, _) = *range; - assert_eq!(firewall_min, 3); - assert_eq!(advertised_min_version(API_KEY_PRODUCE, firewall_min), 0); - assert!(!is_supported_version(API_KEY_PRODUCE, 0)); - assert!(!is_supported_version(API_KEY_PRODUCE, 2)); - - let body = handle_request(API_KEY_PRODUCE, 2, Bytes::new(), &default_broker()) - .expect_response("test request has acks != 0 and expects a response"); - let mut d = Decoder::new(body); - let _topics = d.read_i32().unwrap(); - let _name = d.read_nullable_string().unwrap(); - let _parts = d.read_i32().unwrap(); - assert_eq!(d.read_i32().unwrap(), 0, "partition index"); - assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); -} - -// ── Metadata (spec + SCOPE) ───────────────────────────────────────────────── - -#[test] -fn metadata_v0_empty_topics_returns_zero_length_topic_array() { - let body = handle_request( - API_KEY_METADATA, - 0, - metadata_empty_legacy_body(), - &default_broker(), - ) - .expect_response("test request has acks != 0 and expects a response"); - let mut d = Decoder::new(body); - let _brokers = d.read_i32().unwrap(); - d.read_i32().unwrap(); - d.read_nullable_string().unwrap(); - d.read_i32().unwrap(); - assert_eq!(d.read_i32().unwrap(), 0, "empty request → zero topics"); - assert_eq!(d.remaining(), 0); -} - -#[test] -fn metadata_v3_includes_throttle_time_ms_before_brokers() { - let body = handle_request( - API_KEY_METADATA, - 3, - metadata_empty_legacy_body(), - &default_broker(), - ) - .expect_response("test request has acks != 0 and expects a response"); - let mut d = Decoder::new(body); - assert_eq!(d.read_i32().unwrap(), 0, "throttle_time_ms"); -} - -#[test] -fn metadata_v9_flexible_empty_topics_returns_zero_topics() { - let body = handle_request( - API_KEY_METADATA, - 9, - build_metadata_flexible_request(&[]), - &default_broker(), - ) - .expect_response("test request has acks != 0 and expects a response"); - let mut d = Decoder::new(body); - d.read_i32().unwrap(); // throttle - let broker_count = usize::try_from(d.read_varint().unwrap()) - .unwrap() - .saturating_sub(1); - for _ in 0..broker_count { - d.read_i32().unwrap(); - d.read_compact_nullable_string().unwrap(); - d.read_i32().unwrap(); - d.read_compact_nullable_string().unwrap(); - d.read_tagged_fields().unwrap(); - } - d.read_compact_nullable_string().unwrap(); // cluster_id - d.read_i32().unwrap(); // controller_id - let topic_count = usize::try_from(d.read_varint().unwrap()) - .unwrap() - .saturating_sub(1); - assert_eq!(topic_count, 0); -} - -#[test] -fn metadata_v9_flexible_echoes_each_requested_topic_name() { - let topics = ["orders", "payments", "inventory"]; - let body = handle_request( - API_KEY_METADATA, - 9, - build_metadata_flexible_request(&topics), - &default_broker(), - ) - .expect_response("test request has acks != 0 and expects a response"); - - let mut d = Decoder::new(body); - d.read_i32().unwrap(); - let broker_count = usize::try_from(d.read_varint().unwrap()) - .unwrap() - .saturating_sub(1); - for _ in 0..broker_count { - d.read_i32().unwrap(); - d.read_compact_nullable_string().unwrap(); - d.read_i32().unwrap(); - d.read_compact_nullable_string().unwrap(); - d.read_tagged_fields().unwrap(); - } - d.read_compact_nullable_string().unwrap(); - d.read_i32().unwrap(); - - let topic_count = usize::try_from(d.read_varint().unwrap()) - .unwrap() - .saturating_sub(1); - assert_eq!(topic_count, topics.len()); - - let mut names = Vec::new(); - for _ in 0..topic_count { - assert_eq!( - d.read_i16().unwrap(), - ERROR_UNKNOWN_TOPIC_OR_PARTITION, - "stub gateway returns unknown topic error per topic" - ); - names.push( - d.read_compact_nullable_string() - .unwrap() - .expect("topic name"), - ); - d.read_bool().unwrap(); - assert_eq!( - usize::try_from(d.read_varint().unwrap()) - .unwrap() - .saturating_sub(1), - 0, - "empty partitions array" - ); - d.read_i32().unwrap(); // topic_authorized_operations (v8+) - d.read_tagged_fields().unwrap(); - } - - assert_eq!( - names, - topics - .iter() - .map(|topic| (*topic).to_string()) - .collect::>(), - "metadata must echo requested topic names for client matching" - ); -} - -#[test] -fn metadata_v1_legacy_multiple_topics_echo_names() { - let topics = ["alpha", "beta"]; - let body = handle_request( - API_KEY_METADATA, - 1, - build_metadata_legacy_request(&topics), - &default_broker(), - ) - .expect_response("test request has acks != 0 and expects a response"); - - let mut d = Decoder::new(body); - d.read_i32().unwrap(); - d.read_i32().unwrap(); - d.read_nullable_string().unwrap(); - d.read_i32().unwrap(); - d.read_nullable_string().unwrap(); - d.read_i32().unwrap(); - assert_eq!(d.read_i32().unwrap(), 2); - - for expected in topics { - assert_eq!(d.read_i16().unwrap(), ERROR_UNKNOWN_TOPIC_OR_PARTITION); - assert_eq!( - d.read_nullable_string().unwrap().as_deref(), - Some(expected), - "metadata v1 must echo {expected}" - ); - d.read_bool().unwrap(); - assert_eq!(d.read_i32().unwrap(), 0); - } -} - -// ── Produce (Kafka spec acks semantics) ───────────────────────────────────── - -#[tokio::test] -async fn e2e_produce_v3_acks_all_minus_one_returns_response() { - let (addr, _shutdown) = spawn_test_server().await; - let body = build_produce_v3_body(-1, 0); - let (corr, resp) = round_trip(addr, API_KEY_PRODUCE, 3, 501, &body).await; - assert_eq!(corr, 501); - assert!(!resp.is_empty()); -} - -#[tokio::test] -async fn e2e_produce_v9_flexible_header_with_empty_topics() { - let (addr, _shutdown) = spawn_test_server().await; - let body = build_produce_flexible_empty_request(1); - let (corr, resp) = round_trip(addr, API_KEY_PRODUCE, 9, 502, &body).await; - assert_eq!(corr, 502); - let mut d = Decoder::new(resp); - assert!( - d.read_varint().unwrap() >= 1, - "flexible topics array header" - ); -} - -#[tokio::test] -async fn produce_v3_through_v9_e2e_preserve_correlation_id() { - let (addr, _shutdown) = spawn_test_server().await; - - for version in 3i16..=9 { - let body = if version >= 9 { - build_produce_flexible_empty_request(1) - } else { - build_produce_v3_body(1, 0) - }; - let correlation_id = 510 + i32::from(version); - let (corr, resp) = round_trip(addr, API_KEY_PRODUCE, version, correlation_id, &body).await; - assert_eq!(corr, correlation_id, "Produce v{version} correlation"); - assert!(!resp.is_empty(), "Produce v{version} response"); - } -} - -// ── ListOffsets supported versions ────────────────────────────────────────── - -#[tokio::test] -async fn list_offsets_v1_through_v6_e2e_return_partition_error_zero() { - let (addr, _shutdown) = spawn_test_server().await; - - for version in 1i16..=6 { - let body = build_list_offsets_request(version, "offsets-topic", 0); - let correlation_id = 600 + i32::from(version); - let (corr, resp) = - round_trip(addr, API_KEY_LIST_OFFSETS, version, correlation_id, &body).await; - assert_eq!(corr, correlation_id); - - let flexible = version >= 6; - let mut d = Decoder::new(resp); - if version >= 2 { - d.read_i32().unwrap(); - } - if flexible { - d.read_varint().unwrap(); - d.read_compact_nullable_string().unwrap(); - d.read_varint().unwrap(); - } else { - d.read_i32().unwrap(); - d.read_nullable_string().unwrap(); - d.read_i32().unwrap(); - } - d.read_i32().unwrap(); - assert_eq!( - d.read_i16().unwrap(), - ERROR_NONE, - "ListOffsets v{version} stub partition error" - ); - } -} - -// ── CreateTopics empty create ─────────────────────────────────────────────── - -#[tokio::test] -async fn create_topics_v2_through_v5_empty_request_e2e_succeeds() { - let (addr, _shutdown) = spawn_test_server().await; - - for version in 2i16..=5 { - let body = build_create_topics_empty_request(version); - let correlation_id = 700 + i32::from(version); - let (corr, resp) = - round_trip(addr, API_KEY_CREATE_TOPICS, version, correlation_id, &body).await; - assert_eq!(corr, correlation_id); - let mut d = Decoder::new(resp); - if version >= 2 { - d.read_i32().unwrap(); - } - if version >= 5 { - assert!(d.read_varint().unwrap() >= 1); - } else { - assert_eq!(d.read_i32().unwrap(), 0); - } - } -} - -// ── Fetch flexible boundary ───────────────────────────────────────────────── - -#[tokio::test] -async fn fetch_v4_through_v12_e2e_preserve_correlation_id() { - let (addr, _shutdown) = spawn_test_server().await; - - for version in 4i16..=12 { - let body = request_body_for_scoped_api(API_KEY_FETCH, "Fetch", version); - let correlation_id = 800 + i32::from(version); - let (corr, resp) = round_trip(addr, API_KEY_FETCH, version, correlation_id, &body).await; - assert_eq!(corr, correlation_id, "Fetch v{version} correlation"); - assert!(!resp.is_empty(), "Fetch v{version} response"); - } -} - -#[tokio::test] -async fn apiversions_v4_out_of_range_e2e_returns_unsupported() { - let (addr, _shutdown) = spawn_test_server().await; - let (corr, body) = round_trip(addr, API_KEY_API_VERSIONS, 4, 350, &[]).await; - assert_eq!(corr, 350); - let mut d = Decoder::new(body); - assert_eq!( - d.read_i16().unwrap(), - ERROR_UNSUPPORTED_VERSION, - "ApiVersions v4 must return UNSUPPORTED_VERSION per KIP-511" - ); -} - -#[tokio::test] -async fn metadata_empty_body_e2e_all_topics_request_returns_broker() { - let (addr, _shutdown) = spawn_test_server().await; - let (corr, body) = round_trip(addr, API_KEY_METADATA, 0, 360, &[]).await; - assert_eq!(corr, 360); - let mut d = Decoder::new(body); - assert_eq!(d.read_i32().unwrap(), 1, "one stub broker"); - d.read_i32().unwrap(); // node_id - let host = d.read_nullable_string().unwrap().expect("broker host"); - assert!(!host.is_empty()); - let port = d.read_i32().unwrap(); - assert!(port > 0); -} - -#[tokio::test] -async fn list_offsets_v7_unsupported_e2e_returns_error() { - let (addr, _shutdown) = spawn_test_server().await; - let (corr, body) = round_trip( - addr, - API_KEY_LIST_OFFSETS, - 7, - 370, - &[0x00, 0x00, 0x00, 0x00], - ) - .await; - assert_eq!(corr, 370); - assert!( - scan_for_error_code(&body, ERROR_UNSUPPORTED_VERSION), - "ListOffsets v7 must be rejected" - ); -} - -#[tokio::test] -async fn create_topics_v1_unsupported_e2e_returns_error() { - let (addr, _shutdown) = spawn_test_server().await; - let (corr, body) = round_trip(addr, API_KEY_CREATE_TOPICS, 1, 380, &[]).await; - assert_eq!(corr, 380); - assert!( - scan_for_error_code(&body, ERROR_UNSUPPORTED_VERSION), - "CreateTopics v1 must be rejected" - ); -} - -#[tokio::test] -async fn corrupt_produce_body_e2e_returns_error_without_disconnect() { - let (addr, _shutdown) = spawn_test_server().await; - let mut stream = TcpStream::connect(addr).await.expect("connect"); - - let bad = build_request_frame( - API_KEY_PRODUCE, - 3, - 391, - Some("scope-test"), - &[0xFF, 0xFF, 0xFF], - ); - stream.write_all(&bad).await.expect("corrupt produce"); - let payload = tcp::read_response_frame(&mut stream, 8 * 1024 * 1024).await; - assert!( - scan_for_error_code( - &parse_response_payload(API_KEY_PRODUCE, 3, payload).1, - ERROR_INVALID_REQUEST - ), - "corrupt Produce must surface INVALID_REQUEST" - ); - - let ok = build_request_frame(API_KEY_API_VERSIONS, 1, 392, Some("scope-test"), &[]); - stream.write_all(&ok).await.expect("follow-up"); - let payload = tcp::read_response_frame(&mut stream, 8 * 1024 * 1024).await; - assert_eq!( - parse_response_payload(API_KEY_API_VERSIONS, 1, payload).0, - 392 - ); -} - -#[tokio::test] -async fn corrupt_fetch_body_e2e_returns_error_without_disconnect() { - let (addr, _shutdown) = spawn_test_server().await; - let mut stream = TcpStream::connect(addr).await.expect("connect"); - - let bad = build_request_frame( - API_KEY_FETCH, - 4, - 393, - Some("scope-test"), - &[0xFF, 0xFF, 0xFF], - ); - stream.write_all(&bad).await.expect("corrupt fetch"); - let payload = tcp::read_response_frame(&mut stream, 8 * 1024 * 1024).await; - assert!( - scan_for_error_code( - &parse_response_payload(API_KEY_FETCH, 4, payload).1, - ERROR_INVALID_REQUEST - ), - "corrupt Fetch must surface INVALID_REQUEST" - ); - - let ok = build_request_frame(API_KEY_API_VERSIONS, 1, 394, Some("scope-test"), &[]); - stream.write_all(&ok).await.expect("follow-up"); - let payload = tcp::read_response_frame(&mut stream, 8 * 1024 * 1024).await; - assert_eq!( - parse_response_payload(API_KEY_API_VERSIONS, 1, payload).0, - 394 - ); -} - -// ── Corrupt decode paths for remaining scoped APIs ────────────────────────── - -#[test] -fn corrupt_list_offsets_body_returns_invalid_request_error() { - let body = Bytes::from_static(&[0xFF, 0xFF, 0xFF]); - let resp = handle_request(API_KEY_LIST_OFFSETS, 1, body, &default_broker()) - .expect_response("test request has acks != 0 and expects a response"); - assert!(!resp.is_empty()); - assert!( - scan_for_error_code(&resp, ERROR_INVALID_REQUEST) - || scan_for_error_code(&resp, ERROR_UNSUPPORTED_VERSION), - "corrupt ListOffsets must surface protocol error" - ); -} - -#[test] -fn corrupt_create_topics_body_returns_invalid_request_error() { - let body = Bytes::from_static(&[0xFF, 0xFF, 0xFF]); - let resp = handle_request(API_KEY_CREATE_TOPICS, 2, body, &default_broker()) - .expect_response("test request has acks != 0 and expects a response"); - assert!(!resp.is_empty()); - assert!( - scan_for_error_code(&resp, ERROR_INVALID_REQUEST) - || scan_for_error_code(&resp, ERROR_UNSUPPORTED_VERSION) - ); -} - -#[test] -fn corrupt_metadata_body_returns_zero_topics_not_panic() { - let body = Bytes::from_static(&[0x00, 0x00]); - let resp = handle_request(API_KEY_METADATA, 0, body, &default_broker()) - .expect_response("test request has acks != 0 and expects a response"); - assert!(!resp.is_empty()); - let mut d = Decoder::new(resp); - d.read_i32().unwrap(); - d.read_i32().unwrap(); - d.read_nullable_string().unwrap(); - d.read_i32().unwrap(); - assert_eq!( - d.read_i32().unwrap(), - 0, - "malformed topic list → zero topics" - ); -} - -fn scan_for_error_code(body: &Bytes, code: i16) -> bool { - body.windows(2) - .any(|w| i16::from_be_bytes([w[0], w[1]]) == code) -} - -// ── Flexible encoding boundaries (SCOPE.md) ─────────────────────────────────── - -#[test] -fn request_header_version_switches_at_scope_flexible_boundaries() { - use iggy_gateway_kafka::protocol::header::request_header_version; - - assert_eq!(request_header_version(API_KEY_PRODUCE, 8), 1); - assert_eq!(request_header_version(API_KEY_PRODUCE, 9), 2); - assert_eq!(request_header_version(API_KEY_FETCH, 11), 1); - assert_eq!(request_header_version(API_KEY_FETCH, 12), 2); - assert_eq!(request_header_version(API_KEY_LIST_OFFSETS, 5), 1); - assert_eq!(request_header_version(API_KEY_LIST_OFFSETS, 6), 2); - assert_eq!(request_header_version(API_KEY_METADATA, 8), 1); - assert_eq!(request_header_version(API_KEY_METADATA, 9), 2); - assert_eq!(request_header_version(API_KEY_API_VERSIONS, 2), 1); - assert_eq!(request_header_version(API_KEY_API_VERSIONS, 3), 2); - assert_eq!(request_header_version(API_KEY_CREATE_TOPICS, 4), 1); - assert_eq!(request_header_version(API_KEY_CREATE_TOPICS, 5), 2); -} - -#[test] -fn metadata_v9_request_with_three_topics_yields_three_response_slots() { - let topics = ["a", "b", "c"]; - let body = handle_request( - API_KEY_METADATA, - 9, - build_metadata_flexible_request(&topics), - &default_broker(), - ) - .expect_response("test request has acks != 0 and expects a response"); - let mut d = Decoder::new(body); - d.read_i32().unwrap(); - skip_metadata_v9_prefix(&mut d); - let topic_count = usize::try_from(d.read_varint().unwrap()) - .unwrap() - .saturating_sub(1); - assert_eq!( - topic_count, - topics.len(), - "response topic count must mirror request topic count" - ); -} - -fn skip_metadata_v9_prefix(d: &mut Decoder) { - let broker_count = usize::try_from(d.read_varint().unwrap()) - .unwrap() - .saturating_sub(1); - for _ in 0..broker_count { - d.read_i32().unwrap(); - d.read_compact_nullable_string().unwrap(); - d.read_i32().unwrap(); - d.read_compact_nullable_string().unwrap(); - d.read_tagged_fields().unwrap(); - } - d.read_compact_nullable_string().unwrap(); - d.read_i32().unwrap(); -} - -// ── Handler-level: every in-range version returns bytes ───────────────────── - -#[test] -fn every_in_range_version_returns_non_empty_handler_response() { - for &(api_key, name, min_ver, max_ver) in SCOPED_API_KEYS { - for version in min_ver..=max_ver { - let body = request_body_for_scoped_api(api_key, name, version); - let resp = handle_request(api_key, version, body, &default_broker()) - .expect_response("test request has acks != 0 and expects a response"); - assert!(!resp.is_empty(), "{name} v{version} handler returned empty"); - } - } -} diff --git a/gateways/kafka/tests/server_e2e_tests.rs b/gateways/kafka/tests/server_e2e_tests.rs index 73d0ec7e82..36d3e371d8 100644 --- a/gateways/kafka/tests/server_e2e_tests.rs +++ b/gateways/kafka/tests/server_e2e_tests.rs @@ -23,13 +23,16 @@ mod fixtures; mod server; #[path = "common/tcp.rs"] mod tcp; +#[path = "common/wire.rs"] +mod wire; -use bytes::{BufMut, BytesMut}; +use bytes::{BufMut, Bytes, BytesMut}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpStream; use iggy_gateway_kafka::protocol::api::{ - API_KEY_API_VERSIONS, API_KEY_METADATA, API_KEY_PRODUCE, ERROR_UNSUPPORTED_VERSION, + API_KEY_API_VERSIONS, API_KEY_CREATE_TOPICS, API_KEY_FETCH, API_KEY_LIST_OFFSETS, + API_KEY_METADATA, API_KEY_PRODUCE, ERROR_NONE, ERROR_UNSUPPORTED_VERSION, }; use iggy_gateway_kafka::protocol::codec::Decoder; @@ -37,8 +40,13 @@ use fixtures::load_fixture_body_or_skip; use server::spawn_test_server; use std::time::Duration; use tcp::{ - ByteRead, build_request_frame, parse_response_payload, read_byte_with_timeout, - read_response_frame, round_trip, + ByteRead, build_list_offsets_v0_request_with_topic_t, build_metadata_legacy_request, + build_produce_v3_body, build_request_frame, parse_response_payload, read_byte_with_timeout, + read_response_frame, read_response_frame_with_timeout, round_trip, +}; +use wire::{ + OUT_OF_SCOPE_API_KEYS, build_create_topics_empty_request, build_fetch_empty_topics_request, + build_list_offsets_request, build_produce_flexible_empty_request, }; #[tokio::test] @@ -158,3 +166,315 @@ async fn e2e_oversized_frame_is_rejected() { let n = stream.read(&mut buf).await.unwrap_or(0); assert_eq!(n, 0, "server should close after oversized frame"); } + +// ── Produce acks=0 (broker must stay silent) ──────────────────────────────── + +#[tokio::test] +async fn e2e_produce_v3_acks_zero_sends_no_response() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.expect("connect"); + + let body = build_produce_v3_body(0, 0); + let frame = build_request_frame(API_KEY_PRODUCE, 3, 42, Some("review-test"), &body); + stream + .write_all(&frame) + .await + .expect("write produce acks=0"); + + let response = + read_response_frame_with_timeout(&mut stream, 8 * 1024 * 1024, Duration::from_millis(500)) + .await; + + assert!( + response.is_none(), + "Produce with acks=0 must not receive a response frame (Kafka spec); got {} bytes", + response.as_ref().map_or(0, Bytes::len) + ); +} + +#[tokio::test] +async fn e2e_produce_v3_acks_zero_malformed_topics_sends_no_response() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.expect("connect"); + + // acks=0, claims one topic, no topic bytes - decode fails after acks is read. + let body = build_produce_v3_body(0, 1); + let frame = build_request_frame(API_KEY_PRODUCE, 3, 99, Some("review-test"), &body); + stream + .write_all(&frame) + .await + .expect("write produce acks=0 malformed"); + + let response = + read_response_frame_with_timeout(&mut stream, 8 * 1024 * 1024, Duration::from_millis(500)) + .await; + + assert!( + response.is_none(), + "Produce with acks=0 must stay silent even when the body is malformed; got {} bytes", + response.as_ref().map_or(0, Bytes::len) + ); +} + +#[tokio::test] +async fn e2e_produce_v3_acks_one_still_returns_response() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.expect("connect"); + + let body = build_produce_v3_body(1, 0); + let frame = build_request_frame(API_KEY_PRODUCE, 3, 43, Some("review-test"), &body); + stream + .write_all(&frame) + .await + .expect("write produce acks=1"); + + let response = + read_response_frame_with_timeout(&mut stream, 8 * 1024 * 1024, Duration::from_secs(2)) + .await + .expect("Produce with acks=1 should receive a response"); + + let (corr, resp_body) = parse_response_payload(API_KEY_PRODUCE, 3, response); + assert_eq!(corr, 43); + assert!(!resp_body.is_empty()); +} + +// ── ListOffsets v0 wire shape (old_style_offsets array, not bare i64) ─────── + +#[tokio::test] +async fn e2e_list_offsets_v0_unsupported_version_no_trailing_bytes() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.expect("connect"); + + let request_body = build_list_offsets_v0_request_with_topic_t(); + let frame = build_request_frame( + API_KEY_LIST_OFFSETS, + 0, + 7, + Some("review-test"), + &request_body, + ); + stream + .write_all(&frame) + .await + .expect("write list offsets v0"); + + let payload = + read_response_frame_with_timeout(&mut stream, 8 * 1024 * 1024, Duration::from_secs(2)) + .await + .expect("ListOffsets v0 should still get an error response"); + + let (_corr, body) = parse_response_payload(API_KEY_LIST_OFFSETS, 0, payload); + let mut d = Decoder::new(body); + assert_eq!(d.read_i32().unwrap(), 1); + d.read_nullable_string().unwrap(); + assert_eq!(d.read_i32().unwrap(), 1); + let _partition_index = d.read_i32().expect("partition_index"); + let _error_code = d.read_i16().expect("error_code"); + let offset_count = d.read_i32().expect("old_style_offsets array length"); + assert!( + offset_count >= 0, + "old_style_offsets count must be non-negative, got {offset_count}" + ); + for _ in 0..offset_count { + d.read_i64().expect("old_style_offsets entry"); + } + assert_eq!(d.remaining(), 0); +} + +// ── Metadata topic name echo (must not hardcode a placeholder topic name) ── + +#[tokio::test] +async fn e2e_metadata_v1_response_contains_requested_topic_name() { + let (addr, _shutdown) = spawn_test_server().await; + let topic = "orders"; + let request_body = build_metadata_legacy_request(&[topic]); + let frame = build_request_frame(API_KEY_METADATA, 1, 9, Some("review-test"), &request_body); + + let mut stream = TcpStream::connect(addr).await.expect("connect"); + stream.write_all(&frame).await.expect("write metadata v1"); + + let payload = + read_response_frame_with_timeout(&mut stream, 8 * 1024 * 1024, Duration::from_secs(2)) + .await + .expect("metadata response"); + + let full_response = { + let mut framed = BytesMut::with_capacity(4 + payload.len()); + framed.put_i32(i32::try_from(payload.len()).expect("metadata response fits i32")); + framed.extend_from_slice(&payload); + framed.freeze() + }; + + assert!( + full_response + .windows(topic.len()) + .any(|window| window == topic.as_bytes()), + "metadata response must contain requested topic name {topic:?}; \ + placeholder-only responses break client topic matching" + ); + assert!( + !full_response + .windows(b"unknown-topic".len()) + .any(|window| window == b"unknown-topic"), + "metadata response must not substitute unknown-topic for requested names" + ); +} + +// ── Produce (Kafka spec acks semantics) ───────────────────────────────────── + +#[tokio::test] +async fn e2e_produce_v3_acks_all_minus_one_returns_response() { + let (addr, _shutdown) = spawn_test_server().await; + let body = build_produce_v3_body(-1, 0); + let (corr, resp) = round_trip(addr, API_KEY_PRODUCE, 3, 501, &body).await; + assert_eq!(corr, 501); + assert!(!resp.is_empty()); +} + +#[tokio::test] +async fn e2e_produce_v9_flexible_header_with_empty_topics() { + let (addr, _shutdown) = spawn_test_server().await; + let body = build_produce_flexible_empty_request(1); + let (corr, resp) = round_trip(addr, API_KEY_PRODUCE, 9, 502, &body).await; + assert_eq!(corr, 502); + let mut d = Decoder::new(resp); + assert!( + d.read_varint().unwrap() >= 1, + "flexible topics array header" + ); +} + +#[tokio::test] +async fn produce_v3_through_v9_e2e_preserve_correlation_id() { + let (addr, _shutdown) = spawn_test_server().await; + + for version in 3i16..=9 { + let body = if version >= 9 { + build_produce_flexible_empty_request(1) + } else { + build_produce_v3_body(1, 0) + }; + let correlation_id = 510 + i32::from(version); + let (corr, resp) = round_trip(addr, API_KEY_PRODUCE, version, correlation_id, &body).await; + assert_eq!(corr, correlation_id, "Produce v{version} correlation"); + assert!(!resp.is_empty(), "Produce v{version} response"); + } +} + +// ── ListOffsets supported versions ────────────────────────────────────────── + +#[tokio::test] +async fn list_offsets_v1_through_v6_e2e_return_partition_error_zero() { + let (addr, _shutdown) = spawn_test_server().await; + + for version in 1i16..=6 { + let body = build_list_offsets_request(version, "offsets-topic", 0); + let correlation_id = 600 + i32::from(version); + let (corr, resp) = + round_trip(addr, API_KEY_LIST_OFFSETS, version, correlation_id, &body).await; + assert_eq!(corr, correlation_id); + + let flexible = version >= 6; + let mut d = Decoder::new(resp); + if version >= 2 { + d.read_i32().unwrap(); + } + if flexible { + d.read_varint().unwrap(); + d.read_compact_nullable_string().unwrap(); + d.read_varint().unwrap(); + } else { + d.read_i32().unwrap(); + d.read_nullable_string().unwrap(); + d.read_i32().unwrap(); + } + d.read_i32().unwrap(); + assert_eq!( + d.read_i16().unwrap(), + ERROR_NONE, + "ListOffsets v{version} stub partition error" + ); + } +} + +// ── CreateTopics empty create ──────────────────────────────────────────────── + +#[tokio::test] +async fn create_topics_v2_through_v5_empty_request_e2e_succeeds() { + let (addr, _shutdown) = spawn_test_server().await; + + for version in 2i16..=5 { + let body = build_create_topics_empty_request(version); + let correlation_id = 700 + i32::from(version); + let (corr, resp) = + round_trip(addr, API_KEY_CREATE_TOPICS, version, correlation_id, &body).await; + assert_eq!(corr, correlation_id); + let mut d = Decoder::new(resp); + if version >= 2 { + d.read_i32().unwrap(); + } + if version >= 5 { + assert!(d.read_varint().unwrap() >= 1); + } else { + assert_eq!(d.read_i32().unwrap(), 0); + } + } +} + +// ── Fetch flexible boundary ────────────────────────────────────────────────── + +#[tokio::test] +async fn fetch_v4_through_v12_e2e_preserve_correlation_id() { + let (addr, _shutdown) = spawn_test_server().await; + + for version in 4i16..=12 { + let body = load_fixture_body_or_skip(1, "Fetch", version) + .unwrap_or_else(|| build_fetch_empty_topics_request(version)); + + let correlation_id = 800 + i32::from(version); + let (corr, resp) = round_trip(addr, API_KEY_FETCH, version, correlation_id, &body).await; + assert_eq!(corr, correlation_id, "Fetch v{version} correlation"); + assert!(!resp.is_empty(), "Fetch v{version} response"); + } +} + +#[tokio::test] +async fn metadata_empty_body_e2e_all_topics_request_returns_broker() { + let (addr, _shutdown) = spawn_test_server().await; + let (corr, body) = round_trip(addr, API_KEY_METADATA, 0, 360, &[]).await; + assert_eq!(corr, 360); + let mut d = Decoder::new(body); + assert_eq!(d.read_i32().unwrap(), 1, "one stub broker"); + d.read_i32().unwrap(); // node_id + let host = d.read_nullable_string().unwrap().expect("broker host"); + assert!(!host.is_empty()); + let port = d.read_i32().unwrap(); + assert!(port > 0); +} + +// ── Out-of-scope API keys (SCOPE.md unsupported list) ─────────────────────── + +#[tokio::test] +async fn out_of_scope_api_keys_e2e_respond_then_close() { + let (addr, _shutdown) = spawn_test_server().await; + + for &(api_key, name) in &OUT_OF_SCOPE_API_KEYS[..4] { + let mut stream = TcpStream::connect(addr).await.expect("connect"); + let frame = build_request_frame(api_key, 0, i32::from(api_key), Some("scope-test"), &[]); + stream.write_all(&frame).await.expect("write oos key"); + + let payload = read_response_frame(&mut stream, 8 * 1024 * 1024).await; + let mut d = Decoder::new(parse_response_payload(api_key, 0, payload).1); + assert_eq!( + d.read_i16().unwrap(), + ERROR_UNSUPPORTED_VERSION, + "{name} (key {api_key})" + ); + + assert_eq!( + read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await, + ByteRead::Closed, + "{name} (key {api_key}) must close the connection after the error response" + ); + } +} diff --git a/gateways/kafka/tests/server_integration_tests.rs b/gateways/kafka/tests/server_integration_tests.rs index a8d6d921bc..00495fa009 100644 --- a/gateways/kafka/tests/server_integration_tests.rs +++ b/gateways/kafka/tests/server_integration_tests.rs @@ -33,7 +33,7 @@ async fn tcp_pair() -> (TcpStream, TcpStream) { (client, server) } -/// Raw length-prefixed write (no Kafka response header) — mirrors `server::write_frame`. +/// Raw length-prefixed write (no Kafka response header) - mirrors `server::write_frame`. async fn write_length_prefixed( stream: &mut TcpStream, payload: &[u8], diff --git a/gateways/kafka/tests/version_firewall_tests.rs b/gateways/kafka/tests/version_firewall_tests.rs index 4f86794e57..a77574fece 100644 --- a/gateways/kafka/tests/version_firewall_tests.rs +++ b/gateways/kafka/tests/version_firewall_tests.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -//! Version negotiation firewall — boundary tests for every scoped API key. +//! Version negotiation firewall - boundary tests for every scoped API key. #[path = "common/fixtures.rs"] mod fixtures; @@ -30,22 +30,31 @@ mod wire; use std::time::Duration; -use bytes::Bytes; +use bytes::{BufMut, Bytes, BytesMut}; use tokio::io::AsyncWriteExt; use tokio::net::TcpStream; use iggy_gateway_kafka::protocol::api::{ API_KEY_API_VERSIONS, API_KEY_CREATE_TOPICS, API_KEY_FETCH, API_KEY_LIST_OFFSETS, - API_KEY_METADATA, API_KEY_PRODUCE, ERROR_INVALID_REQUEST, ERROR_UNSUPPORTED_VERSION, - advertised_min_version, handle_request, is_supported_version, supported_api_ranges, + API_KEY_METADATA, API_KEY_PRODUCE, ERROR_INVALID_REQUEST, ERROR_NONE, + ERROR_UNSUPPORTED_VERSION, advertised_min_version, handle_request, is_supported_version, + supported_api_ranges, }; use iggy_gateway_kafka::protocol::codec::Decoder; -use fixtures::load_fixture_body_or_skip; +use fixtures::{fixture_exists, load_fixture_body, load_fixture_body_or_skip}; use scope::{SCOPED_API_KEYS, default_broker}; use server::spawn_test_server; -use tcp::{ByteRead, build_metadata_legacy_request, build_request_frame, read_byte_with_timeout}; +use tcp::{ + ByteRead, build_list_offsets_v0_request_with_topic_t, build_metadata_legacy_request, + build_produce_v3_body, build_request_frame, parse_response_payload, read_byte_with_timeout, + round_trip, scan_for_error_code, +}; use wire::build_metadata_flexible_request_v10; +use wire::{ + OUT_OF_SCOPE_API_KEYS, build_create_topics_empty_request, build_fetch_empty_topics_request, + build_list_offsets_request, +}; #[test] fn supported_ranges_table_has_six_entries() { @@ -347,3 +356,367 @@ fn corrupt_fetch_body_returns_invalid_request_error() { assert_eq!(d.read_i32().unwrap(), 0); assert_eq!(d.read_i16().unwrap(), ERROR_INVALID_REQUEST); } + +// ── ListOffsets v0 wire shape (old_style_offsets array, not bare i64) ─────── + +/// Parse one `ListOffsets` v0 partition entry the way a v0 Kafka client would. +fn parse_list_offsets_v0_partition(d: &mut Decoder) { + let _partition_index = d.read_i32().expect("partition_index"); + let _error_code = d.read_i16().expect("error_code"); + let offset_count = d.read_i32().expect("old_style_offsets array length"); + assert!( + offset_count >= 0, + "old_style_offsets count must be non-negative, got {offset_count}" + ); + for _ in 0..offset_count { + d.read_i64().expect("old_style_offsets entry"); + } +} + +#[test] +fn list_offsets_v0_unsupported_version_is_parseable_by_v0_clients() { + let body = handle_request(API_KEY_LIST_OFFSETS, 0, Bytes::new(), &default_broker()) + .expect_response("test request has acks != 0 and expects a response"); + let mut d = Decoder::new(body); + + assert_eq!(d.read_i32().unwrap(), 1, "topics array length"); + assert_eq!( + d.read_nullable_string().unwrap(), + Some(String::new()), + "placeholder topic name" + ); + assert_eq!(d.read_i32().unwrap(), 1, "partitions array length"); + + parse_list_offsets_v0_partition(&mut d); + + assert_eq!( + d.remaining(), + 0, + "v0 client must consume the full error response without trailing bytes" + ); +} + +#[test] +fn list_offsets_v0_unsupported_version_carries_error_code_in_partition() { + let request_body = build_list_offsets_v0_request_with_topic_t(); + let body = handle_request(API_KEY_LIST_OFFSETS, 0, request_body, &default_broker()) + .expect_response("test request has acks != 0 and expects a response"); + let mut d = Decoder::new(body); + + assert_eq!(d.read_i32().unwrap(), 1); + d.read_nullable_string().unwrap(); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!(d.read_i32().unwrap(), 0, "partition index"); + assert_eq!( + d.read_i16().unwrap(), + ERROR_UNSUPPORTED_VERSION, + "partition error code" + ); + + // partition_index and error_code were already asserted above; only the + // trailing old_style_offsets array remains for this single partition. + let offset_count = d.read_i32().expect("old_style_offsets array length"); + assert!( + offset_count >= 0, + "old_style_offsets count must be non-negative, got {offset_count}" + ); + for _ in 0..offset_count { + d.read_i64().expect("old_style_offsets entry"); + } + assert_eq!(d.remaining(), 0); +} + +// ── Comprehensive scoped-API coverage (correlation id, boundary versions) ── + +fn metadata_empty_legacy_body() -> Bytes { + let mut body = BytesMut::new(); + body.put_i32(0); + body.freeze() +} + +fn request_body_for_scoped_api(api_key: i16, name: &str, version: i16) -> Bytes { + match api_key { + API_KEY_METADATA => metadata_empty_legacy_body(), + API_KEY_PRODUCE => { + if fixture_exists(api_key, name, version) { + load_fixture_body(api_key, name, version) + } else { + build_produce_v3_body(1, 0) + } + } + API_KEY_FETCH => { + if fixture_exists(api_key, name, version) { + load_fixture_body(api_key, name, version) + } else { + build_fetch_empty_topics_request(version) + } + } + API_KEY_LIST_OFFSETS => { + if fixture_exists(api_key, name, version) { + load_fixture_body(api_key, name, version) + } else { + build_list_offsets_request(version, "scope-topic", 0) + } + } + API_KEY_CREATE_TOPICS => build_create_topics_empty_request(version), + _ => Bytes::new(), + } +} + +#[tokio::test] +async fn each_scoped_api_min_version_preserves_correlation_id_e2e() { + let (addr, _shutdown) = spawn_test_server().await; + + for &(api_key, name, min_ver, _max_ver) in SCOPED_API_KEYS { + let correlation_id = 10_000 + i32::from(api_key); + let body = request_body_for_scoped_api(api_key, name, min_ver); + let (corr, resp_body) = round_trip(addr, api_key, min_ver, correlation_id, &body).await; + assert_eq!( + corr, correlation_id, + "{name} v{min_ver} correlation id must round-trip" + ); + assert!( + !resp_body.is_empty(), + "{name} v{min_ver} must return non-empty body" + ); + } +} + +#[tokio::test] +async fn each_scoped_api_max_version_preserves_correlation_id_e2e() { + let (addr, _shutdown) = spawn_test_server().await; + + for &(api_key, name, _min_ver, max_ver) in SCOPED_API_KEYS { + let correlation_id = 20_000 + i32::from(api_key); + let body = request_body_for_scoped_api(api_key, name, max_ver); + let (corr, resp_body) = round_trip(addr, api_key, max_ver, correlation_id, &body).await; + assert_eq!( + corr, correlation_id, + "{name} v{max_ver} correlation id must round-trip" + ); + assert!( + !resp_body.is_empty(), + "{name} v{max_ver} must return non-empty body" + ); + } +} + +#[tokio::test] +async fn apiversions_v0_and_v2_e2e_return_success() { + let (addr, _shutdown) = spawn_test_server().await; + + for version in [0i16, 2] { + let correlation_id = 300 + i32::from(version); + let (corr, body) = + round_trip(addr, API_KEY_API_VERSIONS, version, correlation_id, &[]).await; + assert_eq!(corr, correlation_id); + let mut d = Decoder::new(body); + assert_eq!( + d.read_i16().unwrap(), + ERROR_NONE, + "ApiVersions v{version} must succeed" + ); + } +} + +#[tokio::test] +async fn apiversions_v4_out_of_range_e2e_returns_unsupported() { + let (addr, _shutdown) = spawn_test_server().await; + let (corr, body) = round_trip(addr, API_KEY_API_VERSIONS, 4, 350, &[]).await; + assert_eq!(corr, 350); + let mut d = Decoder::new(body); + assert_eq!( + d.read_i16().unwrap(), + ERROR_UNSUPPORTED_VERSION, + "ApiVersions v4 must return UNSUPPORTED_VERSION per KIP-511" + ); +} + +// ── Out-of-scope API keys (SCOPE.md unsupported list) ─────────────────────── + +#[test] +fn out_of_scope_api_keys_return_unsupported_version_without_panic() { + for &(api_key, name) in OUT_OF_SCOPE_API_KEYS { + let body = handle_request(api_key, 0, Bytes::new(), &default_broker()) + .expect_response("test request has acks != 0 and expects a response"); + let mut d = Decoder::new(body); + assert_eq!( + d.read_i16().unwrap(), + ERROR_UNSUPPORTED_VERSION, + "{name} (key {api_key})" + ); + } +} + +// ── Boundary versions keep the TCP session (except Metadata) ─────────────── + +#[tokio::test] +async fn each_scoped_api_above_max_version_e2e_keeps_connection() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.expect("connect"); + + for &(api_key, name, _min_ver, max_ver) in SCOPED_API_KEYS { + let above = max_ver + 1; + let frame = build_request_frame( + api_key, + above, + 50_000 + i32::from(api_key), + Some("scope-test"), + &[], + ); + stream + .write_all(&frame) + .await + .unwrap_or_else(|_| panic!("write {name} v{above}")); + if api_key == API_KEY_METADATA { + // Unsupported Metadata closes: clamped bodies are unparsable at the client version. + assert_eq!( + read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await, + ByteRead::Closed, + "Metadata v{above} must close the connection" + ); + stream = TcpStream::connect(addr) + .await + .expect("reconnect after Metadata close"); + continue; + } + let payload = tcp::read_response_frame(&mut stream, 8 * 1024 * 1024).await; + assert!( + !payload.is_empty(), + "{name} v{above} must still respond on wire" + ); + } + + let ok = build_request_frame(API_KEY_API_VERSIONS, 1, 89_999, Some("scope-test"), &[]); + stream.write_all(&ok).await.expect("recovery request"); + let payload = tcp::read_response_frame(&mut stream, 8 * 1024 * 1024).await; + assert_eq!( + parse_response_payload(API_KEY_API_VERSIONS, 1, payload).0, + 89_999 + ); +} + +#[tokio::test] +async fn each_scoped_api_below_min_version_e2e_keeps_connection() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.expect("connect"); + + for &(api_key, name, min_ver, _max_ver) in SCOPED_API_KEYS { + let below = min_ver - 1; + let frame = build_request_frame( + api_key, + below, + 40_000 + i32::from(api_key), + Some("scope-test"), + &[], + ); + stream + .write_all(&frame) + .await + .unwrap_or_else(|_| panic!("write {name} v{below}")); + if api_key == API_KEY_METADATA { + assert_eq!( + read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await, + ByteRead::Closed, + "Metadata v{below} must close the connection" + ); + stream = TcpStream::connect(addr) + .await + .expect("reconnect after Metadata close"); + continue; + } + let payload = tcp::read_response_frame(&mut stream, 8 * 1024 * 1024).await; + assert!( + !payload.is_empty(), + "{name} v{below} must still respond on wire" + ); + } + + let ok = build_request_frame(API_KEY_API_VERSIONS, 1, 88_888, Some("scope-test"), &[]); + stream.write_all(&ok).await.expect("recovery request"); + let payload = tcp::read_response_frame(&mut stream, 8 * 1024 * 1024).await; + assert_eq!( + parse_response_payload(API_KEY_API_VERSIONS, 1, payload).0, + 88_888 + ); +} + +#[test] +fn produce_advertises_min_zero_but_firewall_rejects_below_v3() { + let range = SCOPED_API_KEYS + .iter() + .find(|(k, _, _, _)| *k == API_KEY_PRODUCE) + .expect("produce in scope"); + let (_, _, firewall_min, _) = *range; + assert_eq!(firewall_min, 3); + assert_eq!(advertised_min_version(API_KEY_PRODUCE, firewall_min), 0); + assert!(!is_supported_version(API_KEY_PRODUCE, 0)); + assert!(!is_supported_version(API_KEY_PRODUCE, 2)); + + let body = handle_request(API_KEY_PRODUCE, 2, Bytes::new(), &default_broker()) + .expect_response("test request has acks != 0 and expects a response"); + let mut d = Decoder::new(body); + let _topics = d.read_i32().unwrap(); + let _name = d.read_nullable_string().unwrap(); + let _parts = d.read_i32().unwrap(); + assert_eq!(d.read_i32().unwrap(), 0, "partition index"); + assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); +} + +// ── Unsupported-version e2e paths for remaining scoped APIs ───────────────── + +#[tokio::test] +async fn list_offsets_v7_unsupported_e2e_returns_error() { + let (addr, _shutdown) = spawn_test_server().await; + let (corr, body) = round_trip( + addr, + API_KEY_LIST_OFFSETS, + 7, + 370, + &[0x00, 0x00, 0x00, 0x00], + ) + .await; + assert_eq!(corr, 370); + assert!( + scan_for_error_code(&body, ERROR_UNSUPPORTED_VERSION), + "ListOffsets v7 must be rejected" + ); +} + +#[tokio::test] +async fn create_topics_v1_unsupported_e2e_returns_error() { + let (addr, _shutdown) = spawn_test_server().await; + let (corr, body) = round_trip(addr, API_KEY_CREATE_TOPICS, 1, 380, &[]).await; + assert_eq!(corr, 380); + assert!( + scan_for_error_code(&body, ERROR_UNSUPPORTED_VERSION), + "CreateTopics v1 must be rejected" + ); +} + +// ── Corrupt decode paths for remaining scoped APIs ────────────────────────── + +#[test] +fn corrupt_list_offsets_body_returns_invalid_request_error() { + let body = Bytes::from_static(&[0xFF, 0xFF, 0xFF]); + let resp = handle_request(API_KEY_LIST_OFFSETS, 1, body, &default_broker()) + .expect_response("test request has acks != 0 and expects a response"); + assert!(!resp.is_empty()); + assert!( + scan_for_error_code(&resp, ERROR_INVALID_REQUEST) + || scan_for_error_code(&resp, ERROR_UNSUPPORTED_VERSION), + "corrupt ListOffsets must surface protocol error" + ); +} + +#[test] +fn corrupt_create_topics_body_returns_invalid_request_error() { + let body = Bytes::from_static(&[0xFF, 0xFF, 0xFF]); + let resp = handle_request(API_KEY_CREATE_TOPICS, 2, body, &default_broker()) + .expect_response("test request has acks != 0 and expects a response"); + assert!(!resp.is_empty()); + assert!( + scan_for_error_code(&resp, ERROR_INVALID_REQUEST) + || scan_for_error_code(&resp, ERROR_UNSUPPORTED_VERSION) + ); +} diff --git a/gateways/kafka/tools/kafka-tool/src/main.rs b/gateways/kafka/tools/kafka-tool/src/main.rs index 73b2c1cba2..b63f7d9844 100644 --- a/gateways/kafka/tools/kafka-tool/src/main.rs +++ b/gateways/kafka/tools/kafka-tool/src/main.rs @@ -19,7 +19,6 @@ use anyhow::{Context, Result}; use bytes::{BufMut, Bytes, BytesMut}; use clap::{Parser, Subcommand}; use iggy_gateway_kafka::protocol::api::supported_api_ranges; -use iggy_gateway_kafka::protocol::header::request_header_version; use kafka_protocol::messages::*; use kafka_protocol::protocol::{Encodable, StrBytes}; use std::path::PathBuf; @@ -574,8 +573,13 @@ fn build_payload(api_key: i16, version: i16) -> Result { // Build a complete framed Kafka request message ready for TCP transmission. fn build_framed(api_key: i16, version: i16, corr: i32) -> Result { let payload = build_payload(api_key, version)?; - // Header v2 == flexible request encoding; threshold lives in iggy-gateway-kafka. - let flexible = request_header_version(api_key, version) >= 2; + // Header version comes from kafka-protocol's own per-request `HeaderVersion` impl (via + // `ApiKey::request_header_version`), not the gateway's table under test - otherwise a bug + // in the gateway's flexible-version threshold would mis-frame the fixture identically and + // the tests would still pass. + let api = + ApiKey::try_from(api_key).map_err(|()| anyhow::anyhow!("unknown api_key={api_key}"))?; + let flexible = api.request_header_version(version) >= 2; Ok(frame_request( api_key, version, diff --git a/gateways/kafka/tools/kafka-tool/tests/generate_cli_tests.rs b/gateways/kafka/tools/kafka-tool/tests/generate_cli_tests.rs index fd3dccd74b..f29c7d5e48 100644 --- a/gateways/kafka/tools/kafka-tool/tests/generate_cli_tests.rs +++ b/gateways/kafka/tools/kafka-tool/tests/generate_cli_tests.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -//! CLI regression for PR #3519 review: `generate` must accept repeated `--api-key`. use std::process::Command; use std::time::{SystemTime, UNIX_EPOCH}; From 226e5178ef896817bc4201827da87e7e8f77babd Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Fri, 31 Jul 2026 22:04:33 -0400 Subject: [PATCH 41/57] Fixing format errors --- gateways/kafka/src/protocol/header.rs | 2 +- gateways/kafka/src/protocol/requests.rs | 3 +-- gateways/kafka/tools/kafka-tool/tests/generate_cli_tests.rs | 1 - 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/gateways/kafka/src/protocol/header.rs b/gateways/kafka/src/protocol/header.rs index 0a8d34e071..f9c14ec065 100644 --- a/gateways/kafka/src/protocol/header.rs +++ b/gateways/kafka/src/protocol/header.rs @@ -18,7 +18,7 @@ #![allow( clippy::doc_markdown, clippy::missing_const_for_fn, - clippy::missing_errors_doc, + clippy::missing_errors_doc, clippy::match_same_arms )] diff --git a/gateways/kafka/src/protocol/requests.rs b/gateways/kafka/src/protocol/requests.rs index 81e1c1cb2a..c9026b2a50 100644 --- a/gateways/kafka/src/protocol/requests.rs +++ b/gateways/kafka/src/protocol/requests.rs @@ -17,8 +17,7 @@ //! Kafka request decoders for critical API keys -#![allow(clippy::too_many_lines, - clippy::doc_markdown)] +#![allow(clippy::too_many_lines, clippy::doc_markdown)] use crate::error::{KafkaProtocolError, Result}; use crate::protocol::codec::Decoder; use bytes::Bytes; diff --git a/gateways/kafka/tools/kafka-tool/tests/generate_cli_tests.rs b/gateways/kafka/tools/kafka-tool/tests/generate_cli_tests.rs index f29c7d5e48..43299cd55a 100644 --- a/gateways/kafka/tools/kafka-tool/tests/generate_cli_tests.rs +++ b/gateways/kafka/tools/kafka-tool/tests/generate_cli_tests.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. - use std::process::Command; use std::time::{SystemTime, UNIX_EPOCH}; From f016d4a46e2f749c58c01b9fcba7fe24ff6b8a53 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sun, 2 Aug 2026 08:20:52 -0400 Subject: [PATCH 42/57] Harden Kafka gateway stub behavior Tighten the Kafka gateway's wire handling and runtime limits by namespacing env vars, adding connection/idle/shutdown controls, and avoiding large allocations from untrusted frame or collection lengths. This also fixes Produce acks=0 handling for advertised legacy versions, makes Fetch and ListOffsets return retriable stub errors like Produce, and updates tests/docs to match the new behavior. --- Cargo.lock | 1 + gateways/kafka/Cargo.toml | 3 +- gateways/kafka/README.md | 13 +- gateways/kafka/docs/MANUAL_TESTING.md | 4 +- gateways/kafka/docs/SCOPE.md | 4 +- gateways/kafka/docs/TEST_SUITE.md | 166 +++-------- gateways/kafka/src/main.rs | 32 +- gateways/kafka/src/protocol/api.rs | 57 +++- gateways/kafka/src/protocol/codec.rs | 32 ++ gateways/kafka/src/protocol/header.rs | 2 +- gateways/kafka/src/protocol/requests.rs | 22 +- gateways/kafka/src/protocol/responses.rs | 13 +- gateways/kafka/src/server.rs | 273 ++++++++++++++---- gateways/kafka/tests/api_handler_tests.rs | 32 +- .../kafka/tests/broker_advertise_tests.rs | 4 +- gateways/kafka/tests/common/server.rs | 3 + gateways/kafka/tests/decode_safety_tests.rs | 20 ++ .../kafka/tests/decode_validation_tests.rs | 41 +-- gateways/kafka/tests/header_tests.rs | 30 +- .../kafka/tests/listener_robustness_tests.rs | 15 + gateways/kafka/tests/server_e2e_tests.rs | 18 +- .../kafka/tests/server_integration_tests.rs | 42 ++- gateways/kafka/tools/kafka-tool/src/main.rs | 28 +- .../kafka/tools/kafka-tool/src/response.rs | 15 +- 24 files changed, 552 insertions(+), 318 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ed8b6142b4..5b5fe89647 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6790,6 +6790,7 @@ name = "iggy-gateway-kafka" version = "0.1.0" dependencies = [ "bytes", + "kafka-protocol", "socket2 0.6.5", "thiserror 2.0.19", "tokio", diff --git a/gateways/kafka/Cargo.toml b/gateways/kafka/Cargo.toml index 8567e439ad..a06ac51d56 100644 --- a/gateways/kafka/Cargo.toml +++ b/gateways/kafka/Cargo.toml @@ -50,10 +50,11 @@ tracing = { workspace = true } tracing-subscriber = { workspace = true } [dev-dependencies] +kafka-protocol = "0.17" tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "io-util", "time"] } [lints.clippy] enum_glob_use = "deny" -#Ported Kafka wire codec; pedantic cleanup tracked for a follow-up PR. +# Ported Kafka wire codec; pedantic cleanup tracked for a follow-up PR. pedantic = "warn" nursery = "warn" diff --git a/gateways/kafka/README.md b/gateways/kafka/README.md index 9e7b1dbdc3..ead41c2fbf 100644 --- a/gateways/kafka/README.md +++ b/gateways/kafka/README.md @@ -2,7 +2,7 @@ Foundation layer for [apache/iggy#3421](https://github.com/apache/iggy/issues/3421): a TCP listener on the Kafka wire port that decodes requests, validates scoped API keys and versions, and returns stub responses. -> **Stub warning:** Produce does **not** persist records. Valid Produce requests return retriable `NOT_LEADER_OR_FOLLOWER` (6) so clients keep data locally. CreateTopics does **not** create topics; valid requests return `NOT_CONTROLLER` (41). Metadata still reports requested topics as unknown. Persistence lands with the Iggy bridge (see [docs/SCOPE.md](docs/SCOPE.md)). +> **Stub warning:** no API persists or reads real data yet. Produce, Fetch, and ListOffsets return retriable `NOT_LEADER_OR_FOLLOWER` (6) so clients keep data locally / retry elsewhere instead of trusting a fake success. CreateTopics does **not** create topics; valid requests return `NOT_CONTROLLER` (41). Metadata still reports requested topics as unknown. Persistence lands with the Iggy bridge (see [docs/SCOPE.md](docs/SCOPE.md)). ## Run @@ -14,9 +14,12 @@ Default bind: `127.0.0.1:9093`. Environment variables: | Variable | Default | Description | | --- | --- | --- | -| `KAFKA_BIND_ADDR` | `127.0.0.1:9093` | TCP address to listen on | -| `KAFKA_ADVERTISED_HOST` | bind IP | Hostname/IP clients use to reach this broker (required when binding to `0.0.0.0`/`::`) | -| `KAFKA_ADVERTISED_PORT` | bind port | Port advertised in Metadata responses | +| `IGGY_KAFKA_BIND_ADDR` | `127.0.0.1:9093` | TCP address to listen on | +| `IGGY_KAFKA_ADVERTISED_HOST` | bind IP | Hostname/IP clients use to reach this broker (required when binding to `0.0.0.0`/`::`) | +| `IGGY_KAFKA_ADVERTISED_PORT` | bind port | Port advertised in Metadata responses | +| `IGGY_KAFKA_MAX_CONNECTIONS` | `1024` | Maximum concurrent connections before new ones are rejected | +| `IGGY_KAFKA_IDLE_TIMEOUT_SECS` | `600` | Seconds a connection may sit idle before the next frame's length prefix arrives | +| `IGGY_KAFKA_SHUTDOWN_DRAIN_TIMEOUT_SECS` | `25` | Seconds graceful shutdown waits for in-flight connections before abandoning them | ## Test @@ -24,7 +27,7 @@ Default bind: `127.0.0.1:9093`. Environment variables: cargo test -p iggy-gateway-kafka ``` -103 regression tests across 12 suites — see [docs/TEST_SUITE.md](docs/TEST_SUITE.md) for the full catalog. +235 regression tests across 12 suites — see [docs/TEST_SUITE.md](docs/TEST_SUITE.md) for the full catalog. `decode_validation_tests` require wire fixtures under `tools/kafka-tool/kafka_messages/` (gitignored locally; CI generates them via `scripts/ci-wire-fixtures.sh`): diff --git a/gateways/kafka/docs/MANUAL_TESTING.md b/gateways/kafka/docs/MANUAL_TESTING.md index b0c548949a..4607ff0001 100644 --- a/gateways/kafka/docs/MANUAL_TESTING.md +++ b/gateways/kafka/docs/MANUAL_TESTING.md @@ -130,7 +130,7 @@ Follow C1 with A2 on the **same** `nc` session to confirm the connection is not | ID | Test | Steps | Expected | | ---- | ------ | ------- | ---------- | | E1 | Broker advertise address | Start gateway on `127.0.0.1:9093`; Metadata v0 | Broker host=`127.0.0.1`, port=`9093` | -| E2 | Wildcard bind + advertised host | `KAFKA_BIND_ADDR=0.0.0.0:19093` + `KAFKA_ADVERTISED_HOST=kafka.internal`, restart | Metadata broker host/port match advertised values | +| E2 | Wildcard bind + advertised host | `IGGY_KAFKA_BIND_ADDR=0.0.0.0:19093` + `IGGY_KAFKA_ADVERTISED_HOST=kafka.internal`, restart | Metadata broker host/port match advertised values | | E3 | Unknown topic stub | Metadata with topic name `my-topic` | Topic error `3` (UNKNOWN_TOPIC_OR_PARTITION), name `unknown-topic` | | E4 | Multiple topics | Metadata request listing 3 topics | 3 topic entries, each with error 3 | @@ -255,7 +255,7 @@ _________________________________ | Symptom | Likely cause | Fix | | --------- | -------------- | ----- | | `Connection refused` on 9093 | Gateway not running | Start `iggy-gateway-kafka` | -| `decode_validation_tests` panic | Missing fixtures | Run `kafka-message-gen generate` | +| `decode_validation_tests` skips silently | Missing fixtures | Run `kafka-message-gen generate` (set `KAFKA_FIXTURES_REQUIRED=1` to turn skips into failures) | | `ec=35` for in-range version | Version not in `SUPPORTED_RANGES` | Check `SCOPE.md` and `api.rs` | | kcat hangs | Timeout waiting for data | Set `-m 1000`; check gateway logs | | Buffer underflow on Metadata v9+ | Flexible decode mismatch | File issue; check `api.rs` metadata encoder | diff --git a/gateways/kafka/docs/SCOPE.md b/gateways/kafka/docs/SCOPE.md index ee7bc5591c..4e7417073b 100644 --- a/gateways/kafka/docs/SCOPE.md +++ b/gateways/kafka/docs/SCOPE.md @@ -16,7 +16,7 @@ Foundation layer only: a TCP listener on the Kafka wire port that decodes reques | Produce hot path: RecordBatch as opaque `Bytes` | Done | `src/protocol/requests.rs` | | Graceful errors (`UNSUPPORTED_VERSION`, corrupt decode, invalid header) | Done | `src/protocol/api.rs`, `src/server.rs` | | Adversarial decode safety tests | Done | `tests/decode_safety_tests.rs` | -| Regression test suite (103 tests) | Done | `tests/` — catalog in [`TEST_SUITE.md`](TEST_SUITE.md) | +| Regression test suite | Done | `tests/` — see [`TEST_SUITE.md`](TEST_SUITE.md) | | Manual testing procedure | Done | [`MANUAL_TESTING.md`](MANUAL_TESTING.md) | | Wire fixture tool for manual/integration testing | Done | `tools/kafka-tool/` | @@ -118,4 +118,4 @@ Items from the [hybrid architecture review](https://github.com/apache/iggy/discu ### Open questions (ask maintainers before Phase 2) - [ ] Repo placement: `gateways/kafka/` in [apache/iggy](https://github.com/apache/iggy) vs separate proxy repo (affects workspace deps and CI) -- [ ] Confirm bridge dependency strategy with spetz/hubcio ([Discussion #3081](https://github.com/apache/iggy/discussions/3081), [#3252](https://github.com/apache/iggy/discussions/3252)) +- [ ] Confirm bridge dependency strategy ([Discussion #3081](https://github.com/apache/iggy/discussions/3081), [#3252](https://github.com/apache/iggy/discussions/3252)) diff --git a/gateways/kafka/docs/TEST_SUITE.md b/gateways/kafka/docs/TEST_SUITE.md index 7261f05024..10d6a9bca9 100644 --- a/gateways/kafka/docs/TEST_SUITE.md +++ b/gateways/kafka/docs/TEST_SUITE.md @@ -6,8 +6,6 @@ Regression tests live under [`tests/`](../tests/). Run from the workspace root: cargo test -p iggy-gateway-kafka ``` -**Current count:** 103 tests across 12 suites (as of #3421 foundation). - ## Prerequisites ### Wire fixtures (required for `decode_validation_tests` and some handler tests) @@ -16,140 +14,46 @@ cargo test -p iggy-gateway-kafka ./gateways/kafka/scripts/ci-wire-fixtures.sh generate ``` -Fixtures are gitignored under `tools/kafka-tool/kafka_messages/`. CI runs the same script before `rust-gateway` test jobs and removes the directory afterward. Tests that need fixtures skip gracefully when a file is missing (`handler_regression_tests`) or panic with a clear path (`decode_validation_tests`). - ---- - -## Test file catalog - -| File | Suite focus | Test count (approx.) | Depends on fixtures | -| ------ | ------------- | ---------------------- | --------------------- | -| [`codec_tests.rs`](../tests/codec_tests.rs) | Primitive encode/decode round-trips, varint, compact strings, tagged fields | 9 | No | -| [`decode_safety_tests.rs`](../tests/decode_safety_tests.rs) | Adversarial wire input — malformed lengths, truncated bodies | 6 | No | -| [`header_tests.rs`](../tests/header_tests.rs) | Request/response header v1/v2, version lookup table | 10 | No | -| [`api_handler_tests.rs`](../tests/api_handler_tests.rs) | ApiVersions, Metadata stub, unsupported key/version | 7 | No | -| [`golden_wire_fixtures_tests.rs`](../tests/golden_wire_fixtures_tests.rs) | Byte-exact golden responses (ApiVersions v1, Metadata v0) | 2 | No | -| [`decode_validation_tests.rs`](../tests/decode_validation_tests.rs) | kafka-tool fixture decode + response structure per version | 14 | **Yes** | -| [`version_firewall_tests.rs`](../tests/version_firewall_tests.rs) | Version boundary matrix, unsupported keys, corrupt bodies | 17 | Partial | -| [`metadata_regression_tests.rs`](../tests/metadata_regression_tests.rs) | Metadata v0–v9, topic counts, broker advertise | 7 | No | -| [`broker_advertise_tests.rs`](../tests/broker_advertise_tests.rs) | `BrokerAdvertise::from_server_config` parsing | 5 | No | -| [`handler_regression_tests.rs`](../tests/handler_regression_tests.rs) | Every scoped key×version via `handle_request`, stub error codes | 5 | Partial | -| [`server_integration_tests.rs`](../tests/server_integration_tests.rs) | `read_frame` / `write_frame` unit-level I/O | 4 | No | -| [`server_e2e_tests.rs`](../tests/server_e2e_tests.rs) | Full `KafkaServer` TCP round-trips | 8 | Partial | -| [`common/mod.rs`](../tests/common/mod.rs) | Shared helpers (not a test binary) | — | — | - ---- - -## Coverage matrix by API key - -### ApiVersions (key 18, v0–v3) - -| Scenario | Test file | Test name | -| ---------- | ----------- | ----------- | -| Non-flexible response (v1) | `api_handler_tests` | `api_versions_v1_response_non_flexible_format` | -| Flexible response (v3) | `api_handler_tests` | `api_versions_v3_response_flexible_format` | -| Golden byte fixture (v1) | `golden_wire_fixtures_tests` | `golden_apiversions_v1_response_fixture` | -| Exact advertised ranges (v1, v3) | `version_firewall_tests` | `apiversions_advertises_exact_supported_ranges_*` | -| All versions return `error_code=0` | `version_firewall_tests` | `apiversions_all_versions_return_success` | -| Out-of-range version | `version_firewall_tests` | `apiversions_out_of_range_returns_unsupported_in_body` | -| E2E correlation ID preserved | `server_e2e_tests` | `e2e_apiversions_v1_*`, `e2e_apiversions_v3_*` | - -### Metadata (key 3, v0–v9) - -| Scenario | Test file | Test name | -| ---------- | ----------- | ----------- | -| Stub broker (default 127.0.0.1:9093) | `api_handler_tests`, `metadata_regression_tests` | `metadata_response_has_broker_*`, `metadata_v0_empty_*` | -| Unsupported version → topic error 35 | `api_handler_tests`, `version_firewall_tests` | `unsupported_version_returns_protocol_error`, `metadata_*_version_returns_topic_error` | -| Golden byte fixture (v0, 1 topic) | `golden_wire_fixtures_tests` | `golden_metadata_v0_single_topic_response_fixture` | -| v1 controller_id, v2 cluster_id | `metadata_regression_tests` | `metadata_v1_*`, `metadata_v2_*` | -| v9 flexible encoding | `metadata_regression_tests` | `metadata_v9_flexible_encoding` | -| Custom broker advertise | `metadata_regression_tests`, `broker_advertise_tests` | `metadata_uses_custom_*`, `metadata_reflects_parsed_*` | -| E2E round-trip | `server_e2e_tests` | `e2e_metadata_v0_returns_stub_broker` | - -### Produce (key 0, v3–v9) - -| Scenario | Test file | Test name | -| ---------- | ----------- | ----------- | -| Decode all versions (fixture) | `decode_validation_tests` | `produce_all_supported_versions_decode` | -| Response encode all versions | `decode_validation_tests` | `produce_response_encodes_for_all_supported_versions` | -| v3 field layout | `decode_validation_tests` | `produce_response_v3_roundtrip` | -| v8 record_errors array | `decode_validation_tests` | `produce_response_v8_includes_record_errors` | -| Unsupported v2 → error 35 | `version_firewall_tests` | `produce_unsupported_version_returns_error_only` | -| Corrupt body → error 42 | `version_firewall_tests` | `corrupt_produce_body_returns_invalid_request_error` | -| Stub partition error 6 (not leader) | `handler_regression_tests` | `produce_stub_response_returns_retriable_not_leader` | -| E2E round-trip | `server_e2e_tests` | `e2e_produce_v3_round_trip_with_fixture` | - -### Fetch (key 1, v4–v12) - -| Scenario | Test file | Test name | -| ---------- | ----------- | ----------- | -| Decode all versions | `decode_validation_tests` | `fetch_all_supported_versions_decode` | -| Response encode all versions | `decode_validation_tests` | `fetch_response_encodes_for_all_supported_versions` | -| v7 session_id / error_code layout | `decode_validation_tests` | `fetch_response_v7_roundtrip` | -| Unsupported v3 | `version_firewall_tests` | `fetch_unsupported_version_returns_error_only` | -| Corrupt body → error 42 | `version_firewall_tests` | `corrupt_fetch_body_returns_invalid_request_error` | -| Stub partition error 0 | `handler_regression_tests` | `fetch_stub_response_has_zero_partition_error` | - -### ListOffsets (key 2, v1–v6) - -| Scenario | Test file | Test name | -| ---------- | ----------- | ----------- | -| Decode all versions | `decode_validation_tests` | `list_offsets_all_supported_versions_decode` | -| v1 no leader_epoch | `decode_validation_tests` | `list_offsets_response_v1_no_leader_epoch` | -| v4 has leader_epoch | `decode_validation_tests` | `list_offsets_response_v4_has_leader_epoch` | -| Unsupported v0 | `version_firewall_tests` | `list_offsets_unsupported_version_returns_error_only` | -| Stub error 0 | `handler_regression_tests` | `list_offsets_stub_response_has_zero_error` | - -### CreateTopics (key 19, v2–v5) - -| Scenario | Test file | Test name | -| ---------- | ----------- | ----------- | -| Decode all versions | `decode_validation_tests` | `create_topics_all_supported_versions_decode` | -| v2 roundtrip | `decode_validation_tests` | `create_topics_response_v2_roundtrip` | -| v5 flexible roundtrip | `decode_validation_tests` | `create_topics_response_v5_roundtrip` | -| Unsupported v1 | `version_firewall_tests` | `create_topics_unsupported_version_returns_error_only` | -| Stub error 41 (not controller) | `handler_regression_tests` | `create_topics_stub_response_returns_not_controller` | +Fixtures are gitignored under `tools/kafka-tool/kafka_messages/`. CI runs the same script +before `rust-gateway` test jobs and removes the directory afterward. Every fixture-dependent +suite goes through `tests/common/fixtures.rs::load_fixture_body_or_skip`, which skips with a +regeneration hint when a fixture is missing, and panics instead when `KAFKA_FIXTURES_REQUIRED=1` +is set (CI sets this) so a broken generation step can't leave a suite green with zero assertions. --- -## Cross-cutting scenarios - -| Scenario | Test file | Test name | -| ---------- | ----------- | ----------- | -| Version firewall min/max boundaries | `version_firewall_tests` | `is_supported_version_matches_scope_table` | -| Unknown API keys (8, 9, 10, 17, 20, 999) | `version_firewall_tests`, `api_handler_tests` | `unsupported_api_keys_*`, `unknown_api_key_*` | -| Negative i32 array length | `decode_safety_tests` | `negative_i32_array_length_returns_error_not_panic` | -| Oversized collection count | `decode_safety_tests` | `i32_array_length_above_max_returns_collection_too_large` | -| Compact array varint=0 (null array) | `decode_safety_tests` | `compact_array_varint_zero_decodes_as_empty_without_panic` | -| Malformed varint at shift 63 | `decode_safety_tests` | `varint_terminal_byte_with_extra_bits_at_shift_63_is_rejected` | -| Invalid frame length (0) | `server_integration_tests` | `read_frame_rejects_invalid_lengths` | -| Frame exceeds max_frame_size | `server_integration_tests`, `server_e2e_tests` | `read_frame_rejects_invalid_lengths`, `e2e_oversized_frame_is_rejected` | -| Sequential requests on one TCP connection | `server_e2e_tests` | `e2e_sequential_requests_on_one_connection` | -| Unsupported API key returns error then closes connection | `server_e2e_tests` | `e2e_unsupported_api_key_returns_error_then_closes` | -| Negative frame length closes connection | `server_e2e_tests` | `e2e_negative_frame_length_closes_connection` | - ---- - -## CI recommendation - -```bash -# 1. Generate fixtures -cargo run -p kafka-message-gen -- generate \ - --output gateways/kafka/tools/kafka-tool/kafka_messages \ - --api-key 0 --api-key 1 --api-key 2 --api-key 19 - -# 2. Run regression suite -cargo test -p iggy-gateway-kafka - -# 3. Optional lint gate -cargo clippy -p iggy-gateway-kafka -- -D warnings -``` +## Test files + +An exact per-file test count and a full test-name-to-scenario matrix used to live here; both +drifted out of sync with the actual suites more than once as tests were added and consolidated. +Rather than re-derive a snapshot that will drift again, this only lists what each file is for — +`cargo test -p iggy-gateway-kafka -- --list` gives the exact current test names. + +| File | Suite focus | Depends on fixtures | +| ------ | ------------- | --------------------- | +| [`codec_tests.rs`](../tests/codec_tests.rs) | Primitive encode/decode round-trips, varint, compact strings, tagged fields | No | +| [`decode_safety_tests.rs`](../tests/decode_safety_tests.rs) | Adversarial wire input — malformed lengths, truncated bodies, oversized declared counts | No | +| [`header_tests.rs`](../tests/header_tests.rs) | Request/response header v1/v2, flexible-version lookup table | No | +| [`api_handler_tests.rs`](../tests/api_handler_tests.rs) | ApiVersions, Metadata stub, unsupported key/version, `handle_request` dispatch | Partial | +| [`response_negative_tests.rs`](../tests/response_negative_tests.rs) | Error-response encoding and validation for each API | No | +| [`golden_wire_fixtures_tests.rs`](../tests/golden_wire_fixtures_tests.rs) | Byte-exact golden responses (ApiVersions v1, Metadata v0) | No | +| [`decode_validation_tests.rs`](../tests/decode_validation_tests.rs) | kafka-tool fixture decode + response structure per version | **Yes** | +| [`version_firewall_tests.rs`](../tests/version_firewall_tests.rs) | Version boundary matrix, unsupported keys, corrupt bodies | Partial | +| [`broker_advertise_tests.rs`](../tests/broker_advertise_tests.rs) | `BrokerAdvertise::from_server_config` parsing | No | +| [`server_integration_tests.rs`](../tests/server_integration_tests.rs) | `read_frame` / `write_frame` unit-level I/O | No | +| [`server_e2e_tests.rs`](../tests/server_e2e_tests.rs) | Full `KafkaServer` TCP round-trips | Partial | +| [`listener_robustness_tests.rs`](../tests/listener_robustness_tests.rs) | TCP listener robustness — framing, pipelining, concurrency, connection limits | No | + +`tests/common/` holds shared helpers (`fixtures.rs`, `scope.rs`, `server.rs`, `tcp.rs`, `wire.rs`), +compiled per test binary via `#[path]`, not a test binary itself. --- ## Adding new tests -1. **New API key or version range** — update `SUPPORTED_RANGES` in `api.rs`, `SCOPE.md`, and add rows to the coverage matrix above. -2. **New decode path** — add fixture via `kafka-message-gen`, extend `decode_validation_tests.rs`. -3. **New error path** — add to `version_firewall_tests.rs` or `decode_safety_tests.rs`. -4. **New TCP behavior** — add to `server_e2e_tests.rs` using helpers in `tests/common/mod.rs`. +1. **New API key or version range** — update `SUPPORTED_RANGES` in `api.rs` and `SCOPE.md`. +2. **New decode path** — add a fixture via `kafka-message-gen`, extend `decode_validation_tests.rs`. +3. **New error path** — add to `version_firewall_tests.rs`, `decode_safety_tests.rs`, or + `response_negative_tests.rs`. +4. **New TCP behavior** — add to `server_e2e_tests.rs` or `listener_robustness_tests.rs` using + the helpers under `tests/common/`. diff --git a/gateways/kafka/src/main.rs b/gateways/kafka/src/main.rs index 536ddf472a..5b316be7dd 100644 --- a/gateways/kafka/src/main.rs +++ b/gateways/kafka/src/main.rs @@ -27,18 +27,34 @@ async fn main() -> Result<(), Box> { init_tracing(); let mut config = ServerConfig::default(); - if let Ok(bind_addr) = std::env::var("KAFKA_BIND_ADDR") { + if let Ok(bind_addr) = std::env::var("IGGY_KAFKA_BIND_ADDR") { config.bind_addr = bind_addr; } - if let Ok(advertised_host) = std::env::var("KAFKA_ADVERTISED_HOST") { + if let Ok(advertised_host) = std::env::var("IGGY_KAFKA_ADVERTISED_HOST") { config.advertised_host = Some(advertised_host); } - if let Ok(advertised_port) = std::env::var("KAFKA_ADVERTISED_PORT") { - config.advertised_port = Some( - advertised_port - .parse() - .map_err(|e| format!("invalid KAFKA_ADVERTISED_PORT `{advertised_port}`: {e}"))?, - ); + if let Ok(advertised_port) = std::env::var("IGGY_KAFKA_ADVERTISED_PORT") { + config.advertised_port = + Some(advertised_port.parse().map_err(|e| { + format!("invalid IGGY_KAFKA_ADVERTISED_PORT `{advertised_port}`: {e}") + })?); + } + if let Ok(max_connections) = std::env::var("IGGY_KAFKA_MAX_CONNECTIONS") { + config.max_connections = max_connections + .parse() + .map_err(|e| format!("invalid IGGY_KAFKA_MAX_CONNECTIONS `{max_connections}`: {e}"))?; + } + if let Ok(idle_timeout_secs) = std::env::var("IGGY_KAFKA_IDLE_TIMEOUT_SECS") { + let secs: u64 = idle_timeout_secs.parse().map_err(|e| { + format!("invalid IGGY_KAFKA_IDLE_TIMEOUT_SECS `{idle_timeout_secs}`: {e}") + })?; + config.idle_timeout = std::time::Duration::from_secs(secs); + } + if let Ok(drain_secs) = std::env::var("IGGY_KAFKA_SHUTDOWN_DRAIN_TIMEOUT_SECS") { + let secs: u64 = drain_secs.parse().map_err(|e| { + format!("invalid IGGY_KAFKA_SHUTDOWN_DRAIN_TIMEOUT_SECS `{drain_secs}`: {e}") + })?; + config.shutdown_drain_timeout = std::time::Duration::from_secs(secs); } let listener = TcpListener::bind(&config.bind_addr) .await diff --git a/gateways/kafka/src/protocol/api.rs b/gateways/kafka/src/protocol/api.rs index 5c7892a8d2..9c6d86e23e 100644 --- a/gateways/kafka/src/protocol/api.rs +++ b/gateways/kafka/src/protocol/api.rs @@ -18,7 +18,7 @@ use bytes::Bytes; use crate::error::{KafkaProtocolError, Result}; -use crate::protocol::codec::{Decoder, Encoder}; +use crate::protocol::codec::{Decoder, Encoder, PREALLOC_HINT}; use crate::protocol::requests::{ ProduceDecodeResult, decode_create_topics_request, decode_fetch_request, decode_list_offsets_request, decode_produce_request, @@ -183,18 +183,24 @@ pub fn handle_request( /// Produce is the only request the wire protocol allows to go unanswered /// (`acks=0`), so it gets its own path that may return [`HandleOutcome::NoResponse`]. +/// +/// The firewall check runs AFTER decoding `acks`, not before: `ApiVersions` advertises +/// Produce min=0 (see [`advertised_min_version`]) while the firewall's real floor is 3, so a +/// spec-compliant client can legitimately send Produce v0-2 with `acks=0`. Rejecting those +/// versions before reading `acks` would send an error response the client never expects, +/// desyncing the next correlation id it reads. fn handle_produce_request(api_version: i16, body: Bytes) -> HandleOutcome { - if !is_supported_version(API_KEY_PRODUCE, api_version) { - return HandleOutcome::Respond(encode_produce_error_response( - api_version, - ERROR_UNSUPPORTED_VERSION, - )); - } match decode_produce_request(api_version, body) { // acks=0 is fire-and-forget: the client isn't reading a response, so // sending one desyncs the next correlation id it expects. ProduceDecodeResult::Ok(req) if req.acks == 0 => HandleOutcome::NoResponse, ProduceDecodeResult::Ok(req) => { + if !is_supported_version(API_KEY_PRODUCE, api_version) { + return HandleOutcome::Respond(encode_produce_error_response( + api_version, + ERROR_UNSUPPORTED_VERSION, + )); + } HandleOutcome::Respond(encode_produce_response(api_version, &req)) } ProduceDecodeResult::Err { @@ -209,10 +215,12 @@ fn handle_produce_request(api_version: i16, body: Bytes) -> HandleOutcome { } ProduceDecodeResult::Err { error, .. } => { tracing::warn!("Failed to decode Produce request: {:?}", error); - HandleOutcome::Respond(encode_produce_error_response( - api_version, - ERROR_INVALID_REQUEST, - )) + let code = if is_supported_version(API_KEY_PRODUCE, api_version) { + ERROR_INVALID_REQUEST + } else { + ERROR_UNSUPPORTED_VERSION + }; + HandleOutcome::Respond(encode_produce_error_response(api_version, code)) } } } @@ -432,7 +440,7 @@ fn encode_metadata_response( } else { e.write_i32(1); // brokers array length e.write_i32(1); // node_id - // broker.host is config-derived (KAFKA_ADVERTISED_HOST), not request-decoded - use + // broker.host is config-derived (IGGY_KAFKA_ADVERTISED_HOST), not request-decoded - use // the checked variant so an overly long hostname returns an error instead of panicking. if e.write_nullable_string(Some(&broker.host)).is_err() { return encode_error_only_response(ERROR_INVALID_REQUEST); @@ -478,16 +486,20 @@ pub fn encode_error_only_response(error_code: i16) -> Bytes { /// Decodes the requested topic names from a Metadata request body so the /// response can echo them back; clients match metadata by name, not position. +/// +/// A null topics array - `-1` legacy count, or `varint=0` compact count - means "all topics" +/// per the Kafka spec, not a malformed request; both decode to an empty list here since the +/// stub doesn't implement real topic listing yet. pub(crate) fn decode_metadata_request_topics(body: Bytes, api_version: i16) -> Result> { let mut d = Decoder::new(body); let flexible = api_version >= 9; let topics_count = if flexible { d.read_compact_array_count()? } else { - d.read_i32_array_count()? + d.read_i32_array_count_nullable()? }; - let mut topics = Vec::with_capacity(topics_count); + let mut topics = Vec::with_capacity(topics_count.min(PREALLOC_HINT)); for _ in 0..topics_count { if flexible && api_version >= 10 { // MetadataRequestTopic.topic_id: 16-byte UUID before name (v10+). @@ -524,6 +536,23 @@ mod tests { assert!(matches!(err, KafkaProtocolError::NullTopicName)); } + #[test] + fn decode_metadata_request_topics_legacy_null_array_means_all_topics() { + // -1 is the spec-defined "all topics" sentinel for the legacy i32 array count, not a + // malformed request - must decode to an empty list, not InvalidArrayLength. + let body = Bytes::from_static(&[0xff, 0xff, 0xff, 0xff]); // -1 + let topics = decode_metadata_request_topics(body, 0).unwrap(); + assert!(topics.is_empty()); + } + + #[test] + fn decode_metadata_request_topics_legacy_other_negative_counts_still_fail() { + // Only -1 is the null sentinel; any other negative count is genuinely malformed. + let body = Bytes::from_static(&[0xff, 0xff, 0xff, 0xfe]); // -2 + let err = decode_metadata_request_topics(body, 0).unwrap_err(); + assert!(matches!(err, KafkaProtocolError::InvalidArrayLength(-2))); + } + #[test] fn decode_metadata_request_topics_flexible_v10_truncated_topic_id_fails() { let mut enc = Encoder::with_capacity(8); diff --git a/gateways/kafka/src/protocol/codec.rs b/gateways/kafka/src/protocol/codec.rs index 214c79af9e..7146f957d4 100644 --- a/gateways/kafka/src/protocol/codec.rs +++ b/gateways/kafka/src/protocol/codec.rs @@ -35,6 +35,14 @@ use crate::error::{KafkaProtocolError, Result}; /// Matches typical broker limits and prevents OOM from adversarial length prefixes. pub const MAX_COLLECTION_LEN: usize = 65_536; +/// Initial `Vec::with_capacity` hint for wire-decoded array counts. +/// +/// A count up to `MAX_COLLECTION_LEN` is still validated as a count, but pre-reserving on it +/// directly lets a few-byte frame declaring a huge count force a multi-megabyte allocation +/// before any element bytes are checked present. Clamp the reservation; genuine large arrays +/// still grow correctly via `Vec::push`'s amortized doubling once real elements are decoded. +pub const PREALLOC_HINT: usize = 128; + pub struct Decoder { bytes: Bytes, } @@ -117,6 +125,30 @@ impl Decoder { Ok(count) } + /// Legacy nullable array length: signed i32 count, where -1 is the spec-defined null + /// (`all items`) sentinel - used by Metadata's `topics` field to mean "all topics" - treated + /// as empty (0 elements) rather than an error, mirroring how `read_compact_array_count` + /// treats a null compact array. + pub fn read_i32_array_count_nullable(&mut self) -> Result { + let n = self.read_i32()?; + if n == -1 { + return Ok(0); + } + if n < 0 { + return Err(KafkaProtocolError::InvalidArrayLength(n)); + } + // Safe: n is in [0, i32::MAX]; i32::MAX (2_147_483_647) fits in usize + // on all 32-bit and 64-bit platforms this crate targets. + let count = n as usize; + if count > MAX_COLLECTION_LEN { + return Err(KafkaProtocolError::CollectionTooLarge { + count, + max: MAX_COLLECTION_LEN, + }); + } + Ok(count) + } + /// Compact array length: unsigned varint holding `element_count + 1`. /// Per the Kafka spec, varint=0 encodes a null (absent) array; treat as empty (0 elements) /// so optional fields like `forgotten_topics` are skipped rather than rejected. diff --git a/gateways/kafka/src/protocol/header.rs b/gateways/kafka/src/protocol/header.rs index f9c14ec065..932253579a 100644 --- a/gateways/kafka/src/protocol/header.rs +++ b/gateways/kafka/src/protocol/header.rs @@ -111,7 +111,7 @@ fn first_flexible_version_threshold(api_key: i16) -> i16 { 51 => 0, // AlterUserScramCredentials - always flexible 55 => 0, // DescribeQuorum - always flexible 56 => 0, // AlterPartition - always flexible - 57 => 1, // UpdateFeatures + 57 => 0, // UpdateFeatures - always flexible 60 => 0, // DescribeCluster - always flexible 61 => 0, // DescribeProducers - always flexible 64 => 0, // UnregisterBroker - always flexible diff --git a/gateways/kafka/src/protocol/requests.rs b/gateways/kafka/src/protocol/requests.rs index c9026b2a50..833b2ef3aa 100644 --- a/gateways/kafka/src/protocol/requests.rs +++ b/gateways/kafka/src/protocol/requests.rs @@ -19,7 +19,7 @@ #![allow(clippy::too_many_lines, clippy::doc_markdown)] use crate::error::{KafkaProtocolError, Result}; -use crate::protocol::codec::Decoder; +use crate::protocol::codec::{Decoder, PREALLOC_HINT}; use bytes::Bytes; /// Produce Request (API Key 0) @@ -114,7 +114,7 @@ pub fn decode_produce_request(version: i16, body: Bytes) -> ProduceDecodeResult } ); - let mut topics = Vec::with_capacity(topics_count); + let mut topics = Vec::with_capacity(topics_count.min(PREALLOC_HINT)); for _ in 0..topics_count { let topic = produce_decode!( acks_read, @@ -135,7 +135,7 @@ pub fn decode_produce_request(version: i16, body: Bytes) -> ProduceDecodeResult } ); - let mut partitions = Vec::with_capacity(partitions_count); + let mut partitions = Vec::with_capacity(partitions_count.min(PREALLOC_HINT)); for _ in 0..partitions_count { let partition = produce_decode!(acks_read, d.read_i32()); let records = produce_decode!( @@ -226,7 +226,7 @@ pub fn decode_fetch_request(version: i16, body: Bytes) -> Result { d.read_i32_array_count()? }; - let mut topics = Vec::with_capacity(topics_count); + let mut topics = Vec::with_capacity(topics_count.min(PREALLOC_HINT)); for _ in 0..topics_count { let topic = if flexible { d.read_compact_nullable_string()? @@ -242,7 +242,7 @@ pub fn decode_fetch_request(version: i16, body: Bytes) -> Result { d.read_i32_array_count()? }; - let mut partitions = Vec::with_capacity(partitions_count); + let mut partitions = Vec::with_capacity(partitions_count.min(PREALLOC_HINT)); for _ in 0..partitions_count { let partition = d.read_i32()?; @@ -344,7 +344,8 @@ pub struct ListOffsetsPartition { pub partition: i32, pub timestamp: i64, // -2 = earliest, -1 = latest } -/// Collapse to `Result` for tests and callers that only need a successful `decode_list_offsets_request`. +/// Decodes a raw byte stream into a `ListOffsetsRequest`. +/// /// # Errors /// /// Returns an error if the byte stream is malformed or if the API version is unsupported. @@ -362,7 +363,7 @@ pub fn decode_list_offsets_request(version: i16, body: Bytes) -> Result Result Result Bytes { } #[must_use] pub fn encode_fetch_response(version: i16, req: &FetchRequest) -> Bytes { - encode_fetch_response_inner(version, &req.topics, Some(ERROR_NONE), ERROR_NONE) + // Stub: discard payload and return a retriable error so clients don't mistake + // "no real data yet" for a genuinely empty partition (same philosophy as Produce). + encode_fetch_response_inner( + version, + &req.topics, + Some(ERROR_NONE), + ERROR_NOT_LEADER_OR_FOLLOWER, + ) } fn encode_fetch_response_inner( @@ -231,7 +238,9 @@ pub fn encode_list_offsets_error_response(version: i16, error_code: i16) -> Byte } #[must_use] pub fn encode_list_offsets_response(version: i16, req: &ListOffsetsRequest) -> Bytes { - encode_list_offsets_response_inner(version, &req.topics, ERROR_NONE) + // Stub: discard payload and return a retriable error, matching Produce/Fetch - a genuine + // offset lookup requires the same partition-leadership the stub doesn't have yet. + encode_list_offsets_response_inner(version, &req.topics, ERROR_NOT_LEADER_OR_FOLLOWER) } fn encode_list_offsets_response_inner( diff --git a/gateways/kafka/src/server.rs b/gateways/kafka/src/server.rs index 05af1ef44c..2e271824f1 100644 --- a/gateways/kafka/src/server.rs +++ b/gateways/kafka/src/server.rs @@ -22,7 +22,7 @@ use std::time::Duration; use bytes::{BufMut, BytesMut}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; -use tokio::sync::broadcast; +use tokio::sync::{Semaphore, broadcast}; use tokio::time::{timeout, timeout_at}; use tokio_util::task::TaskTracker; use tracing::{debug, error, info, warn}; @@ -43,14 +43,25 @@ const READ_CHUNK: usize = 65536; #[derive(Debug, Clone)] pub struct ServerConfig { pub bind_addr: String, - /// Hostname or IP advertised in Metadata (`KAFKA_ADVERTISED_HOST`). Required when `bind_addr` - /// uses a wildcard address (`0.0.0.0` / `::`). + /// Hostname or IP advertised in Metadata (`IGGY_KAFKA_ADVERTISED_HOST`). Required when + /// `bind_addr` uses a wildcard address (`0.0.0.0` / `::`). pub advertised_host: Option, - /// Port advertised in Metadata (`KAFKA_ADVERTISED_PORT`). Defaults to the bind port. + /// Port advertised in Metadata (`IGGY_KAFKA_ADVERTISED_PORT`). Defaults to the bind port. pub advertised_port: Option, pub max_frame_size: usize, + /// Maximum concurrent connections accepted before new ones are rejected. + pub max_connections: usize, + /// Bound on how long an accepted connection may sit idle before sending the next + /// frame's length prefix. Kafka brokers default `connections.max.idle.ms` to 10 minutes; + /// match that so well-behaved idle clients aren't dropped. + pub idle_timeout: Duration, pub read_timeout: Duration, pub write_timeout: Duration, + /// Cap on how long graceful shutdown waits for in-flight connections to finish. Without + /// this, a connection idling inside `idle_timeout` (10 minutes by default) would otherwise + /// hold shutdown open past typical orchestrator grace periods (e.g. Kubernetes' default + /// 30s `terminationGracePeriodSeconds`). + pub shutdown_drain_timeout: Duration, } impl Default for ServerConfig { @@ -60,8 +71,11 @@ impl Default for ServerConfig { advertised_host: None, advertised_port: None, max_frame_size: 8 * 1024 * 1024, + max_connections: 1024, + idle_timeout: Duration::from_mins(10), read_timeout: Duration::from_secs(15), write_timeout: Duration::from_secs(10), + shutdown_drain_timeout: Duration::from_secs(25), } } } @@ -84,20 +98,21 @@ impl BrokerAdvertise { let trimmed = advertised.trim(); if trimmed.is_empty() { return Err(KafkaProtocolError::InvalidConfig( - "KAFKA_ADVERTISED_HOST must not be empty".into(), + "IGGY_KAFKA_ADVERTISED_HOST must not be empty".into(), )); } if trimmed.len() > i16::MAX as usize { return Err(KafkaProtocolError::InvalidConfig( - "KAFKA_ADVERTISED_HOST exceeds Kafka nullable string limit (32767 bytes)" + "IGGY_KAFKA_ADVERTISED_HOST exceeds Kafka nullable string limit (32767 bytes)" .into(), )); } trimmed.to_string() } else if local_addr.ip().is_unspecified() { return Err(KafkaProtocolError::InvalidConfig( - "binding to a wildcard address (0.0.0.0 or ::) requires KAFKA_ADVERTISED_HOST \ - to be set to a reachable hostname or IP for Metadata broker advertisement" + "binding to a wildcard address (0.0.0.0 or ::) requires \ + IGGY_KAFKA_ADVERTISED_HOST to be set to a reachable hostname or IP for \ + Metadata broker advertisement" .into(), )); } else { @@ -144,7 +159,9 @@ impl KafkaServer { ); let tracker = TaskTracker::new(); - let broker = Arc::clone(&broker); + let conn_limiter = Arc::new(Semaphore::new(self.config.max_connections)); + + let drain_timeout = self.config.shutdown_drain_timeout; loop { tokio::select! { @@ -152,20 +169,17 @@ impl KafkaServer { match result { Ok(()) => { info!("kafka listener shutdown requested"); - tracker.close(); - tracker.wait().await; + drain(&tracker, drain_timeout).await; break; } // Capacity-1 channel: lagged means a signal was sent before we polled - treat as shutdown. Err(broadcast::error::RecvError::Lagged(_)) => { info!("kafka listener shutdown requested (lagged)"); - tracker.close(); - tracker.wait().await; + drain(&tracker, drain_timeout).await; break; } Err(broadcast::error::RecvError::Closed) => { - tracker.close(); - tracker.wait().await; + drain(&tracker, drain_timeout).await; break; } } @@ -173,6 +187,10 @@ impl KafkaServer { accept_result = listener.accept() => { match accept_result { Ok((stream, peer)) => { + let Ok(permit) = Arc::clone(&conn_limiter).try_acquire_owned() else { + warn!(%peer, max_connections = self.config.max_connections, "connection limit reached, rejecting"); + continue; + }; if let Err(e) = stream.set_nodelay(true) { warn!(%peer, "TCP_NODELAY failed: {e}"); } @@ -182,6 +200,7 @@ impl KafkaServer { let cfg = Arc::clone(&self.config); let broker = Arc::clone(&broker); tracker.spawn(async move { + let _permit = permit; if let Err(err) = handle_connection(stream, cfg, peer, broker).await { warn!(%peer, "connection closed with error: {err}"); } @@ -194,7 +213,10 @@ impl KafkaServer { } warn!(%e, "transient accept error, continuing"); } - Err(e) => return Err(e.into()), + Err(e) => { + drain(&tracker, drain_timeout).await; + return Err(e.into()); + } } } @@ -204,12 +226,23 @@ impl KafkaServer { } } -fn is_transient_accept_error(err: &std::io::Error) -> bool { - use std::io::ErrorKind; +/// Close the tracker to new spawns and wait for in-flight connections to finish, but not past +/// `deadline` - an idle connection can otherwise hold shutdown open for up to `idle_timeout` +/// (10 minutes by default), past typical orchestrator grace periods. +async fn drain(tracker: &TaskTracker, deadline: Duration) { + tracker.close(); + if timeout(deadline, tracker.wait()).await.is_err() { + warn!( + ?deadline, + "shutdown drain deadline exceeded; abandoning in-flight connections" + ); + } +} +fn is_transient_accept_error(err: &std::io::Error) -> bool { matches!( err.kind(), - ErrorKind::Interrupted | ErrorKind::ConnectionAborted | ErrorKind::WouldBlock + io::ErrorKind::Interrupted | io::ErrorKind::ConnectionAborted | io::ErrorKind::WouldBlock ) || matches!( err.raw_os_error(), // EMFILE / ENFILE are common across Unix platforms when fd limits are hit. @@ -232,7 +265,13 @@ async fn handle_connection( debug!(%peer, "connection accepted"); loop { - let frame = match read_frame(&mut stream, config.max_frame_size, config.read_timeout).await + let frame = match read_frame( + &mut stream, + config.max_frame_size, + config.idle_timeout, + config.read_timeout, + ) + .await { Ok(f) => f, Err(KafkaProtocolError::Io(ref e)) @@ -367,21 +406,25 @@ fn correlation_id_from_frame(frame: &bytes::Bytes) -> i32 { pub async fn read_frame( stream: &mut TcpStream, max_frame_size: usize, + idle_timeout: Duration, read_timeout: Duration, ) -> Result { let mut len_buf = [0u8; 4]; - // Idle: block until client starts next frame (or EOF). No read_timeout here. - stream.read_exact(&mut len_buf).await?; + // Idle: bounded wait for the client to start the next frame (or EOF). + match timeout(idle_timeout, stream.read_exact(&mut len_buf)).await { + Ok(Ok(_)) => {} + Ok(Err(e)) => return Err(e.into()), + Err(_) => return Err(io::Error::new(io::ErrorKind::TimedOut, "idle timeout").into()), + } let frame_len_i32 = i32::from_be_bytes(len_buf); if frame_len_i32 <= 0 { return Err(KafkaProtocolError::InvalidFrameLength(frame_len_i32)); } - let frame_len = - usize::try_from(frame_len_i32).map_err(|_| KafkaProtocolError::FrameTooLarge { - max_bytes: max_frame_size, - actual_bytes: usize::MAX, - })?; + // frame_len_i32 is validated > 0 above, so it always fits usize on every + // platform this crate targets (32-bit and 64-bit). + #[allow(clippy::cast_sign_loss)] + let frame_len = frame_len_i32 as usize; if frame_len > max_frame_size { return Err(KafkaProtocolError::FrameTooLarge { max_bytes: max_frame_size, @@ -391,17 +434,22 @@ pub async fn read_frame( // In-flight: read_timeout applies only after the length prefix is complete. let deadline = tokio::time::Instant::now() + read_timeout; - // read_buf() exposes all BytesMut spare capacity to the OS; after reserve(n) the - // allocator may give more than n bytes, so the OS can fill past frame_len and silently - // consume bytes belonging to the next pipelined frame. Use read() with a bounded slice - // so each OS call is limited to exactly the remaining bytes needed. - let mut data = BytesMut::with_capacity(frame_len); + // Reserve incrementally, one chunk ahead of what's actually been received, instead of + // BytesMut::with_capacity(frame_len) up front - frame_len comes straight from the wire + // (bounded only by max_frame_size), so an attacker who sends a valid length prefix and + // then no body would otherwise force a full max_frame_size allocation per connection + // before a single body byte arrives (same amplification class as PREALLOC_HINT). + let mut data = BytesMut::with_capacity(frame_len.min(READ_CHUNK)); while data.len() < frame_len { let remaining = frame_len - data.len(); let chunk = remaining.min(READ_CHUNK); - let prev = data.len(); - data.resize(prev + chunk, 0); - let n = match timeout_at(deadline, stream.read(&mut data[prev..prev + chunk])).await { + data.reserve(chunk); + // `.limit(chunk)` bounds how much of BytesMut's spare capacity read_buf may fill, + // so a single OS read still can't consume bytes belonging to the next pipelined + // frame - the same guarantee the old resize()+read() approach had - but without + // pre-zeroing the chunk first, since read_buf only writes into its own spare + // capacity via chunk_mut() rather than requiring pre-initialized memory. + match timeout_at(deadline, stream.read_buf(&mut (&mut data).limit(chunk))).await { Err(_) => return Err(io::Error::new(io::ErrorKind::TimedOut, "read timeout").into()), Ok(Ok(0)) => { return Err( @@ -409,9 +457,8 @@ pub async fn read_frame( ); } Ok(Err(e)) => return Err(e.into()), - Ok(Ok(n)) => n, - }; - data.truncate(prev + n); + Ok(Ok(_)) => {} + } } Ok(data.freeze()) } @@ -493,9 +540,14 @@ mod tests { async fn read_frame_rejects_negative_length() { let (mut client, mut server) = tcp_pair().await; client.write_all(&(-1_i32).to_be_bytes()).await.unwrap(); - let err = read_frame(&mut server, 64, Duration::from_secs(1)) - .await - .unwrap_err(); + let err = read_frame( + &mut server, + 64, + Duration::from_secs(5), + Duration::from_secs(1), + ) + .await + .unwrap_err(); assert!(matches!(err, KafkaProtocolError::InvalidFrameLength(-1))); } @@ -504,9 +556,14 @@ mod tests { let (mut client, mut server) = tcp_pair().await; client.write_all(&(5_i32).to_be_bytes()).await.unwrap(); client.shutdown().await.unwrap(); - let err = read_frame(&mut server, 64, Duration::from_secs(1)) - .await - .unwrap_err(); + let err = read_frame( + &mut server, + 64, + Duration::from_secs(5), + Duration::from_secs(1), + ) + .await + .unwrap_err(); assert!(err.to_string().contains("connection closed")); } @@ -515,12 +572,88 @@ mod tests { let (mut client, mut server) = tcp_pair().await; client.write_all(&(5_i32).to_be_bytes()).await.unwrap(); client.write_all(&[1, 2]).await.unwrap(); - let err = read_frame(&mut server, 64, Duration::from_millis(50)) - .await - .unwrap_err(); + let err = read_frame( + &mut server, + 64, + Duration::from_secs(5), + Duration::from_millis(50), + ) + .await + .unwrap_err(); assert!(err.to_string().contains("read timeout")); } + #[tokio::test] + async fn read_frame_times_out_when_client_sends_nothing() { + let (_client, mut server) = tcp_pair().await; + let err = read_frame( + &mut server, + 64, + Duration::from_millis(50), + Duration::from_secs(1), + ) + .await + .unwrap_err(); + assert!(err.to_string().contains("idle timeout")); + } + + #[tokio::test] + async fn server_shutdown_does_not_stall_past_drain_timeout() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let (tx, rx) = broadcast::channel(1); + let server = KafkaServer::new(ServerConfig { + // Idle timeout is intentionally long - the drain deadline, not the idle timeout, + // must be what bounds shutdown here. + idle_timeout: Duration::from_mins(10), + shutdown_drain_timeout: Duration::from_millis(100), + ..ServerConfig::default() + }); + let handle = tokio::spawn(async move { server.run(listener, rx).await }); + + // Held open, never sends a frame: the in-flight connection task is parked in + // read_frame's idle wait for the full 600s idle_timeout unless drain cuts it short. + let _held = TcpStream::connect(addr).await.unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + + tx.send(()).unwrap(); + let result = tokio::time::timeout(Duration::from_secs(2), handle) + .await + .expect("shutdown must return well within the 600s idle_timeout") + .unwrap(); + assert!(result.is_ok()); + } + + #[tokio::test] + async fn server_rejects_connections_beyond_max_connections() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let (tx, rx) = broadcast::channel(1); + let server = KafkaServer::new(ServerConfig { + max_connections: 1, + ..ServerConfig::default() + }); + let handle = tokio::spawn(async move { server.run(listener, rx).await }); + + // First connection holds the only permit by never sending a frame. + let held = TcpStream::connect(addr).await.unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + + // Second connection should be accepted at the TCP level (backlog) but closed + // immediately by the server once the permit acquisition fails. + let mut rejected = TcpStream::connect(addr).await.unwrap(); + let mut buf = [0u8; 1]; + let n = tokio::time::timeout(Duration::from_secs(1), rejected.read(&mut buf)) + .await + .expect("server should close rejected connection promptly") + .unwrap(); + assert_eq!(n, 0, "rejected connection should be closed with EOF"); + + tx.send(()).unwrap(); + drop(held); + handle.await.unwrap().unwrap(); + } + #[tokio::test] async fn server_run_exits_when_shutdown_channel_closed() { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -564,10 +697,41 @@ mod tests { .await .unwrap(); client.write_all(&payload).await.unwrap(); - let frame = read_frame(&mut server, max_frame_size, Duration::from_secs(1)) + let frame = read_frame( + &mut server, + max_frame_size, + Duration::from_secs(5), + Duration::from_secs(1), + ) + .await + .unwrap(); + assert_eq!(frame.len(), max_frame_size); + } + + #[tokio::test] + async fn read_frame_reassembles_frame_spanning_multiple_read_chunks() { + // frame_len exceeds READ_CHUNK, forcing the incremental reserve()+read_buf loop + // through more than one iteration - guards the switch away from a single + // BytesMut::with_capacity(frame_len) upfront allocation. + let (mut client, mut server) = tcp_pair().await; + let frame_len = READ_CHUNK + 1024; + let payload: Vec = (0..frame_len) + .map(|i| u8::try_from(i % 251).expect("i % 251 < 256")) + .collect(); + client + .write_all(&i32::try_from(frame_len).unwrap().to_be_bytes()) .await .unwrap(); - assert_eq!(frame.len(), max_frame_size); + client.write_all(&payload).await.unwrap(); + let frame = read_frame( + &mut server, + frame_len, + Duration::from_secs(5), + Duration::from_secs(1), + ) + .await + .unwrap(); + assert_eq!(&frame[..], &payload[..]); } #[tokio::test] @@ -575,9 +739,14 @@ mod tests { let (mut client, mut server) = tcp_pair().await; let max_frame_size = 64usize; client.write_all(&65_i32.to_be_bytes()).await.unwrap(); - let err = read_frame(&mut server, max_frame_size, Duration::from_secs(1)) - .await - .unwrap_err(); + let err = read_frame( + &mut server, + max_frame_size, + Duration::from_secs(5), + Duration::from_secs(1), + ) + .await + .unwrap_err(); assert!(matches!( err, KafkaProtocolError::FrameTooLarge { diff --git a/gateways/kafka/tests/api_handler_tests.rs b/gateways/kafka/tests/api_handler_tests.rs index fa70c77a6e..23ed04df02 100644 --- a/gateways/kafka/tests/api_handler_tests.rs +++ b/gateways/kafka/tests/api_handler_tests.rs @@ -38,7 +38,10 @@ use iggy_gateway_kafka::protocol::requests::{ProduceDecodeResult, decode_produce use fixtures::load_fixture_body_or_skip; use scope::default_broker; use tcp::{build_metadata_legacy_request, build_produce_v3_body}; -use wire::{build_metadata_flexible_request, build_metadata_flexible_request_v10}; +use wire::{ + build_metadata_flexible_request, build_metadata_flexible_request_v10, + build_produce_legacy_request, +}; // ── ApiVersions ───────────────────────────────────────────────────────────── @@ -328,7 +331,7 @@ fn produce_stub_response_returns_retriable_not_leader() { } #[test] -fn fetch_stub_response_has_zero_partition_error() { +fn fetch_stub_response_returns_retriable_not_leader() { for version in 4i16..=12 { let Some(body) = load_fixture_body_or_skip(1, "Fetch", version) else { continue; @@ -356,14 +359,14 @@ fn fetch_stub_response_has_zero_partition_error() { let _partition = d.read_i32().unwrap(); assert_eq!( d.read_i16().unwrap(), - ERROR_NONE, + ERROR_NOT_LEADER_OR_FOLLOWER, "Fetch v{version} partition error" ); } } #[test] -fn list_offsets_stub_response_has_zero_error() { +fn list_offsets_stub_response_returns_retriable_not_leader() { for version in 1i16..=6 { let Some(body) = load_fixture_body_or_skip(2, "ListOffsets", version) else { continue; @@ -385,7 +388,11 @@ fn list_offsets_stub_response_has_zero_error() { let _parts = d.read_i32().unwrap(); } let _partition = d.read_i32().unwrap(); - assert_eq!(d.read_i16().unwrap(), ERROR_NONE, "ListOffsets v{version}"); + assert_eq!( + d.read_i16().unwrap(), + ERROR_NOT_LEADER_OR_FOLLOWER, + "ListOffsets v{version}" + ); } } @@ -662,6 +669,21 @@ fn produce_acks_zero_malformed_body_decode_carries_acks() { ); } +#[test] +fn produce_acks_zero_stays_silent_on_advertised_but_unsupported_version() { + // ApiVersions advertises Produce min=0 (KAFKA-18659) while the firewall's real floor is 3 + // (SUPPORTED_RANGES). A spec-compliant client can legitimately send v0-2 with acks=0; the + // firewall must not run before acks is decoded, or this silence contract breaks for exactly + // the versions ApiVersions told the client were fine to use. + for version in 0i16..3 { + let body = build_produce_legacy_request(version, 0, None, None); + assert!( + handle_request(API_KEY_PRODUCE, version, body, &default_broker()).is_no_response(), + "Produce v{version} acks=0 must stay silent even though the firewall doesn't accept v{version}" + ); + } +} + // ── Metadata topic name echo (must not hardcode a placeholder topic name) ── fn read_metadata_v1_topics(d: &mut Decoder, expected_count: i32) -> Vec { diff --git a/gateways/kafka/tests/broker_advertise_tests.rs b/gateways/kafka/tests/broker_advertise_tests.rs index c1236feaf3..4bba7d5faf 100644 --- a/gateways/kafka/tests/broker_advertise_tests.rs +++ b/gateways/kafka/tests/broker_advertise_tests.rs @@ -71,7 +71,7 @@ fn from_server_config_rejects_wildcard_bind_without_advertised_host() { }; let local_addr: SocketAddr = "0.0.0.0:9093".parse().unwrap(); let err = BrokerAdvertise::from_server_config(&config, local_addr).unwrap_err(); - assert!(err.to_string().contains("KAFKA_ADVERTISED_HOST")); + assert!(err.to_string().contains("IGGY_KAFKA_ADVERTISED_HOST")); } #[test] @@ -95,7 +95,7 @@ fn from_server_config_rejects_advertised_host_exceeding_kafka_string_limit() { }; let local_addr: SocketAddr = "127.0.0.1:9093".parse().unwrap(); let err = BrokerAdvertise::from_server_config(&config, local_addr).unwrap_err(); - assert!(err.to_string().contains("KAFKA_ADVERTISED_HOST")); + assert!(err.to_string().contains("IGGY_KAFKA_ADVERTISED_HOST")); } #[test] diff --git a/gateways/kafka/tests/common/server.rs b/gateways/kafka/tests/common/server.rs index ea7891a19d..9fa5311939 100644 --- a/gateways/kafka/tests/common/server.rs +++ b/gateways/kafka/tests/common/server.rs @@ -32,8 +32,11 @@ pub async fn spawn_test_server() -> (SocketAddr, broadcast::Sender<()>) { advertised_host: None, advertised_port: None, max_frame_size: 8 * 1024 * 1024, + max_connections: 1024, + idle_timeout: Duration::from_secs(5), read_timeout: Duration::from_secs(5), write_timeout: Duration::from_secs(5), + shutdown_drain_timeout: Duration::from_secs(5), }) .await } diff --git a/gateways/kafka/tests/decode_safety_tests.rs b/gateways/kafka/tests/decode_safety_tests.rs index 4d047e1426..6bca4911e0 100644 --- a/gateways/kafka/tests/decode_safety_tests.rs +++ b/gateways/kafka/tests/decode_safety_tests.rs @@ -55,6 +55,26 @@ fn i32_array_length_above_max_returns_collection_too_large() { assert!(matches!(err, KafkaProtocolError::CollectionTooLarge { .. })); } +#[test] +fn fetch_max_declared_topics_count_with_empty_body_returns_error_not_large_alloc() { + // Declares the maximum allowed topics_count (65_536) but supplies no element bytes at + // all. Guards against pre-reserving a Vec directly off the wire count before validating + // any element bytes are present - decode must fail fast on the first missing byte, not + // attempt a large upfront allocation. + let mut body = Vec::new(); + body.extend_from_slice(&0_i32.to_be_bytes()); // replica_id + body.extend_from_slice(&0_i32.to_be_bytes()); // max_wait_ms + body.extend_from_slice(&0_i32.to_be_bytes()); // min_bytes + body.extend_from_slice(&0_i32.to_be_bytes()); // max_bytes (version >= 3) + body.push(0); // isolation_level (version >= 4) + let topics_count = i32::try_from(MAX_COLLECTION_LEN).expect("fits i32"); + body.extend_from_slice(&topics_count.to_be_bytes()); + // no topic bytes follow + + let err = decode_fetch_request(4, Bytes::from(body)).unwrap_err(); + assert!(matches!(err, KafkaProtocolError::BufferUnderflow { .. })); +} + #[test] fn produce_decoder_rejects_truncated_flexible_body() { let mut body = Vec::new(); diff --git a/gateways/kafka/tests/decode_validation_tests.rs b/gateways/kafka/tests/decode_validation_tests.rs index 10e6d38208..700d16d56f 100644 --- a/gateways/kafka/tests/decode_validation_tests.rs +++ b/gateways/kafka/tests/decode_validation_tests.rs @@ -25,12 +25,7 @@ //! header v2: [`client_id`] `COMPACT_NULLABLE_STRING` + request-header tagged fields //! [request body] ← properly encoded per spec (flexible or not) -use std::path::PathBuf; - -use bytes::Bytes; - -use iggy_gateway_kafka::protocol::codec::{Decoder, Encoder}; -use iggy_gateway_kafka::protocol::header::{RequestHeader, request_header_version}; +use iggy_gateway_kafka::protocol::codec::Encoder; use iggy_gateway_kafka::protocol::requests::{ decode_create_topics_request, decode_fetch_request, decode_list_offsets_request, decode_produce_request, @@ -40,40 +35,12 @@ use iggy_gateway_kafka::protocol::responses::{ encode_produce_response, }; +#[path = "common/fixtures.rs"] +mod fixtures; #[path = "common/wire.rs"] mod wire; -// ── helpers ─────────────────────────────────────────────────────────────────── - -fn fixtures_dir() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tools/kafka-tool/kafka_messages") -} - -/// Load a kafka-tool `.bin` file and return just the request body bytes, or `None` -/// (after a standardized skip note) when the gitignored fixture is absent, so a fresh -/// clone skips rather than panicking. -fn load_body(api_key: i16, api_name: &str, version: i16) -> Option { - let filename = format!("{api_key:03}_{api_name}_v{version}.bin"); - let path = fixtures_dir().join(&filename); - let Ok(data) = std::fs::read(&path) else { - eprintln!( - "skipping {filename}: wire fixture missing - generate with \ - `gateways/kafka/scripts/ci-wire-fixtures.sh generate` (or the kafka-tool \ - `generate` subcommand)" - ); - return None; - }; - - let frame = Bytes::copy_from_slice(&data[4..]); - let hdr_ver = request_header_version(api_key, version); - let mut decoder = Decoder::new(frame); - RequestHeader::decode_from(&mut decoder, hdr_ver).expect("fixture request header must decode"); - Some( - decoder - .read_bytes(decoder.remaining()) - .expect("fixture request body must decode"), - ) -} +use fixtures::load_fixture_body_or_skip as load_body; // ── Produce (API key 0) ─────────────────────────────────────────────────────── diff --git a/gateways/kafka/tests/header_tests.rs b/gateways/kafka/tests/header_tests.rs index aa8687660d..3cbf52d010 100644 --- a/gateways/kafka/tests/header_tests.rs +++ b/gateways/kafka/tests/header_tests.rs @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +use kafka_protocol::messages::ApiKey; + use iggy_gateway_kafka::protocol::api::{ API_KEY_API_VERSIONS, API_KEY_CREATE_TOPICS, API_KEY_FETCH, API_KEY_LIST_OFFSETS, API_KEY_METADATA, API_KEY_PRODUCE, @@ -107,7 +109,8 @@ fn response_header_v1_encodes_correlation_id_plus_tagged_fields() { // ── Header version lookup ─────────────────────────────────────────────────── -/// Flexible-encoding threshold per API key (mirrors `protocol/header.rs`). +/// Flexible-encoding threshold per API key (mirrors `protocol/header.rs`; cross-checked against +/// the independent `kafka-protocol` crate below rather than trusted on its own). const API_KEY_FLEXIBLE_FROM: &[(i16, i16)] = &[ (0, 9), (1, 12), @@ -163,7 +166,7 @@ const API_KEY_FLEXIBLE_FROM: &[(i16, i16)] = &[ (51, 0), (55, 0), (56, 0), - (57, 1), + (57, 0), (60, 0), (61, 0), (64, 0), @@ -183,6 +186,29 @@ const API_KEY_FLEXIBLE_FROM: &[(i16, i16)] = &[ (80, 0), ]; +#[test] +fn request_header_version_matches_independent_kafka_protocol_crate() { + // API_KEY_FLEXIBLE_FROM is hand-transcribed from header.rs's threshold table, so comparing + // request_header_version only against that same mirror can't catch a value wrong in both + // places (the same transcription mistake copied twice). Cross-check against the third-party + // `kafka-protocol` crate's own per-key header-version logic instead, over every version that + // crate considers actually valid for the key - outside that range a version never appeared + // on the real wire, so there's no independently-meaningful answer to compare against. + for &(api_key, _) in API_KEY_FLEXIBLE_FROM { + let Ok(external) = ApiKey::try_from(api_key) else { + continue; + }; + let range = external.valid_versions(); + for version in range.min..=range.max { + assert_eq!( + request_header_version(api_key, version), + external.request_header_version(version), + "api_key={api_key} version={version}: gateway vs kafka-protocol header version" + ); + } + } +} + #[test] fn request_header_version_hits_every_api_key_match_arm() { for &(api_key, flexible_from) in API_KEY_FLEXIBLE_FROM { diff --git a/gateways/kafka/tests/listener_robustness_tests.rs b/gateways/kafka/tests/listener_robustness_tests.rs index cd62e376fa..ac3309f48c 100644 --- a/gateways/kafka/tests/listener_robustness_tests.rs +++ b/gateways/kafka/tests/listener_robustness_tests.rs @@ -89,8 +89,11 @@ async fn e2e_frame_within_custom_max_frame_size_accepted() { advertised_host: None, advertised_port: None, max_frame_size: max_frame, + max_connections: 1024, + idle_timeout: Duration::from_secs(5), read_timeout: Duration::from_secs(5), write_timeout: Duration::from_secs(5), + shutdown_drain_timeout: Duration::from_secs(5), }) .await; @@ -117,8 +120,11 @@ async fn e2e_frame_exceeding_max_frame_size_closes_connection() { advertised_host: None, advertised_port: None, max_frame_size: max_frame, + max_connections: 1024, + idle_timeout: Duration::from_secs(5), read_timeout: Duration::from_secs(5), write_timeout: Duration::from_secs(5), + shutdown_drain_timeout: Duration::from_secs(5), }) .await; @@ -144,8 +150,11 @@ async fn e2e_truncated_frame_body_closes_connection() { advertised_host: None, advertised_port: None, max_frame_size: 8 * 1024 * 1024, + max_connections: 1024, + idle_timeout: Duration::from_secs(5), read_timeout: Duration::from_secs(1), write_timeout: Duration::from_secs(5), + shutdown_drain_timeout: Duration::from_secs(5), }) .await; let mut stream = TcpStream::connect(addr).await.expect("connect"); @@ -259,8 +268,11 @@ async fn e2e_slow_client_can_complete_request_within_read_timeout() { advertised_host: None, advertised_port: None, max_frame_size: 8 * 1024 * 1024, + max_connections: 1024, + idle_timeout: Duration::from_secs(5), read_timeout: Duration::from_secs(5), write_timeout: Duration::from_secs(5), + shutdown_drain_timeout: Duration::from_secs(5), }) .await; @@ -451,8 +463,11 @@ async fn e2e_quiet_connection_survives_beyond_read_timeout_idle_cap() { advertised_host: None, advertised_port: None, max_frame_size: 8 * 1024 * 1024, + max_connections: 1024, + idle_timeout: Duration::from_secs(5), read_timeout: Duration::from_secs(3), write_timeout: Duration::from_secs(5), + shutdown_drain_timeout: Duration::from_secs(5), }) .await; diff --git a/gateways/kafka/tests/server_e2e_tests.rs b/gateways/kafka/tests/server_e2e_tests.rs index 36d3e371d8..13f237961e 100644 --- a/gateways/kafka/tests/server_e2e_tests.rs +++ b/gateways/kafka/tests/server_e2e_tests.rs @@ -32,7 +32,7 @@ use tokio::net::TcpStream; use iggy_gateway_kafka::protocol::api::{ API_KEY_API_VERSIONS, API_KEY_CREATE_TOPICS, API_KEY_FETCH, API_KEY_LIST_OFFSETS, - API_KEY_METADATA, API_KEY_PRODUCE, ERROR_NONE, ERROR_UNSUPPORTED_VERSION, + API_KEY_METADATA, API_KEY_PRODUCE, ERROR_NOT_LEADER_OR_FOLLOWER, ERROR_UNSUPPORTED_VERSION, }; use iggy_gateway_kafka::protocol::codec::Decoder; @@ -141,16 +141,8 @@ async fn e2e_sequential_requests_on_one_connection() { } } -#[tokio::test] -async fn e2e_negative_frame_length_closes_connection() { - let (addr, _shutdown) = spawn_test_server().await; - let mut stream = TcpStream::connect(addr).await.unwrap(); - stream.write_all(&(-1i32).to_be_bytes()).await.unwrap(); - - let mut buf = [0u8; 1]; - let n = stream.read(&mut buf).await.unwrap_or(0); - assert_eq!(n, 0, "server should close after invalid frame length"); -} +// Negative-frame-length-closes-connection coverage lives in listener_robustness_tests.rs +// (uses a timeout-guarded read helper, so a regression fails fast instead of hanging). #[tokio::test] async fn e2e_oversized_frame_is_rejected() { @@ -364,7 +356,7 @@ async fn produce_v3_through_v9_e2e_preserve_correlation_id() { // ── ListOffsets supported versions ────────────────────────────────────────── #[tokio::test] -async fn list_offsets_v1_through_v6_e2e_return_partition_error_zero() { +async fn list_offsets_v1_through_v6_e2e_return_retriable_not_leader() { let (addr, _shutdown) = spawn_test_server().await; for version in 1i16..=6 { @@ -391,7 +383,7 @@ async fn list_offsets_v1_through_v6_e2e_return_partition_error_zero() { d.read_i32().unwrap(); assert_eq!( d.read_i16().unwrap(), - ERROR_NONE, + ERROR_NOT_LEADER_OR_FOLLOWER, "ListOffsets v{version} stub partition error" ); } diff --git a/gateways/kafka/tests/server_integration_tests.rs b/gateways/kafka/tests/server_integration_tests.rs index 00495fa009..9a8f3c87d6 100644 --- a/gateways/kafka/tests/server_integration_tests.rs +++ b/gateways/kafka/tests/server_integration_tests.rs @@ -70,9 +70,14 @@ async fn read_frame_reads_valid_payload() { frame.extend_from_slice(&payload); client.write_all(&frame).await.unwrap(); - let parsed = read_frame(&mut server, 4096, Duration::from_secs(1)) - .await - .unwrap(); + let parsed = read_frame( + &mut server, + 4096, + Duration::from_secs(5), + Duration::from_secs(1), + ) + .await + .unwrap(); assert_eq!(parsed, payload); } @@ -99,9 +104,14 @@ async fn read_frame_rejects_invalid_lengths() { let (mut client, mut server) = tcp_pair().await; client.write_all(&0i32.to_be_bytes()).await.unwrap(); - let err = read_frame(&mut server, 128, Duration::from_secs(1)) - .await - .expect_err("zero frame must fail"); + let err = read_frame( + &mut server, + 128, + Duration::from_secs(5), + Duration::from_secs(1), + ) + .await + .expect_err("zero frame must fail"); assert!(err.to_string().contains("invalid frame length")); // Ensure connection can still be reused for a second scenario by writing a valid new prefix+payload. @@ -109,9 +119,14 @@ async fn read_frame_rejects_invalid_lengths() { frame.extend_from_slice(&(200i32).to_be_bytes()); frame.resize(4 + 200, 0); client.write_all(&frame).await.unwrap(); - let err = read_frame(&mut server, 64, Duration::from_secs(1)) - .await - .expect_err("large frame must fail"); + let err = read_frame( + &mut server, + 64, + Duration::from_secs(5), + Duration::from_secs(1), + ) + .await + .expect_err("large frame must fail"); assert!(err.to_string().contains("exceeds max frame size")); } @@ -147,9 +162,14 @@ async fn read_frame_does_not_consume_pipelined_frame_bytes() { both.extend_from_slice(payload2); client.write_all(&both).await.unwrap(); + let idle_timeout = Duration::from_secs(5); let timeout = Duration::from_secs(1); - let frame1 = read_frame(&mut server, 4096, timeout).await.unwrap(); - let frame2 = read_frame(&mut server, 4096, timeout).await.unwrap(); + let frame1 = read_frame(&mut server, 4096, idle_timeout, timeout) + .await + .unwrap(); + let frame2 = read_frame(&mut server, 4096, idle_timeout, timeout) + .await + .unwrap(); assert_eq!(&frame1[..], payload1); assert_eq!(&frame2[..], payload2); diff --git a/gateways/kafka/tools/kafka-tool/src/main.rs b/gateways/kafka/tools/kafka-tool/src/main.rs index b63f7d9844..f21d8dc295 100644 --- a/gateways/kafka/tools/kafka-tool/src/main.rs +++ b/gateways/kafka/tools/kafka-tool/src/main.rs @@ -19,8 +19,22 @@ use anyhow::{Context, Result}; use bytes::{BufMut, Bytes, BytesMut}; use clap::{Parser, Subcommand}; use iggy_gateway_kafka::protocol::api::supported_api_ranges; +use kafka_protocol::messages::add_partitions_to_txn_request::*; +use kafka_protocol::messages::create_topics_request::*; +use kafka_protocol::messages::delete_records_request::*; +use kafka_protocol::messages::delete_topics_request::*; +use kafka_protocol::messages::describe_configs_request::*; +use kafka_protocol::messages::fetch_request::*; +use kafka_protocol::messages::join_group_request::*; +use kafka_protocol::messages::list_offsets_request::*; +use kafka_protocol::messages::offset_commit_request::*; +use kafka_protocol::messages::produce_request::*; +use kafka_protocol::messages::txn_offset_commit_request::*; use kafka_protocol::messages::*; use kafka_protocol::protocol::{Encodable, StrBytes}; +use kafka_protocol::records::{ + Compression, Record, RecordBatchEncoder, RecordEncodeOptions, TimestampType, +}; use std::path::PathBuf; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpStream; @@ -272,10 +286,6 @@ fn build_payload(api_key: i16, version: i16) -> Result { r.encode(&mut buf, version).context("Metadata")?; } 0 => { - use kafka_protocol::messages::produce_request::*; - use kafka_protocol::records::{ - Compression, Record, RecordBatchEncoder, RecordEncodeOptions, TimestampType, - }; let rec = Record { transactional: false, control: false, @@ -317,7 +327,6 @@ fn build_payload(api_key: i16, version: i16) -> Result { r.encode(&mut buf, version).context("Produce")?; } 1 => { - use kafka_protocol::messages::fetch_request::*; let fp = FetchPartition::default() .with_partition(0) .with_fetch_offset(0) @@ -343,7 +352,6 @@ fn build_payload(api_key: i16, version: i16) -> Result { r.encode(&mut buf, version).context("Fetch")?; } 2 => { - use kafka_protocol::messages::list_offsets_request::*; let p = ListOffsetsPartition::default() .with_partition_index(0) .with_timestamp(-1); @@ -358,7 +366,6 @@ fn build_payload(api_key: i16, version: i16) -> Result { .context("ListOffsets")?; } 8 => { - use kafka_protocol::messages::offset_commit_request::*; let p = OffsetCommitRequestPartition::default() .with_partition_index(0) .with_committed_offset(42) @@ -386,7 +393,6 @@ fn build_payload(api_key: i16, version: i16) -> Result { .context("FindCoordinator")?; } 11 => { - use kafka_protocol::messages::join_group_request::*; let p = JoinGroupRequestProtocol::default() .with_name(StrBytes::from_static_str("range")) .with_metadata(Bytes::from_static(b"\x00\x00\x00\x01\x00\x0atest-topic")); @@ -444,7 +450,6 @@ fn build_payload(api_key: i16, version: i16) -> Result { .context("SaslHandshake")?; } 19 => { - use kafka_protocol::messages::create_topics_request::*; let t = CreatableTopic::default() .with_name(TopicName::from(StrBytes::from_static_str( "iggy-test-topic", @@ -459,7 +464,6 @@ fn build_payload(api_key: i16, version: i16) -> Result { .context("CreateTopics")?; } 20 => { - use kafka_protocol::messages::delete_topics_request::*; let r = if version >= 6 { DeleteTopicsRequest::default() .with_topics(vec![DeleteTopicState::default().with_name(Some( @@ -476,7 +480,6 @@ fn build_payload(api_key: i16, version: i16) -> Result { r.encode(&mut buf, version).context("DeleteTopics")?; } 21 => { - use kafka_protocol::messages::delete_records_request::*; let p = DeleteRecordsPartition::default() .with_partition_index(0) .with_offset(0); @@ -497,7 +500,6 @@ fn build_payload(api_key: i16, version: i16) -> Result { .context("InitProducerId")?; } 24 => { - use kafka_protocol::messages::add_partitions_to_txn_request::*; let t = AddPartitionsToTxnTopic::default() .with_name(TopicName::from(StrBytes::from_static_str("test-topic"))) .with_partitions(vec![0i32]); @@ -530,7 +532,6 @@ fn build_payload(api_key: i16, version: i16) -> Result { .context("EndTxn")?; } 28 => { - use kafka_protocol::messages::txn_offset_commit_request::*; let p = TxnOffsetCommitRequestPartition::default() .with_partition_index(0) .with_committed_offset(42) @@ -548,7 +549,6 @@ fn build_payload(api_key: i16, version: i16) -> Result { .context("TxnOffsetCommit")?; } 32 => { - use kafka_protocol::messages::describe_configs_request::*; let r = DescribeConfigsResource::default() .with_resource_type(2) .with_resource_name(StrBytes::from_static_str("test-topic")); diff --git a/gateways/kafka/tools/kafka-tool/src/response.rs b/gateways/kafka/tools/kafka-tool/src/response.rs index 67b4e7bd98..4214a2ce23 100644 --- a/gateways/kafka/tools/kafka-tool/src/response.rs +++ b/gateways/kafka/tools/kafka-tool/src/response.rs @@ -18,9 +18,8 @@ //! Kafka response frame parsing and human-readable summaries for `send` / `verify`. use bytes::Bytes; -use iggy_gateway_kafka::protocol::header::response_header_version; use kafka_protocol::messages::{ - ApiVersionsResponse, CreateTopicsResponse, FetchResponse, ListOffsetsResponse, + ApiKey, ApiVersionsResponse, CreateTopicsResponse, FetchResponse, ListOffsetsResponse, MetadataResponse, ProduceResponse, }; use kafka_protocol::protocol::Decodable; @@ -105,9 +104,9 @@ fn is_acceptable_verify_error(api_key: i16, error_code: i16) -> bool { return true; } match api_key { - 0 => error_code == 6, // Produce stub: NOT_LEADER_OR_FOLLOWER - 3 => error_code == 3, // Metadata stub: UNKNOWN_TOPIC_OR_PARTITION - 19 => error_code == 41, // CreateTopics stub: NOT_CONTROLLER + 0..=2 => error_code == 6, // Produce/Fetch/ListOffsets stub: NOT_LEADER_OR_FOLLOWER + 3 => error_code == 3, // Metadata stub: UNKNOWN_TOPIC_OR_PARTITION + 19 => error_code == 41, // CreateTopics stub: NOT_CONTROLLER _ => false, } } @@ -133,7 +132,11 @@ pub fn analyze_response( } let correlation_id = i32::from_be_bytes(payload[0..4].try_into().expect("4 bytes")); - let resp_hdr_ver = response_header_version(api_key, api_version); + // Header version comes from kafka-protocol's own per-response `HeaderVersion` impl, not the + // gateway's table under test - otherwise a bug in the gateway's threshold would identically + // mis-summarize its own responses and this tool would never catch it. + let resp_hdr_ver = + ApiKey::try_from(api_key).map_or(0, |key| key.response_header_version(api_version)); let body_start = if resp_hdr_ver >= 1 { 5 // correlation_id + empty tagged fields (0x00) } else { From ce5296ed13c979cb95bf64b46fb07579b12c8519 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sun, 2 Aug 2026 08:49:52 -0400 Subject: [PATCH 43/57] Fixed few nits --- gateways/kafka/Cargo.toml | 1 - gateways/kafka/tools/kafka-tool/Cargo.toml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/gateways/kafka/Cargo.toml b/gateways/kafka/Cargo.toml index a06ac51d56..314de051c5 100644 --- a/gateways/kafka/Cargo.toml +++ b/gateways/kafka/Cargo.toml @@ -55,6 +55,5 @@ tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "io- [lints.clippy] enum_glob_use = "deny" -# Ported Kafka wire codec; pedantic cleanup tracked for a follow-up PR. pedantic = "warn" nursery = "warn" diff --git a/gateways/kafka/tools/kafka-tool/Cargo.toml b/gateways/kafka/tools/kafka-tool/Cargo.toml index d8815b6817..809704cb76 100644 --- a/gateways/kafka/tools/kafka-tool/Cargo.toml +++ b/gateways/kafka/tools/kafka-tool/Cargo.toml @@ -35,7 +35,7 @@ bytes = { workspace = true } clap = { workspace = true } hex = "0.4" iggy-gateway-kafka = { path = "../.." } -indexmap = "2" +indexmap = { workspace = true } kafka-protocol = "0.17" tokio = { workspace = true } tracing = { workspace = true } From 71aa1a8c969c4745cdfc9fd1166167e3463926af Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sun, 2 Aug 2026 09:13:25 -0400 Subject: [PATCH 44/57] Update MANUAL_TESTING.md This is cheap trick to retrigger CI --- gateways/kafka/docs/MANUAL_TESTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gateways/kafka/docs/MANUAL_TESTING.md b/gateways/kafka/docs/MANUAL_TESTING.md index 4607ff0001..e22b9f171a 100644 --- a/gateways/kafka/docs/MANUAL_TESTING.md +++ b/gateways/kafka/docs/MANUAL_TESTING.md @@ -21,7 +21,7 @@ See also: [SCOPE.md](SCOPE.md) (supported API keys), [TEST_SUITE.md](TEST_SUITE. ### Build and start gateway ```bash -# From iggy workspace root +# From iggy workspace root (or iggy-gateway-kafka subdir) cargo build -p iggy-gateway-kafka # Terminal 1 — start listener (default 127.0.0.1:9093) From 0a6d8cb6b9af31ea921be248c857ed93092d0de7 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Mon, 3 Aug 2026 11:26:35 -0400 Subject: [PATCH 45/57] Normalised Cargo.toml with server ng cargo.toml Updated Gateway Cargo.toml and connectors related Cargo.toml files. --- core/connectors/sinks/s3_sink/Cargo.toml | 4 ++++ core/integration/Cargo.toml | 3 ++- gateways/kafka/Cargo.toml | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/core/connectors/sinks/s3_sink/Cargo.toml b/core/connectors/sinks/s3_sink/Cargo.toml index b706474133..9f50ae816f 100644 --- a/core/connectors/sinks/s3_sink/Cargo.toml +++ b/core/connectors/sinks/s3_sink/Cargo.toml @@ -29,6 +29,10 @@ repository = "https://github.com/apache/iggy" readme = "../../README.md" publish = false +[package.metadata.cargo-machete] +# rust-s3's library name is `s3`, not `rust-s3` - machete's name matching misses `use s3::...`. +ignored = ["rust-s3"] + [lib] crate-type = ["cdylib", "lib"] diff --git a/core/integration/Cargo.toml b/core/integration/Cargo.toml index d0dc17dbab..d28da8adb9 100644 --- a/core/integration/Cargo.toml +++ b/core/integration/Cargo.toml @@ -23,7 +23,8 @@ license = "Apache-2.0" publish = false [package.metadata.cargo-machete] -ignored = ["cfg_aliases"] +# rust-s3's library name is `s3`, not `rust-s3` - machete's name matching misses `use s3::...`. +ignored = ["cfg_aliases", "rust-s3"] # Some tests are failing in CI due to lack of IPv6 interfaces # inside the docker containers. This is a temporary workaround (hopefully). diff --git a/gateways/kafka/Cargo.toml b/gateways/kafka/Cargo.toml index 314de051c5..326b252c39 100644 --- a/gateways/kafka/Cargo.toml +++ b/gateways/kafka/Cargo.toml @@ -55,5 +55,5 @@ tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "io- [lints.clippy] enum_glob_use = "deny" -pedantic = "warn" +pedantic = "deny" nursery = "warn" From c3b38565af9105cf9f83aac881d2ae60c120db0d Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 8 Aug 2026 21:28:02 +0000 Subject: [PATCH 46/57] fix(gateways): harden Kafka wire contract from review findings Close connections when a request version is above the encoder max so clients cannot parse a clamped UNSUPPORTED_VERSION body. Fully decode ApiVersions v3+ and Metadata bodies (reject empty/malformed Metadata), split nullable vs required compact arrays, honor KIP-464 assignments on CreateTopics v2/v3, cancel idle connections on shutdown, and cap kafka-tool response frame allocation. Co-authored-by: ryerraguntla --- gateways/kafka/docs/SCOPE.md | 2 +- gateways/kafka/src/error.rs | 8 + gateways/kafka/src/protocol/api.rs | 375 ++++++++++++------ gateways/kafka/src/protocol/codec.rs | 32 +- gateways/kafka/src/protocol/requests.rs | 14 +- gateways/kafka/src/protocol/responses.rs | 16 +- gateways/kafka/src/server.rs | 173 +++++--- gateways/kafka/tests/api_handler_tests.rs | 160 +++++--- gateways/kafka/tests/common/tcp.rs | 5 +- gateways/kafka/tests/common/wire.rs | 72 +++- gateways/kafka/tests/decode_safety_tests.rs | 13 +- .../kafka/tests/listener_robustness_tests.rs | 6 +- .../kafka/tests/response_negative_tests.rs | 30 +- gateways/kafka/tests/server_e2e_tests.rs | 21 +- .../kafka/tests/version_firewall_tests.rs | 142 ++++--- gateways/kafka/tools/kafka-tool/src/main.rs | 39 +- 16 files changed, 760 insertions(+), 348 deletions(-) diff --git a/gateways/kafka/docs/SCOPE.md b/gateways/kafka/docs/SCOPE.md index 4e7417073b..b741c64a2c 100644 --- a/gateways/kafka/docs/SCOPE.md +++ b/gateways/kafka/docs/SCOPE.md @@ -24,7 +24,7 @@ Source of truth for supported ranges: `SUPPORTED_RANGES` in [`src/protocol/api.r ### Governance model -Expand `SUPPORTED_RANGES` only after a key/version pair is manually tested. ApiVersions advertises exactly what the firewall allows; out-of-range requests receive `UNSUPPORTED_VERSION` (35) without dropping the connection. +Expand `SUPPORTED_RANGES` only after a key/version pair is manually tested. ApiVersions advertises exactly what the firewall allows. Versions below an API's min (but still encodable) receive `UNSUPPORTED_VERSION` (35) in a version-correct body; versions above the encoder max close the connection (a clamped body would be unparsable at the client's version). ApiVersions is the KIP-511 exception: out-of-range still answers with a v0 error body. --- diff --git a/gateways/kafka/src/error.rs b/gateways/kafka/src/error.rs index 16b9de936c..fa6e2df43d 100644 --- a/gateways/kafka/src/error.rs +++ b/gateways/kafka/src/error.rs @@ -44,6 +44,14 @@ pub enum KafkaProtocolError { StringTooLong { length: usize }, #[error("null topic name in request")] NullTopicName, + /// Compact-array prefix `0` is Kafka's null encoding; invalid for non-nullable arrays. + #[error("null compact array where a non-null array is required")] + NullCompactArray, + /// Compact-string prefix `0` is null; invalid where a non-null string is required. + #[error("null compact string where a non-null string is required")] + NullCompactString, + #[error("unexpected bytes remaining in request body")] + UnexpectedTrailingBytes, #[error("io error: {0}")] Io(#[from] std::io::Error), } diff --git a/gateways/kafka/src/protocol/api.rs b/gateways/kafka/src/protocol/api.rs index 9c6d86e23e..d874b1b20c 100644 --- a/gateways/kafka/src/protocol/api.rs +++ b/gateways/kafka/src/protocol/api.rs @@ -196,10 +196,9 @@ fn handle_produce_request(api_version: i16, body: Bytes) -> HandleOutcome { ProduceDecodeResult::Ok(req) if req.acks == 0 => HandleOutcome::NoResponse, ProduceDecodeResult::Ok(req) => { if !is_supported_version(API_KEY_PRODUCE, api_version) { - return HandleOutcome::Respond(encode_produce_error_response( - api_version, - ERROR_UNSUPPORTED_VERSION, - )); + return unsupported_version_response(API_KEY_PRODUCE, api_version, |v| { + encode_produce_error_response(v, ERROR_UNSUPPORTED_VERSION) + }); } HandleOutcome::Respond(encode_produce_response(api_version, &req)) } @@ -215,12 +214,16 @@ fn handle_produce_request(api_version: i16, body: Bytes) -> HandleOutcome { } ProduceDecodeResult::Err { error, .. } => { tracing::warn!("Failed to decode Produce request: {:?}", error); - let code = if is_supported_version(API_KEY_PRODUCE, api_version) { - ERROR_INVALID_REQUEST + if is_supported_version(API_KEY_PRODUCE, api_version) { + HandleOutcome::Respond(encode_produce_error_response( + api_version, + ERROR_INVALID_REQUEST, + )) } else { - ERROR_UNSUPPORTED_VERSION - }; - HandleOutcome::Respond(encode_produce_error_response(api_version, code)) + unsupported_version_response(API_KEY_PRODUCE, api_version, |v| { + encode_produce_error_response(v, ERROR_UNSUPPORTED_VERSION) + }) + } } } } @@ -232,98 +235,140 @@ fn handle_other_request( broker: &BrokerAdvertise, ) -> HandleOutcome { match api_key { - API_KEY_API_VERSIONS => { - if is_supported_version(api_key, api_version) { - HandleOutcome::Respond(encode_api_versions_response(api_version, ERROR_NONE)) - } else { - // KIP-511: reply with v0 when the requested version is not understood. - HandleOutcome::Respond(encode_api_versions_response(0, ERROR_UNSUPPORTED_VERSION)) - } - } - API_KEY_METADATA => { - if is_supported_version(api_key, api_version) { - HandleOutcome::Respond(encode_metadata_response( - api_version, - api_version, - body, - broker, - ERROR_NONE, - )) - } else { - // Clamping the response to MAX_SUPPORTED_METADATA_VERSION leaves a body the - // client parses at its own (unsupported) version, so UNSUPPORTED_VERSION never - // survives. Clients that skip ApiVersions get a naked close instead. - tracing::warn!( - api_version, - max_supported = MAX_SUPPORTED_METADATA_VERSION, - "Metadata version unsupported; closing connection" - ); - HandleOutcome::Close - } - } - API_KEY_FETCH => { - if is_supported_version(api_key, api_version) { - match decode_fetch_request(api_version, body) { - Ok(req) => HandleOutcome::Respond(encode_fetch_response(api_version, &req)), - Err(e) => { - tracing::warn!("Failed to decode Fetch request: {:?}", e); - HandleOutcome::Respond(encode_fetch_error_response( - api_version, - ERROR_INVALID_REQUEST, - )) - } - } - } else { - HandleOutcome::Respond(encode_fetch_error_response( + API_KEY_API_VERSIONS => handle_api_versions(api_version, &body), + API_KEY_METADATA => handle_metadata(api_version, body, broker), + API_KEY_FETCH => handle_versioned_request( + API_KEY_FETCH, + api_version, + body, + decode_fetch_request, + encode_fetch_response, + encode_fetch_error_response, + "Fetch", + ), + API_KEY_LIST_OFFSETS => handle_versioned_request( + API_KEY_LIST_OFFSETS, + api_version, + body, + decode_list_offsets_request, + encode_list_offsets_response, + encode_list_offsets_error_response, + "ListOffsets", + ), + API_KEY_CREATE_TOPICS => handle_versioned_request( + API_KEY_CREATE_TOPICS, + api_version, + body, + decode_create_topics_request, + encode_create_topics_response, + encode_create_topics_error_response, + "CreateTopics", + ), + _ => HandleOutcome::RespondAndClose(encode_error_only_response(ERROR_UNSUPPORTED_VERSION)), + } +} + +fn handle_api_versions(api_version: i16, body: &Bytes) -> HandleOutcome { + if is_supported_version(API_KEY_API_VERSIONS, api_version) { + match decode_api_versions_request(api_version, body) { + Ok(()) => HandleOutcome::Respond(encode_api_versions_response(api_version, ERROR_NONE)), + Err(e) => { + tracing::warn!("Failed to decode ApiVersions request: {:?}", e); + HandleOutcome::Respond(encode_api_versions_response( api_version, - ERROR_UNSUPPORTED_VERSION, + ERROR_INVALID_REQUEST, )) } } - API_KEY_LIST_OFFSETS => { - if is_supported_version(api_key, api_version) { - match decode_list_offsets_request(api_version, body) { - Ok(req) => { - HandleOutcome::Respond(encode_list_offsets_response(api_version, &req)) - } - Err(e) => { - tracing::warn!("Failed to decode ListOffsets request: {:?}", e); - HandleOutcome::Respond(encode_list_offsets_error_response( - api_version, - ERROR_INVALID_REQUEST, - )) - } - } - } else { - HandleOutcome::Respond(encode_list_offsets_error_response( - api_version, - ERROR_UNSUPPORTED_VERSION, - )) - } + } else { + // KIP-511: reply with v0 when the requested version is not understood. + HandleOutcome::Respond(encode_api_versions_response(0, ERROR_UNSUPPORTED_VERSION)) + } +} + +fn handle_metadata(api_version: i16, body: Bytes, broker: &BrokerAdvertise) -> HandleOutcome { + if !is_supported_version(API_KEY_METADATA, api_version) { + // Clamping the response to MAX_SUPPORTED_METADATA_VERSION leaves a body the + // client parses at its own (unsupported) version, so UNSUPPORTED_VERSION never + // survives. Clients that skip ApiVersions get a naked close instead. + tracing::warn!( + api_version, + max_supported = MAX_SUPPORTED_METADATA_VERSION, + "Metadata version unsupported; closing connection" + ); + return HandleOutcome::Close; + } + match decode_metadata_request(api_version, body) { + Ok(topics) => HandleOutcome::Respond(encode_metadata_response( + api_version, + &topics, + broker, + ERROR_NONE, + )), + Err(e) => { + // Metadata has no top-level error field; a malformed body cannot carry + // INVALID_REQUEST in a version-correct way for every client. Close. + tracing::warn!( + ?e, + api_version, + "Failed to decode Metadata request; closing connection" + ); + HandleOutcome::Close } - API_KEY_CREATE_TOPICS => { - if is_supported_version(api_key, api_version) { - match decode_create_topics_request(api_version, body) { - Ok(req) => { - HandleOutcome::Respond(encode_create_topics_response(api_version, &req)) - } - Err(e) => { - tracing::warn!("Failed to decode CreateTopics request: {:?}", e); - HandleOutcome::Respond(encode_create_topics_error_response( - api_version, - ERROR_INVALID_REQUEST, - )) - } - } - } else { - HandleOutcome::Respond(encode_create_topics_error_response( - api_version, - ERROR_UNSUPPORTED_VERSION, - )) + } +} + +fn handle_versioned_request( + api_key: i16, + api_version: i16, + body: Bytes, + decode: impl FnOnce(i16, Bytes) -> Result, + encode_ok: impl FnOnce(i16, &T) -> Bytes, + encode_err: impl Fn(i16, i16) -> Bytes, + api_name: &str, +) -> HandleOutcome { + if is_supported_version(api_key, api_version) { + match decode(api_version, body) { + Ok(req) => HandleOutcome::Respond(encode_ok(api_version, &req)), + Err(e) => { + tracing::warn!("Failed to decode {api_name} request: {:?}", e); + HandleOutcome::Respond(encode_err(api_version, ERROR_INVALID_REQUEST)) } } - _ => HandleOutcome::RespondAndClose(encode_error_only_response(ERROR_UNSUPPORTED_VERSION)), + } else { + unsupported_version_response(api_key, api_version, |version| { + encode_err(version, ERROR_UNSUPPORTED_VERSION) + }) + } +} + +/// Unsupported-version policy for APIs whose encoders only implement up to +/// [`ApiVersionRange::max_version`]. +/// +/// - `api_version > max`: Close. Encoding at the client's raw version would omit later-version +/// fields (`CreateTopics` v7 `TopicId`, Produce v13 UUID topic, …) and the client could not +/// parse the intended `UNSUPPORTED_VERSION` body. +/// - `api_version < min` but still within encoder capability: Respond with an error shaped for +/// that version (e.g. `ListOffsets` v0 `old_style_offsets`, Produce v0–2). +fn unsupported_version_response( + api_key: i16, + api_version: i16, + encode: impl FnOnce(i16) -> Bytes, +) -> HandleOutcome { + let max_version = SUPPORTED_RANGES + .iter() + .find(|r| r.api_key == api_key) + .map_or(0, |r| r.max_version); + if api_version > max_version { + tracing::warn!( + api_key, + api_version, + max_version, + "request version above encoder max; closing connection" + ); + return HandleOutcome::Close; } + HandleOutcome::Respond(encode(api_version)) } #[must_use] @@ -384,29 +429,18 @@ fn encode_api_versions_response(api_version: i16, error_code: i16) -> Bytes { fn encode_metadata_response( response_version: i16, - decode_version: i16, - body: Bytes, + topics: &[String], broker: &BrokerAdvertise, - top_level_error_code: i16, + topic_error_override: i16, ) -> Bytes { let flexible = response_version >= 9; - // Empty body = all-topics request; 0 topics is correct for this stub. - // Non-empty body that fails to decode = malformed request; return 0 topics. - // Kafka Metadata response has no top-level error code field: errors are per-topic only. - // 0 topics is spec-correct and unambiguous for a decode failure. - let (topics, effective_error) = if body.is_empty() { - (Vec::new(), top_level_error_code) - } else { - decode_metadata_request_topics(body, decode_version) - .map_or((Vec::new(), ERROR_INVALID_REQUEST), |names| { - (names, top_level_error_code) - }) - }; let topics_count = topics.len(); - let topic_error = if effective_error == ERROR_NONE { + // Stub has no topic catalog: echo requested names with UNKNOWN_TOPIC_OR_PARTITION, + // or a forced override (unused today; kept for symmetry with other encoders). + let topic_error = if topic_error_override == ERROR_NONE { ERROR_UNKNOWN_TOPIC_OR_PARTITION } else { - effective_error + topic_error_override }; let mut e = Encoder::with_capacity(256); @@ -427,7 +461,7 @@ fn encode_metadata_response( e.write_i32(1); // controller_id (v1+) e.write_varint((topics_count + 1) as u64); - for name in &topics { + for name in topics { e.write_i16(topic_error); e.write_compact_nullable_string(Some(name)); e.write_bool(false); // is_internal (v1+) @@ -458,7 +492,7 @@ fn encode_metadata_response( } e.write_i32(i32::try_from(topics_count).expect("topic count bounded")); - for name in &topics { + for name in topics { e.write_i16(topic_error); e.write_nullable_string_unchecked(Some(name)); if response_version >= 1 { @@ -484,17 +518,46 @@ pub fn encode_error_only_response(error_code: i16) -> Bytes { e.freeze() } -/// Decodes the requested topic names from a Metadata request body so the -/// response can echo them back; clients match metadata by name, not position. +/// Decodes an `ApiVersions` request body. /// -/// A null topics array - `-1` legacy count, or `varint=0` compact count - means "all topics" -/// per the Kafka spec, not a malformed request; both decode to an empty list here since the -/// stub doesn't implement real topic listing yet. -pub(crate) fn decode_metadata_request_topics(body: Bytes, api_version: i16) -> Result> { +/// v0–v2 have an empty body. v3+ requires `ClientSoftwareName`, `ClientSoftwareVersion`, +/// and tagged fields (KIP-511 flexible encoding). +fn decode_api_versions_request(api_version: i16, body: &Bytes) -> Result<()> { + if api_version < 3 { + if !body.is_empty() { + return Err(KafkaProtocolError::UnexpectedTrailingBytes); + } + return Ok(()); + } + let mut d = Decoder::new(body.clone()); + let _client_software_name = d.read_compact_string()?; + let _client_software_version = d.read_compact_string()?; + d.read_tagged_fields()?; + if d.remaining() != 0 { + return Err(KafkaProtocolError::UnexpectedTrailingBytes); + } + Ok(()) +} + +/// Decodes a Metadata request body so the response can echo topic names. +/// +/// Every Metadata version requires at least the topics array count on the wire — an empty +/// body is malformed, not "all topics". A null topics array (`-1` legacy / `varint=0` +/// compact) means "all topics" and decodes to an empty list for this stub. Remaining +/// version-gated fields (`AllowAutoTopicCreation`, authorized-ops flags, tagged fields) +/// are consumed so truncated flexible bodies are rejected. +pub(crate) fn decode_metadata_request(api_version: i16, body: Bytes) -> Result> { + if body.is_empty() { + return Err(KafkaProtocolError::BufferUnderflow { + needed: 1, + remaining: 0, + }); + } let mut d = Decoder::new(body); let flexible = api_version >= 9; let topics_count = if flexible { - d.read_compact_array_count()? + // Metadata topics is a nullable compact array ("all topics" when null). + d.read_compact_array_count_nullable()? } else { d.read_i32_array_count_nullable()? }; @@ -518,9 +581,34 @@ pub(crate) fn decode_metadata_request_topics(body: Bytes, api_version: i16) -> R } } + // allow_auto_topic_creation (v4+) + if api_version >= 4 { + let _allow_auto_topic_creation = d.read_bool()?; + } + // include_cluster_authorized_operations (v8–v10; removed in v11) + if (8..=10).contains(&api_version) { + let _include_cluster_authorized_operations = d.read_bool()?; + } + // include_topic_authorized_operations (v8+) + if api_version >= 8 { + let _include_topic_authorized_operations = d.read_bool()?; + } + if flexible { + d.read_tagged_fields()?; + } + if d.remaining() != 0 { + return Err(KafkaProtocolError::UnexpectedTrailingBytes); + } + Ok(topics) } +/// Compatibility alias used by existing unit tests. +#[cfg(test)] +pub(crate) fn decode_metadata_request_topics(body: Bytes, api_version: i16) -> Result> { + decode_metadata_request(api_version, body) +} + #[cfg(test)] mod tests { use super::*; @@ -564,13 +652,48 @@ mod tests { #[test] fn decode_metadata_request_topics_flexible_invalid_utf8_fails() { - let body = Bytes::from_static(&[ - 0x02, // one topic - 0x02, // string len = 1 - 0xff, // invalid utf-8 - 0x00, // tagged fields - ]); - let err = decode_metadata_request_topics(body, 9).unwrap_err(); + let mut enc = Encoder::with_capacity(16); + enc.write_varint(2); // one topic + enc.write_varint(2); // string len = 1 + enc.write_u8(0xff); // invalid utf-8 + enc.write_empty_tagged_fields(); // per-topic tagged + enc.write_bool(true); // allow_auto_topic_creation + enc.write_bool(false); // include_cluster_authorized_operations + enc.write_bool(false); // include_topic_authorized_operations + enc.write_empty_tagged_fields(); // top-level tagged + let err = decode_metadata_request_topics(enc.freeze(), 9).unwrap_err(); assert!(matches!(err, KafkaProtocolError::InvalidUtf8)); } + + #[test] + fn decode_metadata_request_empty_body_is_malformed() { + let err = decode_metadata_request(0, Bytes::new()).unwrap_err(); + assert!(matches!(err, KafkaProtocolError::BufferUnderflow { .. })); + } + + #[test] + fn decode_metadata_request_flexible_truncated_after_topics_fails() { + // topics = null (all topics) but missing allow_auto / auth flags / tagged fields. + let body = Bytes::from_static(&[0x00]); + let err = decode_metadata_request(9, body).unwrap_err(); + assert!(matches!(err, KafkaProtocolError::BufferUnderflow { .. })); + } + + #[test] + fn decode_api_versions_v3_requires_software_fields() { + let err = decode_api_versions_request(3, &Bytes::new()).unwrap_err(); + assert!(matches!( + err, + KafkaProtocolError::BufferUnderflow { .. } | KafkaProtocolError::NullCompactString + )); + } + + #[test] + fn decode_api_versions_v3_accepts_valid_body() { + let mut enc = Encoder::with_capacity(32); + enc.write_compact_nullable_string(Some("iggy-test")); + enc.write_compact_nullable_string(Some("0.1.0")); + enc.write_empty_tagged_fields(); + decode_api_versions_request(3, &enc.freeze()).unwrap(); + } } diff --git a/gateways/kafka/src/protocol/codec.rs b/gateways/kafka/src/protocol/codec.rs index 7146f957d4..13116e4afd 100644 --- a/gateways/kafka/src/protocol/codec.rs +++ b/gateways/kafka/src/protocol/codec.rs @@ -127,8 +127,8 @@ impl Decoder { /// Legacy nullable array length: signed i32 count, where -1 is the spec-defined null /// (`all items`) sentinel - used by Metadata's `topics` field to mean "all topics" - treated - /// as empty (0 elements) rather than an error, mirroring how `read_compact_array_count` - /// treats a null compact array. + /// as empty (0 elements) rather than an error, mirroring + /// [`Self::read_compact_array_count_nullable`]. pub fn read_i32_array_count_nullable(&mut self) -> Result { let n = self.read_i32()?; if n == -1 { @@ -149,14 +149,30 @@ impl Decoder { Ok(count) } - /// Compact array length: unsigned varint holding `element_count + 1`. - /// Per the Kafka spec, varint=0 encodes a null (absent) array; treat as empty (0 elements) - /// so optional fields like `forgotten_topics` are skipped rather than rejected. + /// Compact array length for a **non-nullable** field: unsigned varint holding + /// `element_count + 1`. Varint `0` is Kafka's null encoding and is rejected here — + /// required arrays (Produce/Fetch/ListOffsets/CreateTopics topic data, nested partitions, + /// etc.) must use `1` for empty. pub fn read_compact_array_count(&mut self) -> Result { + let n = self.read_varint()?; + if n == 0 { + return Err(KafkaProtocolError::NullCompactArray); + } + Self::compact_array_count_from_len_plus_one(n) + } + + /// Compact array length for a **nullable** field: varint `0` means null/absent and is + /// returned as 0 elements (e.g. Metadata topics = "all topics", Fetch forgotten topics + /// when a client encodes null). + pub fn read_compact_array_count_nullable(&mut self) -> Result { let n = self.read_varint()?; if n == 0 { return Ok(0); } + Self::compact_array_count_from_len_plus_one(n) + } + + fn compact_array_count_from_len_plus_one(n: u64) -> Result { let count = usize::try_from(n - 1).map_err(|_| KafkaProtocolError::CollectionTooLarge { count: MAX_COLLECTION_LEN + 1, max: MAX_COLLECTION_LEN, @@ -205,6 +221,12 @@ impl Decoder { Ok(Some(s)) } + /// Non-nullable compact string (flexible versions): varint(len+1), never null. + pub fn read_compact_string(&mut self) -> Result { + self.read_compact_nullable_string()? + .ok_or(KafkaProtocolError::NullCompactString) + } + /// Legacy nullable bytes: i32 length prefix (-1 = null). pub fn read_nullable_bytes(&mut self) -> Result> { let len = self.read_i32()?; diff --git a/gateways/kafka/src/protocol/requests.rs b/gateways/kafka/src/protocol/requests.rs index 833b2ef3aa..6792840487 100644 --- a/gateways/kafka/src/protocol/requests.rs +++ b/gateways/kafka/src/protocol/requests.rs @@ -281,10 +281,11 @@ pub fn decode_fetch_request(version: i16, body: Bytes) -> Result { // forgotten_topics_data (v7+) - skip if version >= 7 { + // forgottenTopicsData is nullable on the wire in practice (null = none forgotten). let forgotten_count = if flexible { - d.read_compact_array_count()? + d.read_compact_array_count_nullable()? } else { - d.read_i32_array_count()? + d.read_i32_array_count_nullable()? }; for _ in 0..forgotten_count { if flexible { @@ -432,6 +433,11 @@ pub struct CreatableTopic { pub name: String, pub num_partitions: i32, pub replication_factor: i16, + /// True when the request included a non-empty manual partition assignment. + /// KIP-464: on v2/v3, `num_partitions = -1` / `replication_factor = -1` are valid + /// only when assignments are present; v4+ allows them as broker-default sentinels + /// even without assignments. + pub has_assignments: bool, } /// Decodes a raw byte stream into a `CreateTopicsRequest`. @@ -462,12 +468,13 @@ pub fn decode_create_topics_request(version: i16, body: Bytes) -> Result 0; for _ in 0..assignments_count { d.read_i32()?; // partition_index let replicas_count = if flexible { @@ -504,6 +511,7 @@ pub fn decode_create_topics_request(version: i16, body: Bytes) -> Result Byt name: String::new(), num_partitions: 1, replication_factor: 1, + has_assignments: false, }]; encode_create_topics_response_inner(version, &topics, error_code) } @@ -324,16 +325,19 @@ pub fn encode_create_topics_response(version: i16, req: &CreateTopicsRequest) -> /// Resolve per-topic CreateTopics error. /// -/// KIP-464: on v4+, `num_partitions = -1` / `replication_factor = -1` mean broker default. -/// Error 37 / 38 only for `0` and values `< -1` (and any non-positive value on v2–v3). -/// When validation passes, the stub returns [`ERROR_NOT_CONTROLLER`] so clients do not -/// believe the topic was created before the Iggy bridge exists. +/// KIP-464: `num_partitions = -1` / `replication_factor = -1` mean broker default when either +/// (a) the version is v4+, or (b) the topic carries a manual partition assignment (valid on +/// v2/v3 as well). Otherwise non-positive values are [`ERROR_INVALID_PARTITIONS`] / +/// [`ERROR_INVALID_REPLICATION_FACTOR`]. When validation passes, the stub returns +/// [`ERROR_NOT_CONTROLLER`] so clients do not believe the topic was created. const fn create_topics_topic_error(version: i16, topic: &CreatableTopic, forced_error: i16) -> i16 { if forced_error != ERROR_NONE { return forced_error; } - let partitions_ok = if version >= 4 { + let broker_default_ok = version >= 4 || topic.has_assignments; + + let partitions_ok = if broker_default_ok { topic.num_partitions == -1 || topic.num_partitions > 0 } else { topic.num_partitions > 0 @@ -342,7 +346,7 @@ const fn create_topics_topic_error(version: i16, topic: &CreatableTopic, forced_ return ERROR_INVALID_PARTITIONS; } - let replication_ok = if version >= 4 { + let replication_ok = if broker_default_ok { topic.replication_factor == -1 || topic.replication_factor > 0 } else { topic.replication_factor > 0 diff --git a/gateways/kafka/src/server.rs b/gateways/kafka/src/server.rs index 2e271824f1..89f33df69b 100644 --- a/gateways/kafka/src/server.rs +++ b/gateways/kafka/src/server.rs @@ -24,6 +24,7 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; use tokio::sync::{Semaphore, broadcast}; use tokio::time::{timeout, timeout_at}; +use tokio_util::sync::CancellationToken; use tokio_util::task::TaskTracker; use tracing::{debug, error, info, warn}; @@ -160,6 +161,9 @@ impl KafkaServer { let tracker = TaskTracker::new(); let conn_limiter = Arc::new(Semaphore::new(self.config.max_connections)); + // Cancelled on shutdown so connection tasks exit instead of sitting in idle waits + // until `idle_timeout` (or forever if that is raised). + let cancel = CancellationToken::new(); let drain_timeout = self.config.shutdown_drain_timeout; @@ -169,17 +173,17 @@ impl KafkaServer { match result { Ok(()) => { info!("kafka listener shutdown requested"); - drain(&tracker, drain_timeout).await; + drain(&tracker, &cancel, drain_timeout).await; break; } // Capacity-1 channel: lagged means a signal was sent before we polled - treat as shutdown. Err(broadcast::error::RecvError::Lagged(_)) => { info!("kafka listener shutdown requested (lagged)"); - drain(&tracker, drain_timeout).await; + drain(&tracker, &cancel, drain_timeout).await; break; } Err(broadcast::error::RecvError::Closed) => { - drain(&tracker, drain_timeout).await; + drain(&tracker, &cancel, drain_timeout).await; break; } } @@ -199,9 +203,12 @@ impl KafkaServer { } let cfg = Arc::clone(&self.config); let broker = Arc::clone(&broker); + let conn_cancel = cancel.child_token(); tracker.spawn(async move { let _permit = permit; - if let Err(err) = handle_connection(stream, cfg, peer, broker).await { + if let Err(err) = + handle_connection(stream, cfg, peer, broker, conn_cancel).await + { warn!(%peer, "connection closed with error: {err}"); } }); @@ -214,7 +221,7 @@ impl KafkaServer { warn!(%e, "transient accept error, continuing"); } Err(e) => { - drain(&tracker, drain_timeout).await; + drain(&tracker, &cancel, drain_timeout).await; return Err(e.into()); } } @@ -226,15 +233,16 @@ impl KafkaServer { } } -/// Close the tracker to new spawns and wait for in-flight connections to finish, but not past -/// `deadline` - an idle connection can otherwise hold shutdown open for up to `idle_timeout` -/// (10 minutes by default), past typical orchestrator grace periods. -async fn drain(tracker: &TaskTracker, deadline: Duration) { +/// Cancel in-flight connections, close the tracker to new spawns, and wait for tasks to finish +/// (bounded by `deadline`). Cancellation is what actually drops idle sockets; `tracker.wait` +/// alone would leave tasks parked in `read_frame` until `idle_timeout`. +async fn drain(tracker: &TaskTracker, cancel: &CancellationToken, deadline: Duration) { + cancel.cancel(); tracker.close(); if timeout(deadline, tracker.wait()).await.is_err() { warn!( ?deadline, - "shutdown drain deadline exceeded; abandoning in-flight connections" + "shutdown drain deadline exceeded; remaining connection tasks will be dropped with the runtime" ); } } @@ -261,27 +269,13 @@ async fn handle_connection( config: Arc, peer: SocketAddr, broker: Arc, + cancel: CancellationToken, ) -> Result<()> { debug!(%peer, "connection accepted"); loop { - let frame = match read_frame( - &mut stream, - config.max_frame_size, - config.idle_timeout, - config.read_timeout, - ) - .await - { - Ok(f) => f, - Err(KafkaProtocolError::Io(ref e)) - if e.kind() == std::io::ErrorKind::UnexpectedEof - || e.kind() == std::io::ErrorKind::ConnectionReset => - { - info!(%peer, "connection closed by client"); - return Ok(()); - } - Err(e) => return Err(e), + let Some(frame) = read_next_frame(&mut stream, &config, &peer, &cancel).await? else { + return Ok(()); }; if frame.len() < 8 { @@ -327,43 +321,88 @@ async fn handle_connection( let body = decoder.read_bytes(decoder.remaining())?; let outcome = handle_request(req.api_key, req.api_version, body, &broker); - let close_after_response = matches!(outcome, HandleOutcome::RespondAndClose(_)); - match outcome { - HandleOutcome::NoResponse => { - // Produce with acks=0: the wire protocol forbids a response. + if dispatch_outcome(&mut stream, &peer, &config, &req, resp_hdr_ver, outcome).await? { + return Ok(()); + } + } +} + +/// Returns `Ok(None)` when shutdown cancellation wins; `Ok(Some(frame))` on a full frame. +async fn read_next_frame( + stream: &mut TcpStream, + config: &ServerConfig, + peer: &SocketAddr, + cancel: &CancellationToken, +) -> Result> { + tokio::select! { + () = cancel.cancelled() => { + debug!(%peer, "connection cancelled by shutdown"); + Ok(None) + } + result = read_frame( + stream, + config.max_frame_size, + config.idle_timeout, + config.read_timeout, + ) => match result { + Ok(frame) => Ok(Some(frame)), + Err(KafkaProtocolError::Io(ref e)) + if e.kind() == std::io::ErrorKind::UnexpectedEof + || e.kind() == std::io::ErrorKind::ConnectionReset => + { + info!(%peer, "connection closed by client"); + Ok(None) } - HandleOutcome::Close => { + Err(e) => Err(e), + }, + } +} + +/// Applies a [`HandleOutcome`]. Returns `true` when the connection should close. +async fn dispatch_outcome( + stream: &mut TcpStream, + peer: &SocketAddr, + config: &ServerConfig, + req: &RequestHeader, + resp_hdr_ver: i16, + outcome: HandleOutcome, +) -> Result { + let close_after_response = matches!(outcome, HandleOutcome::RespondAndClose(_)); + match outcome { + HandleOutcome::NoResponse => { + // Produce with acks=0: the wire protocol forbids a response. + Ok(false) + } + HandleOutcome::Close => { + warn!( + %peer, + api_key = req.api_key, + api_version = req.api_version, + "closing connection: no parseable error response for this request version" + ); + Ok(true) + } + HandleOutcome::Respond(body_response) | HandleOutcome::RespondAndClose(body_response) => { + let resp_header = ResponseHeader { + correlation_id: req.correlation_id, + }; + send_response( + stream, + &resp_header, + resp_hdr_ver, + &body_response, + config.write_timeout, + ) + .await?; + if close_after_response { warn!( %peer, api_key = req.api_key, api_version = req.api_version, - "closing connection: no parseable error response for this request version" + "closing connection after unsupported-version error response" ); - return Ok(()); - } - HandleOutcome::Respond(body_response) - | HandleOutcome::RespondAndClose(body_response) => { - let resp_header = ResponseHeader { - correlation_id: req.correlation_id, - }; - send_response( - &mut stream, - &resp_header, - resp_hdr_ver, - &body_response, - config.write_timeout, - ) - .await?; - if close_after_response { - warn!( - %peer, - api_key = req.api_key, - api_version = req.api_version, - "closing connection after unsupported-version error response" - ); - return Ok(()); - } } + Ok(close_after_response) } } } @@ -603,17 +642,17 @@ mod tests { let addr = listener.local_addr().unwrap(); let (tx, rx) = broadcast::channel(1); let server = KafkaServer::new(ServerConfig { - // Idle timeout is intentionally long - the drain deadline, not the idle timeout, - // must be what bounds shutdown here. + // Idle timeout is intentionally long - cancellation + drain deadline, not the + // idle timeout, must bound shutdown here. idle_timeout: Duration::from_mins(10), - shutdown_drain_timeout: Duration::from_millis(100), + shutdown_drain_timeout: Duration::from_millis(200), ..ServerConfig::default() }); let handle = tokio::spawn(async move { server.run(listener, rx).await }); - // Held open, never sends a frame: the in-flight connection task is parked in - // read_frame's idle wait for the full 600s idle_timeout unless drain cuts it short. - let _held = TcpStream::connect(addr).await.unwrap(); + // Held open, never sends a frame: without cancellation the task would park in + // read_frame's idle wait for the full 600s idle_timeout. + let mut held = TcpStream::connect(addr).await.unwrap(); tokio::time::sleep(Duration::from_millis(50)).await; tx.send(()).unwrap(); @@ -622,6 +661,14 @@ mod tests { .expect("shutdown must return well within the 600s idle_timeout") .unwrap(); assert!(result.is_ok()); + + // Cancellation must close the held socket so the client sees EOF, not a silent stall. + let mut buf = [0u8; 1]; + let n = tokio::time::timeout(Duration::from_secs(1), held.read(&mut buf)) + .await + .expect("held connection must be closed on shutdown") + .unwrap(); + assert_eq!(n, 0, "shutdown must deliver EOF to idle clients"); } #[tokio::test] diff --git a/gateways/kafka/tests/api_handler_tests.rs b/gateways/kafka/tests/api_handler_tests.rs index 23ed04df02..564332fc35 100644 --- a/gateways/kafka/tests/api_handler_tests.rs +++ b/gateways/kafka/tests/api_handler_tests.rs @@ -32,15 +32,16 @@ use iggy_gateway_kafka::protocol::api::{ ERROR_NOT_LEADER_OR_FOLLOWER, ERROR_UNKNOWN_TOPIC_OR_PARTITION, ERROR_UNSUPPORTED_VERSION, handle_request, is_supported_version, supported_api_ranges, }; -use iggy_gateway_kafka::protocol::codec::{Decoder, Encoder}; +use iggy_gateway_kafka::protocol::codec::Decoder; use iggy_gateway_kafka::protocol::requests::{ProduceDecodeResult, decode_produce_request}; use fixtures::load_fixture_body_or_skip; use scope::default_broker; use tcp::{build_metadata_legacy_request, build_produce_v3_body}; use wire::{ + build_api_versions_flexible_request, build_metadata_all_topics_legacy, build_metadata_flexible_request, build_metadata_flexible_request_v10, - build_produce_legacy_request, + build_metadata_legacy_request_for_version, build_produce_legacy_request, }; // ── ApiVersions ───────────────────────────────────────────────────────────── @@ -72,8 +73,13 @@ fn api_versions_v1_response_non_flexible_format() { #[test] fn api_versions_v3_response_flexible_format() { - let body = handle_request(API_KEY_API_VERSIONS, 3, Bytes::new(), &default_broker()) - .expect_response("test request has acks != 0 and expects a response"); + let body = handle_request( + API_KEY_API_VERSIONS, + 3, + build_api_versions_flexible_request("apache-iggy", "0.1.0"), + &default_broker(), + ) + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i16().unwrap(), 0); // error_code @@ -103,8 +109,13 @@ fn api_versions_v3_response_flexible_format() { #[test] fn metadata_response_has_broker_array_and_topic_array() { - let body = handle_request(API_KEY_METADATA, 0, Bytes::new(), &default_broker()) - .expect_response("test request has acks != 0 and expects a response"); + let body = handle_request( + API_KEY_METADATA, + 0, + build_metadata_all_topics_legacy(0), + &default_broker(), + ) + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let broker_count = d.read_i32().unwrap(); @@ -120,6 +131,22 @@ fn metadata_response_has_broker_array_and_topic_array() { assert_eq!(topic_count, 0); } +#[test] +fn metadata_empty_body_closes_connection() { + assert!( + handle_request(API_KEY_METADATA, 0, Bytes::new(), &default_broker()).is_close(), + "empty Metadata body is malformed, not all-topics" + ); +} + +#[test] +fn api_versions_v3_empty_body_returns_invalid_request() { + let body = handle_request(API_KEY_API_VERSIONS, 3, Bytes::new(), &default_broker()) + .expect_response("ApiVersions has a top-level error_code"); + let mut d = Decoder::new(body); + assert_eq!(d.read_i16().unwrap(), ERROR_INVALID_REQUEST); +} + #[test] fn unsupported_metadata_version_closes_connection() { // Above max: a clamped v9 body is unparsable to a v99 client, so Close is the honest contract. @@ -250,24 +277,20 @@ fn create_topics_malformed_body_returns_invalid_request() { } #[test] -fn metadata_null_topic_name_yields_zero_topics() { - let body = handle_request( - API_KEY_METADATA, - 0, - Bytes::from_static(&[ - 0x00, 0x00, 0x00, 0x01, // one topic - 0xff, 0xff, // null topic name - ]), - &default_broker(), - ) - .expect_response("metadata request should still return response"); - let mut d = Decoder::new(body); - assert_eq!(d.read_i32().unwrap(), 1); - d.read_i32().unwrap(); - d.read_nullable_string().unwrap(); - d.read_i32().unwrap(); - assert_eq!(d.read_i32().unwrap(), 0); - assert_eq!(d.remaining(), 0); +fn metadata_null_topic_name_closes_connection() { + assert!( + handle_request( + API_KEY_METADATA, + 0, + Bytes::from_static(&[ + 0x00, 0x00, 0x00, 0x01, // one topic + 0xff, 0xff, // null topic name + ]), + &default_broker(), + ) + .is_close(), + "null topic name cannot be echoed in a Metadata response" + ); } // ── Full handler regression - every scoped API key x version through `handle_request` ── @@ -275,10 +298,30 @@ fn metadata_null_topic_name_yields_zero_topics() { #[test] fn handle_request_succeeds_for_every_supported_version_with_fixture() { for &(api_key, name, min_ver, max_ver) in scope::SCOPED_API_KEYS { - if api_key == 3 || api_key == 18 { - // Metadata / ApiVersions: empty body is valid + if api_key == API_KEY_METADATA { + for version in min_ver..=max_ver { + let body = if version >= 9 { + wire::build_metadata_all_topics_flexible(version) + } else { + build_metadata_all_topics_legacy(version) + }; + let resp = handle_request(api_key, version, body, &default_broker()) + .expect_response("test request has acks != 0 and expects a response"); + assert!( + !resp.is_empty(), + "{name} v{version} returned empty response" + ); + } + continue; + } + if api_key == API_KEY_API_VERSIONS { for version in min_ver..=max_ver { - let resp = handle_request(api_key, version, bytes::Bytes::new(), &default_broker()) + let body = if version >= 3 { + build_api_versions_flexible_request("iggy-test", "0.1.0") + } else { + Bytes::new() + }; + let resp = handle_request(api_key, version, body, &default_broker()) .expect_response("test request has acks != 0 and expects a response"); assert!( !resp.is_empty(), @@ -403,24 +446,18 @@ fn synthetic_topic_name(i: i32) -> String { format!("topic-{i}") } -fn metadata_request_legacy(topic_count: i32) -> Bytes { - let mut enc = Encoder::with_capacity(64); - enc.write_i32(topic_count); - for i in 0..topic_count { - enc.write_nullable_string(Some(&synthetic_topic_name(i))) - .expect("topic name fits"); - } - enc.freeze() +fn metadata_request_legacy(version: i16, topic_count: i32) -> Bytes { + let names: Vec = (0..topic_count).map(synthetic_topic_name).collect(); + let refs: Vec<&str> = names.iter().map(String::as_str).collect(); + build_metadata_legacy_request_for_version(version, &refs) } fn metadata_request_flexible(topic_count: usize) -> Bytes { - let mut enc = Encoder::with_capacity(64); - enc.write_varint((topic_count + 1) as u64); - for i in 0..topic_count { - enc.write_compact_nullable_string(Some(&synthetic_topic_name(i32::try_from(i).unwrap()))); - enc.write_empty_tagged_fields(); - } - enc.freeze() + let names: Vec = (0..topic_count) + .map(|i| synthetic_topic_name(i32::try_from(i).unwrap())) + .collect(); + let refs: Vec<&str> = names.iter().map(String::as_str).collect(); + build_metadata_flexible_request(&refs) } fn read_broker_legacy(d: &mut Decoder) -> (String, i32) { @@ -444,18 +481,17 @@ fn read_broker_flexible(d: &mut Decoder) -> (String, i32) { } #[test] -fn metadata_corrupt_partial_body_returns_zero_topics() { - let body = handle_request( - API_KEY_METADATA, - 0, - Bytes::from_static(&[0x00, 0x00]), - &default_broker(), - ) - .expect_response("test request has acks != 0 and expects a response"); - let mut d = Decoder::new(body); - let _ = read_broker_legacy(&mut d); - assert_eq!(d.read_i32().unwrap(), 0); - assert_eq!(d.remaining(), 0); +fn metadata_corrupt_partial_body_closes_connection() { + assert!( + handle_request( + API_KEY_METADATA, + 0, + Bytes::from_static(&[0x00, 0x00]), + &default_broker(), + ) + .is_close(), + "truncated Metadata body must close; there is no top-level error field" + ); } #[test] @@ -463,7 +499,7 @@ fn metadata_v0_empty_topics_stub_broker() { let body = handle_request( API_KEY_METADATA, 0, - metadata_request_legacy(0), + metadata_request_legacy(0, 0), &default_broker(), ) .expect_response("test request has acks != 0 and expects a response"); @@ -484,7 +520,7 @@ fn metadata_v0_three_topics_each_unknown() { let body = handle_request( API_KEY_METADATA, 0, - metadata_request_legacy(3), + metadata_request_legacy(0, 3), &default_broker(), ) .expect_response("test request has acks != 0 and expects a response"); @@ -506,7 +542,7 @@ fn metadata_v1_includes_controller_id() { let body = handle_request( API_KEY_METADATA, 1, - metadata_request_legacy(0), + metadata_request_legacy(1, 0), &default_broker(), ) .expect_response("test request has acks != 0 and expects a response"); @@ -523,7 +559,7 @@ fn metadata_v2_includes_cluster_id_field() { let body = handle_request( API_KEY_METADATA, 2, - metadata_request_legacy(0), + metadata_request_legacy(2, 0), &default_broker(), ) .expect_response("test request has acks != 0 and expects a response"); @@ -541,7 +577,7 @@ fn metadata_all_legacy_versions_produce_valid_response() { let body = handle_request( API_KEY_METADATA, version, - metadata_request_legacy(1), + metadata_request_legacy(version, 1), &default_broker(), ) .expect_response("test request has acks != 0 and expects a response"); @@ -606,7 +642,7 @@ fn metadata_v8_includes_authorized_operations_legacy() { let body = handle_request( API_KEY_METADATA, 8, - metadata_request_legacy(1), + metadata_request_legacy(8, 1), &default_broker(), ) .expect_response("test request has acks != 0 and expects a response"); @@ -771,10 +807,10 @@ fn metadata_v3_includes_throttle_time_ms_before_brokers() { let body = handle_request( API_KEY_METADATA, 3, - build_metadata_legacy_request(&[]), + build_metadata_legacy_request_for_version(3, &[]), &default_broker(), ) - .expect_response("test request has acks != 0 and expects a response"); + .expect_response("metadata v3 empty topics must succeed"); let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 0, "throttle_time_ms"); } diff --git a/gateways/kafka/tests/common/tcp.rs b/gateways/kafka/tests/common/tcp.rs index bd75a13483..b8d4785a34 100644 --- a/gateways/kafka/tests/common/tcp.rs +++ b/gateways/kafka/tests/common/tcp.rs @@ -133,7 +133,10 @@ pub fn build_list_offsets_v0_request_with_topic_t() -> Bytes { body.freeze() } -/// Legacy Metadata request body listing topic names (non-flexible, v0-v8). +/// Legacy Metadata request body listing topic names (non-flexible, v0–v3 fields only). +/// +/// Prefer version-aware wire helpers when targeting Metadata v4+, which also require +/// `allow_auto_topic_creation` (and later authorized-ops flags). pub fn build_metadata_legacy_request(topic_names: &[&str]) -> Bytes { let mut body = BytesMut::new(); body.put_i32(i32::try_from(topic_names.len()).expect("topic name count fits i32")); diff --git a/gateways/kafka/tests/common/wire.rs b/gateways/kafka/tests/common/wire.rs index 3a873fc72f..4f9cf4ee06 100644 --- a/gateways/kafka/tests/common/wire.rs +++ b/gateways/kafka/tests/common/wire.rs @@ -47,32 +47,86 @@ pub const FLEXIBLE_FROM_VERSION: &[(i16, i16)] = &[ (19, 5), // CreateTopics ]; -/// Metadata v9+ flexible request listing topic names (compact strings). +/// Append Metadata request fields that follow the topics array for `version`. +fn write_metadata_request_trailer(enc: &mut Encoder, version: i16) { + if version >= 4 { + enc.write_bool(true); // allow_auto_topic_creation + } + // include_cluster_authorized_operations exists on v8–v10 only (removed in v11). + if (8..=10).contains(&version) { + enc.write_bool(false); + } + if version >= 8 { + enc.write_bool(false); // include_topic_authorized_operations + } + if version >= 9 { + enc.write_empty_tagged_fields(); + } +} + +/// Metadata v9 flexible request listing topic names (compact strings). /// /// Each topic entry is its own tagged struct per the Kafka protocol schema /// (`topics => name TAG_BUFFER`), so the per-topic tag buffer is written /// right after each name, not once for the whole array. pub fn build_metadata_flexible_request(topic_names: &[&str]) -> Bytes { - let mut enc = Encoder::with_capacity(64); + build_metadata_flexible_request_for_version(9, topic_names) +} + +/// Metadata flexible request for a specific version (v9+). +pub fn build_metadata_flexible_request_for_version(version: i16, topic_names: &[&str]) -> Bytes { + let mut enc = Encoder::with_capacity(96); enc.write_varint((topic_names.len() + 1) as u64); for name in topic_names { + if version >= 10 { + enc.write_bytes(&[0u8; 16]); + } enc.write_compact_nullable_string(Some(name)); enc.write_empty_tagged_fields(); } - enc.write_empty_tagged_fields(); + write_metadata_request_trailer(&mut enc, version); enc.freeze() } /// Metadata v10+ flexible request: each topic entry includes a 16-byte `topic_id` before `name`. pub fn build_metadata_flexible_request_v10(topic_names: &[&str]) -> Bytes { - let mut enc = Encoder::with_capacity(96); - enc.write_varint((topic_names.len() + 1) as u64); + build_metadata_flexible_request_for_version(10, topic_names) +} + +/// `ApiVersions` v3+ flexible body (`ClientSoftwareName`, `ClientSoftwareVersion`, tagged fields). +pub fn build_api_versions_flexible_request(software_name: &str, software_version: &str) -> Bytes { + let mut enc = Encoder::with_capacity(64); + enc.write_compact_nullable_string(Some(software_name)); + enc.write_compact_nullable_string(Some(software_version)); + enc.write_empty_tagged_fields(); + enc.freeze() +} + +/// Legacy Metadata request body for a specific version (v0–v8). +pub fn build_metadata_legacy_request_for_version(version: i16, topic_names: &[&str]) -> Bytes { + let mut enc = Encoder::with_capacity(64); + enc.write_i32(i32::try_from(topic_names.len()).expect("topic name count fits i32")); for name in topic_names { - enc.write_bytes(&[0u8; 16]); - enc.write_compact_nullable_string(Some(name)); - enc.write_empty_tagged_fields(); + enc.write_nullable_string(Some(name)) + .expect("topic name fits"); } - enc.write_empty_tagged_fields(); + write_metadata_request_trailer(&mut enc, version); + enc.freeze() +} + +/// Legacy Metadata "all topics" body (`topics = null` / `-1`) for a specific version. +pub fn build_metadata_all_topics_legacy(version: i16) -> Bytes { + let mut enc = Encoder::with_capacity(16); + enc.write_i32(-1); + write_metadata_request_trailer(&mut enc, version); + enc.freeze() +} + +/// Flexible Metadata "all topics" body (`topics` compact null / varint `0`). +pub fn build_metadata_all_topics_flexible(version: i16) -> Bytes { + let mut enc = Encoder::with_capacity(16); + enc.write_varint(0); // null compact array → all topics + write_metadata_request_trailer(&mut enc, version); enc.freeze() } diff --git a/gateways/kafka/tests/decode_safety_tests.rs b/gateways/kafka/tests/decode_safety_tests.rs index 6bca4911e0..5f9ba767ee 100644 --- a/gateways/kafka/tests/decode_safety_tests.rs +++ b/gateways/kafka/tests/decode_safety_tests.rs @@ -30,10 +30,17 @@ use iggy_gateway_kafka::protocol::requests::{ }; #[test] -fn compact_array_varint_zero_decodes_as_empty_without_panic() { - // Per Kafka spec, compact-array varint=0 means null/absent → 0 elements (not an error). +fn compact_array_varint_zero_rejected_on_non_nullable_array() { + // Compact-array varint=0 is Kafka's null encoding; required (non-nullable) arrays reject it. let mut d = Decoder::new(Bytes::from_static(&[0x00])); - assert_eq!(d.read_compact_array_count().unwrap(), 0); + let err = d.read_compact_array_count().unwrap_err(); + assert!(matches!(err, KafkaProtocolError::NullCompactArray)); +} + +#[test] +fn compact_array_varint_zero_nullable_decodes_as_empty() { + let mut d = Decoder::new(Bytes::from_static(&[0x00])); + assert_eq!(d.read_compact_array_count_nullable().unwrap(), 0); } #[test] diff --git a/gateways/kafka/tests/listener_robustness_tests.rs b/gateways/kafka/tests/listener_robustness_tests.rs index ac3309f48c..c999b2db54 100644 --- a/gateways/kafka/tests/listener_robustness_tests.rs +++ b/gateways/kafka/tests/listener_robustness_tests.rs @@ -213,7 +213,8 @@ async fn e2e_response_frames_have_positive_big_endian_length_prefix() { let (addr, _shutdown) = spawn_test_server().await; let mut stream = TcpStream::connect(addr).await.expect("connect"); - let frame = build_request_frame(API_KEY_API_VERSIONS, 3, 200, Some("len-test"), &[]); + let request = wire::build_api_versions_flexible_request("iggy-test", "0.1.0"); + let frame = build_request_frame(API_KEY_API_VERSIONS, 3, 200, Some("len-test"), &request); stream.write_all(&frame).await.expect("write"); let mut len_buf = [0u8; 4]; @@ -327,7 +328,8 @@ async fn e2e_empty_client_id_request_succeeds() { #[tokio::test] async fn e2e_flexible_apiversions_v3_request_succeeds() { let (addr, _shutdown) = spawn_test_server().await; - let (corr, body) = tcp::round_trip(addr, API_KEY_API_VERSIONS, 3, 401, &[]).await; + let request = wire::build_api_versions_flexible_request("iggy-test", "0.1.0"); + let (corr, body) = tcp::round_trip(addr, API_KEY_API_VERSIONS, 3, 401, &request).await; assert_eq!(corr, 401); let mut d = Decoder::new(body); assert_eq!(d.read_i16().unwrap(), 0); diff --git a/gateways/kafka/tests/response_negative_tests.rs b/gateways/kafka/tests/response_negative_tests.rs index bd33910591..ffac83ab27 100644 --- a/gateways/kafka/tests/response_negative_tests.rs +++ b/gateways/kafka/tests/response_negative_tests.rs @@ -37,6 +37,7 @@ fn create_topics_response_flags_non_positive_partition_count_v2() { name: "bad-topic".to_string(), num_partitions: 0, replication_factor: 1, + has_assignments: false, }], timeout_ms: 5_000, validate_only: false, @@ -59,6 +60,7 @@ fn create_topics_v5_broker_default_partitions_is_not_invalid_partitions() { name: "default-parts".to_string(), num_partitions: -1, replication_factor: 2, + has_assignments: false, }], timeout_ms: 5_000, validate_only: true, @@ -84,6 +86,7 @@ fn create_topics_v5_flags_zero_and_below_minus_one_partition_count() { name: "bad-parts".to_string(), num_partitions, replication_factor: 1, + has_assignments: false, }], timeout_ms: 5_000, validate_only: false, @@ -111,6 +114,7 @@ fn create_topics_v5_flags_invalid_replication_factor() { name: "bad-rf".to_string(), num_partitions: 1, replication_factor, + has_assignments: false, }], timeout_ms: 5_000, validate_only: false, @@ -132,12 +136,13 @@ fn create_topics_v5_flags_invalid_replication_factor() { #[test] fn create_topics_v2_rejects_broker_default_sentinel() { - // KIP-464 defaults apply from v4; on v2, -1 is still INVALID_PARTITIONS. + // KIP-464 defaults apply from v4; on v2 without assignments, -1 is INVALID_PARTITIONS. let req = CreateTopicsRequest { topics: vec![CreatableTopic { name: "legacy".to_string(), num_partitions: -1, replication_factor: 1, + has_assignments: false, }], timeout_ms: 5_000, validate_only: false, @@ -152,6 +157,29 @@ fn create_topics_v2_rejects_broker_default_sentinel() { assert_eq!(d.read_i16().unwrap(), ERROR_INVALID_PARTITIONS); } +#[test] +fn create_topics_v2_with_assignments_allows_broker_default_sentinels() { + // KIP-464: on v2/v3, -1 partitions/replication are valid when assignments are present. + let req = CreateTopicsRequest { + topics: vec![CreatableTopic { + name: "assigned".to_string(), + num_partitions: -1, + replication_factor: -1, + has_assignments: true, + }], + timeout_ms: 5_000, + validate_only: false, + }; + let mut d = Decoder::new(encode_create_topics_response(2, &req)); + assert_eq!(d.read_i32().unwrap(), 0); + assert_eq!(d.read_i32().unwrap(), 1); + assert_eq!( + d.read_nullable_string().unwrap(), + Some("assigned".to_string()) + ); + assert_eq!(d.read_i16().unwrap(), ERROR_NOT_CONTROLLER); +} + #[test] fn create_topics_error_response_carries_explicit_error_code() { let mut d = Decoder::new(encode_create_topics_error_response( diff --git a/gateways/kafka/tests/server_e2e_tests.rs b/gateways/kafka/tests/server_e2e_tests.rs index 13f237961e..c5d6283988 100644 --- a/gateways/kafka/tests/server_e2e_tests.rs +++ b/gateways/kafka/tests/server_e2e_tests.rs @@ -61,7 +61,8 @@ async fn e2e_apiversions_v1_preserves_correlation_id() { #[tokio::test] async fn e2e_apiversions_v3_flexible_preserves_correlation_id() { let (addr, _shutdown) = spawn_test_server().await; - let (corr, body) = round_trip(addr, API_KEY_API_VERSIONS, 3, 42_002, &[]).await; + let request = wire::build_api_versions_flexible_request("iggy-test", "0.1.0"); + let (corr, body) = round_trip(addr, API_KEY_API_VERSIONS, 3, 42_002, &request).await; assert_eq!(corr, 42_002); let mut d = Decoder::new(body); assert_eq!(d.read_i16().unwrap(), 0); @@ -431,9 +432,10 @@ async fn fetch_v4_through_v12_e2e_preserve_correlation_id() { } #[tokio::test] -async fn metadata_empty_body_e2e_all_topics_request_returns_broker() { +async fn metadata_all_topics_null_array_e2e_returns_broker() { let (addr, _shutdown) = spawn_test_server().await; - let (corr, body) = round_trip(addr, API_KEY_METADATA, 0, 360, &[]).await; + let request = wire::build_metadata_all_topics_legacy(0); + let (corr, body) = round_trip(addr, API_KEY_METADATA, 0, 360, &request).await; assert_eq!(corr, 360); let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 1, "one stub broker"); @@ -444,6 +446,19 @@ async fn metadata_empty_body_e2e_all_topics_request_returns_broker() { assert!(port > 0); } +#[tokio::test] +async fn metadata_empty_body_e2e_closes_connection() { + let (addr, _shutdown) = spawn_test_server().await; + let mut stream = TcpStream::connect(addr).await.expect("connect"); + let frame = build_request_frame(API_KEY_METADATA, 0, 361, Some("empty-md"), &[]); + stream.write_all(&frame).await.expect("write"); + assert_eq!( + read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await, + ByteRead::Closed, + "empty Metadata body must close the connection" + ); +} + // ── Out-of-scope API keys (SCOPE.md unsupported list) ─────────────────────── #[tokio::test] diff --git a/gateways/kafka/tests/version_firewall_tests.rs b/gateways/kafka/tests/version_firewall_tests.rs index a77574fece..bd92090a9d 100644 --- a/gateways/kafka/tests/version_firewall_tests.rs +++ b/gateways/kafka/tests/version_firewall_tests.rs @@ -30,7 +30,7 @@ mod wire; use std::time::Duration; -use bytes::{BufMut, Bytes, BytesMut}; +use bytes::Bytes; use tokio::io::AsyncWriteExt; use tokio::net::TcpStream; @@ -50,10 +50,11 @@ use tcp::{ build_produce_v3_body, build_request_frame, parse_response_payload, read_byte_with_timeout, round_trip, scan_for_error_code, }; -use wire::build_metadata_flexible_request_v10; use wire::{ - OUT_OF_SCOPE_API_KEYS, build_create_topics_empty_request, build_fetch_empty_topics_request, - build_list_offsets_request, + OUT_OF_SCOPE_API_KEYS, build_api_versions_flexible_request, build_create_topics_empty_request, + build_fetch_empty_topics_request, build_list_offsets_request, + build_metadata_all_topics_flexible, build_metadata_all_topics_legacy, + build_metadata_flexible_request_v10, }; #[test] @@ -111,8 +112,13 @@ fn apiversions_advertises_exact_supported_ranges_v1() { #[test] fn apiversions_advertises_exact_supported_ranges_v3_flexible() { - let body = handle_request(API_KEY_API_VERSIONS, 3, Bytes::new(), &default_broker()) - .expect_response("test request has acks != 0 and expects a response"); + let body = handle_request( + API_KEY_API_VERSIONS, + 3, + build_api_versions_flexible_request("iggy-test", "0.1.0"), + &default_broker(), + ) + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i16().unwrap(), 0); let count = usize::try_from(d.read_varint().unwrap() - 1).expect("api count fits usize"); @@ -152,13 +158,13 @@ fn apiversions_advertises_produce_min_zero_while_firewall_stays_three() { #[test] fn apiversions_all_versions_return_success() { for version in 0i16..=3 { - let body = handle_request( - API_KEY_API_VERSIONS, - version, - Bytes::new(), - &default_broker(), - ) - .expect_response("test request has acks != 0 and expects a response"); + let request = if version >= 3 { + build_api_versions_flexible_request("iggy-test", "0.1.0") + } else { + Bytes::new() + }; + let body = handle_request(API_KEY_API_VERSIONS, version, request, &default_broker()) + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i16().unwrap(), 0, "ApiVersions v{version}"); } @@ -254,16 +260,36 @@ fn fetch_unsupported_version_returns_well_formed_error_response() { } #[test] -fn fetch_unsupported_version_above_max_uses_top_level_error() { - let body = handle_request(API_KEY_FETCH, 13, Bytes::new(), &default_broker()) - .expect_response("test request has acks != 0 and expects a response"); - let mut d = Decoder::new(body); - assert_eq!(d.read_i32().unwrap(), 0); - assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); - assert_eq!(d.read_i32().unwrap(), 0); - assert_eq!(d.read_varint().unwrap(), 1); - d.read_tagged_fields().unwrap(); - assert_eq!(d.remaining(), 0); +fn fetch_unsupported_version_above_max_closes_connection() { + // Fetch v13+ response shape differs from the v12 encoder; a clamped body is unparsable. + assert!( + handle_request(API_KEY_FETCH, 13, Bytes::new(), &default_broker()).is_close(), + "Fetch above encoder max must close rather than return a clamped body" + ); +} + +#[test] +fn produce_unsupported_version_above_max_closes_connection() { + assert!( + handle_request(API_KEY_PRODUCE, 13, Bytes::new(), &default_broker()).is_close(), + "Produce above encoder max must close rather than return a clamped body" + ); +} + +#[test] +fn create_topics_unsupported_version_above_max_closes_connection() { + assert!( + handle_request(API_KEY_CREATE_TOPICS, 7, Bytes::new(), &default_broker()).is_close(), + "CreateTopics above encoder max must close rather than return a clamped body" + ); +} + +#[test] +fn list_offsets_unsupported_version_above_max_closes_connection() { + assert!( + handle_request(API_KEY_LIST_OFFSETS, 7, Bytes::new(), &default_broker()).is_close(), + "ListOffsets above encoder max must close rather than return a clamped body" + ); } #[test] @@ -428,15 +454,22 @@ fn list_offsets_v0_unsupported_version_carries_error_code_in_partition() { // ── Comprehensive scoped-API coverage (correlation id, boundary versions) ── -fn metadata_empty_legacy_body() -> Bytes { - let mut body = BytesMut::new(); - body.put_i32(0); - body.freeze() -} - fn request_body_for_scoped_api(api_key: i16, name: &str, version: i16) -> Bytes { match api_key { - API_KEY_METADATA => metadata_empty_legacy_body(), + API_KEY_METADATA => { + if version >= 9 { + build_metadata_all_topics_flexible(version) + } else { + build_metadata_all_topics_legacy(version) + } + } + API_KEY_API_VERSIONS => { + if version >= 3 { + build_api_versions_flexible_request("iggy-test", "0.1.0") + } else { + Bytes::new() + } + } API_KEY_PRODUCE => { if fixture_exists(api_key, name, version) { load_fixture_body(api_key, name, version) @@ -551,7 +584,7 @@ fn out_of_scope_api_keys_return_unsupported_version_without_panic() { // ── Boundary versions keep the TCP session (except Metadata) ─────────────── #[tokio::test] -async fn each_scoped_api_above_max_version_e2e_keeps_connection() { +async fn each_scoped_api_above_max_version_e2e_closes_or_kip511() { let (addr, _shutdown) = spawn_test_server().await; let mut stream = TcpStream::connect(addr).await.expect("connect"); @@ -568,23 +601,25 @@ async fn each_scoped_api_above_max_version_e2e_keeps_connection() { .write_all(&frame) .await .unwrap_or_else(|_| panic!("write {name} v{above}")); - if api_key == API_KEY_METADATA { - // Unsupported Metadata closes: clamped bodies are unparsable at the client version. - assert_eq!( - read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await, - ByteRead::Closed, - "Metadata v{above} must close the connection" + if api_key == API_KEY_API_VERSIONS { + // KIP-511: ApiVersions above max still answers with a v0 UNSUPPORTED_VERSION body. + let payload = tcp::read_response_frame(&mut stream, 8 * 1024 * 1024).await; + assert!( + !payload.is_empty(), + "{name} v{above} must still respond on wire" ); - stream = TcpStream::connect(addr) - .await - .expect("reconnect after Metadata close"); continue; } - let payload = tcp::read_response_frame(&mut stream, 8 * 1024 * 1024).await; - assert!( - !payload.is_empty(), - "{name} v{above} must still respond on wire" + // Other APIs: encoding at the client's raw version above our encoder max would omit + // later-version fields, so Close is the honest contract (same as Metadata). + assert_eq!( + read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await, + ByteRead::Closed, + "{name} v{above} must close the connection" ); + stream = TcpStream::connect(addr) + .await + .unwrap_or_else(|_| panic!("reconnect after {name} close")); } let ok = build_request_frame(API_KEY_API_VERSIONS, 1, 89_999, Some("scope-test"), &[]); @@ -666,20 +701,21 @@ fn produce_advertises_min_zero_but_firewall_rejects_below_v3() { // ── Unsupported-version e2e paths for remaining scoped APIs ───────────────── #[tokio::test] -async fn list_offsets_v7_unsupported_e2e_returns_error() { +async fn list_offsets_v7_unsupported_e2e_closes_connection() { let (addr, _shutdown) = spawn_test_server().await; - let (corr, body) = round_trip( - addr, + let mut stream = TcpStream::connect(addr).await.expect("connect"); + let frame = build_request_frame( API_KEY_LIST_OFFSETS, 7, 370, + Some("scope-test"), &[0x00, 0x00, 0x00, 0x00], - ) - .await; - assert_eq!(corr, 370); - assert!( - scan_for_error_code(&body, ERROR_UNSUPPORTED_VERSION), - "ListOffsets v7 must be rejected" + ); + stream.write_all(&frame).await.expect("write"); + assert_eq!( + read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await, + ByteRead::Closed, + "ListOffsets v7 is above encoder max and must close" ); } diff --git a/gateways/kafka/tools/kafka-tool/src/main.rs b/gateways/kafka/tools/kafka-tool/src/main.rs index f21d8dc295..5b052e7844 100644 --- a/gateways/kafka/tools/kafka-tool/src/main.rs +++ b/gateways/kafka/tools/kafka-tool/src/main.rs @@ -668,6 +668,11 @@ async fn connect(host: &str) -> Result { .with_context(|| format!("Cannot connect to {host}")) } +/// Cap on Kafka response frames accepted by the tool. Matches the gateway's default +/// `max_frame_size` so a misconfigured/malicious endpoint cannot force a multi-GiB alloc +/// from a forged 4-byte length prefix. +const MAX_RESPONSE_FRAME_BYTES: usize = 8 * 1024 * 1024; + async fn read_kafka_response(stream: &mut TcpStream) -> std::io::Result> { let mut lb = [0u8; 4]; stream.read_exact(&mut lb).await?; @@ -678,16 +683,30 @@ async fn read_kafka_response(stream: &mut TcpStream) -> std::io::Result> format!("invalid response frame length: {frame_len}"), )); } - let mut body = vec![ - 0u8; - usize::try_from(frame_len).map_err(|_| { - std::io::Error::new( - std::io::ErrorKind::InvalidData, - "response frame length does not fit usize", - ) - })? - ]; - stream.read_exact(&mut body).await?; + let frame_len = usize::try_from(frame_len).map_err(|_| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + "response frame length does not fit usize", + ) + })?; + if frame_len > MAX_RESPONSE_FRAME_BYTES { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("response frame length {frame_len} exceeds max {MAX_RESPONSE_FRAME_BYTES}"), + )); + } + // Grow incrementally so a large declared length cannot force a full reservation before + // any body bytes arrive (same amplification class the gateway `read_frame` avoids). + const CHUNK: usize = 65_536; + let mut body = Vec::with_capacity(frame_len.min(CHUNK)); + let mut remaining = frame_len; + while remaining > 0 { + let chunk = remaining.min(CHUNK); + let start = body.len(); + body.resize(start + chunk, 0); + stream.read_exact(&mut body[start..start + chunk]).await?; + remaining -= chunk; + } Ok(body) } From 570132496b640d8f647c1d1e8b678ebb9b02ad18 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 8 Aug 2026 21:44:47 +0000 Subject: [PATCH 47/57] fix(gateways): reject trailing Kafka request bytes and add fixture canary Request decoders for Produce/Fetch/ListOffsets/CreateTopics now fail on trailing body bytes, matching Metadata/ApiVersions. A fixtures canary fails under KAFKA_FIXTURES_REQUIRED=1 when the wire fixture directory is empty so CI cannot pass green-but-empty skip suites. Co-authored-by: ryerraguntla --- gateways/kafka/src/protocol/requests.rs | 11 +++++ gateways/kafka/tests/common/fixtures.rs | 13 ++++++ gateways/kafka/tests/decode_safety_tests.rs | 14 ++++++ gateways/kafka/tests/fixtures_canary_tests.rs | 46 +++++++++++++++++++ 4 files changed, 84 insertions(+) create mode 100644 gateways/kafka/tests/fixtures_canary_tests.rs diff --git a/gateways/kafka/src/protocol/requests.rs b/gateways/kafka/src/protocol/requests.rs index 6792840487..be760a79a6 100644 --- a/gateways/kafka/src/protocol/requests.rs +++ b/gateways/kafka/src/protocol/requests.rs @@ -81,6 +81,13 @@ macro_rules! produce_decode { }; } +fn ensure_body_exhausted(d: &Decoder) -> Result<()> { + if d.remaining() != 0 { + return Err(KafkaProtocolError::UnexpectedTrailingBytes); + } + Ok(()) +} + pub fn decode_produce_request(version: i16, body: Bytes) -> ProduceDecodeResult { let mut d = Decoder::new(body); let flexible = version >= 9; @@ -161,6 +168,7 @@ pub fn decode_produce_request(version: i16, body: Bytes) -> ProduceDecodeResult if flexible { produce_decode!(acks_read, d.read_tagged_fields()); } + produce_decode!(acks_read, ensure_body_exhausted(&d)); ProduceDecodeResult::Ok(ProduceRequest { transactional_id, @@ -317,6 +325,7 @@ pub fn decode_fetch_request(version: i16, body: Bytes) -> Result { if flexible { d.read_tagged_fields()?; } + ensure_body_exhausted(&d)?; Ok(FetchRequest { max_wait_ms, @@ -413,6 +422,7 @@ pub fn decode_list_offsets_request(version: i16, body: Bytes) -> Result Result Bytes { /// generation script so a fresh clone knows how to produce it. pub const FIXTURE_SKIP_HINT: &str = "generate with `gateways/kafka/scripts/ci-wire-fixtures.sh generate` (or the kafka-tool `generate` subcommand)"; +/// True when the fixtures directory contains at least one `.bin` wire fixture. +pub fn any_wire_fixture_present() -> bool { + let Ok(entries) = std::fs::read_dir(fixtures_dir()) else { + return false; + }; + entries.filter_map(Result::ok).any(|entry| { + entry + .path() + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("bin")) + }) +} + /// Set by CI (after `ci-wire-fixtures.sh generate`) to turn missing-fixture skips into hard /// failures, so a broken generation step cannot leave these suites green with zero assertions. const FIXTURES_REQUIRED_ENV: &str = "KAFKA_FIXTURES_REQUIRED"; diff --git a/gateways/kafka/tests/decode_safety_tests.rs b/gateways/kafka/tests/decode_safety_tests.rs index 5f9ba767ee..d338927f22 100644 --- a/gateways/kafka/tests/decode_safety_tests.rs +++ b/gateways/kafka/tests/decode_safety_tests.rs @@ -43,6 +43,20 @@ fn compact_array_varint_zero_nullable_decodes_as_empty() { assert_eq!(d.read_compact_array_count_nullable().unwrap(), 0); } +#[test] +fn produce_decoder_rejects_trailing_bytes_after_valid_body() { + let mut body = Vec::new(); + body.extend_from_slice(&(-1_i16).to_be_bytes()); // null transactional_id (legacy) + body.extend_from_slice(&1_i16.to_be_bytes()); // acks + body.extend_from_slice(&1000_i32.to_be_bytes()); // timeout_ms + body.extend_from_slice(&0_i32.to_be_bytes()); // empty topics + body.push(0xFF); // trailing garbage + let err = decode_produce_request(3, Bytes::from(body)) + .into_request() + .unwrap_err(); + assert!(matches!(err, KafkaProtocolError::UnexpectedTrailingBytes)); +} + #[test] fn negative_i32_array_length_returns_error_not_panic() { let mut raw = Vec::new(); diff --git a/gateways/kafka/tests/fixtures_canary_tests.rs b/gateways/kafka/tests/fixtures_canary_tests.rs new file mode 100644 index 0000000000..dc4c641675 --- /dev/null +++ b/gateways/kafka/tests/fixtures_canary_tests.rs @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Canary so fixture-backed suites cannot go green-but-empty when CI requires fixtures. + +#[path = "common/fixtures.rs"] +mod fixtures; + +use fixtures::{FIXTURE_SKIP_HINT, any_wire_fixture_present, fixtures_dir}; + +#[test] +fn kafka_wire_fixtures_canary() { + let fixtures_required = + std::env::var("KAFKA_FIXTURES_REQUIRED").is_ok_and(|value| value == "1"); + let present = any_wire_fixture_present(); + + if fixtures_required { + assert!( + present, + "KAFKA_FIXTURES_REQUIRED=1 but no .bin fixtures under {:?}; {FIXTURE_SKIP_HINT}", + fixtures_dir() + ); + return; + } + + if !present { + eprintln!( + "note: no wire fixtures under {:?}; fixture-backed suites will skip ({FIXTURE_SKIP_HINT})", + fixtures_dir() + ); + } +} From d468d4e3c6110c3a6f3ced124e193143cdce4afb Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sun, 9 Aug 2026 03:10:54 -0400 Subject: [PATCH 48/57] Harden Kafka gateway request handling Tightens gateway safety and protocol behavior across config loading, decoding, and response handling. This adds validated IGGY_KAFKA_* parsing (including max frame/read/write timeouts), enforces a per-request decode element budget plus compact-string size limits, and simplifies outcomes so unknown/out-of-scope API keys now close the connection instead of sending an unparseable error body. Produce decode failures now distinguish pre-acks failures (silent to avoid acks=0 stream desync) from post-acks failures (INVALID_REQUEST response). It also updates metadata/version helpers, response write framing, docs, and related regression tests. --- gateways/kafka/src/error.rs | 2 + gateways/kafka/src/main.rs | 131 +++++++++++++----- gateways/kafka/src/protocol/api.rs | 71 +++++----- gateways/kafka/src/protocol/codec.rs | 41 +++++- gateways/kafka/src/protocol/responses.rs | 5 +- gateways/kafka/src/server.rs | 101 +++++--------- gateways/kafka/tests/api_handler_tests.rs | 11 +- gateways/kafka/tests/common/tcp.rs | 21 +++ .../kafka/tests/listener_robustness_tests.rs | 6 +- gateways/kafka/tests/server_e2e_tests.rs | 27 ++-- .../kafka/tests/version_firewall_tests.rs | 84 +++++++---- 11 files changed, 315 insertions(+), 185 deletions(-) diff --git a/gateways/kafka/src/error.rs b/gateways/kafka/src/error.rs index 16b9de936c..afeb643963 100644 --- a/gateways/kafka/src/error.rs +++ b/gateways/kafka/src/error.rs @@ -40,6 +40,8 @@ pub enum KafkaProtocolError { InvalidArrayLength(i32), #[error("collection length {count} exceeds maximum {max}")] CollectionTooLarge { count: usize, max: usize }, + #[error("request element budget exceeded: {count} more requested, {remaining} remaining")] + RequestElementBudgetExceeded { count: usize, remaining: usize }, #[error("string length {length} exceeds i16::MAX")] StringTooLong { length: usize }, #[error("null topic name in request")] diff --git a/gateways/kafka/src/main.rs b/gateways/kafka/src/main.rs index 5b316be7dd..bd37bf458c 100644 --- a/gateways/kafka/src/main.rs +++ b/gateways/kafka/src/main.rs @@ -15,9 +15,13 @@ // specific language governing permissions and limitations // under the License. +use std::fmt::Display; +use std::str::FromStr; +use std::time::Duration; + use tokio::net::TcpListener; use tokio::signal; -use tokio::sync::broadcast; +use tokio::sync::{Semaphore, broadcast}; use iggy_gateway_kafka::server::init_tracing; use iggy_gateway_kafka::{KafkaServer, ServerConfig}; @@ -26,36 +30,8 @@ use iggy_gateway_kafka::{KafkaServer, ServerConfig}; async fn main() -> Result<(), Box> { init_tracing(); - let mut config = ServerConfig::default(); - if let Ok(bind_addr) = std::env::var("IGGY_KAFKA_BIND_ADDR") { - config.bind_addr = bind_addr; - } - if let Ok(advertised_host) = std::env::var("IGGY_KAFKA_ADVERTISED_HOST") { - config.advertised_host = Some(advertised_host); - } - if let Ok(advertised_port) = std::env::var("IGGY_KAFKA_ADVERTISED_PORT") { - config.advertised_port = - Some(advertised_port.parse().map_err(|e| { - format!("invalid IGGY_KAFKA_ADVERTISED_PORT `{advertised_port}`: {e}") - })?); - } - if let Ok(max_connections) = std::env::var("IGGY_KAFKA_MAX_CONNECTIONS") { - config.max_connections = max_connections - .parse() - .map_err(|e| format!("invalid IGGY_KAFKA_MAX_CONNECTIONS `{max_connections}`: {e}"))?; - } - if let Ok(idle_timeout_secs) = std::env::var("IGGY_KAFKA_IDLE_TIMEOUT_SECS") { - let secs: u64 = idle_timeout_secs.parse().map_err(|e| { - format!("invalid IGGY_KAFKA_IDLE_TIMEOUT_SECS `{idle_timeout_secs}`: {e}") - })?; - config.idle_timeout = std::time::Duration::from_secs(secs); - } - if let Ok(drain_secs) = std::env::var("IGGY_KAFKA_SHUTDOWN_DRAIN_TIMEOUT_SECS") { - let secs: u64 = drain_secs.parse().map_err(|e| { - format!("invalid IGGY_KAFKA_SHUTDOWN_DRAIN_TIMEOUT_SECS `{drain_secs}`: {e}") - })?; - config.shutdown_drain_timeout = std::time::Duration::from_secs(secs); - } + let config = load_config()?; + let listener = TcpListener::bind(&config.bind_addr) .await .map_err(|e| format!("failed to bind {}: {e}", config.bind_addr))?; @@ -77,6 +53,77 @@ async fn main() -> Result<(), Box> { Ok(()) } +/// Build [`ServerConfig`] from `IGGY_KAFKA_*` env vars, rejecting values that would silently +/// break the listener (a zero connection cap serves nothing, a zero timeout drops every +/// connection, a connection cap above `Semaphore::MAX_PERMITS` panics at startup). +fn load_config() -> Result { + let mut config = ServerConfig::default(); + + if let Some(bind_addr) = env_var("IGGY_KAFKA_BIND_ADDR") { + config.bind_addr = bind_addr; + } + if let Some(advertised_host) = env_var("IGGY_KAFKA_ADVERTISED_HOST") { + config.advertised_host = Some(advertised_host); + } + if let Some(raw) = env_var("IGGY_KAFKA_ADVERTISED_PORT") { + config.advertised_port = Some(parse_positive("IGGY_KAFKA_ADVERTISED_PORT", &raw)?); + } + if let Some(raw) = env_var("IGGY_KAFKA_MAX_CONNECTIONS") { + let max_connections: usize = parse_positive("IGGY_KAFKA_MAX_CONNECTIONS", &raw)?; + if max_connections > Semaphore::MAX_PERMITS { + return Err(format!( + "IGGY_KAFKA_MAX_CONNECTIONS {max_connections} exceeds maximum {}", + Semaphore::MAX_PERMITS + )); + } + config.max_connections = max_connections; + } + if let Some(raw) = env_var("IGGY_KAFKA_MAX_FRAME_SIZE") { + config.max_frame_size = parse_positive("IGGY_KAFKA_MAX_FRAME_SIZE", &raw)?; + } + if let Some(raw) = env_var("IGGY_KAFKA_IDLE_TIMEOUT_SECS") { + config.idle_timeout = + Duration::from_secs(parse_positive("IGGY_KAFKA_IDLE_TIMEOUT_SECS", &raw)?); + } + if let Some(raw) = env_var("IGGY_KAFKA_READ_TIMEOUT_SECS") { + config.read_timeout = + Duration::from_secs(parse_positive("IGGY_KAFKA_READ_TIMEOUT_SECS", &raw)?); + } + if let Some(raw) = env_var("IGGY_KAFKA_WRITE_TIMEOUT_SECS") { + config.write_timeout = + Duration::from_secs(parse_positive("IGGY_KAFKA_WRITE_TIMEOUT_SECS", &raw)?); + } + // Drain of 0 is valid: abandon in-flight connections immediately on shutdown. + if let Some(raw) = env_var("IGGY_KAFKA_SHUTDOWN_DRAIN_TIMEOUT_SECS") { + let secs: u64 = raw + .parse() + .map_err(|e| format!("invalid IGGY_KAFKA_SHUTDOWN_DRAIN_TIMEOUT_SECS `{raw}`: {e}"))?; + config.shutdown_drain_timeout = Duration::from_secs(secs); + } + + Ok(config) +} + +fn env_var(key: &str) -> Option { + std::env::var(key).ok() +} + +/// Parse a strictly-positive value, rejecting `0` (which for connection caps and timeouts would +/// silently disable the listener) and unparseable input. +fn parse_positive(key: &str, raw: &str) -> Result +where + T: FromStr + Default + PartialEq, + T::Err: Display, +{ + let value: T = raw + .parse() + .map_err(|e| format!("invalid {key} `{raw}`: {e}"))?; + if value == T::default() { + return Err(format!("{key} must be greater than 0")); + } + Ok(value) +} + /// Wait for Ctrl-C (SIGINT) or, on Unix, SIGTERM (`docker stop`). async fn shutdown_signal() { let ctrl_c = async { @@ -101,3 +148,25 @@ async fn shutdown_signal() { () = terminate => {} } } + +#[cfg(test)] +mod tests { + use super::parse_positive; + + #[test] + fn parse_positive_rejects_zero() { + assert!(parse_positive::("KEY", "0").is_err()); + assert!(parse_positive::("KEY", "0").is_err()); + } + + #[test] + fn parse_positive_rejects_non_numeric() { + assert!(parse_positive::("KEY", "abc").is_err()); + } + + #[test] + fn parse_positive_accepts_positive_value() { + assert_eq!(parse_positive::("KEY", "42").unwrap(), 42); + assert_eq!(parse_positive::("KEY", "9093").unwrap(), 9093); + } +} diff --git a/gateways/kafka/src/protocol/api.rs b/gateways/kafka/src/protocol/api.rs index 9c6d86e23e..81cbce706d 100644 --- a/gateways/kafka/src/protocol/api.rs +++ b/gateways/kafka/src/protocol/api.rs @@ -49,8 +49,6 @@ pub const ERROR_INVALID_REPLICATION_FACTOR: i16 = 38; pub const ERROR_NOT_CONTROLLER: i16 = 41; pub const ERROR_INVALID_REQUEST: i16 = 42; -const MAX_SUPPORTED_METADATA_VERSION: i16 = 9; - /// Sentinel for `topic_authorized_operations` / `cluster_authorized_operations` when ACLs are not supported. const AUTHORIZED_OPS_UNKNOWN: i32 = i32::MIN; @@ -59,30 +57,13 @@ const AUTHORIZED_OPS_UNKNOWN: i32 = i32::MIN; pub enum HandleOutcome { /// Write this response body (with a response header). Respond(Bytes), - /// Write this response body (with a response header), then close the TCP connection. - RespondAndClose(Bytes), /// Produce with `acks=0`: write nothing, keep the connection open. NoResponse, - /// Client cannot parse an error at this request wire version; close the TCP connection. + /// No parseable response exists for this request; close the TCP connection. Close, } impl HandleOutcome { - /// Collapse to `Some(body)` for a normal response, or `None` for [`HandleOutcome::NoResponse`]. - /// - /// # Panics - /// - /// Panics on [`HandleOutcome::Close`] - match on `Close` explicitly, or use - /// [`Self::expect_response`] in tests that require a body. - #[must_use] - pub fn into_optional_response(self) -> Option { - match self { - Self::Respond(body) | Self::RespondAndClose(body) => Some(body), - Self::NoResponse => None, - Self::Close => panic!("HandleOutcome::Close has no response body"), - } - } - /// Return the response body, or panic with `msg` if the outcome is not [`Self::Respond`]. /// /// # Panics @@ -91,7 +72,7 @@ impl HandleOutcome { #[must_use] pub fn expect_response(self, msg: &str) -> Bytes { match self { - Self::Respond(body) | Self::RespondAndClose(body) => body, + Self::Respond(body) => body, Self::NoResponse => panic!("{msg}: got NoResponse"), Self::Close => panic!("{msg}: got Close"), } @@ -213,7 +194,22 @@ fn handle_produce_request(api_version: i16, body: Bytes) -> HandleOutcome { ); HandleOutcome::NoResponse } - ProduceDecodeResult::Err { error, .. } => { + ProduceDecodeResult::Err { acks: None, error } => { + // Decode failed before `acks` was read (malformed transactional_id or a truncated + // frame). Whether the client wants a response is unknowable, and an error response + // would desync an acks=0 fire-and-forget client's correlation stream. Frames are + // length-delimited, so the malformed frame does not affect the next frame's boundary; + // stay silent and keep the connection usable rather than risk that desync. + tracing::warn!( + "Failed to decode Produce request before acks was read (no response): {:?}", + error + ); + HandleOutcome::NoResponse + } + ProduceDecodeResult::Err { + acks: Some(_), + error, + } => { tracing::warn!("Failed to decode Produce request: {:?}", error); let code = if is_supported_version(API_KEY_PRODUCE, api_version) { ERROR_INVALID_REQUEST @@ -255,7 +251,7 @@ fn handle_other_request( // survives. Clients that skip ApiVersions get a naked close instead. tracing::warn!( api_version, - max_supported = MAX_SUPPORTED_METADATA_VERSION, + max_supported = supported_max_version(API_KEY_METADATA), "Metadata version unsupported; closing connection" ); HandleOutcome::Close @@ -322,7 +318,9 @@ fn handle_other_request( )) } } - _ => HandleOutcome::RespondAndClose(encode_error_only_response(ERROR_UNSUPPORTED_VERSION)), + // Unknown API key: no api-specific response schema exists, so any body we send is + // misparsed by the client against the schema it expected. Close is unambiguous. + _ => HandleOutcome::Close, } } @@ -334,6 +332,15 @@ pub fn is_supported_version(api_key: i16, api_version: i16) -> bool { .is_some_and(|r| api_version >= r.min_version && api_version <= r.max_version) } +/// Highest version this gateway accepts for `api_key`, from the single firewall table. +#[must_use] +pub fn supported_max_version(api_key: i16) -> Option { + SUPPORTED_RANGES + .iter() + .find(|r| r.api_key == api_key) + .map(|r| r.max_version) +} + /// Min version advertised in `ApiVersions` (may differ from the firewall min). /// /// Produce must advertise min=0 per KAFKA-18659 / `PRODUCE_API_VERSIONS_RESPONSE_MIN_VERSION` @@ -440,11 +447,10 @@ fn encode_metadata_response( } else { e.write_i32(1); // brokers array length e.write_i32(1); // node_id - // broker.host is config-derived (IGGY_KAFKA_ADVERTISED_HOST), not request-decoded - use - // the checked variant so an overly long hostname returns an error instead of panicking. - if e.write_nullable_string(Some(&broker.host)).is_err() { - return encode_error_only_response(ERROR_INVALID_REQUEST); - } + // broker.host is bounded to i16::MAX at BrokerAdvertise::from_server_config, so the + // unchecked writer cannot exceed the length prefix - same guarantee the flexible path + // relies on for its compact string. + e.write_nullable_string_unchecked(Some(&broker.host)); e.write_i32(broker.port); if response_version >= 1 { e.write_nullable_string_unchecked(None); // rack @@ -477,13 +483,6 @@ fn encode_metadata_response( e.freeze() } -#[must_use] -pub fn encode_error_only_response(error_code: i16) -> Bytes { - let mut e = Encoder::with_capacity(2); - e.write_i16(error_code); - e.freeze() -} - /// Decodes the requested topic names from a Metadata request body so the /// response can echo them back; clients match metadata by name, not position. /// diff --git a/gateways/kafka/src/protocol/codec.rs b/gateways/kafka/src/protocol/codec.rs index 7146f957d4..14afc1ae9c 100644 --- a/gateways/kafka/src/protocol/codec.rs +++ b/gateways/kafka/src/protocol/codec.rs @@ -43,19 +43,47 @@ pub const MAX_COLLECTION_LEN: usize = 65_536; /// still grow correctly via `Vec::push`'s amortized doubling once real elements are decoded. pub const PREALLOC_HINT: usize = 128; +/// Cumulative cap on the total array elements decoded from one request frame. +/// +/// `MAX_COLLECTION_LEN` bounds a single array, but not the product across nested arrays +/// (`topics` x `partitions`): an 8 MB frame packed with minimal partition entries decodes into +/// ~1M owned structs before any per-array limit trips. This budget bounds the sum of every array +/// count in one decode, so materialized struct count stays proportional to a fixed ceiling +/// regardless of nesting. Real requests never approach it (a Fetch over thousands of partitions +/// is still far below). +pub const MAX_REQUEST_ELEMENTS: usize = MAX_COLLECTION_LEN; + pub struct Decoder { bytes: Bytes, + element_budget: usize, } impl Decoder { pub fn new(bytes: Bytes) -> Self { - Self { bytes } + Self { + bytes, + element_budget: MAX_REQUEST_ELEMENTS, + } } pub fn remaining(&self) -> usize { self.bytes.remaining() } + /// Debit the cumulative element budget shared across every array in this decode. + fn charge_elements(&mut self, count: usize) -> Result<()> { + match self.element_budget.checked_sub(count) { + Some(remaining) => { + self.element_budget = remaining; + Ok(()) + } + None => Err(KafkaProtocolError::RequestElementBudgetExceeded { + count, + remaining: self.element_budget, + }), + } + } + pub fn read_u8(&mut self) -> Result { self.ensure(1)?; Ok(self.bytes.get_u8()) @@ -122,6 +150,7 @@ impl Decoder { max: MAX_COLLECTION_LEN, }); } + self.charge_elements(count)?; Ok(count) } @@ -146,6 +175,7 @@ impl Decoder { max: MAX_COLLECTION_LEN, }); } + self.charge_elements(count)?; Ok(count) } @@ -167,6 +197,7 @@ impl Decoder { max: MAX_COLLECTION_LEN, }); } + self.charge_elements(count)?; Ok(count) } @@ -197,6 +228,14 @@ impl Decoder { max: MAX_COLLECTION_LEN, } })?; + // Parity with the legacy i16-length string (naturally <= 32767); the compact form is + // otherwise bounded only by the frame, letting one flexible name reach max_frame_size. + if len > MAX_COLLECTION_LEN { + return Err(KafkaProtocolError::CollectionTooLarge { + count: len, + max: MAX_COLLECTION_LEN, + }); + } self.ensure(len)?; let s = std::str::from_utf8(&self.bytes.chunk()[..len]) .map_err(|_| KafkaProtocolError::InvalidUtf8)? diff --git a/gateways/kafka/src/protocol/responses.rs b/gateways/kafka/src/protocol/responses.rs index 902776c618..d435abc471 100644 --- a/gateways/kafka/src/protocol/responses.rs +++ b/gateways/kafka/src/protocol/responses.rs @@ -326,8 +326,9 @@ pub fn encode_create_topics_response(version: i16, req: &CreateTopicsRequest) -> /// /// KIP-464: on v4+, `num_partitions = -1` / `replication_factor = -1` mean broker default. /// Error 37 / 38 only for `0` and values `< -1` (and any non-positive value on v2–v3). -/// When validation passes, the stub returns [`ERROR_NOT_CONTROLLER`] so clients do not -/// believe the topic was created before the Iggy bridge exists. +/// When validation passes, the stub returns [`ERROR_NOT_CONTROLLER`] (including for +/// `validate_only`, which it cannot honestly resolve without a real controller) so clients do +/// not believe the topic was created before the Iggy bridge exists. const fn create_topics_topic_error(version: i16, topic: &CreatableTopic, forced_error: i16) -> i16 { if forced_error != ERROR_NONE { return forced_error; diff --git a/gateways/kafka/src/server.rs b/gateways/kafka/src/server.rs index 2e271824f1..68a9a2a999 100644 --- a/gateways/kafka/src/server.rs +++ b/gateways/kafka/src/server.rs @@ -19,7 +19,7 @@ use std::net::SocketAddr; use std::sync::Arc; use std::time::Duration; -use bytes::{BufMut, BytesMut}; +use bytes::{Buf, BufMut, Bytes, BytesMut}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; use tokio::sync::{Semaphore, broadcast}; @@ -28,10 +28,7 @@ use tokio_util::task::TaskTracker; use tracing::{debug, error, info, warn}; use crate::error::{KafkaProtocolError, Result}; -use crate::protocol::api::{ - BrokerAdvertise, DEFAULT_KAFKA_PORT, ERROR_INVALID_REQUEST, HandleOutcome, - encode_error_only_response, handle_request, -}; +use crate::protocol::api::{BrokerAdvertise, DEFAULT_KAFKA_PORT, HandleOutcome, handle_request}; use crate::protocol::codec::Decoder; use crate::protocol::header::{ RequestHeader, ResponseHeader, request_header_version, response_header_version, @@ -294,27 +291,12 @@ async fn handle_connection( let api_version = i16::from_be_bytes([frame[2], frame[3]]); let req_hdr_ver = request_header_version(api_key, api_version); let resp_hdr_ver = response_header_version(api_key, api_version); - let correlation_id = correlation_id_from_frame(&frame); + // `request_header_version` only ever returns 1 or 2, both of which `decode_from` + // handles, so its `UnsupportedHeaderVersion` arm is unreachable here; any decode error + // is a malformed header and closes the connection. let mut decoder = Decoder::new(frame); - let req = match RequestHeader::decode_from(&mut decoder, req_hdr_ver) { - Ok(req) => req, - Err(KafkaProtocolError::UnsupportedHeaderVersion(_)) => { - warn!(%peer, api_key, api_version, "unsupported request header version"); - let body_response = encode_error_only_response(ERROR_INVALID_REQUEST); - let resp_header = ResponseHeader { correlation_id }; - send_response( - &mut stream, - &resp_header, - 0, - &body_response, - config.write_timeout, - ) - .await?; - return Ok(()); - } - Err(e) => return Err(e), - }; + let req = RequestHeader::decode_from(&mut decoder, req_hdr_ver)?; debug!( %peer, @@ -327,7 +309,6 @@ async fn handle_connection( let body = decoder.read_bytes(decoder.remaining())?; let outcome = handle_request(req.api_key, req.api_version, body, &broker); - let close_after_response = matches!(outcome, HandleOutcome::RespondAndClose(_)); match outcome { HandleOutcome::NoResponse => { // Produce with acks=0: the wire protocol forbids a response. @@ -337,12 +318,11 @@ async fn handle_connection( %peer, api_key = req.api_key, api_version = req.api_version, - "closing connection: no parseable error response for this request version" + "closing connection: no parseable response for this request" ); return Ok(()); } - HandleOutcome::Respond(body_response) - | HandleOutcome::RespondAndClose(body_response) => { + HandleOutcome::Respond(body_response) => { let resp_header = ResponseHeader { correlation_id: req.correlation_id, }; @@ -350,31 +330,25 @@ async fn handle_connection( &mut stream, &resp_header, resp_hdr_ver, - &body_response, + body_response, config.write_timeout, ) .await?; - if close_after_response { - warn!( - %peer, - api_key = req.api_key, - api_version = req.api_version, - "closing connection after unsupported-version error response" - ); - return Ok(()); - } } } } } -/// Write a single length-prefixed Kafka frame using one allocation. -/// Avoids the separate header-encode + payload-concat + length-prefix allocations. +/// Write a single length-prefixed Kafka frame. +/// +/// The length prefix and header are built in a small buffer, then chained with the already-owned +/// `body` so `write_all_buf` streams both without copying the body. The whole frame is written +/// before returning (or the connection errors), so no partial frame ever precedes the next one. async fn send_response( stream: &mut TcpStream, header: &ResponseHeader, header_version: i16, - body: &[u8], + body: Bytes, write_timeout: Duration, ) -> Result<()> { let header_size = ResponseHeader::encoded_size(header_version); @@ -384,20 +358,16 @@ async fn send_response( max_bytes: i32::MAX as usize, actual_bytes: payload_size, })?; - let mut frame = BytesMut::with_capacity(4 + payload_size); - frame.put_i32(payload_len_i32); - header.encode_into(&mut frame, header_version); - frame.put_slice(body); - timeout(write_timeout, stream.write_all(&frame)) + let mut prefix = BytesMut::with_capacity(4 + header_size); + prefix.put_i32(payload_len_i32); + header.encode_into(&mut prefix, header_version); + let mut frame = prefix.freeze().chain(body); + timeout(write_timeout, stream.write_all_buf(&mut frame)) .await .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "write timeout"))??; Ok(()) } -fn correlation_id_from_frame(frame: &bytes::Bytes) -> i32 { - i32::from_be_bytes([frame[4], frame[5], frame[6], frame[7]]) -} - /// Read one length-prefixed Kafka frame from `stream`. /// /// # Errors @@ -506,13 +476,6 @@ mod tests { ))); } - #[test] - fn correlation_id_is_extracted_from_frame() { - let frame = - bytes::Bytes::from_static(&[0x00, 0x12, 0x00, 0x01, 0x11, 0x22, 0x33, 0x44, 0xaa]); - assert_eq!(correlation_id_from_frame(&frame), 0x1122_3344); - } - #[tokio::test] async fn send_response_writes_header_and_body() { let (mut client, mut server) = tcp_pair().await; @@ -521,9 +484,15 @@ mod tests { }; let body = [9u8, 8, 7]; - send_response(&mut server, &header, 1, &body, Duration::from_secs(1)) - .await - .unwrap(); + send_response( + &mut server, + &header, + 1, + Bytes::copy_from_slice(&body), + Duration::from_secs(1), + ) + .await + .unwrap(); let mut len = [0u8; 4]; client.read_exact(&mut len).await.unwrap(); @@ -764,9 +733,15 @@ mod tests { }; let body = [5u8, 6, 7]; - send_response(&mut server, &header, 0, &body, Duration::from_secs(1)) - .await - .unwrap(); + send_response( + &mut server, + &header, + 0, + Bytes::copy_from_slice(&body), + Duration::from_secs(1), + ) + .await + .unwrap(); let mut len = [0u8; 4]; client.read_exact(&mut len).await.unwrap(); diff --git a/gateways/kafka/tests/api_handler_tests.rs b/gateways/kafka/tests/api_handler_tests.rs index 23ed04df02..f30a8a7e66 100644 --- a/gateways/kafka/tests/api_handler_tests.rs +++ b/gateways/kafka/tests/api_handler_tests.rs @@ -138,11 +138,12 @@ fn unsupported_metadata_version_closes_connection() { // ── Misc ──────────────────────────────────────────────────────────────────── #[test] -fn unknown_api_key_returns_error_only_payload() { - let body = handle_request(999, 0, Bytes::new(), &default_broker()) - .expect_response("test request has acks != 0 and expects a response"); - let mut d = Decoder::new(body); - assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); +fn unknown_api_key_closes_connection() { + let outcome = handle_request(999, 0, Bytes::new(), &default_broker()); + assert!( + outcome.is_close(), + "unknown api_key must close (no parseable response schema)" + ); } #[test] diff --git a/gateways/kafka/tests/common/tcp.rs b/gateways/kafka/tests/common/tcp.rs index bd75a13483..cf439bd6cd 100644 --- a/gateways/kafka/tests/common/tcp.rs +++ b/gateways/kafka/tests/common/tcp.rs @@ -119,6 +119,27 @@ pub fn build_produce_v3_body(acks: i16, topics_count: i32) -> Bytes { body.freeze() } +/// Minimal Produce v0-v2 body (no `transactional_id`): acks, timeout, empty topics array. +pub fn build_produce_v2_body(acks: i16, topics_count: i32) -> Bytes { + let mut body = BytesMut::new(); + body.put_i16(acks); + body.put_i32(1_000); // timeout_ms + body.put_i32(topics_count); + body.freeze() +} + +/// Minimal flexible Produce body (v9+): null compact `transactional_id`, acks, timeout, +/// compact topics array, empty tagged fields. +pub fn build_produce_flexible_body(acks: i16, topics_count: u32) -> Bytes { + let mut body = BytesMut::new(); + body.put_u8(0); // null transactional_id (compact string, varint 0) + body.put_i16(acks); + body.put_i32(1_000); // timeout_ms + body.put_u8(u8::try_from(topics_count + 1).expect("small topic count")); // compact array len + body.put_u8(0); // empty tagged fields + body.freeze() +} + /// `ListOffsets` v0 request body for topic "t", partition 0. pub fn build_list_offsets_v0_request_with_topic_t() -> Bytes { let mut body = BytesMut::new(); diff --git a/gateways/kafka/tests/listener_robustness_tests.rs b/gateways/kafka/tests/listener_robustness_tests.rs index ac3309f48c..c911a1aec3 100644 --- a/gateways/kafka/tests/listener_robustness_tests.rs +++ b/gateways/kafka/tests/listener_robustness_tests.rs @@ -501,12 +501,16 @@ async fn corrupt_produce_body_e2e_returns_error_without_disconnect() { let (addr, _shutdown) = spawn_test_server().await; let mut stream = TcpStream::connect(addr).await.expect("connect"); + // acks is readable (=1), so the client expects an error response; the topics array is + // truncated, forcing INVALID_REQUEST. let bad = build_request_frame( API_KEY_PRODUCE, 3, 391, Some("scope-test"), - &[0xFF, 0xFF, 0xFF], + &[ + 0xFF, 0xFF, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, + ], ); stream.write_all(&bad).await.expect("corrupt produce"); let payload = read_response_frame(&mut stream, 8 * 1024 * 1024).await; diff --git a/gateways/kafka/tests/server_e2e_tests.rs b/gateways/kafka/tests/server_e2e_tests.rs index 13f237961e..faf90246da 100644 --- a/gateways/kafka/tests/server_e2e_tests.rs +++ b/gateways/kafka/tests/server_e2e_tests.rs @@ -32,7 +32,7 @@ use tokio::net::TcpStream; use iggy_gateway_kafka::protocol::api::{ API_KEY_API_VERSIONS, API_KEY_CREATE_TOPICS, API_KEY_FETCH, API_KEY_LIST_OFFSETS, - API_KEY_METADATA, API_KEY_PRODUCE, ERROR_NOT_LEADER_OR_FOLLOWER, ERROR_UNSUPPORTED_VERSION, + API_KEY_METADATA, API_KEY_PRODUCE, ERROR_NOT_LEADER_OR_FOLLOWER, }; use iggy_gateway_kafka::protocol::codec::Decoder; @@ -95,23 +95,19 @@ async fn e2e_produce_v3_round_trip_with_fixture() { } #[tokio::test] -async fn e2e_unsupported_api_key_returns_error_then_closes() { +async fn e2e_unsupported_api_key_closes_connection() { let (addr, _shutdown) = spawn_test_server().await; let mut stream = TcpStream::connect(addr).await.unwrap(); + // Unknown api key (8, OffsetCommit) has no response schema this gateway can encode, so the + // server closes the connection without a (misparseable) response body. let frame1 = build_request_frame(8, 2, 99, Some("e2e-test"), &[]); stream.write_all(&frame1).await.unwrap(); - let payload1 = read_response_frame(&mut stream, 8 * 1024 * 1024).await; - let (corr, body) = parse_response_payload(8, 2, payload1); - assert_eq!(corr, 99); - let mut d = Decoder::new(body); - assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); - // The unsupported-version error is terminal: the server closes the connection. assert_eq!( read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await, ByteRead::Closed, - "connection must close after the unsupported-version error response" + "unknown api key must close the connection without a response" ); } @@ -447,7 +443,7 @@ async fn metadata_empty_body_e2e_all_topics_request_returns_broker() { // ── Out-of-scope API keys (SCOPE.md unsupported list) ─────────────────────── #[tokio::test] -async fn out_of_scope_api_keys_e2e_respond_then_close() { +async fn out_of_scope_api_keys_e2e_close() { let (addr, _shutdown) = spawn_test_server().await; for &(api_key, name) in &OUT_OF_SCOPE_API_KEYS[..4] { @@ -455,18 +451,11 @@ async fn out_of_scope_api_keys_e2e_respond_then_close() { let frame = build_request_frame(api_key, 0, i32::from(api_key), Some("scope-test"), &[]); stream.write_all(&frame).await.expect("write oos key"); - let payload = read_response_frame(&mut stream, 8 * 1024 * 1024).await; - let mut d = Decoder::new(parse_response_payload(api_key, 0, payload).1); - assert_eq!( - d.read_i16().unwrap(), - ERROR_UNSUPPORTED_VERSION, - "{name} (key {api_key})" - ); - + // No response schema exists for an out-of-scope key: the server closes without a body. assert_eq!( read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await, ByteRead::Closed, - "{name} (key {api_key}) must close the connection after the error response" + "{name} (key {api_key}) must close the connection without a response" ); } } diff --git a/gateways/kafka/tests/version_firewall_tests.rs b/gateways/kafka/tests/version_firewall_tests.rs index a77574fece..0229a6ac41 100644 --- a/gateways/kafka/tests/version_firewall_tests.rs +++ b/gateways/kafka/tests/version_firewall_tests.rs @@ -47,8 +47,8 @@ use scope::{SCOPED_API_KEYS, default_broker}; use server::spawn_test_server; use tcp::{ ByteRead, build_list_offsets_v0_request_with_topic_t, build_metadata_legacy_request, - build_produce_v3_body, build_request_frame, parse_response_payload, read_byte_with_timeout, - round_trip, scan_for_error_code, + build_produce_flexible_body, build_produce_v2_body, build_produce_v3_body, build_request_frame, + parse_response_payload, read_byte_with_timeout, round_trip, scan_for_error_code, }; use wire::build_metadata_flexible_request_v10; use wire::{ @@ -223,8 +223,13 @@ async fn e2e_metadata_above_max_version_closes_tcp_connection() { #[test] fn produce_unsupported_version_returns_well_formed_error_response() { - let body = handle_request(API_KEY_PRODUCE, 2, Bytes::new(), &default_broker()) - .expect_response("test request has acks != 0 and expects a response"); + let body = handle_request( + API_KEY_PRODUCE, + 2, + build_produce_v2_body(1, 0), + &default_broker(), + ) + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 1); assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); @@ -293,15 +298,12 @@ fn create_topics_unsupported_version_returns_well_formed_error_response() { } #[test] -fn unsupported_api_keys_return_error_only() { +fn unsupported_api_keys_close_connection() { for key in [8, 9, 10, 11, 17, 20, 42, 999] { - let body = handle_request(key, 0, Bytes::new(), &default_broker()) - .expect_response("test request has acks != 0 and expects a response"); - let mut d = Decoder::new(body); - assert_eq!( - d.read_i16().unwrap(), - ERROR_UNSUPPORTED_VERSION, - "api_key {key}" + let outcome = handle_request(key, 0, Bytes::new(), &default_broker()); + assert!( + outcome.is_close(), + "unknown api_key {key} must close (no parseable response schema)" ); } } @@ -332,7 +334,14 @@ fn supported_fetch_versions_accept_valid_fixture() { #[test] fn corrupt_produce_body_returns_invalid_request_error() { - let body = Bytes::from_static(&[0xFF, 0xFF, 0xFF]); + // null transactional_id, acks=1, timeout=0, then a truncated topics array: acks is readable, + // so the client expects (and gets) an error response. + let body = Bytes::from_static(&[ + 0xFF, 0xFF, // null transactional_id + 0x00, 0x01, // acks = 1 + 0x00, 0x00, 0x00, 0x00, // timeout_ms = 0 + 0xFF, 0xFF, 0xFF, // truncated topics count + ]); let resp = handle_request(API_KEY_PRODUCE, 3, body, &default_broker()) .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(resp); @@ -343,6 +352,18 @@ fn corrupt_produce_body_returns_invalid_request_error() { assert_eq!(d.read_i16().unwrap(), ERROR_INVALID_REQUEST); } +#[test] +fn corrupt_produce_body_before_acks_is_silent() { + // Decode fails before acks is read: the client's response expectation is unknowable, and an + // error response could desync an acks=0 fire-and-forget client, so the server stays silent. + let body = Bytes::from_static(&[0xFF, 0xFF]); // null transactional_id, then EOF + let outcome = handle_request(API_KEY_PRODUCE, 3, body, &default_broker()); + assert!( + outcome.is_no_response(), + "produce decode failure before acks must be silent" + ); +} + #[test] fn corrupt_fetch_body_returns_invalid_request_error() { let body = Bytes::from_static(&[0xFF, 0xFF, 0xFF]); @@ -440,8 +461,12 @@ fn request_body_for_scoped_api(api_key: i16, name: &str, version: i16) -> Bytes API_KEY_PRODUCE => { if fixture_exists(api_key, name, version) { load_fixture_body(api_key, name, version) - } else { + } else if version >= 9 { + build_produce_flexible_body(1, 0) + } else if version >= 3 { build_produce_v3_body(1, 0) + } else { + build_produce_v2_body(1, 0) } } API_KEY_FETCH => { @@ -535,16 +560,10 @@ async fn apiversions_v4_out_of_range_e2e_returns_unsupported() { // ── Out-of-scope API keys (SCOPE.md unsupported list) ─────────────────────── #[test] -fn out_of_scope_api_keys_return_unsupported_version_without_panic() { +fn out_of_scope_api_keys_close_without_panic() { for &(api_key, name) in OUT_OF_SCOPE_API_KEYS { - let body = handle_request(api_key, 0, Bytes::new(), &default_broker()) - .expect_response("test request has acks != 0 and expects a response"); - let mut d = Decoder::new(body); - assert_eq!( - d.read_i16().unwrap(), - ERROR_UNSUPPORTED_VERSION, - "{name} (key {api_key})" - ); + let outcome = handle_request(api_key, 0, Bytes::new(), &default_broker()); + assert!(outcome.is_close(), "{name} (key {api_key}) must close"); } } @@ -557,12 +576,15 @@ async fn each_scoped_api_above_max_version_e2e_keeps_connection() { for &(api_key, name, _min_ver, max_ver) in SCOPED_API_KEYS { let above = max_ver + 1; + // Produce is decoded before the firewall check, so it needs a body with a readable acks + // (built for the flexible wire version); the other APIs reject on version pre-decode. + let body = request_body_for_scoped_api(api_key, name, above); let frame = build_request_frame( api_key, above, 50_000 + i32::from(api_key), Some("scope-test"), - &[], + &body, ); stream .write_all(&frame) @@ -603,12 +625,15 @@ async fn each_scoped_api_below_min_version_e2e_keeps_connection() { for &(api_key, name, min_ver, _max_ver) in SCOPED_API_KEYS { let below = min_ver - 1; + // Produce is decoded before the firewall check (to honor acks=0 silence), so it needs a + // body with a readable acks; the other APIs reject on version before touching the body. + let body = request_body_for_scoped_api(api_key, name, below); let frame = build_request_frame( api_key, below, 40_000 + i32::from(api_key), Some("scope-test"), - &[], + &body, ); stream .write_all(&frame) @@ -653,8 +678,13 @@ fn produce_advertises_min_zero_but_firewall_rejects_below_v3() { assert!(!is_supported_version(API_KEY_PRODUCE, 0)); assert!(!is_supported_version(API_KEY_PRODUCE, 2)); - let body = handle_request(API_KEY_PRODUCE, 2, Bytes::new(), &default_broker()) - .expect_response("test request has acks != 0 and expects a response"); + let body = handle_request( + API_KEY_PRODUCE, + 2, + build_produce_v2_body(1, 0), + &default_broker(), + ) + .expect_response("test request has acks != 0 and expects a response"); let mut d = Decoder::new(body); let _topics = d.read_i32().unwrap(); let _name = d.read_nullable_string().unwrap(); From f76517d519490527fa86c3755e71fbced64bf106 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sun, 9 Aug 2026 03:11:43 -0400 Subject: [PATCH 49/57] Documentation updated --- gateways/kafka/README.md | 5 ++++- gateways/kafka/docs/SCOPE.md | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/gateways/kafka/README.md b/gateways/kafka/README.md index ead41c2fbf..b71f3c9a27 100644 --- a/gateways/kafka/README.md +++ b/gateways/kafka/README.md @@ -18,7 +18,10 @@ Default bind: `127.0.0.1:9093`. Environment variables: | `IGGY_KAFKA_ADVERTISED_HOST` | bind IP | Hostname/IP clients use to reach this broker (required when binding to `0.0.0.0`/`::`) | | `IGGY_KAFKA_ADVERTISED_PORT` | bind port | Port advertised in Metadata responses | | `IGGY_KAFKA_MAX_CONNECTIONS` | `1024` | Maximum concurrent connections before new ones are rejected | +| `IGGY_KAFKA_MAX_FRAME_SIZE` | `8388608` | Maximum accepted request frame size in bytes | | `IGGY_KAFKA_IDLE_TIMEOUT_SECS` | `600` | Seconds a connection may sit idle before the next frame's length prefix arrives | +| `IGGY_KAFKA_READ_TIMEOUT_SECS` | `15` | Seconds allowed to read a frame body once its length prefix arrives | +| `IGGY_KAFKA_WRITE_TIMEOUT_SECS` | `10` | Seconds allowed to write a response frame | | `IGGY_KAFKA_SHUTDOWN_DRAIN_TIMEOUT_SECS` | `25` | Seconds graceful shutdown waits for in-flight connections before abandoning them | ## Test @@ -27,7 +30,7 @@ Default bind: `127.0.0.1:9093`. Environment variables: cargo test -p iggy-gateway-kafka ``` -235 regression tests across 12 suites — see [docs/TEST_SUITE.md](docs/TEST_SUITE.md) for the full catalog. +259 regression tests across 12 suites — see [docs/TEST_SUITE.md](docs/TEST_SUITE.md) for the full catalog. `decode_validation_tests` require wire fixtures under `tools/kafka-tool/kafka_messages/` (gitignored locally; CI generates them via `scripts/ci-wire-fixtures.sh`): diff --git a/gateways/kafka/docs/SCOPE.md b/gateways/kafka/docs/SCOPE.md index 4e7417073b..6dcae38b6f 100644 --- a/gateways/kafka/docs/SCOPE.md +++ b/gateways/kafka/docs/SCOPE.md @@ -33,7 +33,7 @@ Expand `SUPPORTED_RANGES` only after a key/version pair is manually tested. ApiV | API key | Name | Min version | Max version | Valid versions | Behavior | | --------- | ------ | ------------- | ------------- | ---------------- | ---------- | | 18 | ApiVersions | 0 | 3 | 0, 1, 2, 3 | Advertise supported ranges; flexible encoding at v3+ | -| 3 | Metadata | 0 | 9 | 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 | Decode topic list count; stub broker from `ServerConfig.bind_addr`; flexible encoding at v9+ | +| 3 | Metadata | 0 | 9 | 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 | Decode topic list count; stub broker host from `advertised_host` or the bound `local_addr` IP; flexible encoding at v9+ | | 0 | Produce | 3 | 9 | 3, 4, 5, 6, 7, 8, 9 | Decode request; stub returns `NOT_LEADER_OR_FOLLOWER` (6) | | 1 | Fetch | 4 | 12 | 4, 5, 6, 7, 8, 9, 10, 11, 12 | Decode request; stub response | | 2 | ListOffsets | 1 | 6 | 1, 2, 3, 4, 5, 6 | Decode request; stub response | From ed268d3d859b554c03589ef32fee890c4941536f Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sun, 9 Aug 2026 03:47:29 -0400 Subject: [PATCH 50/57] Fixing merge conflicts --- gateways/kafka/src/main.rs | 2 +- gateways/kafka/src/protocol/api.rs | 25 +++++++++++++------------ gateways/kafka/src/protocol/codec.rs | 9 ++++++--- gateways/kafka/src/server.rs | 17 ++++------------- 4 files changed, 24 insertions(+), 29 deletions(-) diff --git a/gateways/kafka/src/main.rs b/gateways/kafka/src/main.rs index bd37bf458c..d2b41a6264 100644 --- a/gateways/kafka/src/main.rs +++ b/gateways/kafka/src/main.rs @@ -109,7 +109,7 @@ fn env_var(key: &str) -> Option { } /// Parse a strictly-positive value, rejecting `0` (which for connection caps and timeouts would -/// silently disable the listener) and unparseable input. +/// silently disable the listener) and unparsable input. fn parse_positive(key: &str, raw: &str) -> Result where T: FromStr + Default + PartialEq, diff --git a/gateways/kafka/src/protocol/api.rs b/gateways/kafka/src/protocol/api.rs index b7fa6774df..bccab2b47a 100644 --- a/gateways/kafka/src/protocol/api.rs +++ b/gateways/kafka/src/protocol/api.rs @@ -171,6 +171,12 @@ pub fn handle_request( /// versions before reading `acks` would send an error response the client never expects, /// desyncing the next correlation id it reads. fn handle_produce_request(api_version: i16, body: Bytes) -> HandleOutcome { + // Above the encoder max there is no response parseable at the client's version, so close + // rather than decode (same policy the other APIs apply). Below-min versions still decode so + // acks=0 fire-and-forget clients stay silent per the advertised min=0. + if api_version > supported_max_version(API_KEY_PRODUCE).unwrap_or(i16::MAX) { + return HandleOutcome::Close; + } match decode_produce_request(api_version, body) { // acks=0 is fire-and-forget: the client isn't reading a response, so // sending one desyncs the next correlation id it expects. @@ -260,7 +266,9 @@ fn handle_other_request( encode_create_topics_error_response, "CreateTopics", ), - _ => HandleOutcome::RespondAndClose(encode_error_only_response(ERROR_UNSUPPORTED_VERSION)), + // Unknown API key: no api-specific response schema exists, so any body we send is + // misparsed by the client against the schema it expected. Close is unambiguous. + _ => HandleOutcome::Close, } } @@ -284,12 +292,12 @@ fn handle_api_versions(api_version: i16, body: &Bytes) -> HandleOutcome { fn handle_metadata(api_version: i16, body: Bytes, broker: &BrokerAdvertise) -> HandleOutcome { if !is_supported_version(API_KEY_METADATA, api_version) { - // Clamping the response to MAX_SUPPORTED_METADATA_VERSION leaves a body the - // client parses at its own (unsupported) version, so UNSUPPORTED_VERSION never - // survives. Clients that skip ApiVersions get a naked close instead. + // Clamping the response to the supported max leaves a body the client parses at its own + // (unsupported) version, so UNSUPPORTED_VERSION never survives. Clients that skip + // ApiVersions get a naked close instead. tracing::warn!( api_version, - max_supported = MAX_SUPPORTED_METADATA_VERSION, + max_supported = supported_max_version(API_KEY_METADATA), "Metadata version unsupported; closing connection" ); return HandleOutcome::Close; @@ -515,13 +523,6 @@ fn encode_metadata_response( e.freeze() } -#[must_use] -pub fn encode_error_only_response(error_code: i16) -> Bytes { - let mut e = Encoder::with_capacity(2); - e.write_i16(error_code); - e.freeze() -} - /// Decodes an `ApiVersions` request body. /// /// v0–v2 have an empty body. v3+ requires `ClientSoftwareName`, `ClientSoftwareVersion`, diff --git a/gateways/kafka/src/protocol/codec.rs b/gateways/kafka/src/protocol/codec.rs index 3d3cac24ea..2fb8915af1 100644 --- a/gateways/kafka/src/protocol/codec.rs +++ b/gateways/kafka/src/protocol/codec.rs @@ -188,7 +188,9 @@ impl Decoder { if n == 0 { return Err(KafkaProtocolError::NullCompactArray); } - Self::compact_array_count_from_len_plus_one(n) + let count = Self::compact_array_count_from_len_plus_one(n)?; + self.charge_elements(count)?; + Ok(count) } /// Compact array length for a **nullable** field: varint `0` means null/absent and is @@ -199,7 +201,9 @@ impl Decoder { if n == 0 { return Ok(0); } - Self::compact_array_count_from_len_plus_one(n) + let count = Self::compact_array_count_from_len_plus_one(n)?; + self.charge_elements(count)?; + Ok(count) } fn compact_array_count_from_len_plus_one(n: u64) -> Result { @@ -213,7 +217,6 @@ impl Decoder { max: MAX_COLLECTION_LEN, }); } - self.charge_elements(count)?; Ok(count) } diff --git a/gateways/kafka/src/server.rs b/gateways/kafka/src/server.rs index 8de3a7f07c..8a05215594 100644 --- a/gateways/kafka/src/server.rs +++ b/gateways/kafka/src/server.rs @@ -349,7 +349,6 @@ async fn dispatch_outcome( resp_hdr_ver: i16, outcome: HandleOutcome, ) -> Result { - let close_after_response = matches!(outcome, HandleOutcome::RespondAndClose(_)); match outcome { HandleOutcome::NoResponse => { // Produce with acks=0: the wire protocol forbids a response. @@ -360,11 +359,11 @@ async fn dispatch_outcome( %peer, api_key = req.api_key, api_version = req.api_version, - "closing connection: no parseable error response for this request version" + "closing connection: no parseable response for this request" ); Ok(true) } - HandleOutcome::Respond(body_response) | HandleOutcome::RespondAndClose(body_response) => { + HandleOutcome::Respond(body_response) => { let resp_header = ResponseHeader { correlation_id: req.correlation_id, }; @@ -372,19 +371,11 @@ async fn dispatch_outcome( stream, &resp_header, resp_hdr_ver, - &body_response, + body_response, config.write_timeout, ) .await?; - if close_after_response { - warn!( - %peer, - api_key = req.api_key, - api_version = req.api_version, - "closing connection after unsupported-version error response" - ); - } - Ok(close_after_response) + Ok(false) } } } From 61e0feeb3b09eb198677c69ae7985df5cebcf4d5 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Sun, 9 Aug 2026 07:40:09 -0400 Subject: [PATCH 51/57] Updating documentation to retrigger ci/cd --- gateways/kafka/docs/SCOPE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gateways/kafka/docs/SCOPE.md b/gateways/kafka/docs/SCOPE.md index 6c255621be..fa7995e353 100644 --- a/gateways/kafka/docs/SCOPE.md +++ b/gateways/kafka/docs/SCOPE.md @@ -117,5 +117,5 @@ Items from the [hybrid architecture review](https://github.com/apache/iggy/discu ### Open questions (ask maintainers before Phase 2) -- [ ] Repo placement: `gateways/kafka/` in [apache/iggy](https://github.com/apache/iggy) vs separate proxy repo (affects workspace deps and CI) +- [X] Repo placement: `gateways/kafka/` in [apache/iggy](https://github.com/apache/iggy) vs separate proxy repo (affects workspace deps and CI) - [ ] Confirm bridge dependency strategy ([Discussion #3081](https://github.com/apache/iggy/discussions/3081), [#3252](https://github.com/apache/iggy/discussions/3252)) From ba1f4dc8b2b5a22266e9c6de1f28f59d03312e5f Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Wed, 12 Aug 2026 05:46:01 -0400 Subject: [PATCH 52/57] Initial version -vMigrate Kafka gateway codec to kafka-protocol MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaced the gateway’s hand-rolled Kafka wire codec, request decoders, and header encode/decode logic with `kafka_protocol` (broker feature only), and removed the old `protocol/codec.rs` and `protocol/requests.rs` modules. Response encoding now goes through shared `Encodable` paths with explicit malformed/encode error mapping, and unsupported versions that cannot be encoded at the requested wire version now close the connection instead of emitting downgraded responses. This also updates Produce handling around decode failures and pre-v3 compatibility (including acks=0 silence behavior), switches header version selection to `ApiKey` metadata from `kafka_protocol`, and rewrites tests to use a test-only local wire helper codec plus new expectations for close/silent paths under the version firewall. --- gateways/kafka/Cargo.toml | 5 +- gateways/kafka/src/error.rs | 28 +- gateways/kafka/src/protocol/api.rs | 520 +++++--------- gateways/kafka/src/protocol/codec.rs | 472 ------------ gateways/kafka/src/protocol/header.rs | 222 +----- gateways/kafka/src/protocol/mod.rs | 2 - gateways/kafka/src/protocol/requests.rs | 545 -------------- gateways/kafka/src/protocol/responses.rs | 541 ++++++-------- gateways/kafka/src/server.rs | 63 +- gateways/kafka/tests/api_handler_tests.rs | 32 +- .../kafka/tests/broker_advertise_tests.rs | 6 +- gateways/kafka/tests/codec_tests.rs | 230 ------ gateways/kafka/tests/common/codec.rs | 253 +++++++ gateways/kafka/tests/common/fixtures.rs | 14 +- gateways/kafka/tests/common/tcp.rs | 18 +- gateways/kafka/tests/common/wire.rs | 6 +- gateways/kafka/tests/decode_safety_tests.rs | 347 --------- .../kafka/tests/decode_validation_tests.rs | 678 ------------------ .../kafka/tests/golden_wire_fixtures_tests.rs | 6 +- gateways/kafka/tests/header_tests.rs | 172 +---- .../kafka/tests/listener_robustness_tests.rs | 25 +- .../kafka/tests/response_negative_tests.rs | 233 +++--- gateways/kafka/tests/server_e2e_tests.rs | 36 +- .../kafka/tests/server_integration_tests.rs | 5 +- .../kafka/tests/version_firewall_tests.rs | 262 +++---- gateways/kafka/tools/kafka-tool/src/main.rs | 33 +- 26 files changed, 1043 insertions(+), 3711 deletions(-) delete mode 100644 gateways/kafka/src/protocol/codec.rs delete mode 100644 gateways/kafka/src/protocol/requests.rs delete mode 100644 gateways/kafka/tests/codec_tests.rs create mode 100644 gateways/kafka/tests/common/codec.rs delete mode 100644 gateways/kafka/tests/decode_safety_tests.rs delete mode 100644 gateways/kafka/tests/decode_validation_tests.rs diff --git a/gateways/kafka/Cargo.toml b/gateways/kafka/Cargo.toml index 326b252c39..9123bab575 100644 --- a/gateways/kafka/Cargo.toml +++ b/gateways/kafka/Cargo.toml @@ -34,6 +34,10 @@ path = "src/main.rs" [dependencies] bytes = { workspace = true } +# Broker-role only: decodes requests and encodes responses. Default features also pull in +# client-role codec paths and compression codecs (gzip/lz4/snappy/zstd) this gateway never +# uses, since RecordBatch payloads stay opaque `Bytes` here. +kafka-protocol = { version = "0.17", default-features = false, features = ["broker"] } socket2 = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true, features = [ @@ -50,7 +54,6 @@ tracing = { workspace = true } tracing-subscriber = { workspace = true } [dev-dependencies] -kafka-protocol = "0.17" tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "io-util", "time"] } [lints.clippy] diff --git a/gateways/kafka/src/error.rs b/gateways/kafka/src/error.rs index f8b5585fd1..5ae88937c0 100644 --- a/gateways/kafka/src/error.rs +++ b/gateways/kafka/src/error.rs @@ -30,30 +30,14 @@ pub enum KafkaProtocolError { max_bytes: usize, actual_bytes: usize, }, - #[error("invalid utf8 string")] - InvalidUtf8, - #[error("varint overflows 64 bits")] - InvalidVarint, - #[error("unsupported request header version: {0}")] - UnsupportedHeaderVersion(i16), - #[error("invalid array length: {0}")] - InvalidArrayLength(i32), - #[error("collection length {count} exceeds maximum {max}")] - CollectionTooLarge { count: usize, max: usize }, - #[error("request element budget exceeded: {count} more requested, {remaining} remaining")] - RequestElementBudgetExceeded { count: usize, remaining: usize }, - #[error("string length {length} exceeds i16::MAX")] - StringTooLong { length: usize }, + /// Metadata's per-topic `name` is nullable on the wire (v10+ allows topic-id-only lookups), + /// but this stub only echoes names back, so a null name has no response to build. #[error("null topic name in request")] NullTopicName, - /// Compact-array prefix `0` is Kafka's null encoding; invalid for non-nullable arrays. - #[error("null compact array where a non-null array is required")] - NullCompactArray, - /// Compact-string prefix `0` is null; invalid where a non-null string is required. - #[error("null compact string where a non-null string is required")] - NullCompactString, - #[error("unexpected bytes remaining in request body")] - UnexpectedTrailingBytes, + /// Wraps a `kafka_protocol` decode/encode failure (`anyhow::Error`, no stable variant + /// taxonomy on the wire-format crate's side). + #[error("malformed request: {0}")] + Malformed(String), #[error("io error: {0}")] Io(#[from] std::io::Error), } diff --git a/gateways/kafka/src/protocol/api.rs b/gateways/kafka/src/protocol/api.rs index bccab2b47a..353e98384a 100644 --- a/gateways/kafka/src/protocol/api.rs +++ b/gateways/kafka/src/protocol/api.rs @@ -15,18 +15,21 @@ // specific language governing permissions and limitations // under the License. -use bytes::Bytes; +use bytes::{Buf, Bytes}; +use kafka_protocol::messages::api_versions_response::ApiVersion; +use kafka_protocol::messages::metadata_response::{MetadataResponseBroker, MetadataResponseTopic}; +use kafka_protocol::messages::{ + ApiVersionsRequest, ApiVersionsResponse, BrokerId, CreateTopicsRequest, FetchRequest, + ListOffsetsRequest, MetadataRequest, MetadataResponse, ProduceRequest, TopicName, +}; +use kafka_protocol::protocol::{Decodable, StrBytes}; use crate::error::{KafkaProtocolError, Result}; -use crate::protocol::codec::{Decoder, Encoder, PREALLOC_HINT}; -use crate::protocol::requests::{ - ProduceDecodeResult, decode_create_topics_request, decode_fetch_request, - decode_list_offsets_request, decode_produce_request, -}; use crate::protocol::responses::{ encode_create_topics_error_response, encode_create_topics_response, encode_fetch_error_response, encode_fetch_response, encode_list_offsets_error_response, - encode_list_offsets_response, encode_produce_error_response, encode_produce_response, + encode_list_offsets_response, encode_message, encode_produce_error_response, + encode_produce_response, }; pub const API_KEY_PRODUCE: i16 = 0; @@ -49,9 +52,6 @@ pub const ERROR_INVALID_REPLICATION_FACTOR: i16 = 38; pub const ERROR_NOT_CONTROLLER: i16 = 41; pub const ERROR_INVALID_REQUEST: i16 = 42; -/// Sentinel for `topic_authorized_operations` / `cluster_authorized_operations` when ACLs are not supported. -const AUTHORIZED_OPS_UNKNOWN: i32 = i32::MIN; - /// Result of handling one Kafka request body. #[derive(Debug)] pub enum HandleOutcome { @@ -162,71 +162,89 @@ pub fn handle_request( handle_other_request(api_key, api_version, body, broker) } +/// Decode `T` from the whole request body and reject unconsumed trailing bytes. +/// +/// `kafka_protocol`'s `Decodable` stops once it has read the fields its schema defines; it does +/// not know (or care) whether the caller handed it an exact-length body, so the trailing-bytes +/// check has to live here. +fn decode_exhaustive(version: i16, mut body: Bytes) -> Result { + let value = + T::decode(&mut body, version).map_err(|e| KafkaProtocolError::Malformed(e.to_string()))?; + if body.has_remaining() { + return Err(KafkaProtocolError::Malformed( + "unexpected trailing bytes in request body".to_string(), + )); + } + Ok(value) +} + +/// Turn an encode [`Result`] into a [`HandleOutcome`], closing the connection when encoding +/// fails rather than propagating - there is no parseable response to send in that case. +fn respond_or_close(result: Result, api_name: &str) -> HandleOutcome { + match result { + Ok(body) => HandleOutcome::Respond(body), + Err(error) => { + tracing::warn!(%error, "failed to encode {api_name} response; closing connection"); + HandleOutcome::Close + } + } +} + /// Produce is the only request the wire protocol allows to go unanswered /// (`acks=0`), so it gets its own path that may return [`HandleOutcome::NoResponse`]. /// -/// The firewall check runs AFTER decoding `acks`, not before: `ApiVersions` advertises +/// The firewall check runs AFTER decoding the request, not before: `ApiVersions` advertises /// Produce min=0 (see [`advertised_min_version`]) while the firewall's real floor is 3, so a /// spec-compliant client can legitimately send Produce v0-2 with `acks=0`. Rejecting those /// versions before reading `acks` would send an error response the client never expects, /// desyncing the next correlation id it reads. fn handle_produce_request(api_version: i16, body: Bytes) -> HandleOutcome { // Above the encoder max there is no response parseable at the client's version, so close - // rather than decode (same policy the other APIs apply). Below-min versions still decode so - // acks=0 fire-and-forget clients stay silent per the advertised min=0. + // rather than decode (same policy the other APIs apply). if api_version > supported_max_version(API_KEY_PRODUCE).unwrap_or(i16::MAX) { return HandleOutcome::Close; } - match decode_produce_request(api_version, body) { + // `kafka_protocol`'s ProduceRequest/ProduceResponse schemas only go back to v3, so v0-2 + // (still advertised as the min in ApiVersions per KAFKA-18659) can be neither decoded nor + // encoded by the crate - there is no parseable response at these versions regardless of + // body content. `acks` is always the first i16 on the wire there (`transactional_id` was + // added in v3), so it's peeked by hand: acks=0 must keep the connection open per the wire + // protocol's fire-and-forget rule even though no response can ever be encoded for it. + if api_version < 3 { + let acks = match body.get(0..2) { + Some(&[hi, lo]) => Some(i16::from_be_bytes([hi, lo])), + _ => None, + }; + return match acks { + Some(0) | None => HandleOutcome::NoResponse, + Some(_) => unsupported_version_response(API_KEY_PRODUCE, api_version, |v| { + encode_produce_error_response(v, ERROR_UNSUPPORTED_VERSION) + }), + }; + } + match decode_exhaustive::(api_version, body) { // acks=0 is fire-and-forget: the client isn't reading a response, so // sending one desyncs the next correlation id it expects. - ProduceDecodeResult::Ok(req) if req.acks == 0 => HandleOutcome::NoResponse, - ProduceDecodeResult::Ok(req) => { + Ok(req) if req.acks == 0 => HandleOutcome::NoResponse, + Ok(req) => { if !is_supported_version(API_KEY_PRODUCE, api_version) { return unsupported_version_response(API_KEY_PRODUCE, api_version, |v| { encode_produce_error_response(v, ERROR_UNSUPPORTED_VERSION) }); } - HandleOutcome::Respond(encode_produce_response(api_version, &req)) + respond_or_close(encode_produce_response(api_version, &req), "Produce") } - ProduceDecodeResult::Err { - acks: Some(0), - error, - } => { - tracing::warn!( - "Failed to decode Produce request with acks=0 (no response): {:?}", - error - ); + Err(error) => { + // `kafka_protocol` decodes the whole request in one shot; a failure anywhere gives + // no partial-field access, so `acks` is unknowable here (unlike the pre-migration + // field-by-field decoder, which could still know `acks` on a later-field failure). + // Responding risks desyncing an acks=0 fire-and-forget client's correlation stream, + // so every Produce decode failure now stays silent - a behavior change from the + // hand-rolled decoder, which answered with INVALID_REQUEST when `acks` was known and + // nonzero. + tracing::warn!(%error, "failed to decode Produce request (no response)"); HandleOutcome::NoResponse } - ProduceDecodeResult::Err { acks: None, error } => { - // Decode failed before `acks` was read (malformed transactional_id or a truncated - // frame). Whether the client wants a response is unknowable, and an error response - // would desync an acks=0 fire-and-forget client's correlation stream. Frames are - // length-delimited, so the malformed frame does not affect the next frame's boundary; - // stay silent and keep the connection usable rather than risk that desync. - tracing::warn!( - "Failed to decode Produce request before acks was read (no response): {:?}", - error - ); - HandleOutcome::NoResponse - } - ProduceDecodeResult::Err { - acks: Some(_), - error, - } => { - tracing::warn!("Failed to decode Produce request: {:?}", error); - if is_supported_version(API_KEY_PRODUCE, api_version) { - HandleOutcome::Respond(encode_produce_error_response( - api_version, - ERROR_INVALID_REQUEST, - )) - } else { - unsupported_version_response(API_KEY_PRODUCE, api_version, |v| { - encode_produce_error_response(v, ERROR_UNSUPPORTED_VERSION) - }) - } - } } } @@ -237,13 +255,13 @@ fn handle_other_request( broker: &BrokerAdvertise, ) -> HandleOutcome { match api_key { - API_KEY_API_VERSIONS => handle_api_versions(api_version, &body), + API_KEY_API_VERSIONS => handle_api_versions(api_version, body), API_KEY_METADATA => handle_metadata(api_version, body, broker), API_KEY_FETCH => handle_versioned_request( API_KEY_FETCH, api_version, body, - decode_fetch_request, + decode_exhaustive::, encode_fetch_response, encode_fetch_error_response, "Fetch", @@ -252,7 +270,7 @@ fn handle_other_request( API_KEY_LIST_OFFSETS, api_version, body, - decode_list_offsets_request, + decode_exhaustive::, encode_list_offsets_response, encode_list_offsets_error_response, "ListOffsets", @@ -261,7 +279,7 @@ fn handle_other_request( API_KEY_CREATE_TOPICS, api_version, body, - decode_create_topics_request, + decode_exhaustive::, encode_create_topics_response, encode_create_topics_error_response, "CreateTopics", @@ -272,21 +290,26 @@ fn handle_other_request( } } -fn handle_api_versions(api_version: i16, body: &Bytes) -> HandleOutcome { - if is_supported_version(API_KEY_API_VERSIONS, api_version) { - match decode_api_versions_request(api_version, body) { - Ok(()) => HandleOutcome::Respond(encode_api_versions_response(api_version, ERROR_NONE)), - Err(e) => { - tracing::warn!("Failed to decode ApiVersions request: {:?}", e); - HandleOutcome::Respond(encode_api_versions_response( - api_version, - ERROR_INVALID_REQUEST, - )) - } - } - } else { +fn handle_api_versions(api_version: i16, body: Bytes) -> HandleOutcome { + if !is_supported_version(API_KEY_API_VERSIONS, api_version) { // KIP-511: reply with v0 when the requested version is not understood. - HandleOutcome::Respond(encode_api_versions_response(0, ERROR_UNSUPPORTED_VERSION)) + return respond_or_close( + encode_api_versions_response(0, ERROR_UNSUPPORTED_VERSION), + "ApiVersions", + ); + } + match decode_exhaustive::(api_version, body) { + Ok(_) => respond_or_close( + encode_api_versions_response(api_version, ERROR_NONE), + "ApiVersions", + ), + Err(error) => { + tracing::warn!(%error, "failed to decode ApiVersions request"); + respond_or_close( + encode_api_versions_response(api_version, ERROR_INVALID_REQUEST), + "ApiVersions", + ) + } } } @@ -302,18 +325,16 @@ fn handle_metadata(api_version: i16, body: Bytes, broker: &BrokerAdvertise) -> H ); return HandleOutcome::Close; } - match decode_metadata_request(api_version, body) { - Ok(topics) => HandleOutcome::Respond(encode_metadata_response( - api_version, - &topics, - broker, - ERROR_NONE, - )), - Err(e) => { + match decode_metadata_topics(api_version, body) { + Ok(topics) => respond_or_close( + encode_metadata_response(api_version, &topics, broker, ERROR_NONE), + "Metadata", + ), + Err(error) => { // Metadata has no top-level error field; a malformed body cannot carry // INVALID_REQUEST in a version-correct way for every client. Close. tracing::warn!( - ?e, + %error, api_version, "Failed to decode Metadata request; closing connection" ); @@ -327,16 +348,16 @@ fn handle_versioned_request( api_version: i16, body: Bytes, decode: impl FnOnce(i16, Bytes) -> Result, - encode_ok: impl FnOnce(i16, &T) -> Bytes, - encode_err: impl Fn(i16, i16) -> Bytes, + encode_ok: impl FnOnce(i16, &T) -> Result, + encode_err: impl Fn(i16, i16) -> Result, api_name: &str, ) -> HandleOutcome { if is_supported_version(api_key, api_version) { match decode(api_version, body) { - Ok(req) => HandleOutcome::Respond(encode_ok(api_version, &req)), - Err(e) => { - tracing::warn!("Failed to decode {api_name} request: {:?}", e); - HandleOutcome::Respond(encode_err(api_version, ERROR_INVALID_REQUEST)) + Ok(req) => respond_or_close(encode_ok(api_version, &req), api_name), + Err(error) => { + tracing::warn!(%error, "Failed to decode {api_name} request"); + respond_or_close(encode_err(api_version, ERROR_INVALID_REQUEST), api_name) } } } else { @@ -349,15 +370,20 @@ fn handle_versioned_request( /// Unsupported-version policy for APIs whose encoders only implement up to /// [`ApiVersionRange::max_version`]. /// -/// - `api_version > max`: Close. Encoding at the client's raw version would omit later-version -/// fields (`CreateTopics` v7 `TopicId`, Produce v13 UUID topic, …) and the client could not -/// parse the intended `UNSUPPORTED_VERSION` body. -/// - `api_version < min` but still within encoder capability: Respond with an error shaped for -/// that version (e.g. `ListOffsets` v0 `old_style_offsets`, Produce v0–2). +/// - `api_version > max`: Close. `SUPPORTED_RANGES` is the governance boundary, not just an +/// encoding-capability limit - `kafka_protocol` can often encode versions above our firewall +/// max just fine, but responding there would silently widen what this gateway accepts. +/// - `api_version < min`: Respond with an error shaped for that version when `kafka_protocol` +/// can encode it, otherwise `encode` fails and [`respond_or_close`] closes instead. In +/// practice every `SUPPORTED_RANGES` min was chosen at or above the oldest version +/// `kafka_protocol` implements for that message, so this always closes today (e.g. +/// `ListOffsets` v0's legacy `old_style_offsets` shape predates the crate's schema) - kept +/// generic rather than hard-coded so a future `kafka_protocol` upgrade that widens a schema's +/// floor is picked up automatically instead of silently staying on `Close`. fn unsupported_version_response( api_key: i16, api_version: i16, - encode: impl FnOnce(i16) -> Bytes, + encode: impl FnOnce(i16) -> Result, ) -> HandleOutcome { let max_version = SUPPORTED_RANGES .iter() @@ -372,7 +398,7 @@ fn unsupported_version_response( ); return HandleOutcome::Close; } - HandleOutcome::Respond(encode(api_version)) + respond_or_close(encode(api_version), "unsupported-version") } #[must_use] @@ -405,49 +431,28 @@ pub const fn advertised_min_version(api_key: i16, firewall_min: i16) -> i16 { } } -fn encode_api_versions_response(api_version: i16, error_code: i16) -> Bytes { - let flexible = api_version >= 3; - let ranges = SUPPORTED_RANGES; - let mut e = Encoder::with_capacity(128); - - e.write_i16(error_code); - - if flexible { - e.write_varint((ranges.len() + 1) as u64); - for r in ranges { - e.write_i16(r.api_key); - e.write_i16(advertised_min_version(r.api_key, r.min_version)); - e.write_i16(r.max_version); - e.write_empty_tagged_fields(); - } - } else { - e.write_i32(i32::try_from(ranges.len()).expect("supported range table is small")); - for r in ranges { - e.write_i16(r.api_key); - e.write_i16(advertised_min_version(r.api_key, r.min_version)); - e.write_i16(r.max_version); - } - } - - if api_version >= 1 { - e.write_i32(0); - } - - if flexible { - e.write_empty_tagged_fields(); - } - - e.freeze() +fn encode_api_versions_response(api_version: i16, error_code: i16) -> Result { + let api_keys = SUPPORTED_RANGES + .iter() + .map(|r| { + ApiVersion::default() + .with_api_key(r.api_key) + .with_min_version(advertised_min_version(r.api_key, r.min_version)) + .with_max_version(r.max_version) + }) + .collect(); + let resp = ApiVersionsResponse::default() + .with_error_code(error_code) + .with_api_keys(api_keys); + encode_message(&resp, api_version, 128) } fn encode_metadata_response( response_version: i16, - topics: &[String], + topics: &[StrBytes], broker: &BrokerAdvertise, topic_error_override: i16, -) -> Bytes { - let flexible = response_version >= 9; - let topics_count = topics.len(); +) -> Result { // Stub has no topic catalog: echo requested names with UNKNOWN_TOPIC_OR_PARTITION, // or a forced override (unused today; kept for symmetry with other encoders). let topic_error = if topic_error_override == ERROR_NONE { @@ -456,249 +461,98 @@ fn encode_metadata_response( topic_error_override }; - let mut e = Encoder::with_capacity(256); - - if response_version >= 3 { - e.write_i32(0); // throttle_time_ms (Metadata v3+) - } - - if flexible { - e.write_varint(2); // one broker (N+1) - e.write_i32(1); - e.write_compact_nullable_string(Some(&broker.host)); - e.write_i32(broker.port); - e.write_compact_nullable_string(None); // rack - e.write_empty_tagged_fields(); - - e.write_compact_nullable_string(None); // cluster_id (v2+) - e.write_i32(1); // controller_id (v1+) - - e.write_varint((topics_count + 1) as u64); - for name in topics { - e.write_i16(topic_error); - e.write_compact_nullable_string(Some(name)); - e.write_bool(false); // is_internal (v1+) - e.write_varint(1); // empty partitions array - e.write_i32(AUTHORIZED_OPS_UNKNOWN); // topic_authorized_operations (v8+) - e.write_empty_tagged_fields(); - } - e.write_i32(AUTHORIZED_OPS_UNKNOWN); // cluster_authorized_operations (v8+) - e.write_empty_tagged_fields(); - } else { - e.write_i32(1); // brokers array length - e.write_i32(1); // node_id - // broker.host is bounded to i16::MAX at BrokerAdvertise::from_server_config, so the - // unchecked writer cannot exceed the length prefix - same guarantee the flexible path - // relies on for its compact string. - e.write_nullable_string_unchecked(Some(&broker.host)); - e.write_i32(broker.port); - if response_version >= 1 { - e.write_nullable_string_unchecked(None); // rack - } + let response_topics = topics + .iter() + .map(|name| { + MetadataResponseTopic::default() + .with_error_code(topic_error) + .with_name(Some(TopicName(name.clone()))) + }) + .collect(); - if response_version >= 2 { - e.write_nullable_string_unchecked(None); // cluster_id - } - if response_version >= 1 { - e.write_i32(1); // controller_id - must come before topics array - } + let broker_entry = MetadataResponseBroker::default() + .with_node_id(BrokerId(1)) + .with_host(StrBytes::from_string(broker.host.clone())) + .with_port(broker.port); - e.write_i32(i32::try_from(topics_count).expect("topic count bounded")); - for name in topics { - e.write_i16(topic_error); - e.write_nullable_string_unchecked(Some(name)); - if response_version >= 1 { - e.write_bool(false); // is_internal - } - e.write_i32(0); // partitions array (empty) - if response_version >= 8 { - e.write_i32(AUTHORIZED_OPS_UNKNOWN); // topic_authorized_operations - } - } - if response_version >= 8 { - e.write_i32(AUTHORIZED_OPS_UNKNOWN); // cluster_authorized_operations - } - } + let resp = MetadataResponse::default() + .with_brokers(vec![broker_entry]) + .with_controller_id(BrokerId(1)) + .with_topics(response_topics); - e.freeze() -} - -/// Decodes an `ApiVersions` request body. -/// -/// v0–v2 have an empty body. v3+ requires `ClientSoftwareName`, `ClientSoftwareVersion`, -/// and tagged fields (KIP-511 flexible encoding). -fn decode_api_versions_request(api_version: i16, body: &Bytes) -> Result<()> { - if api_version < 3 { - if !body.is_empty() { - return Err(KafkaProtocolError::UnexpectedTrailingBytes); - } - return Ok(()); - } - let mut d = Decoder::new(body.clone()); - let _client_software_name = d.read_compact_string()?; - let _client_software_version = d.read_compact_string()?; - d.read_tagged_fields()?; - if d.remaining() != 0 { - return Err(KafkaProtocolError::UnexpectedTrailingBytes); - } - Ok(()) + encode_message(&resp, response_version, 256) } /// Decodes a Metadata request body so the response can echo topic names. /// -/// Every Metadata version requires at least the topics array count on the wire — an empty -/// body is malformed, not "all topics". A null topics array (`-1` legacy / `varint=0` -/// compact) means "all topics" and decodes to an empty list for this stub. Remaining -/// version-gated fields (`AllowAutoTopicCreation`, authorized-ops flags, tagged fields) -/// are consumed so truncated flexible bodies are rejected. -pub(crate) fn decode_metadata_request(api_version: i16, body: Bytes) -> Result> { - if body.is_empty() { - return Err(KafkaProtocolError::BufferUnderflow { - needed: 1, - remaining: 0, - }); - } - let mut d = Decoder::new(body); - let flexible = api_version >= 9; - let topics_count = if flexible { - // Metadata topics is a nullable compact array ("all topics" when null). - d.read_compact_array_count_nullable()? - } else { - d.read_i32_array_count_nullable()? - }; - - let mut topics = Vec::with_capacity(topics_count.min(PREALLOC_HINT)); - for _ in 0..topics_count { - if flexible && api_version >= 10 { - // MetadataRequestTopic.topic_id: 16-byte UUID before name (v10+). - let _topic_id = d.read_bytes(16)?; - } - let name = if flexible { - d.read_compact_nullable_string()? - .ok_or(KafkaProtocolError::NullTopicName)? - } else { - d.read_nullable_string()? - .ok_or(KafkaProtocolError::NullTopicName)? - }; - topics.push(name); - if flexible { - d.read_tagged_fields()?; - } - } - - // allow_auto_topic_creation (v4+) - if api_version >= 4 { - let _allow_auto_topic_creation = d.read_bool()?; - } - // include_cluster_authorized_operations (v8–v10; removed in v11) - if (8..=10).contains(&api_version) { - let _include_cluster_authorized_operations = d.read_bool()?; - } - // include_topic_authorized_operations (v8+) - if api_version >= 8 { - let _include_topic_authorized_operations = d.read_bool()?; - } - if flexible { - d.read_tagged_fields()?; - } - if d.remaining() != 0 { - return Err(KafkaProtocolError::UnexpectedTrailingBytes); - } - - Ok(topics) -} - -/// Compatibility alias used by existing unit tests. -#[cfg(test)] -pub(crate) fn decode_metadata_request_topics(body: Bytes, api_version: i16) -> Result> { - decode_metadata_request(api_version, body) +/// A null topics array (`-1` legacy / `varint=0` compact) means "all topics" and decodes to an +/// empty list for this stub. A null per-topic `name` (v10+ allows topic-id-only lookups) has no +/// name to echo, so it errors rather than silently dropping the topic from the response. +fn decode_metadata_topics(api_version: i16, body: Bytes) -> Result> { + let req = decode_exhaustive::(api_version, body)?; + req.topics + .unwrap_or_default() + .into_iter() + .map(|topic| { + topic + .name + .map(|name| name.0) + .ok_or(KafkaProtocolError::NullTopicName) + }) + .collect() } #[cfg(test)] mod tests { use super::*; - use crate::protocol::codec::Encoder; #[test] - fn decode_metadata_request_topics_legacy_null_topic_name_fails() { + fn decode_metadata_topics_legacy_null_topic_name_fails() { let body = Bytes::from_static(&[ 0x00, 0x00, 0x00, 0x01, // one topic 0xff, 0xff, // null topic name ]); - let err = decode_metadata_request_topics(body, 0).unwrap_err(); + let err = decode_metadata_topics(0, body).unwrap_err(); assert!(matches!(err, KafkaProtocolError::NullTopicName)); } #[test] - fn decode_metadata_request_topics_legacy_null_array_means_all_topics() { + fn decode_metadata_topics_legacy_null_array_means_all_topics() { // -1 is the spec-defined "all topics" sentinel for the legacy i32 array count, not a - // malformed request - must decode to an empty list, not InvalidArrayLength. + // malformed request - must decode to an empty list. let body = Bytes::from_static(&[0xff, 0xff, 0xff, 0xff]); // -1 - let topics = decode_metadata_request_topics(body, 0).unwrap(); + let topics = decode_metadata_topics(0, body).unwrap(); assert!(topics.is_empty()); } #[test] - fn decode_metadata_request_topics_legacy_other_negative_counts_still_fail() { - // Only -1 is the null sentinel; any other negative count is genuinely malformed. - let body = Bytes::from_static(&[0xff, 0xff, 0xff, 0xfe]); // -2 - let err = decode_metadata_request_topics(body, 0).unwrap_err(); - assert!(matches!(err, KafkaProtocolError::InvalidArrayLength(-2))); + fn decode_metadata_topics_empty_body_is_malformed() { + assert!(decode_metadata_topics(0, Bytes::new()).is_err()); } #[test] - fn decode_metadata_request_topics_flexible_v10_truncated_topic_id_fails() { - let mut enc = Encoder::with_capacity(8); - enc.write_varint(2); // one topic - enc.write_bytes(&[0u8; 8]); // truncated topic_id, should be 16 bytes - let err = decode_metadata_request_topics(enc.freeze(), 10).unwrap_err(); - assert!(matches!(err, KafkaProtocolError::BufferUnderflow { .. })); - } - - #[test] - fn decode_metadata_request_topics_flexible_invalid_utf8_fails() { - let mut enc = Encoder::with_capacity(16); - enc.write_varint(2); // one topic - enc.write_varint(2); // string len = 1 - enc.write_u8(0xff); // invalid utf-8 - enc.write_empty_tagged_fields(); // per-topic tagged - enc.write_bool(true); // allow_auto_topic_creation - enc.write_bool(false); // include_cluster_authorized_operations - enc.write_bool(false); // include_topic_authorized_operations - enc.write_empty_tagged_fields(); // top-level tagged - let err = decode_metadata_request_topics(enc.freeze(), 9).unwrap_err(); - assert!(matches!(err, KafkaProtocolError::InvalidUtf8)); - } - - #[test] - fn decode_metadata_request_empty_body_is_malformed() { - let err = decode_metadata_request(0, Bytes::new()).unwrap_err(); - assert!(matches!(err, KafkaProtocolError::BufferUnderflow { .. })); - } - - #[test] - fn decode_metadata_request_flexible_truncated_after_topics_fails() { + fn decode_metadata_topics_flexible_truncated_after_topics_fails() { // topics = null (all topics) but missing allow_auto / auth flags / tagged fields. let body = Bytes::from_static(&[0x00]); - let err = decode_metadata_request(9, body).unwrap_err(); - assert!(matches!(err, KafkaProtocolError::BufferUnderflow { .. })); + assert!(decode_metadata_topics(9, body).is_err()); } #[test] fn decode_api_versions_v3_requires_software_fields() { - let err = decode_api_versions_request(3, &Bytes::new()).unwrap_err(); - assert!(matches!( - err, - KafkaProtocolError::BufferUnderflow { .. } | KafkaProtocolError::NullCompactString - )); + assert!(decode_exhaustive::(3, Bytes::new()).is_err()); } #[test] fn decode_api_versions_v3_accepts_valid_body() { - let mut enc = Encoder::with_capacity(32); - enc.write_compact_nullable_string(Some("iggy-test")); - enc.write_compact_nullable_string(Some("0.1.0")); - enc.write_empty_tagged_fields(); - decode_api_versions_request(3, &enc.freeze()).unwrap(); + // Hand-encoded rather than round-tripped through `ApiVersionsRequest::encode`: encoding + // is gated behind the crate's "client" feature, which this broker-only binary doesn't + // enable. + let body = Bytes::from_static(&[ + 0x0a, b'i', b'g', b'g', b'y', b'-', b't', b'e', b's', + b't', // compact string (len 9) + 0x06, b'0', b'.', b'1', b'.', b'0', // compact string (len 5) + 0x00, // empty tagged fields + ]); + decode_exhaustive::(3, body).unwrap(); } } diff --git a/gateways/kafka/src/protocol/codec.rs b/gateways/kafka/src/protocol/codec.rs deleted file mode 100644 index 2fb8915af1..0000000000 --- a/gateways/kafka/src/protocol/codec.rs +++ /dev/null @@ -1,472 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Low-level Kafka primitive encoders/decoders (ported wire codec). - -#![allow(clippy::missing_const_for_fn, clippy::bool_to_int_with_if)] -#![allow( - clippy::missing_errors_doc, - clippy::cast_sign_loss, - clippy::must_use_candidate, - clippy::missing_panics_doc, - clippy::cast_possible_truncation, - clippy::cast_lossless -)] - -use bytes::{Buf, BufMut, Bytes, BytesMut}; - -use crate::error::{KafkaProtocolError, Result}; - -/// Upper bound for Kafka array/collection element counts decoded from the wire. -/// Matches typical broker limits and prevents OOM from adversarial length prefixes. -pub const MAX_COLLECTION_LEN: usize = 65_536; - -/// Initial `Vec::with_capacity` hint for wire-decoded array counts. -/// -/// A count up to `MAX_COLLECTION_LEN` is still validated as a count, but pre-reserving on it -/// directly lets a few-byte frame declaring a huge count force a multi-megabyte allocation -/// before any element bytes are checked present. Clamp the reservation; genuine large arrays -/// still grow correctly via `Vec::push`'s amortized doubling once real elements are decoded. -pub const PREALLOC_HINT: usize = 128; - -/// Cumulative cap on the total array elements decoded from one request frame. -/// -/// `MAX_COLLECTION_LEN` bounds a single array, but not the product across nested arrays -/// (`topics` x `partitions`): an 8 MB frame packed with minimal partition entries decodes into -/// ~1M owned structs before any per-array limit trips. This budget bounds the sum of every array -/// count in one decode, so materialized struct count stays proportional to a fixed ceiling -/// regardless of nesting. Real requests never approach it (a Fetch over thousands of partitions -/// is still far below). -pub const MAX_REQUEST_ELEMENTS: usize = MAX_COLLECTION_LEN; - -pub struct Decoder { - bytes: Bytes, - element_budget: usize, -} - -impl Decoder { - pub fn new(bytes: Bytes) -> Self { - Self { - bytes, - element_budget: MAX_REQUEST_ELEMENTS, - } - } - - pub fn remaining(&self) -> usize { - self.bytes.remaining() - } - - /// Debit the cumulative element budget shared across every array in this decode. - fn charge_elements(&mut self, count: usize) -> Result<()> { - match self.element_budget.checked_sub(count) { - Some(remaining) => { - self.element_budget = remaining; - Ok(()) - } - None => Err(KafkaProtocolError::RequestElementBudgetExceeded { - count, - remaining: self.element_budget, - }), - } - } - - pub fn read_u8(&mut self) -> Result { - self.ensure(1)?; - Ok(self.bytes.get_u8()) - } - - pub fn read_i8(&mut self) -> Result { - self.ensure(1)?; - Ok(self.bytes.get_i8()) - } - - pub fn read_i16(&mut self) -> Result { - self.ensure(2)?; - Ok(self.bytes.get_i16()) - } - - pub fn read_i32(&mut self) -> Result { - self.ensure(4)?; - Ok(self.bytes.get_i32()) - } - - pub fn read_i64(&mut self) -> Result { - self.ensure(8)?; - Ok(self.bytes.get_i64()) - } - - pub fn read_bool(&mut self) -> Result { - Ok(self.read_i8()? != 0) - } - - /// Unsigned varint (Kafka uses this for compact array lengths and tagged-field counts). - /// Value is encoded with 7 bits per byte, LSB first; the high bit of each byte signals - /// that more bytes follow. - pub fn read_varint(&mut self) -> Result { - let mut result: u64 = 0; - let mut shift = 0u32; - loop { - let byte = self.read_u8()?; - if shift == 63 && byte & 0x7E != 0 { - return Err(KafkaProtocolError::InvalidVarint); - } - result |= ((byte & 0x7F) as u64) << shift; - if byte & 0x80 == 0 { - return Ok(result); - } - shift += 7; - if shift >= 64 { - return Err(KafkaProtocolError::InvalidVarint); - } - } - } - - /// Legacy array length: signed i32 count (must be non-negative). - pub fn read_i32_array_count(&mut self) -> Result { - let n = self.read_i32()?; - if n < 0 { - return Err(KafkaProtocolError::InvalidArrayLength(n)); - } - // Safe: n is in [0, i32::MAX]; i32::MAX (2_147_483_647) fits in usize - // on all 32-bit and 64-bit platforms this crate targets. - let count = n as usize; - if count > MAX_COLLECTION_LEN { - return Err(KafkaProtocolError::CollectionTooLarge { - count, - max: MAX_COLLECTION_LEN, - }); - } - self.charge_elements(count)?; - Ok(count) - } - - /// Legacy nullable array length: signed i32 count, where -1 is the spec-defined null - /// (`all items`) sentinel - used by Metadata's `topics` field to mean "all topics" - treated - /// as empty (0 elements) rather than an error, mirroring - /// [`Self::read_compact_array_count_nullable`]. - pub fn read_i32_array_count_nullable(&mut self) -> Result { - let n = self.read_i32()?; - if n == -1 { - return Ok(0); - } - if n < 0 { - return Err(KafkaProtocolError::InvalidArrayLength(n)); - } - // Safe: n is in [0, i32::MAX]; i32::MAX (2_147_483_647) fits in usize - // on all 32-bit and 64-bit platforms this crate targets. - let count = n as usize; - if count > MAX_COLLECTION_LEN { - return Err(KafkaProtocolError::CollectionTooLarge { - count, - max: MAX_COLLECTION_LEN, - }); - } - self.charge_elements(count)?; - Ok(count) - } - - /// Compact array length for a **non-nullable** field: unsigned varint holding - /// `element_count + 1`. Varint `0` is Kafka's null encoding and is rejected here — - /// required arrays (Produce/Fetch/ListOffsets/CreateTopics topic data, nested partitions, - /// etc.) must use `1` for empty. - pub fn read_compact_array_count(&mut self) -> Result { - let n = self.read_varint()?; - if n == 0 { - return Err(KafkaProtocolError::NullCompactArray); - } - let count = Self::compact_array_count_from_len_plus_one(n)?; - self.charge_elements(count)?; - Ok(count) - } - - /// Compact array length for a **nullable** field: varint `0` means null/absent and is - /// returned as 0 elements (e.g. Metadata topics = "all topics", Fetch forgotten topics - /// when a client encodes null). - pub fn read_compact_array_count_nullable(&mut self) -> Result { - let n = self.read_varint()?; - if n == 0 { - return Ok(0); - } - let count = Self::compact_array_count_from_len_plus_one(n)?; - self.charge_elements(count)?; - Ok(count) - } - - fn compact_array_count_from_len_plus_one(n: u64) -> Result { - let count = usize::try_from(n - 1).map_err(|_| KafkaProtocolError::CollectionTooLarge { - count: MAX_COLLECTION_LEN + 1, - max: MAX_COLLECTION_LEN, - })?; - if count > MAX_COLLECTION_LEN { - return Err(KafkaProtocolError::CollectionTooLarge { - count, - max: MAX_COLLECTION_LEN, - }); - } - Ok(count) - } - - /// Legacy nullable string: i16 length prefix (-1 = null). - pub fn read_nullable_string(&mut self) -> Result> { - let len = self.read_i16()?; - if len < 0 { - return Ok(None); - } - let len = len as usize; - self.ensure(len)?; - let s = std::str::from_utf8(&self.bytes.chunk()[..len]) - .map_err(|_| KafkaProtocolError::InvalidUtf8)? - .to_owned(); - self.bytes.advance(len); - Ok(Some(s)) - } - - /// Compact nullable string (flexible versions): varint(len+1) prefix, 0 = null. - pub fn read_compact_nullable_string(&mut self) -> Result> { - let len_plus_one = self.read_varint()?; - if len_plus_one == 0 { - return Ok(None); - } - let len = usize::try_from(len_plus_one - 1).map_err(|_| { - KafkaProtocolError::CollectionTooLarge { - count: MAX_COLLECTION_LEN + 1, - max: MAX_COLLECTION_LEN, - } - })?; - // Parity with the legacy i16-length string (naturally <= 32767); the compact form is - // otherwise bounded only by the frame, letting one flexible name reach max_frame_size. - if len > MAX_COLLECTION_LEN { - return Err(KafkaProtocolError::CollectionTooLarge { - count: len, - max: MAX_COLLECTION_LEN, - }); - } - self.ensure(len)?; - let s = std::str::from_utf8(&self.bytes.chunk()[..len]) - .map_err(|_| KafkaProtocolError::InvalidUtf8)? - .to_owned(); - self.bytes.advance(len); - Ok(Some(s)) - } - - /// Non-nullable compact string (flexible versions): varint(len+1), never null. - pub fn read_compact_string(&mut self) -> Result { - self.read_compact_nullable_string()? - .ok_or(KafkaProtocolError::NullCompactString) - } - - /// Legacy nullable bytes: i32 length prefix (-1 = null). - pub fn read_nullable_bytes(&mut self) -> Result> { - let len = self.read_i32()?; - if len < 0 { - return Ok(None); - } - let len = len as usize; - self.ensure(len)?; - Ok(Some(self.bytes.copy_to_bytes(len))) - } - - /// Compact nullable bytes (flexible versions): varint(len+1) prefix, 0 = null. - pub fn read_compact_nullable_bytes(&mut self) -> Result> { - let len_plus_one = self.read_varint()?; - if len_plus_one == 0 { - return Ok(None); - } - let len = usize::try_from(len_plus_one - 1).map_err(|_| { - KafkaProtocolError::CollectionTooLarge { - count: MAX_COLLECTION_LEN + 1, - max: MAX_COLLECTION_LEN, - } - })?; - self.ensure(len)?; - Ok(Some(self.bytes.copy_to_bytes(len))) - } - - pub fn read_bytes(&mut self, len: usize) -> Result { - self.ensure(len)?; - Ok(self.bytes.copy_to_bytes(len)) - } - - /// Skip over a tagged-fields section. Each field is: tag (varint) + size (varint) + bytes. - /// A count of 0 is the common case (single byte 0x00). - pub fn read_tagged_fields(&mut self) -> Result<()> { - let count = self.read_varint()?; - if count > MAX_COLLECTION_LEN as u64 { - return Err(KafkaProtocolError::CollectionTooLarge { - count: count as usize, - max: MAX_COLLECTION_LEN, - }); - } - let count = count as usize; - - for _ in 0..count { - self.read_varint()?; // tag number - let size = usize::try_from(self.read_varint()?).map_err(|_| { - KafkaProtocolError::CollectionTooLarge { - count: MAX_COLLECTION_LEN + 1, - max: MAX_COLLECTION_LEN, - } - })?; - self.ensure(size)?; - self.bytes.advance(size); - } - Ok(()) - } - - fn ensure(&self, needed: usize) -> Result<()> { - let remaining = self.bytes.remaining(); - if remaining < needed { - return Err(KafkaProtocolError::BufferUnderflow { needed, remaining }); - } - Ok(()) - } -} - -pub struct Encoder { - bytes: BytesMut, -} - -impl Encoder { - pub fn with_capacity(capacity: usize) -> Self { - Self { - bytes: BytesMut::with_capacity(capacity), - } - } - - pub fn write_u8(&mut self, v: u8) { - self.bytes.put_u8(v); - } - - pub fn write_i8(&mut self, v: i8) { - self.bytes.put_i8(v); - } - - pub fn write_i16(&mut self, v: i16) { - self.bytes.put_i16(v); - } - - pub fn write_i32(&mut self, v: i32) { - self.bytes.put_i32(v); - } - - pub fn write_i64(&mut self, v: i64) { - self.bytes.put_i64(v); - } - - pub fn write_bool(&mut self, v: bool) { - self.write_i8(if v { 1 } else { 0 }); - } - - /// Unsigned varint, 7 bits per byte, LSB first. - pub fn write_varint(&mut self, mut v: u64) { - loop { - let byte = (v & 0x7F) as u8; - v >>= 7; - if v == 0 { - self.bytes.put_u8(byte); - return; - } - self.bytes.put_u8(byte | 0x80); - } - } - - /// Legacy nullable string: i16 length prefix, -1 for null. - pub fn write_nullable_string(&mut self, v: Option<&str>) -> Result<()> { - match v { - None => self.write_i16(-1), - Some(s) => { - if s.len() > i16::MAX as usize { - return Err(KafkaProtocolError::StringTooLong { length: s.len() }); - } - self.write_i16(i16::try_from(s.len()).expect("checked above")); - self.bytes.put_slice(s.as_bytes()); - } - } - Ok(()) - } - - /// Infallible variant for response-encoding paths where the string originated from a decoded - /// Kafka request and is therefore already bounded to `i16::MAX` bytes. - pub fn write_nullable_string_unchecked(&mut self, v: Option<&str>) { - match v { - None => self.write_i16(-1), - Some(s) => { - debug_assert!(i16::try_from(s.len()).is_ok()); - self.write_i16(i16::try_from(s.len()).expect("caller guarantees len <= i16::MAX")); - self.bytes.put_slice(s.as_bytes()); - } - } - } - - /// Compact nullable string (flexible versions): varint(len+1), 0 for null. - pub fn write_compact_nullable_string(&mut self, v: Option<&str>) { - match v { - None => self.write_varint(0), - Some(s) => { - self.write_varint((s.len() + 1) as u64); - self.bytes.put_slice(s.as_bytes()); - } - } - } - - /// Write a null bytes field (i32 -1). Infallible; use instead of `write_nullable_bytes(None)`. - pub fn write_null_bytes(&mut self) { - self.write_i32(-1); - } - - /// Legacy nullable bytes: i32 length prefix, -1 for null. - pub fn write_nullable_bytes(&mut self, v: Option<&[u8]>) -> Result<()> { - match v { - None => self.write_i32(-1), - Some(b) => { - if b.len() > i32::MAX as usize { - return Err(KafkaProtocolError::CollectionTooLarge { - count: b.len(), - max: i32::MAX as usize, - }); - } - self.write_i32(i32::try_from(b.len()).expect("checked above")); - self.bytes.put_slice(b); - } - } - Ok(()) - } - - /// Compact nullable bytes (flexible versions): varint(len+1), 0 for null. - pub fn write_compact_nullable_bytes(&mut self, v: Option<&[u8]>) { - match v { - None => self.write_varint(0), - Some(b) => { - self.write_varint((b.len() + 1) as u64); - self.bytes.put_slice(b); - } - } - } - - pub fn write_bytes(&mut self, b: &[u8]) { - self.bytes.put_slice(b); - } - - /// Write an empty tagged-fields section (single 0x00 byte). - pub fn write_empty_tagged_fields(&mut self) { - self.write_varint(0); - } - #[must_use] - pub fn freeze(self) -> Bytes { - self.bytes.freeze() - } -} diff --git a/gateways/kafka/src/protocol/header.rs b/gateways/kafka/src/protocol/header.rs index 932253579a..3016111b91 100644 --- a/gateways/kafka/src/protocol/header.rs +++ b/gateways/kafka/src/protocol/header.rs @@ -15,222 +15,26 @@ // specific language governing permissions and limitations // under the License. -#![allow( - clippy::doc_markdown, - clippy::missing_const_for_fn, - clippy::missing_errors_doc, - clippy::match_same_arms -)] +//! Request/response header version selection. +//! +//! `kafka_protocol::messages::ApiKey` owns the per-API flexible-encoding threshold table +//! (schema-generated from the Kafka message JSON, not hand-transcribed here). These wrappers +//! just add the gateway's policy for API keys the crate doesn't know about: an unrecognized +//! key was never negotiated through `ApiVersions`, so it can only have arrived on header v1 +//! (never flexible) and gets no response (`response_header_version` is unused for it). -use bytes::{BufMut, Bytes}; +use kafka_protocol::messages::ApiKey; -use crate::error::{KafkaProtocolError, Result}; -use crate::protocol::codec::{Decoder, Encoder}; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct RequestHeader { - pub api_key: i16, - pub api_version: i16, - pub correlation_id: i32, - pub client_id: Option, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ResponseHeader { - pub correlation_id: i32, -} - -/// First API version that uses flexible (compact) request encoding for `api_key`. -/// -/// `None` means the API never uses flexible encoding (request header stays v1 for every -/// version). Source of truth for kafka-tool framing and for [`request_header_version`]. -#[must_use] -pub fn first_flexible_version(api_key: i16) -> Option { - let threshold = first_flexible_version_threshold(api_key); - if threshold == i16::MAX { - None - } else { - Some(threshold) - } -} - -/// Threshold used by [`request_header_version`]. `i16::MAX` = never flexible. -fn first_flexible_version_threshold(api_key: i16) -> i16 { - match api_key { - 0 => 9, // Produce - 1 => 12, // Fetch - 2 => 6, // ListOffsets - 3 => 9, // Metadata - 4 => 4, // LeaderAndIsr - 5 => 2, // StopReplica - 6 => 6, // UpdateMetadata - 7 => 3, // ControlledShutdown - 8 => 8, // OffsetCommit - 9 => 6, // OffsetFetch - 10 => 3, // FindCoordinator - 11 => 6, // JoinGroup - 12 => 4, // Heartbeat - 13 => 4, // LeaveGroup - 14 => 4, // SyncGroup - 15 => 5, // DescribeGroups - 16 => 3, // ListGroups - 17 => i16::MAX, // SaslHandshake - never flexible - 18 => 3, // ApiVersions - 19 => 5, // CreateTopics - 20 => 4, // DeleteTopics - 21 => 2, // DeleteRecords - 22 => 2, // InitProducerId - 23 => 4, // OffsetForLeaderEpoch - 24 => 3, // AddPartitionsToTxn - 25 => 3, // AddOffsetsToTxn - 26 => 3, // EndTxn - 27 => 1, // WriteTxnMarkers - 28 => 3, // TxnOffsetCommit - 29 => 2, // DescribeAcls - 30 => 2, // CreateAcls - 31 => 2, // DeleteAcls - 32 => 4, // DescribeConfigs - 33 => 2, // AlterConfigs - 34 => 2, // AlterReplicaLogDirs - 35 => 2, // DescribeLogDirs - 36 => 2, // SaslAuthenticate - 37 => 2, // CreatePartitions - 38 => 2, // CreateDelegationToken - 39 => 2, // RenewDelegationToken - 40 => 2, // ExpireDelegationToken - 41 => 2, // DescribeDelegationToken - 42 => 2, // DeleteGroups - 43 => 2, // ElectLeaders - 44 => 1, // IncrementalAlterConfigs - 45 => 0, // AlterPartitionReassignments - always flexible - 46 => 0, // ListPartitionReassignments - always flexible - 47 => i16::MAX, // OffsetDelete - never flexible - 48 => 1, // DescribeClientQuotas - 49 => 1, // AlterClientQuotas - 50 => 0, // DescribeUserScramCredentials - always flexible - 51 => 0, // AlterUserScramCredentials - always flexible - 55 => 0, // DescribeQuorum - always flexible - 56 => 0, // AlterPartition - always flexible - 57 => 0, // UpdateFeatures - always flexible - 60 => 0, // DescribeCluster - always flexible - 61 => 0, // DescribeProducers - always flexible - 64 => 0, // UnregisterBroker - always flexible - 65 => 0, // DescribeTransactions - always flexible - 66 => 0, // ListTransactions - always flexible - 67 => 0, // AllocateProducerIds - always flexible - 68 => 0, // ConsumerGroupHeartbeat - always flexible - 69 => 0, // ConsumerGroupDescribe - always flexible - 71 => 0, // GetTelemetrySubscriptions - always flexible - 72 => 0, // PushTelemetry - always flexible - 74 => 0, // AssignReplicasToDirs - always flexible - 75 => 0, // DescribeTopicPartitions - always flexible - 76 => 0, // ListClientMetricsResources - always flexible - 77 => 0, // ShareGroupHeartbeat - always flexible (Kafka 4.0) - 78 => 0, // ShareGroupDescribe - always flexible - 79 => 0, // ShareFetch - always flexible - 80 => 0, // ShareAcknowledge - always flexible - _ => i16::MAX, // Unknown API - assume non-flexible - } -} - -/// Returns the request header version to use for a given (api_key, api_version) pair. -/// -/// Header v1 is the standard non-flexible format (nullable string client_id). -/// Header v2 is the flexible format (compact nullable string client_id + empty tagged fields). -/// The threshold at which each API key switches from v1 to v2 is defined by the Kafka protocol. #[must_use] pub fn request_header_version(api_key: i16, api_version: i16) -> i16 { - match first_flexible_version(api_key) { - Some(flexible_from) if api_version >= flexible_from => 2, - _ => 1, - } + ApiKey::try_from(api_key).map_or(1, |key| key.request_header_version(api_version)) } -/// Returns the response header version to use when replying to a given (api_key, api_version). +/// KIP-511 special case for `ApiVersions` (18): always header v0. /// -/// ApiVersions (18) is a special case: the server ALWAYS returns response header v0 (no tagged -/// fields) so that clients that don't yet know the server supports flexible encoding can still -/// parse the discovery response. All other flexible-version APIs use response header v1. +/// `ApiKey::response_header_version` already handles it - clients probing an unknown server +/// must be able to parse the discovery response before they know it supports flexible encoding. #[must_use] pub fn response_header_version(api_key: i16, api_version: i16) -> i16 { - if api_key == 18 { - return 0; - } - i16::from(request_header_version(api_key, api_version) >= 2) -} - -impl RequestHeader { - pub fn decode(bytes: Bytes, header_version: i16) -> Result { - let mut d = Decoder::new(bytes); - Self::decode_from(&mut d, header_version) - } - - /// Decode from a shared `Decoder`. - /// - /// Header v1 (non-flexible): - /// api_key i16 | api_version i16 | correlation_id i32 | client_id NULLABLE_STRING - /// - /// Header v2 (flexible): - /// api_key i16 | api_version i16 | correlation_id i32 - /// | client_id COMPACT_NULLABLE_STRING | _tagged_fields UNSIGNED_VARINT - pub fn decode_from(d: &mut Decoder, header_version: i16) -> Result { - match header_version { - 1 => { - let api_key = d.read_i16()?; - let api_version = d.read_i16()?; - let correlation_id = d.read_i32()?; - let client_id = d.read_nullable_string()?; - Ok(Self { - api_key, - api_version, - correlation_id, - client_id, - }) - } - 2 => { - let api_key = d.read_i16()?; - let api_version = d.read_i16()?; - let correlation_id = d.read_i32()?; - let client_id = d.read_compact_nullable_string()?; - d.read_tagged_fields()?; - Ok(Self { - api_key, - api_version, - correlation_id, - client_id, - }) - } - v => Err(KafkaProtocolError::UnsupportedHeaderVersion(v)), - } - } -} - -impl ResponseHeader { - /// Encode the response header. - /// - /// v0: correlation_id i32 (non-flexible APIs and ApiVersions) - /// v1: correlation_id i32 + empty tagged fields (flexible APIs) - #[must_use] - pub fn encode(&self, header_version: i16) -> Bytes { - let mut e = Encoder::with_capacity(5); - e.write_i32(self.correlation_id); - if header_version >= 1 { - e.write_empty_tagged_fields(); - } - e.freeze() - } - - /// Write this header directly into an existing buffer (avoids a separate heap alloc). - pub fn encode_into(&self, buf: &mut bytes::BytesMut, header_version: i16) { - buf.put_i32(self.correlation_id); - if header_version >= 1 { - buf.put_u8(0); // empty tagged fields - } - } - - /// Byte size of the encoded header for a given version. - #[must_use] - pub fn encoded_size(header_version: i16) -> usize { - if header_version >= 1 { 5 } else { 4 } - } + ApiKey::try_from(api_key).map_or(0, |key| key.response_header_version(api_version)) } diff --git a/gateways/kafka/src/protocol/mod.rs b/gateways/kafka/src/protocol/mod.rs index 3fe1fb544f..4d051f83d9 100644 --- a/gateways/kafka/src/protocol/mod.rs +++ b/gateways/kafka/src/protocol/mod.rs @@ -16,7 +16,5 @@ // under the License. pub mod api; -pub mod codec; pub mod header; -pub mod requests; pub mod responses; diff --git a/gateways/kafka/src/protocol/requests.rs b/gateways/kafka/src/protocol/requests.rs deleted file mode 100644 index be760a79a6..0000000000 --- a/gateways/kafka/src/protocol/requests.rs +++ /dev/null @@ -1,545 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Kafka request decoders for critical API keys - -#![allow(clippy::too_many_lines, clippy::doc_markdown)] -use crate::error::{KafkaProtocolError, Result}; -use crate::protocol::codec::{Decoder, PREALLOC_HINT}; -use bytes::Bytes; - -/// Produce Request (API Key 0) -#[derive(Debug, Clone)] -pub struct ProduceRequest { - pub transactional_id: Option, - pub acks: i16, - pub timeout_ms: i32, - pub topics: Vec, -} - -#[derive(Debug, Clone)] -pub struct ProduceTopicData { - pub topic: String, - pub partitions: Vec, -} - -#[derive(Debug, Clone)] -pub struct ProducePartitionData { - pub partition: i32, - pub records: Option, // Raw RecordBatch bytes -} - -/// Outcome of decoding a Produce request body. -/// -/// Carries `acks` on failure so callers can honor fire-and-forget (`acks=0`) silence -/// even when later fields are malformed. -#[derive(Debug)] -pub enum ProduceDecodeResult { - Ok(ProduceRequest), - Err { - acks: Option, - error: KafkaProtocolError, - }, -} - -impl ProduceDecodeResult { - /// Collapse to `Result` for tests and callers that only need a successful `ProduceRequest`. - /// - /// # Errors - /// - /// Returns an error if the byte stream is malformed or if the API version is unsupported. - pub fn into_request(self) -> Result { - match self { - Self::Ok(req) => Ok(req), - Self::Err { error, .. } => Err(error), - } - } -} - -macro_rules! produce_decode { - ($acks:expr, $expr:expr) => { - match $expr { - Ok(value) => value, - Err(error) => { - return ProduceDecodeResult::Err { acks: $acks, error }; - } - } - }; -} - -fn ensure_body_exhausted(d: &Decoder) -> Result<()> { - if d.remaining() != 0 { - return Err(KafkaProtocolError::UnexpectedTrailingBytes); - } - Ok(()) -} - -pub fn decode_produce_request(version: i16, body: Bytes) -> ProduceDecodeResult { - let mut d = Decoder::new(body); - let flexible = version >= 9; - - // transactional_id (v3+) - let transactional_id = if version >= 3 { - produce_decode!( - None, - if flexible { - d.read_compact_nullable_string() - } else { - d.read_nullable_string() - } - ) - } else { - None - }; - - let acks = produce_decode!(None, d.read_i16()); - let acks_read = Some(acks); - - let timeout_ms = produce_decode!(acks_read, d.read_i32()); - - // topics array - let topics_count = produce_decode!( - acks_read, - if flexible { - d.read_compact_array_count() - } else { - d.read_i32_array_count() - } - ); - - let mut topics = Vec::with_capacity(topics_count.min(PREALLOC_HINT)); - for _ in 0..topics_count { - let topic = produce_decode!( - acks_read, - if flexible { - d.read_compact_nullable_string() - } else { - d.read_nullable_string() - } - .and_then(|name| name.ok_or(KafkaProtocolError::NullTopicName)) - ); - - let partitions_count = produce_decode!( - acks_read, - if flexible { - d.read_compact_array_count() - } else { - d.read_i32_array_count() - } - ); - - let mut partitions = Vec::with_capacity(partitions_count.min(PREALLOC_HINT)); - for _ in 0..partitions_count { - let partition = produce_decode!(acks_read, d.read_i32()); - let records = produce_decode!( - acks_read, - if flexible { - d.read_compact_nullable_bytes() - } else { - d.read_nullable_bytes() - } - ); - partitions.push(ProducePartitionData { partition, records }); - if flexible { - produce_decode!(acks_read, d.read_tagged_fields()); - } - } - - topics.push(ProduceTopicData { topic, partitions }); - if flexible { - produce_decode!(acks_read, d.read_tagged_fields()); - } - } - - if flexible { - produce_decode!(acks_read, d.read_tagged_fields()); - } - produce_decode!(acks_read, ensure_body_exhausted(&d)); - - ProduceDecodeResult::Ok(ProduceRequest { - transactional_id, - acks, - timeout_ms, - topics, - }) -} - -/// Fetch Request (API Key 1) -#[derive(Debug, Clone)] -pub struct FetchRequest { - pub max_wait_ms: i32, - pub min_bytes: i32, - pub max_bytes: i32, - pub isolation_level: i8, - pub topics: Vec, -} - -#[derive(Debug, Clone)] -pub struct FetchTopic { - pub topic: String, - pub partitions: Vec, -} - -#[derive(Debug, Clone)] -pub struct FetchPartition { - pub partition: i32, - pub fetch_offset: i64, - pub partition_max_bytes: i32, -} -/// Decodes a raw byte stream into a `FetchRequest`. -/// -/// # Errors -/// -/// Returns an error if the byte stream is malformed or if the API version is unsupported. -pub fn decode_fetch_request(version: i16, body: Bytes) -> Result { - let mut d = Decoder::new(body); - let flexible = version >= 12; - - let _replica_id = d.read_i32()?; - let max_wait_ms = d.read_i32()?; - let min_bytes = d.read_i32()?; - - let max_bytes = if version >= 3 { - d.read_i32()? - } else { - 52_428_800 // default 50MB - }; - - let isolation_level = if version >= 4 { d.read_i8()? } else { 0 }; - - // session_id and session_epoch (v7+) - read and discard (stub path) - if version >= 7 { - d.read_i32()?; // session_id - d.read_i32()?; // session_epoch - } - - // topics array - let topics_count = if flexible { - d.read_compact_array_count()? - } else { - d.read_i32_array_count()? - }; - - let mut topics = Vec::with_capacity(topics_count.min(PREALLOC_HINT)); - for _ in 0..topics_count { - let topic = if flexible { - d.read_compact_nullable_string()? - .ok_or(KafkaProtocolError::NullTopicName)? - } else { - d.read_nullable_string()? - .ok_or(KafkaProtocolError::NullTopicName)? - }; - - let partitions_count = if flexible { - d.read_compact_array_count()? - } else { - d.read_i32_array_count()? - }; - - let mut partitions = Vec::with_capacity(partitions_count.min(PREALLOC_HINT)); - for _ in 0..partitions_count { - let partition = d.read_i32()?; - - if version >= 9 { - d.read_i32()?; // current_leader_epoch - } - - let fetch_offset = d.read_i64()?; - - if version >= 12 { - d.read_i32()?; // last_fetched_epoch - } - - if version >= 5 { - d.read_i64()?; // log_start_offset - } - - let partition_max_bytes = d.read_i32()?; - - partitions.push(FetchPartition { - partition, - fetch_offset, - partition_max_bytes, - }); - - if flexible { - d.read_tagged_fields()?; - } - } - - topics.push(FetchTopic { topic, partitions }); - if flexible { - d.read_tagged_fields()?; - } - } - - // forgotten_topics_data (v7+) - skip - if version >= 7 { - // forgottenTopicsData is nullable on the wire in practice (null = none forgotten). - let forgotten_count = if flexible { - d.read_compact_array_count_nullable()? - } else { - d.read_i32_array_count_nullable()? - }; - for _ in 0..forgotten_count { - if flexible { - d.read_compact_nullable_string()?; - let partitions_count = d.read_compact_array_count()?; - for _ in 0..partitions_count { - d.read_i32()?; - } - d.read_tagged_fields()?; - } else { - d.read_nullable_string()?; - let partitions_count = d.read_i32_array_count()?; - for _ in 0..partitions_count { - d.read_i32()?; - } - } - } - } - - // rack_id (v11+) - if version >= 11 { - if flexible { - d.read_compact_nullable_string()?; - } else { - d.read_nullable_string()?; - } - } - - if flexible { - d.read_tagged_fields()?; - } - ensure_body_exhausted(&d)?; - - Ok(FetchRequest { - max_wait_ms, - min_bytes, - max_bytes, - isolation_level, - topics, - }) -} - -/// ListOffsets Request (API Key 2) -#[derive(Debug, Clone)] -pub struct ListOffsetsRequest { - pub isolation_level: i8, - pub topics: Vec, -} - -#[derive(Debug, Clone)] -pub struct ListOffsetsTopic { - pub topic: String, - pub partitions: Vec, -} - -#[derive(Debug, Clone)] -pub struct ListOffsetsPartition { - pub partition: i32, - pub timestamp: i64, // -2 = earliest, -1 = latest -} -/// Decodes a raw byte stream into a `ListOffsetsRequest`. -/// -/// # Errors -/// -/// Returns an error if the byte stream is malformed or if the API version is unsupported. -pub fn decode_list_offsets_request(version: i16, body: Bytes) -> Result { - let mut d = Decoder::new(body); - let flexible = version >= 6; - - let _replica_id = d.read_i32()?; - - let isolation_level = if version >= 2 { d.read_i8()? } else { 0 }; - - let topics_count = if flexible { - d.read_compact_array_count()? - } else { - d.read_i32_array_count()? - }; - - let mut topics = Vec::with_capacity(topics_count.min(PREALLOC_HINT)); - for _ in 0..topics_count { - let topic = if flexible { - d.read_compact_nullable_string()? - .ok_or(KafkaProtocolError::NullTopicName)? - } else { - d.read_nullable_string()? - .ok_or(KafkaProtocolError::NullTopicName)? - }; - - let partitions_count = if flexible { - d.read_compact_array_count()? - } else { - d.read_i32_array_count()? - }; - - let mut partitions = Vec::with_capacity(partitions_count.min(PREALLOC_HINT)); - for _ in 0..partitions_count { - let partition = d.read_i32()?; - - if version >= 4 { - d.read_i32()?; // current_leader_epoch - } - - let timestamp = d.read_i64()?; - - if version == 0 { - d.read_i32()?; // max_num_offsets (deprecated) - } - - partitions.push(ListOffsetsPartition { - partition, - timestamp, - }); - - if flexible { - d.read_tagged_fields()?; - } - } - - topics.push(ListOffsetsTopic { topic, partitions }); - if flexible { - d.read_tagged_fields()?; - } - } - - if flexible { - d.read_tagged_fields()?; - } - ensure_body_exhausted(&d)?; - - Ok(ListOffsetsRequest { - isolation_level, - topics, - }) -} - -/// `CreateTopics` Request (API Key 19) -#[derive(Debug, Clone)] -pub struct CreateTopicsRequest { - pub topics: Vec, - pub timeout_ms: i32, - pub validate_only: bool, -} - -#[derive(Debug, Clone)] -pub struct CreatableTopic { - pub name: String, - pub num_partitions: i32, - pub replication_factor: i16, - /// True when the request included a non-empty manual partition assignment. - /// KIP-464: on v2/v3, `num_partitions = -1` / `replication_factor = -1` are valid - /// only when assignments are present; v4+ allows them as broker-default sentinels - /// even without assignments. - pub has_assignments: bool, -} - -/// Decodes a raw byte stream into a `CreateTopicsRequest`. -/// -/// # Errors -/// -/// Returns an error if the byte stream is malformed or if the API version is unsupported. -pub fn decode_create_topics_request(version: i16, body: Bytes) -> Result { - let mut d = Decoder::new(body); - let flexible = version >= 5; - - let topics_count = if flexible { - d.read_compact_array_count()? - } else { - d.read_i32_array_count()? - }; - - let mut topics = Vec::with_capacity(topics_count.min(PREALLOC_HINT)); - for _ in 0..topics_count { - let name = if flexible { - d.read_compact_nullable_string()? - .ok_or(KafkaProtocolError::NullTopicName)? - } else { - d.read_nullable_string()? - .ok_or(KafkaProtocolError::NullTopicName)? - }; - - let num_partitions = d.read_i32()?; - let replication_factor = d.read_i16()?; - - // assignments (COMPACT_ARRAY or ARRAY) - let assignments_count = if flexible { - d.read_compact_array_count()? - } else { - d.read_i32_array_count()? - }; - let has_assignments = assignments_count > 0; - for _ in 0..assignments_count { - d.read_i32()?; // partition_index - let replicas_count = if flexible { - d.read_compact_array_count()? - } else { - d.read_i32_array_count()? - }; - for _ in 0..replicas_count { - d.read_i32()?; // broker_id - } - if flexible { - d.read_tagged_fields()?; - } - } - - // configs (COMPACT_ARRAY or ARRAY) - skip - let configs_count = if flexible { - d.read_compact_array_count()? - } else { - d.read_i32_array_count()? - }; - for _ in 0..configs_count { - if flexible { - d.read_compact_nullable_string()?; // name - d.read_compact_nullable_string()?; // value - d.read_tagged_fields()?; - } else { - d.read_nullable_string()?; - d.read_nullable_string()?; - } - } - - topics.push(CreatableTopic { - name, - num_partitions, - replication_factor, - has_assignments, - }); - - if flexible { - d.read_tagged_fields()?; - } - } - - let timeout_ms = d.read_i32()?; - let validate_only = if version >= 1 { d.read_bool()? } else { false }; - - if flexible { - d.read_tagged_fields()?; - } - ensure_body_exhausted(&d)?; - - Ok(CreateTopicsRequest { - topics, - timeout_ms, - validate_only, - }) -} diff --git a/gateways/kafka/src/protocol/responses.rs b/gateways/kafka/src/protocol/responses.rs index 5d87d0c511..f48c770f9a 100644 --- a/gateways/kafka/src/protocol/responses.rs +++ b/gateways/kafka/src/protocol/responses.rs @@ -16,314 +16,253 @@ // under the License. //! Kafka response encoders (stub implementations). +//! +//! Wire encoding (field order, version gating, compact vs. legacy shapes) is +//! `kafka_protocol`'s responsibility; everything here is stub *policy* - which placeholder +//! values and error codes a request gets back before the Iggy bridge lands. + +use bytes::{Bytes, BytesMut}; +use kafka_protocol::messages::create_topics_request::CreatableTopic; +use kafka_protocol::messages::create_topics_response::CreatableTopicResult; +use kafka_protocol::messages::fetch_response::{FetchableTopicResponse, PartitionData}; +use kafka_protocol::messages::list_offsets_response::{ + ListOffsetsPartitionResponse, ListOffsetsTopicResponse, +}; +use kafka_protocol::messages::produce_response::{PartitionProduceResponse, TopicProduceResponse}; +use kafka_protocol::messages::{ + CreateTopicsRequest, CreateTopicsResponse, FetchRequest, FetchResponse, ListOffsetsRequest, + ListOffsetsResponse, ProduceRequest, ProduceResponse, +}; +use kafka_protocol::protocol::Encodable; -#![allow(clippy::doc_markdown)] - +use crate::error::{KafkaProtocolError, Result}; use crate::protocol::api::{ ERROR_INVALID_PARTITIONS, ERROR_INVALID_REPLICATION_FACTOR, ERROR_NONE, ERROR_NOT_CONTROLLER, ERROR_NOT_LEADER_OR_FOLLOWER, }; -use crate::protocol::codec::Encoder; -use crate::protocol::requests::{ - CreatableTopic, CreateTopicsRequest, FetchPartition, FetchRequest, FetchTopic, - ListOffsetsPartition, ListOffsetsRequest, ListOffsetsTopic, ProducePartitionData, - ProduceRequest, ProduceTopicData, -}; -use bytes::Bytes; -/// Well-formed Produce response with a single placeholder topic/partition. -#[must_use] -pub fn encode_produce_error_response(version: i16, error_code: i16) -> Bytes { - let topics = vec![ProduceTopicData { - topic: String::new(), // TODO topic name will be populated in the end to end functional completion - partitions: vec![ProducePartitionData { - partition: 0, - records: None, - }], - }]; - encode_produce_response_inner(version, &topics, error_code) -} -#[must_use] -pub fn encode_produce_response(version: i16, req: &ProduceRequest) -> Bytes { - // Stub: discard payload and return a retriable error so clients keep data locally - // until the Iggy bridge lands (do not advertise silent success). - encode_produce_response_inner(version, &req.topics, ERROR_NOT_LEADER_OR_FOLLOWER) -} - -fn encode_produce_response_inner( +/// Encode a `kafka_protocol` message, mapping its `anyhow::Error` (the crate has no stable +/// decode/encode error taxonomy) to a variant callers can log or fold into [`HandleOutcome::Close`]. +/// +/// [`HandleOutcome::Close`]: crate::protocol::api::HandleOutcome::Close +pub(crate) fn encode_message( + msg: &T, version: i16, - topics: &[ProduceTopicData], - partition_error: i16, -) -> Bytes { - let flexible = version >= 9; - let mut e = Encoder::with_capacity(512); - - if flexible { - e.write_varint((topics.len() + 1) as u64); - } else { - e.write_i32(i32::try_from(topics.len()).expect("topic count bounded")); - } - - for topic in topics { - if flexible { - e.write_compact_nullable_string(Some(&topic.topic)); - } else { - e.write_nullable_string_unchecked(Some(&topic.topic)); - } - - if flexible { - e.write_varint((topic.partitions.len() + 1) as u64); - } else { - e.write_i32(i32::try_from(topic.partitions.len()).expect("partition count bounded")); - } + capacity: usize, +) -> Result { + let mut buf = BytesMut::with_capacity(capacity); + msg.encode(&mut buf, version) + .map_err(|e| KafkaProtocolError::Malformed(e.to_string()))?; + Ok(buf.freeze()) +} - for p in &topic.partitions { - e.write_i32(p.partition); - e.write_i16(partition_error); - e.write_i64(0); - if version >= 2 { - e.write_i64(-1); - } - if version >= 5 { - e.write_i64(0); - } - if version >= 8 { - if flexible { - e.write_varint(1); - e.write_compact_nullable_string(None); - } else { - e.write_i32(0); - e.write_nullable_string_unchecked(None); - } - } - if flexible { - e.write_empty_tagged_fields(); - } - } +// ── Produce ────────────────────────────────────────────────────────────────── - if flexible { - e.write_empty_tagged_fields(); - } - } +/// Well-formed Produce response with a single placeholder topic/partition. +/// +/// # Errors +/// +/// Returns an error when `kafka_protocol` cannot encode the response at `version`. +pub fn encode_produce_error_response(version: i16, error_code: i16) -> Result { + let resp = ProduceResponse::default().with_responses(vec![ + TopicProduceResponse::default() + .with_partition_responses(vec![produce_partition_response(0, error_code)]), + ]); + encode_message(&resp, version, 512) +} - if version >= 1 { - e.write_i32(0); - } - if flexible { - e.write_empty_tagged_fields(); - } +/// Stub: discard the payload and return a retriable error so clients keep data locally until +/// the Iggy bridge lands (do not advertise silent success). +/// +/// # Errors +/// +/// Returns an error when `kafka_protocol` cannot encode the response at `version`. +pub fn encode_produce_response(version: i16, req: &ProduceRequest) -> Result { + let responses = req + .topic_data + .iter() + .map(|topic| { + TopicProduceResponse::default() + .with_name(topic.name.clone()) + .with_partition_responses( + topic + .partition_data + .iter() + .map(|p| produce_partition_response(p.index, ERROR_NOT_LEADER_OR_FOLLOWER)) + .collect(), + ) + }) + .collect(); + let resp = ProduceResponse::default().with_responses(responses); + encode_message(&resp, version, 512) +} - e.freeze() +fn produce_partition_response(index: i32, error_code: i16) -> PartitionProduceResponse { + PartitionProduceResponse::default() + .with_index(index) + .with_error_code(error_code) + .with_log_start_offset(0) } +// ── Fetch ──────────────────────────────────────────────────────────────────── + /// Well-formed Fetch response. Uses top-level `error_code` at v7+, or a single /// placeholder topic/partition with per-partition `error_code` below v7. -#[must_use] -pub fn encode_fetch_error_response(version: i16, error_code: i16) -> Bytes { +/// +/// # Errors +/// +/// Returns an error when `kafka_protocol` cannot encode the response at `version`. +pub fn encode_fetch_error_response(version: i16, error_code: i16) -> Result { if version >= 7 { - return encode_fetch_response_inner(version, &[], Some(error_code), error_code); + return encode_fetch_response_inner(version, Vec::new(), error_code); } - - let topics = vec![FetchTopic { - topic: String::new(), - partitions: vec![FetchPartition { - partition: 0, - fetch_offset: 0, - partition_max_bytes: 1, - }], - }]; - encode_fetch_response_inner(version, &topics, Some(ERROR_NONE), error_code) + // No top-level error field below v7; the error surfaces on the placeholder partition instead. + let topics = vec![ + FetchableTopicResponse::default() + .with_partitions(vec![fetch_partition_response(0, error_code)]), + ]; + encode_fetch_response_inner(version, topics, ERROR_NONE) } -#[must_use] -pub fn encode_fetch_response(version: i16, req: &FetchRequest) -> Bytes { - // Stub: discard payload and return a retriable error so clients don't mistake - // "no real data yet" for a genuinely empty partition (same philosophy as Produce). - encode_fetch_response_inner( - version, - &req.topics, - Some(ERROR_NONE), - ERROR_NOT_LEADER_OR_FOLLOWER, - ) + +/// Stub: discard the payload and return a retriable error so clients don't mistake "no real +/// data yet" for a genuinely empty partition (same philosophy as Produce). +/// +/// # Errors +/// +/// Returns an error when `kafka_protocol` cannot encode the response at `version`. +pub fn encode_fetch_response(version: i16, req: &FetchRequest) -> Result { + let topics = req + .topics + .iter() + .map(|topic| { + FetchableTopicResponse::default() + .with_topic(topic.topic.clone()) + .with_partitions( + topic + .partitions + .iter() + .map(|p| { + fetch_partition_response(p.partition, ERROR_NOT_LEADER_OR_FOLLOWER) + }) + .collect(), + ) + }) + .collect(); + encode_fetch_response_inner(version, topics, ERROR_NONE) } fn encode_fetch_response_inner( version: i16, - topics: &[crate::protocol::requests::FetchTopic], - top_level_error: Option, - partition_error: i16, -) -> Bytes { - let flexible = version >= 12; - let mut e = Encoder::with_capacity(512); - - if version >= 1 { - e.write_i32(0); - } - if version >= 7 { - e.write_i16(top_level_error.unwrap_or(ERROR_NONE)); - e.write_i32(0); - } - - if flexible { - e.write_varint((topics.len() + 1) as u64); - } else { - e.write_i32(i32::try_from(topics.len()).expect("topic count bounded")); - } - - for topic in topics { - if flexible { - e.write_compact_nullable_string(Some(&topic.topic)); - } else { - e.write_nullable_string_unchecked(Some(&topic.topic)); - } - - if flexible { - e.write_varint((topic.partitions.len() + 1) as u64); - } else { - e.write_i32(i32::try_from(topic.partitions.len()).expect("partition count bounded")); - } - - for partition in &topic.partitions { - e.write_i32(partition.partition); - e.write_i16(partition_error); - e.write_i64(0); // high_watermark - if version >= 4 { - e.write_i64(0); // last_stable_offset - } - if version >= 5 { - e.write_i64(0); // log_start_offset - } - if version >= 4 { - if flexible { - e.write_varint(1); // empty aborted_transactions - } else { - e.write_i32(0); // empty aborted_transactions - } - } - if version >= 11 { - e.write_i32(-1); // preferred_read_replica - } - if flexible { - e.write_compact_nullable_bytes(None); - } else { - e.write_null_bytes(); - } - if flexible { - e.write_empty_tagged_fields(); - } - } + topics: Vec, + top_level_error: i16, +) -> Result { + let resp = FetchResponse::default() + .with_error_code(top_level_error) + .with_responses(topics); + encode_message(&resp, version, 512) +} - if flexible { - e.write_empty_tagged_fields(); - } - } +fn fetch_partition_response(partition: i32, error_code: i16) -> PartitionData { + PartitionData::default() + .with_partition_index(partition) + .with_error_code(error_code) + .with_last_stable_offset(0) + .with_log_start_offset(0) + .with_records(None) +} - if flexible { - e.write_empty_tagged_fields(); - } +// ── ListOffsets ────────────────────────────────────────────────────────────── - e.freeze() +/// Well-formed `ListOffsets` response with a single placeholder topic/partition. +/// +/// `kafka_protocol` has no encodable representation for `ListOffsets` v0 (the legacy +/// `old_style_offsets` shape predates the schema this crate generates from); a v0 request now +/// falls through [`super::api::unsupported_version_response`]'s encode-failure path to `Close` +/// instead of the pre-migration downgraded response. +/// +/// # Errors +/// +/// Returns an error when `kafka_protocol` cannot encode the response at `version` (always the +/// case for `version == 0`). +pub fn encode_list_offsets_error_response(version: i16, error_code: i16) -> Result { + let topics = vec![ + ListOffsetsTopicResponse::default() + .with_partitions(vec![list_offsets_partition_response(0, error_code)]), + ]; + encode_list_offsets_response_inner(version, topics) } -/// Well-formed ListOffsets response with a single placeholder topic/partition. -#[must_use] -pub fn encode_list_offsets_error_response(version: i16, error_code: i16) -> Bytes { - let topics = vec![ListOffsetsTopic { - topic: String::new(), - partitions: vec![ListOffsetsPartition { - partition: 0, - timestamp: -1, - }], - }]; - encode_list_offsets_response_inner(version, &topics, error_code) -} -#[must_use] -pub fn encode_list_offsets_response(version: i16, req: &ListOffsetsRequest) -> Bytes { - // Stub: discard payload and return a retriable error, matching Produce/Fetch - a genuine - // offset lookup requires the same partition-leadership the stub doesn't have yet. - encode_list_offsets_response_inner(version, &req.topics, ERROR_NOT_LEADER_OR_FOLLOWER) +/// Stub: discard the payload and return a retriable error, matching Produce/Fetch - a genuine +/// offset lookup requires the same partition-leadership the stub doesn't have yet. +/// +/// # Errors +/// +/// Returns an error when `kafka_protocol` cannot encode the response at `version`. +pub fn encode_list_offsets_response(version: i16, req: &ListOffsetsRequest) -> Result { + let topics = req + .topics + .iter() + .map(|topic| { + ListOffsetsTopicResponse::default() + .with_name(topic.name.clone()) + .with_partitions( + topic + .partitions + .iter() + .map(|p| { + list_offsets_partition_response( + p.partition_index, + ERROR_NOT_LEADER_OR_FOLLOWER, + ) + }) + .collect(), + ) + }) + .collect(); + encode_list_offsets_response_inner(version, topics) } fn encode_list_offsets_response_inner( version: i16, - topics: &[crate::protocol::requests::ListOffsetsTopic], - partition_error: i16, -) -> Bytes { - let flexible = version >= 6; - let mut e = Encoder::with_capacity(256); - - if version >= 2 { - e.write_i32(0); - } - - if flexible { - e.write_varint((topics.len() + 1) as u64); - } else { - e.write_i32(i32::try_from(topics.len()).expect("topic count bounded")); - } - - for topic in topics { - if flexible { - e.write_compact_nullable_string(Some(&topic.topic)); - } else { - e.write_nullable_string_unchecked(Some(&topic.topic)); - } - - if flexible { - e.write_varint((topic.partitions.len() + 1) as u64); - } else { - e.write_i32(i32::try_from(topic.partitions.len()).expect("partition count bounded")); - } - - for partition in &topic.partitions { - e.write_i32(partition.partition); - e.write_i16(partition_error); - - if version == 0 { - // v0 has no `timestamp`/`offset` fields; it returns the legacy - // `old_style_offsets` ARRAY (i32 count + i64 entries) instead. - // Empty since this stub never resolves a real offset. - e.write_i32(0); - } else { - e.write_i64(-1); // timestamp: -1 = not available (Kafka sentinel) - e.write_i64(0); // offset - if version >= 4 { - e.write_i32(-1); // leader_epoch - } - } - if flexible { - e.write_empty_tagged_fields(); - } - } - - if flexible { - e.write_empty_tagged_fields(); - } - } - - if flexible { - e.write_empty_tagged_fields(); - } + topics: Vec, +) -> Result { + let resp = ListOffsetsResponse::default().with_topics(topics); + encode_message(&resp, version, 256) +} - e.freeze() +fn list_offsets_partition_response( + partition: i32, + error_code: i16, +) -> ListOffsetsPartitionResponse { + ListOffsetsPartitionResponse::default() + .with_partition_index(partition) + .with_error_code(error_code) } -/// Well-formed CreateTopics response with a single placeholder topic. -#[must_use] -pub fn encode_create_topics_error_response(version: i16, error_code: i16) -> Bytes { - let topics = vec![CreatableTopic { - name: String::new(), - num_partitions: 1, - replication_factor: 1, - has_assignments: false, - }]; +// ── CreateTopics ───────────────────────────────────────────────────────────── + +/// Well-formed `CreateTopics` response with a single placeholder topic. +/// +/// # Errors +/// +/// Returns an error when `kafka_protocol` cannot encode the response at `version`. +pub fn encode_create_topics_error_response(version: i16, error_code: i16) -> Result { + let topics = vec![ + CreatableTopic::default() + .with_num_partitions(1) + .with_replication_factor(1), + ]; encode_create_topics_response_inner(version, &topics, error_code) } -#[must_use] -pub fn encode_create_topics_response(version: i16, req: &CreateTopicsRequest) -> Bytes { + +/// # Errors +/// +/// Returns an error when `kafka_protocol` cannot encode the response at `version`. +pub fn encode_create_topics_response(version: i16, req: &CreateTopicsRequest) -> Result { encode_create_topics_response_inner(version, &req.topics, ERROR_NONE) } -/// Resolve per-topic CreateTopics error. +/// Resolve per-topic `CreateTopics` error. /// /// KIP-464: `num_partitions = -1` / `replication_factor = -1` mean broker default when either /// (a) the version is v4+, or (b) the topic carries a manual partition assignment (valid on @@ -335,7 +274,7 @@ const fn create_topics_topic_error(version: i16, topic: &CreatableTopic, forced_ return forced_error; } - let broker_default_ok = version >= 4 || topic.has_assignments; + let broker_default_ok = version >= 4 || !topic.assignments.is_empty(); let partitions_ok = if broker_default_ok { topic.num_partitions == -1 || topic.num_partitions > 0 @@ -362,52 +301,18 @@ fn encode_create_topics_response_inner( version: i16, topics: &[CreatableTopic], topic_error: i16, -) -> Bytes { - let flexible = version >= 5; - let mut e = Encoder::with_capacity(256); - - if version >= 2 { - e.write_i32(0); - } - - if flexible { - e.write_varint((topics.len() + 1) as u64); - } else { - e.write_i32(i32::try_from(topics.len()).expect("topic count bounded")); - } - - for topic in topics { - if flexible { - e.write_compact_nullable_string(Some(&topic.name)); - } else { - e.write_nullable_string_unchecked(Some(&topic.name)); - } - - let error_code = create_topics_topic_error(version, topic, topic_error); - e.write_i16(error_code); - - if version >= 1 { - if flexible { - e.write_compact_nullable_string(None); - } else { - e.write_nullable_string_unchecked(None); - } - } - - if version >= 5 { - e.write_i32(topic.num_partitions); - e.write_i16(topic.replication_factor); - e.write_varint(1); - } - - if flexible { - e.write_empty_tagged_fields(); - } - } - - if flexible { - e.write_empty_tagged_fields(); - } - - e.freeze() +) -> Result { + let results = topics + .iter() + .map(|topic| { + CreatableTopicResult::default() + .with_name(topic.name.clone()) + .with_error_code(create_topics_topic_error(version, topic, topic_error)) + .with_error_message(None) + .with_num_partitions(topic.num_partitions) + .with_replication_factor(topic.replication_factor) + }) + .collect(); + let resp = CreateTopicsResponse::default().with_topics(results); + encode_message(&resp, version, 256) } diff --git a/gateways/kafka/src/server.rs b/gateways/kafka/src/server.rs index 8a05215594..c3d8dc208e 100644 --- a/gateways/kafka/src/server.rs +++ b/gateways/kafka/src/server.rs @@ -20,6 +20,8 @@ use std::sync::Arc; use std::time::Duration; use bytes::{Buf, BufMut, Bytes, BytesMut}; +use kafka_protocol::messages::RequestHeader; +use kafka_protocol::protocol::Decodable; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; use tokio::sync::{Semaphore, broadcast}; @@ -30,10 +32,7 @@ use tracing::{debug, error, info, warn}; use crate::error::{KafkaProtocolError, Result}; use crate::protocol::api::{BrokerAdvertise, DEFAULT_KAFKA_PORT, HandleOutcome, handle_request}; -use crate::protocol::codec::Decoder; -use crate::protocol::header::{ - RequestHeader, ResponseHeader, request_header_version, response_header_version, -}; +use crate::protocol::header::{request_header_version, response_header_version}; use std::io; const READ_CHUNK: usize = 65536; @@ -286,23 +285,25 @@ async fn handle_connection( let req_hdr_ver = request_header_version(api_key, api_version); let resp_hdr_ver = response_header_version(api_key, api_version); - // `request_header_version` only ever returns 1 or 2, both of which `decode_from` - // handles, so its `UnsupportedHeaderVersion` arm is unreachable here; any decode error - // is a malformed header and closes the connection. - let mut decoder = Decoder::new(frame); - let req = RequestHeader::decode_from(&mut decoder, req_hdr_ver)?; + // `request_header_version` only ever returns 1 or 2, both of which `RequestHeader` + // supports, so `decode` cannot fail on the version argument itself; any error here is a + // malformed header and closes the connection. + let mut body = frame; + let req = RequestHeader::decode(&mut body, req_hdr_ver) + .map_err(|e| KafkaProtocolError::Malformed(e.to_string()))?; debug!( %peer, - api_key = req.api_key, - api_version = req.api_version, + api_key = req.request_api_key, + api_version = req.request_api_version, correlation_id = req.correlation_id, client_id = req.client_id.as_deref().unwrap_or(""), "received request" ); - let body = decoder.read_bytes(decoder.remaining())?; - let outcome = handle_request(req.api_key, req.api_version, body, &broker); + // `RequestHeader::decode` advances `body` past the header fields it consumed via + // `Buf::advance`, so `body` is already exactly the request payload. + let outcome = handle_request(req.request_api_key, req.request_api_version, body, &broker); if dispatch_outcome(&mut stream, &peer, &config, &req, resp_hdr_ver, outcome).await? { return Ok(()); } @@ -357,19 +358,16 @@ async fn dispatch_outcome( HandleOutcome::Close => { warn!( %peer, - api_key = req.api_key, - api_version = req.api_version, + api_key = req.request_api_key, + api_version = req.request_api_version, "closing connection: no parseable response for this request" ); Ok(true) } HandleOutcome::Respond(body_response) => { - let resp_header = ResponseHeader { - correlation_id: req.correlation_id, - }; send_response( stream, - &resp_header, + req.correlation_id, resp_hdr_ver, body_response, config.write_timeout, @@ -380,16 +378,24 @@ async fn dispatch_outcome( } } +/// Response header size for a given header version: v0 is `correlation_id` only (4 bytes); v1 +/// adds an empty tagged-fields byte (5 bytes). Kept inline rather than through +/// `kafka_protocol::messages::ResponseHeader` - that type's `Encodable` impl needs its own +/// `BytesMut` allocation, defeating the single-allocation framing this function exists for. +const fn response_header_size(header_version: i16) -> usize { + if header_version >= 1 { 5 } else { 4 } +} + /// Write a single length-prefixed Kafka frame using one allocation. /// Avoids the separate header-encode + payload-concat + length-prefix allocations. async fn send_response( stream: &mut TcpStream, - header: &ResponseHeader, + correlation_id: i32, header_version: i16, body: Bytes, write_timeout: Duration, ) -> Result<()> { - let header_size = ResponseHeader::encoded_size(header_version); + let header_size = response_header_size(header_version); let payload_size = header_size + body.len(); let payload_len_i32 = i32::try_from(payload_size).map_err(|_| KafkaProtocolError::FrameTooLarge { @@ -398,7 +404,10 @@ async fn send_response( })?; let mut prefix = BytesMut::with_capacity(4 + header_size); prefix.put_i32(payload_len_i32); - header.encode_into(&mut prefix, header_version); + prefix.put_i32(correlation_id); + if header_version >= 1 { + prefix.put_u8(0); // empty tagged fields + } let mut frame = prefix.freeze().chain(body); timeout(write_timeout, stream.write_all_buf(&mut frame)) .await @@ -517,14 +526,11 @@ mod tests { #[tokio::test] async fn send_response_writes_header_and_body() { let (mut client, mut server) = tcp_pair().await; - let header = ResponseHeader { - correlation_id: 0x0102_0304, - }; let body = [9u8, 8, 7]; send_response( &mut server, - &header, + 0x0102_0304, 1, Bytes::copy_from_slice(&body), Duration::from_secs(1), @@ -774,14 +780,11 @@ mod tests { #[tokio::test] async fn send_response_v0_writes_correlation_id_only() { let (mut client, mut server) = tcp_pair().await; - let header = ResponseHeader { - correlation_id: 0x0000_00AB, - }; let body = [5u8, 6, 7]; send_response( &mut server, - &header, + 0x0000_00AB, 0, Bytes::copy_from_slice(&body), Duration::from_secs(1), diff --git a/gateways/kafka/tests/api_handler_tests.rs b/gateways/kafka/tests/api_handler_tests.rs index 8d3c32eee9..e2cf324813 100644 --- a/gateways/kafka/tests/api_handler_tests.rs +++ b/gateways/kafka/tests/api_handler_tests.rs @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +#[path = "common/codec.rs"] +mod codec; #[path = "common/fixtures.rs"] mod fixtures; #[path = "common/scope.rs"] @@ -32,9 +34,8 @@ use iggy_gateway_kafka::protocol::api::{ ERROR_NOT_LEADER_OR_FOLLOWER, ERROR_UNKNOWN_TOPIC_OR_PARTITION, ERROR_UNSUPPORTED_VERSION, handle_request, is_supported_version, supported_api_ranges, }; -use iggy_gateway_kafka::protocol::codec::Decoder; -use iggy_gateway_kafka::protocol::requests::{ProduceDecodeResult, decode_produce_request}; +use codec::Decoder; use fixtures::load_fixture_body_or_skip; use scope::default_broker; use tcp::{build_metadata_legacy_request, build_produce_v3_body}; @@ -194,21 +195,21 @@ fn apiversions_unsupported_version_uses_v0_encoding_without_throttle() { } #[test] -fn produce_malformed_body_with_acks_one_returns_invalid_request() { +fn produce_malformed_body_with_acks_one_stays_silent() { + // `kafka_protocol` decodes Produce in one shot, so a decode failure never exposes `acks` + // (unlike the pre-migration field-by-field decoder, which could still answer with + // INVALID_REQUEST once it knew acks was nonzero). Every Produce decode failure now stays + // silent rather than risk desyncing an acks=0 fire-and-forget client's correlation stream. let body = Bytes::from_static(&[ 0xff, 0xff, // null transactional_id 0x00, 0x01, // acks = 1 0x00, 0x00, 0x03, 0xe8, // timeout_ms 0x00, 0x00, 0x00, 0x01, // one topic ]); - let response = handle_request(API_KEY_PRODUCE, 3, body, &default_broker()) - .expect_response("acks=1 malformed produce should get error response"); - let mut d = Decoder::new(response); - assert_eq!(d.read_i32().unwrap(), 1); - assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); - assert_eq!(d.read_i32().unwrap(), 1); - assert_eq!(d.read_i32().unwrap(), 0); - assert_eq!(d.read_i16().unwrap(), ERROR_INVALID_REQUEST); + assert!( + handle_request(API_KEY_PRODUCE, 3, body, &default_broker()).is_no_response(), + "malformed Produce body must stay silent regardless of acks" + ); } #[test] @@ -694,12 +695,11 @@ fn create_topics_stub_response_returns_not_controller() { // ── Produce acks=0 (broker must stay silent even on a malformed body) ────── #[test] -fn produce_acks_zero_malformed_body_decode_carries_acks() { +fn produce_acks_zero_malformed_body_stays_silent() { + // topics array declares 1 element but no topic data follows - decode fails after acks=0 + // was on the wire, though `kafka_protocol`'s one-shot decode no longer exposes that acks + // was read. The handler must stay silent regardless. let body = build_produce_v3_body(0, 1); - match decode_produce_request(3, body.clone()) { - ProduceDecodeResult::Err { acks: Some(0), .. } => {} - other => panic!("expected decode error with acks=0, got {other:?}"), - } assert!( handle_request(API_KEY_PRODUCE, 3, body, &default_broker()).is_no_response(), "handler must not respond when acks=0 even if decode fails after acks" diff --git a/gateways/kafka/tests/broker_advertise_tests.rs b/gateways/kafka/tests/broker_advertise_tests.rs index 4bba7d5faf..5877b41cb5 100644 --- a/gateways/kafka/tests/broker_advertise_tests.rs +++ b/gateways/kafka/tests/broker_advertise_tests.rs @@ -17,11 +17,15 @@ //! `BrokerAdvertise` parsing and metadata reflection. +#[path = "common/codec.rs"] +mod codec; + use std::net::SocketAddr; use iggy_gateway_kafka::ServerConfig; use iggy_gateway_kafka::protocol::api::{API_KEY_METADATA, BrokerAdvertise, handle_request}; -use iggy_gateway_kafka::protocol::codec::{Decoder, Encoder}; + +use codec::{Decoder, Encoder}; #[test] fn default_matches_standard_gateway_port() { diff --git a/gateways/kafka/tests/codec_tests.rs b/gateways/kafka/tests/codec_tests.rs deleted file mode 100644 index e7eeb12733..0000000000 --- a/gateways/kafka/tests/codec_tests.rs +++ /dev/null @@ -1,230 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use bytes::Bytes; - -use iggy_gateway_kafka::error::KafkaProtocolError; -use iggy_gateway_kafka::protocol::codec::{Decoder, Encoder}; - -#[test] -fn codec_round_trip_primitives_and_nullable_fields() { - let mut enc = Encoder::with_capacity(128); - enc.write_i8(-3); - enc.write_i16(42); - enc.write_i32(123_456); - enc.write_i64(9_999_999); - enc.write_nullable_string(Some("client-a")).unwrap(); - enc.write_nullable_string(None).unwrap(); - enc.write_nullable_bytes(Some(&[1, 2, 3])).unwrap(); - enc.write_nullable_bytes(None).unwrap(); - let bytes = enc.freeze(); - - let mut dec = Decoder::new(bytes); - assert_eq!(dec.read_i8().unwrap(), -3); - assert_eq!(dec.read_i16().unwrap(), 42); - assert_eq!(dec.read_i32().unwrap(), 123_456); - assert_eq!(dec.read_i64().unwrap(), 9_999_999); - assert_eq!( - dec.read_nullable_string().unwrap().as_deref(), - Some("client-a") - ); - assert_eq!(dec.read_nullable_string().unwrap(), None); - assert_eq!( - dec.read_nullable_bytes().unwrap().unwrap(), - Bytes::from_static(&[1, 2, 3]) - ); - assert_eq!(dec.read_nullable_bytes().unwrap(), None); -} - -#[test] -fn decoder_returns_underflow_error() { - let mut dec = Decoder::new(Bytes::from_static(&[0x00])); - let err = dec.read_i32().expect_err("must fail"); - assert!(err.to_string().contains("buffer underflow")); -} - -#[test] -fn codec_u8_and_bool() { - let mut enc = Encoder::with_capacity(8); - enc.write_u8(0xFF); - enc.write_bool(true); - enc.write_bool(false); - let bytes = enc.freeze(); - - let mut dec = Decoder::new(bytes); - assert_eq!(dec.read_u8().unwrap(), 0xFF); - assert!(dec.read_bool().unwrap()); - assert!(!dec.read_bool().unwrap()); -} - -#[test] -fn varint_round_trip_small_values() { - for v in [ - 0u64, - 1, - 127, - 128, - 255, - 300, - 16383, - 16384, - u64::from(u32::MAX), - ] { - let mut enc = Encoder::with_capacity(16); - enc.write_varint(v); - let mut dec = Decoder::new(enc.freeze()); - assert_eq!(dec.read_varint().unwrap(), v, "failed for v={v}"); - } -} - -#[test] -fn varint_single_byte_for_values_below_128() { - let mut enc = Encoder::with_capacity(1); - enc.write_varint(42); - let bytes = enc.freeze(); - assert_eq!(bytes.len(), 1); - assert_eq!(bytes[0], 42); -} - -#[test] -fn varint_two_bytes_for_128() { - let mut enc = Encoder::with_capacity(2); - enc.write_varint(128); - let bytes = enc.freeze(); - // 128 = 0x80 → first byte 0x80 | 0x80 = 0x80 (continue), second byte 0x01 - assert_eq!(bytes.as_ref(), &[0x80, 0x01]); -} - -#[test] -fn compact_nullable_string_round_trip() { - let mut enc = Encoder::with_capacity(32); - enc.write_compact_nullable_string(Some("hello")); - enc.write_compact_nullable_string(None); - enc.write_compact_nullable_string(Some("")); - let bytes = enc.freeze(); - - let mut dec = Decoder::new(bytes); - assert_eq!( - dec.read_compact_nullable_string().unwrap().as_deref(), - Some("hello") - ); - assert_eq!(dec.read_compact_nullable_string().unwrap(), None); - assert_eq!( - dec.read_compact_nullable_string().unwrap().as_deref(), - Some("") - ); -} - -#[test] -fn compact_nullable_bytes_round_trip() { - let mut enc = Encoder::with_capacity(32); - enc.write_compact_nullable_bytes(Some(&[10, 20, 30])); - enc.write_compact_nullable_bytes(None); - let bytes = enc.freeze(); - - let mut dec = Decoder::new(bytes); - assert_eq!( - dec.read_compact_nullable_bytes().unwrap().unwrap(), - Bytes::from_static(&[10, 20, 30]) - ); - assert_eq!(dec.read_compact_nullable_bytes().unwrap(), None); -} - -#[test] -fn tagged_fields_empty_section_round_trip() { - let mut enc = Encoder::with_capacity(8); - enc.write_i32(42); - enc.write_empty_tagged_fields(); - enc.write_i16(7); - let bytes = enc.freeze(); - - let mut dec = Decoder::new(bytes); - assert_eq!(dec.read_i32().unwrap(), 42); - dec.read_tagged_fields().unwrap(); // should consume the single 0x00 byte - assert_eq!(dec.read_i16().unwrap(), 7); - assert_eq!(dec.remaining(), 0); -} - -#[test] -fn decoder_rejects_unterminated_varint() { - let mut dec = Decoder::new(Bytes::from_static(&[ - 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, - ])); - let err = dec.read_varint().unwrap_err(); - assert!(matches!(err, KafkaProtocolError::InvalidVarint)); -} - -#[test] -fn compact_array_count_above_max_returns_error() { - let mut enc = Encoder::with_capacity(16); - enc.write_varint(65_538); // count = 65_537 after -1 - let mut dec = Decoder::new(enc.freeze()); - let err = dec.read_compact_array_count().unwrap_err(); - assert!(matches!(err, KafkaProtocolError::CollectionTooLarge { .. })); -} - -#[test] -fn compact_nullable_string_invalid_utf8_fails() { - let raw = vec![2, 0xff]; // len = 1, invalid UTF-8 byte - let mut dec = Decoder::new(Bytes::from(raw)); - let err = dec.read_compact_nullable_string().unwrap_err(); - assert!(matches!(err, KafkaProtocolError::InvalidUtf8)); -} - -#[test] -fn tagged_fields_non_empty_section_is_skipped() { - let mut enc = Encoder::with_capacity(16); - enc.write_varint(1); // one tagged field - enc.write_varint(7); // tag - enc.write_varint(3); // size - enc.write_bytes(&[1, 2, 3]); - enc.write_i16(99); - - let mut dec = Decoder::new(enc.freeze()); - dec.read_tagged_fields().unwrap(); - assert_eq!(dec.read_i16().unwrap(), 99); -} - -#[test] -fn tagged_fields_oversized_count_fails() { - let mut enc = Encoder::with_capacity(16); - enc.write_varint(65_537); - let mut dec = Decoder::new(enc.freeze()); - let err = dec.read_tagged_fields().unwrap_err(); - assert!(matches!(err, KafkaProtocolError::CollectionTooLarge { .. })); -} - -#[test] -fn write_null_bytes_and_write_bytes_round_trip() { - let mut enc = Encoder::with_capacity(16); - enc.write_null_bytes(); - enc.write_bytes(&[9, 8, 7]); - let mut dec = Decoder::new(enc.freeze()); - assert_eq!(dec.read_nullable_bytes().unwrap(), None); - assert_eq!(dec.read_bytes(3).unwrap(), Bytes::from_static(&[9, 8, 7])); -} - -#[test] -fn unchecked_nullable_string_matches_checked_encoding() { - let mut checked = Encoder::with_capacity(16); - checked.write_nullable_string(Some("safe")).unwrap(); - - let mut unchecked = Encoder::with_capacity(16); - unchecked.write_nullable_string_unchecked(Some("safe")); - - assert_eq!(checked.freeze(), unchecked.freeze()); -} diff --git a/gateways/kafka/tests/common/codec.rs b/gateways/kafka/tests/common/codec.rs new file mode 100644 index 0000000000..b6f652d899 --- /dev/null +++ b/gateways/kafka/tests/common/codec.rs @@ -0,0 +1,253 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Minimal Kafka primitive encoder/decoder for test wire fixtures. +//! +//! This is test-only scaffolding, not the gateway's production codec (that decoding/encoding +//! is `kafka_protocol`'s job - see `src/protocol/responses.rs`). It exists here because tests +//! need to hand-build wire bytes `kafka_protocol`'s spec-correct encoder cannot produce +//! (legacy/malformed/version-boundary shapes) and to read gateway response bytes back out at +//! the primitive level, independent of whichever crate encoded them. +#![allow(dead_code, clippy::cast_sign_loss, clippy::missing_const_for_fn)] + +use bytes::{Buf, BufMut, Bytes, BytesMut}; + +pub type Result = std::result::Result; + +pub struct Decoder { + bytes: Bytes, +} + +impl Decoder { + pub fn new(bytes: Bytes) -> Self { + Self { bytes } + } + + pub fn remaining(&self) -> usize { + self.bytes.remaining() + } + + pub fn read_bool(&mut self) -> Result { + self.ensure(1)?; + Ok(self.bytes.get_i8() != 0) + } + + pub fn read_i16(&mut self) -> Result { + self.ensure(2)?; + Ok(self.bytes.get_i16()) + } + + pub fn read_i32(&mut self) -> Result { + self.ensure(4)?; + Ok(self.bytes.get_i32()) + } + + pub fn read_i64(&mut self) -> Result { + self.ensure(8)?; + Ok(self.bytes.get_i64()) + } + + /// Unsigned varint (Kafka uses this for compact array lengths and tagged-field counts). + pub fn read_varint(&mut self) -> Result { + let mut result: u64 = 0; + let mut shift = 0u32; + loop { + self.ensure(1)?; + let byte = self.bytes.get_u8(); + result |= u64::from(byte & 0x7F) << shift; + if byte & 0x80 == 0 { + return Ok(result); + } + shift += 7; + if shift >= 64 { + return Err("varint overflows 64 bits".into()); + } + } + } + + /// Legacy nullable string: i16 length prefix (-1 = null). + pub fn read_nullable_string(&mut self) -> Result> { + let len = self.read_i16()?; + if len < 0 { + return Ok(None); + } + let len = len as usize; + self.ensure(len)?; + let s = std::str::from_utf8(&self.bytes.chunk()[..len]) + .map_err(|e| e.to_string())? + .to_owned(); + self.bytes.advance(len); + Ok(Some(s)) + } + + /// Compact nullable string (flexible versions): varint(len+1) prefix, 0 = null. + pub fn read_compact_nullable_string(&mut self) -> Result> { + let len_plus_one = self.read_varint()?; + if len_plus_one == 0 { + return Ok(None); + } + let len = usize::try_from(len_plus_one - 1).map_err(|e| e.to_string())?; + self.ensure(len)?; + let s = std::str::from_utf8(&self.bytes.chunk()[..len]) + .map_err(|e| e.to_string())? + .to_owned(); + self.bytes.advance(len); + Ok(Some(s)) + } + + /// Legacy nullable bytes: i32 length prefix (-1 = null). + pub fn read_nullable_bytes(&mut self) -> Result> { + let len = self.read_i32()?; + if len < 0 { + return Ok(None); + } + let len = len as usize; + self.ensure(len)?; + Ok(Some(self.bytes.copy_to_bytes(len))) + } + + pub fn read_bytes(&mut self, len: usize) -> Result { + self.ensure(len)?; + Ok(self.bytes.copy_to_bytes(len)) + } + + /// Skip over a tagged-fields section: tag (varint) + size (varint) + bytes, repeated. + pub fn read_tagged_fields(&mut self) -> Result<()> { + let count = self.read_varint()?; + for _ in 0..count { + self.read_varint()?; // tag number + let size = usize::try_from(self.read_varint()?).map_err(|e| e.to_string())?; + self.ensure(size)?; + self.bytes.advance(size); + } + Ok(()) + } + + fn ensure(&self, needed: usize) -> Result<()> { + let remaining = self.bytes.remaining(); + if remaining < needed { + return Err(format!( + "buffer underflow: needed {needed}, remaining {remaining}" + )); + } + Ok(()) + } +} + +pub struct Encoder { + bytes: BytesMut, +} + +impl Encoder { + pub fn with_capacity(capacity: usize) -> Self { + Self { + bytes: BytesMut::with_capacity(capacity), + } + } + + pub fn write_bool(&mut self, v: bool) { + self.write_i8(i8::from(v)); + } + + pub fn write_i8(&mut self, v: i8) { + self.bytes.put_i8(v); + } + + pub fn write_i16(&mut self, v: i16) { + self.bytes.put_i16(v); + } + + pub fn write_i32(&mut self, v: i32) { + self.bytes.put_i32(v); + } + + pub fn write_i64(&mut self, v: i64) { + self.bytes.put_i64(v); + } + + /// Unsigned varint, 7 bits per byte, LSB first. + pub fn write_varint(&mut self, mut v: u64) { + loop { + let byte = (v & 0x7F) as u8; + v >>= 7; + if v == 0 { + self.bytes.put_u8(byte); + return; + } + self.bytes.put_u8(byte | 0x80); + } + } + + /// Legacy nullable string: i16 length prefix, -1 for null. + pub fn write_nullable_string(&mut self, v: Option<&str>) -> Result<()> { + match v { + None => self.write_i16(-1), + Some(s) => { + self.write_i16(i16::try_from(s.len()).map_err(|e| e.to_string())?); + self.bytes.put_slice(s.as_bytes()); + } + } + Ok(()) + } + + /// Compact nullable string (flexible versions): varint(len+1), 0 for null. + pub fn write_compact_nullable_string(&mut self, v: Option<&str>) { + match v { + None => self.write_varint(0), + Some(s) => { + self.write_varint((s.len() + 1) as u64); + self.bytes.put_slice(s.as_bytes()); + } + } + } + + /// Legacy nullable bytes: i32 length prefix, -1 for null. + pub fn write_nullable_bytes(&mut self, v: Option<&[u8]>) -> Result<()> { + match v { + None => self.write_i32(-1), + Some(b) => { + self.write_i32(i32::try_from(b.len()).map_err(|e| e.to_string())?); + self.bytes.put_slice(b); + } + } + Ok(()) + } + + /// Compact nullable bytes (flexible versions): varint(len+1), 0 for null. + pub fn write_compact_nullable_bytes(&mut self, v: Option<&[u8]>) { + match v { + None => self.write_varint(0), + Some(b) => { + self.write_varint((b.len() + 1) as u64); + self.bytes.put_slice(b); + } + } + } + + pub fn write_bytes(&mut self, b: &[u8]) { + self.bytes.put_slice(b); + } + + /// Write an empty tagged-fields section (single 0x00 byte). + pub fn write_empty_tagged_fields(&mut self) { + self.write_varint(0); + } + + pub fn freeze(self) -> Bytes { + self.bytes.freeze() + } +} diff --git a/gateways/kafka/tests/common/fixtures.rs b/gateways/kafka/tests/common/fixtures.rs index 02cce6fed4..0d3f84e30f 100644 --- a/gateways/kafka/tests/common/fixtures.rs +++ b/gateways/kafka/tests/common/fixtures.rs @@ -21,9 +21,10 @@ use std::path::PathBuf; use bytes::Bytes; +use kafka_protocol::messages::RequestHeader; +use kafka_protocol::protocol::Decodable; -use iggy_gateway_kafka::protocol::codec::Decoder; -use iggy_gateway_kafka::protocol::header::{RequestHeader, request_header_version}; +use iggy_gateway_kafka::protocol::header::request_header_version; pub fn fixtures_dir() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tools/kafka-tool/kafka_messages") @@ -83,11 +84,8 @@ pub fn load_fixture_body_or_skip(api_key: i16, api_name: &str, version: i16) -> /// Strip the 4-byte length prefix and Kafka request header from a framed message. pub fn extract_body_from_framed_message(api_key: i16, api_version: i16, data: &[u8]) -> Bytes { - let frame = Bytes::copy_from_slice(&data[4..]); + let mut frame = Bytes::copy_from_slice(&data[4..]); let hdr_ver = request_header_version(api_key, api_version); - let mut decoder = Decoder::new(frame); - RequestHeader::decode_from(&mut decoder, hdr_ver).expect("fixture request header must decode"); - decoder - .read_bytes(decoder.remaining()) - .expect("fixture request body must decode") + RequestHeader::decode(&mut frame, hdr_ver).expect("fixture request header must decode"); + frame } diff --git a/gateways/kafka/tests/common/tcp.rs b/gateways/kafka/tests/common/tcp.rs index 384a5553bd..b2a766d369 100644 --- a/gateways/kafka/tests/common/tcp.rs +++ b/gateways/kafka/tests/common/tcp.rs @@ -16,6 +16,10 @@ // under the License. //! TCP round-trip helpers - compiled into each integration test binary via `#[path]`. +//! +//! Callers must also declare `#[path = "common/codec.rs"] mod codec;` at their own crate root - +//! this file borrows that module via `super::codec` rather than redeclaring it, since `rustc` +//! rejects loading the same file as two distinct modules in one crate. #![allow(dead_code)] use std::net::SocketAddr; @@ -26,9 +30,10 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpStream; use tokio::time; -use iggy_gateway_kafka::protocol::codec::Decoder; use iggy_gateway_kafka::protocol::header::{request_header_version, response_header_version}; +use super::codec::{self, Decoder}; + /// Build a complete length-prefixed Kafka request frame (header + body). pub fn build_request_frame( api_key: i16, @@ -38,16 +43,17 @@ pub fn build_request_frame( body: &[u8], ) -> Bytes { let hdr_ver = request_header_version(api_key, api_version); - let mut enc = iggy_gateway_kafka::protocol::codec::Encoder::with_capacity(64 + body.len()); + let mut enc = codec::Encoder::with_capacity(64 + body.len()); enc.write_i16(api_key); enc.write_i16(api_version); enc.write_i32(correlation_id); + // client_id is the legacy NULLABLE_STRING at every header version, even v2 - only the + // trailing tagged-fields section is new for the "flexible" header. Kafka's RequestHeader + // schema never made client_id itself a compact string. + enc.write_nullable_string(client_id) + .expect("test client_id fits i16"); if hdr_ver >= 2 { - enc.write_compact_nullable_string(client_id); enc.write_empty_tagged_fields(); - } else { - enc.write_nullable_string(client_id) - .expect("test client_id fits i16"); } enc.write_bytes(body); diff --git a/gateways/kafka/tests/common/wire.rs b/gateways/kafka/tests/common/wire.rs index 4f9cf4ee06..ec416fba8a 100644 --- a/gateways/kafka/tests/common/wire.rs +++ b/gateways/kafka/tests/common/wire.rs @@ -16,11 +16,15 @@ // under the License. //! Kafka wire request builders aligned with SCOPE.md / protocol spec. +//! +//! Callers must also declare `#[path = "common/codec.rs"] mod codec;` at their own crate root - +//! this file borrows that module via `super::codec` rather than redeclaring it, since `rustc` +//! rejects loading the same file as two distinct modules in one crate. #![allow(dead_code)] use bytes::Bytes; -use iggy_gateway_kafka::protocol::codec::Encoder; +use super::codec::Encoder; /// Consumer-group and admin keys explicitly out of scope in SCOPE.md. pub const OUT_OF_SCOPE_API_KEYS: &[(i16, &str)] = &[ diff --git a/gateways/kafka/tests/decode_safety_tests.rs b/gateways/kafka/tests/decode_safety_tests.rs deleted file mode 100644 index d338927f22..0000000000 --- a/gateways/kafka/tests/decode_safety_tests.rs +++ /dev/null @@ -1,347 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Adversarial wire-input tests for #3421 - malformed lengths must return errors, never panic. - -#[path = "common/wire.rs"] -mod wire; - -use bytes::Bytes; - -use iggy_gateway_kafka::error::KafkaProtocolError; -use iggy_gateway_kafka::protocol::codec::{Decoder, Encoder, MAX_COLLECTION_LEN}; -use iggy_gateway_kafka::protocol::requests::{ - ProduceDecodeResult, decode_create_topics_request, decode_fetch_request, - decode_list_offsets_request, decode_produce_request, -}; - -#[test] -fn compact_array_varint_zero_rejected_on_non_nullable_array() { - // Compact-array varint=0 is Kafka's null encoding; required (non-nullable) arrays reject it. - let mut d = Decoder::new(Bytes::from_static(&[0x00])); - let err = d.read_compact_array_count().unwrap_err(); - assert!(matches!(err, KafkaProtocolError::NullCompactArray)); -} - -#[test] -fn compact_array_varint_zero_nullable_decodes_as_empty() { - let mut d = Decoder::new(Bytes::from_static(&[0x00])); - assert_eq!(d.read_compact_array_count_nullable().unwrap(), 0); -} - -#[test] -fn produce_decoder_rejects_trailing_bytes_after_valid_body() { - let mut body = Vec::new(); - body.extend_from_slice(&(-1_i16).to_be_bytes()); // null transactional_id (legacy) - body.extend_from_slice(&1_i16.to_be_bytes()); // acks - body.extend_from_slice(&1000_i32.to_be_bytes()); // timeout_ms - body.extend_from_slice(&0_i32.to_be_bytes()); // empty topics - body.push(0xFF); // trailing garbage - let err = decode_produce_request(3, Bytes::from(body)) - .into_request() - .unwrap_err(); - assert!(matches!(err, KafkaProtocolError::UnexpectedTrailingBytes)); -} - -#[test] -fn negative_i32_array_length_returns_error_not_panic() { - let mut raw = Vec::new(); - raw.extend_from_slice(&(-1_i32).to_be_bytes()); - let mut d = Decoder::new(Bytes::from(raw)); - let err = d.read_i32_array_count().unwrap_err(); - assert!(matches!(err, KafkaProtocolError::InvalidArrayLength(-1))); -} - -#[test] -fn i32_array_length_above_max_returns_collection_too_large() { - let mut raw = Vec::new(); - let oversized = i32::try_from(MAX_COLLECTION_LEN + 1).expect("test value fits i32"); - raw.extend_from_slice(&oversized.to_be_bytes()); - let mut d = Decoder::new(Bytes::from(raw)); - let err = d.read_i32_array_count().unwrap_err(); - assert!(matches!(err, KafkaProtocolError::CollectionTooLarge { .. })); -} - -#[test] -fn fetch_max_declared_topics_count_with_empty_body_returns_error_not_large_alloc() { - // Declares the maximum allowed topics_count (65_536) but supplies no element bytes at - // all. Guards against pre-reserving a Vec directly off the wire count before validating - // any element bytes are present - decode must fail fast on the first missing byte, not - // attempt a large upfront allocation. - let mut body = Vec::new(); - body.extend_from_slice(&0_i32.to_be_bytes()); // replica_id - body.extend_from_slice(&0_i32.to_be_bytes()); // max_wait_ms - body.extend_from_slice(&0_i32.to_be_bytes()); // min_bytes - body.extend_from_slice(&0_i32.to_be_bytes()); // max_bytes (version >= 3) - body.push(0); // isolation_level (version >= 4) - let topics_count = i32::try_from(MAX_COLLECTION_LEN).expect("fits i32"); - body.extend_from_slice(&topics_count.to_be_bytes()); - // no topic bytes follow - - let err = decode_fetch_request(4, Bytes::from(body)).unwrap_err(); - assert!(matches!(err, KafkaProtocolError::BufferUnderflow { .. })); -} - -#[test] -fn produce_decoder_rejects_truncated_flexible_body() { - let mut body = Vec::new(); - body.push(0x00); // transactional_id null (compact) - body.extend_from_slice(&1_i16.to_be_bytes()); // acks - body.extend_from_slice(&1000_i32.to_be_bytes()); // timeout - body.push(0x02); // topics compact array: 1 element (varint = count+1) - // truncated before topic name - - let err = decode_produce_request(9, Bytes::from(body)) - .into_request() - .unwrap_err(); - assert!(matches!(err, KafkaProtocolError::BufferUnderflow { .. })); -} - -#[test] -fn write_nullable_string_rejects_oversized_length() { - let mut enc = Encoder::with_capacity(8); - let long = "x".repeat(i16::MAX as usize + 1); - let err = enc.write_nullable_string(Some(&long)).unwrap_err(); - assert!(matches!(err, KafkaProtocolError::StringTooLong { .. })); -} - -#[test] -fn varint_terminal_byte_with_extra_bits_at_shift_63_is_rejected() { - // Nine continuation bytes then terminal 0x7E at shift 63 (bits 1-6 set, bit 7 clear). - let mut d = Decoder::new(Bytes::from_static(&[ - 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x7E, - ])); - let err = d.read_varint().unwrap_err(); - assert!(matches!(err, KafkaProtocolError::InvalidVarint)); -} - -// ── Produce: error preserves acks (so a retry decision can honor acks=0) ──── - -#[test] -fn produce_null_topic_name_preserves_acks_on_error() { - let mut enc = Encoder::with_capacity(32); - enc.write_nullable_string(None::<&str>).unwrap(); - enc.write_i16(1); - enc.write_i32(500); - enc.write_i32(1); - enc.write_nullable_string(None::<&str>).unwrap(); - - match decode_produce_request(3, enc.freeze()) { - ProduceDecodeResult::Err { acks, error } => { - assert_eq!(acks, Some(1)); - assert!(matches!(error, KafkaProtocolError::NullTopicName)); - } - ProduceDecodeResult::Ok(_) => panic!("expected NullTopicName"), - } -} - -#[test] -fn produce_v3_error_before_acks_has_none_acks() { - let mut enc = Encoder::with_capacity(8); - enc.write_i16(1); - match decode_produce_request(3, enc.freeze()) { - ProduceDecodeResult::Err { acks, .. } => assert_eq!(acks, None), - ProduceDecodeResult::Ok(_) => panic!("expected decode error before acks"), - } -} - -#[test] -fn produce_v3_error_after_acks_preserves_acks() { - let mut enc = Encoder::with_capacity(16); - enc.write_nullable_string(None::<&str>).unwrap(); - enc.write_i16(7); - match decode_produce_request(3, enc.freeze()) { - ProduceDecodeResult::Err { acks, .. } => assert_eq!(acks, Some(7)), - ProduceDecodeResult::Ok(_) => panic!("expected decode error after acks"), - } -} - -#[test] -fn produce_v3_error_after_timeout_preserves_acks() { - let mut enc = Encoder::with_capacity(16); - enc.write_nullable_string(None::<&str>).unwrap(); - enc.write_i16(1); - enc.write_i32(500); - match decode_produce_request(3, enc.freeze()) { - ProduceDecodeResult::Err { acks, .. } => assert_eq!(acks, Some(1)), - ProduceDecodeResult::Ok(_) => panic!("expected decode error after timeout"), - } -} - -#[test] -fn produce_v9_error_on_null_topic_preserves_acks() { - let mut enc = Encoder::with_capacity(32); - enc.write_compact_nullable_string(None); - enc.write_i16(2); - enc.write_i32(500); - enc.write_varint(2); - enc.write_compact_nullable_string(None); - match decode_produce_request(9, enc.freeze()) { - ProduceDecodeResult::Err { acks, error } => { - assert_eq!(acks, Some(2)); - assert!(matches!(error, KafkaProtocolError::NullTopicName)); - } - ProduceDecodeResult::Ok(_) => panic!("expected NullTopicName"), - } -} - -#[test] -fn produce_v9_error_on_partition_count_preserves_acks() { - let mut enc = Encoder::with_capacity(64); - enc.write_compact_nullable_string(None); - enc.write_i16(3); - enc.write_i32(500); - enc.write_varint(2); - enc.write_compact_nullable_string(Some("topic")); - match decode_produce_request(9, enc.freeze()) { - ProduceDecodeResult::Err { acks, .. } => assert_eq!(acks, Some(3)), - ProduceDecodeResult::Ok(_) => panic!("expected decode error in partition count"), - } -} - -#[test] -fn produce_v9_error_on_partition_records_preserves_acks() { - let mut enc = Encoder::with_capacity(64); - enc.write_compact_nullable_string(None); - enc.write_i16(4); - enc.write_i32(500); - enc.write_varint(2); - enc.write_compact_nullable_string(Some("topic")); - enc.write_varint(2); - enc.write_i32(0); - match decode_produce_request(9, enc.freeze()) { - ProduceDecodeResult::Err { acks, .. } => assert_eq!(acks, Some(4)), - ProduceDecodeResult::Ok(_) => panic!("expected decode error in records"), - } -} - -// ── Fetch: truncated / null-topic inputs return errors, never panic ──────── - -#[test] -fn fetch_v4_truncated_after_replica_id_returns_error() { - let mut enc = Encoder::with_capacity(4); - enc.write_i32(-1); - let err = decode_fetch_request(4, enc.freeze()).unwrap_err(); - assert!(matches!(err, KafkaProtocolError::BufferUnderflow { .. })); -} - -#[test] -fn fetch_v7_truncated_in_forgotten_topics_returns_error() { - let body = wire::build_fetch_request_with_sections(7, "topic", 0, Some("forgot"), None); - let truncated = body.slice(..body.len() - 2); - let err = decode_fetch_request(7, truncated).unwrap_err(); - assert!(matches!(err, KafkaProtocolError::BufferUnderflow { .. })); -} - -#[test] -fn fetch_v12_flexible_truncated_in_topic_tagged_fields_returns_error() { - let body = wire::build_fetch_request_with_sections(12, "topic", 0, None, None); - let truncated = body.slice(..body.len() - 1); - let err = decode_fetch_request(12, truncated).unwrap_err(); - assert!(matches!(err, KafkaProtocolError::BufferUnderflow { .. })); -} - -#[test] -fn fetch_v12_flexible_null_topic_name_returns_error() { - let mut enc = Encoder::with_capacity(64); - enc.write_i32(-1); - enc.write_i32(100); - enc.write_i32(1); - enc.write_i32(1024); - enc.write_i8(0); - enc.write_i32(0); - enc.write_i32(0); - enc.write_varint(2); - enc.write_compact_nullable_string(None); - let err = decode_fetch_request(12, enc.freeze()).unwrap_err(); - assert!(matches!(err, KafkaProtocolError::NullTopicName)); -} - -#[test] -fn fetch_null_topic_name_returns_error() { - let mut enc = Encoder::with_capacity(64); - enc.write_i32(-1); - enc.write_i32(100); - enc.write_i32(1); - enc.write_i32(i32::MAX); - enc.write_i8(0); - enc.write_i32(1); - enc.write_nullable_string(None::<&str>).unwrap(); - - let err = decode_fetch_request(4, enc.freeze()).unwrap_err(); - assert!(matches!(err, KafkaProtocolError::NullTopicName)); -} - -// ── ListOffsets: truncated / null-topic inputs return errors, never panic ── - -#[test] -fn list_offsets_v4_truncated_in_leader_epoch_returns_error() { - let body = wire::build_list_offsets_branch_request(4, "topic", 1); - let truncated = body.slice(..body.len() - 4); - let err = decode_list_offsets_request(4, truncated).unwrap_err(); - assert!(matches!(err, KafkaProtocolError::BufferUnderflow { .. })); -} - -#[test] -fn list_offsets_v6_flexible_null_topic_name_returns_error() { - let mut enc = Encoder::with_capacity(32); - enc.write_i32(-1); - enc.write_i8(0); - enc.write_varint(2); - enc.write_compact_nullable_string(None); - let err = decode_list_offsets_request(6, enc.freeze()).unwrap_err(); - assert!(matches!(err, KafkaProtocolError::NullTopicName)); -} - -#[test] -fn list_offsets_null_topic_name_returns_error() { - let mut enc = Encoder::with_capacity(32); - enc.write_i32(-1); - enc.write_i8(0); - enc.write_i32(1); - enc.write_nullable_string(None::<&str>).unwrap(); - let err = decode_list_offsets_request(2, enc.freeze()).unwrap_err(); - assert!(matches!(err, KafkaProtocolError::NullTopicName)); -} - -// ── CreateTopics: truncated / null-topic inputs return errors, never panic ─ - -#[test] -fn create_topics_v2_truncated_in_config_value_returns_error() { - let body = wire::build_create_topics_request_with_sections(2, "topic"); - let truncated = body.slice(..body.len() - 3); - let err = decode_create_topics_request(2, truncated).unwrap_err(); - assert!(matches!(err, KafkaProtocolError::BufferUnderflow { .. })); -} - -#[test] -fn create_topics_v5_flexible_null_topic_name_returns_error() { - let mut enc = Encoder::with_capacity(16); - enc.write_varint(2); - enc.write_compact_nullable_string(None); - let err = decode_create_topics_request(5, enc.freeze()).unwrap_err(); - assert!(matches!(err, KafkaProtocolError::NullTopicName)); -} - -#[test] -fn create_topics_null_topic_name_returns_error() { - let mut enc = Encoder::with_capacity(32); - enc.write_i32(1); - enc.write_nullable_string(None::<&str>).unwrap(); - let err = decode_create_topics_request(2, enc.freeze()).unwrap_err(); - assert!(matches!(err, KafkaProtocolError::NullTopicName)); -} diff --git a/gateways/kafka/tests/decode_validation_tests.rs b/gateways/kafka/tests/decode_validation_tests.rs deleted file mode 100644 index 700d16d56f..0000000000 --- a/gateways/kafka/tests/decode_validation_tests.rs +++ /dev/null @@ -1,678 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Validates request decoders and response encoders against the binary fixtures -//! produced by tools/kafka-tool. -//! -//! Frame layout written by kafka-tool (all versions): -//! [4-byte length prefix] -//! [`api_key` i16][`api_version` i16][`correlation_id` i32] -//! header v1: [`client_id`] `NULLABLE_STRING` -//! header v2: [`client_id`] `COMPACT_NULLABLE_STRING` + request-header tagged fields -//! [request body] ← properly encoded per spec (flexible or not) - -use iggy_gateway_kafka::protocol::codec::Encoder; -use iggy_gateway_kafka::protocol::requests::{ - decode_create_topics_request, decode_fetch_request, decode_list_offsets_request, - decode_produce_request, -}; -use iggy_gateway_kafka::protocol::responses::{ - encode_create_topics_response, encode_fetch_response, encode_list_offsets_response, - encode_produce_response, -}; - -#[path = "common/fixtures.rs"] -mod fixtures; -#[path = "common/wire.rs"] -mod wire; - -use fixtures::load_fixture_body_or_skip as load_body; - -// ── Produce (API key 0) ─────────────────────────────────────────────────────── - -#[test] -fn produce_all_supported_versions_decode() { - for version in 3i16..=9 { - let Some(body) = load_body(0, "Produce", version) else { - continue; - }; - let req = decode_produce_request(version, body) - .into_request() - .unwrap_or_else(|e| panic!("Produce v{version} decode failed: {e}")); - - assert_eq!(req.acks, -1, "Produce v{version}: unexpected acks"); - assert_eq!( - req.timeout_ms, 5000, - "Produce v{version}: unexpected timeout_ms" - ); - assert_eq!(req.topics.len(), 1, "Produce v{version}: expected 1 topic"); - assert_eq!( - req.topics[0].topic, "test-topic", - "Produce v{version}: wrong topic name" - ); - assert_eq!( - req.topics[0].partitions.len(), - 1, - "Produce v{version}: expected 1 partition" - ); - assert_eq!( - req.topics[0].partitions[0].partition, 0, - "Produce v{version}: wrong partition index" - ); - assert!( - req.topics[0].partitions[0].records.is_some(), - "Produce v{version}: records should be present" - ); - } -} - -#[test] -fn produce_response_encodes_for_all_supported_versions() { - for version in 3i16..=9 { - let Some(body) = load_body(0, "Produce", version) else { - continue; - }; - let req = decode_produce_request(version, body) - .into_request() - .unwrap_or_else(|e| panic!("Produce v{version} decode failed: {e}")); - let resp = encode_produce_response(version, &req); - assert!( - !resp.is_empty(), - "Produce v{version}: response must not be empty" - ); - } -} - -#[test] -fn produce_response_v3_roundtrip() { - use iggy_gateway_kafka::protocol::codec::Decoder; - let Some(body) = load_body(0, "Produce", 3) else { - return; - }; - let req = decode_produce_request(3, body).into_request().unwrap(); - let resp = encode_produce_response(3, &req); - - let mut d = Decoder::new(resp); - let topic_count = d.read_i32().unwrap(); - assert_eq!(topic_count, 1); - let topic_name = d.read_nullable_string().unwrap().unwrap(); - assert_eq!(topic_name, "test-topic"); - let partition_count = d.read_i32().unwrap(); - assert_eq!(partition_count, 1); - let partition = d.read_i32().unwrap(); - assert_eq!(partition, 0); - let error_code = d.read_i16().unwrap(); - assert_eq!(error_code, 6); // NOT_LEADER_OR_FOLLOWER - stub until Iggy bridge - let base_offset = d.read_i64().unwrap(); - assert_eq!(base_offset, 0); - // log_append_time_ms (v2+) - let _log_append = d.read_i64().unwrap(); - // log_start_offset (v5+) - not present for v3 - let throttle = d.read_i32().unwrap(); - assert_eq!(throttle, 0); -} - -#[test] -fn produce_response_v8_includes_record_errors() { - use iggy_gateway_kafka::protocol::codec::Decoder; - let Some(body) = load_body(0, "Produce", 8) else { - return; - }; - let req = decode_produce_request(8, body).into_request().unwrap(); - let resp = encode_produce_response(8, &req); - - let mut d = Decoder::new(resp); - let topic_count = d.read_i32().unwrap(); - assert_eq!(topic_count, 1); - let _topic_name = d.read_nullable_string().unwrap(); - let partition_count = d.read_i32().unwrap(); - assert_eq!(partition_count, 1); - let _partition = d.read_i32().unwrap(); - let error_code = d.read_i16().unwrap(); - assert_eq!(error_code, 6); // NOT_LEADER_OR_FOLLOWER - stub until Iggy bridge - let _base_offset = d.read_i64().unwrap(); - let _log_append_time = d.read_i64().unwrap(); // v2+ - let _log_start_offset = d.read_i64().unwrap(); // v5+ - let record_errors_count = d.read_i32().unwrap(); // v8+: should be 0 - assert_eq!( - record_errors_count, 0, - "v8 must emit empty record_errors array" - ); - let error_message = d.read_nullable_string().unwrap(); // v8+: should be null - assert!(error_message.is_none(), "v8 error_message must be null"); -} - -#[test] -fn produce_v9_flexible_empty_topics_decode() { - let req = decode_produce_request(9, wire::build_produce_flexible_empty_request(0)) - .into_request() - .expect("flexible produce request should decode"); - assert_eq!(req.acks, 0); - assert_eq!(req.topics.len(), 0); -} - -#[test] -fn produce_v2_skips_transactional_id_branch() { - let req = decode_produce_request(2, wire::build_produce_legacy_request(2, 1, None, None)) - .into_request() - .expect("produce v2 should decode"); - assert_eq!(req.acks, 1); - assert!(req.transactional_id.is_none()); - assert!(req.topics.is_empty()); -} - -#[test] -fn produce_v3_legacy_transactional_id_and_topic_decode() { - let req = decode_produce_request( - 3, - wire::build_produce_legacy_request(3, -1, Some("txn-1"), Some("legacy-topic")), - ) - .into_request() - .expect("produce v3 legacy should decode"); - assert_eq!(req.transactional_id.as_deref(), Some("txn-1")); - assert_eq!(req.topics.len(), 1); - assert_eq!(req.topics[0].topic, "legacy-topic"); - assert!(req.topics[0].partitions[0].records.is_some()); -} - -#[test] -fn produce_v8_legacy_null_records_decode() { - let mut enc = Encoder::with_capacity(64); - enc.write_nullable_string(None::<&str>).unwrap(); - enc.write_i16(1); - enc.write_i32(500); - enc.write_i32(1); - enc.write_nullable_string(Some("topic")).unwrap(); - enc.write_i32(1); - enc.write_i32(0); - enc.write_nullable_bytes(None).unwrap(); - - let req = decode_produce_request(8, enc.freeze()) - .into_request() - .expect("produce v8 with null records should decode"); - assert!(req.topics[0].partitions[0].records.is_none()); -} - -#[test] -fn produce_v9_flexible_transactional_id_and_tagged_fields_decode() { - let req = decode_produce_request( - 9, - wire::build_produce_flexible_request_with_topic("flex-topic"), - ) - .into_request() - .expect("produce v9 flexible should decode"); - assert_eq!(req.transactional_id.as_deref(), Some("txn-1")); - assert_eq!(req.topics[0].topic, "flex-topic"); - assert!(req.topics[0].partitions[0].records.is_some()); -} - -// ── Fetch (API key 1) ───────────────────────────────────────────────────────── - -#[test] -fn fetch_all_supported_versions_decode() { - for version in 4i16..=12 { - let Some(body) = load_body(1, "Fetch", version) else { - continue; - }; - let req = decode_fetch_request(version, body) - .unwrap_or_else(|e| panic!("Fetch v{version} decode failed: {e}")); - - assert_eq!( - req.max_wait_ms, 500, - "Fetch v{version}: unexpected max_wait_ms" - ); - assert_eq!(req.min_bytes, 1, "Fetch v{version}: unexpected min_bytes"); - assert_eq!(req.topics.len(), 1, "Fetch v{version}: expected 1 topic"); - assert_eq!( - req.topics[0].topic, "test-topic", - "Fetch v{version}: wrong topic name" - ); - assert_eq!( - req.topics[0].partitions.len(), - 1, - "Fetch v{version}: expected 1 partition" - ); - assert_eq!( - req.topics[0].partitions[0].partition, 0, - "Fetch v{version}: wrong partition index" - ); - assert_eq!( - req.topics[0].partitions[0].fetch_offset, 0, - "Fetch v{version}: wrong fetch_offset" - ); - } -} - -#[test] -fn fetch_response_encodes_for_all_supported_versions() { - for version in 4i16..=12 { - let Some(body) = load_body(1, "Fetch", version) else { - continue; - }; - let req = decode_fetch_request(version, body) - .unwrap_or_else(|e| panic!("Fetch v{version} decode failed: {e}")); - let resp = encode_fetch_response(version, &req); - assert!( - !resp.is_empty(), - "Fetch v{version}: response must not be empty" - ); - } -} - -#[test] -fn fetch_response_v7_roundtrip() { - use iggy_gateway_kafka::protocol::codec::Decoder; - let Some(body) = load_body(1, "Fetch", 7) else { - return; - }; - let req = decode_fetch_request(7, body).unwrap(); - let resp = encode_fetch_response(7, &req); - - let mut d = Decoder::new(resp); - let throttle_ms = d.read_i32().unwrap(); // v1+ - assert_eq!(throttle_ms, 0); - let error_code = d.read_i16().unwrap(); // v7+ - assert_eq!(error_code, 0); - let session_id = d.read_i32().unwrap(); // v7+ - assert_eq!(session_id, 0); - let topic_count = d.read_i32().unwrap(); - assert_eq!(topic_count, 1); - let topic_name = d.read_nullable_string().unwrap().unwrap(); - assert_eq!(topic_name, "test-topic"); - let partition_count = d.read_i32().unwrap(); - assert_eq!(partition_count, 1); - let partition = d.read_i32().unwrap(); - assert_eq!(partition, 0); - let partition_error = d.read_i16().unwrap(); - assert_eq!(partition_error, 0); - let high_watermark = d.read_i64().unwrap(); - assert_eq!(high_watermark, 0); -} - -#[test] -fn fetch_v12_decodes_forgotten_topics_and_rack_id_sections() { - let req = decode_fetch_request( - 12, - wire::build_fetch_request_with_sections(12, "test-topic", 2, Some("forgotten"), Some("r1")), - ) - .expect("fetch request should decode"); - assert_eq!(req.max_wait_ms, 100); - assert_eq!(req.min_bytes, 1); - assert_eq!(req.max_bytes, i32::MAX); - assert_eq!(req.isolation_level, 0); - assert_eq!(req.topics.len(), 1); - assert_eq!(req.topics[0].topic, "test-topic"); - assert_eq!(req.topics[0].partitions.len(), 1); - assert_eq!(req.topics[0].partitions[0].partition, 2); - assert_eq!(req.topics[0].partitions[0].fetch_offset, 42); - assert_eq!(req.topics[0].partitions[0].partition_max_bytes, 1024); -} - -#[test] -fn fetch_v2_uses_default_max_bytes_when_field_absent() { - let req = decode_fetch_request(2, wire::build_fetch_v2_default_max_bytes_request()) - .expect("fetch v2 should decode"); - assert_eq!(req.max_bytes, 52_428_800); - assert_eq!(req.isolation_level, 0); - assert!(req.topics.is_empty()); -} - -#[test] -fn fetch_v7_legacy_forgotten_topics_and_rack_id_decode() { - let req = decode_fetch_request( - 7, - wire::build_fetch_request_with_sections(7, "topic-a", 1, Some("forgotten"), Some("rack-1")), - ) - .expect("fetch v7 legacy sections should decode"); - assert_eq!(req.topics[0].topic, "topic-a"); - assert_eq!(req.topics[0].partitions[0].partition, 1); -} - -#[test] -fn fetch_v9_leader_epoch_without_v12_fields_decode() { - let req = decode_fetch_request( - 9, - wire::build_fetch_request_with_sections(9, "topic-b", 2, None, None), - ) - .expect("fetch v9 should decode"); - assert_eq!(req.topics[0].partitions[0].fetch_offset, 42); -} - -#[test] -fn fetch_v11_legacy_rack_id_decode() { - let req = decode_fetch_request( - 11, - wire::build_fetch_request_with_sections(11, "topic-c", 3, None, Some("rack-z")), - ) - .expect("fetch v11 legacy rack id should decode"); - assert_eq!(req.max_wait_ms, 100); -} - -#[test] -fn fetch_v3_skips_isolation_level_field() { - let req = decode_fetch_request(3, wire::build_fetch_v3_no_isolation_request()) - .expect("fetch v3 should decode"); - assert_eq!(req.isolation_level, 0); - assert_eq!(req.max_bytes, 1024); -} - -// ── ListOffsets (API key 2) ─────────────────────────────────────────────────── - -#[test] -fn list_offsets_all_supported_versions_decode() { - for version in 1i16..=6 { - let Some(body) = load_body(2, "ListOffsets", version) else { - continue; - }; - let req = decode_list_offsets_request(version, body) - .unwrap_or_else(|e| panic!("ListOffsets v{version} decode failed: {e}")); - - assert_eq!( - req.topics.len(), - 1, - "ListOffsets v{version}: expected 1 topic" - ); - assert_eq!( - req.topics[0].topic, "test-topic", - "ListOffsets v{version}: wrong topic name" - ); - assert_eq!( - req.topics[0].partitions.len(), - 1, - "ListOffsets v{version}: expected 1 partition" - ); - assert_eq!( - req.topics[0].partitions[0].partition, 0, - "ListOffsets v{version}: wrong partition index" - ); - } -} - -#[test] -fn list_offsets_v0_decodes_legacy_max_num_offsets_branch() { - let req = - decode_list_offsets_request(0, wire::build_list_offsets_branch_request(0, "legacy", 4)) - .expect("v0 list offsets should decode"); - assert_eq!(req.isolation_level, 0); - assert_eq!(req.topics.len(), 1); - assert_eq!(req.topics[0].topic, "legacy"); - assert_eq!(req.topics[0].partitions[0].partition, 4); - assert_eq!(req.topics[0].partitions[0].timestamp, -2); -} - -#[test] -fn list_offsets_v6_decodes_flexible_leader_epoch_branch() { - let req = decode_list_offsets_request(6, wire::build_list_offsets_branch_request(6, "flex", 5)) - .expect("v6 list offsets should decode"); - assert_eq!(req.isolation_level, 1); - assert_eq!(req.topics[0].topic, "flex"); - assert_eq!(req.topics[0].partitions[0].partition, 5); - assert_eq!(req.topics[0].partitions[0].timestamp, -2); -} - -#[test] -fn list_offsets_response_encodes_for_all_supported_versions() { - for version in 1i16..=6 { - let Some(body) = load_body(2, "ListOffsets", version) else { - continue; - }; - let req = decode_list_offsets_request(version, body) - .unwrap_or_else(|e| panic!("ListOffsets v{version} decode failed: {e}")); - let resp = encode_list_offsets_response(version, &req); - assert!( - !resp.is_empty(), - "ListOffsets v{version}: response must not be empty" - ); - } -} - -#[test] -fn list_offsets_response_v1_no_leader_epoch() { - use iggy_gateway_kafka::protocol::codec::Decoder; - let Some(body) = load_body(2, "ListOffsets", 1) else { - return; - }; - let req = decode_list_offsets_request(1, body).unwrap(); - let resp = encode_list_offsets_response(1, &req); - - let mut d = Decoder::new(resp); - // v1: no throttle_time_ms - let topic_count = d.read_i32().unwrap(); - assert_eq!(topic_count, 1); - let _topic_name = d.read_nullable_string().unwrap(); - let partition_count = d.read_i32().unwrap(); - assert_eq!(partition_count, 1); - let _partition = d.read_i32().unwrap(); - let error_code = d.read_i16().unwrap(); - assert_eq!(error_code, 0); - let _timestamp = d.read_i64().unwrap(); // v1+ - let _offset = d.read_i64().unwrap(); - // v1 must NOT have a leader_epoch field - assert all bytes consumed - assert_eq!( - d.remaining(), - 0, - "v1 response must have no trailing bytes (leader_epoch must NOT be written)" - ); -} - -#[test] -fn list_offsets_response_v4_has_leader_epoch() { - use iggy_gateway_kafka::protocol::codec::Decoder; - let Some(body) = load_body(2, "ListOffsets", 4) else { - return; - }; - let req = decode_list_offsets_request(4, body).unwrap(); - let resp = encode_list_offsets_response(4, &req); - - let mut d = Decoder::new(resp); - let _throttle = d.read_i32().unwrap(); // v2+ - let topic_count = d.read_i32().unwrap(); - assert_eq!(topic_count, 1); - let _topic_name = d.read_nullable_string().unwrap(); - let partition_count = d.read_i32().unwrap(); - assert_eq!(partition_count, 1); - let _partition = d.read_i32().unwrap(); - let error_code = d.read_i16().unwrap(); - assert_eq!(error_code, 0); - let _timestamp = d.read_i64().unwrap(); - let _offset = d.read_i64().unwrap(); - let leader_epoch = d.read_i32().unwrap(); // v4+ - assert_eq!(leader_epoch, -1, "v4 must have leader_epoch = -1"); - assert_eq!(d.remaining(), 0); -} - -#[test] -fn list_offsets_v1_skips_isolation_level_field() { - let req = decode_list_offsets_request(1, wire::build_list_offsets_branch_request(1, "v1", 0)) - .expect("list offsets v1 should decode"); - assert_eq!(req.isolation_level, 0); -} - -#[test] -fn list_offsets_v2_reads_isolation_level() { - let req = decode_list_offsets_request(2, wire::build_list_offsets_branch_request(2, "v2", 1)) - .expect("list offsets v2 should decode"); - assert_eq!(req.isolation_level, 1); -} - -#[test] -fn list_offsets_v3_skips_leader_epoch_branch() { - let req = decode_list_offsets_request(3, wire::build_list_offsets_branch_request(3, "v3", 2)) - .expect("list offsets v3 should decode"); - assert_eq!(req.topics[0].partitions[0].partition, 2); -} - -#[test] -fn list_offsets_v5_leader_epoch_without_flexible_encoding() { - let req = decode_list_offsets_request(5, wire::build_list_offsets_branch_request(5, "v5", 3)) - .expect("list offsets v5 should decode"); - assert_eq!(req.topics[0].topic, "v5"); -} - -// ── CreateTopics (API key 19) ───────────────────────────────────────────────── - -#[test] -fn create_topics_all_supported_versions_decode() { - for version in 2i16..=5 { - let Some(body) = load_body(19, "CreateTopics", version) else { - continue; - }; - let req = decode_create_topics_request(version, body) - .unwrap_or_else(|e| panic!("CreateTopics v{version} decode failed: {e}")); - - assert_eq!( - req.topics.len(), - 1, - "CreateTopics v{version}: expected 1 topic" - ); - assert_eq!( - req.topics[0].num_partitions, 1, - "CreateTopics v{version}: wrong num_partitions" - ); - assert_eq!( - req.topics[0].replication_factor, 1, - "CreateTopics v{version}: wrong replication_factor" - ); - assert!( - !req.topics[0].name.is_empty(), - "CreateTopics v{version}: topic name must not be empty" - ); - assert_eq!( - req.timeout_ms, 30000, - "CreateTopics v{version}: unexpected timeout_ms" - ); - } -} - -#[test] -fn create_topics_v0_defaults_validate_only_to_false() { - let req = decode_create_topics_request( - 0, - wire::build_create_topics_request_with_sections(0, "legacy-topic"), - ) - .expect("create topics v0 should decode"); - assert_eq!(req.timeout_ms, 5_000); - assert!(!req.validate_only); - assert_eq!(req.topics.len(), 1); - assert_eq!(req.topics[0].name, "legacy-topic"); - assert_eq!(req.topics[0].num_partitions, 3); - assert_eq!(req.topics[0].replication_factor, 1); -} - -#[test] -fn create_topics_v5_decodes_flexible_assignments_and_configs() { - let req = decode_create_topics_request( - 5, - wire::build_create_topics_request_with_sections(5, "flex-topic"), - ) - .expect("create topics v5 should decode"); - assert_eq!(req.timeout_ms, 5_000); - assert!(req.validate_only); - assert_eq!(req.topics.len(), 1); - assert_eq!(req.topics[0].name, "flex-topic"); - assert_eq!(req.topics[0].num_partitions, 3); - assert_eq!(req.topics[0].replication_factor, 1); -} - -#[test] -fn create_topics_response_encodes_for_all_supported_versions() { - for version in 2i16..=5 { - let Some(body) = load_body(19, "CreateTopics", version) else { - continue; - }; - let req = decode_create_topics_request(version, body) - .unwrap_or_else(|e| panic!("CreateTopics v{version} decode failed: {e}")); - let resp = encode_create_topics_response(version, &req); - assert!( - !resp.is_empty(), - "CreateTopics v{version}: response must not be empty" - ); - } -} - -#[test] -fn create_topics_response_v2_roundtrip() { - use iggy_gateway_kafka::protocol::codec::Decoder; - let Some(body) = load_body(19, "CreateTopics", 2) else { - return; - }; - let req = decode_create_topics_request(2, body).unwrap(); - let topic_name = req.topics[0].name.clone(); - let resp = encode_create_topics_response(2, &req); - - let mut d = Decoder::new(resp); - let _throttle = d.read_i32().unwrap(); // v2+ - let topic_count = d.read_i32().unwrap(); - assert_eq!(topic_count, 1); - let resp_topic = d.read_nullable_string().unwrap().unwrap(); - assert_eq!(resp_topic, topic_name); - let error_code = d.read_i16().unwrap(); - assert_eq!(error_code, 41); // NOT_CONTROLLER - stub until Iggy bridge - let error_msg = d.read_nullable_string().unwrap(); // v1+ - assert!(error_msg.is_none()); - assert_eq!(d.remaining(), 0); -} - -#[test] -fn create_topics_response_v5_roundtrip() { - use iggy_gateway_kafka::protocol::codec::Decoder; - let Some(body) = load_body(19, "CreateTopics", 5) else { - return; - }; - let req = decode_create_topics_request(5, body).unwrap(); - let resp = encode_create_topics_response(5, &req); - - let mut d = Decoder::new(resp); - let _throttle = d.read_i32().unwrap(); // v2+ - let topic_count_plus_one = d.read_varint().unwrap(); // flexible compact array - assert_eq!(topic_count_plus_one, 2); // 1 topic → varint = 2 - - let _topic_name = d.read_compact_nullable_string().unwrap(); - let error_code = d.read_i16().unwrap(); - assert_eq!(error_code, 41); // NOT_CONTROLLER - stub until Iggy bridge - let _error_msg = d.read_compact_nullable_string().unwrap(); // v1+ - let num_partitions = d.read_i32().unwrap(); - assert_eq!(num_partitions, 1); - let replication_factor = d.read_i16().unwrap(); - assert_eq!(replication_factor, 1); - let configs_count_plus_one = d.read_varint().unwrap(); // empty compact array - assert_eq!(configs_count_plus_one, 1); // empty = varint(1) - d.read_tagged_fields().unwrap(); // per-entry tagged_fields - d.read_tagged_fields().unwrap(); // top-level tagged_fields - assert_eq!(d.remaining(), 0); -} - -#[test] -fn create_topics_v3_legacy_assignments_decode() { - let req = decode_create_topics_request( - 3, - wire::build_create_topics_request_with_sections(3, "v3-topic"), - ) - .expect("create topics v3 should decode"); - assert_eq!(req.topics[0].name, "v3-topic"); - assert!(req.validate_only); -} - -#[test] -fn create_topics_v4_legacy_configs_decode() { - let req = decode_create_topics_request( - 4, - wire::build_create_topics_request_with_sections(4, "v4-topic"), - ) - .expect("create topics v4 should decode"); - assert_eq!(req.topics[0].replication_factor, 1); -} diff --git a/gateways/kafka/tests/golden_wire_fixtures_tests.rs b/gateways/kafka/tests/golden_wire_fixtures_tests.rs index dad3fc4e98..851f71c2d1 100644 --- a/gateways/kafka/tests/golden_wire_fixtures_tests.rs +++ b/gateways/kafka/tests/golden_wire_fixtures_tests.rs @@ -15,12 +15,16 @@ // specific language governing permissions and limitations // under the License. +#[path = "common/codec.rs"] +mod codec; + use bytes::Bytes; use iggy_gateway_kafka::protocol::api::{ API_KEY_API_VERSIONS, API_KEY_METADATA, BrokerAdvertise, handle_request, }; -use iggy_gateway_kafka::protocol::codec::Encoder; + +use codec::Encoder; #[test] fn golden_apiversions_v1_response_fixture() { diff --git a/gateways/kafka/tests/header_tests.rs b/gateways/kafka/tests/header_tests.rs index 3cbf52d010..91d61ff9fd 100644 --- a/gateways/kafka/tests/header_tests.rs +++ b/gateways/kafka/tests/header_tests.rs @@ -15,111 +15,36 @@ // specific language governing permissions and limitations // under the License. +//! `request_header_version` / `response_header_version` are thin wrappers around +//! `kafka_protocol::messages::ApiKey` (see `src/protocol/header.rs`); decoding/encoding the +//! header bytes themselves is `kafka_protocol::messages::RequestHeader`/`ResponseHeader`'s own +//! tested responsibility, not re-tested here. These tests cover the gateway-specific policy +//! layered on top: the unknown-API-key fallback and the `ApiVersions` response-header special case. + use kafka_protocol::messages::ApiKey; use iggy_gateway_kafka::protocol::api::{ API_KEY_API_VERSIONS, API_KEY_CREATE_TOPICS, API_KEY_FETCH, API_KEY_LIST_OFFSETS, API_KEY_METADATA, API_KEY_PRODUCE, }; -use iggy_gateway_kafka::protocol::codec::Encoder; -use iggy_gateway_kafka::protocol::header::{ - RequestHeader, ResponseHeader, request_header_version, response_header_version, -}; - -// ── Request header v1 (non-flexible) ─────────────────────────────────────── - -#[test] -fn request_header_v1_decodes() { - let mut enc = Encoder::with_capacity(64); - enc.write_i16(18); // api_key: ApiVersions - enc.write_i16(2); // api_version - enc.write_i32(101); - enc.write_nullable_string(Some("kafka-cli")).unwrap(); - let bytes = enc.freeze(); - - let header = RequestHeader::decode(bytes, 1).expect("decode should succeed"); - assert_eq!(header.api_key, 18); - assert_eq!(header.api_version, 2); - assert_eq!(header.correlation_id, 101); - assert_eq!(header.client_id.as_deref(), Some("kafka-cli")); -} - -#[test] -fn request_header_v1_null_client_id() { - let mut enc = Encoder::with_capacity(32); - enc.write_i16(18); - enc.write_i16(1); - enc.write_i32(5); - enc.write_nullable_string(None).unwrap(); - let bytes = enc.freeze(); - - let header = RequestHeader::decode(bytes, 1).unwrap(); - assert_eq!(header.client_id, None); -} - -// ── Request header v2 (flexible - compact client_id + tagged fields) ─────── - -#[test] -fn request_header_v2_decodes() { - let mut enc = Encoder::with_capacity(64); - enc.write_i16(18); // api_key: ApiVersions - enc.write_i16(3); // api_version (flexible threshold for ApiVersions is 3) - enc.write_i32(202); - enc.write_compact_nullable_string(Some("my-client")); - enc.write_empty_tagged_fields(); - let bytes = enc.freeze(); - - let header = RequestHeader::decode(bytes, 2).expect("flexible decode should succeed"); - assert_eq!(header.api_key, 18); - assert_eq!(header.api_version, 3); - assert_eq!(header.correlation_id, 202); - assert_eq!(header.client_id.as_deref(), Some("my-client")); -} - -#[test] -fn request_header_v2_null_client_id() { - let mut enc = Encoder::with_capacity(32); - enc.write_i16(18); - enc.write_i16(3); - enc.write_i32(303); - enc.write_compact_nullable_string(None); - enc.write_empty_tagged_fields(); - let bytes = enc.freeze(); - - let header = RequestHeader::decode(bytes, 2).unwrap(); - assert_eq!(header.client_id, None); -} - -// ── Response header encode ────────────────────────────────────────────────── - -#[test] -fn response_header_v0_encodes_correlation_id_only() { - let header = ResponseHeader { correlation_id: 77 }; - let bytes = header.encode(0); - assert_eq!(bytes.as_ref(), &[0, 0, 0, 77]); -} - -#[test] -fn response_header_v1_encodes_correlation_id_plus_tagged_fields() { - let header = ResponseHeader { correlation_id: 1 }; - let bytes = header.encode(1); - // [0,0,0,1] correlation_id + [0x00] empty tagged fields - assert_eq!(bytes.as_ref(), &[0, 0, 0, 1, 0x00]); -} - -// ── Header version lookup ─────────────────────────────────────────────────── +use iggy_gateway_kafka::protocol::header::{request_header_version, response_header_version}; /// Flexible-encoding threshold per API key (mirrors `protocol/header.rs`; cross-checked against /// the independent `kafka-protocol` crate below rather than trusted on its own). +/// +/// Keys 4-7 (LeaderAndIsr/StopReplica/UpdateMetadata/ControlledShutdown) are inter-broker-only +/// APIs `kafka_protocol` 0.17 does not implement (`ApiKey::try_from` fails for them), so this +/// gateway's wrapper always falls back to header v1 for them - `i16::MAX`, not their legacy +/// threshold from the pre-migration hand-rolled table. const API_KEY_FLEXIBLE_FROM: &[(i16, i16)] = &[ (0, 9), (1, 12), (2, 6), (3, 9), - (4, 4), - (5, 2), - (6, 6), - (7, 3), + (4, i16::MAX), + (5, i16::MAX), + (6, i16::MAX), + (7, i16::MAX), (8, 8), (9, 6), (10, 3), @@ -139,7 +64,10 @@ const API_KEY_FLEXIBLE_FROM: &[(i16, i16)] = &[ (24, 3), (25, 3), (26, 3), - (27, 1), + // WriteTxnMarkers' only valid versions are 1-2 (no v0 on the real wire) and both are + // flexible; `kafka_protocol` encodes this as an unconditional header v2, matching the `0` + // ("always flexible") arm below rather than a real threshold. + (27, 0), (28, 3), (29, 2), (30, 2), @@ -222,8 +150,16 @@ fn request_header_version_hits_every_api_key_match_arm() { assert_eq!(request_header_version(api_key, i16::MAX - 1), 1); } threshold => { - assert_eq!(request_header_version(api_key, threshold - 1), 1); - assert_eq!(request_header_version(api_key, threshold), 2); + assert_eq!( + request_header_version(api_key, threshold - 1), + 1, + "api_key={api_key} threshold={threshold}" + ); + assert_eq!( + request_header_version(api_key, threshold), + 2, + "api_key={api_key} threshold={threshold}" + ); } } } @@ -327,54 +263,6 @@ fn request_header_version_unknown_api_defaults_to_v1() { assert_eq!(request_header_version(-1, 12), 1); } -#[test] -fn request_header_decode_rejects_unsupported_version() { - let bytes = Encoder::with_capacity(0).freeze(); - let err = RequestHeader::decode(bytes, 99).unwrap_err(); - assert!(matches!( - err, - iggy_gateway_kafka::error::KafkaProtocolError::UnsupportedHeaderVersion(99) - )); -} - -#[test] -fn request_header_v1_truncated_payload_fails() { - let mut enc = Encoder::with_capacity(8); - enc.write_i16(18); - enc.write_i16(1); - let err = RequestHeader::decode(enc.freeze(), 1).unwrap_err(); - assert!(err.to_string().contains("buffer underflow")); -} - -#[test] -fn request_header_v2_truncated_before_tagged_fields_fails() { - let mut enc = Encoder::with_capacity(16); - enc.write_i16(18); - enc.write_i16(3); - enc.write_i32(303); - enc.write_compact_nullable_string(Some("c")); - let err = RequestHeader::decode(enc.freeze(), 2).unwrap_err(); - assert!(err.to_string().contains("buffer underflow")); -} - -#[test] -fn response_header_encode_into_matches_encode() { - let header = ResponseHeader { - correlation_id: 1234, - }; - let encoded = header.encode(1); - let mut buf = bytes::BytesMut::new(); - header.encode_into(&mut buf, 1); - assert_eq!(buf.freeze(), encoded); -} - -#[test] -fn response_header_encoded_size_matches_versions() { - assert_eq!(ResponseHeader::encoded_size(0), 4); - assert_eq!(ResponseHeader::encoded_size(1), 5); - assert_eq!(ResponseHeader::encoded_size(2), 5); -} - // ── Flexible-encoding boundaries (SCOPE.md) ───────────────────────────────── #[test] diff --git a/gateways/kafka/tests/listener_robustness_tests.rs b/gateways/kafka/tests/listener_robustness_tests.rs index 74a4a604d3..d6fea55a1c 100644 --- a/gateways/kafka/tests/listener_robustness_tests.rs +++ b/gateways/kafka/tests/listener_robustness_tests.rs @@ -17,6 +17,8 @@ //! TCP listener robustness - framing, pipelining, concurrency, edge cases. +#[path = "common/codec.rs"] +mod codec; #[path = "common/server.rs"] mod server; #[path = "common/tcp.rs"] @@ -35,8 +37,8 @@ use iggy_gateway_kafka::ServerConfig; use iggy_gateway_kafka::protocol::api::{ API_KEY_API_VERSIONS, API_KEY_FETCH, API_KEY_METADATA, API_KEY_PRODUCE, ERROR_INVALID_REQUEST, }; -use iggy_gateway_kafka::protocol::codec::Decoder; +use codec::Decoder; use server::{spawn_test_server, spawn_test_server_with_config}; use tcp::{ ByteRead, build_request_frame, concat_frames, parse_response_payload, read_byte_with_timeout, @@ -499,12 +501,14 @@ async fn e2e_quiet_connection_survives_beyond_read_timeout_idle_cap() { // ── Corrupt body survives on the connection (no disconnect) ──────────────── #[tokio::test] -async fn corrupt_produce_body_e2e_returns_error_without_disconnect() { +async fn corrupt_produce_body_e2e_stays_silent_without_disconnect() { + // `kafka_protocol` decodes Produce in one shot, so a decode failure never exposes whether + // `acks` was nonzero (unlike the pre-migration field-by-field decoder, which could still + // answer with INVALID_REQUEST once it knew acks was nonzero). Every Produce decode failure + // now stays silent - see `api::handle_produce_request` - but must not drop the connection. let (addr, _shutdown) = spawn_test_server().await; let mut stream = TcpStream::connect(addr).await.expect("connect"); - // acks is readable (=1), so the client expects an error response; the topics array is - // truncated, forcing INVALID_REQUEST. let bad = build_request_frame( API_KEY_PRODUCE, 3, @@ -515,13 +519,11 @@ async fn corrupt_produce_body_e2e_returns_error_without_disconnect() { ], ); stream.write_all(&bad).await.expect("corrupt produce"); - let payload = read_response_frame(&mut stream, 8 * 1024 * 1024).await; assert!( - scan_for_error_code( - &parse_response_payload(API_KEY_PRODUCE, 3, payload).1, - ERROR_INVALID_REQUEST - ), - "corrupt Produce must surface INVALID_REQUEST" + read_response_frame_with_timeout(&mut stream, 8 * 1024 * 1024, Duration::from_millis(200)) + .await + .is_none(), + "corrupt Produce must stay silent, not respond with an error" ); let ok = build_request_frame(API_KEY_API_VERSIONS, 1, 392, Some("scope-test"), &[]); @@ -529,7 +531,8 @@ async fn corrupt_produce_body_e2e_returns_error_without_disconnect() { let payload = read_response_frame(&mut stream, 8 * 1024 * 1024).await; assert_eq!( parse_response_payload(API_KEY_API_VERSIONS, 1, payload).0, - 392 + 392, + "connection must stay usable after the silent corrupt Produce" ); } diff --git a/gateways/kafka/tests/response_negative_tests.rs b/gateways/kafka/tests/response_negative_tests.rs index ffac83ab27..58bc110eac 100644 --- a/gateways/kafka/tests/response_negative_tests.rs +++ b/gateways/kafka/tests/response_negative_tests.rs @@ -15,34 +15,46 @@ // specific language governing permissions and limitations // under the License. +#[path = "common/codec.rs"] +mod codec; + +use kafka_protocol::messages::create_topics_request::CreatableTopic; +use kafka_protocol::messages::fetch_request::FetchTopic; +use kafka_protocol::messages::list_offsets_request::ListOffsetsTopic; +use kafka_protocol::messages::produce_request::TopicProduceData; +use kafka_protocol::messages::{ + BrokerId, CreateTopicsRequest, FetchRequest, ListOffsetsRequest, TopicName, +}; +use kafka_protocol::protocol::StrBytes; + use iggy_gateway_kafka::protocol::api::{ ERROR_INVALID_PARTITIONS, ERROR_INVALID_REPLICATION_FACTOR, ERROR_INVALID_REQUEST, ERROR_NOT_CONTROLLER, ERROR_UNSUPPORTED_VERSION, }; -use iggy_gateway_kafka::protocol::codec::Decoder; -use iggy_gateway_kafka::protocol::requests::{ - CreatableTopic, CreateTopicsRequest, FetchPartition, FetchRequest, FetchTopic, - ListOffsetsPartition, ListOffsetsRequest, ListOffsetsTopic, ProducePartitionData, - ProduceRequest, ProduceTopicData, -}; use iggy_gateway_kafka::protocol::responses::{ encode_create_topics_error_response, encode_create_topics_response, - encode_fetch_error_response, encode_list_offsets_error_response, encode_produce_error_response, + encode_fetch_error_response, encode_fetch_response, encode_list_offsets_error_response, + encode_produce_error_response, }; +use codec::Decoder; + +fn topic_name(name: &str) -> TopicName { + TopicName(StrBytes::from_string(name.to_string())) +} + +fn creatable_topic(name: &str, num_partitions: i32, replication_factor: i16) -> CreatableTopic { + CreatableTopic::default() + .with_name(topic_name(name)) + .with_num_partitions(num_partitions) + .with_replication_factor(replication_factor) +} + #[test] fn create_topics_response_flags_non_positive_partition_count_v2() { - let req = CreateTopicsRequest { - topics: vec![CreatableTopic { - name: "bad-topic".to_string(), - num_partitions: 0, - replication_factor: 1, - has_assignments: false, - }], - timeout_ms: 5_000, - validate_only: false, - }; - let mut d = Decoder::new(encode_create_topics_response(2, &req)); + let req = CreateTopicsRequest::default().with_topics(vec![creatable_topic("bad-topic", 0, 1)]); + let body = encode_create_topics_response(2, &req).unwrap(); + let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 0); // throttle assert_eq!(d.read_i32().unwrap(), 1); // topics len assert_eq!( @@ -55,17 +67,11 @@ fn create_topics_response_flags_non_positive_partition_count_v2() { #[test] fn create_topics_v5_broker_default_partitions_is_not_invalid_partitions() { // KIP-464: -1 = broker default on CreateTopics v4+; stub still returns NOT_CONTROLLER. - let req = CreateTopicsRequest { - topics: vec![CreatableTopic { - name: "default-parts".to_string(), - num_partitions: -1, - replication_factor: 2, - has_assignments: false, - }], - timeout_ms: 5_000, - validate_only: true, - }; - let mut d = Decoder::new(encode_create_topics_response(5, &req)); + let req = CreateTopicsRequest::default() + .with_topics(vec![creatable_topic("default-parts", -1, 2)]) + .with_validate_only(true); + let body = encode_create_topics_response(5, &req).unwrap(); + let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 0); // throttle assert_eq!(d.read_varint().unwrap(), 2); // one topic assert_eq!( @@ -81,17 +87,13 @@ fn create_topics_v5_broker_default_partitions_is_not_invalid_partitions() { #[test] fn create_topics_v5_flags_zero_and_below_minus_one_partition_count() { for num_partitions in [0i32, -2] { - let req = CreateTopicsRequest { - topics: vec![CreatableTopic { - name: "bad-parts".to_string(), - num_partitions, - replication_factor: 1, - has_assignments: false, - }], - timeout_ms: 5_000, - validate_only: false, - }; - let mut d = Decoder::new(encode_create_topics_response(5, &req)); + let req = CreateTopicsRequest::default().with_topics(vec![creatable_topic( + "bad-parts", + num_partitions, + 1, + )]); + let body = encode_create_topics_response(5, &req).unwrap(); + let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 0); assert_eq!(d.read_varint().unwrap(), 2); assert_eq!( @@ -109,17 +111,13 @@ fn create_topics_v5_flags_zero_and_below_minus_one_partition_count() { #[test] fn create_topics_v5_flags_invalid_replication_factor() { for replication_factor in [0i16, -2] { - let req = CreateTopicsRequest { - topics: vec![CreatableTopic { - name: "bad-rf".to_string(), - num_partitions: 1, - replication_factor, - has_assignments: false, - }], - timeout_ms: 5_000, - validate_only: false, - }; - let mut d = Decoder::new(encode_create_topics_response(5, &req)); + let req = CreateTopicsRequest::default().with_topics(vec![creatable_topic( + "bad-rf", + 1, + replication_factor, + )]); + let body = encode_create_topics_response(5, &req).unwrap(); + let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 0); assert_eq!(d.read_varint().unwrap(), 2); assert_eq!( @@ -137,17 +135,9 @@ fn create_topics_v5_flags_invalid_replication_factor() { #[test] fn create_topics_v2_rejects_broker_default_sentinel() { // KIP-464 defaults apply from v4; on v2 without assignments, -1 is INVALID_PARTITIONS. - let req = CreateTopicsRequest { - topics: vec![CreatableTopic { - name: "legacy".to_string(), - num_partitions: -1, - replication_factor: 1, - has_assignments: false, - }], - timeout_ms: 5_000, - validate_only: false, - }; - let mut d = Decoder::new(encode_create_topics_response(2, &req)); + let req = CreateTopicsRequest::default().with_topics(vec![creatable_topic("legacy", -1, 1)]); + let body = encode_create_topics_response(2, &req).unwrap(); + let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 0); assert_eq!(d.read_i32().unwrap(), 1); assert_eq!( @@ -160,17 +150,16 @@ fn create_topics_v2_rejects_broker_default_sentinel() { #[test] fn create_topics_v2_with_assignments_allows_broker_default_sentinels() { // KIP-464: on v2/v3, -1 partitions/replication are valid when assignments are present. - let req = CreateTopicsRequest { - topics: vec![CreatableTopic { - name: "assigned".to_string(), - num_partitions: -1, - replication_factor: -1, - has_assignments: true, - }], - timeout_ms: 5_000, - validate_only: false, - }; - let mut d = Decoder::new(encode_create_topics_response(2, &req)); + use kafka_protocol::messages::create_topics_request::CreatableReplicaAssignment; + + let assignment = CreatableReplicaAssignment::default() + .with_partition_index(0) + .with_broker_ids(vec![BrokerId(1)]); + let req = CreateTopicsRequest::default().with_topics(vec![ + creatable_topic("assigned", -1, -1).with_assignments(vec![assignment]), + ]); + let body = encode_create_topics_response(2, &req).unwrap(); + let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 0); assert_eq!(d.read_i32().unwrap(), 1); assert_eq!( @@ -182,10 +171,8 @@ fn create_topics_v2_with_assignments_allows_broker_default_sentinels() { #[test] fn create_topics_error_response_carries_explicit_error_code() { - let mut d = Decoder::new(encode_create_topics_error_response( - 5, - ERROR_UNSUPPORTED_VERSION, - )); + let body = encode_create_topics_error_response(5, ERROR_UNSUPPORTED_VERSION).unwrap(); + let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 0); assert_eq!(d.read_varint().unwrap(), 2); assert_eq!( @@ -197,7 +184,8 @@ fn create_topics_error_response_carries_explicit_error_code() { #[test] fn fetch_error_response_v7_uses_top_level_error_and_no_topics() { - let mut d = Decoder::new(encode_fetch_error_response(7, ERROR_INVALID_REQUEST)); + let body = encode_fetch_error_response(7, ERROR_INVALID_REQUEST).unwrap(); + let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 0); // throttle assert_eq!(d.read_i16().unwrap(), ERROR_INVALID_REQUEST); assert_eq!(d.read_i32().unwrap(), 0); // session_id @@ -207,7 +195,8 @@ fn fetch_error_response_v7_uses_top_level_error_and_no_topics() { #[test] fn fetch_error_response_v12_uses_flexible_empty_topics() { - let mut d = Decoder::new(encode_fetch_error_response(12, ERROR_UNSUPPORTED_VERSION)); + let body = encode_fetch_error_response(12, ERROR_UNSUPPORTED_VERSION).unwrap(); + let mut d = Decoder::new(body); assert_eq!(d.read_i32().unwrap(), 0); // throttle assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); assert_eq!(d.read_i32().unwrap(), 0); // session_id @@ -217,23 +206,31 @@ fn fetch_error_response_v12_uses_flexible_empty_topics() { } #[test] -fn list_offsets_error_response_v0_uses_legacy_old_style_offsets_array() { - let mut d = Decoder::new(encode_list_offsets_error_response( - 0, - ERROR_UNSUPPORTED_VERSION, - )); - assert_eq!(d.read_i32().unwrap(), 1); // topics - assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); - assert_eq!(d.read_i32().unwrap(), 1); // partitions +fn list_offsets_error_response_v6_uses_flexible_shape() { + // v0's legacy `old_style_offsets` shape has no `kafka_protocol` encoder - see + // `responses::encode_list_offsets_error_response` and the version-firewall e2e coverage for + // that behavior. This exercises the lowest version the crate can actually encode. + let body = encode_list_offsets_error_response(6, ERROR_UNSUPPORTED_VERSION).unwrap(); + let mut d = Decoder::new(body); + assert_eq!(d.read_i32().unwrap(), 0); // throttle + assert_eq!(d.read_varint().unwrap(), 2); // one topic + assert_eq!( + d.read_compact_nullable_string().unwrap(), + Some(String::new()) + ); + assert_eq!(d.read_varint().unwrap(), 2); // one partition assert_eq!(d.read_i32().unwrap(), 0); // partition index assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); - assert_eq!(d.read_i32().unwrap(), 0); // old_style_offsets len - assert_eq!(d.remaining(), 0); + assert_eq!(d.read_i64().unwrap(), -1); // timestamp + assert_eq!(d.read_i64().unwrap(), -1); // offset + assert_eq!(d.read_i32().unwrap(), -1); // leader_epoch + d.read_tagged_fields().unwrap(); } #[test] fn produce_error_response_v9_uses_flexible_record_errors_shape() { - let mut d = Decoder::new(encode_produce_error_response(9, ERROR_INVALID_REQUEST)); + let body = encode_produce_error_response(9, ERROR_INVALID_REQUEST).unwrap(); + let mut d = Decoder::new(body); assert_eq!(d.read_varint().unwrap(), 2); // one topic assert_eq!( d.read_compact_nullable_string().unwrap(), @@ -251,58 +248,30 @@ fn produce_error_response_v9_uses_flexible_record_errors_shape() { #[test] fn success_responses_can_still_encode_empty_request_vectors() { - let produce = ProduceRequest { - transactional_id: None, - acks: 1, - timeout_ms: 1_000, - topics: Vec::::new(), - }; + let produce = kafka_protocol::messages::ProduceRequest::default() + .with_acks(1) + .with_timeout_ms(1_000) + .with_topic_data(Vec::::new()); assert!( - !iggy_gateway_kafka::protocol::responses::encode_produce_response(3, &produce).is_empty() + !iggy_gateway_kafka::protocol::responses::encode_produce_response(3, &produce) + .unwrap() + .is_empty() ); - let fetch = FetchRequest { - max_wait_ms: 0, - min_bytes: 0, - max_bytes: 0, - isolation_level: 0, - topics: Vec::::new(), - }; - assert!(!iggy_gateway_kafka::protocol::responses::encode_fetch_response(4, &fetch).is_empty()); + let fetch = FetchRequest::default().with_topics(Vec::::new()); + assert!(!encode_fetch_response(4, &fetch).unwrap().is_empty()); - let list_offsets = ListOffsetsRequest { - isolation_level: 0, - topics: Vec::::new(), - }; + let list_offsets = ListOffsetsRequest::default().with_topics(Vec::::new()); assert!( !iggy_gateway_kafka::protocol::responses::encode_list_offsets_response(1, &list_offsets) + .unwrap() .is_empty() ); - let create_topics = CreateTopicsRequest { - topics: Vec::::new(), - timeout_ms: 0, - validate_only: false, - }; + let create_topics = CreateTopicsRequest::default().with_topics(Vec::::new()); assert!( - !iggy_gateway_kafka::protocol::responses::encode_create_topics_response(2, &create_topics) + !encode_create_topics_response(2, &create_topics) + .unwrap() .is_empty() ); } - -#[allow(clippy::let_unit_value)] -fn _type_anchors() { - let _ = FetchPartition { - partition: 0, - fetch_offset: 0, - partition_max_bytes: 0, - }; - let _ = ListOffsetsPartition { - partition: 0, - timestamp: 0, - }; - let _ = ProducePartitionData { - partition: 0, - records: None, - }; -} diff --git a/gateways/kafka/tests/server_e2e_tests.rs b/gateways/kafka/tests/server_e2e_tests.rs index da961945b2..4f48337437 100644 --- a/gateways/kafka/tests/server_e2e_tests.rs +++ b/gateways/kafka/tests/server_e2e_tests.rs @@ -17,6 +17,8 @@ //! End-to-end TCP tests through `KafkaServer` (full request/response cycle). +#[path = "common/codec.rs"] +mod codec; #[path = "common/fixtures.rs"] mod fixtures; #[path = "common/server.rs"] @@ -34,8 +36,8 @@ use iggy_gateway_kafka::protocol::api::{ API_KEY_API_VERSIONS, API_KEY_CREATE_TOPICS, API_KEY_FETCH, API_KEY_LIST_OFFSETS, API_KEY_METADATA, API_KEY_PRODUCE, ERROR_NOT_LEADER_OR_FOLLOWER, }; -use iggy_gateway_kafka::protocol::codec::Decoder; +use codec::Decoder; use fixtures::load_fixture_body_or_skip; use server::spawn_test_server; use std::time::Duration; @@ -227,10 +229,14 @@ async fn e2e_produce_v3_acks_one_still_returns_response() { assert!(!resp_body.is_empty()); } -// ── ListOffsets v0 wire shape (old_style_offsets array, not bare i64) ─────── +// ── ListOffsets v0 (no encodable representation in kafka_protocol) ───────── #[tokio::test] -async fn e2e_list_offsets_v0_unsupported_version_no_trailing_bytes() { +async fn e2e_list_offsets_v0_closes_connection() { + // `kafka_protocol` has no encoder for ListOffsets v0's legacy `old_style_offsets` shape + // (it predates the schema the crate generates from - see `responses::encode_list_offsets_error_response`), + // so a v0 request - already below the firewall's min=1 - now closes instead of getting the + // pre-migration downgraded response. let (addr, _shutdown) = spawn_test_server().await; let mut stream = TcpStream::connect(addr).await.expect("connect"); @@ -247,27 +253,11 @@ async fn e2e_list_offsets_v0_unsupported_version_no_trailing_bytes() { .await .expect("write list offsets v0"); - let payload = - read_response_frame_with_timeout(&mut stream, 8 * 1024 * 1024, Duration::from_secs(2)) - .await - .expect("ListOffsets v0 should still get an error response"); - - let (_corr, body) = parse_response_payload(API_KEY_LIST_OFFSETS, 0, payload); - let mut d = Decoder::new(body); - assert_eq!(d.read_i32().unwrap(), 1); - d.read_nullable_string().unwrap(); - assert_eq!(d.read_i32().unwrap(), 1); - let _partition_index = d.read_i32().expect("partition_index"); - let _error_code = d.read_i16().expect("error_code"); - let offset_count = d.read_i32().expect("old_style_offsets array length"); - assert!( - offset_count >= 0, - "old_style_offsets count must be non-negative, got {offset_count}" + assert_eq!( + read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await, + ByteRead::Closed, + "ListOffsets v0 has no encodable response shape and must close" ); - for _ in 0..offset_count { - d.read_i64().expect("old_style_offsets entry"); - } - assert_eq!(d.remaining(), 0); } // ── Metadata topic name echo (must not hardcode a placeholder topic name) ── diff --git a/gateways/kafka/tests/server_integration_tests.rs b/gateways/kafka/tests/server_integration_tests.rs index 9a8f3c87d6..5acbbc1bd0 100644 --- a/gateways/kafka/tests/server_integration_tests.rs +++ b/gateways/kafka/tests/server_integration_tests.rs @@ -15,13 +15,16 @@ // specific language governing permissions and limitations // under the License. +#[path = "common/codec.rs"] +mod codec; + use std::time::Duration; use bytes::{Buf, BufMut, BytesMut}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; -use iggy_gateway_kafka::protocol::codec::Encoder; +use codec::Encoder; use iggy_gateway_kafka::server::read_frame; async fn tcp_pair() -> (TcpStream, TcpStream) { diff --git a/gateways/kafka/tests/version_firewall_tests.rs b/gateways/kafka/tests/version_firewall_tests.rs index ecefadf9c9..cb678d3957 100644 --- a/gateways/kafka/tests/version_firewall_tests.rs +++ b/gateways/kafka/tests/version_firewall_tests.rs @@ -17,6 +17,8 @@ //! Version negotiation firewall - boundary tests for every scoped API key. +#[path = "common/codec.rs"] +mod codec; #[path = "common/fixtures.rs"] mod fixtures; #[path = "common/scope.rs"] @@ -40,8 +42,8 @@ use iggy_gateway_kafka::protocol::api::{ ERROR_UNSUPPORTED_VERSION, advertised_min_version, handle_request, is_supported_version, supported_api_ranges, }; -use iggy_gateway_kafka::protocol::codec::Decoder; +use codec::Decoder; use fixtures::{fixture_exists, load_fixture_body, load_fixture_body_or_skip}; use scope::{SCOPED_API_KEYS, default_broker}; use server::spawn_test_server; @@ -228,40 +230,33 @@ async fn e2e_metadata_above_max_version_closes_tcp_connection() { } #[test] -fn produce_unsupported_version_returns_well_formed_error_response() { +fn produce_below_min_version_with_nonzero_acks_closes_connection() { + // Produce v2 is below both the firewall min (3) and `kafka_protocol`'s schema floor (3-13) + // - no encodable response exists at this version, so a client expecting a reply (acks != 0) + // gets a close instead of the pre-migration downgraded error response. acks=0 still keeps + // the connection open - see `produce_advertises_min_zero_but_firewall_rejects_below_v3` and + // `api::handle_produce_request`'s hand-peeked acks path. let body = handle_request( API_KEY_PRODUCE, 2, build_produce_v2_body(1, 0), &default_broker(), - ) - .expect_response("test request has acks != 0 and expects a response"); - let mut d = Decoder::new(body); - assert_eq!(d.read_i32().unwrap(), 1); - assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); - assert_eq!(d.read_i32().unwrap(), 1); - assert_eq!(d.read_i32().unwrap(), 0); - assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); - let _ = d.read_i64().unwrap(); - let _ = d.read_i64().unwrap(); - assert_eq!(d.read_i32().unwrap(), 0); - assert_eq!(d.remaining(), 0); + ); + assert!( + body.is_close(), + "Produce v2 with acks != 0 has no encodable response shape and must close" + ); } #[test] -fn fetch_unsupported_version_returns_well_formed_error_response() { - let body = handle_request(API_KEY_FETCH, 3, Bytes::new(), &default_broker()) - .expect_response("test request has acks != 0 and expects a response"); - let mut d = Decoder::new(body); - assert_eq!(d.read_i32().unwrap(), 0); - assert_eq!(d.read_i32().unwrap(), 1); - assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); - assert_eq!(d.read_i32().unwrap(), 1); - assert_eq!(d.read_i32().unwrap(), 0); - assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); - assert_eq!(d.read_i64().unwrap(), 0); - assert_eq!(d.read_nullable_bytes().unwrap(), None); - assert_eq!(d.remaining(), 0); +fn fetch_below_min_version_closes_connection() { + // Fetch v3 is below both the firewall min (4) and `kafka_protocol`'s schema floor (4-18) - + // no encodable response exists at this version, so this closes instead of the + // pre-migration downgraded error response. + assert!( + handle_request(API_KEY_FETCH, 3, Bytes::new(), &default_broker()).is_close(), + "Fetch v3 has no encodable response shape and must close" + ); } #[test] @@ -298,29 +293,25 @@ fn list_offsets_unsupported_version_above_max_closes_connection() { } #[test] -fn list_offsets_unsupported_version_returns_well_formed_error_response() { - let body = handle_request(API_KEY_LIST_OFFSETS, 0, Bytes::new(), &default_broker()) - .expect_response("test request has acks != 0 and expects a response"); - let mut d = Decoder::new(body); - assert_eq!(d.read_i32().unwrap(), 1); - assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); - assert_eq!(d.read_i32().unwrap(), 1); - assert_eq!(d.read_i32().unwrap(), 0); // partition index - assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); - assert_eq!(d.read_i32().unwrap(), 0); // old_style_offsets empty array (v0 wire) - assert_eq!(d.remaining(), 0); +fn list_offsets_v0_closes_connection() { + // `kafka_protocol` has no encoder for ListOffsets v0's legacy `old_style_offsets` shape (it + // predates the schema the crate generates from), so a v0 request - already below the + // firewall's min=1 - now closes instead of getting the pre-migration downgraded response. + assert!( + handle_request(API_KEY_LIST_OFFSETS, 0, Bytes::new(), &default_broker()).is_close(), + "ListOffsets v0 has no encodable response shape and must close" + ); } #[test] -fn create_topics_unsupported_version_returns_well_formed_error_response() { - let body = handle_request(API_KEY_CREATE_TOPICS, 1, Bytes::new(), &default_broker()) - .expect_response("test request has acks != 0 and expects a response"); - let mut d = Decoder::new(body); - assert_eq!(d.read_i32().unwrap(), 1); - assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); - assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); - assert_eq!(d.read_nullable_string().unwrap(), None); - assert_eq!(d.remaining(), 0); +fn create_topics_below_min_version_closes_connection() { + // CreateTopics v1 is below both the firewall min (2) and `kafka_protocol`'s schema floor + // (2-7) - no encodable response exists at this version, so this closes instead of the + // pre-migration downgraded error response. + assert!( + handle_request(API_KEY_CREATE_TOPICS, 1, Bytes::new(), &default_broker()).is_close(), + "CreateTopics v1 has no encodable response shape and must close" + ); } #[test] @@ -359,23 +350,21 @@ fn supported_fetch_versions_accept_valid_fixture() { } #[test] -fn corrupt_produce_body_returns_invalid_request_error() { - // null transactional_id, acks=1, timeout=0, then a truncated topics array: acks is readable, - // so the client expects (and gets) an error response. +fn corrupt_produce_body_with_acks_stays_silent() { + // `kafka_protocol` decodes Produce in one shot, so a decode failure never exposes `acks` + // (unlike the pre-migration field-by-field decoder, which could still answer with + // INVALID_REQUEST once it knew acks was nonzero). Every Produce decode failure now stays + // silent regardless of whether acks was readable before the truncation. let body = Bytes::from_static(&[ 0xFF, 0xFF, // null transactional_id 0x00, 0x01, // acks = 1 0x00, 0x00, 0x00, 0x00, // timeout_ms = 0 0xFF, 0xFF, 0xFF, // truncated topics count ]); - let resp = handle_request(API_KEY_PRODUCE, 3, body, &default_broker()) - .expect_response("test request has acks != 0 and expects a response"); - let mut d = Decoder::new(resp); - assert_eq!(d.read_i32().unwrap(), 1); - assert_eq!(d.read_nullable_string().unwrap(), Some(String::new())); - assert_eq!(d.read_i32().unwrap(), 1); - assert_eq!(d.read_i32().unwrap(), 0); - assert_eq!(d.read_i16().unwrap(), ERROR_INVALID_REQUEST); + assert!( + handle_request(API_KEY_PRODUCE, 3, body, &default_broker()).is_no_response(), + "malformed Produce body must stay silent regardless of acks" + ); } #[test] @@ -404,73 +393,15 @@ fn corrupt_fetch_body_returns_invalid_request_error() { assert_eq!(d.read_i16().unwrap(), ERROR_INVALID_REQUEST); } -// ── ListOffsets v0 wire shape (old_style_offsets array, not bare i64) ─────── - -/// Parse one `ListOffsets` v0 partition entry the way a v0 Kafka client would. -fn parse_list_offsets_v0_partition(d: &mut Decoder) { - let _partition_index = d.read_i32().expect("partition_index"); - let _error_code = d.read_i16().expect("error_code"); - let offset_count = d.read_i32().expect("old_style_offsets array length"); - assert!( - offset_count >= 0, - "old_style_offsets count must be non-negative, got {offset_count}" - ); - for _ in 0..offset_count { - d.read_i64().expect("old_style_offsets entry"); - } -} - -#[test] -fn list_offsets_v0_unsupported_version_is_parseable_by_v0_clients() { - let body = handle_request(API_KEY_LIST_OFFSETS, 0, Bytes::new(), &default_broker()) - .expect_response("test request has acks != 0 and expects a response"); - let mut d = Decoder::new(body); - - assert_eq!(d.read_i32().unwrap(), 1, "topics array length"); - assert_eq!( - d.read_nullable_string().unwrap(), - Some(String::new()), - "placeholder topic name" - ); - assert_eq!(d.read_i32().unwrap(), 1, "partitions array length"); - - parse_list_offsets_v0_partition(&mut d); - - assert_eq!( - d.remaining(), - 0, - "v0 client must consume the full error response without trailing bytes" - ); -} +// ── ListOffsets v0 (no encodable representation in kafka_protocol) ───────── #[test] -fn list_offsets_v0_unsupported_version_carries_error_code_in_partition() { +fn list_offsets_v0_with_topic_closes_connection() { let request_body = build_list_offsets_v0_request_with_topic_t(); - let body = handle_request(API_KEY_LIST_OFFSETS, 0, request_body, &default_broker()) - .expect_response("test request has acks != 0 and expects a response"); - let mut d = Decoder::new(body); - - assert_eq!(d.read_i32().unwrap(), 1); - d.read_nullable_string().unwrap(); - assert_eq!(d.read_i32().unwrap(), 1); - assert_eq!(d.read_i32().unwrap(), 0, "partition index"); - assert_eq!( - d.read_i16().unwrap(), - ERROR_UNSUPPORTED_VERSION, - "partition error code" - ); - - // partition_index and error_code were already asserted above; only the - // trailing old_style_offsets array remains for this single partition. - let offset_count = d.read_i32().expect("old_style_offsets array length"); assert!( - offset_count >= 0, - "old_style_offsets count must be non-negative, got {offset_count}" + handle_request(API_KEY_LIST_OFFSETS, 0, request_body, &default_broker()).is_close(), + "ListOffsets v0 has no encodable response shape and must close, even with a well-formed body" ); - for _ in 0..offset_count { - d.read_i64().expect("old_style_offsets entry"); - } - assert_eq!(d.remaining(), 0); } // ── Comprehensive scoped-API coverage (correlation id, boundary versions) ── @@ -654,14 +585,17 @@ async fn each_scoped_api_above_max_version_e2e_closes_or_kip511() { } #[tokio::test] -async fn each_scoped_api_below_min_version_e2e_keeps_connection() { +async fn each_scoped_api_below_min_version_e2e_closes_except_api_versions() { + // Every scoped API's firewall min was chosen at or above `kafka_protocol`'s own schema + // floor for that message (Produce 3, Fetch 4, ListOffsets 1's encoder actually starts at 1 + // but the crate's schema floor is 1 too - see below - Metadata 0, CreateTopics 2), so + // `min_ver - 1` has no encodable response under the crate and closes for every API except + // ApiVersions, which always answers per KIP-511 regardless of version validity. let (addr, _shutdown) = spawn_test_server().await; let mut stream = TcpStream::connect(addr).await.expect("connect"); for &(api_key, name, min_ver, _max_ver) in SCOPED_API_KEYS { let below = min_ver - 1; - // Produce is decoded before the firewall check (to honor acks=0 silence), so it needs a - // body with a readable acks; the other APIs reject on version before touching the body. let body = request_body_for_scoped_api(api_key, name, below); let frame = build_request_frame( api_key, @@ -674,22 +608,24 @@ async fn each_scoped_api_below_min_version_e2e_keeps_connection() { .write_all(&frame) .await .unwrap_or_else(|_| panic!("write {name} v{below}")); - if api_key == API_KEY_METADATA { - assert_eq!( - read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await, - ByteRead::Closed, - "Metadata v{below} must close the connection" + + if api_key == API_KEY_API_VERSIONS { + let payload = tcp::read_response_frame(&mut stream, 8 * 1024 * 1024).await; + assert!( + !payload.is_empty(), + "ApiVersions v{below} must still respond per KIP-511" ); - stream = TcpStream::connect(addr) - .await - .expect("reconnect after Metadata close"); continue; } - let payload = tcp::read_response_frame(&mut stream, 8 * 1024 * 1024).await; - assert!( - !payload.is_empty(), - "{name} v{below} must still respond on wire" + + assert_eq!( + read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await, + ByteRead::Closed, + "{name} v{below} has no encodable response shape and must close" ); + stream = TcpStream::connect(addr) + .await + .unwrap_or_else(|_| panic!("reconnect after {name} close")); } let ok = build_request_frame(API_KEY_API_VERSIONS, 1, 88_888, Some("scope-test"), &[]); @@ -713,19 +649,30 @@ fn produce_advertises_min_zero_but_firewall_rejects_below_v3() { assert!(!is_supported_version(API_KEY_PRODUCE, 0)); assert!(!is_supported_version(API_KEY_PRODUCE, 2)); - let body = handle_request( - API_KEY_PRODUCE, - 2, - build_produce_v2_body(1, 0), - &default_broker(), - ) - .expect_response("test request has acks != 0 and expects a response"); - let mut d = Decoder::new(body); - let _topics = d.read_i32().unwrap(); - let _name = d.read_nullable_string().unwrap(); - let _parts = d.read_i32().unwrap(); - assert_eq!(d.read_i32().unwrap(), 0, "partition index"); - assert_eq!(d.read_i16().unwrap(), ERROR_UNSUPPORTED_VERSION); + // acks=0 must keep the connection open even though no response is encodable at v2 - the + // fire-and-forget wire-protocol rule outranks "no parseable response" here. + assert!( + handle_request( + API_KEY_PRODUCE, + 2, + build_produce_v2_body(0, 0), + &default_broker(), + ) + .is_no_response(), + "Produce v2 acks=0 must stay silent, not close" + ); + + // acks != 0 has no encodable response at v2 and must close instead. + assert!( + handle_request( + API_KEY_PRODUCE, + 2, + build_produce_v2_body(1, 0), + &default_broker(), + ) + .is_close(), + "Produce v2 acks != 0 has no encodable response shape and must close" + ); } // ── Unsupported-version e2e paths for remaining scoped APIs ───────────────── @@ -750,13 +697,20 @@ async fn list_offsets_v7_unsupported_e2e_closes_connection() { } #[tokio::test] -async fn create_topics_v1_unsupported_e2e_returns_error() { +async fn create_topics_v1_unsupported_e2e_closes_connection() { + // CreateTopics v1 is below both the firewall min (2) and `kafka_protocol`'s schema floor + // (2-7) - no encodable response exists at this version. let (addr, _shutdown) = spawn_test_server().await; - let (corr, body) = round_trip(addr, API_KEY_CREATE_TOPICS, 1, 380, &[]).await; - assert_eq!(corr, 380); - assert!( - scan_for_error_code(&body, ERROR_UNSUPPORTED_VERSION), - "CreateTopics v1 must be rejected" + let mut stream = TcpStream::connect(addr).await.expect("connect"); + let frame = build_request_frame(API_KEY_CREATE_TOPICS, 1, 380, Some("scope-test"), &[]); + stream + .write_all(&frame) + .await + .expect("write create topics v1"); + assert_eq!( + read_byte_with_timeout(&mut stream, Duration::from_secs(2)).await, + ByteRead::Closed, + "CreateTopics v1 has no encodable response shape and must close" ); } diff --git a/gateways/kafka/tools/kafka-tool/src/main.rs b/gateways/kafka/tools/kafka-tool/src/main.rs index 5b052e7844..45c1a0b8a8 100644 --- a/gateways/kafka/tools/kafka-tool/src/main.rs +++ b/gateways/kafka/tools/kafka-tool/src/main.rs @@ -204,30 +204,6 @@ fn gateway_verify_registry() -> Vec<(i16, &'static str, i16, i16)> { // header v2: [client_id: COMPACT_NULLABLE_STRING] [request_header_tagged_fields] // [payload: bytes] -fn write_unsigned_varint(buf: &mut BytesMut, mut value: u64) { - loop { - let mut byte = (value & 0x7F) as u8; - value >>= 7; - if value != 0 { - byte |= 0x80; - } - buf.put_u8(byte); - if value == 0 { - break; - } - } -} - -fn write_compact_nullable_string(buf: &mut BytesMut, value: Option<&str>) { - match value { - None => write_unsigned_varint(buf, 0), - Some(s) => { - write_unsigned_varint(buf, (s.len() + 1) as u64); - buf.put_slice(s.as_bytes()); - } - } -} - fn frame_request( api_key: i16, api_version: i16, @@ -240,12 +216,13 @@ fn frame_request( header.put_i16(api_key); header.put_i16(api_version); header.put_i32(correlation_id); + // client_id is the legacy NULLABLE_STRING at every header version, even the "flexible" v2 - + // only the trailing tagged-fields section is new there. Kafka's RequestHeader schema never + // made client_id itself a compact string. + header.put_i16(i16::try_from(client_id.len()).expect("client_id fits i16")); + header.put_slice(client_id.as_bytes()); if flexible { - write_compact_nullable_string(&mut header, Some(client_id)); header.put_u8(0); // empty request-header tagged fields - } else { - header.put_i16(i16::try_from(client_id.len()).expect("client_id fits i16")); - header.put_slice(client_id.as_bytes()); } let blen = header.len() + payload.len(); From 3caa4c6f0b94b0f99615c6ba16ec63ecb69be35d Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Wed, 12 Aug 2026 07:16:48 -0400 Subject: [PATCH 53/57] Revert "Merge branch 'master' into feat(gateways)/kafka_to_iggy_listener" This reverts commit 7cee839f16b6b0b1818733f37c7223c63abad14c, reversing changes made to caf878e503bbb25041dec2d87f8fc1f596d04e6c. --- .claude/skills/connectors-overview/SKILL.md | 26 +- .dockerignore | 1 + .../actions/cpp-bazel/pre-merge/action.yml | 2 - .../csharp-dotnet/pre-merge/action.yml | 19 +- .github/actions/go/pre-merge/action.yml | 46 +- .../actions/java-gradle/pre-merge/action.yml | 8 - .github/actions/node-npm/pre-merge/action.yml | 94 +- .github/actions/php/pre-merge/action.yml | 3 - .../python-maturin/pre-merge/action.yml | 15 +- .github/actions/rust/pre-merge/action.yml | 21 +- .../utils/docker-build-test-server/action.yml | 105 + .github/actions/utils/server-start/action.yml | 9 +- .github/config/components.yml | 16 +- .github/dependabot.yml | 1 + .github/workflows/_test.yml | 3 +- .github/workflows/_test_bdd.yml | 52 +- .github/workflows/_test_examples.yml | 20 +- .github/workflows/coverage-baseline.yml | 42 +- .github/workflows/pr-title.yml | 1 + .github/workflows/pre-merge.yml | 2 +- AGENTS.md | 4 +- CONTRIBUTING.md | 22 - Cargo.lock | 74 +- Cargo.toml | 2 + Dockerfile | 3 +- README.md | 10 +- bdd/README.md | 11 +- bdd/docker-compose.cluster.yml | 50 +- bdd/docker-compose.server.yml | 8 +- bdd/docker-compose.vsr.yml | 101 + bdd/go/tests/tcp_test/test_helpers.go | 9 +- .../iggy/bdd/LeaderRedirectionSteps.java | 35 +- bdd/python/tests/test_basic_messaging.py | 3 +- bdd/rust/Cargo.toml | 1 + codecov.yml | 3 + core/ai/mcp/Cargo.toml | 1 + core/bench/Cargo.toml | 12 + .../frontend/scripts/select_index.sh | 2 +- core/bench/src/analytics/report_builder.rs | 4 +- core/bench/src/main.rs | 16 +- core/binary_protocol/Cargo.toml | 1 - core/binary_protocol/src/consensus/command.rs | 2 +- core/binary_protocol/src/consensus/error.rs | 17 - core/binary_protocol/src/consensus/header.rs | 967 +--- core/binary_protocol/src/consensus/mod.rs | 11 +- core/binary_protocol/src/framing.rs | 4 +- core/binary_protocol/src/lib.rs | 14 +- core/binary_protocol/src/namespace.rs | 36 +- .../src/requests/users/login_register.rs | 6 +- .../requests/users/login_register_with_pat.rs | 2 +- .../src/responses/topics/create_topic.rs | 2 +- .../src/responses/users/login_register.rs | 2 +- core/binary_protocol/src/version.rs | 4 +- core/cli/Cargo.toml | 1 + core/common/Cargo.toml | 3 + core/common/src/error/eviction.rs | 4 +- core/common/src/lib.rs | 3 + core/common/src/traits/binary_client.rs | 15 +- .../traits/binary_impls/consumer_groups.rs | 2 + .../src/traits/binary_impls/messages.rs | 16 + core/common/src/traits/binary_impls/mod.rs | 7 +- .../binary_impls/personal_access_tokens.rs | 110 +- core/common/src/traits/binary_impls/system.rs | 1 + core/common/src/traits/binary_impls/users.rs | 108 +- core/common/src/traits/binary_transport.rs | 5 + core/common/src/traits/message_client.rs | 2 +- core/common/src/utils/serde_secret.rs | 95 +- core/configs/Cargo.toml | 1 + core/configs/src/common/defaults.rs | 431 -- core/configs/src/common/displays.rs | 280 - core/configs/src/common/server.rs | 144 - core/configs/src/common/validators.rs | 357 -- core/configs/src/lib.rs | 11 +- .../cache_indexes.rs | 0 core/configs/src/server_config/cluster.rs | 2450 +------- core/configs/src/server_config/defaults.rs | 620 +- core/configs/src/server_config/displays.rs | 311 +- .../src/{common => server_config}/http.rs | 0 core/configs/src/server_config/mod.rs | 14 +- core/configs/src/server_config/quic.rs | 157 +- core/configs/src/server_config/server.rs | 263 +- core/configs/src/server_config/sharding.rs | 383 +- .../src/{common => server_config}/system.rs | 13 +- core/configs/src/server_config/tcp.rs | 2 - core/configs/src/server_config/validators.rs | 1268 +++-- core/configs/src/server_config/websocket.rs | 100 +- core/configs/src/server_ng_config/cluster.rs | 2612 +++++++++ core/configs/src/server_ng_config/defaults.rs | 335 ++ core/configs/src/server_ng_config/displays.rs | 189 + .../message_bus.rs | 38 +- .../metadata.rs | 76 +- core/configs/src/server_ng_config/mod.rs | 45 + .../partition.rs | 56 +- core/configs/src/server_ng_config/quic.rs | 222 + .../configs/src/server_ng_config/server_ng.rs | 229 + core/configs/src/server_ng_config/sharding.rs | 431 ++ core/configs/src/server_ng_config/tcp.rs | 62 + .../src/server_ng_config/validators.rs | 861 +++ .../configs/src/server_ng_config/websocket.rs | 116 + core/connectors/runtime/Cargo.toml | 3 + core/consensus/Cargo.toml | 1 - core/consensus/src/client_table.rs | 9 +- core/consensus/src/dvc_merge.rs | 981 ---- core/consensus/src/impls.rs | 909 +-- core/consensus/src/lib.rs | 9 +- core/consensus/src/observability.rs | 2 +- core/consensus/src/plane_helpers.rs | 1061 +--- core/consensus/src/view_change_quorum.rs | 429 +- core/harness_derive/src/attrs.rs | 7 +- core/harness_derive/src/codegen.rs | 4 +- core/integration/Cargo.toml | 14 +- core/integration/src/bench_utils.rs | 13 +- .../integration/src/harness/config/resolve.rs | 35 +- core/integration/src/harness/config/server.rs | 4 +- core/integration/src/harness/handle/server.rs | 23 +- .../src/harness/orchestrator/builder.rs | 13 +- .../src/harness/orchestrator/harness.rs | 14 +- .../cli/message/test_message_flush_command.rs | 14 +- .../cli/stream/test_stream_purge_command.rs | 2 +- .../cli/system/test_cli_session_scenario.rs | 4 +- .../tests/cli/system/test_me_command.rs | 23 +- .../cli/topic/test_topic_purge_command.rs | 2 +- .../tests/cluster/client_table_restart.rs | 7 + .../cluster/metadata_checkpoint_restart.rs | 10 +- .../tests/cluster/metadata_state_transfer.rs | 2 + .../multi_shard_partition_convergence.rs | 2 + .../tests/cluster/partition_state_transfer.rs | 14 +- core/integration/tests/config_provider/mod.rs | 2 +- core/integration/tests/data_integrity/mod.rs | 13 +- .../verify_after_server_restart.rs | 45 +- .../verify_auto_commit_offset_replicates.rs | 10 +- ...ify_consumer_group_partition_assignment.rs | 81 +- core/integration/tests/mod.rs | 17 +- .../tests/sdk/consumer_group_membership.rs | 2 +- core/integration/tests/sdk/hello_world.rs | 1 + core/integration/tests/sdk/http_refresh.rs | 2 +- core/integration/tests/sdk/mod.rs | 3 + .../integration/tests/sdk/protocol_version.rs | 4 + core/integration/tests/sdk/raw.rs | 11 + .../tests/sdk/send_confirmation.rs | 36 +- .../tests/server/a2a_jwt/jwt_tests.rs | 23 +- .../server/cluster_view_durability_vsr.rs | 4 +- core/integration/tests/server/flush_vsr.rs | 4 +- core/integration/tests/server/general.rs | 19 + core/integration/tests/server/http_client.rs | 4 +- core/integration/tests/server/http_rbac.rs | 4 +- core/integration/tests/server/http_tls.rs | 8 +- core/integration/tests/server/http_vsr.rs | 6 +- .../tests/server/legacy_login_vsr.rs | 6 +- .../tests/server/login_credentials_vsr.rs | 91 - core/integration/tests/server/mod.rs | 36 +- .../server/partition_view_durability_vsr.rs | 4 +- .../tests/server/poll_semantics_vsr.rs | 72 +- core/integration/tests/server/purge_delete.rs | 62 +- core/integration/tests/server/purge_vsr.rs | 101 +- .../scenarios/authentication_scenario.rs | 7 +- .../tests/server/scenarios/bench_scenario.rs | 45 + .../server/scenarios/encryption_scenario.rs | 40 +- .../integration/tests/server/scenarios/mod.rs | 6 +- .../server/scenarios/permissions_scenario.rs | 4 +- .../server/scenarios/purge_delete_scenario.rs | 207 +- .../reconnect_after_restart_scenario.rs | 2 +- .../scenarios/restart_offset_skip_scenario.rs | 14 +- .../stream_size_validation_scenario.rs | 13 +- .../tests/server/scenarios/system_scenario.rs | 7 +- core/integration/tests/server/specific.rs | 39 +- core/integration/tests/server/stats_vsr.rs | 39 - .../tests/server/topic_admission_vsr.rs | 71 +- core/integration/tests/state/file.rs | 164 + core/integration/tests/state/mod.rs | 92 + core/integration/tests/state/system.rs | 212 + .../tests/storage/consumer_offsets.rs | 179 + core/integration/tests/storage/mod.rs | 18 + core/journal/src/lib.rs | 19 - core/journal/src/prepare_journal.rs | 442 +- core/message_bus/src/client_listener/quic.rs | 2 +- core/message_bus/src/config.rs | 50 +- core/message_bus/src/installer/conn_info.rs | 2 +- core/message_bus/src/installer/mod.rs | 2 +- core/message_bus/src/installer/wss.rs | 2 +- core/message_bus/src/lib.rs | 24 +- core/message_bus/src/replica/io.rs | 4 +- core/message_bus/src/transports/quic.rs | 2 +- core/message_bus/src/transports/tls/mod.rs | 4 +- core/message_bus/tests/ws_client_roundtrip.rs | 4 +- core/metadata/Cargo.toml | 2 +- core/metadata/src/impls/metadata.rs | 303 +- core/metadata/src/impls/recovery.rs | 182 +- core/metadata/src/stm/result.rs | 1 - core/metadata/src/stm/snapshot.rs | 59 +- core/metadata/src/stm/stream.rs | 415 +- core/metadata/src/stm/user.rs | 4 +- core/partitions/Cargo.toml | 6 +- core/partitions/src/iggy_partition.rs | 764 +-- core/partitions/src/iggy_partitions.rs | 125 +- core/partitions/src/journal.rs | 184 +- core/partitions/src/lib.rs | 2 +- core/partitions/src/log.rs | 8 - core/partitions/src/messages_writer.rs | 89 +- core/partitions/src/offset_storage.rs | 364 +- core/partitions/src/poll_plan.rs | 85 +- core/partitions/src/state_transfer.rs | 86 +- core/partitions/src/types.rs | 9 - core/sdk/Cargo.toml | 3 + core/sdk/src/clients/client.rs | 35 +- core/sdk/src/http/http_client.rs | 2 +- core/sdk/src/lib.rs | 2 + core/sdk/src/prelude.rs | 24 +- core/sdk/src/quic/quic_client.rs | 149 +- core/sdk/src/session.rs | 2 +- core/sdk/src/tcp/tcp_client.rs | 341 +- core/sdk/src/tcp/tcp_connection_stream.rs | 2 + .../sdk/src/tcp/tcp_connection_stream_kind.rs | 2 + core/sdk/src/tcp/tcp_tls_connection_stream.rs | 2 + core/sdk/src/vsr.rs | 256 +- core/sdk/src/websocket/websocket_client.rs | 126 +- core/server-ng/.dockerignore | 30 + core/server-ng/Cargo.toml | 195 + core/server-ng/Dockerfile | 179 + core/server-ng/build.rs | 86 + core/server-ng/config.toml | 1027 ++++ core/server-ng/server.http | 369 ++ .../server-ng/src/args.rs | 38 +- core/{server => server-ng}/src/auth.rs | 14 +- core/server-ng/src/bootstrap.rs | 4385 ++++++++++++++ .../{server => server-ng}/src/cluster_meta.rs | 4 +- .../src/config_writer.rs | 19 +- .../src/consumer_group.rs | 10 +- core/{server => server-ng}/src/dispatch.rs | 231 +- .../src/dispatch/authz.rs | 8 +- core/{server => server-ng}/src/http.rs | 22 +- .../src/http/admission.rs | 0 core/server-ng/src/http/error.rs | 818 +++ .../src/http/extractor.rs | 0 .../{server => server-ng}/src/http/forward.rs | 14 +- .../src/http/handlers.rs | 4 +- core/{server => server-ng}/src/http/jwks.rs | 0 core/{server => server-ng}/src/http/jwt.rs | 8 +- core/server-ng/src/http/metrics.rs | 297 + core/{server => server-ng}/src/http/reads.rs | 18 +- core/{server => server-ng}/src/http/reply.rs | 4 +- .../{server => server-ng}/src/http/session.rs | 6 +- core/{server => server-ng}/src/http/state.rs | 14 +- core/{server => server-ng}/src/http/submit.rs | 18 +- core/{server => server-ng}/src/http/tls.rs | 24 +- core/{server => server-ng}/src/http/wire.rs | 14 +- core/server-ng/src/lib.rs | 47 + .../src/login_register.rs | 0 core/server-ng/src/main.rs | 89 + .../src/offset_recovery.rs | 61 +- .../src/partition_helpers.rs | 66 +- .../src/partition_reconciler.rs | 289 +- core/{server => server-ng}/src/pat.rs | 24 +- .../src/personal_access_token_cleaner.rs | 6 +- core/{server => server-ng}/src/responses.rs | 70 +- .../src/segment_cleaner.rs | 6 +- .../src/segment_recovery.rs | 30 +- core/server-ng/src/server_error.rs | 391 ++ .../src/session_manager.rs | 8 +- core/{server => server-ng}/src/snapshot.rs | 14 +- .../src/snapshot/procdump.rs | 0 core/{server => server-ng}/src/users.rs | 6 +- core/{server => server-ng}/src/web.rs | 0 core/server-ng/src/wire.rs | 75 + core/{server => server-ng}/tests/sdk_e2e.rs | 18 +- core/server/Cargo.toml | 104 +- core/server/Dockerfile | 11 +- core/server/README.md | 37 +- core/server/config.toml | 495 +- core/server/server.http | 35 +- core/server/src/args.rs | 122 +- core/server/src/binary/dispatch.rs | 456 ++ .../cluster/get_cluster_metadata_handler.rs | 60 + .../server/src/binary/handlers/cluster/mod.rs | 18 + .../create_consumer_group_handler.rs | 67 + .../delete_consumer_group_handler.rs | 56 + .../get_consumer_group_handler.rs | 78 + .../get_consumer_groups_handler.rs | 69 + .../join_consumer_group_handler.rs | 57 + .../leave_consumer_group_handler.rs | 57 + .../binary/handlers/consumer_groups/mod.rs | 25 + .../delete_consumer_offset_handler.rs | 56 + .../get_consumer_offset_handler.rs | 79 + .../binary/handlers/consumer_offsets/mod.rs | 22 + .../store_consumer_offset_handler.rs | 63 + .../messages/flush_unsaved_buffer_handler.rs | 65 + .../src/binary/handlers/messages/mod.rs | 22 + .../messages/poll_messages_handler.rs | 87 + .../messages/send_messages_handler.rs | 203 + core/server/src/binary/handlers/mod.rs | 28 + .../partitions/create_partitions_handler.rs | 61 + .../partitions/delete_partitions_handler.rs | 60 + .../src/binary/handlers/partitions/mod.rs | 21 + .../create_personal_access_token_handler.rs | 73 + .../delete_personal_access_token_handler.rs | 57 + .../get_personal_access_tokens_handler.rs | 60 + ...ogin_with_personal_access_token_handler.rs | 51 + .../handlers/personal_access_tokens/mod.rs | 23 + .../segments/delete_segments_handler.rs | 76 + .../src/binary/handlers/segments/mod.rs | 18 + .../handlers/streams/create_stream_handler.rs | 67 + .../handlers/streams/delete_stream_handler.rs | 56 + .../handlers/streams/get_stream_handler.rs | 121 + .../handlers/streams/get_streams_handler.rs | 62 + .../server/src/binary/handlers/streams/mod.rs | 25 + .../handlers/streams/purge_stream_handler.rs | 56 + .../handlers/streams/update_stream_handler.rs | 56 + .../handlers/system/get_client_handler.rs | 54 + .../handlers/system/get_clients_handler.rs | 44 + .../binary/handlers/system/get_me_handler.rs | 77 + .../handlers/system/get_snapshot_handler.rs | 53 + .../handlers/system/get_stats_handler.rs | 93 + core/server/src/binary/handlers/system/mod.rs | 25 + .../binary/handlers/system/ping_handler.rs | 41 + .../handlers/topics/create_topic_handler.rs | 103 + .../handlers/topics/delete_topic_handler.rs | 56 + .../handlers/topics/get_topic_handler.rs | 76 + .../handlers/topics/get_topics_handler.rs | 61 + core/server/src/binary/handlers/topics/mod.rs | 25 + .../handlers/topics/purge_topic_handler.rs | 56 + .../handlers/topics/update_topic_handler.rs | 56 + .../handlers/users/change_password_handler.rs | 66 + .../handlers/users/create_user_handler.rs | 83 + .../handlers/users/delete_user_handler.rs | 57 + .../binary/handlers/users/get_user_handler.rs | 62 + .../handlers/users/get_users_handler.rs | 53 + .../handlers/users/login_user_handler.rs | 71 + .../handlers/users/logout_user_handler.rs | 44 + core/server/src/binary/handlers/users/mod.rs | 28 + .../users/update_permissions_handler.rs | 59 + .../handlers/users/update_user_handler.rs | 65 + core/server/src/binary/mod.rs | 21 + core/server/src/bootstrap.rs | 5051 ++--------------- .../index_rebuilding/index_rebuilder.rs | 118 + .../server/src/compat/index_rebuilding/mod.rs | 18 + core/server/src/compat/mod.rs | 18 + core/server/src/configs.rs | 21 + core/server/src/diagnostics.rs | 22 + core/server/src/http/consumer_groups.rs | 176 + core/server/src/http/consumer_offsets.rs | 150 + core/server/src/http/diagnostics.rs | 68 + core/server/src/http/error.rs | 780 +-- core/server/src/http/http_server.rs | 399 ++ core/server/src/http/http_shard_wrapper.rs | 232 + core/server/src/http/jwt/json_web_token.rs | 155 + core/server/src/http/jwt/jwks.rs | 357 ++ core/server/src/http/jwt/jwt_manager.rs | 457 ++ core/server/src/http/jwt/middleware.rs | 105 + core/server/src/http/jwt/mod.rs | 24 + core/server/src/http/jwt/storage.rs | 149 + core/server/src/http/mapper.rs | 329 ++ core/server/src/http/messages.rs | 164 + core/server/src/http/metrics.rs | 296 +- core/server/src/http/mod.rs | 40 + core/server/src/http/partitions.rs | 104 + .../server/src/http/personal_access_tokens.rs | 148 + core/server/src/http/segments.rs | 120 + .../server/src/http/shared.rs | 22 +- core/server/src/http/streams.rs | 200 + core/server/src/http/system.rs | 168 + core/server/src/http/topics.rs | 263 + core/server/src/http/users.rs | 331 ++ core/server/src/http/web.rs | 83 + core/server/src/io/mod.rs | 20 + core/server/src/io/storage.rs | 171 + core/server/src/lib.rs | 56 +- core/server/src/main.rs | 591 +- core/server/src/metadata/absorb.rs | 500 ++ core/server/src/metadata/consumer_group.rs | 315 + .../src/metadata/consumer_group_member.rs | 58 + core/server/src/metadata/inner.rs | 54 + core/server/src/metadata/mod.rs | 71 + core/server/src/metadata/ops.rs | 134 + core/server/src/metadata/partition.rs | 39 + core/server/src/metadata/reader.rs | 1920 +++++++ core/server/src/metadata/stream.rs | 64 + core/server/src/metadata/topic.rs | 72 + .../server/src/metadata/user.rs | 22 +- core/server/src/metadata/writer.rs | 617 ++ core/server/src/quic/listener.rs | 239 + core/server/src/quic/mod.rs | 22 + core/server/src/quic/quic_server.rs | 222 + core/server/src/quic/quic_socket.rs | 66 + core/server/src/sender/mod.rs | 257 + core/server/src/sender/quic_sender.rs | 140 + core/server/src/sender/tcp_sender.rs | 90 + core/server/src/sender/tcp_tls_sender.rs | 96 + core/server/src/sender/websocket_sender.rs | 206 + .../server/src/sender/websocket_tls_sender.rs | 185 + core/server/src/server_error.rs | 453 +- core/server/src/shard/builder.rs | 197 + core/server/src/shard/communication.rs | 198 + core/server/src/shard/execution.rs | 732 +++ core/server/src/shard/handlers.rs | 614 ++ core/server/src/shard/mod.rs | 482 ++ core/server/src/shard/system/clients.rs | 92 + core/server/src/shard/system/cluster.rs | 196 + .../src/shard/system/consumer_groups.rs | 205 + .../src/shard/system/consumer_offsets.rs | 489 ++ core/server/src/shard/system/info.rs | 78 + core/server/src/shard/system/messages.rs | 695 +++ core/server/src/shard/system/mod.rs | 35 + core/server/src/shard/system/partitions.rs | 443 ++ .../shard/system/personal_access_tokens.rs | 149 + core/server/src/shard/system/segments.rs | 586 ++ core/server/src/shard/system/snapshot/mod.rs | 257 + .../src/shard/system/snapshot/procdump.rs | 213 + core/server/src/shard/system/stats.rs | 120 + core/server/src/shard/system/storage.rs | 99 + core/server/src/shard/system/streams.rs | 179 + core/server/src/shard/system/topics.rs | 250 + core/server/src/shard/system/users.rs | 301 + core/server/src/shard/system/utils.rs | 255 + core/server/src/shard/systemd.rs | 45 + .../src/shard/task_registry/builders.rs | 42 + .../task_registry/builders/continuous.rs | 109 + .../shard/task_registry/builders/oneshot.rs | 120 + .../shard/task_registry/builders/periodic.rs | 135 + core/server/src/shard/task_registry/mod.rs | 23 + .../src/shard/task_registry/registry.rs | 732 +++ .../src/shard/task_registry/shutdown.rs | 232 + .../src/shard/tasks/continuous/http_server.rs | 40 + .../shard/tasks/continuous/message_pump.rs | 135 + core/server/src/shard/tasks/continuous/mod.rs | 28 + .../src/shard/tasks/continuous/quic_server.rs | 36 + .../src/shard/tasks/continuous/tcp_server.rs | 36 + .../tasks/continuous/websocket_server.rs | 38 + core/server/src/shard/tasks/mod.rs | 20 + .../src/shard/tasks/oneshot/config_writer.rs | 142 + core/server/src/shard/tasks/oneshot/mod.rs | 20 + .../tasks/periodic/heartbeat_verifier.rs | 86 + .../shard/tasks/periodic/jwt_token_cleaner.rs | 60 + .../shard/tasks/periodic/message_cleaner.rs | 120 + .../src/shard/tasks/periodic/message_saver.rs | 71 + core/server/src/shard/tasks/periodic/mod.rs | 36 + .../periodic/personal_access_token_cleaner.rs | 85 + .../tasks/periodic/revocation_timeout.rs | 105 + .../shard/tasks/periodic/sysinfo_printer.rs | 103 + .../shard/tasks/periodic/systemd_watchdog.rs | 47 + .../src/shard/transmission/connector.rs | 106 + core/server/src/shard/transmission/event.rs | 70 + core/server/src/shard/transmission/frame.rs | 116 + core/server/src/shard/transmission/message.rs | 254 + core/server/src/shard/transmission/mod.rs | 21 + core/server/src/state/command.rs | 268 + core/server/src/state/entry.rs | 150 + core/server/src/state/file.rs | 378 ++ .../src/common => server/src/state}/mod.rs | 19 +- core/server/src/state/models.rs | 334 ++ core/server/src/state/system.rs | 633 +++ .../src/streaming/clients/client_manager.rs | 233 + core/server/src/streaming/clients/mod.rs | 18 + .../server/src/streaming/deduplication/mod.rs | 18 + .../src/streaming/diagnostics/metrics.rs | 149 + core/server/src/streaming/diagnostics/mod.rs | 18 + core/server/src/streaming/mod.rs | 31 + .../partitions/consumer_group_offsets.rs | 18 + .../streaming/partitions/consumer_offset.rs | 18 + .../streaming/partitions/consumer_offsets.rs | 18 + .../src/streaming/partitions/helpers.rs | 36 + .../src/streaming/partitions/in_flight.rs | 18 + .../src/streaming/partitions/journal.rs | 212 + .../streaming/partitions/local_partition.rs | 98 + .../streaming/partitions/local_partitions.rs | 213 + core/server/src/streaming/partitions/log.rs | 207 + core/server/src/streaming/partitions/mod.rs | 33 + core/server/src/streaming/partitions/ops.rs | 730 +++ .../src/streaming/partitions/ops_tests.rs | 358 ++ .../src/streaming/partitions/segments.rs | 123 + .../src/streaming/partitions/storage.rs | 346 ++ core/server/src/streaming/persistence/mod.rs | 20 + .../src/streaming/persistence/persister.rs | 166 + core/server/src/streaming/polling_consumer.rs | 129 + .../segments/indexes/index_reader.rs | 19 + .../segments/indexes/index_writer.rs | 18 + .../src/streaming/segments/indexes/mod.rs | 22 + .../src/streaming/segments/memory_journal.rs | 17 + .../segments/messages/messages_reader.rs | 19 + .../segments/messages/messages_writer.rs | 18 + .../src/streaming/segments/messages/mod.rs | 21 + core/server/src/streaming/segments/mod.rs | 34 + core/server/src/streaming/segments/segment.rs | 18 + core/server/src/streaming/segments/storage.rs | 50 + .../src/streaming/segments/types/mod.rs | 19 + core/server/src/streaming/session.rs | 105 + core/server/src/streaming/stats/mod.rs | 18 + core/server/src/streaming/storage.rs | 39 + core/server/src/streaming/streams/mod.rs | 20 + core/server/src/streaming/streams/storage.rs | 65 + core/server/src/streaming/topics/helpers.rs | 33 + core/server/src/streaming/topics/mod.rs | 21 + core/server/src/streaming/topics/storage.rs | 82 + core/server/src/streaming/users/mod.rs | 18 + core/server/src/streaming/users/user.rs | 137 + core/server/src/streaming/utils/address.rs | 75 + core/server/src/streaming/utils/file.rs | 54 + core/server/src/streaming/utils/mod.rs | 22 + core/server/src/streaming/utils/ptr.rs | 70 + core/server/src/systemd.rs | 80 - core/server/src/tcp/connection_handler.rs | 184 + core/server/src/tcp/mod.rs | 73 + core/server/src/tcp/tcp_listener.rs | 171 + core/server/src/tcp/tcp_server.rs | 56 + core/server/src/tcp/tcp_socket.rs | 97 + core/server/src/tcp/tcp_tls_listener.rs | 224 + .../src/websocket/connection_handler.rs | 175 + core/server/src/websocket/mod.rs | 64 + .../src/websocket/websocket_listener.rs | 182 + core/server/src/websocket/websocket_server.rs | 59 + .../src/websocket/websocket_tls_listener.rs | 272 + core/server/src/wire.rs | 161 - core/server_common/src/consensus_message.rs | 237 +- core/server_common/src/indexes_mut.rs | 2 +- core/server_common/src/send_messages2.rs | 256 +- core/server_common/src/sharding/mod.rs | 7 +- core/server_common/src/sharding/namespace.rs | 12 +- core/shard/Cargo.toml | 2 +- core/shard/src/lib.rs | 2365 ++------ core/shard/src/metrics.rs | 12 - core/shard/src/router.rs | 46 +- core/shard_allocator/src/lib.rs | 54 +- core/simulator/Cargo.toml | 2 +- core/simulator/src/client.rs | 158 +- core/simulator/src/deps.rs | 49 - core/simulator/src/lib.rs | 205 +- core/simulator/src/replica.rs | 24 +- core/simulator/src/workload/auditor.rs | 17 +- core/simulator/src/workload/effect.rs | 10 - core/simulator/src/workload/mod.rs | 9 +- .../src/workload/ops/change_password.rs | 4 +- .../src/workload/ops/create_consumer_group.rs | 4 +- .../src/workload/ops/create_partitions.rs | 21 +- .../ops/create_personal_access_token.rs | 4 +- .../src/workload/ops/create_stream.rs | 4 +- .../src/workload/ops/create_topic.rs | 4 +- .../simulator/src/workload/ops/create_user.rs | 4 +- .../src/workload/ops/delete_consumer_group.rs | 4 +- .../workload/ops/delete_consumer_offset.rs | 4 +- .../workload/ops/delete_consumer_offset_2.rs | 4 +- .../src/workload/ops/delete_partitions.rs | 60 +- .../ops/delete_personal_access_token.rs | 4 +- .../src/workload/ops/delete_segments.rs | 4 +- .../src/workload/ops/delete_stream.rs | 4 +- .../src/workload/ops/delete_topic.rs | 4 +- .../simulator/src/workload/ops/delete_user.rs | 4 +- core/simulator/src/workload/ops/mod.rs | 6 +- .../src/workload/ops/purge_stream.rs | 4 +- .../simulator/src/workload/ops/purge_topic.rs | 4 +- .../src/workload/ops/send_messages.rs | 4 +- .../src/workload/ops/store_consumer_offset.rs | 6 +- .../workload/ops/store_consumer_offset_2.rs | 6 +- .../src/workload/ops/update_permissions.rs | 4 +- .../src/workload/ops/update_stream.rs | 4 +- .../src/workload/ops/update_topic.rs | 4 +- .../simulator/src/workload/ops/update_user.rs | 4 +- core/simulator/src/workload/shadow.rs | 113 +- examples/go/README.md | 9 +- examples/java/README.md | 70 +- examples/node/src/tcp-tls/consumer.ts | 1 - examples/node/src/tcp-tls/producer.ts | 16 +- examples/python/getting-started/consumer.py | 52 +- examples/python/getting-started/producer.py | 50 +- foreign/cpp/tests/e2e/client.cpp | 91 +- foreign/cpp/tests/e2e/consumer_group.cpp | 44 +- foreign/cpp/tests/e2e/message.cpp | 7 +- foreign/csharp/Directory.Packages.props | 7 +- .../ClusterRedirectionTests.cs | 27 +- .../ConsumerGroupTests.cs | 11 +- .../FetchMessagesTests.cs | 27 +- .../Fixtures/IggyClusterFixture.cs | 250 + .../Fixtures/IggyServerFixture.cs | 256 +- .../Fixtures/IggyTlsServerFixture.cs | 7 +- .../Fixtures/VsrCluster.cs | 438 -- .../FlushMessagesTests.cs | 61 +- .../HeaderEncryptionIntegrationTests.cs | 74 +- .../Helpers/Eventually.cs | 43 - .../IggyConsumerTests.cs | 94 +- .../IggyPublisherTests.cs | 93 +- .../IggyTlsConnectionTests.cs | 10 +- .../IggyTypedConsumerTests.cs | 20 +- .../IggyTypedPublisherTests.cs | 8 +- .../MessageEncryptionIntegrationTests.cs | 4 +- .../Iggy_SDK.Tests.Integration/OffsetTests.cs | 27 +- .../PartitionsTests.cs | 6 +- .../PersonalAccessTokenTests.cs | 2 +- .../RawCommandTests.cs | 12 +- .../SegmentsTests.cs | 12 +- .../SendMessagesTests.cs | 143 +- .../StreamsTests.cs | 5 +- .../Iggy_SDK.Tests.Integration/SystemTests.cs | 12 +- .../Iggy_SDK.Tests.Integration/TopicsTests.cs | 37 +- .../Iggy_SDK.Tests.Integration/UsersTests.cs | 16 +- .../Vsr/VsrConsumerGroupTests.cs | 228 - .../Vsr/VsrHandshakeTests.cs | 164 - .../Vsr/VsrMessagingTests.cs | 165 - .../Vsr/VsrMetadataTests.cs | 132 - .../Configuration/AutoLoginSettings.cs | 15 - .../Configuration/IggyClientConfigurator.cs | 6 - .../Iggy_SDK/Consumers/IggyConsumer.Rented.cs | 8 +- .../csharp/Iggy_SDK/Consumers/IggyConsumer.cs | 10 +- .../Iggy_SDK/Consumers/IggyConsumerBuilder.cs | 5 +- .../Consumers/IggyConsumerBuilderOfT.cs | 10 +- .../Contracts/SendMessagesResponse.cs | 77 - .../IggyInvalidStatusCodeException.cs | 12 +- .../VsrRequestOutcomeUnknownException.cs | 33 - .../Exceptions/VsrSessionEvictedException.cs | 34 - .../Iggy_SDK/Factory/IggyClientFactory.cs | 17 +- .../Iggy_SDK/IggyClient/IIggyPublisher.cs | 25 +- .../csharp/Iggy_SDK/IggyClient/IIggySystem.cs | 5 +- .../Implementations/HttpMessageStream.cs | 100 +- .../Implementations/TcpMessageStream.Vsr.cs | 978 ---- .../Implementations/TcpMessageStream.cs | 499 +- .../TransientHttpRetryHandler.cs | 109 - foreign/csharp/Iggy_SDK/Iggy_SDK.csproj | 3 +- .../csharp/Iggy_SDK/Mappers/BinaryMapper.cs | 65 +- .../Publishers/BackgroundMessageProcessor.cs | 43 +- .../Iggy_SDK/Publishers/IggyPublisher.cs | 51 +- .../Publishers/IggyPublisherBuilder.cs | 3 +- .../Publishers/IggyPublisherBuilderOfT.cs | 9 +- .../Iggy_SDK/Publishers/IggyPublisherOfT.cs | 35 +- foreign/csharp/Iggy_SDK/Utils/BufferSizes.cs | 1 + foreign/csharp/Iggy_SDK/Utils/CommandCodes.cs | 3 - .../csharp/Iggy_SDK/Utils/ServerAddress.cs | 139 - .../csharp/Iggy_SDK/Vsr/ConsensusSession.cs | 223 - .../Iggy_SDK/Vsr/ConsumerGroupClientState.cs | 301 - .../csharp/Iggy_SDK/Vsr/CredentialBounds.cs | 66 - foreign/csharp/Iggy_SDK/Vsr/LoginRegister.cs | 168 - .../csharp/Iggy_SDK/Vsr/SyncConsumerGroup.cs | 61 - foreign/csharp/Iggy_SDK/Vsr/VsrError.cs | 61 - foreign/csharp/Iggy_SDK/Vsr/VsrHeader.cs | 156 - foreign/csharp/Iggy_SDK/Vsr/VsrOperation.cs | 275 - .../csharp/Iggy_SDK/Vsr/VsrReplyDecoder.cs | 206 - .../ClientTests/IggyClientFactoryTests.cs | 67 - .../ConsumerTests/IggyConsumerBuilderTests.cs | 19 - .../MapperTests/BinaryMapper.cs | 76 - .../IggyPublisherBuilderTests.cs | 47 - .../PublisherTests/IggyTypedPublisherTests.cs | 4 +- .../UtilityTests/SendUnitTests.cs | 7 +- .../VsrTests/ConsensusSessionTests.cs | 218 - .../VsrTests/ConsumerGroupClientStateTests.cs | 176 - .../VsrTests/CredentialBoundsTests.cs | 72 - .../VsrTests/LoginRegisterTests.cs | 165 - .../VsrTests/ServerAddressTests.cs | 85 - .../VsrTests/SyncConsumerGroupTests.cs | 93 - .../Iggy_SDK_Tests/VsrTests/VsrHeaderTests.cs | 283 - .../VsrTests/VsrOperationTests.cs | 167 - .../VsrTests/VsrProtocolDriftTests.cs | 810 --- .../VsrTests/VsrReplyDecoderTests.cs | 293 - .../VsrTests/VsrTestPayloads.cs | 102 - foreign/csharp/README.md | 100 +- foreign/csharp/scripts/pack.sh | 2 +- foreign/go/README.md | 10 +- foreign/go/client/tcp/tcp_connect_test.go | 1 + foreign/go/client/tcp/tcp_core_test.go | 10 +- foreign/go/client/tcp/tcp_testing_test.go | 31 +- foreign/go/internal/vsr/envelope.go | 17 +- foreign/go/internal/vsr/envelope_test.go | 47 +- foreign/go/internal/vsr/header.go | 17 +- foreign/go/internal/vsr/header_test.go | 19 +- foreign/go/internal/vsr/namespace.go | 246 + foreign/go/internal/vsr/namespace_test.go | 376 ++ .../go/internal/vsr/protocol_parity_test.go | 96 +- foreign/go/tests/e2e_helpers_test.go | 5 +- foreign/java/README.md | 3 +- .../async/TcpAsyncPinnedProducerActor.java | 1 + .../bench/report/ServerStatsCollector.java | 1 + .../connector/flink/source/IggySource.java | 1 + .../example/AsyncTcpMessageSendTest.java | 17 +- .../iggy-connector-pinot/integration-test.sh | 2 +- foreign/java/gradle.properties | 2 +- foreign/java/gradle/libs.versions.toml | 2 +- .../client/async/ConsumerGroupsClient.java | 23 - .../iggy/client/async/MessagesClient.java | 22 +- .../client/async/tcp/AsyncIggyTcpClient.java | 288 +- .../async/tcp/AsyncIggyTcpClientBuilder.java | 71 +- .../client/async/tcp/AsyncTcpConnection.java | 768 +-- .../client/async/tcp/ClientRoutingState.java | 159 - .../async/tcp/ConsumerGroupsTcpClient.java | 19 - .../client/async/tcp/IggyAuthenticator.java | 30 +- .../client/async/tcp/IggyFrameDecoder.java | 73 + .../client/async/tcp/IggyFrameEncoder.java | 61 + ...ingHook.java => LoginRedirectionHook.java} | 21 +- .../client/async/tcp/MessagesTcpClient.java | 245 +- .../tcp/PersonalAccessTokensTcpClient.java | 16 +- .../iggy/client/async/tcp/ReconnectPlan.java | 57 - .../iggy/client/async/tcp/UsersTcpClient.java | 15 +- .../iggy/client/async/tcp/package-info.java | 13 +- .../async/tcp/vsr/ConsensusSession.java | 128 - .../client/async/tcp/vsr/VsrFrameDecoder.java | 67 - .../iggy/client/async/tcp/vsr/VsrHeaders.java | 146 - .../client/async/tcp/vsr/VsrLoginCodec.java | 128 - .../client/async/tcp/vsr/VsrOperation.java | 189 - .../async/tcp/vsr/VsrRequestEncoder.java | 112 - .../async/tcp/vsr/VsrResponseHandler.java | 261 - .../client/blocking/ConsumerGroupsClient.java | 11 - .../iggy/client/blocking/MessagesClient.java | 9 +- .../http/ConsumerGroupsHttpClient.java | 6 - .../blocking/http/MessagesHttpClient.java | 10 +- .../blocking/tcp/ConsumerGroupsTcpClient.java | 6 - .../blocking/tcp/IggyTcpClientBuilder.java | 11 + .../blocking/tcp/MessagesTcpClient.java | 6 +- .../ConsumerGroupAssignment.java | 36 - .../apache/iggy/exception/IggyErrorCode.java | 9 +- .../exception/IggyValidationException.java | 1 - .../java/org/apache/iggy/hash/XxHash32.java | 102 - .../apache/iggy/message/SendConfirmation.java | 33 - .../iggy/message/SendMessagesResponse.java | 37 - .../apache/iggy/serde/BytesDeserializer.java | 52 - .../org/apache/iggy/serde/CommandCode.java | 3 +- .../iggy/client/BaseIntegrationTest.java | 19 - .../async/AsyncClientIntegrationTest.java | 3 +- .../async/AsyncConnectionPoolAuthTest.java | 36 +- .../client/async/AsyncConsumerGroupsTest.java | 20 +- .../tcp/AsyncIggyTcpClientBuilderTest.java | 88 +- .../AsyncIggyTcpClientLoginRoutingTest.java | 180 - ...yncIggyTcpClientTransientFailoverTest.java | 337 -- .../AsyncTcpConnectionConcurrencyTest.java | 409 -- .../tcp/AsyncTcpConnectionHeartbeatTest.java | 132 - .../AsyncTcpConnectionRequestTimeoutTest.java | 125 - .../async/tcp/ClientRoutingStateTest.java | 251 - .../async/tcp/IggyFrameDecoderTest.java | 455 ++ .../async/tcp/IggyResponseHandlerTest.java | 60 + .../async/tcp/LoginRoutingHookTest.java | 95 - .../client/async/tcp/ReconnectPlanTest.java | 66 - .../async/tcp/vsr/VsrFrameDecoderTest.java | 118 - .../async/tcp/vsr/VsrRequestEncoderTest.java | 225 - .../async/tcp/vsr/VsrResponseHandlerTest.java | 334 -- .../ConsumerOffsetsClientBaseTest.java | 4 +- .../blocking/MessagesClientBaseTest.java | 108 +- .../client/blocking/SystemClientBaseTest.java | 2 - .../client/blocking/UsersClientBaseTest.java | 3 +- .../tcp/ConsumerGroupsTcpClientTest.java | 86 - .../tcp/IggyTcpClientBuilderTest.java | 16 + .../blocking/tcp/MessagesTcpClientTest.java | 31 - .../iggy/exception/IggyErrorCodeTest.java | 9 +- .../exception/IggyServerExceptionTest.java | 9 - .../IggyValidationExceptionTest.java | 2 - .../org/apache/iggy/hash/XxHash32Test.java | 73 - .../iggy/serde/BytesDeserializerTest.java | 146 - foreign/node/CHANGELOG.md | 95 + foreign/node/README.md | 25 +- foreign/node/docker-compose.yml | 7 +- foreign/node/package-lock.json | 4 +- foreign/node/package.json | 3 +- foreign/node/scripts/check-vsr-protocol.mjs | 52 +- foreign/node/src/bdd/auth.ts | 3 + foreign/node/src/client/client.config.test.ts | 31 +- foreign/node/src/client/client.config.ts | 17 +- .../node/src/client/client.connection.test.ts | 37 +- foreign/node/src/client/client.connection.ts | 17 +- foreign/node/src/client/client.frame.test.ts | 113 +- foreign/node/src/client/client.frame.ts | 40 +- foreign/node/src/client/client.socket.test.ts | 1 + foreign/node/src/client/client.socket.ts | 29 +- foreign/node/src/client/client.type.ts | 7 + foreign/node/src/client/client.utils.test.ts | 31 +- foreign/node/src/client/client.utils.ts | 68 + foreign/node/src/e2e/tcp.cluster.e2e.ts | 21 +- .../node/src/e2e/tcp.consumer-group.e2e.ts | 6 +- .../node/src/e2e/tcp.consumer-stream.e2e.ts | 6 +- foreign/node/src/e2e/tcp.send-message.e2e.ts | 12 + foreign/node/src/e2e/test-client.utils.ts | 2 + foreign/node/src/e2e/tls.system.e2e.ts | 4 + foreign/node/src/wire/command-set.test.ts | 1 + .../src/wire/message/poll-messages.command.ts | 3 +- foreign/node/src/wire/vsr/header.test.ts | 8 + foreign/node/src/wire/vsr/header.ts | 23 +- foreign/node/src/wire/vsr/index.ts | 3 + foreign/node/src/wire/vsr/namespace.test.ts | 342 ++ foreign/node/src/wire/vsr/namespace.ts | 216 + foreign/node/src/wire/vsr/vsr.test.ts | 17 + foreign/php/README.md | 2 +- foreign/php/docker-compose.test.yml | 6 +- foreign/php/iggy-php.stubs.php | 14 +- foreign/php/scripts/test.sh | 2 +- foreign/php/src/client.rs | 5 +- foreign/php/src/send_message.rs | 9 +- foreign/php/tests/IggySdkTest.php | 16 +- foreign/python/Cargo.toml | 5 +- foreign/python/README.md | 36 - foreign/python/apache_iggy.pyi | 263 +- foreign/python/docker-compose.test.yml | 8 +- foreign/python/scripts/test.sh | 2 +- foreign/python/src/client.rs | 135 +- foreign/python/src/config.rs | 423 -- foreign/python/src/consumer.rs | 51 +- foreign/python/src/duration.rs | 65 - foreign/python/src/lib.rs | 8 +- foreign/python/src/receive_message.rs | 2 +- foreign/python/src/send_message.rs | 2 + foreign/python/src/topic.rs | 33 +- foreign/python/tests/conftest.py | 5 - foreign/python/tests/test_client_config.py | 459 -- foreign/python/tests/test_tls.py | 8 +- foreign/python/tests/test_topic.py | 13 +- foreign/python/tests/utils.py | 33 +- justfile | 17 + scripts/bump-version.sh | 2 +- scripts/check-backwards-compat.sh | 260 + scripts/ci/binary-artifacts.sh | 3 - scripts/ci/lib/init.sh | 51 - scripts/ci/license-headers.sh | 3 - scripts/ci/markdownlint.sh | 3 - scripts/ci/python-sdk-version-sync.sh | 3 - scripts/ci/shellcheck.sh | 3 - scripts/ci/skills.sh | 3 - scripts/ci/sync-python-interpreter-version.sh | 3 - scripts/ci/sync-rustc-version.sh | 11 +- scripts/ci/taplo.sh | 3 - scripts/ci/trailing-newline.sh | 3 - scripts/ci/trailing-whitespace.sh | 3 - scripts/ci/uv-lock-check.sh | 3 - scripts/copy-latest-from-master.sh | 2 +- scripts/dashboard/build_release.sh | 2 +- scripts/dashboard/run_dev.sh | 2 +- scripts/extract-version.sh | 5 +- .../run-standard-performance-suite.sh | 2 +- scripts/performance/utils.sh | 2 +- scripts/profile.sh | 2 +- scripts/run-bdd-tests.sh | 46 +- scripts/run-benches.sh | 2 +- scripts/run-examples-from-readme.sh | 41 +- scripts/utils.sh | 26 +- 823 files changed, 57755 insertions(+), 39218 deletions(-) create mode 100644 .github/actions/utils/docker-build-test-server/action.yml create mode 100644 bdd/docker-compose.vsr.yml delete mode 100644 core/configs/src/common/defaults.rs delete mode 100644 core/configs/src/common/displays.rs delete mode 100644 core/configs/src/common/server.rs delete mode 100644 core/configs/src/common/validators.rs rename core/configs/src/{common => server_config}/cache_indexes.rs (100%) rename core/configs/src/{common => server_config}/http.rs (100%) rename core/configs/src/{common => server_config}/system.rs (96%) create mode 100644 core/configs/src/server_ng_config/cluster.rs create mode 100644 core/configs/src/server_ng_config/defaults.rs create mode 100644 core/configs/src/server_ng_config/displays.rs rename core/configs/src/{server_config => server_ng_config}/message_bus.rs (87%) rename core/configs/src/{server_config => server_ng_config}/metadata.rs (74%) create mode 100644 core/configs/src/server_ng_config/mod.rs rename core/configs/src/{server_config => server_ng_config}/partition.rs (81%) create mode 100644 core/configs/src/server_ng_config/quic.rs create mode 100644 core/configs/src/server_ng_config/server_ng.rs create mode 100644 core/configs/src/server_ng_config/sharding.rs create mode 100644 core/configs/src/server_ng_config/tcp.rs create mode 100644 core/configs/src/server_ng_config/validators.rs create mode 100644 core/configs/src/server_ng_config/websocket.rs delete mode 100644 core/consensus/src/dvc_merge.rs delete mode 100644 core/integration/tests/server/login_credentials_vsr.rs create mode 100644 core/integration/tests/server/scenarios/bench_scenario.rs delete mode 100644 core/integration/tests/server/stats_vsr.rs create mode 100644 core/integration/tests/state/file.rs create mode 100644 core/integration/tests/state/mod.rs create mode 100644 core/integration/tests/state/system.rs create mode 100644 core/integration/tests/storage/consumer_offsets.rs create mode 100644 core/integration/tests/storage/mod.rs create mode 100644 core/server-ng/.dockerignore create mode 100644 core/server-ng/Cargo.toml create mode 100644 core/server-ng/Dockerfile create mode 100644 core/server-ng/build.rs create mode 100644 core/server-ng/config.toml create mode 100644 core/server-ng/server.http rename foreign/csharp/Iggy_SDK/Vsr/EvictionReason.cs => core/server-ng/src/args.rs (53%) rename core/{server => server-ng}/src/auth.rs (97%) create mode 100644 core/server-ng/src/bootstrap.rs rename core/{server => server-ng}/src/cluster_meta.rs (98%) rename core/{server => server-ng}/src/config_writer.rs (87%) rename core/{server => server-ng}/src/consumer_group.rs (97%) rename core/{server => server-ng}/src/dispatch.rs (95%) rename core/{server => server-ng}/src/dispatch/authz.rs (98%) rename core/{server => server-ng}/src/http.rs (98%) rename core/{server => server-ng}/src/http/admission.rs (100%) create mode 100644 core/server-ng/src/http/error.rs rename core/{server => server-ng}/src/http/extractor.rs (100%) rename core/{server => server-ng}/src/http/forward.rs (98%) rename core/{server => server-ng}/src/http/handlers.rs (99%) rename core/{server => server-ng}/src/http/jwks.rs (100%) rename core/{server => server-ng}/src/http/jwt.rs (98%) create mode 100644 core/server-ng/src/http/metrics.rs rename core/{server => server-ng}/src/http/reads.rs (95%) rename core/{server => server-ng}/src/http/reply.rs (99%) rename core/{server => server-ng}/src/http/session.rs (98%) rename core/{server => server-ng}/src/http/state.rs (97%) rename core/{server => server-ng}/src/http/submit.rs (97%) rename core/{server => server-ng}/src/http/tls.rs (92%) rename core/{server => server-ng}/src/http/wire.rs (97%) create mode 100644 core/server-ng/src/lib.rs rename core/{server => server-ng}/src/login_register.rs (100%) create mode 100644 core/server-ng/src/main.rs rename core/{server => server-ng}/src/offset_recovery.rs (73%) rename core/{server => server-ng}/src/partition_helpers.rs (94%) rename core/{server => server-ng}/src/partition_reconciler.rs (94%) rename core/{server => server-ng}/src/pat.rs (93%) rename core/{server => server-ng}/src/personal_access_token_cleaner.rs (96%) rename core/{server => server-ng}/src/responses.rs (97%) rename core/{server => server-ng}/src/segment_cleaner.rs (97%) rename core/{server => server-ng}/src/segment_recovery.rs (96%) create mode 100644 core/server-ng/src/server_error.rs rename core/{server => server-ng}/src/session_manager.rs (99%) rename core/{server => server-ng}/src/snapshot.rs (96%) rename core/{server => server-ng}/src/snapshot/procdump.rs (100%) rename core/{server => server-ng}/src/users.rs (98%) rename core/{server => server-ng}/src/web.rs (100%) create mode 100644 core/server-ng/src/wire.rs rename core/{server => server-ng}/tests/sdk_e2e.rs (91%) create mode 100644 core/server/src/binary/dispatch.rs create mode 100644 core/server/src/binary/handlers/cluster/get_cluster_metadata_handler.rs create mode 100644 core/server/src/binary/handlers/cluster/mod.rs create mode 100644 core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs create mode 100644 core/server/src/binary/handlers/consumer_groups/delete_consumer_group_handler.rs create mode 100644 core/server/src/binary/handlers/consumer_groups/get_consumer_group_handler.rs create mode 100644 core/server/src/binary/handlers/consumer_groups/get_consumer_groups_handler.rs create mode 100644 core/server/src/binary/handlers/consumer_groups/join_consumer_group_handler.rs create mode 100644 core/server/src/binary/handlers/consumer_groups/leave_consumer_group_handler.rs create mode 100644 core/server/src/binary/handlers/consumer_groups/mod.rs create mode 100644 core/server/src/binary/handlers/consumer_offsets/delete_consumer_offset_handler.rs create mode 100644 core/server/src/binary/handlers/consumer_offsets/get_consumer_offset_handler.rs create mode 100644 core/server/src/binary/handlers/consumer_offsets/mod.rs create mode 100644 core/server/src/binary/handlers/consumer_offsets/store_consumer_offset_handler.rs create mode 100644 core/server/src/binary/handlers/messages/flush_unsaved_buffer_handler.rs create mode 100644 core/server/src/binary/handlers/messages/mod.rs create mode 100644 core/server/src/binary/handlers/messages/poll_messages_handler.rs create mode 100644 core/server/src/binary/handlers/messages/send_messages_handler.rs create mode 100644 core/server/src/binary/handlers/mod.rs create mode 100644 core/server/src/binary/handlers/partitions/create_partitions_handler.rs create mode 100644 core/server/src/binary/handlers/partitions/delete_partitions_handler.rs create mode 100644 core/server/src/binary/handlers/partitions/mod.rs create mode 100644 core/server/src/binary/handlers/personal_access_tokens/create_personal_access_token_handler.rs create mode 100644 core/server/src/binary/handlers/personal_access_tokens/delete_personal_access_token_handler.rs create mode 100644 core/server/src/binary/handlers/personal_access_tokens/get_personal_access_tokens_handler.rs create mode 100644 core/server/src/binary/handlers/personal_access_tokens/login_with_personal_access_token_handler.rs create mode 100644 core/server/src/binary/handlers/personal_access_tokens/mod.rs create mode 100644 core/server/src/binary/handlers/segments/delete_segments_handler.rs create mode 100644 core/server/src/binary/handlers/segments/mod.rs create mode 100644 core/server/src/binary/handlers/streams/create_stream_handler.rs create mode 100644 core/server/src/binary/handlers/streams/delete_stream_handler.rs create mode 100644 core/server/src/binary/handlers/streams/get_stream_handler.rs create mode 100644 core/server/src/binary/handlers/streams/get_streams_handler.rs create mode 100644 core/server/src/binary/handlers/streams/mod.rs create mode 100644 core/server/src/binary/handlers/streams/purge_stream_handler.rs create mode 100644 core/server/src/binary/handlers/streams/update_stream_handler.rs create mode 100644 core/server/src/binary/handlers/system/get_client_handler.rs create mode 100644 core/server/src/binary/handlers/system/get_clients_handler.rs create mode 100644 core/server/src/binary/handlers/system/get_me_handler.rs create mode 100644 core/server/src/binary/handlers/system/get_snapshot_handler.rs create mode 100644 core/server/src/binary/handlers/system/get_stats_handler.rs create mode 100644 core/server/src/binary/handlers/system/mod.rs create mode 100644 core/server/src/binary/handlers/system/ping_handler.rs create mode 100644 core/server/src/binary/handlers/topics/create_topic_handler.rs create mode 100644 core/server/src/binary/handlers/topics/delete_topic_handler.rs create mode 100644 core/server/src/binary/handlers/topics/get_topic_handler.rs create mode 100644 core/server/src/binary/handlers/topics/get_topics_handler.rs create mode 100644 core/server/src/binary/handlers/topics/mod.rs create mode 100644 core/server/src/binary/handlers/topics/purge_topic_handler.rs create mode 100644 core/server/src/binary/handlers/topics/update_topic_handler.rs create mode 100644 core/server/src/binary/handlers/users/change_password_handler.rs create mode 100644 core/server/src/binary/handlers/users/create_user_handler.rs create mode 100644 core/server/src/binary/handlers/users/delete_user_handler.rs create mode 100644 core/server/src/binary/handlers/users/get_user_handler.rs create mode 100644 core/server/src/binary/handlers/users/get_users_handler.rs create mode 100644 core/server/src/binary/handlers/users/login_user_handler.rs create mode 100644 core/server/src/binary/handlers/users/logout_user_handler.rs create mode 100644 core/server/src/binary/handlers/users/mod.rs create mode 100644 core/server/src/binary/handlers/users/update_permissions_handler.rs create mode 100644 core/server/src/binary/handlers/users/update_user_handler.rs create mode 100644 core/server/src/binary/mod.rs create mode 100644 core/server/src/compat/index_rebuilding/index_rebuilder.rs create mode 100644 core/server/src/compat/index_rebuilding/mod.rs create mode 100644 core/server/src/compat/mod.rs create mode 100644 core/server/src/configs.rs create mode 100644 core/server/src/diagnostics.rs create mode 100644 core/server/src/http/consumer_groups.rs create mode 100644 core/server/src/http/consumer_offsets.rs create mode 100644 core/server/src/http/diagnostics.rs create mode 100644 core/server/src/http/http_server.rs create mode 100644 core/server/src/http/http_shard_wrapper.rs create mode 100644 core/server/src/http/jwt/json_web_token.rs create mode 100644 core/server/src/http/jwt/jwks.rs create mode 100644 core/server/src/http/jwt/jwt_manager.rs create mode 100644 core/server/src/http/jwt/middleware.rs create mode 100644 core/server/src/http/jwt/mod.rs create mode 100644 core/server/src/http/jwt/storage.rs create mode 100644 core/server/src/http/mapper.rs create mode 100644 core/server/src/http/messages.rs create mode 100644 core/server/src/http/mod.rs create mode 100644 core/server/src/http/partitions.rs create mode 100644 core/server/src/http/personal_access_tokens.rs create mode 100644 core/server/src/http/segments.rs rename foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/RedirectionClusterFixture.cs => core/server/src/http/shared.rs (65%) create mode 100644 core/server/src/http/streams.rs create mode 100644 core/server/src/http/system.rs create mode 100644 core/server/src/http/topics.rs create mode 100644 core/server/src/http/users.rs create mode 100644 core/server/src/http/web.rs create mode 100644 core/server/src/io/mod.rs create mode 100644 core/server/src/io/storage.rs create mode 100644 core/server/src/metadata/absorb.rs create mode 100644 core/server/src/metadata/consumer_group.rs create mode 100644 core/server/src/metadata/consumer_group_member.rs create mode 100644 core/server/src/metadata/inner.rs create mode 100644 core/server/src/metadata/mod.rs create mode 100644 core/server/src/metadata/ops.rs create mode 100644 core/server/src/metadata/partition.rs create mode 100644 core/server/src/metadata/reader.rs create mode 100644 core/server/src/metadata/stream.rs create mode 100644 core/server/src/metadata/topic.rs rename foreign/csharp/Iggy_SDK/Vsr/Command2.cs => core/server/src/metadata/user.rs (69%) create mode 100644 core/server/src/metadata/writer.rs create mode 100644 core/server/src/quic/listener.rs create mode 100644 core/server/src/quic/mod.rs create mode 100644 core/server/src/quic/quic_server.rs create mode 100644 core/server/src/quic/quic_socket.rs create mode 100644 core/server/src/sender/mod.rs create mode 100644 core/server/src/sender/quic_sender.rs create mode 100644 core/server/src/sender/tcp_sender.rs create mode 100644 core/server/src/sender/tcp_tls_sender.rs create mode 100644 core/server/src/sender/websocket_sender.rs create mode 100644 core/server/src/sender/websocket_tls_sender.rs create mode 100644 core/server/src/shard/builder.rs create mode 100644 core/server/src/shard/communication.rs create mode 100644 core/server/src/shard/execution.rs create mode 100644 core/server/src/shard/handlers.rs create mode 100644 core/server/src/shard/mod.rs create mode 100644 core/server/src/shard/system/clients.rs create mode 100644 core/server/src/shard/system/cluster.rs create mode 100644 core/server/src/shard/system/consumer_groups.rs create mode 100644 core/server/src/shard/system/consumer_offsets.rs create mode 100644 core/server/src/shard/system/info.rs create mode 100644 core/server/src/shard/system/messages.rs create mode 100644 core/server/src/shard/system/mod.rs create mode 100644 core/server/src/shard/system/partitions.rs create mode 100644 core/server/src/shard/system/personal_access_tokens.rs create mode 100644 core/server/src/shard/system/segments.rs create mode 100644 core/server/src/shard/system/snapshot/mod.rs create mode 100644 core/server/src/shard/system/snapshot/procdump.rs create mode 100644 core/server/src/shard/system/stats.rs create mode 100644 core/server/src/shard/system/storage.rs create mode 100644 core/server/src/shard/system/streams.rs create mode 100644 core/server/src/shard/system/topics.rs create mode 100644 core/server/src/shard/system/users.rs create mode 100644 core/server/src/shard/system/utils.rs create mode 100644 core/server/src/shard/systemd.rs create mode 100644 core/server/src/shard/task_registry/builders.rs create mode 100644 core/server/src/shard/task_registry/builders/continuous.rs create mode 100644 core/server/src/shard/task_registry/builders/oneshot.rs create mode 100644 core/server/src/shard/task_registry/builders/periodic.rs create mode 100644 core/server/src/shard/task_registry/mod.rs create mode 100644 core/server/src/shard/task_registry/registry.rs create mode 100644 core/server/src/shard/task_registry/shutdown.rs create mode 100644 core/server/src/shard/tasks/continuous/http_server.rs create mode 100644 core/server/src/shard/tasks/continuous/message_pump.rs create mode 100644 core/server/src/shard/tasks/continuous/mod.rs create mode 100644 core/server/src/shard/tasks/continuous/quic_server.rs create mode 100644 core/server/src/shard/tasks/continuous/tcp_server.rs create mode 100644 core/server/src/shard/tasks/continuous/websocket_server.rs create mode 100644 core/server/src/shard/tasks/mod.rs create mode 100644 core/server/src/shard/tasks/oneshot/config_writer.rs create mode 100644 core/server/src/shard/tasks/oneshot/mod.rs create mode 100644 core/server/src/shard/tasks/periodic/heartbeat_verifier.rs create mode 100644 core/server/src/shard/tasks/periodic/jwt_token_cleaner.rs create mode 100644 core/server/src/shard/tasks/periodic/message_cleaner.rs create mode 100644 core/server/src/shard/tasks/periodic/message_saver.rs create mode 100644 core/server/src/shard/tasks/periodic/mod.rs create mode 100644 core/server/src/shard/tasks/periodic/personal_access_token_cleaner.rs create mode 100644 core/server/src/shard/tasks/periodic/revocation_timeout.rs create mode 100644 core/server/src/shard/tasks/periodic/sysinfo_printer.rs create mode 100644 core/server/src/shard/tasks/periodic/systemd_watchdog.rs create mode 100644 core/server/src/shard/transmission/connector.rs create mode 100644 core/server/src/shard/transmission/event.rs create mode 100644 core/server/src/shard/transmission/frame.rs create mode 100644 core/server/src/shard/transmission/message.rs create mode 100644 core/server/src/shard/transmission/mod.rs create mode 100644 core/server/src/state/command.rs create mode 100644 core/server/src/state/entry.rs create mode 100644 core/server/src/state/file.rs rename core/{configs/src/common => server/src/state}/mod.rs (70%) create mode 100644 core/server/src/state/models.rs create mode 100644 core/server/src/state/system.rs create mode 100644 core/server/src/streaming/clients/client_manager.rs create mode 100644 core/server/src/streaming/clients/mod.rs create mode 100644 core/server/src/streaming/deduplication/mod.rs create mode 100644 core/server/src/streaming/diagnostics/metrics.rs create mode 100644 core/server/src/streaming/diagnostics/mod.rs create mode 100644 core/server/src/streaming/mod.rs create mode 100644 core/server/src/streaming/partitions/consumer_group_offsets.rs create mode 100644 core/server/src/streaming/partitions/consumer_offset.rs create mode 100644 core/server/src/streaming/partitions/consumer_offsets.rs create mode 100644 core/server/src/streaming/partitions/helpers.rs create mode 100644 core/server/src/streaming/partitions/in_flight.rs create mode 100644 core/server/src/streaming/partitions/journal.rs create mode 100644 core/server/src/streaming/partitions/local_partition.rs create mode 100644 core/server/src/streaming/partitions/local_partitions.rs create mode 100644 core/server/src/streaming/partitions/log.rs create mode 100644 core/server/src/streaming/partitions/mod.rs create mode 100644 core/server/src/streaming/partitions/ops.rs create mode 100644 core/server/src/streaming/partitions/ops_tests.rs create mode 100644 core/server/src/streaming/partitions/segments.rs create mode 100644 core/server/src/streaming/partitions/storage.rs create mode 100644 core/server/src/streaming/persistence/mod.rs create mode 100644 core/server/src/streaming/persistence/persister.rs create mode 100644 core/server/src/streaming/polling_consumer.rs create mode 100644 core/server/src/streaming/segments/indexes/index_reader.rs create mode 100644 core/server/src/streaming/segments/indexes/index_writer.rs create mode 100644 core/server/src/streaming/segments/indexes/mod.rs create mode 100644 core/server/src/streaming/segments/memory_journal.rs create mode 100644 core/server/src/streaming/segments/messages/messages_reader.rs create mode 100644 core/server/src/streaming/segments/messages/messages_writer.rs create mode 100644 core/server/src/streaming/segments/messages/mod.rs create mode 100644 core/server/src/streaming/segments/mod.rs create mode 100644 core/server/src/streaming/segments/segment.rs create mode 100644 core/server/src/streaming/segments/storage.rs create mode 100644 core/server/src/streaming/segments/types/mod.rs create mode 100644 core/server/src/streaming/session.rs create mode 100644 core/server/src/streaming/stats/mod.rs create mode 100644 core/server/src/streaming/storage.rs create mode 100644 core/server/src/streaming/streams/mod.rs create mode 100644 core/server/src/streaming/streams/storage.rs create mode 100644 core/server/src/streaming/topics/helpers.rs create mode 100644 core/server/src/streaming/topics/mod.rs create mode 100644 core/server/src/streaming/topics/storage.rs create mode 100644 core/server/src/streaming/users/mod.rs create mode 100644 core/server/src/streaming/users/user.rs create mode 100644 core/server/src/streaming/utils/address.rs create mode 100644 core/server/src/streaming/utils/file.rs create mode 100644 core/server/src/streaming/utils/mod.rs create mode 100644 core/server/src/streaming/utils/ptr.rs delete mode 100644 core/server/src/systemd.rs create mode 100644 core/server/src/tcp/connection_handler.rs create mode 100644 core/server/src/tcp/mod.rs create mode 100644 core/server/src/tcp/tcp_listener.rs create mode 100644 core/server/src/tcp/tcp_server.rs create mode 100644 core/server/src/tcp/tcp_socket.rs create mode 100644 core/server/src/tcp/tcp_tls_listener.rs create mode 100644 core/server/src/websocket/connection_handler.rs create mode 100644 core/server/src/websocket/mod.rs create mode 100644 core/server/src/websocket/websocket_listener.rs create mode 100644 core/server/src/websocket/websocket_server.rs create mode 100644 core/server/src/websocket/websocket_tls_listener.rs delete mode 100644 core/server/src/wire.rs create mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyClusterFixture.cs delete mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/VsrCluster.cs delete mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/Helpers/Eventually.cs delete mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/Vsr/VsrConsumerGroupTests.cs delete mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/Vsr/VsrHandshakeTests.cs delete mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/Vsr/VsrMessagingTests.cs delete mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/Vsr/VsrMetadataTests.cs delete mode 100644 foreign/csharp/Iggy_SDK/Contracts/SendMessagesResponse.cs delete mode 100644 foreign/csharp/Iggy_SDK/Exceptions/VsrRequestOutcomeUnknownException.cs delete mode 100644 foreign/csharp/Iggy_SDK/Exceptions/VsrSessionEvictedException.cs delete mode 100644 foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.Vsr.cs delete mode 100644 foreign/csharp/Iggy_SDK/IggyClient/Implementations/TransientHttpRetryHandler.cs delete mode 100644 foreign/csharp/Iggy_SDK/Utils/ServerAddress.cs delete mode 100644 foreign/csharp/Iggy_SDK/Vsr/ConsensusSession.cs delete mode 100644 foreign/csharp/Iggy_SDK/Vsr/ConsumerGroupClientState.cs delete mode 100644 foreign/csharp/Iggy_SDK/Vsr/CredentialBounds.cs delete mode 100644 foreign/csharp/Iggy_SDK/Vsr/LoginRegister.cs delete mode 100644 foreign/csharp/Iggy_SDK/Vsr/SyncConsumerGroup.cs delete mode 100644 foreign/csharp/Iggy_SDK/Vsr/VsrError.cs delete mode 100644 foreign/csharp/Iggy_SDK/Vsr/VsrHeader.cs delete mode 100644 foreign/csharp/Iggy_SDK/Vsr/VsrOperation.cs delete mode 100644 foreign/csharp/Iggy_SDK/Vsr/VsrReplyDecoder.cs delete mode 100644 foreign/csharp/Iggy_SDK_Tests/ClientTests/IggyClientFactoryTests.cs delete mode 100644 foreign/csharp/Iggy_SDK_Tests/PublisherTests/IggyPublisherBuilderTests.cs delete mode 100644 foreign/csharp/Iggy_SDK_Tests/VsrTests/ConsensusSessionTests.cs delete mode 100644 foreign/csharp/Iggy_SDK_Tests/VsrTests/ConsumerGroupClientStateTests.cs delete mode 100644 foreign/csharp/Iggy_SDK_Tests/VsrTests/CredentialBoundsTests.cs delete mode 100644 foreign/csharp/Iggy_SDK_Tests/VsrTests/LoginRegisterTests.cs delete mode 100644 foreign/csharp/Iggy_SDK_Tests/VsrTests/ServerAddressTests.cs delete mode 100644 foreign/csharp/Iggy_SDK_Tests/VsrTests/SyncConsumerGroupTests.cs delete mode 100644 foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrHeaderTests.cs delete mode 100644 foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrOperationTests.cs delete mode 100644 foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrProtocolDriftTests.cs delete mode 100644 foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrReplyDecoderTests.cs delete mode 100644 foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrTestPayloads.cs create mode 100644 foreign/go/internal/vsr/namespace.go create mode 100644 foreign/go/internal/vsr/namespace_test.go delete mode 100644 foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ClientRoutingState.java create mode 100644 foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/IggyFrameDecoder.java create mode 100644 foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/IggyFrameEncoder.java rename foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/{LoginRoutingHook.java => LoginRedirectionHook.java} (56%) delete mode 100644 foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ReconnectPlan.java delete mode 100644 foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/ConsensusSession.java delete mode 100644 foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrFrameDecoder.java delete mode 100644 foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrHeaders.java delete mode 100644 foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrLoginCodec.java delete mode 100644 foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrOperation.java delete mode 100644 foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrRequestEncoder.java delete mode 100644 foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandler.java delete mode 100644 foreign/java/java-sdk/src/main/java/org/apache/iggy/consumergroup/ConsumerGroupAssignment.java delete mode 100644 foreign/java/java-sdk/src/main/java/org/apache/iggy/hash/XxHash32.java delete mode 100644 foreign/java/java-sdk/src/main/java/org/apache/iggy/message/SendConfirmation.java delete mode 100644 foreign/java/java-sdk/src/main/java/org/apache/iggy/message/SendMessagesResponse.java delete mode 100644 foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientLoginRoutingTest.java delete mode 100644 foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java delete mode 100644 foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionConcurrencyTest.java delete mode 100644 foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionHeartbeatTest.java delete mode 100644 foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionRequestTimeoutTest.java delete mode 100644 foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/ClientRoutingStateTest.java create mode 100644 foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/IggyFrameDecoderTest.java create mode 100644 foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/IggyResponseHandlerTest.java delete mode 100644 foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/LoginRoutingHookTest.java delete mode 100644 foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/ReconnectPlanTest.java delete mode 100644 foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrFrameDecoderTest.java delete mode 100644 foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrRequestEncoderTest.java delete mode 100644 foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandlerTest.java delete mode 100644 foreign/java/java-sdk/src/test/java/org/apache/iggy/hash/XxHash32Test.java create mode 100644 foreign/node/CHANGELOG.md create mode 100644 foreign/node/src/wire/vsr/namespace.test.ts create mode 100644 foreign/node/src/wire/vsr/namespace.ts delete mode 100644 foreign/python/src/config.rs delete mode 100644 foreign/python/src/duration.rs delete mode 100644 foreign/python/tests/test_client_config.py create mode 100755 scripts/check-backwards-compat.sh delete mode 100644 scripts/ci/lib/init.sh diff --git a/.claude/skills/connectors-overview/SKILL.md b/.claude/skills/connectors-overview/SKILL.md index 230ad5354b..91ff86ec7c 100644 --- a/.claude/skills/connectors-overview/SKILL.md +++ b/.claude/skills/connectors-overview/SKILL.md @@ -96,24 +96,14 @@ The connectors codebase is intentionally repetitive across plugins. Cross-plugin ### Secrets -Any credential-bearing field (connection strings, API keys, bearer tokens, AWS keys) must be `SecretString` from the `secrecy` crate. Plain `String` for a credential is a review-blocker: `SecretString` redacts on `Debug`, so it is what keeps a credential out of a log line that formats the whole config. - -**`serde_secret::serialize_secret` EXPOSES the secret. It does not redact.** It calls `expose_secret()` and writes the plaintext. `SecretString` deliberately has no `Serialize` impl, and that absence is the protection - so adding `serialize_with` is what *unblocks* the derive and turns a compile-time guarantee into plaintext output. Use it only where the plaintext is the point: a wire payload, a persisted config, an API response that exposes credentials by design. - -So the default for a plugin config struct is **derive `Deserialize`, but not `Serialize`**. `Deserialize` is required: the SDK glue deserializes the config into the plugin's own struct (`sdk/src/{sink,source}.rs` call `serde_json::from_str::` under a `DeserializeOwned` bound). - -What never happens is the return trip. The runtime holds plugin configuration as a `serde_json::Value` - parsed from TOML, posted as JSON to the control API, or injected by env var - and hands that across the FFI, so nothing re-serializes the plugin's struct. Leaving `Serialize` off makes that compiler-enforced instead of convention-enforced (`sources/http_source/src/lib.rs::HttpSourceConfig` does this, and comments the omission so nobody adds it back). - -Pattern: +Any credential-bearing field (connection strings, API keys, bearer tokens, AWS keys) must be `SecretString` from the `secrecy` crate, with the workspace serde wrapper applied so `Debug` and serialization both redact. Runtime exposes plugin configs over the `/stats` HTTP surface via serialization - plain `String` leaks the secret to anyone who can hit the endpoint. Plain `String` for a credential is a review-blocker. Pattern (from `sinks/postgres_sink/src/lib.rs::PostgresSinkConfig`): ```rust use secrecy::{ExposeSecret, SecretString}; -// `Deserialize` only. Nothing re-serializes a plugin config, and leaving -// `Serialize` off is what makes the credential unserializable rather than -// merely un-serialized. -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct MyConfig { + #[serde(serialize_with = "iggy_common::serde_secret::serialize_secret")] pub connection_string: SecretString, } @@ -123,13 +113,7 @@ let pool = PgPoolOptions::new() .await?; ``` -If a config struct genuinely needs `Serialize`, `serde_secret::serialize_redacted` (and `serialize_optional_redacted`) write `[REDACTED]` in place of the value. Reach for `serialize_secret` only when the caller must get the real thing back. The sinks and sources listed below predate that helper and use the exposing one; the annotation is inert today, but it is not the protection it looks like. - -Note that none of this protects the credential from the runtime's own control API, which returns plugin configuration verbatim - see #3802. Plugin-side annotations are inert there because the runtime never routes through them. - -Plugin-side uses of the exposing helpers: `sinks/{postgres,mongodb,elasticsearch,influxdb,s3,surrealdb}_sink`, `sources/{postgres,elasticsearch,influxdb}_source`. - -That list is plugin-side only, not an inventory of every caller in the tree, and the others are not all mistakes: `runtime/src/api/config.rs` puts `serialize_secret` on `HttpConfig::api_key` (inert for the same reason), and several `core/common` wire-payload types (login, create-user, change-password, PAT) use these helpers by design, because there the credential *is* the payload. +In-tree uses: `sinks/{postgres,mongodb,elasticsearch,influxdb,delta}_sink`, `sources/{postgres,elasticsearch,influxdb}_source`. ### Errors @@ -209,7 +193,7 @@ Each implemented in at least one in-tree plugin or runtime path. | `flume::unbounded()` channel | `runtime/src/source.rs::spawn_source_handler` / `source_forwarding_loop` | MPSC handoff from SDK async task to runtime loop | | `tokio::sync::watch::channel(())` | `sdk/src/{sink,source}.rs`, `runtime/src/sink.rs`, `runtime/src/manager/*` | One-shot shutdown broadcast | | `dashmap::DashMap` | `runtime/src/manager/sink.rs`, `source.rs::SOURCE_SENDERS`, SDK `INSTANCES` | Lock-free concurrent keyed access | -| `secrecy::SecretString` + `iggy_common::serde_secret::serialize_secret` | `sinks/postgres_sink::PostgresSinkConfig::connection_string` | `Debug` redacts; `serialize_secret` EXPOSES | +| `secrecy::SecretString` + `iggy_common::serde_secret::serialize_secret` | `sinks/postgres_sink::PostgresSinkConfig::connection_string` | Auto-redact on Debug/Display + serialization | ## Drop accounting diff --git a/.dockerignore b/.dockerignore index 02100b3874..b527cb132d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -25,6 +25,7 @@ /target !/target/debug/iggy !/target/debug/iggy-server +!/target/debug/iggy-server-ng !/target/debug/iggy-mcp !/target/debug/iggy-connectors /web/node_modules diff --git a/.github/actions/cpp-bazel/pre-merge/action.yml b/.github/actions/cpp-bazel/pre-merge/action.yml index 20e7e465b1..adce777398 100644 --- a/.github/actions/cpp-bazel/pre-merge/action.yml +++ b/.github/actions/cpp-bazel/pre-merge/action.yml @@ -83,8 +83,6 @@ runs: - name: Setup server for e2e tests if: inputs.task == 'e2e' uses: ./.github/actions/utils/server-start - with: - cargo-bin: iggy-server - name: Run e2e tests if: inputs.task == 'e2e' diff --git a/.github/actions/csharp-dotnet/pre-merge/action.yml b/.github/actions/csharp-dotnet/pre-merge/action.yml index c4dc35bde7..143a157006 100644 --- a/.github/actions/csharp-dotnet/pre-merge/action.yml +++ b/.github/actions/csharp-dotnet/pre-merge/action.yml @@ -78,17 +78,11 @@ runs: - name: Build Iggy server Docker image id: docker_build if: inputs.task == 'e2e' - shell: bash - run: | - cargo build --locked --bin iggy-server --bin iggy - docker build \ - -f core/server/Dockerfile \ - --target runtime-prebuilt \ - -t iggy-server:test \ - --build-arg PREBUILT_IGGY_SERVER=target/debug/iggy-server \ - --build-arg PREBUILT_IGGY_CLI=target/debug/iggy \ - . - echo "docker_image=iggy-server:test" >> "$GITHUB_OUTPUT" + uses: ./.github/actions/utils/docker-build-test-server + with: + image-tag: "iggy-server:test" + libc: "glibc" + profile: "debug" - name: Run integration tests if: inputs.task == 'e2e' @@ -96,7 +90,6 @@ runs: env: IGGY_SERVER_DOCKER_IMAGE: ${{ steps.docker_build.outputs.docker_image }} IGGY_TEST_LOGS_DIR: ./reports/container-logs - IGGY_TEST_CLUSTER_NODES: "3" run: | dotnet test --project Iggy_SDK.Tests.Integration \ --no-build \ @@ -119,7 +112,7 @@ runs: uses: actions/upload-artifact@v7 if: inputs.task == 'e2e' && always() with: - name: dotnet-test-results-${{ inputs.task }} + name: dotnet-test-results path: foreign/csharp/reports retention-days: 7 diff --git a/.github/actions/go/pre-merge/action.yml b/.github/actions/go/pre-merge/action.yml index 56e0162b70..cfb5ba7e62 100644 --- a/.github/actions/go/pre-merge/action.yml +++ b/.github/actions/go/pre-merge/action.yml @@ -128,7 +128,9 @@ runs: if: inputs.task == 'e2e' uses: ./.github/actions/utils/server-start with: - cargo-bin: iggy-server + # TODO: change to iggy-server once legacy server is removed (core/server has VSR support) + cargo-bin: iggy-server-ng + cargo-features: vsr replica-id: "0" pid-file: ${{ runner.temp }}/iggy-go-e2e.pid log-file: ${{ runner.temp }}/iggy-go-e2e.log @@ -182,7 +184,9 @@ runs: if: inputs.task == 'e2e' uses: ./.github/actions/utils/server-start with: - cargo-bin: iggy-server + # TODO: change to iggy-server once legacy server is removed (core/server has VSR support) + cargo-bin: iggy-server-ng + cargo-features: vsr replica-id: "0" pid-file: ${{ runner.temp }}/iggy-go-e2e-tls.pid log-file: ${{ runner.temp }}/iggy-go-e2e-tls.log @@ -211,16 +215,14 @@ runs: pid-file: ${{ steps.iggy-tls.outputs.pid_file }} log-file: ${{ steps.iggy-tls.outputs.log_file }} - # Both replicas of the roster in `core/server/config.toml`, not just this - # one: a two-replica cluster commits on 2 acks (`quorum_replication`), so a - # lone node journals every op and commits none, and each client request - # blocks until it times out. - - name: Start Iggy VSR cluster node 0 + - name: Start Iggy VSR cluster node id: iggy-cluster-0 if: inputs.task == 'e2e' uses: ./.github/actions/utils/server-start with: - cargo-bin: iggy-server + # TODO: change to iggy-server once legacy server is removed (core/server has VSR support) + cargo-bin: iggy-server-ng + cargo-features: vsr replica-id: "0" pid-file: ${{ runner.temp }}/iggy-go-cluster-0.pid log-file: ${{ runner.temp }}/iggy-go-cluster-0.log @@ -229,40 +231,17 @@ runs: IGGY_CLUSTER_ENABLED: "true" IGGY_SYSTEM_PATH: ${{ runner.temp }}/iggy-go-cluster-0-data - - name: Start Iggy VSR cluster node 1 - id: iggy-cluster-1 - if: inputs.task == 'e2e' - uses: ./.github/actions/utils/server-start - with: - cargo-bin: iggy-server - replica-id: "1" - tcp_address: 127.0.0.1:8091 - http_address: 127.0.0.1:3001 - pid-file: ${{ runner.temp }}/iggy-go-cluster-1.pid - log-file: ${{ runner.temp }}/iggy-go-cluster-1.log - wait-timeout-seconds: "90" - env: - IGGY_CLUSTER_ENABLED: "true" - IGGY_SYSTEM_PATH: ${{ runner.temp }}/iggy-go-cluster-1-data - - name: Run cluster e2e tests shell: bash if: inputs.task == 'e2e' env: IGGY_TCP_ADDRESS: 127.0.0.1:8090 run: | - echo "🗳️ Running Go e2e tests against a two-node cluster..." + echo "🗳️ Running Go e2e tests against a single-node cluster..." cd foreign/go go test -v -race ./tests/... - - name: Stop Iggy VSR cluster node 1 - if: always() && inputs.task == 'e2e' - uses: ./.github/actions/utils/server-stop - with: - pid-file: ${{ steps.iggy-cluster-1.outputs.pid_file }} - log-file: ${{ steps.iggy-cluster-1.outputs.log_file }} - - - name: Stop Iggy VSR cluster node 0 + - name: Stop Iggy VSR cluster node if: always() && inputs.task == 'e2e' uses: ./.github/actions/utils/server-stop with: @@ -278,5 +257,4 @@ runs: ${{ steps.iggy.outputs.log_file }} ${{ steps.iggy-tls.outputs.log_file }} ${{ steps.iggy-cluster-0.outputs.log_file }} - ${{ steps.iggy-cluster-1.outputs.log_file }} if-no-files-found: ignore diff --git a/.github/actions/java-gradle/pre-merge/action.yml b/.github/actions/java-gradle/pre-merge/action.yml index 565e33c30f..47ccc4eb7b 100644 --- a/.github/actions/java-gradle/pre-merge/action.yml +++ b/.github/actions/java-gradle/pre-merge/action.yml @@ -84,11 +84,6 @@ runs: if: inputs.task == 'test' id: iggy uses: ./.github/actions/utils/server-start - with: - cargo-bin: iggy-server - wait-timeout-seconds: "90" - env: - IGGY_SYSTEM_PATH: ${{ runner.temp }}/iggy-java-data - name: Test if: inputs.task == 'test' @@ -156,12 +151,9 @@ runs: if: inputs.task == 'test' uses: ./.github/actions/utils/server-start with: - cargo-bin: iggy-server - wait-timeout-seconds: "90" pid-file: ${{ runner.temp }}/iggy-server-tls.pid log-file: ${{ runner.temp }}/iggy-server-tls.log env: - IGGY_SYSTEM_PATH: ${{ runner.temp }}/iggy-java-tls-data IGGY_TCP_TLS_ENABLED: "true" IGGY_TCP_TLS_CERT_FILE: core/certs/iggy_cert.pem IGGY_TCP_TLS_KEY_FILE: core/certs/iggy_key.pem diff --git a/.github/actions/node-npm/pre-merge/action.yml b/.github/actions/node-npm/pre-merge/action.yml index 34b83e0b2f..5dba122baa 100644 --- a/.github/actions/node-npm/pre-merge/action.yml +++ b/.github/actions/node-npm/pre-merge/action.yml @@ -20,7 +20,7 @@ description: Node.js pre-merge testing github iggy actions inputs: task: - description: "Task to run (lint, test, build, e2e)" + description: "Task to run (lint, test, build, e2e, e2e-vsr)" required: true runs: @@ -39,11 +39,11 @@ runs: shell: bash - name: Setup Rust with cache - if: inputs.task == 'e2e' + if: inputs.task == 'e2e' || inputs.task == 'e2e-vsr' uses: ./.github/actions/utils/setup-rust-with-cache - name: Install netcat - if: inputs.task == 'e2e' + if: inputs.task == 'e2e' || inputs.task == 'e2e-vsr' run: sudo apt-get update && sudo apt-get install -y netcat-openbsd shell: bash @@ -95,8 +95,12 @@ runs: if: inputs.task == 'e2e' uses: ./.github/actions/utils/server-start with: - cargo-bin: iggy-server - wait-timeout-seconds: "90" + # Node e2e asserts full cluster metadata (2 nodes from + # config.toml's [[cluster.nodes]] list), so cluster mode must be + # on. --replica-id picks the current node out of that list. + replica-id: "0" + env: + IGGY_CLUSTER_ENABLED: "true" - name: E2E tests if: inputs.task == 'e2e' @@ -104,29 +108,91 @@ runs: cd foreign/node mkdir -p ../../reports/node-coverage/e2e npx c8 --reporter=lcov --reports-dir=../../reports/node-coverage/e2e npm run test:e2e + env: + IGGY_SERVER_HOST: 127.0.0.1 + IGGY_SERVER_TCP_PORT: 8090 shell: bash - - name: Stop Iggy server - if: always() && inputs.task == 'e2e' + - name: Start Iggy VSR server 0 + id: iggy-vsr-0 + if: inputs.task == 'e2e-vsr' + uses: ./.github/actions/utils/server-start + with: + cargo-bin: iggy-server-ng + cargo-features: vsr + replica-id: "0" + pid-file: ${{ runner.temp }}/iggy-node-vsr-0.pid + log-file: ${{ runner.temp }}/iggy-node-vsr-0.log + wait-timeout-seconds: "90" + env: + IGGY_CLUSTER_ENABLED: "true" + IGGY_SYSTEM_PATH: ${{ runner.temp }}/iggy-node-vsr-0-data + + - name: Start Iggy VSR server 1 + id: iggy-vsr-1 + if: inputs.task == 'e2e-vsr' + uses: ./.github/actions/utils/server-start + with: + cargo-bin: iggy-server-ng + cargo-features: vsr + replica-id: "1" + tcp_address: 127.0.0.1:8091 + http_address: 127.0.0.1:3001 + pid-file: ${{ runner.temp }}/iggy-node-vsr-1.pid + log-file: ${{ runner.temp }}/iggy-node-vsr-1.log + wait-timeout-seconds: "90" + env: + IGGY_CLUSTER_ENABLED: "true" + IGGY_SYSTEM_PATH: ${{ runner.temp }}/iggy-node-vsr-1-data + + - name: VSR E2E tests + if: inputs.task == 'e2e-vsr' + run: | + cd foreign/node + mkdir -p ../../reports/node-coverage/e2e-vsr + npx c8 --reporter=lcov \ + --reports-dir=../../reports/node-coverage/e2e-vsr \ + npm run test:e2e:vsr + env: + IGGY_TEST_PROTOCOL: vsr + shell: bash + + - name: Stop Iggy VSR server 1 + if: always() && inputs.task == 'e2e-vsr' uses: ./.github/actions/utils/server-stop with: - pid-file: ${{ steps.iggy.outputs.pid_file }} - log-file: ${{ steps.iggy.outputs.log_file }} + pid-file: ${{ steps.iggy-vsr-1.outputs.pid_file }} + log-file: ${{ steps.iggy-vsr-1.outputs.log_file }} - - name: Upload server logs - if: always() && inputs.task == 'e2e' + - name: Stop Iggy VSR server 0 + if: always() && inputs.task == 'e2e-vsr' + uses: ./.github/actions/utils/server-stop + with: + pid-file: ${{ steps.iggy-vsr-0.outputs.pid_file }} + log-file: ${{ steps.iggy-vsr-0.outputs.log_file }} + + - name: Upload VSR server logs + if: always() && inputs.task == 'e2e-vsr' uses: actions/upload-artifact@v7 with: - name: iggy-node-server-logs - path: ${{ steps.iggy.outputs.log_file }} + name: iggy-node-vsr-server-logs + path: | + ${{ steps.iggy-vsr-0.outputs.log_file }} + ${{ steps.iggy-vsr-1.outputs.log_file }} if-no-files-found: ignore + - name: Stop Iggy server (plain) + if: always() && inputs.task == 'e2e' + uses: ./.github/actions/utils/server-stop + with: + pid-file: ${{ steps.iggy.outputs.pid_file }} + log-file: ${{ steps.iggy.outputs.log_file }} + - name: Start Iggy server (TLS) id: iggy-tls if: inputs.task == 'e2e' uses: ./.github/actions/utils/server-start with: - cargo-bin: iggy-server pid-file: ${{ runner.temp }}/iggy-server-tls.pid log-file: ${{ runner.temp }}/iggy-server-tls.log env: diff --git a/.github/actions/php/pre-merge/action.yml b/.github/actions/php/pre-merge/action.yml index 02b1c70e4b..a0c817cc74 100644 --- a/.github/actions/php/pre-merge/action.yml +++ b/.github/actions/php/pre-merge/action.yml @@ -154,8 +154,6 @@ runs: if: inputs.task == 'test' id: iggy uses: ./.github/actions/utils/server-start - with: - cargo-bin: iggy-server - name: Run PHP SDK tests if: inputs.task == 'test' @@ -182,7 +180,6 @@ runs: id: iggy-tls uses: ./.github/actions/utils/server-start with: - cargo-bin: iggy-server pid-file: ${{ runner.temp }}/iggy-server-tls.pid log-file: ${{ runner.temp }}/iggy-server-tls.log env: diff --git a/.github/actions/python-maturin/pre-merge/action.yml b/.github/actions/python-maturin/pre-merge/action.yml index f97462f34d..a44062fc9a 100644 --- a/.github/actions/python-maturin/pre-merge/action.yml +++ b/.github/actions/python-maturin/pre-merge/action.yml @@ -124,12 +124,14 @@ runs: if: inputs.task == 'test' run: | # test_tls.py spawns the server in a container; build it from the - # same binaries the plain-TCP tests run against. - cargo build --locked --bin iggy-server --bin iggy + # same vsr binaries the plain-TCP tests run against. + # TODO(hubcio): change to iggy-server once legacy server is removed + # (core/server has VSR support) + cargo build --locked --bin iggy-server-ng --bin iggy --features vsr docker build \ - -f core/server/Dockerfile \ + -f core/server-ng/Dockerfile \ --target runtime-prebuilt \ - --build-arg PREBUILT_IGGY_SERVER=target/debug/iggy-server \ + --build-arg PREBUILT_IGGY_SERVER_NG=target/debug/iggy-server-ng \ --build-arg PREBUILT_IGGY_CLI=target/debug/iggy \ -t iggy-server:local . shell: bash @@ -139,7 +141,10 @@ runs: id: iggy uses: ./.github/actions/utils/server-start with: - cargo-bin: iggy-server + # TODO(hubcio): change to iggy-server once legacy server is removed + # (core/server has VSR support) + cargo-bin: iggy-server-ng + cargo-features: vsr - name: Run Python integration tests if: inputs.task == 'test' diff --git a/.github/actions/rust/pre-merge/action.yml b/.github/actions/rust/pre-merge/action.yml index 71248a848b..25a8a8389f 100644 --- a/.github/actions/rust/pre-merge/action.yml +++ b/.github/actions/rust/pre-merge/action.yml @@ -20,7 +20,7 @@ description: Rust pre-merge testing and linting github iggy actions inputs: task: - description: "Task to run (check, check-msrv, fmt, clippy, sort, machete, doctest, verify-publish, test-1, test-2, test-3, miri)" + description: "Task to run (check, check-msrv, fmt, clippy, sort, machete, doctest, verify-publish, test-1, test-2, compat, miri)" required: true component: description: "Component name (for context)" @@ -208,17 +208,13 @@ runs: - name: Build and test with coverage if: startsWith(inputs.task, 'test-') run: | - # Parse partition index from task name (test-1 -> hash:1/3, test-2 -> hash:2/3, ...). - # TEST_PARTITIONS must match the number of test-N tasks in - # .github/config/components.yml. Cluster bootstrap makes each test - # CPU-heavy, so partitions stay small. - TEST_PARTITIONS=3 + # Parse partition index from task name (test-1 -> hash:1/2, test-2 -> hash:2/2) TASK="${{ inputs.task }}" PARTITION_FLAG="" if [[ "$TASK" =~ ^test-([0-9]+)$ ]]; then PARTITION_INDEX="${BASH_REMATCH[1]}" - PARTITION_FLAG="--partition hash:${PARTITION_INDEX}/${TEST_PARTITIONS}" - echo "::notice::Running test partition ${PARTITION_INDEX}/${TEST_PARTITIONS}" + PARTITION_FLAG="--partition hash:${PARTITION_INDEX}/2" + echo "::notice::Running test partition ${PARTITION_INDEX}/2" fi # Read DAG-based affected crate filter (computed in earlier step) @@ -364,6 +360,15 @@ runs: ls -la codecov.json shell: bash + - name: Backwards compatibility check + if: inputs.task == 'compat' && (github.event_name != 'pull_request' || !contains(join(github.event.pull_request.labels.*.name, ','), 'breaking:storage')) + run: | + scripts/check-backwards-compat.sh \ + --master-ref master \ + --pr-ref ${{ github.sha }} \ + --port 8090 --wait-secs 180 + shell: bash + # Miri (UB detector) on the unsafe-heavy crates that don't pull tokio / # compio. Pinned nightly so MIRIFLAGS behavior is stable across CI runs; # bump the date quarterly. Tree-borrows is the future-default aliasing diff --git a/.github/actions/utils/docker-build-test-server/action.yml b/.github/actions/utils/docker-build-test-server/action.yml new file mode 100644 index 0000000000..a82ae762a2 --- /dev/null +++ b/.github/actions/utils/docker-build-test-server/action.yml @@ -0,0 +1,105 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +name: docker-build-test-server +description: Build Iggy server binaries and Docker image with prebuilt binaries +inputs: + dockerfile: + description: "Path to Dockerfile" + required: false + default: "core/server/Dockerfile" + dockerfile-target: + description: "Dockerfile target stage (e.g., runtime-prebuilt)" + required: false + default: "runtime-prebuilt" + image-tag: + description: "Docker image tag (without registry/name)" + required: false + default: "iggy-server:test" + docker-build-context: + description: "Docker build context" + required: false + default: "." + prebuilt-server-binary: + description: "Path to prebuilt iggy-server binary" + required: false + default: "target/debug/iggy-server" + prebuilt-cli-binary: + description: "Path to prebuilt iggy CLI binary" + required: false + default: "target/debug/iggy" + libc: + description: "Libc type (musl or glibc)" + required: false + default: "glibc" + profile: + description: "Build profile (debug or release)" + required: false + default: "debug" +outputs: + docker_image: + description: "Full Docker image name built" + value: ${{ steps.build.outputs.image_tag }} +runs: + using: "composite" + steps: + - name: Build Iggy server and CLI binaries + shell: bash + run: | + echo "Building Iggy server and CLI binaries with cargo..." + cargo build --locked --bin iggy-server --bin iggy + + echo "✅ Binaries built successfully:" + ls -lh ${{ inputs.prebuilt-server-binary }} ${{ inputs.prebuilt-cli-binary }} + + - name: Build Docker image with prebuilt binaries + id: build + shell: bash + run: | + set -euo pipefail + DOCKERFILE="${{ inputs.dockerfile }}" + DOCKERFILE_TARGET="${{ inputs.dockerfile-target }}" + IMAGE_TAG="${{ inputs.image-tag }}" + BUILD_CONTEXT="${{ inputs.docker-build-context }}" + PREBUILT_IGGY_SERVER="${{ inputs.prebuilt-server-binary }}" + PREBUILT_IGGY_CLI="${{ inputs.prebuilt-cli-binary }}" + LIBC="${{ inputs.libc }}" + PROFILE="${{ inputs.profile }}" + + echo "Building Docker image with prebuilt binaries..." + echo " Dockerfile: $DOCKERFILE" + echo " Target: $DOCKERFILE_TARGET" + echo " Image Tag: $IMAGE_TAG" + echo " Context: $BUILD_CONTEXT" + echo " Server Binary: $PREBUILT_IGGY_SERVER" + echo " CLI Binary: $PREBUILT_IGGY_CLI" + echo " Libc: $LIBC" + echo " Profile: $PROFILE" + + docker build \ + -f "$DOCKERFILE" \ + --target "$DOCKERFILE_TARGET" \ + -t "$IMAGE_TAG" \ + --build-arg PREBUILT_IGGY_SERVER="$PREBUILT_IGGY_SERVER" \ + --build-arg PREBUILT_IGGY_CLI="$PREBUILT_IGGY_CLI" \ + --build-arg LIBC="$LIBC" \ + --build-arg PROFILE="$PROFILE" \ + "$BUILD_CONTEXT" + + echo "image_tag=$IMAGE_TAG" >> "$GITHUB_OUTPUT" + echo "✅ Docker image built successfully: $IMAGE_TAG" diff --git a/.github/actions/utils/server-start/action.yml b/.github/actions/utils/server-start/action.yml index 664582f7e0..3eb06a1b97 100644 --- a/.github/actions/utils/server-start/action.yml +++ b/.github/actions/utils/server-start/action.yml @@ -30,6 +30,10 @@ inputs: description: "Cargo profile: release|debug" required: false default: "debug" + cargo-features: + description: "Comma-separated Cargo features" + required: false + default: "" bin: description: "Path to server binary (when mode=bin)" required: false @@ -112,12 +116,15 @@ runs: else OUT="target/debug/$NAME" fi - if [[ ! -x "$OUT" ]]; then + if [[ ! -x "$OUT" || -n "${{ inputs.cargo-features }}" ]]; then echo "Building $NAME with cargo ($PROFILE)…" CARGO_ARGS=(build --locked --bin "$NAME") if [[ "$PROFILE" == "release" ]]; then CARGO_ARGS+=(--release) fi + if [[ -n "${{ inputs.cargo-features }}" ]]; then + CARGO_ARGS+=(--features "${{ inputs.cargo-features }}") + fi cargo "${CARGO_ARGS[@]}" fi BIN_PATH="$OUT" diff --git a/.github/config/components.yml b/.github/config/components.yml index c3c28c6b71..545b5cb5b7 100644 --- a/.github/config/components.yml +++ b/.github/config/components.yml @@ -121,6 +121,7 @@ components: - "rust-cluster" paths: - "core/server/**" + - "core/server-ng/**" rust-cluster: depends_on: @@ -176,7 +177,7 @@ components: - "machete" - "test-1" - "test-2" - - "test-3" + - "compat" - "build-aarch64-gnu" - "build-aarch64-musl" - "build-macos-aarch64" @@ -250,7 +251,7 @@ components: - "ci-infrastructure" # CI changes trigger full regression paths: - "foreign/node/**" - tasks: ["lint", "test", "build", "e2e"] + tasks: ["lint", "test", "build", "e2e", "e2e-vsr"] sdk-go: depends_on: @@ -264,7 +265,8 @@ components: # The SDK compiles against the VSR wire contract, so a change to the # protocol crate or the VSR server must rerun it. - "core/binary_protocol/**" - - "core/server/**" + # TODO: change to core/server once legacy server is removed (core/server has VSR support) + - "core/server-ng/**" # VSR is the only protocol the Go SDK speaks, so there is no separate lane. tasks: ["lint", "test", "build", "e2e"] @@ -305,6 +307,7 @@ components: - "bdd/docker-compose.server.yml" - "bdd/docker-compose.cluster.yml" - "bdd/docker-compose.coverage.yml" + - "bdd/docker-compose.vsr.yml" # Individual BDD tests per SDK - only run when specific SDK changes bdd-rust: @@ -316,7 +319,7 @@ components: paths: - "bdd/rust/**" - "bdd/scenarios/**" - tasks: ["bdd-rust"] + tasks: ["bdd-rust", "bdd-rust-vsr"] bdd-python: depends_on: @@ -354,7 +357,8 @@ components: - "bdd/scenarios/**" # The Go suites run against the VSR server, so its sources gate them. - "core/binary_protocol/**" - - "core/server/**" + # TODO: change to core/server once legacy server is removed (core/server has VSR support) + - "core/server-ng/**" tasks: ["bdd-go", "bdd-go-race"] bdd-node: @@ -367,7 +371,7 @@ components: paths: - "bdd/node/**" - "bdd/scenarios/**" - tasks: ["bdd-node"] + tasks: ["bdd-node", "bdd-node-vsr"] bdd-csharp: depends_on: diff --git a/.github/dependabot.yml b/.github/dependabot.yml index eb28a9fb3b..8d2932f980 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -230,6 +230,7 @@ updates: directories: - "/" - "/core/server" + - "/core/server-ng" - "/core/connectors/runtime" - "/core/ai/mcp" - "/core/bench/dashboard/server" diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index a2b6a61dbb..80edc184bc 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -127,7 +127,8 @@ jobs: if: >- inputs.component == 'sdk-node' && (inputs.task == 'test' || - inputs.task == 'e2e') + inputs.task == 'e2e' || + inputs.task == 'e2e-vsr') uses: codecov/codecov-action@v7.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/_test_bdd.yml b/.github/workflows/_test_bdd.yml index 87ab40756f..5640e1bbb4 100644 --- a/.github/workflows/_test_bdd.yml +++ b/.github/workflows/_test_bdd.yml @@ -51,9 +51,24 @@ jobs: - name: Build server for BDD tests if: startsWith(inputs.component, 'bdd-') && startsWith(inputs.task, 'bdd-') run: | - SERVER_BIN="iggy-server" - echo "Building server binary and CLI for BDD tests..." - cargo build --locked --bin iggy-server --bin iggy + # The VSR lanes need the vsr feature on both the server and the CLI, + # otherwise the CLI cannot frame requests for the VSR wire protocol + # and the healthcheck ping fails. The Go SDK speaks only VSR and the + # Python wheels are vsr-built, so those suites are always on this + # branch. + case "${{ inputs.task }}" in + bdd-rust-vsr|bdd-go|bdd-go-race|bdd-python|bdd-node-vsr) + # TODO: change to iggy-server once legacy server is removed (core/server has VSR support) + SERVER_BIN="iggy-server-ng" + echo "Building the VSR server binary and CLI (--features vsr) for BDD tests..." + cargo build --locked --bin iggy-server-ng --bin iggy --features vsr + ;; + *) + SERVER_BIN="iggy-server" + echo "Building server binary and CLI for BDD tests..." + cargo build --locked --bin iggy-server --bin iggy + ;; + esac echo "Server binary built at: target/debug/${SERVER_BIN}" ls -lh "target/debug/${SERVER_BIN}" @@ -75,20 +90,39 @@ jobs: - name: Run BDD tests if: startsWith(inputs.component, 'bdd-') && startsWith(inputs.task, 'bdd-') run: | - # Extract SDK name from task (format: bdd-). - SDK_NAME=$(echo "${{ inputs.task }}" | sed 's/^bdd-//') - export IGGY_SERVER_PATH="target/debug/iggy-server" - echo "Server binary location: $(ls -lh target/debug/iggy-server)" + # Extract SDK name from task (format: bdd-, or bdd--vsr + # for an explicit vsr lane). Python has no legacy lane, so its + # plain task name runs vsr. + SDK_NAME=$(echo "${{ inputs.task }}" | sed 's/^bdd-//; s/-vsr$//') + EXTRA_FLAGS=() + case "${{ inputs.task }}" in + bdd-rust-vsr|bdd-python|bdd-node-vsr) + EXTRA_FLAGS+=(--vsr) + # TODO: change to iggy-server once legacy server is removed (core/server has VSR support) + export IGGY_SERVER_NG_PATH="target/debug/iggy-server-ng" + echo "Server binary location: $(ls -lh target/debug/iggy-server-ng)" + ;; + bdd-go|bdd-go-race) + # The Go SDK speaks only VSR, so the runner forces the overlay. + # TODO: change to iggy-server once legacy server is removed (core/server has VSR support) + export IGGY_SERVER_NG_PATH="target/debug/iggy-server-ng" + echo "Server binary location: $(ls -lh target/debug/iggy-server-ng)" + ;; + *) + echo "Server binary location: $(ls -lh target/debug/iggy-server)" + ;; + esac echo "Running BDD tests for SDK: $SDK_NAME" echo "Current directory: $(pwd)" echo "CLI binary location: $(ls -lh target/debug/iggy)" - # Export path to the pre-built cli binary (relative to repo root) + # Export path to the pre-built server and cli binaries (relative to repo root) + export IGGY_SERVER_PATH="target/debug/iggy-server" export IGGY_CLI_PATH="target/debug/iggy" export IGGY_ROOT_USERNAME="iggy" export IGGY_ROOT_PASSWORD="iggy" - ./scripts/run-bdd-tests.sh "$SDK_NAME" + ./scripts/run-bdd-tests.sh "${EXTRA_FLAGS[@]}" "$SDK_NAME" - name: Clean up Docker resources (BDD) if: always() && startsWith(inputs.component, 'bdd-') && startsWith(inputs.task, 'bdd-') diff --git a/.github/workflows/_test_examples.yml b/.github/workflows/_test_examples.yml index 1382f9ee55..3fdb104745 100644 --- a/.github/workflows/_test_examples.yml +++ b/.github/workflows/_test_examples.yml @@ -113,12 +113,22 @@ jobs: echo "Building common binaries for all examples tests..." echo "Current directory: $(pwd)" - # Every SDK speaks only the VSR wire protocol, so every lane runs - # against the VSR server. - SERVER_BIN="iggy-server" - echo "Building ${SERVER_BIN}..." - cargo build --locked --bin "${SERVER_BIN}" + # The Go SDK speaks only the VSR wire protocol and the Python wheels + # are vsr-built, so those lanes need the VSR server. Every other + # language still runs against the legacy one until its own migration + # lands. + if [[ "${{ inputs.task }}" == "examples-go" || "${{ inputs.task }}" == "examples-python" ]]; then + # TODO: change to iggy-server once legacy server is removed (core/server has VSR support) + SERVER_BIN="iggy-server-ng" + echo "Building ${SERVER_BIN} (--features vsr)..." + cargo build --locked --bin "${SERVER_BIN}" --features vsr + else + SERVER_BIN="iggy-server" + echo "Building ${SERVER_BIN}..." + cargo build --locked --bin "${SERVER_BIN}" + fi + # For Rust examples, also build CLI and example binaries if [[ "${{ inputs.task }}" == "examples-rust" ]]; then echo "Building additional binaries for Rust examples..." cargo build --locked --bin iggy --examples diff --git a/.github/workflows/coverage-baseline.yml b/.github/workflows/coverage-baseline.yml index 9fed1dc1b0..eaebf82eb2 100644 --- a/.github/workflows/coverage-baseline.yml +++ b/.github/workflows/coverage-baseline.yml @@ -216,16 +216,11 @@ jobs: - name: Build Iggy server Docker image id: docker_build - run: | - cargo build --locked --bin iggy-server --bin iggy - docker build \ - -f core/server/Dockerfile \ - --target runtime-prebuilt \ - -t iggy-server:test \ - --build-arg PREBUILT_IGGY_SERVER=target/debug/iggy-server \ - --build-arg PREBUILT_IGGY_CLI=target/debug/iggy \ - . - echo "docker_image=iggy-server:test" >> "$GITHUB_OUTPUT" + uses: ./.github/actions/utils/docker-build-test-server + with: + image-tag: "iggy-server:test" + libc: "glibc" + profile: "debug" - name: Restore and build working-directory: foreign/csharp @@ -323,12 +318,14 @@ jobs: - name: Build server Docker image for TLS tests run: | # test_tls.py spawns the server in a container; build it from the - # same binaries the plain-TCP tests run against. - cargo build --locked --bin iggy-server --bin iggy + # same vsr binaries the plain-TCP tests run against. + # TODO(hubcio): change to iggy-server once legacy server is removed + # (core/server has VSR support) + cargo build --locked --bin iggy-server-ng --bin iggy --features vsr docker build \ - -f core/server/Dockerfile \ + -f core/server-ng/Dockerfile \ --target runtime-prebuilt \ - --build-arg PREBUILT_IGGY_SERVER=target/debug/iggy-server \ + --build-arg PREBUILT_IGGY_SERVER_NG=target/debug/iggy-server-ng \ --build-arg PREBUILT_IGGY_CLI=target/debug/iggy \ -t iggy-server:local . shell: bash @@ -337,7 +334,10 @@ jobs: id: iggy uses: ./.github/actions/utils/server-start with: - cargo-bin: iggy-server + # TODO(hubcio): change to iggy-server once legacy server is removed + # (core/server has VSR support) + cargo-bin: iggy-server-ng + cargo-features: vsr - name: Run tests run: | @@ -443,9 +443,8 @@ jobs: - name: Start Iggy server id: iggy uses: ./.github/actions/utils/server-start - with: - cargo-bin: iggy-server - wait-timeout-seconds: "90" + env: + IGGY_CLUSTER_ENABLED: true - name: Run unit tests with coverage run: | @@ -458,6 +457,9 @@ jobs: cd foreign/node mkdir -p ../../reports/node-coverage/e2e npx c8 --reporter=lcov --reports-dir=../../reports/node-coverage/e2e npm run test:e2e + env: + IGGY_SERVER_HOST: 127.0.0.1 + IGGY_SERVER_TCP_PORT: 8090 - name: Stop Iggy server if: always() @@ -498,7 +500,9 @@ jobs: id: iggy uses: ./.github/actions/utils/server-start with: - cargo-bin: iggy-server + # TODO: change to iggy-server once legacy server is removed (core/server has VSR support) + cargo-bin: iggy-server-ng + cargo-features: vsr replica-id: "0" wait-timeout-seconds: "90" diff --git a/.github/workflows/pr-title.yml b/.github/workflows/pr-title.yml index 69b4fdcdc3..f485734caa 100644 --- a/.github/workflows/pr-title.yml +++ b/.github/workflows/pr-title.yml @@ -84,6 +84,7 @@ jobs: sdk security server + server-ng shard test web diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index edc9452f58..03b5254675 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -16,7 +16,7 @@ # under the License. # PR gate: detects changed components, builds test matrices, and runs -# lint/test/build/BDD/examples jobs only for affected languages. +# lint/test/build/compat/BDD/examples jobs only for affected languages. # All jobs must pass before merge. name: Pre-merge diff --git a/AGENTS.md b/AGENTS.md index 676ea42422..8631c013f8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -82,7 +82,8 @@ install` (Python) or `prek install` (Rust drop-in) - both read ```text iggy/ ├── core/ -│ ├── server/ Iggy server binary (Viewstamped Replication) +│ ├── server/ Iggy server binary +│ ├── server-ng/ Next-gen server (Viewstamped Replication, WIP) │ ├── sdk/ Rust client SDK │ ├── cli/ iggy CLI │ ├── connectors/ Connectors runtime + SDK + sinks/sources @@ -113,6 +114,7 @@ iggy/ | --------------------- | ---------------------------------------- | | Wire protocol | `core/binary_protocol/` | | Server | `core/server/src/` | +| Next-gen server (WIP) | `core/server-ng/` | | Rust client SDK | `core/sdk/src/` | | Connectors | `core/connectors/` -> connector-* skills | | Integration tests | `core/integration/tests/` | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8297d919dd..f7850662e6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -84,28 +84,6 @@ cargo install prek prek install ``` -The hooks require **bash >= 4.2** and refuse to run on anything older. Every current -Linux distribution already satisfies this. - -#### macOS - -macOS ships bash 3.2 and never updates it, so this is the one platform that needs a -step: - -```bash -brew install bash -``` - -Homebrew's bash has to precede `/bin` on `PATH`, which is the default for a Homebrew -install but not guaranteed. Check with: - -```bash -bash --version -``` - -Git GUIs launched from the Dock get a minimal `PATH` where `/bin` wins, so the hook can -still find bash 3.2 after the install. Committing from a terminal avoids this. - ## Code Style ### Comments: WHY, Not WHAT diff --git a/Cargo.lock b/Cargo.lock index 051b51babc..1cf7f53614 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3198,6 +3198,7 @@ dependencies = [ "static-toml", "strum 0.28.0", "tracing", + "tungstenite 0.30.0", ] [[package]] @@ -3214,7 +3215,6 @@ dependencies = [ name = "consensus" version = "0.1.0" dependencies = [ - "aligned-vec", "bit-set 0.11.1", "bytemuck", "bytes", @@ -6856,7 +6856,6 @@ dependencies = [ "enumset", "secrecy", "thiserror 2.0.19", - "twox-hash", ] [[package]] @@ -7502,6 +7501,7 @@ dependencies = [ "serde", "serde_json", "serial_test", + "server", "sqlx", "sysinfo 0.39.6", "tempfile", @@ -9621,12 +9621,10 @@ dependencies = [ "iggy_common", "journal", "message_bus", - "nix", "papaya", "ringbuffer", "server_common", "smallvec", - "tempfile", "tokio", "tracing", ] @@ -11907,6 +11905,71 @@ dependencies = [ [[package]] name = "server" +version = "0.8.2-edge.1" +dependencies = [ + "ahash 0.8.12", + "anyhow", + "async-channel", + "async_zip", + "axum", + "axum-server", + "bytes", + "chrono", + "clap", + "compio", + "configs", + "ctrlc", + "cyper", + "cyper-axum", + "dashmap", + "dotenvy", + "err_trail", + "error_set", + "figlet-rs", + "flume", + "fs2", + "futures", + "hash32 1.0.0", + "human-repr", + "iggy_binary_protocol", + "iggy_common", + "jsonwebtoken", + "left-right", + "mimalloc", + "mime_guess", + "nix", + "papaya", + "prometheus-client", + "ringbuffer", + "rmp-serde", + "rust-embed", + "rustls", + "rustls-pemfile", + "sd-notify", + "secrecy", + "send_wrapper", + "serde", + "serde_json", + "server_common", + "shard_allocator", + "slab", + "socket2 0.6.5", + "strum 0.28.0", + "sysinfo 0.39.6", + "system_stats", + "tempfile", + "thiserror 2.0.19", + "tokio", + "toml 1.1.3+spec-1.1.0", + "tower-http 0.7.0", + "tracing", + "ulid", + "uuid", + "vergen-git2", +] + +[[package]] +name = "server-ng" version = "0.9.0-edge.2" dependencies = [ "ahash 0.8.12", @@ -11966,7 +12029,6 @@ dependencies = [ "rust-embed", "rustls", "rustls-pemfile", - "sd-notify", "secrecy", "send_wrapper", "serde", @@ -12253,7 +12315,7 @@ dependencies = [ "rand 0.10.2", "rand_xoshiro", "secrecy", - "server", + "server-ng", "server_common", "shard", "strum 0.28.0", diff --git a/Cargo.toml b/Cargo.toml index d6c7d0c544..3e5db8fb62 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,6 +61,7 @@ members = [ "core/partitions", "core/sdk", "core/server", + "core/server-ng", "core/server_common", "core/shard", "core/shard_allocator", @@ -296,6 +297,7 @@ serde_with = { version = "3.21.0", features = ["base64", "macros"] } serde_yaml_ng = "0.10.0" serial_test = "3.5.0" server = { path = "core/server" } +server-ng = { path = "core/server-ng" } server_common = { path = "core/server_common" } shard = { path = "core/shard" } shard_allocator = { path = "core/shard_allocator" } diff --git a/Dockerfile b/Dockerfile index f340b38f40..6f4f99b64a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,7 +43,7 @@ RUN apt-get update && apt-get install -y \ COPY . . RUN npm --prefix web ci && npm --prefix web run build:static RUN cargo build --bin iggy --release -RUN cargo build --bin iggy-server -p server --release +RUN cargo build --bin iggy-server --release FROM debian:trixie-slim RUN apt-get update && apt-get install -y \ @@ -51,6 +51,7 @@ RUN apt-get update && apt-get install -y \ liblzma5 \ libhwloc15 \ && rm -rf /var/lib/apt/lists/* +COPY ./core/configs ./configs COPY --from=builder /build/target/release/iggy . COPY --from=builder /build/target/release/iggy-server . diff --git a/README.md b/README.md index 0e1925eeb8..3db9535827 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ The name is an abbreviation for the Italian Greyhound - small yet extremely fast - Built-in **CLI** to manage the streaming server installable via `cargo install iggy-cli` - Built-in **benchmarking app** to test the performance - **Single binary deployment** (no external dependencies) -- Running as a single node or as a **cluster**, with data replication based on **[Viewstamped Replication (VSR)](https://github.com/apache/iggy/blob/master/assets/vsr.pdf)** +- Running as a single node (clustering based on Viewstamped Replication will be implemented in the near future) ![server](assets/server.png) @@ -117,6 +117,12 @@ We do also publish edge/dev/nightly releases (e.g. `0.7.0-edge.1` or `apache/igg --- +## Roadmap + +- **Clustering** & data replication based on **[VSR](https://github.com/apache/iggy/blob/master/assets/vsr.pdf)** (coming soon) + +--- + ## Supported languages SDK - [Rust](https://crates.io/crates/iggy) @@ -279,7 +285,7 @@ it should only be used for development and testing. `cargo run --bin iggy-server -- --with-default-root-credentials` -Root credentials are only set on the first server startup when the data directory doesn't exist yet. Once the server has been started and persisted data exists, the existing root credentials will be reused, and the `--with-default-root-credentials` flag or environment variables are ignored. They are still validated, though: a half-set pair or an out-of-range value aborts the boot instead of being silently dropped. To reset credentials, delete the data directory. +Root credentials are only set on the first server startup when the data directory doesn't exist yet. Once the server has been started and persisted data exists, the existing root credentials will be reused, and the `--with-default-root-credentials` flag or environment variables will have no effect. To reset credentials, delete the data directory. For configuration options and detailed help: diff --git a/bdd/README.md b/bdd/README.md index b50246b03f..2509006f4f 100644 --- a/bdd/README.md +++ b/bdd/README.md @@ -39,6 +39,7 @@ bdd/ ├── docker-compose.server.yml # Single iggy-server test setup ├── docker-compose.cluster.yml # Leader + follower test setup ├── docker-compose.coverage.yml # Coverage collection overlay +├── docker-compose.vsr.yml # server-ng (VSR) overlay, Rust SDK only ├── Dockerfile # Debug build of Iggy server └── README.md ``` @@ -70,8 +71,14 @@ bdd/ # Run only leader_redirection ../scripts/run-bdd-tests.sh all leader_redirection -# Every suite runs against iggy-server. Build the binaries first: -# cargo build --bin iggy-server --bin iggy +# Run against iggy-server-ng with VSR (Rust SDK only; other SDKs do not +# speak the VSR wire protocol yet). Build the vsr binaries first: +# cargo build --bin iggy-server-ng --bin iggy --features vsr +# NOTE: `target/debug/iggy` is shared between lanes and the vsr flavour +# speaks a different wire protocol. When switching back to the legacy +# lane, rebuild without `--features vsr` first, or the in-container +# healthcheck ping cannot talk to the legacy server. +../scripts/run-bdd-tests.sh --vsr rust # Clean up Docker resources ../scripts/run-bdd-tests.sh clean diff --git a/bdd/docker-compose.cluster.yml b/bdd/docker-compose.cluster.yml index 9976d342b1..063daf0db4 100644 --- a/bdd/docker-compose.cluster.yml +++ b/bdd/docker-compose.cluster.yml @@ -15,20 +15,9 @@ # specific language governing permissions and limitations # under the License. -# Iggy leader + follower cluster test setup: a real 2-node VSR cluster with -# replica 0 as the initial leader. +# Iggy leader + follower cluster test setup. # Activated by: ./scripts/run-bdd-tests.sh leader_redirection # ./scripts/run-bdd-tests.sh all -# -# Server constraints that shape this file: -# - `--replica-id ` is the only CLI arg each node needs; there is no -# `--follower`, and a new container per run makes `--fresh` redundant. -# - root credentials are mandatory once the cluster is enabled and come -# from IGGY_ROOT_USERNAME / IGGY_ROOT_PASSWORD. -# - `cluster.nodes[*].ip` is parsed as a strict IpAddr (no hostnames) and -# cluster listeners bind to that roster IP, so nodes need static IPs and -# healthchecks must ping the roster address rather than loopback. -# - replica-to-replica consensus requires `ports.tcp_replica`. x-cluster-node: &cluster-node image: iggy-bdd-server @@ -50,34 +39,26 @@ x-cluster-node: &cluster-node memlock: soft: -1 hard: -1 + networks: + - iggy-bdd-network x-cluster-topology: &cluster-topology - IGGY_ROOT_USERNAME: iggy - IGGY_ROOT_PASSWORD: iggy IGGY_CLUSTER_ENABLED: "true" IGGY_CLUSTER_NAME: test-cluster IGGY_CLUSTER_NODES_0_NAME: leader-node - IGGY_CLUSTER_NODES_0_IP: 172.28.0.101 + IGGY_CLUSTER_NODES_0_IP: iggy-leader IGGY_CLUSTER_NODES_0_REPLICA_ID: "0" IGGY_CLUSTER_NODES_0_PORTS_TCP: "8091" IGGY_CLUSTER_NODES_0_PORTS_QUIC: "8081" IGGY_CLUSTER_NODES_0_PORTS_HTTP: "3001" IGGY_CLUSTER_NODES_0_PORTS_WEBSOCKET: "8071" - IGGY_CLUSTER_NODES_0_PORTS_TCP_REPLICA: "8191" IGGY_CLUSTER_NODES_1_NAME: follower-node - IGGY_CLUSTER_NODES_1_IP: 172.28.0.102 + IGGY_CLUSTER_NODES_1_IP: iggy-follower IGGY_CLUSTER_NODES_1_REPLICA_ID: "1" IGGY_CLUSTER_NODES_1_PORTS_TCP: "8092" IGGY_CLUSTER_NODES_1_PORTS_QUIC: "8082" IGGY_CLUSTER_NODES_1_PORTS_HTTP: "3002" IGGY_CLUSTER_NODES_1_PORTS_WEBSOCKET: "8072" - IGGY_CLUSTER_NODES_1_PORTS_TCP_REPLICA: "8192" - # http.enabled with cluster.enabled requires a JWT key every node can - # verify; cluster auth provides it (derived from the shared secret). - # Enabling auth activates follower-to-primary forwarding, so the config - # validator requires ports.http on every roster node. - IGGY_CLUSTER_AUTH_ENABLED: "true" - IGGY_CLUSTER_AUTH_SHARED_SECRET: "bdd-vsr-cluster-shared-secret-0123456789" x-cluster-bdd-deps: &cluster-bdd-deps depends_on: @@ -92,9 +73,9 @@ x-cluster-bdd-deps: &cluster-bdd-deps services: iggy-leader: <<: *cluster-node - command: [ "--replica-id", "0" ] + command: [ "--fresh", "--with-default-root-credentials", "--replica-id", "0" ] healthcheck: - test: [ "CMD", "/usr/local/bin/iggy", "--tcp-server-address", "172.28.0.101:8091", "ping" ] + test: [ "CMD", "/usr/local/bin/iggy", "--tcp-server-address", "127.0.0.1:8091", "ping" ] interval: 5s timeout: 3s retries: 30 @@ -107,17 +88,14 @@ services: IGGY_HTTP_ADDRESS: 0.0.0.0:3001 IGGY_QUIC_ADDRESS: 0.0.0.0:8081 IGGY_WEBSOCKET_ADDRESS: 0.0.0.0:8071 - networks: - iggy-bdd-network: - ipv4_address: 172.28.0.101 volumes: - iggy_leader_data:/app/local_data_leader iggy-follower: <<: *cluster-node - command: [ "--replica-id", "1" ] + command: [ "--fresh", "--with-default-root-credentials", "--follower", "--replica-id", "1" ] healthcheck: - test: [ "CMD", "/usr/local/bin/iggy", "--tcp-server-address", "172.28.0.102:8092", "ping" ] + test: [ "CMD", "/usr/local/bin/iggy", "--tcp-server-address", "127.0.0.1:8092", "ping" ] interval: 5s timeout: 3s retries: 30 @@ -130,9 +108,6 @@ services: IGGY_HTTP_ADDRESS: 0.0.0.0:3002 IGGY_QUIC_ADDRESS: 0.0.0.0:8082 IGGY_WEBSOCKET_ADDRESS: 0.0.0.0:8072 - networks: - iggy-bdd-network: - ipv4_address: 172.28.0.102 volumes: - iggy_follower_data:/app/local_data_follower @@ -148,13 +123,6 @@ services: java-bdd: <<: *cluster-bdd-deps -networks: - iggy-bdd-network: - driver: bridge - ipam: - config: - - subnet: 172.28.0.0/24 - volumes: iggy_leader_data: iggy_follower_data: diff --git a/bdd/docker-compose.server.yml b/bdd/docker-compose.server.yml index e22484533c..54158aa1b1 100644 --- a/bdd/docker-compose.server.yml +++ b/bdd/docker-compose.server.yml @@ -19,10 +19,6 @@ # Activated by: ./scripts/run-bdd-tests.sh basic_messaging # ./scripts/run-bdd-tests.sh leader_redirection # ./scripts/run-bdd-tests.sh all -# -# Each run starts from a new container, so `--fresh` would be redundant; root -# credentials arrive through IGGY_ROOT_USERNAME / IGGY_ROOT_PASSWORD rather -# than through `--with-default-root-credentials`. x-server-bdd-deps: &server-bdd-deps depends_on: @@ -44,7 +40,7 @@ services: PREBUILT_IGGY_CLI: ${IGGY_CLI_PATH:-target/debug/iggy} LIBC: glibc PROFILE: debug - command: [] + command: [ "--fresh", "--with-default-root-credentials" ] cap_add: - SYS_NICE security_opt: @@ -61,8 +57,6 @@ services: start_period: 2s environment: - RUST_LOG=info - - IGGY_ROOT_USERNAME=iggy - - IGGY_ROOT_PASSWORD=iggy - IGGY_SYSTEM_PATH=local_data - IGGY_TCP_ADDRESS=0.0.0.0:8090 - IGGY_HTTP_ADDRESS=0.0.0.0:3000 diff --git a/bdd/docker-compose.vsr.yml b/bdd/docker-compose.vsr.yml new file mode 100644 index 0000000000..9aaf51d37f --- /dev/null +++ b/bdd/docker-compose.vsr.yml @@ -0,0 +1,101 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# server-ng (VSR) override. Swaps the legacy iggy-server binary for +# iggy-server-ng built with `--features vsr` and adapts args, credentials, +# and cluster topology to server-ng semantics: +# - server-ng takes no `--fresh` / `--with-default-root-credentials` / +# `--follower` flags; the only CLI arg is `--replica-id `. +# - Root credentials come from IGGY_ROOT_USERNAME / IGGY_ROOT_PASSWORD +# (mandatory when the cluster is enabled). +# - `cluster.nodes[*].ip` is parsed as a strict IpAddr (no hostnames) +# and cluster listeners bind to that roster IP, so nodes get static +# IPs and node healthchecks ping the roster address, not loopback. +# - Replica-to-replica consensus requires `ports.tcp_replica`. +# The leader_redirection feature runs a REAL 2-node VSR cluster here +# (initial leader = replica 0), not the legacy mock `--follower` server. +# +# Activated by: ./scripts/run-bdd-tests.sh --vsr [feature] +# Must be passed LAST so its overrides win over the server/cluster files. + +x-server-ng-image: &server-ng-image + image: iggy-bdd-server-ng + build: + context: .. + dockerfile: core/server-ng/Dockerfile + target: runtime-prebuilt + args: + PREBUILT_IGGY_SERVER_NG: ${IGGY_SERVER_NG_PATH:-target/debug/iggy-server-ng} + PREBUILT_IGGY_CLI: ${IGGY_CLI_PATH:-target/debug/iggy} + +x-vsr-cluster-env: &vsr-cluster-env + IGGY_ROOT_USERNAME: iggy + IGGY_ROOT_PASSWORD: iggy + IGGY_CLUSTER_NODES_0_IP: 172.28.0.101 + IGGY_CLUSTER_NODES_1_IP: 172.28.0.102 + IGGY_CLUSTER_NODES_0_PORTS_TCP_REPLICA: "8191" + IGGY_CLUSTER_NODES_1_PORTS_TCP_REPLICA: "8192" + # http.enabled with cluster.enabled requires a JWT key every node can + # verify; cluster auth provides it (derived from the shared secret). + # Enabling auth activates follower-to-primary forwarding, so the config + # validator requires ports.http on every roster node - those ports are set + # in docker-compose.cluster.yml (merged for the leader_redirection flow). + IGGY_CLUSTER_AUTH_ENABLED: "true" + IGGY_CLUSTER_AUTH_SHARED_SECRET: "bdd-vsr-cluster-shared-secret-0123456789" + +services: + iggy-server: + <<: *server-ng-image + command: [] + environment: + - IGGY_ROOT_USERNAME=iggy + - IGGY_ROOT_PASSWORD=iggy + + # The Node SDK picks the wire protocol at runtime; classic is its default, + # so the VSR lane has to opt in explicitly. + node-bdd: + environment: + - IGGY_TEST_PROTOCOL=vsr + + iggy-leader: + <<: *server-ng-image + command: [ "--replica-id", "0" ] + environment: + <<: *vsr-cluster-env + networks: + iggy-bdd-network: + ipv4_address: 172.28.0.101 + healthcheck: + test: [ "CMD", "/usr/local/bin/iggy", "--tcp-server-address", "172.28.0.101:8091", "ping" ] + + iggy-follower: + <<: *server-ng-image + command: [ "--replica-id", "1" ] + environment: + <<: *vsr-cluster-env + networks: + iggy-bdd-network: + ipv4_address: 172.28.0.102 + healthcheck: + test: [ "CMD", "/usr/local/bin/iggy", "--tcp-server-address", "172.28.0.102:8092", "ping" ] + +networks: + iggy-bdd-network: + driver: bridge + ipam: + config: + - subnet: 172.28.0.0/24 diff --git a/bdd/go/tests/tcp_test/test_helpers.go b/bdd/go/tests/tcp_test/test_helpers.go index efcdd18310..646b741da8 100644 --- a/bdd/go/tests/tcp_test/test_helpers.go +++ b/bdd/go/tests/tcp_test/test_helpers.go @@ -59,10 +59,11 @@ func createClient() iggcon.Client { return cli } -// maxRoutableId is the highest stream or topic id the server's routing -// namespace can address. A larger id cannot be routed at all, so specs that -// want an id the server has never seen must stay inside this range to get the -// server's answer rather than a routing failure. +// maxRoutableId is the highest stream or topic id the wire namespace can +// address. A larger id cannot be routed at all, so the SDK rejects it before +// it reaches the server. Specs that want an id the server has never seen must +// stay inside this range to get the server's answer rather than a local +// rejection. const maxRoutableId = 4095 func createRandomUInt32() uint32 { diff --git a/bdd/java/src/test/java/org/apache/iggy/bdd/LeaderRedirectionSteps.java b/bdd/java/src/test/java/org/apache/iggy/bdd/LeaderRedirectionSteps.java index 4f9645377a..5dc9778b7d 100644 --- a/bdd/java/src/test/java/org/apache/iggy/bdd/LeaderRedirectionSteps.java +++ b/bdd/java/src/test/java/org/apache/iggy/bdd/LeaderRedirectionSteps.java @@ -23,7 +23,6 @@ import io.cucumber.java.en.Given; import io.cucumber.java.en.Then; import io.cucumber.java.en.When; -import org.apache.iggy.client.ConnectionInfo; import org.apache.iggy.client.blocking.tcp.IggyTcpClient; import org.apache.iggy.cluster.ClusterNode; import org.apache.iggy.cluster.ClusterNodeRole; @@ -31,9 +30,6 @@ import org.apache.iggy.exception.IggyException; import org.apache.iggy.stream.StreamDetails; -import java.net.InetAddress; -import java.net.UnknownHostException; -import java.util.Arrays; import java.util.LinkedHashMap; import java.util.Map; import java.util.Optional; @@ -172,15 +168,11 @@ public void clientConnectsWithoutRedirection() { public void bothClientsUseTheSameServer() { IggyTcpClient clientA = client("A"); IggyTcpClient clientB = client("B"); - ConnectionInfo connectionA = clientA.getConnectionInfo(); - ConnectionInfo connectionB = clientB.getConnectionInfo(); - assertTrue( - isSameEndpoint(connectionA, connectionB), - () -> "Both clients should be connected to the same server, got " - + connectionA.serverAddress() - + " and " - + connectionB.serverAddress()); + assertEquals( + clientA.getConnectionInfo().serverAddress(), + clientB.getConnectionInfo().serverAddress(), + "Both clients should be connected to the same server"); clientA.system().ping(); clientB.system().ping(); @@ -250,25 +242,6 @@ private static Optional leaderFromMetadata(IggyTcpClient client) { } } - private static boolean isSameEndpoint(ConnectionInfo left, ConnectionInfo right) { - if (left.port() != right.port()) { - return false; - } - - InetAddress[] leftAddresses = resolveHost(left.host()); - InetAddress[] rightAddresses = resolveHost(right.host()); - return Arrays.stream(leftAddresses) - .anyMatch(leftAddress -> Arrays.stream(rightAddresses).anyMatch(leftAddress::equals)); - } - - private static InetAddress[] resolveHost(String host) { - try { - return InetAddress.getAllByName(host); - } catch (UnknownHostException error) { - throw new AssertionError("Failed to resolve server host " + host, error); - } - } - private static void assertAddressMatchesPort(String address, int port, String description) { assertTrue(address.endsWith(":" + port), description + " " + address + " should use port " + port); } diff --git a/bdd/python/tests/test_basic_messaging.py b/bdd/python/tests/test_basic_messaging.py index d489ef4e90..363c95c4e1 100644 --- a/bdd/python/tests/test_basic_messaging.py +++ b/bdd/python/tests/test_basic_messaging.py @@ -64,7 +64,8 @@ async def _login(): @given("I have no streams in the system") def no_streams_in_system(context): """Ensure no streams exist in the system""" - # Every run gets a new server container, so the system starts empty + # With --fresh flag on server, this should already be clean + # Just verify by attempting to get a stream that shouldn't exist pass diff --git a/bdd/rust/Cargo.toml b/bdd/rust/Cargo.toml index 0b761fb421..f63ac45647 100644 --- a/bdd/rust/Cargo.toml +++ b/bdd/rust/Cargo.toml @@ -25,6 +25,7 @@ publish = false [features] bdd = [] +vsr = ["iggy/vsr"] [dev-dependencies] bytes = { workspace = true } diff --git a/codecov.yml b/codecov.yml index 7c71b59c96..f16b838653 100644 --- a/codecov.yml +++ b/codecov.yml @@ -68,6 +68,9 @@ flag_management: - name: node-e2e paths: - foreign/node/ + - name: node-e2e-vsr + paths: + - foreign/node/ - name: go paths: - foreign/go/ diff --git a/core/ai/mcp/Cargo.toml b/core/ai/mcp/Cargo.toml index 9cc785d274..8f4e55fbea 100644 --- a/core/ai/mcp/Cargo.toml +++ b/core/ai/mcp/Cargo.toml @@ -29,6 +29,7 @@ publish = false [features] systemd = ["dep:sd-notify", "dep:tokio-util"] +vsr = ["iggy/vsr"] [dependencies] axum = { workspace = true } diff --git a/core/bench/Cargo.toml b/core/bench/Cargo.toml index 9842a4413b..faa3269a6e 100644 --- a/core/bench/Cargo.toml +++ b/core/bench/Cargo.toml @@ -31,6 +31,18 @@ publish = false name = "iggy-bench" path = "src/main.rs" +[features] +# Switches the SDK to the vsr Register-handshake framing spoken by server-ng +# clusters. The framing is chosen at compile time, so a default-features bench +# binary cannot talk to a vsr cluster at all (the first request never frames). +# TRAP: `cargo test -p integration --features vsr` does NOT rebuild this +# binary -- the harness spawns whatever the last build produced, and a +# default-featured leftover trips the bench timeout in +# `run_bench_and_wait_for_finish` (one per restart-matrix case). Build the +# workspace (or this crate with --features vsr) first; `just nextest-vsr` +# does. +vsr = ["iggy/vsr"] + [dependencies] async-trait = { workspace = true } bench-report = { workspace = true } diff --git a/core/bench/dashboard/frontend/scripts/select_index.sh b/core/bench/dashboard/frontend/scripts/select_index.sh index 087c383ca4..93f0b699ff 100755 --- a/core/bench/dashboard/frontend/scripts/select_index.sh +++ b/core/bench/dashboard/frontend/scripts/select_index.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/core/bench/src/analytics/report_builder.rs b/core/bench/src/analytics/report_builder.rs index 8e5bad92b6..35119a32e4 100644 --- a/core/bench/src/analytics/report_builder.rs +++ b/core/bench/src/analytics/report_builder.rs @@ -35,8 +35,8 @@ use iggy::prelude::{ }; use tracing::warn; -/// The server synthesizes exactly this cluster name for a non-clustered -/// instance, so it is the single-node sentinel. +/// Both the legacy server and server-ng synthesize exactly this cluster name +/// for a non-clustered instance, so it is the single-node sentinel. const SINGLE_NODE_CLUSTER_NAME: &str = "single-node"; pub struct BenchmarkReportBuilder; diff --git a/core/bench/src/main.rs b/core/bench/src/main.rs index 292356b1cf..70e06be02c 100644 --- a/core/bench/src/main.rs +++ b/core/bench/src/main.rs @@ -33,10 +33,18 @@ use tracing::{error, info}; use tracing_subscriber::{EnvFilter, fmt, layer::SubscriberExt, util::SubscriberInitExt}; use utils::cpu_name::append_cpu_name_lowercase; -/// Which SDK framing this binary speaks, printed in the always-on banner so a -/// server that answers a Register handshake with silence is diagnosed in a -/// second rather than mistaken for a hang. -const SDK_FRAMING: &str = "vsr (Register handshake)"; +/// Which SDK framing this binary was compiled with. +/// +/// One binary name, two wire dialects, and the mismatch is asymmetric: a +/// default-features bench against a vsr server HANGS rather than fails, because +/// the server reads a full 256-byte header before validating anything and the +/// client's own response timeout is itself vsr-gated. Printing the flavor in the +/// always-on banner turns that from a silent hang into a one-second diagnosis. +const SDK_FRAMING: &str = if cfg!(feature = "vsr") { + "vsr (server-ng Register handshake)" +} else { + "legacy (classic server framing)" +}; #[tokio::main] async fn main() -> Result<(), IggyError> { diff --git a/core/binary_protocol/Cargo.toml b/core/binary_protocol/Cargo.toml index 4fcdab1c5d..ec9b25e751 100644 --- a/core/binary_protocol/Cargo.toml +++ b/core/binary_protocol/Cargo.toml @@ -35,7 +35,6 @@ bytes = { workspace = true } enumset = { workspace = true } secrecy = { workspace = true } thiserror = { workspace = true } -twox-hash = { workspace = true } [dev-dependencies] aligned-vec = { workspace = true } diff --git a/core/binary_protocol/src/consensus/command.rs b/core/binary_protocol/src/consensus/command.rs index 8e60655fc1..6dcf1298ae 100644 --- a/core/binary_protocol/src/consensus/command.rs +++ b/core/binary_protocol/src/consensus/command.rs @@ -41,7 +41,7 @@ pub enum Command2 { StartView = 12, Eviction = 13, - // Replica-to-replica auth handshake (the server consensus plane). + // Replica-to-replica auth handshake (server-ng consensus plane). ReplicaHello = 14, ReplicaChallenge = 15, ReplicaFinish = 16, diff --git a/core/binary_protocol/src/consensus/error.rs b/core/binary_protocol/src/consensus/error.rs index b424660534..13af4a92e6 100644 --- a/core/binary_protocol/src/consensus/error.rs +++ b/core/binary_protocol/src/consensus/error.rs @@ -29,23 +29,6 @@ pub enum ConsensusError { #[error("invalid checksum")] InvalidChecksum, - #[error( - "{command:?}: header checksum {found:#034x} does not cover the frame (expected \ - {expected:#034x}){}", - if *found == 0 { - ". A zeroed checksum is the signature of a peer predating the frame seal, \ - which is a hard version break: replicas must be upgraded together, with the \ - cluster down" - } else { - "" - } - )] - FrameChecksumMismatch { - command: Command2, - expected: u128, - found: u128, - }, - #[error("invalid cluster ID")] InvalidCluster, diff --git a/core/binary_protocol/src/consensus/header.rs b/core/binary_protocol/src/consensus/header.rs index 4f8fffe950..a4661cd27a 100644 --- a/core/binary_protocol/src/consensus/header.rs +++ b/core/binary_protocol/src/consensus/header.rs @@ -18,23 +18,6 @@ //! All consensus headers are exactly 256 bytes with `#[repr(C)]` layout. //! Size and field offsets are enforced at compile time. Deserialization //! is a pointer cast (zero-copy) via `bytemuck::try_from_bytes`. -//! -//! # Wire compatibility -//! -//! The replica-to-replica control headers are a BREAKING, non-negotiable change -//! against any build predating [`ConsensusHeader::FRAME_SEALED`]: `checksum` went -//! from a field nobody wrote to one every receiver verifies, so each side reads the -//! other's frames as corrupt. `release` must be zero on every header, so there is no -//! version channel to gate on and no way for the two to detect each other. -//! -//! Replicas must therefore be upgraded together, with the cluster down. A rolling -//! upgrade does not degrade, it stops the cluster: every control frame between a -//! mixed pair is dropped, so no view change reaches a quorum. Nothing enforces this, -//! because there is nothing left to enforce it with; this note is the declaration. -//! -//! `Prepare`, `Request`, `Reply`, and `Eviction` are unaffected. Prepares keep -//! `checksum` as their view-independent identity, and the three client-facing -//! headers are sealed on neither side, so SDKs are untouched. use super::{Command2, ConsensusError, Operation}; use bytemuck::{CheckedBitPattern, NoUninit}; @@ -71,16 +54,6 @@ pub fn read_size_field(header: &[u8]) -> Option { .map(u32::from_le_bytes) } -/// Frame checksum over a raw header: every byte past `checksum` itself. -/// -/// Byte-level twin of [`ConsensusHeader::frame_checksum`], which delegates here so -/// the typed and raw seals cannot disagree. For callers that do not know the -/// concrete header type statically, such as a wire-level test fixture. -#[must_use] -pub fn frame_checksum_bytes(header: &[u8; HEADER_SIZE]) -> u128 { - u128::from(twox_hash::XxHash3_64::oneshot(&header[size_of::()..])) -} - /// Trait implemented by all consensus header types. /// /// Every header is exactly [`HEADER_SIZE`] bytes, `#[repr(C)]`, and supports @@ -105,86 +78,12 @@ pub trait ConsensusHeader: Sized + CheckedBitPattern + NoUninit { command == Self::COMMAND } - /// Whether this header's `checksum` field seals the frame. - /// - /// True for replica-to-replica control frames, whose header carries every - /// decision field: view number, commit point, and the nack bitset that - /// authorises truncation. TCP's 16-bit checksum does not reliably catch a - /// flipped bit on a plaintext replica link. - /// - /// False for three groups: [`PrepareHeader`] / [`RepairPrepareHeader`] spend - /// `checksum` on [`PrepareHeader::identity_checksum`], which excludes `view` so - /// a re-stamped prepare keeps one identity, and a seal cannot share the field; - /// [`RequestHeader`] / [`ReplyHeader`] / [`EvictionHeader`] cross the client - /// boundary, so sealing them is an SDK change on both ends; [`GenericHeader`] is - /// the type-erased pre-dispatch view and defers to the typed parse, where - /// [`Self::verify_frame`] runs. - /// - /// Required, not defaulted: [`Self::seal`] on an unsealed type overwrites the - /// identity checksum with a frame checksum, and in release only a `debug_assert` - /// stands in the way. - const FRAME_SEALED: bool; - /// # Errors /// Returns `ConsensusError` if the header fields are inconsistent. fn validate(&self) -> Result<(), ConsensusError>; fn operation(&self) -> Operation; fn command(&self) -> Command2; fn size(&self) -> u32; - - /// The `checksum` field, whatever this header spends it on. - fn checksum(&self) -> u128; - - /// Overwrite the `checksum` field. - fn set_checksum(&mut self, checksum: u128); - - /// Checksum over every byte of the header past `checksum` itself. - /// - /// `checksum_body` sits inside that range, so sealing the header also - /// pins the body seal, and the two together cover the whole frame. - #[must_use] - fn frame_checksum(&self) -> u128 { - let bytes: &[u8; HEADER_SIZE] = bytemuck::bytes_of(self) - .try_into() - .expect("every consensus header is HEADER_SIZE bytes"); - frame_checksum_bytes(bytes) - } - - /// Stamp [`Self::frame_checksum`]. Call last when building a frame: it covers - /// every other field, `checksum_body` included, so later writes are uncovered. - fn seal(&mut self) { - debug_assert!( - Self::FRAME_SEALED, - "sealing a header whose checksum field means something else", - ); - let checksum = self.frame_checksum(); - self.set_checksum(checksum); - } - - /// Reject a frame whose header does not match its own checksum. - /// - /// Runs before [`Self::validate`] on every typed parse: a header that did not - /// arrive intact cannot have any field believed, `validate`'s included. - /// - /// # Errors - /// [`ConsensusError::FrameChecksumMismatch`] on a bad seal. Unsealed header - /// types return `Ok` unconditionally. - fn verify_frame(&self) -> Result<(), ConsensusError> { - if !Self::FRAME_SEALED { - return Ok(()); - } - let expected = self.frame_checksum(); - let found = self.checksum(); - if found == expected { - Ok(()) - } else { - Err(ConsensusError::FrameChecksumMismatch { - command: self.command(), - expected, - found, - }) - } - } } // GenericHeader - type-erased dispatch @@ -222,15 +121,6 @@ const _: () = { impl ConsensusHeader for GenericHeader { const COMMAND: Command2 = Command2::Reserved; - const FRAME_SEALED: bool = false; - - fn checksum(&self) -> u128 { - self.checksum - } - - fn set_checksum(&mut self, checksum: u128) { - self.checksum = checksum; - } fn operation(&self) -> Operation { Operation::Reserved } @@ -272,6 +162,7 @@ pub struct RequestHeader { pub request: u64, pub operation: Operation, pub operation_padding: [u8; 7], + pub namespace: u64, /// Session fence epoch: the commit op of the latest committed `Register` /// for this `client`. Handed to the client by that register's reply and /// echoed on every subsequent request. @@ -292,7 +183,7 @@ pub struct RequestHeader { /// The submitter's wire value is never trusted. Zero for `Logout`, /// partition-plane, and server-internal ops. pub user_id: u32, - pub reserved: [u8; 60], + pub reserved: [u8; 52], } const _: () = { assert!(size_of::() == HEADER_SIZE); @@ -303,88 +194,9 @@ const _: () = { assert!( offset_of!(RequestHeader, user_id) == offset_of!(RequestHeader, session) + size_of::() ); - assert!(offset_of!(RequestHeader, reserved) + size_of::<[u8; 60]>() == HEADER_SIZE); + assert!(offset_of!(RequestHeader, reserved) + size_of::<[u8; 52]>() == HEADER_SIZE); }; -/// A [`RequestHeader`] AFTER the receiving node resolved its target. -/// -/// The server-internal shape a client request travels in between shards -/// (and follower to primary), never on the client wire. `group` carries the -/// resolved consensus group so the owner shard can route, park, and fence -/// WITHOUT re-decoding the payload -- clients no longer send any namespace -/// (it is derived: plane from `operation`, partition group from the body), -/// so this is where the derivation result lives for the internal hop. -/// -/// Layout: identical to [`RequestHeader`] with `group` claiming the LAST -/// eight reserved bytes (the tail, bytes 248..256); the leading 52 reserved -/// bytes keep their client-wire meaning, so promotion is a same-size copy. -#[repr(C)] -#[derive(Debug, Clone, Copy, CheckedBitPattern, NoUninit)] -pub struct RoutedRequestHeader { - pub checksum: u128, - pub checksum_body: u128, - pub cluster: u128, - pub size: u32, - pub view: u32, - pub release: u32, - pub command: Command2, - pub replica: u8, - pub reserved_frame: [u8; 66], - - pub client: u128, - pub request_checksum: u128, - pub timestamp: u64, - pub request: u64, - pub operation: Operation, - pub operation_padding: [u8; 7], - pub session: u64, - pub user_id: u32, - /// Same offset and meaning as the leading 52 bytes of - /// `RequestHeader::reserved` -- this region CARRIES DATA (the - /// non-replicated op code range), so `group` must not displace it. - pub reserved: [u8; 52], - /// The resolved consensus group id (see `binary_protocol::namespace`), - /// claiming the TAIL of the client header's reserved area. - pub group: u64, -} -const _: () = { - assert!(size_of::() == HEADER_SIZE); - // Every field shared with `RequestHeader` sits at the same offset -- - // including the data-bearing prefix of `reserved` (the non-replicated - // code range) -- so promotion preserves everything a client sent. - assert!(offset_of!(RoutedRequestHeader, client) == offset_of!(RequestHeader, client)); - assert!(offset_of!(RoutedRequestHeader, session) == offset_of!(RequestHeader, session)); - assert!(offset_of!(RoutedRequestHeader, user_id) == offset_of!(RequestHeader, user_id)); - assert!(offset_of!(RoutedRequestHeader, reserved) == offset_of!(RequestHeader, reserved)); - assert!(offset_of!(RoutedRequestHeader, group) + size_of::() == HEADER_SIZE); -}; - -impl Default for RoutedRequestHeader { - fn default() -> Self { - Self { - checksum: 0, - checksum_body: 0, - cluster: 0, - size: 0, - view: 0, - release: 0, - command: Command2::Reserved, - replica: 0, - reserved_frame: [0; 66], - client: 0, - request_checksum: 0, - timestamp: 0, - request: 0, - operation: Operation::Reserved, - operation_padding: [0; 7], - session: 0, - user_id: 0, - reserved: [0; 52], - group: 0, - } - } -} - impl Default for RequestHeader { fn default() -> Self { Self { @@ -403,118 +215,16 @@ impl Default for RequestHeader { request: 0, operation: Operation::Reserved, operation_padding: [0; 7], + namespace: 0, session: 0, user_id: 0, - reserved: [0; 60], - } - } -} - -/// Field rules shared by the client-wire [`RequestHeader`] and the -/// server-internal [`RoutedRequestHeader`]. The routed shape is decoded -/// straight off the peer wire (`MessageBag`), so it must reject everything -/// the client boundary rejects; validating only the command byte there -/// would let a peer frame carry `client = 0` into the client table's hard -/// assert, or `operation = Reserved` into a cached-register replay. -fn validate_request_fields( - client: u128, - operation: Operation, - session: u64, - request: u64, -) -> Result<(), ConsensusError> { - if client == 0 { - return Err(ConsensusError::InvalidField( - "request: client must be != 0".to_string(), - )); - } - // Reserved is the zero value, never a real client op - // (`is_client_allowed` rejects it). Refusing it here rather than after - // the dedup preflight matters: a bound client sending - // `Reserved, request = 0` used to pass validation, reach - // `request_preflight`, hit its own watermark and get its register - // reply replayed before the operation gate ever ran. - if operation == Operation::Reserved { - return Err(ConsensusError::InvalidField( - "operation must not be Reserved".to_string(), - )); - } - // Register: session must be 0, request must be 0. - // NonReplicated: sessionless by design (the `ClientTable` ignores - // these ops and the server routes/auth-gates them by transport id), - // so a pre-register client may legitimately send session 0 -- - // ping must work before authentication. - // Other non-register ops: session must be > 0, request must be > 0. - if operation == Operation::Register { - if session != 0 { - return Err(ConsensusError::InvalidField( - "register: session must be 0".to_string(), - )); - } - if request != 0 { - return Err(ConsensusError::InvalidField( - "register: request must be 0".to_string(), - )); - } - } else if operation != Operation::NonReplicated { - if session == 0 { - return Err(ConsensusError::InvalidField( - "non-register: session must be > 0".to_string(), - )); - } - if request == 0 { - return Err(ConsensusError::InvalidField( - "non-register: request must be > 0".to_string(), - )); - } - } - Ok(()) -} - -impl ConsensusHeader for RoutedRequestHeader { - const COMMAND: Command2 = Command2::Request; - /// The client-wire [`RequestHeader`] this is promoted from is unsealed, and the - /// promotion copies `checksum` verbatim, so there is nothing here to verify. - const FRAME_SEALED: bool = false; - - fn checksum(&self) -> u128 { - self.checksum - } - - fn set_checksum(&mut self, checksum: u128) { - self.checksum = checksum; - } - fn operation(&self) -> Operation { - self.operation - } - fn command(&self) -> Command2 { - self.command - } - fn size(&self) -> u32 { - self.size - } - - fn validate(&self) -> Result<(), ConsensusError> { - if self.command != Command2::Request { - return Err(ConsensusError::InvalidCommand { - expected: Command2::Request, - found: self.command, - }); + reserved: [0; 52], } - validate_request_fields(self.client, self.operation, self.session, self.request) } } impl ConsensusHeader for RequestHeader { const COMMAND: Command2 = Command2::Request; - const FRAME_SEALED: bool = false; - - fn checksum(&self) -> u128 { - self.checksum - } - - fn set_checksum(&mut self, checksum: u128) { - self.checksum = checksum; - } fn operation(&self) -> Operation { self.operation } @@ -532,7 +242,52 @@ impl ConsensusHeader for RequestHeader { found: self.command, }); } - validate_request_fields(self.client, self.operation, self.session, self.request) + if self.client == 0 { + return Err(ConsensusError::InvalidField( + "request: client must be != 0".to_string(), + )); + } + // Reserved is the zero value, never a real client op + // (`is_client_allowed` rejects it). Refusing it here rather than after + // the dedup preflight matters: a bound client sending + // `Reserved, request = 0` used to pass validation, reach + // `request_preflight`, hit its own watermark and get its register + // reply replayed before the operation gate ever ran. + if self.operation == Operation::Reserved { + return Err(ConsensusError::InvalidField( + "operation must not be Reserved".to_string(), + )); + } + // Register: session must be 0, request must be 0. + // NonReplicated: sessionless by design (the `ClientTable` ignores + // these ops and the server routes/auth-gates them by transport id), + // so a pre-register client may legitimately send session 0 -- + // ping must work before authentication. + // Other non-register ops: session must be > 0, request must be > 0. + if self.operation == Operation::Register { + if self.session != 0 { + return Err(ConsensusError::InvalidField( + "register: session must be 0".to_string(), + )); + } + if self.request != 0 { + return Err(ConsensusError::InvalidField( + "register: request must be 0".to_string(), + )); + } + } else if self.operation != Operation::NonReplicated { + if self.session == 0 { + return Err(ConsensusError::InvalidField( + "non-register: session must be > 0".to_string(), + )); + } + if self.request == 0 { + return Err(ConsensusError::InvalidField( + "non-register: request must be > 0".to_string(), + )); + } + } + Ok(()) } } @@ -563,6 +318,7 @@ pub struct ReplyHeader { pub request: u64, pub operation: Operation, pub operation_padding: [u8; 7], + pub namespace: u64, /// Request-level status: 0 = ok; nonzero = the `IggyError` code for a /// failure decided before commit (e.g. a dispatch-time authorization /// denial, or the partition primary rejecting a consumer-offset op). @@ -576,7 +332,7 @@ pub struct ReplyHeader { /// `user_id` in `RequestHeader` / `PrepareHeader`; no existing field offset /// moves and `validate` does not inspect it. pub status: u32, - pub reserved: [u8; 36], + pub reserved: [u8; 28], } const _: () = { assert!(size_of::() == HEADER_SIZE); @@ -584,7 +340,10 @@ const _: () = { offset_of!(ReplyHeader, request_checksum) == offset_of!(ReplyHeader, reserved_frame) + size_of::<[u8; 66]>() ); - assert!(offset_of!(ReplyHeader, reserved) + size_of::<[u8; 36]>() == HEADER_SIZE); + assert!( + offset_of!(ReplyHeader, status) == offset_of!(ReplyHeader, namespace) + size_of::() + ); + assert!(offset_of!(ReplyHeader, reserved) + size_of::<[u8; 28]>() == HEADER_SIZE); }; impl Default for ReplyHeader { @@ -608,23 +367,15 @@ impl Default for ReplyHeader { request: 0, operation: Operation::Reserved, operation_padding: [0; 7], + namespace: 0, status: 0, - reserved: [0; 36], + reserved: [0; 28], } } } impl ConsensusHeader for ReplyHeader { const COMMAND: Command2 = Command2::Reply; - const FRAME_SEALED: bool = false; - - fn checksum(&self) -> u128 { - self.checksum - } - - fn set_checksum(&mut self, checksum: u128) { - self.checksum = checksum; - } fn operation(&self) -> Operation { self.operation } @@ -807,15 +558,6 @@ impl EvictionHeader { impl ConsensusHeader for EvictionHeader { const COMMAND: Command2 = Command2::Eviction; - const FRAME_SEALED: bool = false; - - fn checksum(&self) -> u128 { - self.checksum - } - - fn set_checksum(&mut self, checksum: u128) { - self.checksum = checksum; - } /// Session-level (not per-op): always `Reserved`. fn operation(&self) -> Operation { Operation::Reserved @@ -891,7 +633,7 @@ impl ConsensusHeader for EvictionHeader { /// Primary -> replicas: replicate this operation. #[repr(C)] -#[derive(Debug, Clone, Copy, PartialEq, Eq, CheckedBitPattern, NoUninit)] +#[derive(Debug, Clone, Copy, CheckedBitPattern, NoUninit)] pub struct PrepareHeader { pub checksum: u128, pub checksum_body: u128, @@ -913,12 +655,7 @@ pub struct PrepareHeader { pub request: u64, pub operation: Operation, pub operation_padding: [u8; 7], - /// Consensus group id: which of the node's multiplexed VSR groups this - /// frame belongs to. `METADATA_GROUP` (top bit) for the metadata plane, - /// otherwise the partition's packed stream-topic-partition key. The - /// demux and repair-replay routing key; see `binary_protocol::namespace` - /// for the value-space contract. - pub group: u64, + pub namespace: u64, /// Acting user id, copied verbatim from the admitted `RequestHeader`; see /// that field for the stamping contract. pub user_id: u32, @@ -931,7 +668,8 @@ const _: () = { == offset_of!(PrepareHeader, reserved_frame) + size_of::<[u8; 66]>() ); assert!( - offset_of!(PrepareHeader, user_id) == offset_of!(PrepareHeader, group) + size_of::() + offset_of!(PrepareHeader, user_id) + == offset_of!(PrepareHeader, namespace) + size_of::() ); assert!(offset_of!(PrepareHeader, reserved) + size_of::<[u8; 28]>() == HEADER_SIZE); }; @@ -957,7 +695,7 @@ impl Default for PrepareHeader { request: 0, operation: Operation::Reserved, operation_padding: [0; 7], - group: 0, + namespace: 0, user_id: 0, reserved: [0; 28], } @@ -966,15 +704,6 @@ impl Default for PrepareHeader { impl ConsensusHeader for PrepareHeader { const COMMAND: Command2 = Command2::Prepare; - const FRAME_SEALED: bool = false; - - fn checksum(&self) -> u128 { - self.checksum - } - - fn set_checksum(&mut self, checksum: u128) { - self.checksum = checksum; - } fn operation(&self) -> Operation { self.operation } @@ -992,63 +721,10 @@ impl ConsensusHeader for PrepareHeader { found: self.command, }); } - // Both reserved regions must be zero. They sit inside - // [`Self::identity_checksum`], so a peer that fills them changes the op's - // identity while changing nothing the merge can see; and `dvc_blank` - // classifies a slot by exact struct equality, so a non-zero reserved byte - // turns a blank into a `Valid` header the merge then indexes. - if self.reserved_frame.iter().any(|&byte| byte != 0) { - return Err(ConsensusError::InvalidField( - "prepare: reserved_frame bytes must be zero".to_string(), - )); - } - if self.reserved.iter().any(|&byte| byte != 0) { - return Err(ConsensusError::InvalidField( - "prepare: reserved bytes must be zero".to_string(), - )); - } Ok(()) } } -/// `checksum` of a prepare no producer sealed. -/// -/// Written by a build predating the identity seal, or by the partition plane. -/// Verification skips such entries so an older build's WAL still replays. -pub const CHECKSUM_UNSEALED: u128 = 0; - -/// The frame's body, bounded by `size`. What `checksum_body` covers. -/// -/// Not `&frame[HEADER_SIZE..]`: `Message::try_from` accepts a buffer longer than -/// `size` without trimming, while the WAL scan reads exactly `size`, so slicing to -/// the end makes the two disagree. Empty when `size` overruns the buffer. -#[must_use] -pub fn frame_body(frame: &[u8], size: u32) -> &[u8] { - let end = size as usize; - if end <= HEADER_SIZE || end > frame.len() { - return &[]; - } - &frame[HEADER_SIZE..end] -} - -impl PrepareHeader { - /// Which prepare this is, independent of which view re-sent it. - /// - /// Covers the whole 256-byte header except `checksum` (a field cannot hash - /// itself) and `view`, so a retransmission that re-stamps `view` stays valid. - /// The body reaches the value through the covered `checksum_body`. - /// - /// Lives here, not in the consensus crate, because the WAL scan verifies it too - /// and the two must agree byte for byte: it hashes this struct's layout. - #[must_use] - pub fn identity_checksum(&self) -> u128 { - let mut covered = *self; - covered.checksum = 0; - covered.view = 0; - u128::from(twox_hash::XxHash3_64::oneshot(bytemuck::bytes_of(&covered))) - } -} - // RepairPrepareHeader - repair peer -> recovering replica (journal repair) /// A stored prepare served for journal repair. @@ -1064,15 +740,6 @@ pub struct RepairPrepareHeader(pub PrepareHeader); impl ConsensusHeader for RepairPrepareHeader { const COMMAND: Command2 = Command2::RepairPrepare; - const FRAME_SEALED: bool = false; - - fn checksum(&self) -> u128 { - self.0.checksum - } - - fn set_checksum(&mut self, checksum: u128) { - self.0.checksum = checksum; - } fn operation(&self) -> Operation { self.0.operation } @@ -1090,21 +757,6 @@ impl ConsensusHeader for RepairPrepareHeader { found: self.0.command, }); } - // Same rule as `PrepareHeader::validate`, same reason: the regions sit inside - // `identity_checksum`, and a repaired prepare is journaled and later re-read - // as a DVC suffix entry, where `dvc_blank`'s exact-equality classification is - // what a dirty byte defeats. Not delegated, so the command check above stays - // `RepairPrepare`. - if self.0.reserved_frame.iter().any(|&byte| byte != 0) { - return Err(ConsensusError::InvalidField( - "repair_prepare: reserved_frame bytes must be zero".to_string(), - )); - } - if self.0.reserved.iter().any(|&byte| byte != 0) { - return Err(ConsensusError::InvalidField( - "repair_prepare: reserved bytes must be zero".to_string(), - )); - } Ok(()) } } @@ -1133,7 +785,7 @@ pub struct PrepareOkHeader { pub request: u64, pub operation: Operation, pub operation_padding: [u8; 7], - pub group: u64, + pub namespace: u64, pub reserved: [u8; 48], } const _: () = { @@ -1165,24 +817,14 @@ impl Default for PrepareOkHeader { request: 0, operation: Operation::Reserved, operation_padding: [0; 7], - group: 0, + namespace: 0, reserved: [0; 48], } } } impl ConsensusHeader for PrepareOkHeader { - const FRAME_SEALED: bool = true; - const COMMAND: Command2 = Command2::PrepareOk; - - fn checksum(&self) -> u128 { - self.checksum - } - - fn set_checksum(&mut self, checksum: u128) { - self.checksum = checksum; - } fn operation(&self) -> Operation { self.operation } @@ -1224,7 +866,7 @@ pub struct CommitHeader { pub timestamp_monotonic: u64, pub commit: u64, pub checkpoint_op: u64, - pub group: u64, + pub namespace: u64, pub reserved: [u8; 80], } const _: () = { @@ -1237,17 +879,7 @@ const _: () = { }; impl ConsensusHeader for CommitHeader { - const FRAME_SEALED: bool = true; - const COMMAND: Command2 = Command2::Commit; - - fn checksum(&self) -> u128 { - self.checksum - } - - fn set_checksum(&mut self, checksum: u128) { - self.checksum = checksum; - } fn operation(&self) -> Operation { Operation::Reserved } @@ -1285,30 +917,20 @@ pub struct StartViewChangeHeader { pub replica: u8, pub reserved_frame: [u8; 66], - pub group: u64, + pub namespace: u64, pub reserved: [u8; 120], } const _: () = { assert!(size_of::() == HEADER_SIZE); assert!( - offset_of!(StartViewChangeHeader, group) + offset_of!(StartViewChangeHeader, namespace) == offset_of!(StartViewChangeHeader, reserved_frame) + size_of::<[u8; 66]>() ); assert!(offset_of!(StartViewChangeHeader, reserved) + size_of::<[u8; 120]>() == HEADER_SIZE); }; impl ConsensusHeader for StartViewChangeHeader { - const FRAME_SEALED: bool = true; - const COMMAND: Command2 = Command2::StartViewChange; - - fn checksum(&self) -> u128 { - self.checksum - } - - fn set_checksum(&mut self, checksum: u128) { - self.checksum = checksum; - } fn operation(&self) -> Operation { Operation::Reserved } @@ -1353,31 +975,10 @@ pub struct DoViewChangeHeader { pub op: u64, /// Highest committed op. pub commit: u64, - pub group: u64, + pub namespace: u64, /// View when status was last normal (key for log selection). pub log_view: u32, - pub reserved: [u8; 68], - /// Bit `i` set means the sender proves it never prepared suffix entry `i`, so - /// that entry never reached a replication quorum through this replica. A new - /// primary may truncate only once `quorum_nack_prepare` senders nack an entry; - /// short of that it might be committed and must be preserved. - /// - /// A corrupt local entry is deliberately NOT nacked: the sender cannot tell a - /// prepare it never saw from one it saw and lost, and only the former is proof. - /// Silence costs availability; a false nack costs data. - /// - /// Carved from the tail of the former `reserved` region, with `present_bitset` - /// LAST so both land 16-aligned with no padding and `op`/`commit`/`group`/ - /// `log_view` keep their offsets. A sender with nothing to nack sends zeros, - /// decoding as "nacks nothing": safe, since that can only slow a view change. - pub nack_bitset: u128, - /// Bit `i` set means the sender can serve the BODY of suffix entry `i`, not just - /// its header. The new primary needs one such sender per surviving entry, since - /// a header whose body it cannot fetch is an entry it can never commit. - /// - /// Zero from a sender offering nothing, reading as "offers no bodies": safe, - /// since the new primary waits rather than adopting an entry it cannot complete. - pub present_bitset: u128, + pub reserved: [u8; 100], } const _: () = { assert!(size_of::() == HEADER_SIZE); @@ -1385,38 +986,11 @@ const _: () = { offset_of!(DoViewChangeHeader, op) == offset_of!(DoViewChangeHeader, reserved_frame) + size_of::<[u8; 66]>() ); - // op/commit/group/log_view keep their pre-bitset offsets. - assert!(offset_of!(DoViewChangeHeader, reserved) == 156); - // Both bitsets are last and 16-aligned, so the struct has no padding - // (`NoUninit` would reject any). - assert!(offset_of!(DoViewChangeHeader, nack_bitset) % 16 == 0); - assert!(offset_of!(DoViewChangeHeader, present_bitset) % 16 == 0); - assert!( - offset_of!(DoViewChangeHeader, nack_bitset) - == offset_of!(DoViewChangeHeader, reserved) + size_of::<[u8; 68]>() - ); - assert!(offset_of!(DoViewChangeHeader, present_bitset) + size_of::() == HEADER_SIZE); + assert!(offset_of!(DoViewChangeHeader, reserved) + size_of::<[u8; 100]>() == HEADER_SIZE); }; -/// Suffix headers a `DoViewChange` may carry: one bit per entry in each of the two -/// `u128` bitsets. -/// -/// Mirrors `consensus::DVC_HEADERS_MAX` as a literal so this crate need not depend -/// on the consensus crate, as with `REPLICAS_MAX` in [`EvictionHeader::new`]. -pub const DVC_HEADERS_MAX: usize = 128; - impl ConsensusHeader for DoViewChangeHeader { - const FRAME_SEALED: bool = true; - const COMMAND: Command2 = Command2::DoViewChange; - - fn checksum(&self) -> u128 { - self.checksum - } - - fn set_checksum(&mut self, checksum: u128) { - self.checksum = checksum; - } fn operation(&self) -> Operation { Operation::Reserved } @@ -1449,64 +1023,10 @@ impl ConsensusHeader for DoViewChangeHeader { "commit cannot exceed op".to_string(), )); } - let suffix_len = self.suffix_len()?; - // Bits past the suffix describe entries never sent: unchecked, a peer could - // smuggle a nack for an op the new primary would then truncate. - if suffix_len < DVC_HEADERS_MAX { - let beyond = !((1u128 << suffix_len) - 1); - if self.nack_bitset & beyond != 0 || self.present_bitset & beyond != 0 { - return Err(ConsensusError::InvalidField(format!( - "do_view_change: bitset bits set past the {suffix_len}-entry suffix" - ))); - } - } Ok(()) } } -impl DoViewChangeHeader { - /// Number of `PrepareHeader`s in the body. - /// - /// Zero is valid and means "no suffix": a replica with nothing uncommitted - /// contributes numbers only. - /// - /// # Errors - /// [`ConsensusError::InvalidField`] when `size` is short of the header, is not a - /// whole number of headers, or exceeds what the bitsets can address. - pub fn suffix_len(&self) -> Result { - suffix_len_of("do_view_change", self.size) - } -} - -/// Body length of a suffix-carrying control frame, in whole [`PrepareHeader`]s. -/// -/// Shared by `DoViewChange` and `StartView`: same layout, same `DVC_HEADERS_MAX` -/// bound. `frame` only names the sender in the error text. -/// -/// # Errors -/// [`ConsensusError::InvalidField`] when `size` is short of the header, is not a -/// whole number of headers, or exceeds what a view change can address. -fn suffix_len_of(frame: &str, size: u32) -> Result { - let size = size as usize; - let Some(body_len) = size.checked_sub(HEADER_SIZE) else { - return Err(ConsensusError::InvalidField(format!( - "{frame}: size {size} is shorter than the {HEADER_SIZE}-byte header" - ))); - }; - if body_len % HEADER_SIZE != 0 { - return Err(ConsensusError::InvalidField(format!( - "{frame}: body of {body_len} bytes is not a whole number of headers" - ))); - } - let suffix_len = body_len / HEADER_SIZE; - if suffix_len > DVC_HEADERS_MAX { - return Err(ConsensusError::InvalidField(format!( - "{frame}: {suffix_len} suffix entries exceeds the maximum {DVC_HEADERS_MAX}" - ))); - } - Ok(suffix_len) -} - // StartViewHeader - new view announcement (header-only) /// New primary -> all replicas: start new view. Header-only. @@ -1527,7 +1047,7 @@ pub struct StartViewHeader { pub op: u64, /// max(commit) from all DVCs. pub commit: u64, - pub group: u64, + pub namespace: u64, pub reserved: [u8; 88], /// Sender's incarnation, echoed from the `RequestStartView` this answers so a /// recovering replica can prove the reply post-dates its restart (see @@ -1535,15 +1055,16 @@ pub struct StartViewHeader { /// (a normal view-change completion), which carries no freshness claim. /// /// Carved from the tail of the former `reserved` region and placed LAST so it - /// lands 16-aligned with no padding WITHOUT moving `op`/`commit`/`group`. - /// Zero is "no claim", which is what `handle_start_view` keys on and what the - /// unsolicited completion path sends. NOT mixed-version tolerance: the frame seal - /// drops a pre-seal peer before any field is read (see this module's header). + /// lands 16-aligned with no padding WITHOUT moving `op`/`commit`/`namespace`. + /// A peer that predates it sends zeros, decoding as `incarnation == 0`, which + /// the `handle_start_view` guard treats as no claim rather than as a foreign + /// one, so a mixed-version rolling upgrade is wire-compatible: the pre-upgrade + /// peer's `StartView` is judged by the view checks alone, as before the field. pub incarnation: u128, } const _: () = { assert!(size_of::() == HEADER_SIZE); - // op/commit/group keep their pre-incarnation offsets. + // op/commit/namespace keep their pre-incarnation offsets. assert!( offset_of!(StartViewHeader, op) == offset_of!(StartViewHeader, reserved_frame) + size_of::<[u8; 66]>() @@ -1554,17 +1075,7 @@ const _: () = { }; impl ConsensusHeader for StartViewHeader { - const FRAME_SEALED: bool = true; - const COMMAND: Command2 = Command2::StartView; - - fn checksum(&self) -> u128 { - self.checksum - } - - fn set_checksum(&mut self, checksum: u128) { - self.checksum = checksum; - } fn operation(&self) -> Operation { Operation::Reserved } @@ -1592,26 +1103,10 @@ impl ConsensusHeader for StartViewHeader { "commit cannot exceed op".to_string(), )); } - self.suffix_len()?; Ok(()) } } -impl StartViewHeader { - /// Number of `PrepareHeader`s in the body: the view's suffix, high-to-low op - /// from `op` down toward `commit`. - /// - /// Zero means numbers only, which is what the probe-answer path sends. A backup - /// then falls back to trusting `op`. - /// - /// # Errors - /// [`ConsensusError::InvalidField`] when `size` is short of the header, is not a - /// whole number of headers, or exceeds what a view change can address. - pub fn suffix_len(&self) -> Result { - suffix_len_of("start_view", self.size) - } -} - // RequestStartViewHeader - restarted replica asking for the current view /// Recovering replica -> all replicas: resend me the current `StartView`. @@ -1635,22 +1130,22 @@ pub struct RequestStartViewHeader { pub replica: u8, pub reserved_frame: [u8; 66], - pub group: u64, + pub namespace: u64, pub reserved: [u8; 104], /// The requester's per-boot incarnation, echoed back in the answering /// `StartView` so a reply from a previous incarnation is detectable. /// /// Carved from the tail of the former `reserved` region and placed LAST so it - /// lands 16-aligned with no padding WITHOUT moving `group`. Zero is "no claim - /// to echo"; see [`StartViewHeader::incarnation`] on why that is not - /// mixed-version tolerance. + /// lands 16-aligned with no padding WITHOUT moving `namespace`. A peer that + /// predates it sends zeros, decoding as `incarnation == 0`, so a mixed-version + /// rolling upgrade is wire-compatible. pub incarnation: u128, } const _: () = { assert!(size_of::() == HEADER_SIZE); - // group keeps its pre-incarnation offset. + // namespace keeps its pre-incarnation offset. assert!( - offset_of!(RequestStartViewHeader, group) + offset_of!(RequestStartViewHeader, namespace) == offset_of!(RequestStartViewHeader, reserved_frame) + size_of::<[u8; 66]>() ); // `incarnation` is last and 16-aligned, so the struct has no padding. @@ -1659,17 +1154,7 @@ const _: () = { }; impl ConsensusHeader for RequestStartViewHeader { - const FRAME_SEALED: bool = true; - const COMMAND: Command2 = Command2::RequestStartView; - - fn checksum(&self) -> u128 { - self.checksum - } - - fn set_checksum(&mut self, checksum: u128) { - self.checksum = checksum; - } fn operation(&self) -> Operation { Operation::Reserved } @@ -1701,7 +1186,7 @@ impl ConsensusHeader for RequestStartViewHeader { /// Recovering/holed replica -> a Normal peer: request a repair stream. /// /// Sent to the primary first. Asks for the journaled prepares in -/// `[from_op, to_op]` for `group`. Header-only. The peer answers with +/// `[from_op, to_op]` for `namespace`. Header-only. The peer answers with /// `RepairPrepare` frames in op order, terminated by `RepairDone` or /// `RangeEvicted`. #[derive(Debug, Clone, Copy, PartialEq, Eq, CheckedBitPattern, NoUninit)] @@ -1720,7 +1205,7 @@ pub struct RequestPreparesHeader { pub nonce: u128, pub from_op: u64, pub to_op: u64, - pub group: u64, + pub namespace: u64, pub reserved: [u8; 88], } const _: () = { @@ -1733,17 +1218,7 @@ const _: () = { }; impl ConsensusHeader for RequestPreparesHeader { - const FRAME_SEALED: bool = true; - const COMMAND: Command2 = Command2::RequestPrepares; - - fn checksum(&self) -> u128 { - self.checksum - } - - fn set_checksum(&mut self, checksum: u128) { - self.checksum = checksum; - } fn operation(&self) -> Operation { Operation::Reserved } @@ -1793,7 +1268,7 @@ pub struct RepairRangeReplyHeader { pub nonce: u128, /// `RepairDone`: last op served. `RangeEvicted`: oldest retained op. pub op: u64, - pub group: u64, + pub namespace: u64, pub reserved: [u8; 96], } const _: () = { @@ -1806,17 +1281,7 @@ const _: () = { }; impl ConsensusHeader for RepairRangeReplyHeader { - const FRAME_SEALED: bool = true; - const COMMAND: Command2 = Command2::RepairDone; - - fn checksum(&self) -> u128 { - self.checksum - } - - fn set_checksum(&mut self, checksum: u128) { - self.checksum = checksum; - } // One layout, two commands: `RepairDone` terminates a stream, // `RangeEvicted` prefixes it. Without this widening, `try_into_typed` // rejects `RangeEvicted` frames before `validate` ever sees them. @@ -1873,7 +1338,7 @@ pub struct RequestStateTransferHeader { pub reserved_frame: [u8; 66], pub nonce: u128, - pub group: u64, + pub namespace: u64, pub reserved: [u8; 104], } const _: () = { @@ -1888,17 +1353,7 @@ const _: () = { }; impl ConsensusHeader for RequestStateTransferHeader { - const FRAME_SEALED: bool = true; - const COMMAND: Command2 = Command2::RequestStateTransfer; - - fn checksum(&self) -> u128 { - self.checksum - } - - fn set_checksum(&mut self, checksum: u128) { - self.checksum = checksum; - } fn operation(&self) -> Operation { Operation::Reserved } @@ -1958,7 +1413,7 @@ pub struct StateTransferTargetHeader { /// Serving primary's applied frontier (`commit_min`) when the descriptor /// was built. The receiver's tail repair targets past this. pub commit_op: u64, - pub group: u64, + pub namespace: u64, pub available: u8, /// Set on an `available == 0` refusal that means "not right now" rather than /// "this node is broken". @@ -2004,7 +1459,7 @@ const _: () = { // The pre-existing published offsets. New fields grow into the reserved // tail only; a change that moves one of these is a wire break. assert!(offset_of!(StateTransferTargetHeader, commit_op) == 144); - assert!(offset_of!(StateTransferTargetHeader, group) == 152); + assert!(offset_of!(StateTransferTargetHeader, namespace) == 152); assert!(offset_of!(StateTransferTargetHeader, available) == 160); assert!(offset_of!(StateTransferTargetHeader, unavailable_transient) == 161); assert!(offset_of!(StateTransferTargetHeader, commit_max) == 168); @@ -2012,17 +1467,7 @@ const _: () = { }; impl ConsensusHeader for StateTransferTargetHeader { - const FRAME_SEALED: bool = true; - const COMMAND: Command2 = Command2::StateTransferTarget; - - fn checksum(&self) -> u128 { - self.checksum - } - - fn set_checksum(&mut self, checksum: u128) { - self.checksum = checksum; - } fn operation(&self) -> Operation { Operation::Reserved } @@ -2096,7 +1541,7 @@ pub struct RequestStateChunkHeader { pub nonce: u128, pub offset: u64, - pub group: u64, + pub namespace: u64, pub len: u32, /// Index into the offered state manifest. Range-checked by the serving /// handler against the cached offer (the header cannot know the count). @@ -2113,17 +1558,7 @@ const _: () = { }; impl ConsensusHeader for RequestStateChunkHeader { - const FRAME_SEALED: bool = true; - const COMMAND: Command2 = Command2::RequestStateChunk; - - fn checksum(&self) -> u128 { - self.checksum - } - - fn set_checksum(&mut self, checksum: u128) { - self.checksum = checksum; - } fn operation(&self) -> Operation { Operation::Reserved } @@ -2181,7 +1616,7 @@ pub struct StateChunkHeader { pub nonce: u128, pub offset: u64, - pub group: u64, + pub namespace: u64, /// Index into the offered state manifest. Range-checked by the receiving /// handler against its accepted manifest. pub artifact: u32, @@ -2197,17 +1632,7 @@ const _: () = { }; impl ConsensusHeader for StateChunkHeader { - const FRAME_SEALED: bool = true; - const COMMAND: Command2 = Command2::StateChunk; - - fn checksum(&self) -> u128 { - self.checksum - } - - fn set_checksum(&mut self, checksum: u128) { - self.checksum = checksum; - } fn operation(&self) -> Operation { Operation::Reserved } @@ -2239,12 +1664,9 @@ impl ConsensusHeader for StateChunkHeader { #[cfg(test)] mod tests { use super::{ - Command2, CommitHeader, ConsensusError, ConsensusHeader, DoViewChangeHeader, - EvictionHeader, EvictionReason, GenericHeader, HEADER_SIZE, Operation, PrepareHeader, - PrepareOkHeader, RepairPrepareHeader, RepairRangeReplyHeader, ReplyHeader, RequestHeader, - RequestPreparesHeader, RequestStartViewHeader, RequestStateChunkHeader, - RequestStateTransferHeader, RoutedRequestHeader, StartViewChangeHeader, StartViewHeader, - StateChunkHeader, StateTransferTargetHeader, + Command2, CommitHeader, ConsensusHeader, DoViewChangeHeader, EvictionHeader, + EvictionReason, GenericHeader, HEADER_SIZE, Operation, PrepareHeader, PrepareOkHeader, + ReplyHeader, RequestHeader, StartViewChangeHeader, StartViewHeader, }; use aligned_vec::{AVec, ConstAlign}; @@ -2256,128 +1678,6 @@ mod tests { v } - /// A header-sized frame that satisfies `bytemuck`'s 16-byte alignment. - #[repr(C, align(16))] - struct AlignedFrame([u8; HEADER_SIZE]); - - /// A minimal well-formed header of type `H`: own command and size, everything - /// else zero. Enough for the seal, which reads bytes rather than fields. - fn control_header() -> H { - const COMMAND_OFF: usize = std::mem::offset_of!(GenericHeader, command); - const SIZE_OFF: usize = std::mem::offset_of!(GenericHeader, size); - - let frame_len = u32::try_from(HEADER_SIZE).expect("HEADER_SIZE fits u32"); - let mut frame = AlignedFrame([0u8; HEADER_SIZE]); - frame.0[COMMAND_OFF] = H::COMMAND as u8; - frame.0[SIZE_OFF..SIZE_OFF + 4].copy_from_slice(&frame_len.to_le_bytes()); - *bytemuck::checked::try_from_bytes::(&frame.0).expect("a zeroed frame is a valid header") - } - - /// Seal a header, flip one bit at `offset`, and report `verify_frame`'s verdict. - fn tamper(mut header: H, offset: usize) -> Result<(), ConsensusError> { - header.seal(); - let mut frame = AlignedFrame([0u8; HEADER_SIZE]); - frame.0.copy_from_slice(bytemuck::bytes_of(&header)); - frame.0[offset] ^= 0x01; - let tampered = bytemuck::checked::try_from_bytes::(&frame.0) - .expect("a single flipped bit stays a valid bit pattern here"); - tampered.verify_frame() - } - - #[test] - fn given_a_sealed_control_header_when_verifying_should_accept() { - macro_rules! seals { - ($($header:ty),+ $(,)?) => {$({ - assert!( - <$header>::FRAME_SEALED, - "{} is a replica-to-replica control header and must seal", - stringify!($header), - ); - let mut header = control_header::<$header>(); - header.seal(); - assert_eq!( - header.verify_frame(), - Ok(()), - "{} must accept its own seal", - stringify!($header), - ); - })+}; - } - seals!( - PrepareOkHeader, - CommitHeader, - StartViewChangeHeader, - DoViewChangeHeader, - StartViewHeader, - RequestStartViewHeader, - RequestPreparesHeader, - RepairRangeReplyHeader, - RequestStateTransferHeader, - StateTransferTargetHeader, - RequestStateChunkHeader, - StateChunkHeader, - ); - } - - #[test] - fn given_any_covered_byte_when_flipped_should_reject() { - // Why this seal exists. A `DoViewChange` nack bitset is a new primary's - // authority to truncate: two nacks on three replicas reach - // `quorum_nack_prepare` and discard a committed, client-acked op. The bitsets - // ride the header, and TCP's checksum will not reliably catch one bit. - // Every byte past `checksum` is covered, `checksum_body` included. - for offset in size_of::()..HEADER_SIZE { - let header = control_header::(); - // Skip offsets where the flipped bit is an invalid bit pattern, which - // `try_from_bytes` rejects one layer earlier. - if offset == std::mem::offset_of!(DoViewChangeHeader, command) { - continue; - } - assert!( - matches!( - tamper(header, offset), - Err(ConsensusError::FrameChecksumMismatch { .. }) - ), - "byte {offset} is inside the seal and must be covered" - ); - } - } - - #[test] - fn given_an_unsealed_control_header_when_verifying_should_reject() { - // No presence-keying: a zero checksum is a corrupt frame, not an old one. - // Keying on "does this look sealed" leaves the layer bypassable by zeroing - // the one field that decides whether anything is checked. - let header = control_header::(); - assert_eq!(header.checksum, 0); - assert!(matches!( - header.verify_frame(), - Err(ConsensusError::FrameChecksumMismatch { found: 0, .. }) - )); - } - - #[test] - fn given_an_identity_or_client_header_when_verifying_should_abstain() { - // `PrepareHeader` spends `checksum` on `identity_checksum`, which excludes - // `view` so a re-stamped prepare keeps one identity; the client-facing three - // are sealed on neither side yet. All must parse unchanged. - const { - assert!(!PrepareHeader::FRAME_SEALED); - assert!(!RepairPrepareHeader::FRAME_SEALED); - assert!(!RequestHeader::FRAME_SEALED); - assert!(!ReplyHeader::FRAME_SEALED); - assert!(!EvictionHeader::FRAME_SEALED); - assert!(!GenericHeader::FRAME_SEALED); - } - - let prepare = PrepareHeader { - command: Command2::Prepare, - checksum: 0xdead_beef, - ..Default::default() - }; - assert_eq!(prepare.verify_frame(), Ok(())); - } - #[test] fn all_headers_are_256_bytes() { assert_eq!(size_of::(), 256); @@ -2508,88 +1808,21 @@ mod tests { } // `status` is carved from the reserved tail; the SDK reply funnel peeks it - // at this offset before any body decode, and four foreign SDKs hardcode - // it, so a layout drift must trip here. + // at this offset before any body decode, so a layout drift must trip here. #[test] fn reply_header_status_offset_and_size_pinned() { use std::mem::offset_of; assert_eq!(size_of::(), HEADER_SIZE); - assert_eq!(offset_of!(ReplyHeader, status), 216); - assert_eq!( - offset_of!(ReplyHeader, reserved) + size_of::<[u8; 36]>(), - HEADER_SIZE - ); - } - - // `group` claims the TAIL of the client header's reserved area; the - // leading 52 reserved bytes carry data (the non-replicated op code lives - // in `reserved[0..4]`), so a reshuffle of the carve must trip here rather - // than silently move the code range. - #[test] - fn routed_request_group_claims_reserved_tail() { - use std::mem::offset_of; - assert_eq!( - offset_of!(RoutedRequestHeader, reserved), - offset_of!(RequestHeader, reserved) - ); assert_eq!( - offset_of!(RoutedRequestHeader, group), - offset_of!(RequestHeader, reserved) + 52 + offset_of!(ReplyHeader, status), + offset_of!(ReplyHeader, namespace) + size_of::() ); - assert_eq!(offset_of!(RoutedRequestHeader, group), 248); assert_eq!( - offset_of!(RoutedRequestHeader, group) + size_of::(), + offset_of!(ReplyHeader, reserved) + size_of::<[u8; 28]>(), HEADER_SIZE ); } - // The routed shape is decoded straight off the peer wire, so it must - // enforce the same field rules as the client boundary; a command-only - // validate lets a forged `client = 0` frame reach the client table's - // hard assert. - #[test] - fn routed_request_zero_client_rejected() { - let header = RoutedRequestHeader { - command: Command2::Request, - operation: Operation::SendMessages, - session: 10, - request: 1, - ..RoutedRequestHeader::default() - }; - assert!(header.validate().is_err()); - } - - #[test] - fn routed_request_reserved_operation_rejected() { - let header = RoutedRequestHeader { - command: Command2::Request, - client: 0xCAFE, - session: 10, - request: 1, - ..RoutedRequestHeader::default() - }; - assert!(header.validate().is_err()); - } - - #[test] - fn routed_request_default_fails_validate() { - assert!(RoutedRequestHeader::default().validate().is_err()); - } - - #[test] - fn routed_request_valid_fields_accepted() { - let header = RoutedRequestHeader { - command: Command2::Request, - operation: Operation::SendMessages, - client: 0xCAFE, - session: 10, - request: 1, - group: 7, - ..RoutedRequestHeader::default() - }; - assert!(header.validate().is_ok()); - } - // A nonzero status rides the reserved region, which reply `validate` does // not inspect: a status-bearing reply stays valid so the SDK can peek it. #[test] diff --git a/core/binary_protocol/src/consensus/mod.rs b/core/binary_protocol/src/consensus/mod.rs index 7738256ac3..83bf5ae616 100644 --- a/core/binary_protocol/src/consensus/mod.rs +++ b/core/binary_protocol/src/consensus/mod.rs @@ -45,12 +45,11 @@ mod reply_result; pub use command::Command2; pub use error::ConsensusError; pub use header::{ - CHECKSUM_UNSEALED, CommitHeader, ConsensusHeader, DVC_HEADERS_MAX, DoViewChangeHeader, - EvictionHeader, EvictionReason, GenericHeader, HEADER_SIZE, PrepareHeader, PrepareOkHeader, - RESERVED_COMMAND_LEN, RepairPrepareHeader, RepairRangeReplyHeader, ReplyHeader, RequestHeader, - RequestPreparesHeader, RequestStartViewHeader, RequestStateChunkHeader, - RequestStateTransferHeader, RoutedRequestHeader, SIZE_FIELD_OFFSET, StartViewChangeHeader, - StartViewHeader, StateChunkHeader, StateTransferTargetHeader, frame_body, frame_checksum_bytes, + CommitHeader, ConsensusHeader, DoViewChangeHeader, EvictionHeader, EvictionReason, + GenericHeader, HEADER_SIZE, PrepareHeader, PrepareOkHeader, RESERVED_COMMAND_LEN, + RepairPrepareHeader, RepairRangeReplyHeader, ReplyHeader, RequestHeader, RequestPreparesHeader, + RequestStartViewHeader, RequestStateChunkHeader, RequestStateTransferHeader, SIZE_FIELD_OFFSET, + StartViewChangeHeader, StartViewHeader, StateChunkHeader, StateTransferTargetHeader, read_size_field, }; pub use operation::Operation; diff --git a/core/binary_protocol/src/framing.rs b/core/binary_protocol/src/framing.rs index 9603834d4b..6ead7d6e09 100644 --- a/core/binary_protocol/src/framing.rs +++ b/core/binary_protocol/src/framing.rs @@ -184,7 +184,7 @@ impl<'a> ResponseFrame<'a> { } /// Decoded request frame with request ID for request-response correlation -/// and consensus-level duplicate detection (the server framing). +/// and consensus-level duplicate detection (server-ng framing). /// /// Wire format: `[length:4 LE][code:4 LE][request_id:8 LE][payload:N]` /// where `length` = 4 (code) + 8 (`request_id`) + N (payload). @@ -282,7 +282,7 @@ impl<'a> RequestFrame2<'a> { } /// Decoded response frame with request ID for request-response correlation -/// (the server framing). +/// (server-ng framing). /// /// Wire format: `[status:4 LE][length:4 LE][request_id:8 LE][payload:N]` /// where `status` = 0 for success, non-zero for error code. diff --git a/core/binary_protocol/src/lib.rs b/core/binary_protocol/src/lib.rs index 35109765fe..1f4baf3fb6 100644 --- a/core/binary_protocol/src/lib.rs +++ b/core/binary_protocol/src/lib.rs @@ -71,14 +71,12 @@ pub mod version; pub use codec::{WireDecode, WireEncode}; pub use consensus::{ - CHECKSUM_UNSEALED, Command2, CommitHeader, ConsensusError, ConsensusHeader, DVC_HEADERS_MAX, - DoViewChangeHeader, EvictionHeader, EvictionReason, GenericHeader, HEADER_SIZE, Operation, - PrepareHeader, PrepareOkHeader, RESERVED_COMMAND_LEN, RepairPrepareHeader, - RepairRangeReplyHeader, ReplyHeader, RequestHeader, RequestPreparesHeader, - RequestStartViewHeader, RequestStateChunkHeader, RequestStateTransferHeader, - RoutedRequestHeader, SIZE_FIELD_OFFSET, StartViewChangeHeader, StartViewHeader, - StateChunkHeader, StateTransferTargetHeader, frame_body, frame_checksum_bytes, read_size_field, - result_code, result_section_len, + Command2, CommitHeader, ConsensusError, ConsensusHeader, DoViewChangeHeader, EvictionHeader, + EvictionReason, GenericHeader, HEADER_SIZE, Operation, PrepareHeader, PrepareOkHeader, + RESERVED_COMMAND_LEN, RepairPrepareHeader, RepairRangeReplyHeader, ReplyHeader, RequestHeader, + RequestPreparesHeader, RequestStartViewHeader, RequestStateChunkHeader, + RequestStateTransferHeader, SIZE_FIELD_OFFSET, StartViewChangeHeader, StartViewHeader, + StateChunkHeader, StateTransferTargetHeader, read_size_field, result_code, result_section_len, }; pub use dispatch::{COMMAND_TABLE, CommandMeta, lookup_by_operation, lookup_command}; pub use error::WireError; diff --git a/core/binary_protocol/src/namespace.rs b/core/binary_protocol/src/namespace.rs index 88dbdfa93d..454e81461e 100644 --- a/core/binary_protocol/src/namespace.rs +++ b/core/binary_protocol/src/namespace.rs @@ -15,16 +15,14 @@ // specific language governing permissions and limitations // under the License. -//! Consensus group-id packing constants. +//! Wire-format namespace routing constants. //! -//! Clients send no group or namespace at all -- the server derives the -//! target from the operation and the request payload, stamps it into -//! `RoutedRequestHeader.group` at the dispatch boundary, and every internal -//! layer (sharding hash, consensus demux, repair replay) routes on that -//! stamped value. How stream/topic/partition triples pack into the `u64` -//! is therefore a server-side agreement between the resolver and the -//! sharding layer; the single source of truth lives here in the wire-format -//! crate both already depend on. +//! Both the SDK encoder (which writes `RequestHeader.namespace`) and the +//! server-side sharding layer (which hashes it to a shard) must agree on +//! how stream/topic/partition triples pack into the namespace `u64`. Any +//! drift between the two silently routes writes to the wrong shard, so the +//! single source of truth lives here in the wire-format crate that both +//! sides already depend on. pub const MAX_STREAMS: usize = 4096; pub const MAX_TOPICS: usize = 4096; @@ -63,26 +61,20 @@ pub const PACKED_NAMESPACE_BITS: u32 = STREAM_BITS + TOPIC_BITS + PARTITION_BITS /// Equivalent to `(1 << PACKED_NAMESPACE_BITS) - 1`. pub const PACKED_NAMESPACE_MAX: u64 = (1u64 << PACKED_NAMESPACE_BITS) - 1; -/// Reserved consensus GROUP id for the cluster's metadata plane. +/// Reserved consensus-namespace identifier for the cluster's metadata replica. /// -/// The group-id space is not a free namespace: values inside the packed -/// range are partition groups (the packed stream-topic-partition key), the -/// top bit is the control plane, and 0 is "unset", legal only on client -/// request headers. The packed layout uses only bits -/// `0..PACKED_NAMESPACE_BITS` (compile-asserted below), so the top bit is -/// unreachable from any packed value and routers distinguish metadata's -/// single global consensus group from per-partition groups by value alone. -/// Reserving the BOTTOM of the range instead (Redpanda's raft group 0) -/// only works for allocated ids; ours are derived, and packed 0 is the -/// legal partition `(0, 0, 0)`. -pub const METADATA_GROUP: u64 = 1u64 << 63; +/// The packed layout uses only bits `0..PACKED_NAMESPACE_BITS`, so the top +/// bit is unreachable from any packed namespace value. Routers distinguish +/// metadata's single global consensus group from per-partition consensus +/// groups by value alone. +pub const METADATA_CONSENSUS_NAMESPACE: u64 = 1u64 << 63; // Compile-time invariants. Bumping `MAX_STREAMS`/`MAX_TOPICS`/`MAX_PARTITIONS` // past the values here would silently collapse the sentinel-above-packed-range // guarantee and route writes to the wrong shard; the assertions guard against // that in every build (release included), not only under `cargo test`. const _: () = { - assert!(METADATA_GROUP > PACKED_NAMESPACE_MAX); + assert!(METADATA_CONSENSUS_NAMESPACE > PACKED_NAMESPACE_MAX); assert!(PACKED_NAMESPACE_BITS == STREAM_BITS + TOPIC_BITS + PARTITION_BITS); assert!(PACKED_NAMESPACE_MAX == (1u64 << PACKED_NAMESPACE_BITS) - 1); }; diff --git a/core/binary_protocol/src/requests/users/login_register.rs b/core/binary_protocol/src/requests/users/login_register.rs index f52a2d76cb..b3c3fc3234 100644 --- a/core/binary_protocol/src/requests/users/login_register.rs +++ b/core/binary_protocol/src/requests/users/login_register.rs @@ -22,7 +22,7 @@ use crate::version::ClientVersionInfo; use bytes::{BufMut, BytesMut}; use secrecy::{ExposeSecret, SecretString}; -/// Combined login + register request for the server. +/// Combined login + register request for server-ng. /// /// The server gates on `version_info.protocol_version` (see /// [`crate::version::is_protocol_compatible`]), verifies credentials @@ -49,8 +49,8 @@ use secrecy::{ExposeSecret, SecretString}; /// This wire shape is gated by the `vsr` cargo feature and lives under /// `LOGIN_REGISTER_CODE`. The legacy `LOGIN_USER_CODE` shape (still in use /// by non-`vsr` builds against the legacy `iggy-server`) is untouched. -/// The server speaks VSR framing only; a non-`vsr` SDK cannot log in to -/// the server. Foreign-language SDKs (C++, C#, Python, Go, Java) adopt this +/// server-ng speaks VSR framing only; a non-`vsr` SDK cannot log in to +/// server-ng. Foreign-language SDKs (C++, C#, Python, Go, Java) adopt this /// shape, with their own `sdk_name`, when they wire VSR framing. Bump /// [`crate::version::IGGY_PROTOCOL_VERSION`] on any wire-incompatible /// change. diff --git a/core/binary_protocol/src/requests/users/login_register_with_pat.rs b/core/binary_protocol/src/requests/users/login_register_with_pat.rs index 544cd0de30..5e2ce8f860 100644 --- a/core/binary_protocol/src/requests/users/login_register_with_pat.rs +++ b/core/binary_protocol/src/requests/users/login_register_with_pat.rs @@ -21,7 +21,7 @@ use crate::version::ClientVersionInfo; use bytes::{BufMut, BytesMut}; use secrecy::{ExposeSecret, SecretString}; -/// Combined login-with-PAT + register request for the server. +/// Combined login-with-PAT + register request for server-ng. /// /// Shares the `ClientVersionInfo` prefix with `LoginRegisterRequest` so the /// server gates on the protocol version once before attempting either body diff --git a/core/binary_protocol/src/responses/topics/create_topic.rs b/core/binary_protocol/src/responses/topics/create_topic.rs index 397cc15013..33ea71a750 100644 --- a/core/binary_protocol/src/responses/topics/create_topic.rs +++ b/core/binary_protocol/src/responses/topics/create_topic.rs @@ -19,6 +19,6 @@ /// /// Same `[TopicHeader][PartitionResponse]*` layout as `GetTopicResponse`, /// so the SDK reuses one decoder for both calls. Legacy server's -/// `create_topic_handler` builds this shape directly; the server's metadata +/// `create_topic_handler` builds this shape directly; server-ng's metadata /// STM emits the same bytes from `apply`. pub type CreateTopicResponse = super::GetTopicResponse; diff --git a/core/binary_protocol/src/responses/users/login_register.rs b/core/binary_protocol/src/responses/users/login_register.rs index cd22d7a6b1..dd9714a938 100644 --- a/core/binary_protocol/src/responses/users/login_register.rs +++ b/core/binary_protocol/src/responses/users/login_register.rs @@ -20,7 +20,7 @@ use crate::codec::{WireDecode, WireEncode, read_u32_le, read_u64_le}; use crate::primitives::identifier::WireName; use bytes::{BufMut, BytesMut}; -/// Combined login + register response for the server. +/// Combined login + register response for server-ng. /// /// Returns the authenticated user's ID, the consensus session number /// (commit op number from the Register operation), and the server's diff --git a/core/binary_protocol/src/version.rs b/core/binary_protocol/src/version.rs index 670c460b02..1eaba07c52 100644 --- a/core/binary_protocol/src/version.rs +++ b/core/binary_protocol/src/version.rs @@ -63,8 +63,8 @@ //! `ClientVersionInfo` is the leading bytes of the login-register request //! *body*, which itself rides inside a 256-byte VSR `RequestHeader` (see //! `consensus::header`): `command` = `Command2::Request`, `operation` = -//! `Operation::Register`, client id in `RequestHeader.client`. The client -//! sends no group; the server derives it. A foreign SDK emits that header, +//! `Operation::Register`, `namespace` = `METADATA_CONSENSUS_NAMESPACE`, +//! client id in `RequestHeader.client`. A foreign SDK emits that header, //! then the body starting with this prefix, to reach the gate. //! //! ## Login gate diff --git a/core/cli/Cargo.toml b/core/cli/Cargo.toml index 9bb24742ce..b838b062c5 100644 --- a/core/cli/Cargo.toml +++ b/core/cli/Cargo.toml @@ -51,6 +51,7 @@ login-session = [ "dep:apple-native-keyring-store", "dep:windows-native-keyring-store", ] +vsr = ["iggy/vsr"] [dependencies] anyhow = { workspace = true } diff --git a/core/common/Cargo.toml b/core/common/Cargo.toml index 2e41e25251..e3a4b30d34 100644 --- a/core/common/Cargo.toml +++ b/core/common/Cargo.toml @@ -29,6 +29,9 @@ documentation = "https://iggy.apache.org/docs" repository = "https://github.com/apache/iggy" readme = "README.md" +[features] +vsr = [] + [dependencies] aes-gcm = { workspace = true } async-broadcast = { workspace = true } diff --git a/core/common/src/error/eviction.rs b/core/common/src/error/eviction.rs index ef1fe88db9..8510274252 100644 --- a/core/common/src/error/eviction.rs +++ b/core/common/src/error/eviction.rs @@ -17,7 +17,7 @@ //! Shared grading of a wire [`EvictionReason`] to the typed [`IggyError`] an //! evicted session surfaces. The SDK's binary-transport Eviction-frame decoder -//! (TCP / QUIC / WebSocket) and the server HTTP write path both call this, +//! (TCP / QUIC / WebSocket) and the server-ng HTTP write path both call this, //! so every transport sees one status per reason. use iggy_binary_protocol::consensus::EvictionReason; @@ -32,7 +32,7 @@ use super::iggy_error::IggyError; /// zero minimum or an inverted range), which also falls back to /// re-authentication. /// -/// The two callers extract the fields differently - the server from an aligned +/// The two callers extract the fields differently - server-ng from an aligned /// `EvictionHeader`, the SDK by wire offset off an unaligned buffer - then grade /// through here so the mappings cannot drift apart. #[must_use] diff --git a/core/common/src/lib.rs b/core/common/src/lib.rs index b3bfc1c8fc..dde606a386 100644 --- a/core/common/src/lib.rs +++ b/core/common/src/lib.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +#[cfg(feature = "vsr")] pub mod consumer_group_client_state; mod error; pub mod http; @@ -30,6 +31,7 @@ pub use error::iggy_error::{IggyError, IggyErrorDiscriminants}; // Locking is feature gated, thus only mod level re-export. pub mod locking; pub use chrono::{DateTime, Duration as ChronoDuration, Utc}; +#[cfg(feature = "vsr")] pub use consumer_group_client_state::ConsumerGroupClientState; /// Sentinel `partition_id` in an otherwise-empty poll reply that tells the @@ -55,6 +57,7 @@ pub use iggy_binary_protocol::responses::messages::{ }; pub use traits::binary_client::BinaryClient; pub use traits::binary_transport::BinaryTransport; +#[cfg(feature = "vsr")] pub use traits::binary_transport::{VsrSessionControl, VsrSessionSealed}; pub use traits::client::Client; pub use traits::cluster_client::ClusterClient; diff --git a/core/common/src/traits/binary_client.rs b/core/common/src/traits/binary_client.rs index a38004575d..2d3b618fc2 100644 --- a/core/common/src/traits/binary_client.rs +++ b/core/common/src/traits/binary_client.rs @@ -18,10 +18,15 @@ use crate::{BinaryTransport, Client}; use async_trait::async_trait; -/// A client that can send and receive binary messages. It also exposes the -/// sealed [`VsrSessionControl`](crate::VsrSessionControl) for the SDK's -/// login/logout flows; that surface stays out of [`BinaryTransport`] so -/// external `&dyn BinaryTransport` consumers can't touch consensus session -/// state. +/// A client that can send and receive binary messages. In `vsr` builds it +/// also exposes the sealed [`VsrSessionControl`](crate::VsrSessionControl) +/// for the SDK's login/logout flows; that surface stays out of +/// [`BinaryTransport`] so external `&dyn BinaryTransport` consumers can't +/// touch consensus session state. +#[cfg(feature = "vsr")] #[async_trait] pub trait BinaryClient: BinaryTransport + Client + crate::VsrSessionControl {} + +#[cfg(not(feature = "vsr"))] +#[async_trait] +pub trait BinaryClient: BinaryTransport + Client {} diff --git a/core/common/src/traits/binary_impls/consumer_groups.rs b/core/common/src/traits/binary_impls/consumer_groups.rs index ab4e95ebd5..a47dbb97ad 100644 --- a/core/common/src/traits/binary_impls/consumer_groups.rs +++ b/core/common/src/traits/binary_impls/consumer_groups.rs @@ -159,6 +159,7 @@ impl ConsumerGroupClient for B { // Joining changes this member's assignment (and the group generation); // drop any cached assignment so the next group poll re-syncs. Key // format matches `binary_impls::messages::group_cache_key`. + #[cfg(feature = "vsr")] self.consumer_group_state() .invalidate_assignment(&format!("{stream_id}|{topic_id}|{group_id}")); Ok(()) @@ -184,6 +185,7 @@ impl ConsumerGroupClient for B { .to_bytes(), ) .await?; + #[cfg(feature = "vsr")] { let key = format!("{stream_id}|{topic_id}|{group_id}"); self.consumer_group_state().invalidate_assignment(&key); diff --git a/core/common/src/traits/binary_impls/messages.rs b/core/common/src/traits/binary_impls/messages.rs index ab1f260040..8ffe2fd485 100644 --- a/core/common/src/traits/binary_impls/messages.rs +++ b/core/common/src/traits/binary_impls/messages.rs @@ -24,34 +24,43 @@ use crate::{ Consumer, Identifier, IggyError, IggyMessage, MessageClient, Partitioning, PolledMessages, PollingStrategy, SendMessagesResponse, }; +#[cfg(feature = "vsr")] use crate::{ConsumerKind, PartitioningKind, TopicClient, calculate_32}; use bytes::BytesMut; +#[cfg(feature = "vsr")] use iggy_binary_protocol::codec::WireDecode; use iggy_binary_protocol::codec::WireEncode; +#[cfg(feature = "vsr")] use iggy_binary_protocol::codes::SYNC_CONSUMER_GROUP_CODE; use iggy_binary_protocol::codes::{ FLUSH_UNSAVED_BUFFER_CODE, POLL_MESSAGES_CODE, SEND_MESSAGES_CODE, }; +#[cfg(feature = "vsr")] use iggy_binary_protocol::requests::consumer_groups::SyncConsumerGroupRequest; use iggy_binary_protocol::requests::messages::{ FlushUnsavedBufferRequest, PollMessagesRequest, RawMessage, SendMessagesEncoder, }; +#[cfg(feature = "vsr")] use iggy_binary_protocol::responses::consumer_groups::SyncConsumerGroupResponse; /// Max attempts to resolve a fenced consumer-group poll: one re-sync after the /// coordinator rejects a stale assignment, then retry once. +#[cfg(feature = "vsr")] const GROUP_POLL_MAX_ATTEMPTS: usize = 2; +#[cfg(feature = "vsr")] fn group_cache_key(stream_id: &Identifier, topic_id: &Identifier, group_id: &Identifier) -> String { format!("{stream_id}|{topic_id}|{group_id}") } +#[cfg(feature = "vsr")] fn topic_cache_key(stream_id: &Identifier, topic_id: &Identifier) -> String { format!("{stream_id}|{topic_id}") } /// Sync the requesting member's assignment from the coordinator into the /// transport cache. An empty reply means the client is not a member. +#[cfg(feature = "vsr")] async fn sync_group_assignment( client: &B, stream_id: &Identifier, @@ -97,6 +106,7 @@ async fn sync_group_assignment( /// driven so a member picks up a widened assignment (e.g. after a /// partition-count change) without first hitting an ownership fence. A failed /// per-group sync is logged and skipped so one bad group can't stall the rest. +#[cfg(feature = "vsr")] pub(crate) async fn refresh_group_assignments(client: &B) { for (stream_id, topic_id, group_id) in client.consumer_group_state().registered_groups() { if let Err(error) = sync_group_assignment(client, &stream_id, &topic_id, &group_id).await { @@ -109,6 +119,7 @@ pub(crate) async fn refresh_group_assignments(client: &B) { /// Resolve (and cache) the topic's partition count for client-side produce /// partitioning. +#[cfg(feature = "vsr")] async fn topic_partition_count( client: &B, stream_id: &Identifier, @@ -129,6 +140,7 @@ async fn topic_partition_count( /// Resolve `Balanced` / `MessagesKey` to an explicit `PartitionId` client-side /// (the VSR broker only routes explicit partitions, matching Kafka). +#[cfg(feature = "vsr")] async fn resolve_partitioning( client: &B, stream_id: &Identifier, @@ -168,6 +180,7 @@ async fn resolve_partitioning( /// Poll a consumer group: select one of the member's assigned partitions /// (round-robin) and send an explicit-partition poll. A coordinator fence /// rejection (stale assignment after a rebalance) triggers one re-sync + retry. +#[cfg(feature = "vsr")] async fn poll_group_messages( client: &B, stream_id: &Identifier, @@ -288,6 +301,7 @@ impl MessageClient for B { // VSR: a consumer-group poll without an explicit partition is resolved // client-side from the member's cached assignment (the broker routes // explicit partitions only). + #[cfg(feature = "vsr")] if consumer.kind == ConsumerKind::ConsumerGroup && partition_id.is_none() { return poll_group_messages( self, @@ -326,7 +340,9 @@ impl MessageClient for B { // VSR: resolve Balanced/MessagesKey to an explicit partition client-side. // An explicit `PartitionId` needs no resolution, so borrow the input // directly on that fast path instead of cloning its `value: Vec`. + #[cfg(feature = "vsr")] let resolved_partitioning; + #[cfg(feature = "vsr")] let partitioning = if partitioning.kind == PartitioningKind::PartitionId { partitioning } else { diff --git a/core/common/src/traits/binary_impls/mod.rs b/core/common/src/traits/binary_impls/mod.rs index 58239f5056..23e3c590ec 100644 --- a/core/common/src/traits/binary_impls/mod.rs +++ b/core/common/src/traits/binary_impls/mod.rs @@ -33,17 +33,21 @@ use crate::IggyError; use crate::http::users::defaults::{ MAX_PASSWORD_LENGTH, MAX_USERNAME_LENGTH, MIN_PASSWORD_LENGTH, MIN_USERNAME_LENGTH, }; +#[cfg(feature = "vsr")] use crate::{BinaryClient, ClientState}; use iggy_binary_protocol::WireDecode; +#[cfg(feature = "vsr")] use iggy_binary_protocol::{ClientVersionInfo, IGGY_PROTOCOL_VERSION, WireName}; /// SDK identifier sent in the login-register version prefix. Foreign SDKs /// send their own (e.g. `go-sdk`) once they adopt VSR framing. +#[cfg(feature = "vsr")] pub(crate) const RUST_SDK_NAME: &str = "rust-sdk"; /// Version prefix for both login-register request shapes. `sdk_version` /// comes from [`crate::VsrSessionControl::sdk_version`] so it is the SDK /// crate's version, not this crate's. +#[cfg(feature = "vsr")] pub(crate) fn rust_sdk_version_info(sdk_version: &str) -> Result { Ok(ClientVersionInfo { protocol_version: IGGY_PROTOCOL_VERSION, @@ -54,13 +58,14 @@ pub(crate) fn rust_sdk_version_info(sdk_version: &str) -> Result(client: &B) -> Result<(), IggyError> { if client.get_state().await == ClientState::Authenticated { client.logout_user().await?; diff --git a/core/common/src/traits/binary_impls/personal_access_tokens.rs b/core/common/src/traits/binary_impls/personal_access_tokens.rs index 7e1299f284..202b4f41f9 100644 --- a/core/common/src/traits/binary_impls/personal_access_tokens.rs +++ b/core/common/src/traits/binary_impls/personal_access_tokens.rs @@ -21,22 +21,33 @@ use crate::{ BinaryClient, ClientState, DiagnosticEvent, IdentityInfo, IggyError, PersonalAccessTokenClient, PersonalAccessTokenExpiry, PersonalAccessTokenInfo, RawPersonalAccessToken, }; +#[cfg(feature = "vsr")] use iggy_binary_protocol::MAX_WIRE_NAME_LENGTH; use iggy_binary_protocol::WireName; use iggy_binary_protocol::codec::WireEncode; +#[cfg(feature = "vsr")] use iggy_binary_protocol::codes::LOGIN_REGISTER_WITH_PAT_CODE; +#[cfg(not(feature = "vsr"))] +use iggy_binary_protocol::codes::LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE; use iggy_binary_protocol::codes::{ CREATE_PERSONAL_ACCESS_TOKEN_CODE, DELETE_PERSONAL_ACCESS_TOKEN_CODE, GET_PERSONAL_ACCESS_TOKENS_CODE, }; +#[cfg(not(feature = "vsr"))] +use iggy_binary_protocol::requests::personal_access_tokens::LoginWithPersonalAccessTokenRequest; use iggy_binary_protocol::requests::personal_access_tokens::{ CreatePersonalAccessTokenRequest, DeletePersonalAccessTokenRequest, GetPersonalAccessTokensRequest, }; +#[cfg(feature = "vsr")] use iggy_binary_protocol::requests::users::LoginRegisterWithPatRequest; use iggy_binary_protocol::responses::personal_access_tokens::create_personal_access_token::RawPersonalAccessTokenResponse; use iggy_binary_protocol::responses::personal_access_tokens::get_personal_access_tokens::GetPersonalAccessTokensResponse; +#[cfg(feature = "vsr")] use iggy_binary_protocol::responses::users::LoginRegisterResponse; +#[cfg(not(feature = "vsr"))] +use iggy_binary_protocol::responses::users::login_user::IdentityResponse; +#[cfg(feature = "vsr")] use secrecy::SecretString; #[async_trait::async_trait] @@ -92,52 +103,73 @@ impl PersonalAccessTokenClient for B { &self, token: &str, ) -> Result { - super::logout_before_relogin(self).await?; - // The request stores a `SecretString` rather than a `WireName`, so the - // `WireName` bounds are enforced here to keep the u8 length prefix - // consistent with the realized bytes. - if token.is_empty() || token.len() > MAX_WIRE_NAME_LENGTH { - return Err(IggyError::InvalidFormat); - } - let response = match self - .send_raw_with_response( - LOGIN_REGISTER_WITH_PAT_CODE, - LoginRegisterWithPatRequest { - version_info: super::rust_sdk_version_info(self.sdk_version())?, - token: SecretString::from(token.to_string()), - client_context: None, - } - .to_bytes(), - ) - .await + #[cfg(feature = "vsr")] { - Ok(response) => response, - Err(error) => { - self.reset_vsr_session().await?; - return Err(error); + super::logout_before_relogin(self).await?; + // Same bounds the non-vsr branch gets from `WireName::new(token)`; + // the request stores a `SecretString`, so enforce them here to keep + // the u8 length prefix consistent with the realized bytes. + if token.is_empty() || token.len() > MAX_WIRE_NAME_LENGTH { + return Err(IggyError::InvalidFormat); } - }; - let wire_resp = match super::decode_response::(&response) { - Ok(wire_resp) => wire_resp, - Err(error) => { + let response = match self + .send_raw_with_response( + LOGIN_REGISTER_WITH_PAT_CODE, + LoginRegisterWithPatRequest { + version_info: super::rust_sdk_version_info(self.sdk_version())?, + token: SecretString::from(token.to_string()), + client_context: None, + } + .to_bytes(), + ) + .await + { + Ok(response) => response, + Err(error) => { + self.reset_vsr_session().await?; + return Err(error); + } + }; + let wire_resp = match super::decode_response::(&response) { + Ok(wire_resp) => wire_resp, + Err(error) => { + self.reset_vsr_session().await?; + return Err(error); + } + }; + if let Err(error) = self.bind_vsr_session(wire_resp.session).await { self.reset_vsr_session().await?; return Err(error); } - }; - if let Err(error) = self.bind_vsr_session(wire_resp.session).await { - self.reset_vsr_session().await?; - return Err(error); + tracing::debug!( + server_version = %wire_resp.server_version, + server_protocol_version = wire_resp.server_protocol_version, + "authenticated against iggy server" + ); + self.set_state(ClientState::Authenticated).await; + self.publish_event(DiagnosticEvent::SignedIn).await; + return Ok(IdentityInfo { + user_id: wire_resp.user_id, + access_token: None, + }); } - tracing::debug!( - server_version = %wire_resp.server_version, - server_protocol_version = wire_resp.server_protocol_version, - "authenticated against iggy server" - ); + + #[cfg(not(feature = "vsr"))] + let wire_token = WireName::new(token).map_err(|_| IggyError::InvalidFormat)?; + #[cfg(not(feature = "vsr"))] + let response = self + .send_raw_with_response( + LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE, + LoginWithPersonalAccessTokenRequest { token: wire_token }.to_bytes(), + ) + .await?; + #[cfg(not(feature = "vsr"))] self.set_state(ClientState::Authenticated).await; + #[cfg(not(feature = "vsr"))] self.publish_event(DiagnosticEvent::SignedIn).await; - Ok(IdentityInfo { - user_id: wire_resp.user_id, - access_token: None, - }) + #[cfg(not(feature = "vsr"))] + let wire_resp = super::decode_response::(&response)?; + #[cfg(not(feature = "vsr"))] + Ok(IdentityInfo::from(wire_resp)) } } diff --git a/core/common/src/traits/binary_impls/system.rs b/core/common/src/traits/binary_impls/system.rs index 09f82a997d..f3868fc905 100644 --- a/core/common/src/traits/binary_impls/system.rs +++ b/core/common/src/traits/binary_impls/system.rs @@ -87,6 +87,7 @@ impl SystemClient for B { self.get_heartbeat_interval() } + #[cfg(feature = "vsr")] async fn refresh_consumer_group_assignments(&self) { super::messages::refresh_group_assignments(self).await; } diff --git a/core/common/src/traits/binary_impls/users.rs b/core/common/src/traits/binary_impls/users.rs index a375803668..1a7f15b707 100644 --- a/core/common/src/traits/binary_impls/users.rs +++ b/core/common/src/traits/binary_impls/users.rs @@ -23,18 +23,28 @@ use crate::{ }; use iggy_binary_protocol::WireName; use iggy_binary_protocol::codec::WireEncode; +#[cfg(feature = "vsr")] use iggy_binary_protocol::codes::LOGIN_REGISTER_CODE; +#[cfg(not(feature = "vsr"))] +use iggy_binary_protocol::codes::LOGIN_USER_CODE; use iggy_binary_protocol::codes::{ CHANGE_PASSWORD_CODE, CREATE_USER_CODE, DELETE_USER_CODE, GET_USER_CODE, GET_USERS_CODE, LOGOUT_USER_CODE, UPDATE_PERMISSIONS_CODE, UPDATE_USER_CODE, }; +#[cfg(feature = "vsr")] use iggy_binary_protocol::requests::users::LoginRegisterRequest; +#[cfg(not(feature = "vsr"))] +use iggy_binary_protocol::requests::users::LoginUserRequest; use iggy_binary_protocol::requests::users::{ ChangePasswordRequest, CreateUserRequest, DeleteUserRequest, GetUserRequest, GetUsersRequest, LogoutUserRequest, UpdatePermissionsRequest, UpdateUserRequest, }; +#[cfg(feature = "vsr")] use iggy_binary_protocol::responses::users::LoginRegisterResponse; +#[cfg(not(feature = "vsr"))] +use iggy_binary_protocol::responses::users::login_user::IdentityResponse; use iggy_binary_protocol::responses::users::{GetUsersResponse, UserDetailsResponse}; +#[cfg(feature = "vsr")] use secrecy::SecretString; #[async_trait::async_trait] @@ -177,55 +187,83 @@ impl UserClient for B { async fn login_user(&self, username: &str, password: &str) -> Result { super::validate_username(username)?; super::validate_password(password)?; - super::logout_before_relogin(self).await?; + #[cfg(feature = "vsr")] + { + super::logout_before_relogin(self).await?; + let wire_name = WireName::new(username).map_err(|_| IggyError::InvalidFormat)?; + let response = match self + .send_raw_with_response( + LOGIN_REGISTER_CODE, + LoginRegisterRequest { + version_info: super::rust_sdk_version_info(self.sdk_version())?, + username: wire_name, + password: SecretString::from(password.to_string()), + client_context: None, + } + .to_bytes(), + ) + .await + { + Ok(response) => response, + Err(error) => { + self.reset_vsr_session().await?; + return Err(error); + } + }; + let wire_resp = match super::decode_response::(&response) { + Ok(wire_resp) => wire_resp, + Err(error) => { + self.reset_vsr_session().await?; + return Err(error); + } + }; + if let Err(error) = self.bind_vsr_session(wire_resp.session).await { + self.reset_vsr_session().await?; + return Err(error); + } + tracing::debug!( + server_version = %wire_resp.server_version, + server_protocol_version = wire_resp.server_protocol_version, + "authenticated against iggy server" + ); + self.set_state(ClientState::Authenticated).await; + self.publish_event(DiagnosticEvent::SignedIn).await; + return Ok(IdentityInfo { + user_id: wire_resp.user_id, + access_token: None, + }); + } + + #[cfg(not(feature = "vsr"))] let wire_name = WireName::new(username).map_err(|_| IggyError::InvalidFormat)?; - let response = match self + #[cfg(not(feature = "vsr"))] + let response = self .send_raw_with_response( - LOGIN_REGISTER_CODE, - LoginRegisterRequest { - version_info: super::rust_sdk_version_info(self.sdk_version())?, + LOGIN_USER_CODE, + LoginUserRequest { username: wire_name, - password: SecretString::from(password.to_string()), - client_context: None, + password: password.to_string(), + version: Some(env!("CARGO_PKG_VERSION").to_string()), + context: Some(String::new()), } .to_bytes(), ) - .await - { - Ok(response) => response, - Err(error) => { - self.reset_vsr_session().await?; - return Err(error); - } - }; - let wire_resp = match super::decode_response::(&response) { - Ok(wire_resp) => wire_resp, - Err(error) => { - self.reset_vsr_session().await?; - return Err(error); - } - }; - if let Err(error) = self.bind_vsr_session(wire_resp.session).await { - self.reset_vsr_session().await?; - return Err(error); - } - tracing::debug!( - server_version = %wire_resp.server_version, - server_protocol_version = wire_resp.server_protocol_version, - "authenticated against iggy server" - ); + .await?; + #[cfg(not(feature = "vsr"))] self.set_state(ClientState::Authenticated).await; + #[cfg(not(feature = "vsr"))] self.publish_event(DiagnosticEvent::SignedIn).await; - Ok(IdentityInfo { - user_id: wire_resp.user_id, - access_token: None, - }) + #[cfg(not(feature = "vsr"))] + let wire_resp = super::decode_response::(&response)?; + #[cfg(not(feature = "vsr"))] + Ok(IdentityInfo::from(wire_resp)) } async fn logout_user(&self) -> Result<(), IggyError> { fail_if_not_authenticated(self).await?; self.send_raw_with_response(LOGOUT_USER_CODE, LogoutUserRequest.to_bytes()) .await?; + #[cfg(feature = "vsr")] self.reset_vsr_session().await?; self.set_state(ClientState::Connected).await; self.publish_event(DiagnosticEvent::SignedOut).await; diff --git a/core/common/src/traits/binary_transport.rs b/core/common/src/traits/binary_transport.rs index d3c2bd2e3a..3dacff9ad0 100644 --- a/core/common/src/traits/binary_transport.rs +++ b/core/common/src/traits/binary_transport.rs @@ -18,6 +18,7 @@ use crate::{ClientState, DiagnosticEvent, IggyDuration, IggyError}; use async_trait::async_trait; use bytes::Bytes; +#[cfg(feature = "vsr")] use std::sync::Arc; #[async_trait] @@ -33,6 +34,7 @@ pub trait BinaryTransport { /// Per-transport consumer-group + partitioning cache used to resolve /// partitioning client-side under VSR (the broker never picks a /// partition). Shared via `Arc` so a refresh task can hold it. + #[cfg(feature = "vsr")] fn consumer_group_state(&self) -> Arc; } @@ -40,6 +42,7 @@ pub trait BinaryTransport { /// [`VsrSessionControl`] because they cannot name /// `vsr_session_sealed::Sealed`. The session-mutation methods stay /// in-crate so only the SDK's login/logout flows can call them. +#[cfg(feature = "vsr")] mod vsr_session_sealed { pub trait Sealed {} } @@ -47,6 +50,7 @@ mod vsr_session_sealed { /// VSR-internal session control. Distinct from [`BinaryTransport`] so /// `&dyn BinaryTransport` cannot reach `bind`/`reset` -- mid-session /// mutation corrupts the dedup counter or silently breaks at-most-once. +#[cfg(feature = "vsr")] #[async_trait] pub trait VsrSessionControl: vsr_session_sealed::Sealed + BinaryTransport { async fn bind_vsr_session(&self, session: u64) -> Result<(), IggyError>; @@ -57,4 +61,5 @@ pub trait VsrSessionControl: vsr_session_sealed::Sealed + BinaryTransport { fn sdk_version(&self) -> &'static str; } +#[cfg(feature = "vsr")] pub use vsr_session_sealed::Sealed as VsrSessionSealed; diff --git a/core/common/src/traits/message_client.rs b/core/common/src/traits/message_client.rs index 0c5fa11e4b..23e79c5eaf 100644 --- a/core/common/src/traits/message_client.rs +++ b/core/common/src/traits/message_client.rs @@ -28,7 +28,7 @@ pub trait MessageClient { /// /// Authentication is required, and the permission to poll the messages. /// - /// Polling a consumer group the client is not (or no longer) a member of fails with `ConsumerGroupMemberNotFound` rather than returning an empty batch, so the caller can rejoin. + /// Under the `vsr` feature, polling a consumer group the client is not (or no longer) a member of fails with `ConsumerGroupMemberNotFound` rather than returning an empty batch, so the caller can rejoin. #[allow(clippy::too_many_arguments)] async fn poll_messages( &self, diff --git a/core/common/src/utils/serde_secret.rs b/core/common/src/utils/serde_secret.rs index b10492e438..7f8bd2feb3 100644 --- a/core/common/src/utils/serde_secret.rs +++ b/core/common/src/utils/serde_secret.rs @@ -17,42 +17,22 @@ //! Serde serialization helpers for `SecretString` fields. //! -//! `SecretString` intentionally does not implement `Serialize`, and that -//! absence is the protection: a struct holding one cannot derive `Serialize` -//! at all. Adding `serialize_with` is therefore what *unblocks* the derive, so -//! reaching for a helper here is a decision to serialize a credential, never a -//! way to avoid it. -//! -//! [`serialize_secret`] and [`serialize_optional_secret`] write the plaintext. -//! Use them only where the plaintext is the point: wire protocol payloads, -//! persisted configs, API responses that expose credentials by design. +//! `SecretString` intentionally does not implement `Serialize` to prevent +//! accidental secret exposure. These helpers are for fields that **must** be +//! serialized (e.g., wire protocol payloads, persisted TOML configs, API +//! responses that already expose credentials by design). //! +//! Usage: //! ```ignore //! #[serde(serialize_with = "crate::utils::serde_secret::serialize_secret")] //! pub password: SecretString, //! ``` //! -//! [`serialize_redacted`] and [`serialize_optional_redacted`] write -//! [`REDACTED`] in place of the value, for a struct that must be serializable -//! for unrelated reasons but whose credential no reader is entitled to. -//! -//! **Redacted output is not a config.** Deserializing it hands back the literal -//! [`REDACTED`] as the secret, silently, so a redact-then-reload round trip -//! replaces the credential with the placeholder instead of failing. Nothing -//! in-tree can reach that today: these helpers have no consumers, and the one -//! persist/reload path round-trips a raw `serde_json::Value` rather than a -//! typed struct. If a consumer ever needs the round trip closed mechanically, -//! the shape that cannot be half-applied is a newtype owning both directions, -//! not a paired `deserialize_with` that a caller can forget to add. -//! -//! If neither applies, leave `serialize_with` off and let the missing impl keep -//! the field unserializable. +//! Do **not** add `serialize_with` to fields that should remain redacted in +//! serialized output — rely on `SecretString`'s default behavior instead. use secrecy::{ExposeSecret, SecretString}; -/// Placeholder written in place of a redacted secret. -pub const REDACTED: &str = "[REDACTED]"; - pub fn serialize_secret( secret: &SecretString, serializer: S, @@ -70,28 +50,6 @@ pub fn serialize_optional_secret( } } -/// Writes [`REDACTED`] instead of the secret. -pub fn serialize_redacted( - _secret: &SecretString, - serializer: S, -) -> Result { - serializer.serialize_str(REDACTED) -} - -/// Writes [`REDACTED`] instead of the secret, keeping `None` distinguishable. -/// -/// Whether a credential is configured at all is not itself a secret, and -/// collapsing `Some` to `null` would tell a reader the field is unset. -pub fn serialize_optional_redacted( - secret: &Option, - serializer: S, -) -> Result { - match secret { - Some(_) => serializer.serialize_some(REDACTED), - None => serializer.serialize_none(), - } -} - #[cfg(test)] mod tests { use super::*; @@ -143,43 +101,4 @@ mod tests { let json = serde_json::to_string(&s).unwrap(); assert_eq!(json, r#"{"token":null}"#); } - - #[derive(Serialize)] - struct WithRedactedSecret { - #[serde(serialize_with = "serialize_redacted")] - password: SecretString, - } - - #[derive(Serialize)] - struct WithOptionalRedactedSecret { - #[serde(serialize_with = "serialize_optional_redacted")] - token: Option, - } - - #[test] - fn serialize_redacted_replaces_value_in_json() { - let s = WithRedactedSecret { - password: SecretString::from("my_password"), - }; - let json = serde_json::to_string(&s).unwrap(); - assert_eq!(json, r#"{"password":"[REDACTED]"}"#); - assert!(!json.contains("my_password")); - } - - #[test] - fn serialize_optional_redacted_keeps_some_distinguishable_from_none() { - let present = WithOptionalRedactedSecret { - token: Some(SecretString::from("tok_123")), - }; - let absent = WithOptionalRedactedSecret { token: None }; - - let present_json = serde_json::to_string(&present).unwrap(); - assert_eq!(present_json, r#"{"token":"[REDACTED]"}"#); - assert!(!present_json.contains("tok_123")); - assert_eq!( - serde_json::to_string(&absent).unwrap(), - r#"{"token":null}"#, - "a configured credential must not read as an unset one" - ); - } } diff --git a/core/configs/Cargo.toml b/core/configs/Cargo.toml index 0b2f2079a1..858ea2b176 100644 --- a/core/configs/Cargo.toml +++ b/core/configs/Cargo.toml @@ -37,3 +37,4 @@ server_common = { workspace = true } static-toml = { workspace = true } strum = { workspace = true } tracing = { workspace = true } +tungstenite = { workspace = true } diff --git a/core/configs/src/common/defaults.rs b/core/configs/src/common/defaults.rs deleted file mode 100644 index 2cc6d22007..0000000000 --- a/core/configs/src/common/defaults.rs +++ /dev/null @@ -1,431 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use super::http::{HttpConfig, HttpCorsConfig, HttpJwtConfig, HttpMetricsConfig, HttpTlsConfig}; -use super::server::{ - ConsumerGroupConfig, HeartbeatConfig, MemoryPoolConfig, MessageSaverConfig, - MessagesMaintenanceConfig, PersonalAccessTokenCleanerConfig, PersonalAccessTokenConfig, - TelemetryConfig, TelemetryLogsConfig, TelemetryTracesConfig, -}; -use super::system::{ - BackupConfig, CompatibilityConfig, CompressionConfig, EncryptionConfig, LoggingConfig, - MessageDeduplicationConfig, PartitionConfig, RecoveryConfig, RuntimeConfig, SegmentConfig, - StateConfig, StreamConfig, SystemConfig, TopicConfig, -}; -use configs::ConfigEnvMappings; - -static_toml::static_toml! { - // static_toml resolves relative to CARGO_MANIFEST_DIR (core/configs/). - pub static SERVER_CONFIG = include_toml!("../server/config.toml"); -} - -impl Default for MessagesMaintenanceConfig { - fn default() -> MessagesMaintenanceConfig { - MessagesMaintenanceConfig { - cleaner_enabled: SERVER_CONFIG.data_maintenance.messages.cleaner_enabled, - interval: SERVER_CONFIG - .data_maintenance - .messages - .interval - .parse() - .unwrap(), - } - } -} - -impl Default for HttpConfig { - fn default() -> HttpConfig { - HttpConfig { - enabled: SERVER_CONFIG.http.enabled, - address: SERVER_CONFIG.http.address.parse().unwrap(), - max_request_size: SERVER_CONFIG.http.max_request_size.parse().unwrap(), - web_ui: SERVER_CONFIG.http.web_ui, - cors: HttpCorsConfig::default(), - jwt: HttpJwtConfig::default(), - metrics: HttpMetricsConfig::default(), - tls: HttpTlsConfig::default(), - } - } -} - -impl Default for HttpCorsConfig { - fn default() -> HttpCorsConfig { - HttpCorsConfig { - enabled: SERVER_CONFIG.http.cors.enabled, - allowed_methods: SERVER_CONFIG - .http - .cors - .allowed_methods - .iter() - .map(|s| s.parse().unwrap()) - .collect(), - allowed_origins: SERVER_CONFIG - .http - .cors - .allowed_origins - .iter() - .map(|s| s.parse().unwrap()) - .collect(), - allowed_headers: SERVER_CONFIG - .http - .cors - .allowed_headers - .iter() - .map(|s| s.parse().unwrap()) - .collect(), - exposed_headers: SERVER_CONFIG - .http - .cors - .exposed_headers - .iter() - .map(|s| s.parse().unwrap()) - .collect(), - allow_credentials: SERVER_CONFIG.http.cors.allow_credentials, - allow_private_network: SERVER_CONFIG.http.cors.allow_private_network, - } - } -} - -impl Default for HttpJwtConfig { - fn default() -> HttpJwtConfig { - HttpJwtConfig { - algorithm: SERVER_CONFIG.http.jwt.algorithm.parse().unwrap(), - issuer: SERVER_CONFIG.http.jwt.issuer.parse().unwrap(), - audience: SERVER_CONFIG.http.jwt.audience.parse().unwrap(), - valid_issuers: SERVER_CONFIG - .http - .jwt - .valid_issuers - .iter() - .map(|s| s.parse().unwrap()) - .collect(), - valid_audiences: SERVER_CONFIG - .http - .jwt - .valid_audiences - .iter() - .map(|s| s.parse().unwrap()) - .collect(), - access_token_expiry: SERVER_CONFIG.http.jwt.access_token_expiry.parse().unwrap(), - clock_skew: SERVER_CONFIG.http.jwt.clock_skew.parse().unwrap(), - not_before: SERVER_CONFIG.http.jwt.not_before.parse().unwrap(), - encoding_secret: SERVER_CONFIG.http.jwt.encoding_secret.parse().unwrap(), - decoding_secret: SERVER_CONFIG.http.jwt.decoding_secret.parse().unwrap(), - use_base64_secret: SERVER_CONFIG.http.jwt.use_base_64_secret, - trusted_issuers: None, - } - } -} - -impl Default for HttpMetricsConfig { - fn default() -> HttpMetricsConfig { - HttpMetricsConfig { - enabled: SERVER_CONFIG.http.metrics.enabled, - endpoint: SERVER_CONFIG.http.metrics.endpoint.parse().unwrap(), - } - } -} - -impl Default for HttpTlsConfig { - fn default() -> HttpTlsConfig { - HttpTlsConfig { - enabled: SERVER_CONFIG.http.tls.enabled, - cert_file: SERVER_CONFIG.http.tls.cert_file.parse().unwrap(), - key_file: SERVER_CONFIG.http.tls.key_file.parse().unwrap(), - } - } -} - -impl Default for MessageSaverConfig { - fn default() -> MessageSaverConfig { - MessageSaverConfig { - enabled: SERVER_CONFIG.message_saver.enabled, - enforce_fsync: SERVER_CONFIG.message_saver.enforce_fsync, - interval: SERVER_CONFIG.message_saver.interval.parse().unwrap(), - } - } -} - -impl Default for PersonalAccessTokenConfig { - fn default() -> PersonalAccessTokenConfig { - PersonalAccessTokenConfig { - max_tokens_per_user: SERVER_CONFIG.personal_access_token.max_tokens_per_user as u32, - cleaner: PersonalAccessTokenCleanerConfig::default(), - } - } -} - -impl Default for PersonalAccessTokenCleanerConfig { - fn default() -> PersonalAccessTokenCleanerConfig { - PersonalAccessTokenCleanerConfig { - enabled: SERVER_CONFIG.personal_access_token.cleaner.enabled, - interval: SERVER_CONFIG - .personal_access_token - .cleaner - .interval - .parse() - .unwrap(), - } - } -} - -impl Default for SystemConfig { - fn default() -> Self { - Self { - path: SERVER_CONFIG.system.path.parse().unwrap(), - backup: BackupConfig::default(), - runtime: RuntimeConfig::default(), - logging: LoggingConfig::default(), - stream: StreamConfig::default(), - encryption: EncryptionConfig::default(), - topic: TopicConfig::default(), - partition: PartitionConfig::default(), - segment: SegmentConfig::default(), - state: StateConfig::default(), - compression: CompressionConfig::default(), - message_deduplication: MessageDeduplicationConfig::default(), - recovery: RecoveryConfig::default(), - memory_pool: MemoryPoolConfig::default(), - sharding: S::default(), - } - } -} - -impl Default for BackupConfig { - fn default() -> BackupConfig { - BackupConfig { - path: SERVER_CONFIG.system.backup.path.parse().unwrap(), - compatibility: CompatibilityConfig::default(), - } - } -} - -impl Default for CompatibilityConfig { - fn default() -> Self { - CompatibilityConfig { - path: SERVER_CONFIG - .system - .backup - .compatibility - .path - .parse() - .unwrap(), - } - } -} - -impl Default for HeartbeatConfig { - fn default() -> HeartbeatConfig { - HeartbeatConfig { - enabled: SERVER_CONFIG.heartbeat.enabled, - interval: SERVER_CONFIG.heartbeat.interval.parse().unwrap(), - } - } -} - -impl Default for ConsumerGroupConfig { - fn default() -> ConsumerGroupConfig { - ConsumerGroupConfig { - rebalancing_timeout: SERVER_CONFIG - .consumer_group - .rebalancing_timeout - .parse() - .unwrap(), - rebalancing_check_interval: SERVER_CONFIG - .consumer_group - .rebalancing_check_interval - .parse() - .unwrap(), - } - } -} - -impl Default for RuntimeConfig { - fn default() -> RuntimeConfig { - RuntimeConfig { - path: SERVER_CONFIG.system.runtime.path.parse().unwrap(), - } - } -} - -impl Default for CompressionConfig { - fn default() -> Self { - CompressionConfig { - allow_override: SERVER_CONFIG.system.compression.allow_override, - default_algorithm: SERVER_CONFIG - .system - .compression - .default_algorithm - .parse() - .unwrap(), - } - } -} - -impl Default for LoggingConfig { - fn default() -> LoggingConfig { - LoggingConfig { - path: SERVER_CONFIG.system.logging.path.parse().unwrap(), - level: SERVER_CONFIG.system.logging.level.parse().unwrap(), - file_enabled: SERVER_CONFIG.system.logging.file_enabled, - max_file_size: SERVER_CONFIG.system.logging.max_file_size.parse().unwrap(), - max_total_size: SERVER_CONFIG.system.logging.max_total_size.parse().unwrap(), - rotation_check_interval: SERVER_CONFIG - .system - .logging - .rotation_check_interval - .parse() - .unwrap(), - retention: SERVER_CONFIG.system.logging.retention.parse().unwrap(), - sysinfo_print_interval: SERVER_CONFIG - .system - .logging - .sysinfo_print_interval - .parse() - .unwrap(), - } - } -} - -impl Default for EncryptionConfig { - fn default() -> EncryptionConfig { - EncryptionConfig { - enabled: SERVER_CONFIG.system.encryption.enabled, - key: SERVER_CONFIG.system.encryption.key.parse().unwrap(), - } - } -} - -impl Default for StreamConfig { - fn default() -> StreamConfig { - StreamConfig { - path: SERVER_CONFIG.system.stream.path.parse().unwrap(), - } - } -} - -impl Default for TopicConfig { - fn default() -> TopicConfig { - TopicConfig { - path: SERVER_CONFIG.system.topic.path.parse().unwrap(), - max_size: SERVER_CONFIG.system.topic.max_size.parse().unwrap(), - message_expiry: SERVER_CONFIG.system.topic.message_expiry.parse().unwrap(), - } - } -} - -impl Default for PartitionConfig { - fn default() -> PartitionConfig { - PartitionConfig { - path: SERVER_CONFIG.system.partition.path.parse().unwrap(), - size_of_messages_required_to_save: SERVER_CONFIG - .system - .partition - .size_of_messages_required_to_save - .parse() - .unwrap(), - messages_required_to_save: SERVER_CONFIG.system.partition.messages_required_to_save - as u32, - enforce_fsync: SERVER_CONFIG.system.partition.enforce_fsync, - validate_checksum: SERVER_CONFIG.system.partition.validate_checksum, - } - } -} - -impl Default for SegmentConfig { - fn default() -> SegmentConfig { - SegmentConfig { - size: SERVER_CONFIG.system.segment.size.parse().unwrap(), - preallocate: SERVER_CONFIG.system.segment.preallocate, - cache_indexes: SERVER_CONFIG.system.segment.cache_indexes.parse().unwrap(), - archive_expired: SERVER_CONFIG.system.segment.archive_expired, - } - } -} - -impl Default for StateConfig { - fn default() -> StateConfig { - StateConfig { - enforce_fsync: SERVER_CONFIG.system.state.enforce_fsync, - max_file_operation_retries: SERVER_CONFIG.system.state.max_file_operation_retries - as u32, - retry_delay: SERVER_CONFIG.system.state.retry_delay.parse().unwrap(), - } - } -} - -impl Default for MessageDeduplicationConfig { - fn default() -> MessageDeduplicationConfig { - MessageDeduplicationConfig { - enabled: SERVER_CONFIG.system.message_deduplication.enabled, - max_entries: SERVER_CONFIG.system.message_deduplication.max_entries as u64, - expiry: SERVER_CONFIG - .system - .message_deduplication - .expiry - .parse() - .unwrap(), - } - } -} - -impl Default for RecoveryConfig { - fn default() -> RecoveryConfig { - RecoveryConfig { - recreate_missing_state: SERVER_CONFIG.system.recovery.recreate_missing_state, - } - } -} - -impl Default for MemoryPoolConfig { - fn default() -> MemoryPoolConfig { - Self { - enabled: SERVER_CONFIG.system.memory_pool.enabled, - size: SERVER_CONFIG.system.memory_pool.size.parse().unwrap(), - bucket_capacity: SERVER_CONFIG.system.memory_pool.bucket_capacity as u32, - } - } -} - -impl Default for TelemetryConfig { - fn default() -> TelemetryConfig { - TelemetryConfig { - enabled: SERVER_CONFIG.telemetry.enabled, - service_name: SERVER_CONFIG.telemetry.service_name.parse().unwrap(), - logs: TelemetryLogsConfig::default(), - traces: TelemetryTracesConfig::default(), - } - } -} - -impl Default for TelemetryLogsConfig { - fn default() -> TelemetryLogsConfig { - TelemetryLogsConfig { - transport: SERVER_CONFIG.telemetry.logs.transport.parse().unwrap(), - endpoint: SERVER_CONFIG.telemetry.logs.endpoint.parse().unwrap(), - } - } -} - -impl Default for TelemetryTracesConfig { - fn default() -> TelemetryTracesConfig { - TelemetryTracesConfig { - transport: SERVER_CONFIG.telemetry.traces.transport.parse().unwrap(), - endpoint: SERVER_CONFIG.telemetry.traces.endpoint.parse().unwrap(), - } - } -} diff --git a/core/configs/src/common/displays.rs b/core/configs/src/common/displays.rs deleted file mode 100644 index 3d0b432017..0000000000 --- a/core/configs/src/common/displays.rs +++ /dev/null @@ -1,280 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use super::server::{ - ConsumerGroupConfig, DataMaintenanceConfig, HeartbeatConfig, MessagesMaintenanceConfig, - TelemetryConfig, TelemetryLogsConfig, TelemetryTracesConfig, -}; -use super::system::MessageDeduplicationConfig; -use super::{ - http::{HttpConfig, HttpCorsConfig, HttpJwtConfig, HttpMetricsConfig, HttpTlsConfig}, - server::MessageSaverConfig, - system::{ - CompressionConfig, EncryptionConfig, LoggingConfig, PartitionConfig, SegmentConfig, - StateConfig, StreamConfig, SystemConfig, TopicConfig, - }, -}; -use configs::ConfigEnvMappings; -use std::fmt::{Display, Formatter}; - -impl Display for HttpConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ enabled: {}, address: {}, max_request_size: {}, web_ui: {}, cors: {}, jwt: {}, metrics: {}, tls: {} }}", - self.enabled, - self.address, - self.max_request_size, - self.web_ui, - self.cors, - self.jwt, - self.metrics, - self.tls - ) - } -} - -impl Display for HttpCorsConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ enabled: {}, allowed_methods: {:?}, allowed_origins: {:?}, allowed_headers: {:?}, exposed_headers: {:?}, allow_credentials: {}, allow_private_network: {} }}", - self.enabled, - self.allowed_methods, - self.allowed_origins, - self.allowed_headers, - self.exposed_headers, - self.allow_credentials, - self.allow_private_network - ) - } -} - -impl Display for HttpJwtConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ algorithm: {}, audience: {}, access_token_expiry: {}, use_base64_secret: {} }}", - self.algorithm, self.audience, self.access_token_expiry, self.use_base64_secret - ) - } -} - -impl Display for HttpMetricsConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ enabled: {}, endpoint: {} }}", - self.enabled, self.endpoint - ) - } -} - -impl Display for HttpTlsConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ enabled: {}, cert_file: {}, key_file: {} }}", - self.enabled, self.cert_file, self.key_file - ) - } -} - -impl Display for CompressionConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ allowed_override: {}, default_algorithm: {} }}", - self.allow_override, self.default_algorithm - ) - } -} - -impl Display for DataMaintenanceConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{{ messages: {} }}", self.messages) - } -} - -impl Display for MessagesMaintenanceConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ cleaner_enabled: {}, interval: {} }}", - self.cleaner_enabled, self.interval - ) - } -} - -impl Display for ConsumerGroupConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ rebalancing_timeout: {}, rebalancing_check_interval: {} }}", - self.rebalancing_timeout, self.rebalancing_check_interval - ) - } -} - -impl Display for MessageSaverConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ enabled: {}, enforce_fsync: {}, interval: {} }}", - self.enabled, self.enforce_fsync, self.interval - ) - } -} - -impl Display for HeartbeatConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ enabled: {}, interval: {} }}", - self.enabled, self.interval - ) - } -} - -impl Display for EncryptionConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{{ enabled: {} }}", self.enabled) - } -} - -impl Display for StreamConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{{ path: {} }}", self.path) - } -} - -impl Display for TopicConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ path: {}, max_size: {}, message_expiry: {} }}", - self.path, self.max_size, self.message_expiry - ) - } -} - -impl Display for PartitionConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ path: {}, messages_required_to_save: {}, size_of_messages_required_to_save: {}, enforce_fsync: {}, validate_checksum: {} }}", - self.path, - self.messages_required_to_save, - self.size_of_messages_required_to_save, - self.enforce_fsync, - self.validate_checksum - ) - } -} - -impl Display for MessageDeduplicationConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ enabled: {}, max_entries: {:?}, expiry: {:?} }}", - self.enabled, self.max_entries, self.expiry - ) - } -} - -impl Display for SegmentConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ size_bytes: {}, preallocate: {}, cache_indexes: {}, archive_expired: {} }}", - self.size, self.preallocate, self.cache_indexes, self.archive_expired, - ) - } -} - -impl Display for LoggingConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ path: {}, level: {}, file_enabled: {}, max_file_size: {}, max_total_size: {}, rotation_check_interval: {}, retention: {} }}", - self.path, - self.level, - self.file_enabled, - self.max_file_size.as_human_string_with_zero_as_unlimited(), - self.max_total_size.as_human_string_with_zero_as_unlimited(), - self.rotation_check_interval, - self.retention - ) - } -} - -impl Display for TelemetryConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ enabled: {}, service_name: {}, logs: {}, traces: {} }}", - self.enabled, self.service_name, self.logs, self.traces - ) - } -} - -impl Display for TelemetryLogsConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ transport: {}, endpoint: {} }}", - self.transport, self.endpoint - ) - } -} - -impl Display for StateConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ enforce_fsync: {}, max_file_operation_retries: {}, retry_delay: {} }}", - self.enforce_fsync, self.max_file_operation_retries, self.retry_delay, - ) - } -} - -impl Display for TelemetryTracesConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ transport: {}, endpoint: {} }}", - self.transport, self.endpoint - ) - } -} - -impl Display for SystemConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ path: {}, logging: {}, stream: {}, topic: {}, partition: {}, segment: {}, encryption: {}, state: {} }}", - self.path, - self.logging, - self.stream, - self.topic, - self.partition, - self.segment, - self.encryption, - self.state, - ) - } -} diff --git a/core/configs/src/common/server.rs b/core/configs/src/common/server.rs deleted file mode 100644 index 214ed085d1..0000000000 --- a/core/configs/src/common/server.rs +++ /dev/null @@ -1,144 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use configs::ConfigEnv; -use iggy_common::{IggyByteSize, IggyDuration}; -use serde::{Deserialize, Serialize}; -use serde_with::DisplayFromStr; -use serde_with::serde_as; -use server_common::MemoryPoolConfigOther; -use server_common::log::{TelemetryEndpointSettings, TelemetrySettings}; - -pub use server_common::log::TelemetryTransport; - -/// Configuration for the memory pool. -#[derive(Debug, Deserialize, Serialize, ConfigEnv)] -pub struct MemoryPoolConfig { - pub enabled: bool, - #[config_env(leaf)] - pub size: IggyByteSize, - pub bucket_capacity: u32, -} - -impl MemoryPoolConfig { - pub fn into_other(&self) -> MemoryPoolConfigOther { - MemoryPoolConfigOther { - enabled: self.enabled, - size: self.size, - bucket_capacity: self.bucket_capacity, - } - } -} - -#[serde_as] -#[derive(Debug, Default, Deserialize, Serialize, Clone, ConfigEnv)] -pub struct DataMaintenanceConfig { - pub messages: MessagesMaintenanceConfig, -} - -#[serde_as] -#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] -pub struct MessagesMaintenanceConfig { - pub cleaner_enabled: bool, - #[config_env(leaf)] - #[serde_as(as = "DisplayFromStr")] - pub interval: IggyDuration, -} - -#[serde_as] -#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] -pub struct MessageSaverConfig { - pub enabled: bool, - pub enforce_fsync: bool, - #[config_env(leaf)] - #[serde_as(as = "DisplayFromStr")] - pub interval: IggyDuration, -} - -#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] -pub struct PersonalAccessTokenConfig { - pub max_tokens_per_user: u32, - pub cleaner: PersonalAccessTokenCleanerConfig, -} - -#[serde_as] -#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] -pub struct PersonalAccessTokenCleanerConfig { - pub enabled: bool, - #[config_env(leaf)] - #[serde_as(as = "DisplayFromStr")] - pub interval: IggyDuration, -} - -#[serde_as] -#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] -pub struct HeartbeatConfig { - pub enabled: bool, - #[config_env(leaf)] - #[serde_as(as = "DisplayFromStr")] - pub interval: IggyDuration, -} - -#[serde_as] -#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] -pub struct ConsumerGroupConfig { - #[config_env(leaf)] - #[serde_as(as = "DisplayFromStr")] - pub rebalancing_timeout: IggyDuration, - #[config_env(leaf)] - #[serde_as(as = "DisplayFromStr")] - pub rebalancing_check_interval: IggyDuration, -} - -#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] -pub struct TelemetryConfig { - pub enabled: bool, - pub service_name: String, - pub logs: TelemetryLogsConfig, - pub traces: TelemetryTracesConfig, -} - -#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] -pub struct TelemetryLogsConfig { - #[config_env(leaf)] - pub transport: TelemetryTransport, - pub endpoint: String, -} - -#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] -pub struct TelemetryTracesConfig { - #[config_env(leaf)] - pub transport: TelemetryTransport, - pub endpoint: String, -} - -impl From<&TelemetryConfig> for TelemetrySettings { - fn from(config: &TelemetryConfig) -> Self { - Self { - enabled: config.enabled, - service_name: config.service_name.clone(), - logs: TelemetryEndpointSettings { - transport: config.logs.transport, - endpoint: config.logs.endpoint.clone(), - }, - traces: TelemetryEndpointSettings { - transport: config.traces.transport, - endpoint: config.traces.endpoint.clone(), - }, - } - } -} diff --git a/core/configs/src/common/validators.rs b/core/configs/src/common/validators.rs deleted file mode 100644 index 9c7d54c43c..0000000000 --- a/core/configs/src/common/validators.rs +++ /dev/null @@ -1,357 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use super::COMPONENT; -use super::server::{ - DataMaintenanceConfig, MessageSaverConfig, MessagesMaintenanceConfig, TelemetryConfig, -}; -use super::server::{MemoryPoolConfig, PersonalAccessTokenConfig}; -use super::system::SegmentConfig; -use super::system::{CompressionConfig, LoggingConfig, PartitionConfig}; -use crate::ConfigurationError; -use cpu_allocation::{CpuAllocation, allowed_cpus}; -use err_trail::ErrContext; -use iggy_common::CompressionAlgorithm; -use iggy_common::Validatable; -use std::thread::available_parallelism; -use tracing::warn; - -/// 1 GiB max segment size. -pub const SEGMENT_MAX_SIZE_BYTES: u64 = 1024 * 1024 * 1024; - -impl Validatable for CompressionConfig { - fn validate(&self) -> Result<(), ConfigurationError> { - let compression_alg = &self.default_algorithm; - if *compression_alg != CompressionAlgorithm::None { - // TODO(numinex): Change this message once server side compression is fully developed. - warn!( - "Server started with server-side compression enabled, using algorithm: {compression_alg}, this feature is not implemented yet!" - ); - } - - Ok(()) - } -} - -impl Validatable for TelemetryConfig { - fn validate(&self) -> Result<(), ConfigurationError> { - if !self.enabled { - return Ok(()); - } - - if self.service_name.trim().is_empty() { - eprintln!("telemetry.service_name cannot be empty when telemetry is enabled"); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - if self.logs.endpoint.is_empty() { - eprintln!("telemetry.logs.endpoint cannot be empty when telemetry is enabled"); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - if self.traces.endpoint.is_empty() { - eprintln!("telemetry.traces.endpoint cannot be empty when telemetry is enabled"); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - Ok(()) - } -} - -impl Validatable for PartitionConfig { - fn validate(&self) -> Result<(), ConfigurationError> { - if self.messages_required_to_save == 0 { - eprintln!("Configured system.partition.messages_required_to_save cannot be 0"); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - Ok(()) - } -} - -impl Validatable for SegmentConfig { - fn validate(&self) -> Result<(), ConfigurationError> { - if self.size > SEGMENT_MAX_SIZE_BYTES { - eprintln!( - "Configured system.segment.size {} B is greater than maximum {} B", - self.size.as_bytes_u64(), - SEGMENT_MAX_SIZE_BYTES - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - if !self.size.as_bytes_u64().is_multiple_of(512) { - eprintln!( - "Configured system.segment.size {} B is not a multiple of 512 B", - self.size.as_bytes_u64() - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - Ok(()) - } -} - -impl Validatable for MessageSaverConfig { - fn validate(&self) -> Result<(), ConfigurationError> { - if self.enabled && self.interval.is_zero() { - eprintln!("message_saver.interval cannot be zero when message_saver is enabled"); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - Ok(()) - } -} - -impl Validatable for DataMaintenanceConfig { - fn validate(&self) -> Result<(), ConfigurationError> { - self.messages.validate().error(|e: &ConfigurationError| { - format!("{COMPONENT} (error: {e}) - failed to validate messages maintenance config") - })?; - Ok(()) - } -} - -impl Validatable for MessagesMaintenanceConfig { - fn validate(&self) -> Result<(), ConfigurationError> { - if self.cleaner_enabled && self.interval.is_zero() { - eprintln!("data_maintenance.messages.interval cannot be zero when cleaner is enabled"); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - Ok(()) - } -} - -impl Validatable for PersonalAccessTokenConfig { - fn validate(&self) -> Result<(), ConfigurationError> { - if self.max_tokens_per_user == 0 { - eprintln!("personal_access_token.max_tokens_per_user cannot be 0"); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - if self.cleaner.enabled && self.cleaner.interval.is_zero() { - eprintln!( - "personal_access_token.cleaner.interval cannot be zero when cleaner is enabled" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - Ok(()) - } -} - -impl Validatable for LoggingConfig { - fn validate(&self) -> Result<(), ConfigurationError> { - if self.level.is_empty() { - eprintln!("system.logging.level is supposed be configured"); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - if self.retention.as_secs() < 1 { - eprintln!( - "Configured system.logging.retention {} is less than minimum 1 second", - self.retention - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - if self.rotation_check_interval.as_secs() < 1 { - eprintln!( - "Configured system.logging.rotation_check_interval {} is less than minimum 1 second", - self.rotation_check_interval - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - let max_total_size_unlimited = self.max_total_size.as_bytes_u64() == 0; - if !max_total_size_unlimited - && self.max_file_size.as_bytes_u64() > self.max_total_size.as_bytes_u64() - { - eprintln!( - "Configured system.logging.max_total_size {} is less than system.logging.max_file_size {}", - self.max_total_size, self.max_file_size - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - Ok(()) - } -} - -impl Validatable for MemoryPoolConfig { - fn validate(&self) -> Result<(), ConfigurationError> { - if self.enabled && self.size == 0 { - eprintln!( - "Configured system.memory_pool.enabled is true and system.memory_pool.size is 0" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - const MIN_POOL_SIZE: u64 = 512 * 1024 * 1024; // 512 MiB - const MIN_BUCKET_CAPACITY: u32 = 128; - const DEFAULT_PAGE_SIZE: u64 = 4096; - - if self.enabled && self.size < MIN_POOL_SIZE { - eprintln!( - "Configured system.memory_pool.size {} B ({} MiB) is less than minimum {} B, ({} MiB)", - self.size.as_bytes_u64(), - self.size.as_bytes_u64() / (1024 * 1024), - MIN_POOL_SIZE, - MIN_POOL_SIZE / (1024 * 1024), - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - if self.enabled && !self.size.as_bytes_u64().is_multiple_of(DEFAULT_PAGE_SIZE) { - eprintln!( - "Configured system.memory_pool.size {} B is not a multiple of default page size {} B", - self.size.as_bytes_u64(), - DEFAULT_PAGE_SIZE - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - if self.enabled && self.bucket_capacity < MIN_BUCKET_CAPACITY { - eprintln!( - "Configured system.memory_pool.buffers {} is less than minimum {}", - self.bucket_capacity, MIN_BUCKET_CAPACITY - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - if self.enabled && !self.bucket_capacity.is_power_of_two() { - eprintln!( - "Configured system.memory_pool.buffers {} is not a power of 2", - self.bucket_capacity - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - Ok(()) - } -} - -/// Validate a [`CpuAllocation`] against the machine's available parallelism -/// and, when pinning, the process affinity mask. -pub(crate) fn validate_cpu_allocation( - cpu_allocation: &CpuAllocation, - pin_cores: bool, -) -> Result<(), ConfigurationError> { - let available_cpus = available_parallelism() - .map_err(|_| { - eprintln!("Failed to detect available CPU cores"); - ConfigurationError::InvalidConfigurationValue - })? - .get(); - - match cpu_allocation { - CpuAllocation::All => Ok(()), - CpuAllocation::Count(count) => { - if *count == 0 { - eprintln!("Invalid sharding configuration: cpu_allocation count cannot be 0"); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if *count > available_cpus { - eprintln!( - "Invalid sharding configuration: cpu_allocation count {count} exceeds available CPU cores {available_cpus}" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - Ok(()) - } - CpuAllocation::Range(start, end) => { - if start >= end { - eprintln!( - "Invalid sharding configuration: cpu_allocation range {start}..{end} is invalid (start must be less than end)" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if *end - *start > available_cpus { - eprintln!( - "Invalid sharding configuration: cpu_allocation range {start}..{end} yields {} shards, exceeding available CPU cores {available_cpus}", - *end - *start - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if !pin_cores { - return Ok(()); - } - let allowed = allowed_cpus(); - if let Some(cpu) = (*start..*end).find(|cpu| !allowed.contains(cpu)) { - eprintln!( - "Invalid sharding configuration: cpu_allocation range {start}..{end} includes CPU {cpu}, which is outside the set of cores allowed for this process (affinity/cpuset mask)" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - Ok(()) - } - // NUMA topology validation requires hwlocality (runtime dep). - // Full NUMA validation happens in shard_allocator at startup. - CpuAllocation::NumaAware(_) => Ok(()), - } -} - -#[cfg(test)] -mod cpu_allocation_tests { - use super::*; - - #[test] - fn inverted_range_is_rejected() { - assert!(validate_cpu_allocation(&CpuAllocation::Range(2, 2), true).is_err()); - } - - #[test] - fn pinned_range_within_allowed_set_is_accepted() { - let first = allowed_cpus()[0]; - assert!(validate_cpu_allocation(&CpuAllocation::Range(first, first + 1), true).is_ok()); - } - - #[test] - fn pinned_range_outside_allowed_set_is_rejected() { - let past_last = allowed_cpus().last().copied().unwrap() + 1; - assert!( - validate_cpu_allocation(&CpuAllocation::Range(past_last, past_last + 1), true).is_err() - ); - } - - #[test] - fn pinned_range_wider_than_parallelism_is_rejected() { - // Under a cgroup CPU quota the affinity mask stays full while - // `available_parallelism` shrinks, so membership alone would - // accept this; the shard-count cap must reject it. - let first = allowed_cpus()[0]; - let available = available_parallelism().unwrap().get(); - assert!( - validate_cpu_allocation(&CpuAllocation::Range(first, first + available + 1), true) - .is_err() - ); - } - - #[test] - fn unpinned_range_is_capped_by_shard_count_not_core_ids() { - // Core ids outside the machine are fine unpinned; only the - // resulting shard count matters. - let outside = 1 << 20; - assert!( - validate_cpu_allocation(&CpuAllocation::Range(outside, outside + 1), false).is_ok() - ); - - let available = available_parallelism().unwrap().get(); - assert!(validate_cpu_allocation(&CpuAllocation::Range(0, available + 1), false).is_err()); - } -} diff --git a/core/configs/src/lib.rs b/core/configs/src/lib.rs index e9339b2aa6..914729ef04 100644 --- a/core/configs/src/lib.rs +++ b/core/configs/src/lib.rs @@ -17,15 +17,20 @@ extern crate self as configs; -mod common; mod configs_impl; mod server_config; -pub use common::{COMPONENT, cache_indexes, defaults, displays, http, system, validators}; +mod server_ng_config; pub use configs_derive::ConfigEnv; pub use configs_impl::{ ConfigEnvMappings, ConfigProvider, ConfigurationError, ConfigurationType, EnvVarMapping, FileConfigProvider, TypedEnvProvider, parse_env_value_to_json, }; pub use server_config::{ - cluster, message_bus, metadata, partition, quic, server, sharding, tcp, websocket, + COMPONENT, cache_indexes, cluster, defaults, displays, http, quic, server, sharding, system, + tcp, validators, websocket, +}; +pub use server_ng_config::{ + COMPONENT_NG, cluster as ng_cluster, message_bus, metadata as ng_metadata, + partition as ng_partition, quic as ng_quic, server_ng, sharding as ng_sharding, tcp as ng_tcp, + websocket as ng_websocket, }; diff --git a/core/configs/src/common/cache_indexes.rs b/core/configs/src/server_config/cache_indexes.rs similarity index 100% rename from core/configs/src/common/cache_indexes.rs rename to core/configs/src/server_config/cache_indexes.rs diff --git a/core/configs/src/server_config/cluster.rs b/core/configs/src/server_config/cluster.rs index ad767c6b20..af6f26c513 100644 --- a/core/configs/src/server_config/cluster.rs +++ b/core/configs/src/server_config/cluster.rs @@ -15,257 +15,23 @@ // specific language governing permissions and limitations // under the License. -//! Cluster schema: node topology plus the VSR consensus tunables. - -use super::defaults::SERVER_CONFIG; -use crate::ConfigurationError; -use crate::http::HttpJwtConfig; use configs::ConfigEnv; -use iggy_common::{IggyDuration, Validatable}; -use ipnet::{IpNet, Ipv4Net}; use serde::{Deserialize, Serialize}; -use serde_with::{DisplayFromStr, serde_as}; -use std::cmp::Reverse; -use std::fmt; -use std::net::{IpAddr, Ipv6Addr, SocketAddr}; -use std::str::FromStr; -use std::time::Duration; - -/// Absolute floor for the backup liveness window, independent of the -/// commit-broadcast rate. The primary signals liveness through its commit -/// broadcast (`commit_broadcast_interval`, 500ms by default); 2s spans several -/// broadcasts, so a single delayed one never elects. The per-config -/// `MIN_HEARTBEAT_TO_COMMIT_BROADCAST_RATIO` check scales the same headroom -/// when the broadcast interval is retuned. -pub const MIN_CLUSTER_HEARTBEAT_TIMEOUT: Duration = Duration::from_secs(2); - -/// The backup liveness window (`heartbeat_timeout`) must span at least this -/// many commit broadcasts (`commit_broadcast_interval`), so one dropped or -/// delayed broadcast never trips a view change on a healthy primary. -const MIN_HEARTBEAT_TO_COMMIT_BROADCAST_RATIO: u32 = 4; - -/// The view-change status backstop (`view_change_status_timeout`) must span at -/// least this many retransmit intervals (`view_change_retransmit_interval`), so -/// a few dropped `StartViewChange` / `DoViewChange` messages retransmit rather -/// than escalating a progressing view change into a fresh cluster-wide election. -const MIN_STATUS_TO_RETRANSMIT_RATIO: u32 = 4; - -/// Default recovering-replica probe-attempt ceiling. Duplicated here rather -/// than imported so `core/configs` keeps off a build-time edge onto -/// `core/consensus` (mirroring [`super::partition`]); `core/server`'s -/// bootstrap static-asserts it equal to `consensus::PROBE_ATTEMPTS_MAX`. -pub const DEFAULT_VIEW_PROBE_ATTEMPTS_MAX: u32 = 5; - -/// Upper bound on `view_probe_attempts_max`. A recovering replica probes once -/// per `request_start_view_retransmit_interval`, so hundreds of attempts would -/// stall the election fallback for minutes on a full-cluster restart; this is a -/// typo guard, not a sizing endorsement. -const MAX_VIEW_PROBE_ATTEMPTS: u32 = 100; - -/// Default per-round repair-serving chunk. Duplicated here rather than imported -/// so `core/configs` keeps off a build-time edge onto `core/shard` (mirroring -/// [`DEFAULT_VIEW_PROBE_ATTEMPTS_MAX`]); `core/server`'s bootstrap -/// static-asserts it equal to `shard::REPAIR_CHUNK_MAX`. -pub const DEFAULT_REPAIR_CHUNK_MAX: usize = 128; - -/// `size_of::()`. Duplicated here for the same reason as -/// [`DEFAULT_REPAIR_CHUNK_MAX`]; `core/server`'s bootstrap static-asserts it -/// against the real header. Used to reject a `[message_bus] max_message_size` -/// too small to carry a single state-transfer chunk. -pub const STATE_CHUNK_HEADER_LEN: u64 = 256; - -/// Upper bound on `repair_chunk_max`. A chunk rides the per-peer bus queue, so -/// the load-bearing rule is `repair_chunk_max < message_bus.peer_queue_capacity` -/// (enforced at the top level); this standalone ceiling is a typo guard. -const MAX_REPAIR_CHUNK_MAX: usize = 1024; - -/// Upper bound on per-node `advertised_addresses` selectors. Must mirror the -/// `#[config_env(max_elements = 16)]` cap on the field: env overrides above -/// the cap do not exist, so a TOML roster exceeding it could never be -/// replicated through the env path. Also bounds the cross-node conflict scan -/// (quadratic in pooled entries) and the per-request longest-prefix walk. -const MAX_ADVERTISED_SELECTORS: usize = 16; - -/// Length floor for the replica-auth PSK, in raw bytes. The 32-byte MAC key -/// is KDF-derived from these bytes at use-site, so any encoding clearing this -/// length is accepted. -const MIN_SHARED_SECRET_LEN: usize = 32; - -/// DNS caps a full name at 255 octets on the wire, which leaves 253 -/// characters of presentation text (RFC 1035). -const MAX_HOSTNAME_LEN: usize = 253; - -/// Per-label limit from RFC 1035. -const MAX_HOSTNAME_LABEL_LEN: usize = 63; - -/// serde fallback for configs written before the field existed; the value -/// itself lives in `core/server/config.toml` like every other default. -fn default_heartbeat_timeout() -> IggyDuration { - SERVER_CONFIG.cluster.heartbeat_timeout.parse().unwrap() -} - -/// serde fallback for configs written before the field existed; the value -/// itself lives in `core/server/config.toml` like every other default. -fn default_commit_broadcast_interval() -> IggyDuration { - SERVER_CONFIG - .cluster - .commit_broadcast_interval - .parse() - .unwrap() -} - -/// serde fallback for configs written before the field existed; the value -/// itself lives in `core/server/config.toml` like every other default. -fn default_prepare_retransmit_interval() -> IggyDuration { - SERVER_CONFIG - .cluster - .prepare_retransmit_interval - .parse() - .unwrap() -} - -/// serde fallback for configs written before the field existed; the value -/// itself lives in `core/server/config.toml` like every other default. -fn default_view_change_retransmit_interval() -> IggyDuration { - SERVER_CONFIG - .cluster - .view_change_retransmit_interval - .parse() - .unwrap() -} - -/// serde fallback for configs written before the field existed; the value -/// itself lives in `core/server/config.toml` like every other default. -fn default_view_change_status_timeout() -> IggyDuration { - SERVER_CONFIG - .cluster - .view_change_status_timeout - .parse() - .unwrap() -} - -/// serde fallback for configs written before the field existed; the value -/// itself lives in `core/server/config.toml` like every other default. -fn default_request_start_view_retransmit_interval() -> IggyDuration { - SERVER_CONFIG - .cluster - .request_start_view_retransmit_interval - .parse() - .unwrap() -} - -/// serde fallback for configs written before the field existed; the value -/// itself lives in `core/server/config.toml` like every other default. -fn default_view_probe_attempts_max() -> u32 { - SERVER_CONFIG.cluster.view_probe_attempts_max as u32 -} - -/// serde fallback for configs written before the field existed; the value -/// itself lives in `core/server/config.toml` like every other default. -fn default_repair_retry_interval() -> IggyDuration { - SERVER_CONFIG.cluster.repair_retry_interval.parse().unwrap() -} - -/// serde fallback for configs written before the field existed; the value -/// itself lives in `core/server/config.toml` like every other default. -fn default_repair_chunk_max() -> usize { - SERVER_CONFIG.cluster.repair_chunk_max as usize -} -#[serde_as] #[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] #[serde(deny_unknown_fields)] pub struct ClusterConfig { pub enabled: bool, pub name: String, - /// Backup-side liveness window for a plane's primary. A replica that - /// sees no primary traffic for this long starts a view change - /// (`normal_heartbeat_timeout`). Raise it on oversubscribed hosts where - /// scheduling stalls fake primary death; sub-`MIN_CLUSTER_HEARTBEAT_TIMEOUT` - /// values (including the `0` / `disabled` / `unlimited` sentinels, which - /// all parse to zero) are rejected at boot. - #[serde(default = "default_heartbeat_timeout")] - #[serde_as(as = "DisplayFromStr")] - #[config_env(leaf)] - pub heartbeat_timeout: IggyDuration, - /// How often the primary broadcasts its commit point to every backup, the - /// cluster's primary-liveness signal. Each broadcast resets the backups' - /// `heartbeat_timeout` window, so that window must span several broadcasts: - /// boot rejects `heartbeat_timeout < MIN_HEARTBEAT_TO_COMMIT_BROADCAST_RATIO - /// * commit_broadcast_interval`. Sizes the consensus `CommitMessage` timer. - /// Zero (and the `0` / `disabled` / `unlimited` sentinels, which all parse - /// to zero) is rejected at boot. - #[serde(default = "default_commit_broadcast_interval")] - #[serde_as(as = "DisplayFromStr")] - #[config_env(leaf)] - pub commit_broadcast_interval: IggyDuration, - /// How often the primary retransmits prepares a backup has not yet acked. - /// Lower recovers faster from a dropped prepare at the cost of replica - /// traffic. Sizes the consensus `Prepare` timer. Zero (and the `0` / - /// `disabled` / `unlimited` sentinels, which all parse to zero) is rejected - /// at boot. - #[serde(default = "default_prepare_retransmit_interval")] - #[serde_as(as = "DisplayFromStr")] - #[config_env(leaf)] - pub prepare_retransmit_interval: IggyDuration, - /// How often a plane retransmits its `StartViewChange` / `DoViewChange` - /// while a view change is running. Lower converges a healthy election - /// faster at the cost of replica traffic. Sizes both consensus view-change - /// retransmit timers, which are deliberately equal. Zero (and the `0` / - /// `disabled` / `unlimited` sentinels, which all parse to zero) is rejected - /// at boot. - #[serde(default = "default_view_change_retransmit_interval")] - #[serde_as(as = "DisplayFromStr")] - #[config_env(leaf)] - pub view_change_retransmit_interval: IggyDuration, - /// Backstop for a stalled view change: one that does not conclude within - /// this window escalates to a fresh cluster-wide election. Must span - /// several `view_change_retransmit_interval`s so a few dropped view-change - /// messages retransmit rather than escalate: boot rejects - /// `view_change_status_timeout < MIN_STATUS_TO_RETRANSMIT_RATIO * - /// view_change_retransmit_interval`. Zero (and the `0` / `disabled` / - /// `unlimited` sentinels, which all parse to zero) is rejected at boot. - #[serde(default = "default_view_change_status_timeout")] - #[serde_as(as = "DisplayFromStr")] - #[config_env(leaf)] - pub view_change_status_timeout: IggyDuration, - /// How often a recovering or view-change backup re-requests the current - /// view's `StartView` from its primary (`RequestStartView`). Sizes the - /// consensus `RequestStartView` timer. Zero (and the `0` / `disabled` / - /// `unlimited` sentinels, which all parse to zero) is rejected at boot. - #[serde(default = "default_request_start_view_retransmit_interval")] - #[serde_as(as = "DisplayFromStr")] - #[config_env(leaf)] - pub request_start_view_retransmit_interval: IggyDuration, - /// How many consecutive unanswered `RequestStartView` probes a recovering - /// replica tolerates before it falls back to an election (a full-cluster - /// restart leaves nobody settled to answer). Must be >= 1 and <= - /// `MAX_VIEW_PROBE_ATTEMPTS`. - #[serde(default = "default_view_probe_attempts_max")] - pub view_probe_attempts_max: u32, - /// How long a stalled journal-repair stream waits before re-requesting its - /// remaining window from the serving peer. Repair frames are - /// fire-and-forget over the lossy bus, so a session with no retry wedges - /// forever on a single dropped frame. Paces both the metadata and - /// partition repair loops. Sizes the retry threshold in consensus ticks. - /// Zero (and the `0` / `disabled` / `unlimited` sentinels, which all parse - /// to zero) is rejected at boot. - #[serde(default = "default_repair_retry_interval")] - #[serde_as(as = "DisplayFromStr")] - #[config_env(leaf)] - pub repair_retry_interval: IggyDuration, - /// Prepares a peer serves per repair round before the requester walks to - /// the next chunk. Each frame rides the per-peer message-bus queue, so this - /// must stay below `message_bus.peer_queue_capacity` or a full round - /// overruns the queue and silently drops frames (enforced at the top - /// level). Applies to both the metadata and partition repair planes. Must - /// be > 0 and <= `MAX_REPAIR_CHUNK_MAX`. - #[serde(default = "default_repair_chunk_max")] - pub repair_chunk_max: usize, /// Full roster of cluster members. Intended to be byte-identical across /// every node so operators ship one config. The running node's identity /// is supplied out-of-band via the `--replica-id` CLI flag, which /// selects the entry in this list that describes the current node. + // + // TODO(hubcio): IGGY-155 `register-replica` CLI (a validated roster + // append) is deferred - it is convenience only over a manual TOML edit, + // and `ClusterConfig::validate` already rejects a malformed roster at + // boot. Add it only if scripted/automated roster edits become a need. #[serde(default)] pub nodes: Vec, /// Replica-to-replica authentication settings (PSK + BLAKE3 handshake). @@ -305,25 +71,11 @@ pub struct ClusterAuthConfig { #[serde(default, skip_serializing)] #[config_env(secret)] pub shared_secret: String, - /// Retiring pre-shared key, accepted for VERIFICATION only during a key - /// rotation window; every MAC this node produces uses [`Self::shared_secret`]. - /// - /// Enables rolling PSK rotation without an auth outage, three rolls: - /// 1. every node gets `shared_secret = old, previous_shared_secret = new`; - /// 2. every node gets `shared_secret = new, previous_shared_secret = old`; - /// 3. every node gets `shared_secret = new` alone, closing the window. - /// - /// Leave empty (default) outside a rotation. Same 32-byte minimum and - /// provisioning rules as `shared_secret` - /// (`IGGY_CLUSTER_AUTH_PREVIOUS_SHARED_SECRET`). - #[serde(default, skip_serializing)] - #[config_env(secret)] - pub previous_shared_secret: String, } /// Replica-to-replica TLS for the consensus (`tcp_replica`) port. /// -/// Mirrors the legacy [`crate::tcp::TcpTlsConfig`] shape plus `ca_file`: +/// Mirrors the legacy [`super::tcp::TcpTlsConfig`] shape plus `ca_file`: /// the replica plane DIALS its peers (a TLS client role the /// client-facing server plane never has), so the dialer needs a trust /// anchor to verify the acceptor's certificate against. @@ -358,23 +110,9 @@ pub struct ClusterTlsConfig { } #[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] -#[serde(deny_unknown_fields)] pub struct ClusterNodeConfig { pub name: String, pub ip: String, - /// Optional client-facing address: a literal IP or a DNS hostname, - /// validated as [`AdvertisedAddress`] at boot. Replica traffic continues - /// to use [`Self::ip`]. - #[serde(default)] - pub advertised_address: Option, - /// Client-network-scoped overrides of [`Self::advertised_address`], - /// resolved by longest-prefix match over the client's IP (see - /// [`AdvertisedAddressSelector`]). Empty by default, so existing configs - /// keep the single catch-all address. At most `MAX_ADVERTISED_SELECTORS` - /// per node (validated), matching the env-override cap below. - #[serde(default)] - #[config_env(max_elements = 16)] - pub advertised_addresses: Vec, /// Numeric replica ID for VSR consensus (0-based). /// /// Must be unique across [`ClusterConfig::nodes`] and strictly less than @@ -383,165 +121,6 @@ pub struct ClusterNodeConfig { pub ports: TransportPorts, } -/// One client-network-scoped advertised address: clients whose IP falls -/// inside `client_cidr` are told `address` instead of the node's catch-all -/// [`ClusterNodeConfig::advertised_address`]. -/// -/// Typical split-network case: the roster `ip` is VPC-private and -/// `advertised_address` is public; a selector with the VPC CIDR keeps -/// in-VPC clients on the private address while everyone else stays on the -/// public one. Selection is longest-prefix match across a node's selectors. -/// Selection sees the transport-level peer address, so clients arriving -/// through a proxy or load balancer match the proxy's network, not their -/// own. -#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] -#[serde(deny_unknown_fields)] -pub struct AdvertisedAddressSelector { - /// Client network this selector matches, in CIDR notation - /// (`10.0.0.0/16`, `2001:db8::/32`). Must parse at boot; duplicate - /// networks within one node are rejected. A v4-mapped v6 network - /// (`::ffff:10.0.0.0/104`) canonicalizes to its v4 form (`10.0.0.0/8`), - /// matching how client IPs canonicalize before matching. - pub client_cidr: String, - /// Address advertised to matching clients: a literal IP or a DNS - /// hostname, validated as [`AdvertisedAddress`] at boot (no port; ports - /// come from [`ClusterNodeConfig::ports`]). - pub address: String, -} - -/// A roster node with its advertised-address selectors and catch-all parsed -/// once, built wherever a roster is assembled for serving clients -/// (listener/shard start). Per-request resolution never re-parses config -/// strings: everything is snapshotted here, so mutating the source config -/// after conversion has no effect on what clients are told. Entries that do -/// not parse are dropped at build time; validation already rejects them -/// whenever the cluster is enabled, and a disabled cluster never consults -/// the roster. -#[derive(Debug, Clone)] -pub struct ResolvedClusterNode { - config: ClusterNodeConfig, - /// Truncated, canonicalized selector networks with their parsed - /// addresses, in declaration order. - selectors: Vec<(IpNet, AdvertisedAddress)>, - /// Parsed catch-all: [`ClusterNodeConfig::advertised_address`], else the - /// roster [`ClusterNodeConfig::ip`]. `None` when the configured value - /// does not parse - a set `advertised_address` never falls through to - /// the private roster ip. - catch_all: Option, - /// Parsed roster [`ClusterNodeConfig::ip`], the replica-plane dial - /// address. `None` when the roster ip is not a literal IP (boot only - /// requires it non-empty); internal forwarding then has no dial target. - replica_ip: Option, -} - -impl From for ResolvedClusterNode { - fn from(config: ClusterNodeConfig) -> Self { - let selectors = config - .advertised_addresses - .iter() - .filter_map(|selector| { - let network = selector.client_cidr.parse::().ok()?; - let address = selector.address.parse::().ok()?; - Some((canonical_ip_net(network.trunc()), address)) - }) - .collect(); - let catch_all = match config.advertised_address.as_deref() { - Some(advertised_address) => advertised_address.parse().ok(), - None => config.ip.parse().ok(), - }; - let replica_ip = config.ip.parse().ok(); - Self { - config, - selectors, - catch_all, - replica_ip, - } - } -} - -impl ResolvedClusterNode { - /// The roster entry this node was built from. Read-only: resolution runs - /// on the boot-parsed snapshot, never on the config strings. - #[must_use] - pub fn config(&self) -> &ClusterNodeConfig { - &self.config - } - - /// The roster `ip` as a dialable address for the replica plane and - /// internal request forwarding. Never routed through the advertised - /// ladder: this is what servers dial, not what clients are told. - #[must_use] - pub fn replica_ip(&self) -> Option { - self.replica_ip - } - - /// The client-facing address for a client connecting from `client_ip`: - /// longest-prefix match over the selector networks, then the parsed - /// catch-all. `None` when no selector matches and the catch-all did not - /// parse; callers choose whether to fail closed (redirect URLs) or to - /// publish [`Self::raw_advertised_fallback`] verbatim (cluster metadata). - #[must_use] - pub fn advertised_for(&self, client_ip: Option) -> Option<&AdvertisedAddress> { - client_ip - .and_then(|client_ip| self.selector_address(client_ip)) - .or(self.catch_all.as_ref()) - } - - /// The catch-all ladder ([`ClusterNodeConfig::advertised_address`], else - /// the roster [`ClusterNodeConfig::ip`]) as configured, unparsed. Cluster - /// metadata publishes this verbatim when [`Self::advertised_for`] finds - /// nothing: the roster `ip` is only validated non-empty, and Docker - /// service names with underscores exist in the wild. - #[must_use] - pub fn raw_advertised_fallback(&self) -> &str { - self.config - .advertised_address - .as_deref() - .unwrap_or(&self.config.ip) - } - - /// Longest-prefix match over the boot-parsed selector networks. The - /// client IP is canonicalized first so a v4-mapped v6 peer - /// (`::ffff:10.0.0.7`, the shape a dual-stack listener reports) matches - /// v4 networks. `min_by_key` keeps the first of equal-length matches, so - /// resolution stays declaration-order deterministic even though a - /// validated config cannot produce two matching networks of equal length - /// (equal-length distinct networks are disjoint, duplicates are - /// rejected). - fn selector_address(&self, client_ip: IpAddr) -> Option<&AdvertisedAddress> { - let client_ip = client_ip.to_canonical(); - self.selectors - .iter() - .filter(|(network, _)| network.contains(&client_ip)) - .min_by_key(|(network, _)| Reverse(network.prefix_len())) - .map(|(_, address)| address) - } -} - -/// Network-side mirror of the `IpAddr::to_canonical` applied to client IPs -/// before matching: a selector network written in v4-mapped v6 form -/// (`::ffff:10.0.0.0/104`) becomes its v4 equivalent (`10.0.0.0/8`), since a -/// canonicalized client could never match the v6 spelling. Prefixes shorter -/// than 96 bits cannot drop the `::ffff:` mapping and stay v6 (they match -/// native v6 clients only). -fn canonical_ip_net(network: IpNet) -> IpNet { - if let IpNet::V6(v6_network) = network - && v6_network.prefix_len() >= 96 - && let IpAddr::V4(v4_address) = v6_network.addr().to_canonical() - && let Ok(v4_network) = Ipv4Net::new(v4_address, v6_network.prefix_len() - 96) - { - return IpNet::V4(v4_network); - } - network -} - -/// Per-node listener ports advertised in the cluster roster. In cluster mode -/// the roster is the single source of ports: every enabled transport needs -/// an explicit per-node port (validated at startup, no fallback to the -/// transport's top-level `address` port). The roster entry's `ip` is the -/// advertised address only: tcp/ws/quic/http bind the interface from their own -/// `address` config, and followers forward HTTP requests to the primary at -/// `ip:http`. #[derive(Debug, Deserialize, Serialize, Clone, Default, ConfigEnv)] pub struct TransportPorts { pub tcp: Option, @@ -552,832 +131,6 @@ pub struct TransportPorts { pub tcp_replica: Option, } -/// A validated client-facing node address: a literal IP or a DNS hostname. -/// -/// Hostnames follow RFC 1123: ASCII letters, digits and hyphens in labels of -/// 1-63 characters that do not start or end with a hyphen, at most -/// [`MAX_HOSTNAME_LEN`] characters total, no port and no trailing dot. Names -/// consisting solely of digits and dots are rejected as malformed IPv4 rather -/// than accepted as hostnames, so `10.0.0.256` fails loudly instead of being -/// handed to DNS. Hostnames normalize to lowercase and IPs to their canonical -/// form ([`IpAddr`]), so textual variants of one address (`Broker.Example.COM`, -/// `2001:DB8::1`, `[2001:db8::1]`) compare equal. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub enum AdvertisedAddress { - Ip(IpAddr), - Hostname(String), -} - -impl AdvertisedAddress { - /// Render `host:port` for a URL or endpoint listing, bracketing IPv6 - /// hosts (`[::1]:8080`) so the port separator stays unambiguous. - pub fn authority(&self, port: u16) -> String { - match self { - Self::Ip(ip) => SocketAddr::new(*ip, port).to_string(), - Self::Hostname(hostname) => format!("{hostname}:{port}"), - } - } -} - -impl FromStr for AdvertisedAddress { - type Err = AdvertisedAddressError; - - fn from_str(address: &str) -> Result { - if address.is_empty() { - return Err(AdvertisedAddressError::Empty); - } - if let Ok(ip) = address.parse::() { - return Ok(Self::Ip(ip)); - } - // URL-style bracketed IPv6 (`[2001:db8::1]`) is unambiguous; accept - // it and store the inner address. - if let Some(inner) = address - .strip_prefix('[') - .and_then(|rest| rest.strip_suffix(']')) - && let Ok(ip) = inner.parse::() - { - return Ok(Self::Ip(IpAddr::V6(ip))); - } - if let Some((host, port)) = address.rsplit_once(':') { - // `host:port` and `[v6]:port` are the common misconfigurations; - // anything else with a colon can only be a broken IPv6 literal, - // since ':' never appears in a hostname. - let bracketed_host = host.starts_with('[') && host.ends_with(']'); - if !port.is_empty() - && port.bytes().all(|byte| byte.is_ascii_digit()) - && (bracketed_host || !host.contains(':')) - { - return Err(AdvertisedAddressError::PortNotAllowed); - } - return Err(AdvertisedAddressError::MalformedIpv6); - } - if address.len() > MAX_HOSTNAME_LEN { - return Err(AdvertisedAddressError::HostnameTooLong { - length: address.len(), - }); - } - let mut all_labels_numeric = true; - for label in address.split('.') { - if label.is_empty() { - return Err(AdvertisedAddressError::EmptyLabel); - } - if label.len() > MAX_HOSTNAME_LABEL_LEN { - return Err(AdvertisedAddressError::LabelTooLong { - label: label.to_owned(), - }); - } - if label.starts_with('-') || label.ends_with('-') { - return Err(AdvertisedAddressError::LabelHyphen { - label: label.to_owned(), - }); - } - if let Some(character) = label - .chars() - .find(|character| !character.is_ascii_alphanumeric() && *character != '-') - { - return Err(AdvertisedAddressError::InvalidCharacter { character }); - } - all_labels_numeric &= label.bytes().all(|byte| byte.is_ascii_digit()); - } - if all_labels_numeric { - return Err(AdvertisedAddressError::MalformedIpv4); - } - // DNS resolution is case-insensitive; normalizing here makes equality - // (and thus endpoint-conflict detection) case-insensitive too. - Ok(Self::Hostname(address.to_ascii_lowercase())) - } -} - -impl fmt::Display for AdvertisedAddress { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Ip(ip) => write!(formatter, "{ip}"), - Self::Hostname(hostname) => write!(formatter, "{hostname}"), - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum AdvertisedAddressError { - Empty, - PortNotAllowed, - MalformedIpv4, - MalformedIpv6, - HostnameTooLong { length: usize }, - EmptyLabel, - LabelTooLong { label: String }, - LabelHyphen { label: String }, - InvalidCharacter { character: char }, -} - -impl fmt::Display for AdvertisedAddressError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Empty => write!(formatter, "address cannot be empty"), - Self::PortNotAllowed => write!( - formatter, - "address must not include a port; ports are configured in cluster.nodes.ports" - ), - Self::MalformedIpv4 => write!( - formatter, - "address consists only of digits and dots but is not a valid IPv4 address" - ), - Self::MalformedIpv6 => write!( - formatter, - "address contains ':' but is not a valid IPv6 address, and ':' cannot appear in a hostname" - ), - Self::HostnameTooLong { length } => write!( - formatter, - "hostname is {length} characters long; the limit is {MAX_HOSTNAME_LEN}" - ), - Self::EmptyLabel => write!( - formatter, - "hostname contains an empty label (leading, trailing, or doubled dot)" - ), - Self::LabelTooLong { label } => write!( - formatter, - "hostname label '{label}' exceeds {MAX_HOSTNAME_LABEL_LEN} characters" - ), - Self::LabelHyphen { label } => write!( - formatter, - "hostname label '{label}' cannot start or end with a hyphen" - ), - Self::InvalidCharacter { character } => write!( - formatter, - "character '{character}' is not allowed in a hostname (allowed: ASCII letters, digits, '-', '.')" - ), - } - } -} - -impl std::error::Error for AdvertisedAddressError {} - -/// Whether cluster-wide JWT key material exists: a configured `http.jwt` -/// secret, or the signing key derived from the cluster PSK. When it does, a -/// bearer minted on any node verifies on every node - the invariant -/// follower-to-primary HTTP forwarding depends on. Callers gate `http.enabled` -/// themselves; this covers only the key material. -/// -/// Forwarding targets resolve from the roster (`ip:ports.http`); the config -/// validator unconditionally requires a roster port for every enabled -/// transport, so a forward never dials a node without a declared http port. -pub fn http_forwarding_key_material(jwt: &HttpJwtConfig, cluster: &ClusterConfig) -> bool { - cluster.enabled - && ((cluster.auth.enabled && !cluster.auth.shared_secret.is_empty()) - || !jwt.encoding_secret.is_empty() - || !jwt.decoding_secret.is_empty()) -} - -impl Validatable for ClusterConfig { - fn validate(&self) -> Result<(), ConfigurationError> { - // Ahead of the enabled gate: the top-level rule against - // message_bus.peer_queue_capacity holds repair_chunk_max unconditionally, - // so a single-node config skipping these would be bound by the - // cross-section rule while its own floor and ceiling went unchecked. - if self.repair_chunk_max == 0 { - eprintln!("Invalid cluster configuration: cluster.repair_chunk_max must be > 0"); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if self.repair_chunk_max > MAX_REPAIR_CHUNK_MAX { - eprintln!( - "Invalid cluster configuration: cluster.repair_chunk_max ({}) exceeds the maximum \ - ({MAX_REPAIR_CHUNK_MAX})", - self.repair_chunk_max - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - if !self.enabled { - return Ok(()); - } - - if self.name.trim().is_empty() { - eprintln!("Invalid cluster configuration: cluster name cannot be empty"); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - // `0` / `disabled` / `unlimited` all parse to a zero duration and - // land here too: there is no way to switch the liveness window off. - if self.heartbeat_timeout.get_duration() < MIN_CLUSTER_HEARTBEAT_TIMEOUT { - eprintln!( - "Invalid cluster configuration: cluster.heartbeat_timeout '{}' must be at least {}s \ - (the primary signals liveness through its commit broadcast; a shorter window \ - elects on every scheduling hiccup)", - self.heartbeat_timeout, - MIN_CLUSTER_HEARTBEAT_TIMEOUT.as_secs() - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - // The commit broadcast is the cluster's liveness feed and the prepare - // retransmit its recovery timer; both size consensus timers that have - // to advance. `0` / `disabled` / `unlimited` all collapse to a zero - // duration, which would stall the timer - reject them. - if self.commit_broadcast_interval.get_duration().is_zero() { - eprintln!( - "Invalid cluster configuration: cluster.commit_broadcast_interval must be nonzero \ - (it drives the primary's liveness broadcast)" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if self.prepare_retransmit_interval.get_duration().is_zero() { - eprintln!( - "Invalid cluster configuration: cluster.prepare_retransmit_interval must be \ - nonzero (it drives prepare retransmission)" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - // The liveness window must span several commit broadcasts so a single - // delayed broadcast never trips a view change on a healthy primary. - let min_heartbeat = self - .commit_broadcast_interval - .get_duration() - .saturating_mul(MIN_HEARTBEAT_TO_COMMIT_BROADCAST_RATIO); - if self.heartbeat_timeout.get_duration() < min_heartbeat { - eprintln!( - "Invalid cluster configuration: cluster.heartbeat_timeout '{}' must be at least \ - {}x cluster.commit_broadcast_interval '{}' so the liveness window spans several \ - broadcasts", - self.heartbeat_timeout, - MIN_HEARTBEAT_TO_COMMIT_BROADCAST_RATIO, - self.commit_broadcast_interval - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - // The three view-change timers each size a consensus timer that has to - // advance; `0` / `disabled` / `unlimited` all collapse to zero and - // would stall it - reject them. - if self - .view_change_retransmit_interval - .get_duration() - .is_zero() - { - eprintln!( - "Invalid cluster configuration: cluster.view_change_retransmit_interval must be \ - nonzero (it drives StartViewChange / DoViewChange retransmission)" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if self.view_change_status_timeout.get_duration().is_zero() { - eprintln!( - "Invalid cluster configuration: cluster.view_change_status_timeout must be nonzero \ - (it backstops a stalled view change)" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if self - .request_start_view_retransmit_interval - .get_duration() - .is_zero() - { - eprintln!( - "Invalid cluster configuration: cluster.request_start_view_retransmit_interval \ - must be nonzero (it drives RequestStartView retransmission)" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - // The status backstop must span several retransmits so a few dropped - // view-change messages retransmit rather than escalating a progressing - // view change into a fresh cluster-wide election. - let min_status = self - .view_change_retransmit_interval - .get_duration() - .saturating_mul(MIN_STATUS_TO_RETRANSMIT_RATIO); - if self.view_change_status_timeout.get_duration() < min_status { - eprintln!( - "Invalid cluster configuration: cluster.view_change_status_timeout '{}' must be at \ - least {}x cluster.view_change_retransmit_interval '{}' so a stalled view change \ - retransmits before it escalates to an election", - self.view_change_status_timeout, - MIN_STATUS_TO_RETRANSMIT_RATIO, - self.view_change_retransmit_interval - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - // A recovering replica needs at least one probe before it may give up - // and elect; the ceiling is a typo guard (see MAX_VIEW_PROBE_ATTEMPTS). - if self.view_probe_attempts_max == 0 { - eprintln!( - "Invalid cluster configuration: cluster.view_probe_attempts_max must be >= 1" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if self.view_probe_attempts_max > MAX_VIEW_PROBE_ATTEMPTS { - eprintln!( - "Invalid cluster configuration: cluster.view_probe_attempts_max ({}) exceeds the \ - maximum ({MAX_VIEW_PROBE_ATTEMPTS})", - self.view_probe_attempts_max - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - // The repair retry interval sizes a tick threshold that has to advance; - // `0` / `disabled` / `unlimited` all collapse to zero and would wedge - // every stalled repair stream - reject them. - if self.repair_retry_interval.get_duration().is_zero() { - eprintln!( - "Invalid cluster configuration: cluster.repair_retry_interval must be nonzero \ - (it paces stalled-repair retries)" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - if self.nodes.is_empty() { - eprintln!( - "Invalid cluster configuration: cluster.nodes must contain at least one entry when cluster is enabled" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - // VSR needs every replica to have a stable, unique id strictly - // less than the total replica count. Duplicate ids would split the - // cluster into two replicas claiming the same slot; out-of-range - // ids never win a primary election. Both are unrecoverable at - // runtime - fail fast at startup. - let total_replicas = u8::try_from(self.nodes.len()).map_err(|_| { - eprintln!("Invalid cluster configuration: more than 255 replicas is unsupported"); - ConfigurationError::InvalidConfigurationValue - })?; - - let mut seen_ids = std::collections::HashSet::new(); - let mut seen_names = std::collections::HashSet::new(); - let mut used_endpoints = std::collections::HashSet::new(); - let mut advertised_endpoints: Vec = Vec::new(); - - for node in &self.nodes { - if node.name.trim().is_empty() { - eprintln!("Invalid cluster configuration: node name cannot be empty"); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - if node.ip.trim().is_empty() { - eprintln!( - "Invalid cluster configuration: IP cannot be empty for node '{}'", - node.name - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - if !seen_names.insert(node.name.clone()) { - eprintln!( - "Invalid cluster configuration: duplicate node name '{}' found", - node.name - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - if node.replica_id >= total_replicas { - eprintln!( - "Invalid cluster configuration: replica_id {} for node '{}' must be < total replica count {total_replicas}", - node.replica_id, node.name - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - if !seen_ids.insert(node.replica_id) { - eprintln!( - "Invalid cluster configuration: duplicate replica_id {} (two nodes claim the same slot)", - node.replica_id - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - let client_ports = [ - ("TCP", node.ports.tcp), - ("QUIC", node.ports.quic), - ("HTTP", node.ports.http), - ("WebSocket", node.ports.websocket), - ]; - let replica_port = ("TCP_REPLICA", node.ports.tcp_replica); - - for (name, port_opt) in client_ports.into_iter().chain([replica_port]) { - if let Some(port) = port_opt { - if port == 0 { - eprintln!( - "Invalid cluster configuration: {} port cannot be 0 for node '{}'", - name, node.name - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - let endpoint = format!("{}:{}", node.ip, port); - if !used_endpoints.insert(endpoint.clone()) { - eprintln!( - "Invalid cluster configuration: port conflict - {endpoint} is already bound (node '{}', transport {name})", - node.name - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - } - } - - // An advertised address must parse strictly (IP or RFC 1123 - // hostname): the value is handed verbatim to every client via - // cluster metadata and redirect URLs, so a bad one poisons them - // all. The roster `ip` predates this check and is only validated - // as non-empty (Docker service names with underscores exist in - // the wild), so when it backs the client endpoints an unparsable - // value falls back to raw-string comparison instead of failing - // boot. - let client_address = match node.advertised_address.as_deref() { - Some(advertised_address) => match advertised_address.parse::() { - Ok(address) => Some(address), - Err(error) => { - eprintln!( - "Invalid cluster configuration: advertised_address '{advertised_address}' for node '{}': {error}", - node.name - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - }, - None => node.ip.parse::().ok(), - }; - - if node.advertised_addresses.len() > MAX_ADVERTISED_SELECTORS { - eprintln!( - "Invalid cluster configuration: node '{}' declares {} advertised_addresses \ - selectors, exceeding the maximum ({MAX_ADVERTISED_SELECTORS})", - node.name, - node.advertised_addresses.len() - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - // Selector CIDRs and addresses feed clients the same way the - // catch-all advertised address does, so they get the same strict - // parse. Networks are compared truncated (`10.0.1.0/16` == - // `10.0.0.0/16`) and canonicalized (`::ffff:10.0.0.0/104` == - // `10.0.0.0/8`) since matching truncates and canonicalizes too. - // Parsed before the catch-all enters the conflict pool because - // every entry's effective client set depends on the node's full - // selector list. - let mut selectors = Vec::with_capacity(node.advertised_addresses.len()); - let mut seen_selector_cidrs = std::collections::HashSet::new(); - for selector in &node.advertised_addresses { - let client_cidr = match selector.client_cidr.parse::() { - Ok(client_cidr) => canonical_ip_net(client_cidr.trunc()), - Err(error) => { - eprintln!( - "Invalid cluster configuration: advertised_addresses client_cidr '{}' for node '{}': {error}", - selector.client_cidr, node.name - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - }; - if !seen_selector_cidrs.insert(client_cidr) { - eprintln!( - "Invalid cluster configuration: duplicate advertised_addresses client_cidr '{}' for node '{}'", - selector.client_cidr, node.name - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - let address = match selector.address.parse::() { - Ok(address) => address, - Err(error) => { - eprintln!( - "Invalid cluster configuration: advertised_addresses address '{}' for node '{}': {error}", - selector.address, node.name - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - }; - selectors.push((client_cidr, address)); - } - let selector_ranges: Vec = selectors - .iter() - .map(|(network, _)| ClientAddressRange::from(network)) - .collect(); - - // Endpoint conflicts are checked across every node's selectors - // and catch-all on effective client sets - the clients an entry - // actually wins after this node's longest-prefix match. Two - // nodes may reuse one host:port as long as the winning sets stay - // disjoint (that is the feature, and it includes a per-subnet - // override shadowing the same node's wider selector); a conflict - // means some client wins both entries and would resolve both - // nodes to one endpoint. The catch-all is an implicit - // match-everything-else selector, so it pools the same way. A - // roster ip that fails the strict parse skips the pool: it can - // never equal a parsed host, and two raw ips sharing host:port - // are already rejected by the bind-endpoint check above. - if let Some(address) = &client_address { - let catch_all_clients = EffectiveClients::for_catch_all(&selector_ranges); - for (name, port) in &client_ports { - if let Some(port) = port { - insert_advertised_endpoint( - &mut advertised_endpoints, - AdvertisedEndpoint { - node_name: &node.name, - transport: name, - network: None, - clients: catch_all_clients.clone(), - host: address.clone(), - port: *port, - }, - )?; - } - } - } - - for (selector_index, (client_cidr, address)) in selectors.iter().enumerate() { - let sibling_ranges: Vec = selector_ranges - .iter() - .enumerate() - .filter(|(other_index, _)| *other_index != selector_index) - .map(|(_, range)| *range) - .collect(); - let clients = EffectiveClients::for_selector(client_cidr, &sibling_ranges); - for (name, port) in &client_ports { - if let Some(port) = port { - insert_advertised_endpoint( - &mut advertised_endpoints, - AdvertisedEndpoint { - node_name: &node.name, - transport: name, - network: Some(*client_cidr), - clients: clients.clone(), - host: address.clone(), - port: *port, - }, - )?; - } - } - } - } - - // Replica-auth PSK (only reached when the cluster is enabled; the early - // return above skips these while it is disabled). When auth is enabled - // the key is mandatory; any configured key must clear the length floor - - // a typo guard that fires with auth off too, though only while the - // cluster itself is enabled. - let secret_len = self.auth.shared_secret.len(); - if self.auth.enabled && self.auth.shared_secret.is_empty() { - eprintln!( - "Invalid cluster configuration: cluster.auth.shared_secret must be set when cluster.auth.enabled is true" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if !self.auth.shared_secret.is_empty() && secret_len < MIN_SHARED_SECRET_LEN { - eprintln!( - "Invalid cluster configuration: cluster.auth.shared_secret must be >= {MIN_SHARED_SECRET_LEN} bytes" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - // Rotation window key: same typo guard as the primary, plus a - // distinctness check - a window equal to the primary means the - // operator rolled the config without changing the key, so the - // "rotation" would silently be a no-op. - if !self.auth.previous_shared_secret.is_empty() { - if self.auth.previous_shared_secret.len() < MIN_SHARED_SECRET_LEN { - eprintln!( - "Invalid cluster configuration: cluster.auth.previous_shared_secret must be >= {MIN_SHARED_SECRET_LEN} bytes" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if self.auth.previous_shared_secret == self.auth.shared_secret { - eprintln!( - "Invalid cluster configuration: cluster.auth.previous_shared_secret must differ from cluster.auth.shared_secret (an identical window is a no-op rotation)" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - } - - // Replica TLS. Both cert modes run one-directional TLS (no client - // certificate anywhere), so TLS only authenticates the acceptor to - // the dialer; peer authentication comes solely from the PSK - // handshake. Without it any TLS-capable host could register as a - // replica - require auth in both modes. CA mode (the default) - // additionally needs all three PEM paths: cert/key for this node's - // acceptor side, ca_file as the dialer's trust anchor. - if self.tls.enabled { - if !self.auth.enabled { - eprintln!( - "Invalid cluster configuration: cluster.tls.enabled = true requires cluster.auth.enabled = true (TLS authenticates the acceptor only; the PSK handshake authenticates the peer)" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if !self.tls.self_signed { - for (field, value) in [ - ("cert_file", &self.tls.cert_file), - ("key_file", &self.tls.key_file), - ("ca_file", &self.tls.ca_file), - ] { - if value.trim().is_empty() { - eprintln!( - "Invalid cluster configuration: cluster.tls.{field} must be set when cluster.tls.enabled = true and self_signed = false" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - } - } - } - - Ok(()) - } -} - -/// One advertised client endpoint and the clients it wins, pooled by -/// [`ClusterConfig::validate`] so selectors and catch-all conflict-check -/// against each other. `network: None` is the catch-all (`advertised_address`, -/// or the roster `ip` as fallback); `clients` is the entry's effective set -/// after its node's longest-prefix shadowing. -struct AdvertisedEndpoint<'roster> { - node_name: &'roster str, - transport: &'static str, - network: Option, - clients: EffectiveClients, - host: AdvertisedAddress, - port: u16, -} - -impl AdvertisedEndpoint<'_> { - /// True when some client would resolve both entries to one host:port on - /// two different nodes. Effective client sets already encode each node's - /// longest-prefix shadowing, so nested networks conflict only where the - /// wider entry still wins some client that the other node's entry also - /// wins. Entries of one node never conflict: their effective sets are - /// disjoint by construction. - fn conflicts_with(&self, other: &Self) -> bool { - self.node_name != other.node_name - && self.port == other.port - && self.host == other.host - && self.clients.overlaps(&other.clients) - } - - fn authority(&self) -> String { - self.host.authority(self.port) - } - - fn network_description(&self) -> String { - match self.network { - Some(network) => format!("client_cidr {network}"), - None => "every client network (catch-all)".to_owned(), - } - } -} - -/// The clients an advertised entry actually wins under its node's -/// longest-prefix match, built by [`ClusterConfig::validate`] for the -/// cross-node conflict scan. -#[derive(Clone)] -struct EffectiveClients { - /// Sorted disjoint ranges of winning client addresses. - ranges: Vec, - /// The catch-all also wins clients whose peer address the transport - /// could not produce ([`ResolvedClusterNode::advertised_for`] with no - /// client IP), so two catch-all overlap even when selectors cover both - /// address families. - serves_unknown_peers: bool, -} - -impl EffectiveClients { - /// A selector wins its network minus the sibling networks nested inside - /// it (longer prefixes take the node's LPM). `sibling_ranges` must - /// exclude the selector's own network. - fn for_selector(network: &IpNet, sibling_ranges: &[ClientAddressRange]) -> Self { - Self { - ranges: ClientAddressRange::from(network).subtract_nested(sibling_ranges), - serves_unknown_peers: false, - } - } - - /// The catch-all wins every client no selector matches, in both address - /// families, plus unknown-peer clients. - fn for_catch_all(selector_ranges: &[ClientAddressRange]) -> Self { - let mut ranges = ClientAddressRange::FULL_IPV4.subtract_nested(selector_ranges); - ranges.extend(ClientAddressRange::FULL_IPV6.subtract_nested(selector_ranges)); - Self { - ranges, - serves_unknown_peers: true, - } - } - - fn overlaps(&self, other: &Self) -> bool { - if self.serves_unknown_peers && other.serves_unknown_peers { - return true; - } - self.ranges.iter().any(|range| { - other.ranges.iter().any(|other_range| { - range.is_ipv4 == other_range.is_ipv4 - && range.first <= other_range.last - && other_range.first <= range.last - }) - }) - } -} - -/// Inclusive range of client addresses within one family. Client IPs -/// canonicalize to v4 before matching, so v4 and v6 networks match disjoint -/// client populations and a range never spans families. -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -struct ClientAddressRange { - is_ipv4: bool, - first: u128, - last: u128, -} - -impl ClientAddressRange { - const FULL_IPV4: Self = Self { - is_ipv4: true, - first: 0, - last: u32::MAX as u128, - }; - const FULL_IPV6: Self = Self { - is_ipv4: false, - first: 0, - last: u128::MAX, - }; - - /// `self` minus every range nested inside it, as sorted disjoint - /// leftovers. CIDR networks are nested or disjoint, never partially - /// overlapping, so a range outside `self` is either disjoint from it - /// (subtracts nothing) or contains it (a shorter prefix, which loses - /// LPM and also subtracts nothing). - fn subtract_nested(self, ranges: &[Self]) -> Vec { - let mut nested: Vec = ranges - .iter() - .filter(|range| { - range.is_ipv4 == self.is_ipv4 - && range.first >= self.first - && range.last <= self.last - }) - .copied() - .collect(); - nested.sort_unstable(); - let mut remaining = Vec::new(); - let mut cursor = Some(self.first); - for nested_range in nested { - let Some(next_free) = cursor else { break }; - if nested_range.first > next_free { - remaining.push(Self { - is_ipv4: self.is_ipv4, - first: next_free, - last: nested_range.first - 1, - }); - } - cursor = nested_range - .last - .checked_add(1) - .map(|after| after.max(next_free)); - } - if let Some(next_free) = cursor - && next_free <= self.last - { - remaining.push(Self { - is_ipv4: self.is_ipv4, - first: next_free, - last: self.last, - }); - } - remaining - } -} - -impl From<&IpNet> for ClientAddressRange { - fn from(network: &IpNet) -> Self { - match network { - IpNet::V4(network) => Self { - is_ipv4: true, - first: u128::from(u32::from(network.network())), - last: u128::from(u32::from(network.broadcast())), - }, - IpNet::V6(network) => Self { - is_ipv4: false, - first: u128::from(network.network()), - last: u128::from(network.broadcast()), - }, - } - } -} - -fn insert_advertised_endpoint<'roster>( - advertised_endpoints: &mut Vec>, - endpoint: AdvertisedEndpoint<'roster>, -) -> Result<(), ConfigurationError> { - if let Some(existing) = advertised_endpoints - .iter() - .find(|existing| existing.conflicts_with(&endpoint)) - { - eprintln!( - "Invalid cluster configuration: advertised client endpoint conflict - {} is advertised for {} (node '{}', transport {}) and for {} (node '{}', transport {}); their effective client sets overlap after longest-prefix shadowing, so a client in the overlap would resolve both nodes to one endpoint", - endpoint.authority(), - endpoint.network_description(), - endpoint.node_name, - endpoint.transport, - existing.network_description(), - existing.node_name, - existing.transport, - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - advertised_endpoints.push(endpoint); - Ok(()) -} - #[cfg(test)] mod tests { use super::*; @@ -1392,21 +145,10 @@ mod tests { let config = ClusterConfig { enabled: true, name: "iggy-cluster".to_owned(), - heartbeat_timeout: default_heartbeat_timeout(), - commit_broadcast_interval: default_commit_broadcast_interval(), - prepare_retransmit_interval: default_prepare_retransmit_interval(), - view_change_retransmit_interval: default_view_change_retransmit_interval(), - view_change_status_timeout: default_view_change_status_timeout(), - request_start_view_retransmit_interval: default_request_start_view_retransmit_interval( - ), - view_probe_attempts_max: default_view_probe_attempts_max(), - repair_retry_interval: default_repair_retry_interval(), - repair_chunk_max: default_repair_chunk_max(), nodes: Vec::new(), auth: ClusterAuthConfig { enabled: true, shared_secret: "current-psk-MUST-NOT-be-persisted".to_owned(), - previous_shared_secret: "retiring-psk-MUST-NOT-be-persisted".to_owned(), }, tls: ClusterTlsConfig::default(), }; @@ -1420,1184 +162,4 @@ mod tests { "shared_secret field present in serialized config: {serialized}" ); } - - #[test] - fn cluster_node_rejects_unknown_fields() { - let error = serde_json::from_str::( - r#"{ - "name": "node-0", - "ip": "10.0.0.1", - "advertise_address": "203.0.113.1", - "replica_id": 0, - "ports": {} - }"#, - ) - .expect_err("misspelled advertised_address must be rejected"); - - assert!( - error - .to_string() - .contains("unknown field `advertise_address`"), - "unexpected deserialization error: {error}" - ); - } - - #[test] - fn advertised_addresses_env_expansion_is_capped() { - // The selectors Vec nests inside the nodes Vec, so the derive's - // index ceilings multiply; without the field's max_elements cap the - // default of 256x256 adds ~131k leaked mappings to every boot. - let mappings = ::env_mappings(); - assert!( - mappings - .iter() - .any(|mapping| mapping.env_name.contains("ADVERTISED_ADDRESSES_15_")), - "selector index 15 must stay reachable by env override" - ); - assert!( - !mappings - .iter() - .any(|mapping| mapping.env_name.contains("ADVERTISED_ADDRESSES_16_")), - "selector env expansion must stop at max_elements = 16" - ); - } -} - -#[cfg(test)] -mod advertised_address_tests { - use super::*; - - #[test] - fn parses_ip_literals_to_canonical_form() { - assert_eq!( - "203.0.113.1".parse::(), - Ok(AdvertisedAddress::Ip("203.0.113.1".parse().unwrap())) - ); - for equivalent_address in ["2001:DB8::1", "2001:db8:0:0:0:0:0:1", "[2001:db8::1]"] { - assert_eq!( - equivalent_address.parse::(), - Ok(AdvertisedAddress::Ip("2001:db8::1".parse().unwrap())), - "'{equivalent_address}' must parse to canonical 2001:db8::1" - ); - } - } - - #[test] - fn normalizes_hostname_to_lowercase() { - let address = "Broker-1.Example.COM".parse::(); - assert_eq!( - address, - Ok(AdvertisedAddress::Hostname( - "broker-1.example.com".to_owned() - )) - ); - } - - #[test] - fn authority_brackets_ipv6_hosts_only() { - let cases = [ - ("203.0.113.1", "203.0.113.1:8090"), - ("2001:db8::1", "[2001:db8::1]:8090"), - ("broker-1.example.com", "broker-1.example.com:8090"), - ]; - for (host, expected_authority) in cases { - let address = host.parse::().expect("valid address"); - assert_eq!(address.authority(8090), expected_authority); - } - } - - #[test] - fn rejects_port_suffixes() { - for address_with_port in ["example.com:8090", "10.0.0.1:8090", "[2001:db8::1]:8090"] { - assert_eq!( - address_with_port.parse::(), - Err(AdvertisedAddressError::PortNotAllowed), - "'{address_with_port}' must be rejected as host:port" - ); - } - } - - #[test] - fn rejects_dotted_numeric_strings_as_malformed_ipv4() { - for malformed_ip in ["10.0.0.256", "192.168.1", "12345"] { - assert_eq!( - malformed_ip.parse::(), - Err(AdvertisedAddressError::MalformedIpv4), - "'{malformed_ip}' must not pass as a hostname" - ); - } - } - - #[test] - fn rejects_broken_ipv6_literals() { - for broken_ipv6 in ["2001:db8:::1", "[2001:db8::zz]", "::1::2"] { - assert_eq!( - broken_ipv6.parse::(), - Err(AdvertisedAddressError::MalformedIpv6), - "'{broken_ipv6}' must be rejected as malformed IPv6" - ); - } - } -} - -#[cfg(test)] -mod advertised_for_tests { - use super::*; - - fn node_with_selectors(selectors: Vec) -> ClusterNodeConfig { - ClusterNodeConfig { - name: "node-0".to_owned(), - ip: "10.0.1.5".to_owned(), - advertised_address: Some("203.0.113.10".to_owned()), - advertised_addresses: selectors, - replica_id: 0, - ports: TransportPorts::default(), - } - } - - fn selector(client_cidr: &str, address: &str) -> AdvertisedAddressSelector { - AdvertisedAddressSelector { - client_cidr: client_cidr.to_owned(), - address: address.to_owned(), - } - } - - fn resolved(node: ClusterNodeConfig) -> ResolvedClusterNode { - node.into() - } - - fn ip(address: &str) -> IpAddr { - address.parse().unwrap() - } - - #[test] - fn falls_back_to_advertised_address_without_selectors() { - let node = node_with_selectors(Vec::new()); - assert_eq!( - resolved(node).advertised_for(Some(ip("10.0.0.7"))), - Some(&AdvertisedAddress::Ip(ip("203.0.113.10"))) - ); - } - - #[test] - fn falls_back_to_roster_ip_without_advertised_address() { - let mut node = node_with_selectors(Vec::new()); - node.advertised_address = None; - assert_eq!( - resolved(node).advertised_for(Some(ip("10.0.0.7"))), - Some(&AdvertisedAddress::Ip(ip("10.0.1.5"))) - ); - } - - #[test] - fn is_none_when_no_fallback_parses() { - let mut node = node_with_selectors(Vec::new()); - node.advertised_address = None; - node.ip = "iggy_node".to_owned(); - assert_eq!(resolved(node).advertised_for(Some(ip("10.0.0.7"))), None); - } - - #[test] - fn matching_selector_beats_advertised_address() { - let node = node_with_selectors(vec![selector("10.0.0.0/16", "10.0.1.5")]); - assert_eq!( - resolved(node).advertised_for(Some(ip("10.0.200.7"))), - Some(&AdvertisedAddress::Ip(ip("10.0.1.5"))) - ); - } - - #[test] - fn unmatched_client_falls_back_to_advertised_address() { - let node = node_with_selectors(vec![selector("10.0.0.0/16", "10.0.1.5")]); - assert_eq!( - resolved(node).advertised_for(Some(ip("192.168.0.7"))), - Some(&AdvertisedAddress::Ip(ip("203.0.113.10"))) - ); - } - - #[test] - fn unknown_client_ip_falls_back_to_advertised_address() { - let node = node_with_selectors(vec![selector("10.0.0.0/16", "10.0.1.5")]); - assert_eq!( - resolved(node).advertised_for(None), - Some(&AdvertisedAddress::Ip(ip("203.0.113.10"))) - ); - } - - #[test] - fn longest_prefix_wins_regardless_of_declaration_order() { - let node = resolved(node_with_selectors(vec![ - selector("10.0.0.0/8", "10.255.255.1"), - selector("10.0.0.0/16", "10.0.1.5"), - ])); - assert_eq!( - node.advertised_for(Some(ip("10.0.200.7"))), - Some(&AdvertisedAddress::Ip(ip("10.0.1.5"))), - "the /16 must win over the /8 even though it is declared second" - ); - assert_eq!( - node.advertised_for(Some(ip("10.9.0.7"))), - Some(&AdvertisedAddress::Ip(ip("10.255.255.1"))), - "a client outside the /16 but inside the /8 must match the /8" - ); - } - - #[test] - fn equal_prefix_matches_resolve_deterministically_to_first_declared() { - // No validated config reaches this state: these networks truncate to - // one /16, which validation rejects as a duplicate. Pinned anyway so - // a future relaxation of that rule cannot make resolution - // order-dependent. - let node = node_with_selectors(vec![ - selector("10.0.1.0/16", "10.0.1.5"), - selector("10.0.2.0/16", "10.0.2.5"), - ]); - assert_eq!( - resolved(node).advertised_for(Some(ip("10.0.200.7"))), - Some(&AdvertisedAddress::Ip(ip("10.0.1.5"))) - ); - } - - #[test] - fn v4_mapped_v6_client_matches_v4_cidr() { - // A dual-stack listener reports v4 peers as `::ffff:a.b.c.d`. - let node = node_with_selectors(vec![selector("10.0.0.0/16", "10.0.1.5")]); - assert_eq!( - resolved(node).advertised_for(Some(ip("::ffff:10.0.0.7"))), - Some(&AdvertisedAddress::Ip(ip("10.0.1.5"))) - ); - } - - #[test] - fn v4_mapped_v6_selector_cidr_matches_v4_client() { - // The mirror case: the CIDR side canonicalizes at build, so - // `::ffff:10.0.0.0/104` matches like `10.0.0.0/8` instead of being - // a silently dead selector. - let node = node_with_selectors(vec![selector("::ffff:10.0.0.0/104", "10.0.1.5")]); - assert_eq!( - resolved(node).advertised_for(Some(ip("10.0.0.7"))), - Some(&AdvertisedAddress::Ip(ip("10.0.1.5"))) - ); - } - - #[test] - fn v6_selector_matches_v6_client() { - let node = node_with_selectors(vec![selector("2001:db8::/32", "2001:db8::1")]); - assert_eq!( - resolved(node).advertised_for(Some(ip("2001:db8::7"))), - Some(&AdvertisedAddress::Ip(ip("2001:db8::1"))) - ); - } - - #[test] - fn selector_address_may_be_a_hostname() { - let node = node_with_selectors(vec![selector("10.0.0.0/16", "Broker.Internal.Example")]); - assert_eq!( - resolved(node).advertised_for(Some(ip("10.0.0.7"))), - Some(&AdvertisedAddress::Hostname( - "broker.internal.example".to_owned() - )) - ); - } -} - -#[cfg(test)] -mod cluster_validate_tests { - use super::*; - - fn node(name: &str, id: u8) -> ClusterNodeConfig { - ClusterNodeConfig { - name: name.to_string(), - ip: "127.0.0.1".to_string(), - advertised_address: None, - advertised_addresses: Vec::new(), - replica_id: id, - ports: TransportPorts::default(), - } - } - - fn selector(client_cidr: &str, address: &str) -> AdvertisedAddressSelector { - AdvertisedAddressSelector { - client_cidr: client_cidr.to_owned(), - address: address.to_owned(), - } - } - - fn cfg(nodes: Vec) -> ClusterConfig { - ClusterConfig { - enabled: true, - name: "iggy-cluster".to_string(), - heartbeat_timeout: default_heartbeat_timeout(), - commit_broadcast_interval: default_commit_broadcast_interval(), - prepare_retransmit_interval: default_prepare_retransmit_interval(), - view_change_retransmit_interval: default_view_change_retransmit_interval(), - view_change_status_timeout: default_view_change_status_timeout(), - request_start_view_retransmit_interval: default_request_start_view_retransmit_interval( - ), - view_probe_attempts_max: default_view_probe_attempts_max(), - repair_retry_interval: default_repair_retry_interval(), - repair_chunk_max: default_repair_chunk_max(), - nodes, - auth: ClusterAuthConfig::default(), - tls: ClusterTlsConfig::default(), - } - } - - #[test] - fn validate_rejects_sub_minimum_heartbeat_timeout() { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.heartbeat_timeout = IggyDuration::new(Duration::from_millis(500)); - assert!(c.validate().is_err()); - // The "disabled" / "unlimited" sentinels collapse to zero and must - // be rejected the same way. - c.heartbeat_timeout = IggyDuration::new(Duration::ZERO); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_rejects_zero_commit_broadcast_interval() { - // `0` / `disabled` / `unlimited` all collapse to zero and stall the - // liveness broadcast. - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.commit_broadcast_interval = IggyDuration::new(Duration::ZERO); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_rejects_zero_prepare_retransmit_interval() { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.prepare_retransmit_interval = IggyDuration::new(Duration::ZERO); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_rejects_heartbeat_below_commit_broadcast_ratio() { - // 3s clears the absolute 2s floor but is still < 4x the 1s broadcast, - // so the ratio rule is what rejects here, not the floor. - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.heartbeat_timeout = IggyDuration::new(Duration::from_secs(3)); - c.commit_broadcast_interval = IggyDuration::new(Duration::from_secs(1)); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_accepts_heartbeat_at_commit_broadcast_ratio() { - // Exactly 4x the broadcast (and above the 2s floor) must pass. - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.heartbeat_timeout = IggyDuration::new(Duration::from_secs(4)); - c.commit_broadcast_interval = IggyDuration::new(Duration::from_secs(1)); - assert!(c.validate().is_ok()); - } - - #[test] - fn validate_rejects_zero_view_change_retransmit_interval() { - // `0` / `disabled` / `unlimited` all collapse to zero and stall the - // view-change retransmit timers. - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.view_change_retransmit_interval = IggyDuration::new(Duration::ZERO); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_rejects_zero_view_change_status_timeout() { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.view_change_status_timeout = IggyDuration::new(Duration::ZERO); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_rejects_zero_request_start_view_retransmit_interval() { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.request_start_view_retransmit_interval = IggyDuration::new(Duration::ZERO); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_rejects_view_change_status_below_retransmit_ratio() { - // 3s is nonzero but still < 4x the 1s retransmit, so the ratio rule is - // what rejects here, not the zero check. - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.view_change_retransmit_interval = IggyDuration::new(Duration::from_secs(1)); - c.view_change_status_timeout = IggyDuration::new(Duration::from_secs(3)); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_accepts_view_change_status_at_retransmit_ratio() { - // Exactly 4x the retransmit interval must pass. - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.view_change_retransmit_interval = IggyDuration::new(Duration::from_secs(1)); - c.view_change_status_timeout = IggyDuration::new(Duration::from_secs(4)); - assert!(c.validate().is_ok()); - } - - #[test] - fn validate_rejects_zero_view_probe_attempts_max() { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.view_probe_attempts_max = 0; - assert!(c.validate().is_err()); - } - - #[test] - fn validate_rejects_view_probe_attempts_above_ceiling() { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.view_probe_attempts_max = MAX_VIEW_PROBE_ATTEMPTS + 1; - assert!(c.validate().is_err()); - } - - #[test] - fn validate_accepts_view_probe_attempts_at_ceiling() { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.view_probe_attempts_max = MAX_VIEW_PROBE_ATTEMPTS; - assert!(c.validate().is_ok()); - } - - #[test] - fn validate_rejects_zero_repair_retry_interval() { - // `0` / `disabled` / `unlimited` all collapse to zero and would wedge - // stalled repair streams. - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.repair_retry_interval = IggyDuration::new(Duration::ZERO); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_rejects_zero_repair_chunk_max() { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.repair_chunk_max = 0; - assert!(c.validate().is_err()); - } - - #[test] - fn validate_rejects_repair_chunk_max_above_ceiling() { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.repair_chunk_max = MAX_REPAIR_CHUNK_MAX + 1; - assert!(c.validate().is_err()); - } - - #[test] - fn validate_accepts_repair_chunk_max_at_ceiling() { - // Section-level validate only; the cross-section rule against - // message_bus.peer_queue_capacity lives in the top-level validate. - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.repair_chunk_max = MAX_REPAIR_CHUNK_MAX; - assert!(c.validate().is_ok()); - } - - #[test] - fn validate_rejects_empty_nodes() { - let c = cfg(vec![]); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_rejects_duplicate_replica_ids() { - let c = cfg(vec![node("n1", 0), node("n2", 0)]); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_rejects_duplicate_names() { - let c = cfg(vec![node("n1", 0), node("n1", 1)]); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_rejects_out_of_range_replica_id() { - // 2 nodes total, so id 2 is out of range. - let c = cfg(vec![node("n1", 0), node("n2", 2)]); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_accepts_unique_contiguous_replica_ids() { - let c = cfg(vec![node("n1", 0), node("n2", 1), node("n3", 2)]); - assert!(c.validate().is_ok()); - } - - #[test] - fn validate_skips_checks_when_disabled() { - let mut c = cfg(vec![]); - c.enabled = false; - assert!(c.validate().is_ok()); - } - - // repair_chunk_max is also read by the unconditional top-level check - // against message_bus.peer_queue_capacity, so its own bounds apply with - // the cluster off too. - #[test] - fn validate_rejects_zero_repair_chunk_max_when_disabled() { - let mut c = cfg(vec![]); - c.enabled = false; - c.repair_chunk_max = 0; - assert!(c.validate().is_err()); - } - - #[test] - fn validate_rejects_repair_chunk_max_above_ceiling_when_disabled() { - let mut c = cfg(vec![]); - c.enabled = false; - c.repair_chunk_max = MAX_REPAIR_CHUNK_MAX + 1; - assert!(c.validate().is_err()); - } - - #[test] - fn validate_rejects_duplicate_tcp_replica_port() { - let ports = TransportPorts { - tcp: None, - quic: None, - http: None, - websocket: None, - tcp_replica: Some(9090), - }; - let mut n1 = node("n1", 0); - n1.ports = ports.clone(); - let mut n2 = node("n2", 1); - n2.ports = ports; - let c = cfg(vec![n1, n2]); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_rejects_cross_transport_port_reuse() { - let mut n1 = node("n1", 0); - n1.ports = TransportPorts { - tcp: Some(8090), - quic: None, - http: Some(8090), - websocket: None, - tcp_replica: None, - }; - let c = cfg(vec![n1]); - assert!( - c.validate().is_err(), - "same port on TCP and HTTP of the same node must be rejected" - ); - } - - #[test] - fn validate_accepts_same_port_on_different_ips() { - let mut n1 = node("n1", 0); - n1.ip = "127.0.0.1".to_string(); - n1.ports = TransportPorts { - tcp: Some(8090), - quic: None, - http: None, - websocket: None, - tcp_replica: None, - }; - let mut n2 = node("n2", 1); - n2.ip = "127.0.0.2".to_string(); - n2.ports = TransportPorts { - tcp: Some(8090), - quic: None, - http: None, - websocket: None, - tcp_replica: None, - }; - let c = cfg(vec![n1, n2]); - assert!(c.validate().is_ok()); - } - - #[test] - fn validate_rejects_duplicate_advertised_client_endpoint() { - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_address = Some("203.0.113.1".to_owned()); - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "10.0.0.2".to_owned(); - n2.advertised_address = n1.advertised_address.clone(); - n2.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, n2]).validate().is_err()); - } - - #[test] - fn validate_rejects_equivalent_ipv6_advertised_client_endpoints() { - for equivalent_address in ["2001:DB8::1", "2001:db8:0:0:0:0:0:1", "[2001:db8::1]"] { - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_address = Some("2001:db8::1".to_owned()); - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "10.0.0.2".to_owned(); - n2.advertised_address = Some(equivalent_address.to_owned()); - n2.ports.tcp = Some(8090); - - assert!( - cfg(vec![n1, n2]).validate().is_err(), - "{equivalent_address} must conflict with 2001:db8::1" - ); - } - } - - #[test] - fn validate_rejects_equivalent_ipv6_client_endpoints_from_node_ip() { - let mut n1 = node("n1", 0); - n1.ip = "2001:db8::1".to_owned(); - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "2001:db8:0:0:0:0:0:1".to_owned(); - n2.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, n2]).validate().is_err()); - } - - #[test] - fn validate_accepts_distinct_advertised_client_endpoints() { - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_address = Some("203.0.113.1".to_owned()); - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "10.0.0.2".to_owned(); - n2.advertised_address = Some("203.0.113.2".to_owned()); - n2.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, n2]).validate().is_ok()); - } - - #[test] - fn validate_accepts_hostname_advertised_address() { - let mut n1 = node("n1", 0); - n1.advertised_address = Some("iggy-node-1.example.com".to_owned()); - n1.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, node("n2", 1)]).validate().is_ok()); - } - - #[test] - fn validate_rejects_malformed_advertised_addresses() { - let oversized_label = format!("{}.example.com", "a".repeat(64)); - let oversized_hostname = format!("{}example.com", "a.".repeat(130)); - for advertised_address in [ - "", - " 203.0.113.1", - "10.0.0.256", - "192.168.1", - "example.com:8090", - "[2001:db8::1]:8090", - "2001:db8:::1", - "iggy_node.example.com", - "-node.example.com", - "node-.example.com", - ".example.com", - "example..com", - "example.com.", - "ex\u{e4}mple.com", - oversized_label.as_str(), - oversized_hostname.as_str(), - ] { - let mut n1 = node("n1", 0); - n1.advertised_address = Some(advertised_address.to_owned()); - - assert!( - cfg(vec![n1, node("n2", 1)]).validate().is_err(), - "'{advertised_address}' must be rejected" - ); - } - } - - #[test] - fn validate_rejects_case_variant_hostname_advertised_endpoints() { - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_address = Some("broker.example.com".to_owned()); - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "10.0.0.2".to_owned(); - n2.advertised_address = Some("Broker.Example.COM".to_owned()); - n2.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, n2]).validate().is_err()); - } - - #[test] - fn validate_rejects_node_ip_hostname_clashing_with_advertised_hostname() { - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_address = Some("broker.example.com".to_owned()); - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "broker.example.com".to_owned(); - n2.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, n2]).validate().is_err()); - } - - #[test] - fn validate_accepts_distinct_hostname_advertised_endpoints() { - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_address = Some("broker-1.example.com".to_owned()); - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "10.0.0.2".to_owned(); - n2.advertised_address = Some("broker-2.example.com".to_owned()); - n2.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, n2]).validate().is_ok()); - } - - #[test] - fn validate_accepts_selectors_with_distinct_cidrs() { - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_address = Some("203.0.113.1".to_owned()); - n1.advertised_addresses = vec![ - selector("10.0.0.0/16", "10.0.0.1"), - selector("10.0.0.0/8", "broker-1.internal.example"), - ]; - n1.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, node("n2", 1)]).validate().is_ok()); - } - - #[test] - fn validate_rejects_malformed_selector_cidr() { - for client_cidr in ["10.0.0.0", "10.0.0.0/33", "not-a-cidr", ""] { - let mut n1 = node("n1", 0); - n1.advertised_addresses = vec![selector(client_cidr, "10.0.0.1")]; - - assert!( - cfg(vec![n1, node("n2", 1)]).validate().is_err(), - "client_cidr '{client_cidr}' must be rejected" - ); - } - } - - #[test] - fn validate_rejects_malformed_selector_address() { - for address in ["", "10.0.0.1:8090", "10.0.0.256", "iggy_node"] { - let mut n1 = node("n1", 0); - n1.advertised_addresses = vec![selector("10.0.0.0/16", address)]; - - assert!( - cfg(vec![n1, node("n2", 1)]).validate().is_err(), - "selector address '{address}' must be rejected" - ); - } - } - - #[test] - fn validate_rejects_duplicate_selector_cidr_within_a_node() { - // `10.0.1.0/16` truncates to `10.0.0.0/16`: the two selectors match - // the identical client set, so the second could never win LPM. - let mut n1 = node("n1", 0); - n1.advertised_addresses = vec![ - selector("10.0.0.0/16", "10.0.0.1"), - selector("10.0.1.0/16", "10.0.0.2"), - ]; - - assert!(cfg(vec![n1, node("n2", 1)]).validate().is_err()); - } - - #[test] - fn validate_accepts_selector_count_at_the_cap() { - let mut n1 = node("n1", 0); - n1.advertised_addresses = (0..MAX_ADVERTISED_SELECTORS) - .map(|index| selector(&format!("10.{index}.0.0/16"), &format!("192.0.2.{index}"))) - .collect(); - - assert!(cfg(vec![n1, node("n2", 1)]).validate().is_ok()); - } - - #[test] - fn validate_rejects_selector_count_above_the_cap() { - // The env-override path stops expanding selector indices at the same - // ceiling, so a TOML roster exceeding it could never be replicated - // byte-identically through env vars. - let mut n1 = node("n1", 0); - n1.advertised_addresses = (0..=MAX_ADVERTISED_SELECTORS) - .map(|index| selector(&format!("10.{index}.0.0/16"), &format!("192.0.2.{index}"))) - .collect(); - - assert!(cfg(vec![n1, node("n2", 1)]).validate().is_err()); - } - - #[test] - fn validate_rejects_selector_endpoint_conflict_within_one_cidr() { - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_addresses = vec![selector("10.0.0.0/16", "10.0.7.7")]; - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "10.0.0.2".to_owned(); - n2.advertised_addresses = vec![selector("10.0.0.0/16", "10.0.7.7")]; - n2.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, n2]).validate().is_err()); - } - - #[test] - fn validate_accepts_identical_selector_endpoint_across_different_cidrs() { - // Reusing one host:port across DIFFERENT client networks is the - // feature (e.g. each network NATs the address to its local node). - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_addresses = vec![selector("10.1.0.0/16", "192.0.2.10")]; - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "10.0.0.2".to_owned(); - n2.advertised_addresses = vec![selector("10.2.0.0/16", "192.0.2.10")]; - n2.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, n2]).validate().is_ok()); - } - - #[test] - fn validate_rejects_v4_mapped_v6_selector_cidr_duplicating_its_v4_form() { - // `::ffff:10.0.0.0/104` canonicalizes to `10.0.0.0/8` (matching how - // client IPs canonicalize before LPM), so these two selectors match - // the identical client set. - let mut n1 = node("n1", 0); - n1.advertised_addresses = vec![ - selector("10.0.0.0/8", "10.0.0.1"), - selector("::ffff:10.0.0.0/104", "10.0.0.2"), - ]; - - assert!(cfg(vec![n1, node("n2", 1)]).validate().is_err()); - } - - #[test] - fn validate_rejects_selector_endpoint_clashing_with_another_nodes_catch_all() { - // The catch-all matches every client, so a 10.0.0.0/16 client would - // resolve both nodes to 192.0.2.10:8090. - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_addresses = vec![selector("10.0.0.0/16", "192.0.2.10")]; - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "10.0.0.2".to_owned(); - n2.advertised_address = Some("192.0.2.10".to_owned()); - n2.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, n2]).validate().is_err()); - } - - #[test] - fn validate_rejects_selector_endpoint_clashing_with_another_nodes_roster_ip() { - // Without an advertised_address the roster ip backs the catch-all, - // so the same cross-set conflict applies to it. - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_addresses = vec![selector("10.0.0.0/16", "10.0.0.2")]; - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "10.0.0.2".to_owned(); - n2.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, n2]).validate().is_err()); - } - - #[test] - fn validate_rejects_identical_selector_endpoint_across_nested_cidrs() { - // LPM runs per node, not cluster-wide: n1 has no longer prefix of - // its own shadowing the /16 overlap, so a 10.0.0.0/16 client wins - // n1's /8 and n2's /16, resolving both to 192.0.2.10:8090. - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_addresses = vec![selector("10.0.0.0/8", "192.0.2.10")]; - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "10.0.0.2".to_owned(); - n2.advertised_addresses = vec![selector("10.0.0.0/16", "192.0.2.10")]; - n2.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, n2]).validate().is_err()); - } - - #[test] - fn validate_accepts_nested_cidr_reuse_shadowed_by_same_node_longer_prefix() { - // n1's /16 selector shadows its /8 within 10.0.0.0/16, so n1's /8 - // entry wins only 10.0.0.0/8 minus 10.0.0.0/16 - disjoint from n2's - // /16. No client resolves both nodes to 192.0.2.10:8090. - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_addresses = vec![ - selector("10.0.0.0/8", "192.0.2.10"), - selector("10.0.0.0/16", "192.0.2.20"), - ]; - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "10.0.0.2".to_owned(); - n2.advertised_addresses = vec![selector("10.0.0.0/16", "192.0.2.10")]; - n2.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, n2]).validate().is_ok()); - } - - #[test] - fn validate_rejects_partially_shadowed_nested_cidr_reuse() { - // n1's /24 shadow carves only part of the /16 overlap: a client in - // 10.0.0.0/16 outside 10.0.0.0/24 still wins n1's /8 entry and n2's - // /16 entry, resolving both nodes to 192.0.2.10:8090. - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_addresses = vec![ - selector("10.0.0.0/8", "192.0.2.10"), - selector("10.0.0.0/24", "192.0.2.20"), - ]; - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "10.0.0.2".to_owned(); - n2.advertised_addresses = vec![selector("10.0.0.0/16", "192.0.2.10")]; - n2.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, n2]).validate().is_err()); - } - - #[test] - fn validate_accepts_catch_all_reuse_shadowed_by_same_node_selector() { - // n1's /16 selector shadows its catch-all within 10.0.0.0/16, so - // the catch-all never wins a client inside n2's /24. Without the - // shadow the same pair conflicts (see - // validate_rejects_selector_endpoint_clashing_with_another_nodes_catch_all). - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_address = Some("192.0.2.10".to_owned()); - n1.advertised_addresses = vec![selector("10.0.0.0/16", "192.0.2.20")]; - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "10.0.0.2".to_owned(); - n2.advertised_addresses = vec![selector("10.0.0.0/24", "192.0.2.10")]; - n2.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, n2]).validate().is_ok()); - } - - #[test] - fn validate_accepts_selector_reusing_a_fully_shadowed_catch_all_address() { - // n1's selectors cover both address families, so its catch-all wins - // known peers nowhere; only unknown-peer clients reach it, and they - // never match n2's selector. - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_address = Some("192.0.2.10".to_owned()); - n1.advertised_addresses = vec![ - selector("0.0.0.0/0", "192.0.2.20"), - selector("::/0", "192.0.2.30"), - ]; - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "10.0.0.2".to_owned(); - n2.advertised_addresses = vec![selector("10.0.0.0/16", "192.0.2.10")]; - n2.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, n2]).validate().is_ok()); - } - - #[test] - fn validate_accepts_catch_all_spelling_another_nodes_selector_address_when_self_shadowed() { - // Split-network NAT roster: n2's catch-all spells n1's 10/8 selector - // address, but n2's own 10/8 selector shadows its catch-all inside - // 10/8 (outside it n1 serves its own catch-all), so no client - // resolves both nodes to 192.0.2.10:8090. - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_addresses = vec![selector("10.0.0.0/8", "192.0.2.10")]; - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "10.0.0.2".to_owned(); - n2.advertised_address = Some("192.0.2.10".to_owned()); - n2.advertised_addresses = vec![selector("10.0.0.0/8", "192.0.2.20")]; - n2.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, n2]).validate().is_ok()); - } - - #[test] - fn validate_rejects_duplicate_catch_all_even_when_fully_shadowed() { - // A client whose peer address the transport cannot produce always - // falls to the catch-all, so duplicate catch-all conflict even when - // selectors cover every known network. - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_address = Some("192.0.2.10".to_owned()); - n1.advertised_addresses = vec![ - selector("0.0.0.0/0", "192.0.2.20"), - selector("::/0", "192.0.2.30"), - ]; - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "10.0.0.2".to_owned(); - n2.advertised_address = Some("192.0.2.10".to_owned()); - n2.advertised_addresses = vec![ - selector("0.0.0.0/0", "192.0.2.40"), - selector("::/0", "192.0.2.50"), - ]; - n2.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, n2]).validate().is_err()); - } - - #[test] - fn validate_accepts_selector_reusing_its_own_nodes_catch_all_address() { - // Redundant but harmless: within one node the selector and the - // catch-all cannot resolve a client to two different nodes. - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_address = Some("192.0.2.10".to_owned()); - n1.advertised_addresses = vec![selector("10.0.0.0/16", "192.0.2.10")]; - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "10.0.0.2".to_owned(); - n2.ports.tcp = Some(8091); - - assert!(cfg(vec![n1, n2]).validate().is_ok()); - } - - #[test] - fn validate_rejects_zero_tcp_replica_port() { - let ports = TransportPorts { - tcp: None, - quic: None, - http: None, - websocket: None, - tcp_replica: Some(0), - }; - let mut n1 = node("n1", 0); - n1.ports = ports; - let c = cfg(vec![n1]); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_accepts_empty_secret_when_auth_disabled() { - // Default: no secret, auth off -> legacy mode, must pass. - let c = cfg(vec![node("n1", 0), node("n2", 1)]); - assert!(c.validate().is_ok()); - } - - #[test] - fn validate_rejects_missing_secret_when_auth_enabled() { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.auth.enabled = true; - assert!(c.validate().is_err()); - } - - #[test] - fn validate_rejects_short_secret_when_auth_enabled() { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.auth.enabled = true; - c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN - 1); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_rejects_short_secret_even_when_auth_disabled() { - // Typo guard: a configured-but-short key fails even with auth off. - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN - 1); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_accepts_valid_secret_when_auth_enabled() { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.auth.enabled = true; - c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); - assert!(c.validate().is_ok()); - } - - #[test] - fn validate_accepts_valid_rotation_window() { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.auth.enabled = true; - c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); - c.auth.previous_shared_secret = "b".repeat(MIN_SHARED_SECRET_LEN); - assert!(c.validate().is_ok()); - } - - #[test] - fn validate_rejects_short_previous_secret() { - // Same typo guard as the primary key. - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.auth.enabled = true; - c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); - c.auth.previous_shared_secret = "b".repeat(MIN_SHARED_SECRET_LEN - 1); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_rejects_rotation_window_equal_to_primary() { - // An identical window is a no-op rotation: the operator rolled the - // config without changing the key. - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.auth.enabled = true; - c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); - c.auth.previous_shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); - assert!(c.validate().is_err()); - } - - fn tls_files() -> ClusterTlsConfig { - ClusterTlsConfig { - enabled: true, - self_signed: false, - cert_file: "cert.pem".to_string(), - key_file: "key.pem".to_string(), - ca_file: "ca.pem".to_string(), - } - } - - #[test] - fn validate_rejects_tls_ca_mode_with_missing_files() { - // Auth on so the failure exercises the file check, not the auth gate. - for missing in ["cert_file", "key_file", "ca_file"] { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.auth.enabled = true; - c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); - c.tls = tls_files(); - match missing { - "cert_file" => c.tls.cert_file.clear(), - "key_file" => c.tls.key_file.clear(), - _ => c.tls.ca_file.clear(), - } - assert!(c.validate().is_err(), "missing {missing} must be rejected"); - } - } - - #[test] - fn validate_rejects_tls_self_signed_without_auth() { - // Accept-any certificate without the PSK handshake = MITM-able. - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.tls = ClusterTlsConfig { - enabled: true, - self_signed: true, - ..ClusterTlsConfig::default() - }; - assert!(c.validate().is_err()); - } - - #[test] - fn validate_accepts_tls_self_signed_with_auth() { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.auth.enabled = true; - c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); - c.tls = ClusterTlsConfig { - enabled: true, - self_signed: true, - ..ClusterTlsConfig::default() - }; - assert!(c.validate().is_ok()); - } - - #[test] - fn validate_rejects_tls_ca_mode_without_auth() { - // TLS never authenticates the dialer (no client certificates); - // only the PSK handshake does, so it is mandatory with TLS on. - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.tls = tls_files(); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_accepts_tls_ca_mode_with_auth() { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.auth.enabled = true; - c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); - c.tls = tls_files(); - assert!(c.validate().is_ok()); - } } diff --git a/core/configs/src/server_config/defaults.rs b/core/configs/src/server_config/defaults.rs index 33422f752d..155239cd37 100644 --- a/core/configs/src/server_config/defaults.rs +++ b/core/configs/src/server_config/defaults.rs @@ -15,167 +15,65 @@ // specific language governing permissions and limitations // under the License. -//! `Default` impls for the sections this module owns (`tcp`, `websocket`, -//! `quic`, `cluster`, `metadata`, `partition`, `message_bus`), sourced -//! from `core/server/config.toml` via [`SERVER_CONFIG`]. Sections drawn -//! from [`crate::common`] (`http`, `system`, `telemetry`, -//! `consumer_group`, `data_maintenance`, `message_saver`, -//! `personal_access_token`, `heartbeat`) delegate to the `Default` impls -//! in [`crate::common::defaults`]. - use super::cluster::{ ClusterAuthConfig, ClusterConfig, ClusterNodeConfig, ClusterTlsConfig, TransportPorts, }; -use super::message_bus::MessageBusConfig; -use super::metadata::MetadataConfig; -use super::partition::PartitionConfig; +use super::http::{HttpConfig, HttpCorsConfig, HttpJwtConfig, HttpMetricsConfig, HttpTlsConfig}; use super::quic::{QuicCertificateConfig, QuicConfig, QuicSocketConfig}; -use super::server::ServerSystemConfig; -use super::server::{ExtraConfig, ServerConfig}; -use super::tcp::{TcpConfig, TcpSocketConfig, TcpTlsConfig}; -use super::websocket::{WebSocketConfig, WebSocketTlsConfig}; -use crate::common::http::HttpConfig; -use crate::common::server::{ - ConsumerGroupConfig, DataMaintenanceConfig, HeartbeatConfig, MessageSaverConfig, - PersonalAccessTokenConfig, TelemetryConfig, +use super::server::{ + ConsumerGroupConfig, DataMaintenanceConfig, HeartbeatConfig, MemoryPoolConfig, + MessageSaverConfig, MessagesMaintenanceConfig, PersonalAccessTokenCleanerConfig, + PersonalAccessTokenConfig, ServerConfig, TelemetryConfig, TelemetryLogsConfig, + TelemetryTracesConfig, +}; +use super::system::{ + BackupConfig, CompatibilityConfig, CompressionConfig, EncryptionConfig, LoggingConfig, + MessageDeduplicationConfig, PartitionConfig, RecoveryConfig, RuntimeConfig, SegmentConfig, + StateConfig, StreamConfig, SystemConfig, TopicConfig, }; +use super::tcp::TcpSocketConfig; +use super::tcp::{TcpConfig, TcpTlsConfig}; +use super::websocket::{WebSocketConfig, WebSocketTlsConfig}; +use configs::ConfigEnvMappings; +use iggy_common::IggyByteSize; +use iggy_common::IggyDuration; use std::sync::Arc; +use std::time::Duration; -// Same embedded TOML the shared sections read; re-exported so sibling -// modules reach it as `super::defaults::SERVER_CONFIG`. -pub use crate::common::defaults::SERVER_CONFIG; +static_toml::static_toml! { + // static_toml resolves relative to CARGO_MANIFEST_DIR (core/configs/). + pub static SERVER_CONFIG = include_toml!("../server/config.toml"); +} impl Default for ServerConfig { fn default() -> ServerConfig { ServerConfig { consumer_group: ConsumerGroupConfig::default(), data_maintenance: DataMaintenanceConfig::default(), - extra: ExtraConfig::default(), heartbeat: HeartbeatConfig::default(), message_saver: MessageSaverConfig::default(), personal_access_token: PersonalAccessTokenConfig::default(), - system: Arc::new(ServerSystemConfig::default()), + system: Arc::new(SystemConfig::default()), quic: QuicConfig::default(), tcp: TcpConfig::default(), websocket: WebSocketConfig::default(), http: HttpConfig::default(), telemetry: TelemetryConfig::default(), cluster: ClusterConfig::default(), - metadata: MetadataConfig::default(), - partition: PartitionConfig::default(), - message_bus: MessageBusConfig::default(), } } } -impl Default for ClusterConfig { - fn default() -> ClusterConfig { - ClusterConfig { - enabled: SERVER_CONFIG.cluster.enabled, - name: SERVER_CONFIG.cluster.name.parse().unwrap(), - heartbeat_timeout: SERVER_CONFIG.cluster.heartbeat_timeout.parse().unwrap(), - commit_broadcast_interval: SERVER_CONFIG - .cluster - .commit_broadcast_interval - .parse() - .unwrap(), - prepare_retransmit_interval: SERVER_CONFIG - .cluster - .prepare_retransmit_interval - .parse() - .unwrap(), - view_change_retransmit_interval: SERVER_CONFIG - .cluster - .view_change_retransmit_interval +impl Default for MessagesMaintenanceConfig { + fn default() -> MessagesMaintenanceConfig { + MessagesMaintenanceConfig { + cleaner_enabled: SERVER_CONFIG.data_maintenance.messages.cleaner_enabled, + interval: SERVER_CONFIG + .data_maintenance + .messages + .interval .parse() .unwrap(), - view_change_status_timeout: SERVER_CONFIG - .cluster - .view_change_status_timeout - .parse() - .unwrap(), - request_start_view_retransmit_interval: SERVER_CONFIG - .cluster - .request_start_view_retransmit_interval - .parse() - .unwrap(), - view_probe_attempts_max: SERVER_CONFIG.cluster.view_probe_attempts_max as u32, - repair_retry_interval: SERVER_CONFIG - .cluster - .repair_retry_interval - .parse() - .unwrap(), - repair_chunk_max: SERVER_CONFIG.cluster.repair_chunk_max as usize, - nodes: SERVER_CONFIG - .cluster - .nodes - .iter() - .map(|node| ClusterNodeConfig { - name: node.name.parse().unwrap(), - ip: node.ip.parse().unwrap(), - advertised_address: None, - advertised_addresses: Vec::new(), - replica_id: u8::try_from(node.replica_id).expect( - "static_toml replica_id must fit in u8 (0..=255); \ - fix core/server/config.toml", - ), - ports: TransportPorts { - tcp: Some(u16::try_from(node.ports.tcp).expect( - "static_toml cluster.nodes.ports.tcp must fit in u16 (0..=65535); \ - fix core/server/config.toml", - )), - quic: Some(u16::try_from(node.ports.quic).expect( - "static_toml cluster.nodes.ports.quic must fit in u16 (0..=65535); \ - fix core/server/config.toml", - )), - http: Some(u16::try_from(node.ports.http).expect( - "static_toml cluster.nodes.ports.http must fit in u16 (0..=65535); \ - fix core/server/config.toml", - )), - websocket: Some(u16::try_from(node.ports.websocket).expect( - "static_toml cluster.nodes.ports.websocket must fit in u16 (0..=65535); \ - fix core/server/config.toml", - )), - tcp_replica: Some(u16::try_from(node.ports.tcp_replica).expect( - "static_toml cluster.nodes.ports.tcp_replica must fit in u16 (0..=65535); \ - fix core/server/config.toml", - )), - }, - }) - .collect(), - auth: ClusterAuthConfig::default(), - tls: ClusterTlsConfig::default(), - } - } -} - -impl Default for MetadataConfig { - fn default() -> MetadataConfig { - // Read from the embedded TOML so the Default impl and the on-disk - // schema cannot drift (same pattern as MessageBusConfig below). - let metadata = &SERVER_CONFIG.metadata; - MetadataConfig { - prepare_queue_depth: metadata.prepare_queue_depth as usize, - journal_slots: metadata.journal_slots as usize, - clients_table_max: metadata.clients_table_max as usize, - } - } -} - -impl Default for PartitionConfig { - fn default() -> PartitionConfig { - // Read from the embedded TOML so the Default impl and the on-disk - // schema cannot drift (same pattern as MetadataConfig above). - let partition = &SERVER_CONFIG.partition; - PartitionConfig { - prepare_queue_depth: partition.prepare_queue_depth as usize, - evicted_ring_capacity: partition.evicted_ring_capacity as usize, - evicted_ring_bytes_max: partition.evicted_ring_bytes_max.parse().unwrap(), - transfer_served_cache_bytes_max: partition - .transfer_served_cache_bytes_max - .parse() - .unwrap(), - transfer_artifact_bytes_max: partition.transfer_artifact_bytes_max.parse().unwrap(), } } } @@ -250,22 +148,18 @@ impl Default for TcpTlsConfig { impl Default for TcpSocketConfig { fn default() -> TcpSocketConfig { TcpSocketConfig { - override_defaults: SERVER_CONFIG.tcp.socket.override_defaults, - recv_buffer_size: SERVER_CONFIG.tcp.socket.recv_buffer_size.parse().unwrap(), - send_buffer_size: SERVER_CONFIG.tcp.socket.send_buffer_size.parse().unwrap(), - keepalive: SERVER_CONFIG.tcp.socket.keepalive, - nodelay: SERVER_CONFIG.tcp.socket.nodelay, - linger: SERVER_CONFIG.tcp.socket.linger.parse().unwrap(), + override_defaults: false, + recv_buffer_size: IggyByteSize::from(100_000_u64), + send_buffer_size: IggyByteSize::from(100_000_u64), + keepalive: false, + nodelay: false, + linger: IggyDuration::new(Duration::new(0, 0)), } } } impl Default for WebSocketConfig { fn default() -> WebSocketConfig { - // The size knobs are optional in the schema (commented-out by - // default), so they map to `None` here when absent; every other - // field comes from the embedded TOML so the Default impl and - // the on-disk schema cannot drift. WebSocketConfig { enabled: SERVER_CONFIG.websocket.enabled, address: SERVER_CONFIG.websocket.address.parse().unwrap(), @@ -274,7 +168,7 @@ impl Default for WebSocketConfig { max_write_buffer_size: None, max_message_size: None, max_frame_size: None, - accept_unmasked_frames: SERVER_CONFIG.websocket.accept_unmasked_frames, + accept_unmasked_frames: false, tls: WebSocketTlsConfig::default(), } } @@ -291,20 +185,430 @@ impl Default for WebSocketTlsConfig { } } -impl Default for MessageBusConfig { - fn default() -> MessageBusConfig { - // Read every field from the embedded TOML so the Default impl - // and the on-disk schema cannot drift. Sibling impls in this - // file follow the same pattern. - let bus = &SERVER_CONFIG.message_bus; - MessageBusConfig { - max_batch: bus.max_batch as usize, - max_message_size: bus.max_message_size.parse().unwrap(), - peer_queue_capacity: bus.peer_queue_capacity as usize, - reconnect_period: bus.reconnect_period.parse().unwrap(), - close_peer_timeout: bus.close_peer_timeout.parse().unwrap(), - close_grace: bus.close_grace.parse().unwrap(), - handshake_grace: bus.handshake_grace.parse().unwrap(), +impl Default for HttpConfig { + fn default() -> HttpConfig { + HttpConfig { + enabled: SERVER_CONFIG.http.enabled, + address: SERVER_CONFIG.http.address.parse().unwrap(), + max_request_size: SERVER_CONFIG.http.max_request_size.parse().unwrap(), + web_ui: SERVER_CONFIG.http.web_ui, + cors: HttpCorsConfig::default(), + jwt: HttpJwtConfig::default(), + metrics: HttpMetricsConfig::default(), + tls: HttpTlsConfig::default(), + } + } +} + +impl Default for HttpCorsConfig { + fn default() -> HttpCorsConfig { + HttpCorsConfig { + enabled: SERVER_CONFIG.http.cors.enabled, + allowed_methods: SERVER_CONFIG + .http + .cors + .allowed_methods + .iter() + .map(|s| s.parse().unwrap()) + .collect(), + allowed_origins: SERVER_CONFIG + .http + .cors + .allowed_origins + .iter() + .map(|s| s.parse().unwrap()) + .collect(), + allowed_headers: SERVER_CONFIG + .http + .cors + .allowed_headers + .iter() + .map(|s| s.parse().unwrap()) + .collect(), + exposed_headers: SERVER_CONFIG + .http + .cors + .exposed_headers + .iter() + .map(|s| s.parse().unwrap()) + .collect(), + allow_credentials: SERVER_CONFIG.http.cors.allow_credentials, + allow_private_network: SERVER_CONFIG.http.cors.allow_private_network, + } + } +} + +impl Default for HttpJwtConfig { + fn default() -> HttpJwtConfig { + HttpJwtConfig { + algorithm: SERVER_CONFIG.http.jwt.algorithm.parse().unwrap(), + issuer: SERVER_CONFIG.http.jwt.issuer.parse().unwrap(), + audience: SERVER_CONFIG.http.jwt.audience.parse().unwrap(), + valid_issuers: SERVER_CONFIG + .http + .jwt + .valid_issuers + .iter() + .map(|s| s.parse().unwrap()) + .collect(), + valid_audiences: SERVER_CONFIG + .http + .jwt + .valid_audiences + .iter() + .map(|s| s.parse().unwrap()) + .collect(), + access_token_expiry: SERVER_CONFIG.http.jwt.access_token_expiry.parse().unwrap(), + clock_skew: SERVER_CONFIG.http.jwt.clock_skew.parse().unwrap(), + not_before: SERVER_CONFIG.http.jwt.not_before.parse().unwrap(), + encoding_secret: SERVER_CONFIG.http.jwt.encoding_secret.parse().unwrap(), + decoding_secret: SERVER_CONFIG.http.jwt.decoding_secret.parse().unwrap(), + use_base64_secret: SERVER_CONFIG.http.jwt.use_base_64_secret, + trusted_issuers: None, + } + } +} + +impl Default for HttpMetricsConfig { + fn default() -> HttpMetricsConfig { + HttpMetricsConfig { + enabled: SERVER_CONFIG.http.metrics.enabled, + endpoint: SERVER_CONFIG.http.metrics.endpoint.parse().unwrap(), + } + } +} + +impl Default for HttpTlsConfig { + fn default() -> HttpTlsConfig { + HttpTlsConfig { + enabled: SERVER_CONFIG.http.tls.enabled, + cert_file: SERVER_CONFIG.http.tls.cert_file.parse().unwrap(), + key_file: SERVER_CONFIG.http.tls.key_file.parse().unwrap(), + } + } +} + +impl Default for MessageSaverConfig { + fn default() -> MessageSaverConfig { + MessageSaverConfig { + enabled: SERVER_CONFIG.message_saver.enabled, + enforce_fsync: SERVER_CONFIG.message_saver.enforce_fsync, + interval: SERVER_CONFIG.message_saver.interval.parse().unwrap(), + } + } +} + +impl Default for PersonalAccessTokenConfig { + fn default() -> PersonalAccessTokenConfig { + PersonalAccessTokenConfig { + max_tokens_per_user: SERVER_CONFIG.personal_access_token.max_tokens_per_user as u32, + cleaner: PersonalAccessTokenCleanerConfig::default(), + } + } +} + +impl Default for PersonalAccessTokenCleanerConfig { + fn default() -> PersonalAccessTokenCleanerConfig { + PersonalAccessTokenCleanerConfig { + enabled: SERVER_CONFIG.personal_access_token.cleaner.enabled, + interval: SERVER_CONFIG + .personal_access_token + .cleaner + .interval + .parse() + .unwrap(), + } + } +} + +impl Default for SystemConfig { + fn default() -> Self { + Self { + path: SERVER_CONFIG.system.path.parse().unwrap(), + backup: BackupConfig::default(), + runtime: RuntimeConfig::default(), + logging: LoggingConfig::default(), + stream: StreamConfig::default(), + encryption: EncryptionConfig::default(), + topic: TopicConfig::default(), + partition: PartitionConfig::default(), + segment: SegmentConfig::default(), + state: StateConfig::default(), + compression: CompressionConfig::default(), + message_deduplication: MessageDeduplicationConfig::default(), + recovery: RecoveryConfig::default(), + memory_pool: MemoryPoolConfig::default(), + sharding: S::default(), + } + } +} + +impl Default for BackupConfig { + fn default() -> BackupConfig { + BackupConfig { + path: SERVER_CONFIG.system.backup.path.parse().unwrap(), + compatibility: CompatibilityConfig::default(), + } + } +} + +impl Default for CompatibilityConfig { + fn default() -> Self { + CompatibilityConfig { + path: SERVER_CONFIG + .system + .backup + .compatibility + .path + .parse() + .unwrap(), + } + } +} + +impl Default for HeartbeatConfig { + fn default() -> HeartbeatConfig { + HeartbeatConfig { + enabled: SERVER_CONFIG.heartbeat.enabled, + interval: SERVER_CONFIG.heartbeat.interval.parse().unwrap(), + } + } +} + +impl Default for ConsumerGroupConfig { + fn default() -> ConsumerGroupConfig { + ConsumerGroupConfig { + rebalancing_timeout: SERVER_CONFIG + .consumer_group + .rebalancing_timeout + .parse() + .unwrap(), + rebalancing_check_interval: SERVER_CONFIG + .consumer_group + .rebalancing_check_interval + .parse() + .unwrap(), + } + } +} + +impl Default for RuntimeConfig { + fn default() -> RuntimeConfig { + RuntimeConfig { + path: SERVER_CONFIG.system.runtime.path.parse().unwrap(), + } + } +} + +impl Default for CompressionConfig { + fn default() -> Self { + CompressionConfig { + allow_override: SERVER_CONFIG.system.compression.allow_override, + default_algorithm: SERVER_CONFIG + .system + .compression + .default_algorithm + .parse() + .unwrap(), + } + } +} + +impl Default for LoggingConfig { + fn default() -> LoggingConfig { + LoggingConfig { + path: SERVER_CONFIG.system.logging.path.parse().unwrap(), + level: SERVER_CONFIG.system.logging.level.parse().unwrap(), + file_enabled: SERVER_CONFIG.system.logging.file_enabled, + max_file_size: SERVER_CONFIG.system.logging.max_file_size.parse().unwrap(), + max_total_size: SERVER_CONFIG.system.logging.max_total_size.parse().unwrap(), + rotation_check_interval: SERVER_CONFIG + .system + .logging + .rotation_check_interval + .parse() + .unwrap(), + retention: SERVER_CONFIG.system.logging.retention.parse().unwrap(), + sysinfo_print_interval: SERVER_CONFIG + .system + .logging + .sysinfo_print_interval + .parse() + .unwrap(), + } + } +} + +impl Default for EncryptionConfig { + fn default() -> EncryptionConfig { + EncryptionConfig { + enabled: SERVER_CONFIG.system.encryption.enabled, + key: SERVER_CONFIG.system.encryption.key.parse().unwrap(), + } + } +} + +impl Default for StreamConfig { + fn default() -> StreamConfig { + StreamConfig { + path: SERVER_CONFIG.system.stream.path.parse().unwrap(), + } + } +} + +impl Default for TopicConfig { + fn default() -> TopicConfig { + TopicConfig { + path: SERVER_CONFIG.system.topic.path.parse().unwrap(), + max_size: SERVER_CONFIG.system.topic.max_size.parse().unwrap(), + message_expiry: SERVER_CONFIG.system.topic.message_expiry.parse().unwrap(), + } + } +} + +impl Default for PartitionConfig { + fn default() -> PartitionConfig { + PartitionConfig { + path: SERVER_CONFIG.system.partition.path.parse().unwrap(), + size_of_messages_required_to_save: SERVER_CONFIG + .system + .partition + .size_of_messages_required_to_save + .parse() + .unwrap(), + messages_required_to_save: SERVER_CONFIG.system.partition.messages_required_to_save + as u32, + enforce_fsync: SERVER_CONFIG.system.partition.enforce_fsync, + validate_checksum: SERVER_CONFIG.system.partition.validate_checksum, + } + } +} + +impl Default for SegmentConfig { + fn default() -> SegmentConfig { + SegmentConfig { + size: SERVER_CONFIG.system.segment.size.parse().unwrap(), + cache_indexes: SERVER_CONFIG.system.segment.cache_indexes.parse().unwrap(), + archive_expired: SERVER_CONFIG.system.segment.archive_expired, + } + } +} + +impl Default for StateConfig { + fn default() -> StateConfig { + StateConfig { + enforce_fsync: SERVER_CONFIG.system.state.enforce_fsync, + max_file_operation_retries: SERVER_CONFIG.system.state.max_file_operation_retries + as u32, + retry_delay: SERVER_CONFIG.system.state.retry_delay.parse().unwrap(), + } + } +} + +impl Default for MessageDeduplicationConfig { + fn default() -> MessageDeduplicationConfig { + MessageDeduplicationConfig { + enabled: SERVER_CONFIG.system.message_deduplication.enabled, + max_entries: SERVER_CONFIG.system.message_deduplication.max_entries as u64, + expiry: SERVER_CONFIG + .system + .message_deduplication + .expiry + .parse() + .unwrap(), + } + } +} + +impl Default for RecoveryConfig { + fn default() -> RecoveryConfig { + RecoveryConfig { + recreate_missing_state: SERVER_CONFIG.system.recovery.recreate_missing_state, + } + } +} + +impl Default for MemoryPoolConfig { + fn default() -> MemoryPoolConfig { + Self { + enabled: SERVER_CONFIG.system.memory_pool.enabled, + size: SERVER_CONFIG.system.memory_pool.size.parse().unwrap(), + bucket_capacity: SERVER_CONFIG.system.memory_pool.bucket_capacity as u32, + } + } +} + +impl Default for TelemetryConfig { + fn default() -> TelemetryConfig { + TelemetryConfig { + enabled: SERVER_CONFIG.telemetry.enabled, + service_name: SERVER_CONFIG.telemetry.service_name.parse().unwrap(), + logs: TelemetryLogsConfig::default(), + traces: TelemetryTracesConfig::default(), + } + } +} + +impl Default for TelemetryLogsConfig { + fn default() -> TelemetryLogsConfig { + TelemetryLogsConfig { + transport: SERVER_CONFIG.telemetry.logs.transport.parse().unwrap(), + endpoint: SERVER_CONFIG.telemetry.logs.endpoint.parse().unwrap(), + } + } +} + +impl Default for TelemetryTracesConfig { + fn default() -> TelemetryTracesConfig { + TelemetryTracesConfig { + transport: SERVER_CONFIG.telemetry.traces.transport.parse().unwrap(), + endpoint: SERVER_CONFIG.telemetry.traces.endpoint.parse().unwrap(), + } + } +} + +impl Default for ClusterConfig { + fn default() -> ClusterConfig { + ClusterConfig { + enabled: SERVER_CONFIG.cluster.enabled, + name: SERVER_CONFIG.cluster.name.parse().unwrap(), + nodes: SERVER_CONFIG + .cluster + .nodes + .iter() + .map(|node| ClusterNodeConfig { + name: node.name.parse().unwrap(), + ip: node.ip.parse().unwrap(), + replica_id: u8::try_from(node.replica_id).expect( + "static_toml replica_id must fit in u8 (0..=255); \ + fix core/server/config.toml", + ), + ports: TransportPorts { + tcp: Some(u16::try_from(node.ports.tcp).expect( + "static_toml cluster.nodes.ports.tcp must fit in u16 (0..=65535); \ + fix core/server/config.toml", + )), + quic: Some(u16::try_from(node.ports.quic).expect( + "static_toml cluster.nodes.ports.quic must fit in u16 (0..=65535); \ + fix core/server/config.toml", + )), + http: Some(u16::try_from(node.ports.http).expect( + "static_toml cluster.nodes.ports.http must fit in u16 (0..=65535); \ + fix core/server/config.toml", + )), + websocket: Some(u16::try_from(node.ports.websocket).expect( + "static_toml cluster.nodes.ports.websocket must fit in u16 (0..=65535); \ + fix core/server/config.toml", + )), + tcp_replica: Some(u16::try_from(node.ports.tcp_replica).expect( + "static_toml cluster.nodes.ports.tcp_replica must fit in u16 (0..=65535); \ + fix core/server/config.toml", + )), + }, + }) + .collect(), + auth: ClusterAuthConfig::default(), + tls: ClusterTlsConfig::default(), } } } diff --git a/core/configs/src/server_config/displays.rs b/core/configs/src/server_config/displays.rs index 2fac643c36..f7693965d9 100644 --- a/core/configs/src/server_config/displays.rs +++ b/core/configs/src/server_config/displays.rs @@ -15,101 +15,258 @@ // specific language governing permissions and limitations // under the License. -//! `Display` impls for the sections this module owns. -//! -//! Sections drawn from [`crate::common`] pick up [`Display`] from -//! [`crate::displays`]; this module only adds the top-level -//! [`ServerConfig`] formatter and the [`MessageBusConfig`] section -//! formatter. - -use super::message_bus::MessageBusConfig; -use super::metadata::MetadataConfig; -use super::partition::PartitionConfig; -use super::quic::{QuicCertificateConfig, QuicConfig, QuicSocketConfig}; -use super::server::{ExtraConfig, NamespaceConfig, ServerConfig}; -use super::tcp::{TcpConfig, TcpSocketConfig, TcpTlsConfig}; +use super::quic::{QuicCertificateConfig, QuicConfig}; +use super::server::{ + ConsumerGroupConfig, DataMaintenanceConfig, HeartbeatConfig, MessagesMaintenanceConfig, + TelemetryConfig, TelemetryLogsConfig, TelemetryTracesConfig, +}; +use super::system::MessageDeduplicationConfig; +use super::{ + http::{HttpConfig, HttpCorsConfig, HttpJwtConfig, HttpMetricsConfig, HttpTlsConfig}, + server::{MessageSaverConfig, ServerConfig}, + system::{ + CompressionConfig, EncryptionConfig, LoggingConfig, PartitionConfig, SegmentConfig, + StateConfig, StreamConfig, SystemConfig, TopicConfig, + }, + tcp::{TcpConfig, TcpSocketConfig, TcpTlsConfig}, +}; +use configs::ConfigEnvMappings; use std::fmt::{Display, Formatter}; +impl Display for HttpConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ enabled: {}, address: {}, max_request_size: {}, web_ui: {}, cors: {}, jwt: {}, metrics: {}, tls: {} }}", + self.enabled, + self.address, + self.max_request_size, + self.web_ui, + self.cors, + self.jwt, + self.metrics, + self.tls + ) + } +} + +impl Display for HttpCorsConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ enabled: {}, allowed_methods: {:?}, allowed_origins: {:?}, allowed_headers: {:?}, exposed_headers: {:?}, allow_credentials: {}, allow_private_network: {} }}", + self.enabled, + self.allowed_methods, + self.allowed_origins, + self.allowed_headers, + self.exposed_headers, + self.allow_credentials, + self.allow_private_network + ) + } +} + +impl Display for HttpJwtConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ algorithm: {}, audience: {}, access_token_expiry: {}, use_base64_secret: {} }}", + self.algorithm, self.audience, self.access_token_expiry, self.use_base64_secret + ) + } +} + +impl Display for HttpMetricsConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ enabled: {}, endpoint: {} }}", + self.enabled, self.endpoint + ) + } +} + +impl Display for HttpTlsConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ enabled: {}, cert_file: {}, key_file: {} }}", + self.enabled, self.cert_file, self.key_file + ) + } +} + +impl Display for QuicConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ enabled: {}, address: {}, max_concurrent_bidi_streams: {}, datagram_send_buffer_size: {}, initial_mtu: {}, send_window: {}, receive_window: {}, keep_alive_interval: {}, max_idle_timeout: {}, certificate: {} }}", + self.enabled, + self.address, + self.max_concurrent_bidi_streams, + self.datagram_send_buffer_size, + self.initial_mtu, + self.send_window, + self.receive_window, + self.keep_alive_interval, + self.max_idle_timeout, + self.certificate + ) + } +} + +impl Display for QuicCertificateConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ self_signed: {}, cert_file: {}, key_file: {} }}", + self.self_signed, self.cert_file, self.key_file + ) + } +} + +impl Display for CompressionConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ allowed_override: {}, default_algorithm: {} }}", + self.allow_override, self.default_algorithm + ) + } +} + +impl Display for DataMaintenanceConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{{ messages: {} }}", self.messages) + } +} + +impl Display for MessagesMaintenanceConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ cleaner_enabled: {}, interval: {} }}", + self.cleaner_enabled, self.interval + ) + } +} + impl Display for ServerConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!( f, - "{{ consumer_group: {}, data_maintenance: {}, extra: {}, message_saver: {}, \ - heartbeat: {}, system: {}, quic: {}, tcp: {}, http: {}, telemetry: {}, \ - metadata: {}, message_bus: {}, partition: {} }}", + "{{ consumer_group: {}, data_maintenance: {}, message_saver: {}, heartbeat: {}, system: {}, quic: {}, tcp: {}, http: {}, telemetry: {} }}", self.consumer_group, self.data_maintenance, - self.extra, self.message_saver, self.heartbeat, self.system, self.quic, self.tcp, self.http, - self.telemetry, - self.metadata, - self.message_bus, - self.partition, + self.telemetry ) } } -impl Display for PartitionConfig { +impl Display for ConsumerGroupConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ rebalancing_timeout: {}, rebalancing_check_interval: {} }}", + self.rebalancing_timeout, self.rebalancing_check_interval + ) + } +} + +impl Display for MessageSaverConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!( f, - "{{ prepare_queue_depth: {}, evicted_ring_capacity: {}, \ - evicted_ring_bytes_max: {}, transfer_served_cache_bytes_max: {}, \ - transfer_artifact_bytes_max: {} }}", - self.prepare_queue_depth, - self.evicted_ring_capacity, - self.evicted_ring_bytes_max, - self.transfer_served_cache_bytes_max, - self.transfer_artifact_bytes_max, + "{{ enabled: {}, enforce_fsync: {}, interval: {} }}", + self.enabled, self.enforce_fsync, self.interval ) } } -impl Display for MetadataConfig { +impl Display for HeartbeatConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!( f, - "{{ prepare_queue_depth: {}, journal_slots: {}, clients_table_max: {} }}", - self.prepare_queue_depth, self.journal_slots, self.clients_table_max, + "{{ enabled: {}, interval: {} }}", + self.enabled, self.interval ) } } -impl Display for MessageBusConfig { +impl Display for EncryptionConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{{ enabled: {} }}", self.enabled) + } +} + +impl Display for StreamConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{{ path: {} }}", self.path) + } +} + +impl Display for TopicConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!( f, - "{{ max_batch: {}, max_message_size: {}, peer_queue_capacity: {}, \ - reconnect_period: {}, close_peer_timeout: {}, close_grace: {}, \ - handshake_grace: {} }}", - self.max_batch, - self.max_message_size, - self.peer_queue_capacity, - self.reconnect_period, - self.close_peer_timeout, - self.close_grace, - self.handshake_grace, + "{{ path: {}, max_size: {}, message_expiry: {} }}", + self.path, self.max_size, self.message_expiry ) } } -impl Display for ExtraConfig { +impl Display for PartitionConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{{ namespace: {} }}", self.namespace) + write!( + f, + "{{ path: {}, messages_required_to_save: {}, size_of_messages_required_to_save: {}, enforce_fsync: {}, validate_checksum: {} }}", + self.path, + self.messages_required_to_save, + self.size_of_messages_required_to_save, + self.enforce_fsync, + self.validate_checksum + ) + } +} + +impl Display for MessageDeduplicationConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ enabled: {}, max_entries: {:?}, expiry: {:?} }}", + self.enabled, self.max_entries, self.expiry + ) } } -impl Display for NamespaceConfig { +impl Display for SegmentConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!( f, - "{{ max_streams: {}, max_topics: {}, max_partitions: {} }}", - self.max_streams, self.max_topics, self.max_partitions + "{{ size_bytes: {}, cache_indexes: {}, archive_expired: {} }}", + self.size, self.cache_indexes, self.archive_expired, + ) + } +} + +impl Display for LoggingConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ path: {}, level: {}, file_enabled: {}, max_file_size: {}, max_total_size: {}, rotation_check_interval: {}, retention: {} }}", + self.path, + self.level, + self.file_enabled, + self.max_file_size.as_human_string_with_zero_as_unlimited(), + self.max_total_size.as_human_string_with_zero_as_unlimited(), + self.rotation_check_interval, + self.retention ) } } @@ -138,7 +295,7 @@ impl Display for TcpSocketConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!( f, - "{{ override_defaults: {}, recv_buffer_size: {}, send_buffer_size: {}, keepalive: {}, nodelay: {}, linger: {} }}", + "{{ override defaults: {}, recv buffer size: {}, send buffer size {}, keepalive: {}, nodelay: {}, linger: {} }}", self.override_defaults, self.recv_buffer_size, self.send_buffer_size, @@ -149,41 +306,59 @@ impl Display for TcpSocketConfig { } } -impl Display for QuicConfig { +impl Display for TelemetryConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!( f, - "{{ enabled: {}, address: {}, max_concurrent_bidi_streams: {}, datagram_send_buffer_size: {}, initial_mtu: {}, send_window: {}, receive_window: {}, keep_alive_interval: {}, max_idle_timeout: {}, certificate: {} }}", - self.enabled, - self.address, - self.max_concurrent_bidi_streams, - self.datagram_send_buffer_size, - self.initial_mtu, - self.send_window, - self.receive_window, - self.keep_alive_interval, - self.max_idle_timeout, - self.certificate + "{{ enabled: {}, service_name: {}, logs: {}, traces: {} }}", + self.enabled, self.service_name, self.logs, self.traces ) } } -impl Display for QuicCertificateConfig { +impl Display for TelemetryLogsConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!( f, - "{{ self_signed: {}, cert_file: {}, key_file: {} }}", - self.self_signed, self.cert_file, self.key_file + "{{ transport: {}, endpoint: {} }}", + self.transport, self.endpoint + ) + } +} + +impl Display for StateConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ enforce_fsync: {}, max_file_operation_retries: {}, retry_delay: {} }}", + self.enforce_fsync, self.max_file_operation_retries, self.retry_delay, ) } } -impl Display for QuicSocketConfig { +impl Display for TelemetryTracesConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!( f, - "{{ override_defaults: {}, recv_buffer_size: {}, send_buffer_size: {}, keepalive: {} }}", - self.override_defaults, self.recv_buffer_size, self.send_buffer_size, self.keepalive + "{{ transport: {}, endpoint: {} }}", + self.transport, self.endpoint + ) + } +} + +impl Display for SystemConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ path: {}, logging: {}, stream: {}, topic: {}, partition: {}, segment: {}, encryption: {}, state: {} }}", + self.path, + self.logging, + self.stream, + self.topic, + self.partition, + self.segment, + self.encryption, + self.state, ) } } diff --git a/core/configs/src/common/http.rs b/core/configs/src/server_config/http.rs similarity index 100% rename from core/configs/src/common/http.rs rename to core/configs/src/server_config/http.rs diff --git a/core/configs/src/server_config/mod.rs b/core/configs/src/server_config/mod.rs index 718aee5cdd..e88b2f03eb 100644 --- a/core/configs/src/server_config/mod.rs +++ b/core/configs/src/server_config/mod.rs @@ -15,23 +15,17 @@ // specific language governing permissions and limitations // under the License. -//! On-disk config schema for the `iggy-server` binary. -//! -//! Composes the shared section vocabulary from [`crate::common`] with the -//! transport, cluster, metadata and bus sections this server owns. -//! [`server::ServerConfig`] is the root type the bootstrap loads. - +pub mod cache_indexes; pub mod cluster; pub mod defaults; pub mod displays; -pub mod message_bus; -pub mod metadata; -pub mod partition; +pub mod http; pub mod quic; pub mod server; pub mod sharding; +pub mod system; pub mod tcp; pub mod validators; pub mod websocket; -pub use crate::common::COMPONENT; +pub const COMPONENT: &str = "CONFIG"; diff --git a/core/configs/src/server_config/quic.rs b/core/configs/src/server_config/quic.rs index b6a52f3077..502d1bf7a4 100644 --- a/core/configs/src/server_config/quic.rs +++ b/core/configs/src/server_config/quic.rs @@ -15,12 +15,9 @@ // specific language governing permissions and limitations // under the License. -//! QUIC listener schema. - -use super::COMPONENT; -use crate::ConfigurationError; use configs::ConfigEnv; -use iggy_common::{IggyByteSize, IggyDuration, Validatable}; +use iggy_common::IggyByteSize; +use iggy_common::IggyDuration; use serde::{Deserialize, Serialize}; use serde_with::DisplayFromStr; use serde_with::serde_as; @@ -65,153 +62,3 @@ pub struct QuicCertificateConfig { pub cert_file: String, pub key_file: String, } - -/// Validates the field range constraints the runtime conversion in -/// `core::message_bus::config::build_quic_tuning` previously enforced -/// via `expect(...)`. Surfacing them here turns boot-time misconfig -/// into a `ConfigurationError` instead of a panic in the bus crate. -impl Validatable for QuicConfig { - fn validate(&self) -> Result<(), ConfigurationError> { - // QUIC requires at least one bidi stream per connection. - if self.max_concurrent_bidi_streams == 0 { - eprintln!("{COMPONENT} quic.max_concurrent_bidi_streams must be >= 1"); - return Err(ConfigurationError::InvalidConfigurationValue); - } - // quinn-proto stores stream counts as u32 internally. - if u32::try_from(self.max_concurrent_bidi_streams).is_err() { - eprintln!( - "{COMPONENT} quic.max_concurrent_bidi_streams ({}) does not fit in u32", - self.max_concurrent_bidi_streams - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - // The datagram send buffer is materialized as a `Vec` of this - // length in compio-quic, so it must fit in `usize` on the target - // platform. - if usize::try_from(self.datagram_send_buffer_size.as_bytes_u64()).is_err() { - eprintln!( - "{COMPONENT} quic.datagram_send_buffer_size ({} bytes) does not fit in usize on this target", - self.datagram_send_buffer_size.as_bytes_u64() - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - // RFC 9000 §14: minimum required MTU is 1200 bytes; quinn stores - // initial_mtu as u16 (max 65535). - let initial_mtu = self.initial_mtu.as_bytes_u64(); - if initial_mtu < 1200 { - eprintln!( - "{COMPONENT} quic.initial_mtu ({initial_mtu}) is below the QUIC minimum of 1200 bytes", - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if u16::try_from(initial_mtu).is_err() { - eprintln!("{COMPONENT} quic.initial_mtu ({initial_mtu}) exceeds u16::MAX (65535)",); - return Err(ConfigurationError::InvalidConfigurationValue); - } - // quinn VarInt for `receive_window` accepts u32; rejecting - // out-of-range values here surfaces a config error rather than - // panicking inside the bus crate's runtime conversion. - if u32::try_from(self.receive_window.as_bytes_u64()).is_err() { - eprintln!( - "{COMPONENT} quic.receive_window ({} bytes) does not fit in u32 (quinn VarInt limit)", - self.receive_window.as_bytes_u64() - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - // `send_window` is u64-sized in QuicTuning, but quinn's VarInt - // protocol-level cap is 2^62 - 1; reject anything above that. - const QUINN_VARINT_MAX: u64 = (1u64 << 62) - 1; - if self.send_window.as_bytes_u64() > QUINN_VARINT_MAX { - eprintln!( - "{COMPONENT} quic.send_window ({} bytes) exceeds quinn VarInt max (2^62 - 1)", - self.send_window.as_bytes_u64() - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn baseline() -> QuicConfig { - QuicConfig { - enabled: false, - address: String::new(), - max_concurrent_bidi_streams: 1, - datagram_send_buffer_size: IggyByteSize::from(100_u64 * 1024), - initial_mtu: IggyByteSize::from(8_u64 * 1024), - send_window: IggyByteSize::from(64_u64 * 1024 * 1024), - receive_window: IggyByteSize::from(64_u64 * 1024 * 1024), - keep_alive_interval: IggyDuration::from(std::time::Duration::from_secs(10)), - max_idle_timeout: IggyDuration::from(std::time::Duration::from_secs(30)), - certificate: QuicCertificateConfig { - self_signed: false, - cert_file: String::new(), - key_file: String::new(), - }, - socket: QuicSocketConfig { - override_defaults: false, - recv_buffer_size: IggyByteSize::from(0_u64), - send_buffer_size: IggyByteSize::from(0_u64), - keepalive: false, - }, - } - } - - #[test] - fn baseline_validates() { - baseline().validate().expect("baseline is valid"); - } - - #[test] - fn rejects_zero_max_concurrent_bidi_streams() { - let mut c = baseline(); - c.max_concurrent_bidi_streams = 0; - assert!(c.validate().is_err()); - } - - #[test] - fn rejects_max_concurrent_bidi_streams_above_u32() { - let mut c = baseline(); - c.max_concurrent_bidi_streams = u64::from(u32::MAX) + 1; - assert!(c.validate().is_err()); - } - - #[test] - fn rejects_initial_mtu_below_qiuc_minimum() { - let mut c = baseline(); - c.initial_mtu = IggyByteSize::from(1199_u64); - assert!(c.validate().is_err()); - } - - #[test] - fn rejects_initial_mtu_above_u16() { - let mut c = baseline(); - c.initial_mtu = IggyByteSize::from(u64::from(u16::MAX) + 1); - assert!(c.validate().is_err()); - } - - #[test] - fn rejects_receive_window_above_u32() { - let mut c = baseline(); - c.receive_window = IggyByteSize::from(u64::from(u32::MAX) + 1); - assert!(c.validate().is_err()); - } - - #[test] - fn rejects_send_window_above_quinn_varint_max() { - let mut c = baseline(); - c.send_window = IggyByteSize::from(1_u64 << 62); - assert!(c.validate().is_err()); - } - - #[test] - fn accepts_initial_mtu_at_quic_minimum() { - let mut c = baseline(); - c.initial_mtu = IggyByteSize::from(1200_u64); - assert!(c.validate().is_ok()); - } -} diff --git a/core/configs/src/server_config/server.rs b/core/configs/src/server_config/server.rs index 4dbd7e81de..2ef47c23ec 100644 --- a/core/configs/src/server_config/server.rs +++ b/core/configs/src/server_config/server.rs @@ -17,85 +17,161 @@ use super::COMPONENT; use super::cluster::ClusterConfig; -use super::message_bus::MessageBusConfig; -use super::metadata::MetadataConfig; -use super::partition::PartitionConfig; +use super::http::HttpConfig; use super::quic::QuicConfig; +use super::system::SystemConfig; use super::tcp::TcpConfig; use super::websocket::WebSocketConfig; use crate::ConfigurationError; -use crate::common::http::HttpConfig; -use crate::common::system::SystemConfig; use configs::{ConfigEnv, ConfigEnvMappings, ConfigProvider, FileConfigProvider, TypedEnvProvider}; use err_trail::ErrContext; use figment::providers::{Format, Toml}; use figment::value::Dict; use figment::{Metadata, Profile, Provider}; -use iggy_common::Validatable; +use iggy_common::{IggyByteSize, IggyDuration, Validatable}; use serde::{Deserialize, Serialize}; -use server_common::sharding::{MAX_PARTITIONS, MAX_STREAMS, MAX_TOPICS}; +use serde_with::DisplayFromStr; +use serde_with::serde_as; +use server_common::MemoryPoolConfigOther; +use server_common::log::{TelemetryEndpointSettings, TelemetrySettings}; use std::env; use std::sync::Arc; -pub use crate::common::server::{ - ConsumerGroupConfig, DataMaintenanceConfig, HeartbeatConfig, MemoryPoolConfig, - MessageSaverConfig, MessagesMaintenanceConfig, PersonalAccessTokenCleanerConfig, - PersonalAccessTokenConfig, TelemetryConfig, TelemetryLogsConfig, TelemetryTracesConfig, - TelemetryTransport, -}; +pub use server_common::log::TelemetryTransport; const DEFAULT_CONFIG_PATH: &str = "core/server/config.toml"; -/// [`SystemConfig`] bound to this crate's own -/// [`super::sharding::ShardingConfig`]. `core/server` names this alias -/// wherever it refers to the system config. -pub type ServerSystemConfig = SystemConfig; - -/// Top-level on-disk config schema for the `iggy-server` binary. -/// -/// Composes the shared section types from [`crate::common`] with the -/// transport, cluster, metadata and [`MessageBusConfig`] sections owned -/// by [`super`]. #[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] #[config_env(prefix = "IGGY_", name = "iggy-server-config")] pub struct ServerConfig { pub consumer_group: ConsumerGroupConfig, pub data_maintenance: DataMaintenanceConfig, - #[serde(default)] - pub extra: ExtraConfig, pub message_saver: MessageSaverConfig, pub personal_access_token: PersonalAccessTokenConfig, pub heartbeat: HeartbeatConfig, - pub system: Arc, + pub system: Arc, pub quic: QuicConfig, pub tcp: TcpConfig, pub http: HttpConfig, pub websocket: WebSocketConfig, pub telemetry: TelemetryConfig, pub cluster: ClusterConfig, - pub metadata: MetadataConfig, - pub partition: PartitionConfig, - pub message_bus: MessageBusConfig, } +/// Configuration for the memory pool. +#[derive(Debug, Deserialize, Serialize, ConfigEnv)] +pub struct MemoryPoolConfig { + pub enabled: bool, + #[config_env(leaf)] + pub size: IggyByteSize, + pub bucket_capacity: u32, +} + +impl MemoryPoolConfig { + pub fn into_other(&self) -> MemoryPoolConfigOther { + MemoryPoolConfigOther { + enabled: self.enabled, + size: self.size, + bucket_capacity: self.bucket_capacity, + } + } +} + +#[serde_as] #[derive(Debug, Default, Deserialize, Serialize, Clone, ConfigEnv)] -pub struct ExtraConfig { - pub namespace: NamespaceConfig, +pub struct DataMaintenanceConfig { + pub messages: MessagesMaintenanceConfig, } +#[serde_as] #[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] -pub struct NamespaceConfig { - pub max_streams: usize, - pub max_topics: usize, - pub max_partitions: usize, +pub struct MessagesMaintenanceConfig { + pub cleaner_enabled: bool, + #[config_env(leaf)] + #[serde_as(as = "DisplayFromStr")] + pub interval: IggyDuration, } -impl Default for NamespaceConfig { - fn default() -> Self { +#[serde_as] +#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] +pub struct MessageSaverConfig { + pub enabled: bool, + pub enforce_fsync: bool, + #[config_env(leaf)] + #[serde_as(as = "DisplayFromStr")] + pub interval: IggyDuration, +} + +#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] +pub struct PersonalAccessTokenConfig { + pub max_tokens_per_user: u32, + pub cleaner: PersonalAccessTokenCleanerConfig, +} + +#[serde_as] +#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] +pub struct PersonalAccessTokenCleanerConfig { + pub enabled: bool, + #[config_env(leaf)] + #[serde_as(as = "DisplayFromStr")] + pub interval: IggyDuration, +} + +#[serde_as] +#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] +pub struct HeartbeatConfig { + pub enabled: bool, + #[config_env(leaf)] + #[serde_as(as = "DisplayFromStr")] + pub interval: IggyDuration, +} + +#[serde_as] +#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] +pub struct ConsumerGroupConfig { + #[config_env(leaf)] + #[serde_as(as = "DisplayFromStr")] + pub rebalancing_timeout: IggyDuration, + #[config_env(leaf)] + #[serde_as(as = "DisplayFromStr")] + pub rebalancing_check_interval: IggyDuration, +} + +#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] +pub struct TelemetryConfig { + pub enabled: bool, + pub service_name: String, + pub logs: TelemetryLogsConfig, + pub traces: TelemetryTracesConfig, +} + +#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] +pub struct TelemetryLogsConfig { + #[config_env(leaf)] + pub transport: TelemetryTransport, + pub endpoint: String, +} + +#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] +pub struct TelemetryTracesConfig { + #[config_env(leaf)] + pub transport: TelemetryTransport, + pub endpoint: String, +} + +impl From<&TelemetryConfig> for TelemetrySettings { + fn from(config: &TelemetryConfig) -> Self { Self { - max_streams: MAX_STREAMS, - max_topics: MAX_TOPICS, - max_partitions: MAX_PARTITIONS, + enabled: config.enabled, + service_name: config.service_name.clone(), + logs: TelemetryEndpointSettings { + transport: config.logs.transport, + endpoint: config.logs.endpoint.clone(), + }, + traces: TelemetryEndpointSettings { + transport: config.traces.transport, + endpoint: config.traces.endpoint.clone(), + }, } } } @@ -103,36 +179,49 @@ impl Default for NamespaceConfig { impl ServerConfig { /// Load server configuration from file and environment variables. /// - /// The path comes from `IGGY_CONFIG_PATH` or defaults to - /// `core/server/config.toml`; missing on-disk paths fall through - /// to the embedded default TOML; env-var overrides flow through the - /// [`ServerConfigEnvProvider`]; the result is validated before - /// returning. - /// - /// # Errors - /// Returns [`ConfigurationError`] when the config cannot be parsed - /// from the configured source(s) or fails [`Validatable::validate`]. + /// Uses compile-time generated env var mappings for unambiguous resolution. pub async fn load() -> Result { + Self::load_with_path( + DEFAULT_CONFIG_PATH, + include_str!("../../../server/config.toml"), + ) + .await + } + + pub async fn load_with_path( + default_config_path: &str, + default_config: &'static str, + ) -> Result { let config_path = - env::var("IGGY_CONFIG_PATH").unwrap_or_else(|_| DEFAULT_CONFIG_PATH.to_string()); - let provider = ServerConfig::config_provider(&config_path); - let cfg: ServerConfig = - provider + env::var("IGGY_CONFIG_PATH").unwrap_or_else(|_| default_config_path.to_string()); + let config_provider = + ServerConfig::config_provider_with_default(&config_path, default_config); + let server_config: ServerConfig = + config_provider .load_config() .await .error(|e: &configs::ConfigurationError| { - format!("{COMPONENT} (error: {e}) - failed to load server config") + format!("{COMPONENT} (error: {e}) - failed to load config") })?; - cfg.validate().error(|e: &configs::ConfigurationError| { - format!("{COMPONENT} (error: {e}) - failed to validate server config") - })?; - Ok(cfg) + server_config + .validate() + .error(|e: &configs::ConfigurationError| { + format!("{COMPONENT} (error: {e}) - failed to validate server config") + })?; + Ok(server_config) } - /// Build the file-backed config provider with the embedded default - /// TOML and the type-safe env-var provider attached. + /// Create a config provider using compile-time generated env var mappings. pub fn config_provider(config_path: &str) -> FileConfigProvider { - let default_config = Toml::string(include_str!("../../../server/config.toml")); + Self::config_provider_with_default(config_path, include_str!("../../../server/config.toml")) + } + + /// Create a config provider using compile-time generated env var mappings. + pub fn config_provider_with_default( + config_path: &str, + default_config: &'static str, + ) -> FileConfigProvider { + let default_config = Toml::string(default_config); FileConfigProvider::new( config_path.to_string(), ServerConfigEnvProvider::default(), @@ -141,16 +230,16 @@ impl ServerConfig { ) } - /// All recognised env var names for [`ServerConfig`]. + /// Returns all valid environment variable names for ServerConfig. pub fn all_env_var_names() -> Vec<&'static str> { ::all_env_var_names() } } -/// Type-safe environment provider for [`ServerConfig`]. +/// Type-safe environment provider using compile-time generated mappings. /// -/// Uses the [`ConfigEnvMappings`] trait generated by `#[derive(ConfigEnv)]` -/// to look up known env var names directly, eliminating path ambiguity. +/// Uses the `ConfigEnvMappings` trait generated by `#[derive(ConfigEnv)]` +/// to directly look up known environment variable names, eliminating path ambiguity. #[derive(Debug, Clone)] pub struct ServerConfigEnvProvider { provider: TypedEnvProvider, @@ -177,47 +266,3 @@ impl Provider for ServerConfigEnvProvider { }) } } - -#[cfg(test)] -mod tests { - use super::*; - use figment::Figment; - - /// The embedded default TOML deserializes into a fully populated - /// [`ServerConfig`] and passes validation. Exercises the - /// `include_str!` resolution and the deserialization of every - /// section without depending on an async runtime in `dev-deps`. - #[test] - fn embedded_default_toml_deserializes_and_validates() { - let toml_str = include_str!("../../../server/config.toml"); - let cfg: ServerConfig = Figment::new() - .merge(Toml::string(toml_str)) - .extract() - .expect("embedded TOML deserializes"); - cfg.validate().expect("embedded default validates"); - - // Spot-check: defaults match the runtime crate's invariants. - assert_eq!(cfg.message_bus.max_batch, 256); - assert_eq!(cfg.message_bus.peer_queue_capacity, 256); - } - - #[test] - fn default_impl_validates() { - let cfg = ServerConfig::default(); - cfg.validate().expect("Default impl validates"); - } - - #[test] - fn env_prefix_is_iggy() { - assert_eq!(ServerConfig::ENV_PREFIX, "IGGY_"); - } - - #[test] - fn all_env_var_names_include_message_bus_section() { - let names = ServerConfig::all_env_var_names(); - assert!( - names.iter().any(|n| n.starts_with("IGGY_MESSAGE_BUS_")), - "expected at least one IGGY_MESSAGE_BUS_* env var, got: {names:?}" - ); - } -} diff --git a/core/configs/src/server_config/sharding.rs b/core/configs/src/server_config/sharding.rs index b690d45fb1..87a411e1fe 100644 --- a/core/configs/src/server_config/sharding.rs +++ b/core/configs/src/server_config/sharding.rs @@ -15,59 +15,21 @@ // specific language governing permissions and limitations // under the License. -//! Sharding config: the full thread-per-core + bus surface, with -//! defaults read from the embedded `core/server/config.toml`. - -use iggy_common::IggyDuration; -use iggy_common::Validatable; use serde::{Deserialize, Serialize}; -use serde_with::{DisplayFromStr, serde_as}; -use std::time::Duration; use super::defaults::SERVER_CONFIG; -use crate::ConfigurationError; -use crate::common::validators::validate_cpu_allocation; use configs::ConfigEnv; -// Re-exported so callers reach these through `configs::sharding::*` -// alongside the rest of the section. +// `CpuAllocation`/`NumaConfig` are pure config types and live in their own +// leaf crate so both `configs` and `shard_allocator` can share them without +// pulling each other's heavier dependency trees. Re-exported here to keep the +// `configs::sharding::*` path stable for existing callers. pub use cpu_allocation::{CpuAllocation, NumaConfig}; -/// Maximum permitted per-shard inbox depth. The channel is allocated -/// up-front per shard, so a runaway value here OOMs the process at boot. -/// `1 << 20` (~1M frames) is several orders of magnitude above any -/// realistic backpressure target and still fits comfortably in process -/// address space. -pub const INBOX_CAPACITY_MAX: usize = 1 << 20; - -/// Hard upper bound on `shutdown_drain_timeout`. A drain that never -/// completes wedges process exit; capping at 10 minutes guarantees the -/// watchdog eventually force-tears the bus even with a pathological -/// config typo. -pub const SHUTDOWN_DRAIN_TIMEOUT_MAX: Duration = Duration::from_secs(600); - -/// Hard upper bound on `shutdown_poll_interval`. A poll interval longer -/// than the drain timeout makes the flag effectively unobservable; cap -/// at 5s so Ctrl-C latency stays bounded regardless of config. -pub const SHUTDOWN_POLL_INTERVAL_MAX: Duration = Duration::from_secs(5); - -/// Hard upper bound on `shutdown_join_timeout`. Comfortably above the -/// drain cap so a full drain always fits inside the join budget, while -/// still guaranteeing process exit against a pathological config typo. -pub const SHUTDOWN_JOIN_TIMEOUT_MAX: Duration = Duration::from_secs(900); - -/// Hard upper bound on `reconcile_periodic_interval`. A tick longer -/// than ~30s makes post-failure recovery latency operator-visible; the -/// cap reins in pathological typos without disturbing reasonable -/// production values. -pub const RECONCILE_PERIODIC_INTERVAL_MAX: Duration = Duration::from_secs(30); - -// Every omitted field falls back to the frozen `Default`, so a partial -// `[system.sharding]` table resolves each key independently instead of -// failing on the first missing one (parity with the legacy type). -#[serde_as] +/// Sharding config for the legacy `core/server`. That server consumes only +/// `cpu_allocation` and `pin_cores`; the bus / shutdown / reconcile knobs are +/// server-ng concepts and live in [`crate::server_ng_config::sharding`]. #[derive(Debug, Deserialize, Serialize, ConfigEnv)] -#[serde(default)] pub struct ShardingConfig { #[serde(default)] #[config_env(leaf)] @@ -83,60 +45,6 @@ pub struct ShardingConfig { /// `false` drops both the CPU and memory-node bindings (and logs a /// warning, since NUMA placement without pinning is meaningless). pub pin_cores: bool, - /// Per-shard inter-shard inbox channel capacity. Bounded by design. - /// Drops on full inbox of consensus frames are recovered by VSR - /// retransmit. Drops of cross-shard client Reply frames are terminal: - /// the client never receives the reply (no in-protocol retransmit). - /// Both frame classes share this one channel, so a consensus burst - /// can starve client-reply forwards: size against the worst-case sum - /// of consensus working set + peak client-reply fan-out per shard - /// occurring together. - /// - // TODO(hubcio): split into two priority lanes - one bounded queue for - // consensus frames (drops recovered by VSR retransmit) and one for - // client `Reply` frames (drops terminal, must be sized for worst-case - // fan-out). Current single-channel design is the minimum-viable - // wiring so `frame_drops_total{variant,reason}` surfaces under load - // and yields real numbers to size the split against. - pub inbox_capacity: usize, - /// Wall-clock budget for a single shard's bus drain on shutdown. - /// Drives `IggyMessageBus::shutdown(..)` from the per-shard watchdog - /// and the parallel-join survivor path. Sized larger than typical - /// TCP RTT times in-flight write-batch so writers receive their full - /// last `write_vectored_all` budget before the connection registry - /// force-tears the bus. Slow-fsync hosts may need to extend this past - /// the default; the cap is `SHUTDOWN_DRAIN_TIMEOUT_MAX` so a config - /// typo cannot wedge process exit. - #[serde_as(as = "DisplayFromStr")] - #[config_env(leaf)] - pub shutdown_drain_timeout: IggyDuration, - /// Poll cadence for the cross-thread shutdown flag and for the - /// `await_metadata_bundle` / `broadcast_metadata_bundle` poll loops. - /// Trades off Ctrl-C latency against idle wakeup cost; the default - /// keeps shutdown observably prompt without measurable scheduler - /// overhead. Capped at `SHUTDOWN_POLL_INTERVAL_MAX` so the flag - /// remains effectively observable regardless of config. - #[serde_as(as = "DisplayFromStr")] - #[config_env(leaf)] - pub shutdown_poll_interval: IggyDuration, - /// Hard wall-clock deadline for joining shard threads at process - /// exit. A shard whose pump or listener wedges past this budget is - /// abandoned with an error log instead of blocking exit forever. - /// Must be at least `shutdown_drain_timeout` (abandoning a shard - /// mid-drain would interrupt its WAL fsync / replica drain) and at - /// most [`SHUTDOWN_JOIN_TIMEOUT_MAX`]. - #[serde_as(as = "DisplayFromStr")] - #[config_env(leaf)] - pub shutdown_join_timeout: IggyDuration, - /// Safety-tick cadence for the partition reconciliation loop; the - /// reconciler also wakes immediately on every - /// `LifecycleFrame::MetadataCommitTick` from shard 0. This periodic - /// fallback covers dropped wake-ups (the wake channel is capacity-1) - /// and the initial post-bootstrap convergence window. Values above - /// [`RECONCILE_PERIODIC_INTERVAL_MAX`] are rejected by the validator. - #[serde_as(as = "DisplayFromStr")] - #[config_env(leaf)] - pub reconcile_periodic_interval: IggyDuration, } impl Default for ShardingConfig { @@ -144,283 +52,6 @@ impl Default for ShardingConfig { Self { cpu_allocation: CpuAllocation::default(), pin_cores: SERVER_CONFIG.system.sharding.pin_cores, - inbox_capacity: SERVER_CONFIG.system.sharding.inbox_capacity as usize, - shutdown_drain_timeout: SERVER_CONFIG - .system - .sharding - .shutdown_drain_timeout - .parse() - .unwrap(), - shutdown_poll_interval: SERVER_CONFIG - .system - .sharding - .shutdown_poll_interval - .parse() - .unwrap(), - shutdown_join_timeout: SERVER_CONFIG - .system - .sharding - .shutdown_join_timeout - .parse() - .unwrap(), - reconcile_periodic_interval: SERVER_CONFIG - .system - .sharding - .reconcile_periodic_interval - .parse() - .unwrap(), - } - } -} - -impl Validatable for ShardingConfig { - fn validate(&self) -> Result<(), ConfigurationError> { - if self.inbox_capacity == 0 { - eprintln!( - "Invalid sharding configuration: inbox_capacity must be > 0 (crossfire silently \ - rounds 0 to 1, masking config errors)" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if self.inbox_capacity > INBOX_CAPACITY_MAX { - eprintln!( - "Invalid sharding configuration: inbox_capacity {} exceeds the {} cap (each \ - shard preallocates a channel of this size; oversizing here OOMs the process at \ - boot)", - self.inbox_capacity, INBOX_CAPACITY_MAX - ); - return Err(ConfigurationError::InvalidConfigurationValue); } - - let drain = self.shutdown_drain_timeout.get_duration(); - if drain.is_zero() { - eprintln!( - "Invalid sharding configuration: shutdown_drain_timeout must be > 0 (a zero \ - budget force-tears the bus mid-WAL-fsync on every shutdown)" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if drain > SHUTDOWN_DRAIN_TIMEOUT_MAX { - eprintln!( - "Invalid sharding configuration: shutdown_drain_timeout {:?} exceeds the {:?} \ - cap (an unbounded drain wedges process exit on bus stall)", - drain, SHUTDOWN_DRAIN_TIMEOUT_MAX - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - let poll = self.shutdown_poll_interval.get_duration(); - if poll.is_zero() { - eprintln!( - "Invalid sharding configuration: shutdown_poll_interval must be > 0 (a zero \ - cadence busy-loops every shard's watchdog and metadata-handoff poller)" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if poll > SHUTDOWN_POLL_INTERVAL_MAX { - eprintln!( - "Invalid sharding configuration: shutdown_poll_interval {:?} exceeds the {:?} \ - cap (a coarse cadence stalls Ctrl-C handling and metadata handoff abort)", - poll, SHUTDOWN_POLL_INTERVAL_MAX - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if poll > drain { - eprintln!( - "Invalid sharding configuration: shutdown_poll_interval {:?} must be <= \ - shutdown_drain_timeout {:?} (a poll cadence coarser than the drain budget makes \ - the shutdown flag effectively unobservable)", - poll, drain - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - let join = self.shutdown_join_timeout.get_duration(); - if join < drain { - eprintln!( - "Invalid sharding configuration: shutdown_join_timeout {:?} must be >= \ - shutdown_drain_timeout {:?} (a join budget shorter than the drain abandons \ - shards mid-drain, interrupting the WAL fsync / replica drain)", - join, drain - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if join > SHUTDOWN_JOIN_TIMEOUT_MAX { - eprintln!( - "Invalid sharding configuration: shutdown_join_timeout {:?} exceeds the {:?} \ - cap (an unbounded join budget wedges process exit on a stuck shard)", - join, SHUTDOWN_JOIN_TIMEOUT_MAX - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - let reconcile = self.reconcile_periodic_interval.get_duration(); - if reconcile.is_zero() { - eprintln!( - "Invalid sharding configuration: reconcile_periodic_interval resolves to zero. \ - Note that \"0\", \"none\", \"unlimited\", and \"disabled\" all parse to zero. The \ - periodic reconcile tick is a safety net for dropped commit-wakes and cannot be \ - turned off; set a positive duration (default \"1s\", max {RECONCILE_PERIODIC_INTERVAL_MAX:?})." - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if reconcile > RECONCILE_PERIODIC_INTERVAL_MAX { - eprintln!( - "Invalid sharding configuration: reconcile_periodic_interval {:?} exceeds the \ - {:?} cap (a long tick makes post-failure convergence latency operator-visible)", - reconcile, RECONCILE_PERIODIC_INTERVAL_MAX - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - validate_cpu_allocation(&self.cpu_allocation, self.pin_cores) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::server_config::server::ServerConfig; - use figment::Figment; - use figment::providers::{Format, Toml}; - - #[test] - fn defaults_validate() { - assert!(ShardingConfig::default().validate().is_ok()); - } - - #[test] - fn zero_drain_is_rejected() { - let cfg = ShardingConfig { - shutdown_drain_timeout: IggyDuration::new(Duration::ZERO), - ..ShardingConfig::default() - }; - assert!(cfg.validate().is_err()); - } - - #[test] - fn over_cap_drain_is_rejected() { - let cfg = ShardingConfig { - shutdown_drain_timeout: IggyDuration::new( - SHUTDOWN_DRAIN_TIMEOUT_MAX + Duration::from_secs(1), - ), - ..ShardingConfig::default() - }; - assert!(cfg.validate().is_err()); - } - - #[test] - fn zero_poll_is_rejected() { - let cfg = ShardingConfig { - shutdown_poll_interval: IggyDuration::new(Duration::ZERO), - ..ShardingConfig::default() - }; - assert!(cfg.validate().is_err()); - } - - #[test] - fn over_cap_poll_is_rejected() { - let cfg = ShardingConfig { - shutdown_poll_interval: IggyDuration::new( - SHUTDOWN_POLL_INTERVAL_MAX + Duration::from_secs(1), - ), - ..ShardingConfig::default() - }; - assert!(cfg.validate().is_err()); - } - - #[test] - fn poll_greater_than_drain_is_rejected() { - let cfg = ShardingConfig { - shutdown_drain_timeout: IggyDuration::new(Duration::from_millis(20)), - shutdown_poll_interval: IggyDuration::new(Duration::from_millis(50)), - ..ShardingConfig::default() - }; - assert!(cfg.validate().is_err()); - } - - #[test] - fn join_shorter_than_drain_is_rejected() { - // A join budget under the drain would abandon shards mid-drain. - let cfg = ShardingConfig { - shutdown_drain_timeout: IggyDuration::new(Duration::from_secs(10)), - shutdown_join_timeout: IggyDuration::new(Duration::from_secs(5)), - ..ShardingConfig::default() - }; - assert!(cfg.validate().is_err()); - } - - #[test] - fn over_cap_join_is_rejected() { - let cfg = ShardingConfig { - shutdown_join_timeout: IggyDuration::new( - SHUTDOWN_JOIN_TIMEOUT_MAX + Duration::from_secs(1), - ), - ..ShardingConfig::default() - }; - assert!(cfg.validate().is_err()); - } - - #[test] - fn join_equal_to_drain_is_accepted() { - let cfg = ShardingConfig { - shutdown_drain_timeout: IggyDuration::new(Duration::from_secs(10)), - shutdown_join_timeout: IggyDuration::new(Duration::from_secs(10)), - ..ShardingConfig::default() - }; - assert!(cfg.validate().is_ok()); - } - - // Guards the single source of truth: the sharding defaults resolve - // from the embedded TOML, not hard-coded Rust values. - #[test] - fn embedded_toml_resolves_sharding_defaults() { - let toml_str = include_str!("../../../server/config.toml"); - let config: ServerConfig = Figment::new() - .merge(Toml::string(toml_str)) - .extract() - .expect("embedded TOML deserializes"); - config.validate().expect("embedded config validates"); - - let sharding = &config.system.sharding; - assert!(sharding.pin_cores); - assert_eq!(sharding.inbox_capacity, 1024); - assert_eq!(sharding.shutdown_drain_timeout, "10 s".parse().unwrap()); - assert_eq!(sharding.shutdown_poll_interval, "50 ms".parse().unwrap()); - assert_eq!(sharding.shutdown_join_timeout, "30 s".parse().unwrap()); - assert_eq!(sharding.reconcile_periodic_interval, "1 s".parse().unwrap()); - } - - // Extract straight from a raw table (no embedded base layer) so the - // struct-level `#[serde(default)]` is what fills the gaps, not the - // provider's embedded-TOML fallback. - #[test] - fn partial_table_fills_missing_fields_with_frozen_defaults() { - let sharding: ShardingConfig = Figment::new() - .merge(Toml::string("pin_cores = false")) - .extract() - .expect("partial sharding table deserializes"); - - assert!(!sharding.pin_cores); - assert_eq!(sharding.inbox_capacity, 1024); - assert_eq!(sharding.shutdown_drain_timeout, "10 s".parse().unwrap()); - assert_eq!(sharding.shutdown_poll_interval, "50 ms".parse().unwrap()); - assert_eq!(sharding.shutdown_join_timeout, "30 s".parse().unwrap()); - assert_eq!(sharding.reconcile_periodic_interval, "1 s".parse().unwrap()); - } - - #[test] - fn empty_table_yields_all_frozen_defaults() { - let sharding: ShardingConfig = Figment::new() - .merge(Toml::string("")) - .extract() - .expect("empty sharding table deserializes"); - - assert!(sharding.pin_cores); - assert_eq!(sharding.inbox_capacity, 1024); - assert_eq!(sharding.shutdown_drain_timeout, "10 s".parse().unwrap()); - assert_eq!(sharding.shutdown_poll_interval, "50 ms".parse().unwrap()); - assert_eq!(sharding.shutdown_join_timeout, "30 s".parse().unwrap()); - assert_eq!(sharding.reconcile_periodic_interval, "1 s".parse().unwrap()); } } diff --git a/core/configs/src/common/system.rs b/core/configs/src/server_config/system.rs similarity index 96% rename from core/configs/src/common/system.rs rename to core/configs/src/server_config/system.rs index d72913fc5c..60437a94fb 100644 --- a/core/configs/src/common/system.rs +++ b/core/configs/src/server_config/system.rs @@ -17,6 +17,7 @@ use super::cache_indexes::CacheIndexesConfig; use super::server::MemoryPoolConfig; +use super::sharding::ShardingConfig; use configs::{ConfigEnv, ConfigEnvMappings}; use iggy_common::IggyByteSize; use iggy_common::IggyError; @@ -32,11 +33,13 @@ use server_common::log::LoggingSettings; pub const INDEX_EXTENSION: &str = "index"; pub const LOG_EXTENSION: &str = "log"; -// Generic over the sharding config so every server flavour binds its own -// `ShardingConfig` (different knob sets, different default source) while -// sharing this whole struct and its path helpers. +// Generic over the sharding config so the legacy server and `server-ng` each +// bind their own `ShardingConfig` (different knob sets, different default +// source) while sharing this whole struct and its path helpers. The default +// type param keeps bare `SystemConfig` meaning the legacy variant, so existing +// callers compile unchanged. #[derive(Debug, Deserialize, Serialize, ConfigEnv)] -pub struct SystemConfig { +pub struct SystemConfig { pub path: String, pub backup: BackupConfig, pub state: StateConfig, @@ -175,8 +178,6 @@ pub struct RecoveryConfig { pub struct SegmentConfig { #[config_env(leaf)] pub size: IggyByteSize, - #[serde(default)] - pub preallocate: bool, #[config_env(leaf)] pub cache_indexes: CacheIndexesConfig, pub archive_expired: bool, diff --git a/core/configs/src/server_config/tcp.rs b/core/configs/src/server_config/tcp.rs index 8e0acafdb8..7d81f20930 100644 --- a/core/configs/src/server_config/tcp.rs +++ b/core/configs/src/server_config/tcp.rs @@ -15,8 +15,6 @@ // specific language governing permissions and limitations // under the License. -//! TCP listener schema. - use configs::ConfigEnv; use iggy_common::{IggyByteSize, IggyDuration}; use serde::{Deserialize, Serialize}; diff --git a/core/configs/src/server_config/validators.rs b/core/configs/src/server_config/validators.rs index 6784ef294e..d35763bc6a 100644 --- a/core/configs/src/server_config/validators.rs +++ b/core/configs/src/server_config/validators.rs @@ -15,27 +15,45 @@ // specific language governing permissions and limitations // under the License. -//! [`Validatable`] for [`ServerConfig`]. -//! -//! Delegates section by section, including -//! [`super::message_bus::MessageBusConfig::validate`], then applies the -//! cross-section invariants: topic vs segment sizing, JWT gating when -//! HTTP is enabled, and server-default expiry sanity. - use super::COMPONENT; -use super::cluster::STATE_CHUNK_HEADER_LEN; -use super::server::{ExtraConfig, NamespaceConfig, ServerConfig}; +use super::cluster::ClusterConfig; +use super::server::{ + DataMaintenanceConfig, MessageSaverConfig, MessagesMaintenanceConfig, TelemetryConfig, +}; +use super::server::{MemoryPoolConfig, PersonalAccessTokenConfig, ServerConfig}; +use super::sharding::{CpuAllocation, ShardingConfig}; +use super::system::SegmentConfig; +use super::system::{CompressionConfig, LoggingConfig, PartitionConfig}; use crate::ConfigurationError; +use cpu_allocation::allowed_cpus; use err_trail::ErrContext; -use iggy_common::{IggyExpiry, MaxTopicSize, Validatable}; -use server_common::sharding::IggyNamespace; +use iggy_common::CompressionAlgorithm; +use iggy_common::IggyExpiry; +use iggy_common::MaxTopicSize; +use iggy_common::Validatable; +use std::thread::available_parallelism; use tracing::warn; -/// compio-ws (tungstenite 0.29) `write_buffer_size` default. Used to -/// evaluate the `max_write_buffer_size > write_buffer_size` invariant -/// when the operator leaves `write_buffer_size` unset; keep in sync -/// with the defaults documented in the shipped config.toml. -const WS_DEFAULT_WRITE_BUFFER_SIZE: u64 = 128 * 1024; +/// 1 GiB max segment size. Canonical definition; re-exported by core/server streaming. +pub const SEGMENT_MAX_SIZE_BYTES: u64 = 1024 * 1024 * 1024; + +/// Return `Err(reason)` when `alloc` would yield a host-dependent shard +/// count, disqualifying it for cluster mode where every node must derive +/// the same count from its byte-identical config. +/// +/// Deterministic variants (`Count(n)`, `Range(s, e)`, explicit +/// `NumaAware`) return `Ok`. +fn host_dependent_cpu_allocation(alloc: &CpuAllocation) -> Result<(), &'static str> { + match alloc { + CpuAllocation::All => Err("'all' (follows host CPU count)"), + CpuAllocation::NumaAware(numa) if numa.nodes.is_empty() && numa.cores_per_node == 0 => { + Err("'numa:auto' (follows host NUMA topology)") + } + CpuAllocation::Count(_) | CpuAllocation::Range(_, _) | CpuAllocation::NumaAware(_) => { + Ok(()) + } + } +} impl Validatable for ServerConfig { fn validate(&self) -> Result<(), ConfigurationError> { @@ -57,9 +75,6 @@ impl Validatable for ServerConfig { "{COMPONENT} (error: {e}) - failed to validate personal access token config" ) })?; - self.extra.validate().error(|e: &ConfigurationError| { - format!("{COMPONENT} (error: {e}) - failed to validate extra config") - })?; self.system .segment .validate() @@ -84,755 +99,876 @@ impl Validatable for ServerConfig { self.cluster.validate().error(|e: &ConfigurationError| { format!("{COMPONENT} (error: {e}) - failed to validate cluster config") })?; - self.metadata.validate().error(|e: &ConfigurationError| { - format!("{COMPONENT} (error: {e}) - failed to validate metadata config") - })?; - self.partition.validate().error(|e: &ConfigurationError| { - format!("{COMPONENT} (error: {e}) - failed to validate partition config") - })?; + + // Cluster consensus routing (`calculate_shard_from_consensus_ns`) + // hashes namespaces modulo the local shard count. Every node must + // agree on that count or control-plane messages (StartViewChange, + // DoViewChange, StartView, Commit) route to different shards on + // different nodes, splitting the view-change quorum. + // + // Because `cluster.nodes` is byte-identical across every host, the + // only way shard counts can drift is if `system.sharding.cpu_allocation` + // depends on host topology. We can't prove divergence from a single + // node at load time (peers may be homogeneous), so this is a warning + // rather than a hard error; operators running heterogeneous hardware + // must pin a deterministic value themselves. + if self.cluster.enabled + && let Err(reason) = host_dependent_cpu_allocation(&self.system.sharding.cpu_allocation) + { + warn!( + "cluster.enabled = true with host-dependent system.sharding.cpu_allocation ({reason}); \ + if peers resolve this to different shard counts, view-change quorum will split. \ + Pin a deterministic value (count, explicit range, or explicit numa) on heterogeneous hardware." + ); + } + self.system .logging .validate() .error(|e: &ConfigurationError| { format!("{COMPONENT} (error: {e}) - failed to validate logging config") })?; - self.message_saver - .validate() - .error(|e: &ConfigurationError| { - format!("{COMPONENT} (error: {e}) - failed to validate message saver config") - })?; let topic_size = match self.system.topic.max_size { MaxTopicSize::Custom(size) => Ok(size.as_bytes_u64()), MaxTopicSize::Unlimited => Ok(u64::MAX), MaxTopicSize::ServerDefault => { - eprintln!("system.topic.max_size cannot be ServerDefault in the server config"); + eprintln!("system.topic.max_size cannot be ServerDefault in server config"); Err(ConfigurationError::InvalidConfigurationValue) } }?; if let IggyExpiry::ServerDefault = self.system.topic.message_expiry { - eprintln!("system.topic.message_expiry cannot be ServerDefault in the server config"); + eprintln!("system.topic.message_expiry cannot be ServerDefault in server config"); return Err(ConfigurationError::InvalidConfigurationValue); } - // A zero duration encodes to wire value 0, the same value the wire uses - // for ServerDefault, so it would silently collide with that sentinel. - if let IggyExpiry::ExpireDuration(duration) = self.system.topic.message_expiry - && duration.as_micros() == 0 + if self.http.enabled + && let IggyExpiry::ServerDefault = self.http.jwt.access_token_expiry { + eprintln!("http.jwt.access_token_expiry cannot be ServerDefault when HTTP is enabled"); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + if topic_size < self.system.segment.size.as_bytes_u64() { eprintln!( - "system.topic.message_expiry is a zero duration, which collides with the server-default sentinel on the wire; use \"none\" to never expire or a positive duration" + "system.topic.max_size ({} B) must be >= system.segment.size ({} B)", + topic_size, + self.system.segment.size.as_bytes_u64() ); return Err(ConfigurationError::InvalidConfigurationValue); } - if self.http.enabled - && let IggyExpiry::ServerDefault = self.http.jwt.access_token_expiry - { - eprintln!("http.jwt.access_token_expiry cannot be ServerDefault when HTTP is enabled"); + Ok(()) + } +} + +impl Validatable for CompressionConfig { + fn validate(&self) -> Result<(), ConfigurationError> { + let compression_alg = &self.default_algorithm; + if *compression_alg != CompressionAlgorithm::None { + // TODO(numinex): Change this message once server side compression is fully developed. + warn!( + "Server started with server-side compression enabled, using algorithm: {compression_alg}, this feature is not implemented yet!" + ); + } + + Ok(()) + } +} + +impl Validatable for TelemetryConfig { + fn validate(&self) -> Result<(), ConfigurationError> { + if !self.enabled { + return Ok(()); + } + + if self.service_name.trim().is_empty() { + eprintln!("telemetry.service_name cannot be empty when telemetry is enabled"); return Err(ConfigurationError::InvalidConfigurationValue); } - if self.http.enabled - && self.http.tls.enabled - && (self.http.tls.cert_file.is_empty() || self.http.tls.key_file.is_empty()) - { + if self.logs.endpoint.is_empty() { + eprintln!("telemetry.logs.endpoint cannot be empty when telemetry is enabled"); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + if self.traces.endpoint.is_empty() { + eprintln!("telemetry.traces.endpoint cannot be empty when telemetry is enabled"); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + Ok(()) + } +} + +impl Validatable for PartitionConfig { + fn validate(&self) -> Result<(), ConfigurationError> { + if self.messages_required_to_save == 0 { + eprintln!("Configured system.partition.messages_required_to_save cannot be 0"); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + Ok(()) + } +} + +impl Validatable for SegmentConfig { + fn validate(&self) -> Result<(), ConfigurationError> { + if self.size > SEGMENT_MAX_SIZE_BYTES { eprintln!( - "http.tls.enabled=true requires non-empty http.tls.cert_file and http.tls.key_file" + "Configured system.segment.size {} B is greater than maximum {} B", + self.size.as_bytes_u64(), + SEGMENT_MAX_SIZE_BYTES ); return Err(ConfigurationError::InvalidConfigurationValue); } - // Cluster mode has no port fallbacks: the roster is the single source - // of listener ports, so every enabled transport needs an explicit - // per-node port. Falling back to the port of a transport's top-level - // `address` would hand two same-host nodes the same socket and fail - // only at bind time, and a portless node would silently degrade every - // follower-to-primary HTTP forward through it to a fail-closed 503. - if self.cluster.enabled { - for node in &self.cluster.nodes { - let required_ports = [ - ("tcp", true, node.ports.tcp), - ("quic", self.quic.enabled, node.ports.quic), - ("http", self.http.enabled, node.ports.http), - ("websocket", self.websocket.enabled, node.ports.websocket), - ("tcp_replica", true, node.ports.tcp_replica), - ]; - for (transport, enabled, port) in required_ports { - if enabled && port.is_none() { - eprintln!( - "cluster node '{}' has no ports.{transport}; cluster mode requires an explicit roster port for every enabled transport", - node.name - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - } - } + if !self.size.as_bytes_u64().is_multiple_of(512) { + eprintln!( + "Configured system.segment.size {} B is not a multiple of 512 B", + self.size.as_bytes_u64() + ); + return Err(ConfigurationError::InvalidConfigurationValue); } - if topic_size < self.system.segment.size.as_bytes_u64() { + Ok(()) + } +} + +impl Validatable for MessageSaverConfig { + fn validate(&self) -> Result<(), ConfigurationError> { + if self.enabled && self.interval.is_zero() { + eprintln!("message_saver.interval cannot be zero when message_saver is enabled"); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + Ok(()) + } +} + +impl Validatable for DataMaintenanceConfig { + fn validate(&self) -> Result<(), ConfigurationError> { + self.messages.validate().error(|e: &ConfigurationError| { + format!("{COMPONENT} (error: {e}) - failed to validate messages maintenance config") + })?; + Ok(()) + } +} + +impl Validatable for MessagesMaintenanceConfig { + fn validate(&self) -> Result<(), ConfigurationError> { + if self.cleaner_enabled && self.interval.is_zero() { + eprintln!("data_maintenance.messages.interval cannot be zero when cleaner is enabled"); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + Ok(()) + } +} + +impl Validatable for PersonalAccessTokenConfig { + fn validate(&self) -> Result<(), ConfigurationError> { + if self.max_tokens_per_user == 0 { + eprintln!("personal_access_token.max_tokens_per_user cannot be 0"); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + if self.cleaner.enabled && self.cleaner.interval.is_zero() { eprintln!( - "system.topic.max_size ({} B) must be >= system.segment.size ({} B)", - topic_size, - self.system.segment.size.as_bytes_u64() + "personal_access_token.cleaner.interval cannot be zero when cleaner is enabled" ); return Err(ConfigurationError::InvalidConfigurationValue); } - // A received segment artifact can be one whole batch larger than the - // segment cap (rotation checks the cap AFTER appending), and the real - // batch bound is the BUS frame cap -- the server never enforces - // `MAX_PAYLOAD_SIZE`. An artifact ceiling under that floor refuses a - // legal segment, and the manifest check is all-or-nothing, so the - // partition livelocks re-requesting the same segment from every peer at - // the backoff ceiling. Caught here so it is a boot error rather than one - // partition that silently never rejoins. - let artifact_floor = self - .system - .segment - .size - .as_bytes_u64() - .saturating_add(self.message_bus.max_message_size.as_bytes_u64()); - if self.partition.transfer_artifact_bytes_max.as_bytes_u64() < artifact_floor { + Ok(()) + } +} + +impl Validatable for LoggingConfig { + fn validate(&self) -> Result<(), ConfigurationError> { + if self.level.is_empty() { + eprintln!("system.logging.level is supposed be configured"); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + if self.retention.as_secs() < 1 { eprintln!( - "{COMPONENT} partition.transfer_artifact_bytes_max ({} B) must be at least \ - system.segment.size ({} B) + message_bus.max_message_size ({} B) = \ - {artifact_floor} B: a segment may close one whole batch past its cap, and an \ - artifact ceiling below that refuses a legal segment and livelocks the \ - partition's rejoin", - self.partition.transfer_artifact_bytes_max.as_bytes_u64(), - self.system.segment.size.as_bytes_u64(), - self.message_bus.max_message_size.as_bytes_u64(), + "Configured system.logging.retention {} is less than minimum 1 second", + self.retention ); return Err(ConfigurationError::InvalidConfigurationValue); } - self.message_bus - .validate() - .error(|e: &ConfigurationError| { - format!("{COMPONENT} (error: {e}) - failed to validate message_bus config") - })?; + if self.rotation_check_interval.as_secs() < 1 { + eprintln!( + "Configured system.logging.rotation_check_interval {} is less than minimum 1 second", + self.rotation_check_interval + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + let max_total_size_unlimited = self.max_total_size.as_bytes_u64() == 0; + if !max_total_size_unlimited + && self.max_file_size.as_bytes_u64() > self.max_total_size.as_bytes_u64() + { + eprintln!( + "Configured system.logging.max_total_size {} is less than system.logging.max_file_size {}", + self.max_total_size, self.max_file_size + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } - // Repair frames ride the bounded per-peer message-bus queue. A repair - // round of cluster.repair_chunk_max frames that meets or overruns - // message_bus.peer_queue_capacity drops its own tail silently, wedging - // the repair loop into slow retries. Keep the chunk strictly below the - // queue; this also floors peer_queue_capacity, which is otherwise only - // checked for > 0. - if self.cluster.repair_chunk_max >= self.message_bus.peer_queue_capacity { + Ok(()) + } +} + +impl Validatable for MemoryPoolConfig { + fn validate(&self) -> Result<(), ConfigurationError> { + if self.enabled && self.size == 0 { eprintln!( - "{COMPONENT} cluster.repair_chunk_max ({}) must be < message_bus.peer_queue_capacity ({}): repair frames ride the per-peer bus queue, so a chunk that fills or overruns it drops frames and wedges repair", - self.cluster.repair_chunk_max, self.message_bus.peer_queue_capacity + "Configured system.memory_pool.enabled is true and system.memory_pool.size is 0" ); return Err(ConfigurationError::InvalidConfigurationValue); } - // State-transfer chunks ride the same bus. A cap that cannot carry one - // header plus a byte of payload makes every rejoin that needs a - // transfer impossible, and the failure surfaces only as a replica - // connection tearing down when the frame is rejected on the read side. - let bus_cap = self.message_bus.max_message_size.as_bytes_u64(); - if bus_cap <= STATE_CHUNK_HEADER_LEN { + const MIN_POOL_SIZE: u64 = 512 * 1024 * 1024; // 512 MiB + const MIN_BUCKET_CAPACITY: u32 = 128; + const DEFAULT_PAGE_SIZE: u64 = 4096; + + if self.enabled && self.size < MIN_POOL_SIZE { eprintln!( - "{COMPONENT} message_bus.max_message_size ({bus_cap}) must exceed the {STATE_CHUNK_HEADER_LEN}-byte state-chunk header: state transfer serves artifact chunks over this bus, and a frame above the cap is rejected by the receiving transport, which tears down the whole replica connection" + "Configured system.memory_pool.size {} B ({} MiB) is less than minimum {} B, ({} MiB)", + self.size.as_bytes_u64(), + self.size.as_bytes_u64() / (1024 * 1024), + MIN_POOL_SIZE, + MIN_POOL_SIZE / (1024 * 1024), ); return Err(ConfigurationError::InvalidConfigurationValue); } - // WS frame chain: websocket.max_frame_size <= websocket.max_message_size - // <= message_bus.max_message_size. The bus's WS / WSS install path takes - // its frame tuning from [websocket], so a WS ceiling above the bus's own - // frame cap would admit messages the bus read-side validator then tears - // the connection down over. An absent knob defers to the compio-ws - // default (16 MiB frame / 64 MiB message), which satisfies the chain - // against the shipped bus cap in practice. - let bus_max_message_size = self.message_bus.max_message_size.as_bytes_u64(); - if let (Some(frame), Some(message)) = ( - self.websocket.max_frame_size, - self.websocket.max_message_size, - ) && frame.as_bytes_u64() > message.as_bytes_u64() - { + if self.enabled && !self.size.as_bytes_u64().is_multiple_of(DEFAULT_PAGE_SIZE) { eprintln!( - "{COMPONENT} websocket.max_frame_size ({}) exceeds websocket.max_message_size ({})", - frame.as_bytes_u64(), - message.as_bytes_u64() + "Configured system.memory_pool.size {} B is not a multiple of default page size {} B", + self.size.as_bytes_u64(), + DEFAULT_PAGE_SIZE ); return Err(ConfigurationError::InvalidConfigurationValue); } - if let Some(message) = self.websocket.max_message_size - && message.as_bytes_u64() > bus_max_message_size - { + + if self.enabled && self.bucket_capacity < MIN_BUCKET_CAPACITY { eprintln!( - "{COMPONENT} websocket.max_message_size ({}) exceeds message_bus.max_message_size ({})", - message.as_bytes_u64(), - bus_max_message_size + "Configured system.memory_pool.buffers {} is less than minimum {}", + self.bucket_capacity, MIN_BUCKET_CAPACITY ); return Err(ConfigurationError::InvalidConfigurationValue); } - if let Some(frame) = self.websocket.max_frame_size - && frame.as_bytes_u64() > bus_max_message_size - { + + if self.enabled && !self.bucket_capacity.is_power_of_two() { eprintln!( - "{COMPONENT} websocket.max_frame_size ({}) exceeds message_bus.max_message_size ({})", - frame.as_bytes_u64(), - bus_max_message_size + "Configured system.memory_pool.buffers {} is not a power of 2", + self.bucket_capacity ); return Err(ConfigurationError::InvalidConfigurationValue); } - // "0", "unlimited" and "none" all parse to a zero IggyByteSize. A - // zero WS tunable is never usable: zero message or frame ceilings - // reject every inbound frame, and zero buffers starve the - // compio-ws pipeline. Reject at boot instead of shipping a - // listener that cannot serve a single message. - for (key, size) in [ - ("read_buffer_size", self.websocket.read_buffer_size), - ("write_buffer_size", self.websocket.write_buffer_size), - ( - "max_write_buffer_size", - self.websocket.max_write_buffer_size, - ), - ("max_message_size", self.websocket.max_message_size), - ("max_frame_size", self.websocket.max_frame_size), - ] { - if let Some(size) = size - && size.as_bytes_u64() == 0 - { + Ok(()) + } +} + +/// Validate a [`CpuAllocation`] against the machine's available parallelism +/// and, when pinning, the process affinity mask. Shared by the legacy and +/// server-ng sharding configs, which both carry these two knobs. +pub(crate) fn validate_cpu_allocation( + cpu_allocation: &CpuAllocation, + pin_cores: bool, +) -> Result<(), ConfigurationError> { + let available_cpus = available_parallelism() + .map_err(|_| { + eprintln!("Failed to detect available CPU cores"); + ConfigurationError::InvalidConfigurationValue + })? + .get(); + + match cpu_allocation { + CpuAllocation::All => Ok(()), + CpuAllocation::Count(count) => { + if *count == 0 { + eprintln!("Invalid sharding configuration: cpu_allocation count cannot be 0"); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if *count > available_cpus { eprintln!( - "{COMPONENT} websocket.{key} must be non-zero (\"0\", \"unlimited\" and \"none\" all parse to zero)" + "Invalid sharding configuration: cpu_allocation count {count} exceeds available CPU cores {available_cpus}" ); return Err(ConfigurationError::InvalidConfigurationValue); } + Ok(()) } - - // tungstenite asserts `max_write_buffer_size > write_buffer_size` - // during connection setup, so a violating pair panics on every - // accepted socket. Enforce the invariant at boot; an unset - // write_buffer_size runs at the compio-ws default. - if let Some(max_write) = self.websocket.max_write_buffer_size { - let write_buffer_size = self - .websocket - .write_buffer_size - .map_or(WS_DEFAULT_WRITE_BUFFER_SIZE, |size| size.as_bytes_u64()); - if max_write.as_bytes_u64() <= write_buffer_size { + CpuAllocation::Range(start, end) => { + if start >= end { + eprintln!( + "Invalid sharding configuration: cpu_allocation range {start}..{end} is invalid (start must be less than end)" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if *end - *start > available_cpus { + eprintln!( + "Invalid sharding configuration: cpu_allocation range {start}..{end} yields {} shards, exceeding available CPU cores {available_cpus}", + *end - *start + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if !pin_cores { + return Ok(()); + } + let allowed = allowed_cpus(); + if let Some(cpu) = (*start..*end).find(|cpu| !allowed.contains(cpu)) { eprintln!( - "{COMPONENT} websocket.max_write_buffer_size ({}) must exceed websocket.write_buffer_size ({write_buffer_size})", - max_write.as_bytes_u64() + "Invalid sharding configuration: cpu_allocation range {start}..{end} includes CPU {cpu}, which is outside the set of cores allowed for this process (affinity/cpuset mask)" ); return Err(ConfigurationError::InvalidConfigurationValue); } + Ok(()) } + // NUMA topology validation requires hwlocality (runtime dep). + // Full NUMA validation happens in shard_allocator at startup. + CpuAllocation::NumaAware(_) => Ok(()), + } +} - self.quic.validate().error(|e: &ConfigurationError| { - format!("{COMPONENT} (error: {e}) - failed to validate quic config") - })?; +impl Validatable for ShardingConfig { + fn validate(&self) -> Result<(), ConfigurationError> { + validate_cpu_allocation(&self.cpu_allocation, self.pin_cores) + } +} - // Both knobs below sit on shared section structs, so the rejects live - // here rather than in those types' own `Validatable` impls. `0` / - // `disabled` / `unlimited` all parse to the same zero duration. - if self - .consumer_group - .rebalancing_timeout - .get_duration() - .is_zero() - { - eprintln!( - "{COMPONENT} consumer_group.rebalancing_timeout must be nonzero: it is the deadline after which a pending revocation completes without the source client committing what it was served, so zero force-transfers every partition on the next reconciler tick and reopens the duplicate-delivery window" - ); +/// Length floor for the replica-auth PSK, in raw bytes. The 32-byte MAC key +/// is KDF-derived from these bytes at use-site, so any encoding clearing this +/// length is accepted. +const MIN_SHARED_SECRET_LEN: usize = 32; + +impl Validatable for ClusterConfig { + fn validate(&self) -> Result<(), ConfigurationError> { + if !self.enabled { + return Ok(()); + } + + if self.name.trim().is_empty() { + eprintln!("Invalid cluster configuration: cluster name cannot be empty"); return Err(ConfigurationError::InvalidConfigurationValue); } - if self.heartbeat.enabled && self.heartbeat.interval.get_duration().is_zero() { + + if self.nodes.is_empty() { eprintln!( - "{COMPONENT} heartbeat.interval must be nonzero when heartbeat.enabled: it sizes both the verifier's sleep and the staleness window, so zero spins the verifier and reaps every live session on its first pass" + "Invalid cluster configuration: cluster.nodes must contain at least one entry when cluster is enabled" ); return Err(ConfigurationError::InvalidConfigurationValue); } - reject_unsupported_and_warn_inert(self)?; + // VSR needs every replica to have a stable, unique id strictly + // less than the total replica count. Duplicate ids would split the + // cluster into two replicas claiming the same slot; out-of-range + // ids never win a primary election. Both are unrecoverable at + // runtime - fail fast at startup. + let total_replicas = u8::try_from(self.nodes.len()).map_err(|_| { + eprintln!("Invalid cluster configuration: more than 255 replicas is unsupported"); + ConfigurationError::InvalidConfigurationValue + })?; - Ok(()) - } -} + let mut seen_ids = std::collections::HashSet::new(); + let mut seen_names = std::collections::HashSet::new(); + let mut used_endpoints = std::collections::HashSet::new(); -/// The server parses the whole config surface but does not yet honor every -/// knob. Make the still-inert ones loud at boot: reject the unsupported -/// features (all off by default, so only a deliberate opt-in trips this) and -/// warn once for tuning knobs the server silently ignores. Warnings fire only -/// when a knob deviates from its [`ServerConfig::default`] baseline, so a -/// pristine config.toml boots without noise. The guard test below pins the -/// compared knobs against drift. -fn reject_unsupported_and_warn_inert(config: &ServerConfig) -> Result<(), ConfigurationError> { - if config.system.message_deduplication.enabled { - eprintln!("system.message_deduplication.enabled is not supported"); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if config.system.segment.archive_expired { - eprintln!("system.segment.archive_expired is not supported"); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if config.system.recovery.recreate_missing_state { - eprintln!("system.recovery.recreate_missing_state is not supported"); - return Err(ConfigurationError::InvalidConfigurationValue); - } + for node in &self.nodes { + if node.name.trim().is_empty() { + eprintln!("Invalid cluster configuration: node name cannot be empty"); + return Err(ConfigurationError::InvalidConfigurationValue); + } - let defaults = ServerConfig::default(); + if node.ip.trim().is_empty() { + eprintln!( + "Invalid cluster configuration: IP cannot be empty for node '{}'", + node.name + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } - if config.tcp.socket.override_defaults { - warn!("tcp.socket tuning is set but not applied"); - } - if config.quic.socket.override_defaults { - warn!("quic.socket tuning is set but not applied"); - } - if config.tcp.ipv6 { - warn!("tcp.ipv6 is ignored; IPv4 vs IPv6 is decided by the tcp.address string"); - } - if config.tcp.socket_migration != defaults.tcp.socket_migration { - warn!("tcp.socket_migration is not implemented"); - } - if config.system.segment.cache_indexes != defaults.system.segment.cache_indexes { - warn!("system.segment.cache_indexes is not applied"); - } - if config.system.logging.sysinfo_print_interval - != defaults.system.logging.sysinfo_print_interval - { - warn!("system.logging.sysinfo_print_interval is not applied"); - } - if config.system.backup.path != defaults.system.backup.path - || config.system.backup.compatibility.path != defaults.system.backup.compatibility.path - { - warn!("backup is not supported"); - } - // default_algorithm deviation is already warned by the delegated legacy - // CompressionConfig::validate; only allow_override needs a signal here. - if config.system.compression.allow_override != defaults.system.compression.allow_override { - warn!( - "system.compression.allow_override is inert; live compression is per-topic from the request" - ); - } - if config.system.state.enforce_fsync != defaults.system.state.enforce_fsync - || config.system.state.max_file_operation_retries - != defaults.system.state.max_file_operation_retries - || config.system.state.retry_delay != defaults.system.state.retry_delay - { - warn!( - "system.state tuning (enforce_fsync, max_file_operation_retries, retry_delay) is not applied" - ); - } - if config.consumer_group.rebalancing_check_interval - != defaults.consumer_group.rebalancing_check_interval - { - warn!( - "consumer_group.rebalancing_check_interval is not applied; rebalancing cadence uses system.sharding.reconcile_periodic_interval" - ); - } - if config.message_saver.interval != defaults.message_saver.interval - || config.message_saver.enforce_fsync != defaults.message_saver.enforce_fsync - { - warn!("periodic message_saver is not implemented; only shutdown-flush is active"); - } + if !seen_names.insert(node.name.clone()) { + eprintln!( + "Invalid cluster configuration: duplicate node name '{}' found", + node.name + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } - Ok(()) -} + if node.replica_id >= total_replicas { + eprintln!( + "Invalid cluster configuration: replica_id {} for node '{}' must be < total replica count {total_replicas}", + node.replica_id, node.name + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } -impl Validatable for ExtraConfig { - fn validate(&self) -> Result<(), ConfigurationError> { - self.namespace.validate().error(|e: &ConfigurationError| { - format!("{COMPONENT} (error: {e}) - failed to validate namespace config") - })?; - Ok(()) - } -} + if !seen_ids.insert(node.replica_id) { + eprintln!( + "Invalid cluster configuration: duplicate replica_id {} (two nodes claim the same slot)", + node.replica_id + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + let port_list = [ + ("TCP", node.ports.tcp), + ("QUIC", node.ports.quic), + ("HTTP", node.ports.http), + ("WebSocket", node.ports.websocket), + ("TCP_REPLICA", node.ports.tcp_replica), + ]; + + for (name, port_opt) in &port_list { + if let Some(port) = port_opt { + if *port == 0 { + eprintln!( + "Invalid cluster configuration: {} port cannot be 0 for node '{}'", + name, node.name + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + let endpoint = format!("{}:{}", node.ip, port); + if !used_endpoints.insert(endpoint.clone()) { + eprintln!( + "Invalid cluster configuration: port conflict - {endpoint} is already bound (node '{}', transport {name})", + node.name + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + } + } + } + + // Replica-auth PSK (only reached when the cluster is enabled; the early + // return above skips these while it is disabled). When auth is enabled + // the key is mandatory; any configured key must clear the length floor - + // a typo guard that fires with auth off too, though only while the + // cluster itself is enabled. + let secret_len = self.auth.shared_secret.len(); + if self.auth.enabled && self.auth.shared_secret.is_empty() { + eprintln!( + "Invalid cluster configuration: cluster.auth.shared_secret must be set when cluster.auth.enabled is true" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if !self.auth.shared_secret.is_empty() && secret_len < MIN_SHARED_SECRET_LEN { + eprintln!( + "Invalid cluster configuration: cluster.auth.shared_secret must be >= {MIN_SHARED_SECRET_LEN} bytes" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + // Replica TLS. Both cert modes run one-directional TLS (no client + // certificate anywhere), so TLS only authenticates the acceptor to + // the dialer; peer authentication comes solely from the PSK + // handshake. Without it any TLS-capable host could register as a + // replica - require auth in both modes. CA mode (the default) + // additionally needs all three PEM paths: cert/key for this node's + // acceptor side, ca_file as the dialer's trust anchor. + if self.tls.enabled { + if !self.auth.enabled { + eprintln!( + "Invalid cluster configuration: cluster.tls.enabled = true requires cluster.auth.enabled = true (TLS authenticates the acceptor only; the PSK handshake authenticates the peer)" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if !self.tls.self_signed { + for (field, value) in [ + ("cert_file", &self.tls.cert_file), + ("key_file", &self.tls.key_file), + ("ca_file", &self.tls.ca_file), + ] { + if value.trim().is_empty() { + eprintln!( + "Invalid cluster configuration: cluster.tls.{field} must be set when cluster.tls.enabled = true and self_signed = false" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + } + } + } -impl Validatable for NamespaceConfig { - fn validate(&self) -> Result<(), ConfigurationError> { - IggyNamespace::validate_capacity(self.max_streams, self.max_topics, self.max_partitions) - .map_err(|error| { - eprintln!("extra.namespace is invalid: {error}"); - ConfigurationError::InvalidConfigurationValue - })?; Ok(()) } } #[cfg(test)] -mod tests { - use super::super::cluster::{ClusterNodeConfig, TransportPorts}; +mod cluster_validate_tests { use super::*; - use figment::Figment; - use figment::providers::{Format, Toml}; + use crate::server_config::cluster::{ + ClusterAuthConfig, ClusterConfig, ClusterNodeConfig, ClusterTlsConfig, TransportPorts, + }; - const DEFAULT_CONFIG: &str = include_str!("../../../server/config.toml"); - - /// Deep-merge a partial override over the shipped default, mirroring the - /// file-over-embedded layering the runtime loader performs. - fn config_with_override(override_toml: &str) -> ServerConfig { - Figment::new() - .merge(Toml::string(DEFAULT_CONFIG)) - .merge(Toml::string(override_toml)) - .extract() - .expect("config deserializes") + fn node(name: &str, id: u8) -> ClusterNodeConfig { + ClusterNodeConfig { + name: name.to_string(), + ip: "127.0.0.1".to_string(), + replica_id: id, + ports: TransportPorts::default(), + } } - #[test] - fn given_shipped_default_config_when_validating_should_pass() { - let config: ServerConfig = Figment::new() - .merge(Toml::string(DEFAULT_CONFIG)) - .extract() - .expect("default config deserializes"); - config.validate().expect("pristine config must validate"); + fn cfg(nodes: Vec) -> ClusterConfig { + ClusterConfig { + enabled: true, + name: "iggy-cluster".to_string(), + nodes, + auth: ClusterAuthConfig::default(), + tls: ClusterTlsConfig::default(), + } } #[test] - fn given_message_deduplication_enabled_when_validating_should_reject() { - let config = config_with_override("[system.message_deduplication]\nenabled = true\n"); - assert!(config.validate().is_err()); + fn validate_rejects_empty_nodes() { + let c = cfg(vec![]); + assert!(c.validate().is_err()); } #[test] - fn given_web_ui_enabled_when_validating_should_pass() { - let config = config_with_override("[http]\nweb_ui = true\n"); - config - .validate() - .expect("web_ui is served by the server and must validate"); + fn validate_rejects_duplicate_replica_ids() { + let c = cfg(vec![node("n1", 0), node("n2", 0)]); + assert!(c.validate().is_err()); } #[test] - fn given_archive_expired_enabled_when_validating_should_reject() { - let config = config_with_override("[system.segment]\narchive_expired = true\n"); - assert!(config.validate().is_err()); + fn validate_rejects_duplicate_names() { + let c = cfg(vec![node("n1", 0), node("n1", 1)]); + assert!(c.validate().is_err()); } #[test] - fn given_recreate_missing_state_enabled_when_validating_should_reject() { - let config = config_with_override("[system.recovery]\nrecreate_missing_state = true\n"); - assert!(config.validate().is_err()); + fn validate_rejects_out_of_range_replica_id() { + // 2 nodes total, so id 2 is out of range. + let c = cfg(vec![node("n1", 0), node("n2", 2)]); + assert!(c.validate().is_err()); } #[test] - fn given_zero_message_expiry_when_validating_should_reject() { - let config = config_with_override("[system.topic]\nmessage_expiry = \"0s\"\n"); - assert!(config.validate().is_err()); + fn validate_accepts_unique_contiguous_replica_ids() { + let c = cfg(vec![node("n1", 0), node("n2", 1), node("n3", 2)]); + assert!(c.validate().is_ok()); } #[test] - fn given_peer_queue_capacity_not_above_repair_chunk_max_when_validating_should_reject() { - // The default repair_chunk_max (128) must stay strictly below - // peer_queue_capacity; shrinking the queue to the chunk size is the - // silent wedged-repair footgun this cross-section guard closes. - let config = config_with_override("[message_bus]\npeer_queue_capacity = 128\n"); - assert!(config.validate().is_err()); + fn validate_skips_checks_when_disabled() { + let mut c = cfg(vec![]); + c.enabled = false; + assert!(c.validate().is_ok()); } #[test] - fn given_repair_chunk_max_at_peer_queue_capacity_when_validating_should_reject() { - let config = config_with_override("[cluster]\nrepair_chunk_max = 256\n"); - assert!(config.validate().is_err()); + fn validate_rejects_duplicate_tcp_replica_port() { + let ports = TransportPorts { + tcp: None, + quic: None, + http: None, + websocket: None, + tcp_replica: Some(9090), + }; + let mut n1 = node("n1", 0); + n1.ports = ports.clone(); + let mut n2 = node("n2", 1); + n2.ports = ports; + let c = cfg(vec![n1, n2]); + assert!(c.validate().is_err()); } #[test] - fn given_repair_chunk_max_below_peer_queue_capacity_when_validating_should_pass() { - let config = config_with_override("[cluster]\nrepair_chunk_max = 255\n"); - config - .validate() - .expect("a chunk below the peer queue capacity must validate"); + fn validate_rejects_cross_transport_port_reuse() { + let mut n1 = node("n1", 0); + n1.ports = TransportPorts { + tcp: Some(8090), + quic: None, + http: Some(8090), + websocket: None, + tcp_replica: None, + }; + let c = cfg(vec![n1]); + assert!( + c.validate().is_err(), + "same port on TCP and HTTP of the same node must be rejected" + ); } #[test] - fn given_ws_frame_size_above_ws_message_size_when_validating_should_reject() { - let config = config_with_override( - "[websocket]\nmax_message_size = \"1 MiB\"\nmax_frame_size = \"2 MiB\"\n", - ); - assert!(config.validate().is_err()); + fn validate_accepts_same_port_on_different_ips() { + let mut n1 = node("n1", 0); + n1.ip = "127.0.0.1".to_string(); + n1.ports = TransportPorts { + tcp: Some(8090), + quic: None, + http: None, + websocket: None, + tcp_replica: None, + }; + let mut n2 = node("n2", 1); + n2.ip = "127.0.0.2".to_string(); + n2.ports = TransportPorts { + tcp: Some(8090), + quic: None, + http: None, + websocket: None, + tcp_replica: None, + }; + let c = cfg(vec![n1, n2]); + assert!(c.validate().is_ok()); } - // The shipped bus cap is 64 MiB, so a 128 MiB WS ceiling breaks the chain. #[test] - fn given_ws_message_size_above_bus_max_message_size_when_validating_should_reject() { - let config = config_with_override("[websocket]\nmax_message_size = \"128 MiB\"\n"); - assert!(config.validate().is_err()); + fn validate_rejects_zero_tcp_replica_port() { + let ports = TransportPorts { + tcp: None, + quic: None, + http: None, + websocket: None, + tcp_replica: Some(0), + }; + let mut n1 = node("n1", 0); + n1.ports = ports; + let c = cfg(vec![n1]); + assert!(c.validate().is_err()); } #[test] - fn given_ws_frame_size_above_bus_max_message_size_when_validating_should_reject() { - let config = config_with_override("[websocket]\nmax_frame_size = \"128 MiB\"\n"); - assert!(config.validate().is_err()); + fn validate_accepts_empty_secret_when_auth_disabled() { + // Default: no secret, auth off -> legacy mode, must pass. + let c = cfg(vec![node("n1", 0), node("n2", 1)]); + assert!(c.validate().is_ok()); } #[test] - fn given_ws_frame_chain_in_ascending_order_when_validating_should_pass() { - let config = config_with_override( - "[websocket]\nmax_message_size = \"32 MiB\"\nmax_frame_size = \"16 MiB\"\n", - ); - config - .validate() - .expect("frame <= message <= bus cap must validate"); + fn validate_rejects_missing_secret_when_auth_enabled() { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.auth.enabled = true; + assert!(c.validate().is_err()); } - // "unlimited" is not a supported sentinel for the WS size knobs: it - // parses to zero, which as a cap would reject every message. #[test] - fn given_zero_ws_size_when_validating_should_reject() { - let config = config_with_override("[websocket]\nmax_message_size = \"unlimited\"\n"); - assert!(config.validate().is_err()); + fn validate_rejects_short_secret_when_auth_enabled() { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.auth.enabled = true; + c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN - 1); + assert!(c.validate().is_err()); } - // tungstenite panics on this pair at connection setup; boot must - // reject it first. #[test] - fn given_max_write_buffer_at_write_buffer_when_validating_should_reject() { - let config = config_with_override( - "[websocket]\nwrite_buffer_size = \"256 KiB\"\nmax_write_buffer_size = \"256 KiB\"\n", - ); - assert!(config.validate().is_err()); + fn validate_rejects_short_secret_even_when_auth_disabled() { + // Typo guard: a configured-but-short key fails even with auth off. + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN - 1); + assert!(c.validate().is_err()); } #[test] - fn given_max_write_buffer_below_default_write_buffer_when_validating_should_reject() { - let config = config_with_override("[websocket]\nmax_write_buffer_size = \"64 KiB\"\n"); - assert!(config.validate().is_err()); + fn validate_accepts_valid_secret_when_auth_enabled() { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.auth.enabled = true; + c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); + assert!(c.validate().is_ok()); } - #[test] - fn given_max_write_buffer_above_write_buffer_when_validating_should_pass() { - let config = config_with_override( - "[websocket]\nwrite_buffer_size = \"128 KiB\"\nmax_write_buffer_size = \"1 MiB\"\n", - ); - config - .validate() - .expect("max write buffer above write buffer must validate"); + fn tls_files() -> ClusterTlsConfig { + ClusterTlsConfig { + enabled: true, + self_signed: false, + cert_file: "cert.pem".to_string(), + key_file: "key.pem".to_string(), + ca_file: "ca.pem".to_string(), + } } - // The size knobs are strictly typed; a malformed string must fail - // deserialization at load rather than degrade to the compio-ws default. #[test] - fn given_malformed_ws_size_string_when_deserializing_should_reject() { - let result: Result = Figment::new() - .merge(Toml::string(DEFAULT_CONFIG)) - .merge(Toml::string( - "[websocket]\nmax_message_size = \"not-a-size\"\n", - )) - .extract(); - assert!( - result.is_err(), - "malformed websocket.max_message_size must fail config load" - ); + fn validate_rejects_tls_ca_mode_with_missing_files() { + // Auth on so the failure exercises the file check, not the auth gate. + for missing in ["cert_file", "key_file", "ca_file"] { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.auth.enabled = true; + c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); + c.tls = tls_files(); + match missing { + "cert_file" => c.tls.cert_file.clear(), + "key_file" => c.tls.key_file.clear(), + _ => c.tls.ca_file.clear(), + } + assert!(c.validate().is_err(), "missing {missing} must be rejected"); + } } - // The shipped config is single-node (cluster.enabled = false), where the - // cross-section rule above is the only repair_chunk_max check that used to - // run; its structural bounds have to hold there too. #[test] - fn given_single_node_zero_repair_chunk_max_when_validating_should_reject() { - let config = config_with_override("[cluster]\nrepair_chunk_max = 0\n"); - assert!(config.validate().is_err()); + fn validate_rejects_tls_self_signed_without_auth() { + // Accept-any certificate without the PSK handshake = MITM-able. + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.tls = ClusterTlsConfig { + enabled: true, + self_signed: true, + ..ClusterTlsConfig::default() + }; + assert!(c.validate().is_err()); } #[test] - fn given_single_node_repair_chunk_max_above_ceiling_when_validating_should_reject() { - // Queue widened past the chunk so the cross-section rule passes and - // only the structural ceiling can reject. - let config = config_with_override( - "[cluster]\nrepair_chunk_max = 2000\n\n[message_bus]\npeer_queue_capacity = 4096\n", - ); - assert!(config.validate().is_err()); + fn validate_accepts_tls_self_signed_with_auth() { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.auth.enabled = true; + c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); + c.tls = ClusterTlsConfig { + enabled: true, + self_signed: true, + ..ClusterTlsConfig::default() + }; + assert!(c.validate().is_ok()); } #[test] - fn given_zero_rebalancing_timeout_when_validating_should_reject() { - let config = config_with_override("[consumer_group]\nrebalancing_timeout = \"0\"\n"); - assert!(config.validate().is_err()); + fn validate_rejects_tls_ca_mode_without_auth() { + // TLS never authenticates the dialer (no client certificates); + // only the PSK handshake does, so it is mandatory with TLS on. + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.tls = tls_files(); + assert!(c.validate().is_err()); } #[test] - fn given_disabled_rebalancing_timeout_when_validating_should_reject() { - // "disabled" reads like an opt-out but parses to the same zero - // duration, which force-transfers every revocation instead. - let config = config_with_override("[consumer_group]\nrebalancing_timeout = \"disabled\"\n"); - assert!(config.validate().is_err()); + fn validate_accepts_tls_ca_mode_with_auth() { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.auth.enabled = true; + c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); + c.tls = tls_files(); + assert!(c.validate().is_ok()); } +} + +#[cfg(test)] +mod cluster_shards_count_determinism_tests { + use super::*; + use crate::server_config::sharding::NumaConfig; #[test] - fn given_zero_heartbeat_interval_when_heartbeat_enabled_should_reject() { - let config = config_with_override("[heartbeat]\nenabled = true\ninterval = \"0\"\n"); - assert!(config.validate().is_err()); + fn all_is_rejected() { + let err = host_dependent_cpu_allocation(&CpuAllocation::All).unwrap_err(); + assert!(err.contains("all")); } #[test] - fn given_zero_heartbeat_interval_when_heartbeat_disabled_should_pass() { - let config = config_with_override("[heartbeat]\nenabled = false\ninterval = \"0\"\n"); - config - .validate() - .expect("a disabled heartbeat never reads its interval"); + fn numa_auto_is_rejected() { + let err = host_dependent_cpu_allocation(&CpuAllocation::NumaAware(NumaConfig::default())) + .unwrap_err(); + assert!(err.contains("numa:auto")); } - /// The warn-helper baseline is [`ServerConfig::default`], but the reused - /// legacy sections source that default from the legacy server config.toml, - /// not this NG file. Pin the knobs the helper compares so any drift between - /// the two config.toml files fails here instead of as a spurious boot warn. #[test] - fn given_shipped_ng_config_when_compared_to_default_should_match_warned_knobs() { - let shipped: ServerConfig = Figment::new() - .merge(Toml::string(DEFAULT_CONFIG)) - .extract() - .expect("default config deserializes"); - let defaults = ServerConfig::default(); - - assert_eq!(shipped.tcp.socket_migration, defaults.tcp.socket_migration); - assert_eq!( - shipped.system.segment.cache_indexes, - defaults.system.segment.cache_indexes - ); - assert_eq!( - shipped.system.logging.sysinfo_print_interval, - defaults.system.logging.sysinfo_print_interval - ); - assert_eq!(shipped.system.backup.path, defaults.system.backup.path); - assert_eq!( - shipped.system.backup.compatibility.path, - defaults.system.backup.compatibility.path - ); - assert_eq!( - shipped.system.compression.allow_override, - defaults.system.compression.allow_override - ); - assert_eq!( - shipped.system.state.enforce_fsync, - defaults.system.state.enforce_fsync - ); - assert_eq!( - shipped.system.state.max_file_operation_retries, - defaults.system.state.max_file_operation_retries - ); - assert_eq!( - shipped.system.state.retry_delay, - defaults.system.state.retry_delay - ); - assert_eq!( - shipped.consumer_group.rebalancing_check_interval, - defaults.consumer_group.rebalancing_check_interval - ); - assert_eq!( - shipped.message_saver.interval, - defaults.message_saver.interval - ); - assert_eq!( - shipped.message_saver.enforce_fsync, - defaults.message_saver.enforce_fsync - ); + fn count_is_accepted() { + assert!(host_dependent_cpu_allocation(&CpuAllocation::Count(4)).is_ok()); } - // http.enabled needs a non-ServerDefault JWT expiry to clear the sibling - // check above; ServerConfig::default() already satisfies that. - fn https_config(cert_file: &str, key_file: &str) -> ServerConfig { - let mut cfg = ServerConfig::default(); - cfg.http.enabled = true; - cfg.http.tls.enabled = true; - cfg.http.tls.cert_file = cert_file.to_string(); - cfg.http.tls.key_file = key_file.to_string(); - cfg + #[test] + fn range_is_accepted() { + assert!(host_dependent_cpu_allocation(&CpuAllocation::Range(0, 4)).is_ok()); } #[test] - fn validate_rejects_tls_enabled_with_empty_cert_file() { - let cfg = https_config("", "key.pem"); - assert!(cfg.validate().is_err()); + fn explicit_numa_is_accepted() { + let numa = NumaConfig { + nodes: vec![0, 1], + cores_per_node: 4, + avoid_hyperthread: true, + }; + assert!(host_dependent_cpu_allocation(&CpuAllocation::NumaAware(numa)).is_ok()); } +} + +#[cfg(test)] +mod sharding_cpu_range_tests { + use super::*; #[test] - fn validate_accepts_tls_enabled_with_both_files_set() { - let cfg = https_config("cert.pem", "key.pem"); - assert!(cfg.validate().is_ok()); + fn inverted_range_is_rejected() { + let cfg = ShardingConfig { + cpu_allocation: CpuAllocation::Range(2, 2), + pin_cores: true, + }; + assert!(cfg.validate().is_err()); } - fn cluster_node(replica_id: u8, http: Option) -> ClusterNodeConfig { - ClusterNodeConfig { - name: format!("node-{replica_id}"), - ip: "127.0.0.1".to_string(), - advertised_address: None, - advertised_addresses: Vec::new(), - replica_id, - ports: TransportPorts { - tcp: Some(8090 + u16::from(replica_id)), - quic: Some(8080 + u16::from(replica_id)), - http, - websocket: Some(8070 + u16::from(replica_id)), - tcp_replica: Some(9090 + u16::from(replica_id)), - }, - } - } - - fn clustered_http_config(nodes: Vec) -> ServerConfig { - let mut cfg = ServerConfig::default(); - cfg.http.enabled = true; - cfg.cluster.enabled = true; - cfg.cluster.name = "test-cluster".to_string(); - cfg.cluster.nodes = nodes; - cfg - } - - // Keyless cluster+http boots: forwarding degrades to off instead of - // failing the whole server. #[test] - fn validate_accepts_cluster_http_without_jwt_secret_or_cluster_auth() { - let cfg = clustered_http_config(vec![ - cluster_node(0, Some(3000)), - cluster_node(1, Some(3001)), - ]); + fn pinned_range_within_allowed_set_is_accepted() { + let first = allowed_cpus()[0]; + let cfg = ShardingConfig { + cpu_allocation: CpuAllocation::Range(first, first + 1), + pin_cores: true, + }; assert!(cfg.validate().is_ok()); } - // Cluster mode has no port fallbacks, so a portless roster node is - // invalid even when forwarding is off (keyless). #[test] - fn validate_rejects_keyless_cluster_http_with_portless_roster_node() { - let cfg = clustered_http_config(vec![cluster_node(0, Some(3000)), cluster_node(1, None)]); + fn pinned_range_outside_allowed_set_is_rejected() { + let past_last = allowed_cpus().last().copied().unwrap() + 1; + let cfg = ShardingConfig { + cpu_allocation: CpuAllocation::Range(past_last, past_last + 1), + pin_cores: true, + }; assert!(cfg.validate().is_err()); } - // The explicit-port rule covers every enabled transport, not just http. #[test] - fn validate_rejects_cluster_node_without_port_for_enabled_quic() { - let mut cfg = clustered_http_config(vec![ - cluster_node(0, Some(3000)), - cluster_node(1, Some(3001)), - ]); - cfg.quic.enabled = true; - cfg.cluster.nodes[1].ports.quic = None; + fn pinned_range_wider_than_parallelism_is_rejected() { + // Under a cgroup CPU quota the affinity mask stays full while + // `available_parallelism` shrinks, so membership alone would + // accept this; the shard-count cap must reject it. + let first = allowed_cpus()[0]; + let available = available_parallelism().unwrap().get(); + let cfg = ShardingConfig { + cpu_allocation: CpuAllocation::Range(first, first + available + 1), + pin_cores: true, + }; assert!(cfg.validate().is_err()); } - // A disabled transport never binds, so its roster port may stay unset. #[test] - fn validate_accepts_cluster_node_without_port_for_disabled_quic() { - let mut cfg = clustered_http_config(vec![ - cluster_node(0, Some(3000)), - cluster_node(1, Some(3001)), - ]); - cfg.quic.enabled = false; - cfg.cluster.nodes[1].ports.quic = None; + fn unpinned_range_is_capped_by_shard_count_not_core_ids() { + // Core ids outside the machine are fine unpinned; only the + // resulting shard count matters. + let cfg = ShardingConfig { + cpu_allocation: CpuAllocation::Range(1 << 20, (1 << 20) + 1), + pin_cores: false, + }; assert!(cfg.validate().is_ok()); - } - #[test] - fn validate_accepts_cluster_http_with_configured_jwt_secrets() { - let mut cfg = clustered_http_config(vec![ - cluster_node(0, Some(3000)), - cluster_node(1, Some(3001)), - ]); - cfg.http.jwt.encoding_secret = "0123456789abcdef0123456789abcdef".to_string(); - cfg.http.jwt.decoding_secret = "0123456789abcdef0123456789abcdef".to_string(); - assert!(cfg.validate().is_ok()); + let available = available_parallelism().unwrap().get(); + let cfg = ShardingConfig { + cpu_allocation: CpuAllocation::Range(0, available + 1), + pin_cores: false, + }; + assert!(cfg.validate().is_err()); } +} + +#[cfg(test)] +mod sharding_embedded_default_tests { + use super::*; + use figment::Figment; + use figment::providers::{Format, Toml}; + // Guards the single source of truth: the legacy sharding defaults resolve + // from the embedded legacy TOML, not hard-coded Rust values. #[test] - fn validate_accepts_cluster_http_with_cluster_auth_as_jwt_key_source() { - let mut cfg = clustered_http_config(vec![ - cluster_node(0, Some(3000)), - cluster_node(1, Some(3001)), - ]); - cfg.cluster.auth.enabled = true; - cfg.cluster.auth.shared_secret = "0123456789abcdef0123456789abcdef".to_string(); - assert!(cfg.validate().is_ok()); + fn legacy_embedded_toml_resolves_sharding_defaults() { + let toml_str = include_str!("../../../server/config.toml"); + let config: ServerConfig = Figment::new() + .merge(Toml::string(toml_str)) + .extract() + .expect("embedded legacy TOML deserializes"); + config.validate().expect("embedded legacy config validates"); + + assert!(config.system.sharding.pin_cores); } } diff --git a/core/configs/src/server_config/websocket.rs b/core/configs/src/server_config/websocket.rs index 85120daa61..30f7dc5c64 100644 --- a/core/configs/src/server_config/websocket.rs +++ b/core/configs/src/server_config/websocket.rs @@ -15,76 +15,28 @@ // specific language governing permissions and limitations // under the License. -//! WebSocket listener schema. -//! -//! This section is the live frame-tuning source for the WS / WSS -//! plane: the message bus folds the -//! `Option` knobs below into a compio-ws -//! `WebSocketConfig` once at bus construction. The sizes are strictly -//! typed, so a malformed size string fails config load instead of -//! being silently ignored at conversion time. The conversion itself -//! lives in `core/message_bus` because the standalone `tungstenite` -//! dependency and the compio-ws re-export are different major versions -//! with incompatible config types. - use configs::ConfigEnv; use iggy_common::IggyByteSize; use serde::{Deserialize, Serialize}; -use serde_with::{DisplayFromStr, serde_as}; use std::fmt::{Display, Formatter}; +use tungstenite::protocol::WebSocketConfig as TungsteniteConfig; -#[serde_as] #[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] pub struct WebSocketConfig { pub enabled: bool, pub address: String, - - /// Target minimum size of the frame read buffer. `None` keeps the - /// compio-ws default (currently 128 KiB). - #[config_env(leaf)] #[serde(default)] - #[serde_as(as = "Option")] - pub read_buffer_size: Option, - - /// Target buffer size for batched writes before compio-ws flushes. - /// `None` keeps the compio-ws default (currently 128 KiB). - #[config_env(leaf)] + pub read_buffer_size: Option, #[serde(default)] - #[serde_as(as = "Option")] - pub write_buffer_size: Option, - - /// Hard ceiling on the write buffer; writes past it error instead - /// of buffering. Must exceed [`Self::write_buffer_size`] by at - /// least one frame. `None` keeps the compio-ws default - /// (unlimited). - #[config_env(leaf)] + pub write_buffer_size: Option, #[serde(default)] - #[serde_as(as = "Option")] - pub max_write_buffer_size: Option, - - /// Hard upper bound on a single inbound WebSocket message - /// (post-fragment-reassembly). `None` keeps the compio-ws default - /// (currently 64 MiB). - #[config_env(leaf)] + pub max_write_buffer_size: Option, #[serde(default)] - #[serde_as(as = "Option")] - pub max_message_size: Option, - - /// Hard upper bound on a single inbound WebSocket frame - /// (pre-fragment-reassembly). `None` keeps the compio-ws default - /// (currently 16 MiB). - #[config_env(leaf)] + pub max_message_size: Option, #[serde(default)] - #[serde_as(as = "Option")] - pub max_frame_size: Option, - - /// Whether to accept unmasked frames from clients in violation of - /// RFC 6455 client-to-server framing rules. Strict (`false`) by - /// default; enable only for non-browser test clients that emit - /// unmasked frames. + pub max_frame_size: Option, #[serde(default)] pub accept_unmasked_frames: bool, - #[serde(default)] pub tls: WebSocketTlsConfig, } @@ -97,6 +49,46 @@ pub struct WebSocketTlsConfig { pub key_file: String, } +impl WebSocketConfig { + pub fn to_tungstenite_config(&self) -> TungsteniteConfig { + let mut config = TungsteniteConfig::default(); + + if let Some(read_buf_size_str) = &self.read_buffer_size + && let Ok(byte_size) = read_buf_size_str.parse::() + { + config = config.read_buffer_size(byte_size.as_bytes_u64() as usize); + } + + if let Some(write_buf_size_str) = &self.write_buffer_size + && let Ok(byte_size) = write_buf_size_str.parse::() + { + config = config.write_buffer_size(byte_size.as_bytes_u64() as usize); + } + + if let Some(max_write_buf_size_str) = &self.max_write_buffer_size + && let Ok(byte_size) = max_write_buf_size_str.parse::() + { + config = config.max_write_buffer_size(byte_size.as_bytes_u64() as usize); + } + + if let Some(msg_size_str) = &self.max_message_size + && let Ok(byte_size) = msg_size_str.parse::() + { + config = config.max_message_size(Some(byte_size.as_bytes_u64() as usize)); + } + + if let Some(frame_size_str) = &self.max_frame_size + && let Ok(byte_size) = frame_size_str.parse::() + { + config = config.max_frame_size(Some(byte_size.as_bytes_u64() as usize)); + } + + config = config.accept_unmasked_frames(self.accept_unmasked_frames); + + config + } +} + impl Display for WebSocketConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!( diff --git a/core/configs/src/server_ng_config/cluster.rs b/core/configs/src/server_ng_config/cluster.rs new file mode 100644 index 0000000000..9a6b38fc62 --- /dev/null +++ b/core/configs/src/server_ng_config/cluster.rs @@ -0,0 +1,2612 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Server-ng cluster schema. +//! +//! Field shape mirrors the legacy [`crate::cluster::ClusterConfig`] +//! plus the ng-only `heartbeat_timeout` knob; the type is forked into +//! `server_ng_config` so server-ng can evolve its cluster surface +//! (VSR consensus tunables) independently of the legacy server. + +use super::defaults::SERVER_NG_CONFIG; +use crate::ConfigurationError; +use crate::http::HttpJwtConfig; +use configs::ConfigEnv; +use iggy_common::{IggyDuration, Validatable}; +use ipnet::{IpNet, Ipv4Net}; +use serde::{Deserialize, Serialize}; +use serde_with::{DisplayFromStr, serde_as}; +use std::cmp::Reverse; +use std::fmt; +use std::net::{IpAddr, Ipv6Addr, SocketAddr}; +use std::str::FromStr; +use std::time::Duration; + +/// Absolute floor for the backup liveness window, independent of the +/// commit-broadcast rate. The primary signals liveness through its commit +/// broadcast (`commit_broadcast_interval`, 500ms by default); 2s spans several +/// broadcasts, so a single delayed one never elects. The per-config +/// `MIN_HEARTBEAT_TO_COMMIT_BROADCAST_RATIO` check scales the same headroom +/// when the broadcast interval is retuned. +pub const MIN_CLUSTER_HEARTBEAT_TIMEOUT: Duration = Duration::from_secs(2); + +/// The backup liveness window (`heartbeat_timeout`) must span at least this +/// many commit broadcasts (`commit_broadcast_interval`), so one dropped or +/// delayed broadcast never trips a view change on a healthy primary. +const MIN_HEARTBEAT_TO_COMMIT_BROADCAST_RATIO: u32 = 4; + +/// The view-change status backstop (`view_change_status_timeout`) must span at +/// least this many retransmit intervals (`view_change_retransmit_interval`), so +/// a few dropped `StartViewChange` / `DoViewChange` messages retransmit rather +/// than escalating a progressing view change into a fresh cluster-wide election. +const MIN_STATUS_TO_RETRANSMIT_RATIO: u32 = 4; + +/// Default recovering-replica probe-attempt ceiling. Duplicated here rather +/// than imported so `core/configs` keeps off a build-time edge onto +/// `core/consensus` (mirroring [`super::partition`]); `core/server-ng`'s +/// bootstrap static-asserts it equal to `consensus::PROBE_ATTEMPTS_MAX`. +pub const DEFAULT_VIEW_PROBE_ATTEMPTS_MAX: u32 = 5; + +/// Upper bound on `view_probe_attempts_max`. A recovering replica probes once +/// per `request_start_view_retransmit_interval`, so hundreds of attempts would +/// stall the election fallback for minutes on a full-cluster restart; this is a +/// typo guard, not a sizing endorsement. +const MAX_VIEW_PROBE_ATTEMPTS: u32 = 100; + +/// Default per-round repair-serving chunk. Duplicated here rather than imported +/// so `core/configs` keeps off a build-time edge onto `core/shard` (mirroring +/// [`DEFAULT_VIEW_PROBE_ATTEMPTS_MAX`]); `core/server-ng`'s bootstrap +/// static-asserts it equal to `shard::REPAIR_CHUNK_MAX`. +pub const DEFAULT_REPAIR_CHUNK_MAX: usize = 128; + +/// `size_of::()`. Duplicated here for the same reason as +/// [`DEFAULT_REPAIR_CHUNK_MAX`]; `core/server-ng`'s bootstrap static-asserts it +/// against the real header. Used to reject a `[message_bus] max_message_size` +/// too small to carry a single state-transfer chunk. +pub const STATE_CHUNK_HEADER_LEN: u64 = 256; + +/// Upper bound on `repair_chunk_max`. A chunk rides the per-peer bus queue, so +/// the load-bearing rule is `repair_chunk_max < message_bus.peer_queue_capacity` +/// (enforced at the top level); this standalone ceiling is a typo guard. +const MAX_REPAIR_CHUNK_MAX: usize = 1024; + +/// Upper bound on per-node `advertised_addresses` selectors. Must mirror the +/// `#[config_env(max_elements = 16)]` cap on the field: env overrides above +/// the cap do not exist, so a TOML roster exceeding it could never be +/// replicated through the env path. Also bounds the cross-node conflict scan +/// (quadratic in pooled entries) and the per-request longest-prefix walk. +const MAX_ADVERTISED_SELECTORS: usize = 16; + +/// Length floor for the replica-auth PSK, in raw bytes. The 32-byte MAC key +/// is KDF-derived from these bytes at use-site, so any encoding clearing this +/// length is accepted. +const MIN_SHARED_SECRET_LEN: usize = 32; + +/// DNS caps a full name at 255 octets on the wire, which leaves 253 +/// characters of presentation text (RFC 1035). +const MAX_HOSTNAME_LEN: usize = 253; + +/// Per-label limit from RFC 1035. +const MAX_HOSTNAME_LABEL_LEN: usize = 63; + +/// serde fallback for configs written before the field existed; the value +/// itself lives in `core/server-ng/config.toml` like every other default. +fn default_heartbeat_timeout() -> IggyDuration { + SERVER_NG_CONFIG.cluster.heartbeat_timeout.parse().unwrap() +} + +/// serde fallback for configs written before the field existed; the value +/// itself lives in `core/server-ng/config.toml` like every other default. +fn default_commit_broadcast_interval() -> IggyDuration { + SERVER_NG_CONFIG + .cluster + .commit_broadcast_interval + .parse() + .unwrap() +} + +/// serde fallback for configs written before the field existed; the value +/// itself lives in `core/server-ng/config.toml` like every other default. +fn default_prepare_retransmit_interval() -> IggyDuration { + SERVER_NG_CONFIG + .cluster + .prepare_retransmit_interval + .parse() + .unwrap() +} + +/// serde fallback for configs written before the field existed; the value +/// itself lives in `core/server-ng/config.toml` like every other default. +fn default_view_change_retransmit_interval() -> IggyDuration { + SERVER_NG_CONFIG + .cluster + .view_change_retransmit_interval + .parse() + .unwrap() +} + +/// serde fallback for configs written before the field existed; the value +/// itself lives in `core/server-ng/config.toml` like every other default. +fn default_view_change_status_timeout() -> IggyDuration { + SERVER_NG_CONFIG + .cluster + .view_change_status_timeout + .parse() + .unwrap() +} + +/// serde fallback for configs written before the field existed; the value +/// itself lives in `core/server-ng/config.toml` like every other default. +fn default_request_start_view_retransmit_interval() -> IggyDuration { + SERVER_NG_CONFIG + .cluster + .request_start_view_retransmit_interval + .parse() + .unwrap() +} + +/// serde fallback for configs written before the field existed; the value +/// itself lives in `core/server-ng/config.toml` like every other default. +fn default_view_probe_attempts_max() -> u32 { + SERVER_NG_CONFIG.cluster.view_probe_attempts_max as u32 +} + +/// serde fallback for configs written before the field existed; the value +/// itself lives in `core/server-ng/config.toml` like every other default. +fn default_repair_retry_interval() -> IggyDuration { + SERVER_NG_CONFIG + .cluster + .repair_retry_interval + .parse() + .unwrap() +} + +/// serde fallback for configs written before the field existed; the value +/// itself lives in `core/server-ng/config.toml` like every other default. +fn default_repair_chunk_max() -> usize { + SERVER_NG_CONFIG.cluster.repair_chunk_max as usize +} + +#[serde_as] +#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] +#[serde(deny_unknown_fields)] +pub struct ClusterConfig { + pub enabled: bool, + pub name: String, + /// Backup-side liveness window for a plane's primary. A replica that + /// sees no primary traffic for this long starts a view change + /// (`normal_heartbeat_timeout`). Raise it on oversubscribed hosts where + /// scheduling stalls fake primary death; sub-`MIN_CLUSTER_HEARTBEAT_TIMEOUT` + /// values (including the `0` / `disabled` / `unlimited` sentinels, which + /// all parse to zero) are rejected at boot. + #[serde(default = "default_heartbeat_timeout")] + #[serde_as(as = "DisplayFromStr")] + #[config_env(leaf)] + pub heartbeat_timeout: IggyDuration, + /// How often the primary broadcasts its commit point to every backup, the + /// cluster's primary-liveness signal. Each broadcast resets the backups' + /// `heartbeat_timeout` window, so that window must span several broadcasts: + /// boot rejects `heartbeat_timeout < MIN_HEARTBEAT_TO_COMMIT_BROADCAST_RATIO + /// * commit_broadcast_interval`. Sizes the consensus `CommitMessage` timer. + /// Zero (and the `0` / `disabled` / `unlimited` sentinels, which all parse + /// to zero) is rejected at boot. + #[serde(default = "default_commit_broadcast_interval")] + #[serde_as(as = "DisplayFromStr")] + #[config_env(leaf)] + pub commit_broadcast_interval: IggyDuration, + /// How often the primary retransmits prepares a backup has not yet acked. + /// Lower recovers faster from a dropped prepare at the cost of replica + /// traffic. Sizes the consensus `Prepare` timer. Zero (and the `0` / + /// `disabled` / `unlimited` sentinels, which all parse to zero) is rejected + /// at boot. + #[serde(default = "default_prepare_retransmit_interval")] + #[serde_as(as = "DisplayFromStr")] + #[config_env(leaf)] + pub prepare_retransmit_interval: IggyDuration, + /// How often a plane retransmits its `StartViewChange` / `DoViewChange` + /// while a view change is running. Lower converges a healthy election + /// faster at the cost of replica traffic. Sizes both consensus view-change + /// retransmit timers, which are deliberately equal. Zero (and the `0` / + /// `disabled` / `unlimited` sentinels, which all parse to zero) is rejected + /// at boot. + #[serde(default = "default_view_change_retransmit_interval")] + #[serde_as(as = "DisplayFromStr")] + #[config_env(leaf)] + pub view_change_retransmit_interval: IggyDuration, + /// Backstop for a stalled view change: one that does not conclude within + /// this window escalates to a fresh cluster-wide election. Must span + /// several `view_change_retransmit_interval`s so a few dropped view-change + /// messages retransmit rather than escalate: boot rejects + /// `view_change_status_timeout < MIN_STATUS_TO_RETRANSMIT_RATIO * + /// view_change_retransmit_interval`. Zero (and the `0` / `disabled` / + /// `unlimited` sentinels, which all parse to zero) is rejected at boot. + #[serde(default = "default_view_change_status_timeout")] + #[serde_as(as = "DisplayFromStr")] + #[config_env(leaf)] + pub view_change_status_timeout: IggyDuration, + /// How often a recovering or view-change backup re-requests the current + /// view's `StartView` from its primary (`RequestStartView`). Sizes the + /// consensus `RequestStartView` timer. Zero (and the `0` / `disabled` / + /// `unlimited` sentinels, which all parse to zero) is rejected at boot. + #[serde(default = "default_request_start_view_retransmit_interval")] + #[serde_as(as = "DisplayFromStr")] + #[config_env(leaf)] + pub request_start_view_retransmit_interval: IggyDuration, + /// How many consecutive unanswered `RequestStartView` probes a recovering + /// replica tolerates before it falls back to an election (a full-cluster + /// restart leaves nobody settled to answer). Must be >= 1 and <= + /// `MAX_VIEW_PROBE_ATTEMPTS`. + #[serde(default = "default_view_probe_attempts_max")] + pub view_probe_attempts_max: u32, + /// How long a stalled journal-repair stream waits before re-requesting its + /// remaining window from the serving peer. Repair frames are + /// fire-and-forget over the lossy bus, so a session with no retry wedges + /// forever on a single dropped frame. Paces both the metadata and + /// partition repair loops. Sizes the retry threshold in consensus ticks. + /// Zero (and the `0` / `disabled` / `unlimited` sentinels, which all parse + /// to zero) is rejected at boot. + #[serde(default = "default_repair_retry_interval")] + #[serde_as(as = "DisplayFromStr")] + #[config_env(leaf)] + pub repair_retry_interval: IggyDuration, + /// Prepares a peer serves per repair round before the requester walks to + /// the next chunk. Each frame rides the per-peer message-bus queue, so this + /// must stay below `message_bus.peer_queue_capacity` or a full round + /// overruns the queue and silently drops frames (enforced at the top + /// level). Applies to both the metadata and partition repair planes. Must + /// be > 0 and <= `MAX_REPAIR_CHUNK_MAX`. + #[serde(default = "default_repair_chunk_max")] + pub repair_chunk_max: usize, + /// Full roster of cluster members. Intended to be byte-identical across + /// every node so operators ship one config. The running node's identity + /// is supplied out-of-band via the `--replica-id` CLI flag, which + /// selects the entry in this list that describes the current node. + #[serde(default)] + pub nodes: Vec, + /// Replica-to-replica authentication settings (PSK + BLAKE3 handshake). + #[serde(default)] + pub auth: ClusterAuthConfig, + /// Replica-to-replica TLS settings for the consensus (`tcp_replica`) port. + #[serde(default)] + pub tls: ClusterTlsConfig, +} + +/// Replica-to-replica authentication for the consensus (`tcp_replica`) port. +#[derive(Debug, Default, Deserialize, Serialize, Clone, ConfigEnv)] +#[serde(deny_unknown_fields)] +pub struct ClusterAuthConfig { + /// When true, every replica peer must complete the authenticated handshake + /// or be rejected, and [`Self::shared_secret`] is mandatory. When false + /// (default) the replica handshake stays in legacy unauthenticated mode and + /// `shared_secret` is not used for authentication. A configured non-empty + /// `shared_secret` must still meet the 32-byte minimum whenever the cluster + /// is enabled (a short value fails boot even with auth off). + /// + /// Enabling auth is a coordinated-restart change, and not the only one: the + /// consensus `cluster_id` is derived from `ClusterConfig::name` + /// unconditionally, so a mixed-version roster fails to connect regardless of + /// this flag. Flip every node in one restart. + #[serde(default)] + pub enabled: bool, + /// Cluster-wide pre-shared key for replica-to-replica authentication. + /// + /// At least 32 bytes of CSPRNG output, byte-identical across every node. + /// Provisioned out-of-band, normally via `IGGY_CLUSTER_AUTH_SHARED_SECRET` + /// rather than the on-disk config. + // skip_serializing keeps the PSK out of the runtime `current_config.toml` + // (and the `ServerConfig` diagnostic snapshot that cats it). The live + // secret is read from env / on-disk config at boot, never from the + // snapshot, so it must never be persisted there. Deserialize is retained. + #[serde(default, skip_serializing)] + #[config_env(secret)] + pub shared_secret: String, + /// Retiring pre-shared key, accepted for VERIFICATION only during a key + /// rotation window; every MAC this node produces uses [`Self::shared_secret`]. + /// + /// Enables rolling PSK rotation without an auth outage, three rolls: + /// 1. every node gets `shared_secret = old, previous_shared_secret = new`; + /// 2. every node gets `shared_secret = new, previous_shared_secret = old`; + /// 3. every node gets `shared_secret = new` alone, closing the window. + /// + /// Leave empty (default) outside a rotation. Same 32-byte minimum and + /// provisioning rules as `shared_secret` + /// (`IGGY_CLUSTER_AUTH_PREVIOUS_SHARED_SECRET`). + #[serde(default, skip_serializing)] + #[config_env(secret)] + pub previous_shared_secret: String, +} + +/// Replica-to-replica TLS for the consensus (`tcp_replica`) port. +/// +/// Mirrors the legacy [`crate::tcp::TcpTlsConfig`] shape plus `ca_file`: +/// the replica plane DIALS its peers (a TLS client role the +/// client-facing server plane never has), so the dialer needs a trust +/// anchor to verify the acceptor's certificate against. +#[derive(Debug, Default, Deserialize, Serialize, Clone, ConfigEnv)] +#[serde(deny_unknown_fields)] +pub struct ClusterTlsConfig { + /// When true every replica connection is wrapped in TLS (1.3 only) + /// before the replica handshake runs. Requires `cluster.auth.enabled`: + /// TLS carries no client certificates, so it authenticates the + /// acceptor only; the PSK handshake authenticates the peer while TLS + /// supplies confidentiality. Enabling is a coordinated-restart + /// change: a TLS dialer cannot talk to a plaintext acceptor or vice + /// versa. Flip every node in one restart. + #[serde(default)] + pub enabled: bool, + /// When true the node auto-generates a self-signed certificate at + /// boot and the dialer accepts ANY peer certificate. With the + /// default `false`, `cert_file` / `key_file` / `ca_file` are all + /// required. + #[serde(default)] + pub self_signed: bool, + /// PEM certificate chain presented by this node's acceptor side. + #[serde(default)] + pub cert_file: String, + /// PEM private key matching `cert_file`. + #[serde(default)] + pub key_file: String, + /// PEM trust anchor(s) the dialer verifies peer certificates + /// against. Unused when `self_signed` is true. + #[serde(default)] + pub ca_file: String, +} + +#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] +#[serde(deny_unknown_fields)] +pub struct ClusterNodeConfig { + pub name: String, + pub ip: String, + /// Optional client-facing address: a literal IP or a DNS hostname, + /// validated as [`AdvertisedAddress`] at boot. Replica traffic continues + /// to use [`Self::ip`]. + #[serde(default)] + pub advertised_address: Option, + /// Client-network-scoped overrides of [`Self::advertised_address`], + /// resolved by longest-prefix match over the client's IP (see + /// [`AdvertisedAddressSelector`]). Empty by default, so existing configs + /// keep the single catch-all address. At most `MAX_ADVERTISED_SELECTORS` + /// per node (validated), matching the env-override cap below. + #[serde(default)] + #[config_env(max_elements = 16)] + pub advertised_addresses: Vec, + /// Numeric replica ID for VSR consensus (0-based). + /// + /// Must be unique across [`ClusterConfig::nodes`] and strictly less than + /// `nodes.len()`. Validated by [`ClusterConfig::validate`]. + pub replica_id: u8, + pub ports: TransportPorts, +} + +/// One client-network-scoped advertised address: clients whose IP falls +/// inside `client_cidr` are told `address` instead of the node's catch-all +/// [`ClusterNodeConfig::advertised_address`]. +/// +/// Typical split-network case: the roster `ip` is VPC-private and +/// `advertised_address` is public; a selector with the VPC CIDR keeps +/// in-VPC clients on the private address while everyone else stays on the +/// public one. Selection is longest-prefix match across a node's selectors. +/// Selection sees the transport-level peer address, so clients arriving +/// through a proxy or load balancer match the proxy's network, not their +/// own. +#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] +#[serde(deny_unknown_fields)] +pub struct AdvertisedAddressSelector { + /// Client network this selector matches, in CIDR notation + /// (`10.0.0.0/16`, `2001:db8::/32`). Must parse at boot; duplicate + /// networks within one node are rejected. A v4-mapped v6 network + /// (`::ffff:10.0.0.0/104`) canonicalizes to its v4 form (`10.0.0.0/8`), + /// matching how client IPs canonicalize before matching. + pub client_cidr: String, + /// Address advertised to matching clients: a literal IP or a DNS + /// hostname, validated as [`AdvertisedAddress`] at boot (no port; ports + /// come from [`ClusterNodeConfig::ports`]). + pub address: String, +} + +/// A roster node with its advertised-address selectors and catch-all parsed +/// once, built wherever a roster is assembled for serving clients +/// (listener/shard start). Per-request resolution never re-parses config +/// strings: everything is snapshotted here, so mutating the source config +/// after conversion has no effect on what clients are told. Entries that do +/// not parse are dropped at build time; validation already rejects them +/// whenever the cluster is enabled, and a disabled cluster never consults +/// the roster. +#[derive(Debug, Clone)] +pub struct ResolvedClusterNode { + config: ClusterNodeConfig, + /// Truncated, canonicalized selector networks with their parsed + /// addresses, in declaration order. + selectors: Vec<(IpNet, AdvertisedAddress)>, + /// Parsed catch-all: [`ClusterNodeConfig::advertised_address`], else the + /// roster [`ClusterNodeConfig::ip`]. `None` when the configured value + /// does not parse - a set `advertised_address` never falls through to + /// the private roster ip. + catch_all: Option, + /// Parsed roster [`ClusterNodeConfig::ip`], the replica-plane dial + /// address. `None` when the roster ip is not a literal IP (boot only + /// requires it non-empty); internal forwarding then has no dial target. + replica_ip: Option, +} + +impl From for ResolvedClusterNode { + fn from(config: ClusterNodeConfig) -> Self { + let selectors = config + .advertised_addresses + .iter() + .filter_map(|selector| { + let network = selector.client_cidr.parse::().ok()?; + let address = selector.address.parse::().ok()?; + Some((canonical_ip_net(network.trunc()), address)) + }) + .collect(); + let catch_all = match config.advertised_address.as_deref() { + Some(advertised_address) => advertised_address.parse().ok(), + None => config.ip.parse().ok(), + }; + let replica_ip = config.ip.parse().ok(); + Self { + config, + selectors, + catch_all, + replica_ip, + } + } +} + +impl ResolvedClusterNode { + /// The roster entry this node was built from. Read-only: resolution runs + /// on the boot-parsed snapshot, never on the config strings. + #[must_use] + pub fn config(&self) -> &ClusterNodeConfig { + &self.config + } + + /// The roster `ip` as a dialable address for the replica plane and + /// internal request forwarding. Never routed through the advertised + /// ladder: this is what servers dial, not what clients are told. + #[must_use] + pub fn replica_ip(&self) -> Option { + self.replica_ip + } + + /// The client-facing address for a client connecting from `client_ip`: + /// longest-prefix match over the selector networks, then the parsed + /// catch-all. `None` when no selector matches and the catch-all did not + /// parse; callers choose whether to fail closed (redirect URLs) or to + /// publish [`Self::raw_advertised_fallback`] verbatim (cluster metadata). + #[must_use] + pub fn advertised_for(&self, client_ip: Option) -> Option<&AdvertisedAddress> { + client_ip + .and_then(|client_ip| self.selector_address(client_ip)) + .or(self.catch_all.as_ref()) + } + + /// The catch-all ladder ([`ClusterNodeConfig::advertised_address`], else + /// the roster [`ClusterNodeConfig::ip`]) as configured, unparsed. Cluster + /// metadata publishes this verbatim when [`Self::advertised_for`] finds + /// nothing: the roster `ip` is only validated non-empty, and Docker + /// service names with underscores exist in the wild. + #[must_use] + pub fn raw_advertised_fallback(&self) -> &str { + self.config + .advertised_address + .as_deref() + .unwrap_or(&self.config.ip) + } + + /// Longest-prefix match over the boot-parsed selector networks. The + /// client IP is canonicalized first so a v4-mapped v6 peer + /// (`::ffff:10.0.0.7`, the shape a dual-stack listener reports) matches + /// v4 networks. `min_by_key` keeps the first of equal-length matches, so + /// resolution stays declaration-order deterministic even though a + /// validated config cannot produce two matching networks of equal length + /// (equal-length distinct networks are disjoint, duplicates are + /// rejected). + fn selector_address(&self, client_ip: IpAddr) -> Option<&AdvertisedAddress> { + let client_ip = client_ip.to_canonical(); + self.selectors + .iter() + .filter(|(network, _)| network.contains(&client_ip)) + .min_by_key(|(network, _)| Reverse(network.prefix_len())) + .map(|(_, address)| address) + } +} + +/// Network-side mirror of the `IpAddr::to_canonical` applied to client IPs +/// before matching: a selector network written in v4-mapped v6 form +/// (`::ffff:10.0.0.0/104`) becomes its v4 equivalent (`10.0.0.0/8`), since a +/// canonicalized client could never match the v6 spelling. Prefixes shorter +/// than 96 bits cannot drop the `::ffff:` mapping and stay v6 (they match +/// native v6 clients only). +fn canonical_ip_net(network: IpNet) -> IpNet { + if let IpNet::V6(v6_network) = network + && v6_network.prefix_len() >= 96 + && let IpAddr::V4(v4_address) = v6_network.addr().to_canonical() + && let Ok(v4_network) = Ipv4Net::new(v4_address, v6_network.prefix_len() - 96) + { + return IpNet::V4(v4_network); + } + network +} + +/// Per-node listener ports advertised in the cluster roster. In cluster mode +/// the roster is the single source of ports: every enabled transport needs +/// an explicit per-node port (validated at startup, no fallback to the +/// transport's top-level `address` port). The roster entry's `ip` is the +/// advertised address only: tcp/ws/quic/http bind the interface from their own +/// `address` config, and followers forward HTTP requests to the primary at +/// `ip:http`. +#[derive(Debug, Deserialize, Serialize, Clone, Default, ConfigEnv)] +pub struct TransportPorts { + pub tcp: Option, + pub quic: Option, + pub http: Option, + pub websocket: Option, + /// Dedicated port for replica-to-replica consensus traffic. + pub tcp_replica: Option, +} + +/// A validated client-facing node address: a literal IP or a DNS hostname. +/// +/// Hostnames follow RFC 1123: ASCII letters, digits and hyphens in labels of +/// 1-63 characters that do not start or end with a hyphen, at most +/// [`MAX_HOSTNAME_LEN`] characters total, no port and no trailing dot. Names +/// consisting solely of digits and dots are rejected as malformed IPv4 rather +/// than accepted as hostnames, so `10.0.0.256` fails loudly instead of being +/// handed to DNS. Hostnames normalize to lowercase and IPs to their canonical +/// form ([`IpAddr`]), so textual variants of one address (`Broker.Example.COM`, +/// `2001:DB8::1`, `[2001:db8::1]`) compare equal. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum AdvertisedAddress { + Ip(IpAddr), + Hostname(String), +} + +impl AdvertisedAddress { + /// Render `host:port` for a URL or endpoint listing, bracketing IPv6 + /// hosts (`[::1]:8080`) so the port separator stays unambiguous. + pub fn authority(&self, port: u16) -> String { + match self { + Self::Ip(ip) => SocketAddr::new(*ip, port).to_string(), + Self::Hostname(hostname) => format!("{hostname}:{port}"), + } + } +} + +impl FromStr for AdvertisedAddress { + type Err = AdvertisedAddressError; + + fn from_str(address: &str) -> Result { + if address.is_empty() { + return Err(AdvertisedAddressError::Empty); + } + if let Ok(ip) = address.parse::() { + return Ok(Self::Ip(ip)); + } + // URL-style bracketed IPv6 (`[2001:db8::1]`) is unambiguous; accept + // it and store the inner address. + if let Some(inner) = address + .strip_prefix('[') + .and_then(|rest| rest.strip_suffix(']')) + && let Ok(ip) = inner.parse::() + { + return Ok(Self::Ip(IpAddr::V6(ip))); + } + if let Some((host, port)) = address.rsplit_once(':') { + // `host:port` and `[v6]:port` are the common misconfigurations; + // anything else with a colon can only be a broken IPv6 literal, + // since ':' never appears in a hostname. + let bracketed_host = host.starts_with('[') && host.ends_with(']'); + if !port.is_empty() + && port.bytes().all(|byte| byte.is_ascii_digit()) + && (bracketed_host || !host.contains(':')) + { + return Err(AdvertisedAddressError::PortNotAllowed); + } + return Err(AdvertisedAddressError::MalformedIpv6); + } + if address.len() > MAX_HOSTNAME_LEN { + return Err(AdvertisedAddressError::HostnameTooLong { + length: address.len(), + }); + } + let mut all_labels_numeric = true; + for label in address.split('.') { + if label.is_empty() { + return Err(AdvertisedAddressError::EmptyLabel); + } + if label.len() > MAX_HOSTNAME_LABEL_LEN { + return Err(AdvertisedAddressError::LabelTooLong { + label: label.to_owned(), + }); + } + if label.starts_with('-') || label.ends_with('-') { + return Err(AdvertisedAddressError::LabelHyphen { + label: label.to_owned(), + }); + } + if let Some(character) = label + .chars() + .find(|character| !character.is_ascii_alphanumeric() && *character != '-') + { + return Err(AdvertisedAddressError::InvalidCharacter { character }); + } + all_labels_numeric &= label.bytes().all(|byte| byte.is_ascii_digit()); + } + if all_labels_numeric { + return Err(AdvertisedAddressError::MalformedIpv4); + } + // DNS resolution is case-insensitive; normalizing here makes equality + // (and thus endpoint-conflict detection) case-insensitive too. + Ok(Self::Hostname(address.to_ascii_lowercase())) + } +} + +impl fmt::Display for AdvertisedAddress { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Ip(ip) => write!(formatter, "{ip}"), + Self::Hostname(hostname) => write!(formatter, "{hostname}"), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AdvertisedAddressError { + Empty, + PortNotAllowed, + MalformedIpv4, + MalformedIpv6, + HostnameTooLong { length: usize }, + EmptyLabel, + LabelTooLong { label: String }, + LabelHyphen { label: String }, + InvalidCharacter { character: char }, +} + +impl fmt::Display for AdvertisedAddressError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Empty => write!(formatter, "address cannot be empty"), + Self::PortNotAllowed => write!( + formatter, + "address must not include a port; ports are configured in cluster.nodes.ports" + ), + Self::MalformedIpv4 => write!( + formatter, + "address consists only of digits and dots but is not a valid IPv4 address" + ), + Self::MalformedIpv6 => write!( + formatter, + "address contains ':' but is not a valid IPv6 address, and ':' cannot appear in a hostname" + ), + Self::HostnameTooLong { length } => write!( + formatter, + "hostname is {length} characters long; the limit is {MAX_HOSTNAME_LEN}" + ), + Self::EmptyLabel => write!( + formatter, + "hostname contains an empty label (leading, trailing, or doubled dot)" + ), + Self::LabelTooLong { label } => write!( + formatter, + "hostname label '{label}' exceeds {MAX_HOSTNAME_LABEL_LEN} characters" + ), + Self::LabelHyphen { label } => write!( + formatter, + "hostname label '{label}' cannot start or end with a hyphen" + ), + Self::InvalidCharacter { character } => write!( + formatter, + "character '{character}' is not allowed in a hostname (allowed: ASCII letters, digits, '-', '.')" + ), + } + } +} + +impl std::error::Error for AdvertisedAddressError {} + +/// Whether cluster-wide JWT key material exists: a configured `http.jwt` +/// secret, or the signing key derived from the cluster PSK. When it does, a +/// bearer minted on any node verifies on every node - the invariant +/// follower-to-primary HTTP forwarding depends on. Callers gate `http.enabled` +/// themselves; this covers only the key material. +/// +/// Forwarding targets resolve from the roster (`ip:ports.http`); the config +/// validator unconditionally requires a roster port for every enabled +/// transport, so a forward never dials a node without a declared http port. +pub fn http_forwarding_key_material(jwt: &HttpJwtConfig, cluster: &ClusterConfig) -> bool { + cluster.enabled + && ((cluster.auth.enabled && !cluster.auth.shared_secret.is_empty()) + || !jwt.encoding_secret.is_empty() + || !jwt.decoding_secret.is_empty()) +} + +impl Validatable for ClusterConfig { + fn validate(&self) -> Result<(), ConfigurationError> { + // Ahead of the enabled gate: the top-level rule against + // message_bus.peer_queue_capacity holds repair_chunk_max unconditionally, + // so a single-node config skipping these would be bound by the + // cross-section rule while its own floor and ceiling went unchecked. + if self.repair_chunk_max == 0 { + eprintln!("Invalid cluster configuration: cluster.repair_chunk_max must be > 0"); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if self.repair_chunk_max > MAX_REPAIR_CHUNK_MAX { + eprintln!( + "Invalid cluster configuration: cluster.repair_chunk_max ({}) exceeds the maximum \ + ({MAX_REPAIR_CHUNK_MAX})", + self.repair_chunk_max + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + if !self.enabled { + return Ok(()); + } + + if self.name.trim().is_empty() { + eprintln!("Invalid cluster configuration: cluster name cannot be empty"); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + // `0` / `disabled` / `unlimited` all parse to a zero duration and + // land here too: there is no way to switch the liveness window off. + if self.heartbeat_timeout.get_duration() < MIN_CLUSTER_HEARTBEAT_TIMEOUT { + eprintln!( + "Invalid cluster configuration: cluster.heartbeat_timeout '{}' must be at least {}s \ + (the primary signals liveness through its commit broadcast; a shorter window \ + elects on every scheduling hiccup)", + self.heartbeat_timeout, + MIN_CLUSTER_HEARTBEAT_TIMEOUT.as_secs() + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + // The commit broadcast is the cluster's liveness feed and the prepare + // retransmit its recovery timer; both size consensus timers that have + // to advance. `0` / `disabled` / `unlimited` all collapse to a zero + // duration, which would stall the timer - reject them. + if self.commit_broadcast_interval.get_duration().is_zero() { + eprintln!( + "Invalid cluster configuration: cluster.commit_broadcast_interval must be nonzero \ + (it drives the primary's liveness broadcast)" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if self.prepare_retransmit_interval.get_duration().is_zero() { + eprintln!( + "Invalid cluster configuration: cluster.prepare_retransmit_interval must be \ + nonzero (it drives prepare retransmission)" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + // The liveness window must span several commit broadcasts so a single + // delayed broadcast never trips a view change on a healthy primary. + let min_heartbeat = self + .commit_broadcast_interval + .get_duration() + .saturating_mul(MIN_HEARTBEAT_TO_COMMIT_BROADCAST_RATIO); + if self.heartbeat_timeout.get_duration() < min_heartbeat { + eprintln!( + "Invalid cluster configuration: cluster.heartbeat_timeout '{}' must be at least \ + {}x cluster.commit_broadcast_interval '{}' so the liveness window spans several \ + broadcasts", + self.heartbeat_timeout, + MIN_HEARTBEAT_TO_COMMIT_BROADCAST_RATIO, + self.commit_broadcast_interval + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + // The three view-change timers each size a consensus timer that has to + // advance; `0` / `disabled` / `unlimited` all collapse to zero and + // would stall it - reject them. + if self + .view_change_retransmit_interval + .get_duration() + .is_zero() + { + eprintln!( + "Invalid cluster configuration: cluster.view_change_retransmit_interval must be \ + nonzero (it drives StartViewChange / DoViewChange retransmission)" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if self.view_change_status_timeout.get_duration().is_zero() { + eprintln!( + "Invalid cluster configuration: cluster.view_change_status_timeout must be nonzero \ + (it backstops a stalled view change)" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if self + .request_start_view_retransmit_interval + .get_duration() + .is_zero() + { + eprintln!( + "Invalid cluster configuration: cluster.request_start_view_retransmit_interval \ + must be nonzero (it drives RequestStartView retransmission)" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + // The status backstop must span several retransmits so a few dropped + // view-change messages retransmit rather than escalating a progressing + // view change into a fresh cluster-wide election. + let min_status = self + .view_change_retransmit_interval + .get_duration() + .saturating_mul(MIN_STATUS_TO_RETRANSMIT_RATIO); + if self.view_change_status_timeout.get_duration() < min_status { + eprintln!( + "Invalid cluster configuration: cluster.view_change_status_timeout '{}' must be at \ + least {}x cluster.view_change_retransmit_interval '{}' so a stalled view change \ + retransmits before it escalates to an election", + self.view_change_status_timeout, + MIN_STATUS_TO_RETRANSMIT_RATIO, + self.view_change_retransmit_interval + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + // A recovering replica needs at least one probe before it may give up + // and elect; the ceiling is a typo guard (see MAX_VIEW_PROBE_ATTEMPTS). + if self.view_probe_attempts_max == 0 { + eprintln!( + "Invalid cluster configuration: cluster.view_probe_attempts_max must be >= 1" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if self.view_probe_attempts_max > MAX_VIEW_PROBE_ATTEMPTS { + eprintln!( + "Invalid cluster configuration: cluster.view_probe_attempts_max ({}) exceeds the \ + maximum ({MAX_VIEW_PROBE_ATTEMPTS})", + self.view_probe_attempts_max + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + // The repair retry interval sizes a tick threshold that has to advance; + // `0` / `disabled` / `unlimited` all collapse to zero and would wedge + // every stalled repair stream - reject them. + if self.repair_retry_interval.get_duration().is_zero() { + eprintln!( + "Invalid cluster configuration: cluster.repair_retry_interval must be nonzero \ + (it paces stalled-repair retries)" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + if self.nodes.is_empty() { + eprintln!( + "Invalid cluster configuration: cluster.nodes must contain at least one entry when cluster is enabled" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + // VSR needs every replica to have a stable, unique id strictly + // less than the total replica count. Duplicate ids would split the + // cluster into two replicas claiming the same slot; out-of-range + // ids never win a primary election. Both are unrecoverable at + // runtime - fail fast at startup. + let total_replicas = u8::try_from(self.nodes.len()).map_err(|_| { + eprintln!("Invalid cluster configuration: more than 255 replicas is unsupported"); + ConfigurationError::InvalidConfigurationValue + })?; + + let mut seen_ids = std::collections::HashSet::new(); + let mut seen_names = std::collections::HashSet::new(); + let mut used_endpoints = std::collections::HashSet::new(); + let mut advertised_endpoints: Vec = Vec::new(); + + for node in &self.nodes { + if node.name.trim().is_empty() { + eprintln!("Invalid cluster configuration: node name cannot be empty"); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + if node.ip.trim().is_empty() { + eprintln!( + "Invalid cluster configuration: IP cannot be empty for node '{}'", + node.name + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + if !seen_names.insert(node.name.clone()) { + eprintln!( + "Invalid cluster configuration: duplicate node name '{}' found", + node.name + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + if node.replica_id >= total_replicas { + eprintln!( + "Invalid cluster configuration: replica_id {} for node '{}' must be < total replica count {total_replicas}", + node.replica_id, node.name + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + if !seen_ids.insert(node.replica_id) { + eprintln!( + "Invalid cluster configuration: duplicate replica_id {} (two nodes claim the same slot)", + node.replica_id + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + let client_ports = [ + ("TCP", node.ports.tcp), + ("QUIC", node.ports.quic), + ("HTTP", node.ports.http), + ("WebSocket", node.ports.websocket), + ]; + let replica_port = ("TCP_REPLICA", node.ports.tcp_replica); + + for (name, port_opt) in client_ports.into_iter().chain([replica_port]) { + if let Some(port) = port_opt { + if port == 0 { + eprintln!( + "Invalid cluster configuration: {} port cannot be 0 for node '{}'", + name, node.name + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + let endpoint = format!("{}:{}", node.ip, port); + if !used_endpoints.insert(endpoint.clone()) { + eprintln!( + "Invalid cluster configuration: port conflict - {endpoint} is already bound (node '{}', transport {name})", + node.name + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + } + } + + // An advertised address must parse strictly (IP or RFC 1123 + // hostname): the value is handed verbatim to every client via + // cluster metadata and redirect URLs, so a bad one poisons them + // all. The roster `ip` predates this check and is only validated + // as non-empty (Docker service names with underscores exist in + // the wild), so when it backs the client endpoints an unparsable + // value falls back to raw-string comparison instead of failing + // boot. + let client_address = match node.advertised_address.as_deref() { + Some(advertised_address) => match advertised_address.parse::() { + Ok(address) => Some(address), + Err(error) => { + eprintln!( + "Invalid cluster configuration: advertised_address '{advertised_address}' for node '{}': {error}", + node.name + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + }, + None => node.ip.parse::().ok(), + }; + + if node.advertised_addresses.len() > MAX_ADVERTISED_SELECTORS { + eprintln!( + "Invalid cluster configuration: node '{}' declares {} advertised_addresses \ + selectors, exceeding the maximum ({MAX_ADVERTISED_SELECTORS})", + node.name, + node.advertised_addresses.len() + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + // Selector CIDRs and addresses feed clients the same way the + // catch-all advertised address does, so they get the same strict + // parse. Networks are compared truncated (`10.0.1.0/16` == + // `10.0.0.0/16`) and canonicalized (`::ffff:10.0.0.0/104` == + // `10.0.0.0/8`) since matching truncates and canonicalizes too. + // Parsed before the catch-all enters the conflict pool because + // every entry's effective client set depends on the node's full + // selector list. + let mut selectors = Vec::with_capacity(node.advertised_addresses.len()); + let mut seen_selector_cidrs = std::collections::HashSet::new(); + for selector in &node.advertised_addresses { + let client_cidr = match selector.client_cidr.parse::() { + Ok(client_cidr) => canonical_ip_net(client_cidr.trunc()), + Err(error) => { + eprintln!( + "Invalid cluster configuration: advertised_addresses client_cidr '{}' for node '{}': {error}", + selector.client_cidr, node.name + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + }; + if !seen_selector_cidrs.insert(client_cidr) { + eprintln!( + "Invalid cluster configuration: duplicate advertised_addresses client_cidr '{}' for node '{}'", + selector.client_cidr, node.name + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + let address = match selector.address.parse::() { + Ok(address) => address, + Err(error) => { + eprintln!( + "Invalid cluster configuration: advertised_addresses address '{}' for node '{}': {error}", + selector.address, node.name + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + }; + selectors.push((client_cidr, address)); + } + let selector_ranges: Vec = selectors + .iter() + .map(|(network, _)| ClientAddressRange::from(network)) + .collect(); + + // Endpoint conflicts are checked across every node's selectors + // and catch-all on effective client sets - the clients an entry + // actually wins after this node's longest-prefix match. Two + // nodes may reuse one host:port as long as the winning sets stay + // disjoint (that is the feature, and it includes a per-subnet + // override shadowing the same node's wider selector); a conflict + // means some client wins both entries and would resolve both + // nodes to one endpoint. The catch-all is an implicit + // match-everything-else selector, so it pools the same way. A + // roster ip that fails the strict parse skips the pool: it can + // never equal a parsed host, and two raw ips sharing host:port + // are already rejected by the bind-endpoint check above. + if let Some(address) = &client_address { + let catch_all_clients = EffectiveClients::for_catch_all(&selector_ranges); + for (name, port) in &client_ports { + if let Some(port) = port { + insert_advertised_endpoint( + &mut advertised_endpoints, + AdvertisedEndpoint { + node_name: &node.name, + transport: name, + network: None, + clients: catch_all_clients.clone(), + host: address.clone(), + port: *port, + }, + )?; + } + } + } + + for (selector_index, (client_cidr, address)) in selectors.iter().enumerate() { + let sibling_ranges: Vec = selector_ranges + .iter() + .enumerate() + .filter(|(other_index, _)| *other_index != selector_index) + .map(|(_, range)| *range) + .collect(); + let clients = EffectiveClients::for_selector(client_cidr, &sibling_ranges); + for (name, port) in &client_ports { + if let Some(port) = port { + insert_advertised_endpoint( + &mut advertised_endpoints, + AdvertisedEndpoint { + node_name: &node.name, + transport: name, + network: Some(*client_cidr), + clients: clients.clone(), + host: address.clone(), + port: *port, + }, + )?; + } + } + } + } + + // Replica-auth PSK (only reached when the cluster is enabled; the early + // return above skips these while it is disabled). When auth is enabled + // the key is mandatory; any configured key must clear the length floor - + // a typo guard that fires with auth off too, though only while the + // cluster itself is enabled. + let secret_len = self.auth.shared_secret.len(); + if self.auth.enabled && self.auth.shared_secret.is_empty() { + eprintln!( + "Invalid cluster configuration: cluster.auth.shared_secret must be set when cluster.auth.enabled is true" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if !self.auth.shared_secret.is_empty() && secret_len < MIN_SHARED_SECRET_LEN { + eprintln!( + "Invalid cluster configuration: cluster.auth.shared_secret must be >= {MIN_SHARED_SECRET_LEN} bytes" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + // Rotation window key: same typo guard as the primary, plus a + // distinctness check - a window equal to the primary means the + // operator rolled the config without changing the key, so the + // "rotation" would silently be a no-op. + if !self.auth.previous_shared_secret.is_empty() { + if self.auth.previous_shared_secret.len() < MIN_SHARED_SECRET_LEN { + eprintln!( + "Invalid cluster configuration: cluster.auth.previous_shared_secret must be >= {MIN_SHARED_SECRET_LEN} bytes" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if self.auth.previous_shared_secret == self.auth.shared_secret { + eprintln!( + "Invalid cluster configuration: cluster.auth.previous_shared_secret must differ from cluster.auth.shared_secret (an identical window is a no-op rotation)" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + } + + // Replica TLS. Both cert modes run one-directional TLS (no client + // certificate anywhere), so TLS only authenticates the acceptor to + // the dialer; peer authentication comes solely from the PSK + // handshake. Without it any TLS-capable host could register as a + // replica - require auth in both modes. CA mode (the default) + // additionally needs all three PEM paths: cert/key for this node's + // acceptor side, ca_file as the dialer's trust anchor. + if self.tls.enabled { + if !self.auth.enabled { + eprintln!( + "Invalid cluster configuration: cluster.tls.enabled = true requires cluster.auth.enabled = true (TLS authenticates the acceptor only; the PSK handshake authenticates the peer)" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if !self.tls.self_signed { + for (field, value) in [ + ("cert_file", &self.tls.cert_file), + ("key_file", &self.tls.key_file), + ("ca_file", &self.tls.ca_file), + ] { + if value.trim().is_empty() { + eprintln!( + "Invalid cluster configuration: cluster.tls.{field} must be set when cluster.tls.enabled = true and self_signed = false" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + } + } + } + + Ok(()) + } +} + +/// One advertised client endpoint and the clients it wins, pooled by +/// [`ClusterConfig::validate`] so selectors and catch-all conflict-check +/// against each other. `network: None` is the catch-all (`advertised_address`, +/// or the roster `ip` as fallback); `clients` is the entry's effective set +/// after its node's longest-prefix shadowing. +struct AdvertisedEndpoint<'roster> { + node_name: &'roster str, + transport: &'static str, + network: Option, + clients: EffectiveClients, + host: AdvertisedAddress, + port: u16, +} + +impl AdvertisedEndpoint<'_> { + /// True when some client would resolve both entries to one host:port on + /// two different nodes. Effective client sets already encode each node's + /// longest-prefix shadowing, so nested networks conflict only where the + /// wider entry still wins some client that the other node's entry also + /// wins. Entries of one node never conflict: their effective sets are + /// disjoint by construction. + fn conflicts_with(&self, other: &Self) -> bool { + self.node_name != other.node_name + && self.port == other.port + && self.host == other.host + && self.clients.overlaps(&other.clients) + } + + fn authority(&self) -> String { + self.host.authority(self.port) + } + + fn network_description(&self) -> String { + match self.network { + Some(network) => format!("client_cidr {network}"), + None => "every client network (catch-all)".to_owned(), + } + } +} + +/// The clients an advertised entry actually wins under its node's +/// longest-prefix match, built by [`ClusterConfig::validate`] for the +/// cross-node conflict scan. +#[derive(Clone)] +struct EffectiveClients { + /// Sorted disjoint ranges of winning client addresses. + ranges: Vec, + /// The catch-all also wins clients whose peer address the transport + /// could not produce ([`ResolvedClusterNode::advertised_for`] with no + /// client IP), so two catch-all overlap even when selectors cover both + /// address families. + serves_unknown_peers: bool, +} + +impl EffectiveClients { + /// A selector wins its network minus the sibling networks nested inside + /// it (longer prefixes take the node's LPM). `sibling_ranges` must + /// exclude the selector's own network. + fn for_selector(network: &IpNet, sibling_ranges: &[ClientAddressRange]) -> Self { + Self { + ranges: ClientAddressRange::from(network).subtract_nested(sibling_ranges), + serves_unknown_peers: false, + } + } + + /// The catch-all wins every client no selector matches, in both address + /// families, plus unknown-peer clients. + fn for_catch_all(selector_ranges: &[ClientAddressRange]) -> Self { + let mut ranges = ClientAddressRange::FULL_IPV4.subtract_nested(selector_ranges); + ranges.extend(ClientAddressRange::FULL_IPV6.subtract_nested(selector_ranges)); + Self { + ranges, + serves_unknown_peers: true, + } + } + + fn overlaps(&self, other: &Self) -> bool { + if self.serves_unknown_peers && other.serves_unknown_peers { + return true; + } + self.ranges.iter().any(|range| { + other.ranges.iter().any(|other_range| { + range.is_ipv4 == other_range.is_ipv4 + && range.first <= other_range.last + && other_range.first <= range.last + }) + }) + } +} + +/// Inclusive range of client addresses within one family. Client IPs +/// canonicalize to v4 before matching, so v4 and v6 networks match disjoint +/// client populations and a range never spans families. +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +struct ClientAddressRange { + is_ipv4: bool, + first: u128, + last: u128, +} + +impl ClientAddressRange { + const FULL_IPV4: Self = Self { + is_ipv4: true, + first: 0, + last: u32::MAX as u128, + }; + const FULL_IPV6: Self = Self { + is_ipv4: false, + first: 0, + last: u128::MAX, + }; + + /// `self` minus every range nested inside it, as sorted disjoint + /// leftovers. CIDR networks are nested or disjoint, never partially + /// overlapping, so a range outside `self` is either disjoint from it + /// (subtracts nothing) or contains it (a shorter prefix, which loses + /// LPM and also subtracts nothing). + fn subtract_nested(self, ranges: &[Self]) -> Vec { + let mut nested: Vec = ranges + .iter() + .filter(|range| { + range.is_ipv4 == self.is_ipv4 + && range.first >= self.first + && range.last <= self.last + }) + .copied() + .collect(); + nested.sort_unstable(); + let mut remaining = Vec::new(); + let mut cursor = Some(self.first); + for nested_range in nested { + let Some(next_free) = cursor else { break }; + if nested_range.first > next_free { + remaining.push(Self { + is_ipv4: self.is_ipv4, + first: next_free, + last: nested_range.first - 1, + }); + } + cursor = nested_range + .last + .checked_add(1) + .map(|after| after.max(next_free)); + } + if let Some(next_free) = cursor + && next_free <= self.last + { + remaining.push(Self { + is_ipv4: self.is_ipv4, + first: next_free, + last: self.last, + }); + } + remaining + } +} + +impl From<&IpNet> for ClientAddressRange { + fn from(network: &IpNet) -> Self { + match network { + IpNet::V4(network) => Self { + is_ipv4: true, + first: u128::from(u32::from(network.network())), + last: u128::from(u32::from(network.broadcast())), + }, + IpNet::V6(network) => Self { + is_ipv4: false, + first: u128::from(network.network()), + last: u128::from(network.broadcast()), + }, + } + } +} + +fn insert_advertised_endpoint<'roster>( + advertised_endpoints: &mut Vec>, + endpoint: AdvertisedEndpoint<'roster>, +) -> Result<(), ConfigurationError> { + if let Some(existing) = advertised_endpoints + .iter() + .find(|existing| existing.conflicts_with(&endpoint)) + { + eprintln!( + "Invalid cluster configuration: advertised client endpoint conflict - {} is advertised for {} (node '{}', transport {}) and for {} (node '{}', transport {}); their effective client sets overlap after longest-prefix shadowing, so a client in the overlap would resolve both nodes to one endpoint", + endpoint.authority(), + endpoint.network_description(), + endpoint.node_name, + endpoint.transport, + existing.network_description(), + existing.node_name, + existing.transport, + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + advertised_endpoints.push(endpoint); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn shared_secret_is_never_serialized() { + // Regression guard: the runtime current_config.toml (and the + // ServerConfig diagnostic snapshot that cats it) are produced by + // serializing this struct, so the PSK must not survive serialize. + // skip_serializing is format-agnostic, so a JSON dump proves the toml + // path too. + let config = ClusterConfig { + enabled: true, + name: "iggy-cluster".to_owned(), + heartbeat_timeout: default_heartbeat_timeout(), + commit_broadcast_interval: default_commit_broadcast_interval(), + prepare_retransmit_interval: default_prepare_retransmit_interval(), + view_change_retransmit_interval: default_view_change_retransmit_interval(), + view_change_status_timeout: default_view_change_status_timeout(), + request_start_view_retransmit_interval: default_request_start_view_retransmit_interval( + ), + view_probe_attempts_max: default_view_probe_attempts_max(), + repair_retry_interval: default_repair_retry_interval(), + repair_chunk_max: default_repair_chunk_max(), + nodes: Vec::new(), + auth: ClusterAuthConfig { + enabled: true, + shared_secret: "current-psk-MUST-NOT-be-persisted".to_owned(), + previous_shared_secret: "retiring-psk-MUST-NOT-be-persisted".to_owned(), + }, + tls: ClusterTlsConfig::default(), + }; + let serialized = serde_json::to_string(&config).expect("serialize cluster config"); + assert!( + !serialized.contains("MUST-NOT-be-persisted"), + "PSK leaked into serialized config: {serialized}" + ); + assert!( + !serialized.contains("shared_secret"), + "shared_secret field present in serialized config: {serialized}" + ); + } + + #[test] + fn cluster_node_rejects_unknown_fields() { + let error = serde_json::from_str::( + r#"{ + "name": "node-0", + "ip": "10.0.0.1", + "advertise_address": "203.0.113.1", + "replica_id": 0, + "ports": {} + }"#, + ) + .expect_err("misspelled advertised_address must be rejected"); + + assert!( + error + .to_string() + .contains("unknown field `advertise_address`"), + "unexpected deserialization error: {error}" + ); + } + + #[test] + fn advertised_addresses_env_expansion_is_capped() { + // The selectors Vec nests inside the nodes Vec, so the derive's + // index ceilings multiply; without the field's max_elements cap the + // default of 256x256 adds ~131k leaked mappings to every boot. + let mappings = ::env_mappings(); + assert!( + mappings + .iter() + .any(|mapping| mapping.env_name.contains("ADVERTISED_ADDRESSES_15_")), + "selector index 15 must stay reachable by env override" + ); + assert!( + !mappings + .iter() + .any(|mapping| mapping.env_name.contains("ADVERTISED_ADDRESSES_16_")), + "selector env expansion must stop at max_elements = 16" + ); + } +} + +#[cfg(test)] +mod advertised_address_tests { + use super::*; + + #[test] + fn parses_ip_literals_to_canonical_form() { + assert_eq!( + "203.0.113.1".parse::(), + Ok(AdvertisedAddress::Ip("203.0.113.1".parse().unwrap())) + ); + for equivalent_address in ["2001:DB8::1", "2001:db8:0:0:0:0:0:1", "[2001:db8::1]"] { + assert_eq!( + equivalent_address.parse::(), + Ok(AdvertisedAddress::Ip("2001:db8::1".parse().unwrap())), + "'{equivalent_address}' must parse to canonical 2001:db8::1" + ); + } + } + + #[test] + fn normalizes_hostname_to_lowercase() { + let address = "Broker-1.Example.COM".parse::(); + assert_eq!( + address, + Ok(AdvertisedAddress::Hostname( + "broker-1.example.com".to_owned() + )) + ); + } + + #[test] + fn authority_brackets_ipv6_hosts_only() { + let cases = [ + ("203.0.113.1", "203.0.113.1:8090"), + ("2001:db8::1", "[2001:db8::1]:8090"), + ("broker-1.example.com", "broker-1.example.com:8090"), + ]; + for (host, expected_authority) in cases { + let address = host.parse::().expect("valid address"); + assert_eq!(address.authority(8090), expected_authority); + } + } + + #[test] + fn rejects_port_suffixes() { + for address_with_port in ["example.com:8090", "10.0.0.1:8090", "[2001:db8::1]:8090"] { + assert_eq!( + address_with_port.parse::(), + Err(AdvertisedAddressError::PortNotAllowed), + "'{address_with_port}' must be rejected as host:port" + ); + } + } + + #[test] + fn rejects_dotted_numeric_strings_as_malformed_ipv4() { + for malformed_ip in ["10.0.0.256", "192.168.1", "12345"] { + assert_eq!( + malformed_ip.parse::(), + Err(AdvertisedAddressError::MalformedIpv4), + "'{malformed_ip}' must not pass as a hostname" + ); + } + } + + #[test] + fn rejects_broken_ipv6_literals() { + for broken_ipv6 in ["2001:db8:::1", "[2001:db8::zz]", "::1::2"] { + assert_eq!( + broken_ipv6.parse::(), + Err(AdvertisedAddressError::MalformedIpv6), + "'{broken_ipv6}' must be rejected as malformed IPv6" + ); + } + } +} + +#[cfg(test)] +mod advertised_for_tests { + use super::*; + + fn node_with_selectors(selectors: Vec) -> ClusterNodeConfig { + ClusterNodeConfig { + name: "node-0".to_owned(), + ip: "10.0.1.5".to_owned(), + advertised_address: Some("203.0.113.10".to_owned()), + advertised_addresses: selectors, + replica_id: 0, + ports: TransportPorts::default(), + } + } + + fn selector(client_cidr: &str, address: &str) -> AdvertisedAddressSelector { + AdvertisedAddressSelector { + client_cidr: client_cidr.to_owned(), + address: address.to_owned(), + } + } + + fn resolved(node: ClusterNodeConfig) -> ResolvedClusterNode { + node.into() + } + + fn ip(address: &str) -> IpAddr { + address.parse().unwrap() + } + + #[test] + fn falls_back_to_advertised_address_without_selectors() { + let node = node_with_selectors(Vec::new()); + assert_eq!( + resolved(node).advertised_for(Some(ip("10.0.0.7"))), + Some(&AdvertisedAddress::Ip(ip("203.0.113.10"))) + ); + } + + #[test] + fn falls_back_to_roster_ip_without_advertised_address() { + let mut node = node_with_selectors(Vec::new()); + node.advertised_address = None; + assert_eq!( + resolved(node).advertised_for(Some(ip("10.0.0.7"))), + Some(&AdvertisedAddress::Ip(ip("10.0.1.5"))) + ); + } + + #[test] + fn is_none_when_no_fallback_parses() { + let mut node = node_with_selectors(Vec::new()); + node.advertised_address = None; + node.ip = "iggy_node".to_owned(); + assert_eq!(resolved(node).advertised_for(Some(ip("10.0.0.7"))), None); + } + + #[test] + fn matching_selector_beats_advertised_address() { + let node = node_with_selectors(vec![selector("10.0.0.0/16", "10.0.1.5")]); + assert_eq!( + resolved(node).advertised_for(Some(ip("10.0.200.7"))), + Some(&AdvertisedAddress::Ip(ip("10.0.1.5"))) + ); + } + + #[test] + fn unmatched_client_falls_back_to_advertised_address() { + let node = node_with_selectors(vec![selector("10.0.0.0/16", "10.0.1.5")]); + assert_eq!( + resolved(node).advertised_for(Some(ip("192.168.0.7"))), + Some(&AdvertisedAddress::Ip(ip("203.0.113.10"))) + ); + } + + #[test] + fn unknown_client_ip_falls_back_to_advertised_address() { + let node = node_with_selectors(vec![selector("10.0.0.0/16", "10.0.1.5")]); + assert_eq!( + resolved(node).advertised_for(None), + Some(&AdvertisedAddress::Ip(ip("203.0.113.10"))) + ); + } + + #[test] + fn longest_prefix_wins_regardless_of_declaration_order() { + let node = resolved(node_with_selectors(vec![ + selector("10.0.0.0/8", "10.255.255.1"), + selector("10.0.0.0/16", "10.0.1.5"), + ])); + assert_eq!( + node.advertised_for(Some(ip("10.0.200.7"))), + Some(&AdvertisedAddress::Ip(ip("10.0.1.5"))), + "the /16 must win over the /8 even though it is declared second" + ); + assert_eq!( + node.advertised_for(Some(ip("10.9.0.7"))), + Some(&AdvertisedAddress::Ip(ip("10.255.255.1"))), + "a client outside the /16 but inside the /8 must match the /8" + ); + } + + #[test] + fn equal_prefix_matches_resolve_deterministically_to_first_declared() { + // No validated config reaches this state: these networks truncate to + // one /16, which validation rejects as a duplicate. Pinned anyway so + // a future relaxation of that rule cannot make resolution + // order-dependent. + let node = node_with_selectors(vec![ + selector("10.0.1.0/16", "10.0.1.5"), + selector("10.0.2.0/16", "10.0.2.5"), + ]); + assert_eq!( + resolved(node).advertised_for(Some(ip("10.0.200.7"))), + Some(&AdvertisedAddress::Ip(ip("10.0.1.5"))) + ); + } + + #[test] + fn v4_mapped_v6_client_matches_v4_cidr() { + // A dual-stack listener reports v4 peers as `::ffff:a.b.c.d`. + let node = node_with_selectors(vec![selector("10.0.0.0/16", "10.0.1.5")]); + assert_eq!( + resolved(node).advertised_for(Some(ip("::ffff:10.0.0.7"))), + Some(&AdvertisedAddress::Ip(ip("10.0.1.5"))) + ); + } + + #[test] + fn v4_mapped_v6_selector_cidr_matches_v4_client() { + // The mirror case: the CIDR side canonicalizes at build, so + // `::ffff:10.0.0.0/104` matches like `10.0.0.0/8` instead of being + // a silently dead selector. + let node = node_with_selectors(vec![selector("::ffff:10.0.0.0/104", "10.0.1.5")]); + assert_eq!( + resolved(node).advertised_for(Some(ip("10.0.0.7"))), + Some(&AdvertisedAddress::Ip(ip("10.0.1.5"))) + ); + } + + #[test] + fn v6_selector_matches_v6_client() { + let node = node_with_selectors(vec![selector("2001:db8::/32", "2001:db8::1")]); + assert_eq!( + resolved(node).advertised_for(Some(ip("2001:db8::7"))), + Some(&AdvertisedAddress::Ip(ip("2001:db8::1"))) + ); + } + + #[test] + fn selector_address_may_be_a_hostname() { + let node = node_with_selectors(vec![selector("10.0.0.0/16", "Broker.Internal.Example")]); + assert_eq!( + resolved(node).advertised_for(Some(ip("10.0.0.7"))), + Some(&AdvertisedAddress::Hostname( + "broker.internal.example".to_owned() + )) + ); + } +} + +#[cfg(test)] +mod cluster_validate_tests { + use super::*; + + fn node(name: &str, id: u8) -> ClusterNodeConfig { + ClusterNodeConfig { + name: name.to_string(), + ip: "127.0.0.1".to_string(), + advertised_address: None, + advertised_addresses: Vec::new(), + replica_id: id, + ports: TransportPorts::default(), + } + } + + fn selector(client_cidr: &str, address: &str) -> AdvertisedAddressSelector { + AdvertisedAddressSelector { + client_cidr: client_cidr.to_owned(), + address: address.to_owned(), + } + } + + fn cfg(nodes: Vec) -> ClusterConfig { + ClusterConfig { + enabled: true, + name: "iggy-cluster".to_string(), + heartbeat_timeout: default_heartbeat_timeout(), + commit_broadcast_interval: default_commit_broadcast_interval(), + prepare_retransmit_interval: default_prepare_retransmit_interval(), + view_change_retransmit_interval: default_view_change_retransmit_interval(), + view_change_status_timeout: default_view_change_status_timeout(), + request_start_view_retransmit_interval: default_request_start_view_retransmit_interval( + ), + view_probe_attempts_max: default_view_probe_attempts_max(), + repair_retry_interval: default_repair_retry_interval(), + repair_chunk_max: default_repair_chunk_max(), + nodes, + auth: ClusterAuthConfig::default(), + tls: ClusterTlsConfig::default(), + } + } + + #[test] + fn validate_rejects_sub_minimum_heartbeat_timeout() { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.heartbeat_timeout = IggyDuration::new(Duration::from_millis(500)); + assert!(c.validate().is_err()); + // The "disabled" / "unlimited" sentinels collapse to zero and must + // be rejected the same way. + c.heartbeat_timeout = IggyDuration::new(Duration::ZERO); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_rejects_zero_commit_broadcast_interval() { + // `0` / `disabled` / `unlimited` all collapse to zero and stall the + // liveness broadcast. + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.commit_broadcast_interval = IggyDuration::new(Duration::ZERO); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_rejects_zero_prepare_retransmit_interval() { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.prepare_retransmit_interval = IggyDuration::new(Duration::ZERO); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_rejects_heartbeat_below_commit_broadcast_ratio() { + // 3s clears the absolute 2s floor but is still < 4x the 1s broadcast, + // so the ratio rule is what rejects here, not the floor. + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.heartbeat_timeout = IggyDuration::new(Duration::from_secs(3)); + c.commit_broadcast_interval = IggyDuration::new(Duration::from_secs(1)); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_accepts_heartbeat_at_commit_broadcast_ratio() { + // Exactly 4x the broadcast (and above the 2s floor) must pass. + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.heartbeat_timeout = IggyDuration::new(Duration::from_secs(4)); + c.commit_broadcast_interval = IggyDuration::new(Duration::from_secs(1)); + assert!(c.validate().is_ok()); + } + + #[test] + fn validate_rejects_zero_view_change_retransmit_interval() { + // `0` / `disabled` / `unlimited` all collapse to zero and stall the + // view-change retransmit timers. + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.view_change_retransmit_interval = IggyDuration::new(Duration::ZERO); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_rejects_zero_view_change_status_timeout() { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.view_change_status_timeout = IggyDuration::new(Duration::ZERO); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_rejects_zero_request_start_view_retransmit_interval() { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.request_start_view_retransmit_interval = IggyDuration::new(Duration::ZERO); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_rejects_view_change_status_below_retransmit_ratio() { + // 3s is nonzero but still < 4x the 1s retransmit, so the ratio rule is + // what rejects here, not the zero check. + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.view_change_retransmit_interval = IggyDuration::new(Duration::from_secs(1)); + c.view_change_status_timeout = IggyDuration::new(Duration::from_secs(3)); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_accepts_view_change_status_at_retransmit_ratio() { + // Exactly 4x the retransmit interval must pass. + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.view_change_retransmit_interval = IggyDuration::new(Duration::from_secs(1)); + c.view_change_status_timeout = IggyDuration::new(Duration::from_secs(4)); + assert!(c.validate().is_ok()); + } + + #[test] + fn validate_rejects_zero_view_probe_attempts_max() { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.view_probe_attempts_max = 0; + assert!(c.validate().is_err()); + } + + #[test] + fn validate_rejects_view_probe_attempts_above_ceiling() { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.view_probe_attempts_max = MAX_VIEW_PROBE_ATTEMPTS + 1; + assert!(c.validate().is_err()); + } + + #[test] + fn validate_accepts_view_probe_attempts_at_ceiling() { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.view_probe_attempts_max = MAX_VIEW_PROBE_ATTEMPTS; + assert!(c.validate().is_ok()); + } + + #[test] + fn validate_rejects_zero_repair_retry_interval() { + // `0` / `disabled` / `unlimited` all collapse to zero and would wedge + // stalled repair streams. + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.repair_retry_interval = IggyDuration::new(Duration::ZERO); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_rejects_zero_repair_chunk_max() { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.repair_chunk_max = 0; + assert!(c.validate().is_err()); + } + + #[test] + fn validate_rejects_repair_chunk_max_above_ceiling() { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.repair_chunk_max = MAX_REPAIR_CHUNK_MAX + 1; + assert!(c.validate().is_err()); + } + + #[test] + fn validate_accepts_repair_chunk_max_at_ceiling() { + // Section-level validate only; the cross-section rule against + // message_bus.peer_queue_capacity lives in the top-level validate. + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.repair_chunk_max = MAX_REPAIR_CHUNK_MAX; + assert!(c.validate().is_ok()); + } + + #[test] + fn validate_rejects_empty_nodes() { + let c = cfg(vec![]); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_rejects_duplicate_replica_ids() { + let c = cfg(vec![node("n1", 0), node("n2", 0)]); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_rejects_duplicate_names() { + let c = cfg(vec![node("n1", 0), node("n1", 1)]); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_rejects_out_of_range_replica_id() { + // 2 nodes total, so id 2 is out of range. + let c = cfg(vec![node("n1", 0), node("n2", 2)]); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_accepts_unique_contiguous_replica_ids() { + let c = cfg(vec![node("n1", 0), node("n2", 1), node("n3", 2)]); + assert!(c.validate().is_ok()); + } + + #[test] + fn validate_skips_checks_when_disabled() { + let mut c = cfg(vec![]); + c.enabled = false; + assert!(c.validate().is_ok()); + } + + // repair_chunk_max is also read by the unconditional top-level check + // against message_bus.peer_queue_capacity, so its own bounds apply with + // the cluster off too. + #[test] + fn validate_rejects_zero_repair_chunk_max_when_disabled() { + let mut c = cfg(vec![]); + c.enabled = false; + c.repair_chunk_max = 0; + assert!(c.validate().is_err()); + } + + #[test] + fn validate_rejects_repair_chunk_max_above_ceiling_when_disabled() { + let mut c = cfg(vec![]); + c.enabled = false; + c.repair_chunk_max = MAX_REPAIR_CHUNK_MAX + 1; + assert!(c.validate().is_err()); + } + + #[test] + fn validate_rejects_duplicate_tcp_replica_port() { + let ports = TransportPorts { + tcp: None, + quic: None, + http: None, + websocket: None, + tcp_replica: Some(9090), + }; + let mut n1 = node("n1", 0); + n1.ports = ports.clone(); + let mut n2 = node("n2", 1); + n2.ports = ports; + let c = cfg(vec![n1, n2]); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_rejects_cross_transport_port_reuse() { + let mut n1 = node("n1", 0); + n1.ports = TransportPorts { + tcp: Some(8090), + quic: None, + http: Some(8090), + websocket: None, + tcp_replica: None, + }; + let c = cfg(vec![n1]); + assert!( + c.validate().is_err(), + "same port on TCP and HTTP of the same node must be rejected" + ); + } + + #[test] + fn validate_accepts_same_port_on_different_ips() { + let mut n1 = node("n1", 0); + n1.ip = "127.0.0.1".to_string(); + n1.ports = TransportPorts { + tcp: Some(8090), + quic: None, + http: None, + websocket: None, + tcp_replica: None, + }; + let mut n2 = node("n2", 1); + n2.ip = "127.0.0.2".to_string(); + n2.ports = TransportPorts { + tcp: Some(8090), + quic: None, + http: None, + websocket: None, + tcp_replica: None, + }; + let c = cfg(vec![n1, n2]); + assert!(c.validate().is_ok()); + } + + #[test] + fn validate_rejects_duplicate_advertised_client_endpoint() { + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_address = Some("203.0.113.1".to_owned()); + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "10.0.0.2".to_owned(); + n2.advertised_address = n1.advertised_address.clone(); + n2.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, n2]).validate().is_err()); + } + + #[test] + fn validate_rejects_equivalent_ipv6_advertised_client_endpoints() { + for equivalent_address in ["2001:DB8::1", "2001:db8:0:0:0:0:0:1", "[2001:db8::1]"] { + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_address = Some("2001:db8::1".to_owned()); + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "10.0.0.2".to_owned(); + n2.advertised_address = Some(equivalent_address.to_owned()); + n2.ports.tcp = Some(8090); + + assert!( + cfg(vec![n1, n2]).validate().is_err(), + "{equivalent_address} must conflict with 2001:db8::1" + ); + } + } + + #[test] + fn validate_rejects_equivalent_ipv6_client_endpoints_from_node_ip() { + let mut n1 = node("n1", 0); + n1.ip = "2001:db8::1".to_owned(); + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "2001:db8:0:0:0:0:0:1".to_owned(); + n2.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, n2]).validate().is_err()); + } + + #[test] + fn validate_accepts_distinct_advertised_client_endpoints() { + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_address = Some("203.0.113.1".to_owned()); + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "10.0.0.2".to_owned(); + n2.advertised_address = Some("203.0.113.2".to_owned()); + n2.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, n2]).validate().is_ok()); + } + + #[test] + fn validate_accepts_hostname_advertised_address() { + let mut n1 = node("n1", 0); + n1.advertised_address = Some("iggy-node-1.example.com".to_owned()); + n1.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, node("n2", 1)]).validate().is_ok()); + } + + #[test] + fn validate_rejects_malformed_advertised_addresses() { + let oversized_label = format!("{}.example.com", "a".repeat(64)); + let oversized_hostname = format!("{}example.com", "a.".repeat(130)); + for advertised_address in [ + "", + " 203.0.113.1", + "10.0.0.256", + "192.168.1", + "example.com:8090", + "[2001:db8::1]:8090", + "2001:db8:::1", + "iggy_node.example.com", + "-node.example.com", + "node-.example.com", + ".example.com", + "example..com", + "example.com.", + "ex\u{e4}mple.com", + oversized_label.as_str(), + oversized_hostname.as_str(), + ] { + let mut n1 = node("n1", 0); + n1.advertised_address = Some(advertised_address.to_owned()); + + assert!( + cfg(vec![n1, node("n2", 1)]).validate().is_err(), + "'{advertised_address}' must be rejected" + ); + } + } + + #[test] + fn validate_rejects_case_variant_hostname_advertised_endpoints() { + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_address = Some("broker.example.com".to_owned()); + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "10.0.0.2".to_owned(); + n2.advertised_address = Some("Broker.Example.COM".to_owned()); + n2.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, n2]).validate().is_err()); + } + + #[test] + fn validate_rejects_node_ip_hostname_clashing_with_advertised_hostname() { + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_address = Some("broker.example.com".to_owned()); + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "broker.example.com".to_owned(); + n2.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, n2]).validate().is_err()); + } + + #[test] + fn validate_accepts_distinct_hostname_advertised_endpoints() { + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_address = Some("broker-1.example.com".to_owned()); + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "10.0.0.2".to_owned(); + n2.advertised_address = Some("broker-2.example.com".to_owned()); + n2.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, n2]).validate().is_ok()); + } + + #[test] + fn validate_accepts_selectors_with_distinct_cidrs() { + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_address = Some("203.0.113.1".to_owned()); + n1.advertised_addresses = vec![ + selector("10.0.0.0/16", "10.0.0.1"), + selector("10.0.0.0/8", "broker-1.internal.example"), + ]; + n1.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, node("n2", 1)]).validate().is_ok()); + } + + #[test] + fn validate_rejects_malformed_selector_cidr() { + for client_cidr in ["10.0.0.0", "10.0.0.0/33", "not-a-cidr", ""] { + let mut n1 = node("n1", 0); + n1.advertised_addresses = vec![selector(client_cidr, "10.0.0.1")]; + + assert!( + cfg(vec![n1, node("n2", 1)]).validate().is_err(), + "client_cidr '{client_cidr}' must be rejected" + ); + } + } + + #[test] + fn validate_rejects_malformed_selector_address() { + for address in ["", "10.0.0.1:8090", "10.0.0.256", "iggy_node"] { + let mut n1 = node("n1", 0); + n1.advertised_addresses = vec![selector("10.0.0.0/16", address)]; + + assert!( + cfg(vec![n1, node("n2", 1)]).validate().is_err(), + "selector address '{address}' must be rejected" + ); + } + } + + #[test] + fn validate_rejects_duplicate_selector_cidr_within_a_node() { + // `10.0.1.0/16` truncates to `10.0.0.0/16`: the two selectors match + // the identical client set, so the second could never win LPM. + let mut n1 = node("n1", 0); + n1.advertised_addresses = vec![ + selector("10.0.0.0/16", "10.0.0.1"), + selector("10.0.1.0/16", "10.0.0.2"), + ]; + + assert!(cfg(vec![n1, node("n2", 1)]).validate().is_err()); + } + + #[test] + fn validate_accepts_selector_count_at_the_cap() { + let mut n1 = node("n1", 0); + n1.advertised_addresses = (0..MAX_ADVERTISED_SELECTORS) + .map(|index| selector(&format!("10.{index}.0.0/16"), &format!("192.0.2.{index}"))) + .collect(); + + assert!(cfg(vec![n1, node("n2", 1)]).validate().is_ok()); + } + + #[test] + fn validate_rejects_selector_count_above_the_cap() { + // The env-override path stops expanding selector indices at the same + // ceiling, so a TOML roster exceeding it could never be replicated + // byte-identically through env vars. + let mut n1 = node("n1", 0); + n1.advertised_addresses = (0..=MAX_ADVERTISED_SELECTORS) + .map(|index| selector(&format!("10.{index}.0.0/16"), &format!("192.0.2.{index}"))) + .collect(); + + assert!(cfg(vec![n1, node("n2", 1)]).validate().is_err()); + } + + #[test] + fn validate_rejects_selector_endpoint_conflict_within_one_cidr() { + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_addresses = vec![selector("10.0.0.0/16", "10.0.7.7")]; + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "10.0.0.2".to_owned(); + n2.advertised_addresses = vec![selector("10.0.0.0/16", "10.0.7.7")]; + n2.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, n2]).validate().is_err()); + } + + #[test] + fn validate_accepts_identical_selector_endpoint_across_different_cidrs() { + // Reusing one host:port across DIFFERENT client networks is the + // feature (e.g. each network NATs the address to its local node). + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_addresses = vec![selector("10.1.0.0/16", "192.0.2.10")]; + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "10.0.0.2".to_owned(); + n2.advertised_addresses = vec![selector("10.2.0.0/16", "192.0.2.10")]; + n2.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, n2]).validate().is_ok()); + } + + #[test] + fn validate_rejects_v4_mapped_v6_selector_cidr_duplicating_its_v4_form() { + // `::ffff:10.0.0.0/104` canonicalizes to `10.0.0.0/8` (matching how + // client IPs canonicalize before LPM), so these two selectors match + // the identical client set. + let mut n1 = node("n1", 0); + n1.advertised_addresses = vec![ + selector("10.0.0.0/8", "10.0.0.1"), + selector("::ffff:10.0.0.0/104", "10.0.0.2"), + ]; + + assert!(cfg(vec![n1, node("n2", 1)]).validate().is_err()); + } + + #[test] + fn validate_rejects_selector_endpoint_clashing_with_another_nodes_catch_all() { + // The catch-all matches every client, so a 10.0.0.0/16 client would + // resolve both nodes to 192.0.2.10:8090. + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_addresses = vec![selector("10.0.0.0/16", "192.0.2.10")]; + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "10.0.0.2".to_owned(); + n2.advertised_address = Some("192.0.2.10".to_owned()); + n2.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, n2]).validate().is_err()); + } + + #[test] + fn validate_rejects_selector_endpoint_clashing_with_another_nodes_roster_ip() { + // Without an advertised_address the roster ip backs the catch-all, + // so the same cross-set conflict applies to it. + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_addresses = vec![selector("10.0.0.0/16", "10.0.0.2")]; + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "10.0.0.2".to_owned(); + n2.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, n2]).validate().is_err()); + } + + #[test] + fn validate_rejects_identical_selector_endpoint_across_nested_cidrs() { + // LPM runs per node, not cluster-wide: n1 has no longer prefix of + // its own shadowing the /16 overlap, so a 10.0.0.0/16 client wins + // n1's /8 and n2's /16, resolving both to 192.0.2.10:8090. + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_addresses = vec![selector("10.0.0.0/8", "192.0.2.10")]; + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "10.0.0.2".to_owned(); + n2.advertised_addresses = vec![selector("10.0.0.0/16", "192.0.2.10")]; + n2.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, n2]).validate().is_err()); + } + + #[test] + fn validate_accepts_nested_cidr_reuse_shadowed_by_same_node_longer_prefix() { + // n1's /16 selector shadows its /8 within 10.0.0.0/16, so n1's /8 + // entry wins only 10.0.0.0/8 minus 10.0.0.0/16 - disjoint from n2's + // /16. No client resolves both nodes to 192.0.2.10:8090. + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_addresses = vec![ + selector("10.0.0.0/8", "192.0.2.10"), + selector("10.0.0.0/16", "192.0.2.20"), + ]; + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "10.0.0.2".to_owned(); + n2.advertised_addresses = vec![selector("10.0.0.0/16", "192.0.2.10")]; + n2.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, n2]).validate().is_ok()); + } + + #[test] + fn validate_rejects_partially_shadowed_nested_cidr_reuse() { + // n1's /24 shadow carves only part of the /16 overlap: a client in + // 10.0.0.0/16 outside 10.0.0.0/24 still wins n1's /8 entry and n2's + // /16 entry, resolving both nodes to 192.0.2.10:8090. + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_addresses = vec![ + selector("10.0.0.0/8", "192.0.2.10"), + selector("10.0.0.0/24", "192.0.2.20"), + ]; + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "10.0.0.2".to_owned(); + n2.advertised_addresses = vec![selector("10.0.0.0/16", "192.0.2.10")]; + n2.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, n2]).validate().is_err()); + } + + #[test] + fn validate_accepts_catch_all_reuse_shadowed_by_same_node_selector() { + // n1's /16 selector shadows its catch-all within 10.0.0.0/16, so + // the catch-all never wins a client inside n2's /24. Without the + // shadow the same pair conflicts (see + // validate_rejects_selector_endpoint_clashing_with_another_nodes_catch_all). + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_address = Some("192.0.2.10".to_owned()); + n1.advertised_addresses = vec![selector("10.0.0.0/16", "192.0.2.20")]; + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "10.0.0.2".to_owned(); + n2.advertised_addresses = vec![selector("10.0.0.0/24", "192.0.2.10")]; + n2.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, n2]).validate().is_ok()); + } + + #[test] + fn validate_accepts_selector_reusing_a_fully_shadowed_catch_all_address() { + // n1's selectors cover both address families, so its catch-all wins + // known peers nowhere; only unknown-peer clients reach it, and they + // never match n2's selector. + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_address = Some("192.0.2.10".to_owned()); + n1.advertised_addresses = vec![ + selector("0.0.0.0/0", "192.0.2.20"), + selector("::/0", "192.0.2.30"), + ]; + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "10.0.0.2".to_owned(); + n2.advertised_addresses = vec![selector("10.0.0.0/16", "192.0.2.10")]; + n2.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, n2]).validate().is_ok()); + } + + #[test] + fn validate_accepts_catch_all_spelling_another_nodes_selector_address_when_self_shadowed() { + // Split-network NAT roster: n2's catch-all spells n1's 10/8 selector + // address, but n2's own 10/8 selector shadows its catch-all inside + // 10/8 (outside it n1 serves its own catch-all), so no client + // resolves both nodes to 192.0.2.10:8090. + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_addresses = vec![selector("10.0.0.0/8", "192.0.2.10")]; + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "10.0.0.2".to_owned(); + n2.advertised_address = Some("192.0.2.10".to_owned()); + n2.advertised_addresses = vec![selector("10.0.0.0/8", "192.0.2.20")]; + n2.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, n2]).validate().is_ok()); + } + + #[test] + fn validate_rejects_duplicate_catch_all_even_when_fully_shadowed() { + // A client whose peer address the transport cannot produce always + // falls to the catch-all, so duplicate catch-all conflict even when + // selectors cover every known network. + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_address = Some("192.0.2.10".to_owned()); + n1.advertised_addresses = vec![ + selector("0.0.0.0/0", "192.0.2.20"), + selector("::/0", "192.0.2.30"), + ]; + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "10.0.0.2".to_owned(); + n2.advertised_address = Some("192.0.2.10".to_owned()); + n2.advertised_addresses = vec![ + selector("0.0.0.0/0", "192.0.2.40"), + selector("::/0", "192.0.2.50"), + ]; + n2.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, n2]).validate().is_err()); + } + + #[test] + fn validate_accepts_selector_reusing_its_own_nodes_catch_all_address() { + // Redundant but harmless: within one node the selector and the + // catch-all cannot resolve a client to two different nodes. + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_address = Some("192.0.2.10".to_owned()); + n1.advertised_addresses = vec![selector("10.0.0.0/16", "192.0.2.10")]; + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "10.0.0.2".to_owned(); + n2.ports.tcp = Some(8091); + + assert!(cfg(vec![n1, n2]).validate().is_ok()); + } + + #[test] + fn validate_rejects_zero_tcp_replica_port() { + let ports = TransportPorts { + tcp: None, + quic: None, + http: None, + websocket: None, + tcp_replica: Some(0), + }; + let mut n1 = node("n1", 0); + n1.ports = ports; + let c = cfg(vec![n1]); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_accepts_empty_secret_when_auth_disabled() { + // Default: no secret, auth off -> legacy mode, must pass. + let c = cfg(vec![node("n1", 0), node("n2", 1)]); + assert!(c.validate().is_ok()); + } + + #[test] + fn validate_rejects_missing_secret_when_auth_enabled() { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.auth.enabled = true; + assert!(c.validate().is_err()); + } + + #[test] + fn validate_rejects_short_secret_when_auth_enabled() { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.auth.enabled = true; + c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN - 1); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_rejects_short_secret_even_when_auth_disabled() { + // Typo guard: a configured-but-short key fails even with auth off. + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN - 1); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_accepts_valid_secret_when_auth_enabled() { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.auth.enabled = true; + c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); + assert!(c.validate().is_ok()); + } + + #[test] + fn validate_accepts_valid_rotation_window() { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.auth.enabled = true; + c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); + c.auth.previous_shared_secret = "b".repeat(MIN_SHARED_SECRET_LEN); + assert!(c.validate().is_ok()); + } + + #[test] + fn validate_rejects_short_previous_secret() { + // Same typo guard as the primary key. + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.auth.enabled = true; + c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); + c.auth.previous_shared_secret = "b".repeat(MIN_SHARED_SECRET_LEN - 1); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_rejects_rotation_window_equal_to_primary() { + // An identical window is a no-op rotation: the operator rolled the + // config without changing the key. + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.auth.enabled = true; + c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); + c.auth.previous_shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); + assert!(c.validate().is_err()); + } + + fn tls_files() -> ClusterTlsConfig { + ClusterTlsConfig { + enabled: true, + self_signed: false, + cert_file: "cert.pem".to_string(), + key_file: "key.pem".to_string(), + ca_file: "ca.pem".to_string(), + } + } + + #[test] + fn validate_rejects_tls_ca_mode_with_missing_files() { + // Auth on so the failure exercises the file check, not the auth gate. + for missing in ["cert_file", "key_file", "ca_file"] { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.auth.enabled = true; + c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); + c.tls = tls_files(); + match missing { + "cert_file" => c.tls.cert_file.clear(), + "key_file" => c.tls.key_file.clear(), + _ => c.tls.ca_file.clear(), + } + assert!(c.validate().is_err(), "missing {missing} must be rejected"); + } + } + + #[test] + fn validate_rejects_tls_self_signed_without_auth() { + // Accept-any certificate without the PSK handshake = MITM-able. + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.tls = ClusterTlsConfig { + enabled: true, + self_signed: true, + ..ClusterTlsConfig::default() + }; + assert!(c.validate().is_err()); + } + + #[test] + fn validate_accepts_tls_self_signed_with_auth() { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.auth.enabled = true; + c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); + c.tls = ClusterTlsConfig { + enabled: true, + self_signed: true, + ..ClusterTlsConfig::default() + }; + assert!(c.validate().is_ok()); + } + + #[test] + fn validate_rejects_tls_ca_mode_without_auth() { + // TLS never authenticates the dialer (no client certificates); + // only the PSK handshake does, so it is mandatory with TLS on. + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.tls = tls_files(); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_accepts_tls_ca_mode_with_auth() { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.auth.enabled = true; + c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); + c.tls = tls_files(); + assert!(c.validate().is_ok()); + } +} diff --git a/core/configs/src/server_ng_config/defaults.rs b/core/configs/src/server_ng_config/defaults.rs new file mode 100644 index 0000000000..40695a63e6 --- /dev/null +++ b/core/configs/src/server_ng_config/defaults.rs @@ -0,0 +1,335 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! `Default` impls for the server-ng config surface. +//! +//! Sections that fork (`tcp`, `websocket`, `quic`, `cluster`, +//! `message_bus`) have their own `Default` impls here, sourced from +//! `core/server-ng/config.toml` via [`SERVER_NG_CONFIG`]. Sections that +//! still reuse legacy types (`http`, `system`, `telemetry`, +//! `consumer_group`, `data_maintenance`, `message_saver`, +//! `personal_access_token`, `heartbeat`) delegate to the legacy +//! `Default` impls; overrides land at the consumer level once +//! [`super::server_ng::ServerNgConfig::load`] is wired into server-ng's +//! bootstrap. + +use super::cluster::{ + ClusterAuthConfig, ClusterConfig, ClusterNodeConfig, ClusterTlsConfig, TransportPorts, +}; +use super::message_bus::MessageBusConfig; +use super::metadata::MetadataConfig; +use super::partition::PartitionConfig; +use super::quic::{QuicCertificateConfig, QuicConfig, QuicSocketConfig}; +use super::server_ng::NgSystemConfig; +use super::server_ng::{ExtraConfig, ServerNgConfig}; +use super::tcp::{TcpConfig, TcpSocketConfig, TcpTlsConfig}; +use super::websocket::{WebSocketConfig, WebSocketTlsConfig}; +use crate::server_config::http::HttpConfig; +use crate::server_config::server::{ + ConsumerGroupConfig, DataMaintenanceConfig, HeartbeatConfig, MessageSaverConfig, + PersonalAccessTokenConfig, TelemetryConfig, +}; +use std::sync::Arc; + +static_toml::static_toml! { + // static_toml resolves relative to CARGO_MANIFEST_DIR (core/configs/). + pub static SERVER_NG_CONFIG = include_toml!("../server-ng/config.toml"); +} + +impl Default for ServerNgConfig { + fn default() -> ServerNgConfig { + ServerNgConfig { + consumer_group: ConsumerGroupConfig::default(), + data_maintenance: DataMaintenanceConfig::default(), + extra: ExtraConfig::default(), + heartbeat: HeartbeatConfig::default(), + message_saver: MessageSaverConfig::default(), + personal_access_token: PersonalAccessTokenConfig::default(), + system: Arc::new(NgSystemConfig::default()), + quic: QuicConfig::default(), + tcp: TcpConfig::default(), + websocket: WebSocketConfig::default(), + http: HttpConfig::default(), + telemetry: TelemetryConfig::default(), + cluster: ClusterConfig::default(), + metadata: MetadataConfig::default(), + partition: PartitionConfig::default(), + message_bus: MessageBusConfig::default(), + } + } +} + +impl Default for ClusterConfig { + fn default() -> ClusterConfig { + ClusterConfig { + enabled: SERVER_NG_CONFIG.cluster.enabled, + name: SERVER_NG_CONFIG.cluster.name.parse().unwrap(), + heartbeat_timeout: SERVER_NG_CONFIG.cluster.heartbeat_timeout.parse().unwrap(), + commit_broadcast_interval: SERVER_NG_CONFIG + .cluster + .commit_broadcast_interval + .parse() + .unwrap(), + prepare_retransmit_interval: SERVER_NG_CONFIG + .cluster + .prepare_retransmit_interval + .parse() + .unwrap(), + view_change_retransmit_interval: SERVER_NG_CONFIG + .cluster + .view_change_retransmit_interval + .parse() + .unwrap(), + view_change_status_timeout: SERVER_NG_CONFIG + .cluster + .view_change_status_timeout + .parse() + .unwrap(), + request_start_view_retransmit_interval: SERVER_NG_CONFIG + .cluster + .request_start_view_retransmit_interval + .parse() + .unwrap(), + view_probe_attempts_max: SERVER_NG_CONFIG.cluster.view_probe_attempts_max as u32, + repair_retry_interval: SERVER_NG_CONFIG + .cluster + .repair_retry_interval + .parse() + .unwrap(), + repair_chunk_max: SERVER_NG_CONFIG.cluster.repair_chunk_max as usize, + nodes: SERVER_NG_CONFIG + .cluster + .nodes + .iter() + .map(|node| ClusterNodeConfig { + name: node.name.parse().unwrap(), + ip: node.ip.parse().unwrap(), + advertised_address: None, + advertised_addresses: Vec::new(), + replica_id: u8::try_from(node.replica_id).expect( + "static_toml replica_id must fit in u8 (0..=255); \ + fix core/server-ng/config.toml", + ), + ports: TransportPorts { + tcp: Some(u16::try_from(node.ports.tcp).expect( + "static_toml cluster.nodes.ports.tcp must fit in u16 (0..=65535); \ + fix core/server-ng/config.toml", + )), + quic: Some(u16::try_from(node.ports.quic).expect( + "static_toml cluster.nodes.ports.quic must fit in u16 (0..=65535); \ + fix core/server-ng/config.toml", + )), + http: Some(u16::try_from(node.ports.http).expect( + "static_toml cluster.nodes.ports.http must fit in u16 (0..=65535); \ + fix core/server-ng/config.toml", + )), + websocket: Some(u16::try_from(node.ports.websocket).expect( + "static_toml cluster.nodes.ports.websocket must fit in u16 (0..=65535); \ + fix core/server-ng/config.toml", + )), + tcp_replica: Some(u16::try_from(node.ports.tcp_replica).expect( + "static_toml cluster.nodes.ports.tcp_replica must fit in u16 (0..=65535); \ + fix core/server-ng/config.toml", + )), + }, + }) + .collect(), + auth: ClusterAuthConfig::default(), + tls: ClusterTlsConfig::default(), + } + } +} + +impl Default for MetadataConfig { + fn default() -> MetadataConfig { + // Read from the embedded TOML so the Default impl and the on-disk + // schema cannot drift (same pattern as MessageBusConfig below). + let metadata = &SERVER_NG_CONFIG.metadata; + MetadataConfig { + prepare_queue_depth: metadata.prepare_queue_depth as usize, + journal_slots: metadata.journal_slots as usize, + clients_table_max: metadata.clients_table_max as usize, + } + } +} + +impl Default for PartitionConfig { + fn default() -> PartitionConfig { + // Read from the embedded TOML so the Default impl and the on-disk + // schema cannot drift (same pattern as MetadataConfig above). + let partition = &SERVER_NG_CONFIG.partition; + PartitionConfig { + prepare_queue_depth: partition.prepare_queue_depth as usize, + evicted_ring_capacity: partition.evicted_ring_capacity as usize, + evicted_ring_bytes_max: partition.evicted_ring_bytes_max.parse().unwrap(), + transfer_served_cache_bytes_max: partition + .transfer_served_cache_bytes_max + .parse() + .unwrap(), + transfer_artifact_bytes_max: partition.transfer_artifact_bytes_max.parse().unwrap(), + } + } +} + +impl Default for QuicConfig { + fn default() -> QuicConfig { + QuicConfig { + enabled: SERVER_NG_CONFIG.quic.enabled, + address: SERVER_NG_CONFIG.quic.address.parse().unwrap(), + max_concurrent_bidi_streams: SERVER_NG_CONFIG.quic.max_concurrent_bidi_streams as u64, + datagram_send_buffer_size: SERVER_NG_CONFIG + .quic + .datagram_send_buffer_size + .parse() + .unwrap(), + initial_mtu: SERVER_NG_CONFIG.quic.initial_mtu.parse().unwrap(), + send_window: SERVER_NG_CONFIG.quic.send_window.parse().unwrap(), + receive_window: SERVER_NG_CONFIG.quic.receive_window.parse().unwrap(), + keep_alive_interval: SERVER_NG_CONFIG.quic.keep_alive_interval.parse().unwrap(), + max_idle_timeout: SERVER_NG_CONFIG.quic.max_idle_timeout.parse().unwrap(), + certificate: QuicCertificateConfig::default(), + socket: QuicSocketConfig::default(), + } + } +} + +impl Default for QuicSocketConfig { + fn default() -> QuicSocketConfig { + QuicSocketConfig { + override_defaults: SERVER_NG_CONFIG.quic.socket.override_defaults, + recv_buffer_size: SERVER_NG_CONFIG + .quic + .socket + .recv_buffer_size + .parse() + .unwrap(), + send_buffer_size: SERVER_NG_CONFIG + .quic + .socket + .send_buffer_size + .parse() + .unwrap(), + keepalive: SERVER_NG_CONFIG.quic.socket.keepalive, + } + } +} + +impl Default for QuicCertificateConfig { + fn default() -> QuicCertificateConfig { + QuicCertificateConfig { + self_signed: SERVER_NG_CONFIG.quic.certificate.self_signed, + cert_file: SERVER_NG_CONFIG.quic.certificate.cert_file.parse().unwrap(), + key_file: SERVER_NG_CONFIG.quic.certificate.key_file.parse().unwrap(), + } + } +} + +impl Default for TcpConfig { + fn default() -> TcpConfig { + TcpConfig { + enabled: SERVER_NG_CONFIG.tcp.enabled, + address: SERVER_NG_CONFIG.tcp.address.parse().unwrap(), + ipv6: SERVER_NG_CONFIG.tcp.ipv_6, + tls: TcpTlsConfig::default(), + socket: TcpSocketConfig::default(), + socket_migration: SERVER_NG_CONFIG.tcp.socket_migration, + } + } +} + +impl Default for TcpTlsConfig { + fn default() -> TcpTlsConfig { + TcpTlsConfig { + enabled: SERVER_NG_CONFIG.tcp.tls.enabled, + self_signed: SERVER_NG_CONFIG.tcp.tls.self_signed, + cert_file: SERVER_NG_CONFIG.tcp.tls.cert_file.parse().unwrap(), + key_file: SERVER_NG_CONFIG.tcp.tls.key_file.parse().unwrap(), + } + } +} + +impl Default for TcpSocketConfig { + fn default() -> TcpSocketConfig { + TcpSocketConfig { + override_defaults: SERVER_NG_CONFIG.tcp.socket.override_defaults, + recv_buffer_size: SERVER_NG_CONFIG + .tcp + .socket + .recv_buffer_size + .parse() + .unwrap(), + send_buffer_size: SERVER_NG_CONFIG + .tcp + .socket + .send_buffer_size + .parse() + .unwrap(), + keepalive: SERVER_NG_CONFIG.tcp.socket.keepalive, + nodelay: SERVER_NG_CONFIG.tcp.socket.nodelay, + linger: SERVER_NG_CONFIG.tcp.socket.linger.parse().unwrap(), + } + } +} + +impl Default for WebSocketConfig { + fn default() -> WebSocketConfig { + // The size knobs are optional in the schema (commented-out by + // default), so they map to `None` here when absent; every other + // field comes from the embedded TOML so the Default impl and + // the on-disk schema cannot drift. + WebSocketConfig { + enabled: SERVER_NG_CONFIG.websocket.enabled, + address: SERVER_NG_CONFIG.websocket.address.parse().unwrap(), + read_buffer_size: None, + write_buffer_size: None, + max_write_buffer_size: None, + max_message_size: None, + max_frame_size: None, + accept_unmasked_frames: SERVER_NG_CONFIG.websocket.accept_unmasked_frames, + tls: WebSocketTlsConfig::default(), + } + } +} + +impl Default for WebSocketTlsConfig { + fn default() -> WebSocketTlsConfig { + WebSocketTlsConfig { + enabled: SERVER_NG_CONFIG.websocket.tls.enabled, + self_signed: SERVER_NG_CONFIG.websocket.tls.self_signed, + cert_file: SERVER_NG_CONFIG.websocket.tls.cert_file.parse().unwrap(), + key_file: SERVER_NG_CONFIG.websocket.tls.key_file.parse().unwrap(), + } + } +} + +impl Default for MessageBusConfig { + fn default() -> MessageBusConfig { + // Read every field from the embedded TOML so the Default impl + // and the on-disk schema cannot drift. Sibling impls in this + // file follow the same pattern. + let bus = &SERVER_NG_CONFIG.message_bus; + MessageBusConfig { + max_batch: bus.max_batch as usize, + max_message_size: bus.max_message_size.parse().unwrap(), + peer_queue_capacity: bus.peer_queue_capacity as usize, + reconnect_period: bus.reconnect_period.parse().unwrap(), + close_peer_timeout: bus.close_peer_timeout.parse().unwrap(), + close_grace: bus.close_grace.parse().unwrap(), + handshake_grace: bus.handshake_grace.parse().unwrap(), + } + } +} diff --git a/core/configs/src/server_ng_config/displays.rs b/core/configs/src/server_ng_config/displays.rs new file mode 100644 index 0000000000..ab487129b6 --- /dev/null +++ b/core/configs/src/server_ng_config/displays.rs @@ -0,0 +1,189 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! `Display` impls for the server-ng config surface. +//! +//! Reused section types pick up [`Display`] from +//! [`crate::displays`]; this module only adds the top-level +//! [`ServerNgConfig`] formatter and the new [`MessageBusConfig`] +//! section formatter. + +use super::message_bus::MessageBusConfig; +use super::metadata::MetadataConfig; +use super::partition::PartitionConfig; +use super::quic::{QuicCertificateConfig, QuicConfig, QuicSocketConfig}; +use super::server_ng::{ExtraConfig, NamespaceConfig, ServerNgConfig}; +use super::tcp::{TcpConfig, TcpSocketConfig, TcpTlsConfig}; +use std::fmt::{Display, Formatter}; + +impl Display for ServerNgConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ consumer_group: {}, data_maintenance: {}, extra: {}, message_saver: {}, \ + heartbeat: {}, system: {}, quic: {}, tcp: {}, http: {}, telemetry: {}, \ + metadata: {}, message_bus: {}, partition: {} }}", + self.consumer_group, + self.data_maintenance, + self.extra, + self.message_saver, + self.heartbeat, + self.system, + self.quic, + self.tcp, + self.http, + self.telemetry, + self.metadata, + self.message_bus, + self.partition, + ) + } +} + +impl Display for PartitionConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ prepare_queue_depth: {}, evicted_ring_capacity: {}, \ + evicted_ring_bytes_max: {}, transfer_served_cache_bytes_max: {}, \ + transfer_artifact_bytes_max: {} }}", + self.prepare_queue_depth, + self.evicted_ring_capacity, + self.evicted_ring_bytes_max, + self.transfer_served_cache_bytes_max, + self.transfer_artifact_bytes_max, + ) + } +} + +impl Display for MetadataConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ prepare_queue_depth: {}, journal_slots: {}, clients_table_max: {} }}", + self.prepare_queue_depth, self.journal_slots, self.clients_table_max, + ) + } +} + +impl Display for MessageBusConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ max_batch: {}, max_message_size: {}, peer_queue_capacity: {}, \ + reconnect_period: {}, close_peer_timeout: {}, close_grace: {}, \ + handshake_grace: {} }}", + self.max_batch, + self.max_message_size, + self.peer_queue_capacity, + self.reconnect_period, + self.close_peer_timeout, + self.close_grace, + self.handshake_grace, + ) + } +} + +impl Display for ExtraConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{{ namespace: {} }}", self.namespace) + } +} + +impl Display for NamespaceConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ max_streams: {}, max_topics: {}, max_partitions: {} }}", + self.max_streams, self.max_topics, self.max_partitions + ) + } +} + +impl Display for TcpConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ enabled: {}, address: {}, ipv6: {}, tls: {}, socket: {}, socket_migration: {} }}", + self.enabled, self.address, self.ipv6, self.tls, self.socket, self.socket_migration + ) + } +} + +impl Display for TcpTlsConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ enabled: {}, self_signed: {}, cert_file: {}, key_file: {} }}", + self.enabled, self.self_signed, self.cert_file, self.key_file + ) + } +} + +impl Display for TcpSocketConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ override_defaults: {}, recv_buffer_size: {}, send_buffer_size: {}, keepalive: {}, nodelay: {}, linger: {} }}", + self.override_defaults, + self.recv_buffer_size, + self.send_buffer_size, + self.keepalive, + self.nodelay, + self.linger, + ) + } +} + +impl Display for QuicConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ enabled: {}, address: {}, max_concurrent_bidi_streams: {}, datagram_send_buffer_size: {}, initial_mtu: {}, send_window: {}, receive_window: {}, keep_alive_interval: {}, max_idle_timeout: {}, certificate: {} }}", + self.enabled, + self.address, + self.max_concurrent_bidi_streams, + self.datagram_send_buffer_size, + self.initial_mtu, + self.send_window, + self.receive_window, + self.keep_alive_interval, + self.max_idle_timeout, + self.certificate + ) + } +} + +impl Display for QuicCertificateConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ self_signed: {}, cert_file: {}, key_file: {} }}", + self.self_signed, self.cert_file, self.key_file + ) + } +} + +impl Display for QuicSocketConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ override_defaults: {}, recv_buffer_size: {}, send_buffer_size: {}, keepalive: {} }}", + self.override_defaults, self.recv_buffer_size, self.send_buffer_size, self.keepalive + ) + } +} diff --git a/core/configs/src/server_config/message_bus.rs b/core/configs/src/server_ng_config/message_bus.rs similarity index 87% rename from core/configs/src/server_config/message_bus.rs rename to core/configs/src/server_ng_config/message_bus.rs index 91a84d6a01..3fcd2b1615 100644 --- a/core/configs/src/server_config/message_bus.rs +++ b/core/configs/src/server_ng_config/message_bus.rs @@ -29,13 +29,13 @@ //! and `[websocket]` + `[websocket.tls]` respectively. //! - WebSocket frame-layer tuning (buffer sizes, message / frame //! ceilings, unmasked-frame acceptance) lives in `[websocket]` -//! ([`super::websocket::WebSocketConfig`]): the bus IS the server's +//! ([`super::websocket::WebSocketConfig`]): the bus IS server-ng's //! WS / WSS install path, so the listener section carries the frame //! tuning and the runtime folds it into a compio-ws //! `WebSocketConfig` once at bus construction. The //! `websocket.max_*` <= `message_bus.max_message_size` chain is //! enforced as a cross-section check in -//! [`super::server::ServerConfig`]'s validator. +//! [`super::server_ng::ServerNgConfig`]'s validator. //! //! Tunables the bus owns directly: bus-internal abstractions the //! operator does not see anywhere else in the schema (batch sizing, @@ -48,10 +48,10 @@ //! this section. //! //! Construction of the runtime type from this struct happens in the -//! follow-up PR that wires `core/server` to call -//! [`super::server::ServerConfig::load`]. +//! follow-up PR that wires `core/server-ng` to call +//! [`super::server_ng::ServerNgConfig::load`]. -use super::COMPONENT; +use super::COMPONENT_NG; use crate::ConfigurationError; use configs::ConfigEnv; use iggy_common::{IggyByteSize, IggyDuration, Validatable}; @@ -68,7 +68,7 @@ use serde_with::{DisplayFromStr, serde_as}; /// `IggyMessageBus::with_config`. A unit test below pins the literal so /// any future bump on the runtime side surfaces as a configs-build /// failure until both are reconciled. -pub const IOV_MAX_LIMIT: usize = 512; +pub const IOV_MAX_LIMIT_NG: usize = 512; /// Tunables for the message bus that ships consensus traffic between /// replicas and SDK-client traffic between shards. @@ -77,7 +77,7 @@ pub const IOV_MAX_LIMIT: usize = 512; pub struct MessageBusConfig { /// Maximum number of `BusMessage` entries the writer task coalesces /// into a single `writev(2)` call. Higher values amortise syscalls - /// at the cost of tail latency. Capped at [`IOV_MAX_LIMIT`]. + /// at the cost of tail latency. Capped at [`IOV_MAX_LIMIT_NG`]. pub max_batch: usize, /// Wire-level cap on a single framed message. Read-side validator; @@ -127,38 +127,38 @@ pub struct MessageBusConfig { impl Validatable for MessageBusConfig { fn validate(&self) -> Result<(), ConfigurationError> { if self.max_batch == 0 { - eprintln!("{COMPONENT} message_bus.max_batch must be > 0"); + eprintln!("{COMPONENT_NG} message_bus.max_batch must be > 0"); return Err(ConfigurationError::InvalidConfigurationValue); } - if self.max_batch > IOV_MAX_LIMIT { + if self.max_batch > IOV_MAX_LIMIT_NG { eprintln!( - "{COMPONENT} message_bus.max_batch ({}) exceeds IOV_MAX_LIMIT ({IOV_MAX_LIMIT})", + "{COMPONENT_NG} message_bus.max_batch ({}) exceeds IOV_MAX_LIMIT ({IOV_MAX_LIMIT_NG})", self.max_batch ); return Err(ConfigurationError::InvalidConfigurationValue); } if self.peer_queue_capacity == 0 { - eprintln!("{COMPONENT} message_bus.peer_queue_capacity must be > 0"); + eprintln!("{COMPONENT_NG} message_bus.peer_queue_capacity must be > 0"); return Err(ConfigurationError::InvalidConfigurationValue); } if self.max_message_size.as_bytes_u64() == 0 { - eprintln!("{COMPONENT} message_bus.max_message_size must be > 0"); + eprintln!("{COMPONENT_NG} message_bus.max_message_size must be > 0"); return Err(ConfigurationError::InvalidConfigurationValue); } if self.handshake_grace.as_micros() == 0 { - eprintln!("{COMPONENT} message_bus.handshake_grace must be > 0"); + eprintln!("{COMPONENT_NG} message_bus.handshake_grace must be > 0"); return Err(ConfigurationError::InvalidConfigurationValue); } if self.close_grace.as_micros() == 0 { - eprintln!("{COMPONENT} message_bus.close_grace must be > 0"); + eprintln!("{COMPONENT_NG} message_bus.close_grace must be > 0"); return Err(ConfigurationError::InvalidConfigurationValue); } if self.close_peer_timeout.as_micros() == 0 { - eprintln!("{COMPONENT} message_bus.close_peer_timeout must be > 0"); + eprintln!("{COMPONENT_NG} message_bus.close_peer_timeout must be > 0"); return Err(ConfigurationError::InvalidConfigurationValue); } if self.reconnect_period.as_micros() == 0 { - eprintln!("{COMPONENT} message_bus.reconnect_period must be > 0"); + eprintln!("{COMPONENT_NG} message_bus.reconnect_period must be > 0"); return Err(ConfigurationError::InvalidConfigurationValue); } Ok(()) @@ -188,14 +188,14 @@ mod tests { #[test] fn rejects_max_batch_above_iov_max() { let mut c = baseline(); - c.max_batch = IOV_MAX_LIMIT + 1; + c.max_batch = IOV_MAX_LIMIT_NG + 1; assert!(c.validate().is_err()); } #[test] fn accepts_max_batch_at_iov_max() { let mut c = baseline(); - c.max_batch = IOV_MAX_LIMIT; + c.max_batch = IOV_MAX_LIMIT_NG; assert!(c.validate().is_ok()); } @@ -220,7 +220,7 @@ mod tests { /// `core/configs` does not depend on `core/message_bus`. #[test] fn iov_max_limit_matches_runtime_crate() { - assert_eq!(IOV_MAX_LIMIT, 512); + assert_eq!(IOV_MAX_LIMIT_NG, 512); } #[test] diff --git a/core/configs/src/server_config/metadata.rs b/core/configs/src/server_ng_config/metadata.rs similarity index 74% rename from core/configs/src/server_config/metadata.rs rename to core/configs/src/server_ng_config/metadata.rs index 2ba8d821fb..f38840f893 100644 --- a/core/configs/src/server_config/metadata.rs +++ b/core/configs/src/server_ng_config/metadata.rs @@ -28,7 +28,7 @@ //! between forced checkpoints) //! - `clients_table_max` -> `consensus::CLIENTS_TABLE_MAX` (the VSR //! client-table slot count; independent of the two above). The -//! HTTP session cap tracks it at half. +//! server-ng HTTP session cap tracks it at half. //! //! The first two interlock through the forced-checkpoint margin //! (`max(64, prepare_queue_depth)` at bootstrap): while a checkpoint @@ -39,11 +39,11 @@ //! The defaults are duplicated literals rather than imports so //! `core/configs` does not grow build-time edges onto `core/consensus` //! and `core/journal` (the runtime crates are the consumers of this -//! config, mirroring the `IOV_MAX_LIMIT` precedent in -//! [`super::message_bus`]). `core/server`'s bootstrap pins these +//! config, mirroring the `IOV_MAX_LIMIT_NG` precedent in +//! [`super::message_bus`]). `core/server-ng`'s bootstrap pins these //! literals against the runtime constants with static asserts. -use super::COMPONENT; +use super::COMPONENT_NG; use crate::ConfigurationError; use configs::ConfigEnv; use iggy_common::Validatable; @@ -60,15 +60,10 @@ pub const DEFAULT_METADATA_JOURNAL_SLOTS: usize = 1024; /// margin is `max(this, prepare_queue_depth)`. pub const METADATA_CHECKPOINT_MARGIN_FLOOR: usize = 64; -/// Upper bound on `prepare_queue_depth`. -/// -/// Pinned by the view-change wire format, not by memory: a `DoViewChange` carries -/// the sender's uncommitted suffix plus one nack bit and one present bit per entry, -/// each bitset a single `u128` (`consensus::DVC_HEADERS_MAX` = 128). The suffix -/// spans `commit_max..=op`, which this depth bounds, so a deeper queue produces -/// entries the new primary can neither adopt nor prove dead. The reserved head slot -/// leaves room for the head op. -pub const MAX_METADATA_PREPARE_QUEUE_DEPTH: usize = 127; +/// Upper bound on `prepare_queue_depth`. Every queued prepare pins a +/// full message buffer; four thousand in-flight metadata ops is far past +/// any sane deployment and a likely unit typo. +pub const MAX_METADATA_PREPARE_QUEUE_DEPTH: usize = 4096; /// Upper bound on `journal_slots`. Each slot costs index memory and every /// checkpoint rewrites the live WAL suffix; a million slots is the sanity @@ -78,7 +73,7 @@ pub const MAX_METADATA_JOURNAL_SLOTS: usize = 1 << 20; /// Mirrors `consensus::CLIENTS_TABLE_MAX`, the VSR client-table slot count. pub const DEFAULT_METADATA_CLIENTS_TABLE_MAX: usize = 8192; -/// Floor on `clients_table_max`. The HTTP session cap derives as +/// Floor on `clients_table_max`. The server-ng HTTP session cap derives as /// `clients_table_max / 2`; below two that floors to zero and HTTP could /// register no sessions at all. pub const MIN_METADATA_CLIENTS_TABLE_MAX: usize = 2; @@ -106,7 +101,7 @@ pub struct MetadataConfig { /// Slot count of the VSR client table: how many distinct clients /// (TCP/QUIC/WS virtual clients and HTTP sessions together) hold live /// session state before the oldest-committed entry is evicted. The - /// HTTP session cap tracks this at half, so raising it lifts + /// server-ng HTTP session cap tracks this at half, so raising it lifts /// both. pub clients_table_max: usize, } @@ -127,23 +122,19 @@ impl MetadataConfig { impl Validatable for MetadataConfig { fn validate(&self) -> Result<(), ConfigurationError> { if self.prepare_queue_depth == 0 { - eprintln!("{COMPONENT} metadata.prepare_queue_depth must be > 0"); + eprintln!("{COMPONENT_NG} metadata.prepare_queue_depth must be > 0"); return Err(ConfigurationError::InvalidConfigurationValue); } if self.prepare_queue_depth > MAX_METADATA_PREPARE_QUEUE_DEPTH { eprintln!( - "{COMPONENT} metadata.prepare_queue_depth ({}) exceeds the maximum \ - ({MAX_METADATA_PREPARE_QUEUE_DEPTH}). The ceiling is the view-change wire, not memory: \ - a DoViewChange describes the uncommitted suffix with one bit per op in a u128 \ - bitset, and this depth bounds that suffix. Deeper produces entries a new \ - primary can neither adopt nor prove dead. Lowered from 256; not raisable.", + "{COMPONENT_NG} metadata.prepare_queue_depth ({}) exceeds the maximum ({MAX_METADATA_PREPARE_QUEUE_DEPTH})", self.prepare_queue_depth ); return Err(ConfigurationError::InvalidConfigurationValue); } if self.journal_slots > MAX_METADATA_JOURNAL_SLOTS { eprintln!( - "{COMPONENT} metadata.journal_slots ({}) exceeds the maximum ({MAX_METADATA_JOURNAL_SLOTS})", + "{COMPONENT_NG} metadata.journal_slots ({}) exceeds the maximum ({MAX_METADATA_JOURNAL_SLOTS})", self.journal_slots ); return Err(ConfigurationError::InvalidConfigurationValue); @@ -155,21 +146,21 @@ impl Validatable for MetadataConfig { let min_slots = 4 * self.checkpoint_margin(); if self.journal_slots < min_slots { eprintln!( - "{COMPONENT} metadata.journal_slots ({}) must be >= 4 * max({METADATA_CHECKPOINT_MARGIN_FLOOR}, prepare_queue_depth) = {min_slots}", + "{COMPONENT_NG} metadata.journal_slots ({}) must be >= 4 * max({METADATA_CHECKPOINT_MARGIN_FLOOR}, prepare_queue_depth) = {min_slots}", self.journal_slots ); return Err(ConfigurationError::InvalidConfigurationValue); } if self.clients_table_max < MIN_METADATA_CLIENTS_TABLE_MAX { eprintln!( - "{COMPONENT} metadata.clients_table_max ({}) must be >= {MIN_METADATA_CLIENTS_TABLE_MAX}", + "{COMPONENT_NG} metadata.clients_table_max ({}) must be >= {MIN_METADATA_CLIENTS_TABLE_MAX}", self.clients_table_max ); return Err(ConfigurationError::InvalidConfigurationValue); } if self.clients_table_max > MAX_METADATA_CLIENTS_TABLE_MAX { eprintln!( - "{COMPONENT} metadata.clients_table_max ({}) exceeds the maximum ({MAX_METADATA_CLIENTS_TABLE_MAX})", + "{COMPONENT_NG} metadata.clients_table_max ({}) exceeds the maximum ({MAX_METADATA_CLIENTS_TABLE_MAX})", self.clients_table_max ); return Err(ConfigurationError::InvalidConfigurationValue); @@ -196,52 +187,33 @@ mod tests { #[test] fn margin_tracks_deep_prepare_queue() { let config = MetadataConfig { - prepare_queue_depth: MAX_METADATA_PREPARE_QUEUE_DEPTH, + prepare_queue_depth: 256, journal_slots: 4096, clients_table_max: DEFAULT_METADATA_CLIENTS_TABLE_MAX, }; assert!(config.validate().is_ok()); - assert_eq!(config.checkpoint_margin(), MAX_METADATA_PREPARE_QUEUE_DEPTH); + assert_eq!(config.checkpoint_margin(), 256); } #[test] fn journal_must_outsize_margin() { - // Deepest permitted queue: margin becomes the depth, and the journal - // must hold 4x that. At exactly 4x the boundary is accepted... - let min_slots = 4 * MAX_METADATA_PREPARE_QUEUE_DEPTH; + // Deep queue, journal kept at the old default: margin becomes 256, + // 4 * 256 = 1024 == journal_slots, boundary accepted... let boundary = MetadataConfig { - prepare_queue_depth: MAX_METADATA_PREPARE_QUEUE_DEPTH, - journal_slots: min_slots, + prepare_queue_depth: 256, + journal_slots: 1024, clients_table_max: DEFAULT_METADATA_CLIENTS_TABLE_MAX, }; assert!(boundary.validate().is_ok()); // ...one slot fewer is refused. let starved = MetadataConfig { - prepare_queue_depth: MAX_METADATA_PREPARE_QUEUE_DEPTH, - journal_slots: min_slots - 1, + prepare_queue_depth: 256, + journal_slots: 1023, clients_table_max: DEFAULT_METADATA_CLIENTS_TABLE_MAX, }; assert!(starved.validate().is_err()); } - #[test] - fn prepare_queue_depth_capped_by_view_change_bitset_width() { - // Not a memory guard: it keeps every uncommitted suffix entry addressable by - // a `u128` bitset in a `DoViewChange`. One past it must be refused, or a view - // change meets an entry it can neither adopt nor prove dead. - let over = MetadataConfig { - prepare_queue_depth: MAX_METADATA_PREPARE_QUEUE_DEPTH + 1, - journal_slots: MAX_METADATA_JOURNAL_SLOTS, - clients_table_max: DEFAULT_METADATA_CLIENTS_TABLE_MAX, - }; - assert!(over.validate().is_err()); - assert_eq!( - MAX_METADATA_PREPARE_QUEUE_DEPTH + 1, - 128, - "cap must leave the head op a slot inside the 128-bit bitset" - ); - } - #[test] fn zero_depth_is_refused() { let config = MetadataConfig { diff --git a/core/configs/src/server_ng_config/mod.rs b/core/configs/src/server_ng_config/mod.rs new file mode 100644 index 0000000000..a1b9b774eb --- /dev/null +++ b/core/configs/src/server_ng_config/mod.rs @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! On-disk schema for the `server-ng` binary. +//! +//! Mirrors the legacy server-config section surface verbatim +//! (operator-facing schema is unchanged) and adds a `message_bus` section +//! for inter-shard / inter-replica bus tunables previously hardcoded in +//! the `core/message_bus` runtime crate. +//! +//! Scaffolding-only at the time of introduction: the type is defined and +//! loadable but no binary calls [`server_ng::ServerNgConfig::load`]; the +//! wiring PR for `core/server-ng`'s bootstrap and the message_bus crate's +//! runtime type is a separate change. + +pub mod cluster; +pub mod defaults; +pub mod displays; +pub mod message_bus; +pub mod metadata; +pub mod partition; +pub mod quic; +pub mod server_ng; +pub mod sharding; +pub mod tcp; +pub mod validators; +pub mod websocket; + +/// Component tag used in error messages for the server-ng config surface. +/// Mirrors [`crate::COMPONENT`] (`"CONFIG"`). +pub const COMPONENT_NG: &str = "CONFIG_NG"; diff --git a/core/configs/src/server_config/partition.rs b/core/configs/src/server_ng_config/partition.rs similarity index 81% rename from core/configs/src/server_config/partition.rs rename to core/configs/src/server_ng_config/partition.rs index 37a5933858..011e82a9b8 100644 --- a/core/configs/src/server_config/partition.rs +++ b/core/configs/src/server_ng_config/partition.rs @@ -27,18 +27,18 @@ //! (the per-partition journal-repair retention ring's dual ceilings) //! //! Distinct from `[metadata]` (a single, shard-0-global VSR plane) because -//! partition pipelines exist PER PARTITION: the request queue (`depth * 2` slots) -//! pins full inbound produce batches, so pinned memory scales with the partition -//! count. The default mirrors the runtime constant; the ceiling matches metadata's, -//! since both planes ship the same `DoViewChange` suffix over the same bitsets (see -//! [`MAX_PARTITION_PREPARE_QUEUE_DEPTH`]). +//! partition pipelines exist PER PARTITION. The default mirrors the runtime +//! constant so a default deployment is byte-identical; the ceiling is far +//! below metadata's because the request queue (`depth * 2` slots) pins full +//! inbound produce batches, so pinned memory scales with the partition count +//! (see [`MAX_PARTITION_PREPARE_QUEUE_DEPTH`]). //! //! The default is a duplicated literal rather than an import so //! `core/configs` does not grow a build-time edge onto `core/consensus` -//! (mirroring [`super::metadata`]). `core/server`'s bootstrap pins the +//! (mirroring [`super::metadata`]). `core/server-ng`'s bootstrap pins the //! literal against the runtime constant with a static assert. -use super::COMPONENT; +use super::COMPONENT_NG; use crate::ConfigurationError; use configs::ConfigEnv; use iggy_common::{IggyByteSize, Validatable}; @@ -47,19 +47,13 @@ use serde::{Deserialize, Serialize}; /// Mirrors `consensus::PIPELINE_PREPARE_QUEUE_MAX`. pub const DEFAULT_PARTITION_PREPARE_QUEUE_DEPTH: usize = 32; -/// Upper bound on `prepare_queue_depth`. -/// -/// Pinned by the view-change wire format, and equal to -/// [`super::metadata::MAX_METADATA_PREPARE_QUEUE_DEPTH`] for that reason: a -/// `DoViewChange` carries the sender's uncommitted suffix spanning `commit..=op` -/// with one nack bit and one present bit per entry, each bitset a single `u128` -/// (`consensus::DVC_HEADERS_MAX` = 128). This depth bounds `op - commit`, so a -/// deeper queue produces entries the new primary can neither adopt nor prove dead. -/// The reserved head slot leaves room for the head op. -/// -/// The memory bound (`depth * 2 * partition_count * batch_size` of pinned produce -/// batches) still holds and is looser, so the wire is what decides. -pub const MAX_PARTITION_PREPARE_QUEUE_DEPTH: usize = 127; +/// Upper bound on `prepare_queue_depth`. Unlike the single metadata pipeline, +/// a pipeline exists per partition, and each queued request pins a full +/// inbound produce batch (a 4 KiB floor up to megabytes). Worst-case pinned +/// memory therefore scales as `depth * 2 * partition_count * batch_size`, so +/// this ceiling sits far below metadata's 4096: it is a typo guard, not a +/// sizing endorsement. +pub const MAX_PARTITION_PREPARE_QUEUE_DEPTH: usize = 256; /// Mirrors the free const `shard::PARTITION_ARTIFACT_LEN_DEFAULT` (segment /// ceiling plus the one whole batch a segment may close past it). @@ -143,39 +137,35 @@ pub struct PartitionConfig { impl Validatable for PartitionConfig { fn validate(&self) -> Result<(), ConfigurationError> { if self.prepare_queue_depth == 0 { - eprintln!("{COMPONENT} partition.prepare_queue_depth must be > 0"); + eprintln!("{COMPONENT_NG} partition.prepare_queue_depth must be > 0"); return Err(ConfigurationError::InvalidConfigurationValue); } if self.prepare_queue_depth > MAX_PARTITION_PREPARE_QUEUE_DEPTH { eprintln!( - "{COMPONENT} partition.prepare_queue_depth ({}) exceeds the maximum \ - ({MAX_PARTITION_PREPARE_QUEUE_DEPTH}). The ceiling is the view-change wire, not memory: \ - a DoViewChange describes the uncommitted suffix with one bit per op in a u128 \ - bitset, and this depth bounds that suffix. Deeper produces entries a new \ - primary can neither adopt nor prove dead. Lowered from 256; not raisable.", + "{COMPONENT_NG} partition.prepare_queue_depth ({}) exceeds the maximum ({MAX_PARTITION_PREPARE_QUEUE_DEPTH})", self.prepare_queue_depth ); return Err(ConfigurationError::InvalidConfigurationValue); } if self.evicted_ring_capacity == 0 { - eprintln!("{COMPONENT} partition.evicted_ring_capacity must be > 0"); + eprintln!("{COMPONENT_NG} partition.evicted_ring_capacity must be > 0"); return Err(ConfigurationError::InvalidConfigurationValue); } if self.evicted_ring_capacity > MAX_EVICTED_RING_CAPACITY { eprintln!( - "{COMPONENT} partition.evicted_ring_capacity ({}) exceeds the maximum ({MAX_EVICTED_RING_CAPACITY})", + "{COMPONENT_NG} partition.evicted_ring_capacity ({}) exceeds the maximum ({MAX_EVICTED_RING_CAPACITY})", self.evicted_ring_capacity ); return Err(ConfigurationError::InvalidConfigurationValue); } // The FLOOR on `transfer_artifact_bytes_max` cannot live here (it needs // `system.segment.size` and the bus cap); it is enforced in the - // `ServerConfig` validator, which is what turns that misconfiguration + // `ServerNgConfig` validator, which is what turns that misconfiguration // into a boot error instead of a silent per-partition rejoin livelock. let served_cache = self.transfer_served_cache_bytes_max.as_bytes_u64(); if served_cache == 0 || served_cache > MAX_TRANSFER_BYTES { eprintln!( - "{COMPONENT} partition.transfer_served_cache_bytes_max ({served_cache} bytes) \ + "{COMPONENT_NG} partition.transfer_served_cache_bytes_max ({served_cache} bytes) \ must be > 0 and <= {MAX_TRANSFER_BYTES} bytes" ); return Err(ConfigurationError::InvalidConfigurationValue); @@ -183,19 +173,19 @@ impl Validatable for PartitionConfig { let artifact_bytes = self.transfer_artifact_bytes_max.as_bytes_u64(); if artifact_bytes == 0 || artifact_bytes > MAX_TRANSFER_BYTES { eprintln!( - "{COMPONENT} partition.transfer_artifact_bytes_max ({artifact_bytes} bytes) \ + "{COMPONENT_NG} partition.transfer_artifact_bytes_max ({artifact_bytes} bytes) \ must be > 0 and <= {MAX_TRANSFER_BYTES} bytes" ); return Err(ConfigurationError::InvalidConfigurationValue); } let ring_bytes = self.evicted_ring_bytes_max.as_bytes_u64(); if ring_bytes == 0 { - eprintln!("{COMPONENT} partition.evicted_ring_bytes_max must be > 0"); + eprintln!("{COMPONENT_NG} partition.evicted_ring_bytes_max must be > 0"); return Err(ConfigurationError::InvalidConfigurationValue); } if ring_bytes > MAX_EVICTED_RING_BYTES { eprintln!( - "{COMPONENT} partition.evicted_ring_bytes_max ({ring_bytes} bytes) exceeds the maximum ({MAX_EVICTED_RING_BYTES} bytes)" + "{COMPONENT_NG} partition.evicted_ring_bytes_max ({ring_bytes} bytes) exceeds the maximum ({MAX_EVICTED_RING_BYTES} bytes)" ); return Err(ConfigurationError::InvalidConfigurationValue); } diff --git a/core/configs/src/server_ng_config/quic.rs b/core/configs/src/server_ng_config/quic.rs new file mode 100644 index 0000000000..d86402921c --- /dev/null +++ b/core/configs/src/server_ng_config/quic.rs @@ -0,0 +1,222 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Server-ng QUIC listener schema. +//! +//! Field shape mirrors the legacy [`crate::quic::QuicConfig`] verbatim; +//! the type is forked into `server_ng_config` so server-ng can evolve +//! its QUIC surface independently of the legacy server. No semantic +//! change at fork time. + +use super::COMPONENT_NG; +use crate::ConfigurationError; +use configs::ConfigEnv; +use iggy_common::{IggyByteSize, IggyDuration, Validatable}; +use serde::{Deserialize, Serialize}; +use serde_with::DisplayFromStr; +use serde_with::serde_as; + +#[serde_as] +#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] +pub struct QuicConfig { + pub enabled: bool, + pub address: String, + pub max_concurrent_bidi_streams: u64, + #[config_env(leaf)] + pub datagram_send_buffer_size: IggyByteSize, + #[config_env(leaf)] + pub initial_mtu: IggyByteSize, + #[config_env(leaf)] + pub send_window: IggyByteSize, + #[config_env(leaf)] + pub receive_window: IggyByteSize, + #[config_env(leaf)] + #[serde_as(as = "DisplayFromStr")] + pub keep_alive_interval: IggyDuration, + #[config_env(leaf)] + #[serde_as(as = "DisplayFromStr")] + pub max_idle_timeout: IggyDuration, + pub certificate: QuicCertificateConfig, + pub socket: QuicSocketConfig, +} + +#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] +pub struct QuicSocketConfig { + pub override_defaults: bool, + #[config_env(leaf)] + pub recv_buffer_size: IggyByteSize, + #[config_env(leaf)] + pub send_buffer_size: IggyByteSize, + pub keepalive: bool, +} + +#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] +pub struct QuicCertificateConfig { + pub self_signed: bool, + pub cert_file: String, + pub key_file: String, +} + +/// Validates the field range constraints the runtime conversion in +/// `core::message_bus::config::build_quic_tuning` previously enforced +/// via `expect(...)`. Surfacing them here turns boot-time misconfig +/// into a `ConfigurationError` instead of a panic in the bus crate. +impl Validatable for QuicConfig { + fn validate(&self) -> Result<(), ConfigurationError> { + // QUIC requires at least one bidi stream per connection. + if self.max_concurrent_bidi_streams == 0 { + eprintln!("{COMPONENT_NG} quic.max_concurrent_bidi_streams must be >= 1"); + return Err(ConfigurationError::InvalidConfigurationValue); + } + // quinn-proto stores stream counts as u32 internally. + if u32::try_from(self.max_concurrent_bidi_streams).is_err() { + eprintln!( + "{COMPONENT_NG} quic.max_concurrent_bidi_streams ({}) does not fit in u32", + self.max_concurrent_bidi_streams + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + // The datagram send buffer is materialized as a `Vec` of this + // length in compio-quic, so it must fit in `usize` on the target + // platform. + if usize::try_from(self.datagram_send_buffer_size.as_bytes_u64()).is_err() { + eprintln!( + "{COMPONENT_NG} quic.datagram_send_buffer_size ({} bytes) does not fit in usize on this target", + self.datagram_send_buffer_size.as_bytes_u64() + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + // RFC 9000 §14: minimum required MTU is 1200 bytes; quinn stores + // initial_mtu as u16 (max 65535). + let initial_mtu = self.initial_mtu.as_bytes_u64(); + if initial_mtu < 1200 { + eprintln!( + "{COMPONENT_NG} quic.initial_mtu ({initial_mtu}) is below the QUIC minimum of 1200 bytes", + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if u16::try_from(initial_mtu).is_err() { + eprintln!("{COMPONENT_NG} quic.initial_mtu ({initial_mtu}) exceeds u16::MAX (65535)",); + return Err(ConfigurationError::InvalidConfigurationValue); + } + // quinn VarInt for `receive_window` accepts u32; rejecting + // out-of-range values here surfaces a config error rather than + // panicking inside the bus crate's runtime conversion. + if u32::try_from(self.receive_window.as_bytes_u64()).is_err() { + eprintln!( + "{COMPONENT_NG} quic.receive_window ({} bytes) does not fit in u32 (quinn VarInt limit)", + self.receive_window.as_bytes_u64() + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + // `send_window` is u64-sized in QuicTuning, but quinn's VarInt + // protocol-level cap is 2^62 - 1; reject anything above that. + const QUINN_VARINT_MAX: u64 = (1u64 << 62) - 1; + if self.send_window.as_bytes_u64() > QUINN_VARINT_MAX { + eprintln!( + "{COMPONENT_NG} quic.send_window ({} bytes) exceeds quinn VarInt max (2^62 - 1)", + self.send_window.as_bytes_u64() + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn baseline() -> QuicConfig { + QuicConfig { + enabled: false, + address: String::new(), + max_concurrent_bidi_streams: 1, + datagram_send_buffer_size: IggyByteSize::from(100_u64 * 1024), + initial_mtu: IggyByteSize::from(8_u64 * 1024), + send_window: IggyByteSize::from(64_u64 * 1024 * 1024), + receive_window: IggyByteSize::from(64_u64 * 1024 * 1024), + keep_alive_interval: IggyDuration::from(std::time::Duration::from_secs(10)), + max_idle_timeout: IggyDuration::from(std::time::Duration::from_secs(30)), + certificate: QuicCertificateConfig { + self_signed: false, + cert_file: String::new(), + key_file: String::new(), + }, + socket: QuicSocketConfig { + override_defaults: false, + recv_buffer_size: IggyByteSize::from(0_u64), + send_buffer_size: IggyByteSize::from(0_u64), + keepalive: false, + }, + } + } + + #[test] + fn baseline_validates() { + baseline().validate().expect("baseline is valid"); + } + + #[test] + fn rejects_zero_max_concurrent_bidi_streams() { + let mut c = baseline(); + c.max_concurrent_bidi_streams = 0; + assert!(c.validate().is_err()); + } + + #[test] + fn rejects_max_concurrent_bidi_streams_above_u32() { + let mut c = baseline(); + c.max_concurrent_bidi_streams = u64::from(u32::MAX) + 1; + assert!(c.validate().is_err()); + } + + #[test] + fn rejects_initial_mtu_below_qiuc_minimum() { + let mut c = baseline(); + c.initial_mtu = IggyByteSize::from(1199_u64); + assert!(c.validate().is_err()); + } + + #[test] + fn rejects_initial_mtu_above_u16() { + let mut c = baseline(); + c.initial_mtu = IggyByteSize::from(u64::from(u16::MAX) + 1); + assert!(c.validate().is_err()); + } + + #[test] + fn rejects_receive_window_above_u32() { + let mut c = baseline(); + c.receive_window = IggyByteSize::from(u64::from(u32::MAX) + 1); + assert!(c.validate().is_err()); + } + + #[test] + fn rejects_send_window_above_quinn_varint_max() { + let mut c = baseline(); + c.send_window = IggyByteSize::from(1_u64 << 62); + assert!(c.validate().is_err()); + } + + #[test] + fn accepts_initial_mtu_at_quic_minimum() { + let mut c = baseline(); + c.initial_mtu = IggyByteSize::from(1200_u64); + assert!(c.validate().is_ok()); + } +} diff --git a/core/configs/src/server_ng_config/server_ng.rs b/core/configs/src/server_ng_config/server_ng.rs new file mode 100644 index 0000000000..3484d5f36c --- /dev/null +++ b/core/configs/src/server_ng_config/server_ng.rs @@ -0,0 +1,229 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::COMPONENT_NG; +use super::cluster::ClusterConfig; +use super::message_bus::MessageBusConfig; +use super::metadata::MetadataConfig; +use super::partition::PartitionConfig; +use super::quic::QuicConfig; +use super::tcp::TcpConfig; +use super::websocket::WebSocketConfig; +use crate::ConfigurationError; +use crate::server_config::http::HttpConfig; +use crate::server_config::server::{ + ConsumerGroupConfig, DataMaintenanceConfig, HeartbeatConfig, MessageSaverConfig, + PersonalAccessTokenConfig, TelemetryConfig, +}; +use crate::server_config::system::SystemConfig; +use configs::{ConfigEnv, ConfigEnvMappings, ConfigProvider, FileConfigProvider, TypedEnvProvider}; +use err_trail::ErrContext; +use figment::providers::{Format, Toml}; +use figment::value::Dict; +use figment::{Metadata, Profile, Provider}; +use iggy_common::Validatable; +use serde::{Deserialize, Serialize}; +use server_common::sharding::{MAX_PARTITIONS, MAX_STREAMS, MAX_TOPICS}; +use std::env; +use std::sync::Arc; + +const DEFAULT_CONFIG_PATH: &str = "core/server-ng/config.toml"; + +/// The `server-ng` flavour of [`SystemConfig`], bound to this crate's own +/// [`super::sharding::ShardingConfig`]. `core/server-ng` names this alias +/// wherever it refers to the system config. +pub type NgSystemConfig = SystemConfig; + +/// Top-level on-disk config schema for the `server-ng` binary. +/// +/// Mirrors the legacy [`crate::server::ServerConfig`] section surface +/// verbatim (operator-facing schema is unchanged) and adds a +/// [`MessageBusConfig`] section for inter-shard / inter-replica bus +/// tunables. +/// +/// Section types are reused directly from the legacy server-config +/// modules; only [`MessageBusConfig`] and this composer are net-new +/// code at the time of introduction. +/// +/// At the time of introduction this type is NOT consumed by +/// `core/server-ng`'s bootstrap. The wiring PR is a separate change. +#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] +#[config_env(prefix = "IGGY_", name = "iggy-server-ng-config")] +pub struct ServerNgConfig { + pub consumer_group: ConsumerGroupConfig, + pub data_maintenance: DataMaintenanceConfig, + #[serde(default)] + pub extra: ExtraConfig, + pub message_saver: MessageSaverConfig, + pub personal_access_token: PersonalAccessTokenConfig, + pub heartbeat: HeartbeatConfig, + pub system: Arc, + pub quic: QuicConfig, + pub tcp: TcpConfig, + pub http: HttpConfig, + pub websocket: WebSocketConfig, + pub telemetry: TelemetryConfig, + pub cluster: ClusterConfig, + pub metadata: MetadataConfig, + pub partition: PartitionConfig, + pub message_bus: MessageBusConfig, +} + +#[derive(Debug, Default, Deserialize, Serialize, Clone, ConfigEnv)] +pub struct ExtraConfig { + pub namespace: NamespaceConfig, +} + +#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] +pub struct NamespaceConfig { + pub max_streams: usize, + pub max_topics: usize, + pub max_partitions: usize, +} + +impl Default for NamespaceConfig { + fn default() -> Self { + Self { + max_streams: MAX_STREAMS, + max_topics: MAX_TOPICS, + max_partitions: MAX_PARTITIONS, + } + } +} + +impl ServerNgConfig { + /// Load server-ng configuration from file and environment variables. + /// + /// Mirrors [`crate::server::ServerConfig::load`]: the path comes + /// from `IGGY_CONFIG_PATH` or defaults to + /// `core/server-ng/config.toml`; missing on-disk paths fall through + /// to the embedded default TOML; env-var overrides flow through the + /// [`ServerNgConfigEnvProvider`]; the result is validated before + /// returning. + /// + /// # Errors + /// Returns [`ConfigurationError`] when the config cannot be parsed + /// from the configured source(s) or fails [`Validatable::validate`]. + pub async fn load() -> Result { + let config_path = + env::var("IGGY_CONFIG_PATH").unwrap_or_else(|_| DEFAULT_CONFIG_PATH.to_string()); + let provider = ServerNgConfig::config_provider(&config_path); + let cfg: ServerNgConfig = + provider + .load_config() + .await + .error(|e: &configs::ConfigurationError| { + format!("{COMPONENT_NG} (error: {e}) - failed to load server-ng config") + })?; + cfg.validate().error(|e: &configs::ConfigurationError| { + format!("{COMPONENT_NG} (error: {e}) - failed to validate server-ng config") + })?; + Ok(cfg) + } + + /// Build the file-backed config provider with the embedded default + /// TOML and the type-safe env-var provider attached. + pub fn config_provider(config_path: &str) -> FileConfigProvider { + let default_config = Toml::string(include_str!("../../../server-ng/config.toml")); + FileConfigProvider::new( + config_path.to_string(), + ServerNgConfigEnvProvider::default(), + true, + Some(default_config), + ) + } + + /// All recognised env var names for [`ServerNgConfig`]. + pub fn all_env_var_names() -> Vec<&'static str> { + ::all_env_var_names() + } +} + +/// Type-safe environment provider for [`ServerNgConfig`]. +/// +/// Uses the [`ConfigEnvMappings`] trait generated by `#[derive(ConfigEnv)]` +/// to look up known env var names directly, eliminating path ambiguity. +#[derive(Debug, Clone)] +pub struct ServerNgConfigEnvProvider { + provider: TypedEnvProvider, +} + +impl Default for ServerNgConfigEnvProvider { + fn default() -> Self { + Self { + provider: TypedEnvProvider::from_config(ServerNgConfig::ENV_PREFIX), + } + } +} + +impl Provider for ServerNgConfigEnvProvider { + fn metadata(&self) -> Metadata { + Metadata::named(ServerNgConfig::ENV_PROVIDER_NAME) + } + + fn data(&self) -> Result, figment::Error> { + self.provider.deserialize().map_err(|e| { + figment::Error::from(format!( + "Cannot deserialize environment variables for server-ng config: {e}" + )) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use figment::Figment; + + /// The embedded default TOML deserializes into a fully populated + /// [`ServerNgConfig`] and passes validation. Exercises the + /// `include_str!` resolution and the deserialization of every + /// section without depending on an async runtime in `dev-deps`. + #[test] + fn embedded_default_toml_deserializes_and_validates() { + let toml_str = include_str!("../../../server-ng/config.toml"); + let cfg: ServerNgConfig = Figment::new() + .merge(Toml::string(toml_str)) + .extract() + .expect("embedded TOML deserializes"); + cfg.validate().expect("embedded default validates"); + + // Spot-check: defaults match the runtime crate's invariants. + assert_eq!(cfg.message_bus.max_batch, 256); + assert_eq!(cfg.message_bus.peer_queue_capacity, 256); + } + + #[test] + fn default_impl_validates() { + let cfg = ServerNgConfig::default(); + cfg.validate().expect("Default impl validates"); + } + + #[test] + fn env_prefix_is_iggy() { + assert_eq!(ServerNgConfig::ENV_PREFIX, "IGGY_"); + } + + #[test] + fn all_env_var_names_include_message_bus_section() { + let names = ServerNgConfig::all_env_var_names(); + assert!( + names.iter().any(|n| n.starts_with("IGGY_MESSAGE_BUS_")), + "expected at least one IGGY_MESSAGE_BUS_* env var, got: {names:?}" + ); + } +} diff --git a/core/configs/src/server_ng_config/sharding.rs b/core/configs/src/server_ng_config/sharding.rs new file mode 100644 index 0000000000..f5c2257d43 --- /dev/null +++ b/core/configs/src/server_ng_config/sharding.rs @@ -0,0 +1,431 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Sharding config for `server-ng`. Forked from the legacy +//! [`crate::server_config::sharding`] because the two servers own different +//! knob sets and different default sources: this type carries the full +//! thread-per-core + bus surface and reads its defaults from the server-ng +//! TOML, while the legacy type keeps only `cpu_allocation` + `pin_cores`. + +use iggy_common::IggyDuration; +use iggy_common::Validatable; +use serde::{Deserialize, Serialize}; +use serde_with::{DisplayFromStr, serde_as}; +use std::time::Duration; + +use super::defaults::SERVER_NG_CONFIG; +use crate::ConfigurationError; +use crate::server_config::validators::validate_cpu_allocation; +use configs::ConfigEnv; + +// Re-exported so the `configs::ng_sharding::*` path mirrors the legacy +// `configs::sharding::*` surface for callers. +pub use cpu_allocation::{CpuAllocation, NumaConfig}; + +/// Maximum permitted per-shard inbox depth. The channel is allocated +/// up-front per shard, so a runaway value here OOMs the process at boot. +/// `1 << 20` (~1M frames) is several orders of magnitude above any +/// realistic backpressure target and still fits comfortably in process +/// address space. +pub const INBOX_CAPACITY_MAX: usize = 1 << 20; + +/// Hard upper bound on `shutdown_drain_timeout`. A drain that never +/// completes wedges process exit; capping at 10 minutes guarantees the +/// watchdog eventually force-tears the bus even with a pathological +/// config typo. +pub const SHUTDOWN_DRAIN_TIMEOUT_MAX: Duration = Duration::from_secs(600); + +/// Hard upper bound on `shutdown_poll_interval`. A poll interval longer +/// than the drain timeout makes the flag effectively unobservable; cap +/// at 5s so Ctrl-C latency stays bounded regardless of config. +pub const SHUTDOWN_POLL_INTERVAL_MAX: Duration = Duration::from_secs(5); + +/// Hard upper bound on `shutdown_join_timeout`. Comfortably above the +/// drain cap so a full drain always fits inside the join budget, while +/// still guaranteeing process exit against a pathological config typo. +pub const SHUTDOWN_JOIN_TIMEOUT_MAX: Duration = Duration::from_secs(900); + +/// Hard upper bound on `reconcile_periodic_interval`. A tick longer +/// than ~30s makes post-failure recovery latency operator-visible; the +/// cap reins in pathological typos without disturbing reasonable +/// production values. +pub const RECONCILE_PERIODIC_INTERVAL_MAX: Duration = Duration::from_secs(30); + +// Every omitted field falls back to the frozen `Default`, so a partial +// `[system.sharding]` table resolves each key independently instead of +// failing on the first missing one (parity with the legacy type). +#[serde_as] +#[derive(Debug, Deserialize, Serialize, ConfigEnv)] +#[serde(default)] +pub struct ShardingConfig { + #[serde(default)] + #[config_env(leaf)] + pub cpu_allocation: CpuAllocation, + /// Whether shard threads are pinned to dedicated CPU cores + /// (`sched_setaffinity`). Pinning maximizes cache locality when this + /// server owns its cores (dedicated host, `numa:` allocations). Set to + /// `false` when the server shares cores with other workloads — e.g. a + /// multi-tenant host slicing CPU via cgroup quotas — where every process + /// pinning to the same low-numbered cores would pile onto one core while + /// the rest sit idle; unpinned shards let the kernel scheduler place + /// threads freely within the allowed set. With a NUMA-aware allocation, + /// `false` drops both the CPU and memory-node bindings (and logs a + /// warning, since NUMA placement without pinning is meaningless). + pub pin_cores: bool, + /// Per-shard inter-shard inbox channel capacity. Bounded by design. + /// Drops on full inbox of consensus frames are recovered by VSR + /// retransmit. Drops of cross-shard client Reply frames are terminal: + /// the client never receives the reply (no in-protocol retransmit). + /// Both frame classes share this one channel, so a consensus burst + /// can starve client-reply forwards: size against the worst-case sum + /// of consensus working set + peak client-reply fan-out per shard + /// occurring together. + /// + // TODO(hubcio): split into two priority lanes - one bounded queue for + // consensus frames (drops recovered by VSR retransmit) and one for + // client `Reply` frames (drops terminal, must be sized for worst-case + // fan-out). Current single-channel design is the minimum-viable + // wiring so `frame_drops_total{variant,reason}` surfaces under load + // and yields real numbers to size the split against. + pub inbox_capacity: usize, + /// Wall-clock budget for a single shard's bus drain on shutdown. + /// Drives `IggyMessageBus::shutdown(..)` from the per-shard watchdog + /// and the parallel-join survivor path. Sized larger than typical + /// TCP RTT times in-flight write-batch so writers receive their full + /// last `write_vectored_all` budget before the connection registry + /// force-tears the bus. Slow-fsync hosts may need to extend this past + /// the default; the cap is `SHUTDOWN_DRAIN_TIMEOUT_MAX` so a config + /// typo cannot wedge process exit. + #[serde_as(as = "DisplayFromStr")] + #[config_env(leaf)] + pub shutdown_drain_timeout: IggyDuration, + /// Poll cadence for the cross-thread shutdown flag and for the + /// `await_metadata_bundle` / `broadcast_metadata_bundle` poll loops. + /// Trades off Ctrl-C latency against idle wakeup cost; the default + /// keeps shutdown observably prompt without measurable scheduler + /// overhead. Capped at `SHUTDOWN_POLL_INTERVAL_MAX` so the flag + /// remains effectively observable regardless of config. + #[serde_as(as = "DisplayFromStr")] + #[config_env(leaf)] + pub shutdown_poll_interval: IggyDuration, + /// Hard wall-clock deadline for joining shard threads at process + /// exit. A shard whose pump or listener wedges past this budget is + /// abandoned with an error log instead of blocking exit forever. + /// Must be at least `shutdown_drain_timeout` (abandoning a shard + /// mid-drain would interrupt its WAL fsync / replica drain) and at + /// most [`SHUTDOWN_JOIN_TIMEOUT_MAX`]. + #[serde_as(as = "DisplayFromStr")] + #[config_env(leaf)] + pub shutdown_join_timeout: IggyDuration, + /// Safety-tick cadence for the partition reconciliation loop; the + /// reconciler also wakes immediately on every + /// `LifecycleFrame::MetadataCommitTick` from shard 0. This periodic + /// fallback covers dropped wake-ups (the wake channel is capacity-1) + /// and the initial post-bootstrap convergence window. Values above + /// [`RECONCILE_PERIODIC_INTERVAL_MAX`] are rejected by the validator. + #[serde_as(as = "DisplayFromStr")] + #[config_env(leaf)] + pub reconcile_periodic_interval: IggyDuration, +} + +impl Default for ShardingConfig { + fn default() -> Self { + Self { + cpu_allocation: CpuAllocation::default(), + pin_cores: SERVER_NG_CONFIG.system.sharding.pin_cores, + inbox_capacity: SERVER_NG_CONFIG.system.sharding.inbox_capacity as usize, + shutdown_drain_timeout: SERVER_NG_CONFIG + .system + .sharding + .shutdown_drain_timeout + .parse() + .unwrap(), + shutdown_poll_interval: SERVER_NG_CONFIG + .system + .sharding + .shutdown_poll_interval + .parse() + .unwrap(), + shutdown_join_timeout: SERVER_NG_CONFIG + .system + .sharding + .shutdown_join_timeout + .parse() + .unwrap(), + reconcile_periodic_interval: SERVER_NG_CONFIG + .system + .sharding + .reconcile_periodic_interval + .parse() + .unwrap(), + } + } +} + +impl Validatable for ShardingConfig { + fn validate(&self) -> Result<(), ConfigurationError> { + if self.inbox_capacity == 0 { + eprintln!( + "Invalid sharding configuration: inbox_capacity must be > 0 (crossfire silently \ + rounds 0 to 1, masking config errors)" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if self.inbox_capacity > INBOX_CAPACITY_MAX { + eprintln!( + "Invalid sharding configuration: inbox_capacity {} exceeds the {} cap (each \ + shard preallocates a channel of this size; oversizing here OOMs the process at \ + boot)", + self.inbox_capacity, INBOX_CAPACITY_MAX + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + let drain = self.shutdown_drain_timeout.get_duration(); + if drain.is_zero() { + eprintln!( + "Invalid sharding configuration: shutdown_drain_timeout must be > 0 (a zero \ + budget force-tears the bus mid-WAL-fsync on every shutdown)" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if drain > SHUTDOWN_DRAIN_TIMEOUT_MAX { + eprintln!( + "Invalid sharding configuration: shutdown_drain_timeout {:?} exceeds the {:?} \ + cap (an unbounded drain wedges process exit on bus stall)", + drain, SHUTDOWN_DRAIN_TIMEOUT_MAX + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + let poll = self.shutdown_poll_interval.get_duration(); + if poll.is_zero() { + eprintln!( + "Invalid sharding configuration: shutdown_poll_interval must be > 0 (a zero \ + cadence busy-loops every shard's watchdog and metadata-handoff poller)" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if poll > SHUTDOWN_POLL_INTERVAL_MAX { + eprintln!( + "Invalid sharding configuration: shutdown_poll_interval {:?} exceeds the {:?} \ + cap (a coarse cadence stalls Ctrl-C handling and metadata handoff abort)", + poll, SHUTDOWN_POLL_INTERVAL_MAX + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if poll > drain { + eprintln!( + "Invalid sharding configuration: shutdown_poll_interval {:?} must be <= \ + shutdown_drain_timeout {:?} (a poll cadence coarser than the drain budget makes \ + the shutdown flag effectively unobservable)", + poll, drain + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + let join = self.shutdown_join_timeout.get_duration(); + if join < drain { + eprintln!( + "Invalid sharding configuration: shutdown_join_timeout {:?} must be >= \ + shutdown_drain_timeout {:?} (a join budget shorter than the drain abandons \ + shards mid-drain, interrupting the WAL fsync / replica drain)", + join, drain + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if join > SHUTDOWN_JOIN_TIMEOUT_MAX { + eprintln!( + "Invalid sharding configuration: shutdown_join_timeout {:?} exceeds the {:?} \ + cap (an unbounded join budget wedges process exit on a stuck shard)", + join, SHUTDOWN_JOIN_TIMEOUT_MAX + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + let reconcile = self.reconcile_periodic_interval.get_duration(); + if reconcile.is_zero() { + eprintln!( + "Invalid sharding configuration: reconcile_periodic_interval resolves to zero. \ + Note that \"0\", \"none\", \"unlimited\", and \"disabled\" all parse to zero. The \ + periodic reconcile tick is a safety net for dropped commit-wakes and cannot be \ + turned off; set a positive duration (default \"1s\", max {RECONCILE_PERIODIC_INTERVAL_MAX:?})." + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if reconcile > RECONCILE_PERIODIC_INTERVAL_MAX { + eprintln!( + "Invalid sharding configuration: reconcile_periodic_interval {:?} exceeds the \ + {:?} cap (a long tick makes post-failure convergence latency operator-visible)", + reconcile, RECONCILE_PERIODIC_INTERVAL_MAX + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + validate_cpu_allocation(&self.cpu_allocation, self.pin_cores) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::server_ng_config::server_ng::ServerNgConfig; + use figment::Figment; + use figment::providers::{Format, Toml}; + + #[test] + fn defaults_validate() { + assert!(ShardingConfig::default().validate().is_ok()); + } + + #[test] + fn zero_drain_is_rejected() { + let cfg = ShardingConfig { + shutdown_drain_timeout: IggyDuration::new(Duration::ZERO), + ..ShardingConfig::default() + }; + assert!(cfg.validate().is_err()); + } + + #[test] + fn over_cap_drain_is_rejected() { + let cfg = ShardingConfig { + shutdown_drain_timeout: IggyDuration::new( + SHUTDOWN_DRAIN_TIMEOUT_MAX + Duration::from_secs(1), + ), + ..ShardingConfig::default() + }; + assert!(cfg.validate().is_err()); + } + + #[test] + fn zero_poll_is_rejected() { + let cfg = ShardingConfig { + shutdown_poll_interval: IggyDuration::new(Duration::ZERO), + ..ShardingConfig::default() + }; + assert!(cfg.validate().is_err()); + } + + #[test] + fn over_cap_poll_is_rejected() { + let cfg = ShardingConfig { + shutdown_poll_interval: IggyDuration::new( + SHUTDOWN_POLL_INTERVAL_MAX + Duration::from_secs(1), + ), + ..ShardingConfig::default() + }; + assert!(cfg.validate().is_err()); + } + + #[test] + fn poll_greater_than_drain_is_rejected() { + let cfg = ShardingConfig { + shutdown_drain_timeout: IggyDuration::new(Duration::from_millis(20)), + shutdown_poll_interval: IggyDuration::new(Duration::from_millis(50)), + ..ShardingConfig::default() + }; + assert!(cfg.validate().is_err()); + } + + #[test] + fn join_shorter_than_drain_is_rejected() { + // A join budget under the drain would abandon shards mid-drain. + let cfg = ShardingConfig { + shutdown_drain_timeout: IggyDuration::new(Duration::from_secs(10)), + shutdown_join_timeout: IggyDuration::new(Duration::from_secs(5)), + ..ShardingConfig::default() + }; + assert!(cfg.validate().is_err()); + } + + #[test] + fn over_cap_join_is_rejected() { + let cfg = ShardingConfig { + shutdown_join_timeout: IggyDuration::new( + SHUTDOWN_JOIN_TIMEOUT_MAX + Duration::from_secs(1), + ), + ..ShardingConfig::default() + }; + assert!(cfg.validate().is_err()); + } + + #[test] + fn join_equal_to_drain_is_accepted() { + let cfg = ShardingConfig { + shutdown_drain_timeout: IggyDuration::new(Duration::from_secs(10)), + shutdown_join_timeout: IggyDuration::new(Duration::from_secs(10)), + ..ShardingConfig::default() + }; + assert!(cfg.validate().is_ok()); + } + + // Guards the single source of truth: the server-ng sharding defaults + // resolve from the embedded server-ng TOML, not hard-coded Rust values. + #[test] + fn ng_embedded_toml_resolves_sharding_defaults() { + let toml_str = include_str!("../../../server-ng/config.toml"); + let config: ServerNgConfig = Figment::new() + .merge(Toml::string(toml_str)) + .extract() + .expect("embedded server-ng TOML deserializes"); + config + .validate() + .expect("embedded server-ng config validates"); + + let sharding = &config.system.sharding; + assert!(sharding.pin_cores); + assert_eq!(sharding.inbox_capacity, 1024); + assert_eq!(sharding.shutdown_drain_timeout, "10 s".parse().unwrap()); + assert_eq!(sharding.shutdown_poll_interval, "50 ms".parse().unwrap()); + assert_eq!(sharding.shutdown_join_timeout, "30 s".parse().unwrap()); + assert_eq!(sharding.reconcile_periodic_interval, "1 s".parse().unwrap()); + } + + // Extract straight from a raw table (no embedded base layer) so the + // struct-level `#[serde(default)]` is what fills the gaps, not the + // provider's embedded-TOML fallback. + #[test] + fn partial_table_fills_missing_fields_with_frozen_defaults() { + let sharding: ShardingConfig = Figment::new() + .merge(Toml::string("pin_cores = false")) + .extract() + .expect("partial sharding table deserializes"); + + assert!(!sharding.pin_cores); + assert_eq!(sharding.inbox_capacity, 1024); + assert_eq!(sharding.shutdown_drain_timeout, "10 s".parse().unwrap()); + assert_eq!(sharding.shutdown_poll_interval, "50 ms".parse().unwrap()); + assert_eq!(sharding.shutdown_join_timeout, "30 s".parse().unwrap()); + assert_eq!(sharding.reconcile_periodic_interval, "1 s".parse().unwrap()); + } + + #[test] + fn empty_table_yields_all_frozen_defaults() { + let sharding: ShardingConfig = Figment::new() + .merge(Toml::string("")) + .extract() + .expect("empty sharding table deserializes"); + + assert!(sharding.pin_cores); + assert_eq!(sharding.inbox_capacity, 1024); + assert_eq!(sharding.shutdown_drain_timeout, "10 s".parse().unwrap()); + assert_eq!(sharding.shutdown_poll_interval, "50 ms".parse().unwrap()); + assert_eq!(sharding.shutdown_join_timeout, "30 s".parse().unwrap()); + assert_eq!(sharding.reconcile_periodic_interval, "1 s".parse().unwrap()); + } +} diff --git a/core/configs/src/server_ng_config/tcp.rs b/core/configs/src/server_ng_config/tcp.rs new file mode 100644 index 0000000000..7fbb34bb06 --- /dev/null +++ b/core/configs/src/server_ng_config/tcp.rs @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Server-ng TCP listener schema. +//! +//! Field shape mirrors the legacy [`crate::tcp::TcpConfig`] verbatim; +//! the type is forked into `server_ng_config` so server-ng can evolve +//! its TCP/TLS surface (additional knobs, removed knobs) independently +//! of the legacy server. No semantic change at fork time. + +use configs::ConfigEnv; +use iggy_common::{IggyByteSize, IggyDuration}; +use serde::{Deserialize, Serialize}; +use serde_with::DisplayFromStr; +use serde_with::serde_as; + +#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] +pub struct TcpConfig { + pub enabled: bool, + pub address: String, + pub ipv6: bool, + pub tls: TcpTlsConfig, + pub socket: TcpSocketConfig, + pub socket_migration: bool, +} + +#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] +pub struct TcpTlsConfig { + pub enabled: bool, + pub self_signed: bool, + pub cert_file: String, + pub key_file: String, +} + +#[serde_as] +#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] +pub struct TcpSocketConfig { + pub override_defaults: bool, + #[config_env(leaf)] + pub recv_buffer_size: IggyByteSize, + #[config_env(leaf)] + pub send_buffer_size: IggyByteSize, + pub keepalive: bool, + pub nodelay: bool, + #[config_env(leaf)] + #[serde_as(as = "DisplayFromStr")] + pub linger: IggyDuration, +} diff --git a/core/configs/src/server_ng_config/validators.rs b/core/configs/src/server_ng_config/validators.rs new file mode 100644 index 0000000000..f7408b7d35 --- /dev/null +++ b/core/configs/src/server_ng_config/validators.rs @@ -0,0 +1,861 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! [`Validatable`] for [`ServerNgConfig`]. +//! +//! Mirrors the section-by-section delegation of +//! [`crate::validators`]'s `impl Validatable for ServerConfig`, plus a +//! call into [`super::message_bus::MessageBusConfig::validate`] for the +//! new section. The cross-section invariants (topic vs segment sizing, +//! JWT gating when HTTP is enabled, server-default expiry sanity) are +//! mirrored exactly so server-ng inherits the same boot-time safety +//! net. + +use super::COMPONENT_NG; +use super::cluster::STATE_CHUNK_HEADER_LEN; +use super::server_ng::{ExtraConfig, NamespaceConfig, ServerNgConfig}; +use crate::ConfigurationError; +use err_trail::ErrContext; +use iggy_common::{IggyExpiry, MaxTopicSize, Validatable}; +use server_common::sharding::IggyNamespace; +use tracing::warn; + +/// compio-ws (tungstenite 0.29) `write_buffer_size` default. Used to +/// evaluate the `max_write_buffer_size > write_buffer_size` invariant +/// when the operator leaves `write_buffer_size` unset; keep in sync +/// with the defaults documented in the shipped config.toml. +const WS_DEFAULT_WRITE_BUFFER_SIZE: u64 = 128 * 1024; + +impl Validatable for ServerNgConfig { + fn validate(&self) -> Result<(), ConfigurationError> { + self.system + .memory_pool + .validate() + .error(|e: &ConfigurationError| { + format!("{COMPONENT_NG} (error: {e}) - failed to validate memory pool config") + })?; + self.data_maintenance + .validate() + .error(|e: &ConfigurationError| { + format!("{COMPONENT_NG} (error: {e}) - failed to validate data maintenance config") + })?; + self.personal_access_token + .validate() + .error(|e: &ConfigurationError| { + format!( + "{COMPONENT_NG} (error: {e}) - failed to validate personal access token config" + ) + })?; + self.extra.validate().error(|e: &ConfigurationError| { + format!("{COMPONENT_NG} (error: {e}) - failed to validate extra config") + })?; + self.system + .segment + .validate() + .error(|e: &ConfigurationError| { + format!("{COMPONENT_NG} (error: {e}) - failed to validate segment config") + })?; + self.system + .compression + .validate() + .error(|e: &ConfigurationError| { + format!("{COMPONENT_NG} (error: {e}) - failed to validate compression config") + })?; + self.telemetry.validate().error(|e: &ConfigurationError| { + format!("{COMPONENT_NG} (error: {e}) - failed to validate telemetry config") + })?; + self.system + .sharding + .validate() + .error(|e: &ConfigurationError| { + format!("{COMPONENT_NG} (error: {e}) - failed to validate sharding config") + })?; + self.cluster.validate().error(|e: &ConfigurationError| { + format!("{COMPONENT_NG} (error: {e}) - failed to validate cluster config") + })?; + self.metadata.validate().error(|e: &ConfigurationError| { + format!("{COMPONENT_NG} (error: {e}) - failed to validate metadata config") + })?; + self.partition.validate().error(|e: &ConfigurationError| { + format!("{COMPONENT_NG} (error: {e}) - failed to validate partition config") + })?; + self.system + .logging + .validate() + .error(|e: &ConfigurationError| { + format!("{COMPONENT_NG} (error: {e}) - failed to validate logging config") + })?; + self.message_saver + .validate() + .error(|e: &ConfigurationError| { + format!("{COMPONENT_NG} (error: {e}) - failed to validate message saver config") + })?; + + let topic_size = match self.system.topic.max_size { + MaxTopicSize::Custom(size) => Ok(size.as_bytes_u64()), + MaxTopicSize::Unlimited => Ok(u64::MAX), + MaxTopicSize::ServerDefault => { + eprintln!("system.topic.max_size cannot be ServerDefault in server-ng config"); + Err(ConfigurationError::InvalidConfigurationValue) + } + }?; + + if let IggyExpiry::ServerDefault = self.system.topic.message_expiry { + eprintln!("system.topic.message_expiry cannot be ServerDefault in server-ng config"); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + // A zero duration encodes to wire value 0, the same value the wire uses + // for ServerDefault, so it would silently collide with that sentinel. + if let IggyExpiry::ExpireDuration(duration) = self.system.topic.message_expiry + && duration.as_micros() == 0 + { + eprintln!( + "system.topic.message_expiry is a zero duration, which collides with the server-default sentinel on the wire; use \"none\" to never expire or a positive duration" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + if self.http.enabled + && let IggyExpiry::ServerDefault = self.http.jwt.access_token_expiry + { + eprintln!("http.jwt.access_token_expiry cannot be ServerDefault when HTTP is enabled"); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + if self.http.enabled + && self.http.tls.enabled + && (self.http.tls.cert_file.is_empty() || self.http.tls.key_file.is_empty()) + { + eprintln!( + "http.tls.enabled=true requires non-empty http.tls.cert_file and http.tls.key_file" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + // Cluster mode has no port fallbacks: the roster is the single source + // of listener ports, so every enabled transport needs an explicit + // per-node port. Falling back to the port of a transport's top-level + // `address` would hand two same-host nodes the same socket and fail + // only at bind time, and a portless node would silently degrade every + // follower-to-primary HTTP forward through it to a fail-closed 503. + if self.cluster.enabled { + for node in &self.cluster.nodes { + let required_ports = [ + ("tcp", true, node.ports.tcp), + ("quic", self.quic.enabled, node.ports.quic), + ("http", self.http.enabled, node.ports.http), + ("websocket", self.websocket.enabled, node.ports.websocket), + ("tcp_replica", true, node.ports.tcp_replica), + ]; + for (transport, enabled, port) in required_ports { + if enabled && port.is_none() { + eprintln!( + "cluster node '{}' has no ports.{transport}; cluster mode requires an explicit roster port for every enabled transport", + node.name + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + } + } + } + + if topic_size < self.system.segment.size.as_bytes_u64() { + eprintln!( + "system.topic.max_size ({} B) must be >= system.segment.size ({} B)", + topic_size, + self.system.segment.size.as_bytes_u64() + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + // A received segment artifact can be one whole batch larger than the + // segment cap (rotation checks the cap AFTER appending), and the real + // batch bound is the BUS frame cap -- server-ng never enforces + // `MAX_PAYLOAD_SIZE`. An artifact ceiling under that floor refuses a + // legal segment, and the manifest check is all-or-nothing, so the + // partition livelocks re-requesting the same segment from every peer at + // the backoff ceiling. Caught here so it is a boot error rather than one + // partition that silently never rejoins. + let artifact_floor = self + .system + .segment + .size + .as_bytes_u64() + .saturating_add(self.message_bus.max_message_size.as_bytes_u64()); + if self.partition.transfer_artifact_bytes_max.as_bytes_u64() < artifact_floor { + eprintln!( + "{COMPONENT_NG} partition.transfer_artifact_bytes_max ({} B) must be at least \ + system.segment.size ({} B) + message_bus.max_message_size ({} B) = \ + {artifact_floor} B: a segment may close one whole batch past its cap, and an \ + artifact ceiling below that refuses a legal segment and livelocks the \ + partition's rejoin", + self.partition.transfer_artifact_bytes_max.as_bytes_u64(), + self.system.segment.size.as_bytes_u64(), + self.message_bus.max_message_size.as_bytes_u64(), + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + self.message_bus + .validate() + .error(|e: &ConfigurationError| { + format!("{COMPONENT_NG} (error: {e}) - failed to validate message_bus config") + })?; + + // Repair frames ride the bounded per-peer message-bus queue. A repair + // round of cluster.repair_chunk_max frames that meets or overruns + // message_bus.peer_queue_capacity drops its own tail silently, wedging + // the repair loop into slow retries. Keep the chunk strictly below the + // queue; this also floors peer_queue_capacity, which is otherwise only + // checked for > 0. + if self.cluster.repair_chunk_max >= self.message_bus.peer_queue_capacity { + eprintln!( + "{COMPONENT_NG} cluster.repair_chunk_max ({}) must be < message_bus.peer_queue_capacity ({}): repair frames ride the per-peer bus queue, so a chunk that fills or overruns it drops frames and wedges repair", + self.cluster.repair_chunk_max, self.message_bus.peer_queue_capacity + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + // State-transfer chunks ride the same bus. A cap that cannot carry one + // header plus a byte of payload makes every rejoin that needs a + // transfer impossible, and the failure surfaces only as a replica + // connection tearing down when the frame is rejected on the read side. + let bus_cap = self.message_bus.max_message_size.as_bytes_u64(); + if bus_cap <= STATE_CHUNK_HEADER_LEN { + eprintln!( + "{COMPONENT_NG} message_bus.max_message_size ({bus_cap}) must exceed the {STATE_CHUNK_HEADER_LEN}-byte state-chunk header: state transfer serves artifact chunks over this bus, and a frame above the cap is rejected by the receiving transport, which tears down the whole replica connection" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + // WS frame chain: websocket.max_frame_size <= websocket.max_message_size + // <= message_bus.max_message_size. The bus's WS / WSS install path takes + // its frame tuning from [websocket], so a WS ceiling above the bus's own + // frame cap would admit messages the bus read-side validator then tears + // the connection down over. An absent knob defers to the compio-ws + // default (16 MiB frame / 64 MiB message), which satisfies the chain + // against the shipped bus cap in practice. + let bus_max_message_size = self.message_bus.max_message_size.as_bytes_u64(); + if let (Some(frame), Some(message)) = ( + self.websocket.max_frame_size, + self.websocket.max_message_size, + ) && frame.as_bytes_u64() > message.as_bytes_u64() + { + eprintln!( + "{COMPONENT_NG} websocket.max_frame_size ({}) exceeds websocket.max_message_size ({})", + frame.as_bytes_u64(), + message.as_bytes_u64() + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if let Some(message) = self.websocket.max_message_size + && message.as_bytes_u64() > bus_max_message_size + { + eprintln!( + "{COMPONENT_NG} websocket.max_message_size ({}) exceeds message_bus.max_message_size ({})", + message.as_bytes_u64(), + bus_max_message_size + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if let Some(frame) = self.websocket.max_frame_size + && frame.as_bytes_u64() > bus_max_message_size + { + eprintln!( + "{COMPONENT_NG} websocket.max_frame_size ({}) exceeds message_bus.max_message_size ({})", + frame.as_bytes_u64(), + bus_max_message_size + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + // "0", "unlimited" and "none" all parse to a zero IggyByteSize. A + // zero WS tunable is never usable: zero message or frame ceilings + // reject every inbound frame, and zero buffers starve the + // compio-ws pipeline. Reject at boot instead of shipping a + // listener that cannot serve a single message. + for (key, size) in [ + ("read_buffer_size", self.websocket.read_buffer_size), + ("write_buffer_size", self.websocket.write_buffer_size), + ( + "max_write_buffer_size", + self.websocket.max_write_buffer_size, + ), + ("max_message_size", self.websocket.max_message_size), + ("max_frame_size", self.websocket.max_frame_size), + ] { + if let Some(size) = size + && size.as_bytes_u64() == 0 + { + eprintln!( + "{COMPONENT_NG} websocket.{key} must be non-zero (\"0\", \"unlimited\" and \"none\" all parse to zero)" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + } + + // tungstenite asserts `max_write_buffer_size > write_buffer_size` + // during connection setup, so a violating pair panics on every + // accepted socket. Enforce the invariant at boot; an unset + // write_buffer_size runs at the compio-ws default. + if let Some(max_write) = self.websocket.max_write_buffer_size { + let write_buffer_size = self + .websocket + .write_buffer_size + .map_or(WS_DEFAULT_WRITE_BUFFER_SIZE, |size| size.as_bytes_u64()); + if max_write.as_bytes_u64() <= write_buffer_size { + eprintln!( + "{COMPONENT_NG} websocket.max_write_buffer_size ({}) must exceed websocket.write_buffer_size ({write_buffer_size})", + max_write.as_bytes_u64() + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + } + + self.quic.validate().error(|e: &ConfigurationError| { + format!("{COMPONENT_NG} (error: {e}) - failed to validate quic config") + })?; + + // Both knobs below are live in server-ng but sit on structs the legacy + // server shares, so the rejects live here instead of in a `Validatable` + // impl that would tighten legacy boots too. `0` / `disabled` / + // `unlimited` all parse to the same zero duration. + if self + .consumer_group + .rebalancing_timeout + .get_duration() + .is_zero() + { + eprintln!( + "{COMPONENT_NG} consumer_group.rebalancing_timeout must be nonzero: it is the deadline after which a pending revocation completes without the source client committing what it was served, so zero force-transfers every partition on the next reconciler tick and reopens the duplicate-delivery window" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if self.heartbeat.enabled && self.heartbeat.interval.get_duration().is_zero() { + eprintln!( + "{COMPONENT_NG} heartbeat.interval must be nonzero when heartbeat.enabled: it sizes both the verifier's sleep and the staleness window, so zero spins the verifier and reaps every live session on its first pass" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + reject_unsupported_and_warn_inert(self)?; + + Ok(()) + } +} + +/// server-ng parses the whole legacy config surface but does not yet honor +/// every knob. Make the still-inert ones loud at boot: reject the unsupported +/// features (all off by default, so only a deliberate opt-in trips this) and +/// warn once for tuning knobs server-ng silently ignores. Warnings fire only +/// when a knob deviates from its [`ServerNgConfig::default`] baseline, so a +/// pristine config.toml boots without noise. Baseline caveat: only the tcp/quic +/// fork sections take that default from the ng config.toml. The reused legacy +/// sections (`system.*`, `consumer_group.*`, `message_saver.*`) take theirs from +/// the legacy server config.toml; those match ng's shipped values today but are +/// not schema-locked, so editing such a knob in the ng config.toml could surface +/// a spurious warn. The guard test below pins the compared knobs against drift. +fn reject_unsupported_and_warn_inert(config: &ServerNgConfig) -> Result<(), ConfigurationError> { + if config.system.message_deduplication.enabled { + eprintln!("system.message_deduplication.enabled is not supported in server-ng"); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if config.system.segment.archive_expired { + eprintln!("system.segment.archive_expired is not supported in server-ng"); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if config.system.recovery.recreate_missing_state { + eprintln!("system.recovery.recreate_missing_state is not supported in server-ng"); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + let defaults = ServerNgConfig::default(); + + if config.tcp.socket.override_defaults { + warn!("tcp.socket tuning is set but not applied in server-ng"); + } + if config.quic.socket.override_defaults { + warn!("quic.socket tuning is set but not applied in server-ng"); + } + if config.tcp.ipv6 { + warn!( + "tcp.ipv6 is ignored in server-ng; IPv4 vs IPv6 is decided by the tcp.address string" + ); + } + if config.tcp.socket_migration != defaults.tcp.socket_migration { + warn!("tcp.socket_migration is not implemented in server-ng"); + } + if config.system.partition.validate_checksum != defaults.system.partition.validate_checksum { + warn!( + "system.partition.validate_checksum is not applied in server-ng; nothing verifies checksums on load" + ); + } + if config.system.segment.cache_indexes != defaults.system.segment.cache_indexes { + warn!("system.segment.cache_indexes is not applied in server-ng"); + } + if config.system.logging.sysinfo_print_interval + != defaults.system.logging.sysinfo_print_interval + { + warn!("system.logging.sysinfo_print_interval is not applied in server-ng"); + } + if config.system.backup.path != defaults.system.backup.path + || config.system.backup.compatibility.path != defaults.system.backup.compatibility.path + { + warn!("backup is not supported in server-ng"); + } + // default_algorithm deviation is already warned by the delegated legacy + // CompressionConfig::validate; only allow_override needs a signal here. + if config.system.compression.allow_override != defaults.system.compression.allow_override { + warn!( + "system.compression.allow_override is inert in server-ng; live compression is per-topic from the request" + ); + } + if config.system.state.enforce_fsync != defaults.system.state.enforce_fsync + || config.system.state.max_file_operation_retries + != defaults.system.state.max_file_operation_retries + || config.system.state.retry_delay != defaults.system.state.retry_delay + { + warn!( + "system.state tuning (enforce_fsync, max_file_operation_retries, retry_delay) is not applied in server-ng" + ); + } + if config.consumer_group.rebalancing_check_interval + != defaults.consumer_group.rebalancing_check_interval + { + warn!( + "consumer_group.rebalancing_check_interval is not applied in server-ng; rebalancing cadence uses system.sharding.reconcile_periodic_interval" + ); + } + if config.message_saver.interval != defaults.message_saver.interval + || config.message_saver.enforce_fsync != defaults.message_saver.enforce_fsync + { + warn!( + "periodic message_saver is not implemented in server-ng; only shutdown-flush is active" + ); + } + + Ok(()) +} + +impl Validatable for ExtraConfig { + fn validate(&self) -> Result<(), ConfigurationError> { + self.namespace.validate().error(|e: &ConfigurationError| { + format!("{COMPONENT_NG} (error: {e}) - failed to validate namespace config") + })?; + Ok(()) + } +} + +impl Validatable for NamespaceConfig { + fn validate(&self) -> Result<(), ConfigurationError> { + IggyNamespace::validate_capacity(self.max_streams, self.max_topics, self.max_partitions) + .map_err(|error| { + eprintln!("extra.namespace is invalid: {error}"); + ConfigurationError::InvalidConfigurationValue + })?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::super::cluster::{ClusterNodeConfig, TransportPorts}; + use super::*; + use figment::Figment; + use figment::providers::{Format, Toml}; + + const DEFAULT_CONFIG: &str = include_str!("../../../server-ng/config.toml"); + + /// Deep-merge a partial override over the shipped default, mirroring the + /// file-over-embedded layering the runtime loader performs. + fn config_with_override(override_toml: &str) -> ServerNgConfig { + Figment::new() + .merge(Toml::string(DEFAULT_CONFIG)) + .merge(Toml::string(override_toml)) + .extract() + .expect("config deserializes") + } + + #[test] + fn given_shipped_default_config_when_validating_should_pass() { + let config: ServerNgConfig = Figment::new() + .merge(Toml::string(DEFAULT_CONFIG)) + .extract() + .expect("default config deserializes"); + config + .validate() + .expect("pristine server-ng config must validate"); + } + + #[test] + fn given_message_deduplication_enabled_when_validating_should_reject() { + let config = config_with_override("[system.message_deduplication]\nenabled = true\n"); + assert!(config.validate().is_err()); + } + + #[test] + fn given_web_ui_enabled_when_validating_should_pass() { + let config = config_with_override("[http]\nweb_ui = true\n"); + config + .validate() + .expect("web_ui is now served by server-ng and must validate"); + } + + #[test] + fn given_archive_expired_enabled_when_validating_should_reject() { + let config = config_with_override("[system.segment]\narchive_expired = true\n"); + assert!(config.validate().is_err()); + } + + #[test] + fn given_recreate_missing_state_enabled_when_validating_should_reject() { + let config = config_with_override("[system.recovery]\nrecreate_missing_state = true\n"); + assert!(config.validate().is_err()); + } + + #[test] + fn given_zero_message_expiry_when_validating_should_reject() { + let config = config_with_override("[system.topic]\nmessage_expiry = \"0s\"\n"); + assert!(config.validate().is_err()); + } + + #[test] + fn given_peer_queue_capacity_not_above_repair_chunk_max_when_validating_should_reject() { + // The default repair_chunk_max (128) must stay strictly below + // peer_queue_capacity; shrinking the queue to the chunk size is the + // silent wedged-repair footgun this cross-section guard closes. + let config = config_with_override("[message_bus]\npeer_queue_capacity = 128\n"); + assert!(config.validate().is_err()); + } + + #[test] + fn given_repair_chunk_max_at_peer_queue_capacity_when_validating_should_reject() { + let config = config_with_override("[cluster]\nrepair_chunk_max = 256\n"); + assert!(config.validate().is_err()); + } + + #[test] + fn given_repair_chunk_max_below_peer_queue_capacity_when_validating_should_pass() { + let config = config_with_override("[cluster]\nrepair_chunk_max = 255\n"); + config + .validate() + .expect("a chunk below the peer queue capacity must validate"); + } + + #[test] + fn given_ws_frame_size_above_ws_message_size_when_validating_should_reject() { + let config = config_with_override( + "[websocket]\nmax_message_size = \"1 MiB\"\nmax_frame_size = \"2 MiB\"\n", + ); + assert!(config.validate().is_err()); + } + + // The shipped bus cap is 64 MiB, so a 128 MiB WS ceiling breaks the chain. + #[test] + fn given_ws_message_size_above_bus_max_message_size_when_validating_should_reject() { + let config = config_with_override("[websocket]\nmax_message_size = \"128 MiB\"\n"); + assert!(config.validate().is_err()); + } + + #[test] + fn given_ws_frame_size_above_bus_max_message_size_when_validating_should_reject() { + let config = config_with_override("[websocket]\nmax_frame_size = \"128 MiB\"\n"); + assert!(config.validate().is_err()); + } + + #[test] + fn given_ws_frame_chain_in_ascending_order_when_validating_should_pass() { + let config = config_with_override( + "[websocket]\nmax_message_size = \"32 MiB\"\nmax_frame_size = \"16 MiB\"\n", + ); + config + .validate() + .expect("frame <= message <= bus cap must validate"); + } + + // "unlimited" is not a supported sentinel for the WS size knobs: it + // parses to zero, which as a cap would reject every message. + #[test] + fn given_zero_ws_size_when_validating_should_reject() { + let config = config_with_override("[websocket]\nmax_message_size = \"unlimited\"\n"); + assert!(config.validate().is_err()); + } + + // tungstenite panics on this pair at connection setup; boot must + // reject it first. + #[test] + fn given_max_write_buffer_at_write_buffer_when_validating_should_reject() { + let config = config_with_override( + "[websocket]\nwrite_buffer_size = \"256 KiB\"\nmax_write_buffer_size = \"256 KiB\"\n", + ); + assert!(config.validate().is_err()); + } + + #[test] + fn given_max_write_buffer_below_default_write_buffer_when_validating_should_reject() { + let config = config_with_override("[websocket]\nmax_write_buffer_size = \"64 KiB\"\n"); + assert!(config.validate().is_err()); + } + + #[test] + fn given_max_write_buffer_above_write_buffer_when_validating_should_pass() { + let config = config_with_override( + "[websocket]\nwrite_buffer_size = \"128 KiB\"\nmax_write_buffer_size = \"1 MiB\"\n", + ); + config + .validate() + .expect("max write buffer above write buffer must validate"); + } + + // The size knobs are strictly typed; a malformed string must fail + // deserialization at load rather than degrade to the compio-ws default. + #[test] + fn given_malformed_ws_size_string_when_deserializing_should_reject() { + let result: Result = Figment::new() + .merge(Toml::string(DEFAULT_CONFIG)) + .merge(Toml::string( + "[websocket]\nmax_message_size = \"not-a-size\"\n", + )) + .extract(); + assert!( + result.is_err(), + "malformed websocket.max_message_size must fail config load" + ); + } + + // The shipped config is single-node (cluster.enabled = false), where the + // cross-section rule above is the only repair_chunk_max check that used to + // run; its structural bounds have to hold there too. + #[test] + fn given_single_node_zero_repair_chunk_max_when_validating_should_reject() { + let config = config_with_override("[cluster]\nrepair_chunk_max = 0\n"); + assert!(config.validate().is_err()); + } + + #[test] + fn given_single_node_repair_chunk_max_above_ceiling_when_validating_should_reject() { + // Queue widened past the chunk so the cross-section rule passes and + // only the structural ceiling can reject. + let config = config_with_override( + "[cluster]\nrepair_chunk_max = 2000\n\n[message_bus]\npeer_queue_capacity = 4096\n", + ); + assert!(config.validate().is_err()); + } + + #[test] + fn given_zero_rebalancing_timeout_when_validating_should_reject() { + let config = config_with_override("[consumer_group]\nrebalancing_timeout = \"0\"\n"); + assert!(config.validate().is_err()); + } + + #[test] + fn given_disabled_rebalancing_timeout_when_validating_should_reject() { + // "disabled" reads like an opt-out but parses to the same zero + // duration, which force-transfers every revocation instead. + let config = config_with_override("[consumer_group]\nrebalancing_timeout = \"disabled\"\n"); + assert!(config.validate().is_err()); + } + + #[test] + fn given_zero_heartbeat_interval_when_heartbeat_enabled_should_reject() { + let config = config_with_override("[heartbeat]\nenabled = true\ninterval = \"0\"\n"); + assert!(config.validate().is_err()); + } + + #[test] + fn given_zero_heartbeat_interval_when_heartbeat_disabled_should_pass() { + let config = config_with_override("[heartbeat]\nenabled = false\ninterval = \"0\"\n"); + config + .validate() + .expect("a disabled heartbeat never reads its interval"); + } + + /// The warn-helper baseline is [`ServerNgConfig::default`], but the reused + /// legacy sections source that default from the legacy server config.toml, + /// not this NG file. Pin the knobs the helper compares so any drift between + /// the two config.toml files fails here instead of as a spurious boot warn. + #[test] + fn given_shipped_ng_config_when_compared_to_default_should_match_warned_knobs() { + let shipped: ServerNgConfig = Figment::new() + .merge(Toml::string(DEFAULT_CONFIG)) + .extract() + .expect("default config deserializes"); + let defaults = ServerNgConfig::default(); + + assert_eq!(shipped.tcp.socket_migration, defaults.tcp.socket_migration); + assert_eq!( + shipped.system.partition.validate_checksum, + defaults.system.partition.validate_checksum + ); + assert_eq!( + shipped.system.segment.cache_indexes, + defaults.system.segment.cache_indexes + ); + assert_eq!( + shipped.system.logging.sysinfo_print_interval, + defaults.system.logging.sysinfo_print_interval + ); + assert_eq!(shipped.system.backup.path, defaults.system.backup.path); + assert_eq!( + shipped.system.backup.compatibility.path, + defaults.system.backup.compatibility.path + ); + assert_eq!( + shipped.system.compression.allow_override, + defaults.system.compression.allow_override + ); + assert_eq!( + shipped.system.state.enforce_fsync, + defaults.system.state.enforce_fsync + ); + assert_eq!( + shipped.system.state.max_file_operation_retries, + defaults.system.state.max_file_operation_retries + ); + assert_eq!( + shipped.system.state.retry_delay, + defaults.system.state.retry_delay + ); + assert_eq!( + shipped.consumer_group.rebalancing_check_interval, + defaults.consumer_group.rebalancing_check_interval + ); + assert_eq!( + shipped.message_saver.interval, + defaults.message_saver.interval + ); + assert_eq!( + shipped.message_saver.enforce_fsync, + defaults.message_saver.enforce_fsync + ); + } + + // http.enabled needs a non-ServerDefault JWT expiry to clear the sibling + // check above; ServerNgConfig::default() already satisfies that. + fn https_config(cert_file: &str, key_file: &str) -> ServerNgConfig { + let mut cfg = ServerNgConfig::default(); + cfg.http.enabled = true; + cfg.http.tls.enabled = true; + cfg.http.tls.cert_file = cert_file.to_string(); + cfg.http.tls.key_file = key_file.to_string(); + cfg + } + + #[test] + fn validate_rejects_tls_enabled_with_empty_cert_file() { + let cfg = https_config("", "key.pem"); + assert!(cfg.validate().is_err()); + } + + #[test] + fn validate_accepts_tls_enabled_with_both_files_set() { + let cfg = https_config("cert.pem", "key.pem"); + assert!(cfg.validate().is_ok()); + } + + fn cluster_node(replica_id: u8, http: Option) -> ClusterNodeConfig { + ClusterNodeConfig { + name: format!("node-{replica_id}"), + ip: "127.0.0.1".to_string(), + advertised_address: None, + advertised_addresses: Vec::new(), + replica_id, + ports: TransportPorts { + tcp: Some(8090 + u16::from(replica_id)), + quic: Some(8080 + u16::from(replica_id)), + http, + websocket: Some(8070 + u16::from(replica_id)), + tcp_replica: Some(9090 + u16::from(replica_id)), + }, + } + } + + fn clustered_http_config(nodes: Vec) -> ServerNgConfig { + let mut cfg = ServerNgConfig::default(); + cfg.http.enabled = true; + cfg.cluster.enabled = true; + cfg.cluster.name = "test-cluster".to_string(); + cfg.cluster.nodes = nodes; + cfg + } + + // Keyless cluster+http boots: forwarding degrades to off instead of + // failing the whole server. + #[test] + fn validate_accepts_cluster_http_without_jwt_secret_or_cluster_auth() { + let cfg = clustered_http_config(vec![ + cluster_node(0, Some(3000)), + cluster_node(1, Some(3001)), + ]); + assert!(cfg.validate().is_ok()); + } + + // Cluster mode has no port fallbacks, so a portless roster node is + // invalid even when forwarding is off (keyless). + #[test] + fn validate_rejects_keyless_cluster_http_with_portless_roster_node() { + let cfg = clustered_http_config(vec![cluster_node(0, Some(3000)), cluster_node(1, None)]); + assert!(cfg.validate().is_err()); + } + + // The explicit-port rule covers every enabled transport, not just http. + #[test] + fn validate_rejects_cluster_node_without_port_for_enabled_quic() { + let mut cfg = clustered_http_config(vec![ + cluster_node(0, Some(3000)), + cluster_node(1, Some(3001)), + ]); + cfg.quic.enabled = true; + cfg.cluster.nodes[1].ports.quic = None; + assert!(cfg.validate().is_err()); + } + + // A disabled transport never binds, so its roster port may stay unset. + #[test] + fn validate_accepts_cluster_node_without_port_for_disabled_quic() { + let mut cfg = clustered_http_config(vec![ + cluster_node(0, Some(3000)), + cluster_node(1, Some(3001)), + ]); + cfg.quic.enabled = false; + cfg.cluster.nodes[1].ports.quic = None; + assert!(cfg.validate().is_ok()); + } + + #[test] + fn validate_accepts_cluster_http_with_configured_jwt_secrets() { + let mut cfg = clustered_http_config(vec![ + cluster_node(0, Some(3000)), + cluster_node(1, Some(3001)), + ]); + cfg.http.jwt.encoding_secret = "0123456789abcdef0123456789abcdef".to_string(); + cfg.http.jwt.decoding_secret = "0123456789abcdef0123456789abcdef".to_string(); + assert!(cfg.validate().is_ok()); + } + + #[test] + fn validate_accepts_cluster_http_with_cluster_auth_as_jwt_key_source() { + let mut cfg = clustered_http_config(vec![ + cluster_node(0, Some(3000)), + cluster_node(1, Some(3001)), + ]); + cfg.cluster.auth.enabled = true; + cfg.cluster.auth.shared_secret = "0123456789abcdef0123456789abcdef".to_string(); + assert!(cfg.validate().is_ok()); + } +} diff --git a/core/configs/src/server_ng_config/websocket.rs b/core/configs/src/server_ng_config/websocket.rs new file mode 100644 index 0000000000..9bfed9b9ee --- /dev/null +++ b/core/configs/src/server_ng_config/websocket.rs @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Server-ng WebSocket listener schema. +//! +//! Unlike the legacy [`crate::websocket::WebSocketConfig`] it was +//! forked from, this section is the live frame-tuning source for +//! server-ng's WS / WSS plane: the message bus folds the +//! `Option` knobs below into a compio-ws +//! `WebSocketConfig` once at bus construction. The sizes are strictly +//! typed, so a malformed size string fails config load instead of +//! being silently ignored at conversion time. The conversion itself +//! lives in `core/message_bus` because the standalone `tungstenite` +//! dependency and the compio-ws re-export are different major versions +//! with incompatible config types. + +use configs::ConfigEnv; +use iggy_common::IggyByteSize; +use serde::{Deserialize, Serialize}; +use serde_with::{DisplayFromStr, serde_as}; +use std::fmt::{Display, Formatter}; + +#[serde_as] +#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] +pub struct WebSocketConfig { + pub enabled: bool, + pub address: String, + + /// Target minimum size of the frame read buffer. `None` keeps the + /// compio-ws default (currently 128 KiB). + #[config_env(leaf)] + #[serde(default)] + #[serde_as(as = "Option")] + pub read_buffer_size: Option, + + /// Target buffer size for batched writes before compio-ws flushes. + /// `None` keeps the compio-ws default (currently 128 KiB). + #[config_env(leaf)] + #[serde(default)] + #[serde_as(as = "Option")] + pub write_buffer_size: Option, + + /// Hard ceiling on the write buffer; writes past it error instead + /// of buffering. Must exceed [`Self::write_buffer_size`] by at + /// least one frame. `None` keeps the compio-ws default + /// (unlimited). + #[config_env(leaf)] + #[serde(default)] + #[serde_as(as = "Option")] + pub max_write_buffer_size: Option, + + /// Hard upper bound on a single inbound WebSocket message + /// (post-fragment-reassembly). `None` keeps the compio-ws default + /// (currently 64 MiB). + #[config_env(leaf)] + #[serde(default)] + #[serde_as(as = "Option")] + pub max_message_size: Option, + + /// Hard upper bound on a single inbound WebSocket frame + /// (pre-fragment-reassembly). `None` keeps the compio-ws default + /// (currently 16 MiB). + #[config_env(leaf)] + #[serde(default)] + #[serde_as(as = "Option")] + pub max_frame_size: Option, + + /// Whether to accept unmasked frames from clients in violation of + /// RFC 6455 client-to-server framing rules. Strict (`false`) by + /// default; enable only for non-browser test clients that emit + /// unmasked frames. + #[serde(default)] + pub accept_unmasked_frames: bool, + + #[serde(default)] + pub tls: WebSocketTlsConfig, +} + +#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] +pub struct WebSocketTlsConfig { + pub enabled: bool, + pub self_signed: bool, + pub cert_file: String, + pub key_file: String, +} + +impl Display for WebSocketConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ enabled: {}, address: {}, read_buffer_size: {:?}, write_buffer_size: {:?}, max_write_buffer_size: {:?}, max_message_size: {:?}, max_frame_size: {:?}, accept_unmasked_frames: {} }}", + self.enabled, + self.address, + self.read_buffer_size, + self.write_buffer_size, + self.max_write_buffer_size, + self.max_message_size, + self.max_frame_size, + self.accept_unmasked_frames + ) + } +} diff --git a/core/connectors/runtime/Cargo.toml b/core/connectors/runtime/Cargo.toml index c9b7f5a5eb..9533516678 100644 --- a/core/connectors/runtime/Cargo.toml +++ b/core/connectors/runtime/Cargo.toml @@ -29,6 +29,9 @@ repository = "https://github.com/apache/iggy" readme = "README.md" publish = false +[features] +vsr = ["iggy/vsr"] + [dependencies] async-trait = { workspace = true } axum = { workspace = true } diff --git a/core/consensus/Cargo.toml b/core/consensus/Cargo.toml index 342ce604d8..1ba5610c24 100644 --- a/core/consensus/Cargo.toml +++ b/core/consensus/Cargo.toml @@ -45,7 +45,6 @@ tracing = { workspace = true } twox-hash = { workspace = true } [dev-dependencies] -aligned-vec = { workspace = true } futures = { workspace = true } [lints.clippy] diff --git a/core/consensus/src/client_table.rs b/core/consensus/src/client_table.rs index 665053f990..4413b8643c 100644 --- a/core/consensus/src/client_table.rs +++ b/core/consensus/src/client_table.rs @@ -437,7 +437,7 @@ impl ClientTable { /// Resize the table to `max_clients` slots. Boot-only: reallocating a /// populated table would silently drop live sessions, so this must run - /// before any client registers (the server bootstrap applies the configured + /// before any client registers (server-ng bootstrap applies the configured /// `[metadata] clients_table_max` here). /// /// # Panics @@ -1019,12 +1019,7 @@ impl From for ClientTableWireError { } /// Format tag for [`ClientTable::encode`]; bump on layout change. -/// -/// That includes any `ReplyHeader` layout move -- cached replies are embedded -/// as raw wire bytes, so an artifact written under an older header layout must -/// be refused, not silently misread. `ICT2`: `status` sits at offset 216 (the -/// pre-`ICT2` layout carried a `namespace` word before it). -pub const CLIENT_TABLE_MAGIC: [u8; 4] = *b"ICT2"; +pub const CLIENT_TABLE_MAGIC: [u8; 4] = *b"ICT1"; /// Per-entry fixed fields in the wire encoding: `client(u128) epoch(u64) /// user_id(u32) watermark(u64) watermark_checksum(u128) ring_len(u8)`. diff --git a/core/consensus/src/dvc_merge.rs b/core/consensus/src/dvc_merge.rs deleted file mode 100644 index 19bda15e3c..0000000000 --- a/core/consensus/src/dvc_merge.rs +++ /dev/null @@ -1,981 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Merging a `DoViewChange` quorum into the new view's log: for every op that -//! might be uncommitted, does the new view keep it or discard it? -//! -//! Keeping an op that was never committed costs a wasted slot. Discarding one -//! that WAS committed loses acknowledged client data, so the only proof accepted -//! for discarding is a nack quorum: enough replicas stating they never prepared -//! it that a replication quorum provably never formed. Absent that the op is -//! kept, and if no replica offers its body the view does not start. Stalling is -//! visible and recoverable; losing the op is neither. - -#[cfg(test)] -use crate::view_change_quorum::DvcSuffix; -use crate::view_change_quorum::{DvcQuorumArray, StoredDvc, dvc_count, dvc_iter}; -use iggy_binary_protocol::{CHECKSUM_UNSEALED, PrepareHeader}; - -/// Sizes the merge needs from the replica. -#[derive(Debug, Clone, Copy)] -pub struct MergeQuorums { - /// `DoViewChange` messages needed before a view may start. - pub view_change: usize, - /// Nacks needed to prove an op uncommitted, so it may be discarded. - pub nack_prepare: usize, - /// Cluster size, which bounds how many more DVCs could still arrive. - pub replica_count: usize, - /// Cluster-wide pipeline ceiling: an op further than this below a sender's head - /// cannot still be uncommitted, since no node could have kept it in flight. - /// - /// NOT this node's configured depth. The bound applies to a *peer's* head op, - /// and a local depth larger than that peer's manufactures a commit the peer - /// never made. Every node's depth is pinned below `DVC_HEADERS_MAX`, so the - /// ceiling holds for all of them. - pub prepare_queue_ceiling: u64, -} - -/// What the collected DVCs say about starting the view. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum MergeOutcome { - /// Fewer than `view_change` DVCs so far. - AwaitingQuorum, - /// Quorum is in, but some op is neither provably dead nor recoverable and an - /// unreported replica could still settle it. Wait for them. - AwaitingRepair { - /// The op that cannot yet be decided. - undecided_op: u64, - }, - /// Every replica reported and an op is still neither provably dead nor - /// recoverable. No further message changes that: data loss already happened, - /// and truncating here would turn it from detected into silent. - Deadlocked { - /// The op that cannot be decided. - undecided_op: u64, - }, - /// The view can start. - Ready(MergedLog), -} - -/// The log the new primary adopts. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct MergedLog { - /// New head op. Below the highest op any canonical sender reported when a - /// nack quorum proved the ops above it dead. - pub op_head: u64, - /// Highest op the quorum proves committed. The merge never discards at or - /// below this. - pub commit_max: u64, - /// Canonical headers for `commit_max..=op_head`, ordered high-to-low op. - /// The new primary installs these over its own log. - pub headers: Vec, - /// Headers non-canonical senders report committed and the canonical chain - /// corroborates. See [`committed_elsewhere`]. - pub committed_elsewhere: Vec, -} - -/// Highest op the quorum proves committed. -/// -/// Three independent lower bounds, because a single sender's view of the commit -/// point can lag arbitrarily while the cluster's cannot: -/// * each sender's own reported commit, -/// * the `commit` its head prepare carries, stamped by that op's primary, -/// * its head minus the cluster-wide pipeline ceiling, since nothing further back -/// than one pipeline can still be in flight. -/// -/// The lowest op in a sender's suffix is deliberately NOT a fourth bound, and -/// re-adding one is a data-loss bug. It would be sound only if the suffix floor -/// were the commit point by construction; here it is computed, and two paths in -/// `build_dvc_suffix` raise it above the sender's commit (ops are 1-based, so -/// commit 0 floors at op 1; the `DVC_HEADERS_MAX` clamp drops the bottom of an -/// over-wide window). Nothing on the wire distinguishes the two. -/// -/// Nothing is lost by omitting it: the snapshot is tagged with the `(op, commit)` -/// the header is stamped with and dropped on a mismatch, so the floor either -/// equals `dvc.commit`, already the first bound, or exceeds it, the unsound case. -#[must_use] -pub fn merge_commit_max(quorum: &DvcQuorumArray, prepare_queue_ceiling: u64) -> u64 { - let mut commit_max = 0; - for dvc in dvc_iter(quorum) { - commit_max = commit_max.max(dvc.commit); - commit_max = commit_max.max(dvc.op.saturating_sub(prepare_queue_ceiling)); - if let Some(head) = dvc.suffix.headers().first() { - commit_max = commit_max.max(head.commit); - } - } - commit_max -} - -/// Highest `log_view` any sender reported. -/// -/// Senders at this `log_view` were in every earlier view change, so their headers -/// already reflect the truncations those views decided. That makes them canonical, -/// and a lower-`log_view` sender disagreeing is evidence against its own header. -fn log_view_canonical(quorum: &DvcQuorumArray) -> Option { - dvc_iter(quorum).map(|dvc| dvc.log_view).max() -} - -/// Per-op tally over the whole quorum. -struct OpVerdict<'a> { - canonical: Option<&'a PrepareHeader>, - /// Senders holding the canonical header AND able to serve its body. - copies: usize, - nacks: usize, - /// Canonical senders disagree about this op, so no header here is trustworthy. - conflict: bool, -} - -/// What the canonical senders say about one op. -struct CanonicalAt<'a> { - header: Option<&'a PrepareHeader>, - /// Two senders at the canonical `log_view` disagree here. - conflict: bool, -} - -/// The canonical header at `op`, and whether the canonical senders agree. -/// -/// Every canonical sender is consulted, not just the first: they were all in -/// normal status in that view and a primary prepares one thing per op, so a -/// disagreement is not a vote but proof that one header is wrong with no way to -/// tell which. The caller treats the op as undecidable. -fn canonical_header_at<'a>(canonical_senders: &[&'a StoredDvc], op: u64) -> CanonicalAt<'a> { - let mut header: Option<&'a PrepareHeader> = None; - let mut conflict = false; - for dvc in canonical_senders { - let Some(index) = dvc.suffix.index_of(dvc.op, op) else { - continue; - }; - let Some(candidate) = dvc.suffix.valid_header_at(index) else { - continue; - }; - match header { - Some(existing) if existing.checksum != candidate.checksum => conflict = true, - Some(_) => {} - None => header = Some(candidate), - } - } - CanonicalAt { header, conflict } -} - -/// Tally every sender's position on `op`. -fn tally_op<'a>( - quorum: &'a DvcQuorumArray, - canonical_senders: &[&'a StoredDvc], - canonical_log_view: u32, - op: u64, -) -> OpVerdict<'a> { - let CanonicalAt { - header: canonical, - conflict, - } = canonical_header_at(canonical_senders, op); - let mut copies = 0; - let mut nacks = 0; - - for dvc in dvc_iter(quorum) { - // The sender's log stops below this op, so it never prepared it. - if dvc.op < op { - nacks += 1; - continue; - } - let Some(index) = dvc.suffix.index_of(dvc.op, op) else { - // The sender said nothing about this op, so it abstains: no nack, no - // copy. Counting silence as a nack is sound only when every DVC carries - // headers, since then falling outside a window means being above it. - // Two cases here are not, and abstaining costs a slower view change - // where nacking costs data. - // - // An empty suffix is not a vote. It comes from a replica with - // nothing uncommitted, or one whose snapshot no longer matches its - // log; reading it as - // agreement with someone else's nack discards a committed op on one real - // nack plus one silence. And with `dvc.op >= op` established above, a - // non-empty suffix not covering `op` puts `op` below the sender's window - // floor, at or below its own commit point, so nacking it is backwards. - // Only the defensive `DVC_HEADERS_MAX` clamp reaches that. - continue; - }; - - let held = dvc.suffix.valid_header_at(index); - if let (Some(held), Some(canonical)) = (held, canonical) - && dvc.suffix.offers_body(index) - && held.checksum == canonical.checksum - { - copies += 1; - } - - if dvc.suffix.nacks(index) { - // Explicit: the sender proves it never prepared this op. - nacks += 1; - } else if let Some(held) = held { - // Only a sender BEHIND the canonical log_view can implicitly nack. - // Without this, corrupting one canonical header in transit turns every - // honest sender's correct header into an implicit nack against the - // garbage: a nack quorum on three replicas. A same-log_view - // disagreement is evidence, not a vote, and goes through `conflict`. - let may_nack_implicitly = dvc.log_view < canonical_log_view; - match canonical { - // Implicit: the sender holds a DIFFERENT prepare, so not this one. - Some(canonical) if may_nack_implicitly && held.checksum != canonical.checksum => { - nacks += 1; - } - // Implicit: no canonical sender holds anything here, so a newer - // view already truncated this op and the sender holds a corpse. - None if may_nack_implicitly => nacks += 1, - _ => {} - } - } - } - - OpVerdict { - canonical, - copies, - nacks, - conflict, - } -} - -/// Collect the canonical headers for `commit_max..=op_head`, high-to-low. -/// -/// Checks the hash chain as it walks: a break means the canonical senders agreed -/// on individual ops but not on one history, which no later step would notice. -fn canonical_headers( - canonical_senders: &[&StoredDvc], - op_head: u64, - commit_max: u64, -) -> Option> { - if op_head == 0 { - return Some(Vec::new()); - } - let floor = commit_max.max(1); - let mut headers = Vec::new(); - let mut child: Option = None; - let mut op = op_head; - loop { - let at = canonical_header_at(canonical_senders, op); - if at.conflict { - return None; - } - let header = *at.header?; - if let Some(child) = child - && child.parent != header.checksum - { - tracing::error!( - op, - child_op = child.op, - "view-change headers do not hash-chain; refusing to install" - ); - return None; - } - child = Some(header); - headers.push(header); - if op <= floor { - break; - } - op -= 1; - } - Some(headers) -} - -/// Headers a non-canonical sender reports committed, corroborated by the canonical -/// chain. -/// -/// Needed because header repair stops at a gap, so an op missing below the new -/// primary's commit point can never be repaired into place. -/// -/// But a sender's `commit` is not proof: `commit_max` advances from the primary's -/// claim without checking the local log matches, and reconcile leaves a divergent -/// entry at or below the applied floor in place. So a replica can honestly report -/// `commit >= N` holding a header at N that never committed. Nothing journals these, -/// but they pin the repair gate: the genuine repaired prepare then mismatches and is -/// discarded, and the view change stalls at N. -/// -/// So each candidate must be the `parent` the entry one op above names, walking down -/// from the canonical window: the canonical senders vouch, not the offering sender. -/// One that chains to nothing is dropped, leaving the op unconstrained for repair -/// rather than pinned to an unconfirmable header. Unsealed on either side passes. -/// -/// `None` when two senders report *different* prepares committed at one op: as in -/// [`canonical_header_at`], undecidable. -fn committed_elsewhere( - quorum: &DvcQuorumArray, - canonical_log_view: u32, - already_installed: &[PrepareHeader], -) -> Option> { - let mut claimed: Vec = Vec::new(); - for dvc in dvc_iter(quorum).filter(|dvc| dvc.log_view < canonical_log_view) { - for (index, header) in dvc.suffix.headers().iter().enumerate() { - if header.op > dvc.commit { - continue; - } - if dvc.suffix.valid_header_at(index).is_none() { - continue; - } - if already_installed - .iter() - .any(|installed| installed.op == header.op) - { - continue; - } - if let Some(queued) = claimed.iter().find(|queued| queued.op == header.op) { - if queued.checksum != header.checksum { - tracing::error!( - op = header.op, - "replicas disagree about the prepare committed at op {}; refusing to \ - choose one to install", - header.op - ); - return None; - } - continue; - } - claimed.push(*header); - } - } - - // Descending, so the entry a candidate must parent is already canonical or - // already corroborated. - claimed.sort_unstable_by_key(|header| std::cmp::Reverse(header.op)); - let mut extra: Vec = Vec::new(); - for header in claimed { - let child = already_installed - .iter() - .chain(extra.iter()) - .find(|candidate| candidate.op == header.op + 1); - let Some(child) = child else { - tracing::warn!( - op = header.op, - "op {} reported committed but nothing in the view's log chains to it; \ - leaving it unconstrained for repair", - header.op - ); - continue; - }; - if child.parent != header.checksum - && child.parent != CHECKSUM_UNSEALED - && header.checksum != CHECKSUM_UNSEALED - { - tracing::warn!( - op = header.op, - child_op = child.op, - "op {} reported committed but the entry above does not name it as parent; \ - dropping the claim", - header.op - ); - continue; - } - extra.push(header); - } - Some(extra) -} - -/// Decide whether the new view can start, and with what log. -/// -/// Walks every op that might be uncommitted, from the proven commit point to the -/// highest a canonical sender reported, stopping at the first proved dead. -#[must_use] -pub fn merge_dvc_quorum(quorum: &DvcQuorumArray, quorums: MergeQuorums) -> MergeOutcome { - let received = dvc_count(quorum); - if received < quorums.view_change { - return MergeOutcome::AwaitingQuorum; - } - - let Some(canonical_log_view) = log_view_canonical(quorum) else { - return MergeOutcome::AwaitingQuorum; - }; - let canonical_senders: Vec<&StoredDvc> = dvc_iter(quorum) - .filter(|dvc| dvc.log_view == canonical_log_view) - .collect(); - debug_assert!( - !canonical_senders.is_empty(), - "the max log_view must be held by at least one sender" - ); - - let commit_max = merge_commit_max(quorum, quorums.prepare_queue_ceiling); - let op_head_max = canonical_senders - .iter() - .map(|dvc| dvc.op) - .max() - .unwrap_or(commit_max) - .max(commit_max); - - if op_head_max == 0 { - // Nothing was ever prepared, so nothing to decide. Ops are 1-based, and - // scanning op 0 reads every sender's absent entry as a nack, manufacturing - // a nack quorum for an op that does not exist. - return MergeOutcome::Ready(MergedLog { - op_head: 0, - commit_max: 0, - headers: Vec::new(), - committed_elsewhere: Vec::new(), - }); - } - - let mut op_head = op_head_max; - // Start at the proven commit point, or op 1 when nothing is committed. The - // commit point is scanned so the adopted log is anchored on a servable header. - let mut op = commit_max.max(1); - while op <= op_head_max { - let verdict = tally_op(quorum, &canonical_senders, canonical_log_view, op); - - if verdict.nacks >= quorums.nack_prepare { - if op <= commit_max { - // A nack quorum for a committed op is impossible under quorum - // intersection, so a peer lied or a bitset is wrong. Refuse the - // view rather than assert, so one bad peer stalls the group instead - // of panicking a node into a restart loop. - tracing::error!( - op, - commit_max, - nacks = verdict.nacks, - nack_quorum = quorums.nack_prepare, - "nack quorum for an op the quorum proves committed; refusing the view" - ); - return MergeOutcome::Deadlocked { undecided_op: op }; - } - op_head = op - 1; - break; - } - - if verdict.conflict { - // Senders all in normal status in the same view disagree about what it - // prepared here. One header is wrong with no way to tell which, so - // refuse rather than pick. - tracing::error!( - op, - log_view = canonical_log_view, - "replicas at the same log_view disagree about op {op}; refusing to choose a \ - canonical header for it" - ); - return if received < quorums.replica_count { - MergeOutcome::AwaitingRepair { undecided_op: op } - } else { - MergeOutcome::Deadlocked { undecided_op: op } - }; - } - - if verdict.canonical.is_none() || verdict.copies == 0 { - // Neither provably dead nor recoverable. An outstanding replica may - // hold the body or supply the deciding nack. - return if received < quorums.replica_count { - MergeOutcome::AwaitingRepair { undecided_op: op } - } else { - tracing::error!( - op, - canonical = verdict.canonical.is_some(), - copies = verdict.copies, - nacks = verdict.nacks, - nack_quorum = quorums.nack_prepare, - "every replica reported and op {op} is neither recoverable nor provably \ - uncommitted; the view cannot start" - ); - MergeOutcome::Deadlocked { undecided_op: op } - }; - } - - op += 1; - } - - debug_assert!(op_head >= commit_max); - let Some(headers) = canonical_headers(&canonical_senders, op_head, commit_max) else { - return MergeOutcome::Deadlocked { - undecided_op: op_head, - }; - }; - let Some(committed_elsewhere) = committed_elsewhere(quorum, canonical_log_view, &headers) - else { - // Two senders disagree about a committed op. Not a repair problem: these - // install without a nack quorum, and no message resolves which is true. - return MergeOutcome::Deadlocked { - undecided_op: op_head, - }; - }; - - MergeOutcome::Ready(MergedLog { - op_head, - commit_max, - headers, - committed_elsewhere, - }) -} - -/// Build a suffix for a sender that holds every op in `commit..=op` with a -/// servable body. Test helper. -#[cfg(test)] -#[must_use] -pub fn suffix_all_present(headers: Vec) -> DvcSuffix { - // `1 << 128` overflows, and a full-width suffix is exactly what the clamp - // produces, so the widest case cannot use the shift. Past the width is left to - // `DvcSuffix::new`, which rejects it by name rather than as an overflow. - let count = u32::try_from(headers.len()).unwrap_or(u32::MAX).min(128); - let mask = u128::MAX.checked_shr(128 - count).unwrap_or(0); - DvcSuffix::new(headers, 0, mask) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::DVC_HEADERS_MAX; - use crate::view_change_quorum::{dvc_blank, dvc_quorum_array_empty, dvc_record}; - use iggy_binary_protocol::{Command2, Operation}; - - /// Three replicas: replication 2, view-change 2, nack 2. - fn quorums_r3() -> MergeQuorums { - MergeQuorums { - view_change: 2, - nack_prepare: 2, - replica_count: 3, - prepare_queue_ceiling: 32, - } - } - - /// A prepare whose checksum derives from its op, so the hash chain connects. - fn prepare(op: u64, view: u32) -> PrepareHeader { - PrepareHeader { - command: Command2::Prepare, - operation: Operation::CreateStream, - op, - view, - checksum: u128::from(op) | (u128::from(view) << 64), - parent: if op <= 1 { - 0 - } else { - u128::from(op - 1) | (u128::from(view) << 64) - }, - // Left at zero so each test drives `commit_max` through the bound it is - // about; `merge_commit_max` honours this field, covered separately below. - commit: 0, - ..Default::default() - } - } - - /// Headers for `low..=high`, ordered high-to-low as a suffix requires. - fn suffix_headers(low: u64, high: u64, view: u32) -> Vec { - (low..=high).rev().map(|op| prepare(op, view)).collect() - } - - fn dvc(replica: u8, log_view: u32, op: u64, commit: u64, suffix: DvcSuffix) -> StoredDvc { - StoredDvc { - replica, - log_view, - op, - commit, - suffix, - } - } - - #[test] - fn given_agreeing_quorum_when_merging_should_adopt_the_shared_head() { - let mut quorum = dvc_quorum_array_empty(); - dvc_record( - &mut quorum, - dvc(0, 1, 5, 3, suffix_all_present(suffix_headers(3, 5, 1))), - ); - dvc_record( - &mut quorum, - dvc(1, 1, 5, 3, suffix_all_present(suffix_headers(3, 5, 1))), - ); - - let MergeOutcome::Ready(log) = merge_dvc_quorum(&quorum, quorums_r3()) else { - panic!("an agreeing quorum must be ready"); - }; - assert_eq!(log.op_head, 5); - assert_eq!(log.commit_max, 3); - assert_eq!( - log.headers.iter().map(|h| h.op).collect::>(), - vec![5, 4, 3], - "headers run high-to-low from the head down to commit_max" - ); - } - - #[test] - fn given_nack_quorum_above_commit_when_merging_should_truncate_to_the_nacked_op() { - // Both survivors hold 1..=3 and never prepared 4, so the head drops to 3. - let mut quorum = dvc_quorum_array_empty(); - let mut headers = suffix_headers(2, 4, 1); - headers[0] = dvc_blank(4); - let nack_op_four = DvcSuffix::new(headers.clone(), 0b001, 0b110); - dvc_record(&mut quorum, dvc(0, 1, 4, 2, nack_op_four.clone())); - dvc_record(&mut quorum, dvc(1, 1, 4, 2, nack_op_four)); - - let MergeOutcome::Ready(log) = merge_dvc_quorum(&quorum, quorums_r3()) else { - panic!("a nack quorum must decide the view"); - }; - assert_eq!(log.op_head, 3, "op 4 is provably uncommitted"); - assert!(log.headers.iter().all(|header| header.op <= 3)); - } - - #[test] - fn given_one_nack_short_of_quorum_when_merging_should_keep_the_op() { - // Replica 0 never saw op 4; replica 1 holds it and can serve it. One nack - // is short of the quorum of 2, so op 4 survives. - let mut quorum = dvc_quorum_array_empty(); - let mut holed = suffix_headers(2, 4, 1); - holed[0] = dvc_blank(4); - dvc_record( - &mut quorum, - dvc(0, 1, 4, 2, DvcSuffix::new(holed, 0b001, 0b110)), - ); - dvc_record( - &mut quorum, - dvc(1, 1, 4, 2, suffix_all_present(suffix_headers(2, 4, 1))), - ); - - let MergeOutcome::Ready(log) = merge_dvc_quorum(&quorum, quorums_r3()) else { - panic!("one nack must not decide the view"); - }; - assert_eq!(log.op_head, 4, "a single nack cannot discard op 4"); - assert!( - log.headers.iter().any(|header| header.op == 4), - "the surviving op must be installed" - ); - } - - #[test] - fn given_committed_op_when_nacked_by_quorum_should_refuse_rather_than_truncate() { - // A nack quorum at or below the proven commit point is impossible under - // quorum intersection. If it appears anyway, refuse; never truncate. - let mut quorum = dvc_quorum_array_empty(); - let blanks = vec![dvc_blank(3)]; - let all_nacked = DvcSuffix::new(blanks, 0b1, 0b0); - dvc_record(&mut quorum, dvc(0, 1, 3, 3, all_nacked.clone())); - dvc_record(&mut quorum, dvc(1, 1, 3, 3, all_nacked)); - - assert_eq!( - merge_dvc_quorum(&quorum, quorums_r3()), - MergeOutcome::Deadlocked { undecided_op: 3 }, - "a committed op must never be truncated" - ); - } - - #[test] - fn given_blank_commit_point_from_every_sender_should_deadlock() { - // The commit point is scanned and may not be discarded, so a sender that - // reports it blank is deferring to a peer. When every sender defers there - // is no peer left and the view cannot start. - // - // Nothing in the merge can rescue this, which is why the senders must not - // produce it: a replica keeps the header at its own commit point through - // compaction (the metadata checkpoint drain stops one op short, a - // partition answers from its evicted ring). - let mut quorum = dvc_quorum_array_empty(); - let blank_at_commit = DvcSuffix::new(vec![dvc_blank(5)], 0, 0); - for replica in 0..3 { - dvc_record(&mut quorum, dvc(replica, 1, 5, 5, blank_at_commit.clone())); - } - - assert_eq!( - merge_dvc_quorum(&quorum, quorums_r3()), - MergeOutcome::Deadlocked { undecided_op: 5 }, - "a blank commit point is neither adoptable nor discardable" - ); - } - - #[test] - fn given_blank_commit_point_from_one_sender_should_adopt_the_peer_header() { - // The same suffix stops being fatal the moment one sender still holds the - // header: that one is canonical and serves the body, and the deferring - // sender neither nacks it nor conflicts with it. - let mut quorum = dvc_quorum_array_empty(); - dvc_record( - &mut quorum, - dvc(0, 1, 5, 5, DvcSuffix::new(vec![dvc_blank(5)], 0, 0)), - ); - dvc_record( - &mut quorum, - dvc(1, 1, 5, 5, suffix_all_present(suffix_headers(5, 5, 1))), - ); - - let MergeOutcome::Ready(log) = merge_dvc_quorum(&quorum, quorums_r3()) else { - panic!("one surviving copy of the commit point is enough to start the view"); - }; - assert_eq!(log.op_head, 5); - assert_eq!(log.commit_max, 5); - } - - #[test] - fn given_header_without_a_servable_body_when_replicas_outstanding_should_await_repair() { - // Both senders have op 4's header, neither can serve its body, and replica 2 - // has not reported. A head whose body nobody holds would wedge the view. - let mut quorum = dvc_quorum_array_empty(); - let headers = suffix_headers(2, 4, 1); - let header_only = DvcSuffix::new(headers, 0, 0b110); - dvc_record(&mut quorum, dvc(0, 1, 4, 2, header_only.clone())); - dvc_record(&mut quorum, dvc(1, 1, 4, 2, header_only)); - - assert_eq!( - merge_dvc_quorum(&quorum, quorums_r3()), - MergeOutcome::AwaitingRepair { undecided_op: 4 } - ); - } - - #[test] - fn given_all_replicas_reported_and_op_undecidable_should_deadlock() { - let mut quorum = dvc_quorum_array_empty(); - let headers = suffix_headers(2, 4, 1); - let header_only = DvcSuffix::new(headers, 0, 0b110); - dvc_record(&mut quorum, dvc(0, 1, 4, 2, header_only.clone())); - dvc_record(&mut quorum, dvc(1, 1, 4, 2, header_only.clone())); - dvc_record(&mut quorum, dvc(2, 1, 4, 2, header_only)); - - assert_eq!( - merge_dvc_quorum(&quorum, quorums_r3()), - MergeOutcome::Deadlocked { undecided_op: 4 }, - "with every replica in, an unrecoverable op stalls the view forever" - ); - } - - #[test] - fn given_lower_log_view_sender_when_merging_should_prefer_the_canonical_log() { - // Replica 1 is at the newer log_view, so its op 4 is canonical and - // replica 0's stale op 4 counts as an implicit nack against itself. - let mut quorum = dvc_quorum_array_empty(); - dvc_record( - &mut quorum, - dvc(0, 1, 4, 2, suffix_all_present(suffix_headers(2, 4, 1))), - ); - dvc_record( - &mut quorum, - dvc(1, 2, 4, 2, suffix_all_present(suffix_headers(2, 4, 2))), - ); - - let MergeOutcome::Ready(log) = merge_dvc_quorum(&quorum, quorums_r3()) else { - panic!("the canonical log must win"); - }; - assert_eq!(log.op_head, 4); - assert!( - log.headers.iter().all(|header| header.view == 2), - "installed headers must come from the canonical log_view" - ); - } - - #[test] - fn given_committed_header_on_a_stale_sender_should_be_installed_anyway() { - // Replica 0 is behind on log_view but reports op 2 committed. The canonical - // window starts at 3, so op 2 is otherwise unreachable across the gap. Its - // header is genuine (a `view` stamp is when the entry was appended, not the - // sender's `log_view`), so op 3 names it as parent. - let mut quorum = dvc_quorum_array_empty(); - dvc_record( - &mut quorum, - dvc(0, 1, 2, 2, suffix_all_present(suffix_headers(2, 2, 2))), - ); - dvc_record( - &mut quorum, - dvc(1, 2, 4, 3, suffix_all_present(suffix_headers(3, 4, 2))), - ); - - let MergeOutcome::Ready(log) = merge_dvc_quorum(&quorum, quorums_r3()) else { - panic!("the view must start"); - }; - assert!( - log.committed_elsewhere.iter().any(|header| header.op == 2), - "a committed header from a stale sender must still be installed" - ); - } - - #[test] - fn given_a_committed_claim_the_canonical_log_does_not_chain_to_should_be_dropped() { - // A replica can honestly report `commit >= 2` while holding a header at 2 - // that never committed. Installing it pins the repair gate: the genuine - // repaired prepare then mismatches, and the view change stalls at op 2. - let mut quorum = dvc_quorum_array_empty(); - let mut impostor = prepare(2, 2); - impostor.checksum ^= 0xFF; - dvc_record( - &mut quorum, - dvc(0, 1, 2, 2, suffix_all_present(vec![impostor])), - ); - dvc_record( - &mut quorum, - dvc(1, 2, 4, 3, suffix_all_present(suffix_headers(3, 4, 2))), - ); - - let MergeOutcome::Ready(log) = merge_dvc_quorum(&quorum, quorums_r3()) else { - panic!("the view must start"); - }; - assert!( - log.committed_elsewhere.is_empty(), - "a claim the canonical log does not chain to must not pin the repair gate" - ); - } - - #[test] - fn given_a_corrupted_canonical_header_should_not_let_honest_senders_nack_it() { - // Transit corruption of one canonical sender's suffix entry must not turn - // every other sender's CORRECT header at that op into an implicit nack - // against the garbage, reaching a nack quorum through the one path that does - // not verify. Senders at the canonical log_view cannot legitimately disagree, - // since a primary prepares one thing per op, so it is evidence, not a vote. - let mut quorum = dvc_quorum_array_empty(); - - let mut corrupted = suffix_headers(2, 4, 1); - corrupted[0].checksum ^= 0xFFFF; - dvc_record(&mut quorum, dvc(0, 1, 4, 2, suffix_all_present(corrupted))); - // Two honest senders at the same log_view holding the real op 4. - dvc_record( - &mut quorum, - dvc(1, 1, 4, 2, suffix_all_present(suffix_headers(2, 4, 1))), - ); - dvc_record( - &mut quorum, - dvc(2, 1, 4, 2, suffix_all_present(suffix_headers(2, 4, 1))), - ); - - let outcome = merge_dvc_quorum(&quorum, quorums_r3()); - if let MergeOutcome::Ready(log) = &outcome { - assert_eq!( - log.op_head, 4, - "op 4 is held by two honest senders and must not be discarded" - ); - } - assert!( - !matches!(&outcome, MergeOutcome::Ready(log) if log.op_head < 4), - "a corrupted canonical header must never authorise truncating op 4, got {outcome:?}" - ); - } - - #[test] - fn given_a_mixed_version_quorum_when_one_upgraded_sender_nacks_should_not_truncate() { - // A silent sender must never count as agreement with someone else's nack. - // Only replica 1 proves it never held op 4; replica 0 sends no suffix and so - // says nothing. One real nack is short of the quorum of two, so op 4 has to - // survive -- reading the empty suffix as a second nack discards it. - let mut quorum = dvc_quorum_array_empty(); - dvc_record(&mut quorum, dvc(0, 1, 4, 2, DvcSuffix::empty())); - let mut holed = suffix_headers(2, 4, 1); - holed[0] = dvc_blank(4); - dvc_record( - &mut quorum, - dvc(1, 1, 4, 2, DvcSuffix::new(holed, 0b001, 0b110)), - ); - dvc_record( - &mut quorum, - dvc(2, 1, 4, 2, suffix_all_present(suffix_headers(2, 4, 1))), - ); - - let MergeOutcome::Ready(log) = merge_dvc_quorum(&quorum, quorums_r3()) else { - panic!("op 4 is recoverable from replica 2, so the view must start"); - }; - assert_eq!( - log.op_head, 4, - "an empty suffix must not stand in for the second nack" - ); - } - - #[test] - fn given_a_clamped_suffix_floor_when_merging_should_not_raise_commit_max() { - // `build_dvc_suffix` clamps a window wider than `DVC_HEADERS_MAX` from below, - // so its floor stops being the sender's commit point with nothing on the wire - // saying so. Reading that floor as a commit point marks every op between the - // real commit and the clamp committed: applied and replied to without a - // replication quorum, and unreachable by later truncation via `Deadlocked`. - let mut quorum = dvc_quorum_array_empty(); - // Sender at op 400, commit 200, whose window clamped to 273..=400. - let clamped = suffix_headers(273, 400, 1); - assert_eq!(clamped.len(), DVC_HEADERS_MAX); - dvc_record( - &mut quorum, - dvc(0, 1, 400, 200, suffix_all_present(clamped.clone())), - ); - dvc_record( - &mut quorum, - dvc(1, 1, 400, 200, suffix_all_present(clamped)), - ); - - // A ceiling wide enough not to bind, so this asserts the floor rule alone. In - // production the ceiling is `PREPARE_QUEUE_CEILING`, which already puts - // `commit_max` at or above any clamped floor. - assert_eq!( - merge_commit_max(&quorum, 1000), - 200, - "a clamped window floor is a scan bound, not a proven commit point" - ); - } - - #[test] - fn given_a_sender_at_commit_zero_when_merging_should_not_commit_op_one() { - // The other floor-raising path: ops are 1-based, so a sender with nothing - // committed still floors its window at op 1. Every sender says commit 0, so - // treating op 1 as committed would ack a client for an unheld op. - let mut quorum = dvc_quorum_array_empty(); - let floored = suffix_headers(1, 1, 1); - dvc_record( - &mut quorum, - dvc(0, 1, 1, 0, suffix_all_present(floored.clone())), - ); - dvc_record(&mut quorum, dvc(1, 1, 1, 0, suffix_all_present(floored))); - - assert_eq!( - merge_commit_max(&quorum, 32), - 0, - "nothing is committed, so the merge must prove nothing committed" - ); - } - - #[test] - fn given_disagreeing_committed_elsewhere_headers_should_refuse_the_view() { - // These headers install unconditionally, with no nack quorum behind them, so - // first-wins is the defect `canonical_header_at` refuses for the canonical - // range: only one of two committed claims can be true. - let mut quorum = dvc_quorum_array_empty(); - // Canonical sender at the higher log_view. Its window starts at the proven - // commit point, so ops below it come only from a stale sender. - dvc_record( - &mut quorum, - dvc(0, 2, 6, 5, suffix_all_present(suffix_headers(5, 6, 2))), - ); - // Two senders behind on log_view but level on op, so no nack and no conflict - // inside the canonical range. They disagree only at op 3, which both report - // committed; `prepare` derives the checksum from `(op, view)`. - dvc_record( - &mut quorum, - dvc(1, 1, 6, 5, suffix_all_present(suffix_headers(3, 6, 2))), - ); - let mut divergent = suffix_headers(3, 6, 2); - *divergent.last_mut().expect("suffix is non-empty") = prepare(3, 5); - dvc_record(&mut quorum, dvc(2, 1, 6, 5, suffix_all_present(divergent))); - - assert!( - matches!( - merge_dvc_quorum(&quorum, quorums_r3()), - MergeOutcome::Deadlocked { .. } - ), - "two committed claims at one op must refuse the view, not pick one" - ); - } - - #[test] - fn given_head_header_claiming_a_higher_commit_should_raise_commit_max() { - // The head prepare's `commit` was stamped by the primary that prepared it, so - // it proves a commit point even when every sender's own tracking lags. - // Without it the merge rescans committed ops and could accept nacks for them. - let mut quorum = dvc_quorum_array_empty(); - let mut headers = suffix_headers(2, 4, 1); - headers[0].commit = 3; - dvc_record( - &mut quorum, - dvc(0, 1, 4, 2, suffix_all_present(headers.clone())), - ); - dvc_record(&mut quorum, dvc(1, 1, 4, 2, suffix_all_present(headers))); - - assert_eq!( - merge_commit_max(&quorum, 32), - 3, - "the head header's commit field is a commit_max lower bound" - ); - } -} diff --git a/core/consensus/src/impls.rs b/core/consensus/src/impls.rs index e55c9c3b9f..d23378f154 100644 --- a/core/consensus/src/impls.rs +++ b/core/consensus/src/impls.rs @@ -18,25 +18,24 @@ use crate::oneshot::{self, Receiver, Sender}; use crate::vsr_timeout::{TimeoutKind, TimeoutManager}; use crate::{ - AckLogEvent, Consensus, ControlActionLogEvent, DvcQuorumArray, DvcSuffix, IgnoreReason, - MergeOutcome, MergeQuorums, MergedLog, Pipeline, PlaneKind, PrepareLogEvent, Project, - ReplicaLogContext, SimEventKind, StoredDvc, ViewChangeLogEvent, ViewChangeReason, VsrState, - dvc_count, dvc_iter, dvc_quorum_array_empty, dvc_record, dvc_reset, dvc_suffix_decode, - emit_replica_event, emit_sim_event, merge_dvc_quorum, seal_prepare_checksum, + AckLogEvent, Consensus, ControlActionLogEvent, DvcQuorumArray, IgnoreReason, Pipeline, + PlaneKind, PrepareLogEvent, Project, ReplicaLogContext, SimEventKind, StoredDvc, + ViewChangeLogEvent, ViewChangeReason, VsrState, dvc_count, dvc_max_commit, + dvc_quorum_array_empty, dvc_record, dvc_reset, dvc_select_winner, emit_replica_event, + emit_sim_event, }; use bit_set::BitSet; use clock::{Clock, IggySystemClock}; use iggy_binary_protocol::{ Command2, ConsensusHeader, DoViewChangeHeader, GenericHeader, PrepareHeader, PrepareOkHeader, - ReplyHeader, RequestStartViewHeader, RoutedRequestHeader, StartViewChangeHeader, - StartViewHeader, frame_body, + ReplyHeader, RequestHeader, RequestStartViewHeader, StartViewChangeHeader, StartViewHeader, }; use iggy_common::IggyTimestamp; use iggy_common::calculate_checksum; use message_bus::IggyMessageBus; use message_bus::MessageBus; use server_common::Message; -use server_common::sharding::{IggyNamespace, METADATA_GROUP}; +use server_common::sharding::{IggyNamespace, METADATA_CONSENSUS_NAMESPACE}; use std::cell::{Cell, RefCell}; use std::collections::VecDeque; use std::rc::Rc; @@ -128,7 +127,7 @@ impl Sequencer for LocalSequencer { /// Default in-flight prepare-queue depth. /// -/// [`LocalPipeline::new`] uses it, and the server config default +/// [`LocalPipeline::new`] uses it, and the server-ng config default /// (`DEFAULT_METADATA_PREPARE_QUEUE_DEPTH`) is static-asserted equal to it at /// bootstrap. Operators raise the running bound via `[metadata] /// prepare_queue_depth`; the pipeline then carries its own capacity (see @@ -147,27 +146,6 @@ pub const PIPELINE_REQUEST_QUEUE_MAX: usize = 64; /// Maximum number of replicas in a cluster. pub const REPLICAS_MAX: usize = 32; -/// Ceiling on [`VsrConsensus::quorum_replication`]. -/// -/// Past three acks marginal durability is small and every extra ack sits on the -/// commit path, so wide clusters spend the difference on the view-change quorum. -pub const QUORUM_REPLICATION_MAX: usize = 3; - -/// Headers a `DoViewChange` may carry, and so the widest uncommitted suffix a -/// view change can reason about. -/// -/// Pinned by the wire: `DoViewChangeHeader`'s nack and present bitsets are one -/// `u128` each, one bit per entry. The suffix spans `commit_max..=op`, bounded by -/// `prepare_queue_max`, so capping that depth here keeps every suffix addressable. -pub const DVC_HEADERS_MAX: usize = 128; - -/// Deepest prepare queue any node in the cluster may be configured with. -/// -/// One less than [`DVC_HEADERS_MAX`]: the suffix spans `commit..=op` and the head -/// needs the reserved slot. Config ceilings and [`LocalPipeline::with_capacities`] -/// both enforce it, so it holds for a peer as well as for this node. -pub const PREPARE_QUEUE_CEILING: usize = DVC_HEADERS_MAX - 1; - /// Unanswered `RequestStartView` probes tolerated before a recovering /// replica gives up waiting for a settled primary and falls back to an /// election (a full-cluster restart leaves nobody able to answer). @@ -264,7 +242,7 @@ impl PipelineEntry { /// Accepted request waiting in `request_queue` for a prepare slot. #[derive(Debug)] pub struct RequestEntry { - pub message: Message, + pub message: Message, // TODO: populate from monotonic clock at push, promote to `pub` for // age-based filtering. Currently `0`; `pub(crate)` blocks sort-on-stub. #[allow(dead_code)] @@ -278,7 +256,7 @@ pub struct RequestEntry { impl RequestEntry { #[must_use] - pub const fn new(message: Message) -> Self { + pub const fn new(message: Message) -> Self { Self { message, received_at: 0, @@ -293,7 +271,7 @@ impl RequestEntry { /// instead of being bounced with a transient error. #[must_use] pub fn with_subscriber( - message: Message, + message: Message, ) -> (Self, Receiver>) { let (sender, receiver) = oneshot::channel(); let entry = Self { @@ -318,7 +296,7 @@ pub struct LocalPipeline { /// Requests awaiting a prepare slot; cap [`Self::request_queue_max`]. request_queue: VecDeque, /// Depth bound for `prepare_queue`; [`PIPELINE_PREPARE_QUEUE_MAX`] - /// unless the operator overrode it (`[metadata]` in the server + /// unless the operator overrode it (`[metadata]` in the server-ng /// config). prepare_queue_max: usize, /// Depth bound for `request_queue`; [`PIPELINE_REQUEST_QUEUE_MAX`] @@ -347,8 +325,7 @@ impl LocalPipeline { /// `SnapshotCoordinator` in `core/metadata`). /// /// # Panics - /// If a depth is zero, or if the prepare depth would let the uncommitted - /// suffix outgrow what a `DoViewChange` can address. + /// If a depth is zero — a zero-depth pipeline can never admit an op. #[must_use] pub fn with_capacities(prepare_queue_max: usize, request_queue_max: usize) -> Self { assert!( @@ -356,16 +333,6 @@ impl LocalPipeline { "pipeline queue depths must be non-zero \ (prepare={prepare_queue_max}, request={request_queue_max})" ); - // Each `DoViewChange` bitset addresses one suffix entry with one bit of a - // `u128`, and the suffix spans `commit..=op`. Deeper, and the builder clamps - // its window from below, leaving undecidable ops and a stalled view change. - // Config ceilings also enforce this; a stall is worth a loud boot. - assert!( - prepare_queue_max < DVC_HEADERS_MAX, - "prepare queue depth {prepare_queue_max} would produce an uncommitted suffix wider \ - than a DoViewChange can address (max {})", - DVC_HEADERS_MAX - 1, - ); Self { prepare_queue: VecDeque::with_capacity(prepare_queue_max), request_queue: VecDeque::with_capacity(request_queue_max), @@ -756,7 +723,7 @@ pub enum CommitOutcome { #[derive(Debug, Clone)] pub enum VsrAction { /// Send `StartViewChange` to all replicas. - SendStartViewChange { view: u32, group: u64 }, + SendStartViewChange { view: u32, namespace: u64 }, /// Send `DoViewChange` to primary. SendDoViewChange { view: u32, @@ -764,18 +731,13 @@ pub enum VsrAction { log_view: u32, op: u64, commit: u64, - group: u64, - /// The sender's uncommitted suffix, snapshotted for this view. Carried on - /// the action rather than re-read by the dispatcher so the wire bytes match - /// this replica's own `StoredDvc`: a merge seeing two versions of one - /// sender's suffix could adopt a header no replica holds. - suffix: DvcSuffix, + namespace: u64, }, /// Broadcast a `RequestStartView` probe (recovering replica asking for /// the current view's `StartView`; only that view's primary answers). /// Stamped with the prober's view so peers can fence stale duplicates /// out of the probed-primary election path. - SendRequestStartView { view: u32, group: u64 }, + SendRequestStartView { view: u32, namespace: u64 }, /// Send `StartView`, as the view's primary. /// /// `incarnation` echoes the requester's nonce when this answers a @@ -792,14 +754,7 @@ pub enum VsrAction { commit: u64, incarnation: u128, target: Option, - group: u64, - /// The view's suffix, high-to-low op from `op` down toward `commit`. - /// - /// Lets a backup check the head it is told to adopt against real headers, - /// and gives it canonical checksums to verify repaired bodies against. - /// Empty on the probe-answer path, where the primary reports its own - /// frontier rather than concluding a view change; the backup trusts `op`. - suffix: Vec, + namespace: u64, }, /// Send `PrepareOK` for each op in `[from_op, to_op]` that is present in the WAL. /// @@ -811,7 +766,7 @@ pub enum VsrAction { from_op: u64, to_op: u64, target: u8, - group: u64, + namespace: u64, }, /// Retransmit uncommitted prepares from the WAL to replicas that haven't acked. /// @@ -838,7 +793,7 @@ pub enum VsrAction { SendCommit { view: u32, commit: u64, - group: u64, + namespace: u64, timestamp_monotonic: u64, }, } @@ -874,7 +829,7 @@ where cluster: u128, replica: u8, replica_count: u8, - group: u64, + namespace: u64, view: Cell, @@ -968,29 +923,6 @@ where /// built-in default. probe_attempts_max: Cell, - /// This replica's own uncommitted suffix, with the `(op, commit)` the journal - /// was at when it was read. - /// - /// Installed by the shard via [`Self::set_local_dvc_suffix`] before any handler - /// that could enter a view change. Snapshotted rather than recomputed per send - /// so a retransmit is byte-identical: a nack is a durable claim about this - /// replica's log, and silently retracting one lets the new primary assemble a - /// quorum that never simultaneously existed. - /// - /// Tagged by `(op, commit)`, not by view, because that is what the suffix - /// describes: a view advance leaves the log alone so the snapshot survives, - /// while anything moving the head or commit point makes the tag mismatch, - /// which reads as no snapshot at all. - local_dvc_suffix: RefCell>, - - /// The log a DVC quorum settled on, parked until this replica's journal can - /// serve all of it. - /// - /// Non-`None` means "primary-elect, repairing": decided but not started, so - /// this replica prepares and announces nothing. Cleared by - /// [`VsrConsensus::start_pending_view`], or by `reset_view_change_state`. - pending_view_log: RefCell>, - /// Tracks DVC messages received (only used by primary candidate) /// Stores metadata; actual log comes from message do_view_change_from_all_replicas: RefCell, @@ -1020,7 +952,7 @@ impl> VsrConsensus { cluster: u128, replica: u8, replica_count: u8, - group: u64, + namespace: u64, message_bus: B, pipeline: P, ) -> Self { @@ -1028,7 +960,7 @@ impl> VsrConsensus { cluster, replica, replica_count, - group, + namespace, message_bus, pipeline, ConsensusClock::system(), @@ -1045,7 +977,7 @@ impl> VsrConsensus { cluster: u128, replica: u8, replica_count: u8, - group: u64, + namespace: u64, message_bus: B, pipeline: P, clock: ConsensusClock, @@ -1056,25 +988,25 @@ impl> VsrConsensus { ); assert!(replica_count >= 1, "need at least 1 replica"); // Consensus-control routing distinguishes metadata frames from - // partition frames by the group id: metadata uses the sentinel, + // partition frames by namespace value: metadata uses the sentinel, // partitions use `IggyNamespace::inner()` which lives strictly - // inside the packed range. A group outside both ranges would + // inside the packed range. A namespace outside both ranges would // route to neither and silently warn-drop on every receiving peer. debug_assert!( - group == METADATA_GROUP || IggyNamespace::is_packable(group), - "VsrConsensus group must be METADATA_GROUP or a packable \ - IggyNamespace; got {group:#x}" + namespace == METADATA_CONSENSUS_NAMESPACE || IggyNamespace::is_packable(namespace), + "VsrConsensus namespace must be METADATA_CONSENSUS_NAMESPACE or a packable \ + IggyNamespace; got {namespace:#x}" ); // TODO: Verify that XOR-based seeding provides sufficient jitter diversity // across groups. Consider using a proper hash (e.g., Murmur3) of - // (replica_id, group) for production. - let timeout_seed = u128::from(replica) ^ u128::from(group); + // (replica_id, namespace) for production. + let timeout_seed = u128::from(replica) ^ u128::from(namespace); let prepare_queue_max = pipeline.prepare_queue_max(); Self { cluster, replica, replica_count, - group, + namespace, view: Cell::new(0), log_view: Cell::new(0), view_durable: Cell::new(0), @@ -1098,8 +1030,6 @@ impl> VsrConsensus { start_view_change_from_all_replicas: RefCell::new(BitSet::with_capacity(REPLICAS_MAX)), probe_attempts: Cell::new(0), probe_attempts_max: Cell::new(PROBE_ATTEMPTS_MAX), - local_dvc_suffix: RefCell::new(None), - pending_view_log: RefCell::new(None), do_view_change_from_all_replicas: RefCell::new(dvc_quorum_array_empty()), do_view_change_quorum: Cell::new(false), sent_own_start_view_change: Cell::new(false), @@ -1319,45 +1249,10 @@ impl> VsrConsensus { (self.replica_count as usize - 1) / 2 } - /// Replicas that must ack before an op is committed. - /// - /// Capped at [`QUORUM_REPLICATION_MAX`] to keep a wide cluster's commit path - /// cheap; the view-change quorum grows so the two still sum above the count. - #[must_use] - pub const fn quorum_replication(&self) -> usize { - if self.replica_count == 2 { - // =1 would intersect, but =2 keeps a two-replica cluster durable. - return 2; - } - let half_rounded_up = (self.replica_count as usize).div_ceil(2); - if half_rounded_up < QUORUM_REPLICATION_MAX { - half_rounded_up - } else { - QUORUM_REPLICATION_MAX - } - } - - /// Replicas that must send a `DoViewChange` before a view can start. - /// - /// Pays for the cheaper replication quorum, which is the far hotter path. - #[must_use] - pub const fn quorum_view_change(&self) -> usize { - if self.replica_count == 2 { - // Avoids a single-replica view change special case. - return 2; - } - self.replica_count as usize - self.quorum_replication() + 1 - } - - /// Nacks required to prove an op was never committed, so the new primary may - /// truncate it. - /// - /// Sized so a nack quorum and a replication quorum cannot both exist for one - /// op. This is what makes truncation safe, so it is the one quorum that must - /// never be loosened. + /// Quorum size = f + 1 = `max_faulty` + 1 #[must_use] - pub const fn quorum_nack_prepare(&self) -> usize { - self.replica_count as usize - self.quorum_replication() + 1 + pub const fn quorum(&self) -> usize { + self.max_faulty() + 1 } /// Highest op locally executed (state machine applied, client table updated). @@ -1536,8 +1431,8 @@ impl> VsrConsensus { } #[must_use] - pub const fn group(&self) -> u64 { - self.group + pub const fn namespace(&self) -> u64 { + self.namespace } #[must_use] @@ -1638,64 +1533,6 @@ impl> VsrConsensus { } } - /// Install this replica's uncommitted-suffix snapshot for the current view. - /// - /// Called by the shard, which owns the journal. Consensus keeps the snapshot - /// rather than deriving it so the copy in this replica's `StoredDvc` and the - /// copy on the wire are the same bytes: a merge seeing two versions of one - /// sender's suffix could adopt a header no replica holds. - /// - /// Installing twice for one view overwrites: the shard refreshes before each - /// handler, and a later suffix is at least as complete (repair only adds). - pub fn set_local_dvc_suffix(&self, suffix: DvcSuffix) { - let (op, commit) = self.local_dvc_suffix_tag(); - *self.local_dvc_suffix.borrow_mut() = Some((op, commit, suffix)); - } - - /// Drop the cached suffix snapshot. - /// - /// The `(op, commit)` tag tracks how far the log reaches, not what it still - /// contains, so a mutation that removes entries without moving either - /// (truncating a diverging uncommitted range) leaves a snapshot reading as - /// current while offering bodies this replica can no longer serve. A peer that - /// picks it as a body source then waits out the whole view change. - /// - /// Call from the mutation site. The next refresh re-reads the journal. - pub fn invalidate_local_dvc_suffix(&self) { - self.local_dvc_suffix.borrow_mut().take(); - } - - /// The `(op, commit)` a snapshot must match to still describe this log. - /// `commit` is clamped to `op` exactly as the outgoing DVC clamps it. - fn local_dvc_suffix_tag(&self) -> (u64, u64) { - let op = self.sequencer.current_sequence(); - (op, self.commit_max.get().min(op)) - } - - /// This replica's suffix snapshot, or an empty one when none matches the log's - /// current head and commit point. Empty is the safe direction: it nacks nothing - /// and offers no bodies, so it can only stall a view change, never authorise a - /// truncation. - #[must_use] - pub fn local_dvc_suffix(&self) -> DvcSuffix { - let tag = self.local_dvc_suffix_tag(); - match &*self.local_dvc_suffix.borrow() { - Some((op, commit, suffix)) if (*op, *commit) == tag => suffix.clone(), - _ => DvcSuffix::empty(), - } - } - - /// True when no snapshot matches the log's current head and commit point, - /// so the shard must read one from the journal before this replica votes. - #[must_use] - pub fn local_dvc_suffix_stale(&self) -> bool { - let tag = self.local_dvc_suffix_tag(); - !matches!( - &*self.local_dvc_suffix.borrow(), - Some((op, commit, _)) if (*op, *commit) == tag - ) - } - /// True when the current `(view, log_view)` is not yet in the superblock, so a /// view-scoped send would advertise a view a crash could lose. The split-brain /// gate: the dispatcher persists first when this holds. `commit_max` is @@ -1778,9 +1615,6 @@ impl> VsrConsensus { self.reset_dvc_quorum(); self.sent_own_start_view_change.set(false); self.sent_own_do_view_change.set(false); - // A merge parked for the superseded view may describe a different log, so - // drop it and let the new attempt re-derive from the DVCs it collects. - self.pending_view_log.borrow_mut().take(); self.loopback_queue.borrow_mut().clear(); let mut pipeline = self.pipeline.borrow_mut(); pipeline.cancel_all_subscribers(); @@ -1862,7 +1696,7 @@ impl> VsrConsensus { if self.state_transfer_stage.get() != StateTransferStage::Idle { tracing::info!( replica = self.replica, - namespace_raw = self.group, + namespace_raw = self.namespace, "view probe exhausted; abandoning state transfer (cluster bootstrap)" ); self.set_state_transfer_stage(StateTransferStage::Idle); @@ -1877,7 +1711,7 @@ impl> VsrConsensus { .reset(TimeoutKind::RequestStartViewMessage); actions.push(VsrAction::SendRequestStartView { view: self.view.get(), - group: self.group, + namespace: self.namespace, }); } } @@ -1887,7 +1721,7 @@ impl> VsrConsensus { .reset(TimeoutKind::RequestStartViewMessage); actions.push(VsrAction::SendRequestStartView { view: self.view.get(), - group: self.group, + namespace: self.namespace, }); } _ => { @@ -1951,7 +1785,7 @@ impl> VsrConsensus { if self.observed_newer_view.get() > self.view.get() { tracing::info!( replica = self.replica, - namespace_raw = self.group, + namespace_raw = self.namespace, view = self.view.get(), observed_newer_view = self.observed_newer_view.get(), "heartbeat timed out behind a newer view; probing to catch up" @@ -1996,7 +1830,7 @@ impl> VsrConsensus { let action = VsrAction::SendStartViewChange { view: new_view, - group: self.group, + namespace: self.namespace, }; emit_sim_event( SimEventKind::ControlMessageScheduled, @@ -2020,7 +1854,7 @@ impl> VsrConsensus { let action = VsrAction::SendStartViewChange { view: self.view.get(), - group: self.group, + namespace: self.namespace, }; emit_sim_event( SimEventKind::ControlMessageScheduled, @@ -2051,13 +1885,16 @@ impl> VsrConsensus { .borrow_mut() .reset(TimeoutKind::DoViewChangeMessage); - // NOT the snapshot the first send used: `build_do_view_change` re-reads - // `local_dvc_suffix()`, whose `(op, commit)` tag can have moved since, in - // which case it answers EMPTY, retracting every nack and body offer already - // sent. Survivable only because `dvc_record` drops a duplicate sender, so the - // candidate keeps the first vote. Allow a retransmit to replace a seated vote - // and this must pin the snapshot instead. - let action = self.build_do_view_change(self.primary_index(self.view.get())); + let current_op = self.sequencer.current_sequence(); + let action = VsrAction::SendDoViewChange { + view: self.view.get(), + target: self.primary_index(self.view.get()), + log_view: self.log_view.get(), + op: current_op, + // commit_max clamped to op: see `handle_start_view_change`. + commit: self.commit_max.get().min(current_op), + namespace: self.namespace, + }; emit_sim_event( SimEventKind::ControlMessageScheduled, &ControlActionLogEvent::from_vsr_action( @@ -2101,7 +1938,7 @@ impl> VsrConsensus { let action = VsrAction::SendStartViewChange { view: next_view, - group: self.group, + namespace: self.namespace, }; emit_sim_event( SimEventKind::ControlMessageScheduled, @@ -2210,7 +2047,7 @@ impl> VsrConsensus { if self.observed_newer_view.get() > self.view.get() { tracing::info!( replica = self.replica, - namespace_raw = self.group, + namespace_raw = self.namespace, view = self.view.get(), observed_newer_view = self.observed_newer_view.get(), "stale primary-by-index behind a newer view; probing to catch up" @@ -2231,7 +2068,7 @@ impl> VsrConsensus { vec![VsrAction::SendCommit { view: self.view.get(), commit: self.commit_min.get(), - group: self.group, + namespace: self.namespace, timestamp_monotonic: ts, }] } @@ -2243,13 +2080,16 @@ impl> VsrConsensus { /// that will be the primary in the new view." /// /// # Panics - /// If `header.group` does not match this replica's namespace. + /// If `header.namespace` does not match this replica's namespace. pub fn handle_start_view_change( &self, plane: PlaneKind, header: &StartViewChangeHeader, ) -> Vec { - assert_eq!(header.group, self.group, "SVC routed to wrong group"); + assert_eq!( + header.namespace, self.namespace, + "SVC routed to wrong group" + ); // A recovering replica is quorum-invisible: it lost (or cannot trust) // its durable state, so it must not vote history into existence. The // election proceeds among the peers; its conclusion reaches this @@ -2300,7 +2140,7 @@ impl> VsrConsensus { // Send our own SVC let action = VsrAction::SendStartViewChange { view: msg_view, - group: self.group, + namespace: self.namespace, }; emit_sim_event( SimEventKind::ControlMessageScheduled, @@ -2326,14 +2166,36 @@ impl> VsrConsensus { let primary_candidate = self.primary_index(self.view.get()); let current_op = self.sequencer.current_sequence(); - let commit = self.dvc_commit(); + // DVC carries commit_max (highest known-committed), not commit_min + // (locally applied). The new primary floors its pipeline rebuild at + // max(commit) across the quorum; only commit_max bounds that range + // to pipeline depth (every replica holds op - commit_max <= depth). + // commit_min can lag far behind and overflow the rebuild. The + // committed-but-unapplied tail (commit_min..commit_max] is replayed + // by the new primary's CommitJournal, not the pipeline. + // + // Clamp to op: a backup learns commit_max from a heartbeat before + // receiving the prepares, so commit_max can exceed its op. The wire + // contract `DoViewChangeHeader::validate` rejects commit > op and + // drops such a DVC (view-change liveness stall). The clamp is + // lossless for the rebuild floor: quorum intersection guarantees + // some sender whose op covers the true commit point carries it, so + // max(commit) across the quorum is unchanged. + let commit = self.commit_max.get().min(current_op); // Start DVC timeout self.timeouts .borrow_mut() .start(TimeoutKind::DoViewChangeMessage); - let action = self.build_do_view_change(primary_candidate); + let action = VsrAction::SendDoViewChange { + view: self.view.get(), + target: primary_candidate, + log_view: self.log_view.get(), + op: current_op, + commit, + namespace: self.namespace, + }; emit_sim_event( SimEventKind::ControlMessageScheduled, &ControlActionLogEvent::from_vsr_action( @@ -2350,19 +2212,15 @@ impl> VsrConsensus { log_view: self.log_view.get(), op: current_op, commit, - suffix: self.local_dvc_suffix(), }; dvc_record( &mut self.do_view_change_from_all_replicas.borrow_mut(), own_dvc, ); - // `complete_view_change_as_primary` latches only once the merge - // decides, so an undecidable quorum stays open to later DVCs. - if !self.do_view_change_quorum.get() - && dvc_count(&self.do_view_change_from_all_replicas.borrow()) - >= self.quorum_view_change() - { + // Check if we now have quorum + if dvc_count(&self.do_view_change_from_all_replicas.borrow()) >= self.quorum() { + self.do_view_change_quorum.set(true); actions.extend(self.complete_view_change_as_primary(plane)); } } @@ -2377,77 +2235,17 @@ impl> VsrConsensus { /// replicas (including itself), it sets its view-number to that in the messages /// and selects as the new log the one contained in the message with the largest v'..." /// - /// The `commit` this replica advertises in a `DoViewChange`. - /// - /// `commit_max`, not `commit_min`: the new primary floors its pipeline rebuild - /// at `max(commit)` across the quorum, and only `commit_max` bounds that range - /// to the pipeline depth. `commit_min` can lag far enough to overflow the - /// rebuild; `CommitJournal` replays the committed-but-unapplied tail instead. - /// - /// Clamped to `op`, since a backup learns `commit_max` from a heartbeat before - /// the prepares and `DoViewChangeHeader::validate` rejects `commit > op`. - /// Lossless for the rebuild floor: quorum intersection guarantees some sender - /// whose head covers the true commit point carries it. - fn dvc_commit(&self) -> u64 { - let op = self.sequencer.current_sequence(); - self.commit_max.get().min(op) - } - - /// Build this replica's `DoViewChange` for the current view. - fn build_do_view_change(&self, target: u8) -> VsrAction { - VsrAction::SendDoViewChange { - view: self.view.get(), - target, - log_view: self.log_view.get(), - op: self.sequencer.current_sequence(), - commit: self.dvc_commit(), - group: self.group, - suffix: self.local_dvc_suffix(), - } - } - - /// Decode a peer's suffix, or `None` to drop the whole `DoViewChange`. - /// - /// A suffix that will not decode makes the numbers untrustworthy too. Dropping - /// the message lets the sender's retransmit try again, rather than seating a - /// vote whose nacks and offered bodies cannot be placed against an op. - fn decode_peer_suffix( - &self, - header: &DoViewChangeHeader, - suffix_body: &[u8], - ) -> Option { - match dvc_suffix_decode( - suffix_body, - header.op, - header.nack_bitset, - header.present_bitset, - ) { - Ok(suffix) => Some(suffix), - Err(error) => { - tracing::warn!( - replica = self.replica, - from_replica = header.replica, - view = header.view, - op = header.op, - "dropping do_view_change with an unreadable suffix: {error}" - ); - None - } - } - } - - /// `suffix_body` is the sender's uncommitted-suffix headers. Empty from a peer - /// unable to snapshot one, which then contributes numbers only. - /// /// # Panics - /// If `header.group` does not match this replica's namespace. + /// If `header.namespace` does not match this replica's namespace. pub fn handle_do_view_change( &self, plane: PlaneKind, header: &DoViewChangeHeader, - suffix_body: &[u8], ) -> Vec { - assert_eq!(header.group, self.group, "DVC routed to wrong group"); + assert_eq!( + header.namespace, self.namespace, + "DVC routed to wrong group" + ); // Quorum-invisible while recovering (see handle_start_view_change): // a recovering replica must not collect DVCs and crown itself. if self.status.get() == Status::Recovering { @@ -2458,9 +2256,6 @@ impl> VsrConsensus { let msg_log_view = header.log_view; let msg_op = header.op; let msg_commit = header.commit; - let Some(msg_suffix) = self.decode_peer_suffix(header, suffix_body) else { - return Vec::new(); - }; // Ignore DVCs for old views if msg_view < self.view.get() { @@ -2502,7 +2297,7 @@ impl> VsrConsensus { // Send our own SVC let action = VsrAction::SendStartViewChange { view: msg_view, - group: self.group, + namespace: self.namespace, }; emit_sim_event( SimEventKind::ControlMessageScheduled, @@ -2537,7 +2332,6 @@ impl> VsrConsensus { log_view: self.log_view.get(), op: current_op, commit, - suffix: self.local_dvc_suffix(), }; dvc_record( &mut self.do_view_change_from_all_replicas.borrow_mut(), @@ -2551,16 +2345,14 @@ impl> VsrConsensus { log_view: msg_log_view, op: msg_op, commit: msg_commit, - suffix: msg_suffix, }; dvc_record(&mut self.do_view_change_from_all_replicas.borrow_mut(), dvc); - // `complete_view_change_as_primary` latches only once the merge decides, - // so an undecidable quorum re-merges as each further DVC lands. + // Check if quorum achieved if !self.do_view_change_quorum.get() - && dvc_count(&self.do_view_change_from_all_replicas.borrow()) - >= self.quorum_view_change() + && dvc_count(&self.do_view_change_from_all_replicas.borrow()) >= self.quorum() { + self.do_view_change_quorum.set(true); actions.extend(self.complete_view_change_as_primary(plane)); } @@ -2581,7 +2373,7 @@ impl> VsrConsensus { pub fn begin_view_probe(&self) { tracing::info!( replica = self.replica, - namespace_raw = self.group, + namespace_raw = self.namespace, "beginning view probe" ); self.status.set(Status::Recovering); @@ -2631,7 +2423,7 @@ impl> VsrConsensus { ); tracing::info!( replica = self.replica, - namespace_raw = self.group, + namespace_raw = self.namespace, ?from, ?to, "state transfer stage" @@ -2657,7 +2449,7 @@ impl> VsrConsensus { header: &RequestStartViewHeader, ) -> Vec { assert_eq!( - header.group, self.group, + header.namespace, self.namespace, "RequestStartView routed to wrong group" ); if self.status.get() != Status::Normal { @@ -2698,10 +2490,7 @@ impl> VsrConsensus { commit: self.commit_max.get(), incarnation: header.incarnation, target: Some(header.replica), - // A probe answer reports this primary's settled frontier, not a - // freshly merged log, so there is no canonical suffix to publish. - suffix: Vec::new(), - group: self.group, + namespace: self.namespace, }] } @@ -2730,44 +2519,6 @@ impl> VsrConsensus { .stop(TimeoutKind::RequestStartViewMessage); } - /// Decide which head to adopt from a `StartView`, and record the view's - /// canonical headers when it carried any. - /// - /// Headers go in `pending_view_log`, not the journal: a journal entry is a - /// header plus its body, and a backup adopting a view usually holds neither. - /// Keeping them lets the repair ingest reject a body that disagrees with what - /// the view decided, which is what makes fetching by op number safe. - /// - /// Falls back to the announced `op` on an empty body (probe answer, stale-view - /// correction). - fn adopt_start_view_suffix(&self, header: &StartViewHeader, suffix_body: &[u8]) -> u64 { - let suffix = match dvc_suffix_decode(suffix_body, header.op, 0, 0) { - Ok(suffix) => suffix, - Err(error) => { - tracing::warn!( - replica = self.replica, - from_replica = header.replica, - view = header.view, - op = header.op, - "start_view suffix did not decode, falling back to the announced op: {error}" - ); - return header.op; - } - }; - let headers = suffix.headers(); - if headers.is_empty() { - return header.op; - } - - *self.pending_view_log.borrow_mut() = Some(MergedLog { - op_head: header.op, - commit_max: header.commit, - headers: headers.to_vec(), - committed_elsewhere: Vec::new(), - }); - header.op - } - /// Handle a received `StartView` message (backups only). /// /// "When other replicas receive the STARTVIEW message, they replace their log @@ -2776,7 +2527,7 @@ impl> VsrConsensus { /// their status to normal, and send `PrepareOK` for any uncommitted ops." /// /// # Panics - /// If `header.group` does not match this replica's namespace. + /// If `header.namespace` does not match this replica's namespace. /// # Client-table maintenance /// /// Backups maintain the client-table during normal operation via @@ -2786,15 +2537,8 @@ impl> VsrConsensus { /// /// Gap: if a backup never received a prepare (lost message), /// `commit_journal` stops at the gap. Requires message repair. - /// `suffix_body` is the message body: the view's canonical headers, empty - /// when the announcement carries numbers only. - pub fn handle_start_view( - &self, - plane: PlaneKind, - header: &StartViewHeader, - suffix_body: &[u8], - ) -> Vec { - assert_eq!(header.group, self.group, "SV routed to wrong group"); + pub fn handle_start_view(&self, plane: PlaneKind, header: &StartViewHeader) -> Vec { + assert_eq!(header.namespace, self.namespace, "SV routed to wrong group"); let from_replica = header.replica; let msg_view = header.view; let msg_op = header.op; @@ -2819,7 +2563,7 @@ impl> VsrConsensus { // incarnation is set (partition plane, tests). // // A zero `header.incarnation` makes no claim either way: it is what an - // unsolicited StartView carries. + // unsolicited StartView carries, and what a peer predating the field sends. // Classifying it stale would have this replica reject a current StartView // from a healthy primary purely because that primary is older, so it falls // through to the view checks that governed before the field existed. @@ -2896,11 +2640,11 @@ impl> VsrConsensus { // Stale pipeline entries from the old view must be discarded self.pipeline.borrow_mut().clear(); - // Cross-check the announced head against the headers published with it: a - // suffix head disagreeing with `header.op` means an inconsistently built - // frame, and either value leaves this replica chasing an unservable head. - let announced = self.adopt_start_view_suffix(header, suffix_body); - self.sequencer.set_sequence(announced); + // TODO: StartView should carry uncommitted headers so backup installs + // into WAL and sets op WAL-verified. Today we trust msg_op, correct + // for truncation (sequencer > msg_op) but wrong when behind + // (sequencer < msg_op): gap is unreachable without message repair. + self.sequencer.set_sequence(msg_op); // Update timeouts for normal backup operation { @@ -2932,7 +2676,7 @@ impl> VsrConsensus { from_op: msg_commit + 1, to_op: msg_op, target: from_replica, - group: self.group, + namespace: self.namespace, }; emit_sim_event( SimEventKind::ControlMessageScheduled, @@ -2956,9 +2700,12 @@ impl> VsrConsensus { /// to prevent old/replayed messages from suppressing view changes. /// /// # Panics - /// If `header.group` does not match this replica's namespace. + /// If `header.namespace` does not match this replica's namespace. pub fn handle_commit(&self, header: &iggy_binary_protocol::CommitHeader) -> CommitOutcome { - assert_eq!(header.group, self.group, "Commit routed to wrong group"); + assert_eq!( + header.namespace, self.namespace, + "Commit routed to wrong group" + ); if self.is_primary() { // A heartbeat from the primary of an OLDER view means that @@ -3036,83 +2783,22 @@ impl> VsrConsensus { /// contains entries for all committed ops it received. /// /// Gap: missing prepares (lost messages) require message repair. - /// - /// Re-entrant, called again for every `DoViewChange` landing while the merge is - /// undecided. Every non-`Ready` outcome leaves this replica untouched, so a - /// re-run costs only the merge. fn complete_view_change_as_primary(&self, plane: PlaneKind) -> Vec { - let merged = { - let dvc_array = self.do_view_change_from_all_replicas.borrow(); - merge_dvc_quorum(&dvc_array, self.merge_quorums()) - }; + let dvc_array = self.do_view_change_from_all_replicas.borrow(); - let merged = match merged { - MergeOutcome::Ready(merged) => merged, - // Every non-ready outcome keeps this replica in `ViewChange` with its - // log untouched. Picking a winner unconditionally and letting the - // pipeline rebuild truncate what it cannot find locally discards - // committed ops; an unavailable cluster that says so is the better - // failure. - // - // None of these latch `do_view_change_quorum`: an undecidable quorum is - // not a decision, and the replicas still to report are what would - // settle it. The flag belongs only where the quorum is decidable. - MergeOutcome::AwaitingQuorum => return Vec::new(), - MergeOutcome::AwaitingRepair { undecided_op } => { - tracing::debug!( - replica = self.replica, - view = self.view.get(), - undecided_op, - "view change waiting on more DoViewChange messages to decide an op" - ); - return Vec::new(); - } - MergeOutcome::Deadlocked { undecided_op } => { - tracing::error!( - replica = self.replica, - view = self.view.get(), - undecided_op, - "view change cannot start: op {undecided_op} is neither recoverable from any \ - replica nor provably uncommitted" - ); - return Vec::new(); - } + let Some(winner) = dvc_select_winner(&dvc_array) else { + return Vec::new(); }; - // The pipeline must hold the whole uncommitted range, and the merge decides - // that range against a cluster-wide ceiling, so a node configured shallower - // than its peers can be handed a range it cannot rebuild. - // - // Refuse rather than panic: a further DoViewChange can raise `commit_max` - // and shrink the range, and otherwise the status timeout escalates. A panic - // would restart into the same merge. - if merged.op_head.saturating_sub(merged.commit_max) > self.prepare_queue_max as u64 { - tracing::error!( - replica = self.replica, - view = self.view.get(), - commit_max = merged.commit_max, - op_head = merged.op_head, - prepare_queue_max = self.prepare_queue_max, - "view change cannot start: the merged log claims {} in-flight ops, more than this \ - replica's pipeline holds; refusing the view", - merged.op_head - merged.commit_max, - ); - return Vec::new(); - } + let new_op = winner.op; + let max_commit = dvc_max_commit(&dvc_array); - // Quorum closed now the merge decided: re-merging after parking could - // produce a different log than the one already being repaired toward. - self.do_view_change_quorum.set(true); - - // The merged log is authoritative but this replica may not hold every body - // yet. Park it, let the shard repair up to it, and `start_pending_view` - // finishes once the journal covers the range. Until then this replica stays - // in `ViewChange` and prepares nothing, so no client op is stamped onto an - // unproven log. `log_view` does NOT advance here; see `start_pending_view`. - let max_commit = merged.commit_max; - let new_op = merged.op_head; + // Update state + self.log_view.set(self.view.get()); + self.status.set(Status::Normal); + self.ceded_primaryship.set(false); self.advance_commit_max(max_commit); - *self.pending_view_log.borrow_mut() = Some(merged); + self.sequencer.set_sequence(new_op); // Stale pipeline entries are invalid in new view; reconciliation // replays from journal. @@ -3132,137 +2818,6 @@ impl> VsrConsensus { // the loopback queue directly. self.loopback_queue.borrow_mut().clear(); - tracing::info!( - replica = self.replica, - view = self.view.get(), - op_head = new_op, - commit_max = max_commit, - "view-change quorum merged; repairing up to the merged log before starting the view" - ); - emit_replica_event( - SimEventKind::ReplicaStateChanged, - &ReplicaLogContext::from_consensus(self, plane), - ); - - // No sends yet: `SendStartView` promises this replica can serve every op in - // the merged log, and a backup adopting the announced head asks it for the - // bodies behind it. - Vec::new() - } - - /// Sizes handed to the DVC merge. - const fn merge_quorums(&self) -> MergeQuorums { - MergeQuorums { - view_change: self.quorum_view_change(), - nack_prepare: self.quorum_nack_prepare(), - replica_count: self.replica_count as usize, - // The cluster-wide ceiling, not `self.prepare_queue_max`: this node's - // config says nothing about how deep a peer's pipeline is. - prepare_queue_ceiling: PREPARE_QUEUE_CEILING as u64, - } - } - - /// The merged log this replica is repairing toward, if a view change is - /// mid-transition. The shard reads it for the op range it must cover before the - /// view can start, and for which peers offered the bodies. - /// - /// Clones two `Vec`. Prefer [`Self::view_log_is_pending`] / - /// [`Self::with_pending_view_log`]; clone only to hold it across an `.await` or - /// across [`Self::start_pending_view`], which takes the cell. - #[must_use] - pub fn pending_view_log(&self) -> Option { - self.pending_view_log.borrow().clone() - } - - /// Whether a merge is parked, without cloning it. - #[must_use] - pub fn view_log_is_pending(&self) -> bool { - self.pending_view_log.borrow().is_some() - } - - /// Read the parked merge in place. The closure must not re-enter consensus: the - /// `RefCell` stays borrowed for its whole body. - pub fn with_pending_view_log(&self, read: impl FnOnce(&MergedLog) -> T) -> Option { - self.pending_view_log.borrow().as_ref().map(read) - } - - /// Replicas that offered a body for `op`, most-recent-log_view first. - /// - /// Only meaningful while a merge is parked. These peers and nobody else: a - /// cleared present bit means the body was never held or cannot be read back, - /// and the view change is blocked on the round-trip. - #[must_use] - pub fn pending_view_body_sources(&self, op: u64) -> Vec { - let quorum = self.do_view_change_from_all_replicas.borrow(); - let mut sources: Vec<(u32, u8)> = dvc_iter(&quorum) - .filter(|dvc| dvc.replica != self.replica) - .filter_map(|dvc| { - let index = dvc.suffix.index_of(dvc.op, op)?; - dvc.suffix - .offers_body(index) - .then_some((dvc.log_view, dvc.replica)) - }) - .collect(); - sources.sort_unstable_by_key(|(log_view, _)| std::cmp::Reverse(*log_view)); - sources.into_iter().map(|(_, replica)| replica).collect() - } - - /// Finish the parked view change: this replica's journal now covers the merged - /// log, so it can serve any op it is about to announce. - /// - /// Called by the shard after repair progress. No-op when nothing is parked. - /// - /// # Panics - /// If the merged uncommitted range exceeds pipeline capacity, which needs a head - /// more than one pipeline depth above the proven commit point. - pub fn start_pending_view(&self, plane: PlaneKind) -> Vec { - // A backup's parked log (the `StartView` suffix) is only what its ingest - // verifies bodies against. It must never take this path: starting the view - // claims the primaryship of a view this replica did not win. - if !self.is_primary_for_view(self.view.get()) { - return Vec::new(); - } - let Some(merged) = self.pending_view_log.borrow_mut().take() else { - return Vec::new(); - }; - let new_op = merged.op_head; - let max_commit = merged.commit_max; - - // The one view-change exit that skips `reset_view_change_state`, so the DVCs - // (a suffix `Vec` per sender, per group led) would be held for the whole - // primaryship. Nothing reads the array after the view starts: a late - // same-view DVC returns at the status gate, a higher-view one resets first. - // - // `dvc_reset`, not `reset_dvc_quorum`: the latter also clears the - // `do_view_change_quorum` latch, which from here means "log decided" and is - // what stops `handle_do_view_change_timeout` retransmitting. - dvc_reset(&mut self.do_view_change_from_all_replicas.borrow_mut()); - self.invalidate_local_dvc_suffix(); - - self.status.set(Status::Normal); - self.ceded_primaryship.set(false); - self.sequencer.set_sequence(new_op); - if let Some(head) = merged.headers.first() { - // Keep the hash chain continuous: the next prepare must chain onto the - // head this view adopted, not onto whatever was appended last. - self.set_last_prepare_checksum(head.checksum); - } - for header in &merged.headers { - self.observe_prepare_timestamp(header.timestamp); - } - // Only now, with the merged head installed above. `log_view` claims "my log - // IS the log this view decided", and it selects the canonical senders of the - // next view change, whose headers outrank everyone else's. - // - // Raising it at merge time breaks that claim for the whole parked window, - // which can end in supersession or a crash (`log_view` is durable): the - // replica then votes as canonical carrying its own stale head, and ops the - // merge decided to keep fall outside the next scan range, discarded with no - // nack required. Merge-time assignment is only truthful where every merged - // header is installed there; parking installs nothing and the repair ingest - // only fills holes, so a parked replica still holds its old view's log. - self.log_view.set(self.view.get()); - // Update timeouts for normal primary operation { let mut timeouts = self.timeouts.borrow_mut(); @@ -3291,9 +2846,7 @@ impl> VsrConsensus { commit: max_commit, incarnation: 0, target: None, - group: self.group, - // `merged` was taken out of the parked slot, so hand the headers over. - suffix: merged.headers, + namespace: self.namespace, }; emit_sim_event( SimEventKind::ControlMessageScheduled, @@ -3311,12 +2864,10 @@ impl> VsrConsensus { // The new primary must rebuild its pipeline from the journal so that // incoming PrepareOk messages can be matched and commits can proceed. if max_commit < new_op { - // `complete_view_change_as_primary` already refused a non-fitting - // range. Asserted so the sites cannot drift; this one cannot decline. - debug_assert!( + assert!( (new_op - max_commit) <= self.prepare_queue_max as u64, "view change: uncommitted range {}..={} ({} ops) exceeds pipeline capacity ({}); \ - the merged log claims more in-flight ops than the pipeline can hold", + DVC winner claims more in-flight ops than the pipeline can hold", max_commit + 1, new_op, new_op - max_commit, @@ -3397,7 +2948,7 @@ impl> VsrConsensus { // Record the ack from this replica let ack_count = entry.add_ack(header.replica); - let quorum = self.quorum_replication(); + let quorum = self.quorum(); let quorum_reached = ack_count >= quorum && !entry.ok_quorum_received; // Check if we've reached quorum @@ -3474,7 +3025,7 @@ impl> VsrConsensus { } } -impl Project, VsrConsensus> for Message +impl Project, VsrConsensus> for Message where B: MessageBus, P: Pipeline, @@ -3497,8 +3048,12 @@ where // stores the same value and the scan verifies it after a crash. The body is // never re-stamped (`restamp_prepare_view` patches only `view`), so this // survives view-change retransmits. The header `checksum` and its `parent` - // chain are sealed too, for both planes, by `seal_prepare_checksum` below; - // they exclude `view`, which is what lets a restamp leave them valid. + // chain stay `0`: activating them needs the retransmit path to re-seal a + // re-stamped header, a separate change. Whoever activates it must also + // audit every `set_last_prepare_checksum` caller for cross-plane carry -- + // the repair router in `shard` drops metadata-plane frames it cannot + // journal precisely so one cannot stamp a PARTITION consensus, which is + // inert only while these values are structurally zero. // // Metadata plane only. A partition produce prepare already carries a verified // `batch_checksum` over the same bytes, so a second full-payload pass is pure @@ -3507,35 +3062,15 @@ where // sealed region before the entry is journaled. Leaving those prepares at `0` // is the designed "nothing to verify" sentinel, so a future durable partition // journal skips verification instead of failing every entry as corrupt. - // - // TODO(consensus): a partition prepare's `checksum` covers its header alone, - // so two at one op with matching header fields are indistinguishable however - // far their batch bytes diverge. The merge then counts a divergent replica as - // holding a servable copy, and the partition repair ingest (no merged-log - // identity gate) short-circuits `verify_prepare_integrity`'s body branch on - // the zero. Two closures, both larger than they look: - // - // 1. The batch checksum, recomputed after `stamp_prepare_for_persistence`. - // But stamping runs per replica after replication and folds `base_offset` - // in, so identity would change at stamp time and the journaled entry would - // no longer match the pipeline entry `handle_prepare_ok` compares. - // 2. The stamp-invariant cover: everything past the 256-byte command header, - // which stamping never touches. Identical on every replica, safe to seal - // here, but costs a produce-path pass and retires the "0 means nothing to - // verify" sentinel that lets an existing WAL replay. - // - // Bounded by `size`, the range every verifier re-reads; the prepare - // inherits it verbatim below. - let checksum_body = if consensus.group == METADATA_GROUP { - u128::from(calculate_checksum(frame_body( - self.as_slice(), - self.header().size, - ))) + let checksum_body = if consensus.namespace == METADATA_CONSENSUS_NAMESPACE { + u128::from(calculate_checksum( + &self.as_slice()[size_of::()..], + )) } else { 0 }; - let prepared = self.transmute_header(|old, new| { + self.transmute_header(|old, new| { *new = PrepareHeader { cluster: consensus.cluster, size: old.size, @@ -3551,12 +3086,12 @@ where op, timestamp, operation: old.operation, - // The GROUP's own id, never the request's: a routed request - // header can carry group 0, and journaling that would make - // the stored prepare route to the wrong plane when repair - // later ships it verbatim (live replication masked this; - // repair replay is what broke). - group: consensus.group, + // The GROUP's namespace, never the request's: a client + // RequestHeader carries namespace 0, and journaling that + // would make the stored prepare route to the wrong plane + // when repair later ships it verbatim (live replication + // masked this; repair replay is what broke). + namespace: consensus.namespace, checksum_body, // Copied verbatim: carries the stamped acting user for client // ops (and the authenticated user on Register), so the in-apply @@ -3564,11 +3099,7 @@ where user_id: old.user_id, ..Default::default() } - }); - // Last, because the checksum covers every other field. Gives the op the - // stable identity the view-change merge compares across replicas; `parent` - // chains it, so the log is hash-linked rather than nominally so. - seal_prepare_checksum(prepared) + }) } } @@ -3595,13 +3126,12 @@ where commit: consensus.commit_max.get(), timestamp: old.timestamp, operation: old.operation, - group: old.group, + namespace: old.namespace, // PrepareOk is header-only; the frame is exactly the header, so // `size` is the header size. size: std::mem::size_of::() as u32, ..Default::default() }; - new.seal(); }) } } @@ -3614,7 +3144,7 @@ where type MessageBus = B; #[rustfmt::skip] // Scuffed formatter. TODO: Make the naming less ambiguous for `Message`. type Message = Message where H: ConsensusHeader; - type RoutedRequestHeader = RoutedRequestHeader; + type RequestHeader = RequestHeader; type ReplicateHeader = PrepareHeader; type AckHeader = PrepareOkHeader; @@ -3652,20 +3182,20 @@ mod request_queue_tests { use super::*; use iggy_binary_protocol::{Command2, Operation}; - fn make_request(client: u128, request_num: u64) -> Message { - let header_size = std::mem::size_of::(); - let mut msg = Message::::new(header_size); - let header = bytemuck::checked::try_from_bytes_mut::( + fn make_request(client: u128, request_num: u64) -> Message { + let header_size = std::mem::size_of::(); + let mut msg = Message::::new(header_size); + let header = bytemuck::checked::try_from_bytes_mut::( &mut msg.as_mut_slice()[..header_size], ) .expect("zeroed bytes are valid"); - *header = RoutedRequestHeader { + *header = RequestHeader { command: Command2::Request, client, session: 1, request: request_num, operation: Operation::SendMessages, - ..RoutedRequestHeader::default() + ..RequestHeader::default() }; msg } @@ -3970,7 +3500,7 @@ mod timestamp_clamp_tests { 1, 0, 1, - METADATA_GROUP, + METADATA_CONSENSUS_NAMESPACE, NoopBus, LocalPipeline::new(), lagging_clock, @@ -4002,7 +3532,7 @@ mod timestamp_clamp_tests { 1, 0, 1, - METADATA_GROUP, + METADATA_CONSENSUS_NAMESPACE, NoopBus, LocalPipeline::new(), leading_clock, @@ -4035,7 +3565,7 @@ mod timestamp_clamp_tests { header.commit = op; header.replica = replica; header.incarnation = incarnation; - header.group = METADATA_GROUP; + header.namespace = METADATA_CONSENSUS_NAMESPACE; header.size = size as u32; msg } @@ -4056,7 +3586,7 @@ mod timestamp_clamp_tests { 1, 0, 3, - METADATA_GROUP, + METADATA_CONSENSUS_NAMESPACE, NoopBus, LocalPipeline::new(), ConsensusClock::system(), @@ -4070,7 +3600,7 @@ mod timestamp_clamp_tests { let stale = make_start_view(1, 4, 1, STALE); assert!( consensus - .handle_start_view(PlaneKind::Metadata, stale.header(), &[]) + .handle_start_view(PlaneKind::Metadata, stale.header()) .is_empty(), "a StartView echoing a previous incarnation must be ignored while recovering" ); @@ -4090,7 +3620,7 @@ mod timestamp_clamp_tests { let fresh = make_start_view(1, 4, 1, CURRENT); assert!( !consensus - .handle_start_view(PlaneKind::Metadata, fresh.header(), &[]) + .handle_start_view(PlaneKind::Metadata, fresh.header()) .is_empty(), "a StartView echoing our current incarnation must be adopted" ); @@ -4122,14 +3652,14 @@ mod timestamp_clamp_tests { LocalPipeline::new(), ConsensusClock::new(Rc::new(FixedClock(100_000))), ); - let header_size = size_of::(); + let header_size = size_of::(); let body = b"produce payload"; - let mut msg = Message::::new(header_size + body.len()); + let mut msg = Message::::new(header_size + body.len()); msg.as_mut_slice()[header_size..].copy_from_slice(body); - let header = bytemuck::checked::try_from_bytes_mut::( + let header = bytemuck::checked::try_from_bytes_mut::( &mut msg.as_mut_slice()[..header_size], ) - .expect("zeroed bytes are a valid RoutedRequestHeader"); + .expect("zeroed bytes are a valid RequestHeader"); header.command = Command2::Request; header.client = 1; header.request = 1; @@ -4139,7 +3669,7 @@ mod timestamp_clamp_tests { }; assert_ne!( - seal(METADATA_GROUP), + seal(METADATA_CONSENSUS_NAMESPACE), 0, "a metadata prepare must be sealed: the WAL scan verifies it after a crash" ); @@ -4168,7 +3698,7 @@ mod timestamp_clamp_tests { 1, 0, 3, - METADATA_GROUP, + METADATA_CONSENSUS_NAMESPACE, NoopBus, LocalPipeline::new(), ConsensusClock::system(), @@ -4182,11 +3712,7 @@ mod timestamp_clamp_tests { // head covers every op it told us was committed. assert!( consensus - .handle_start_view( - PlaneKind::Metadata, - make_start_view(7, 104, 1, 0).header(), - &[] - ) + .handle_start_view(PlaneKind::Metadata, make_start_view(7, 104, 1, 0).header()) .is_empty(), "an equal-view StartView below the commit floor must be skipped" ); @@ -4200,11 +3726,7 @@ mod timestamp_clamp_tests { // Adopt it and drop the discarded suffix. assert!( !consensus - .handle_start_view( - PlaneKind::Metadata, - make_start_view(7, 105, 1, 0).header(), - &[] - ) + .handle_start_view(PlaneKind::Metadata, make_start_view(7, 105, 1, 0).header()) .is_empty(), "an equal-view StartView at or above the commit floor must be adopted, \ even when its head is behind a WAL suffix the view already discarded" @@ -4224,8 +3746,14 @@ mod timestamp_clamp_tests { /// pins the predicate the dispatch sites and the debug tripwire both read. #[test] fn given_view_change_when_needs_superblock_persist_should_track_durability() { - let mut consensus = - VsrConsensus::new(1, 0, 3, METADATA_GROUP, NoopBus, LocalPipeline::new()); + let mut consensus = VsrConsensus::new( + 1, + 0, + 3, + METADATA_CONSENSUS_NAMESPACE, + NoopBus, + LocalPipeline::new(), + ); assert!( !consensus.needs_superblock_persist(), "fresh replica: view == view_durable == 0" @@ -4466,82 +3994,3 @@ mod state_transfer_stage_tests { assert_eq!(consensus.status(), Status::Recovering); } } - -#[cfg(test)] -mod quorum_tests { - //! Pin the three quorum sizes for replica counts 1 through 8. The - //! intersection asserts are the safety properties: replication and - //! view-change quorums must overlap, so a committed op is visible to the - //! next view, and replication and nack quorums must overlap, so an op that - //! may have committed can never gather a nack quorum. - - use super::*; - use crate::LocalPipeline; - use server_common::MESSAGE_ALIGN; - use server_common::iobuf::Frozen; - - struct NoopBus; - - impl MessageBus for NoopBus { - async fn send_to_client( - &self, - _client_id: u128, - _data: Frozen, - ) -> Result<(), message_bus::SendError> { - Ok(()) - } - - async fn send_to_replica( - &self, - _replica: u8, - _data: Frozen, - ) -> Result<(), message_bus::SendError> { - Ok(()) - } - - fn set_connection_lost_fn(&self, _f: message_bus::ConnectionLostFn) {} - fn set_replica_forward_fn(&self, _f: message_bus::ReplicaForwardFn) {} - fn set_client_forward_fn(&self, _f: message_bus::ClientForwardFn) {} - fn track_background(&self, _handle: message_bus::JoinHandle<()>) {} - } - - fn consensus_with_replica_count(replica_count: u8) -> VsrConsensus { - VsrConsensus::new( - 1, - 0, - replica_count, - METADATA_GROUP, - NoopBus, - LocalPipeline::new(), - ) - } - - #[test] - fn given_any_replica_count_when_sizing_quorums_should_intersect() { - for replica_count in 1u8..=REPLICAS_MAX_U8 { - let consensus = consensus_with_replica_count(replica_count); - let count = usize::from(replica_count); - - assert!( - consensus.quorum_replication() + consensus.quorum_view_change() > count, - "replication+view-change must intersect at replica_count={replica_count}" - ); - assert!( - consensus.quorum_nack_prepare() + consensus.quorum_replication() > count, - "nack+replication must intersect at replica_count={replica_count}" - ); - assert!(consensus.quorum_replication() <= count); - assert!(consensus.quorum_view_change() <= count); - assert!(consensus.quorum_nack_prepare() <= count); - } - } - - /// `REPLICAS_MAX` as a `u8` for loop bounds. - const REPLICAS_MAX_U8: u8 = { - assert!(REPLICAS_MAX <= u8::MAX as usize); - #[allow(clippy::cast_possible_truncation)] - { - REPLICAS_MAX as u8 - } - }; -} diff --git a/core/consensus/src/lib.rs b/core/consensus/src/lib.rs index 02cca677b0..4d7a53d53e 100644 --- a/core/consensus/src/lib.rs +++ b/core/consensus/src/lib.rs @@ -27,7 +27,7 @@ pub trait Project { pub trait Pipeline { type Entry; /// Accepted-but-not-yet-prepared client request. For `LocalPipeline`, - /// `RequestEntry` wrapping `Message`. + /// `RequestEntry` wrapping `Message`. type Request; fn push(&mut self, entry: Self::Entry); @@ -93,7 +93,7 @@ pub trait Pipeline { } } -pub type RequestMessage = ::Message<::RoutedRequestHeader>; +pub type RequestMessage = ::Message<::RequestHeader>; pub type ReplicateMessage = ::Message<::ReplicateHeader>; pub type AckMessage = ::Message<::AckHeader>; @@ -102,7 +102,7 @@ pub trait Consensus: Sized { #[rustfmt::skip] // Scuffed formatter. type Message: ConsensusMessage where H: ConsensusHeader; - type RoutedRequestHeader: ConsensusHeader; + type RequestHeader: ConsensusHeader; type ReplicateHeader: ConsensusHeader; type AckHeader: ConsensusHeader; @@ -180,9 +180,6 @@ pub use observability::*; mod view_change_quorum; pub use view_change_quorum::*; - -mod dvc_merge; -pub use dvc_merge::*; mod vsr_state; pub use vsr_state::{VsrState, VsrStateError}; mod vsr_timeout; diff --git a/core/consensus/src/observability.rs b/core/consensus/src/observability.rs index e5c58326bb..4cd041bc4f 100644 --- a/core/consensus/src/observability.rs +++ b/core/consensus/src/observability.rs @@ -249,7 +249,7 @@ impl ReplicaLogContext { plane, cluster_id: consensus.cluster(), replica_id: consensus.replica(), - namespace: NamespaceLogContext::from_raw(plane, consensus.group()), + namespace: NamespaceLogContext::from_raw(plane, consensus.namespace()), view: consensus.view(), log_view: consensus.log_view(), commit: consensus.commit_max(), diff --git a/core/consensus/src/plane_helpers.rs b/core/consensus/src/plane_helpers.rs index 87f3b65e6e..4515f2ba7d 100644 --- a/core/consensus/src/plane_helpers.rs +++ b/core/consensus/src/plane_helpers.rs @@ -19,71 +19,10 @@ use crate::{ Consensus, IgnoreReason, Pipeline, PipelineEntry, PlaneKind, PrepareOkOutcome, Sequencer, Status, VsrConsensus, }; -use iggy_binary_protocol::{ - CHECKSUM_UNSEALED, Command2, ConsensusHeader, GenericHeader, PrepareHeader, PrepareOkHeader, - ReplyHeader, RoutedRequestHeader, frame_body, -}; +use iggy_binary_protocol::{Command2, PrepareHeader, PrepareOkHeader, ReplyHeader, RequestHeader}; use message_bus::{MessageBus, SendError}; -use server_common::{ - MESSAGE_ALIGN, Message, - iobuf::{Frozen, Owned}, -}; -use std::{error::Error, fmt, mem::size_of, ops::AsyncFnOnce}; - -/// Failure to route or forward a prepare through the replication chain. -#[derive(Debug)] -#[non_exhaustive] -pub enum ChainReplicationError { - MalformedPrepare, - UnexpectedCommand { command: Command2 }, - CommittedPrepare { op: u64, commit_min: u64 }, - SelfRoute { replica: u8 }, - Transport(SendError), -} - -impl ChainReplicationError { - #[must_use] - pub const fn is_transport(&self) -> bool { - matches!(self, Self::Transport(_)) - } -} - -impl fmt::Display for ChainReplicationError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::MalformedPrepare => formatter.write_str("malformed prepare frame"), - Self::UnexpectedCommand { command } => { - write!(formatter, "expected prepare command, found {command:?}") - } - Self::CommittedPrepare { op, commit_min } => write!( - formatter, - "prepare op {op} is not above committed op {commit_min}" - ), - Self::SelfRoute { replica } => { - write!( - formatter, - "replication chain routes replica {replica} to itself" - ) - } - Self::Transport(error) => error.fmt(formatter), - } - } -} - -impl Error for ChainReplicationError { - fn source(&self) -> Option<&(dyn Error + 'static)> { - match self { - Self::Transport(error) => Some(error), - _ => None, - } - } -} - -impl From for ChainReplicationError { - fn from(error: SendError) -> Self { - Self::Transport(error) - } -} +use server_common::{Message, iobuf::Owned}; +use std::ops::AsyncFnOnce; /// Shared pipeline-first request flow (metadata + partitions). /// @@ -136,166 +75,43 @@ where /// /// # Errors /// -/// Returns an error if the prepare cannot be routed or the bus cannot deliver -/// it to the next replica. +/// Returns `SendError` if the bus fails to deliver to the next replica. /// Callers decide error policy (VSR retransmits from WAL via prepare timeout). +/// +/// # Panics +/// - If `header.command` is not `Command2::Prepare`. +/// - If `header.op <= consensus.commit_min()`. +/// - If the computed next replica equals self. #[allow(clippy::future_not_send)] pub async fn replicate_to_next_in_chain( consensus: &VsrConsensus, message: &Message, -) -> Result<(), ChainReplicationError> -where - B: MessageBus, - P: Pipeline, -{ - let Some(next) = replication_target(consensus, message.header())? else { - return Ok(()); - }; - let frozen = message.deep_copy().into_generic().into_frozen(); - consensus - .message_bus() - .send_to_replica(next, frozen) - .await - .map_err(Into::into) -} - -/// Forward an already validated frozen prepare to the next replica without -/// copying its payload. -/// -/// # Errors -/// -/// Returns an error if the frame is malformed, cannot be routed, or the bus -/// cannot deliver it to the next replica. -#[allow(clippy::future_not_send)] -pub async fn replicate_frozen_to_next_in_chain( - consensus: &VsrConsensus, - message: Frozen, -) -> Result<(), ChainReplicationError> +) -> Result<(), SendError> where B: MessageBus, P: Pipeline, { - let header = frozen_prepare_header(&message)?; - let Some(next) = replication_target(consensus, &header)? else { - return Ok(()); - }; - consensus - .message_bus() - .send_to_replica(next, message) - .await - .map_err(Into::into) -} - -fn frozen_prepare_header( - message: &Frozen, -) -> Result { - let header_bytes = message - .as_slice() - .get(..size_of::()) - .ok_or(ChainReplicationError::MalformedPrepare)?; - let header = bytemuck::checked::try_from_bytes::(header_bytes) - .copied() - .map_err(|_| ChainReplicationError::MalformedPrepare)?; - header - .validate() - .map_err(|_| ChainReplicationError::MalformedPrepare)?; - let frame_size = - usize::try_from(header.size).map_err(|_| ChainReplicationError::MalformedPrepare)?; - if !(size_of::()..=message.len()).contains(&frame_size) { - return Err(ChainReplicationError::MalformedPrepare); - } - Ok(header) -} + let header = *message.header(); -fn replication_target( - consensus: &VsrConsensus, - header: &PrepareHeader, -) -> Result, ChainReplicationError> -where - B: MessageBus, - P: Pipeline, -{ - if header.command != Command2::Prepare { - return Err(ChainReplicationError::UnexpectedCommand { - command: header.command, - }); - } - let commit_min = consensus.commit_min(); - if header.op <= commit_min { - return Err(ChainReplicationError::CommittedPrepare { - op: header.op, - commit_min, - }); - } + assert_eq!(header.command, Command2::Prepare); + assert!(header.op > consensus.commit_min()); let next = (consensus.replica() + 1) % consensus.replica_count(); let primary = consensus.primary_index(header.view); if next == primary { - return Ok(None); - } - - if next == consensus.replica() { - return Err(ChainReplicationError::SelfRoute { - replica: consensus.replica(), - }); - } - Ok(Some(next)) -} - -/// Re-stamp a stored prepare with the current view before retransmission. -/// The prepare identity excludes `view`, so the payload and operation identity -/// remain unchanged. -#[must_use] -pub fn restamp_prepare_view( - stored: Frozen, - view: u32, -) -> Option> { - const VIEW_OFFSET: usize = std::mem::offset_of!(PrepareHeader, view); - - let header = bytemuck::checked::try_from_bytes::( - stored.as_slice().get(..size_of::())?, - ) - .ok()?; - if header.view == view { - return Some(stored); + return Ok(()); } - let mut owned = Owned::::copy_from_slice(stored.as_slice()); - owned.as_mut_slice()[VIEW_OFFSET..VIEW_OFFSET + size_of::()] - .copy_from_slice(&view.to_ne_bytes()); - Message::::try_from(owned) - .ok() - .map(Message::into_frozen) -} + assert_ne!(next, consensus.replica()); -/// Recompute a prepare's integrity fields and report the first that disagrees. -/// -/// Everywhere else `checksum` is an opaque token: the pipeline, the merge, and the -/// repair ingest compare it for equality without asking whether it describes the -/// bytes it arrived with, so a corrupted frame is admitted whenever its flipped -/// value satisfies those comparisons, then journaled and re-served to peers. -/// -/// `frame` is the whole message. The body range comes from [`frame_body`], not the -/// caller, so no ingress point can verify a different span than the producer sealed. -/// [`CHECKSUM_UNSEALED`] skips the partition plane, which carries `batch_checksum` -/// over the same bytes instead. -/// -/// # Errors -/// Returns a static description of which field failed. -pub fn verify_prepare_integrity(header: &PrepareHeader, frame: &[u8]) -> Result<(), &'static str> { - if header.checksum != CHECKSUM_UNSEALED && header.identity_checksum() != header.checksum { - return Err("prepare header does not match its own checksum"); - } - if header.checksum_body != 0 - && u128::from(iggy_common::calculate_checksum(frame_body( - frame, - header.size, - ))) != header.checksum_body - { - return Err("prepare body does not match its checksum"); - } - Ok(()) + // Chain replication to the next replica is N=1, so the freeze-once + // trick does not apply: the caller has already appended `message` to + // its local journal (durability-before-ack) and kept a reference for + // this forward, so we deep_copy a fresh Frozen here. Future refactor + // could freeze once and share the backing with the journal path. + let frozen = message.deep_copy().into_generic().into_frozen(); + consensus.message_bus().send_to_replica(next, frozen).await } /// Shared preflight checks for `on_replicate`. @@ -353,23 +169,6 @@ where Ok(current_op) } -/// Stamp [`PrepareHeader::identity_checksum`] into a freshly built prepare. -/// -/// Call once, after every other field is final: the checksum covers them. -/// `checksum_body` in particular, since that is how the body reaches the value. -/// -/// # Panics -/// If the message is shorter than its own header. -#[must_use] -pub fn seal_prepare_checksum(mut message: Message) -> Message { - let checksum = message.header().identity_checksum(); - let bytes = &mut message.as_mut_slice()[..size_of::()]; - let header = bytemuck::checked::try_from_bytes_mut::(bytes) - .expect("a prepare header round-trips its own bit pattern"); - header.checksum = checksum; - message -} - /// Shared preflight checks for `on_ack`. /// /// # Errors @@ -550,6 +349,7 @@ where timestamp: prepare_header.timestamp, request: prepare_header.request, operation: prepare_header.operation, + namespace: prepare_header.namespace, ..Default::default() }; // `BytesMut` makes no alignment guarantee, so never cast into it. @@ -584,7 +384,7 @@ where #[must_use] #[allow(clippy::cast_possible_truncation)] pub fn build_result_rejection_reply( - request_header: &RoutedRequestHeader, + request_header: &RequestHeader, commit: u64, code: u32, ) -> Message { @@ -612,6 +412,7 @@ pub fn build_result_rejection_reply( timestamp: request_header.timestamp, request: request_header.request, operation: request_header.operation, + namespace: request_header.namespace, ..Default::default() }; buffer[..header_size].copy_from_slice(bytemuck::bytes_of(&header)); @@ -635,7 +436,7 @@ pub fn build_result_rejection_reply( #[allow(clippy::needless_pass_by_value, clippy::cast_possible_truncation)] pub fn build_reply_from_request( consensus: &VsrConsensus, - request_header: &RoutedRequestHeader, + request_header: &RequestHeader, body: bytes::Bytes, ) -> Message where @@ -665,6 +466,7 @@ where timestamp: request_header.timestamp, request: request_header.request, operation: request_header.operation, + namespace: request_header.namespace, ..Default::default() }; buffer[..header_size].copy_from_slice(bytemuck::bytes_of(&header)); @@ -690,7 +492,7 @@ where /// If the constructed message buffer is not valid. pub fn build_deny_reply_from_request( consensus: &VsrConsensus, - request_header: &RoutedRequestHeader, + request_header: &RequestHeader, status: u32, ) -> Message where @@ -709,7 +511,7 @@ where } /// [`build_deny_reply_from_request`] for layers that hold no consensus group -/// for the request's group (a shard fencing a frame aimed at a torn-down +/// for the request's namespace (a shard fencing a frame aimed at a torn-down /// or never-materialised partition). /// /// Replica-stamped fields (`cluster`, `view`, `replica`) echo the request @@ -722,7 +524,7 @@ where #[must_use] #[allow(clippy::cast_possible_truncation)] pub fn build_deny_reply_from_request_header( - request_header: &RoutedRequestHeader, + request_header: &RequestHeader, status: u32, ) -> Message { let header_size = std::mem::size_of::(); @@ -741,6 +543,7 @@ pub fn build_deny_reply_from_request_header( timestamp: request_header.timestamp, request: request_header.request, operation: request_header.operation, + namespace: request_header.namespace, ..Default::default() }; buffer[..header_size].copy_from_slice(bytemuck::bytes_of(&header)); @@ -817,18 +620,14 @@ pub async fn send_prepare_ok( prepare_checksum: header.checksum, request: header.request, operation: header.operation, - group: header.group, + namespace: header.namespace, size: std::mem::size_of::() as u32, ..Default::default() }; - let message: Message = Message::::new(std::mem::size_of::< - PrepareOkHeader, - >()) - .transmute_header(|_, new| { - *new = prepare_ok_header; - new.seal(); - }); + let message: Message = + Message::::new(std::mem::size_of::()) + .transmute_header(|_, new| *new = prepare_ok_header); let primary = consensus.primary_index(consensus.view()); consensus @@ -840,18 +639,10 @@ pub async fn send_prepare_ok( mod tests { use super::*; use crate::{Consensus, LocalPipeline, VsrAction}; - use aligned_vec::{AVec, ConstAlign}; use iggy_binary_protocol::{ConsensusHeader, Operation, StartViewChangeHeader}; - use iggy_common::calculate_checksum; use message_bus::SendError; use server_common::{MESSAGE_ALIGN, iobuf::Frozen}; - /// `PrepareHeader`'s alignment, which every suffix body has to satisfy. - const BODY_ALIGN: usize = align_of::(); - - /// A control-message body, aligned for the headers packed into it. - type Body = AVec>; - #[derive(Debug, Default)] struct NoopBus; @@ -962,7 +753,7 @@ mod tests { new.size = std::mem::size_of::() as u32; new.view = 1; new.replica = 0; - new.group = 0; + new.namespace = 0; }); let actions = consensus.handle_start_view_change(PlaneKind::Metadata, svc.header()); @@ -1002,7 +793,7 @@ mod tests { new.size = std::mem::size_of::() as u32; new.view = 1; new.replica = 0; - new.group = 0; + new.namespace = 0; }); let actions = consensus.handle_start_view_change(PlaneKind::Metadata, svc.header()); @@ -1025,7 +816,7 @@ mod tests { checksum: 0, checksum_body: 0, cluster: 0, - size: std::mem::size_of::() as u32, + size: 0, view: 1, release: 0, command: Command2::DoViewChange, @@ -1033,11 +824,9 @@ mod tests { reserved_frame: [0; 66], op: dvc_op, commit, - group: 0, + namespace: 0, log_view: 0, - reserved: [0; 68], - nack_bitset: 0, - present_bitset: 0, + reserved: [0; 100], }; assert!(header(dvc_commit).validate().is_ok()); assert!( @@ -1046,138 +835,6 @@ mod tests { ); } - #[test] - fn given_restamped_view_when_sealing_should_keep_the_same_identity() { - // `restamp_prepare_view` rewrites `view` on retransmission. If the identity - // moved with it, one op would carry different checksums per receiving view - // and the merge would read them as competing prepares nacking each other. - let base = PrepareHeader { - command: Command2::Prepare, - operation: iggy_binary_protocol::Operation::CreateStream, - op: 9, - view: 4, - client: 11, - request: 2, - timestamp: 1234, - checksum_body: 99, - ..Default::default() - }; - let restamped = PrepareHeader { view: 12, ..base }; - assert_eq!( - base.identity_checksum(), - restamped.identity_checksum(), - "view must not participate in a prepare's identity" - ); - } - - #[test] - fn given_matching_view_when_restamping_should_reuse_frozen_prepare() { - let message = prepare_message(9, 7, 11).transmute_header(|old, new: &mut PrepareHeader| { - *new = old; - new.view = 4; - }); - let frozen = message.into_frozen(); - let original_ptr = frozen.as_slice().as_ptr(); - - let restamped = restamp_prepare_view(frozen, 4).expect("valid prepare"); - let header = bytemuck::checked::try_from_bytes::( - &restamped[..size_of::()], - ) - .expect("restamped prepare header"); - - assert_eq!(restamped.as_slice().as_ptr(), original_ptr); - assert_eq!(header.view, 4); - } - - #[test] - fn given_new_view_when_restamping_should_only_change_view() { - let message = prepare_message(9, 7, 11).transmute_header(|old, new: &mut PrepareHeader| { - *new = old; - new.view = 4; - new.client = 17; - new.request = 23; - }); - let expected_identity = message.header().identity_checksum(); - let expected_op = message.header().op; - let expected_client = message.header().client; - let expected_request = message.header().request; - - let restamped = restamp_prepare_view(message.into_frozen(), 12).expect("valid prepare"); - let header = bytemuck::checked::try_from_bytes::( - &restamped[..size_of::()], - ) - .expect("restamped prepare header"); - - assert_eq!(header.view, 12); - assert_eq!(header.identity_checksum(), expected_identity); - assert_eq!(header.op, expected_op); - assert_eq!(header.client, expected_client); - assert_eq!(header.request, expected_request); - } - - #[test] - fn given_truncated_buffer_when_restamping_should_reject() { - let malformed: Frozen = Owned::::copy_from_slice(&[0]).into(); - - assert!(restamp_prepare_view(malformed, 1).is_none()); - } - - #[test] - fn given_truncated_buffer_when_reading_frozen_prepare_should_reject() { - let malformed: Frozen = Owned::::copy_from_slice(&[0]).into(); - - assert!(matches!( - frozen_prepare_header(&malformed), - Err(ChainReplicationError::MalformedPrepare) - )); - } - - #[test] - fn given_committed_prepare_when_selecting_replication_target_should_reject() { - let consensus = VsrConsensus::new(1, 0, 3, 0, NoopBus, LocalPipeline::new()); - consensus.init(); - let prepare = prepare_message(0, 0, 0); - - assert!(matches!( - replication_target(&consensus, prepare.header()), - Err(ChainReplicationError::CommittedPrepare { - op: 0, - commit_min: 0 - }) - )); - } - - #[test] - fn given_different_prepares_at_one_op_when_sealing_should_differ() { - // The distinction the merge depends on: two prepares at one op number are - // told apart, so a canonical header is distinguishable from a stale one. - let first = PrepareHeader { - command: Command2::Prepare, - operation: iggy_binary_protocol::Operation::CreateStream, - op: 5, - client: 1, - request: 1, - timestamp: 100, - ..Default::default() - }; - let second = PrepareHeader { client: 2, ..first }; - assert_ne!( - first.identity_checksum(), - second.identity_checksum(), - "distinct prepares at the same op must not share an identity" - ); - - let body_differs = PrepareHeader { - checksum_body: 7, - ..first - }; - assert_ne!( - first.identity_checksum(), - body_differs.identity_checksum(), - "the body reaches the identity through checksum_body" - ); - } - #[test] fn loopback_push_and_drain() { let consensus = VsrConsensus::new(1, 0, 3, 0, NoopBus, LocalPipeline::new()); @@ -1237,590 +894,81 @@ mod tests { assert_eq!(typed.header().command, Command2::PrepareOk); } - /// A sender's suffix and matching body bytes for a replica that holds every op - /// in `commit..=op` and can serve each body. Checksums derive from `(op, view)` - /// so the hash chain connects, which the merge checks. - fn dvc_with_full_suffix( - replica: u8, - view: u32, - log_view: u32, - op: u64, - commit: u64, - ) -> (iggy_binary_protocol::DoViewChangeHeader, Body) { - dvc_with_suffix(replica, view, log_view, op, commit, None) - } - - /// As [`dvc_with_full_suffix`], but `withhold_body` names one op whose present - /// bit is cleared: header held, body unservable. A quorum where every sender - /// withholds the same op decides nothing yet. - fn dvc_with_suffix( - replica: u8, - view: u32, - log_view: u32, - op: u64, - commit: u64, - withhold_body: Option, - ) -> (iggy_binary_protocol::DoViewChangeHeader, Body) { - use iggy_binary_protocol::DoViewChangeHeader; - - let headers = suffix_headers(commit, op, log_view); - let body = encode_body(&headers); - let mut present = if headers.is_empty() { - 0 - } else { - (1u128 << headers.len()) - 1 - }; - if let Some(withheld) = withhold_body - && let Some(index) = headers.iter().position(|header| header.op == withheld) - { - present &= !(1u128 << index); - } - let header = DoViewChangeHeader { - checksum: 0, - checksum_body: 0, - cluster: 0, - size: u32::try_from(std::mem::size_of::() + body.len()) - .expect("synthetic DVC frame fits u32"), - view, - release: 0, - command: Command2::DoViewChange, - replica, - reserved_frame: [0; 66], - op, - commit, - group: 0, - log_view, - reserved: [0; 68], - nack_bitset: 0, - present_bitset: present, - }; - (header, body) - } + #[test] + fn loopback_cleared_on_complete_view_change_as_primary() { + use iggy_binary_protocol::{DoViewChangeHeader, StartViewChangeHeader}; - /// Headers for `low..=high`, high-to-low as a suffix requires, sealed and - /// chained the way a real producer writes them. - /// - /// Built ascending so each `parent` is the previous entry's real identity, then - /// reversed. The decoder recomputes both, so fabricated checksums are rejected - /// before the code under test sees them. - fn suffix_headers(low: u64, high: u64, view: u32) -> Vec { - if high == 0 { - return Vec::new(); - } - let mut parent = 0u128; - let mut ascending = Vec::new(); - for op in low.max(1)..=high { - let mut header = PrepareHeader { - command: Command2::Prepare, - operation: iggy_binary_protocol::Operation::CreateStream, - op, - view, - parent, - // Strictly increasing with op, so the suffix reads decreasing. - timestamp: op, - // Zero so the DVC's own commit drives `commit_max`. - commit: 0, - ..Default::default() - }; - header.checksum = header.identity_checksum(); - parent = header.checksum; - ascending.push(header); - } - ascending.reverse(); - ascending - } + // 3 replicas, replica 0 is primary for view 0 (and view 3: 3 % 3 = 0). + let consensus = VsrConsensus::new(1, 0, 3, 0, NoopBus, LocalPipeline::new()); + consensus.init(); - fn svc_header(replica: u8, view: u32) -> iggy_binary_protocol::StartViewChangeHeader { - iggy_binary_protocol::StartViewChangeHeader { + // SVC from replica 1, view 3. Replica 0 advances to view 3 + // (reset_view_change_state clears loopback), records own SVC+DVC and + // replica 1's SVC. DVC quorum needs 2; have 1. + let svc = StartViewChangeHeader { checksum: 0, checksum_body: 0, cluster: 0, - size: u32::try_from(std::mem::size_of::< - iggy_binary_protocol::StartViewChangeHeader, - >()) - .expect("header fits u32"), - view, + size: 0, + view: 3, release: 0, command: Command2::StartViewChange, - replica, + replica: 1, reserved_frame: [0; 66], - group: 0, + namespace: 0, reserved: [0; 120], - } - } - - /// Install the suffix this replica would read from its own journal. - fn install_local_suffix( - consensus: &VsrConsensus, - op: u64, - commit: u64, - log_view: u32, - ) { - let headers = suffix_headers(commit, op, log_view); - consensus.set_local_dvc_suffix(crate::dvc_merge::suffix_all_present(headers)); - } - - #[test] - fn given_an_undecidable_quorum_when_a_later_dvc_decides_it_should_start_the_view() { - // Reaching a view-change quorum is not the same as deciding a log. Latching - // `do_view_change_quorum` at the quorum makes every non-Ready outcome - // terminal: later DoViewChanges are recorded, but the guard that calls the - // merge is already false, so the view burns its status timeout for nothing. - // - // 5 replicas, view_change quorum 3, replica 0 is primary for view 5. - let consensus = VsrConsensus::new(1, 0, 5, 0, NoopBus, LocalPipeline::new()); - consensus.init(); - consensus.restore_commit_state(2, 2); - consensus.sequencer().set_sequence(4); - // This replica holds op 4's header but cannot serve its body. Suffix entries - // run high-to-low, so bit 0 is op 4: clearing it offers ops 3 and 2 only. - let local = suffix_headers(2, 4, 0); - consensus.set_local_dvc_suffix(crate::view_change_quorum::DvcSuffix::new(local, 0, 0b110)); - - let _ = consensus.handle_start_view_change(PlaneKind::Metadata, &svc_header(1, 5)); - - // Two peers report, reaching the quorum of 3. All three hold op 4's header, - // none can serve its body, and two replicas have yet to report. - for replica in [1u8, 2] { - let (dvc, body) = dvc_with_suffix(replica, 5, 0, 4, 2, Some(4)); - let actions = consensus.handle_do_view_change(PlaneKind::Metadata, &dvc, &body); - assert!(actions.is_empty()); - } - assert!( - consensus.pending_view_log().is_none(), - "op 4 is neither servable nor provably dead, so nothing may be parked yet" - ); - - // Replica 3 arrives holding the body: the deciding message, still merged. - let (dvc, body) = dvc_with_full_suffix(3, 5, 0, 4, 2); - let _ = consensus.handle_do_view_change(PlaneKind::Metadata, &dvc, &body); - - let pending = consensus - .pending_view_log() - .expect("the DVC that supplies the missing body must complete the merge"); - assert_eq!(pending.op_head, 4); - assert_eq!(pending.commit_max, 2); - } - - #[test] - fn given_a_sealed_prepare_when_verifying_integrity_should_accept() { - let message = Message::::new(size_of::()).transmute_header( - |_, header: &mut PrepareHeader| { - header.command = Command2::Prepare; - header.op = 7; - header.size = u32::try_from(size_of::()).expect("header fits u32"); - }, - ); - let sealed = seal_prepare_checksum(message); - assert_eq!(verify_prepare_integrity(sealed.header(), &[]), Ok(())); - } - - #[test] - fn given_a_prepare_whose_header_was_altered_when_verifying_should_reject() { - // Downstream compares `checksum` as an opaque token, so without this a frame - // corrupted in transit is journaled and then re-served to peers from the WAL. - let message = Message::::new(size_of::()).transmute_header( - |_, header: &mut PrepareHeader| { - header.command = Command2::Prepare; - header.op = 7; - header.size = u32::try_from(size_of::()).expect("header fits u32"); - }, - ); - let mut sealed = seal_prepare_checksum(message); - let bytes = &mut sealed.as_mut_slice()[..size_of::()]; - let header = bytemuck::checked::try_from_bytes_mut::(bytes) - .expect("a prepare header round-trips its own bit pattern"); - header.op = 8; - assert!(verify_prepare_integrity(&header.clone(), &[]).is_err()); - } - - #[test] - fn given_an_unsealed_prepare_when_verifying_should_abstain() { - // The partition plane leaves `checksum` at `CHECKSUM_UNSEALED` and carries a - // verified `batch_checksum` over the same bytes instead. - let header = PrepareHeader { - command: Command2::Prepare, - op: 7, - ..Default::default() }; - assert_eq!(header.checksum, CHECKSUM_UNSEALED); - assert_eq!(verify_prepare_integrity(&header, &[]), Ok(())); - } - - /// A frame carrying `body`, with `size` covering exactly header + body and - /// `checksum_body` sealed over it, as the metadata projection does. - /// `trailing` bytes of garbage past the sealed frame, which `size` does not - /// cover. The buffer is `MESSAGE_ALIGN`ed: `PrepareHeader` holds `u128`s, so a - /// `Vec` would only be 16-aligned by the allocator's good graces and miri - /// rejects the cast. - fn sealed_frame(body: &[u8], trailing: usize) -> Owned { - let size = size_of::() + body.len(); - let mut frame = Owned::::zeroed(size + trailing); - let bytes = frame.as_mut_slice(); - bytes[size_of::()..size].copy_from_slice(body); - bytes[size..].fill(0xAA); - let header = bytemuck::checked::from_bytes_mut::( - &mut bytes[..size_of::()], - ); - header.command = Command2::Prepare; - header.op = 7; - header.size = u32::try_from(size).expect("fits u32"); - header.checksum_body = u128::from(calculate_checksum(body)); - frame - } - - fn frame_header(frame: &Owned) -> PrepareHeader { - *bytemuck::checked::from_bytes::( - &frame.as_slice()[..size_of::()], - ) - } - - #[test] - fn given_a_prepare_whose_body_was_altered_when_verifying_should_reject() { - let mut frame = sealed_frame(b"body", 0); - let header = frame_header(&frame); - assert_eq!(verify_prepare_integrity(&header, frame.as_slice()), Ok(())); - - *frame - .as_mut_slice() - .last_mut() - .expect("the frame has a body") ^= 1; - assert!(verify_prepare_integrity(&header, frame.as_slice()).is_err()); - } - - #[test] - fn given_bytes_past_the_frame_size_when_verifying_should_ignore_them() { - // `try_from` accepts a buffer longer than `size` without trimming; hashing to - // the end would reject a correctly sealed prepare and disagree with the WAL scan. - let padded = sealed_frame(b"body", 16); - let header = frame_header(&padded); - assert_eq!( - verify_prepare_integrity(&header, padded.as_slice()), - Ok(()), - "only the bytes `size` covers are the body" - ); - } - - #[test] - fn given_a_size_that_overruns_the_buffer_when_verifying_should_reject() { - // Truncated frame, header still claims the full length: the body it names is - // not there to hash. - let frame = sealed_frame(b"body", 0); - let header = frame_header(&frame); - - let truncated = &frame.as_slice()[..frame.as_slice().len() - 1]; - assert!(verify_prepare_integrity(&header, truncated).is_err()); - } - - #[test] - fn given_a_parked_merge_when_not_yet_started_should_not_advance_log_view() { - // `log_view` claims "my log IS the log this view decided", which is what - // makes a sender canonical next time. Raising it when the merge parks, before - // the merged head is installed, lets a primary-elect that never finishes - // repair vote as canonical carrying its own stale head, and ops the merge - // kept then fall outside the next scan range, dropped with no nack. - let consensus = VsrConsensus::new(1, 0, 3, 0, NoopBus, LocalPipeline::new()); - consensus.init(); - consensus.restore_commit_state(2, 2); - consensus.sequencer().set_sequence(3); - install_local_suffix(&consensus, 3, 2, 0); - assert_eq!(consensus.log_view(), 0); - - let _ = consensus.handle_start_view_change(PlaneKind::Metadata, &svc_header(1, 3)); - let (dvc, body) = dvc_with_full_suffix(2, 3, 0, 3, 2); - let _ = consensus.handle_do_view_change(PlaneKind::Metadata, &dvc, &body); - - assert!(consensus.pending_view_log().is_some(), "the merge parks"); - assert_eq!( - consensus.log_view(), - 0, - "a parked merge has installed nothing, so log_view must still \ - describe the log this replica actually holds" - ); - assert_eq!(consensus.view(), 3, "the view itself did advance"); - - let _ = consensus.start_pending_view(PlaneKind::Metadata); - assert_eq!( - consensus.log_view(), - 3, - "installing the merged head is what earns the log_view claim" - ); - } - - #[test] - fn loopback_cleared_on_complete_view_change_as_primary() { - // 3 replicas, replica 0 is primary for view 0 (and view 3: 3 % 3 = 0). - let consensus = VsrConsensus::new(1, 0, 3, 0, NoopBus, LocalPipeline::new()); - consensus.init(); - consensus.restore_commit_state(2, 2); - consensus.sequencer().set_sequence(3); - install_local_suffix(&consensus, 3, 2, 0); - - // SVC from replica 1, view 3. Replica 0 advances, records own SVC+DVC and - // replica 1's SVC. DVC quorum needs 2; have 1. - let _ = consensus.handle_start_view_change(PlaneKind::Metadata, &svc_header(1, 3)); + let _ = consensus.handle_start_view_change(PlaneKind::Metadata, &svc); // Stale loopback queued between SVC and DVC quorum. let stale_msg = Message::::new(std::mem::size_of::()); consensus.push_loopback(stale_msg.into_generic()); - // DVC from replica 2 forms the quorum and the merge settles the log. - let (dvc, body) = dvc_with_full_suffix(2, 3, 0, 3, 2); - let actions = consensus.handle_do_view_change(PlaneKind::Metadata, &dvc, &body); - - // Parked: nothing is announced until the journal can serve it. - assert!( - actions.is_empty(), - "a merged view change must announce nothing until repair completes" - ); - let pending = consensus - .pending_view_log() - .expect("a decidable quorum must park a merged log"); - assert_eq!(pending.op_head, 3); - assert_eq!(pending.commit_max, 2); - assert_eq!(consensus.status(), Status::ViewChange); - - // Stale loopback must be cleared. - let mut buf = Vec::new(); - consensus.drain_loopback_into(&mut buf); - assert!( - buf.is_empty(), - "loopback queue must be empty after view change completion" - ); - - // Journal now covers the merged log, so the view starts and announces. - let actions = consensus.start_pending_view(PlaneKind::Metadata); - assert!( - actions - .iter() - .any(|a| matches!(a, crate::VsrAction::SendStartView { .. })), - "expected SendStartView once the view starts" - ); - assert_eq!(consensus.status(), Status::Normal); - assert!( - consensus.pending_view_log().is_none(), - "starting the view must consume the parked log" - ); - } - - /// Refusing to start a view must not be terminal. - /// - /// A parked merge leaves the replica in `ViewChange` announcing nothing if the - /// bodies never arrive, which is the intended trade against losing data but has - /// to stay recoverable: the status timeout fires, escalates, and drops the parked - /// log. Reusing a log merged for a superseded view would leak a truncation - /// decided there into a view that never voted for it. - /// A `StartView` from the view's primary, optionally carrying the view's - /// canonical headers. - fn start_view_with_suffix( - replica: u8, - view: u32, - op: u64, - commit: u64, - with_suffix: bool, - ) -> (iggy_binary_protocol::StartViewHeader, Body) { - use iggy_binary_protocol::StartViewHeader; - - let body = if with_suffix { - encode_body(&suffix_headers(commit, op, view)) - } else { - Body::new(BODY_ALIGN) - }; - let header = StartViewHeader { + // DVC from replica 2, view 3, quorum, complete_view_change_as_primary fires. + let dvc = DoViewChangeHeader { checksum: 0, checksum_body: 0, cluster: 0, - size: u32::try_from(std::mem::size_of::() + body.len()) - .expect("synthetic StartView fits u32"), - view, + size: 0, + view: 3, release: 0, - command: Command2::StartView, - replica, + command: Command2::DoViewChange, + replica: 2, reserved_frame: [0; 66], - op, - commit, - group: 0, - reserved: [0; 88], - incarnation: 0, + op: 0, + commit: 0, + namespace: 0, + log_view: 0, + reserved: [0; 100], }; - (header, body) - } - - /// Encode headers as a control-message body. - /// - /// Aligned, because `dvc_suffix_decode` uses a checked `bytemuck` cast per - /// 256-byte chunk: a `Vec` body reports `MalformedHeader` for entry 0 - /// instead of the failure under test. glibc over-aligns these; Miri does not. - fn encode_body(headers: &[PrepareHeader]) -> Body { - let mut body = Body::with_capacity(BODY_ALIGN, std::mem::size_of_val(headers)); - for header in headers { - body.extend_from_slice(bytemuck::bytes_of(header)); - } - body - } - - #[test] - fn given_a_corrupted_suffix_entry_when_decoding_should_reject_the_frame() { - // The worst failure mode: a flipped bit in a canonical sender's header makes - // it canonical for the view, so honest senders read as disagreeing and can - // reach a nack quorum against a committed op. Recomputing keeps that out. - let mut headers = suffix_headers(2, 4, 1); - headers[0].timestamp ^= 0xFF; - let body = encode_body(&headers); - - let error = crate::dvc_suffix_decode(&body, 4, 0, 0) - .expect_err("a header that does not match its own checksum must be rejected"); - assert_eq!(error, crate::DvcSuffixError::ChecksumMismatch { index: 0 }); - } - - #[test] - fn given_a_broken_suffix_chain_when_decoding_should_reject_the_frame() { - // Well-sealed entries that do not link: a log, not a bag of records. - let mut headers = suffix_headers(2, 4, 1); - headers[0].parent ^= 0xFF; - headers[0].checksum = headers[0].identity_checksum(); - let body = encode_body(&headers); - - let error = crate::dvc_suffix_decode(&body, 4, 0, 0) - .expect_err("a suffix whose entries do not chain must be rejected"); - assert_eq!(error, crate::DvcSuffixError::ChainBreak { index: 1 }); - } - - #[test] - fn given_a_suffix_with_mixed_view_stamps_when_decoding_should_be_accepted() { - // A stitched suffix: a held op keeps the view that delivered it, a repaired - // neighbour carries the view that decided it. Rejecting drops the sender's - // vote forever (the retransmit is byte-identical) and the cluster can fail to - // elect. No re-seal: `identity_checksum` excludes `view`. - let mut headers = suffix_headers(2, 4, 2); - headers[1].view = 1; - let body = encode_body(&headers); - - let suffix = crate::dvc_suffix_decode(&body, 4, 0, 0) - .expect("a stitched suffix with mixed view stamps must decode"); - assert_eq!(suffix.len(), 3); - } - - #[test] - fn given_an_unsealed_suffix_when_decoding_should_be_accepted() { - // The on-disk sentinel: suffixes are read out of the journal, which may hold - // pre-seal entries, and partition-plane prepares are unsealed by construction. - let headers: Vec = suffix_headers(2, 4, 1) - .into_iter() - .map(|mut header| { - header.checksum = 0; - header.parent = 0; - header - }) - .collect(); - let body = encode_body(&headers); - - let suffix = - crate::dvc_suffix_decode(&body, 4, 0, 0).expect("an unsealed suffix must still decode"); - assert_eq!(suffix.len(), 3); - } - - #[test] - fn given_start_view_with_suffix_when_adopted_should_record_the_canonical_headers() { - // The backup keeps the view's headers so its repair ingest can reject a body - // that disagrees with the view's decision, and so a disagreeing local entry - // is reported rather than silently blocking its own repair forever. - let consensus = VsrConsensus::new(1, 0, 3, 0, NoopBus, LocalPipeline::new()); - consensus.init(); - - // Replica 1 is primary for view 1 (1 % 3). - let (header, body) = start_view_with_suffix(1, 1, 5, 3, true); - let actions = consensus.handle_start_view(PlaneKind::Metadata, &header, &body); - - assert!(!actions.is_empty(), "a valid StartView must be adopted"); - assert_eq!(consensus.status(), Status::Normal); - assert_eq!(consensus.sequencer().current_sequence(), 5); - - let recorded = consensus - .pending_view_log() - .expect("an adopted StartView carrying a suffix must record its headers"); - assert_eq!(recorded.op_head, 5); - assert_eq!(recorded.commit_max, 3); - assert_eq!( - recorded.headers.iter().map(|h| h.op).collect::>(), - vec![5, 4, 3], - "headers run high-to-low from the head down to the announced commit" - ); - } - - #[test] - fn given_start_view_without_suffix_when_adopted_should_trust_the_announced_op() { - // Probe answers and stale-view corrections carry numbers only: a backup must - // still adopt, and record nothing it could mistake for the view's decision. - let consensus = VsrConsensus::new(1, 0, 3, 0, NoopBus, LocalPipeline::new()); - consensus.init(); - - let (header, body) = start_view_with_suffix(1, 1, 5, 3, false); - assert!(body.is_empty()); - let actions = consensus.handle_start_view(PlaneKind::Metadata, &header, &body); + let actions = consensus.handle_do_view_change(PlaneKind::Metadata, &dvc); + // View change complete → SendStartView action. assert!( - !actions.is_empty(), - "a numbers-only StartView must still adopt" - ); - assert_eq!(consensus.sequencer().current_sequence(), 5); - assert!( - consensus.pending_view_log().is_none(), - "no suffix means no canonical headers to verify against" + actions + .iter() + .any(|a| matches!(a, crate::VsrAction::SendStartView { .. })), + "expected SendStartView after DVC quorum" ); - } - - #[test] - fn given_parked_view_change_when_status_timeout_fires_should_escalate_and_drop_merged_log() { - let consensus = VsrConsensus::new(1, 0, 3, 0, NoopBus, LocalPipeline::new()); - consensus.init(); - consensus.restore_commit_state(2, 2); - consensus.sequencer().set_sequence(3); - install_local_suffix(&consensus, 3, 2, 0); - - let _ = consensus.handle_start_view_change(PlaneKind::Metadata, &svc_header(1, 3)); - let (dvc, body) = dvc_with_full_suffix(2, 3, 0, 3, 2); - let _ = consensus.handle_do_view_change(PlaneKind::Metadata, &dvc, &body); - - // Parked: no shard here reports coverage, so the view never starts. - assert!(consensus.pending_view_log().is_some()); - assert_eq!(consensus.status(), Status::ViewChange); - let parked_view = consensus.view(); - - // `VIEW_CHANGE_STATUS_TICKS` is 500; tick past it. Escalation shows as the - // view advancing, since the 50-tick SVC retransmit also emits a send. - let mut escalated = false; - for _ in 0..600 { - let _ = consensus.tick(PlaneKind::Metadata); - if consensus.view() > parked_view { - escalated = true; - break; - } - } + // Stale loopback must be cleared. + let mut buf = Vec::new(); + consensus.drain_loopback_into(&mut buf); assert!( - escalated, - "a parked view change must still escalate on the status timeout" - ); - assert!( - consensus.view() > parked_view, - "escalation must advance the view past {parked_view}, got {}", - consensus.view() - ); - assert!( - consensus.pending_view_log().is_none(), - "the superseded merged log must be dropped, not carried into the next view" + buf.is_empty(), + "loopback queue must be empty after view change completion" ); - assert_eq!(consensus.status(), Status::ViewChange); } - /// A merged log may claim an uncommitted range up to the *configured* + /// A DVC winner may claim an uncommitted range up to the *configured* /// prepare depth. With a pipeline deeper than the default const, the new /// primary schedules the rebuild rather than panicking on the old /// `PIPELINE_PREPARE_QUEUE_MAX` bound. #[test] #[allow(clippy::cast_possible_truncation)] - fn given_view_change_range_above_default_when_starting_view_should_rebuild() { + fn given_view_change_range_above_default_when_complete_as_primary_should_rebuild() { + use iggy_binary_protocol::{DoViewChangeHeader, StartViewChangeHeader}; + let depth = crate::PIPELINE_PREPARE_QUEUE_MAX * 2; // Strictly above the default const, still within the configured depth. let winner_op = (crate::PIPELINE_PREPARE_QUEUE_MAX + 8) as u64; @@ -1835,23 +983,48 @@ mod tests { LocalPipeline::with_capacities(depth, depth * 2), ); consensus.init(); - consensus.sequencer().set_sequence(winner_op); - install_local_suffix(&consensus, winner_op, 1, 0); // SVC from replica 1 moves replica 0 into view 3 and records its own DVC. - let _ = consensus.handle_start_view_change(PlaneKind::Metadata, &svc_header(1, 3)); + let svc = StartViewChangeHeader { + checksum: 0, + checksum_body: 0, + cluster: 0, + size: 0, + view: 3, + release: 0, + command: Command2::StartViewChange, + replica: 1, + reserved_frame: [0; 66], + namespace: 0, + reserved: [0; 120], + }; + let _ = consensus.handle_start_view_change(PlaneKind::Metadata, &svc); - // DVC from replica 2 claims the same deep log, forming quorum. - let (dvc, body) = dvc_with_full_suffix(2, 3, 0, winner_op, 1); - let _ = consensus.handle_do_view_change(PlaneKind::Metadata, &dvc, &body); + // DVC from replica 2 claims a log head far past commit, forming quorum. + let dvc = DoViewChangeHeader { + checksum: 0, + checksum_body: 0, + cluster: 0, + size: 0, + view: 3, + release: 0, + command: Command2::DoViewChange, + replica: 2, + reserved_frame: [0; 66], + op: winner_op, + commit: 0, + namespace: 0, + log_view: 0, + reserved: [0; 100], + }; + let actions = consensus.handle_do_view_change(PlaneKind::Metadata, &dvc); - let actions = consensus.start_pending_view(PlaneKind::Metadata); assert!( actions.iter().any(|action| matches!( action, - VsrAction::RebuildPipeline { from_op: 2, to_op } if *to_op == winner_op + VsrAction::RebuildPipeline { from_op: 1, to_op } if *to_op == winner_op )), - "expected RebuildPipeline over the uncommitted range, got {actions:?}" + "expected RebuildPipeline over the full uncommitted range" ); } @@ -1990,11 +1163,12 @@ mod tests { consensus.init(); consensus.advance_commit_max(4); - let request = RoutedRequestHeader { + let request = RequestHeader { command: Command2::Request, operation: Operation::DeleteConsumerOffset2, client: 42, request: 7, + namespace: 9, ..Default::default() }; let status = 3021; @@ -2007,6 +1181,7 @@ mod tests { assert_eq!(header.commit, 4); assert_eq!(header.client, 42); assert_eq!(header.request, 7); + assert_eq!(header.namespace, 9); assert_eq!(header.operation, Operation::DeleteConsumerOffset2); assert_eq!( header.size as usize, diff --git a/core/consensus/src/view_change_quorum.rs b/core/consensus/src/view_change_quorum.rs index 95da481b31..2eaff15b36 100644 --- a/core/consensus/src/view_change_quorum.rs +++ b/core/consensus/src/view_change_quorum.rs @@ -16,201 +16,27 @@ // under the License. use crate::REPLICAS_MAX; -use iggy_binary_protocol::{ - CHECKSUM_UNSEALED, Command2, ConsensusHeader, DVC_HEADERS_MAX, Operation, PrepareHeader, -}; - -/// Write prepare headers into a control-message body, high-to-low op. -/// -/// Shared by `DoViewChange` and `StartView`, which both carry a suffix as a plain -/// run of 256-byte headers; stating the layout once keeps them from drifting. -/// -/// # Panics -/// When `dst` is not exactly `headers.len()` headers wide. -pub fn encode_prepare_headers(headers: &[PrepareHeader], dst: &mut [u8]) { - let stride = size_of::(); - assert_eq!( - dst.len(), - std::mem::size_of_val(headers), - "control-message body buffer must fit the headers exactly" - ); - for (index, header) in headers.iter().enumerate() { - dst[index * stride..(index + 1) * stride].copy_from_slice(bytemuck::bytes_of(header)); - } -} - -/// Placeholder standing in for a suffix entry the sender does not hold. -/// -/// The suffix stays consecutive so a `(head_op, op)` pair indexes it arithmetically -/// and one bitset bit lines up with one op. A gap is transmitted, not omitted. -/// -/// `Operation::Reserved` is the marker, since no real prepare carries it, and every -/// other field is zero. [`dvc_header_kind`] insists on exactly that, so arbitrary -/// bytes cannot pass as a blank the merge would index. -#[must_use] -pub fn dvc_blank(op: u64) -> PrepareHeader { - PrepareHeader { - command: Command2::Prepare, - operation: Operation::Reserved, - op, - ..Default::default() - } -} - -/// What a suffix slot says about the sender's log at that op. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum DvcHeaderKind { - /// No header for this op, or one the sender cannot vouch for. The nack bit is - /// what distinguishes "never prepared" (proof) from "lost it" (no proof). - Blank, - /// A real prepare header the sender holds. - Valid, -} - -/// Classify a suffix slot, by exact comparison against the canonical blank. -#[must_use] -pub fn dvc_header_kind(header: &PrepareHeader) -> DvcHeaderKind { - if *header == dvc_blank(header.op) { - DvcHeaderKind::Blank - } else { - DvcHeaderKind::Valid - } -} - -/// A sender's uncommitted suffix: the headers spanning `commit..=op`, high-to-low, -/// plus one nack bit and one present bit per entry. -/// -/// Index 0 is the head (`StoredDvc::op`); index `i` is op `op - i`. Empty when the -/// sender has nothing uncommitted, or could not snapshot a suffix for this log. -#[derive(Debug, Clone, Default)] -pub struct DvcSuffix { - headers: Vec, - nack_bitset: u128, - present_bitset: u128, -} - -// These all read through the `Vec`, and neither `Vec::len` nor `Deref` is const on -// the pinned toolchain, so clippy's suggestion does not compile. -#[allow(clippy::missing_const_for_fn)] -impl DvcSuffix { - /// # Panics - /// When `headers` exceeds [`DVC_HEADERS_MAX`], or a bitset sets a bit past the - /// suffix. Sender-side programming errors; the same conditions off the wire go - /// through `DoViewChangeHeader::validate`. - #[must_use] - pub fn new(headers: Vec, nack_bitset: u128, present_bitset: u128) -> Self { - assert!( - headers.len() <= DVC_HEADERS_MAX, - "DVC suffix of {} entries exceeds the addressable maximum {DVC_HEADERS_MAX}", - headers.len() - ); - if headers.len() < DVC_HEADERS_MAX { - let beyond = !((1u128 << headers.len()) - 1); - assert_eq!( - nack_bitset & beyond, - 0, - "nack bit set past the {}-entry suffix", - headers.len() - ); - assert_eq!( - present_bitset & beyond, - 0, - "present bit set past the {}-entry suffix", - headers.len() - ); - } - Self { - headers, - nack_bitset, - present_bitset, - } - } - - /// A sender contributing numbers only: no headers, nacks, or offered bodies. - #[must_use] - pub fn empty() -> Self { - Self::default() - } - - #[must_use] - pub fn len(&self) -> usize { - self.headers.len() - } - - #[must_use] - pub fn is_empty(&self) -> bool { - self.headers.is_empty() - } - - #[must_use] - pub fn headers(&self) -> &[PrepareHeader] { - &self.headers - } - - #[must_use] - pub const fn nack_bitset(&self) -> u128 { - self.nack_bitset - } - - #[must_use] - pub const fn present_bitset(&self) -> u128 { - self.present_bitset - } - - /// Bytes the headers occupy on the wire. - #[must_use] - pub fn encoded_len(&self) -> usize { - self.headers.len() * size_of::() - } - - /// Write the headers into a `DoViewChange` body, high-to-low op. - /// - /// Paired with [`dvc_suffix_decode`], so the wire ordering is stated once. - /// - /// # Panics - /// When `dst` is not exactly [`Self::encoded_len`] bytes. - pub fn encode_into(&self, dst: &mut [u8]) { - encode_prepare_headers(&self.headers, dst); - } - - /// Slot index for `op`. `None` when `op` falls outside this sender's window. - #[must_use] - pub fn index_of(&self, head_op: u64, op: u64) -> Option { - let distance = usize::try_from(head_op.checked_sub(op)?).ok()?; - (distance < self.headers.len()).then_some(distance) - } - - /// The header at `index`, or `None` for a blank or out-of-range slot. - #[must_use] - pub fn valid_header_at(&self, index: usize) -> Option<&PrepareHeader> { - let header = self.headers.get(index)?; - matches!(dvc_header_kind(header), DvcHeaderKind::Valid).then_some(header) - } - - /// Whether the sender proves it never prepared the entry at `index`. - #[must_use] - pub fn nacks(&self, index: usize) -> bool { - index < self.headers.len() && self.nack_bitset & (1u128 << index) != 0 - } - - /// Whether the sender can serve the body of the entry at `index`. - #[must_use] - pub fn offers_body(&self, index: usize) -> bool { - index < self.headers.len() && self.present_bitset & (1u128 << index) != 0 - } -} /// Stored information from a `DoViewChange` message. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Copy)] pub struct StoredDvc { pub replica: u8, /// The view when the replica's status was last normal. pub log_view: u32, pub op: u64, pub commit: u64, - /// The sender's uncommitted suffix. Empty from a silent sender, which counts - /// toward the quorum and `max(commit)` but neither nacks nor offers bodies. - pub suffix: DvcSuffix, +} + +impl StoredDvc { + /// Compare for log selection: highest `log_view`, then highest op. + #[must_use] + pub const fn is_better_than(&self, other: &Self) -> bool { + if self.log_view == other.log_view { + self.op > other.op + } else { + self.log_view > other.log_view + } + } } /// Array type for storing DVC messages from all replicas. @@ -218,13 +44,12 @@ pub type DvcQuorumArray = [Option; REPLICAS_MAX]; /// Create an empty DVC quorum array. #[must_use] -pub fn dvc_quorum_array_empty() -> DvcQuorumArray { - // `[None; REPLICAS_MAX]` needs `StoredDvc: Copy`, ruled out by the `Vec`. - std::array::from_fn(|_| None) +pub const fn dvc_quorum_array_empty() -> DvcQuorumArray { + [None; REPLICAS_MAX] } /// Record a DVC in the array. Returns true if this is a new entry (not duplicate). -pub fn dvc_record(array: &mut DvcQuorumArray, dvc: StoredDvc) -> bool { +pub const fn dvc_record(array: &mut DvcQuorumArray, dvc: StoredDvc) -> bool { let slot = &mut array[dvc.replica as usize]; if slot.is_some() { return false; // Duplicate @@ -239,203 +64,43 @@ pub fn dvc_count(array: &DvcQuorumArray) -> usize { array.iter().filter(|m| m.is_some()).count() } -/// Reset the DVC quorum array. -pub fn dvc_reset(array: &mut DvcQuorumArray) { - *array = dvc_quorum_array_empty(); +/// Check if a specific replica has sent a DVC. +#[must_use] +pub fn dvc_has_from(array: &DvcQuorumArray, replica: u8) -> bool { + array.get(replica as usize).is_some_and(Option::is_some) } -/// Iterator over all stored DVCs. -pub fn dvc_iter(array: &DvcQuorumArray) -> impl Iterator { - array.iter().filter_map(|m| m.as_ref()) +/// Select the winning DVC (best log) from the quorum. +/// Returns the DVC with: highest `log_view`, then highest op. +#[must_use] +pub fn dvc_select_winner(array: &DvcQuorumArray) -> Option<&StoredDvc> { + array + .iter() + .filter_map(|m| m.as_ref()) + .max_by(|a, b| match a.log_view.cmp(&b.log_view) { + std::cmp::Ordering::Equal => a.op.cmp(&b.op), + other => other, + }) } -/// Why a `DoViewChange` body could not be read as a suffix. -/// -/// Dropped whole rather than partially trusted: the merge indexes arithmetically -/// from the head op, so one bad offset misattributes a header, nack, or body. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum DvcSuffixError { - /// Body length is not a whole number of headers. - NotHeaderMultiple { body_len: usize }, - /// More entries than the bitsets can address. - TooManyEntries { count: usize }, - /// An entry is not a valid `PrepareHeader` bit pattern. - MalformedHeader { index: usize }, - /// Entries are not consecutive descending from the head op. - OpOutOfOrder { - index: usize, - expected: u64, - found: u64, - }, - /// A bitset addresses an entry the body does not contain. - BitsetBeyondSuffix { count: usize }, - /// An entry's identity checksum does not match its own contents. - ChecksumMismatch { index: usize }, - /// A lower entry claims a timestamp at or after the entry above it. - TimestampNotDecreasing { index: usize }, - /// Consecutive entries do not hash-chain. - ChainBreak { index: usize }, +/// Get the maximum commit number across all DVCs. +#[must_use] +pub fn dvc_max_commit(array: &DvcQuorumArray) -> u64 { + array + .iter() + .filter_map(|m| m.as_ref()) + .map(|dvc| dvc.commit) + .max() + .unwrap_or(0) } -impl std::fmt::Display for DvcSuffixError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::NotHeaderMultiple { body_len } => write!( - f, - "do_view_change body of {body_len} bytes is not a whole number of {} -byte headers", - size_of::() - ), - Self::TooManyEntries { count } => write!( - f, - "do_view_change suffix of {count} entries exceeds the maximum {DVC_HEADERS_MAX}" - ), - Self::MalformedHeader { index } => { - write!( - f, - "do_view_change suffix entry {index} is not a prepare header" - ) - } - Self::OpOutOfOrder { - index, - expected, - found, - } => write!( - f, - "do_view_change suffix entry {index} carries op {found}, expected {expected}" - ), - Self::BitsetBeyondSuffix { count } => write!( - f, - "do_view_change bitset addresses an entry past the {count}-entry suffix" - ), - Self::ChecksumMismatch { index } => write!( - f, - "do_view_change suffix entry {index} does not match its own identity checksum" - ), - Self::TimestampNotDecreasing { index } => write!( - f, - "do_view_change suffix entry {index} does not predate the entry above it" - ), - Self::ChainBreak { index } => write!( - f, - "do_view_change suffix entry {index} does not chain to the entry above it" - ), - } - } +/// Reset the DVC quorum array. +pub const fn dvc_reset(array: &mut DvcQuorumArray) { + *array = dvc_quorum_array_empty(); } -impl std::error::Error for DvcSuffixError {} - -/// Read a suffix out of a `DoViewChange` body. -/// -/// `head_op` is the sender's `header.op`; entries run consecutively down from it, -/// blanks included, so slot `i` is unambiguously op `head_op - i`. An empty body -/// yields an empty suffix, which is what a sender with nothing to describe sends. -/// -/// `body` must carry [`PrepareHeader`]'s alignment: the cast below is checked, so an -/// unaligned body reports every entry as [`DvcSuffixError::MalformedHeader`] instead -/// of what is actually wrong. Real frames clear this because the body starts a whole -/// number of 256-byte headers into an aligned buffer; the debug assert catches a -/// hand-built one. -/// -/// # Errors -/// [`DvcSuffixError`] when the body is not a consecutive run of valid prepare -/// headers descending from `head_op`, or a bitset addresses a missing entry. -pub fn dvc_suffix_decode( - body: &[u8], - head_op: u64, - nack_bitset: u128, - present_bitset: u128, -) -> Result { - debug_assert!( - body.is_empty() - || body - .as_ptr() - .addr() - .is_multiple_of(align_of::()), - "suffix body must be aligned for PrepareHeader" - ); - let header_size = size_of::(); - if !body.len().is_multiple_of(header_size) { - return Err(DvcSuffixError::NotHeaderMultiple { - body_len: body.len(), - }); - } - let count = body.len() / header_size; - if count > DVC_HEADERS_MAX { - return Err(DvcSuffixError::TooManyEntries { count }); - } - if count < DVC_HEADERS_MAX { - let beyond = !((1u128 << count) - 1); - if nack_bitset & beyond != 0 || present_bitset & beyond != 0 { - return Err(DvcSuffixError::BitsetBeyondSuffix { count }); - } - } - - let mut headers = Vec::with_capacity(count); - // The entry above the current one, skipping blanks: high-to-low, so the child. - let mut child: Option = None; - for index in 0..count { - let chunk = &body[index * header_size..(index + 1) * header_size]; - let header = bytemuck::checked::try_from_bytes::(chunk) - .map_err(|_| DvcSuffixError::MalformedHeader { index })?; - // The cast proves the enums, not the reserved regions. A dirty reserved byte - // with `checksum == 0` reads as Valid here (blanks are classified by exact - // struct equality) AND skips the identity recompute below, so it conflicts - // with every honest header at that op and no canonical one can be picked. - header - .validate() - .map_err(|_| DvcSuffixError::MalformedHeader { index })?; - let expected = head_op - .checked_sub(index as u64) - .ok_or(DvcSuffixError::OpOutOfOrder { - index, - expected: 0, - found: header.op, - })?; - if header.op != expected { - return Err(DvcSuffixError::OpOutOfOrder { - index, - expected, - found: header.op, - }); - } - - if matches!(dvc_header_kind(header), DvcHeaderKind::Valid) { - // Recompute rather than trust the field. Otherwise one bit flipped in - // transit becomes a canonical header no replica holds, honest senders - // read as disagreeing, and a corrupted frame turns into a nack quorum - // against a committed op. The on-disk sentinel is skipped: a suffix is read - // out of the journal, which may hold entries a pre-seal build wrote, and - // partition-plane prepares are unsealed by construction. - if header.checksum != CHECKSUM_UNSEALED && header.identity_checksum() != header.checksum - { - return Err(DvcSuffixError::ChecksumMismatch { index }); - } - if let Some(child) = child { - // Timestamps never run forwards down the log, and consecutive entries - // hash-chain. A frame breaking either describes a log that cannot exist. - // - // `view` is NOT checked. Monotone along one log, but a suffix is two: - // `build_dvc_suffix` stitches the journal over the adopted view's - // headers, and `restamp_prepare_view` rewrites `view` in place, so a - // held op keeps whichever view delivered it. A hole below one is enough - // to make an honest sender's suffix look regressed, and rejecting drops - // its vote forever (the retransmit is byte-identical). Nothing reads a - // suffix entry's `view`; the merge ranks by the message's `log_view`. - if header.timestamp >= child.timestamp { - return Err(DvcSuffixError::TimestampNotDecreasing { index }); - } - if header.op + 1 == child.op - && header.checksum != CHECKSUM_UNSEALED - && child.parent != header.checksum - { - return Err(DvcSuffixError::ChainBreak { index }); - } - } - child = Some(*header); - } - headers.push(*header); - } - - Ok(DvcSuffix::new(headers, nack_bitset, present_bitset)) +/// Iterator over all stored DVCs. +// TODO: add #[must_use] -- pure iterator query, callers should not ignore. +pub fn dvc_iter(array: &DvcQuorumArray) -> impl Iterator { + array.iter().filter_map(|m| m.as_ref()) } diff --git a/core/harness_derive/src/attrs.rs b/core/harness_derive/src/attrs.rs index ee4ff391e2..7a895b2bb1 100644 --- a/core/harness_derive/src/attrs.rs +++ b/core/harness_derive/src/attrs.rs @@ -636,8 +636,11 @@ mod tests { #[test] fn parse_server_executable_path() { - let attrs: IggyTestAttrs = syn::parse_quote!(server(executable_path = "iggy-server")); - assert_eq!(attrs.server.executable_path.as_deref(), Some("iggy-server")); + let attrs: IggyTestAttrs = syn::parse_quote!(server(executable_path = "iggy-server-ng")); + assert_eq!( + attrs.server.executable_path.as_deref(), + Some("iggy-server-ng") + ); } #[test] diff --git a/core/harness_derive/src/codegen.rs b/core/harness_derive/src/codegen.rs index 76e2b664ad..95e8e987b7 100644 --- a/core/harness_derive/src/codegen.rs +++ b/core/harness_derive/src/codegen.rs @@ -125,8 +125,8 @@ fn generate_variants(attrs: &IggyTestAttrs) -> Vec { } /// No transport is gated out of the VSR test matrix. Retained as the single -/// seam where a transport could be excluded from the matrix if the server -/// ever drops support for one again. +/// seam where a transport could be excluded from `--features vsr` if one is +/// ever unsupported by the next-gen server again. fn vsr_transport_cfg(_transport: Transport) -> TokenStream { quote!() } diff --git a/core/integration/Cargo.toml b/core/integration/Cargo.toml index bb7bdc51b2..d4d0bdbd53 100644 --- a/core/integration/Cargo.toml +++ b/core/integration/Cargo.toml @@ -34,6 +34,7 @@ ignored = ["cfg_aliases", "rust-s3"] ci-qemu = [] default = ["login-session"] login-session = ["dep:zbus-secret-service-keyring-store"] +vsr = ["dep:consensus", "dep:journal", "iggy/vsr"] [dependencies] assert_cmd = { workspace = true } @@ -45,9 +46,9 @@ bytes = { workspace = true } compio = { workspace = true } configs = { workspace = true } configs_derive = { workspace = true } -# Decodes a metadata replica's durable `VsrState` off disk in the superblock -# recovery test. -consensus = { workspace = true } +# vsr-only: decode a metadata replica's durable `VsrState` off disk in the +# superblock recovery test. +consensus = { workspace = true, optional = true } ctor = { workspace = true } deltalake = { workspace = true } dtor = { workspace = true } @@ -63,9 +64,9 @@ iggy_common = { workspace = true } # `build_label` function — keeping the test and production label format in lock-step. iggy_connector_doris_sink = { path = "../connectors/sinks/doris_sink" } iggy_connector_sdk = { workspace = true, features = ["api"] } -# Locates and decodes the on-disk superblock slot files in the recovery test -# (`SLOT_FILE_NAMES`, `decode_slots`). -journal = { workspace = true } +# vsr-only: locate and decode the on-disk superblock slot files in the recovery +# test (`SLOT_FILE_NAMES`, `decode_slots`). +journal = { workspace = true, optional = true } jsonwebtoken = { workspace = true } keyring-core = { workspace = true } lazy_static = { workspace = true } @@ -88,6 +89,7 @@ secrecy = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } serial_test = { workspace = true } +server = { workspace = true } sqlx = { workspace = true } sysinfo = { workspace = true } tempfile = { workspace = true } diff --git a/core/integration/src/bench_utils.rs b/core/integration/src/bench_utils.rs index dcfbde1c84..de2de07d1b 100644 --- a/core/integration/src/bench_utils.rs +++ b/core/integration/src/bench_utils.rs @@ -34,9 +34,10 @@ const DEFAULT_NUMBER_OF_STREAMS: u64 = 8; // Generous for a few MB of traffic even in debug builds, and deliberately // UNDER nextest's harness timeout (`.config/nextest.toml` sigkills at // 60s x 5): a longer wait here would never fire, taking the capture dump and -// the stale-binary hint below with it. Exists because a stale prebuilt -// iggy-bench speaking an outdated protocol hangs both sides silently -// instead of erroring. +// the `--features vsr` hint below with it. Exists because a protocol mismatch +// (an SDK framing the server does not speak, e.g. a default-features +// iggy-bench against a vsr cluster) hangs both sides silently instead of +// erroring. const BENCH_WAIT_TIMEOUT: Duration = Duration::from_secs(240); pub fn run_bench_and_wait_for_finish( @@ -155,9 +156,9 @@ pub fn run_bench_and_wait_for_finish( assert!( !timed_out, - "iggy-bench did not finish within {BENCH_WAIT_TIMEOUT:?}; the harness \ - spawns the prebuilt binary, so make sure iggy-bench was rebuilt \ - alongside the server (a stale binary hangs instead of erroring)" + "iggy-bench did not finish within {BENCH_WAIT_TIMEOUT:?}; if the server \ + runs in vsr mode, make sure iggy-bench was built with --features vsr \ + (the SDK framing is chosen at compile time)" ); assert!(status.is_some_and(|status| status.success())); } diff --git a/core/integration/src/harness/config/resolve.rs b/core/integration/src/harness/config/resolve.rs index e5f36f60a4..99d89f4d30 100644 --- a/core/integration/src/harness/config/resolve.rs +++ b/core/integration/src/harness/config/resolve.rs @@ -17,15 +17,15 @@ //! Runtime validation and resolution of config paths to environment variables. -use configs::server::ServerConfig; -use configs::{ConfigEnvMappings, EnvVarMapping}; +use configs::ConfigEnvMappings; +use server::configs::server::ServerConfig; use std::collections::HashMap; /// Resolve config paths to environment variable names. /// /// Takes a map of dot-notation config paths (e.g., "segment.size") and their values, -/// validates them against the `ServerConfig` mappings, and returns the -/// corresponding environment variable names with values. +/// validates them against the `ServerConfig` mappings, and returns the corresponding +/// environment variable names with values. /// /// # Implicit defaults /// @@ -51,8 +51,13 @@ pub fn resolve_config_paths( path.as_str() }; - let mapping = find_mapping(resolved_path) - .or_else(|| find_mapping(&format!("system.{}", resolved_path))); + let mapping = ServerConfig::find_by_config_path(resolved_path) + .or_else(|| ServerConfig::find_by_config_path(&format!("system.{}", resolved_path))) + // Fields that exist only in the next-gen server's config (e.g. + // `metadata.*`, `cluster.*`, `message_bus.*`). Env names share + // the `IGGY_` prefix, so the resolved variable reaches whichever + // binary the harness spawns; the legacy server ignores unknowns. + .or_else(|| configs::server_ng::ServerNgConfig::find_by_config_path(resolved_path)); match mapping { Some(m) => { @@ -91,18 +96,24 @@ pub fn resolve_config_paths( } // Auto-enable override_defaults when socket settings are customized - if needs_tcp_socket_override && let Some(m) = find_mapping("tcp.socket.override_defaults") { + if needs_tcp_socket_override + && let Some(m) = ServerConfig::find_by_config_path("tcp.socket.override_defaults") + { env_vars .entry(m.env_name.to_string()) .or_insert_with(|| "true".to_string()); } - if needs_quic_socket_override && let Some(m) = find_mapping("quic.socket.override_defaults") { + if needs_quic_socket_override + && let Some(m) = ServerConfig::find_by_config_path("quic.socket.override_defaults") + { env_vars .entry(m.env_name.to_string()) .or_insert_with(|| "true".to_string()); } // Auto-enable encryption when key is set - if needs_encryption_enabled && let Some(m) = find_mapping("system.encryption.enabled") { + if needs_encryption_enabled + && let Some(m) = ServerConfig::find_by_config_path("system.encryption.enabled") + { env_vars .entry(m.env_name.to_string()) .or_insert_with(|| "true".to_string()); @@ -111,10 +122,6 @@ pub fn resolve_config_paths( Ok(env_vars) } -fn find_mapping(path: &str) -> Option<&'static EnvVarMapping> { - ServerConfig::find_by_config_path(path) -} - fn levenshtein(a: &str, b: &str) -> usize { let a_len = a.len(); let b_len = b.len(); @@ -177,7 +184,7 @@ fn find_similar_paths(unknown: &str) -> Vec { }) .collect(); - candidates.sort_by_key(|(path, score)| (*score, *path)); + candidates.sort_by_key(|(_, score)| *score); candidates .into_iter() diff --git a/core/integration/src/harness/config/server.rs b/core/integration/src/harness/config/server.rs index 07c7f8e940..14c4433c3c 100644 --- a/core/integration/src/harness/config/server.rs +++ b/core/integration/src/harness/config/server.rs @@ -54,12 +54,12 @@ mod tests { fn test_server_config_builder() { let config = TestServerConfig::builder() .quic_enabled(false) - .executable_path("iggy-server") + .executable_path("iggy-server-ng") .extra_envs(HashMap::from([("FOO".to_string(), "BAR".to_string())])) .build(); assert!(!config.quic_enabled); - assert_eq!(config.executable_path.as_deref(), Some("iggy-server")); + assert_eq!(config.executable_path.as_deref(), Some("iggy-server-ng")); assert_eq!(config.extra_envs.get("FOO"), Some(&"BAR".to_string())); } diff --git a/core/integration/src/harness/handle/server.rs b/core/integration/src/harness/handle/server.rs index 9e23d077ef..34c5498072 100644 --- a/core/integration/src/harness/handle/server.rs +++ b/core/integration/src/harness/handle/server.rs @@ -92,7 +92,15 @@ impl std::fmt::Debug for ServerHandle { impl ServerHandle { fn default_server_binary() -> &'static str { - "iggy-server" + #[cfg(feature = "vsr")] + { + "iggy-server-ng" + } + + #[cfg(not(feature = "vsr"))] + { + "iggy-server" + } } fn launched_binary(&self) -> String { @@ -829,9 +837,13 @@ impl TestBinary for ServerHandle { // trusts (rcgen self-signed certs share the same subject DN), which // rustls rejects as `BadSignature`. Generate only when absent so all // nodes and clients share one keypair; this also keeps the - // certificate stable across a restart. + // certificate stable across a restart. The legacy single-node + // harness keeps its regenerate-per-start behavior. + #[cfg(feature = "vsr")] let should_generate = !(cert_dir.join("test_cert.pem").exists() && cert_dir.join("test_key.pem").exists()); + #[cfg(not(feature = "vsr"))] + let should_generate = true; if should_generate { generate_test_certificates(cert_dir.to_str().unwrap()).map_err(|e| { TestBinaryError::InvalidState { @@ -883,6 +895,13 @@ impl TestBinary for ServerHandle { } command.envs(&self.envs); + // Legacy clustering elects node 0 externally and requires explicit followers. + // VSR/server-ng elects its own primary and should see symmetric node startup. + #[cfg(not(feature = "vsr"))] + if self.server_id > 0 { + command.arg("--follower"); + } + // `--replica-id` is the single identity input expected by the // server when cluster mode is enabled; all other cluster config is // byte-identical across nodes. Pass the harness's `server_id` diff --git a/core/integration/src/harness/orchestrator/builder.rs b/core/integration/src/harness/orchestrator/builder.rs index 4eb8b3d78f..c73cd273fa 100644 --- a/core/integration/src/harness/orchestrator/builder.rs +++ b/core/integration/src/harness/orchestrator/builder.rs @@ -297,7 +297,7 @@ fn build_servers( fn default_cluster_node_count() -> usize { // Suite-wide override: run every test that does not pin `cluster_nodes` // against an N-node cluster (e.g. `IGGY_TEST_CLUSTER_NODES=1` probes the - // whole vsr suite on a single the server node). Explicit attrs win. + // whole vsr suite on a single server-ng node). Explicit attrs win. if let Some(count) = std::env::var("IGGY_TEST_CLUSTER_NODES") .ok() .and_then(|value| value.parse::().ok()) @@ -306,7 +306,15 @@ fn default_cluster_node_count() -> usize { return count; } - 3 + #[cfg(feature = "vsr")] + { + 3 + } + + #[cfg(not(feature = "vsr"))] + { + 1 + } } fn build_cluster_envs( @@ -324,6 +332,7 @@ fn build_cluster_envs( envs.insert("IGGY_CLUSTER_ENABLED".to_string(), "true".to_string()); envs.insert("IGGY_CLUSTER_NAME".to_string(), cluster_name.to_string()); + #[cfg(feature = "vsr")] envs.insert( "IGGY_MESSAGE_BUS_RECONNECT_PERIOD".to_string(), "100ms".to_string(), diff --git a/core/integration/src/harness/orchestrator/harness.rs b/core/integration/src/harness/orchestrator/harness.rs index 03e8257003..4a661578a9 100644 --- a/core/integration/src/harness/orchestrator/harness.rs +++ b/core/integration/src/harness/orchestrator/harness.rs @@ -26,12 +26,16 @@ use crate::harness::handle::{ use crate::harness::traits::{Restartable, TestBinary}; use futures::executor::block_on; use iggy::prelude::{ClientWrapper, IggyClient}; +#[cfg(feature = "vsr")] use iggy_common::Client; use iggy_common::TransportProtocol; use std::path::Path; use std::sync::Arc; +#[cfg(feature = "vsr")] use std::time::{Duration, Instant}; +#[cfg(feature = "vsr")] use tokio::time::{sleep, timeout}; +#[cfg(feature = "vsr")] use tracing::warn; use wiremock::matchers::{method, path}; use wiremock::{Mock, MockServer, ResponseTemplate}; @@ -150,13 +154,20 @@ impl TestHarness { self.jwks_server = Some(mock_server); } - // Cluster startup can hit a transient replica-handshake blip that + // Legacy single-server harness: plain start, no cluster readiness. + #[cfg(not(feature = "vsr"))] + for server in &mut self.servers { + server.start()?; + } + + // Cluster startup (vsr) can hit a transient replica-handshake blip that // leaves the mesh incomplete (a peer link drops mid-handshake, so a // node never reaches "all peers connected"). Rather than fail the whole // test on a startup blip, retry spawn + mesh-readiness a few times, // tearing down and respawning between attempts. `ServerHandle::start` // truncates the captured stdout (`File::create`), so the readiness // log-grep never matches a stale marker from a prior attempt. + #[cfg(feature = "vsr")] { const CLUSTER_STARTUP_ATTEMPTS: usize = 3; for attempt in 1..=CLUSTER_STARTUP_ATTEMPTS { @@ -208,6 +219,7 @@ impl TestHarness { Ok(()) } + #[cfg(feature = "vsr")] async fn wait_for_cluster_ready(&self) -> Result<(), TestBinaryError> { { if self.servers.len() <= 1 { diff --git a/core/integration/tests/cli/message/test_message_flush_command.rs b/core/integration/tests/cli/message/test_message_flush_command.rs index 6b1dfde463..9658943dbf 100644 --- a/core/integration/tests/cli/message/test_message_flush_command.rs +++ b/core/integration/tests/cli/message/test_message_flush_command.rs @@ -29,7 +29,10 @@ use iggy::prelude::Client; use iggy::prelude::Identifier; use iggy::prelude::IggyExpiry; use iggy::prelude::MaxTopicSize; +#[cfg(feature = "vsr")] use predicates::str::contains; +#[cfg(not(feature = "vsr"))] +use predicates::str::diff; use serial_test::parallel; use std::str::FromStr; @@ -147,12 +150,21 @@ impl IggyCmdTestCase for TestMessageFetchCmd { } ); - // The server has no on-demand flush primitive: FLUSH_UNSAVED_BUFFER + // server-ng has no on-demand flush primitive: FLUSH_UNSAVED_BUFFER // surfaces a typed FeatureUnavailable (see flush_vsr.rs), so the CLI // reports a flush problem instead of success. + #[cfg(feature = "vsr")] command_state.failure().stderr(contains(format!( "Problem flushing messages {identification_part}" ))); + + #[cfg(not(feature = "vsr"))] + { + let message = format!( + "Executing flush messages {identification_part}\nFlushed messages {identification_part}\n" + ); + command_state.success().stdout(diff(message)); + } } async fn verify_server_state(&self, client: &dyn Client) { diff --git a/core/integration/tests/cli/stream/test_stream_purge_command.rs b/core/integration/tests/cli/stream/test_stream_purge_command.rs index e0e7bace5c..0a472120aa 100644 --- a/core/integration/tests/cli/stream/test_stream_purge_command.rs +++ b/core/integration/tests/cli/stream/test_stream_purge_command.rs @@ -115,7 +115,7 @@ impl IggyCmdTestCase for TestStreamPurgeCmd { } async fn verify_server_state(&self, client: &dyn Client) { - // The server purge is eventually consistent: the partition reset (and + // server-ng purge is eventually consistent: the partition reset (and // its stats zeroing) runs in the reconciler after the metadata commit // the purge command awaited. Legacy is synchronous and satisfies this // on the first poll. diff --git a/core/integration/tests/cli/system/test_cli_session_scenario.rs b/core/integration/tests/cli/system/test_cli_session_scenario.rs index 8b7d245af5..407f40238e 100644 --- a/core/integration/tests/cli/system/test_cli_session_scenario.rs +++ b/core/integration/tests/cli/system/test_cli_session_scenario.rs @@ -91,12 +91,12 @@ pub async fn should_be_successful() { iggy_cmd_test .execute_test(TestMeCmd::new( TransportProtocol::Tcp, - Scenario::FailureDueToSessionTimeout, + Scenario::FailureDueToSessionTimeout(server_address), )) .await; // After the session timed out, logging in again with username and password // must recover: the CLI drops the dead session token and recreates it, - // rather than wedging on the expired credential (regression for the server, + // rather than wedging on the expired credential (regression for server-ng, // where the terminal auth failure is opaque and self-heal never happened). iggy_cmd_test .execute_test(TestLoginCmd::new( diff --git a/core/integration/tests/cli/system/test_me_command.rs b/core/integration/tests/cli/system/test_me_command.rs index cb67e457e6..a0bc269fc9 100644 --- a/core/integration/tests/cli/system/test_me_command.rs +++ b/core/integration/tests/cli/system/test_me_command.rs @@ -30,7 +30,7 @@ pub(super) enum Scenario { SuccessWithCredentials, SuccessWithoutCredentials, FailureWithoutCredentials, - FailureDueToSessionTimeout, + FailureDueToSessionTimeout(String), } // Helper trait to add command-specific methods to TransportProtocol @@ -78,7 +78,7 @@ impl IggyCmdTestCase for TestMeCmd { match &self.scenario { Scenario::SuccessWithCredentials => command.with_env_credentials(), Scenario::FailureWithoutCredentials => command.disable_backtrace(), - Scenario::FailureDueToSessionTimeout => command.disable_backtrace(), + Scenario::FailureDueToSessionTimeout(_) => command.disable_backtrace(), _ => command, } } @@ -99,12 +99,19 @@ impl IggyCmdTestCase for TestMeCmd { .failure() .stderr(diff("Error: CommandError(Iggy command line tool error\n\nCaused by:\n Missing iggy server credentials)\n")); } - Scenario::FailureDueToSessionTimeout => { - // An expired or invalid stored session surfaces as a generic - // login-with-token failure. - command_state.failure().stderr(diff( - "Error: CommandError(Problem with server login with token)\n", - )); + Scenario::FailureDueToSessionTimeout(server_address) => { + #[cfg(not(feature = "vsr"))] + command_state.failure().stderr(diff(format!("Error: CommandError(Login session expired for Iggy server: {server_address}, please login again or use other authentication method)\n"))); + // server-ng maps an expired/invalid stored session to a generic + // login-with-token failure rather than the legacy "session + // expired" message. + #[cfg(feature = "vsr")] + { + let _ = server_address; + command_state.failure().stderr(diff( + "Error: CommandError(Problem with server login with token)\n", + )); + } } } } diff --git a/core/integration/tests/cli/topic/test_topic_purge_command.rs b/core/integration/tests/cli/topic/test_topic_purge_command.rs index 4d1323c6f6..ab36525ebd 100644 --- a/core/integration/tests/cli/topic/test_topic_purge_command.rs +++ b/core/integration/tests/cli/topic/test_topic_purge_command.rs @@ -141,7 +141,7 @@ impl IggyCmdTestCase for TestTopicPurgeCmd { } async fn verify_server_state(&self, client: &dyn Client) { - // The server purge is eventually consistent: the partition reset (and + // server-ng purge is eventually consistent: the partition reset (and // its stats zeroing) runs in the reconciler after the metadata commit // the purge command awaited. Legacy is synchronous and satisfies this // on the first poll. diff --git a/core/integration/tests/cluster/client_table_restart.rs b/core/integration/tests/cluster/client_table_restart.rs index d2ea0ea59a..4e572eb5c4 100644 --- a/core/integration/tests/cluster/client_table_restart.rs +++ b/core/integration/tests/cluster/client_table_restart.rs @@ -71,6 +71,8 @@ //! work later settles on an explicit resume handshake, adjust `resume_request` //! to speak it -- but it must stay credential-bearing. +#![cfg(feature = "vsr")] + use bytes::Bytes; use iggy::prelude::*; use iggy_binary_protocol::codec::{WireDecode, WireEncode}; @@ -78,6 +80,7 @@ use iggy_binary_protocol::consensus::{ Command2, Operation, ReplyHeader, RequestHeader, read_size_field, result_code, result_section_len, }; +use iggy_binary_protocol::namespace::METADATA_CONSENSUS_NAMESPACE; use iggy_binary_protocol::requests::streams::CreateStreamRequest; use iggy_binary_protocol::requests::users::LoginRegisterRequest; use iggy_binary_protocol::responses::users::LoginRegisterResponse; @@ -275,6 +278,10 @@ fn request_header( client: CLIENT_ID, session, request, + namespace: match operation { + Operation::Register => METADATA_CONSENSUS_NAMESPACE, + _ => 0, + }, ..Default::default() } } diff --git a/core/integration/tests/cluster/metadata_checkpoint_restart.rs b/core/integration/tests/cluster/metadata_checkpoint_restart.rs index 499b66bae9..75ae60f6eb 100644 --- a/core/integration/tests/cluster/metadata_checkpoint_restart.rs +++ b/core/integration/tests/cluster/metadata_checkpoint_restart.rs @@ -45,6 +45,8 @@ //! the `forced checkpoint completed` markers below pin it rather than trust //! it. +#![cfg(feature = "vsr")] + use super::client_table_restart::{ commit_request, create_stream_payload, register, resume_request, tcp_addr, tcp_addrs, }; @@ -117,10 +119,8 @@ async fn await_checkpoint_on_all_nodes(harness: &TestHarness, generation: usize) // a checkpoint, so the transfer descriptor's `commit_op == snapshot_seq` and // the post-install tail repair has nothing to fetch (`commit_min == // commit_max` skips it). The below-floor retry then proves the reply ring -// rode the transferred table: request 191's reply was minted at op 192, and -// replay starts at `snapshot_seq + 1`, so no node re-executes it. The -// checkpoint drain keeps op 192's entry as the commit-point header a -// `DoViewChange` needs, but never replays it. +// rode the transferred table: request 191's reply was minted at op 192, which +// every node drained out of its WAL at that same checkpoint. #[iggy_harness(cluster_nodes = 3, server(metadata.journal_slots = "256"))] async fn given_drained_journal_when_node_restarts_should_install_snapshot_only( harness: &mut TestHarness, @@ -327,7 +327,7 @@ async fn wait_for_stream(harness: &TestHarness, stream: &str) -> IggyClient { } // Metadata checkpoint-fold recovery across a solo restart, over -// `iggy-server`'s production snapshot and WAL path. +// `iggy-server-ng`'s production snapshot and WAL path. // // Between checkpoints a replica recovers its metadata by replaying the WAL. Once the // WAL fills, the `SnapshotCoordinator` checkpoints: it persists `snapshot.bin`, pairs diff --git a/core/integration/tests/cluster/metadata_state_transfer.rs b/core/integration/tests/cluster/metadata_state_transfer.rs index 79d78fd9c7..2759464125 100644 --- a/core/integration/tests/cluster/metadata_state_transfer.rs +++ b/core/integration/tests/cluster/metadata_state_transfer.rs @@ -32,6 +32,8 @@ //! functional assert (post-restart continuation commits cluster-wide) rides //! on top. +#![cfg(feature = "vsr")] + use super::client_table_restart::{ commit_request, create_stream_payload, register, resume_request, tcp_addr, tcp_addrs, }; diff --git a/core/integration/tests/cluster/multi_shard_partition_convergence.rs b/core/integration/tests/cluster/multi_shard_partition_convergence.rs index 5379074b94..a26fbd293f 100644 --- a/core/integration/tests/cluster/multi_shard_partition_convergence.rs +++ b/core/integration/tests/cluster/multi_shard_partition_convergence.rs @@ -48,6 +48,8 @@ //! owner, a park queue that never drains, or a fence that denies forever -- //! since every one of those surfaces as a failed send or a short poll. +#![cfg(feature = "vsr")] + use iggy::prelude::*; use integration::harness::TestHarness; use integration::iggy_harness; diff --git a/core/integration/tests/cluster/partition_state_transfer.rs b/core/integration/tests/cluster/partition_state_transfer.rs index fbc5c93919..13413d8866 100644 --- a/core/integration/tests/cluster/partition_state_transfer.rs +++ b/core/integration/tests/cluster/partition_state_transfer.rs @@ -27,6 +27,8 @@ //! `RangeEvicted`, the repaired window cannot connect to recovered state, //! and `complete_repair` returns the `FloorRefused` conversion trigger. +#![cfg(feature = "vsr")] + use std::path::{Path, PathBuf}; use std::str::FromStr; use std::time::{Duration, Instant}; @@ -38,7 +40,7 @@ use tokio::time::sleep; const STREAM_NAME: &str = "partition-transfer-stream"; const TOPIC_NAME: &str = "partition-transfer-topic"; -/// Partition ids are 0-based (CreateTopic assigns them from 0). +/// server-ng partition ids are 0-based (CreateTopic assigns them from 0). const PARTITION_ID: u32 = 0; /// Enough batches to push the evicted ring (capacity 64) well past the /// window a rejoiner could repair from op 1. @@ -125,8 +127,7 @@ async fn given_evicted_ring_when_fresh_node_joins_late_should_state_transfer_par await_marker(harness, 2, INSTALL_MARKER).await; // Disk proof on the rejoined node: transferred segment bytes and a - // persisted consumer-offset file (leading LE u64 offset, trailing - // checksum; see `partitions::offset_storage::encode_offset_record`). + // persisted consumer-offset file (a single LE u64). let data_path = harness.node(2).data_path(); // Each transferred batch is at least its 256-byte header; anything below // this floor is a truncated install, not the seeded 200 batches. @@ -146,11 +147,8 @@ async fn given_evicted_ring_when_fresh_node_joins_late_should_state_transfer_par let offsets_file = find_consumer_offset_file(&data_path) .expect("transferred consumer offset file exists on node 2"); let bytes = std::fs::read(&offsets_file).expect("read transferred consumer offset"); - let offset_bytes = bytes - .first_chunk::<8>() - .expect("offset file starts with a u64 offset"); assert_eq!( - u64::from_le_bytes(*offset_bytes), + u64::from_le_bytes(bytes.as_slice().try_into().expect("offset file is one u64")), STORED_CONSUMER_OFFSET, "the stored consumer offset must survive the transfer" ); @@ -612,7 +610,7 @@ fn find_consumer_offset_file(root: &Path) -> Option { path.parent() .and_then(Path::file_name) .is_some_and(|name| name == "consumers") - && std::fs::metadata(path).is_ok_and(|metadata| metadata.len() >= 8) + && std::fs::metadata(path).is_ok_and(|metadata| metadata.len() == 8) }) } diff --git a/core/integration/tests/config_provider/mod.rs b/core/integration/tests/config_provider/mod.rs index 32d0d9aec6..b2d2c6d366 100644 --- a/core/integration/tests/config_provider/mod.rs +++ b/core/integration/tests/config_provider/mod.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use configs::server::ServerConfig; use configs::{ConfigEnvMappings, ConfigProvider, TypedEnvProvider}; use configs_derive::ConfigEnv; use figment::providers::{Format, Toml}; @@ -23,6 +22,7 @@ use figment::value::Dict; use figment::{Figment, Provider}; use serde::{Deserialize, Serialize}; use serial_test::serial; +use server::configs::server::ServerConfig; use std::env; use std::path::PathBuf; diff --git a/core/integration/tests/data_integrity/mod.rs b/core/integration/tests/data_integrity/mod.rs index 2418ef39b5..5a3815d2d0 100644 --- a/core/integration/tests/data_integrity/mod.rs +++ b/core/integration/tests/data_integrity/mod.rs @@ -16,26 +16,29 @@ // under the License. // Partially vsr-gated inside the module: the remaining gates cover -// `flush_unsaved_buffer`, which the server answers `FeatureUnavailable` and +// `flush_unsaved_buffer`, which server-ng answers `FeatureUnavailable` and // which the eager-flush server envs replace under vsr. The bench-fill test // itself runs under vsr since PARTITION-plane state transfer landed, but the // harness spawns `iggy-bench` off disk with no cargo build-graph edge, so the -// binary must be freshly built or a stale build hangs on login. +// binary must have been built `--features vsr` or its login hangs on the +// framing mismatch. mod verify_after_server_restart; mod verify_user_login_after_restart; // Not restart-based: it creates a user + PAT, stops the server, and greps the -// data dir for plaintext. No replica catch-up needed, and the server hashes the +// data dir for plaintext. No replica catch-up needed, and server-ng hashes the // password / PAT before either reaches the WAL, so it runs under vsr too. mod verify_no_plaintext_credentials_on_disk; -// The cooperative-rebalance matrix exercises the server's consumer-group -// rebalancing (a VSR capability). Green at 95/95. +// The cooperative-rebalance matrix runs under vsr too: it exercises server-ng's +// consumer-group rebalancing (a VSR feature). Green at 95/95. mod verify_consumer_group_partition_assignment; // Cross-replica on-disk data identity is VSR-only. +#[cfg(feature = "vsr")] mod verify_cluster_replica_data_identical; // Auto-commit offset replication is inherently a multi-node (VSR) property: the // backup only holds the offset if the poll's auto-commit rode consensus. +#[cfg(feature = "vsr")] mod verify_auto_commit_offset_replicates; diff --git a/core/integration/tests/data_integrity/verify_after_server_restart.rs b/core/integration/tests/data_integrity/verify_after_server_restart.rs index 1bed4408e8..71a6164945 100644 --- a/core/integration/tests/data_integrity/verify_after_server_restart.rs +++ b/core/integration/tests/data_integrity/verify_after_server_restart.rs @@ -40,16 +40,18 @@ fn build_server_config(cache_setting: &str) -> TestServerConfig { "IGGY_SYSTEM_SEGMENT_CACHE_INDEXES".to_string(), cache_setting.to_string(), ); - // The server flushes on the journal thresholds (no flush primitive), so + // server-ng flushes on the journal thresholds (no flush primitive), so // force every committed batch straight to disk: the restart asserts // below need everything durable, and the explicit flush calls are // cfg'd out under vsr (`flush_unsaved_buffer` answers // FeatureUnavailable there and is slated for removal). Legacy keeps its // shipped buffered defaults; the flush loops below are its barrier. + #[cfg(feature = "vsr")] extra_envs.insert( "IGGY_SYSTEM_PARTITION_MESSAGES_REQUIRED_TO_SAVE".to_string(), "1".to_string(), ); + #[cfg(feature = "vsr")] extra_envs.insert( "IGGY_SYSTEM_PARTITION_ENFORCE_FSYNC".to_string(), "true".to_string(), @@ -59,23 +61,20 @@ fn build_server_config(cache_setting: &str) -> TestServerConfig { // TODO(numminex) - Move the message generation method from benchmark run to a special method. // -// The durability barrier is the eager-flush envs in `build_server_config` -// (`flush_unsaved_buffer` answers FeatureUnavailable on VSR, so there is no -// explicit flush loop), and `iggy-bench` must be freshly built: the harness -// spawns the prebuilt binary, and a stale one never completes a frame -// against the server, tripping the bench timeout in -// `run_bench_and_wait_for_finish`. +// Under vsr this runs against a 3-node cluster and needs two adaptations: +// the durability barrier is the eager-flush envs in `build_server_config` +// (`flush_unsaved_buffer` answers FeatureUnavailable there, so the explicit +// flush loops are cfg'd out), and `iggy-bench` must be built with +// `--features vsr` because the SDK framing is chosen at compile time. A +// default-features bench binary never completes a frame against server-ng +// and the run trips the bench timeout in `run_bench_and_wait_for_finish`. #[test_matrix( [cache_all(), cache_open_segment(), cache_none()] )] #[tokio::test] #[parallel] async fn should_fill_data_and_verify_after_restart(cache_setting: &'static str) { - // Restart scenarios run single-node: restarting a node in a multi-node - // cluster trips a known partitions-plane view-change stall, tracked - // separately. let mut harness = TestHarness::builder() - .cluster_nodes(1) .server(build_server_config(cache_setting)) .build() .unwrap(); @@ -105,6 +104,16 @@ async fn should_fill_data_and_verify_after_restart(cache_setting: &'static str) let client = harness.tcp_root_client().await.unwrap(); let topic_id = Identifier::numeric(0).unwrap(); + // Durability barrier on the legacy server only; server-ng persists + // eagerly via the config envs and answers FeatureUnavailable here. + #[cfg(not(feature = "vsr"))] + for i in 0..7 { + let stream_id = Identifier::numeric(i).unwrap(); + client + .flush_unsaved_buffer(&stream_id, &topic_id, 0, true) + .await + .unwrap(); + } // Create consumer groups to test persistence let consumer_group_names = ["test-cg-1", "test-cg-2", "test-cg-3"]; @@ -211,6 +220,19 @@ async fn should_fill_data_and_verify_after_restart(cache_setting: &'static str) // Connect and login to server let client = harness.tcp_root_client().await.unwrap(); + // Durability barrier on the legacy server only (see the first loop). + #[cfg(not(feature = "vsr"))] + { + let topic_id = Identifier::numeric(0).unwrap(); + for i in 0..7 { + let stream_id = Identifier::numeric(i).unwrap(); + client + .flush_unsaved_buffer(&stream_id, &topic_id, 0, true) + .await + .unwrap(); + } + } + // Save stats from the second server (should have double the data) let stats = client.get_stats().await.unwrap(); let actual_messages_size_bytes = stats.messages_size_bytes; @@ -303,7 +325,6 @@ async fn should_fill_data_and_verify_after_restart(cache_setting: &'static str) #[parallel] async fn should_handle_resource_deletion_and_restart() { let mut harness = TestHarness::builder() - .cluster_nodes(1) .server(TestServerConfig::default()) .build() .unwrap(); diff --git a/core/integration/tests/data_integrity/verify_auto_commit_offset_replicates.rs b/core/integration/tests/data_integrity/verify_auto_commit_offset_replicates.rs index cb953b1b8c..6f26dd983f 100644 --- a/core/integration/tests/data_integrity/verify_auto_commit_offset_replicates.rs +++ b/core/integration/tests/data_integrity/verify_auto_commit_offset_replicates.rs @@ -150,10 +150,8 @@ async fn run(harness: &TestHarness) { /// The u64 offset persisted under any `offsets/consumers/` file in a node's /// data dir, or `None` when no such file has been written yet. Walks the tree so /// it is robust to the stream/topic/partition id layout; the test drives exactly -/// one consumer, so at most one such file exists. Reads the leading u64 of the -/// record: the file is offset + trailing checksum (see -/// `partitions::offset_storage::encode_offset_record`), and a shorter read -/// (persist truncates before writing) is treated as not-yet-written. +/// one consumer, so at most one such file exists. A zero-length read (persist +/// truncates before writing the 8 bytes) is treated as not-yet-written. fn read_replicated_consumer_offset(data_dir: &Path) -> Option { let mut stack: Vec = vec![data_dir.to_path_buf()]; while let Some(dir) = stack.pop() { @@ -180,9 +178,9 @@ fn read_replicated_consumer_offset(data_dir: &Path) -> Option { .is_some_and(|name| name == "offsets"); if is_consumer_offset && let Ok(bytes) = std::fs::read(&path) - && let Some(offset_bytes) = bytes.first_chunk::<8>() + && let Ok(array) = <[u8; 8]>::try_from(bytes.as_slice()) { - return Some(u64::from_le_bytes(*offset_bytes)); + return Some(u64::from_le_bytes(array)); } } } diff --git a/core/integration/tests/data_integrity/verify_consumer_group_partition_assignment.rs b/core/integration/tests/data_integrity/verify_consumer_group_partition_assignment.rs index 8fac77b964..b92c5bcf4d 100644 --- a/core/integration/tests/data_integrity/verify_consumer_group_partition_assignment.rs +++ b/core/integration/tests/data_integrity/verify_consumer_group_partition_assignment.rs @@ -15,25 +15,6 @@ // specific language governing permissions and limitations // under the License. -//! Consumer-group partition assignment specs. -//! -//! # Why most specs here ask for a 60s server heartbeat -//! -//! `run_heartbeat_verifier` evicts a connection idle past `1.2 x -//! heartbeat.interval`, and a harness client NEVER pings on its own: the SDK's -//! pinger is spawned by `IggyClient::connect`, which the harness builder does -//! not call. A spec that joins a member and then spends its setup elsewhere -//! therefore races its own subject being reaped, and the failure surfaces as a -//! short `members_count` or a `StaleClient` rather than as anything about -//! assignment. At the former 2s interval the 16-consumer spec sat 2.27s into a -//! 2.4s deadline over quic -- under 5% margin. -//! -//! These specs assert ASSIGNMENT, not liveness, so the deadline is pushed out -//! of reach instead of being raced. The two that genuinely drive eviction keep -//! the short interval and are marked as such: they build members with -//! [`create_stale_tcp_client`], whose 1h client-side heartbeat means only the -//! server's verifier can ever remove them. - use iggy::prelude::*; use integration::iggy_harness; use std::collections::HashSet; @@ -79,8 +60,6 @@ async fn create_tcp_client(server_addr: &str) -> IggyClient { #[iggy_harness(server( heartbeat.enabled = true, - // Deliberately short: this spec drives the server's eviction path (see the - // module note), so the verifier must be able to reap a stale member. heartbeat.interval = "2s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true @@ -277,7 +256,7 @@ async fn should_not_duplicate_partition_assignments_after_stale_client_cleanup( #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "60s", + heartbeat.interval = "2s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -355,7 +334,7 @@ async fn should_not_reshuffle_partitions_when_new_member_joins(harness: &TestHar #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "60s", + heartbeat.interval = "2s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -437,7 +416,7 @@ async fn should_skip_revoked_partitions_in_round_robin(harness: &TestHarness) { #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "60s", + heartbeat.interval = "2s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -597,7 +576,7 @@ async fn should_not_lose_messages_with_concurrent_polls_during_partition_add( #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "60s", + heartbeat.interval = "2s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -745,7 +724,7 @@ async fn should_handle_partition_add_then_consumer_disconnect_then_new_join(harn #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "60s", + heartbeat.interval = "2s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -882,7 +861,7 @@ async fn should_handle_partition_delete_while_multiple_consumers_polling(harness #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "60s", + heartbeat.interval = "2s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -1064,7 +1043,7 @@ async fn should_reach_even_distribution_after_multiple_joins(harness: &TestHarne #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "60s", + heartbeat.interval = "2s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -1185,7 +1164,7 @@ async fn should_split_evenly_when_consumer_joins_after_partitions_added(harness: #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "60s", + heartbeat.interval = "2s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -1286,7 +1265,7 @@ async fn should_not_duplicate_messages_when_partitions_added_during_polling(harn #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "60s", + heartbeat.interval = "2s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -1418,7 +1397,7 @@ async fn should_handle_delete_partitions_with_uncommitted_work(harness: &TestHar #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "60s", + heartbeat.interval = "2s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -1556,7 +1535,7 @@ async fn should_handle_rapid_partition_changes_with_active_consumers(harness: &T #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "60s", + heartbeat.interval = "2s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -1630,7 +1609,7 @@ async fn should_rebalance_after_adding_partitions(harness: &TestHarness) { #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "60s", + heartbeat.interval = "2s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -1718,7 +1697,7 @@ async fn should_rebalance_after_deleting_partitions(harness: &TestHarness) { #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "60s", + heartbeat.interval = "2s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -1909,8 +1888,6 @@ async fn should_timeout_revocation(harness: &TestHarness) { #[iggy_harness(server( heartbeat.enabled = true, - // Deliberately short: this spec drives the server's eviction path (see the - // module note), so the verifier must be able to reap a stale member. heartbeat.interval = "2s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true @@ -2332,7 +2309,7 @@ fn assert_balanced_partition_distribution(cg: &ConsumerGroupDetails, expected_to #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "60s", + heartbeat.interval = "2s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -2441,7 +2418,7 @@ async fn should_not_return_same_message_to_two_consumers_during_rebalance(harnes #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "60s", + heartbeat.interval = "2s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -2513,7 +2490,7 @@ async fn should_complete_revocation_on_auto_commit(harness: &TestHarness) { #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "60s", + heartbeat.interval = "2s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -2571,7 +2548,7 @@ async fn should_transfer_never_polled_partitions_immediately(harness: &TestHarne #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "60s", + heartbeat.interval = "2s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -2663,7 +2640,7 @@ async fn should_rebalance_when_member_with_pending_revocation_leaves(harness: &T #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "60s", + heartbeat.interval = "2s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -2821,7 +2798,7 @@ async fn should_not_produce_duplicate_messages_with_sequential_consumer_joins( #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "60s", + heartbeat.interval = "2s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -2930,7 +2907,7 @@ async fn should_wait_for_manual_commit_before_completing_revocation(harness: &Te #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "60s", + heartbeat.interval = "2s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -3023,7 +3000,7 @@ async fn should_redistribute_when_revocation_target_leaves(harness: &TestHarness #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "60s", + heartbeat.interval = "2s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -3123,7 +3100,7 @@ async fn should_distribute_partitions_evenly_with_concurrent_joins(harness: &Tes #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "60s", + heartbeat.interval = "2s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -3229,7 +3206,7 @@ async fn should_not_assign_partition_to_wrong_member_after_slab_reuse(harness: & #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "60s", + heartbeat.interval = "2s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -3360,7 +3337,7 @@ async fn should_not_complete_other_members_revocations_on_leave(harness: &TestHa #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "60s", + heartbeat.interval = "2s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -3453,7 +3430,7 @@ async fn should_distribute_16_partitions_evenly_across_16_consumers(harness: &Te #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "60s", + heartbeat.interval = "2s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -3520,7 +3497,7 @@ async fn should_distribute_excess_evenly_when_multiple_idle_members_join(harness #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "60s", + heartbeat.interval = "2s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -3585,7 +3562,7 @@ async fn should_distribute_remainder_fairly_with_uneven_ratio(harness: &TestHarn #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "60s", + heartbeat.interval = "2s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -3646,7 +3623,7 @@ async fn should_collect_excess_from_multiple_overassigned_members(harness: &Test #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "60s", + heartbeat.interval = "2s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -3701,7 +3678,7 @@ async fn should_not_starve_any_member_in_large_scale_rebalance(harness: &TestHar #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "60s", + heartbeat.interval = "2s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] diff --git a/core/integration/tests/mod.rs b/core/integration/tests/mod.rs index 7bdcd52353..efb0cc8cab 100644 --- a/core/integration/tests/mod.rs +++ b/core/integration/tests/mod.rs @@ -30,18 +30,29 @@ use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::{EnvFilter, fmt}; -// Drives the `iggy` CLI binary against a running server. Single-node and the -// default 3-node cluster both pass. +// Drives the `iggy` CLI binary against a running server, in both legacy and +// vsr/server-ng modes. Under vsr, single-node and the default 3-node cluster +// both pass. A few cases are mode-split where server-ng diverges from legacy by +// design: flush returns FeatureUnavailable, the session-timeout message +// differs, and purge is eventually consistent so server state is polled. mod cli; // Raw-wire spec tests for VSR session continuity across a node restart -// (IGGY-137). +// (IGGY-137); the module is vsr-only by construction (file-level cfg). mod cluster; mod config_provider; mod connectors; +// Runs under vsr; the one gap (`verify_after_server_restart`) is gated inside +// the module: its rejoin window exceeds journal retention (state transfer). mod data_integrity; mod mcp; mod sdk; mod server; +// Unit-tests the legacy `server` crate's state-file machinery directly +// (`server::state::file::FileState` etc.). server-ng replaces that layer with +// the metadata WAL, so this suite is legacy-only by construction, not a gap. +#[cfg(not(feature = "vsr"))] +mod state; +mod storage; lazy_static! { static ref TESTS_FAILED: AtomicBool = AtomicBool::new(false); diff --git a/core/integration/tests/sdk/consumer_group_membership.rs b/core/integration/tests/sdk/consumer_group_membership.rs index ab07bb3f1f..91800cf765 100644 --- a/core/integration/tests/sdk/consumer_group_membership.rs +++ b/core/integration/tests/sdk/consumer_group_membership.rs @@ -148,7 +148,7 @@ async fn given_group_member_holds_no_partitions_when_group_deleted_should_surfac // End-to-end wire pin for the consumer-group join/leave error ladder. The // metadata STM unit tests pin the committed result codes; this pins that -// the server actually emits them over the wire, so a client observes the same +// server-ng actually emits them over the wire, so a client observes the same // codes the legacy server returns. Binary transports only: the HTTP client // has no join/leave (stateless sessions carry no member identity, the SDK // returns FeatureUnavailable client-side), so the ladder cannot run there. diff --git a/core/integration/tests/sdk/hello_world.rs b/core/integration/tests/sdk/hello_world.rs index 10a646a3e9..c7ad212893 100644 --- a/core/integration/tests/sdk/hello_world.rs +++ b/core/integration/tests/sdk/hello_world.rs @@ -24,6 +24,7 @@ async fn hello_world(harness: &TestHarness) { client.ping().await.unwrap(); } +#[cfg(feature = "vsr")] #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic])] async fn hello_world(harness: &TestHarness) { let client = harness.new_client().await.unwrap(); diff --git a/core/integration/tests/sdk/http_refresh.rs b/core/integration/tests/sdk/http_refresh.rs index e33e2a96ba..98236fb385 100644 --- a/core/integration/tests/sdk/http_refresh.rs +++ b/core/integration/tests/sdk/http_refresh.rs @@ -16,7 +16,7 @@ // under the License. //! End-to-end coverage for the SDK HTTP client's `refresh_access_token` -//! against a live the server listener: the reissued token must replace the one +//! against a live server-ng listener: the reissued token must replace the one //! the client holds and keep it authenticated. use iggy::http::http_client::HttpClient; diff --git a/core/integration/tests/sdk/mod.rs b/core/integration/tests/sdk/mod.rs index a59335337d..ade2ec45c7 100644 --- a/core/integration/tests/sdk/mod.rs +++ b/core/integration/tests/sdk/mod.rs @@ -16,10 +16,13 @@ // under the License. mod consumer_group; +#[cfg(feature = "vsr")] mod consumer_group_membership; mod hello_world; +#[cfg(feature = "vsr")] mod http_refresh; mod producer; +#[cfg(feature = "vsr")] mod protocol_version; mod raw; mod send_confirmation; diff --git a/core/integration/tests/sdk/protocol_version.rs b/core/integration/tests/sdk/protocol_version.rs index 91bdb3bddf..b24db05f23 100644 --- a/core/integration/tests/sdk/protocol_version.rs +++ b/core/integration/tests/sdk/protocol_version.rs @@ -22,9 +22,12 @@ //! frame carrying `IncompatibleProtocol` plus the accepted window; a body //! without a decodable prefix with `MalformedLogin` and a zero window. +#![cfg(feature = "vsr")] + use iggy::prelude::*; use iggy_binary_protocol::codec::WireEncode; use iggy_binary_protocol::consensus::{Command2, Operation, RequestHeader}; +use iggy_binary_protocol::namespace::METADATA_CONSENSUS_NAMESPACE; use iggy_binary_protocol::requests::users::LoginRegisterRequest; use iggy_binary_protocol::{ ClientVersionInfo, HEADER_SIZE, IGGY_PROTOCOL_VERSION, IGGY_PROTOCOL_VERSION_MIN, WireName, @@ -85,6 +88,7 @@ async fn assert_login_evicted( client: 0xC0FFEE, session: 0, request: 0, + namespace: METADATA_CONSENSUS_NAMESPACE, ..Default::default() }; diff --git a/core/integration/tests/sdk/raw.rs b/core/integration/tests/sdk/raw.rs index 29ff474ec0..0bf0049470 100644 --- a/core/integration/tests/sdk/raw.rs +++ b/core/integration/tests/sdk/raw.rs @@ -22,6 +22,16 @@ use iggy_binary_protocol::codes::{GET_STATS_CODE, LOGIN_USER_CODE, PING_CODE}; use iggy_binary_protocol::requests::system::{GetStatsRequest, PingRequest}; use integration::iggy_harness; +#[cfg(not(feature = "vsr"))] +#[iggy_harness(test_client_transport = [Tcp, Quic, Http, WebSocket])] +async fn given_authenticated_client_when_sending_raw_request_should_round_trip( + harness: &TestHarness, +) { + let client = harness.root_client().await.unwrap(); + assert_raw_round_trip(&client).await; +} + +#[cfg(feature = "vsr")] #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic])] async fn given_authenticated_client_when_sending_raw_request_should_round_trip( harness: &TestHarness, @@ -82,6 +92,7 @@ async fn assert_raw_round_trip(client: &IggyClient) { // follow-up ping is the point of the case: the server must answer // with a deny frame, not drop the frame and leave the connection // wedged until the read timeout. + #[cfg(feature = "vsr")] { let error = client .send_binary_request(60_000, Bytes::new()) diff --git a/core/integration/tests/sdk/send_confirmation.rs b/core/integration/tests/sdk/send_confirmation.rs index 61d1c0e7e6..616ee700a1 100644 --- a/core/integration/tests/sdk/send_confirmation.rs +++ b/core/integration/tests/sdk/send_confirmation.rs @@ -16,7 +16,7 @@ // under the License. //! Commit confirmations for `SendMessages`: which partition a batch landed in -//! and at which offset. The server answers a committed send with a confirmation +//! and at which offset. server-ng answers a committed send with a confirmation //! payload; the legacy server answers with an empty body, which the SDK reports //! as no confirmations rather than as a decode failure. @@ -28,11 +28,14 @@ const TOPIC_NAME: &str = "confirmation-topic"; const MESSAGES_COUNT: u32 = 10; const PARTITIONS_COUNT: u32 = 3; -// Partition ids are 0-based (CreateTopic assigns them from 0). +// server-ng partition ids are 0-based (CreateTopic assigns them from 0). +#[cfg(feature = "vsr")] const PARTITION_ID: u32 = 0; /// Chunking for the direct producer: `CHUNKS * CHUNK_LENGTH` messages exceed /// one request, so the send is split and every chunk confirms separately. +#[cfg(feature = "vsr")] const CHUNK_LENGTH: u32 = 4; +#[cfg(feature = "vsr")] const CHUNKS: u32 = 3; fn batch(count: u32) -> Vec { @@ -68,6 +71,7 @@ async fn create_stream_and_topic(client: &IggyClient, partitions_count: u32) -> (stream.id, topic.id) } +#[cfg(feature = "vsr")] fn sole_confirmation(response: &SendMessagesResponse) -> &SendMessagesConfirmationResponse { assert_eq!( response.confirmations.len(), @@ -79,6 +83,7 @@ fn sole_confirmation(response: &SendMessagesResponse) -> &SendMessagesConfirmati /// Each transport carries the reply body on its own path, so the full /// confirmation shape is pinned on all three. +#[cfg(feature = "vsr")] #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic])] async fn given_explicit_partition_when_sending_two_batches_should_confirm_advancing_base_offset( harness: &TestHarness, @@ -123,6 +128,7 @@ async fn given_explicit_partition_when_sending_two_batches_should_confirm_advanc client.logout_user().await.unwrap(); } +#[cfg(feature = "vsr")] #[iggy_harness] async fn given_balanced_partitioning_when_sending_should_confirm_a_partition_of_the_topic( harness: &TestHarness, @@ -161,6 +167,7 @@ async fn given_balanced_partitioning_when_sending_should_confirm_a_partition_of_ client.logout_user().await.unwrap(); } +#[cfg(feature = "vsr")] #[iggy_harness] async fn given_messages_key_partitioning_when_sending_should_confirm_a_partition_of_the_topic( harness: &TestHarness, @@ -193,6 +200,7 @@ async fn given_messages_key_partitioning_when_sending_should_confirm_a_partition client.logout_user().await.unwrap(); } +#[cfg(feature = "vsr")] #[iggy_harness] async fn given_direct_producer_when_send_splits_into_chunks_should_confirm_every_chunk( harness: &TestHarness, @@ -234,3 +242,27 @@ async fn given_direct_producer_when_send_splits_into_chunks_should_confirm_every client.logout_user().await.unwrap(); } + +#[cfg(not(feature = "vsr"))] +#[iggy_harness] +async fn given_legacy_server_when_sending_should_report_no_confirmations(harness: &TestHarness) { + let client = harness.root_client().await.unwrap(); + + create_stream_and_topic(&client, PARTITIONS_COUNT).await; + let mut messages = batch(MESSAGES_COUNT); + let response = client + .send_messages( + &Identifier::named(STREAM_NAME).unwrap(), + &Identifier::named(TOPIC_NAME).unwrap(), + &Partitioning::balanced(), + &mut messages, + ) + .await + .expect("send_messages"); + + assert!( + response.confirmations.is_empty(), + "the legacy server reports no offsets; a synthetic entry would be \ + indistinguishable from a genuine commit at offset 0" + ); +} diff --git a/core/integration/tests/server/a2a_jwt/jwt_tests.rs b/core/integration/tests/server/a2a_jwt/jwt_tests.rs index 1b7d915a9f..4d0b2b1943 100644 --- a/core/integration/tests/server/a2a_jwt/jwt_tests.rs +++ b/core/integration/tests/server/a2a_jwt/jwt_tests.rs @@ -25,6 +25,7 @@ use iggy_common::{StreamClient, UserClient}; use integration::iggy_harness; use jsonwebtoken::{Algorithm, EncodingKey, Header, encode}; use serde::{Deserialize, Serialize}; +use server::http::jwt::json_web_token::Audience; const TEST_ISSUER: &str = "https://test-issuer.com"; const TEST_AUDIENCE: &str = "iggy"; @@ -68,16 +69,6 @@ async fn seed_a2a_user( Ok(()) } -/// The `aud` claim as RFC 7519 allows it on the wire: one string, or an array -/// of them. Untagged serialization emits each shape verbatim, which is what the -/// server's own audience parser reads back. -#[derive(Debug, Serialize, Deserialize)] -#[serde(untagged)] -enum Audience { - Single(String), - Multiple(Vec), -} - /// Test claims structure for JWT tokens /// Supports both single string and array audience per RFC 7519 #[derive(Debug, Serialize, Deserialize)] @@ -105,7 +96,7 @@ fn create_valid_jwt(exp_seconds: u64) -> String { let claims = TestClaims { jti: uuid::Uuid::now_v7().to_string(), iss: TEST_ISSUER.to_string(), - aud: Audience::Single(TEST_AUDIENCE.to_string()), + aud: Audience::from(TEST_AUDIENCE), sub: "external-a2a-user-123".to_string(), exp: now + exp_seconds, iat: now, @@ -125,7 +116,7 @@ fn create_valid_jwt_with_array_aud(exp_seconds: u64) -> String { let claims = TestClaims { jti: uuid::Uuid::now_v7().to_string(), iss: TEST_ISSUER.to_string(), - aud: Audience::Multiple(vec![ + aud: Audience::from(vec![ "some-other-service".to_string(), TEST_AUDIENCE.to_string(), "another-service".to_string(), @@ -149,7 +140,7 @@ fn create_expired_jwt() -> String { let claims = TestClaims { jti: uuid::Uuid::now_v7().to_string(), iss: TEST_ISSUER.to_string(), - aud: Audience::Single(TEST_AUDIENCE.to_string()), + aud: Audience::from(TEST_AUDIENCE), sub: "external-a2a-user-123".to_string(), exp: now.saturating_sub(3600), iat: now.saturating_sub(7200), @@ -169,7 +160,7 @@ fn create_unknown_issuer_jwt() -> String { let claims = TestClaims { jti: uuid::Uuid::now_v7().to_string(), iss: "https://unknown-issuer.com".to_string(), - aud: Audience::Single(TEST_AUDIENCE.to_string()), + aud: Audience::from(TEST_AUDIENCE), sub: "external-a2a-user-123".to_string(), exp: now + 3600, iat: now, @@ -193,7 +184,7 @@ fn create_algorithm_confusion_jwt() -> String { let claims = TestClaims { jti: uuid::Uuid::now_v7().to_string(), iss: TEST_ISSUER.to_string(), - aud: Audience::Single(TEST_AUDIENCE.to_string()), + aud: Audience::from(TEST_AUDIENCE), sub: "external-a2a-user-123".to_string(), exp: now + 3600, iat: now, @@ -215,7 +206,7 @@ fn create_jwt_with_kid(kid: &str) -> String { let claims = TestClaims { jti: uuid::Uuid::now_v7().to_string(), iss: TEST_ISSUER.to_string(), - aud: Audience::Single(TEST_AUDIENCE.to_string()), + aud: Audience::from(TEST_AUDIENCE), sub: "external-a2a-user-123".to_string(), exp: now + 3600, iat: now, diff --git a/core/integration/tests/server/cluster_view_durability_vsr.rs b/core/integration/tests/server/cluster_view_durability_vsr.rs index bf59636a90..a2cc801f43 100644 --- a/core/integration/tests/server/cluster_view_durability_vsr.rs +++ b/core/integration/tests/server/cluster_view_durability_vsr.rs @@ -16,7 +16,7 @@ // under the License. //! Metadata-plane view durability across a real view change and process restarts, -//! over `iggy-server`'s production superblock path. +//! over `iggy-server-ng`'s production superblock path. //! //! The superblock exists so a replica recovers a view it already acted in from its //! OWN disk after a crash, instead of inferring a stale view from the WAL or @@ -41,7 +41,7 @@ //! so recovery is exercised without wedging the cluster below quorum. //! //! vsr-only: a metadata view change has no analog on the single-process legacy -//! server, and the superblock is the server's durable consensus record. +//! server, and the superblock is server-ng's durable consensus record. use std::path::Path; use std::time::{Duration, Instant}; diff --git a/core/integration/tests/server/flush_vsr.rs b/core/integration/tests/server/flush_vsr.rs index 56ff451c32..36c840724f 100644 --- a/core/integration/tests/server/flush_vsr.rs +++ b/core/integration/tests/server/flush_vsr.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -//! Flush contract against the server (vsr): the server has no on-demand flush +//! Flush contract against server-ng (vsr): the server has no on-demand flush //! primitive, so `FLUSH_UNSAVED_BUFFER` must surface a typed //! `FeatureUnavailable` over the SDK rather than the non-replicated catch-all's //! empty-ok, which would fake a durability guarantee. @@ -27,7 +27,7 @@ use iggy::prelude::*; use integration::iggy_harness; -// Partition ids are 0-based (CreateTopic assigns them from 0). +// server-ng partition ids are 0-based (CreateTopic assigns them from 0). const PARTITION_ID: u32 = 0; #[iggy_harness( diff --git a/core/integration/tests/server/general.rs b/core/integration/tests/server/general.rs index d21d118ab4..c9a5ad6068 100644 --- a/core/integration/tests/server/general.rs +++ b/core/integration/tests/server/general.rs @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +#[cfg(not(feature = "vsr"))] +use crate::server::scenarios::bench_scenario; use crate::server::scenarios::{ authentication_scenario, consumer_timestamp_polling_scenario, invalid_consumer_offset_scenario, message_headers_scenario, permissions_scenario, snapshot_scenario, @@ -101,6 +103,23 @@ async fn stream_size_validation(harness: &TestHarness) { stream_size_validation_scenario::run(harness).await; } +// Blocked under vsr: pushes 8 MiB through the data plane, which drains the +// in-memory partition journal to disk segments; benchmarks are out of +// scope for the vsr test pass. +#[cfg(not(feature = "vsr"))] +#[iggy_harness( + test_client_transport = [Tcp, Http, Quic, WebSocket], + server( + tcp.socket.override_defaults = true, + tcp.socket.nodelay = true, + quic.max_idle_timeout = "500s", + quic.keep_alive_interval = "15s" + ) +)] +async fn bench(harness: &TestHarness) { + bench_scenario::run(harness).await; +} + #[iggy_harness( test_client_transport = [Tcp, Http, Quic, WebSocket], server( diff --git a/core/integration/tests/server/http_client.rs b/core/integration/tests/server/http_client.rs index 7393ffef36..e254fbfd7e 100644 --- a/core/integration/tests/server/http_client.rs +++ b/core/integration/tests/server/http_client.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -//! Shared HTTP transport plumbing for the server REST suites (`http_vsr`, +//! Shared HTTP transport plumbing for the server-ng REST suites (`http_vsr`, //! `http_rbac`): one authenticated `reqwest` session with the login-retry gate //! and the generic verb helpers. Each suite keeps its own request shapes and //! assertions as extension methods on [`HttpClient`], so the wire-contract and @@ -38,7 +38,7 @@ pub const REQUEST_TIMEOUT: Duration = Duration::from_secs(30); /// One authenticated HTTP session against the test server's shard-0 listener: a /// `reqwest` client, the listener base URL, and the bearer to send. The bearer -/// is either a login JWT or a raw personal access token (the server resolves +/// is either a login JWT or a raw personal access token (server-ng resolves /// either on the `Authorization: Bearer` header). pub struct HttpClient { pub client: reqwest::Client, diff --git a/core/integration/tests/server/http_rbac.rs b/core/integration/tests/server/http_rbac.rs index 6a113d03c9..e07fffb5d6 100644 --- a/core/integration/tests/server/http_rbac.rs +++ b/core/integration/tests/server/http_rbac.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -//! HTTP wire-contract residue for the server's shard-0 REST listener. The RBAC +//! HTTP wire-contract residue for server-ng's shard-0 REST listener. The RBAC //! authorization matrix itself (who may do what, over every transport) lives in //! the cross-transport `permissions_scenario` suite; this file keeps only what //! the SDK abstracts away and only raw HTTP can show: @@ -42,7 +42,7 @@ use integration::iggy_harness; use reqwest::{Response, StatusCode}; use serde_json::{Value, json}; -// Partition ids are 0-based (CreateTopic assigns them from 0). +// server-ng partition ids are 0-based (CreateTopic assigns them from 0). const PARTITION_ID: u32 = 0; // Explicit consumer id in the poll query (`Consumer::default()` is numeric 0). const CONSUMER_ID: u32 = 1; diff --git a/core/integration/tests/server/http_tls.rs b/core/integration/tests/server/http_tls.rs index 15e5233f85..f36e4a058d 100644 --- a/core/integration/tests/server/http_tls.rs +++ b/core/integration/tests/server/http_tls.rs @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -//! End-to-end HTTPS proof for the server shard-0 REST listener. The TLS -//! accept pump ([`server::http::tls`]) and the `start()` HTTPS branch +//! End-to-end HTTPS proof for the server-ng shard-0 REST listener. The TLS +//! accept pump ([`server_ng::http::tls`]) and the `start()` HTTPS branch //! unit-test in isolation; this is the only path that drives a live rustls //! client over the wire and proves the response was actually served over //! HTTP/2, negotiated via ALPN. A failure here (h1 fallback, handshake @@ -71,7 +71,7 @@ fn cert_asset(file: &str) -> PathBuf { .unwrap_or_else(|error| panic!("canonicalize repo cert asset {file}: {error}")) } -/// Boot an iggy-server cluster with `[http.tls]` enabled against the repo +/// Boot an iggy-server-ng cluster with `[http.tls]` enabled against the repo /// loopback cert, then prove a real HTTPS request is served over HTTP/2. Two /// nodes rather than one because the `/cluster/metadata` assertion below /// needs an enabled cluster roster (a single-node harness runs with the @@ -113,7 +113,7 @@ async fn given_http_tls_enabled_when_pinging_should_serve_https_over_http2() { harness .start() .await - .expect("start the server cluster with HTTPS enabled"); + .expect("start server-ng cluster with HTTPS enabled"); let addr = harness .server() diff --git a/core/integration/tests/server/http_vsr.rs b/core/integration/tests/server/http_vsr.rs index ca4acf4c37..5d6e1e2ea5 100644 --- a/core/integration/tests/server/http_vsr.rs +++ b/core/integration/tests/server/http_vsr.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -//! HTTP data-plane gate for the server: produce, poll, and consumer-offset +//! HTTP data-plane gate for server-ng: produce, poll, and consumer-offset //! routes exercised over raw `reqwest` (not the SDK HTTP client) so the wire //! contract itself is under test - exact status codes, the //! `iggy-durability` header, the body-size cap, and cross-request isolation @@ -36,7 +36,7 @@ use std::str::FromStr; use std::time::{Duration, Instant}; use tokio::time::sleep; -// Partition ids are 0-based (CreateTopic assigns them from 0). +// server-ng partition ids are 0-based (CreateTopic assigns them from 0). const PARTITION_ID: u32 = 0; /// Explicit consumer id shared by the offset store body and the read/delete @@ -826,7 +826,7 @@ async fn given_valid_access_token_when_refreshing_should_issue_working_token_wit "the refreshed token must authenticate" ); - // Stateless by design: the server has no replicated revocation list (P3), so + // Stateless by design: server-ng has no replicated revocation list (P3), so // refreshing never invalidates the token it was minted from - the old token // lives to its natural exp. Deliberate, not a bug; the same posture as // logout, which ends a session without revoking its bearer. diff --git a/core/integration/tests/server/legacy_login_vsr.rs b/core/integration/tests/server/legacy_login_vsr.rs index 9e129e928e..4161a4c3b2 100644 --- a/core/integration/tests/server/legacy_login_vsr.rs +++ b/core/integration/tests/server/legacy_login_vsr.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -//! Legacy login codes against the server (vsr). The server authenticates only +//! Legacy login codes against server-ng (vsr). server-ng authenticates only //! through the Register handshake, so the pre-register `LOGIN_USER` (38) and //! `LOGIN_WITH_PERSONAL_ACCESS_TOKEN` (44) codes -- which the vsr SDK never //! emits (its typed login methods send the register codes, its raw path @@ -27,9 +27,12 @@ //! TCP socket: a header-only non-replicated frame carrying the code in the //! reserved command slot. +#![cfg(feature = "vsr")] + use iggy_binary_protocol::HEADER_SIZE; use iggy_binary_protocol::codes::{LOGIN_USER_CODE, LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE}; use iggy_binary_protocol::consensus::{Command2, Operation, RequestHeader}; +use iggy_binary_protocol::namespace::METADATA_CONSENSUS_NAMESPACE; use integration::harness::TestHarness; use integration::iggy_harness; use std::mem::offset_of; @@ -69,6 +72,7 @@ async fn assert_legacy_login_code_evicted(harness: &TestHarness, code: u32) { client: 0xC0FFEE, session: 0, request: 0, + namespace: METADATA_CONSENSUS_NAMESPACE, ..Default::default() }; // A non-replicated command code travels in the first 4 reserved bytes. diff --git a/core/integration/tests/server/login_credentials_vsr.rs b/core/integration/tests/server/login_credentials_vsr.rs deleted file mode 100644 index 0f05c9beb4..0000000000 --- a/core/integration/tests/server/login_credentials_vsr.rs +++ /dev/null @@ -1,91 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Credential rejections on the register handshake against the server (vsr). -//! -//! A username/password body that fails verification falls through to the PAT -//! decode attempt, so the terminal rejection has to carry the credential -//! failure that actually happened. Reporting the payload shape instead -//! (`MalformedLogin` -> `InvalidFormat`) tells a client its request was -//! malformed when the request was fine and the password was not. - -use iggy::prelude::*; -use integration::iggy_harness; - -#[iggy_harness(test_client_transport = [Tcp])] -async fn given_wrong_password_when_logging_in_should_reject_invalid_credentials( - harness: &TestHarness, -) { - let client = harness - .new_client_for(TransportProtocol::Tcp) - .await - .expect("tcp client"); - - let result = client - .login_user("iggy", "definitely-not-the-password") - .await; - - let expected = IggyError::InvalidCredentials.as_code(); - assert!( - matches!(&result, Err(error) if error.as_code() == expected), - "a wrong password must surface Err(InvalidCredentials), got {result:?}" - ); -} - -#[iggy_harness(test_client_transport = [Tcp])] -async fn given_unknown_username_when_logging_in_should_reject_invalid_credentials( - harness: &TestHarness, -) { - let client = harness - .new_client_for(TransportProtocol::Tcp) - .await - .expect("tcp client"); - - let result = client.login_user("no-such-user", "irrelevant").await; - - // Same rejection as a wrong password: an unknown username must not be - // distinguishable from a bad password. - let expected = IggyError::InvalidCredentials.as_code(); - assert!( - matches!(&result, Err(error) if error.as_code() == expected), - "an unknown username must surface Err(InvalidCredentials), got {result:?}" - ); -} - -#[iggy_harness(test_client_transport = [Tcp])] -async fn given_rejected_login_when_retrying_with_valid_credentials_should_succeed( - harness: &TestHarness, -) { - let client = harness - .new_client_for(TransportProtocol::Tcp) - .await - .expect("tcp client"); - - let rejected = client.login_user("iggy", "wrong").await; - assert!(rejected.is_err(), "the first login must be rejected"); - - // The rejection evicts the session, so a usable client has to reconnect; - // the point is that a bad password does not poison the account. - let retry = harness - .new_client_for(TransportProtocol::Tcp) - .await - .expect("tcp client"); - retry - .login_user("iggy", "iggy") - .await - .expect("valid credentials must still authenticate after a rejection"); -} diff --git a/core/integration/tests/server/mod.rs b/core/integration/tests/server/mod.rs index f31ca33a02..ed6ce07437 100644 --- a/core/integration/tests/server/mod.rs +++ b/core/integration/tests/server/mod.rs @@ -16,55 +16,63 @@ // under the License. // a2a_jwt exercises trusted-issuer (JWKS) tokens; both the legacy verifier and -// the server's ported trusted-issuer path verify them. +// server-ng's ported trusted-issuer path verify them. mod a2a_jwt; mod cg; -// Flush (FLUSH_UNSAVED_BUFFER) has no the server primitive; it must deny typed. +// Flush (FLUSH_UNSAVED_BUFFER) has no server-ng primitive; it must deny typed. +#[cfg(feature = "vsr")] mod flush_vsr; -// Legacy login codes (LOGIN_USER / LOGIN_WITH_PAT) have no the server handler; +// Legacy login codes (LOGIN_USER / LOGIN_WITH_PAT) have no server-ng handler; // they must evict typed (MalformedLogin), not stall or reply empty-ok. +#[cfg(feature = "vsr")] mod legacy_login_vsr; -// A failed credential login must report the credential failure, not the -// payload shape it fell through to. -mod login_credentials_vsr; // Poll addressing + timestamp semantics: typed PartitionNotFound on a bad // partition id, at-or-after timestamp polls. +#[cfg(feature = "vsr")] mod poll_semantics_vsr; // Create-topic static bounds deny typed before consensus. +#[cfg(feature = "vsr")] mod topic_admission_vsr; -// Stats aggregates the cross-shard connected-client count, not a hardcoded 0. -mod stats_vsr; // Purge durability: applied generation survives restart; journal-resident // purged batches stay fenced behind the purge floor. +#[cfg(feature = "vsr")] mod purge_vsr; // Shared HTTP transport plumbing (session + verb helpers) for the raw-HTTP -// server suites below. +// server-ng suites below. +#[cfg(feature = "vsr")] mod http_client; -// Raw-HTTP data-plane contract against the server's shard-0 listener. +// Raw-HTTP data-plane contract against server-ng's shard-0 listener. +#[cfg(feature = "vsr")] mod http_vsr; -// Raw-HTTP wire-contract residue against the server (status codes + typed error +// Raw-HTTP wire-contract residue against server-ng (status codes + typed error // bodies); the RBAC matrix lives in permissions_scenario. +#[cfg(feature = "vsr")] mod http_rbac; -// End-to-end HTTPS: the server serves the REST listener over TLS and negotiates +// End-to-end HTTPS: server-ng serves the REST listener over TLS and negotiates // HTTP/2 via ALPN. +#[cfg(feature = "vsr")] mod http_tls; // Binary GetClusterMetadata must serve the real roster from a VSR cluster. +#[cfg(feature = "vsr")] mod cluster_metadata_vsr; // A metadata view change must persist the advanced view and recover it from disk // across a replica restart. +#[cfg(feature = "vsr")] mod cluster_view_durability_vsr; // A partition view change must persist the advanced view in that group's own // superblock and recover it from disk across a replica restart. +#[cfg(feature = "vsr")] mod partition_view_durability_vsr; // 80-case race matrix with hardcoded HTTP variants (test_matrix bypasses // the harness transport filter). mod concurrent_addition; mod general; -// The per-shard segment cleaner deletes expired / oversize segments from disk. +// The per-shard segment cleaner deletes expired / oversize segments from disk +// under both the legacy server and server-ng. mod message_cleanup; mod message_retrieval; // Server restarts, consumer-group barriers, and DeleteSegments maintenance. -// The full restart matrix (consumer variants included) runs under the server: +// The full restart matrix (consumer variants included) runs under server-ng: // a restarted replica rejoins via the view probe + journal repair. mod purge_delete; mod scenarios; diff --git a/core/integration/tests/server/partition_view_durability_vsr.rs b/core/integration/tests/server/partition_view_durability_vsr.rs index 0ea467e5ae..0071473998 100644 --- a/core/integration/tests/server/partition_view_durability_vsr.rs +++ b/core/integration/tests/server/partition_view_durability_vsr.rs @@ -29,7 +29,7 @@ //! partition data path rides along. //! //! vsr-only: partition consensus groups and their superblocks exist only on -//! `iggy-server`. +//! `iggy-server-ng`. use std::path::{Path, PathBuf}; use std::str::FromStr; @@ -44,7 +44,7 @@ use tokio::time::sleep; const STREAM_NAME: &str = "partition-view-durability-stream"; const TOPIC_NAME: &str = "partition-view-durability-topic"; -/// Partition ids are 0-based (CreateTopic assigns them from 0). +/// server-ng partition ids are 0-based (CreateTopic assigns them from 0). const PARTITION_ID: u32 = 0; const MESSAGES_COUNT: u32 = 10; diff --git a/core/integration/tests/server/poll_semantics_vsr.rs b/core/integration/tests/server/poll_semantics_vsr.rs index 9d211e9a6a..1f48eef36b 100644 --- a/core/integration/tests/server/poll_semantics_vsr.rs +++ b/core/integration/tests/server/poll_semantics_vsr.rs @@ -15,12 +15,10 @@ // specific language governing permissions and limitations // under the License. -//! Poll semantics against the server (vsr): a poll aimed at a partition id the +//! Poll semantics against server-ng (vsr): a poll aimed at a partition id the //! topic does not have must surface a typed `PartitionNotFound`, not an empty -//! poll a consumer would read as end-of-partition; a poll whose stream or -//! topic does not resolve must surface the legacy `StreamIdNotFound` / -//! `TopicIdNotFound` the same way; the partition addressing error on -//! `get_consumer_offset` must not decode as "no offset stored"; and a +//! poll a consumer would read as end-of-partition; the same addressing error +//! on `get_consumer_offset` must not decode as "no offset stored"; and a //! timestamp poll must be at-or-after, including the message stamped exactly at //! the queried timestamp (the timestamp replies report per message). @@ -89,72 +87,10 @@ async fn given_missing_partition_when_polling_should_reject_partition_not_found( assert_eq!(valid.messages.len(), 0, "empty topic polls empty"); } -#[iggy_harness( - test_client_transport = [Tcp], - server(tcp.socket.override_defaults = true, tcp.socket.nodelay = true) -)] -async fn given_missing_stream_when_polling_should_reject_stream_not_found(harness: &TestHarness) { - let client = harness.tcp_root_client().await.expect("tcp root client"); - let stream_id = Identifier::from_str_value("no-such-stream").expect("stream identifier"); - let topic_id = Identifier::from_str_value("no-such-topic").expect("topic identifier"); - - let result = client - .poll_messages( - &stream_id, - &topic_id, - Some(0), - &Consumer::default(), - &PollingStrategy::offset(0), - 1, - false, - ) - .await; - - let expected = IggyError::StreamIdNotFound(Identifier::default()).as_code(); - assert!( - matches!(&result, Err(error) if error.as_code() == expected), - "polling a missing stream must surface Err(StreamIdNotFound), got {result:?}" - ); -} - -#[iggy_harness( - test_client_transport = [Tcp], - server(tcp.socket.override_defaults = true, tcp.socket.nodelay = true) -)] -async fn given_missing_topic_when_polling_should_reject_topic_not_found(harness: &TestHarness) { - let client = harness.tcp_root_client().await.expect("tcp root client"); - client - .create_stream("topicless-stream") - .await - .expect("create stream"); - let stream_id = Identifier::from_str_value("topicless-stream").expect("stream identifier"); - let topic_id = Identifier::from_str_value("no-such-topic").expect("topic identifier"); - - let result = client - .poll_messages( - &stream_id, - &topic_id, - Some(0), - &Consumer::default(), - &PollingStrategy::offset(0), - 1, - false, - ) - .await; - - let expected = - IggyError::TopicIdNotFound(Identifier::default(), Identifier::default()).as_code(); - assert!( - matches!(&result, Err(error) if error.as_code() == expected), - "polling a missing topic of an existing stream must surface Err(TopicIdNotFound), \ - got {result:?}" - ); -} - /// `get_consumer_offset` answered an unknown partition with an empty body, /// which the SDK decodes as `None` - the same value a consumer that simply has /// no stored offset yet gets back, so a client could not tell a typo from a -/// fresh consumer. Legacy swallows this one too; the server surfaces the code +/// fresh consumer. Legacy swallows this one too; server-ng surfaces the code /// the poll path already surfaces for the identical addressing error. #[iggy_harness( test_client_transport = [Tcp], diff --git a/core/integration/tests/server/purge_delete.rs b/core/integration/tests/server/purge_delete.rs index c597b9ed52..5b631ae608 100644 --- a/core/integration/tests/server/purge_delete.rs +++ b/core/integration/tests/server/purge_delete.rs @@ -19,17 +19,12 @@ use crate::server::scenarios::purge_delete_scenario; use integration::iggy_harness; use test_case::test_matrix; -// Restart scenarios run single-node: restarting a node in a multi-node cluster -// trips a known partitions-plane view-change stall, tracked separately. -#[iggy_harness( - cluster_nodes = 1, - server( - segment.size = "5KiB", - segment.cache_indexes = ["all", "none", "open_segment"], - partition.messages_required_to_save = "1", - partition.enforce_fsync = "true", - ) -)] +#[iggy_harness(server( + segment.size = "5KiB", + segment.cache_indexes = ["all", "none", "open_segment"], + partition.messages_required_to_save = "1", + partition.enforce_fsync = "true", +))] #[test_matrix([restart_off(), restart_on()])] async fn should_delete_segments_and_validate_filesystem( harness: &mut TestHarness, @@ -38,15 +33,12 @@ async fn should_delete_segments_and_validate_filesystem( purge_delete_scenario::run(harness, restart_server).await; } -#[iggy_harness( - cluster_nodes = 1, - server( - segment.size = "5KiB", - segment.cache_indexes = ["all", "none", "open_segment"], - partition.messages_required_to_save = "1", - partition.enforce_fsync = "true", - ) -)] +#[iggy_harness(server( + segment.size = "5KiB", + segment.cache_indexes = ["all", "none", "open_segment"], + partition.messages_required_to_save = "1", + partition.enforce_fsync = "true", +))] #[test_matrix([restart_off(), restart_on()])] async fn should_delete_segments_without_consumers(harness: &mut TestHarness, restart_server: bool) { purge_delete_scenario::run_no_consumers(harness, restart_server).await; @@ -65,15 +57,12 @@ async fn should_delete_segments_with_consumer_group_barrier(harness: &TestHarnes purge_delete_scenario::run_consumer_group_barrier(&client, &data_path).await; } -#[iggy_harness( - cluster_nodes = 1, - server( - segment.size = "5KiB", - segment.cache_indexes = ["all", "none", "open_segment"], - partition.messages_required_to_save = "1", - partition.enforce_fsync = "true", - ) -)] +#[iggy_harness(server( + segment.size = "5KiB", + segment.cache_indexes = ["all", "none", "open_segment"], + partition.messages_required_to_save = "1", + partition.enforce_fsync = "true", +))] #[test_matrix([restart_off(), restart_on()])] async fn should_block_deletion_until_all_consumers_pass_segment( harness: &mut TestHarness, @@ -82,15 +71,12 @@ async fn should_block_deletion_until_all_consumers_pass_segment( purge_delete_scenario::run_multi_consumer_barrier(harness, restart_server).await; } -#[iggy_harness( - cluster_nodes = 1, - server( - segment.size = "5KiB", - segment.cache_indexes = ["all", "none", "open_segment"], - partition.messages_required_to_save = "1", - partition.enforce_fsync = "true", - ) -)] +#[iggy_harness(server( + segment.size = "5KiB", + segment.cache_indexes = ["all", "none", "open_segment"], + partition.messages_required_to_save = "1", + partition.enforce_fsync = "true", +))] // The scenario asserts the exact [0, 7, 14, 21] layout only on the legacy path; // under vsr it verifies the framing-agnostic purge outcome (offsets cleared, // files deleted, partition reset to a single segment at offset 0). diff --git a/core/integration/tests/server/purge_vsr.rs b/core/integration/tests/server/purge_vsr.rs index 6a5eac2088..5cd8028549 100644 --- a/core/integration/tests/server/purge_vsr.rs +++ b/core/integration/tests/server/purge_vsr.rs @@ -15,14 +15,11 @@ // specific language governing permissions and limitations // under the License. -//! Server purge durability: the applied purge generation survives a +//! server-ng purge durability: the applied purge generation survives a //! restart (`purge.gen`), and purged journal-resident batches stay fenced //! behind the purge floor instead of resurfacing through the shutdown flush. -//! Plus read-your-purge: the counters a purge acks are visible to the very -//! next read, without waiting for the reconciler's on-disk reset. use crate::server::scenarios::purge_delete_scenario; -use iggy::prelude::*; use integration::iggy_harness; // Single node: the tests reason about ONE replica's on-disk state across a @@ -52,99 +49,3 @@ async fn given_journal_resident_messages_when_purged_should_not_resurface( ) { purge_delete_scenario::run_resident_purge_no_resurface(harness).await; } - -// No sleep, no poll: the purge acks on commit and the segment prune runs later -// on the reconciler, so the reset of the counters `get_topic` / `get_stream` -// read has to happen in the replicated apply. A retry loop here would pass -// against the pre-apply behavior too. -#[iggy_harness( - test_client_transport = [Tcp], - server(tcp.socket.override_defaults = true, tcp.socket.nodelay = true) -)] -async fn given_purged_topic_when_getting_topic_immediately_should_report_zero_stats( - harness: &TestHarness, -) { - const STREAM: &str = "purge-stats-stream"; - const TOPIC: &str = "purge-stats-topic"; - - let client = harness.tcp_root_client().await.expect("tcp root client"); - client.create_stream(STREAM).await.expect("create stream"); - let stream_id = Identifier::from_str_value(STREAM).expect("stream identifier"); - let topic_id = Identifier::from_str_value(TOPIC).expect("topic identifier"); - client - .create_topic( - &stream_id, - TOPIC, - 1, - CompressionAlgorithm::None, - None, - IggyExpiry::NeverExpire, - MaxTopicSize::ServerDefault, - ) - .await - .expect("create topic"); - - let mut messages: Vec = (0..10) - .map(|index| { - IggyMessage::builder() - .payload(format!("message-{index}").into()) - .build() - .expect("build message") - }) - .collect(); - client - .send_messages( - &stream_id, - &topic_id, - &Partitioning::partition_id(0), - &mut messages, - ) - .await - .expect("send messages"); - - let before = client - .get_topic(&stream_id, &topic_id) - .await - .expect("get topic before purge") - .expect("topic exists before purge"); - assert_eq!( - before.messages_count, 10, - "the send must be counted before the purge, or the assert below proves nothing" - ); - assert!(before.size.as_bytes_u64() > 0); - - client - .purge_topic(&stream_id, &topic_id) - .await - .expect("purge topic"); - - let topic = client - .get_topic(&stream_id, &topic_id) - .await - .expect("get topic after purge") - .expect("purge keeps the topic"); - assert_eq!( - topic.messages_count, 0, - "a read right after the purge ack must not report pre-purge messages" - ); - assert_eq!(topic.size.as_bytes_u64(), 0); - assert_eq!( - topic.partitions.len(), - 1, - "purge keeps the partition, it only empties it" - ); - assert_eq!(topic.partitions[0].messages_count, 0); - assert_eq!(topic.partitions[0].size.as_bytes_u64(), 0); - assert_eq!(topic.partitions[0].current_offset, 0); - - let stream = client - .get_stream(&stream_id) - .await - .expect("get stream after purge") - .expect("purge keeps the stream"); - assert_eq!( - stream.messages_count, 0, - "the stream rollup must drop with its purged topic" - ); - assert_eq!(stream.size.as_bytes_u64(), 0); -} diff --git a/core/integration/tests/server/scenarios/authentication_scenario.rs b/core/integration/tests/server/scenarios/authentication_scenario.rs index 0932e55861..e39d6cd6a4 100644 --- a/core/integration/tests/server/scenarios/authentication_scenario.rs +++ b/core/integration/tests/server/scenarios/authentication_scenario.rs @@ -127,16 +127,17 @@ async fn test_all_commands_require_auth(client: &IggyClient) { ) { continue; } - // The server serves `GetClusterMetadata` pre-auth so a client can + // server-ng serves `GetClusterMetadata` pre-auth so a client can // locate the cluster leader before signing in; the legacy server // still auth-gates it. + #[cfg(feature = "vsr")] if code == GET_CLUSTER_METADATA_CODE { continue; } // Stateful - not supported on HTTP. `SYNC_CONSUMER_GROUP` is // SDK-internal (issued during poll partition resolution), with no // top-level client method to invoke unauthenticated here; its auth - // gate is exercised through the server dispatch allowlist instead. + // gate is exercised through the server-ng dispatch allowlist instead. if matches!( code, JOIN_CONSUMER_GROUP_CODE | LEAVE_CONSUMER_GROUP_CODE | SYNC_CONSUMER_GROUP_CODE @@ -152,7 +153,7 @@ async fn test_all_commands_require_auth(client: &IggyClient) { } // v2 consumer-offset ops are registered in the dispatch table for the // consensus/simulator pathway but are not wired into the legacy binary - // server's dispatch. They'll move into the server alongside the rest of + // server's dispatch. They'll move into server-ng alongside the rest of // the v2 surface; re-enable these codes here once that lands. if matches!( code, diff --git a/core/integration/tests/server/scenarios/bench_scenario.rs b/core/integration/tests/server/scenarios/bench_scenario.rs new file mode 100644 index 0000000000..b5a333e67c --- /dev/null +++ b/core/integration/tests/server/scenarios/bench_scenario.rs @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use iggy::prelude::*; +use iggy_common::TransportProtocol; +use integration::bench_utils::run_bench_and_wait_for_finish; +use integration::harness::TestHarness; + +pub async fn run(harness: &TestHarness) { + let transport = harness.transport().expect("Transport not set"); + let server = harness.server(); + let server_addr = match transport { + TransportProtocol::Tcp => server.raw_tcp_addr().expect("TCP address not available"), + TransportProtocol::Http => server + .http_addr() + .expect("HTTP address not available") + .to_string(), + TransportProtocol::Quic => server + .quic_addr() + .expect("QUIC address not available") + .to_string(), + TransportProtocol::WebSocket => server + .websocket_addr() + .expect("WebSocket address not available") + .to_string(), + }; + let data_size = IggyByteSize::from(8 * 1024 * 1024); + + run_bench_and_wait_for_finish(&server_addr, &transport, "pinned-producer", data_size); + run_bench_and_wait_for_finish(&server_addr, &transport, "pinned-consumer", data_size); +} diff --git a/core/integration/tests/server/scenarios/encryption_scenario.rs b/core/integration/tests/server/scenarios/encryption_scenario.rs index 4316a1fd90..79c1c2e995 100644 --- a/core/integration/tests/server/scenarios/encryption_scenario.rs +++ b/core/integration/tests/server/scenarios/encryption_scenario.rs @@ -31,11 +31,7 @@ use test_case::test_matrix; #[tokio::test] #[parallel] async fn should_fill_data_with_headers_and_verify_after_restart_using_api(encryption: bool) { - // Restart scenarios run single-node: restarting a node in a multi-node - // cluster trips a known partitions-plane view-change stall, tracked - // separately. let mut harness = TestHarness::builder() - .cluster_nodes(1) .server(build_server_config(encryption)) .build() .unwrap(); @@ -97,9 +93,20 @@ async fn should_fill_data_with_headers_and_verify_after_restart_using_api(encryp .await .unwrap(); - // No flush primitive exists (FLUSH_UNSAVED_BUFFER denies typed); the - // eager-flush envs in `build_server_config` make every committed batch hit - // disk instead. + // server-ng has no flush primitive (FLUSH_UNSAVED_BUFFER denies typed); + // the eager-flush envs in `build_server_config` make every committed + // batch hit disk instead. + #[cfg(not(feature = "vsr"))] + client + .flush_unsaved_buffer( + &Identifier::named(stream_name).unwrap(), + &Identifier::named(topic_name).unwrap(), + 0, + true, + ) + .await + .unwrap(); + tokio::time::sleep(tokio::time::Duration::from_millis(500)).await; // Verify on-disk encryption of headers and payload @@ -244,9 +251,20 @@ async fn should_fill_data_with_headers_and_verify_after_restart_using_api(encryp .await .unwrap(); - // No flush primitive exists (FLUSH_UNSAVED_BUFFER denies typed); the - // eager-flush envs in `build_server_config` make every committed batch hit - // disk instead. + // server-ng has no flush primitive (FLUSH_UNSAVED_BUFFER denies typed); + // the eager-flush envs in `build_server_config` make every committed + // batch hit disk instead. + #[cfg(not(feature = "vsr"))] + client + .flush_unsaved_buffer( + &Identifier::named(stream_name).unwrap(), + &Identifier::named(topic_name).unwrap(), + 0, + true, + ) + .await + .unwrap(); + tokio::time::sleep(tokio::time::Duration::from_millis(500)).await; let polled = client @@ -480,7 +498,7 @@ fn encryption_disabled() -> bool { fn build_server_config(encryption: bool) -> TestServerConfig { let mut extra_envs = HashMap::new(); - // The server flushes on the journal thresholds (no flush primitive), so + // server-ng flushes on the journal thresholds (no flush primitive), so // force every committed batch straight to disk for the on-disk asserts. extra_envs.insert( "IGGY_SYSTEM_PARTITION_MESSAGES_REQUIRED_TO_SAVE".to_string(), diff --git a/core/integration/tests/server/scenarios/mod.rs b/core/integration/tests/server/scenarios/mod.rs index 0143621e9c..5287c30539 100644 --- a/core/integration/tests/server/scenarios/mod.rs +++ b/core/integration/tests/server/scenarios/mod.rs @@ -16,6 +16,8 @@ // under the License. pub mod authentication_scenario; +#[cfg(not(feature = "vsr"))] +pub mod bench_scenario; pub mod concurrent_produce_consume_scenario; pub mod concurrent_scenario; pub mod consumer_group_auto_commit_reconnection_scenario; @@ -27,7 +29,7 @@ pub mod consumer_group_with_multiple_clients_polling_messages_scenario; pub mod consumer_group_with_single_client_polling_messages_scenario; pub mod consumer_timestamp_polling_scenario; // Cross-protocol PAT visibility (create via HTTP, list via TCP across shards, -// and the reverse). Runs under vsr too: the server serves the PAT routes on its +// and the reverse). Runs under vsr too: server-ng serves the PAT routes on its // shard-0 HTTP listener and the create/delete commit through the metadata STM, // so the token replicates to every shard a TCP client may land on. pub mod cross_protocol_pat_scenario; @@ -77,7 +79,7 @@ const MESSAGES_COUNT: u32 = 1337; /// /// `send_messages` acks at consensus commit while the owning shard applies /// the batch asynchronously (see the materialisation race note at the top -/// of `server/src/partition_reconciler.rs`), so the first read after a +/// of `server-ng/src/partition_reconciler.rs`), so the first read after a /// send burst can observe fewer messages than were acked. Retrying absorbs /// that convergence window without weakening the caller's assertion: real /// message loss still returns short and fails it once the deadline expires. diff --git a/core/integration/tests/server/scenarios/permissions_scenario.rs b/core/integration/tests/server/scenarios/permissions_scenario.rs index cdbb514b07..76df9d80f3 100644 --- a/core/integration/tests/server/scenarios/permissions_scenario.rs +++ b/core/integration/tests/server/scenarios/permissions_scenario.rs @@ -69,7 +69,7 @@ pub async fn run(harness: &TestHarness) { // Missing resource behavior tests test_missing_resource_behavior(harness, &root_client).await; - // RBAC surface ported from the raw-HTTP the server suite (server::http_rbac), + // RBAC surface ported from the raw-HTTP server-ng suite (server::http_rbac), // asserting the exact typed IggyError over every transport. test_consumer_offset_permissions(harness, &root_client).await; test_change_password_reply_path(harness, &root_client).await; @@ -2474,7 +2474,7 @@ async fn test_consumer_offset_permissions(harness: &TestHarness, root_client: &I IggyError::Unauthorized, "no perms: delete_consumer_offset", ); - // GET is enumeration-safe: legacy answers Ok(None), the server denies typed. + // GET is enumeration-safe: legacy answers Ok(None), server-ng denies typed. assert_unauthorized( client .get_consumer_offset(&consumer, &stream_id, &topic_id, Some(0)) diff --git a/core/integration/tests/server/scenarios/purge_delete_scenario.rs b/core/integration/tests/server/scenarios/purge_delete_scenario.rs index 30cda49ac4..c790aae63f 100644 --- a/core/integration/tests/server/scenarios/purge_delete_scenario.rs +++ b/core/integration/tests/server/scenarios/purge_delete_scenario.rs @@ -31,22 +31,44 @@ const PARTITION_ID: u32 = 0; const LOG_EXTENSION: &str = "log"; const INDEX_EXTENSION: &str = "index"; -/// The server persists the actual `SendMessages2` batch framing: a 256-byte -/// command header per append (each send below is a single-message batch) plus -/// a 48-byte per-message header, and a 24-byte sparse index entry per flush -/// (one per message with messages_required_to_save = 1). See -/// `server_common::send_messages2` and `stream_size_validation_scenario`. +/// Payload chosen so IGGY_MESSAGE_HEADER_SIZE + payload = 1000B per message on disk. +/// +/// Rotation mechanics (with segment.size = 5KiB = 5120B, messages_required_to_save = 1): +/// `is_full()` checks `size >= 5120` BEFORE persisting the current message. +/// After 6 persisted messages (6000B >= 5120) the next arrival sees is_full=true, +/// gets persisted into the same segment, then rotation fires. +/// Result: 7 messages per sealed segment (7000B on disk). const PAYLOAD_SIZE: usize = 936; +#[cfg(not(feature = "vsr"))] +const MESSAGE_ON_DISK_SIZE: u64 = IGGY_MESSAGE_HEADER_SIZE as u64 + PAYLOAD_SIZE as u64; +#[cfg(not(feature = "vsr"))] +const INDEX_SIZE_PER_MSG: u64 = INDEX_SIZE as u64; +// server-ng persists the actual `SendMessages2` batch framing: a 256-byte +// command header per append (each send below is a single-message batch) plus +// a 48-byte per-message header, and a 24-byte sparse index entry per flush +// (one per message with messages_required_to_save = 1). See +// `server_common::send_messages2` and `stream_size_validation_scenario`. +#[cfg(feature = "vsr")] const NG_BATCH_HEADER_SIZE: u64 = 256; +#[cfg(feature = "vsr")] const NG_MESSAGE_HEADER_SIZE: u64 = 48; +#[cfg(feature = "vsr")] const MESSAGE_ON_DISK_SIZE: u64 = NG_BATCH_HEADER_SIZE + NG_MESSAGE_HEADER_SIZE + PAYLOAD_SIZE as u64; +#[cfg(feature = "vsr")] const INDEX_SIZE_PER_MSG: u64 = 24; const TOTAL_MESSAGES: u32 = 25; +/// 3 sealed segments (7 msgs each) + 1 active (4 msgs at offsets 21-24). +#[cfg(not(feature = "vsr"))] +const EXPECTED_SEGMENT_OFFSETS: &[u64] = &[0, 7, 14, 21]; +#[cfg(not(feature = "vsr"))] +const MSGS_PER_SEALED_SEGMENT: u64 = 7; /// 5 sealed segments (5 msgs each at 1240B on disk; the post-append size /// check seals at 6200B >= 5KiB) + 1 empty active segment at offset 25. +#[cfg(feature = "vsr")] const EXPECTED_SEGMENT_OFFSETS: &[u64] = &[0, 5, 10, 15, 20, 25]; +#[cfg(feature = "vsr")] const MSGS_PER_SEALED_SEGMENT: u64 = 5; /// Single consumer barrier: oldest-first deletion, barrier advancement, and edge cases. @@ -137,29 +159,11 @@ pub async fn run(harness: &mut TestHarness, restart_server: bool) { // reflect the true partition max (24), not messages_count - 1 (17). { let max_offset = (TOTAL_MESSAGES - 1) as u64; - // Short poll, not a one-shot read: the restart cells reconnect, and a - // read issued before the SDK settles on the leader can land on a replica - // that has not applied the offset op yet, which answers "no offset" - // rather than redirecting. Measured sub-millisecond on every converging - // run, so 2s is a transient allowance -- an offset that is genuinely - // gone still fails here. - let offset_deadline = std::time::Instant::now() + std::time::Duration::from_secs(2); - let offset_info = loop { - // A read racing the SDK's post-restart re-sign-in answers - // `Unauthenticated`; retry it inside the window like an absent - // offset rather than panicking on the Result. - if let Ok(Some(info)) = client - .get_consumer_offset(&consumer, &stream_ident, &topic_ident, Some(PARTITION_ID)) - .await - { - break info; - } - assert!( - std::time::Instant::now() < offset_deadline, - "consumer offset must exist after segment deletion" - ); - tokio::time::sleep(std::time::Duration::from_millis(25)).await; - }; + let offset_info = client + .get_consumer_offset(&consumer, &stream_ident, &topic_ident, Some(PARTITION_ID)) + .await + .unwrap() + .expect("consumer offset must exist after segment deletion"); assert_eq!(offset_info.stored_offset, stored_offset); assert_eq!( offset_info.current_offset, @@ -370,9 +374,18 @@ pub async fn run_no_consumers(harness: &mut TestHarness, restart_server: bool) { let partition_path = partition_path(&data_path, stream_id, topic_id); - // Capture the real layout rather than hardcoding boundaries: this path - // only needs a sealed segment plus the active one. + // server-ng's per-message on-disk framing differs from legacy, so the + // segment boundaries are not hardcodable. Capture the real layout once; + // legacy still verifies it matches the calculated offsets + file sizes. let layout = get_sorted_segment_offsets(&partition_path); + #[cfg(not(feature = "vsr"))] + { + assert_eq!( + layout, EXPECTED_SEGMENT_OFFSETS, + "Segment layout must match calculated offsets" + ); + assert_segment_file_sizes(&partition_path, EXPECTED_SEGMENT_OFFSETS); + } assert!( layout.len() >= 2, "expected at least one sealed segment plus the active one, got {layout:?}" @@ -392,9 +405,11 @@ pub async fn run_no_consumers(harness: &mut TestHarness, restart_server: bool) { maybe_restart(harness, restart_server).await; let first_surviving = layout[i + 1]; - // Deletion is asynchronous (metadata commit -> reconciler), so - // converge before asserting. + // server-ng deletes asynchronously (metadata commit -> reconciler); + // legacy deletes synchronously. Converge before asserting. await_segment_layout(&partition_path, &layout[i + 1..]).await; + #[cfg(not(feature = "vsr"))] + assert_segment_file_sizes(&partition_path, &layout[i + 1..]); await_polled_offsets( &client, &stream_ident, @@ -416,6 +431,8 @@ pub async fn run_no_consumers(harness: &mut TestHarness, restart_server: bool) { .unwrap(); await_segment_layout(&partition_path, std::slice::from_ref(&active)).await; + #[cfg(not(feature = "vsr"))] + assert_segment_file_sizes(&partition_path, std::slice::from_ref(&active)); await_polled_offsets( &client, &stream_ident, @@ -848,6 +865,15 @@ pub async fn run_purge_topic(harness: &mut TestHarness, restart_server: bool) { let partition_path = partition_path(&data_path, stream_id, topic_id); + // Exact layout is legacy-framing-specific; the purge outcome asserted below + // (offsets cleared, files deleted, partition reset to a single empty segment + // at offset 0, new messages from offset 0) is framing-agnostic. + #[cfg(not(feature = "vsr"))] + assert_eq!( + get_sorted_segment_offsets(&partition_path), + EXPECTED_SEGMENT_OFFSETS + ); + // --- Store individual consumer offset at 13 --- let consumer = Consumer { kind: ConsumerKind::Consumer, @@ -956,15 +982,16 @@ pub async fn run_purge_topic(harness: &mut TestHarness, restart_server: bool) { let drained_before_restart = is_dir_empty(&consumers_dir) && is_dir_empty(&groups_dir); maybe_restart(harness, restart_server).await; - // Purge is asynchronous (metadata commit -> reconciler -> pump). The - // pump's purge resets the partition to a single segment at offset 0 and - // clears consumer offsets + files in the same frame, so converging on the - // [0] layout means the whole purge landed. + // server-ng purges asynchronously (metadata commit -> reconciler -> pump); + // legacy purges synchronously. The pump's purge resets the partition to a + // single segment at offset 0 and clears consumer offsets + files in the + // same frame, so converging on the [0] layout means the whole purge landed. + #[cfg(feature = "vsr")] await_segment_layout(&partition_path, &[0]).await; // --- Verify consumer offsets cleared (memory + disk) --- - // ZERO tolerance everywhere except one cell: restart where the kill landed - // mid-purge. There boot plants the [0] layout itself (fencing a torn + // ZERO tolerance everywhere except one cell: vsr + restart where the kill + // landed mid-purge. There boot plants the [0] layout itself (fencing a torn // chain, or recovering an already-drained directory) with the offset files // still present, so the layout gate above is satisfied BEFORE the // reconciler's re-purge clears them (the kill preceded the purge.gen @@ -974,41 +1001,26 @@ pub async fn run_purge_topic(harness: &mut TestHarness, restart_server: bool) { // would hide a regression that clears them one frame late. Kept short -- // a client-visible stale offset after purge-then-restart is a real // (bounded) window, not something to paper over with a long tolerance. - // 5s, not 2s: the re-purge after a restart is floor-bounded by the - // reconciler's 1s PERIODIC tick, not by a wake -- measured at 1.06-1.11s in - // isolation against 0.5-0.8ms for every non-restart cell. 2s left under one - // tick of slack, so metadata repair under load pushed it over. Still a - // bounded window on purpose: widen only with a measurement, and if this - // starts needing more, the wake is missing rather than the budget too small. - let poll_window = if restart_server && !drained_before_restart { - std::time::Duration::from_secs(5) + let poll_window = if cfg!(feature = "vsr") && restart_server && !drained_before_restart { + std::time::Duration::from_secs(2) } else { std::time::Duration::ZERO }; let offsets_deadline = std::time::Instant::now() + poll_window; loop { - // Errors retry inside the window instead of panicking: the restart cells - // reconnect mid-loop, so the first read after the server comes back can - // answer `Unauthenticated` while the SDK is still re-signing in. A - // transient here is "not converged yet", not a verdict. - let reads = futures::future::join( - client.get_consumer_offset(&consumer, &stream_ident, &topic_ident, Some(PARTITION_ID)), - client.get_consumer_offset( + let consumer_offset = client + .get_consumer_offset(&consumer, &stream_ident, &topic_ident, Some(PARTITION_ID)) + .await + .unwrap(); + let group_offset = client + .get_consumer_offset( &group_consumer_ref, &stream_ident, &topic_ident, Some(PARTITION_ID), - ), - ) - .await; - let (Ok(consumer_offset), Ok(group_offset)) = reads else { - assert!( - std::time::Instant::now() < offsets_deadline, - "consumer offset reads never succeeded after purge: {reads:?}" - ); - tokio::time::sleep(std::time::Duration::from_millis(25)).await; - continue; - }; + ) + .await + .unwrap(); let consumer_files: Vec<_> = read_dir(&consumers_dir) .map(|e| e.filter_map(|e| e.ok().map(|e| e.file_name())).collect()) .unwrap_or_default(); @@ -1031,7 +1043,7 @@ pub async fn run_purge_topic(harness: &mut TestHarness, restart_server: bool) { } // --- Verify partition reset: single empty segment at offset 0 --- - assert_fresh_empty_partition(&partition_path).await; + assert_fresh_empty_partition(&partition_path); // --- Verify new messages start at offset 0 --- let new_msg_count = 3u32; @@ -1078,12 +1090,15 @@ pub async fn run_purge_topic(harness: &mut TestHarness, restart_server: bool) { /// post-purge data. Without that file a restart re-reads applied=0 against /// the replayed committed generation and silently wipes the new messages on /// its first pass. +#[cfg(feature = "vsr")] pub async fn run_purge_survives_restart(harness: &mut TestHarness) { let client = build_root_client(harness); client.connect().await.unwrap(); - client.create_stream(STREAM_NAME).await.unwrap(); + let data_path = harness.server().data_path().to_path_buf(); + + let stream = client.create_stream(STREAM_NAME).await.unwrap(); let stream_ident = Identifier::named(STREAM_NAME).unwrap(); - client + let topic = client .create_topic( &stream_ident, TOPIC_NAME, @@ -1096,21 +1111,14 @@ pub async fn run_purge_survives_restart(harness: &mut TestHarness) { .await .unwrap(); let topic_ident = Identifier::named(TOPIC_NAME).unwrap(); + let partition_path = partition_path(&data_path, stream.id, topic.id); send_messages(&client, &stream_ident, &topic_ident, 10).await; client .purge_topic(&stream_ident, &topic_ident) .await .unwrap(); - // The poll going empty is the barrier, not the segment layout: 10 messages - // never rotate the default 1.07 GB segment, so the directory holds one - // `0.log` before AND after the purge and a layout gate on `[0]` passes on - // its first read, before the purge has applied. `purge_topic` returns on - // the metadata commit while the reconciler stages the reset and the pump - // applies it, so an unsynchronized send races that window and is either - // wiped or fenced below the purge floor -- silently, since the offset it - // was acked at never becomes visible. - poll_exactly(&client, &stream_ident, &topic_ident, 0).await; + await_segment_layout(&partition_path, &[0]).await; send_messages(&client, &stream_ident, &topic_ident, 3).await; poll_exactly(&client, &stream_ident, &topic_ident, 3).await; @@ -1135,6 +1143,7 @@ pub async fn run_purge_survives_restart(harness: &mut TestHarness) { /// in-memory journal as consensus history, and the graceful-shutdown flush /// walks them again. The purge floor must fence them out of the segment so /// the restart recovers only the post-purge appends. +#[cfg(feature = "vsr")] pub async fn run_resident_purge_no_resurface(harness: &mut TestHarness) { let client = build_root_client(harness); client.connect().await.unwrap(); @@ -1190,6 +1199,7 @@ pub async fn run_resident_purge_no_resurface(harness: &mut TestHarness) { /// messages are served, so an extra resurfaced message fails the count /// instead of being cropped by the poll size. Panics after /// [`POLL_CONVERGENCE_TIMEOUT`] with the last observed count. +#[cfg(feature = "vsr")] async fn poll_exactly( client: &IggyClient, stream_ident: &Identifier, @@ -1250,9 +1260,11 @@ async fn await_stored_offset( /// Wait for the partition's on-disk segment layout to converge to `expected`. /// -/// `DeleteSegments` is eventually-consistent: the client call returns after -/// the metadata `TruncatePartition` commit, and the partition reconciler -/// performs the on-disk deletion on its next pass. +/// server-ng's `DeleteSegments` is eventually-consistent: the client call +/// returns after the metadata `TruncatePartition` commit, and the partition +/// reconciler performs the on-disk deletion on its next pass. Legacy deletes +/// synchronously, so it asserts immediately. +#[cfg(feature = "vsr")] async fn await_segment_layout(partition_path: &str, expected: &[u64]) { for _ in 0..200 { if get_sorted_segment_offsets(partition_path).as_slice() == expected { @@ -1269,9 +1281,11 @@ async fn await_segment_layout(partition_path: &str, expected: &[u64]) { /// Assert the layout stays at `expected` when no deletion must happen. /// -/// Sleeps past a reconciler pass first, since an erroneous deletion would -/// land asynchronously. +/// The vsr side sleeps past a reconciler pass first, since an erroneous +/// deletion would land asynchronously; legacy deletes synchronously, so an +/// immediate assert suffices. async fn assert_layout_stable(partition_path: &str, expected: &[u64]) { + #[cfg(feature = "vsr")] tokio::time::sleep(std::time::Duration::from_millis(1500)).await; assert_eq!( get_sorted_segment_offsets(partition_path).as_slice(), @@ -1280,6 +1294,14 @@ async fn assert_layout_stable(partition_path: &str, expected: &[u64]) { ); } +#[cfg(not(feature = "vsr"))] +async fn await_segment_layout(partition_path: &str, expected: &[u64]) { + assert_eq!( + get_sorted_segment_offsets(partition_path).as_slice(), + expected + ); +} + async fn maybe_restart(harness: &mut TestHarness, restart_server: bool) { if restart_server { harness.restart_server().await.unwrap(); @@ -1378,10 +1400,7 @@ async fn poll_all_offsets( kind: ConsumerKind::Consumer, id: Identifier::numeric(99).unwrap(), }; - // An errored poll reads as "nothing yet" so the caller's retry loop keeps - // going: the restart cells reconnect mid-scenario and the first poll after - // the server returns can answer `Unauthenticated` while the SDK re-signs in. - client + let polled = client .poll_messages( stream_ident, topic_ident, @@ -1392,8 +1411,8 @@ async fn poll_all_offsets( false, ) .await - .map(|polled| polled.messages.iter().map(|m| m.header.offset).collect()) - .unwrap_or_default() + .unwrap(); + polled.messages.iter().map(|m| m.header.offset).collect() } /// Asserts that each segment's `.log` and `.index` files have the exact expected size. @@ -1474,12 +1493,12 @@ fn is_dir_empty(dir: &str) -> bool { /// Asserts the partition directory contains exactly one .log and one .index file at offset 0, /// both with size 0 — the expected state after a full purge or segment reset. -/// -/// Awaits the layout rather than reading once: a state-transfer install unlinks -/// the old chain before planting the replacement, so a replica that learns the -/// purge that way exposes a window with no `.log` at all. -async fn assert_fresh_empty_partition(partition_path: &str) { - await_segment_layout(partition_path, &[0]).await; +fn assert_fresh_empty_partition(partition_path: &str) { + assert_eq!( + get_sorted_segment_offsets(partition_path), + [0], + "Partition must contain a single segment at offset 0" + ); assert_eq!( count_files_with_ext(partition_path, INDEX_EXTENSION), 1, @@ -1505,7 +1524,7 @@ async fn assert_fresh_empty_partition(partition_path: &str) { /// /// `get_sorted_segment_offsets` only checks .log files -- this additionally /// verifies that the .index file count matches, catching stale .index files -/// left behind. The server unlinks a segment's .log and .index files across +/// left behind. server-ng unlinks a segment's .log and .index files across /// separate awaits, so a layout that already converged on .log files can /// transiently show one extra .index file. async fn assert_no_orphaned_segment_files(partition_path: &str, expected_count: usize) { diff --git a/core/integration/tests/server/scenarios/reconnect_after_restart_scenario.rs b/core/integration/tests/server/scenarios/reconnect_after_restart_scenario.rs index 32b4d0cd02..bc2da26a73 100644 --- a/core/integration/tests/server/scenarios/reconnect_after_restart_scenario.rs +++ b/core/integration/tests/server/scenarios/reconnect_after_restart_scenario.rs @@ -627,7 +627,7 @@ pub async fn run_ring_overflow_rejoin(harness: &mut TestHarness) { // count the overflow stops happening and the test silently passes without // covering the floor path. Fail loud instead. const _: () = assert!( - RING_OVERFLOW_OPS as usize > configs::partition::DEFAULT_EVICTED_RING_CAPACITY, + RING_OVERFLOW_OPS as usize > configs::ng_partition::DEFAULT_EVICTED_RING_CAPACITY, "RING_OVERFLOW_OPS must exceed the default evicted ring capacity, or this scenario no longer exercises RangeEvicted", ); diff --git a/core/integration/tests/server/scenarios/restart_offset_skip_scenario.rs b/core/integration/tests/server/scenarios/restart_offset_skip_scenario.rs index 03bf5774f2..31378d0b0f 100644 --- a/core/integration/tests/server/scenarios/restart_offset_skip_scenario.rs +++ b/core/integration/tests/server/scenarios/restart_offset_skip_scenario.rs @@ -95,9 +95,17 @@ pub async fn run(harness: &mut TestHarness) { .await .unwrap(); - // No flush primitive exists (FLUSH_UNSAVED_BUFFER denies typed); graceful - // shutdown flushes the committed journal, which this scenario's restart - // exercises instead. + // Explicitly flush to disk, then wait for message_saver to persist. + // server-ng has no flush primitive (FLUSH_UNSAVED_BUFFER denies typed); + // its graceful shutdown flushes the committed journal, which this + // scenario's restart exercises instead. + #[cfg(not(feature = "vsr"))] + setup_client + .flush_unsaved_buffer( + &stream_id, &topic_id, 0, true, // fsync + ) + .await + .unwrap(); tokio::time::sleep(Duration::from_secs(2)).await; drop(setup_client); diff --git a/core/integration/tests/server/scenarios/stream_size_validation_scenario.rs b/core/integration/tests/server/scenarios/stream_size_validation_scenario.rs index fb16f39918..ffa38a84a7 100644 --- a/core/integration/tests/server/scenarios/stream_size_validation_scenario.rs +++ b/core/integration/tests/server/scenarios/stream_size_validation_scenario.rs @@ -36,14 +36,21 @@ const T1_NAME: &str = "test-topic-1"; const S2_NAME: &str = "test-stream-2"; const T2_NAME: &str = "test-topic-2"; const MESSAGE_PAYLOAD_SIZE_BYTES: u64 = 57; +#[cfg(not(feature = "vsr"))] +const MSG_SIZE: u64 = IGGY_MESSAGE_HEADER_SIZE as u64 + MESSAGE_PAYLOAD_SIZE_BYTES; // number of bytes in a single message const MSGS_COUNT: u64 = 117; // number of messages in a single topic after one pass of appending -// The server accounts the actual on-disk batch framing: one 256-byte +#[cfg(not(feature = "vsr"))] +const MSGS_SIZE: u64 = MSG_SIZE * MSGS_COUNT; // number of bytes in a single topic after one pass of appending +// server-ng accounts the actual on-disk batch framing: one 256-byte // `SendMessages2` command header per append pass plus a 48-byte per-message // header (`server_common::send_messages2::{COMMAND_HEADER_SIZE, -// MESSAGE_HEADER_SIZE}`). Each pass below sends all `MSGS_COUNT` messages in -// one batch. +// MESSAGE_HEADER_SIZE}`), instead of the legacy 64-byte per-message header. +// Each pass below sends all `MSGS_COUNT` messages in one batch. +#[cfg(feature = "vsr")] const NG_BATCH_HEADER_SIZE: u64 = 256; +#[cfg(feature = "vsr")] const NG_MESSAGE_HEADER_SIZE: u64 = 48; +#[cfg(feature = "vsr")] const MSGS_SIZE: u64 = NG_BATCH_HEADER_SIZE + (NG_MESSAGE_HEADER_SIZE + MESSAGE_PAYLOAD_SIZE_BYTES) * MSGS_COUNT; diff --git a/core/integration/tests/server/scenarios/system_scenario.rs b/core/integration/tests/server/scenarios/system_scenario.rs index 5e1b81ba4d..2e11ba4991 100644 --- a/core/integration/tests/server/scenarios/system_scenario.rs +++ b/core/integration/tests/server/scenarios/system_scenario.rs @@ -289,8 +289,11 @@ pub async fn run(harness: &TestHarness) { assert_eq!(topic.name, TOPIC_NAME); assert_eq!(topic.partitions_count, PARTITIONS_COUNT); assert_eq!(topic.partitions.len(), PARTITIONS_COUNT as usize); - // The exact byte size tracks the on-disk batch framing, so only its - // presence is asserted here. + // The exact byte size is framing-specific: legacy counts a 64-byte header + // per message; server-ng counts its on-disk batch framing. + #[cfg(not(feature = "vsr"))] + assert_eq!(topic.size, 100502); + #[cfg(feature = "vsr")] assert!(topic.size > 0); assert_eq!(topic.messages_count, MESSAGES_COUNT as u64); let topic_partition = topic.partitions.get((PARTITION_ID) as usize).unwrap(); diff --git a/core/integration/tests/server/specific.rs b/core/integration/tests/server/specific.rs index d0508a4e9d..0c455d045e 100644 --- a/core/integration/tests/server/specific.rs +++ b/core/integration/tests/server/specific.rs @@ -72,12 +72,21 @@ async fn producer_reconnect_after_server_restart(harness: &mut TestHarness) { reconnect_after_restart_scenario::run_producer(harness).await; } -// QUIC is excluded on an SDK gap: after the restart the QUIC client redirects -// to the new leader, reconnects, and signs in, but the long-lived consumer's -// polls then return nothing for the whole window -- the post-reconnect request -// path wedges (QUIC also lacks the TCP client's mid-connection failover). TCP -// and WebSocket run. -#[iggy_harness( +// QUIC stays vsr-gated on an SDK gap: after the restart the QUIC client +// redirects to the new leader, reconnects, and signs in, but the long-lived +// consumer's polls then return nothing for the whole window -- the +// post-reconnect request path wedges (QUIC also lacks the TCP client's +// mid-connection failover). TCP and WebSocket run. +#[cfg_attr(not(feature = "vsr"), iggy_harness( + test_client_transport = [Tcp, WebSocket, Quic], + server( + tcp.socket.override_defaults = true, + tcp.socket.nodelay = true, + quic.max_idle_timeout = "500s", + quic.keep_alive_interval = "15s" + ) +))] +#[cfg_attr(feature = "vsr", iggy_harness( test_client_transport = [Tcp, WebSocket], server( tcp.socket.override_defaults = true, @@ -85,7 +94,7 @@ async fn producer_reconnect_after_server_restart(harness: &mut TestHarness) { quic.max_idle_timeout = "500s", quic.keep_alive_interval = "15s" ) -)] +))] async fn consumer_reconnect_after_server_restart(harness: &mut TestHarness) { reconnect_after_restart_scenario::run_consumer(harness).await; } @@ -98,8 +107,10 @@ async fn single_message_restart_offset_zero(harness: &mut TestHarness) { reconnect_after_restart_scenario::run_single_message_offset_zero_restart(harness).await; } -// Exercises the rejoin probe's election fallback across all replicas, which a -// plain single-node restart does not reach. +// Full-cluster restart is vsr-only by construction: it exercises the rejoin +// probe's election fallback across all replicas, which a single-process +// legacy server has no equivalent of (plain restart covers it there). +#[cfg(feature = "vsr")] #[iggy_harness(server( partition.messages_required_to_save = "1", partition.enforce_fsync = true @@ -108,8 +119,10 @@ async fn full_cluster_restart_recovers_and_serves(harness: &mut TestHarness) { reconnect_after_restart_scenario::run_full_cluster_restart(harness).await; } -// Exercises `RangeEvicted` + the commit floor: the rejoin window exceeds the -// peers' evicted ring, so journal repair alone cannot cover it. +// vsr-only: exercises `RangeEvicted` + the commit floor, which only exist +// on the replicated plane (the rejoin window exceeds the peers' evicted +// ring, so journal repair alone cannot cover it). +#[cfg(feature = "vsr")] #[iggy_harness(server( partition.messages_required_to_save = "1", partition.enforce_fsync = true @@ -158,8 +171,8 @@ async fn restart_offset_skip(harness: &mut TestHarness) { /// Test configuration: /// - 8 producers total (2 per protocol: TCP, HTTP, QUIC, WebSocket) /// - All producers write to the same partition for maximum lock contention -// Concurrency race test: runs over the three VSR transports (TCP/QUIC/ -// WebSocket -- HTTP/REST carries no VSR framing). +// Concurrency race test: under vsr it runs over the three VSR transports +// (TCP/QUIC/WebSocket -- HTTP/REST carries no VSR framing), legacy runs all four. #[iggy_harness(server( segment.size = "512B", message_saver.interval = "1s", diff --git a/core/integration/tests/server/stats_vsr.rs b/core/integration/tests/server/stats_vsr.rs deleted file mode 100644 index beaa97831f..0000000000 --- a/core/integration/tests/server/stats_vsr.rs +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Stats against the server (vsr): `clients_count` must report the cross-shard -//! connected-client total gathered by the `ListClients` broadcast, not the -//! hardcoded 0 the sync single-shard read used to answer. - -use iggy::prelude::*; -use integration::iggy_harness; - -#[iggy_harness( - test_client_transport = [Tcp], - server(tcp.socket.override_defaults = true, tcp.socket.nodelay = true) -)] -async fn given_connected_clients_when_getting_stats_should_count_clients(harness: &TestHarness) { - let clients = harness.tcp_root_clients(2).await.expect("tcp root clients"); - - let stats = clients[0].get_stats().await.expect("get stats"); - - assert_eq!( - stats.clients_count, 2, - "stats must count both connected clients, got {}", - stats.clients_count - ); -} diff --git a/core/integration/tests/server/topic_admission_vsr.rs b/core/integration/tests/server/topic_admission_vsr.rs index dac97dd969..8f7c7da157 100644 --- a/core/integration/tests/server/topic_admission_vsr.rs +++ b/core/integration/tests/server/topic_admission_vsr.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -//! Topic admission and echo semantics against the server (vsr). Create bounds +//! Topic admission and echo semantics against server-ng (vsr). Create bounds //! must be rejected with typed errors before consensus: partitions count above //! `MAX_PARTITIONS_PER_REQUEST` denies with `TooManyPartitions` (for create //! topic, create partitions and delete partitions alike); a custom @@ -23,10 +23,8 @@ //! `InvalidTopicSize`; `ServerDefault` and `Unlimited` sizes pass. Update //! stores `max_topic_size` and `message_expiry` verbatim and gets echo the //! stored value (never the node default frozen at update time), matching -//! legacy wire behavior. Deleting more partitions than the topic has rejects -//! with `InvalidPartitionsCount` as a committed result instead of silently -//! acking a no-op. Listing topics of a missing stream replies with an empty -//! list, as the legacy server does. +//! legacy wire behavior. Listing topics of a missing stream replies with an +//! empty list, as the legacy server does. use std::str::FromStr; @@ -256,69 +254,6 @@ async fn given_out_of_bounds_partitions_count_when_mutating_should_reject_typed( ); } -#[iggy_harness( - test_client_transport = [Tcp], - server(tcp.socket.override_defaults = true, tcp.socket.nodelay = true) -)] -async fn given_over_count_when_deleting_partitions_should_reject_invalid_partitions_count( - harness: &TestHarness, -) { - let client = harness.tcp_root_client().await.expect("tcp root client"); - client - .create_stream("over-count-stream") - .await - .expect("create stream"); - let stream_id = Identifier::from_str_value("over-count-stream").expect("stream identifier"); - create_topic_with( - &client, - &stream_id, - "over-count-topic", - 3, - MaxTopicSize::ServerDefault, - ) - .await - .expect("create topic"); - let topic_id = Identifier::from_str_value("over-count-topic").expect("topic identifier"); - - // Deleting more partitions than the topic has must reject with the legacy - // typed error, not silently no-op and ack. - let invalid_count = IggyError::InvalidPartitionsCount.as_code(); - let result = client.delete_partitions(&stream_id, &topic_id, 4).await; - assert!( - matches!(&result, Err(error) if error.as_code() == invalid_count), - "deleting 4 partitions of a 3-partition topic must deny with \ - InvalidPartitionsCount, got {result:?}" - ); - let topic = client - .get_topic(&stream_id, &topic_id) - .await - .expect("get topic") - .expect("topic exists"); - assert_eq!( - topic.partitions_count, 3, - "the rejected over-count delete must not remove any partition" - ); - - client - .delete_partitions(&stream_id, &topic_id, 3) - .await - .expect("deleting exactly the topic's partition count is accepted"); - let topic = client - .get_topic(&stream_id, &topic_id) - .await - .expect("get topic") - .expect("topic exists"); - assert_eq!(topic.partitions_count, 0, "all partitions are gone"); - - // Same rejection once the topic is already empty (any count exceeds 0). - let result = client.delete_partitions(&stream_id, &topic_id, 1).await; - assert!( - matches!(&result, Err(error) if error.as_code() == invalid_count), - "deleting from a zero-partition topic must deny with \ - InvalidPartitionsCount, got {result:?}" - ); -} - #[iggy_harness( test_client_transport = [Tcp], server(tcp.socket.override_defaults = true, tcp.socket.nodelay = true) diff --git a/core/integration/tests/state/file.rs b/core/integration/tests/state/file.rs new file mode 100644 index 0000000000..2e604625c5 --- /dev/null +++ b/core/integration/tests/state/file.rs @@ -0,0 +1,164 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::state::StateSetup; +use bytes::Bytes; +use iggy_binary_protocol::WireEncode; +use iggy_binary_protocol::WireName; +use iggy_binary_protocol::requests::{streams::CreateStreamRequest, users::CreateUserRequest}; +use server::state::command::EntryCommand; +use server::state::entry::StateEntry; +use server::state::models::{CreateStreamWithId, CreateUserWithId}; + +#[compio::test] +async fn should_be_empty_given_initialized_state() { + let setup = StateSetup::init().await; + let state = setup.state(); + state.init().await.unwrap(); + let entries = state.load_entries().await.unwrap(); + assert!(entries.is_empty()); +} + +#[compio::test] +async fn should_apply_single_entry() { + let setup = StateSetup::init().await; + let state = setup.state(); + state.init().await.unwrap(); + + let user_id = 0; + let command = EntryCommand::CreateUser(CreateUserWithId { + user_id: 1, + command: CreateUserRequest { + username: WireName::new("test").unwrap(), + password: "secret".to_string(), + status: 1, + permissions: None, + }, + }); + let command_bytes = command.to_bytes(); + + state.apply(user_id, &command).await.unwrap(); + + let mut entries = state.load_entries().await.unwrap(); + assert_eq!(entries.len(), 1); + let entry = entries.remove(0); + assert_entry(entry, 0, setup.version(), user_id, command_bytes); +} + +#[compio::test] +async fn should_apply_encrypted_entry() { + let setup = StateSetup::init_with_encryptor().await; + let state = setup.state(); + state.init().await.unwrap(); + + let user_id = 0; + let command = EntryCommand::CreateUser(CreateUserWithId { + user_id: 1, + command: CreateUserRequest { + username: WireName::new("test").unwrap(), + password: "secret".to_string(), + status: 1, + permissions: None, + }, + }); + let command_bytes = command.to_bytes(); + + state.apply(user_id, &command).await.unwrap(); + + let mut entries = state.load_entries().await.unwrap(); + assert_eq!(entries.len(), 1); + let entry = entries.remove(0); + assert_entry(entry, 0, setup.version(), user_id, command_bytes); +} + +#[compio::test] +async fn should_apply_multiple_entries() { + let setup = StateSetup::init().await; + let state = setup.state(); + let entries = state.init().await.unwrap(); + + assert!(entries.is_empty()); + assert_eq!(state.current_index(), 0); + assert_eq!(state.entries_count(), 0); + assert_eq!(state.term(), 0); + + let first_user_id = 0; // Root user + let created_user_id = 1; // First created user + let create_user = EntryCommand::CreateUser(CreateUserWithId { + user_id: created_user_id, + command: CreateUserRequest { + username: WireName::new("test").unwrap(), + password: "secret".to_string(), + status: 1, + permissions: None, + }, + }); + let create_user_bytes = create_user.to_bytes(); + + state.apply(first_user_id, &create_user).await.unwrap(); + + assert_eq!(state.current_index(), 0); + assert_eq!(state.entries_count(), 1); + + let second_user_id = 1; + let stream_id = 1; + let create_stream = EntryCommand::CreateStream(CreateStreamWithId { + stream_id, + command: CreateStreamRequest { + name: WireName::new("test").unwrap(), + }, + }); + let create_stream_bytes = create_stream.to_bytes(); + + state.apply(second_user_id, &create_stream).await.unwrap(); + + assert_eq!(state.current_index(), 1); + assert_eq!(state.entries_count(), 2); + + let mut entries = state.load_entries().await.unwrap(); + assert_eq!(entries.len(), 2); + + let create_user_entry = entries.remove(0); + assert_entry( + create_user_entry, + 0, + setup.version(), + first_user_id, + create_user_bytes, + ); + + let create_stream_entry = entries.remove(0); + assert_entry( + create_stream_entry, + 1, + setup.version(), + second_user_id, + create_stream_bytes, + ); +} + +fn assert_entry(entry: StateEntry, index: u64, version: u32, user_id: u32, command: Bytes) { + assert_eq!(entry.index, index); + assert_eq!(entry.term, 0); + assert_eq!(entry.version, version); + assert_eq!(entry.flags, 0); + assert!(entry.checksum > 0); + assert!(entry.timestamp.as_micros() > 0); + assert_eq!(entry.user_id, user_id); + assert_eq!(entry.command, command); + assert!(entry.context.is_empty()); +} diff --git a/core/integration/tests/state/mod.rs b/core/integration/tests/state/mod.rs new file mode 100644 index 0000000000..b6623bcba3 --- /dev/null +++ b/core/integration/tests/state/mod.rs @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use compio::fs::create_dir; +use iggy::prelude::{Aes256GcmEncryptor, EncryptorKind}; +use iggy_common::SemanticVersion; +use server::state::file::FileState; +use server::streaming::persistence::persister::{FileWithSyncPersister, PersisterKind}; +use server::streaming::utils::file::overwrite; +use std::str::FromStr; +use std::sync::Arc; +use std::sync::atomic::{AtomicU32, AtomicU64}; +use uuid::Uuid; + +mod file; +mod system; + +pub struct StateSetup { + directory_path: String, + state: FileState, + version: u32, +} + +impl StateSetup { + pub async fn init() -> StateSetup { + StateSetup::create(None).await + } + + pub async fn init_with_encryptor() -> StateSetup { + StateSetup::create(Some(&[1; 32])).await + } + + pub async fn create(encryption_key: Option<&[u8]>) -> StateSetup { + let directory_path = format!("state_{}", Uuid::now_v7().to_u128_le()); + let messages_file_path = format!("{directory_path}/log"); + create_dir(&directory_path).await.unwrap(); + overwrite(&messages_file_path).await.unwrap(); + + let version = SemanticVersion::from_str("1.2.3").unwrap(); + let persister = PersisterKind::FileWithSync(FileWithSyncPersister {}); + let encryptor = encryption_key + .map(|key| EncryptorKind::Aes256Gcm(Aes256GcmEncryptor::new(key).unwrap())); + let state_current_index = Arc::new(AtomicU64::new(0)); + let state_entries_count = Arc::new(AtomicU64::new(0)); + let state_current_leader = Arc::new(AtomicU32::new(0)); + let state_term = Arc::new(AtomicU64::new(0)); + let state = FileState::new( + &messages_file_path, + &version, + Arc::new(persister), + encryptor, + state_current_index, + state_entries_count, + state_current_leader, + state_term, + ); + + Self { + directory_path, + state, + version: version.get_numeric_version().unwrap(), + } + } + + pub fn state(&self) -> &FileState { + &self.state + } + + pub fn version(&self) -> u32 { + self.version + } +} + +impl Drop for StateSetup { + fn drop(&mut self) { + std::fs::remove_dir_all(&self.directory_path).unwrap(); + } +} diff --git a/core/integration/tests/state/system.rs b/core/integration/tests/state/system.rs new file mode 100644 index 0000000000..f6491fcd25 --- /dev/null +++ b/core/integration/tests/state/system.rs @@ -0,0 +1,212 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::state::StateSetup; +use iggy_binary_protocol::requests::{ + consumer_groups::CreateConsumerGroupRequest, partitions::CreatePartitionsRequest, + personal_access_tokens::CreatePersonalAccessTokenRequest, streams::CreateStreamRequest, + streams::DeleteStreamRequest, topics::CreateTopicRequest, users::CreateUserRequest, +}; +use iggy_binary_protocol::{WireIdentifier, WireName}; +use server::state::command::EntryCommand; +use server::state::models::{ + CreateConsumerGroupWithId, CreatePersonalAccessTokenWithHash, CreateStreamWithId, + CreateTopicWithId, CreateUserWithId, +}; +use server::state::system::SystemState; + +#[compio::test] +async fn should_be_initialized_based_on_state_entries() { + let setup = StateSetup::init().await; + let state = setup.state(); + state.init().await.unwrap(); + + let user_id = 0; + let create_user = CreateUserRequest { + username: WireName::new("user").unwrap(), + password: "secret".to_string(), + status: 1, // Active + permissions: None, + }; + + let stream1_id = 1u32; + let create_stream1 = CreateStreamRequest { + name: WireName::new("stream1").unwrap(), + }; + + let topic1_id = 1u32; + let create_topic1 = CreateTopicRequest { + stream_id: WireIdentifier::numeric(stream1_id), + partitions_count: 1, + compression_algorithm: 1, // None compression + message_expiry: 0, // NeverExpire + max_topic_size: 0, // ServerDefault + replication_factor: 0, // None + name: WireName::new("topic1").unwrap(), + }; + + let stream2_id = 2u32; + let create_stream2 = CreateStreamRequest { + name: WireName::new("stream2").unwrap(), + }; + + let topic2_id = 2u32; + let create_topic2 = CreateTopicRequest { + stream_id: WireIdentifier::numeric(stream2_id), + partitions_count: 1, + compression_algorithm: 1, + message_expiry: 0, + max_topic_size: 0, + replication_factor: 0, + name: WireName::new("topic2").unwrap(), + }; + + let create_partitions = CreatePartitionsRequest { + stream_id: WireIdentifier::numeric(stream1_id), + topic_id: WireIdentifier::numeric(topic1_id), + partitions_count: 2, + }; + + let delete_stream = DeleteStreamRequest { + stream_id: WireIdentifier::numeric(stream2_id), + }; + + let create_personal_access_token = CreatePersonalAccessTokenWithHash { + command: CreatePersonalAccessTokenRequest { + name: WireName::new("test").unwrap(), + expiry: 0, // NeverExpire + }, + hash: "hash".to_string(), + }; + + let create_consumer_group = CreateConsumerGroupRequest { + stream_id: WireIdentifier::numeric(stream1_id), + topic_id: WireIdentifier::numeric(topic1_id), + name: WireName::new("test").unwrap(), + }; + + let group_id = 1u32; + + state + .apply( + user_id, + &EntryCommand::CreateUser(CreateUserWithId { + user_id, + command: create_user, + }), + ) + .await + .unwrap(); + state + .apply( + user_id, + &EntryCommand::CreateStream(CreateStreamWithId { + stream_id: stream1_id, + command: create_stream1, + }), + ) + .await + .unwrap(); + state + .apply( + user_id, + &EntryCommand::CreateTopic(CreateTopicWithId { + topic_id: topic1_id, + command: create_topic1, + }), + ) + .await + .unwrap(); + state + .apply( + user_id, + &EntryCommand::CreateStream(CreateStreamWithId { + stream_id: stream2_id, + command: create_stream2, + }), + ) + .await + .unwrap(); + state + .apply( + user_id, + &EntryCommand::CreateTopic(CreateTopicWithId { + topic_id: topic2_id, + command: create_topic2, + }), + ) + .await + .unwrap(); + state + .apply(user_id, &EntryCommand::CreatePartitions(create_partitions)) + .await + .unwrap(); + state + .apply(user_id, &EntryCommand::DeleteStream(delete_stream)) + .await + .unwrap(); + state + .apply( + user_id, + &EntryCommand::CreatePersonalAccessToken(create_personal_access_token), + ) + .await + .unwrap(); + state + .apply( + user_id, + &EntryCommand::CreateConsumerGroup(CreateConsumerGroupWithId { + group_id, + command: create_consumer_group, + }), + ) + .await + .unwrap(); + + let entries = state.load_entries().await.unwrap(); + assert_eq!(entries.len(), 9); + + let mut system = SystemState::init(entries).await.unwrap(); + + assert_eq!(system.users.len(), 1); + let mut user = system.users.remove(&user_id).unwrap(); + assert_eq!(user.id, user_id); + assert_eq!(user.username, "user"); + assert_eq!(user.password_hash, "secret"); + assert_eq!(user.personal_access_tokens.len(), 1); + + let personal_access_token = user.personal_access_tokens.remove("test").unwrap(); + assert_eq!(personal_access_token.token_hash, "hash"); + assert_eq!(personal_access_token.name, "test"); + + assert_eq!(system.streams.len(), 1); + let mut stream = system.streams.remove(&stream1_id).unwrap(); + assert_eq!(stream.id, stream1_id); + assert_eq!(stream.name, "stream1"); + assert_eq!(stream.topics.len(), 1); + + let mut topic = stream.topics.remove(&topic1_id).unwrap(); + assert_eq!(topic.id, topic1_id); + assert_eq!(topic.name, "topic1"); + assert_eq!(topic.partitions.len(), 3); + + assert_eq!(topic.consumer_groups.len(), 1); + let consumer_group = topic.consumer_groups.remove(&group_id).unwrap(); + + assert_eq!(consumer_group.id, group_id); + assert_eq!(consumer_group.name, "test"); +} diff --git a/core/integration/tests/storage/consumer_offsets.rs b/core/integration/tests/storage/consumer_offsets.rs new file mode 100644 index 0000000000..f0e36f2233 --- /dev/null +++ b/core/integration/tests/storage/consumer_offsets.rs @@ -0,0 +1,179 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use iggy_common::{ConsumerKind, IggyError}; +use server::streaming::partitions::storage::{load_consumer_group_offsets, load_consumer_offsets}; +use std::path::Path; +use std::sync::atomic::Ordering; + +fn write_offset_file(dir: &Path, name: &str, offset: u64) { + std::fs::write(dir.join(name), offset.to_le_bytes()).unwrap(); +} + +#[test] +fn load_consumer_offsets_valid_files() { + let dir = tempfile::tempdir().unwrap(); + write_offset_file(dir.path(), "1", 100); + write_offset_file(dir.path(), "2", 200); + write_offset_file(dir.path(), "3", 300); + + let offsets = load_consumer_offsets(dir.path().to_str().unwrap()).unwrap(); + + assert_eq!(offsets.len(), 3); + assert_eq!(offsets[0].consumer_id, 1); + assert_eq!(offsets[0].offset.load(Ordering::Relaxed), 100); + assert_eq!(offsets[0].kind, ConsumerKind::Consumer); + assert_eq!(offsets[1].consumer_id, 2); + assert_eq!(offsets[1].offset.load(Ordering::Relaxed), 200); + assert_eq!(offsets[2].consumer_id, 3); + assert_eq!(offsets[2].offset.load(Ordering::Relaxed), 300); +} + +#[test] +fn load_consumer_offsets_skips_non_numeric_files() { + let dir = tempfile::tempdir().unwrap(); + write_offset_file(dir.path(), ".DS_Store", 0); + write_offset_file(dir.path(), "backup.bak", 0); + write_offset_file(dir.path(), "1", 42); + + let offsets = load_consumer_offsets(dir.path().to_str().unwrap()).unwrap(); + + assert_eq!(offsets.len(), 1); + assert_eq!(offsets[0].consumer_id, 1); + assert_eq!(offsets[0].offset.load(Ordering::Relaxed), 42); +} + +#[test] +fn load_consumer_offsets_skips_truncated_files() { + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("1"), [0u8; 3]).unwrap(); + std::fs::write(dir.path().join("2"), []).unwrap(); + write_offset_file(dir.path(), "3", 500); + + let offsets = load_consumer_offsets(dir.path().to_str().unwrap()).unwrap(); + + assert_eq!(offsets.len(), 1); + assert_eq!(offsets[0].consumer_id, 3); + assert_eq!(offsets[0].offset.load(Ordering::Relaxed), 500); +} + +#[test] +fn load_consumer_offsets_empty_dir() { + let dir = tempfile::tempdir().unwrap(); + + let offsets = load_consumer_offsets(dir.path().to_str().unwrap()).unwrap(); + + assert!(offsets.is_empty()); +} + +#[test] +fn load_consumer_offsets_skips_directories() { + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir(dir.path().join("123")).unwrap(); + write_offset_file(dir.path(), "1", 77); + + let offsets = load_consumer_offsets(dir.path().to_str().unwrap()).unwrap(); + + assert_eq!(offsets.len(), 1); + assert_eq!(offsets[0].consumer_id, 1); + assert_eq!(offsets[0].offset.load(Ordering::Relaxed), 77); +} + +#[test] +fn load_consumer_offsets_nonexistent_dir() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().to_str().unwrap().to_string(); + drop(dir); + + let result = load_consumer_offsets(&path); + + assert!(result.is_err()); + assert!(matches!( + result.unwrap_err(), + IggyError::CannotReadConsumerOffsets(_) + )); +} + +#[test] +fn load_consumer_group_offsets_valid_files() { + let dir = tempfile::tempdir().unwrap(); + write_offset_file(dir.path(), "1", 500); + write_offset_file(dir.path(), "2", 600); + + let offsets = load_consumer_group_offsets(dir.path().to_str().unwrap()).unwrap(); + + assert_eq!(offsets.len(), 2); + for (group_id, offset) in &offsets { + assert_eq!(offset.kind, ConsumerKind::ConsumerGroup); + assert_eq!(offset.consumer_id, group_id.0 as u32); + } + let ids: Vec = offsets.iter().map(|(_, co)| co.consumer_id).collect(); + assert!(ids.contains(&1)); + assert!(ids.contains(&2)); +} + +#[test] +fn load_consumer_group_offsets_skips_non_numeric_files() { + let dir = tempfile::tempdir().unwrap(); + write_offset_file(dir.path(), ".DS_Store", 0); + write_offset_file(dir.path(), "notes.txt", 0); + write_offset_file(dir.path(), "5", 999); + + let offsets = load_consumer_group_offsets(dir.path().to_str().unwrap()).unwrap(); + + assert_eq!(offsets.len(), 1); + assert_eq!(offsets[0].0.0, 5); + assert_eq!(offsets[0].1.consumer_id, 5); + assert_eq!(offsets[0].1.offset.load(Ordering::Relaxed), 999); +} + +#[test] +fn load_consumer_group_offsets_skips_truncated_files() { + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("1"), [0u8; 4]).unwrap(); + write_offset_file(dir.path(), "2", 750); + + let offsets = load_consumer_group_offsets(dir.path().to_str().unwrap()).unwrap(); + + assert_eq!(offsets.len(), 1); + assert_eq!(offsets[0].0.0, 2); + assert_eq!(offsets[0].1.offset.load(Ordering::Relaxed), 750); +} + +#[test] +fn load_consumer_group_offsets_empty_dir() { + let dir = tempfile::tempdir().unwrap(); + + let offsets = load_consumer_group_offsets(dir.path().to_str().unwrap()).unwrap(); + + assert!(offsets.is_empty()); +} + +#[test] +fn load_consumer_group_offsets_nonexistent_dir() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().to_str().unwrap().to_string(); + drop(dir); + + let result = load_consumer_group_offsets(&path); + + assert!(result.is_err()); + assert!(matches!( + result.unwrap_err(), + IggyError::CannotReadConsumerOffsets(_) + )); +} diff --git a/core/integration/tests/storage/mod.rs b/core/integration/tests/storage/mod.rs new file mode 100644 index 0000000000..1c05a5a3b0 --- /dev/null +++ b/core/integration/tests/storage/mod.rs @@ -0,0 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod consumer_offsets; diff --git a/core/journal/src/lib.rs b/core/journal/src/lib.rs index 1d223d7e6c..9de342b6c1 100644 --- a/core/journal/src/lib.rs +++ b/core/journal/src/lib.rs @@ -46,25 +46,6 @@ where None } - /// Remove every entry at or above `from_op`, returning how many went, and - /// leave the snapshot watermark where it is. - /// - /// Not `drain` with a different range: `drain` advances the watermark past what - /// it removed, which would mark the removed ops evictable when a suffix - /// truncation needs them refillable. - /// - /// Required, not defaulted: an `Unsupported` default hides a missing impl until - /// mid-view-change, where the caller can only wedge or start a view over a log it - /// cannot serve. - /// - /// # Errors - /// I/O error if the rewrite fails. - fn truncate_from(&self, from_op: u64) -> impl Future>; - - /// Highest op the index holds. Not derivable from [`Self::header`]: a caller - /// looking for a suffix ABOVE some op has no bound to probe up to. - fn last_op(&self) -> Option; - /// Remove entries with ops in `ops` from the journal, /// returning the removed entries sorted by op. /// diff --git a/core/journal/src/prepare_journal.rs b/core/journal/src/prepare_journal.rs index 9e665db836..3f4f332ccd 100644 --- a/core/journal/src/prepare_journal.rs +++ b/core/journal/src/prepare_journal.rs @@ -18,7 +18,7 @@ use crate::file_storage::FileStorage; use crate::{Journal, JournalHandle}; use compio::io::AsyncWriteAtExt; -use iggy_binary_protocol::consensus::{CHECKSUM_UNSEALED, Command2, PrepareHeader}; +use iggy_binary_protocol::consensus::{Command2, PrepareHeader}; use server_common::{MESSAGE_ALIGN, Message, iobuf::Owned}; use std::cell::{Cell, OnceCell, Ref, RefCell}; use std::fmt; @@ -66,7 +66,7 @@ pub(crate) const SLOT_COUNT: usize = 1024; /// Default in-memory index size, in slots. Overridable per journal via /// [`PrepareJournal::open_with_slots`] (operator knob: `[metadata] -/// journal_slots` in the server config). +/// journal_slots` in the server-ng config). pub const DEFAULT_SLOT_COUNT: usize = SLOT_COUNT; /// Error type for journal operations. @@ -391,7 +391,7 @@ impl PrepareJournal { /// `slot_count` bounds how many committed-but-unsnapshotted entries the /// journal holds before a forced checkpoint must reclaim WAL space; the /// caller owns keeping it above its checkpoint margin + prepare-queue - /// depth (validated at config load for the server `[metadata]` knob). + /// depth (validated at config load for the server-ng `[metadata]` knob). /// /// # Errors /// Returns `JournalError::Io` if the WAL file cannot be opened or read, @@ -412,7 +412,7 @@ impl PrepareJournal { Self::scan(storage, snapshot_op, slot_count).await } - #[allow(clippy::future_not_send, clippy::too_many_lines)] + #[allow(clippy::future_not_send)] async fn scan( storage: FileStorage, snapshot_op: u64, @@ -423,8 +423,6 @@ impl PrepareJournal { let mut offsets: Vec> = vec![None; slot_count]; let mut last_op: Option = None; let mut unsealed_entries: u64 = 0; - // Previous entry's `(op, checksum)`, for the parent-chain check. - let mut chain_previous: Option<(u64, u128)> = None; let mut pos: u64 = 0; let mut header_buf = vec![0u8; HEADER_SIZE]; // Reused 16-aligned scratch (PrepareHeader has u128 fields). Avoids @@ -479,57 +477,14 @@ impl PrepareJournal { // verify against and is skipped, not rejected: see // [`CHECKSUM_BODY_UNSEALED`]. // - // The header's own integrity field is checked first, since a flipped - // header field is the more dangerous of the two: recovery derives - // `commit_watermark = max(header.commit)`, so a corrupted `commit` applies - // uncommitted ops as committed, diverging from the group. `size` and `op` - // are equally load-bearing for the scan itself. - if header.checksum != CHECKSUM_UNSEALED && header.identity_checksum() != header.checksum - { - if pos + entry_size < file_len { - return Err(JournalError::Io(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "interior WAL corruption at pos {pos} (op {}, operation {:?}): \ - prepare header checksum mismatch with {} bytes of entries \ - following; refusing to truncate and discard the committed suffix", - header.op, - header.operation, - file_len - (pos + entry_size), - ), - ))); - } - truncate_or_fail(&storage, pos, "prepare header checksum mismatch at tail").await?; - break; - } - - // The hash chain, checked only where meaningful: consecutive ops with both - // ends sealed. A gap means compaction dropped the predecessor, and an - // unsealed end has nothing to chain from, so neither is evidence of damage. - if let Some((previous_op, previous_checksum)) = chain_previous - && previous_op + 1 == header.op - && previous_checksum != CHECKSUM_UNSEALED - && header.parent != previous_checksum - { - if pos + entry_size < file_len { - return Err(JournalError::Io(io::Error::new( - io::ErrorKind::InvalidData, - format!( - "interior WAL corruption at pos {pos}: op {} does not chain to op \ - {previous_op} (parent {} != {previous_checksum}) with {} bytes of \ - entries following; refusing to truncate and discard the committed \ - suffix", - header.op, - header.parent, - file_len - (pos + entry_size), - ), - ))); - } - truncate_or_fail(&storage, pos, "prepare parent chain break at tail").await?; - break; - } - chain_previous = Some((header.op, header.checksum)); - + // TODO(wal-integrity): the header `checksum` and its `parent` chain stay + // unverified, since the producer does not seal them yet (blocked on + // re-sealing re-stamped retransmits), so a bit-flip in a + // structurally-valid header field slips through. Recovery derives + // `commit_watermark = max(header.commit)`, so a flipped `commit` makes it + // apply prepared-but-uncommitted ops as committed, the very ops a view + // change may have truncated cluster-wide, diverging this replica. Seal + // and verify the header checksum + parent chain. if header.checksum_body == CHECKSUM_BODY_UNSEALED { // Skip the body read too, so a WAL written entirely by a pre-sealing // build scans without touching its payload. @@ -742,128 +697,6 @@ impl PrepareJournal { clippy::future_not_send )] impl Journal for PrepareJournal { - fn last_op(&self) -> Option { - self.last_op.get() - } - - /// Remove every entry at or above `from_op`, leaving the snapshot floor where - /// it is. Returns how many entries went. - /// - /// Deliberately not `drain`, which compacts a committed prefix and advances - /// `snapshot_op` past its range. Doing that to a suffix would declare everything - /// below the head snapshotted, letting `append` evict live entries repair cannot - /// put back, when those ops are exactly the ones that must stay refillable. - /// - /// For the one caller that needs it: a backup whose uncommitted entries disagree - /// with the log a view change settled on. They cannot be corrected in place, and - /// journal repair skips their ops as already-present, so dropping them is what - /// lets the primary's retransmission refill the range. - /// - /// # Errors - /// I/O error if the rewrite fails. Past the rename the journal is poisoned on any - /// failure, as in `drain`: serving a pre-truncation offset or appending at a stale - /// `write_offset` is worse than a hard stop. `from_op` must be at least 1. - async fn truncate_from(&self, from_op: u64) -> io::Result { - if let Some(state) = self.poisoned.get() { - return Err(Self::poisoned_io_error(state)); - } - if from_op == 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "truncate_from: ops are 1-based, so 0 would discard the whole journal", - )); - } - // Shares the drain guard: both rewrite the same WAL through the same tmp - // path, so letting them overlap would race the swap. - if self.drain_in_flight.replace(true) { - return Err(io::Error::new( - io::ErrorKind::ResourceBusy, - "drain or truncate already in flight: concurrent rewrites would race the WAL", - )); - } - let _guard = DrainInFlightGuard(&self.drain_in_flight); - - let mut removed = 0usize; - let mut live: Vec<(PrepareHeader, u64)> = Vec::new(); - { - let headers = self.headers.borrow(); - let offsets = self.offsets.borrow(); - for slot in 0..self.slot_count { - if let (Some(header), Some(offset)) = (&headers[slot], offsets[slot]) { - if header.op >= from_op { - removed += 1; - } else { - live.push((*header, offset)); - } - } - } - } - if removed == 0 { - return Ok(0); - } - live.sort_unstable_by_key(|(header, _)| header.op); - - let wal_path = self.storage.path(); - let tmp_path = wal_path.with_extension("wal.tmp"); - let tmp_guard = TmpFileGuard::new(tmp_path.clone()); - { - let mut tmp = compio::fs::File::create(&tmp_path).await?; - let mut write_pos: u64 = 0; - for (header, old_offset) in &live { - let size = header.size as usize; - let buf = vec![0u8; size]; - let buf = self.storage.read_at(*old_offset, buf).await?; - let (result, _buf) = tmp.write_all_at(buf, write_pos).await.into(); - result?; - write_pos += size as u64; - } - tmp.sync_all().await?; - } - - // COMMIT POINT, same as `drain`: past the rename the on-disk WAL is the new - // one while the in-memory index still describes the old layout, so every - // fallible step below poisons rather than serving stale offsets. - compio::fs::rename(&tmp_path, wal_path).await?; - tmp_guard.defuse(); - - if let Some(parent) = wal_path.parent() { - let dir = match compio::fs::File::open(parent).await { - Ok(dir) => dir, - Err(error) => { - return Err(self.poison("truncate_from: open parent dir for fsync", error)); - } - }; - if let Err(error) = dir.sync_all().await { - return Err(self.poison("truncate_from: parent dir fsync", error)); - } - } - if let Err(error) = self.storage.reopen().await { - return Err(self.poison("truncate_from: storage reopen after rename", error)); - } - - // `snapshot_op` is deliberately untouched. See the doc comment. - let mut headers = self.headers.borrow_mut(); - let mut offsets = self.offsets.borrow_mut(); - let mut pos: u64 = 0; - for (header, _) in &live { - let slot = slot_for_op(header.op, self.slot_count); - offsets[slot] = Some(pos); - pos += u64::from(header.size); - } - for slot in 0..self.slot_count { - if let Some(header) = &headers[slot] - && header.op >= from_op - { - headers[slot] = None; - offsets[slot] = None; - } - } - // Unlike a prefix drain, removing a suffix moves the head. - self.last_op.set(live.last().map(|(header, _)| header.op)); - - Ok(removed) - } - type Header = PrepareHeader; type Entry = Message; type HeaderRef<'a> = Ref<'a, PrepareHeader>; @@ -1214,257 +1047,6 @@ mod tests { Message::try_from(buffer).unwrap() } - /// A prepare with both integrity fields sealed and its parent chained, as a live - /// producer writes them. `make_entry` leaves `checksum` zero, read as unsealed. - fn make_identity_sealed_prepare( - op: u64, - body_size: usize, - parent: u128, - ) -> Message { - let mut message = make_prepare(op, body_size); - let bytes = message.as_mut_slice(); - let header = bytemuck::checked::from_bytes_mut::(&mut bytes[..HEADER_SIZE]); - header.parent = parent; - header.view = 1; - let checksum = header.identity_checksum(); - header.checksum = checksum; - message - } - - /// Byte offset of `field_offset` within the entry for `op`, at a fixed stride. - const fn header_field_offset(op: u64, body_size: usize, field_offset: usize) -> usize { - (op as usize - 1) * (HEADER_SIZE + body_size) + field_offset - } - - #[compio::test] - async fn truncate_from_removes_the_suffix_and_keeps_the_snapshot_floor() { - // The property that makes this not-a-drain: the floor must not move, or the - // removed ops become evictable and repair can never put them back. - let dir = tempdir().unwrap(); - let path = dir.path().join("journal.wal"); - let journal = PrepareJournal::open(&path, 0).await.unwrap(); - for op in 1..=5u64 { - journal - .append(make_prepare(op, 64).deep_copy()) - .await - .unwrap(); - } - assert_eq!(journal.last_op(), Some(5)); - let floor_before = journal.snapshot_op(); - - let removed = journal.truncate_from(3).await.unwrap(); - assert_eq!(removed, 3, "ops 3, 4 and 5 must go"); - assert_eq!( - journal.snapshot_op(), - floor_before, - "truncating a suffix must not advance the snapshot floor" - ); - assert_eq!( - journal.last_op(), - Some(2), - "the head follows the truncation" - ); - for op in 1..=2u64 { - assert!( - journal.header(op as usize).is_some(), - "op {op} must survive" - ); - } - for op in 3..=5u64 { - assert!( - journal.header(op as usize).is_none(), - "op {op} must be gone" - ); - } - } - - #[compio::test] - async fn truncate_from_leaves_a_refillable_range() { - // The whole point: after truncation the ops can be appended again. A raised - // floor would either reject that or silently evict a live entry. - let dir = tempdir().unwrap(); - let path = dir.path().join("journal.wal"); - let journal = PrepareJournal::open(&path, 0).await.unwrap(); - for op in 1..=4u64 { - journal - .append(make_prepare(op, 64).deep_copy()) - .await - .unwrap(); - } - journal.truncate_from(3).await.unwrap(); - - for op in 3..=4u64 { - journal - .append(make_prepare(op, 64).deep_copy()) - .await - .expect("a truncated op must be appendable again"); - } - assert_eq!(journal.last_op(), Some(4)); - assert!(journal.header(3).is_some()); - assert!(journal.header(4).is_some()); - } - - #[compio::test] - async fn truncate_from_survives_reopen() { - // The rewrite has to be durable, not just reflected in the index. - const BODY: usize = 64; - let dir = tempdir().unwrap(); - let path = dir.path().join("journal.wal"); - { - let journal = PrepareJournal::open(&path, 0).await.unwrap(); - let mut parent = 0u128; - for op in 1..=4u64 { - let entry = make_identity_sealed_prepare(op, BODY, parent); - parent = entry.header().checksum; - journal.append(entry.deep_copy()).await.unwrap(); - } - assert_eq!(journal.truncate_from(3).await.unwrap(), 2); - } - let journal = PrepareJournal::open(&path, 0).await.unwrap(); - assert_eq!( - journal.last_op(), - Some(2), - "the truncation must be on disk, not only in the index" - ); - assert!(journal.header(3).is_none()); - } - - #[compio::test] - async fn scan_accepts_a_sealed_and_chained_wal() { - // Everything below only means something if the happy path still opens. - const BODY: usize = 64; - let dir = tempdir().unwrap(); - let path = dir.path().join("journal.wal"); - { - let journal = PrepareJournal::open(&path, 0).await.unwrap(); - let mut parent = 0u128; - for op in 1..=3u64 { - let entry = make_identity_sealed_prepare(op, BODY, parent); - parent = entry.header().checksum; - journal.append(entry.deep_copy()).await.unwrap(); - } - } - let journal = PrepareJournal::open(&path, 0).await.unwrap(); - assert_eq!(journal.last_op(), Some(3)); - assert_eq!( - journal.unsealed_entry_count(), - 0, - "sealed entries must not be counted as unsealed" - ); - } - - #[compio::test] - async fn scan_truncates_tail_entry_with_header_checksum_mismatch() { - // A flipped `commit` leaves the header structurally valid, so only the identity - // checksum catches it. Recovery derives its watermark from `max(header.commit)`, - // so an undetected flip applies prepared-but-uncommitted ops as committed. - const BODY: usize = 64; - let dir = tempdir().unwrap(); - let path = dir.path().join("journal.wal"); - { - let journal = PrepareJournal::open(&path, 0).await.unwrap(); - let first = make_identity_sealed_prepare(1, BODY, 0); - let parent = first.header().checksum; - journal.append(first.deep_copy()).await.unwrap(); - journal - .append(make_identity_sealed_prepare(2, BODY, parent).deep_copy()) - .await - .unwrap(); - } - - let commit_offset = - header_field_offset(2, BODY, std::mem::offset_of!(PrepareHeader, commit)); - let mut bytes = std::fs::read(&path).unwrap(); - bytes[commit_offset] ^= 0xFF; - std::fs::write(&path, &bytes).unwrap(); - - let journal = PrepareJournal::open(&path, 0).await.unwrap(); - assert_eq!( - journal.last_op(), - Some(1), - "a header-checksum mismatch on the tail entry must truncate it" - ); - assert!(journal.header(2).is_none()); - } - - #[compio::test] - async fn scan_refuses_boot_on_interior_header_checksum_mismatch() { - const BODY: usize = 64; - let dir = tempdir().unwrap(); - let path = dir.path().join("journal.wal"); - { - let journal = PrepareJournal::open(&path, 0).await.unwrap(); - let mut parent = 0u128; - for op in 1..=3u64 { - let entry = make_identity_sealed_prepare(op, BODY, parent); - parent = entry.header().checksum; - journal.append(entry.deep_copy()).await.unwrap(); - } - } - - let commit_offset = - header_field_offset(2, BODY, std::mem::offset_of!(PrepareHeader, commit)); - let mut bytes = std::fs::read(&path).unwrap(); - bytes[commit_offset] ^= 0xFF; - std::fs::write(&path, &bytes).unwrap(); - - let error = PrepareJournal::open(&path, 0).await.unwrap_err(); - let message = error.to_string(); - assert!( - message.contains("interior WAL corruption"), - "an interior header flip must refuse boot rather than discard the \ - committed suffix, got: {message}" - ); - } - - #[compio::test] - async fn scan_detects_a_parent_chain_break() { - // Both entries are individually well sealed; only the link is wrong. Catching - // this is what makes the log a chain rather than a bag of valid records. - const BODY: usize = 64; - let dir = tempdir().unwrap(); - let path = dir.path().join("journal.wal"); - { - let journal = PrepareJournal::open(&path, 0).await.unwrap(); - journal - .append(make_identity_sealed_prepare(1, BODY, 0).deep_copy()) - .await - .unwrap(); - // Op 2 chains to a parent that is not op 1's checksum. - journal - .append(make_identity_sealed_prepare(2, BODY, 0xDEAD_BEEF).deep_copy()) - .await - .unwrap(); - } - - let journal = PrepareJournal::open(&path, 0).await.unwrap(); - assert_eq!( - journal.last_op(), - Some(1), - "op 2 does not chain to op 1 and must be truncated as a torn tail" - ); - } - - #[compio::test] - async fn scan_skips_verification_for_unsealed_entries() { - // A WAL from a pre-sealing build must still open: `checksum` reads as the - // unsealed sentinel, so neither the identity nor the chain is checked. - const BODY: usize = 32; - let dir = tempdir().unwrap(); - let path = dir.path().join("journal.wal"); - { - let journal = PrepareJournal::open(&path, 0).await.unwrap(); - for op in 1..=2u64 { - journal - .append(make_unsealed_prepare(op, BODY).deep_copy()) - .await - .unwrap(); - } - } - let journal = PrepareJournal::open(&path, 0).await.unwrap(); - assert_eq!(journal.last_op(), Some(2)); - } - #[compio::test] async fn scan_truncates_entry_with_body_checksum_mismatch() { let dir = tempdir().unwrap(); diff --git a/core/message_bus/src/client_listener/quic.rs b/core/message_bus/src/client_listener/quic.rs index 923eb0b5b7..63cb6ac78c 100644 --- a/core/message_bus/src/client_listener/quic.rs +++ b/core/message_bus/src/client_listener/quic.rs @@ -22,7 +22,7 @@ //! invoking the supplied callback, so the callback receives a //! ready-for-traffic [`compio_quic::Connection`] plus its //! `(SendStream, RecvStream)` pair. No ALPN is advertised; protocol -//! version is validated by the caller (the server) inside the LOGIN +//! version is validated by the caller (server-ng) inside the LOGIN //! command. //! //! 0-RTT data is refused at accept time diff --git a/core/message_bus/src/config.rs b/core/message_bus/src/config.rs index c31d14672d..1d30a0420f 100644 --- a/core/message_bus/src/config.rs +++ b/core/message_bus/src/config.rs @@ -18,7 +18,7 @@ //! Runtime tunables for the message bus. //! //! Single source of truth for these knobs is the on-disk schema -//! [`configs::server::ServerConfig`]. The bus consumes that +//! [`configs::server_ng::ServerNgConfig`]. The bus consumes that //! schema at construction (see [`crate::IggyMessageBus::with_config`]) //! and converts the schema-typed fields //! ([`iggy_common::IggyDuration`] / [`iggy_common::IggyByteSize`]) @@ -27,10 +27,10 @@ //! //! The WebSocket frame-layer config the bus consumes lives under the //! schema's `[websocket]` block (buffer sizes, message / frame -//! ceilings, unmasked-frame acceptance): the bus IS the server's +//! ceilings, unmasked-frame acceptance): the bus IS server-ng's //! WS / WSS install path, so the listener section carries the frame -//! tuning (see `configs::websocket`). -//! [`From<&ServerConfig> for MessageBusConfig`](MessageBusConfig) +//! tuning (see `configs::ng_websocket`). +//! [`From<&ServerNgConfig> for MessageBusConfig`](MessageBusConfig) //! folds that section into [`WebSocketConfig`] once at boot. //! //! Liveness detection is NOT done via TCP keepalive on the bus: SDK @@ -39,17 +39,17 @@ //! than by `SO_KEEPALIVE`. //! //! Neither plane is authenticated at the bus layer: identity and -//! credential checks belong to the caller (`core/server`) via +//! credential checks belong to the caller (`core/server-ng`) via //! `LOGIN_*` commands. This struct therefore carries no secret / //! token-source state. pub use compio::ws::tungstenite::protocol::WebSocketConfig; -use configs::server::ServerConfig; +use configs::server_ng::ServerNgConfig; use std::time::Duration; /// Pre-converted QUIC transport tuning derived from -/// [`ServerConfig::quic`](configs::quic::QuicConfig). +/// [`ServerNgConfig::quic`](configs::ng_quic::QuicConfig). /// /// Threaded into [`crate::transports::quic::transport_config_from`] at /// every bind site so the schema's `[quic]` block actually drives @@ -109,8 +109,8 @@ pub const IOV_MAX_LIMIT: usize = 512; /// Pre-converted runtime tunables in effect on a `IggyMessageBus` /// instance. /// -/// Built from a fully-validated [`ServerConfig`] via -/// [`From<&ServerConfig>`] at boot. All fields are runtime-typed +/// Built from a fully-validated [`ServerNgConfig`] via +/// [`From<&ServerNgConfig>`] at boot. All fields are runtime-typed /// (`Duration`, `usize`, `tungstenite::WebSocketConfig`) so hot paths /// read them directly without `.get_duration()` / `.as_bytes_u64()` /// conversion. @@ -177,9 +177,9 @@ pub struct MessageBusConfig { /// Threaded into `compio_ws::accept_async_with_config` on the WS /// install path and into `WssTransportConn::ws_handshake` for WSS. /// Built once at boot by `build_ws_config` (see the - /// [`From<&ServerConfig> for MessageBusConfig`](MessageBusConfig) impl below) + /// [`From<&ServerNgConfig> for MessageBusConfig`](MessageBusConfig) impl below) /// from the schema's `[websocket]` section, the live frame-tuning - /// source for the server's WS plane. + /// source for server-ng's WS plane. /// /// The [`WebSocketConfig`] type is re-exported from `compio_ws`'s /// vendored `tungstenite` so callers do not need a direct dep on @@ -187,23 +187,23 @@ pub struct MessageBusConfig { pub ws_config: WebSocketConfig, /// QUIC transport tuning, pre-converted from - /// [`ServerConfig::quic`](configs::quic::QuicConfig) at boot. + /// [`ServerNgConfig::quic`](configs::ng_quic::QuicConfig) at boot. pub quic: QuicTuning, } -impl From<&ServerConfig> for MessageBusConfig { - fn from(cfg: &ServerConfig) -> Self { +impl From<&ServerNgConfig> for MessageBusConfig { + fn from(cfg: &ServerNgConfig) -> Self { let bus = &cfg.message_bus; - // Production load goes through `ServerConfig::validate()`, which + // Production load goes through `ServerNgConfig::validate()`, which // already exercises `bus.validate()`. This debug-assert catches - // direct callers (tests, simulators) that build a `ServerConfig` + // direct callers (tests, simulators) that build a `ServerNgConfig` // by hand and forget to validate before converting. debug_assert!( >::validate(bus) .is_ok(), - "MessageBusConfig::from(&ServerConfig) called on an unvalidated bus config", + "MessageBusConfig::from(&ServerNgConfig) called on an unvalidated bus config", ); Self { max_batch: bus.max_batch, @@ -225,7 +225,7 @@ impl From<&ServerConfig> for MessageBusConfig { } } -/// Convert the schema's [`configs::quic::QuicConfig`] +/// Convert the schema's [`configs::ng_quic::QuicConfig`] /// (`IggyByteSize` / `IggyDuration` typed) into the runtime /// [`QuicTuning`] (plain integer / `Duration` fields). /// @@ -235,7 +235,7 @@ impl From<&ServerConfig> for MessageBusConfig { /// `unwrap_or` arms below are still bounded saturations that keep /// the build unconditionally infallible if a future caller skips /// validation in dev / test code. -fn build_quic_tuning(quic: &configs::quic::QuicConfig) -> QuicTuning { +fn build_quic_tuning(quic: &configs::ng_quic::QuicConfig) -> QuicTuning { QuicTuning { max_concurrent_bidi_streams: u32::try_from(quic.max_concurrent_bidi_streams) .unwrap_or(u32::MAX), @@ -251,12 +251,12 @@ fn build_quic_tuning(quic: &configs::quic::QuicConfig) -> QuicTuning { impl Default for QuicTuning { /// Mirrors the `[quic]` defaults in - /// `core/server/config.toml`: 64 MiB send/receive windows, + /// `core/server-ng/config.toml`: 64 MiB send/receive windows, /// 30 s idle timeout, 10 s keep-alive, 8 KiB initial MTU, 100 KiB /// datagram send buffer, single bidi stream per peer. /// /// Intended for tests and direct callers; production builds - /// derive the field from [`ServerConfig`] so the values stay in + /// derive the field from [`ServerNgConfig`] so the values stay in /// lock-step with the on-disk schema. fn default() -> Self { Self { @@ -283,7 +283,7 @@ impl Default for QuicTuning { /// Conversion to `usize` saturates on platforms where `IggyByteSize` /// would overflow, but on supported targets `usize` is at least 32 /// bits, so saturation is unreachable in practice. -fn build_ws_config(websocket: &configs::websocket::WebSocketConfig) -> WebSocketConfig { +fn build_ws_config(websocket: &configs::ng_websocket::WebSocketConfig) -> WebSocketConfig { let mut ws = WebSocketConfig::default(); if let Some(sz) = websocket.read_buffer_size { ws = ws.read_buffer_size(byte_size_to_usize(sz)); @@ -311,7 +311,7 @@ fn byte_size_to_usize(sz: iggy_common::IggyByteSize) -> usize { impl Default for MessageBusConfig { fn default() -> Self { - Self::from(&ServerConfig::default()) + Self::from(&ServerNgConfig::default()) } } @@ -321,13 +321,13 @@ mod tests { /// `QuicTuning::default()` carries hand-coded literals that must /// match the schema-derived path through - /// `From<&ServerConfig> for MessageBusConfig`. If the embedded + /// `From<&ServerNgConfig> for MessageBusConfig`. If the embedded /// TOML or the literals drift, every test that uses /// `QuicTuning::default()` (e.g. `quic_client_roundtrip`) silently /// observes different bytes than production. Pin both sides here. #[test] fn quic_tuning_default_matches_schema() { - let schema_quic = MessageBusConfig::from(&ServerConfig::default()).quic; + let schema_quic = MessageBusConfig::from(&ServerNgConfig::default()).quic; let literal = QuicTuning::default(); assert_eq!( diff --git a/core/message_bus/src/installer/conn_info.rs b/core/message_bus/src/installer/conn_info.rs index 10cf776d45..8584d86698 100644 --- a/core/message_bus/src/installer/conn_info.rs +++ b/core/message_bus/src/installer/conn_info.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -//! Per-client transport metadata exposed to the caller (`the server`). +//! Per-client transport metadata exposed to the caller (`server-ng`). //! //! Constructed by the listener / install path on shard 0 with whatever //! is known at accept time (`client_id`, `peer_addr`, `transport`) and diff --git a/core/message_bus/src/installer/mod.rs b/core/message_bus/src/installer/mod.rs index 641fd74e6f..3a04c2ccaa 100644 --- a/core/message_bus/src/installer/mod.rs +++ b/core/message_bus/src/installer/mod.rs @@ -109,7 +109,7 @@ pub trait ConnectionInstaller { /// handshake, then installs WS reader / writer tasks via /// [`install_client_ws`] on success. On handshake failure /// the fd is closed by dropping the wrapping `TcpStream`. No - /// subprotocol negotiation: the caller (the server) gates command + /// subprotocol negotiation: the caller (server-ng) gates command /// access via the LOGIN allowlist. fn install_client_ws_fd(&self, fd: DupedFd, meta: ClientConnMeta, on_request: RequestHandler); diff --git a/core/message_bus/src/installer/wss.rs b/core/message_bus/src/installer/wss.rs index d65f66fd34..32db26cb4c 100644 --- a/core/message_bus/src/installer/wss.rs +++ b/core/message_bus/src/installer/wss.rs @@ -37,7 +37,7 @@ use tracing::warn; /// then run inside the transport's `run` body on the per-connection /// install task; the install path stays thin. No subprotocol /// negotiation: client identity is established post-handshake by the -/// LOGIN command on the caller (the server). +/// LOGIN command on the caller (server-ng). /// /// `TCP_NODELAY` is applied pre-handshake for symmetry with /// [`super::tcp_tls::install_client_tcp_tls`]. `SO_KEEPALIVE` is diff --git a/core/message_bus/src/lib.rs b/core/message_bus/src/lib.rs index 644591110a..7f3c84a654 100644 --- a/core/message_bus/src/lib.rs +++ b/core/message_bus/src/lib.rs @@ -27,13 +27,13 @@ //! - **SDK-client plane**: ephemeral client connections. Available //! transports: TCP, TCP-TLS, WebSocket, WSS, QUIC. Each request //! carries a `(client: u128, request: u64)` pair in `RequestHeader`; -//! downstream consumers in `core/server` are free to use it for +//! downstream consumers in `core/server-ng` are free to use it for //! tracing, idempotency, or correlation. //! //! # Auth //! //! Neither plane is authenticated at the bus layer. Both connect first -//! and let the caller (`core/server`) gate command access via +//! and let the caller (`core/server-ng`) gate command access via //! application-level LOGIN commands: //! //! - SDK-client plane: `LOGIN_USER` / `LOGIN_WITH_PERSONAL_ACCESS_TOKEN`, @@ -101,7 +101,7 @@ pub use lifecycle::{ pub use transports::tls::TlsServerCredentials; pub use compio::runtime::JoinHandle; -use configs::server::ServerConfig; +use configs::server_ng::ServerNgConfig; use iggy_binary_protocol::{GenericHeader, ReplyHeader}; use server_common::{MESSAGE_ALIGN, Message, iobuf::Frozen}; use std::array; @@ -144,7 +144,7 @@ pub const OWNER_NONE: u16 = u16::MAX; /// Shared atomic owner table mapping `replica_id` to `owning_shard_id`. /// -/// One Arc-cloned instance is allocated per the server process at +/// One Arc-cloned instance is allocated per server-ng process at /// bootstrap and shared across every shard's [`IggyMessageBus`]. The owning shard /// stamps its id into a slot when an inbound replica connection passes /// the registry-insert race; the same shard CAS-clears the slot when @@ -723,7 +723,7 @@ pub struct IggyMessageBus { impl IggyMessageBus { /// Construct a bus with default tunables (derived from - /// [`ServerConfig::default`]). + /// [`ServerNgConfig::default`]). #[must_use] pub fn new(shard_id: u16) -> Self { Self::with_tunables(shard_id, MessageBusConfig::default()) @@ -743,10 +743,10 @@ impl IggyMessageBus { Self::with_tunables(shard_id, cfg) } - /// Construct a bus from the validated the server schema. + /// Construct a bus from the validated server-ng schema. /// /// Production constructor: takes a fully-validated - /// [`ServerConfig`] and derives the runtime [`MessageBusConfig`] + /// [`ServerNgConfig`] and derives the runtime [`MessageBusConfig`] /// internally. Field conversions ([`iggy_common::IggyDuration`] -> [`Duration`], /// [`iggy_common::IggyByteSize`] -> `usize`, schema WS knobs -> /// tungstenite [`WebSocketConfig`]) happen once here so hot paths @@ -759,19 +759,19 @@ impl IggyMessageBus { /// surfaces operator misconfiguration loudly rather than letting /// every `writev` fail silently with `EMSGSIZE` once traffic starts. #[must_use] - pub fn with_config(shard_id: u16, cfg: &ServerConfig) -> Self { + pub fn with_config(shard_id: u16, cfg: &ServerNgConfig) -> Self { Self::with_tunables(shard_id, MessageBusConfig::from(cfg)) } - /// Production constructor for multi-shard the server: same as + /// Production constructor for multi-shard server-ng: same as /// [`Self::with_config`] but takes a pre-allocated /// [`ReplicaOwnerTable`] Arc. Bootstrap allocates one table per - /// server process and clones the Arc into every shard so all + /// server-ng process and clones the Arc into every shard so all /// buses see the same atomic slots. #[must_use] pub fn with_config_and_owner_table( shard_id: u16, - cfg: &ServerConfig, + cfg: &ServerNgConfig, owner_table: Arc, ) -> Self { Self::with_tunables_and_owner_table(shard_id, MessageBusConfig::from(cfg), owner_table) @@ -781,7 +781,7 @@ impl IggyMessageBus { /// /// Used by the public constructors above and by tests that need to /// patch a single field on the derived [`MessageBusConfig`] without - /// round-tripping through [`ServerConfig`]. + /// round-tripping through [`ServerNgConfig`]. /// /// # Panics /// diff --git a/core/message_bus/src/replica/io.rs b/core/message_bus/src/replica/io.rs index 290db8e2d2..d36ec4d18f 100644 --- a/core/message_bus/src/replica/io.rs +++ b/core/message_bus/src/replica/io.rs @@ -59,7 +59,7 @@ use crate::{ /// The cert chain is the leaf-first sequence rustls expects; the key /// is the server's private key in DER form. Tests use rcgen to mint a /// throwaway pair; production callers load real PKI material via -/// `core/server`'s `[quic.certificate]` config section. +/// `core/server-ng`'s `[quic.certificate]` config section. pub struct QuicServerCredentials { pub cert_chain: Vec>, pub key_der: PrivateKeyDer<'static>, @@ -331,7 +331,7 @@ pub async fn start_on_shard_zero( /// Leaves the WS, QUIC, TCP-TLS, and WSS listener slots unconfigured /// (`None`). Convenience entry for TCP-only deployments and existing /// tests; prefer the full [`start_on_shard_zero`] in production where -/// the optional planes come from `core/server`'s config. +/// the optional planes come from `core/server-ng`'s config. /// /// # Errors /// diff --git a/core/message_bus/src/transports/quic.rs b/core/message_bus/src/transports/quic.rs index df571c9941..8fdf49e830 100644 --- a/core/message_bus/src/transports/quic.rs +++ b/core/message_bus/src/transports/quic.rs @@ -527,7 +527,7 @@ impl TransportConn for QuicTransportConn { /// Applies [`transport_config_from`] and disables 0-RTT; otherwise /// inherits upstream defaults including `migration: true`. No ALPN is /// advertised; protocol-version validation lives in the LOGIN command -/// on the caller (the server). +/// on the caller (server-ng). /// /// # Errors /// diff --git a/core/message_bus/src/transports/tls/mod.rs b/core/message_bus/src/transports/tls/mod.rs index 888f9358e6..c92ca70de4 100644 --- a/core/message_bus/src/transports/tls/mod.rs +++ b/core/message_bus/src/transports/tls/mod.rs @@ -29,7 +29,7 @@ //! `ServerConfig::builder()` / `ClientConfig::builder()` call. The //! workspace pins `rustls = { features = ["ring"] }`, so the //! provider is `rustls::crypto::ring::default_provider()`. Multiple -//! call sites (transports, tests, the server bootstrap) may attempt to +//! call sites (transports, tests, server-ng bootstrap) may attempt to //! install it; the wrapper here is idempotent and safe under races //! between concurrent first-use sites. @@ -245,7 +245,7 @@ impl ServerCertVerifier for AcceptAnyServerCert { /// rustls 0.23 requires a default provider before any /// `ServerConfig::builder()` or `ClientConfig::builder()` call. This /// helper centralises the install so transports, tests, and -/// the server bootstrap converge on the same provider without +/// server-ng bootstrap converge on the same provider without /// per-call-site `let _ = ...install_default()`. pub fn install_default_crypto_provider() { // Race-safe: rustls's `install_default` returns Err if a provider diff --git a/core/message_bus/tests/ws_client_roundtrip.rs b/core/message_bus/tests/ws_client_roundtrip.rs index a66f5a621f..88214becfd 100644 --- a/core/message_bus/tests/ws_client_roundtrip.rs +++ b/core/message_bus/tests/ws_client_roundtrip.rs @@ -26,7 +26,7 @@ //! server's `bus.send_to_client` reply lands on the client's reader. //! Verifies the full bidirectional plane through the reader / writer //! two-task split. Pre-LOGIN command gating is the caller's -//! responsibility (the server) and is not exercised here. +//! responsibility (server-ng) and is not exercised here. mod common; @@ -123,7 +123,7 @@ async fn handshake_succeeds_and_round_trip_completes() { async fn handshake_succeeds_without_subprotocol_header() { // Without the subprotocol gate, a client that sends NO // Sec-WebSocket-Protocol header must still complete the upgrade. - // Pre-LOGIN command gating is enforced by the caller (the server), + // Pre-LOGIN command gating is enforced by the caller (server-ng), // not at the bus layer. let bus = Rc::new(IggyMessageBus::new(0)); let on_request: RequestHandler = Rc::new(|_, _| {}); diff --git a/core/metadata/Cargo.toml b/core/metadata/Cargo.toml index cd83798bc6..681ff73ead 100644 --- a/core/metadata/Cargo.toml +++ b/core/metadata/Cargo.toml @@ -32,7 +32,7 @@ publish = false # Simulator-only seed helper (`Streams::seed_single_partition`): applies # CreateStream + CreateTopic straight onto the writer STM, bypassing metadata # consensus, so the deterministic dispatch shell can resolve a namespace -# without driving the reconciler. A `-p iggy-server` build excludes it; +# without driving the reconciler. A `-p iggy-server-ng` build excludes it; # `cargo build --workspace` unifies features so the shared `metadata` unit # compiles it in when the simulator requests it. No production caller. simulator = [] diff --git a/core/metadata/src/impls/metadata.rs b/core/metadata/src/impls/metadata.rs index c4bb620e08..9643493fa8 100644 --- a/core/metadata/src/impls/metadata.rs +++ b/core/metadata/src/impls/metadata.rs @@ -34,7 +34,6 @@ use consensus::{ is_caught_up_primary, panic_if_hash_chain_would_break_in_same_view, peek_committable_head, pipeline_prepare_common, register_preflight, replicate_preflight, replicate_to_next_in_chain, request_preflight, send_eviction_to_client, send_prepare_ok as send_prepare_ok_common, - verify_prepare_integrity, }; use iggy_binary_protocol::WireIdentifier; use iggy_binary_protocol::primitives::partition_assignment::CreatedPartitionAssignment; @@ -44,7 +43,7 @@ use iggy_binary_protocol::requests::topics::CreateTopicRequest as WireCreateTopi use iggy_binary_protocol::requests::topics::CreateTopicWithAssignmentsRequest as PersistedCreateTopicRequest; use iggy_binary_protocol::{ Command2, ConsensusHeader, EvictionReason, GenericHeader, Operation, PrepareHeader, - PrepareOkHeader, ReplyHeader, RoutedRequestHeader, WireDecode, WireEncode, WireName, + PrepareOkHeader, ReplyHeader, RequestHeader, WireDecode, WireEncode, WireName, }; use iggy_common::IggyError; use iggy_common::UserId; @@ -415,29 +414,17 @@ impl SnapshotCoordinator { Ok(checksum) } - /// Drain the snapshotted prefix below `last_op` to reclaim WAL space. Runs - /// only after the pairing is durable (see [`Self::persist_snapshot`]). - /// - /// `last_op` itself is retained, one entry the snapshot has already - /// superseded. It is this replica's commit point, and a `DoViewChange` - /// carries a header for every op from there up. Draining it inclusively - /// leaves that entry blank, and blank at the commit point is the one slot - /// the merge can neither adopt nor discard: a quorum of senders that all - /// checkpointed at the same op deadlocks the view change - /// (`dvc_merge::merge_dvc_quorum`). Reclaiming one more entry is not worth - /// a group that cannot elect. + /// Drain the snapshotted prefix `0..=last_op` to reclaim WAL space. Runs only + /// after the pairing is durable (see [`Self::persist_snapshot`]). #[allow(clippy::future_not_send)] async fn drain( &self, journal: &J, last_op: u64, ) -> Result<(), SnapshotError> { - let Some(drain_to) = last_op.checked_sub(1) else { - return Ok(()); - }; journal .handle() - .drain(0..=drain_to) + .drain(0..=last_op) .await .map_err(SnapshotError::Io)?; Ok(()) @@ -641,7 +628,7 @@ pub fn apply_committed_prepare( /// Late-bound callback invoked after every committed op on shard 0's metadata /// commit path (via `gated_apply`, including a gated no-op). /// -/// Wired by the server bootstrap once the metadata bundle has broadcast; +/// Wired by server-ng bootstrap once the metadata bundle has broadcast; /// receives the committed [`Operation`] so the recipient can filter (the /// partition reconciliation loop only cares about partition-shaped /// events). Wrapped in [`RefCell`] for late binding; the per-shard @@ -660,7 +647,7 @@ pub struct IggyMetadata { /// the WAL at all. They receive a `MetadataHandoff::Waiter` factory /// bundle from shard 0 over the bootstrap broadcast channel and /// reconstruct `mux_stm` from the in-memory snapshot it carries (see - /// `server/src/bootstrap.rs` `await_metadata_bundle` / + /// `server-ng/src/bootstrap.rs` `await_metadata_bundle` / /// `broadcast_metadata_bundle`). pub journal: Option, /// `Some` on shard 0, `None` on other shards. @@ -728,7 +715,7 @@ pub struct IggyMetadata { /// after `gated_apply` returns (including a gated `Unauthorized` no-op that /// never reaches [`crate::stm::StateMachine::update`]) in both /// [`Plane::on_ack`] and [`Self::commit_journal`]. `None` until - /// [`Self::set_commit_notifier`] runs (the server bootstrap on shard + /// [`Self::set_commit_notifier`] runs (server-ng bootstrap on shard /// 0 sets it; peer shards and tests leave it `None`). commit_notifier: RefCell>, /// Resolved byte value for `MaxTopicSize::ServerDefault` (`0` on the @@ -946,10 +933,7 @@ where Error = iggy_common::IggyError, >, { - async fn on_request( - &self, - message: as Consensus>::Message, - ) { + async fn on_request(&self, message: as Consensus>::Message) { let Some(consensus) = require_shard_zero(self.consensus.as_ref(), "on_request", "consensus") else { @@ -1054,23 +1038,6 @@ where let header = *message.header(); - // Before anything trusts `checksum` as an identity token, and before the WAL - // takes the bytes. Every live prepare travels this path: unverified, a frame - // corrupted between primary and backup is journaled as-is and re-served to - // peers, which the interior-corruption boot refusal turns into an unbootable - // node on the next restart. - if let Err(reason) = verify_prepare_integrity(&header, message.as_slice()) { - warn!( - target: "iggy.metadata.diag", - plane = "metadata", - replica_id = consensus.replica(), - view = consensus.view(), - op = header.op, - "discarding prepare: {reason}" - ); - return; - } - let current_op = match replicate_preflight(consensus, &header) { Ok(current_op) => current_op, Err(reason) => { @@ -1896,7 +1863,7 @@ where } let request = build_register_request_message(consensus, client_id, user_id); - // Wire path runs `RoutedRequestHeader::validate` at network boundary; + // Wire path runs `RequestHeader::validate` at network boundary; // in-process skips it. debug_assert pins drift. debug_assert!( { @@ -1998,7 +1965,7 @@ where /// commit. fn answer_preflight( consensus: &VsrConsensus, - request_header: &RoutedRequestHeader, + request_header: &RequestHeader, outcome: PreflightOutcome, ) -> Option, MetadataSubmitError>> { let client_id = request_header.client; @@ -2311,10 +2278,10 @@ where // Build the prepare directly so the `client = 0` header skips the // client-header validation in `prepare_request` / `Project::project` // (the in-process path `build_prepare_message` documents). - let header = RoutedRequestHeader { + let header = RequestHeader { client: 0, - group: server_common::sharding::METADATA_GROUP, - ..RoutedRequestHeader::default() + namespace: server_common::sharding::METADATA_CONSENSUS_NAMESPACE, + ..RequestHeader::default() }; let prepare = build_prepare_message( consensus, @@ -2357,7 +2324,7 @@ where #[allow(clippy::future_not_send)] pub async fn submit_request_in_process( &self, - message: Message, + message: Message, ) -> Result, MetadataSubmitError> { let request_header = *message.header(); let client_id = request_header.client; @@ -3124,7 +3091,7 @@ where return; }; // Serialize whole checkpoints against each other. In-process metadata submits - // each run on their own spawned task (`bus.spawn` in the server's metadata submit + // each run on their own spawned task (`bus.spawn` in server-ng's metadata submit // handler), so at the checkpoint margin two can enter here concurrently; without // this lock they would run concurrent `persist_snapshot`s over the single // `snapshot.bin` and concurrently `drain` the WAL, which rewrites through a @@ -3233,7 +3200,7 @@ where #[allow(clippy::too_many_lines)] fn prepare_request( &self, - mut message: Message, + mut message: Message, ) -> Result, iggy_common::IggyError> { let consensus = self.consensus.as_ref().unwrap(); let operation = message.header().operation; @@ -3264,8 +3231,8 @@ where if let Some(acting_user_id) = resolve_acting_user_id(operation, client_id, &self.client_table)? { - let request_header = bytemuck::checked::from_bytes_mut::( - &mut message.as_mut_slice()[..size_of::()], + let request_header = bytemuck::checked::from_bytes_mut::( + &mut message.as_mut_slice()[..size_of::()], ); request_header.user_id = acting_user_id; } @@ -3278,7 +3245,7 @@ where // authz gate. The default arm projects the mutated buffer directly and // is order-independent. let header = *message.header(); - let body = &message.as_slice()[size_of::()..header.size as usize]; + let body = &message.as_slice()[size_of::()..header.size as usize]; match header.operation { Operation::CreateTopic => { @@ -3511,36 +3478,36 @@ where } } -/// In-process Register `Message`. Mirrors +/// In-process Register `Message`. Mirrors /// `SimClient::register`: `session=0`, `request=0` per -/// [`RoutedRequestHeader::validate`]; empty body. +/// [`RequestHeader::validate`]; empty body. /// /// `cluster` + `view` from `consensus` for self-consistency before /// `Project::project` overwrites. `release = 0` matches wire today; both /// paths should switch to `consensus.release()` once /// `ClientReleaseTooLow/TooHigh` lands. /// -/// Buffer is `size_of::()`; `prepare_request` transmutes into +/// Buffer is `size_of::()`; `prepare_request` transmutes into /// `PrepareHeader` (also 256 bytes), no realloc. fn build_register_request_message( consensus: &VsrConsensus, client_id: u128, user_id: u32, -) -> Message +) -> Message where B: MessageBus, P: Pipeline, { - let header_size = size_of::(); - let mut msg = Message::::new(header_size); - let header = bytemuck::checked::try_from_bytes_mut::( + let header_size = size_of::(); + let mut msg = Message::::new(header_size); + let header = bytemuck::checked::try_from_bytes_mut::( &mut msg.as_mut_slice()[..header_size], ) - .expect("zeroed bytes are a valid RoutedRequestHeader"); - *header = RoutedRequestHeader { + .expect("zeroed bytes are a valid RequestHeader"); + *header = RequestHeader { command: Command2::Request, operation: Operation::Register, - size: u32::try_from(header_size).expect("RoutedRequestHeader size fits u32"), + size: u32::try_from(header_size).expect("RequestHeader size fits u32"), cluster: consensus.cluster(), view: consensus.view(), release: 0, @@ -3553,8 +3520,8 @@ where // prepare is re-routed on each peer by namespace; a `0` here would // hash to a non-zero shard with no metadata consensus and be // silently dropped (see `shard::router::route_typed`). - group: server_common::sharding::METADATA_GROUP, - ..RoutedRequestHeader::default() + namespace: server_common::sharding::METADATA_CONSENSUS_NAMESPACE, + ..RequestHeader::default() }; msg } @@ -3564,21 +3531,21 @@ fn build_logout_request_message( client_id: u128, session: u64, request: u64, -) -> Message +) -> Message where B: MessageBus, P: Pipeline, { - let header_size = size_of::(); - let mut msg = Message::::new(header_size); - let header = bytemuck::checked::try_from_bytes_mut::( + let header_size = size_of::(); + let mut msg = Message::::new(header_size); + let header = bytemuck::checked::try_from_bytes_mut::( &mut msg.as_mut_slice()[..header_size], ) - .expect("zeroed bytes are a valid RoutedRequestHeader"); - *header = RoutedRequestHeader { + .expect("zeroed bytes are a valid RequestHeader"); + *header = RequestHeader { command: Command2::Request, operation: Operation::Logout, - size: u32::try_from(header_size).expect("RoutedRequestHeader size fits u32"), + size: u32::try_from(header_size).expect("RequestHeader size fits u32"), cluster: consensus.cluster(), view: consensus.view(), release: 0, @@ -3586,8 +3553,8 @@ where session, request, // Metadata consensus group (see `build_register_request_message`). - group: server_common::sharding::METADATA_GROUP, - ..RoutedRequestHeader::default() + namespace: server_common::sharding::METADATA_CONSENSUS_NAMESPACE, + ..RequestHeader::default() }; msg } @@ -3597,21 +3564,21 @@ fn build_complete_revocation_request_message( client_id: u128, request: u64, body: &[u8], -) -> Message +) -> Message where B: MessageBus, P: Pipeline, { - let header_size = size_of::(); + let header_size = size_of::(); let total = header_size + body.len(); - let mut msg = Message::::new(total); + let mut msg = Message::::new(total); { let slice = msg.as_mut_slice(); slice[header_size..total].copy_from_slice(body); let header = - bytemuck::checked::try_from_bytes_mut::(&mut slice[..header_size]) - .expect("zeroed bytes are a valid RoutedRequestHeader"); - *header = RoutedRequestHeader { + bytemuck::checked::try_from_bytes_mut::(&mut slice[..header_size]) + .expect("zeroed bytes are a valid RequestHeader"); + *header = RequestHeader { command: Command2::Request, operation: Operation::CompleteConsumerGroupRevocation, size: u32::try_from(total).expect("request size fits u32"), @@ -3623,8 +3590,8 @@ where // there is no real session (the commit path skips reply-caching). session: 1, request, - group: server_common::sharding::METADATA_GROUP, - ..RoutedRequestHeader::default() + namespace: server_common::sharding::METADATA_CONSENSUS_NAMESPACE, + ..RequestHeader::default() }; } msg @@ -3647,14 +3614,14 @@ where /// a few fixed-width fields, so this cannot happen in practice. #[must_use] pub fn build_truncate_partition_client_message( - template: &RoutedRequestHeader, + template: &RequestHeader, client_id: u128, session: u64, stream_id: u32, topic_id: u32, partition_id: u32, up_to_offset: u64, -) -> Message { +) -> Message { build_truncate_partition_client_message_with_identifiers( template, client_id, @@ -3678,14 +3645,14 @@ pub fn build_truncate_partition_client_message( /// a few small fields, so this cannot happen in practice. #[must_use] pub fn build_truncate_partition_client_message_with_identifiers( - template: &RoutedRequestHeader, + template: &RequestHeader, client_id: u128, session: u64, stream_id: WireIdentifier, topic_id: WireIdentifier, partition_id: u32, up_to_offset: u64, -) -> Message { +) -> Message { let body = TruncatePartitionRequest { stream_id, topic_id, @@ -3693,16 +3660,16 @@ pub fn build_truncate_partition_client_message_with_identifiers( up_to_offset, } .to_bytes(); - let header_size = size_of::(); + let header_size = size_of::(); let total = header_size + body.len(); - let mut msg = Message::::new(total); + let mut msg = Message::::new(total); { let slice = msg.as_mut_slice(); slice[header_size..total].copy_from_slice(&body); let header = - bytemuck::checked::try_from_bytes_mut::(&mut slice[..header_size]) - .expect("zeroed bytes are a valid RoutedRequestHeader"); - *header = RoutedRequestHeader { + bytemuck::checked::try_from_bytes_mut::(&mut slice[..header_size]) + .expect("zeroed bytes are a valid RequestHeader"); + *header = RequestHeader { command: Command2::Request, operation: Operation::TruncatePartition, size: u32::try_from(total).expect("request size fits u32"), @@ -3712,8 +3679,8 @@ pub fn build_truncate_partition_client_message_with_identifiers( client: client_id, session, request: template.request, - group: server_common::sharding::METADATA_GROUP, - ..RoutedRequestHeader::default() + namespace: server_common::sharding::METADATA_CONSENSUS_NAMESPACE, + ..RequestHeader::default() }; } msg @@ -3721,7 +3688,7 @@ pub fn build_truncate_partition_client_message_with_identifiers( fn build_prepare_message( consensus: &VsrConsensus, - request: &RoutedRequestHeader, + request: &RequestHeader, operation: Operation, body: &[u8], ) -> Message @@ -3768,7 +3735,7 @@ where operation, // The group's namespace, never the request's: clients send 0, and a // journaled 0 mis-routes the entry when repair replays it verbatim. - group: consensus.group(), + namespace: consensus.namespace(), // Carry the acting user id so the in-apply RBAC gate sees the same // identity on every replica. The default projection copies it (see // `Project::project`); this helper builds prepares for the ops it @@ -3785,12 +3752,7 @@ where ..Default::default() }; - // Last, because the identity checksum covers every other field. Same contract as - // the wire path in `Project::project`; skipping it would leave the rewritten - // prepares (CreateTopic/CreatePartitions assignments, the UpdateTopic default-size - // rewrite, the PAT-cleaner delete) as the only ops the merge cannot tell apart - // from a competing prepare. - consensus::seal_prepare_checksum(prepare) + prepare } /// Eviction reason for a request `prepare_request` rejected as structurally @@ -3804,7 +3766,7 @@ const fn eviction_reason_for_invalid(operation: Operation) -> EvictionReason { } /// Resolve the acting user id to stamp into a client op's replicated -/// `RoutedRequestHeader`, so the in-apply RBAC gate (`crate::stm::authz`) reads the +/// `RequestHeader`, so the in-apply RBAC gate (`crate::stm::authz`) reads the /// same identity on every replica (WAL replay has no session table). /// /// - `Ok(Some(id))`: overwrite the header's `user_id` with the committed @@ -3881,7 +3843,7 @@ fn log_commit_reply_outcome(outcome: CommitReply, client_id: u128, op: u64) { /// A cached REJECTION replays untouched: it carries no secret, so serving it /// is both safe and useful. fn unreplayable_secret_refusal( - request_header: &RoutedRequestHeader, + request_header: &RequestHeader, cached: &Frozen<{ server_common::MESSAGE_ALIGN }>, commit: u64, client_id: u128, @@ -4170,7 +4132,7 @@ mod tests { 1, 0, 1, - server_common::sharding::METADATA_GROUP, + server_common::sharding::METADATA_CONSENSUS_NAMESPACE, NoopBus, LocalPipeline::new(), ); @@ -4277,7 +4239,7 @@ mod tests { 1, 0, 1, - server_common::sharding::METADATA_GROUP, + server_common::sharding::METADATA_CONSENSUS_NAMESPACE, NoopBus, LocalPipeline::new(), ); @@ -4285,7 +4247,7 @@ mod tests { IggyMetadata::new(Some(consensus), None, None, None, TestMux::default(), None) } - fn create_topic_request(client: u128, wire_user_id: u32) -> Message { + fn create_topic_request(client: u128, wire_user_id: u32) -> Message { let body = CreateTopicRequest { stream_id: WireIdentifier::numeric(1), partitions_count: 1, @@ -4296,15 +4258,15 @@ mod tests { name: WireName::new("t").unwrap(), } .to_bytes(); - let header_size = size_of::(); + let header_size = size_of::(); let total = header_size + body.len(); - let mut message = Message::::new(total); + let mut message = Message::::new(total); { let slice = message.as_mut_slice(); slice[header_size..total].copy_from_slice(&body); let header = - bytemuck::checked::from_bytes_mut::(&mut slice[..header_size]); - *header = RoutedRequestHeader { + bytemuck::checked::from_bytes_mut::(&mut slice[..header_size]); + *header = RequestHeader { command: Command2::Request, operation: Operation::CreateTopic, size: u32::try_from(total).unwrap(), @@ -4312,7 +4274,7 @@ mod tests { session: 1, request: 1, user_id: wire_user_id, - group: server_common::sharding::METADATA_GROUP, + namespace: server_common::sharding::METADATA_CONSENSUS_NAMESPACE, ..Default::default() }; } @@ -4440,7 +4402,7 @@ mod tests { 1, 0, 1, - server_common::sharding::METADATA_GROUP, + server_common::sharding::METADATA_CONSENSUS_NAMESPACE, NoopBus, LocalPipeline::new(), ); @@ -4535,43 +4497,39 @@ mod tests { reply } - fn pat_create_request(client: u128, request: u64) -> Message { - let header_size = size_of::(); - let mut message = Message::::new(header_size); - let header = bytemuck::checked::from_bytes_mut::( + fn pat_create_request(client: u128, request: u64) -> Message { + let header_size = size_of::(); + let mut message = Message::::new(header_size); + let header = bytemuck::checked::from_bytes_mut::( &mut message.as_mut_slice()[..header_size], ); - *header = RoutedRequestHeader { + *header = RequestHeader { command: Command2::Request, operation: Operation::CreatePersonalAccessToken, size: u32::try_from(header_size).unwrap(), client, session: 1, request, - group: server_common::sharding::METADATA_GROUP, + namespace: server_common::sharding::METADATA_CONSENSUS_NAMESPACE, ..Default::default() }; message } - fn create_stream_request( - client: u128, - request: u64, - name: &str, - ) -> Message { + fn create_stream_request(client: u128, request: u64, name: &str) -> Message { let body = iggy_binary_protocol::requests::streams::CreateStreamRequest { name: WireName::new(name).unwrap(), } .to_bytes(); - let header_size = size_of::(); + let header_size = size_of::(); let total = header_size + body.len(); - let mut message = Message::::new(total); + let mut message = Message::::new(total); { let slice = message.as_mut_slice(); slice[header_size..total].copy_from_slice(&body); let header = - bytemuck::checked::from_bytes_mut::(&mut slice[..header_size]); - *header = RoutedRequestHeader { + bytemuck::checked::from_bytes_mut::(&mut slice[..header_size]); + *header = RequestHeader { command: Command2::Request, operation: Operation::CreateStream, size: u32::try_from(total).unwrap(), @@ -4579,7 +4537,7 @@ mod tests { session: 1, request, user_id: 0, - group: server_common::sharding::METADATA_GROUP, + namespace: server_common::sharding::METADATA_CONSENSUS_NAMESPACE, ..Default::default() }; } @@ -4621,7 +4579,7 @@ mod tests { 1, 0, 1, - server_common::sharding::METADATA_GROUP, + server_common::sharding::METADATA_CONSENSUS_NAMESPACE, StallBus::default(), LocalPipeline::new(), ); @@ -4730,87 +4688,6 @@ mod tests { ); } - /// A checkpoint reclaims the WAL prefix the snapshot supersedes, but must - /// stop one op short of the checkpoint op itself. - /// - /// That op is the replica's commit point, and its `DoViewChange` suffix is - /// floored there. The merge scans the commit point and may not discard it, - /// so a sender with no header to put there is deferring to a peer; when - /// every sender has checkpointed at the same op the view change deadlocks - /// (`dvc_merge::merge_dvc_quorum`). Checkpoints fire on local journal - /// occupancy, which is symmetric across replicas seeing the same ops, so - /// "every sender" is the ordinary case, not a coincidence. - #[compio::test] - async fn checkpoint_drain_retains_the_commit_point_header() { - const CLIENT: u128 = 1; - const SESSION: u64 = 1; - const ACTING_USER: u32 = 7; - const OPS: u64 = 5; - const CHECKPOINT_OP: u64 = 3; - - let dir = tempfile::tempdir().unwrap(); - std::fs::create_dir_all(dir.path().join(crate::impls::METADATA_DIR)).unwrap(); - let journal = - journal::prepare_journal::PrepareJournal::open(&dir.path().join("journal.wal"), 0) - .await - .unwrap(); - let consensus = VsrConsensus::new( - 1, - 0, - 1, - server_common::sharding::METADATA_GROUP, - NoopBus, - LocalPipeline::new(), - ); - consensus.init(); - let md: IggyMetadata<_, journal::prepare_journal::PrepareJournal, (), TestMux> = - IggyMetadata::new( - Some(consensus), - Some(journal), - None, - None, - TestMux::default(), - Some(dir.path().to_path_buf()), - ); - let consensus = md.consensus.as_ref().unwrap(); - md.client_table.borrow_mut().commit_register( - CLIENT, - ACTING_USER, - register_reply(CLIENT, SESSION), - ); - - for op in 1..=OPS { - let prepare = md - .prepare_request(create_stream_request(CLIENT, op, &format!("s{op}"))) - .expect("CreateStream is client-allowed"); - consensus.pipeline_message(PlaneKind::Metadata, &prepare); - md.on_replicate(prepare).await; - } - - let journal = md.journal.as_ref().unwrap(); - md.coordinator - .as_ref() - .expect("data_dir present arms the coordinator") - .drain(journal, CHECKPOINT_OP) - .await - .expect("drain the snapshotted prefix"); - - let header_at = |op: u64| journal.header(usize::try_from(op).expect("test ops fit usize")); - for op in 1..CHECKPOINT_OP { - assert!( - header_at(op).is_none(), - "op {op} is below the checkpoint and must be reclaimed" - ); - } - assert!( - header_at(CHECKPOINT_OP).is_some(), - "the checkpoint op is the commit point and must stay describable in a DVC" - ); - for op in CHECKPOINT_OP + 1..=OPS { - assert!(header_at(op).is_some(), "op {op} was never snapshotted"); - } - } - /// Reproduces the single-node "metadata prepare queue is full" wedge /// /// `checkpoint_if_needed` runs inside `on_replicate`, once per submit. @@ -4852,7 +4729,7 @@ mod tests { 1, 0, 1, - server_common::sharding::METADATA_GROUP, + server_common::sharding::METADATA_CONSENSUS_NAMESPACE, NoopBus, LocalPipeline::new(), ); @@ -4999,7 +4876,7 @@ mod tests { 1, 0, 1, - server_common::sharding::METADATA_GROUP, + server_common::sharding::METADATA_CONSENSUS_NAMESPACE, NoopBus, LocalPipeline::new(), ); @@ -5124,7 +5001,7 @@ mod tests { 1, 0, 1, - server_common::sharding::METADATA_GROUP, + server_common::sharding::METADATA_CONSENSUS_NAMESPACE, NoopBus, LocalPipeline::new(), ); @@ -5260,7 +5137,7 @@ mod tests { 1, 0, 1, - server_common::sharding::METADATA_GROUP, + server_common::sharding::METADATA_CONSENSUS_NAMESPACE, NoopBus, LocalPipeline::new(), ); diff --git a/core/metadata/src/impls/recovery.rs b/core/metadata/src/impls/recovery.rs index 3b2e009afd..286d286e20 100644 --- a/core/metadata/src/impls/recovery.rs +++ b/core/metadata/src/impls/recovery.rs @@ -23,9 +23,8 @@ use consensus::{ ClientTable, ClientTableDecodeError, VsrState, VsrStateError, build_reply_message, build_reply_message_with, }; -use iggy_binary_protocol::consensus::{CHECKSUM_UNSEALED, Operation, PrepareHeader}; +use iggy_binary_protocol::consensus::{Operation, PrepareHeader}; use iggy_common::IggyError; -use journal::Journal as _; use journal::prepare_journal::{JournalError, PrepareJournal}; use journal::superblock::{ PingPongSuperblock, SLOT_FILE_NAMES, SuperblockContents, SuperblockStore, @@ -308,13 +307,6 @@ pub struct RecoveredMetadata { /// they stay journal-only until the recovered primary re-replicates them /// (or a backup sees the commit point advance past them). pub last_journaled_op: Option, - /// First op replay could not connect to its predecessor, `None` when the - /// replayed range is one unbroken chain. - /// - /// `Some(op)` means entries at and above `op` were truncated and must come back - /// from the cluster. `last_journaled_op` stops below it, which keeps the restored - /// head, the re-pipeline range, and the recovery barrier honest. - pub chain_break_op: Option, } /// Recover metadata state from disk. @@ -553,35 +545,10 @@ where let mut last_applied_op: Option = None; let mut last_journaled_op: Option = None; - let mut chain_break_op: Option = None; - let mut previous: Option = None; for header in &headers_to_replay { - // Stop at the first op that does not connect to the one before it. Applying - // across a hole replays effects onto a state machine that never saw the - // missing op, and nothing downstream re-checks it. - // - // The WAL scan does not cover this: it only fires on CONSECUTIVE ops with - // both ends sealed, so a gap reaches here. The first replayed op is exempt, - // since a snapshot records no checksum for its parent to chain to. - if let Some(previous) = previous { - let gap = previous.op + 1 != header.op; - let broken_chain = previous.checksum != CHECKSUM_UNSEALED - && header.checksum != CHECKSUM_UNSEALED - && header.parent != previous.checksum; - if gap || broken_chain { - tracing::error!( - op = header.op, - previous_op = previous.op, - gap, - broken_chain, - "metadata WAL does not connect at this op; stopping replay and dropping the \ - suffix for VSR repair" - ); - chain_break_op = Some(header.op); - break; - } - } - previous = Some(*header); + // TODO: Check hash chain integrity against `previous_header`. On a + // same-view break, stop replay here and mark the remaining entries for + // repair via VSR instead of panicking. last_journaled_op = Some(header.op); if header.op > commit_watermark { @@ -659,22 +626,6 @@ where last_applied_op = Some(header.op); } - // `truncate_from`, never `drain`: the removed ops must stay refillable, so the - // snapshot watermark stays put. Leaving them resident would make `append` refuse - // the slot, failing repair on exactly the ops it exists to fix. - if let Some(break_op) = chain_break_op { - let removed = journal - .truncate_from(break_op) - .await - .map_err(RecoveryError::Io)?; - tracing::warn!( - break_op, - removed, - last_journaled_op, - "dropped the disconnected metadata WAL suffix; the cluster re-supplies these ops" - ); - } - Ok(RecoveredMetadata { journal, snapshot, @@ -685,7 +636,6 @@ where client_table, last_applied_op, last_journaled_op, - chain_break_op, }) } @@ -1026,130 +976,6 @@ mod tests { assert_eq!(recovered.journal.last_op(), Some(3)); } - /// A prepare sealed the way a live primary seals one: `parent` chains to the - /// previous op's identity and `checksum` is that identity. - fn make_chained_prepare(op: u64, commit: u64, parent: u128) -> Message { - let mut message = make_prepare_with_commit(op, commit, 32); - let header = bytemuck::checked::from_bytes_mut::( - &mut message.as_mut_slice()[..HEADER_SIZE], - ); - header.parent = parent; - let checksum = header.identity_checksum(); - header.checksum = checksum; - message - } - - #[compio::test] - async fn recover_stops_at_a_gap_and_drops_the_disconnected_suffix() { - // Ops 1-3 then 5: op 4 never landed. Replaying 5 over a state machine that - // never saw 4 diverges silently, and the WAL scan waves this through -- - // its chain check only fires on CONSECUTIVE ops, since a gap is also what - // ordinary compaction leaves behind. - let dir = tempdir().unwrap(); - let metadata_dir = dir.path().join("metadata"); - std::fs::create_dir_all(&metadata_dir).unwrap(); - - { - let journal = PrepareJournal::open(&metadata_dir.join("journal.wal"), 0) - .await - .unwrap(); - for op in 1..=3u64 { - journal - .append(make_prepare_with_commit(op, op, 32)) - .await - .unwrap(); - } - journal - .append(make_prepare_with_commit(5, 5, 32)) - .await - .unwrap(); - journal.storage_ref().fsync().await.unwrap(); - } - - let recovered = recover::( - dir.path(), - CLUSTERED, - journal::prepare_journal::DEFAULT_SLOT_COUNT, - CLIENTS_TABLE_MAX, - |_| {}, - ) - .await - .unwrap(); - - assert_eq!(recovered.chain_break_op, Some(5)); - assert_eq!( - recovered.last_applied_op, - Some(3), - "op 5 must not apply across the hole at op 4" - ); - assert_eq!( - recovered.last_journaled_op, - Some(3), - "the restored head stops below the break, so nothing re-pipelines it" - ); - assert_eq!( - recovered.journal.last_op(), - Some(3), - "the disconnected entry is dropped so repair can journal the cluster's op 5" - ); - assert_eq!( - recovered.journal.snapshot_op(), - 0, - "truncating a suffix must leave the watermark, or the ops stop being refillable" - ); - } - - #[compio::test] - async fn recover_stops_at_a_broken_chain_between_consecutive_ops() { - // Consecutive and sealed on both ends, but op 3 names a parent that is not - // op 2: a fork left by a crash mid view change. Ops are appended out of - // ascending file order so the scan's own chain check does not fire first. - let dir = tempdir().unwrap(); - let metadata_dir = dir.path().join("metadata"); - std::fs::create_dir_all(&metadata_dir).unwrap(); - - { - let journal = PrepareJournal::open(&metadata_dir.join("journal.wal"), 0) - .await - .unwrap(); - let first = make_chained_prepare(1, 1, 0); - let first_checksum = first.header().checksum; - journal.append(first).await.unwrap(); - let second = make_chained_prepare(2, 2, first_checksum); - journal.append(second).await.unwrap(); - // Parent of a prepare that is not op 2. - journal - .append(make_chained_prepare(3, 3, 0xdead_beef)) - .await - .unwrap(); - journal.storage_ref().fsync().await.unwrap(); - } - - let recovered = recover::( - dir.path(), - CLUSTERED, - journal::prepare_journal::DEFAULT_SLOT_COUNT, - CLIENTS_TABLE_MAX, - |_| {}, - ) - .await; - - // The WAL scan reaches this first and refuses boot: consecutive ops, both - // sealed, chain broken, with no entry after it is only a tail. Either - // outcome is a refusal to apply the fork; what must never happen is a - // clean recovery that replayed op 3. - match recovered { - Err(RecoveryError::Journal(_) | RecoveryError::Io(_)) => {} - Ok(recovered) => { - assert!( - recovered.last_applied_op < Some(3), - "op 3 forks the chain and must not be applied" - ); - } - Err(other) => panic!("unexpected recovery error: {other:?}"), - } - } - #[compio::test] async fn recover_applies_only_the_committed_prefix() { let dir = tempdir().unwrap(); diff --git a/core/metadata/src/stm/result.rs b/core/metadata/src/stm/result.rs index 6a610bf3a5..67a0a1383d 100644 --- a/core/metadata/src/stm/result.rs +++ b/core/metadata/src/stm/result.rs @@ -183,7 +183,6 @@ result_enum!(CreatePartitionsResult { result_enum!(DeletePartitionsResult { StreamNotFound = 1009, TopicNotFound = 2010, - InvalidPartitionsCount = 2019, }); // `TruncatePartition` is the committed form of a client `DeleteSegments`; an // unresolvable target commits as a rejection so the request sequence stays diff --git a/core/metadata/src/stm/snapshot.rs b/core/metadata/src/stm/snapshot.rs index e9fadca173..ca28a30a4a 100644 --- a/core/metadata/src/stm/snapshot.rs +++ b/core/metadata/src/stm/snapshot.rs @@ -53,10 +53,6 @@ pub enum SnapshotError { commit_op: u64, table_frontier: u64, }, - /// The snapshot was written under a different format version. Refuse it - /// rather than reinterpret embedded raw bytes (the client table's cached - /// replies are wire `ReplyHeader` frames) under the wrong layout. - UnsupportedVersion { found: u32, supported: u32 }, } /// Stage at which snapshot persistence failed. @@ -108,13 +104,6 @@ impl fmt::Display for SnapshotError { commit_op {commit_op}, table frontier {table_frontier}" ) } - Self::UnsupportedVersion { found, supported } => { - write!( - f, - "unsupported metadata snapshot version {found}; this build reads only \ - version {supported}" - ) - } } } } @@ -127,8 +116,7 @@ impl std::error::Error for SnapshotError { Self::Io(e) | Self::Persist { source: e, .. } => Some(e), Self::ChecksumMismatch { .. } | Self::Truncated { .. } - | Self::IncoherentManifest { .. } - | Self::UnsupportedVersion { .. } => None, + | Self::IncoherentManifest { .. } => None, } } } @@ -150,17 +138,10 @@ impl From for SnapshotError { /// replicas with identical state must serialize identically. Regression guards: /// `stream::tests::populated_streams_snapshot_reencode_is_byte_stable` and /// `impls::metadata::tests::populated_snapshot_reencode_and_checksum_are_stable`. -/// Current [`MetadataSnapshot::version`]. Bump whenever the serialized form -/// changes meaning without changing shape -- in particular the client table's -/// cached replies, which are embedded as raw `ReplyHeader` wire bytes msgpack -/// cannot introspect. Version 2: `status` sits at reply-header offset 216 -/// (version 1 carried a `namespace` word before it). -pub const METADATA_SNAPSHOT_VERSION: u32 = 2; - #[derive(Debug, Clone, Serialize, Deserialize)] pub struct MetadataSnapshot { - /// Snapshot format version; [`MetadataSnapshot::decode`] refuses any other - /// value (see [`METADATA_SNAPSHOT_VERSION`]). + /// Snapshot format version for forward/backward compatibility. + /// TODO(krishvishal): Properly handle versioning for snapshot. This is a placeholder for now. pub version: u32, /// Timestamp when the snapshot was created (microseconds since epoch). pub created_at: u64, @@ -190,7 +171,7 @@ impl MetadataSnapshot { #[must_use] pub const fn new(sequence_number: u64) -> Self { Self { - version: METADATA_SNAPSHOT_VERSION, + version: 1, // Deterministic placeholder. The real creation time is stamped by // `Snapshot::create` from the consensus-injected clock (see // `VsrConsensus::clock_realtime_micros`) so a replayed simulator @@ -215,18 +196,9 @@ impl MetadataSnapshot { /// Decode a snapshot from msgpack bytes. /// /// # Errors - /// Returns `SnapshotError::Deserialize` if msgpack deserialization fails, - /// or `SnapshotError::UnsupportedVersion` if the snapshot was written - /// under a different format version. + /// Returns `SnapshotError::Deserialize` if msgpack deserialization fails. pub fn decode(bytes: &[u8]) -> Result { - let snapshot: Self = rmp_serde::from_slice(bytes).map_err(SnapshotError::Deserialize)?; - if snapshot.version != METADATA_SNAPSHOT_VERSION { - return Err(SnapshotError::UnsupportedVersion { - found: snapshot.version, - supported: METADATA_SNAPSHOT_VERSION, - }); - } - Ok(snapshot) + rmp_serde::from_slice(bytes).map_err(SnapshotError::Deserialize) } } @@ -468,25 +440,6 @@ mod tests { assert!(decoded.client_table.is_none()); } - // The client table's cached replies are embedded as raw `ReplyHeader` - // wire bytes msgpack cannot introspect, so a snapshot from a different - // format version must be refused, never reinterpreted under the current - // header layout. - #[test] - fn decode_refuses_a_snapshot_from_another_format_version() { - let mut snapshot = MetadataSnapshot::new(42); - snapshot.version = METADATA_SNAPSHOT_VERSION - 1; - - let encoded = snapshot.encode().unwrap(); - assert!(matches!( - MetadataSnapshot::decode(&encoded), - Err(SnapshotError::UnsupportedVersion { - found, - supported: METADATA_SNAPSHOT_VERSION, - }) if found == METADATA_SNAPSHOT_VERSION - 1 - )); - } - #[test] fn roundtrip_with_data() { let ts = IggyTimestamp::from(1_694_968_446_131_680_u64); diff --git a/core/metadata/src/stm/stream.rs b/core/metadata/src/stm/stream.rs index e4dfc1b12c..0a97ddc8e2 100644 --- a/core/metadata/src/stm/stream.rs +++ b/core/metadata/src/stm/stream.rs @@ -183,7 +183,7 @@ pub struct Topic { /// key (keyed by group id) can't be inherited by a recreated group. /// /// Ceiling: the partition-plane offset key is `u32`, so a group id must stay - /// within `u32::MAX` (the wire rewrite in `the server` clamps past-ceiling + /// within `u32::MAX` (the wire rewrite in `server-ng` clamps past-ceiling /// ids to `u32::MAX` rather than panic). ~4 billion group creates on a /// single topic is unreachable in practice, but the cap is real -- past it /// clamped wire ids all collide on `u32::MAX`, including with a live @@ -362,22 +362,7 @@ impl Stream { pub struct StatsRegistry { streams: std::sync::Mutex>>, topics: std::sync::Mutex>>, - partitions: std::sync::Mutex>, -} - -/// Shared partition counters plus the purge generation they were last reset for. -#[derive(Debug)] -struct PartitionEntry { - stats: Arc, - /// Highest [`Partition::purge_generation`] this entry's counters were reset - /// for, the registry's mirror of the partition plane's - /// `applied_purge_generation` gate. - /// - /// Load-bearing: an apply runs on BOTH left-right buffers and the second run - /// is deferred to the next metadata publish, which can be long after the - /// purge acked. Counters are shared side state (one `Arc` across buffers), - /// so an ungated second reset would wipe messages sent since the purge. - purged_generation: u64, + partitions: std::sync::Mutex>>, } impl StatsRegistry { @@ -422,11 +407,7 @@ impl StatsRegistry { .lock() .expect("stats registry mutex poisoned") .entry((stream_id, topic_id, partition_id)) - .or_insert_with(|| PartitionEntry { - stats: Arc::new(PartitionStats::new(parent)), - purged_generation: 0, - }) - .stats + .or_insert_with(|| Arc::new(PartitionStats::new(parent))) .clone() } @@ -445,63 +426,7 @@ impl StatsRegistry { .lock() .expect("stats registry mutex poisoned") .get(&(stream_id, topic_id, partition_id)) - .map(|entry| entry.stats.clone()) - } - - /// Reset the counters of every partition a purge just advanced, so a client - /// that reads right after the ack sees the purge instead of pre-purge - /// totals. The on-disk reset stays async (the reconciler resets each - /// partition on every replica once it observes the committed generation); - /// this only moves the counters to the shape that reset converges on. - /// - /// Reset, never decrement: `zero_out_all` swaps in 0 and rolls each parent - /// back by exactly what it swapped out, so a replayed purge entry over an - /// already-zeroed registry cannot underflow a parent total. The generation - /// gate on top makes the replay a no-op outright. - /// - /// The entry is created when missing so the gate is recorded even for a - /// partition this node has not materialized yet. A fresh entry holds no - /// segment, and `ensure_initial_segment` counts the one it plants, hence - /// the segment is restored only for a partition that already had storage -- - /// inventing one here would double-count against that later bump. - // The guard spans a read-modify-write of one entry (check the gate, stamp - // it, take the `Arc`), so it cannot collapse into the single chained - // expression the drop-tightening lint asks for. - #[allow(clippy::significant_drop_tightening)] - fn reset_purged_partitions( - &self, - stream_id: usize, - topic_id: usize, - parent: &Arc, - partitions: &[Partition], - ) { - for partition in partitions { - // Guard dropped before the counters move: `zero_out_all` cascades a - // rollback into the parent topic and stream totals, which the - // registry map has no part in. - let stats = { - let mut entries = self - .partitions - .lock() - .expect("stats registry mutex poisoned"); - let entry = entries - .entry((stream_id, topic_id, partition.id)) - .or_insert_with(|| PartitionEntry { - stats: Arc::new(PartitionStats::new(Arc::clone(parent))), - purged_generation: 0, - }); - if entry.purged_generation >= partition.purge_generation { - continue; - } - entry.purged_generation = partition.purge_generation; - entry.stats.clone() - }; - let had_storage = stats.segments_count_inconsistent() > 0; - stats.zero_out_all(); - if had_storage { - stats.increment_segments_count(1); - } - } + .cloned() } fn remove_stream(&self, id: usize) { @@ -1583,31 +1508,28 @@ impl StateHandler for PurgeStreamRequest { type State = StreamsInner; fn apply(&self, state: &mut StreamsInner, _timestamp: IggyTimestamp) -> ApplyReply { // Stream purge = topic purge over every topic in the stream: advance - // each partition's monotonic purge generation, clear the delete - // watermark, and reset the partition counters; every replica's - // reconciler observes the committed generation and resets the partition - // to a single empty segment at offset 0 with cleared offsets (see - // `PurgeTopicRequest`). Metadata shape stays intact. - let Some(stream_id) = state.resolve_stream_id(&self.stream_id) else { - return ApplyReply::err(PurgeStreamResult::StreamNotFound); - }; - let Some(stream) = state.items.get_mut(stream_id) else { - return ApplyReply::err(PurgeStreamResult::StreamNotFound); - }; - let mut advanced = false; - for (topic_id, topic) in &mut stream.topics { - for partition in &mut topic.partitions { - partition.purge_generation = partition.purge_generation.wrapping_add(1); - partition.deleted_up_to_offset = 0; - advanced = true; + // each partition's monotonic purge generation and clear the delete + // watermark; every replica's reconciler observes the committed + // generation and resets the partition to a single empty segment at + // offset 0 with cleared offsets (see `PurgeTopicRequest`). Metadata + // shape stays intact. + let advanced = { + let Some(stream_id) = state.resolve_stream_id(&self.stream_id) else { + return ApplyReply::err(PurgeStreamResult::StreamNotFound); + }; + let Some(stream) = state.items.get_mut(stream_id) else { + return ApplyReply::err(PurgeStreamResult::StreamNotFound); + }; + let mut advanced = false; + for (_, topic) in &mut stream.topics { + for partition in &mut topic.partitions { + partition.purge_generation = partition.purge_generation.wrapping_add(1); + partition.deleted_up_to_offset = 0; + advanced = true; + } } - state.stats_registry.reset_purged_partitions( - stream_id, - topic_id, - &topic.stats, - &topic.partitions, - ); - } + advanced + }; if advanced { state.revision = state.revision.wrapping_add(1); } @@ -1850,34 +1772,25 @@ impl StateHandler for PurgeTopicRequest { // offsets at 0 and drops the consumer-offset barrier that bounded the // trim, and the reconciler re-stages any nonzero watermark on every // pass -- a surviving one would delete post-purge segments. - // - // The shared partition counters are reset here too: they are read back - // by `get_topic` / `get_stream` on any node that applied this commit, - // and leaving them until the reconciler runs makes a purge ack followed - // by a read report pre-purge totals. - let Some(stream_id) = state.resolve_stream_id(&self.stream_id) else { - return ApplyReply::err(PurgeTopicResult::StreamNotFound); - }; - let Some(topic_id) = state.resolve_topic_id(stream_id, &self.topic_id) else { - return ApplyReply::err(PurgeTopicResult::TopicNotFound); - }; - let Some(stream) = state.items.get_mut(stream_id) else { - return ApplyReply::err(PurgeTopicResult::StreamNotFound); - }; - let Some(topic) = stream.topics.get_mut(topic_id) else { - return ApplyReply::err(PurgeTopicResult::TopicNotFound); + let advanced = { + let Some(stream_id) = state.resolve_stream_id(&self.stream_id) else { + return ApplyReply::err(PurgeTopicResult::StreamNotFound); + }; + let Some(topic_id) = state.resolve_topic_id(stream_id, &self.topic_id) else { + return ApplyReply::err(PurgeTopicResult::TopicNotFound); + }; + let Some(stream) = state.items.get_mut(stream_id) else { + return ApplyReply::err(PurgeTopicResult::StreamNotFound); + }; + let Some(topic) = stream.topics.get_mut(topic_id) else { + return ApplyReply::err(PurgeTopicResult::TopicNotFound); + }; + for partition in &mut topic.partitions { + partition.purge_generation = partition.purge_generation.wrapping_add(1); + partition.deleted_up_to_offset = 0; + } + !topic.partitions.is_empty() }; - for partition in &mut topic.partitions { - partition.purge_generation = partition.purge_generation.wrapping_add(1); - partition.deleted_up_to_offset = 0; - } - let advanced = !topic.partitions.is_empty(); - state.stats_registry.reset_purged_partitions( - stream_id, - topic_id, - &topic.stats, - &topic.partitions, - ); if advanced { state.revision = state.revision.wrapping_add(1); } @@ -1979,12 +1892,8 @@ impl StateHandler for DeletePartitionsRequest { }; let count_to_delete = self.partitions_count as usize; - if count_to_delete > topic.partitions.len() { - return ApplyReply::err(DeletePartitionsResult::InvalidPartitionsCount); - } - // Zero count is rejected pre-consensus; a replayed legacy entry still - // applies as the historical ok no-op. - if count_to_delete > 0 { + let did_delete = count_to_delete > 0 && count_to_delete <= topic.partitions.len(); + if did_delete { let retained = topic.partitions.len() - count_to_delete; topic.partitions.truncate(retained); // Members assigned the removed partitions must give them up. @@ -1994,6 +1903,8 @@ impl StateHandler for DeletePartitionsRequest { state .stats_registry .remove_partitions_from(stream_id, topic_id, retained); + } + if did_delete { state.revision = state.revision.wrapping_add(1); } ApplyReply::ok(Bytes::new()) @@ -2554,63 +2465,6 @@ mod tests { assert!(apply.body.is_empty()); } - /// Over-count deletes were acked ok as a silent no-op; they must commit the - /// legacy `InvalidPartitionsCount` rejection. Zero stays an ok no-op at the - /// apply (rejected pre-consensus; a replayed entry keeps its historical ack). - #[test] - fn given_delete_partitions_counts_when_applied_should_reject_over_count() { - let cases: &[(u32, u32, u32, usize)] = &[ - // (partitions in topic, count to delete, expected code, remaining) - ( - 3, - 4, - u32::from(DeletePartitionsResult::InvalidPartitionsCount), - 3, - ), - ( - 0, - 1, - u32::from(DeletePartitionsResult::InvalidPartitionsCount), - 0, - ), - (3, 0, 0, 3), - (3, 3, 0, 0), - (3, 2, 0, 1), - ]; - for &(partitions_count, count_to_delete, expected_code, expected_remaining) in cases { - let mut inner = StreamsInner::new(); - create_stream(&mut inner, "stream"); - let create_topic = CreateTopicWithAssignmentsRequest { - request: make_topic_request(0, partitions_count, "topic"), - partitions: (0..partitions_count) - .map(|partition_id| CreatedPartitionAssignment { - partition_id, - consensus_group_id: 1, - }) - .collect(), - }; - let _ = StateHandler::apply(&create_topic, &mut inner, IggyTimestamp::now()); - - let delete = DeletePartitionsRequest { - stream_id: WireIdentifier::numeric(0), - topic_id: WireIdentifier::numeric(0), - partitions_count: count_to_delete, - }; - let apply = StateHandler::apply(&delete, &mut inner, IggyTimestamp::now()); - - assert_eq!( - apply.code, expected_code, - "deleting {count_to_delete} of {partitions_count} partitions" - ); - assert!(apply.body.is_empty()); - assert_eq!( - inner.items[0].topics[0].partitions.len(), - expected_remaining, - "deleting {count_to_delete} of {partitions_count} partitions" - ); - } - } - #[test] fn given_live_stream_when_apply_purge_stream_should_return_ok_with_empty_body() { let mut inner = StreamsInner::new(); @@ -2687,181 +2541,6 @@ mod tests { ); } - /// A purge acks on commit while the on-disk reset waits for the reconciler, - /// so the counters `get_topic` / `get_stream` read must move in the apply or - /// a read right after the ack reports pre-purge totals. - #[test] - fn given_counted_partition_when_apply_purge_topic_should_zero_the_scope() { - let mut inner = inner_with_registered_partition(); - let stats = inner.stats_registry.partition_get(0, 0, 0).expect("stats"); - stats.increment_segments_count(1); - stats.increment_messages_count(7); - stats.increment_size_bytes(512); - stats.set_current_offset(6); - assert_eq!( - inner.items[0].topics[0].stats.messages_count_inconsistent(), - 7, - "partition counters must roll up before the purge, or the test proves nothing" - ); - - let purge = PurgeTopicRequest { - stream_id: WireIdentifier::numeric(0), - topic_id: WireIdentifier::numeric(0), - }; - let apply = StateHandler::apply(&purge, &mut inner, IggyTimestamp::now()); - assert_eq!(apply.code, 0); - - assert_eq!(stats.messages_count_inconsistent(), 0); - assert_eq!(stats.size_bytes_inconsistent(), 0); - assert_eq!(stats.current_offset(), 0); - assert_eq!( - stats.segments_count_inconsistent(), - 1, - "a purged partition keeps the one empty segment the reset lands on" - ); - let topic_stats = &inner.items[0].topics[0].stats; - assert_eq!(topic_stats.messages_count_inconsistent(), 0); - assert_eq!(topic_stats.size_bytes_inconsistent(), 0); - let stream_stats = &inner.items[0].stats; - assert_eq!(stream_stats.messages_count_inconsistent(), 0); - assert_eq!(stream_stats.size_bytes_inconsistent(), 0); - } - - /// A stream purge walks every topic, so every topic's partitions must reset, - /// not just the first one. - #[test] - fn given_counted_partitions_when_apply_purge_stream_should_zero_every_topic() { - let mut inner = inner_with_registered_partition(); - let create_topic = CreateTopicWithAssignmentsRequest { - request: make_topic_request(0, 1, "metrics"), - partitions: vec![CreatedPartitionAssignment { - partition_id: 0, - consensus_group_id: 2, - }], - }; - let _ = StateHandler::apply(&create_topic, &mut inner, IggyTimestamp::now()); - let second_topic_stats = inner.items[0].topics[1].stats.clone(); - inner.stats_registry.partition(0, 1, 0, second_topic_stats); - - let counters: Vec> = (0..2) - .map(|topic_id| { - let stats = inner - .stats_registry - .partition_get(0, topic_id, 0) - .expect("stats"); - stats.increment_segments_count(1); - stats.increment_messages_count(9); - stats.increment_size_bytes(64); - stats - }) - .collect(); - assert_eq!(inner.items[0].stats.messages_count_inconsistent(), 18); - - let purge = PurgeStreamRequest { - stream_id: WireIdentifier::numeric(0), - }; - let apply = StateHandler::apply(&purge, &mut inner, IggyTimestamp::now()); - assert_eq!(apply.code, 0); - - for stats in &counters { - assert_eq!(stats.messages_count_inconsistent(), 0); - assert_eq!(stats.size_bytes_inconsistent(), 0); - assert_eq!(stats.segments_count_inconsistent(), 1); - } - assert_eq!(inner.items[0].stats.messages_count_inconsistent(), 0); - assert_eq!(inner.items[0].stats.size_bytes_inconsistent(), 0); - } - - /// The left-right buffers absorb every op twice and the second absorb is - /// deferred to the next metadata publish, which can land long after the - /// purge acked. Counters are shared side state, so the deferred replay must - /// leave post-purge traffic alone -- and must not decrement a parent total - /// it already rolled back. - #[test] - fn given_purged_buffer_when_other_buffer_replays_purge_should_keep_new_counters() { - let mut first = inner_with_registered_partition(); - let mut second = first.clone(); - let stats = first.stats_registry.partition_get(0, 0, 0).expect("stats"); - stats.increment_segments_count(1); - stats.increment_messages_count(10); - stats.increment_size_bytes(320); - - let purge = PurgeTopicRequest { - stream_id: WireIdentifier::numeric(0), - topic_id: WireIdentifier::numeric(0), - }; - let _ = StateHandler::apply(&purge, &mut first, IggyTimestamp::now()); - assert_eq!(stats.messages_count_inconsistent(), 0); - - // Sent after the ack, before the deferred absorb on the other buffer. - stats.increment_messages_count(4); - stats.increment_size_bytes(128); - - let _ = StateHandler::apply(&purge, &mut second, IggyTimestamp::now()); - assert_eq!( - second.items[0].topics[0].partitions[0].purge_generation, 1, - "the replay computes the same generation, so the gate is what stops it" - ); - assert_eq!( - stats.messages_count_inconsistent(), - 4, - "the deferred replay must not wipe post-purge counters" - ); - assert_eq!(stats.size_bytes_inconsistent(), 128); - let topic_stats = first.items[0].topics[0].stats.clone(); - assert_eq!( - topic_stats.messages_count_inconsistent(), - 4, - "a second rollback of the same total would underflow the parent" - ); - assert_eq!(topic_stats.size_bytes_inconsistent(), 128); - - // A genuinely new purge still resets: the gate is per generation. - let _ = StateHandler::apply(&purge, &mut first, IggyTimestamp::now()); - assert_eq!(stats.messages_count_inconsistent(), 0); - assert_eq!(topic_stats.messages_count_inconsistent(), 0); - } - - /// Boot replays the metadata WAL before any partition materializes, so the - /// purge has no counters to reset -- but it must still record the gate, or - /// the deferred second absorb wipes whatever the partition loaded since. - #[test] - fn given_unmaterialized_partition_when_apply_purge_should_gate_the_replay() { - let mut inner = StreamsInner::new(); - create_stream(&mut inner, "alpha"); - let create_topic = CreateTopicWithAssignmentsRequest { - request: make_topic_request(0, 1, "logs"), - partitions: vec![CreatedPartitionAssignment { - partition_id: 0, - consensus_group_id: 1, - }], - }; - let _ = StateHandler::apply(&create_topic, &mut inner, IggyTimestamp::now()); - let mut replay = inner.clone(); - - let purge = PurgeTopicRequest { - stream_id: WireIdentifier::numeric(0), - topic_id: WireIdentifier::numeric(0), - }; - let _ = StateHandler::apply(&purge, &mut inner, IggyTimestamp::now()); - - // The data plane materializes the partition afterwards and counts what - // it plants; the purge must not have invented a segment for it. - let topic_stats = inner.items[0].topics[0].stats.clone(); - let stats = inner.stats_registry.partition(0, 0, 0, topic_stats); - assert_eq!(stats.segments_count_inconsistent(), 0); - stats.increment_segments_count(1); - stats.increment_messages_count(5); - - let _ = StateHandler::apply(&purge, &mut replay, IggyTimestamp::now()); - assert_eq!( - stats.messages_count_inconsistent(), - 5, - "the gate recorded at apply must survive into the partition's entry" - ); - assert_eq!(stats.segments_count_inconsistent(), 1); - } - #[test] fn given_missing_topic_when_apply_purge_topic_should_return_topic_not_found() { let mut inner = StreamsInner::new(); diff --git a/core/metadata/src/stm/user.rs b/core/metadata/src/stm/user.rs index 7dffbeb8ef..18b7a2e567 100644 --- a/core/metadata/src/stm/user.rs +++ b/core/metadata/src/stm/user.rs @@ -267,7 +267,7 @@ impl Users { "root username length {length} outside {MIN_USERNAME_LENGTH}..={MAX_USERNAME_LENGTH}; fix IGGY_ROOT_USERNAME" ); - // Boot-only invariant: the server calls this before listeners and + // Boot-only invariant: server-ng calls this before listeners and // consensus traffic start, on shard 0 initialization. The read/apply // split cannot race another user creation in that phase. let username = WireName::new(username).expect("root username must be valid"); @@ -571,7 +571,7 @@ impl StateHandler for ChangePasswordRequest { }; // An empty `new_password` is the primary's signal that the caller's - // current password did not match (see the server + // current password did not match (see server-ng // `verify_and_rewrite_change_password`): the accept path always // replicates a non-empty Argon2 hash, so this is unambiguous. Rejecting // here (rather than denying pre-consensus) commits the op as a no-op, diff --git a/core/partitions/Cargo.toml b/core/partitions/Cargo.toml index 4e29b1a3b6..0acc918ba1 100644 --- a/core/partitions/Cargo.toml +++ b/core/partitions/Cargo.toml @@ -32,7 +32,7 @@ publish = false # Simulator-only detector hook (`IggyPartitions::hold_borrow_across_await`): # deliberately holds a `with_partition` borrow across an `.await` so the # dispatch shell can prove its borrow-across-await detector. A -# `-p iggy-server` build excludes it; `cargo build --workspace` unifies +# `-p iggy-server-ng` build excludes it; `cargo build --workspace` unifies # features so the shared `partitions` unit compiles it in when the simulator # requests it. No production caller. simulator = [] @@ -48,7 +48,6 @@ iggy_binary_protocol = { workspace = true } iggy_common = { workspace = true } journal = { workspace = true } message_bus = { workspace = true } -nix = { workspace = true } papaya = { workspace = true } ringbuffer = { workspace = true } server_common = { workspace = true } @@ -56,9 +55,6 @@ smallvec = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } -[dev-dependencies] -tempfile = { workspace = true } - [lints.clippy] enum_glob_use = "deny" pedantic = "deny" diff --git a/core/partitions/src/iggy_partition.rs b/core/partitions/src/iggy_partition.rs index e5ed479a19..148c4078b9 100644 --- a/core/partitions/src/iggy_partition.rs +++ b/core/partitions/src/iggy_partition.rs @@ -40,9 +40,8 @@ use consensus::{ ReplicaLogContext, RequestLogEvent, Sequencer, SimEventKind, VsrConsensus, ack_preflight, ack_quorum_reached, build_deny_reply_from_request, build_reply_from_request, build_reply_message, drain_committable_prefix, emit_namespace_progress_event, - emit_partition_diag, emit_sim_event, fence_old_prepare_by_commit, - replicate_frozen_to_next_in_chain, replicate_preflight, restamp_prepare_view, - send_prepare_ok as send_prepare_ok_common, verify_prepare_integrity, + emit_partition_diag, emit_sim_event, fence_old_prepare_by_commit, replicate_preflight, + replicate_to_next_in_chain, send_prepare_ok as send_prepare_ok_common, }; use iggy_binary_protocol::requests::consumer_offsets::{ DeleteConsumerOffset2Request, DeleteConsumerOffsetRequest, StoreConsumerOffset2Request, @@ -54,7 +53,7 @@ use iggy_binary_protocol::responses::messages::{ use iggy_binary_protocol::{ AckLevel, GenericHeader, Operation, PrepareHeader, WireDecode, WireEncode, WireIdentifier, }; -use iggy_binary_protocol::{PrepareOkHeader, RoutedRequestHeader}; +use iggy_binary_protocol::{PrepareOkHeader, RequestHeader}; use iggy_common::{ ConsumerGroupId, ConsumerGroupOffsets, ConsumerKind, ConsumerOffset, ConsumerOffsets, IggyByteSize, IggyError, IggyExpiry, IggyTimestamp, PartitionStats, PollingKind, @@ -70,8 +69,8 @@ use server_common::{ MESSAGE_ALIGN, Message, SegmentStorage, iobuf::{Frozen, Owned}, send_messages2::{ - ChecksumMode, SendMessages2Header, convert_request_message, decode_prepare_slice, - decode_prepare_slice_trusted, stamp_prepare_for_persistence, + ChecksumMode, convert_request_message, decode_prepare_slice, decode_prepare_slice_trusted, + stamp_prepare_for_persistence, verify_received_send_messages, }, sharding::IggyNamespace, }; @@ -279,7 +278,7 @@ where // are the ones diagnostics actually key on. fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("IggyPartition") - .field("namespace", &self.consensus.group()) + .field("namespace", &self.consensus.namespace()) .field("offset", &self.offset) .field("dirty_offset", &self.dirty_offset) .field("should_increment_offset", &self.should_increment_offset) @@ -293,12 +292,12 @@ where } /// Post-preflight dispatch in `on_request`: replicate via VSR or take the -/// `NoAck` leader-local fast path. `RoutedRequestHeader` is boxed to avoid the +/// `NoAck` leader-local fast path. `RequestHeader` is boxed to avoid the /// 277-byte inline variant tripping clippy's `large_enum_variant`. enum Disposition { Replicate(Message), NoAck { - request_header: Box, + request_header: Box, kind: ConsumerKind, consumer_id: u32, offset: Option, @@ -719,7 +718,7 @@ where target: "iggy.partitions.diag", plane = "partitions", replica_id = self.consensus.replica(), - namespace_raw = self.consensus.group(), + namespace_raw = self.consensus.namespace(), view = state.view, log_view = state.log_view, superblock_write_failures = failures, @@ -764,31 +763,14 @@ where return; } tracing::info!( - namespace_raw = self.consensus().group(), + namespace_raw = self.consensus().namespace(), offset_frontier = frontier, "restored partition offset frontier from its superblock" ); self.offset.store(recovered_end, Ordering::Release); self.dirty_offset.store(recovered_end, Ordering::Relaxed); self.should_increment_offset = true; - } - - /// Copy this incarnation's offset counter into the shared - /// [`PartitionStats`], making it the value readers (offset validation, - /// `get_topic`, `get_stats`) see. - /// - /// Called from [`IggyPartitions::insert`](crate::IggyPartitions::insert) - /// only: when the instance BECOMES the addressable one, never while - /// building it. The stats registry keys on the namespace, not the - /// incarnation, so every build of a namespace holds the same `Arc` as - /// whatever is already serving it -- and a build is not guaranteed to be - /// adopted. Seeding from the build instead leaves a zeroed `current_offset` - /// on the live incarnation, which then rejects every - /// `store_consumer_offset` above 0 with `InvalidOffset` until the next send - /// re-seeds it. - pub(crate) fn publish_current_offset(&self) { - self.stats - .set_current_offset(self.offset.load(Ordering::Acquire)); + self.stats.set_current_offset(recovered_end); } /// The next message offset this replica will mint, `0` while the offset @@ -1316,7 +1298,7 @@ where #[allow(clippy::future_not_send)] async fn apply_consumer_offset_no_ack( &self, - request_header: Box, + request_header: Box, kind: ConsumerKind, consumer_id: u32, offset: Option, @@ -1434,7 +1416,6 @@ where &mut self, consumer: PollingConsumer, args: &PollingArgs, - validate_checksum: bool, ) -> PollPlan { // Reads the durable commit frontier (`self.offset`, stored only on // commit). Also used below as the poll's high-water bound: this function @@ -1546,7 +1527,6 @@ where segments, start_position, namespace_raw: self.namespace().inner(), - validate_checksum, }; // Snapshot the resident journal tail now (on the pump, under the // borrow) so the straddle splice runs off-task on owned data with no @@ -1633,9 +1613,68 @@ where &mut self, message: Message, ) -> Result { - self.stamp_and_append_messages(message) + let header = *message.header(); + if header.operation != Operation::SendMessages { + return Err(IggyError::CannotAppendMessage); + } + + let dirty_offset = if self.should_increment_offset { + self.dirty_offset.load(Ordering::Relaxed) + 1 + } else { + 0 + }; + + // Reuse the prepare's monotonic timestamp, assigned once by the primary + // in `project()` (`next_monotonic_timestamp`) and replicated verbatim to + // every backup. Sourcing it here instead of a fresh local `now()` makes + // the persisted `base_timestamp` (and the `batch_checksum` derived from + // it) byte-identical across replicas; a local `now()` diverges per node. + let batch_timestamp = header.timestamp; + let (message, batch, batch_messages_count) = + stamp_prepare_for_persistence(message, dirty_offset, batch_timestamp) + .map_err(|_| IggyError::CannotAppendMessage)?; + + if batch_messages_count == 0 { + return Ok(AppendResult::new(0, 0, 0)); + } + + let batch_messages_size = + u64::try_from(batch.total_size()).map_err(|_| IggyError::CannotAppendMessage)?; + + let last_dirty_offset = dirty_offset + u64::from(batch_messages_count) - 1; + + if !self.should_increment_offset { + self.should_increment_offset = true; + } + self.dirty_offset + .store(last_dirty_offset, Ordering::Relaxed); + + let segment_index = self.log.segments().len() - 1; + let current_position = self.log.segments()[segment_index].current_position; + self.log.segments_mut()[segment_index].current_position = current_position + .checked_add(batch_messages_size) + .ok_or(IggyError::CannotAppendMessage)?; + + let journal = self.log.journal_mut(); + journal.info.messages_count += batch_messages_count; + journal.info.size += IggyByteSize::from(batch_messages_size); + journal.info.current_offset = last_dirty_offset; + if journal.info.first_timestamp == 0 { + journal.info.first_timestamp = batch.base_timestamp; + } + journal.info.end_timestamp = batch.base_timestamp; + journal.info.max_timestamp = journal.info.max_timestamp.max(batch.base_timestamp); + journal + .inner + .append(message.into_frozen()) .await - .map(|journaled| journaled.result) + .map_err(|_| IggyError::CannotAppendMessage)?; + + Ok(AppendResult::new( + dirty_offset, + last_dirty_offset, + batch_messages_count, + )) } #[allow(clippy::cast_possible_truncation)] @@ -1677,40 +1716,9 @@ where B: MessageBus, SB: SuperblockStore, { - async fn stamp_and_append_messages( - &mut self, - message: Message, - ) -> Result { - let header = *message.header(); - if header.operation != Operation::SendMessages { - return Err(IggyError::CannotAppendMessage); - } - - let dirty_offset = if self.should_increment_offset { - self.dirty_offset - .load(Ordering::Relaxed) - .checked_add(1) - .ok_or(IggyError::CannotAppendMessage)? - } else { - 0 - }; - - // Reuse the prepare's monotonic timestamp, assigned once by the primary - // in `project()` (`next_monotonic_timestamp`) and replicated verbatim to - // every backup. Sourcing it here instead of a fresh local `now()` makes - // the persisted `base_timestamp` (and the `batch_checksum` derived from - // it) byte-identical across replicas. A local `now()` diverges per node. - let batch_timestamp = header.timestamp; - let (message, batch, batch_messages_count) = - stamp_prepare_for_persistence(message, dirty_offset, batch_timestamp) - .map_err(|_| IggyError::CannotAppendMessage)?; - - debug_assert_eq!(batch.message_count, batch_messages_count); - self.append_stamped_messages(message, batch).await - } #[must_use] fn namespace(&self) -> IggyNamespace { - IggyNamespace::from_raw(self.consensus.group()) + IggyNamespace::from_raw(self.consensus.namespace()) } fn partition_dir(&self) -> Option { @@ -1812,9 +1820,9 @@ where /// Panics if called when this partition's consensus instance is not the /// primary, is not in normal status, or is currently syncing. #[allow(clippy::future_not_send, clippy::too_many_lines)] - pub async fn on_request(&mut self, message: Message) { + pub async fn on_request(&mut self, message: Message) { self.clear_pending_consumer_offset_commits_if_view_changed(); - let namespace = IggyNamespace::from_raw(message.header().group); + let namespace = IggyNamespace::from_raw(message.header().namespace); let client_id = message.header().client; let request = message.header().request; @@ -2088,22 +2096,6 @@ where pub async fn on_replicate(&mut self, message: Message) { self.clear_pending_consumer_offset_commits_if_view_changed(); let header = *message.header(); - // Same reason as the metadata plane: `checksum` is compared as an opaque token - // downstream, so a corrupted frame passes whenever its flipped value satisfies - // those comparisons. - if let Err(reason) = verify_prepare_integrity(&header, message.as_slice()) { - emit_partition_diag( - tracing::Level::WARN, - &PartitionDiagEvent::new( - ReplicaLogContext::from_consensus(self.consensus(), PlaneKind::Partitions), - "discarding prepare that failed its own integrity check", - ) - .with_operation(header.operation) - .with_op(header.op) - .with_reason(reason), - ); - return; - } let current_op = { let consensus = self.consensus(); match replicate_preflight(consensus, &header) { @@ -2155,53 +2147,11 @@ where .with_operation(header.operation) .with_op(header.op), ); - let Some(journaled) = self.log.journal().inner.repair_entry(header.op) else { - emit_partition_diag( - tracing::Level::ERROR, - &PartitionDiagEvent::new( - self.diag_ctx(), - "journal header exists without matching prepare bytes", - ) - .with_operation(header.operation) - .with_op(header.op), - ); - return; - }; - if !journaled_prepare_matches_retransmit(&journaled, &message) { - emit_partition_diag( - tracing::Level::WARN, - &PartitionDiagEvent::new( - self.diag_ctx(), - "rejecting retransmitted prepare that differs from the journaled entry", - ) - .with_operation(header.operation) - .with_op(header.op), - ); - return; - } - let Some(frozen_for_forward) = restamp_prepare_view(journaled, header.view) else { - emit_partition_diag( - tracing::Level::ERROR, - &PartitionDiagEvent::new( - self.diag_ctx(), - "failed to restamp journaled prepare for retransmission", - ) - .with_operation(header.operation) - .with_op(header.op), - ); - return; - }; + let clone_for_forward = message.clone(); let consensus = self.consensus(); - if let Err(error) = - replicate_frozen_to_next_in_chain(consensus, frozen_for_forward).await - { - let is_transport_error = error.is_transport(); + if let Err(error) = replicate_to_next_in_chain(consensus, &clone_for_forward).await { emit_partition_diag( - if is_transport_error { - tracing::Level::WARN - } else { - tracing::Level::ERROR - }, + tracing::Level::WARN, &PartitionDiagEvent::new( self.diag_ctx(), "failed to re-forward retransmitted prepare to next in chain", @@ -2210,9 +2160,6 @@ where .with_op(header.op) .with_error(error.to_string()), ); - if !is_transport_error { - return; - } } self.send_prepare_ok(&header).await; return; @@ -2288,63 +2235,78 @@ where ); } } - // Forward only after apply_replicated_operation journals the prepare. - // The journal and network share the frozen allocation, so the bytes - // retained for repair are exactly the bytes sent downstream. - let replicated_result = if is_backup && header.operation == Operation::SendMessages { - self.append_received_send_messages_to_journal(message).await - } else { - self.apply_replicated_operation(message).await - }; - let frozen_for_forward = match replicated_result { - Ok(frozen) => frozen, - Err(error) => { + // First blob-integrity check on the replicated path. The consensus + // layer never validates the body (PrepareHeader integrity fields are + // inert zeros) and the batch checksum is recomputed locally at stamp, + // so a follower must verify each message's stamp-invariant per-message + // checksum before journaling transit bytes. Follower-only: the primary + // (and single-node self-replicate) produced these bytes and already + // checked the client batch at ingest, so they must not pay this pass. + // Fail closed on mismatch - drop without journaling, forwarding, or + // acking; the primary retransmits on prepare-timeout. + if is_backup + && header.operation == Operation::SendMessages + && let Err(error) = verify_received_send_messages(message.as_slice()) + { + emit_partition_diag( + tracing::Level::WARN, + &PartitionDiagEvent::new( + self.diag_ctx(), + "rejecting replicated send_messages: per-message checksum mismatch", + ) + .with_operation(header.operation) + .with_op(header.op) + .with_error(error.to_string()), + ); + return; + } + + // Durability-before-ack: clone for chain-replicate, forward only + // AFTER apply_replicated_operation persists. Forward-first would + // give downstream an op whose WAL entry we never wrote, that violates + // tail-ahead-of-head. Clone is cheap (Arc bumps in common case). + let clone_for_forward = message.clone(); + let replicated_result = self.apply_replicated_operation(message).await; + if replicated_result.is_ok() { + let consensus = self.consensus(); + // Backup only: advance sequencer + checksum after journal append. + // Pre-advance on failing apply would leave consensus claiming op N + // while journal has nothing; retransmit of N would silently drop + // as is_old_prepare (header.op <= current_sequence). Primary must + // NOT re-set here: push_prepare_entry already advanced, and a + // sibling request pipelined during the apply await would be + // rewound to a stale op + parent, projecting a duplicate next. + if is_backup { + consensus.sequencer().set_sequence(header.op); + consensus.set_last_prepare_checksum(header.checksum); + consensus.observe_prepare_timestamp(header.timestamp); + } + if let Err(error) = replicate_to_next_in_chain(consensus, &clone_for_forward).await { emit_partition_diag( tracing::Level::WARN, &PartitionDiagEvent::new( self.diag_ctx(), - "failed to apply replicated partition operation", + "failed to replicate prepare to next in chain", ) .with_operation(header.operation) .with_op(header.op) .with_error(error.to_string()), ); - return; } - }; - - let consensus = self.consensus(); - // Backup only: advance sequencer + checksum after journal append. - // Pre-advance on failing apply would leave consensus claiming op N - // while the journal has nothing. Retransmit of N would silently drop - // as is_old_prepare (header.op <= current_sequence). The primary does - // not re-set here because push_prepare_entry already advanced it. A - // sibling request pipelined during the apply await would otherwise be - // rewound to a stale op + parent, projecting a duplicate next. - if is_backup { - consensus.sequencer().set_sequence(header.op); - consensus.set_last_prepare_checksum(header.checksum); - consensus.observe_prepare_timestamp(header.timestamp); } - if let Err(error) = replicate_frozen_to_next_in_chain(consensus, frozen_for_forward).await { - let is_transport_error = error.is_transport(); + + if let Err(error) = replicated_result { emit_partition_diag( - if is_transport_error { - tracing::Level::WARN - } else { - tracing::Level::ERROR - }, + tracing::Level::WARN, &PartitionDiagEvent::new( self.diag_ctx(), - "failed to replicate prepare to next in chain", + "failed to apply replicated partition operation", ) .with_operation(header.operation) .with_op(header.op) .with_error(error.to_string()), ); - if !is_transport_error { - return; - } + return; } { @@ -2473,14 +2435,14 @@ where async fn apply_replicated_operation( &mut self, message: Message, - ) -> Result, IggyError> { + ) -> Result<(), IggyError> { let header = *message.header(); let replica_id = self.consensus.replica(); - let namespace_raw = self.consensus.group(); + let namespace_raw = self.consensus.namespace(); match header.operation { Operation::SendMessages => { - let frozen = self.append_send_messages_to_journal(message).await?; + self.append_send_messages_to_journal(message).await?; debug!( target: "iggy.partitions.diag", plane = "partitions", @@ -2490,7 +2452,7 @@ where operation = ?header.operation, "replicated send_messages appended to partition journal" ); - Ok(frozen) + Ok(()) } Operation::StoreConsumerOffset | Operation::DeleteConsumerOffset @@ -2511,11 +2473,10 @@ where // the `journal.info` accounting: it counts SendMessages // batches for segment-commit thresholds, which do not // apply to offset ops. - let frozen = message.into_frozen(); self.log .journal() .inner - .append(frozen.clone()) + .append(message.clone().into_frozen()) .await .map_err(|_| IggyError::CannotAppendMessage)?; @@ -2547,7 +2508,7 @@ where offset = ?offset, "replicated consumer offset journaled and staged" ); - Ok(frozen) + Ok(()) } _ => { warn!( @@ -2559,7 +2520,7 @@ where operation = ?header.operation, "unexpected replicated partition operation" ); - Err(IggyError::InvalidCommand) + Ok(()) } } } @@ -2567,190 +2528,10 @@ where async fn append_send_messages_to_journal( &mut self, message: Message, - ) -> Result, IggyError> { - let write_lock = self.write_lock.clone(); - let _guard = write_lock.lock().await; - self.stamp_and_append_messages(message) - .await - .map(|journaled| journaled.prepare) - } - - async fn append_received_send_messages_to_journal( - &mut self, - message: Message, - ) -> Result, IggyError> { - let write_lock = self.write_lock.clone(); - let _guard = write_lock.lock().await; - let header = *message.header(); - if header.operation != Operation::SendMessages { - return Err(IggyError::CannotAppendMessage); - } - let validated = decode_prepare_slice(message.as_slice())?.header; - if validated.message_count == 0 { - return Err(IggyError::InvalidCommand); - } - let expected_offset = if self.should_increment_offset { - self.dirty_offset - .load(Ordering::Relaxed) - .checked_add(1) - .ok_or(IggyError::CannotAppendMessage)? - } else { - 0 - }; - if (validated.base_offset, validated.base_timestamp) != (expected_offset, header.timestamp) - { - return Err(IggyError::CannotAppendMessage); - } - self.append_stamped_messages(message, validated) - .await - .map(|journaled| journaled.prepare) - } - - async fn append_stamped_messages( - &mut self, - message: Message, - batch: SendMessages2Header, - ) -> Result { - let batch_messages_count = batch.message_count; - if batch_messages_count == 0 { - return Err(IggyError::CannotAppendMessage); - } - - let batch_messages_size = - u64::try_from(batch.total_size()).map_err(|_| IggyError::CannotAppendMessage)?; - let last_dirty_offset = batch - .base_offset - .checked_add(u64::from(batch_messages_count) - 1) - .ok_or(IggyError::CannotAppendMessage)?; - - let segment_index = self.log.segments().len() - 1; - let current_position = self.log.segments()[segment_index].current_position; - let next_position = current_position - .checked_add(batch_messages_size) - .ok_or(IggyError::CannotAppendMessage)?; - - let mut journal_info = self.log.journal().info; - journal_info.messages_count = journal_info - .messages_count - .checked_add(batch_messages_count) - .ok_or(IggyError::CannotAppendMessage)?; - journal_info.size = IggyByteSize::from( - journal_info - .size - .as_bytes_u64() - .checked_add(batch_messages_size) - .ok_or(IggyError::CannotAppendMessage)?, - ); - journal_info.current_offset = last_dirty_offset; - if journal_info.first_timestamp == 0 { - journal_info.first_timestamp = batch.base_timestamp; - } - journal_info.end_timestamp = batch.base_timestamp; - journal_info.max_timestamp = journal_info.max_timestamp.max(batch.base_timestamp); - - let frozen = message.into_frozen(); - self.log - .journal() - .inner - .append(frozen.clone()) - .await - .map_err(|_| IggyError::CannotAppendMessage)?; - - self.should_increment_offset = true; - self.dirty_offset - .store(last_dirty_offset, Ordering::Relaxed); - self.log.segments_mut()[segment_index].current_position = next_position; - self.log.journal_mut().info = journal_info; - - Ok(JournaledMessages { - result: AppendResult::new(batch.base_offset, last_dirty_offset, batch_messages_count), - prepare: frozen, - }) - } - - /// Drop an uncommitted view-divergent suffix and restore every append cursor - /// from the retained prefix as one write-locked operation. - /// - /// # Errors - /// - /// Returns an error if a retained batch is invalid, the restored segment - /// position overflows, or the journal cannot truncate the suffix. - pub async fn truncate_uncommitted_from(&mut self, from_op: u64) -> Result { + ) -> Result<(), IggyError> { let write_lock = self.write_lock.clone(); let _guard = write_lock.lock().await; - - let mut entries = self.log.journal().inner.resident_entries(); - entries.sort_unstable_by_key(peek_op); - let mut retained_info = JournalInfo::default(); - let mut retained_next_offset = 0; - let mut rewind_next_offset = None; - for entry in &entries { - if peek_operation(entry) != Operation::SendMessages { - continue; - } - let batch = decode_prepare_slice_trusted(entry.as_slice()) - .map_err(|_| IggyError::InvalidCommand)?; - if batch.message_count() == 0 { - continue; - } - if peek_op(entry) >= from_op { - rewind_next_offset = Some( - rewind_next_offset.map_or(batch.header.base_offset, |offset: u64| { - offset.min(batch.header.base_offset) - }), - ); - continue; - } - accumulate_committed_info( - &mut retained_info, - batch.header.base_offset, - batch.header.base_timestamp, - batch.header.total_size() as u64, - batch.message_count(), - ); - retained_next_offset = retained_next_offset.max( - batch - .header - .base_offset - .saturating_add(u64::from(batch.message_count())), - ); - } - - let active = self.log.active_segment(); - let active_size = active.size.as_bytes_u64(); - let durable_next_offset = if active_size == 0 { - active.start_offset - } else { - active.end_offset.saturating_add(1) - }; - let minimum_next_offset = durable_next_offset - .max(retained_next_offset) - .max( - self.recovered_durable_offset - .map_or(0, |offset| offset.saturating_add(1)), - ) - .max(self.installed_frontier.unwrap_or(0)); - let restored_position = active_size - .checked_add(retained_info.size.as_bytes_u64()) - .ok_or(IggyError::CannotAppendMessage)?; - let removed = self - .log - .journal() - .inner - .truncate_from(from_op) - .await - .map_err(|_| IggyError::CannotAppendMessage)?; - - self.log.journal_mut().info = retained_info; - self.log.active_segment_mut().current_position = restored_position; - if let Some(next_offset) = rewind_next_offset { - let next_offset = next_offset.max(minimum_next_offset); - self.dirty_offset - .store(next_offset.saturating_sub(1), Ordering::Relaxed); - self.should_increment_offset = next_offset > 0; - } - self.consensus.invalidate_local_dvc_suffix(); - Ok(removed) + self.append_messages(message).await.map(|_| ()) } async fn commit_messages(&mut self, config: &PartitionsConfig) -> Result<(), IggyError> { @@ -3081,7 +2862,7 @@ where send_client_replies: bool, ) { let replica_id = self.consensus.replica(); - let namespace_raw = self.consensus.group(); + let namespace_raw = self.consensus.namespace(); let drained_count = drained.len(); if let (Some(first), Some(last)) = (drained.first(), drained.last()) { debug!( @@ -3163,7 +2944,7 @@ where if send_client_replies && !is_auto_commit_client(prepare_header.client) { let body = match prepare_header.operation { Operation::SendMessages => { - send_messages_reply_body(prepare_header.group, batch_stats) + send_messages_reply_body(prepare_header.namespace, batch_stats) } operation => committed_reply_body(operation), }; @@ -3381,13 +3162,13 @@ where fn parse_consumer_offset_request( operation: Operation, - message: &Message, + message: &Message, ) -> Result<(ConsumerKind, u32, Option, AckLevel), IggyError> { let total_size = usize::try_from(message.header().size).map_err(|_| IggyError::InvalidCommand)?; let body = message .as_slice() - .get(std::mem::size_of::()..total_size) + .get(std::mem::size_of::()..total_size) .ok_or(IggyError::InvalidCommand)?; Self::parse_consumer_offset_payload(operation, body) } @@ -3398,7 +3179,7 @@ where /// so nothing replicates. async fn send_partition_deny_or_log( consensus: &VsrConsensus, - header: &RoutedRequestHeader, + header: &RequestHeader, status: u32, send_fail_label: &'static str, ) { @@ -3721,7 +3502,6 @@ where messages_size_bytes, config.enforce_fsync, false, - config.preallocate_segments.then_some(config.segment_size), ) .await .map_err(|_| IggyError::CannotCreateSegmentLogFile(messages_path.clone()))?, @@ -3973,7 +3753,6 @@ where messages_size_bytes, config.enforce_fsync, false, - config.preallocate_segments.then_some(config.segment_size), ) .await .map_err(|_| IggyError::CannotCreateSegmentLogFile(messages_path.clone()))?, @@ -4203,18 +3982,7 @@ where guard.clear(); paths }; - // Sweep the directories too, not just the map-derived paths: a purge is a - // full reset, and an offset file the live map never held -- a pre-purge - // op re-persisted by journal repair on a restarted replica -- would - // otherwise survive for boot to hydrate back. - let strayed = - crate::state_transfer::strayed_offset_files(self.consumer_offsets_path.as_deref(), &[]) - .into_iter() - .chain(crate::state_transfer::strayed_offset_files( - self.consumer_group_offsets_path.as_deref(), - &[], - )); - for path in consumer_paths.into_iter().chain(group_paths).chain(strayed) { + for path in consumer_paths.into_iter().chain(group_paths) { let _ = delete_persisted_offset(&path).await; } // Directory fsync so those unlinks stick, mirroring the install path: a @@ -4387,7 +4155,7 @@ where Err(error) => Err(error), } } else { - self.apply_replicated_operation(message).await.map(|_| ()) + self.apply_replicated_operation(message).await }; if let Err(error) = applied { warn!( @@ -4576,7 +4344,8 @@ where let op = message.header().op; let (base_offset, base_timestamp, total_size, message_count) = { - let batch = decode_prepare_slice(message.as_slice())?; + let batch = + decode_prepare_slice(message.as_slice()).map_err(|_| IggyError::InvalidCommand)?; ( batch.header.base_offset, batch.header.base_timestamp, @@ -4585,7 +4354,7 @@ where ) }; if message_count == 0 { - return Err(IggyError::InvalidCommand); + return Ok(None); } // Purge floor: the same fence every other journal-apply path honors. A @@ -4605,49 +4374,33 @@ where return Ok(None); } - let last_offset = base_offset - .checked_add(u64::from(message_count) - 1) - .ok_or(IggyError::CannotAppendMessage)?; + let last_offset = base_offset + u64::from(message_count) - 1; + + self.should_increment_offset = true; let dirty = self.dirty_offset.load(Ordering::Relaxed); + self.dirty_offset + .store(dirty.max(last_offset), Ordering::Relaxed); let segment_index = self.log.segments().len() - 1; let current_position = self.log.segments()[segment_index].current_position; - let next_position = current_position + self.log.segments_mut()[segment_index].current_position = current_position .checked_add(total_size) .ok_or(IggyError::CannotAppendMessage)?; - let mut journal_info = self.log.journal().info; - journal_info.messages_count = journal_info - .messages_count - .checked_add(message_count) - .ok_or(IggyError::CannotAppendMessage)?; - journal_info.size = IggyByteSize::from( - journal_info - .size - .as_bytes_u64() - .checked_add(total_size) - .ok_or(IggyError::CannotAppendMessage)?, - ); - journal_info.current_offset = last_offset; - if journal_info.first_timestamp == 0 { - journal_info.first_timestamp = base_timestamp; + let journal = self.log.journal_mut(); + journal.info.messages_count += message_count; + journal.info.size += IggyByteSize::from(total_size); + journal.info.current_offset = last_offset; + if journal.info.first_timestamp == 0 { + journal.info.first_timestamp = base_timestamp; } - journal_info.end_timestamp = base_timestamp; - journal_info.max_timestamp = journal_info.max_timestamp.max(base_timestamp); - - let frozen = message.into_frozen(); - self.log - .journal() + journal.info.end_timestamp = base_timestamp; + journal.info.max_timestamp = journal.info.max_timestamp.max(base_timestamp); + journal .inner - .append(frozen) + .append(message.into_frozen()) .await .map_err(|_| IggyError::CannotAppendMessage)?; - - self.should_increment_offset = true; - self.dirty_offset - .store(dirty.max(last_offset), Ordering::Relaxed); - self.log.segments_mut()[segment_index].current_position = next_position; - self.log.journal_mut().info = journal_info; Ok(Some(base_offset)) } @@ -4724,30 +4477,6 @@ fn peek_op(entry: &Frozen<4096>) -> u64 { .op } -/// Match a retransmit against immutable, validated journal bytes. Only the view -/// may change, so exact body equality replaces another checksum pass. -fn journaled_prepare_matches_retransmit( - journaled: &Frozen<4096>, - incoming: &Message, -) -> bool { - const VIEW_OFFSET: usize = std::mem::offset_of!(PrepareHeader, view); - - let stored = journaled.as_slice(); - let received = incoming.as_slice(); - let header_size = std::mem::size_of::(); - if stored.len() != received.len() || stored.len() < header_size { - return false; - } - - let view_end = VIEW_OFFSET + std::mem::size_of::(); - if stored[..VIEW_OFFSET] != received[..VIEW_OFFSET] - || stored[view_end..header_size] != received[view_end..header_size] - { - return false; - } - stored[header_size..] == received[header_size..] -} - /// Success reply body for a committed partition op other than `SendMessages` /// (which confirms its offsets through [`send_messages_reply_body`]). /// @@ -4812,11 +4541,6 @@ struct CommittedBatchStats { size_bytes: u64, } -struct JournaledMessages { - result: AppendResult, - prepare: Frozen<4096>, -} - impl CommittedBatchStats { /// Offset of the batch's last message. The batch carries a contiguous /// offset run, and the sole constructor rejects an empty one, so the @@ -5352,7 +5076,7 @@ mod tests { client_id: u128, request_id: u64, consumer_id: u32, - ) -> Message { + ) -> Message { let body = DeleteConsumerOffset2Request { consumer: WireConsumer::consumer(WireIdentifier::Numeric(consumer_id)), stream_id: WireIdentifier::Numeric(1), @@ -5361,17 +5085,17 @@ mod tests { ack: AckLevel::Quorum, } .to_bytes(); - let header_size = std::mem::size_of::(); + let header_size = std::mem::size_of::(); let total = header_size + body.len(); - let mut message = Message::::new(total); + let mut message = Message::::new(total); message.as_mut_slice()[header_size..].copy_from_slice(&body); - message.transmute_header(|_, header: &mut RoutedRequestHeader| { + message.transmute_header(|_, header: &mut RequestHeader| { header.command = Command2::Request; header.operation = Operation::DeleteConsumerOffset2; header.client = client_id; header.session = 1; header.request = request_id; - header.group = IggyNamespace::new(1, 1, 0).inner(); + header.namespace = IggyNamespace::new(1, 1, 0).inner(); header.size = u32::try_from(total).expect("request size fits u32"); }) } @@ -5463,10 +5187,7 @@ mod tests { let path = format!("{dir}/{consumer_id}"); let read_disk = |p: &str| -> u64 { let bytes = std::fs::read(p).expect("offset file exists"); - match crate::offset_storage::decode_offset_record(&bytes) { - crate::offset_storage::OffsetRecord::Value { offset, .. } => offset, - other => panic!("offset file must hold a readable value, got {other:?}"), - } + u64::from_le_bytes(bytes.try_into().expect("offset file is 8 bytes")) }; // Reordered auto-commits: the later op (109) trails the earlier (114). @@ -5549,10 +5270,7 @@ mod tests { let path = format!("{dir}/{consumer_id}"); let read_disk = |p: &str| -> u64 { let bytes = std::fs::read(p).expect("offset file exists"); - match crate::offset_storage::decode_offset_record(&bytes) { - crate::offset_storage::OffsetRecord::Value { offset, .. } => offset, - other => panic!("offset file must hold a readable value, got {other:?}"), - } + u64::from_le_bytes(bytes.try_into().expect("offset file is 8 bytes")) }; // Simulate the previous process run: the file already holds 114. @@ -5715,7 +5433,6 @@ mod tests { // open exhausts retries -> the walk must fault-close before segment two. let plan = DiskReadPlan { partition_dir: PartitionDirResolution::Resolved(partition_dir), - validate_checksum: true, segments: vec![ DiskSegment { start_offset: 0, @@ -5801,7 +5518,6 @@ mod tests { let plan = DiskReadPlan { partition_dir: PartitionDirResolution::Resolved(partition_dir), - validate_checksum: true, segments: vec![ DiskSegment { start_offset: 0, @@ -5834,77 +5550,6 @@ mod tests { let _ = std::fs::remove_dir_all(&dir); } - /// A segment whose bytes decode cleanly but do not match their own - /// `batch_checksum`: bit rot at rest, not a torn write. Unverified, the batch is - /// served and a consumer reads data provably not what was written. - /// - /// Detection only, per the operator knob: the poll fails closed and reports, with - /// no attempt to repair. - #[compio::test] - async fn read_disk_faults_closed_on_batch_checksum_mismatch() { - let namespace = IggyNamespace::new(1, 1, 0); - let dir = std::env::temp_dir().join(format!( - "iggy-read-disk-bitrot-{}-{}", - std::process::id(), - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .expect("system clock after epoch") - .as_nanos(), - )); - compio::fs::create_dir_all(&dir) - .await - .expect("create temp partition dir"); - let partition_dir = dir.to_string_lossy().into_owned(); - - // Structurally valid with one payload byte flipped, so every length and - // offset still decodes and only the checksum disagrees. - let mut record = build_segment_record(namespace, 0); - let last = record.len() - 1; - record[last] ^= 0x01; - let record_len = record.len() as u64; - let path = format!("{partition_dir}/{:0>20}.log", 0u64); - { - let mut file = compio::fs::File::create(&path) - .await - .expect("create segment file"); - let (written, _) = file.write_all_at(record, 0).await.into(); - written.expect("write segment record"); - file.sync_all().await.expect("flush segment file"); - } - - let plan = |validate_checksum| DiskReadPlan { - partition_dir: PartitionDirResolution::Resolved(partition_dir.clone()), - validate_checksum, - segments: vec![DiskSegment { - start_offset: 0, - persisted: record_len, - read_state: None, - }], - start_position: 0, - namespace_raw: namespace.inner(), - }; - let query = MessageLookup::Offset { - offset: 0, - count: 10, - ceiling: u64::MAX, - }; - - let outcome = plan(true).read_disk(query).await; - assert!( - matches!(outcome, DiskReadOutcome::Faulted), - "a batch that fails its own checksum must fault-close" - ); - - // What the opt-out costs. The shipped default is `true` because of it. - let outcome = plan(false).read_disk(query).await; - assert!( - matches!(outcome, DiskReadOutcome::Matched { .. }), - "verification off is an explicit opt-out: the corrupt batch is served" - ); - - let _ = std::fs::remove_dir_all(&dir); - } - /// A simulated (file-less) partition has no segment files by design, so a /// disk poll with no dir must stay `Empty`: the caller then serves the /// resident journal tier, the sim's only tier. @@ -5919,7 +5564,6 @@ mod tests { }], start_position: 0, namespace_raw: IggyNamespace::new(1, 1, 0).inner(), - validate_checksum: true, }; let outcome = plan @@ -5950,7 +5594,6 @@ mod tests { }], start_position: 0, namespace_raw: IggyNamespace::new(1, 1, 0).inner(), - validate_checksum: true, }; let outcome = plan @@ -6009,7 +5652,6 @@ mod tests { let plan = DiskReadPlan { partition_dir: PartitionDirResolution::Resolved(partition_dir.clone()), - validate_checksum: true, segments: vec![DiskSegment { start_offset: 0, persisted: record_len, @@ -6040,7 +5682,6 @@ mod tests { let plan = DiskReadPlan { partition_dir: PartitionDirResolution::Resolved(partition_dir.clone()), - validate_checksum: true, segments: vec![DiskSegment { start_offset: 0, persisted: record_len, @@ -6100,7 +5741,6 @@ mod tests { let handle = SealedSegmentHandle::default(); let plan = DiskReadPlan { partition_dir: PartitionDirResolution::Resolved(partition_dir.clone()), - validate_checksum: true, segments: vec![DiskSegment { start_offset: 0, persisted: record_len, @@ -6184,7 +5824,6 @@ mod tests { let handle = SealedSegmentHandle::default(); let plan = DiskReadPlan { partition_dir: PartitionDirResolution::Resolved(partition_dir.clone()), - validate_checksum: true, segments: vec![DiskSegment { start_offset: 0, persisted: log_len, @@ -6283,7 +5922,6 @@ mod tests { let handle = SealedSegmentHandle::default(); let plan = DiskReadPlan { partition_dir: PartitionDirResolution::Resolved(partition_dir.clone()), - validate_checksum: true, segments: vec![DiskSegment { start_offset: 0, persisted: log_len, @@ -6361,7 +5999,6 @@ mod tests { let handle = Rc::clone(&partition.log.sealed_read_state()[0]); let plan = DiskReadPlan { partition_dir: PartitionDirResolution::Resolved(partition_dir.clone()), - validate_checksum: true, segments: vec![DiskSegment { start_offset: 0, persisted: record_len, @@ -6409,7 +6046,6 @@ mod tests { // unlinked pre-purge inode. let resumed = DiskReadPlan { partition_dir: PartitionDirResolution::Resolved(partition_dir.clone()), - validate_checksum: true, segments: vec![DiskSegment { start_offset: 0, persisted: record_len, @@ -6438,9 +6074,7 @@ mod tests { messages_required_to_save: 1, size_of_messages_required_to_save: IggyByteSize::from(1024 * 1024), enforce_fsync: false, - validate_checksum: true, segment_size: IggyByteSize::from(1024 * 1024), - preallocate_segments: false, encryptor: None, } } @@ -6792,50 +6426,6 @@ mod tests { let _ = std::fs::remove_dir_all(&partition_dir); } - /// A replica that missed the purge entirely: the metadata plane has it - /// committed, this replica never applied it, so its frontier still measures - /// the PRE-purge offset space. The reset offer is the only thing that can - /// converge it, and journal repair cannot bridge the floor the purge moved, - /// so refusing it strands the replica on pre-purge data for good. - /// - /// Distinguished from the lagging-origin case above by `next_offset == 0`: - /// nothing has been appended since the purge, so there is no post-purge - /// data for the offer to rewind. - #[compio::test] - async fn given_replica_that_missed_the_purge_when_offered_the_reset_should_install() { - let partition_dir = transfer_fence_dir("missed-purge-reset").await; - let mut partition = test_partition(); - partition.set_partition_dir(partition_dir.clone()); - partition.should_increment_offset = true; - partition.offset.store(99, Ordering::Release); - assert_eq!( - partition.applied_purge_generation(), - 0, - "a replica that missed the purge has not recorded its generation" - ); - - let reset = crate::state_transfer::ConsumerOffsetsWire { - purge_generation: 1, - next_offset: 0, - consumers: Vec::new(), - groups: Vec::new(), - }; - let installed = partition - .install_state_transfer(&repair_config(), 12, Vec::new(), &reset.encode(), 1) - .await; - - assert!( - !matches!( - installed, - Err(crate::state_transfer::PartitionInstallError::OfferRewindsDurableData { .. }) - ), - "the reset for a purge this replica never applied must pass the rewind \ - fence, got {installed:?}" - ); - - let _ = std::fs::remove_dir_all(&partition_dir); - } - /// Primary-by-index at view 0 with nothing committed refuses to serve: an /// empty group is trivially "caught up", so this gate is the only thing /// separating a real primary from a phantom whose directory vanished, whose @@ -7084,7 +6674,7 @@ mod purge_floor_tests { header.operation = Operation::SendMessages; header.op = op; header.timestamp = op; - header.group = namespace.inner(); + header.namespace = namespace.inner(); header.size = u32::try_from(total).expect("prepare size fits u32"); }); partition @@ -7119,7 +6709,7 @@ mod purge_floor_tests { header.command = Command2::Prepare; header.operation = Operation::StoreConsumerOffset2; header.op = op; - header.group = IggyNamespace::new(1, 1, 0).inner(); + header.namespace = IggyNamespace::new(1, 1, 0).inner(); header.size = u32::try_from(total).expect("prepare size fits u32"); }); partition @@ -7449,7 +7039,7 @@ mod purge_floor_tests { header.command = Command2::Prepare; header.operation = Operation::SendMessages; header.op = 1; - header.group = namespace.inner(); + header.namespace = namespace.inner(); header.size = u32::try_from(total).expect("prepare size fits u32"); }); diff --git a/core/partitions/src/iggy_partitions.rs b/core/partitions/src/iggy_partitions.rs index 45c0df4b53..a25aed657b 100644 --- a/core/partitions/src/iggy_partitions.rs +++ b/core/partitions/src/iggy_partitions.rs @@ -23,7 +23,7 @@ use crate::{IggyPartition, Partition, PollingArgs, PollingConsumer}; use ahash::AHashSet; use consensus::{Consensus, Plane, PlaneIdentity, VsrConsensus}; use iggy_binary_protocol::{ - Command2, ConsensusHeader, Operation, PrepareHeader, PrepareOkHeader, RoutedRequestHeader, + Command2, ConsensusHeader, Operation, PrepareHeader, PrepareOkHeader, RequestHeader, }; use journal::superblock::{PingPongSuperblock, SuperblockStore}; use message_bus::MessageBus; @@ -200,12 +200,6 @@ where /// Insert a new partition and return its local index. /// - /// Insertion is the moment a build becomes the addressable incarnation, - /// so this is also where its offset counter is published into the shared - /// `PartitionStats` ([`IggyPartition::publish_current_offset`]). No - /// earlier point is safe: a build that is never adopted must leave the - /// live incarnation's counters alone. - /// /// # Safety discipline (compiler cannot enforce) /// /// Must only be called from the shard's pump task (i.e. inside @@ -226,7 +220,6 @@ where 0, "IggyPartitions::insert while a with_partition borrow is live" ); - partition.publish_current_offset(); // Safety: pump-only invariant, caller responsibility. let partitions = unsafe { &mut *self.partitions.get() }; let local_idx = LocalIdx::new(partitions.len()); @@ -359,7 +352,7 @@ where // by the moved partition's namespace key in O(1); the previous // linear value-scan turned bulk DeleteStream into O(K²) on the // pump task, stalling client traffic for ~10k-partition topics. - let moved_ns = IggyNamespace::from_raw(partitions[idx].consensus().group()); + let moved_ns = IggyNamespace::from_raw(partitions[idx].consensus().namespace()); let entry = self.namespace_map_mut().get_mut(&moved_ns).expect( "IggyPartitions invariant: swapped-in partition missing namespace_to_local entry", ); @@ -419,10 +412,8 @@ where // `build_poll_plan` touches the partition's sealed-read-handle LRU, so it // needs `&mut`. Sound on the pump: it is fully synchronous (no `.await` // inside), so no sibling task can realloc the partitions vec under it. - // Read the knob first: the `&mut` borrow below covers `self.config` too. - let validate_checksum = self.config.validate_checksum; let partition = self.get_mut_by_ns(namespace)?; - Some(partition.build_poll_plan(consumer, args, validate_checksum)) + Some(partition.build_poll_plan(consumer, args)) } /// Read a consumer's stored offset + the partition commit offset. Fully @@ -505,11 +496,8 @@ where B: MessageBus, SB: SuperblockStore, { - async fn on_request( - &self, - message: as Consensus>::Message, - ) { - let namespace = IggyNamespace::from_raw(message.header().group); + async fn on_request(&self, message: as Consensus>::Message) { + let namespace = IggyNamespace::from_raw(message.header().namespace); if self.is_tombstoned(&namespace) { warn!( target: "iggy.partitions.diag", @@ -562,20 +550,20 @@ where } async fn on_replicate(&self, message: as Consensus>::Message) { - let group = IggyNamespace::from_raw(message.header().group); - if self.is_tombstoned(&group) { + let namespace = IggyNamespace::from_raw(message.header().namespace); + if self.is_tombstoned(&namespace) { warn!( target: "iggy.partitions.diag", - namespace_raw = group.inner(), + namespace_raw = namespace.inner(), "dropping prepare: namespace tombstoned" ); return; } - let Some(partition) = self.get_mut_by_ns(&group) else { + let Some(partition) = self.get_mut_by_ns(&namespace) else { warn!( target: "iggy.partitions.diag", plane = "partitions", - namespace_raw = group.inner(), + namespace_raw = namespace.inner(), op = message.header().op, operation = ?message.header().operation, "partition not initialized for namespace" @@ -587,21 +575,21 @@ where #[allow(clippy::too_many_lines)] async fn on_ack(&self, message: as Consensus>::Message) { - let group = IggyNamespace::from_raw(message.header().group); - if self.is_tombstoned(&group) { + let namespace = IggyNamespace::from_raw(message.header().namespace); + if self.is_tombstoned(&namespace) { warn!( target: "iggy.partitions.diag", - namespace_raw = group.inner(), + namespace_raw = namespace.inner(), "dropping prepare-ok: namespace tombstoned" ); return; } let config = self.config.clone(); - let Some(partition) = self.get_mut_by_ns(&group) else { + let Some(partition) = self.get_mut_by_ns(&namespace) else { warn!( target: "iggy.partitions.diag", plane = "partitions", - namespace_raw = group.inner(), + namespace_raw = namespace.inner(), op = message.header().op, "partition not initialized for namespace" ); @@ -666,28 +654,6 @@ mod tests { ) } - /// `build_partition` for a replicated group. The replica count is what - /// decides whether the journal retains evicted entries for repair, so a - /// single-replica partition cannot exercise anything that reads the ring. - fn build_replicated_partition() -> IggyPartition { - let namespace = IggyNamespace::new(1, 1, 0); - let consensus = VsrConsensus::new( - TEST_CLUSTER, - 0, - 3, - namespace.inner(), - IggyMessageBus::new(0), - LocalPipeline::new(), - ); - consensus.init(); - IggyPartition::with_in_memory_storage( - Arc::new(PartitionStats::default()), - consensus, - IggyByteSize::from(1024 * 1024), - false, - ) - } - /// One-message `SendMessages` journal entry stamped at `op` / `base_offset`. /// Reuses the production blob builder + checksum stamping so the entry /// decodes through `decode_prepare_slice` and indexes into `offset_to_op`, @@ -830,67 +796,6 @@ mod tests { ); } - /// A flush evicts the committed prefix up to and INCLUDING `commit_max`, so - /// a caught-up replica keeps no resident header at its own commit point. The - /// `DoViewChange` suffix is floored there and cannot nack it, so reading the - /// resident headers alone sends the commit point out blank, which a quorum of - /// senders turns into a view change that never starts. - /// - /// The entry is still servable (`repair_entry` answers from the evicted - /// ring), so the suffix reads through `repair_header`, over the same range. - #[compio::test] - async fn evicted_commit_point_still_answers_for_the_view_change_suffix() { - let namespace = IggyNamespace::new(1, 1, 0); - let partition = build_replicated_partition(); - - for offset in 0..=2u64 { - partition - .log - .journal() - .inner - .append(build_send_messages_entry(namespace, offset + 1, offset)) - .await - .expect("append journal entry"); - } - - let commit_max = 3; - let prefix = partition.log.journal().inner.committed_prefix(commit_max); - assert_eq!(prefix.len(), 3, "the whole log is committed and flushable"); - partition - .log - .journal() - .inner - .evict_prefix(prefix.len()) - .await; - - assert!( - partition - .log - .journal() - .inner - .header_by_op(commit_max) - .is_none(), - "the flush evicted the commit point from the resident headers", - ); - assert!( - partition - .log - .journal() - .inner - .repair_entry(commit_max) - .is_some(), - "yet the entry is still servable from the evicted ring", - ); - - let header = partition - .log - .journal() - .inner - .repair_header(commit_max) - .expect("the commit point must stay describable for the DVC suffix"); - assert_eq!(header.op, commit_max); - } - /// The resident journal holds replicated-but-uncommitted prepares ahead of /// the commit frontier. A poll must clamp at `ceiling` (the commit offset) /// so it never returns a dirty read of view-change-rollbackable data, even diff --git a/core/partitions/src/journal.rs b/core/partitions/src/journal.rs index 689e2b727f..f29a951f27 100644 --- a/core/partitions/src/journal.rs +++ b/core/partitions/src/journal.rs @@ -25,7 +25,6 @@ use std::io; use std::{ cell::{Cell, UnsafeCell}, collections::{BTreeMap, HashMap, VecDeque}, - ops::RangeInclusive, }; use tracing::warn; @@ -189,11 +188,11 @@ where /// Running byte total of the buffers held by `evicted_ring`. evicted_ring_bytes: Cell, /// Entry-count ceiling for `evicted_ring`. Defaults to - /// [`EVICTED_RING_CAPACITY`]; the server overrides it from config at + /// [`EVICTED_RING_CAPACITY`]; server-ng overrides it from config at /// partition build. evicted_ring_capacity: Cell, /// Byte ceiling for `evicted_ring`. Defaults to - /// [`EVICTED_RING_BYTES_MAX`]; the server overrides it from config at + /// [`EVICTED_RING_BYTES_MAX`]; server-ng overrides it from config at /// partition build. evicted_ring_bytes_max: Cell, /// Single-replica groups have nobody to repair; retaining evicted @@ -368,64 +367,6 @@ impl PartitionJournal { .map(|(_, entry)| entry.clone()) } - /// The header at `op`, over exactly the range [`Self::repair_entry`] serves. - /// - /// NOT [`Self::header_by_op`], which reads the resident headers alone. The - /// committed prefix is evicted from those the moment its bytes reach a - /// segment, up to and including `commit_max`, so a `DoViewChange` built off - /// the resident headers reports its own commit point blank. The merge scans - /// the commit point and cannot discard it, so a quorum of such senders is - /// undecidable and the view never starts (`dvc_merge::merge_dvc_quorum`). - /// The entry is still servable from the evicted ring, which is what makes - /// the blank wrong rather than merely pessimistic. - /// - /// The ring drops from the front, so the highest evicted op -- the commit - /// point of the last flush -- is the last thing it forgets. - pub fn repair_header(&self, op: u64) -> Option { - if let Some(header) = self.header_by_op(op) { - return Some(header); - } - let ring = unsafe { &*self.evicted_ring.get() }; - let (_, entry) = ring.iter().find(|(ring_op, _)| *ring_op == op)?; - let header_bytes = entry.as_slice().get(..PREPARE_HEADER_SIZE)?; - bytemuck::checked::try_from_bytes::(header_bytes) - .ok() - .copied() - } - - /// Every repairable header with an op in `ops`, in ONE pass over the resident - /// headers and ONE over the evicted ring. - /// - /// [`Self::repair_header`] is two linear scans, so probing it per op costs - /// O(window x (headers + ring)), and the `DoViewChange` suffix build does - /// exactly that, up to `DVC_HEADERS_MAX` probes, on every SVC/DVC arrival and - /// non-Normal tick, on the pump. Result size is bounded by what the journal - /// holds, not by the width of `ops`. Resident wins over ring, as `repair_header` - /// probes. - #[must_use] - pub fn repair_headers_in(&self, ops: RangeInclusive) -> BTreeMap { - let mut found = BTreeMap::new(); - { - let headers = unsafe { &*self.headers.get() }; - for header in headers.iter().filter(|header| ops.contains(&header.op)) { - found.insert(header.op, *header); - } - } - let ring = unsafe { &*self.evicted_ring.get() }; - for (op, entry) in ring.iter().filter(|(op, _)| ops.contains(op)) { - if found.contains_key(op) { - continue; - } - let Some(header_bytes) = entry.as_slice().get(..PREPARE_HEADER_SIZE) else { - continue; - }; - if let Ok(header) = bytemuck::checked::try_from_bytes::(header_bytes) { - found.insert(*op, *header); - } - } - found - } - /// Oldest op this journal can still serve for repair (ring front, else /// resident head), or `None` when it holds nothing at all. pub fn repair_retained_from(&self) -> Option { @@ -967,59 +908,6 @@ impl Journal for PartitionJournal Option { self.bytes_by_op(header.op).await } - - /// Appends are in op order and every rewrite preserves it, so the tail header - /// carries the highest op. - fn last_op(&self) -> Option { - let headers = unsafe { &*self.headers.get() }; - headers.last().map(|header| header.op) - } - - /// Drop every entry at or above `from_op`, rebuilding the indexes. Same - /// drain-and-re-append shape as `evict_prefix`, from the other end and retaining - /// nothing: `append` has no slot-collision check here, so a superseded entry left - /// in place sits beside the new view's prepare at the same op and - /// `committed_prefix`, which walks positionally, flushes the stale one. - /// - /// Dropped entries do NOT enter the evicted repair ring: it answers repair for - /// committed ops, and these are ones the view just decided against. - async fn truncate_from(&self, from_op: u64) -> io::Result { - if from_op == 0 { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - "truncate_from: ops are 1-based, so 0 would discard the whole journal", - )); - } - let all_entries = { - let inner = unsafe { &*self.inner.get() }; - inner.storage.drain() - }; - // Positional against `headers` until the clear below (see the length-lock - // invariant on `append_with_meta`), so the ops are captured first. - let ops: Vec = { - let headers = unsafe { &*self.headers.get() }; - headers.iter().map(|header| header.op).collect() - }; - { - unsafe { &mut *self.headers.get() }.clear(); - unsafe { &mut *self.op_to_storage_offset.get() }.clear(); - unsafe { &mut *self.offset_to_op.get() }.clear(); - unsafe { &mut *self.timestamp_to_op.get() }.clear(); - } - - let mut removed = 0usize; - for (op, entry) in ops.into_iter().zip(all_entries) { - if op >= from_op { - removed += 1; - continue; - } - // Replays bytes this journal already accepted once, so it cannot fail. - self.append_with_meta(entry) - .await - .expect("re-appending a retained journal entry must not fail"); - } - Ok(removed) - } } pub fn select_batch_slice( @@ -1276,74 +1164,6 @@ mod tests { ); } - #[compio::test] - async fn truncate_from_drops_the_suffix_and_keeps_the_prefix_readable() { - let journal = PartitionJournal::::default(); - for op in 1..=5 { - journal - .append(build_prepare(op, HEADER_SIZE + 16).into_frozen()) - .await - .expect("append"); - } - - let removed = journal.truncate_from(4).await.expect("truncate"); - assert_eq!(removed, 2, "ops 4 and 5 must go"); - assert_eq!(journal.last_op(), Some(3)); - for op in 1..=3u64 { - let header = journal - .header_by_op(op) - .expect("a retained op must survive"); - assert!( - journal.entry(&header).await.is_some(), - "a retained entry must still read back after the rewrite" - ); - } - for op in 4..=5u64 { - assert!(journal.header_by_op(op).is_none(), "op {op} must be gone"); - } - - // The point of dropping them: the primary's retransmission refills the range. - journal - .append(build_prepare(4, HEADER_SIZE + 16).into_frozen()) - .await - .expect("a truncated op must be appendable again"); - assert_eq!(journal.last_op(), Some(4)); - } - - #[compio::test] - async fn repair_headers_in_serves_the_commit_point_from_the_evicted_ring() { - // Blank AT the commit point is the one slot a merge can neither adopt nor - // discard, so a quorum that all flushed there deadlocks. A flushed replica has - // no resident header there, so the ring must answer. - let journal = PartitionJournal::::default(); - for op in 1..=4 { - journal - .append(build_prepare(op, HEADER_SIZE + 16).into_frozen()) - .await - .expect("append"); - } - // `commit_messages` evicts the committed prefix inclusively, so the commit - // point's own resident header goes with it. - journal.evict_prefix(2).await; - assert!( - journal.header_by_op(2).is_none(), - "the resident header at the commit point is gone after the flush" - ); - - let window = journal.repair_headers_in(2..=4); - assert!( - window.contains_key(&2), - "the commit point must still be describable, or the view change deadlocks" - ); - for op in 3..=4u64 { - assert!(window.contains_key(&op), "op {op} is resident and in range"); - } - assert!( - !window.contains_key(&1), - "ops outside the window must not be reported" - ); - } - #[compio::test] async fn committed_prefix_reads_then_evict_retains_uncommitted_tail() { // A backup journals ops ahead of the commit frontier. Reading the diff --git a/core/partitions/src/lib.rs b/core/partitions/src/lib.rs index 2b62402cf6..75b4c70ca8 100644 --- a/core/partitions/src/lib.rs +++ b/core/partitions/src/lib.rs @@ -25,7 +25,7 @@ mod iggy_partitions; mod journal; mod log; mod messages_writer; -pub mod offset_storage; +mod offset_storage; mod poll_plan; mod segment; pub mod state_transfer; diff --git a/core/partitions/src/log.rs b/core/partitions/src/log.rs index 574b37108f..34d5b93075 100644 --- a/core/partitions/src/log.rs +++ b/core/partitions/src/log.rs @@ -116,14 +116,6 @@ where ) -> impl Future>> { self.inner.drain(ops) } - - fn truncate_from(&self, from_op: u64) -> impl Future> { - self.inner.truncate_from(from_op) - } - - fn last_op(&self) -> Option { - self.inner.last_op() - } } impl Default for JournalState { diff --git a/core/partitions/src/messages_writer.rs b/core/partitions/src/messages_writer.rs index cf6fc33650..f6ad70cc18 100644 --- a/core/partitions/src/messages_writer.rs +++ b/core/partitions/src/messages_writer.rs @@ -21,16 +21,11 @@ use compio::{ }; use iggy_common::{IggyByteSize, IggyError}; use server_common::iobuf::Frozen; -#[cfg(target_os = "linux")] -use std::os::fd::AsFd; use std::{ rc::Rc, sync::atomic::{AtomicU64, Ordering}, }; -use tracing::{error, warn}; - -#[cfg(target_os = "linux")] -use nix::fcntl::{FallocateFlags, fallocate}; +use tracing::error; const MAX_IOV_COUNT: usize = 1024; @@ -53,7 +48,6 @@ impl MessagesWriter { messages_size_bytes: Rc, fsync: bool, file_exists: bool, - preallocate_size: Option, ) -> Result { let mut opts = OpenOptions::new(); opts.write(true); @@ -65,13 +59,6 @@ impl MessagesWriter { .await .map_err(|_| IggyError::CannotReadFile)?; - if let Some(preallocate_size) = preallocate_size { - #[cfg(target_os = "linux")] - preallocate_file(&file, file_path, preallocate_size.as_bytes_u64()).await; - #[cfg(not(target_os = "linux"))] - preallocate_file(&file, file_path, preallocate_size.as_bytes_u64()); - } - if file_exists { file.sync_all() .await @@ -161,58 +148,6 @@ impl MessagesWriter { } } -#[cfg(target_os = "linux")] -async fn preallocate_file(file: &File, file_path: &str, len: u64) { - let Ok(len) = i64::try_from(len) else { - warn!( - target: "iggy.partitions.storage", - file = file_path, - preallocate_len = len, - "file preallocation size is unsupported, using buffered allocation" - ); - return; - }; - - let file = match file.as_fd().try_clone_to_owned() { - Ok(file) => file, - Err(error) => { - warn!( - target: "iggy.partitions.storage", - file = file_path, - preallocate_len = len, - %error, - "file descriptor duplication failed, using buffered allocation" - ); - return; - } - }; - - // Remote filesystems can make fallocate block. The duplicated descriptor - // lets the blocking pool reserve extents without stalling the shard thread. - let result = compio::runtime::spawn_blocking(move || { - fallocate(file, FallocateFlags::FALLOC_FL_KEEP_SIZE, 0, len) - }) - .await; - if let Err(error) = result { - warn!( - target: "iggy.partitions.storage", - file = file_path, - preallocate_len = len, - %error, - "file preallocation failed, using buffered allocation" - ); - } -} - -#[cfg(not(target_os = "linux"))] -fn preallocate_file(_file: &File, file_path: &str, _len: u64) { - warn!( - target: "iggy.partitions.storage", - file = file_path, - "file preallocation is unavailable on this platform, using buffered allocation" - ); -} - async fn write_frozen_chunked( file: &File, file_path: &str, @@ -243,25 +178,3 @@ async fn write_frozen_chunked( Ok(()) } - -#[cfg(test)] -mod tests { - use super::*; - - #[compio::test] - async fn preallocated_file_keeps_logical_length() { - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("segment.log"); - let writer = MessagesWriter::new( - path.to_str().unwrap(), - Rc::new(AtomicU64::new(0)), - false, - false, - Some(IggyByteSize::from(1024 * 1024_u64)), - ) - .await - .unwrap(); - - assert_eq!(writer.file.metadata().await.unwrap().len(), 0); - } -} diff --git a/core/partitions/src/offset_storage.rs b/core/partitions/src/offset_storage.rs index 618f987eb1..c3377d824b 100644 --- a/core/partitions/src/offset_storage.rs +++ b/core/partitions/src/offset_storage.rs @@ -17,97 +17,23 @@ use compio::{ fs::{OpenOptions, create_dir_all, remove_file}, - io::{AsyncReadAt, AsyncReadAtExt, AsyncWriteAtExt}, + io::{AsyncReadAtExt, AsyncWriteAtExt}, }; -use iggy_common::{IggyError, calculate_checksum}; +use iggy_common::IggyError; use std::path::Path; use tracing::warn; const OFFSET_SIZE: usize = core::mem::size_of::(); -const CHECKSUM_SIZE: usize = core::mem::size_of::(); - -/// Bytes a consumer-offset file holds: the offset, then a checksum over it. -/// -/// The offset is a consumer cursor reloaded unchanged on every restart, so a -/// flipped bit silently rewinds the consumer into redelivery or skips it forward. -pub const OFFSET_RECORD_SIZE: usize = OFFSET_SIZE + CHECKSUM_SIZE; /// Per-partition file recording the purge generation this replica last applied -/// locally, in the partition dir beside the segments it fences. -/// -/// Two LE u64s: the applied generation, then the `created_revision` of the -/// partition incarnation it was applied for. +/// locally, in the partition dir beside the segments it fences. Two LE u64s: +/// the applied generation, then the `created_revision` of the partition +/// incarnation it was applied for. pub const PURGE_GENERATION_FILE: &str = "purge.gen"; /// `[generation][created_revision]`, both LE u64. const PURGE_GENERATION_RECORD_SIZE: usize = 2 * OFFSET_SIZE; -/// What a consumer-offset file was found to hold. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum OffsetRecord { - /// A usable offset. `checksummed` is false for a bare offset predating the - /// checksum, read as-is and upgraded by the next write. - Value { offset: u64, checksummed: bool }, - /// Shorter than the value: a crash between `persist_offset`'s truncate and write. - Torn, - /// The checksum does not describe the value stored beside it. - Corrupt { - offset: u64, - expected: u64, - found: u64, - }, -} - -/// Encode a consumer offset for persistence. -#[must_use] -pub fn encode_offset_record(offset: u64) -> [u8; OFFSET_RECORD_SIZE] { - let mut record = [0u8; OFFSET_RECORD_SIZE]; - record[..OFFSET_SIZE].copy_from_slice(&offset.to_le_bytes()); - let checksum = calculate_checksum(&record[..OFFSET_SIZE]); - record[OFFSET_SIZE..].copy_from_slice(&checksum.to_le_bytes()); - record -} - -/// Decode whatever a consumer-offset file contained. -/// -/// A file of exactly one offset predates the checksum and is accepted. A partly -/// written checksum region reads as the bare offset for the same reason: the record -/// is written in one call, so the low bytes are the complete new value. -#[must_use] -pub fn decode_offset_record(bytes: &[u8]) -> OffsetRecord { - let Some(value) = bytes.first_chunk::() else { - return OffsetRecord::Torn; - }; - let offset = u64::from_le_bytes(*value); - let Some(stored) = bytes - .get(OFFSET_SIZE..) - .and_then(<[u8]>::first_chunk::) - else { - return OffsetRecord::Value { - offset, - checksummed: false, - }; - }; - let found = u64::from_le_bytes(*stored); - let expected = calculate_checksum(value); - if found == expected { - OffsetRecord::Value { - offset, - checksummed: true, - } - } else { - OffsetRecord::Corrupt { - offset, - expected, - found, - } - } -} - -/// Overwrite a consumer-offset file with `offset` and a checksum over it. -/// -/// # Errors -/// [`IggyError`] when the directory, file, or write cannot be created or completed. pub async fn persist_offset(path: &str, offset: u64, enforce_fsync: bool) -> Result<(), IggyError> { // No `exists()` probe first: that is a BLOCKING `std::path` stat on the pump // in front of every write, which serialises a batched fan-out on stats @@ -126,7 +52,8 @@ pub async fn persist_offset(path: &str, offset: u64, enforce_fsync: bool) -> Res .open(path) .await .map_err(|_| IggyError::CannotOpenConsumerOffsetsFile(path.to_owned()))?; - file.write_all_at(encode_offset_record(offset), 0) + let buf = offset.to_le_bytes(); + file.write_all_at(buf, 0) .await .0 .map_err(|_| IggyError::CannotWriteToFile)?; @@ -140,52 +67,25 @@ pub async fn persist_offset(path: &str, offset: u64, enforce_fsync: bool) -> Res Ok(()) } -/// Monotone counterpart of [`persist_offset`] for a server auto-commit op. -/// -/// Folds `max(current_on_disk, offset)` and returns the value now on disk, skipping -/// the write when the file already holds it. Disk-tier polls replicate their -/// auto-committed offsets in IO-completion order, so a committed op can carry a lower -/// offset than an earlier one, and a plain overwrite would leave the file rewound for -/// a restart to reload and re-deliver. The on-disk value is committed-only, so the -/// fold is identical on every replica applying the same op order. -/// -/// The read makes this the cold-key path only: once the caller's persisted-offset -/// tracker knows the file's value, warm commits persist with a blind -/// [`persist_offset`] and skip covered offsets without reading. +/// Monotone counterpart of [`persist_offset`] for a server auto-commit op: +/// folds `max(current_on_disk, offset)` and returns the value now on disk, +/// skipping the write when the file already holds it. Disk-tier polls +/// replicate their auto-committed offsets in IO-completion order, so a +/// committed op can carry a lower offset than an earlier one; a plain +/// overwrite would leave the file rewound and a restart would reload the +/// stale value and re-deliver. The on-disk value is committed-only (this path +/// never writes the eager serving map), so the fold is identical on every +/// replica applying the same op order. /// -/// A file that fails its checksum folds as absent and is overwritten. Its value is -/// untrusted and the boot loader already discarded it, so nothing is preserved by -/// refusing, and refusing is not survivable: the caller reads a failed commit as -/// divergence and aborts the shard, and the key is cold on every boot, so one damaged -/// file aborts every boot. Redelivery is within at-least-once. -/// -/// # Errors -/// [`IggyError`] when the file cannot be read or written. +/// The read makes this the cold-key path only: once the caller's +/// persisted-offset tracker knows the file's value, warm commits persist with +/// a blind [`persist_offset`] and skip covered offsets without any file read. pub async fn persist_offset_max( path: &str, offset: u64, enforce_fsync: bool, ) -> Result { - let on_disk = match read_offset_record(path).await? { - Some(OffsetRecord::Value { offset, .. }) => Some(offset), - Some(OffsetRecord::Corrupt { - offset: stored, - expected, - found, - }) => { - tracing::error!( - path, - stored, - expected, - found, - fold_to = offset, - "consumer offset file failed its checksum; overwriting it with the committed \ - offset. This consumer may see redelivery." - ); - None - } - Some(OffsetRecord::Torn) | None => None, - }; + let on_disk = read_persisted_offset(path).await?; let effective = on_disk.map_or(offset, |current| current.max(offset)); if on_disk != Some(effective) { persist_offset(path, effective, enforce_fsync).await?; @@ -194,11 +94,10 @@ pub async fn persist_offset_max( } /// Durably record the purge generation a partition has locally applied, keyed -/// to the incarnation (`created_revision`) it was applied for. -/// -/// Truncate+write like [`persist_offset`] but ALWAYS data-synced, regardless of -/// the consumer-offset fsync knob: purges are rare, the record is 16 bytes, and -/// a generation lost from the page cache in a crash makes the reconciler +/// to the incarnation (`created_revision`) it was applied for. Truncate+write +/// like [`persist_offset`] but ALWAYS data-synced, regardless of the +/// consumer-offset fsync knob: purges are rare, the record is 16 bytes, and a +/// generation lost from the page cache in a crash makes the reconciler /// re-purge on restart, wiping messages appended after the purge. A failure /// leaves the previous record on disk so the caller keeps its in-memory /// applied generation old and retries. @@ -297,29 +196,32 @@ pub async fn read_purge_generation(path: &str, created_revision: u64) -> Result< Ok(generation) } -/// Read whatever a consumer-offset file holds. `None` only when absent; a short file -/// reports [`OffsetRecord::Torn`] and the caller folds it as the boot loader does. -/// -/// Real I/O errors propagate: unlike a failed checksum, an unreadable file may still -/// hold an intact cursor, and folding it as absent would rewind the consumer. -async fn read_offset_record(path: &str) -> Result, IggyError> { - // Absence answered by the open, not a `Path::exists()` probe: that is a BLOCKING - // stat on the pump before every cold-key commit (see `persist_offset`). - let file = match OpenOptions::new().read(true).open(path).await { - Ok(file) => file, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), - Err(_) => return Err(IggyError::CannotOpenConsumerOffsetsFile(path.to_owned())), - }; - // One short read, not `read_exact` twice: `decode_offset_record` classifies any - // length, so the returned count separates a legacy 8-byte file from a full - // record. `..read` matters: the zero padding would otherwise decode as a - // checksum and the legacy file as `Corrupt`. - let compio::BufResult(read, buf) = file.read_at(vec![0u8; OFFSET_RECORD_SIZE], 0).await; - let read = read.map_err(|_| IggyError::CannotReadConsumerOffsets(path.to_owned()))?; - if read == 0 { +/// Read a single persisted consumer offset. `None` if the file is absent or +/// torn (shorter than 8 bytes): a crash between `persist_offset`'s truncate +/// and write leaves a short file, and the boot-time loader already skips such +/// files, so the commit-path reader must agree or a torn file turns every +/// later commit-apply into an error. Real I/O errors still propagate: mapping +/// them to `None` would silently rewind a valid higher offset. +async fn read_persisted_offset(path: &str) -> Result, IggyError> { + if !Path::new(path).exists() { return Ok(None); } - Ok(Some(decode_offset_record(&buf[..read]))) + let file = OpenOptions::new() + .read(true) + .open(path) + .await + .map_err(|_| IggyError::CannotOpenConsumerOffsetsFile(path.to_owned()))?; + let buf = vec![0u8; OFFSET_SIZE]; + let compio::BufResult(read, buf) = file.read_exact_at(buf, 0).await; + match read { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::UnexpectedEof => return Ok(None), + Err(_) => return Err(IggyError::CannotReadConsumerOffsets(path.to_owned())), + } + let bytes: [u8; OFFSET_SIZE] = buf + .try_into() + .map_err(|_| IggyError::CannotReadConsumerOffsets(path.to_owned()))?; + Ok(Some(u64::from_le_bytes(bytes))) } /// Unlink a persisted consumer-offset file. A no-op if the file is absent. @@ -354,158 +256,52 @@ mod tests { dir } - #[test] - fn offset_record_round_trips() { - let record = encode_offset_record(114); - assert_eq!(record.len(), OFFSET_RECORD_SIZE); - assert_eq!( - decode_offset_record(&record), - OffsetRecord::Value { - offset: 114, - checksummed: true - } - ); - } - - #[test] - fn offset_record_accepts_a_bare_value_written_before_the_checksum() { - assert_eq!( - decode_offset_record(&114u64.to_le_bytes()), - OffsetRecord::Value { - offset: 114, - checksummed: false - } - ); - } - - #[test] - fn offset_record_rejects_either_half_flipped() { - // The point of the checksum: a flipped bit rewinds a consumer into redelivery - // or skips it forward, and nothing ever notices. - let mut value_flipped = encode_offset_record(114); - value_flipped[0] ^= 0x01; - assert!(matches!( - decode_offset_record(&value_flipped), - OffsetRecord::Corrupt { offset: 115, .. } - )); - - let mut checksum_flipped = encode_offset_record(114); - checksum_flipped[OFFSET_SIZE] ^= 0x01; - assert!(matches!( - decode_offset_record(&checksum_flipped), - OffsetRecord::Corrupt { offset: 114, .. } - )); - } - - #[test] - fn offset_record_partly_written_is_torn_below_the_value_and_bare_above_it() { - assert_eq!(decode_offset_record(&[]), OffsetRecord::Torn); - assert_eq!(decode_offset_record(&[0xAB; 7]), OffsetRecord::Torn); - - // One `write_all_at` writes the whole record, so a torn tail keeps the value. - let record = encode_offset_record(114); - assert_eq!( - decode_offset_record(&record[..OFFSET_SIZE + 3]), - OffsetRecord::Value { - offset: 114, - checksummed: false - } - ); - } - - #[compio::test] - async fn read_offset_record_reports_a_corrupt_file_as_corrupt() { - let dir = unique_temp_dir(); - let path = dir.join("42").to_string_lossy().into_owned(); - - persist_offset(&path, 114, false).await.expect("persist"); - let mut bytes = std::fs::read(&path).expect("offset file exists"); - bytes[0] ^= 0x01; - std::fs::write(&path, &bytes).expect("corrupt the file"); - - let result = read_offset_record(&path).await; - assert!( - matches!(result, Ok(Some(OffsetRecord::Corrupt { .. }))), - "a corrupt cursor must be distinguishable from an absent one, got {result:?}" - ); - - let _ = std::fs::remove_dir_all(&dir); - } - - #[compio::test] - async fn read_offset_record_reads_a_legacy_bare_value() { - let dir = unique_temp_dir(); - let path = dir.join("42").to_string_lossy().into_owned(); - std::fs::write(&path, 114u64.to_le_bytes()).expect("write legacy file"); - - let read = read_offset_record(&path).await.expect("legacy file"); - assert_eq!( - read, - Some(OffsetRecord::Value { - offset: 114, - checksummed: false - }) - ); - - let _ = std::fs::remove_dir_all(&dir); - } - #[compio::test] - async fn read_offset_record_absent_file_is_none() { + async fn read_persisted_offset_absent_file_is_none() { let dir = unique_temp_dir(); let path = dir.join("42").to_string_lossy().into_owned(); - let read = read_offset_record(&path).await.expect("absent file"); + let read = read_persisted_offset(&path).await.expect("absent file"); assert_eq!(read, None); let _ = std::fs::remove_dir_all(&dir); } #[compio::test] - async fn read_offset_record_round_trips_persisted_value() { + async fn read_persisted_offset_round_trips_persisted_value() { let dir = unique_temp_dir(); let path = dir.join("42").to_string_lossy().into_owned(); persist_offset(&path, 114, false).await.expect("persist"); - let read = read_offset_record(&path).await.expect("valid file"); - assert_eq!( - read, - Some(OffsetRecord::Value { - offset: 114, - checksummed: true - }) - ); + let read = read_persisted_offset(&path).await.expect("valid file"); + assert_eq!(read, Some(114)); let _ = std::fs::remove_dir_all(&dir); } #[compio::test] - async fn read_offset_record_torn_file_is_torn_not_error() { + async fn read_persisted_offset_torn_file_is_none_not_error() { let dir = unique_temp_dir(); let path = dir.join("42").to_string_lossy().into_owned(); std::fs::write(&path, [0xAB, 0xCD, 0xEF]).expect("write torn file"); - let read = read_offset_record(&path) + let read = read_persisted_offset(&path) .await .expect("torn file must not error the commit path"); - assert_eq!( - read, - Some(OffsetRecord::Torn), - "a short file folds as absent, like the boot loader, but is not silence" - ); + assert_eq!(read, None, "short read maps to None like the boot loader"); let _ = std::fs::remove_dir_all(&dir); } #[compio::test] - async fn read_offset_record_real_io_error_propagates() { + async fn read_persisted_offset_real_io_error_propagates() { // A directory opens read-only but every read fails with EISDIR: a real // I/O error, not a short read. It must surface as Err, never as None // (a blanket None would silently rewind a valid higher offset). let dir = unique_temp_dir(); let path = dir.to_string_lossy().into_owned(); - let result = read_offset_record(&path).await; + let result = read_persisted_offset(&path).await; assert!( matches!(result, Err(IggyError::CannotReadConsumerOffsets(_))), "real I/O error must propagate, got {result:?}", @@ -606,48 +402,8 @@ mod tests { persist_offset_max(&path, 7, false) .await .expect("torn file folds as absent"); - let read = read_offset_record(&path).await.expect("repaired file"); - assert_eq!( - read, - Some(OffsetRecord::Value { - offset: 7, - checksummed: true - }) - ); - - let _ = std::fs::remove_dir_all(&dir); - } - - /// The crash-loop guard: a fold against a failed-checksum file must repair it, - /// not fail the commit. The caller aborts the shard on a failed commit, and the - /// key is cold on every boot, so the abort would repeat. - #[compio::test] - async fn persist_offset_max_overwrites_a_corrupt_file_instead_of_failing() { - let dir = unique_temp_dir(); - let path = dir.join("42").to_string_lossy().into_owned(); - - persist_offset(&path, 114, false).await.expect("persist"); - let mut bytes = std::fs::read(&path).expect("offset file exists"); - bytes[0] ^= 0x01; - std::fs::write(&path, &bytes).expect("corrupt the file"); - - let folded = persist_offset_max(&path, 7, false) - .await - .expect("a corrupt file must not fail the commit"); - assert_eq!( - folded, 7, - "the untrusted stored value must not win the fold" - ); - - let read = read_offset_record(&path).await.expect("repaired file"); - assert_eq!( - read, - Some(OffsetRecord::Value { - offset: 7, - checksummed: true - }), - "the corrupt file must be repaired in place, not left to trip the next commit" - ); + let read = read_persisted_offset(&path).await.expect("repaired file"); + assert_eq!(read, Some(7)); let _ = std::fs::remove_dir_all(&dir); } diff --git a/core/partitions/src/poll_plan.rs b/core/partitions/src/poll_plan.rs index 928222d124..b618cd84ab 100644 --- a/core/partitions/src/poll_plan.rs +++ b/core/partitions/src/poll_plan.rs @@ -39,13 +39,13 @@ use iggy_common::{ ConsumerGroupId, ConsumerGroupOffsets, ConsumerKind, ConsumerOffset, ConsumerOffsets, IggyError, }; use server_common::iobuf::{Frozen, Owned}; -use server_common::send_messages2::{BatchIntegrity, COMMAND_HEADER_SIZE, decode_batch_slice_with}; +use server_common::send_messages2::{COMMAND_HEADER_SIZE, decode_batch_slice}; use std::cell::{Cell, RefCell}; use std::hash::Hash; use std::rc::Rc; use std::sync::Arc; use std::sync::atomic::Ordering; -use tracing::{error, warn}; +use tracing::warn; /// Byte cap for materializing a sealed segment's sparse index into its shared /// read-state handle. Index density is one entry per flush: at the default @@ -120,9 +120,6 @@ pub struct DiskReadPlan { pub(crate) segments: Vec, pub(crate) start_position: u64, pub(crate) namespace_raw: u64, - /// Whether to verify each batch's `batch_checksum` against the bytes read. - /// Detection only; a mismatch fails the poll closed and repairs nothing. - pub(crate) validate_checksum: bool, } pub struct DiskSegment { @@ -286,18 +283,12 @@ impl PollPlan { crate::journal::select_resident(&resident_tail.entries, query) .unwrap_or_else(|| (PollFragments::new(), None)) } - // Disk read stopped on a fault. Fail-closed: return an empty poll - // WITHOUT the journal-forward fallback. Falling forward here would - // splice the next resident op over the unreadable run and silently - // skip live data. - // - // TODO(partitions): the poll reply has no error channel, so this - // reaches the consumer as an ordinary empty poll. Fair for a transient - // IO fault, wrong for a batch that failed its own checksum: data - // damaged at rest never reads again, so the consumer waits forever. - // Surfacing it needs a status on the poll reply, an SDK-visible change - // on every client. Until then the ERROR in `walk_disk_chunk` is the - // only signal, and it is server-side only. + // Disk read stopped on an IO fault. Fail-closed: return an empty + // poll WITHOUT the journal-forward fallback. Falling forward + // here would splice the next resident op over the unreadable run + // and silently skip live data; the fault instead surfaces as a + // visibly stuck consumer that recovers on a later poll once the + // segment reads again. DiskReadOutcome::Faulted => (PollFragments::new(), None), // Straddle: continue past the last disk match into the resident // tail (gate + race argument live on `straddle_continuation`). @@ -542,27 +533,14 @@ impl DiskReadPlan { faulted = true; break 'walk; }; - let ChunkWalk { consumed, corrupt } = walk_disk_chunk( + let consumed = walk_disk_chunk( &chunk, query, count, &mut matched, &mut fragments, &mut last_matching_offset, - if self.validate_checksum { - BatchIntegrity::Verify - } else { - BatchIntegrity::LayoutOnly - }, - self.namespace_raw, ); - if corrupt { - // A batch that does not match its own checksum. Fail closed like - // an IO fault: serving it hands a consumer data provably not what - // was written, and skipping ahead punches a silent gap. - faulted = true; - break 'walk; - } if consumed == 0 { if (len as u64) >= persisted - position { // The whole remainder fit yet no complete batch @@ -905,7 +883,6 @@ pub fn upsert_offset_max( /// chunk, pushing matching fragments. Returns bytes consumed: the start /// of the first batch that did not fully fit in the chunk (the caller /// re-reads from there), or the chunk end when everything decoded. -#[allow(clippy::too_many_arguments)] fn walk_disk_chunk( chunk: &Frozen<4096>, query: MessageLookup, @@ -913,37 +890,15 @@ fn walk_disk_chunk( matched: &mut u32, fragments: &mut PollFragments<4096>, last_matching_offset: &mut Option, - integrity: BatchIntegrity, - namespace_raw: u64, -) -> ChunkWalk { +) -> usize { let bytes: &[u8] = chunk; let mut cursor = 0usize; while *matched < count && cursor + COMMAND_HEADER_SIZE <= bytes.len() { - let batch = match decode_batch_slice_with(&bytes[cursor..], integrity) { - Ok(batch) => batch, - Err(IggyError::InvalidBatchChecksum(found, expected, base_offset)) => { - // Distinguished from the incomplete-tail case below: this batch is - // entirely present and fails its own checksum, so it is damaged at rest. - error!( - target: "iggy.partitions.diag", - plane = "partitions", - namespace_raw, - base_offset, - expected, - found, - position = cursor, - "disk poll: batch checksum mismatch; segment is corrupt at rest" - ); - return ChunkWalk { - consumed: cursor.min(bytes.len()), - corrupt: true, - }; - } - Err(_) => { - // Incomplete tail batch: hand the position back to re-read or bail. - break; - } + let Ok(batch) = decode_batch_slice(&bytes[cursor..]) else { + // Incomplete tail batch (or corrupt data): hand the position + // back so the caller can re-read or bail. + break; }; let total_size = batch.header.total_size(); @@ -964,17 +919,7 @@ fn walk_disk_chunk( cursor += total_size; } - ChunkWalk { - consumed: cursor.min(bytes.len()), - corrupt: false, - } -} - -/// How far [`walk_disk_chunk`] got, and whether it stopped on corruption rather -/// than on a batch that simply did not fit in the chunk. -struct ChunkWalk { - consumed: usize, - corrupt: bool, + cursor.min(bytes.len()) } #[cfg(test)] diff --git a/core/partitions/src/state_transfer.rs b/core/partitions/src/state_transfer.rs index fd9e17e6f4..22bb8cd861 100644 --- a/core/partitions/src/state_transfer.rs +++ b/core/partitions/src/state_transfer.rs @@ -1512,15 +1512,16 @@ where .retain(|start_offset, _| live.contains_key(start_offset)); } - // An empty chain at frontier 0 tells the receiver to unlink its own, so - // serve it only when a recorded purge says the emptiness is the truth. - // `install_state_transfer`'s `purge_advances` check re-decides that - // against the metadata plane and refuses the rest. + // Second phantom gate, on the BUILT offer rather than on `commit_max`. + // The gate above keys on `commit_max() == 0`, which a replica that lifted + // its commit floor through an offsets-only repair window clears while + // still holding zero bytes; such a replica passes `is_caught_up_primary` + // and would hand a data-holding peer an empty chain at frontier 0, + // making it unlink its own. An offer with no segments AND no offset + // space is indistinguishable from that phantom, and a group genuinely + // in that state has nothing worth transferring anyway. let offsets_wire = self.offsets_wire_snapshot(); - if segments.is_empty() - && offsets_wire.next_offset == 0 - && offsets_wire.purge_generation == 0 - { + if segments.is_empty() && offsets_wire.next_offset == 0 { return Err(PartitionTransferUnavailable::NothingCommitted); } let offsets_bytes = Rc::new(offsets_wire.encode()); @@ -1985,15 +1986,7 @@ where // purge disables the `OfferRewindsDurableData` refusal below -- the // one guard standing between an offer that rewinds this replica's // offset space and its durable data. - // Second disjunct: this replica has NOT applied the committed purge, so - // its frontier still measures the pre-purge offset space and cannot be - // compared against a post-purge offer. Restricted to `next_offset == 0` - // -- the state a purge leaves before anything is appended -- so an - // origin that merely lags within the same purge era still fails the - // fence rather than rewinding this replica's durable post-purge data. - let purge_advances = offsets_wire.purge_generation > committed_purge_generation - || (self.applied_purge_generation < committed_purge_generation - && offsets_wire.next_offset == 0); + let purge_advances = offsets_wire.purge_generation > committed_purge_generation; let local_next_offset = self.offset_frontier(); if !purge_advances && local_next_offset > 0 && offsets_wire.next_offset < local_next_offset { @@ -2101,7 +2094,7 @@ where tracing::error!( target: "iggy.partitions.diag", plane = "partitions", - namespace_raw = self.consensus().group(), + namespace_raw = self.consensus().namespace(), frontier = self.offset_frontier(), "state-transfer install could not record the installed offset frontier; \ the durable record stays at the pre-swap claim until the next view change" @@ -2155,7 +2148,7 @@ where // Unlink the old segment chain oldest-first (a crash mid-loop leaves // the NEWEST suffix, which is contiguous) and drop the in-memory // vectors in lockstep, exactly as `purge` does. - let namespace_raw = self.consensus().group(); + let namespace_raw = self.consensus().namespace(); while let Some((_, mut storage)) = self.log.retire_front() { let (messages_path, index_path) = storage.segment_and_index_paths(); let _ = storage.shutdown(); @@ -2331,7 +2324,6 @@ where messages_w.size_counter(), config.enforce_fsync, true, - config.preallocate_segments.then_some(config.segment_size), ) .await .map_err(|source| PartitionInstallError::SegmentOpen { @@ -2381,7 +2373,7 @@ where // keys are minted from u32 wire ids, so assert it. let old_consumer_paths: Vec = { let guard = self.consumer_offsets.pin(); - let mut paths: Vec = guard + let paths = guard .iter() .filter_map(|(key, _)| { let narrowed = u32::try_from(*key).ok(); @@ -2390,20 +2382,11 @@ where }) .collect(); guard.clear(); - // The map is not the whole truth about what is on disk: a repaired - // pre-purge offset op persists a file this incarnation never held, - // and a purged origin offering `next_offset = 0` drops every - // incoming entry, so a map-only sweep leaves the old table for boot - // to resurrect. - paths.extend(strayed_offset_files( - self.consumer_offsets_path.as_deref(), - &offsets_wire.consumers, - )); paths }; let old_group_paths: Vec = { let guard = self.consumer_group_offsets.pin(); - let mut paths: Vec = guard + let paths = guard .iter() .filter_map(|(key, _)| { let narrowed = u32::try_from(key.0).ok(); @@ -2417,10 +2400,6 @@ where }) .collect(); guard.clear(); - paths.extend(strayed_offset_files( - self.consumer_group_offsets_path.as_deref(), - &offsets_wire.groups, - )); paths }; for path in old_consumer_paths.into_iter().chain(old_group_paths) { @@ -2434,7 +2413,7 @@ where tracing::warn!( target: "iggy.partitions.diag", plane = "partitions", - namespace_raw = self.consensus().group(), + namespace_raw = self.consensus().namespace(), path = %path, %error, "failed to unlink a superseded consumer-offset file during install" @@ -2597,7 +2576,7 @@ where tracing::warn!( target: "iggy.partitions.diag", plane = "partitions", - namespace_raw = self.consensus().group(), + namespace_raw = self.consensus().namespace(), purge_generation = offsets_wire.purge_generation, %error, "state-transfer install could not record the offered purge \ @@ -2701,7 +2680,7 @@ where tracing::error!( target: "iggy.partitions.diag", plane = "partitions", - namespace_raw = self.consensus().group(), + namespace_raw = self.consensus().namespace(), partition_dir, %error, "converge sweep cannot list the partition directory" @@ -2715,7 +2694,7 @@ where Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} Err(error) => { warn_unlink( - self.consensus().group(), + self.consensus().namespace(), &path.display().to_string(), &error, ); @@ -3018,35 +2997,6 @@ pub fn offered_purge_generation(offsets_bytes: &[u8]) -> u64 { .unwrap_or_default() } -/// Offset files under `dir` whose id is absent from `incoming`. -/// -/// The install's own map cannot name these: a pre-purge offset op replayed by -/// journal repair persists a file this incarnation never held, and a purged -/// origin offers `next_offset = 0`, which drops every incoming entry. Left -/// behind, boot hydrates them back. -/// -/// A file whose name is not a bare u32 is left alone rather than guessed at: -/// every offset file is named by its id, so anything else is not ours. -pub(crate) fn strayed_offset_files(dir: Option<&str>, incoming: &[(u32, u64)]) -> Vec { - let Some(dir) = dir else { - return Vec::new(); - }; - let Ok(entries) = std::fs::read_dir(dir) else { - return Vec::new(); - }; - entries - .filter_map(Result::ok) - .filter_map(|entry| { - let name = entry.file_name().into_string().ok()?; - let id: u32 = name.parse().ok()?; - incoming - .iter() - .all(|(incoming_id, _)| *incoming_id != id) - .then(|| format!("{dir}/{name}")) - }) - .collect() -} - /// Stamp over every `SEGMENT_LOG` entry of a manifest, keying /// [`ReuseScanMemo`]. Equal digests mean the two offers expect byte-identical /// staged files, so a scan already done for one answers the other; the offsets diff --git a/core/partitions/src/types.rs b/core/partitions/src/types.rs index 5bc67b8e7d..f052ff75f0 100644 --- a/core/partitions/src/types.rs +++ b/core/partitions/src/types.rs @@ -269,17 +269,8 @@ pub struct PartitionsConfig { pub size_of_messages_required_to_save: IggyByteSize, /// Whether to enforce fsync after writes. pub enforce_fsync: bool, - /// Whether a disk poll verifies each batch's `batch_checksum` against the bytes - /// it just read. - /// - /// Detection only: a mismatch fails the poll closed and is reported, with no - /// attempt to repair. The alternative is serving bytes provably not the ones - /// written, which reads to a consumer as ordinary data. - pub validate_checksum: bool, /// Maximum size of a single segment before rotation. pub segment_size: IggyByteSize, - /// Whether local message files reserve the configured segment size on open. - pub preallocate_segments: bool, /// Server-side at-rest encryption. Applied ONCE, on the primary at /// ingestion, so the ciphertext replicates verbatim: every replica /// journals, acks, and persists identical bytes (checksums and the diff --git a/core/sdk/Cargo.toml b/core/sdk/Cargo.toml index 398680d768..e5562b27f6 100644 --- a/core/sdk/Cargo.toml +++ b/core/sdk/Cargo.toml @@ -29,6 +29,9 @@ documentation = "https://iggy.apache.org/docs" repository = "https://github.com/apache/iggy" readme = "README.md" +[features] +vsr = ["iggy_common/vsr"] + [dependencies] async-broadcast = { workspace = true } async-dropper = { workspace = true } diff --git a/core/sdk/src/clients/client.rs b/core/sdk/src/clients/client.rs index e91a601a5d..dd7f91fd98 100644 --- a/core/sdk/src/clients/client.rs +++ b/core/sdk/src/clients/client.rs @@ -39,9 +39,8 @@ use iggy_common::locking::{IggyRwLock, IggyRwLockFn}; use iggy_common::{BinaryTransport, Client, HttpMethod, SystemClient}; use iggy_common::{ConnectionStringUtils, DiagnosticEvent, Partitioner, TransportProtocol}; use std::fmt::Debug; -use std::sync::{Arc, Mutex}; +use std::sync::Arc; use tokio::spawn; -use tokio::task::JoinHandle; use tokio::time::sleep; use tracing::log::warn; use tracing::{debug, error, info}; @@ -65,7 +64,6 @@ pub struct IggyClient { pub(crate) client: IggyRwLock, partitioner: Option>, pub(crate) encryptor: Option>, - heartbeat_handle: Mutex>>, } impl Default for IggyClient { @@ -94,7 +92,6 @@ impl IggyClient { client, partitioner: None, encryptor: None, - heartbeat_handle: Mutex::new(None), } } @@ -134,7 +131,6 @@ impl IggyClient { client, partitioner, encryptor, - heartbeat_handle: Mutex::new(None), } } @@ -240,19 +236,6 @@ impl IggyClient { } } -impl Drop for IggyClient { - fn drop(&mut self) { - let heartbeat_handle = self - .heartbeat_handle - .get_mut() - .unwrap_or_else(|error| error.into_inner()) - .take(); - if let Some(handle) = heartbeat_handle { - handle.abort(); - } - } -} - #[async_trait] impl Client for IggyClient { async fn connect(&self) -> Result<(), IggyError> { @@ -263,20 +246,8 @@ impl Client for IggyClient { heartbeat_interval = client.heartbeat_interval().await; } - let mut heartbeat_handle = self - .heartbeat_handle - .lock() - .unwrap_or_else(|error| error.into_inner()); - if heartbeat_handle - .as_ref() - .is_some_and(|handle| !handle.is_finished()) - { - return Ok(()); - } - - drop(heartbeat_handle.take()); let client = self.client.clone(); - *heartbeat_handle = Some(spawn(async move { + spawn(async move { loop { debug!("Sending the heartbeat..."); if let Err(error) = client.read().await.ping().await { @@ -297,7 +268,7 @@ impl Client for IggyClient { } sleep(heartbeat_interval.get_duration()).await } - })); + }); Ok(()) } diff --git a/core/sdk/src/http/http_client.rs b/core/sdk/src/http/http_client.rs index f0c7510196..5cd4bb4133 100644 --- a/core/sdk/src/http/http_client.rs +++ b/core/sdk/src/http/http_client.rs @@ -307,7 +307,7 @@ impl HttpClient { /// - Legacy server: one-shot. The presented token is revoked as it is /// consumed, so a concurrent in-flight request still carrying the old /// token may fail with 401. - /// - the server: stateless. The old token stays valid until its natural + /// - server-ng: stateless. The old token stays valid until its natural /// expiry; refreshing never revokes it. pub async fn refresh_access_token(&self) -> Result { // Release the read guard before `set_token_from_identity` takes the diff --git a/core/sdk/src/lib.rs b/core/sdk/src/lib.rs index fd8b22efb2..50deca0e51 100644 --- a/core/sdk/src/lib.rs +++ b/core/sdk/src/lib.rs @@ -27,9 +27,11 @@ pub mod quic; pub mod session; pub mod stream_builder; pub mod tcp; +#[cfg(feature = "vsr")] mod vsr; pub mod websocket; /// Rust SDK version sent in the login-register version prefix; must be this /// crate's version, see `VsrSessionControl::sdk_version`. +#[cfg(feature = "vsr")] pub(crate) const SDK_VERSION: &str = env!("CARGO_PKG_VERSION"); diff --git a/core/sdk/src/prelude.rs b/core/sdk/src/prelude.rs index 51b34e34ab..ee0dc06d64 100644 --- a/core/sdk/src/prelude.rs +++ b/core/sdk/src/prelude.rs @@ -51,18 +51,18 @@ pub use iggy_common::{ Aes256GcmEncryptor, Args, ArgsOptional, AutoLogin, CacheMetrics, CacheMetricsKey, ClientError, ClientInfoDetails, ClusterMetadata, ClusterNode, ClusterNodeRole, ClusterNodeStatus, CompressionAlgorithm, Consumer, ConsumerGroup, ConsumerGroupDetails, ConsumerGroupMember, - ConsumerKind, Credentials, EncryptorKind, GlobalPermissions, HeaderField, HeaderKey, - HeaderKind, HeaderValue, HttpClientConfig, HttpClientConfigBuilder, HttpMethod, IdKind, - Identifier, IdentityInfo, IggyByteSize, IggyDuration, IggyError, IggyExpiry, IggyIndexView, - IggyMessage, IggyMessageHeader, IggyMessageHeaderView, IggyMessageView, - IggyMessageViewIterator, IggyTimestamp, MaxTopicSize, Partition, Partitioner, Partitioning, - Permissions, PersonalAccessTokenExpiry, PollMessages, PolledMessages, PollingKind, - PollingStrategy, QuicClientConfig, QuicClientConfigBuilder, QuicClientReconnectionConfig, - SendMessages, SendMessagesConfirmationResponse, SendMessagesResponse, Sizeable, - SnapshotCompression, Stats, Stream, StreamDetails, StreamPermissions, SystemSnapshotType, - TcpClientConfig, TcpClientConfigBuilder, TcpClientReconnectionConfig, Topic, TopicDetails, - TopicPermissions, TransportEndpoints, TransportProtocol, UserId, UserInfo, UserInfoDetails, - UserStatus, Validatable, WebSocketClientConfig, WebSocketClientConfigBuilder, + ConsumerKind, EncryptorKind, GlobalPermissions, HeaderField, HeaderKey, HeaderKind, + HeaderValue, HttpClientConfig, HttpClientConfigBuilder, HttpMethod, IdKind, Identifier, + IdentityInfo, IggyByteSize, IggyDuration, IggyError, IggyExpiry, IggyIndexView, IggyMessage, + IggyMessageHeader, IggyMessageHeaderView, IggyMessageView, IggyMessageViewIterator, + IggyTimestamp, MaxTopicSize, Partition, Partitioner, Partitioning, Permissions, + PersonalAccessTokenExpiry, PollMessages, PolledMessages, PollingKind, PollingStrategy, + QuicClientConfig, QuicClientConfigBuilder, QuicClientReconnectionConfig, SendMessages, + SendMessagesConfirmationResponse, SendMessagesResponse, Sizeable, SnapshotCompression, Stats, + Stream, StreamDetails, StreamPermissions, SystemSnapshotType, TcpClientConfig, + TcpClientConfigBuilder, TcpClientReconnectionConfig, Topic, TopicDetails, TopicPermissions, + TransportEndpoints, TransportProtocol, UserId, UserInfo, UserInfoDetails, UserStatus, + Validatable, WebSocketClientConfig, WebSocketClientConfigBuilder, WebSocketClientReconnectionConfig, defaults, locking, }; pub use iggy_common::{ diff --git a/core/sdk/src/quic/quic_client.rs b/core/sdk/src/quic/quic_client.rs index 7bf52ee660..9a731f6082 100644 --- a/core/sdk/src/quic/quic_client.rs +++ b/core/sdk/src/quic/quic_client.rs @@ -17,7 +17,9 @@ use crate::leader_aware::{LeaderRedirectionState, check_and_redirect_to_leader}; use crate::prelude::AutoLogin; +#[cfg(feature = "vsr")] use crate::session::ConsensusSession; +#[cfg(feature = "vsr")] use iggy_common::VsrSessionControl as _; use iggy_common::{BinaryClient, BinaryTransport, Client, PersonalAccessTokenClient, UserClient}; @@ -26,6 +28,7 @@ use crate::quic::skip_server_verification::SkipServerVerification; use async_broadcast::{Receiver, Sender, broadcast}; use async_trait::async_trait; use bytes::Bytes; +#[cfg(feature = "vsr")] use iggy_binary_protocol::codes::{LOGIN_REGISTER_CODE, LOGIN_REGISTER_WITH_PAT_CODE}; use iggy_common::{ ClientState, ConnectionString, ConnectionStringUtils, Credentials, DiagnosticEvent, @@ -38,12 +41,17 @@ use secrecy::ExposeSecret; use std::net::{SocketAddr, ToSocketAddrs}; use std::str::FromStr; use std::sync::Arc; +#[cfg(feature = "vsr")] use std::sync::Mutex as StdMutex; use std::time::Duration; use tokio::sync::Mutex; use tokio::time::sleep; use tracing::{error, info, trace, warn}; +#[cfg(not(feature = "vsr"))] +const REQUEST_INITIAL_BYTES_LENGTH: usize = 4; +#[cfg(not(feature = "vsr"))] +const RESPONSE_INITIAL_BYTES_LENGTH: usize = 8; const NAME: &str = "Iggy"; /// Bound on how long a single QUIC request waits for its response, mirroring the @@ -60,6 +68,7 @@ const RESPONSE_READ_TIMEOUT: Duration = Duration::from_secs(30); /// view-change cancel). Unlike a silent timeout, this reply arrives promptly, so /// a short pause keeps the replay from spinning while the primary catches up. /// Bounded overall by `RESPONSE_READ_TIMEOUT`. +#[cfg(feature = "vsr")] const NOT_READY_RETRY_INTERVAL: Duration = Duration::from_millis(50); /// QUIC client for interacting with the Iggy API. @@ -73,10 +82,13 @@ pub struct QuicClient { pub(crate) connected_at: Mutex>, leader_redirection_state: Mutex, pub(crate) current_server_address: Mutex, + #[cfg(feature = "vsr")] // See `core/sdk/src/tcp/tcp_client.rs` for the `tokio::sync::Mutex` -> // `std::sync::Mutex` rationale (pure-CPU critical section). consensus_session: Arc>, + #[cfg(feature = "vsr")] skip_auto_login_once: Mutex, + #[cfg(feature = "vsr")] consumer_group_state: Arc, } @@ -148,6 +160,7 @@ impl BinaryTransport for QuicClient { return Err(IggyError::Disconnected); } + #[cfg(feature = "vsr")] if matches!(self.config.auto_login, AutoLogin::Disabled) && !is_login_register_code(code) { // Without auto-login a reconnect cannot re-establish the session, so // non-login requests are not recovered here - their transient replay @@ -159,7 +172,9 @@ impl BinaryTransport for QuicClient { } self.disconnect().await?; + #[cfg(feature = "vsr")] let skip_auto_login = is_login_register_code(code); + #[cfg(feature = "vsr")] if skip_auto_login { *self.skip_auto_login_once.lock().await = true; } @@ -169,6 +184,7 @@ impl BinaryTransport for QuicClient { server_address, self.config.client_address ); let reconnect = self.connect().await; + #[cfg(feature = "vsr")] if skip_auto_login && reconnect.is_err() { *self.skip_auto_login_once.lock().await = false; } @@ -180,13 +196,16 @@ impl BinaryTransport for QuicClient { self.config.heartbeat_interval } + #[cfg(feature = "vsr")] fn consumer_group_state(&self) -> Arc { Arc::clone(&self.consumer_group_state) } } +#[cfg(feature = "vsr")] impl iggy_common::VsrSessionSealed for QuicClient {} +#[cfg(feature = "vsr")] #[async_trait::async_trait] impl iggy_common::VsrSessionControl for QuicClient { async fn bind_vsr_session(&self, session: u64) -> Result<(), IggyError> { @@ -283,8 +302,11 @@ impl QuicClient { connected_at: Mutex::new(None), leader_redirection_state: Mutex::new(LeaderRedirectionState::new()), current_server_address: Mutex::new(server_address), + #[cfg(feature = "vsr")] consensus_session: Arc::new(StdMutex::new(ConsensusSession::new())), + #[cfg(feature = "vsr")] skip_auto_login_once: Mutex::new(false), + #[cfg(feature = "vsr")] consumer_group_state: Arc::new(iggy_common::ConsumerGroupClientState::new()), }) } @@ -319,7 +341,43 @@ impl QuicClient { return Err(IggyError::EmptyResponse); } - crate::vsr::decode_response(Bytes::from(buffer)) + #[cfg(feature = "vsr")] + { + crate::vsr::decode_response(Bytes::from(buffer)) + } + + #[cfg(not(feature = "vsr"))] + { + let status = u32::from_le_bytes( + buffer[..4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); + if status != 0 { + error!( + "Received an invalid response with status: {} ({}).", + status, + IggyError::from_code_as_string(status) + ); + + return Err(IggyError::from_code(status)); + } + + let length = u32::from_le_bytes( + buffer[4..RESPONSE_INITIAL_BYTES_LENGTH] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); + trace!("Status: OK. Response length: {}", length); + if length <= 1 { + return Ok(Bytes::new()); + } + + Ok(Bytes::copy_from_slice( + &buffer[RESPONSE_INITIAL_BYTES_LENGTH + ..RESPONSE_INITIAL_BYTES_LENGTH + length as usize], + )) + } } async fn connect(&self) -> Result<(), IggyError> { @@ -434,6 +492,7 @@ impl QuicClient { self.connected_at.lock().await.replace(now); self.publish_event(DiagnosticEvent::Connected).await; + #[cfg(feature = "vsr")] let skip_auto_login = { let mut guard = self.skip_auto_login_once.lock().await; std::mem::take(&mut *guard) @@ -446,11 +505,20 @@ impl QuicClient { // Leadership still matters without auto-login: the caller // signs in manually, and a login against a non-leader // replays for its whole read timeout. `GetClusterMetadata` - // is sessionless and pre-auth, so the check works on the - // unauthenticated connection. - self.handle_leader_redirection().await? + // is sessionless and pre-auth on server-ng, so the check + // works on the unauthenticated connection. vsr-only: the + // legacy server auth-gates cluster metadata, so this check + // would bounce `Unauthenticated` into the reconnect path + // and recurse back into `connect`. + #[cfg(feature = "vsr")] + { + self.handle_leader_redirection().await? + } + #[cfg(not(feature = "vsr"))] + false } AutoLogin::Enabled(credentials) => { + #[cfg(feature = "vsr")] if skip_auto_login { info!("Skipping automatic sign-in for a retried login/register request."); false @@ -489,6 +557,35 @@ impl QuicClient { } } + self.handle_leader_redirection().await? + } + #[cfg(not(feature = "vsr"))] + { + info!( + "{NAME} client: {} is signing in...", + self.config.client_address + ); + self.set_state(ClientState::Authenticating).await; + match credentials { + Credentials::UsernamePassword(username, password) => { + self.login_user(username, password.expose_secret()).await?; + self.publish_event(DiagnosticEvent::SignedIn).await; + info!( + "{NAME} client: {} has signed in with the user credentials, username: {username}", + self.config.client_address + ); + } + Credentials::PersonalAccessToken(token) => { + self.login_with_personal_access_token(token.expose_secret()) + .await?; + self.publish_event(DiagnosticEvent::SignedIn).await; + info!( + "{NAME} client: {} has signed in with a personal access token.", + self.config.client_address + ); + } + } + self.handle_leader_redirection().await? } } @@ -551,6 +648,7 @@ impl QuicClient { } self.endpoint.wait_idle().await; + #[cfg(feature = "vsr")] self.reset_vsr_session().await?; self.set_state(ClientState::Shutdown).await; self.publish_event(DiagnosticEvent::Shutdown).await; @@ -570,6 +668,7 @@ impl QuicClient { self.set_state(ClientState::Disconnected).await; self.connection.lock().await.take(); self.endpoint.wait_idle().await; + #[cfg(feature = "vsr")] self.reset_vsr_session().await?; self.publish_event(DiagnosticEvent::Disconnected).await; let now = IggyTimestamp::now(); @@ -605,6 +704,7 @@ impl QuicClient { let connection = self.connection.clone(); let response_buffer_size = self.config.response_buffer_size; + #[cfg(feature = "vsr")] let consensus_session = self.consensus_session.clone(); // SAFETY: we run code holding the `connection` lock in a task so we can't be cancelled while holding the lock. tokio::spawn(async move { @@ -614,7 +714,9 @@ impl QuicClient { return Err(IggyError::NotConnected); }; - let (request_header, request_size) = { + #[cfg(feature = "vsr")] + { + let (request_header, request_size) = { let mut consensus_session = consensus_session .lock() .expect("consensus session mutex poisoned"); @@ -692,6 +794,42 @@ impl QuicClient { Err(error) => return Err(error), } } + } + + #[cfg(not(feature = "vsr"))] + { + let payload_length = payload.len() + REQUEST_INITIAL_BYTES_LENGTH; + let (mut send, mut recv) = connection.open_bi().await.map_err(|error| { + error!("Failed to open a bidirectional stream: {error}"); + IggyError::QuicError + })?; + trace!("Sending a QUIC request with code: {code}"); + send.write_all(&(payload_length as u32).to_le_bytes()) + .await + .map_err(|error| { + error!("Failed to write payload length: {error}"); + IggyError::QuicError + })?; + send.write_all(&code.to_le_bytes()).await.map_err(|error| { + error!("Failed to write payload code: {error}"); + IggyError::QuicError + })?; + send.write_all(&payload).await.map_err(|error| { + error!("Failed to write payload: {error}"); + IggyError::QuicError + })?; + send.finish().map_err(|error| { + error!("Failed to finish sending data: {error}"); + IggyError::QuicError + })?; + trace!("Sent a QUIC request with code: {code}, waiting for a response..."); + QuicClient::handle_response( + &mut recv, + response_buffer_size as usize, + RESPONSE_READ_TIMEOUT, + ) + .await + } }) .await .map_err(|e| { @@ -701,6 +839,7 @@ impl QuicClient { } } +#[cfg(feature = "vsr")] const fn is_login_register_code(code: u32) -> bool { matches!(code, LOGIN_REGISTER_CODE | LOGIN_REGISTER_WITH_PAT_CODE) } diff --git a/core/sdk/src/session.rs b/core/sdk/src/session.rs index a26e101e00..ac56c0b337 100644 --- a/core/sdk/src/session.rs +++ b/core/sdk/src/session.rs @@ -24,7 +24,7 @@ //! The SDK tracks the `(client_id, session)` pair and a monotonically //! increasing `request_id` counter. These values populate the consensus //! headers (`RequestHeader.client`, `.session`, `.request`) when the -//! transport sends requests through the server. +//! transport sends requests through server-ng. //! //! ## Lifecycle //! diff --git a/core/sdk/src/tcp/tcp_client.rs b/core/sdk/src/tcp/tcp_client.rs index d68e3d698e..f827b9f55b 100644 --- a/core/sdk/src/tcp/tcp_client.rs +++ b/core/sdk/src/tcp/tcp_client.rs @@ -18,14 +18,21 @@ use crate::leader_aware::{LeaderRedirectionState, check_and_redirect_to_leader}; use crate::prelude::Client; use crate::prelude::TcpClientConfig; +#[cfg(feature = "vsr")] use crate::session::ConsensusSession; use crate::tcp::tcp_connection_stream::TcpConnectionStream; use crate::tcp::tcp_connection_stream_kind::ConnectionStreamKind; use crate::tcp::tcp_tls_connection_stream::TcpTlsConnectionStream; use async_broadcast::{Receiver, Sender, broadcast}; use async_trait::async_trait; +#[cfg(not(feature = "vsr"))] +use bytes::BufMut; use bytes::{Bytes, BytesMut}; +#[cfg(feature = "vsr")] use iggy_binary_protocol::codes::{LOGIN_REGISTER_CODE, LOGIN_REGISTER_WITH_PAT_CODE}; +#[cfg(not(feature = "vsr"))] +use iggy_common::IggyErrorDiscriminants; +#[cfg(feature = "vsr")] use iggy_common::VsrSessionControl as _; use iggy_common::{ AutoLogin, ClientState, ConnectionString, ConnectionStringUtils, Credentials, DiagnosticEvent, @@ -37,6 +44,7 @@ use secrecy::ExposeSecret; use std::net::SocketAddr; use std::str::FromStr; use std::sync::Arc; +#[cfg(feature = "vsr")] use std::sync::Mutex as StdMutex; use tokio::net::TcpStream; use tokio::sync::Mutex; @@ -44,11 +52,16 @@ use tokio::time::sleep; use tokio_rustls::{TlsConnector, TlsStream}; use tracing::{error, info, trace, warn}; +#[cfg(not(feature = "vsr"))] +const REQUEST_INITIAL_BYTES_LENGTH: usize = 4; +#[cfg(not(feature = "vsr"))] +const RESPONSE_INITIAL_BYTES_LENGTH: usize = 8; const NAME: &str = "Iggy"; /// Upper bound for awaiting a reply on the lockstep VSR connection. Far /// beyond any healthy round-trip; only trips when the server loses the /// reply entirely (e.g. stalled replication quorum), which would otherwise /// hold the stream lock forever and wedge the client. +#[cfg(feature = "vsr")] const RESPONSE_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); /// Backoff before replaying a request the server answered with an explicit @@ -56,6 +69,7 @@ const RESPONSE_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_sec /// view-change cancel). The reply arrives promptly, so a short pause keeps the /// replay from spinning while the primary catches up. Bounded by /// `RESPONSE_READ_TIMEOUT`. +#[cfg(feature = "vsr")] const NOT_READY_RETRY_INTERVAL: std::time::Duration = std::time::Duration::from_millis(50); /// How long a request replays `TransientNotCommitted` on the SAME connection @@ -64,6 +78,7 @@ const NOT_READY_RETRY_INTERVAL: std::time::Duration = std::time::Duration::from_ /// forever, so replaying alone never recovers; periodically consult the /// roster and fail over to the leader. Bounded by `RESPONSE_READ_TIMEOUT` /// overall. +#[cfg(feature = "vsr")] const TRANSIENT_FAILOVER_CHECK_INTERVAL: std::time::Duration = std::time::Duration::from_secs(2); /// TCP client for interacting with the Iggy API. @@ -78,12 +93,15 @@ pub struct TcpClient { pub(crate) connected_at: Mutex>, leader_redirection_state: Mutex, pub(crate) current_server_address: Mutex, + #[cfg(feature = "vsr")] // `std::sync::Mutex` (not `tokio::sync::Mutex`): the critical section // is `encode_request_header`, which is pure CPU and never awaits. The // tokio variant would pay a waker alloc + internal semaphore on // contention with zero correctness benefit. consensus_session: Arc>, + #[cfg(feature = "vsr")] skip_auto_login_once: Mutex, + #[cfg(feature = "vsr")] consumer_group_state: Arc, } @@ -153,18 +171,21 @@ impl BinaryTransport for TcpClient { return Err(IggyError::Disconnected); } + #[cfg(feature = "vsr")] if matches!(self.config.auto_login, AutoLogin::Disabled) && !is_login_register_code(code) { // Without auto-login a reconnect cannot re-establish the session, // so non-login requests fail fast. Login/register itself is the // exception: the server stays deliberately silent on transient - // register failures (the server `surface_login_failure`) and + // register failures (server-ng `surface_login_failure`) and // relies on the client timing out and replaying the request. return Err(error); } self.disconnect().await?; + #[cfg(feature = "vsr")] let skip_auto_login = is_login_register_code(code); + #[cfg(feature = "vsr")] if skip_auto_login { *self.skip_auto_login_once.lock().await = true; } @@ -179,6 +200,7 @@ impl BinaryTransport for TcpClient { } let reconnect = self.connect().await; + #[cfg(feature = "vsr")] if skip_auto_login && reconnect.is_err() { *self.skip_auto_login_once.lock().await = false; } @@ -190,13 +212,16 @@ impl BinaryTransport for TcpClient { self.config.heartbeat_interval } + #[cfg(feature = "vsr")] fn consumer_group_state(&self) -> Arc { Arc::clone(&self.consumer_group_state) } } +#[cfg(feature = "vsr")] impl iggy_common::VsrSessionSealed for TcpClient {} +#[cfg(feature = "vsr")] #[async_trait::async_trait] impl iggy_common::VsrSessionControl for TcpClient { async fn bind_vsr_session(&self, session: u64) -> Result<(), IggyError> { @@ -286,12 +311,56 @@ impl TcpClient { connected_at: Mutex::new(None), leader_redirection_state: Mutex::new(LeaderRedirectionState::new()), current_server_address: Mutex::new(server_address), + #[cfg(feature = "vsr")] consensus_session: Arc::new(StdMutex::new(ConsensusSession::new())), + #[cfg(feature = "vsr")] skip_auto_login_once: Mutex::new(false), + #[cfg(feature = "vsr")] consumer_group_state: Arc::new(iggy_common::ConsumerGroupClientState::new()), }) } + #[cfg(not(feature = "vsr"))] + async fn handle_response( + status: u32, + length: u32, + stream: &mut ConnectionStreamKind, + ) -> Result { + if status != 0 { + // TEMP: See https://github.com/apache/iggy/pull/604 for context. + if status == IggyErrorDiscriminants::TopicNameAlreadyExists as u32 + || status == IggyErrorDiscriminants::StreamNameAlreadyExists as u32 + || status == IggyErrorDiscriminants::UserAlreadyExists as u32 + || status == IggyErrorDiscriminants::PersonalAccessTokenAlreadyExists as u32 + || status == IggyErrorDiscriminants::ConsumerGroupNameAlreadyExists as u32 + { + tracing::debug!( + "Received a server resource already exists response: {} ({})", + status, + IggyError::from_code_as_string(status) + ) + } else { + error!( + "Received an invalid response with status: {} ({}).", + status, + IggyError::from_code_as_string(status), + ); + } + + return Err(IggyError::from_code(status)); + } + + trace!("Status: OK. Response length: {}", length); + if length <= 1 { + return Ok(Bytes::new()); + } + + let mut response_buffer = BytesMut::with_capacity(length as usize); + response_buffer.put_bytes(0, length as usize); + stream.read(&mut response_buffer).await?; + Ok(response_buffer.freeze()) + } + async fn connect(&self) -> Result<(), IggyError> { loop { match self.get_state().await { @@ -473,6 +542,7 @@ impl TcpClient { self.set_state(ClientState::Connected).await; self.connected_at.lock().await.replace(now); self.publish_event(DiagnosticEvent::Connected).await; + #[cfg(feature = "vsr")] let skip_auto_login = { let mut guard = self.skip_auto_login_once.lock().await; std::mem::take(&mut *guard) @@ -485,11 +555,20 @@ impl TcpClient { // Leadership still matters without auto-login: the caller // signs in manually, and a login against a non-leader // replays for its whole read timeout. `GetClusterMetadata` - // is sessionless and pre-auth, so the check works on the - // unauthenticated connection. - self.handle_leader_redirection().await? + // is sessionless and pre-auth on server-ng, so the check + // works on the unauthenticated connection. vsr-only: the + // legacy server auth-gates cluster metadata, so this check + // would bounce `Unauthenticated` into the reconnect path + // and recurse back into `connect`. + #[cfg(feature = "vsr")] + { + self.handle_leader_redirection().await? + } + #[cfg(not(feature = "vsr"))] + false } AutoLogin::Enabled(credentials) => { + #[cfg(feature = "vsr")] if skip_auto_login { info!("Skipping automatic sign-in for a retried login/register request."); false @@ -521,6 +600,28 @@ impl TcpClient { } } + self.handle_leader_redirection().await? + } + #[cfg(not(feature = "vsr"))] + { + info!("{NAME} client: {client_address} is signing in..."); + self.set_state(ClientState::Authenticating).await; + match credentials { + Credentials::UsernamePassword(username, password) => { + self.login_user(username, password.expose_secret()).await?; + info!( + "{NAME} client: {client_address} has signed in with the user credentials, username: {username}", + ); + } + Credentials::PersonalAccessToken(token) => { + self.login_with_personal_access_token(token.expose_secret()) + .await?; + info!( + "{NAME} client: {client_address} has signed in with a personal access token.", + ); + } + } + self.handle_leader_redirection().await? } } @@ -580,6 +681,7 @@ impl TcpClient { info!("{NAME} client: {client_address} is disconnecting from server..."); self.set_state(ClientState::Disconnected).await; self.stream.lock().await.take(); + #[cfg(feature = "vsr")] self.reset_vsr_session().await?; self.publish_event(DiagnosticEvent::Disconnected).await; let now = IggyTimestamp::now(); @@ -598,6 +700,7 @@ impl TcpClient { if let Some(mut stream) = stream { stream.shutdown().await?; } + #[cfg(feature = "vsr")] self.reset_vsr_session().await?; self.set_state(ClientState::Shutdown).await; self.publish_event(DiagnosticEvent::Shutdown).await; @@ -622,62 +725,126 @@ impl TcpClient { _ => {} } - // One overall deadline bounds the request across transient replays - // AND leader failovers, matching the previous single-connection - // budget. Login/register replays stay on this connection for the - // whole budget: the connect flow owns leader redirection for the - // sign-in handshake, and reconnecting from underneath it would - // recurse. - let overall_deadline = tokio::time::Instant::now() + RESPONSE_READ_TIMEOUT; - let mut preencoded = None; - loop { - let transient_deadline = if is_login_register_code(code) { - overall_deadline - } else { - overall_deadline - .min(tokio::time::Instant::now() + TRANSIENT_FAILOVER_CHECK_INTERVAL) - }; - let (header, result) = self - .send_raw_vsr_attempt( - code, - payload.clone(), - preencoded, - transient_deadline, - overall_deadline, - ) - .await; - match result { - Err(IggyError::TransientNotAccepted) - if tokio::time::Instant::now() < overall_deadline - && !is_login_register_code(code) => - { - // The server explicitly did NOT admit the request, so - // re-issuing it -- same id on this session, or a fresh - // id under a new session after a failover -- cannot - // double-apply. Keep the encoded id for same-session - // replays; a redirect re-registers, so the id is - // re-encoded under the new session. - // (`TransientNotCommitted` never reaches this branch: - // its outcome is unknown, so the attempt loop replays - // it same-session for the whole budget and then the - // error propagates to the caller.) - preencoded = header; - if let Ok(true) = self.handle_leader_redirection().await { - self.connect().await?; - preencoded = None; + #[cfg(feature = "vsr")] + { + // One overall deadline bounds the request across transient replays + // AND leader failovers, matching the previous single-connection + // budget. Login/register replays stay on this connection for the + // whole budget: the connect flow owns leader redirection for the + // sign-in handshake, and reconnecting from underneath it would + // recurse. + let overall_deadline = tokio::time::Instant::now() + RESPONSE_READ_TIMEOUT; + let mut preencoded = None; + loop { + let transient_deadline = if is_login_register_code(code) { + overall_deadline + } else { + overall_deadline + .min(tokio::time::Instant::now() + TRANSIENT_FAILOVER_CHECK_INTERVAL) + }; + let (header, result) = self + .send_raw_vsr_attempt( + code, + payload.clone(), + preencoded, + transient_deadline, + overall_deadline, + ) + .await; + match result { + Err(IggyError::TransientNotAccepted) + if tokio::time::Instant::now() < overall_deadline + && !is_login_register_code(code) => + { + // The server explicitly did NOT admit the request, so + // re-issuing it -- same id on this session, or a fresh + // id under a new session after a failover -- cannot + // double-apply. Keep the encoded id for same-session + // replays; a redirect re-registers, so the id is + // re-encoded under the new session. + // (`TransientNotCommitted` never reaches this branch: + // its outcome is unknown, so the attempt loop replays + // it same-session for the whole budget and then the + // error propagates to the caller.) + preencoded = header; + if let Ok(true) = self.handle_leader_redirection().await { + self.connect().await?; + preencoded = None; + } + } + Err(IggyError::Disconnected) => { + // Reply stream state is unknown (timed out or torn + // mid-frame); a late reply would desync framing for the + // next request, so drop the connection and let callers + // reconnect. + self.stream.lock().await.take(); + self.set_state(ClientState::Disconnected).await; + return Err(IggyError::Disconnected); } + other => return other, } - Err(IggyError::Disconnected) => { - // Reply stream state is unknown (timed out or torn - // mid-frame); a late reply would desync framing for the - // next request, so drop the connection and let callers - // reconnect. - self.stream.lock().await.take(); - self.set_state(ClientState::Disconnected).await; - return Err(IggyError::Disconnected); + } + } + + #[cfg(not(feature = "vsr"))] + { + let stream = self.stream.clone(); + // SAFETY: we run code holding the `stream` lock in a task so we can't be cancelled while holding the lock. + let result = tokio::spawn(async move { + let mut stream = stream.lock().await; + if let Some(stream) = stream.as_mut() { + let payload_length = payload.len() + REQUEST_INITIAL_BYTES_LENGTH; + trace!("Sending a TCP request of size {payload_length} with code: {code}"); + stream.write(&(payload_length as u32).to_le_bytes()).await?; + stream.write(&code.to_le_bytes()).await?; + stream.write(&payload).await?; + stream.flush().await?; + trace!("Sent a TCP request with code: {code}, waiting for a response..."); + let mut response_buffer = [0u8; RESPONSE_INITIAL_BYTES_LENGTH]; + let read_bytes = stream.read(&mut response_buffer).await.map_err(|error| { + error!( + "Failed to read response for TCP request with code: {code}: {error}", + code = code, + error = error + ); + IggyError::Disconnected + })?; + + if read_bytes != RESPONSE_INITIAL_BYTES_LENGTH { + error!("Received an invalid or empty response."); + return Err(IggyError::EmptyResponse); + } + + let status = u32::from_le_bytes( + response_buffer[..4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); + let length = u32::from_le_bytes( + response_buffer[4..] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); + return TcpClient::handle_response(status, length, stream).await; } - other => return other, + + error!("Cannot send data. Client is not connected."); + Err(IggyError::NotConnected) + }) + .await + .map_err(|e| { + error!("Task execution failed during TCP request: {}", e); + IggyError::TcpError + })?; + + if matches!(result, Err(IggyError::Disconnected)) { + // Reply stream state is unknown (timed out or torn mid-frame); + // a late reply would desync framing for the next request, so + // drop the connection and let callers reconnect. + self.stream.lock().await.take(); + self.set_state(ClientState::Disconnected).await; } + result } } @@ -688,6 +855,7 @@ impl TcpClient { /// full request budget -- so a short transient window cannot tear down a /// connection that is merely slow to reply. Returns the header used so the /// caller can replay the same id on a later attempt. + #[cfg(feature = "vsr")] async fn send_raw_vsr_attempt( &self, code: u32, @@ -858,6 +1026,7 @@ impl TcpClient { } } +#[cfg(feature = "vsr")] const fn is_login_register_code(code: u32) -> bool { matches!(code, LOGIN_REGISTER_CODE | LOGIN_REGISTER_WITH_PAT_CODE) } @@ -868,6 +1037,26 @@ const fn is_login_register_code(code: u32) -> bool { #[cfg(test)] mod tests { use super::*; + #[cfg(not(feature = "vsr"))] + use tokio::io::AsyncWriteExt; + #[cfg(not(feature = "vsr"))] + use tokio::net::TcpListener; + + #[cfg(not(feature = "vsr"))] + async fn make_dummy_stream(data: &[u8]) -> ConnectionStreamKind { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + + let data = data.to_vec(); + tokio::spawn(async move { + let (mut server_side, _) = listener.accept().await.unwrap(); + server_side.write_all(&data).await.unwrap(); + }); + + let client = tokio::net::TcpStream::connect(addr).await.unwrap(); + let client_addr = client.local_addr().unwrap(); + ConnectionStreamKind::Tcp(TcpConnectionStream::new(client_addr, client)) + } #[test] fn should_fail_with_empty_connection_string() { @@ -1136,4 +1325,44 @@ mod tests { IggyDuration::from_str("5s").unwrap() ); } + + #[cfg(not(feature = "vsr"))] + #[tokio::test] + async fn should_return_error_when_status_is_non_zero() { + let mut stream = make_dummy_stream(&[1u8; 10]).await; + let tcp_client = TcpClient::handle_response(1, 0, &mut stream).await; + assert!(tcp_client.is_err()); + } + + #[cfg(not(feature = "vsr"))] + #[tokio::test] + async fn should_return_ok_when_status_is_zero() { + let mut stream = make_dummy_stream(&[1u8; 10]).await; + let tcp_client = TcpClient::handle_response(0, 0, &mut stream).await; + assert!(tcp_client.is_ok()); + } + + #[cfg(not(feature = "vsr"))] + #[tokio::test] + async fn should_return_ok_when_length_is_less_than_data() { + let mut stream = make_dummy_stream(&[1u8; 10]).await; + let tcp_client = TcpClient::handle_response(0, 5, &mut stream).await; + assert!(tcp_client.is_ok()); + } + + #[cfg(not(feature = "vsr"))] + #[tokio::test] + async fn should_return_ok_when_length_is_equal_to_one() { + let mut stream = make_dummy_stream(&[1u8; 10]).await; + let tcp_client = TcpClient::handle_response(0, 1, &mut stream).await; + assert_eq!(tcp_client.unwrap(), Bytes::new()); + } + + #[cfg(not(feature = "vsr"))] + #[tokio::test] + async fn should_return_err_when_length_exceeds_data() { + let mut stream = make_dummy_stream(&[1u8; 10]).await; + let tcp_client = TcpClient::handle_response(0, 50, &mut stream).await; + assert!(tcp_client.is_err()); + } } diff --git a/core/sdk/src/tcp/tcp_connection_stream.rs b/core/sdk/src/tcp/tcp_connection_stream.rs index 1d87d38161..299e0fb289 100644 --- a/core/sdk/src/tcp/tcp_connection_stream.rs +++ b/core/sdk/src/tcp/tcp_connection_stream.rs @@ -17,6 +17,7 @@ use crate::tcp::tcp_stream::ConnectionStream; use async_trait::async_trait; +#[cfg(feature = "vsr")] use bytes::BytesMut; use iggy_common::IggyError; use std::net::SocketAddr; @@ -42,6 +43,7 @@ impl TcpConnectionStream { } } + #[cfg(feature = "vsr")] pub async fn read_buf(&mut self, buf: &mut BytesMut, len: usize) -> Result<(), IggyError> { let target_len = buf.len() + len; while buf.len() < target_len { diff --git a/core/sdk/src/tcp/tcp_connection_stream_kind.rs b/core/sdk/src/tcp/tcp_connection_stream_kind.rs index ce51a7d6c6..2617ea5987 100644 --- a/core/sdk/src/tcp/tcp_connection_stream_kind.rs +++ b/core/sdk/src/tcp/tcp_connection_stream_kind.rs @@ -18,6 +18,7 @@ use crate::tcp::tcp_connection_stream::TcpConnectionStream; use crate::tcp::tcp_stream::ConnectionStream; use crate::tcp::tcp_tls_connection_stream::TcpTlsConnectionStream; +#[cfg(feature = "vsr")] use bytes::BytesMut; use iggy_common::IggyError; @@ -36,6 +37,7 @@ impl ConnectionStreamKind { } } + #[cfg(feature = "vsr")] pub async fn read_buf(&mut self, buf: &mut BytesMut, len: usize) -> Result<(), IggyError> { match self { Self::Tcp(c) => c.read_buf(buf, len).await, diff --git a/core/sdk/src/tcp/tcp_tls_connection_stream.rs b/core/sdk/src/tcp/tcp_tls_connection_stream.rs index a49a2a6ec1..b01ddf1df6 100644 --- a/core/sdk/src/tcp/tcp_tls_connection_stream.rs +++ b/core/sdk/src/tcp/tcp_tls_connection_stream.rs @@ -17,6 +17,7 @@ use crate::tcp::tcp_stream::ConnectionStream; use async_trait::async_trait; +#[cfg(feature = "vsr")] use bytes::BytesMut; use iggy_common::IggyError; use std::net::SocketAddr; @@ -39,6 +40,7 @@ impl TcpTlsConnectionStream { } } + #[cfg(feature = "vsr")] pub async fn read_buf(&mut self, buf: &mut BytesMut, len: usize) -> Result<(), IggyError> { let target_len = buf.len() + len; while buf.len() < target_len { diff --git a/core/sdk/src/vsr.rs b/core/sdk/src/vsr.rs index e0c0efcbdc..48fd28e314 100644 --- a/core/sdk/src/vsr.rs +++ b/core/sdk/src/vsr.rs @@ -17,14 +17,28 @@ use crate::session::ConsensusSession; use bytes::{BufMut, Bytes, BytesMut}; +use iggy_binary_protocol::codec::WireDecode; use iggy_binary_protocol::codes::{ - LOGIN_REGISTER_CODE, LOGIN_REGISTER_WITH_PAT_CODE, LOGOUT_USER_CODE, + DELETE_CONSUMER_OFFSET_2_CODE, DELETE_CONSUMER_OFFSET_CODE, DELETE_SEGMENTS_CODE, + LOGIN_REGISTER_CODE, LOGIN_REGISTER_WITH_PAT_CODE, LOGOUT_USER_CODE, SEND_MESSAGES_CODE, + STORE_CONSUMER_OFFSET_2_CODE, STORE_CONSUMER_OFFSET_CODE, }; use iggy_binary_protocol::consensus::{ Command2, EvictionHeader, EvictionReason, GenericHeader, HEADER_SIZE, Operation, ReplyHeader, RequestHeader, read_size_field, result_code, result_section_len, }; -use iggy_common::{IggyError, calculate_checksum, eviction_reason_to_error}; +use iggy_binary_protocol::namespace::{ + MAX_PARTITIONS, MAX_STREAMS, MAX_TOPICS, METADATA_CONSENSUS_NAMESPACE, PARTITION_MASK, + PARTITION_SHIFT, STREAM_MASK, STREAM_SHIFT, TOPIC_MASK, TOPIC_SHIFT, +}; +use iggy_binary_protocol::requests::consumer_offsets::{ + DeleteConsumerOffset2Request, DeleteConsumerOffsetRequest, StoreConsumerOffset2Request, + StoreConsumerOffsetRequest, +}; +use iggy_binary_protocol::requests::messages::SendMessagesHeader; +use iggy_binary_protocol::requests::segments::DeleteSegmentsRequest; +use iggy_binary_protocol::{WireIdentifier, WirePartitioning}; +use iggy_common::{IggyError, eviction_reason_to_error}; const NON_REPLICATED_CODE_RANGE: std::ops::Range = 0..4; @@ -111,20 +125,12 @@ pub(crate) fn encode_request_header( } } }; - // Stamped only for ops the server's `ClientTable` dedups. Partition ops are - // at-least-once with no reply cache to poison, and theirs are the large payloads, - // already covered client-side by `batch_checksum` over the same bytes. - // NonReplicated ops bypass dedup too. - let request_checksum = if operation.is_partition() || operation == Operation::NonReplicated { - 0 - } else { - u128::from(calculate_checksum(payload)) - }; + let namespace = namespace_for_request(code, payload, operation)?; let total_size = HEADER_SIZE .checked_add(payload.len()) .ok_or(IggyError::InvalidConfiguration)?; let size = u32::try_from(total_size).map_err(|_| IggyError::InvalidConfiguration)?; - let mut reserved = [0; 60]; + let mut reserved = [0; 52]; if operation == Operation::NonReplicated { reserved[NON_REPLICATED_CODE_RANGE].copy_from_slice(&code.to_le_bytes()); } @@ -135,11 +141,7 @@ pub(crate) fn encode_request_header( client: session.client_id(), request: request_id, session: session_id, - // Lets the client table tell a genuine retry from a `request` number reused - // for different arguments. Zero means unstamped, which is what an SDK - // predating this sends. A server that rewrites the body (PAT, password) - // carries it through untouched, so it keeps describing what the client sent. - request_checksum, + namespace, // Zeroed: the field is "informational" -- the server copies it into // `ReplyHeader.timestamp` for RTT but nothing else reads it. Paying // a `clock_gettime` syscall per encoded request (formerly held the @@ -343,11 +345,131 @@ fn read_window_field(header_bytes: &[u8; HEADER_SIZE], offset: usize) -> u32 { u32::from_le_bytes(value) } +fn namespace_for_request( + code: u32, + payload: &Bytes, + operation: Operation, +) -> Result { + // Control-plane requests target the metadata replica (shard 0). The + // router's `route_typed` only short-circuits to shard 0 when the + // namespace value equals `METADATA_CONSENSUS_NAMESPACE`; sending plain + // `0` falls into `route_consensus_control` which hashes the namespace + // and lands a Register on a peer shard whose `submit_register_in_process` + // panics ("consensus only exists on shard 0"). + if operation == Operation::Register || operation == Operation::Logout { + return Ok(METADATA_CONSENSUS_NAMESPACE); + } + if operation == Operation::NonReplicated || operation.is_metadata() { + return Ok(0); + } + + let namespace = match code { + SEND_MESSAGES_CODE => { + if payload.len() < 4 { + return Err(IggyError::InvalidCommand); + } + let metadata_length = u32::from_le_bytes( + payload[..4] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ) as usize; + if payload.len() < 4 + metadata_length { + return Err(IggyError::InvalidCommand); + } + let header = SendMessagesHeader::decode_from(&payload[4..4 + metadata_length]) + .map_err(|_| IggyError::InvalidCommand)?; + namespace_from_partitioning(&header.stream_id, &header.topic_id, &header.partitioning)? + } + STORE_CONSUMER_OFFSET_CODE => { + let request = StoreConsumerOffsetRequest::decode_from(payload) + .map_err(|_| IggyError::InvalidCommand)?; + namespace_from_partition(&request.stream_id, &request.topic_id, request.partition_id)? + } + DELETE_CONSUMER_OFFSET_CODE => { + let request = DeleteConsumerOffsetRequest::decode_from(payload) + .map_err(|_| IggyError::InvalidCommand)?; + namespace_from_partition(&request.stream_id, &request.topic_id, request.partition_id)? + } + STORE_CONSUMER_OFFSET_2_CODE => { + let request = StoreConsumerOffset2Request::decode_from(payload) + .map_err(|_| IggyError::InvalidCommand)?; + namespace_from_partition(&request.stream_id, &request.topic_id, request.partition_id)? + } + DELETE_CONSUMER_OFFSET_2_CODE => { + let request = DeleteConsumerOffset2Request::decode_from(payload) + .map_err(|_| IggyError::InvalidCommand)?; + namespace_from_partition(&request.stream_id, &request.topic_id, request.partition_id)? + } + DELETE_SEGMENTS_CODE => { + let request = DeleteSegmentsRequest::decode_from(payload) + .map_err(|_| IggyError::InvalidCommand)?; + namespace_from_partition( + &request.stream_id, + &request.topic_id, + Some(request.partition_id), + )? + } + _ => return Err(IggyError::FeatureUnavailable), + }; + + Ok(namespace) +} + +fn namespace_from_partitioning( + stream_id: &WireIdentifier, + topic_id: &WireIdentifier, + partitioning: &WirePartitioning, +) -> Result { + let WirePartitioning::PartitionId(partition_id) = partitioning else { + return Err(IggyError::FeatureUnavailable); + }; + namespace_from_partition(stream_id, topic_id, Some(*partition_id)) +} + +fn namespace_from_partition( + stream_id: &WireIdentifier, + topic_id: &WireIdentifier, + partition_id: Option, +) -> Result { + let partition_id = partition_id.ok_or(IggyError::InvalidIdentifier)?; + let Some(stream_id) = stream_id.as_u32() else { + return Ok(0); + }; + let Some(topic_id) = topic_id.as_u32() else { + return Ok(0); + }; + validate_namespace_field(stream_id, MAX_STREAMS)?; + validate_namespace_field(topic_id, MAX_TOPICS)?; + validate_namespace_field(partition_id, MAX_PARTITIONS)?; + Ok(pack_namespace( + stream_id as usize, + topic_id as usize, + partition_id as usize, + )) +} + +fn validate_namespace_field(value: u32, exclusive_max: usize) -> Result<(), IggyError> { + let value = usize::try_from(value).map_err(|_| IggyError::InvalidIdentifier)?; + if value >= exclusive_max { + return Err(IggyError::InvalidIdentifier); + } + Ok(()) +} + +fn pack_namespace(stream_id: usize, topic_id: usize, partition_id: usize) -> u64 { + ((stream_id as u64) & STREAM_MASK) << STREAM_SHIFT + | ((topic_id as u64) & TOPIC_MASK) << TOPIC_SHIFT + | ((partition_id as u64) & PARTITION_MASK) << PARTITION_SHIFT +} + #[cfg(test)] mod tests { use super::*; use crate::session::ConsensusSession; - use iggy_binary_protocol::codes::{CREATE_STREAM_CODE, GET_STREAM_CODE, PING_CODE}; + use iggy_binary_protocol::codes::{ + CREATE_STREAM_CODE, GET_STREAM_CODE, LOGOUT_USER_CODE, PING_CODE, + }; + use iggy_binary_protocol::requests::messages::SendMessagesHeader; use iggy_binary_protocol::requests::streams::CreateStreamRequest; use iggy_binary_protocol::requests::users::LoginRegisterRequest; use iggy_binary_protocol::version::IGGY_PROTOCOL_VERSION; @@ -358,6 +480,34 @@ mod tests { *bytemuck::checked::try_from_bytes::(&bytes[..HEADER_SIZE]).unwrap() } + #[test] + fn register_request_uses_zero_request_and_session() { + let mut session = ConsensusSession::with_client_id(7); + let request = LoginRegisterRequest { + version_info: ClientVersionInfo { + protocol_version: IGGY_PROTOCOL_VERSION, + sdk_name: WireName::new("rust-sdk").unwrap(), + sdk_version: WireName::new("1.0.0").unwrap(), + }, + username: WireName::new("admin").unwrap(), + password: SecretString::from("secret"), + client_context: None, + }; + + let bytes = + encode_contiguous_request(&mut session, LOGIN_REGISTER_CODE, &request.to_bytes()) + .unwrap(); + let header = decode_request_header(&bytes); + + assert_eq!(header.operation, Operation::Register); + assert_eq!(header.request, 0); + assert_eq!(header.session, 0); + assert_eq!(header.client, 7); + // Register is routed to the metadata replica (shard 0). The router's + // namespace==METADATA short-circuit needs the sentinel, not 0. + assert_eq!(header.namespace, METADATA_CONSENSUS_NAMESPACE); + } + #[test] fn second_register_on_bound_session_re_arms_instead_of_panicking() { let request = LoginRegisterRequest { @@ -483,27 +633,7 @@ mod tests { assert_eq!(decode_request_header(&first).request, 1); assert_eq!(decode_request_header(&second).request, 2); assert_eq!(decode_request_header(&second).session, 99); - } - - #[test] - fn request_checksum_is_stamped_only_for_deduped_operations() { - // The stamp exists to stop a reused `request` number returning the wrong - // cached reply, so it is worth its hashing pass only where `ClientTable` - // dedups. Partition payloads are the large ones and carry `batch_checksum` - // over the same bytes already; hashing them again is pure cost. - let mut session = ConsensusSession::with_client_id(42); - session.bind(99); - let payload = Bytes::from_static(b"payload"); - - let deduped = - encode_contiguous_request(&mut session, CREATE_STREAM_CODE, &payload).unwrap(); - assert_eq!( - decode_request_header(&deduped).request_checksum, - u128::from(calculate_checksum(&payload)), - ); - - let ping = encode_contiguous_request(&mut session, PING_CODE, &Bytes::new()).unwrap(); - assert_eq!(decode_request_header(&ping).request_checksum, 0); + assert_eq!(decode_request_header(&second).namespace, 0); } #[test] @@ -523,6 +653,7 @@ mod tests { PING_CODE ); assert_eq!(header.session, 99); + assert_eq!(header.namespace, 0); } #[test] @@ -536,6 +667,9 @@ mod tests { assert_eq!(header.operation, Operation::Logout); assert_eq!(header.request, 1); assert_eq!(header.session, 99); + // Logout, like Register, is routed to shard 0 via the metadata + // sentinel rather than namespace 0. + assert_eq!(header.namespace, METADATA_CONSENSUS_NAMESPACE); } #[test] @@ -606,6 +740,50 @@ mod tests { } } + #[test] + fn namespace_defers_named_identifiers_to_server_resolution() { + let stream = WireIdentifier::named("stream").unwrap(); + let topic = WireIdentifier::numeric(1); + let namespace = namespace_from_partition(&stream, &topic, Some(0)).unwrap(); + assert_eq!(namespace, 0); + } + + #[test] + fn namespace_rejects_out_of_range_fields() { + let stream = WireIdentifier::numeric(MAX_STREAMS as u32); + let topic = WireIdentifier::numeric(1); + let err = namespace_from_partition(&stream, &topic, Some(0)).unwrap_err(); + assert!(matches!(err, IggyError::InvalidIdentifier)); + + let stream = WireIdentifier::numeric(1); + let partition_id = u32::try_from(MAX_PARTITIONS).unwrap(); + let err = namespace_from_partition(&stream, &topic, Some(partition_id)).unwrap_err(); + assert!(matches!(err, IggyError::InvalidIdentifier)); + } + + #[test] + fn send_messages_with_numeric_partition_builds_namespace() { + let header = SendMessagesHeader { + stream_id: WireIdentifier::numeric(2), + topic_id: WireIdentifier::numeric(3), + partitioning: WirePartitioning::PartitionId(4), + messages_count: 0, + }; + let mut payload = BytesMut::new(); + payload.put_u32_le(header.metadata_length() as u32); + header.encode(&mut payload); + + let namespace = namespace_for_request( + SEND_MESSAGES_CODE, + &payload.freeze(), + Operation::SendMessages, + ) + .unwrap(); + assert_eq!((namespace >> STREAM_SHIFT) & STREAM_MASK, 2); + assert_eq!((namespace >> TOPIC_SHIFT) & TOPIC_MASK, 3); + assert_eq!((namespace >> PARTITION_SHIFT) & PARTITION_MASK, 4); + } + #[test] fn metadata_success_reply_strips_result_section_and_returns_payload() { let mut body = BytesMut::new(); diff --git a/core/sdk/src/websocket/websocket_client.rs b/core/sdk/src/websocket/websocket_client.rs index 514ab6d21a..537d68de22 100644 --- a/core/sdk/src/websocket/websocket_client.rs +++ b/core/sdk/src/websocket/websocket_client.rs @@ -16,6 +16,7 @@ // under the License. use crate::leader_aware::{LeaderRedirectionState, check_and_redirect_to_leader}; +#[cfg(feature = "vsr")] use crate::session::ConsensusSession; use crate::websocket::websocket_connection_stream::WebSocketConnectionStream; use crate::websocket::websocket_stream_kind::WebSocketStreamKind; @@ -26,7 +27,13 @@ use crate::prelude::Client; use async_broadcast::{Receiver, Sender, broadcast}; use async_trait::async_trait; use bytes::Bytes; +#[cfg(not(feature = "vsr"))] +use bytes::{BufMut, BytesMut}; +#[cfg(feature = "vsr")] use iggy_binary_protocol::codes::{LOGIN_REGISTER_CODE, LOGIN_REGISTER_WITH_PAT_CODE}; +#[cfg(not(feature = "vsr"))] +use iggy_common::IggyErrorDiscriminants; +#[cfg(feature = "vsr")] use iggy_common::VsrSessionControl as _; use iggy_common::{ AutoLogin, ClientState, ConnectionString, Credentials, DiagnosticEvent, IggyDuration, @@ -36,6 +43,7 @@ use iggy_common::{BinaryClient, BinaryTransport, PersonalAccessTokenClient, User use secrecy::ExposeSecret; use std::net::SocketAddr; use std::sync::Arc; +#[cfg(feature = "vsr")] use std::sync::Mutex as StdMutex; use tokio::net::TcpStream; use tokio::sync::Mutex; @@ -46,11 +54,16 @@ use tokio_tungstenite::{ }; use tracing::{debug, error, info, trace, warn}; +#[cfg(not(feature = "vsr"))] +const REQUEST_INITIAL_BYTES_LENGTH: usize = 4; +#[cfg(not(feature = "vsr"))] +const RESPONSE_INITIAL_BYTES_LENGTH: usize = 8; const NAME: &str = "WebSocket"; /// Bound on how long a single VSR reply read may block. The connection is /// lockstep and the read runs in the caller's task while holding the stream /// lock, so an unanswered read (lost server reply) would wedge every later /// request on this client forever. On expiry the stream is dropped. +#[cfg(feature = "vsr")] const RESPONSE_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); /// Backoff before replaying a request the server answered with an explicit @@ -58,6 +71,7 @@ const RESPONSE_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_sec /// view-change cancel). The reply arrives promptly, so a short pause keeps the /// replay from spinning while the primary catches up. Bounded by /// `RESPONSE_READ_TIMEOUT`. +#[cfg(feature = "vsr")] const NOT_READY_RETRY_INTERVAL: std::time::Duration = std::time::Duration::from_millis(50); #[derive(Debug)] @@ -70,10 +84,13 @@ pub struct WebSocketClient { pub(crate) connected_at: Mutex>, leader_redirection_state: Mutex, pub(crate) current_server_address: Mutex, + #[cfg(feature = "vsr")] // See `core/sdk/src/tcp/tcp_client.rs` for the `tokio::sync::Mutex` -> // `std::sync::Mutex` rationale (pure-CPU critical section). consensus_session: Arc>, + #[cfg(feature = "vsr")] skip_auto_login_once: Mutex, + #[cfg(feature = "vsr")] consumer_group_state: Arc, } @@ -145,13 +162,16 @@ impl BinaryTransport for WebSocketClient { return Err(IggyError::Disconnected); } + #[cfg(feature = "vsr")] if matches!(self.config.auto_login, AutoLogin::Disabled) { return Err(error); } self.disconnect().await?; + #[cfg(feature = "vsr")] let skip_auto_login = is_login_register_code(code); + #[cfg(feature = "vsr")] if skip_auto_login { *self.skip_auto_login_once.lock().await = true; } @@ -165,6 +185,7 @@ impl BinaryTransport for WebSocketClient { } let reconnect = self.connect().await; + #[cfg(feature = "vsr")] if skip_auto_login && reconnect.is_err() { *self.skip_auto_login_once.lock().await = false; } @@ -176,13 +197,16 @@ impl BinaryTransport for WebSocketClient { self.config.heartbeat_interval } + #[cfg(feature = "vsr")] fn consumer_group_state(&self) -> Arc { Arc::clone(&self.consumer_group_state) } } +#[cfg(feature = "vsr")] impl iggy_common::VsrSessionSealed for WebSocketClient {} +#[cfg(feature = "vsr")] #[async_trait::async_trait] impl iggy_common::VsrSessionControl for WebSocketClient { async fn bind_vsr_session(&self, session: u64) -> Result<(), IggyError> { @@ -231,8 +255,11 @@ impl WebSocketClient { connected_at: Mutex::new(None), leader_redirection_state: Mutex::new(LeaderRedirectionState::new()), current_server_address: Mutex::new(server_address), + #[cfg(feature = "vsr")] consensus_session: Arc::new(StdMutex::new(ConsensusSession::new())), + #[cfg(feature = "vsr")] skip_auto_login_once: Mutex::new(false), + #[cfg(feature = "vsr")] consumer_group_state: Arc::new(iggy_common::ConsumerGroupClientState::new()), }) } @@ -509,15 +536,25 @@ impl WebSocketClient { match &self.config.auto_login { // Leadership still matters without auto-login: the caller signs in // manually, and a login against a non-leader replays for its whole - // read timeout. `GetClusterMetadata` is sessionless and pre-auth, - // so the check works on the unauthenticated connection. + // read timeout. `GetClusterMetadata` is sessionless and pre-auth on + // server-ng, so the check works on the unauthenticated connection. + // vsr-only: the legacy server auth-gates cluster metadata, so this + // check would bounce `Unauthenticated` into the reconnect path and + // recurse back into `connect`. + #[cfg(feature = "vsr")] AutoLogin::Disabled => self.handle_leader_redirection().await, + #[cfg(not(feature = "vsr"))] + AutoLogin::Disabled => Ok(false), AutoLogin::Enabled(_) => { // Check leadership BEFORE signing in: register/login are // consensus ops a backup answers with a transient frame, so // signing in against a non-leader replays for the whole read // timeout instead of failing over. `GetClusterMetadata` is - // sessionless and pre-auth. + // sessionless and pre-auth on server-ng. vsr-only: the legacy + // server auth-gates cluster metadata, so this pre-login check + // would bounce `Unauthenticated` into the reconnect path and + // recurse back into `connect`. + #[cfg(feature = "vsr")] if self.handle_leader_redirection().await? { return Ok(true); } @@ -564,6 +601,7 @@ impl WebSocketClient { async fn auto_login(&self) -> Result<(), IggyError> { let client_address = self.get_client_address_value().await; + #[cfg(feature = "vsr")] let skip_auto_login = { let mut guard = self.skip_auto_login_once.lock().await; std::mem::take(&mut *guard) @@ -575,6 +613,7 @@ impl WebSocketClient { Ok(()) } AutoLogin::Enabled(credentials) => { + #[cfg(feature = "vsr")] if skip_auto_login { info!("Skipping automatic sign-in for a retried login/register request."); return Ok(()); @@ -612,6 +651,7 @@ impl WebSocketClient { self.set_state(ClientState::Disconnected).await; self.stream.lock().await.take(); + #[cfg(feature = "vsr")] self.reset_vsr_session().await?; self.publish_event(DiagnosticEvent::Disconnected).await; @@ -635,6 +675,7 @@ impl WebSocketClient { let _ = stream.shutdown().await; } + #[cfg(feature = "vsr")] self.reset_vsr_session().await?; self.set_state(ClientState::Shutdown).await; self.publish_event(DiagnosticEvent::Shutdown).await; @@ -665,6 +706,7 @@ impl WebSocketClient { return Err(IggyError::NotConnected); } + #[cfg(feature = "vsr")] { // Encode the request ONCE: `next_request_id` advances here, so a // transient replay must reuse the same id for the server's dedup. @@ -743,9 +785,87 @@ impl WebSocketClient { } } } + + #[cfg(not(feature = "vsr"))] + { + let stream = stream_guard.as_mut().ok_or_else(|| { + trace!("Cannot send data. Client is not connected."); + IggyError::NotConnected + })?; + let payload_length = payload.len() + REQUEST_INITIAL_BYTES_LENGTH; + let mut request = + BytesMut::with_capacity(4 + REQUEST_INITIAL_BYTES_LENGTH + payload.len()); + request.put_u32_le(payload_length as u32); + request.put_u32_le(code); + request.put_slice(&payload); + trace!( + "Sending {NAME} message with code: {}, payload size: {} bytes", + code, + payload.len() + ); + stream.write(&request).await?; + stream.flush().await?; + + let mut response_initial_buffer = vec![0u8; RESPONSE_INITIAL_BYTES_LENGTH]; + stream.read(&mut response_initial_buffer).await?; + + let status = u32::from_le_bytes([ + response_initial_buffer[0], + response_initial_buffer[1], + response_initial_buffer[2], + response_initial_buffer[3], + ]); + + let length = u32::from_le_bytes([ + response_initial_buffer[4], + response_initial_buffer[5], + response_initial_buffer[6], + response_initial_buffer[7], + ]) as usize; + + trace!( + "Received {NAME} response status: {}, length: {} bytes", + status, length + ); + + if status != 0 { + // TEMP: See https://github.com/apache/iggy/pull/604 for context. + if status == IggyErrorDiscriminants::TopicNameAlreadyExists as u32 + || status == IggyErrorDiscriminants::StreamNameAlreadyExists as u32 + || status == IggyErrorDiscriminants::UserAlreadyExists as u32 + || status == IggyErrorDiscriminants::PersonalAccessTokenAlreadyExists as u32 + || status == IggyErrorDiscriminants::ConsumerGroupNameAlreadyExists as u32 + { + debug!( + "Received a server resource already exists response: {} ({})", + status, + IggyError::from_code_as_string(status) + ) + } else { + error!( + "Received an invalid response with status: {} ({}).", + status, + IggyError::from_code_as_string(status), + ); + } + + return Err(IggyError::from_code(status)); + } + + if length == 0 { + return Ok(Bytes::new()); + } + + let mut response_buffer = vec![0u8; length]; + stream.read(&mut response_buffer).await?; + + trace!("Received {NAME} response payload, size: {} bytes", length); + Ok(Bytes::from(response_buffer)) + } } } +#[cfg(feature = "vsr")] const fn is_login_register_code(code: u32) -> bool { matches!(code, LOGIN_REGISTER_CODE | LOGIN_REGISTER_WITH_PAT_CODE) } diff --git a/core/server-ng/.dockerignore b/core/server-ng/.dockerignore new file mode 100644 index 0000000000..e2fdd5f0dd --- /dev/null +++ b/core/server-ng/.dockerignore @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +.config +.github +/assets +/local_data +/licenses +/certs +/helm +/web +Dockerfile +docker-compose.yml +.dockerignore +.git +.gitignore diff --git a/core/server-ng/Cargo.toml b/core/server-ng/Cargo.toml new file mode 100644 index 0000000000..4b66dc09df --- /dev/null +++ b/core/server-ng/Cargo.toml @@ -0,0 +1,195 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "server-ng" +version = "0.9.0-edge.2" +edition = "2024" +license = "Apache-2.0" +publish = false + +[package.metadata.cargo-udeps.ignore] +normal = ["tracing-appender"] + +[package.metadata.cargo-machete] +ignored = [ + "ahash", + "anyhow", + "argon2", + "async-channel", + "async_zip", + "axum", + "axum-server", + "bytes", + "chrono", + "ctrlc", + "cyper", + "cyper-axum", + "dashmap", + "err_trail", + "error_set", + "figlet-rs", + "hash32", + "human-repr", + "hwlocality", + "jsonwebtoken", + "left-right", + "mimalloc", + "mime_guess", + "nix", + "opentelemetry", + "opentelemetry-appender-tracing", + "opentelemetry-otlp", + "opentelemetry-semantic-conventions", + "opentelemetry_sdk", + "papaya", + "rand", + "ringbuffer", + "rmp-serde", + "rolling-file", + "rust-embed", + "rustls", + "rustls-pemfile", + "send_wrapper", + "serde", + "slab", + "socket2", + "strum", + "sysinfo", + "tempfile", + "tracing-appender", + "tracing-opentelemetry", + "ulid", + "uuid", + "vergen-git2", +] + +[[bin]] +name = "iggy-server-ng" +path = "src/main.rs" + +[features] +default = ["mimalloc", "iggy-web"] +disable-mimalloc = [] +mimalloc = ["dep:mimalloc"] +iggy-web = ["dep:rust-embed", "dep:mime_guess"] +vsr = ["iggy_common/vsr"] + +[dependencies] +ahash = { workspace = true } +argon2 = { workspace = true } +async-channel = { workspace = true } +async_zip = { workspace = true } +axum = { workspace = true } +axum-server = { workspace = true } +blake3 = { workspace = true } +bytemuck = { workspace = true } +bytes = { workspace = true } +chrono = { workspace = true } +clap = { workspace = true } +compio = { workspace = true } +configs = { workspace = true } +consensus = { workspace = true } +crossfire = { workspace = true } +ctrlc = { workspace = true } +cyper = { workspace = true } +cyper-axum = { workspace = true } +cyper-core = { workspace = true } +dashmap = { workspace = true } +dotenvy = { workspace = true } +err_trail = { workspace = true } +error_set = { workspace = true } +figlet-rs = { workspace = true } +fs2 = { workspace = true } +futures = { workspace = true } +hash32 = { workspace = true } +human-repr = { workspace = true } +hyper = { workspace = true } +hyper-util = { workspace = true } +iggy_binary_protocol = { workspace = true } +iggy_common = { workspace = true } +journal = { workspace = true } +jsonwebtoken = { workspace = true } +left-right = { workspace = true } +message_bus = { workspace = true } +metadata = { workspace = true } +mimalloc = { workspace = true, optional = true } +mime_guess = { workspace = true, optional = true } +nix = { workspace = true } +opentelemetry = { workspace = true } +opentelemetry-appender-tracing = { workspace = true } +opentelemetry-otlp = { workspace = true } +opentelemetry-semantic-conventions = { workspace = true } +opentelemetry_sdk = { workspace = true } +papaya = { workspace = true } +partitions = { workspace = true } +prometheus-client = { workspace = true } +rand = { workspace = true } +ringbuffer = { workspace = true } +rmp-serde = { workspace = true } +rolling-file = { workspace = true } +rust-embed = { workspace = true, optional = true } +rustls = { workspace = true } +rustls-pemfile = { workspace = true } +secrecy = { workspace = true } +send_wrapper = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +server_common = { workspace = true } +shard = { workspace = true } +shard_allocator = { workspace = true } +slab = { workspace = true } +socket2 = { workspace = true } +strum = { workspace = true } +sysinfo = { workspace = true } +system_stats = { workspace = true } +tempfile = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true } +toml = { workspace = true } +tower-http = { workspace = true } +tracing = { workspace = true } +tracing-appender = { workspace = true } +tracing-opentelemetry = { workspace = true } +ulid = { workspace = true } +uuid = { workspace = true } + +[target.'cfg(not(target_env = "musl"))'.dependencies] +hwlocality = { workspace = true } + +[target.'cfg(target_env = "musl")'.dependencies] +hwlocality = { workspace = true, features = ["vendored"] } + +[build-dependencies] +vergen-git2 = { workspace = true } + +[dev-dependencies] +assert_cmd = { workspace = true } +bytemuck = { workspace = true } +# Reconciler unit tests assert on `ShardMetrics` snapshots and +# `IggyShard::parked_frame_count`, gated to test/simulator so they cannot grow +# production callers. `shard`'s own `cfg(test)` is false when compiled as our +# dependency, so the feature is how those accessors become visible. The resolver +# keeps a dev-dependency's features out of non-test targets, so a production +# build still links `shard` without `simulator`. +shard = { workspace = true, features = ["simulator"] } +tokio = { workspace = true, features = ["full", "test-util"] } + +[lints.clippy] +enum_glob_use = "deny" +pedantic = "deny" +nursery = "warn" diff --git a/core/server-ng/Dockerfile b/core/server-ng/Dockerfile new file mode 100644 index 0000000000..a1bb1a91c7 --- /dev/null +++ b/core/server-ng/Dockerfile @@ -0,0 +1,179 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Apache Iggy (Incubating) is an effort undergoing incubation at the Apache +# Software Foundation (ASF), sponsored by the Apache Incubator PMC. +# +# Incubation is required of all newly accepted projects until a further review +# indicates that the infrastructure, communications, and decision making +# process have stabilized in a manner consistent with other successful ASF +# projects. +# +# While incubation status is not necessarily a reflection of the completeness +# or stability of the code, it does indicate that the project has yet to be +# fully endorsed by the ASF. + +ARG RUST_VERSION=1.97.1 +ARG ALPINE_VERSION=3.23 + +# ── from-source path ───────────────────────────────────────────────────────── +FROM --platform=$BUILDPLATFORM lukemathwalker/cargo-chef:latest-rust-${RUST_VERSION}-alpine${ALPINE_VERSION} AS chef +WORKDIR /app +RUN apk add --no-cache musl-dev pkgconfig + +FROM --platform=$BUILDPLATFORM chef AS planner +COPY . . +RUN cargo chef prepare --recipe-path recipe.json + +FROM --platform=$BUILDPLATFORM chef AS builder +ARG PROFILE=release +ARG TARGETPLATFORM +ARG LIBC=musl +ARG IGGY_CI_BUILD +ENV IGGY_CI_BUILD=${IGGY_CI_BUILD} + +RUN apk add --no-cache zig make autoconf automake libtool pkgconfig hwloc-dev xz-dev xz-static nodejs npm && \ + cargo install cargo-zigbuild --version '=0.22.1' --locked + +COPY rust-toolchain.toml rust-toolchain.toml + +RUN rustup target add \ + x86_64-unknown-linux-musl \ + aarch64-unknown-linux-musl \ + x86_64-unknown-linux-gnu \ + aarch64-unknown-linux-gnu + +# Allow pkg-config to work in cross-compilation mode (needed for hwlocality-sys) +ENV PKG_CONFIG_ALLOW_CROSS=1 + +COPY --from=planner /app/recipe.json recipe.json + +RUN --mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARGETPLATFORM}-${LIBC} \ + --mount=type=cache,target=/usr/local/cargo/git,id=cargo-git-${TARGETPLATFORM}-${LIBC} \ + --mount=type=cache,target=/app/target,id=cargo-target-${TARGETPLATFORM}-${LIBC} \ + case "$TARGETPLATFORM:$LIBC" in \ + "linux/amd64:musl") RUST_TARGET="x86_64-unknown-linux-musl" ;; \ + "linux/arm64:musl") RUST_TARGET="aarch64-unknown-linux-musl" ;; \ + "linux/amd64:glibc") RUST_TARGET="x86_64-unknown-linux-gnu" ;; \ + "linux/arm64:glibc") RUST_TARGET="aarch64-unknown-linux-gnu" ;; \ + *) echo "Unsupported $TARGETPLATFORM/$LIBC" && exit 1 ;; \ + esac && \ + if [ "$PROFILE" = "debug" ]; then \ + cargo chef cook --recipe-path recipe.json --target ${RUST_TARGET} --zigbuild \ + --features vsr -p server-ng -p iggy-cli; \ + else \ + cargo chef cook --recipe-path recipe.json --target ${RUST_TARGET} --zigbuild --release \ + --features vsr -p server-ng -p iggy-cli; \ + fi + +COPY . . + +# Build Web UI static files for embedding +RUN npm --prefix web ci && npm --prefix web run build:static + +RUN --mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARGETPLATFORM}-${LIBC} \ + --mount=type=cache,target=/usr/local/cargo/git,id=cargo-git-${TARGETPLATFORM}-${LIBC} \ + --mount=type=cache,target=/app/target,id=cargo-target-${TARGETPLATFORM}-${LIBC} \ + case "$TARGETPLATFORM:$LIBC" in \ + "linux/amd64:musl") RUST_TARGET="x86_64-unknown-linux-musl" ;; \ + "linux/arm64:musl") RUST_TARGET="aarch64-unknown-linux-musl" ;; \ + "linux/amd64:glibc") RUST_TARGET="x86_64-unknown-linux-gnu" ;; \ + "linux/arm64:glibc") RUST_TARGET="aarch64-unknown-linux-gnu" ;; \ + *) echo "Unsupported $TARGETPLATFORM/$LIBC" && exit 1 ;; \ + esac && \ + if [ "$PROFILE" = "debug" ]; then \ + cargo zigbuild --locked --target ${RUST_TARGET} --features vsr --bin iggy-server-ng --bin iggy && \ + cp /app/target/${RUST_TARGET}/debug/iggy-server-ng /app/iggy-server-ng && \ + cp /app/target/${RUST_TARGET}/debug/iggy /app/iggy; \ + else \ + cargo zigbuild --locked --target ${RUST_TARGET} --features vsr --bin iggy-server-ng --bin iggy --release && \ + cp /app/target/${RUST_TARGET}/release/iggy-server-ng /app/iggy-server-ng && \ + cp /app/target/${RUST_TARGET}/release/iggy /app/iggy; \ + fi + +# ── prebuilt path (FAST) ────────────────────────────────────────────────────── +FROM debian:trixie-slim AS prebuilt +WORKDIR /out +ARG PREBUILT_IGGY_SERVER_NG +ARG PREBUILT_IGGY_CLI +COPY ${PREBUILT_IGGY_SERVER_NG} /out/iggy-server-ng +COPY ${PREBUILT_IGGY_CLI} /out/iggy +RUN chmod +x /out/iggy-server-ng /out/iggy + +# ── final images ────────────────────────────────────────────────────────────── +FROM debian:trixie-slim AS runtime-prebuilt +ARG TARGETPLATFORM +ARG PREBUILT_IGGY_SERVER_NG +ARG PREBUILT_IGGY_CLI +WORKDIR /app +RUN apt-get update && apt-get install -y \ + libhwloc-dev \ + libudev-dev \ + pkg-config \ + && rm -rf /var/lib/apt/lists/* +COPY --from=prebuilt /out/iggy-server-ng /usr/local/bin/iggy-server-ng +COPY --from=prebuilt /out/iggy /usr/local/bin/iggy +RUN echo "═══════════════════════════════════════════════════════════════" && \ + echo " IGGY SERVER-NG BUILD SUMMARY " && \ + echo "═══════════════════════════════════════════════════════════════" && \ + echo "Build Type: PREBUILT BINARIES" && \ + echo "Platform: ${TARGETPLATFORM:-linux/amd64}" && \ + echo "Source Path: ${PREBUILT_IGGY_SERVER_NG:-not specified}" && \ + echo "Binary Info:" && \ + (command -v file >/dev/null 2>&1 && file /usr/local/bin/iggy-server-ng | sed 's/^/ /' || \ + echo " $(ldd /usr/local/bin/iggy-server-ng 2>&1 | head -1)") && \ + echo "Binary Size:" && \ + ls -lh /usr/local/bin/iggy-server-ng /usr/local/bin/iggy | awk '{print " " $9 ": " $5}' && \ + echo "Build Date: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" && \ + echo "Container Base: debian:trixie-slim" && \ + echo "═══════════════════════════════════════════════════════════════" +ENTRYPOINT ["iggy-server-ng"] + +FROM debian:trixie-slim AS runtime +ARG TARGETPLATFORM +ARG PROFILE=release +ARG LIBC=musl +WORKDIR /app +RUN apt-get update && apt-get install -y \ + libhwloc15 \ + libudev1 \ + pkg-config \ + && rm -rf /var/lib/apt/lists/* +COPY --from=builder /app/iggy-server-ng /usr/local/bin/iggy-server-ng +COPY --from=builder /app/iggy /usr/local/bin/iggy +RUN echo "═══════════════════════════════════════════════════════════════" && \ + echo " IGGY SERVER-NG BUILD SUMMARY " && \ + echo "═══════════════════════════════════════════════════════════════" && \ + echo "Build Type: FROM SOURCE" && \ + echo "Platform: ${TARGETPLATFORM:-linux/amd64}" && \ + echo "Profile: ${PROFILE}" && \ + echo "Libc: ${LIBC}" && \ + case "${TARGETPLATFORM:-linux/amd64}:${LIBC}" in \ + "linux/amd64:musl") echo "Target: x86_64-unknown-linux-musl" ;; \ + "linux/arm64:musl") echo "Target: aarch64-unknown-linux-musl" ;; \ + "linux/amd64:glibc") echo "Target: x86_64-unknown-linux-gnu" ;; \ + "linux/arm64:glibc") echo "Target: aarch64-unknown-linux-gnu" ;; \ + *) echo "Target: unknown" ;; \ + esac && \ + echo "Binary Info:" && \ + (command -v file >/dev/null 2>&1 && file /usr/local/bin/iggy-server-ng | sed 's/^/ /' || \ + echo " $(ldd /usr/local/bin/iggy-server-ng 2>&1 | head -1)") && \ + echo "Binary Size:" && \ + ls -lh /usr/local/bin/iggy-server-ng /usr/local/bin/iggy | awk '{print " " $9 ": " $5}' && \ + echo "Build Date: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" && \ + echo "═══════════════════════════════════════════════════════════════" +ENTRYPOINT ["iggy-server-ng"] diff --git a/core/server-ng/build.rs b/core/server-ng/build.rs new file mode 100644 index 0000000000..4d07885ebb --- /dev/null +++ b/core/server-ng/build.rs @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::path::PathBuf; +use std::{env, error}; +use vergen_git2::{Build, Cargo, Emitter, Git2, Rustc, Sysinfo}; + +const WEB_ASSETS_PATH: &str = "web/build/static"; +const WEB_INDEX_FILE: &str = "web/build/static/index.html"; + +fn main() -> Result<(), Box> { + verify_web_assets_if_enabled(); + emit_vergen_instructions()?; + Ok(()) +} + +/// Returns the workspace root (iggy/), two levels up from core/server-ng. +fn workspace_root() -> PathBuf { + PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()) + .parent() + .and_then(|p| p.parent()) + .expect("server-ng crate must be at core/server-ng within workspace") + .to_path_buf() +} + +fn emit_vergen_instructions() -> Result<(), Box> { + if option_env!("IGGY_CI_BUILD") != Some("true") { + println!("cargo:info=Skipping vergen because IGGY_CI_BUILD is not set to 'true'"); + return Ok(()); + } + + Emitter::default() + .add_instructions(&Build::all_build())? + .add_instructions(&Cargo::all_cargo())? + .add_instructions(&Git2::all_git())? + .add_instructions(&Rustc::all_rustc())? + .add_instructions(&Sysinfo::all_sysinfo())? + .emit()?; + + let configs_path = workspace_root() + .join("core/configs") + .canonicalize() + .unwrap_or_else(|e| panic!("Failed to canonicalize configs path: {e}")); + + println!("cargo:rerun-if-changed={}", configs_path.display()); + Ok(()) +} + +fn verify_web_assets_if_enabled() { + if env::var("CARGO_FEATURE_IGGY_WEB").is_err() { + return; + } + + let assets_dir = workspace_root().join(WEB_ASSETS_PATH); + let index_file = workspace_root().join(WEB_INDEX_FILE); + + println!("cargo:rerun-if-changed={}", assets_dir.display()); + + if !assets_dir.exists() || !index_file.exists() { + println!( + "cargo:info=Web UI assets not found at {}. \ + To build them, run: npm --prefix web ci && npm --prefix web run build:static", + assets_dir.display() + ); + return; + } + + println!( + "cargo:info=Web UI assets verified at {}", + assets_dir.display() + ); +} diff --git a/core/server-ng/config.toml b/core/server-ng/config.toml new file mode 100644 index 0000000000..10a529930a --- /dev/null +++ b/core/server-ng/config.toml @@ -0,0 +1,1027 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Configuration for consumer group cooperative partition rebalancing. +[consumer_group] +# Maximum time a partition can remain in pending revocation before being force-transferred to the target member. +rebalancing_timeout = "30s" +# How often the periodic checker scans for timed-out pending revocations. +# TODO(hubcio): inert in server-ng, which paces the scan from +# system.sharding.reconcile_periodic_interval instead. Boot warns when set. +rebalancing_check_interval = "5s" + +[data_maintenance.messages] +# Enables or disables the expired message cleaner process. +cleaner_enabled = false + +# Interval for running the message cleaner. +interval = "1 m" + +# HTTP server configuration +[http] +# Determines if the HTTP server is active. +# `true` enables the server, allowing it to handle HTTP requests. +# `false` disables the server, preventing it from handling HTTP requests. +# In cluster mode, followers forward control-plane requests (streams, topics, +# users, ...) to the current primary when a cluster-wide JWT key exists (see +# http.jwt / cluster.auth below). +# TODO: forwarding does not cover the partition-plane APIs yet - message +# produce and consumer-offset writes are never forwarded and must reach the +# partition's primary node directly (message polls read locally on any node). +enabled = true + +# Specifies the network address and port for the HTTP server. +# The format is "HOST:PORT". For example, "127.0.0.1:3000" listens on localhost only on port 3000. +# In cluster mode the HOST still picks the bind interface, while the port +# comes from this node's cluster.nodes ports.http entry. +address = "127.0.0.1:3000" + +# Maximum size of the request body in bytes. For security reasons, the default limit is 2 MB. +max_request_size = "2 MB" + +# Enables the embedded Web UI dashboard at '/ui'. +# When set to `true` and the server is compiled with the 'iggy-web' feature, +# the Svelte dashboard will be served at the '/ui' endpoint, providing a +# browser-based interface for managing streams, topics, and viewing messages. +# If the server is compiled without 'iggy-web' feature and this is set to `true`, +# a warning will be logged at startup but the server will continue to run. +# `true` enables the embedded Web UI (requires server built with 'iggy-web' feature). +# `false` disables the embedded Web UI (default). +web_ui = false + +# Configuration for Cross-Origin Resource Sharing (CORS). +[http.cors] +# Controls whether CORS is enabled for the HTTP server. +# `true` allows handling cross-origin requests with specified rules. +# `false` blocks cross-origin requests, enhancing security. +enabled = true + +# Specifies which HTTP methods are allowed when CORS is enabled. +# For example, ["GET", "POST"] would allow only GET and POST requests. +allowed_methods = ["GET", "POST", "PUT", "DELETE"] + +# Defines which origins are permitted to make cross-origin requests. +# An asterisk "*" as the first entry allows all origins (any entries after it +# are ignored); "*" in any other position fails the config. Specific domains +# can be listed to restrict access. +allowed_origins = ["*"] + +# Lists allowed headers that can be used in CORS requests. +# For example, ["content-type"] permits only the content-type header. +allowed_headers = ["content-type", "authorization"] + +# Headers that browsers are allowed to access in CORS responses. +# `iggy-view` carries the current VSR view number; exposing it lets browser +# clients read it on cross-origin responses. +exposed_headers = ["iggy-view"] + +# Determines if credentials like cookies or HTTP auth can be included in CORS requests. +# `true` allows credentials to be included, useful for authenticated sessions; +# it requires explicit (non-wildcard) allowed_origins, allowed_headers, and +# exposed_headers. +# `false` prevents credentials, enhancing privacy and security. +allow_credentials = false + +# Allows or blocks requests from private networks in CORS. +# `true` permits requests from private networks. +# `false` disallows such requests, providing additional security. +allow_private_network = false + +# JWT (JSON Web Token) configuration for HTTP. +[http.jwt] +# Specifies the algorithm used for signing JWTs. +# For example, "HS256" indicates HMAC with SHA-256. +algorithm = "HS256" + +# The issuer of the JWT, typically a URL or an identifier of the issuing entity. +issuer = "iggy.apache.org" + +# Intended audience for the JWT, usually the recipient or system intended to process the token. +audience = "iggy.apache.org" + +# Lists valid issuers for JWT validation to ensure tokens are from trusted sources. +valid_issuers = ["iggy.apache.org"] + +# Lists valid audiences for JWT validation to confirm tokens are for the intended recipient. +valid_audiences = ["iggy.apache.org"] + +# Expiry time for access tokens. +access_token_expiry = "1 h" + +# Tolerance for timing discrepancies during token validation. +clock_skew = "5 s" + +# Time before which the token should not be considered valid. +not_before = "0 s" + +# Secret key for encoding JWTs. +# If left empty, a secure random secret will be generated on each server start. +# In cluster mode a configured secret (identical on every node) makes bearers +# valid cluster-wide and activates follower-to-primary HTTP forwarding; with +# cluster.auth enabled the key is instead derived from the shared PSK. Without +# either, tokens are node-local and forwarding stays disabled. +encoding_secret = "" + +# Secret key for decoding JWTs. +# If left empty, a secure random secret will be generated on each server start. +decoding_secret = "" + +# Indicates if the secret key is base64 encoded. +# `true` means the secret is base64 encoded. +# `false` means the secret is in plain text. +use_base64_secret = false + +# Trusted issuers for A2A (Application-to-Application) authentication. Opt-in: +# with none configured the listener accepts only self-issued HS256 tokens. +# `issuer`, `audience` and `jwks_url` are required per entry; `user_id` is +# optional but defaults to 0 (root), which is rejected - set it to the non-zero +# iggy user every token from that issuer is remapped onto. +# +# Operational note: enabling an issuer opens an outbound JWKS fetch that is +# reachable before a token's signature is verified - a token naming this issuer +# with an unknown key id can trigger a fetch to `jwks_url`. The target is fixed +# (not attacker-chosen); concurrent misses coalesce onto one fetch and repeats +# are rate-limited to at most one outbound request per issuer per short window, +# so an unknown-key-id flood cannot amplify. The same window bounds how quickly a +# freshly rotated signing key is picked up. +# [[http.jwt.trusted_issuers]] +# issuer = "test-issuer" +# jwks_url = "http://127.0.0.1:8081/.well-known/jwks.json" +# audience = "iggy.apache.org" +# user_id = 1 + +# Metrics configuration for HTTP. +[http.metrics] +# Enable or disable the metrics endpoint. +# `true` makes metrics available at the specified endpoint. +# `false` disables metrics collection. +enabled = true + +# Specifies the endpoint for accessing metrics, e.g., "/metrics". +endpoint = "/metrics" + +# TLS (Transport Layer Security) configuration for HTTP. +[http.tls] +# Controls the use of TLS for encrypted HTTP connections. +# `true` enables TLS, enhancing security. +# `false` disables TLS, which may be appropriate in secure internal networks. +enabled = false + +# Path to the TLS certificate file. +cert_file = "core/certs/iggy_cert.pem" + +# Path to the TLS key file. +key_file = "core/certs/iggy_key.pem" + +# TCP server configuration. +[tcp] +# Determines if the TCP server is active. +# `true` enables the TCP server for handling TCP connections. +# `false` disables it, preventing any TCP communication. +enabled = true + +# Defines the network address and port for the TCP server. +# For example, "127.0.0.1:8090" listens on localhost only on port 8090. +address = "127.0.0.1:8090" + +# Enable TCP socket migration across shards. +# TODO(hubcio): inert in server-ng, not implemented. Boot warns when set. +socket_migration = true + +# Whether to use ipv4 or ipv6 +# TODO(hubcio): inert in server-ng, which takes the family from the +# tcp.address string. Boot warns when set. +ipv6 = false + +# TLS configuration for the TCP server. +[tcp.tls] +# Enables or disables TLS for TCP connections. +# `true` secures TCP connections with TLS. +# `false` leaves TCP connections unencrypted. +enabled = false + +# Enables or disables self-signed certificate generation. +# `true` generates a self-signed certificate if cert files don't exist. +# `false` requires certificate files to exist at the specified paths. +self_signed = true + +# Path to the TLS certificate file. +cert_file = "core/certs/iggy_cert.pem" + +# Path to the TLS key file. +key_file = "core/certs/iggy_key.pem" + +# Configuration for the TCP socket +# TODO(hubcio): the whole section is inert in server-ng, which leaves the OS +# defaults in place. Boot warns when override_defaults is set. +[tcp.socket] +# Whether to overwrite the OS-default socket parameters +override_defaults = false + +# SO_RCVBUF: maximum size of the receive buffer, can be clamped by the OS +recv_buffer_size = "100 KB" + +# SO_SNDBUF: maximum size of the send buffer, can be clamped by the OS +send_buffer_size = "100 KB" + +# SO_KEEPALIVE: whether to regularly send a keepalive packet maintaining the connection +keepalive = false + +# TCP_NODELAY: enable/disable the Nagle algorithm which buffers data before sending segments +nodelay = false + +# SO_LINGER: delay to wait for while data is being transmitted before closing the socket after a +# close or shutdown call has been received +linger = "0 s" + +# QUIC protocol configuration. +[quic] +# Controls whether the QUIC server is enabled. +# `true` enables QUIC for fast, secure connections. +# `false` disables QUIC, possibly for compatibility or simplicity. +enabled = true + +# Network address and port for the QUIC server. +# For example, "127.0.0.1:8080" binds to localhost on port 8080. +address = "127.0.0.1:8080" + +# Maximum number of simultaneous bidirectional streams per QUIC +# connection. The message bus opens exactly one bidi stream per peer +# (no multiplexing), so any value above 1 is wasted on extra +# preallocated quinn-proto state. +max_concurrent_bidi_streams = 1 + +# Size of the buffer for sending datagrams in QUIC. Binary-aligned to +# match QuicTuning::default() (102_400 bytes) and the rest of this [quic] +# block, which uses MiB throughout (`send_window`, `receive_window`). +datagram_send_buffer_size = "100 KiB" + +# Initial Maximum Transmission Unit (MTU) for QUIC connections. Binary +# units for the same reason as `datagram_send_buffer_size`. +initial_mtu = "8 KiB" + +# Send-flow window per connection. Sized to fit a single max-size +# framed message (`message_bus.max_message_size`) without +# head-of-line wait. +send_window = "64 MiB" + +# Receive-flow window per connection. Symmetric with `send_window`. +receive_window = "64 MiB" + +# Interval for sending QUIC keep-alive PINGs. One third of +# `max_idle_timeout` so up to two consecutive losses fit before the +# idle timer closes the connection. Set to "0 s" to disable. +keep_alive_interval = "10 s" + +# Maximum idle time before a QUIC connection is closed. Set to +# "0 s" to disable (not recommended). +max_idle_timeout = "30 s" + +# QUIC certificate configuration. +[quic.certificate] +# Indicates whether the QUIC certificate is self-signed. +# `true` for self-signed certificates, often used in internal or testing environments. +# `false` for certificates issued by a certificate authority, common in production. +self_signed = true + +# Path to the QUIC TLS certificate file. +cert_file = "core/certs/iggy_cert.pem" + +# Path to the QUIC TLS key file. +key_file = "core/certs/iggy_key.pem" + +# Configuration for the QUIC socket +# TODO(hubcio): the whole section is inert in server-ng, which leaves the OS +# defaults in place. Boot warns when override_defaults is set. +[quic.socket] +# Whether to override the OS-default socket parameters +override_defaults = false + +# SO_RCVBUF: maximum size of the receive buffer, can be clamped by the OS +recv_buffer_size = "64 KB" + +# SO_SNDBUF: maximum size of the send buffer, can be clamped by the OS +send_buffer_size = "64 KB" + +# SO_KEEPALIVE: whether to regularly send a keepalive packet maintaining the connection +keepalive = false + +# Message saver configuration. +[message_saver] +# Enables or disables the background process for saving buffered data to disk. +# `true` ensures data is periodically written to disk. +# `false` turns off automatic saving, relying on other triggers for data persistence. +enabled = true + +# Controls whether data saving is synchronous (enforce fsync) or asynchronous. +# `true` for synchronous saving, ensuring data integrity at the cost of performance. +# `false` for asynchronous saving, improving performance but with delayed data writing. +# TODO(hubcio): inert in server-ng, which only flushes on shutdown and has no +# periodic saver to configure. Boot warns when set. +enforce_fsync = true + +# Interval for running the message saver. +# TODO(hubcio): inert in server-ng, see enforce_fsync above. Boot warns when set. +interval = "30 s" + +# Personal access token configuration. +[personal_access_token] +# Sets the maximum number of active tokens allowed per user. +max_tokens_per_user = 100 + +# Personal access token cleaner configuration. +[personal_access_token.cleaner] +# Enables or disables the token cleaner process. +# `true` activates periodic token cleaning. +# `false` disables it, tokens remain active until manually revoked or expired. +enabled = true + +# Interval for running the token cleaner. +interval = "1 m" + +# Heartbeat configuration +[heartbeat] +# Enables or disables the client heartbeat verification process. +enabled = false +# Interval for expected client heartbeats +interval = "5 s" + +# OpenTelemetry configuration +[telemetry] +# Enables or disables telemetry. +enabled = false +# Service name for telemetry. +service_name = "iggy" + +# OpenTelemetry logs configuration +[telemetry.logs] +# Transport for sending logs. Options: "grpc", "http". +transport = "grpc" +# Endpoint for sending logs. +endpoint = "http://localhost:7281/v1/logs" + +# OpenTelemetry traces configuration +[telemetry.traces] +# Transport for sending traces. Options: "grpc", "http". +transport = "grpc" +# Endpoint for sending traces. +endpoint = "http://localhost:7281/v1/traces" + +# System configuration. +[system] +# Base path for system data storage. +path = "local_data" + +# Backup configuration +# TODO(hubcio): backup is not supported in server-ng; both paths below are +# inert. Boot warns when either is set. +[system.backup] +# Path for storing backup. +path = "backup" + +# Compatibility conversion configuration +[system.backup.compatibility] +# Subpath of the backup directory where converted segment data is stored after compatibility conversion. +path = "compatibility" + +# TODO(hubcio): the three tunables below are inert in server-ng, whose state +# writes do not go through the legacy retrying file layer. Boot warns when any +# is set. +[system.state] +# Determines whether to enforce file synchronization on state updates (boolean). +# `true` ensures immediate writing of data to disk for durability. +# `false` allows the OS to manage write operations, which can improve performance. +enforce_fsync = false + +# Maximum number of retries for a failed file operation (e.g., append, overwrite). +# This defines how many times the system will attempt the operation before failing. +max_file_operation_retries = 1 + +# Delay between retries in case of a failed file operation. +# This helps to avoid immediate repeated attempts and can reduce load. +retry_delay = "1 s" + +# Runtime configuration. +[system.runtime] +# Path for storing runtime data. +# Specifies the directory where any runtime data is stored, relative to `system.path`. +path = "runtime" + +# Logging configuration. +[system.logging] +# Path for storing log files. +path = "logs" + +# Log filtering directive using the same syntax as the RUST_LOG environment variable. +# Supports simple levels ("trace", "debug", "info", "warn", "error", "off" or "none") +# as well as complex directives like "warn,server=debug,iggy=trace". +# Note: RUST_LOG environment variable always takes precedence over this setting. +level = "info" + +# Whether to write logs to file. When false, logs are only written to stdout. +# When enabled, logs are stored in {system.path}/{system.logging.path} (default: local_data/logs). +file_enabled = true + +# Maximum size of a single log file before rotation occurs. When a log +# file reaches this size, it will be rotated (closed and a new file +# created). This setting works together with max_total_size to control +# log storage. You can set it to 0 to enable unlimited size of single +# log, but all logs will be written to a single file, thus disabling +# log rotation. Please configure 0 with caution, esp. RUST_LOG > debug +max_file_size = "500 MB" + +# Maximum total size of all log files. When this size is reached, +# the oldest log files will be deleted first. Set it to 0 to allow +# an unlimited number of archived logs. This does not disable time +# based log rotation or per-log-file size limits. +max_total_size = "4 GB" + +# Time interval for checking log rotation status. Avoid less than 1s. +rotation_check_interval = "1 h" + +# Time to retain log files before deletion. Avoid less than 1s, too. +retention = "7 days" + +# Interval for printing system information to the log. +# TODO(hubcio): inert in server-ng, which has no sysinfo printer. Boot warns +# when set. +sysinfo_print_interval = "10 s" + +# Encryption configuration +[system.encryption] +# Determines whether server-side data encryption for the messages payloads and state commands is enabled (boolean). +# `true` enables encryption for stored data using AES-256-GCM. +# `false` means data is stored without encryption. +enabled = false + +# The encryption key used when encryption is enabled (string). +# Should be a 32 bytes length key, provided as a base64 encoded string. +# This key is required and used only if encryption is enabled. +key = "" + +# Compression configuration +[system.compression] +# Allows overriding the default compression algorithm per data segment (boolean). +# `true` permits different compression algorithms for individual segments. +# `false` means all data segments use the default compression algorithm. +# TODO(hubcio): inert in server-ng, where live compression is already per-topic +# from the request. Boot warns when set. +allow_override = false + +# The default compression algorithm used for data storage (string). +# "none" indicates no compression, other values can specify different algorithms. +default_algorithm = "none" + +# Stream configuration +[system.stream] +# Path for storing stream-related data (string). +# Specifies the directory where stream data is stored, relative to `system.path`. +path = "streams" + +# Topic configuration - default settings for new topics +[system.topic] +# Path for storing topic-related data, relative to `stream.path`. +path = "topics" + +# Messages can be deleted based on two independent policies: +# 1. Size-based: delete oldest segments when topic exceeds max_size +# 2. Time-based: delete segments older than message_expiry +# Both can be active simultaneously. Per-topic overrides via CreateTopic/UpdateTopic. + +# Maximum topic size before oldest segments are deleted. +# "unlimited" or "0" = no size limit (messages kept indefinitely). +# When 90% of this limit is reached, oldest segments are removed to make room. +# Applies to sealed segments only (active segment is protected). +# Example: "10 GiB" +max_size = "unlimited" + +# Maximum age of messages before segments are deleted. +# "none" = no time limit (messages kept indefinitely). +# Applies to sealed segments only (active segment is protected). +# Example: "7 days", "2 days 4 hours 15 minutes" +message_expiry = "none" + +# Partition configuration +[system.partition] +# Path for storing partition-related data (string). +# Specifies the directory where partition data is stored, relative to `topic.path`. +path = "partitions" + +# Determines whether to enforce file synchronization on partition updates (boolean). +# `true` ensures immediate writing of data to disk for durability. +# `false` allows the OS to manage write operations, which can improve performance. +enforce_fsync = false + +# Enables checksum validation for data integrity (boolean). +# TODO(hubcio): inert in server-ng, which never verifies checksums on load +# whatever this says - `true` buys no corruption guard here. Boot warns when +# set. +validate_checksum = false + +# The count threshold of buffered messages before triggering a save to disk. +# Together with `size_of_messages_required_to_save` it defines the threshold. +# This is a soft limit - actual count may be higher depending on last batch size. +# Minimum value is 1. +messages_required_to_save = 1024 + +# The size threshold of buffered messages before triggering a save to disk. +# Together with `messages_required_to_save` it defines the threshold. +# This is a soft limit - actual size may be higher depending on last batch size. +size_of_messages_required_to_save = "1 MiB" + +# Segment configuration +[system.segment] +# Defines the soft limit for the size of a storage segment. +# When a segment reaches this size, a new segment is created for subsequent data. +# Example: if `size` is set "1GiB", the actual segment size may be 1GiB + the size of remaining messages in received batch. +# Maximum size is 1 GiB. Size has to be a multiple of 512 B. +size = "1 GiB" + +# Configures whether expired segments are archived (boolean) or just deleted without archiving. +# Unsupported in server-ng: setting this to `true` aborts boot. +archive_expired = false + +# Controls whether to cache indexes (time and positional) for segment access. +# Possible values: +# - "true" or "all": keeps indexes in memory, speeding up data retrieval at the cost of memory +# - "open_segment": keeps indexes in memory only for the currently open segment +# - "false" or "none": reads indexes from disk, which can conserve memory at the cost of access speed +# TODO(hubcio): inert in server-ng, which picks its own index residency. Boot +# warns when set. +cache_indexes = "open_segment" + +# Message deduplication configuration +[system.message_deduplication] +# Controls whether message deduplication is enabled (boolean). +# `true` activates deduplication, ignoring messages with duplicate IDs. +# `false` treats each message as unique, even if IDs are duplicated. +# Unsupported in server-ng: setting this to `true` aborts boot. +enabled = false +# Maximum number of ID entries in the deduplication cache (u64). +max_entries = 10000 +# Maximum age of ID entries in the deduplication cache in human-readable format. +expiry = "1 m" + +# Recovery configuration in case of lost data +[system.recovery] +# Controls whether streams/topics/partitions should be recreated if the expected data for existing state is missing (boolean). +# Unsupported in server-ng: setting this to `true` aborts boot. +recreate_missing_state = false + +# Memory pool configuration +[system.memory_pool] +# Enables or disables the memory pool (boolean). +# `true` enables the memory pool. +# `false` disables the memory pool. +enabled = true + +# Size of the memory pool (string). +# Example: "512 MiB" or "1 GiB". +# This defines the maximum, total memory allocated for the memory pool. +# Note: This number has to be multiplication of 4096 (default linux page size). +# Minimum size is 512 MiB due to internal implementation details. +size = "4 GiB" + +# Maximum number of buffers in each bucket (u32). +# There are 32 buckets in the memory pool. Each bucket can hold up to this number of buffers +# and holds different buffer sizes, from 256 B to 512 MiB. +# Note: This number has to be a power of 2. Minimum value is 128 due to internal implementation details. +bucket_capacity = 8192 + +# Cluster configuration +[cluster] +# Enables or disables cluster mode (boolean). +# When enabled, this node will participate in the cluster and coordinate with other nodes. +enabled = false + +# Unique cluster name (string). +# All nodes in the same cluster must share the same name. +# This prevents accidental cross-cluster communication. +name = "iggy-cluster" + +# Backup-side liveness window for a consensus plane's primary (duration). +# A replica that sees no primary traffic for this long starts a view change. +# Raise it on oversubscribed hosts where scheduling stalls fake primary +# death. Must be at least "2s" and at least 4x commit_broadcast_interval: the +# primary signals liveness through its commit broadcast, and the window must +# span several broadcasts so one delayed broadcast never trips an election. +heartbeat_timeout = "5s" + +# How often the primary broadcasts its commit point to every backup (duration). +# This is the cluster's liveness signal: each broadcast resets every backup's +# heartbeat_timeout window and carries the latest commit point forward. Must be +# nonzero and, with heartbeat_timeout, satisfy heartbeat_timeout >= 4x this +# value. Drives the consensus CommitMessage timer. +commit_broadcast_interval = "500ms" + +# How often the primary retransmits prepares that backups have not yet acked +# (duration). Lower values recover faster from a dropped prepare at the cost of +# more replica traffic; must be nonzero. Drives the consensus Prepare timer. +prepare_retransmit_interval = "250ms" + +# How often a replica retransmits its StartViewChange / DoViewChange while a +# view change is in progress (duration). Lower values converge a healthy +# election faster at the cost of more replica traffic; must be nonzero. Drives +# both consensus view-change retransmit timers. +view_change_retransmit_interval = "500ms" + +# Backstop for a stalled view change (duration): one that does not conclude +# within this window escalates to a fresh cluster-wide election. Must be nonzero +# and at least 4x view_change_retransmit_interval, so a few dropped view-change +# messages retransmit rather than prematurely escalate. +view_change_status_timeout = "5s" + +# How often a recovering or view-change backup re-requests the current view's +# StartView from its primary (duration); must be nonzero. Drives the consensus +# RequestStartView timer. +request_start_view_retransmit_interval = "1s" + +# How many consecutive unanswered RequestStartView probes a recovering replica +# tolerates before falling back to an election (integer). A full-cluster restart +# leaves nobody settled to answer, so the replica elects on its recovered log. +# Must be between 1 and 100. +view_probe_attempts_max = 5 + +# How long a stalled journal-repair stream waits before re-requesting its +# remaining window from the serving peer (duration). Repair frames are +# fire-and-forget over the lossy bus, so a session with no retry wedges forever +# on a single dropped frame. Paces both the metadata and partition repair loops; +# must be nonzero. +repair_retry_interval = "1s" + +# Prepares a peer serves per repair round before the requester walks to the next +# chunk (integer). Each frame rides the per-peer message-bus queue, so this must +# stay strictly below message_bus.peer_queue_capacity or a full round overruns +# the queue and drops frames. Must be > 0 and <= 1024. +repair_chunk_max = 128 + +# Replica-to-replica authentication (PSK + BLAKE3 keyed-MAC handshake). +[cluster.auth] +# When true, every replica peer must complete the authenticated handshake or be +# rejected, and shared_secret becomes mandatory. Off by default = legacy +# unauthenticated replica traffic. Enabling it is a coordinated-restart change. +# With http enabled and no http.jwt secrets configured, the PSK also becomes +# the JWT key source, making bearers valid cluster-wide and activating +# follower-to-primary HTTP forwarding. +enabled = false + +# Cluster-wide pre-shared key, >= 32 bytes of CSPRNG output, byte-identical on +# every node. Prefer the IGGY_CLUSTER_AUTH_SHARED_SECRET env var (masked in +# logs, never persisted) over storing it on disk. Ignored when enabled = false. +shared_secret = "" + +# Retiring pre-shared key, accepted for verification only during a rolling key +# rotation (this node keeps signing with shared_secret). Rotate in three rolls: +# 1) shared_secret = old + previous_shared_secret = new on every node, +# 2) shared_secret = new + previous_shared_secret = old on every node, +# 3) shared_secret = new alone. Leave empty outside a rotation. Same length +# floor and env-var preference as shared_secret +# (IGGY_CLUSTER_AUTH_PREVIOUS_SHARED_SECRET). +previous_shared_secret = "" + +# Replica-to-replica TLS for the consensus (tcp_replica) port. +[cluster.tls] +# When true every replica connection is wrapped in TLS 1.3 (ALPN +# "iggy-replica") before the replica handshake runs. Requires +# cluster.auth.enabled: TLS carries no client certificates, so it +# authenticates the acceptor only; the PSK handshake authenticates the +# peer, TLS supplies confidentiality. Off by default = plaintext replica +# traffic. Enabling it is a coordinated-restart change: a TLS dialer +# cannot talk to a plaintext acceptor or vice versa. +enabled = false + +# When true the node auto-generates a self-signed certificate at boot and +# the dialer accepts ANY peer certificate. When false (default), +# cert_file / key_file / ca_file are all required. +self_signed = false + +# PEM certificate chain presented by this node's acceptor side. +cert_file = "" + +# PEM private key matching cert_file. +key_file = "" + +# PEM trust anchor(s) the dialer verifies peer certificates against. +# Unused when self_signed = true. +ca_file = "" + +# Full roster of cluster members. Byte-identical on every node. The running +# node's identity is resolved at launch from the '--replica-id ' CLI +# flag, which selects the entry in this list that describes the current +# node. All other entries are remote peers. +# +# 'ip' is the node's roster address. Replica-to-replica traffic and +# follower-to-primary HTTP forwarding use it. It is not the bind interface for +# tcp/quic/http/websocket, which comes from each transport's own 'address' +# setting above; the roster supplies those transports their port only. A +# cluster spread across hosts therefore needs each transport's 'address' set to +# '0.0.0.0' or the routable NIC; the defaults below listen on loopback only, +# and a bind that cannot serve the advertised 'ip' is warned about at startup. +# +# Each node may also set 'advertised_address': the client-facing address +# handed out in cluster metadata and leader redirects. Set it when 'ip' is +# a private replica-network address unreachable by clients (Docker, +# Kubernetes, NAT). Accepts a literal IPv4/IPv6 address or a DNS hostname +# (RFC 1123: ASCII letters, digits, '-' and '.'; no port, no trailing dot). +# When unset, clients receive 'ip'. +# +# When different client networks need different addresses (a public +# 'advertised_address' would route in-VPC clients out through the public +# side), add per-network 'advertised_addresses' selectors: clients whose +# peer IP falls inside 'client_cidr' are handed 'address' instead of the +# catch-all. 'address' takes the same forms as 'advertised_address' +# (literal IP or RFC 1123 hostname, never a port - ports always come from +# 'ports'). At most 16 selectors per node; boot also rejects duplicate +# 'client_cidr' entries on one node (compared truncated, so '10.0.1.0/16' +# duplicates '10.0.0.0/16') and any two nodes advertising one host:port +# to overlapping client sets - reusing a host:port across nodes is legal +# only when no client would resolve both nodes to it. +# +# The longest matching prefix wins; clients matching no selector fall +# back to 'advertised_address', then 'ip'. Matching is per address +# family: '0.0.0.0/0' matches no IPv6 client and '::/0' matches no IPv4 +# client, so covering both families takes one selector per family (or the +# catch-all). IPv4-mapped IPv6 CIDRs ('::ffff:10.0.0.0/104') match like +# their IPv4 form only at prefix length 96 or longer; shorter ones match +# native IPv6 clients only. Matching sees the transport-level peer +# address, so clients behind a proxy or load balancer match the proxy's +# network, not their own. +# +# Every 'address' must be routable from inside its own 'client_cidr': +# leader-aware SDK clients redial whatever address metadata advertises, +# so a selector pointing at a host its own clients cannot reach strands +# them mid-redirect. Prefer literal IPs over hostnames - the SDKs differ +# in how they compare an advertised hostname against the address they +# dialed, and a mismatch costs a reconnect on every fresh connect. +# +# Note for rolling upgrades: older server binaries reject a TOML config +# containing 'advertised_addresses' but silently ignore the equivalent +# 'IGGY_CLUSTER_NODES_*_ADVERTISED_ADDRESSES_*' env vars; either way, +# upgrade every binary first, then add selectors. Mid-upgrade, an env-var +# roster would serve selector addresses from upgraded nodes and the +# catch-all from the rest. +# +# [[cluster.nodes]] +# name = "iggy-node-1" +# ip = "10.0.1.5" # replica plane + last-resort fallback +# advertised_address = "203.0.113.10" # catch-all for unmatched clients +# replica_id = 0 +# ports = { tcp = 8090, http = 3000, tcp_replica = 9090 } +# +# [[cluster.nodes.advertised_addresses]] +# client_cidr = "10.0.0.0/16" # in-VPC clients stay private +# address = "10.0.1.5" +# +# In cluster mode, 'ports' is the single source of listener ports: every +# enabled transport needs an explicit per-node port, otherwise the server +# refuses to start. +[[cluster.nodes]] +name = "iggy-node-1" +ip = "127.0.0.1" +replica_id = 0 +ports = { tcp = 8090, quic = 8080, http = 3000, websocket = 8092, tcp_replica = 9090 } + +[[cluster.nodes]] +name = "iggy-node-2" +ip = "127.0.0.1" +replica_id = 1 +ports = { tcp = 8091, quic = 8081, http = 3001, websocket = 8093, tcp_replica = 9091 } + +# Example additional node (commented out). tcp skips 8092-8094: those are the +# websocket ports of the three nodes, which collide once nodes share a host. +# [[cluster.nodes]] +# name = "iggy-node-3" +# ip = "192.168.1.100" +# advertised_address = "iggy-node-3.example.com" +# replica_id = 2 +# ports = { tcp = 8095, quic = 8082, http = 3002, websocket = 8094, tcp_replica = 9092 } + +# Sharding configuration +[system.sharding] +# CPU allocation - controls the number of shards and their CPU affinity. +# Possible values: +# - "all": Use all available CPU cores (default) +# - numeric value (e.g. 4): Use 4 shards (4 threads pinned to cores 0, 1, 2, 3) +# - range (e.g. "5..8"): Use 3 shards with affinity to cores 5, 6, 7 +# - numa settings: +# + "numa:auto": Use all available numa node, cores +# + "numa:nodes=0,1;cores=4;no_ht=true": Use NUMA node 0 and 1, each nodes use 4 cores, and no hyperthreads +cpu_allocation = "numa:auto" + +# Whether shard threads are pinned to dedicated CPU cores (default: true). +# Pinned cores are drawn from the process's allowed CPU set (affinity/cpuset +# mask), so the server cooperates with systemd `AllowedCPUs=` and container +# cpusets. Set to false when the server shares cores with other workloads +# (e.g. a multi-tenant host slicing CPU via cgroup quotas): unpinned shards +# let the kernel scheduler place threads freely instead of piling every +# process onto the same low-numbered cores. +pin_cores = true + +# Per-shard inter-shard inbox capacity. Bounded by design: consensus-frame +# drops recover via VSR retransmit, but cross-shard client-reply drops are +# terminal. Size for the worst-case sum of both: the consensus working set +# (~ the prepare queue depth of the planes the shard hosts - [metadata] on +# shard 0, [partition] elsewhere - times replica_count times directions) plus +# peak client-reply fan-out per shard. Both depths are tunable, so raising +# either raises the capacity needed here. +inbox_capacity = 1024 + +# Wall-clock budget for a single shard's bus drain on shutdown. Drives +# the per-shard watchdog and the parallel-join survivor path; sized +# larger than typical TCP RTT times in-flight write-batch so writers +# receive their full last `write_vectored_all` budget before the +# connection registry force-tears the bus. Slow-fsync hosts may need +# to extend this past the default. +shutdown_drain_timeout = "10 s" + +# Poll cadence for the cross-thread shutdown flag and for the +# metadata-handoff loops. Trades off Ctrl-C latency against idle wakeup +# cost; the default keeps shutdown observably prompt without measurable +# scheduler overhead. Must be less than or equal to shutdown_drain_timeout. +shutdown_poll_interval = "50 ms" + +# Hard wall-clock deadline for joining shard threads at process exit. A +# shard whose pump or listener wedges past this budget is abandoned with +# an error log instead of blocking exit forever. Must be at least +# shutdown_drain_timeout, or shards would be abandoned mid-drain. +shutdown_join_timeout = "30 s" + +# Safety-tick cadence for the partition reconciliation loop. The reconciler +# also wakes on every metadata commit from shard 0, so this only covers +# dropped wake-ups and the initial post-bootstrap convergence window. +reconcile_periodic_interval = "1 s" + +# WebSocket listener configuration. The frame-tuning knobs below are the +# live source for server-ng's WS / WSS plane; they are folded into a +# compio-ws WebSocketConfig once at bus construction. Each size knob is +# optional: commenting it out keeps the compio-ws (tungstenite) default +# noted next to it. A malformed size string fails config load. +[websocket] +enabled = true +address = "127.0.0.1:8092" + +# Target minimum size of the frame read buffer. compio-ws default: "128 KiB". +# read_buffer_size = "128 KiB" + +# Target buffer size for batched writes before flush. compio-ws +# default: "128 KiB". +# write_buffer_size = "128 KiB" + +# Hard ceiling on the write buffer; writes past it error instead of +# buffering, so it must exceed write_buffer_size by at least one message. +# compio-ws default: unlimited. +# max_write_buffer_size = "128 MiB" + +# Hard upper bound on a single inbound WebSocket message +# (post-fragment-reassembly). Must not exceed message_bus.max_message_size. +# compio-ws default: "64 MiB". +# max_message_size = "64 MiB" + +# Hard upper bound on a single inbound WebSocket frame +# (pre-fragment-reassembly). Must not exceed max_message_size. +# compio-ws default: "16 MiB". +# max_frame_size = "16 MiB" + +# Whether to accept unmasked frames from clients in violation of +# RFC 6455 client-to-server framing rules. Strict (false) by default. +accept_unmasked_frames = false + +[websocket.tls] +enabled = false +self_signed = true +cert_file = "core/certs/iggy_cert.pem" +key_file = "core/certs/iggy_key.pem" + +# Metadata consensus plane tunables (shard 0's VSR replica: users, +# streams, topics, sessions). Size these together: a deeper prepare queue +# admits more concurrent in-flight metadata ops (e.g. login storms), and +# the journal must hold enough slots that a forced checkpoint (triggered +# when remaining slots fall to the checkpoint margin, which itself is +# max(64, prepare_queue_depth)) stays rare. Validation enforces +# journal_slots >= 4 * max(64, prepare_queue_depth). +[metadata] +# Depth of the metadata prepare queue: how many uncommitted metadata ops +# may be in flight at once. Submits beyond it are rejected with the +# transient "metadata prepare queue is full" and retried by the SDK. +prepare_queue_depth = 32 + +# Size of the metadata WAL's in-memory index, in slots (one committed but +# not-yet-snapshotted op per slot). Larger values buy more headroom +# between forced checkpoints at the cost of memory and bigger WAL +# rewrites per checkpoint. +journal_slots = 1024 + +# Slot count of the VSR client table: how many distinct clients (TCP/QUIC/WS +# virtual clients and HTTP sessions together) hold live session state at once. +# When full, the client whose last commit is oldest is evicted and its next +# request re-registers. The HTTP session cap tracks this at half, so raising +# it lifts both. Must be between 2 and 65536. +clients_table_max = 8192 + +# Per-partition consensus plane tunables. Unlike [metadata] (one shard-0 +# plane), a pipeline exists per partition, so raising this multiplies pinned +# request-buffer memory by the partition count. Keep it modest. +[partition] +# Depth of a partition's prepare queue: how many uncommitted produce / +# consumer-offset ops may be in flight at once for that partition. Submits past +# it spill into a request queue of twice this depth; once both are full the +# server drops the request without a reply and the client retries on its own +# request timeout. Must be > 0 and <= 256. +prepare_queue_depth = 32 + +# Entries the evicted ring retains per multi-replica partition for journal +# repair after a peer rejoins. Larger widens the window a restarting peer can be +# served from the ring before falling back to bulk sync, at the cost of pinned +# memory per partition. Must be > 0 and <= 65536. Single-replica partitions +# retain nothing regardless. +evicted_ring_capacity = 4096 + +# Byte ceiling for the evicted ring per partition; whichever ring cap (this or +# evicted_ring_capacity) trips first evicts. Bounds the ring memory a burst of +# large batches can pin. Must be > 0 and <= "256 MiB". +evicted_ring_bytes_max = "16 MiB" + +# Byte budget for segment payloads a SERVING shard keeps resident to answer +# state-transfer chunk requests. PER SHARD, and shard count defaults to core +# count, so the process-wide high-water is this times the core count on top of +# page cache -- keep that product in mind before raising it. The default is a +# FIXED 2176 MiB: two sealed segments at the SHIPPED system.segment.size of +# 1 GiB, each of which can close one whole message_bus.max_message_size past +# its target, which is why it is not 2 GiB. It does not track your segment +# size. How many groups this shard serves at once IS derived from yours: +# floor(this / max(partition.transfer_artifact_bytes_max, +# system.segment.size + 64 MiB)), minimum one. So raising either that knob or +# system.segment.size without raising this lowers concurrency and can take it +# to one, serialising rejoins, and nothing at boot warns about it. +# Below one segment a single rejoining node thrashes the cache by itself and +# every miss re-reads and re-hashes a whole segment to serve one 256 KiB chunk. +# Running under the budget costs re-reads, not failures. +# Must be > 0 and <= "64 GiB". +transfer_served_cache_bytes_max = "2176 MiB" + +# Alloc ceiling for ONE received state-transfer artifact, per shard. The +# receiver holds it resident through verify, walk and staging write, and up to +# four transfers run at once. MUST cover system.segment.size plus +# message_bus.max_message_size (a segment may close one whole batch past its +# cap): under that, a legal segment is refused, the whole manifest with it, and +# the partition livelocks re-requesting it from every peer. Boot validates the +# floor. Raising this above the floor for headroom also DIVIDES the serving +# concurrency derived from transfer_served_cache_bytes_max above, so raise that +# in step. Must be > 0 and <= "64 GiB". +transfer_artifact_bytes_max = "1088 MiB" + +# Message bus configuration. +# Tunables for the inter-shard / inter-replica internal bus that ships +# consensus traffic between replicas and SDK-client traffic between +# shards. These knobs are consensus-liveness-critical (max_batch gates +# throughput under backpressure). Defaults match +# core::message_bus::config::MessageBusConfig::default(). + +[message_bus] +# Maximum number of BusMessage entries coalesced into a single writev(2) +# call. Hard upper bound: IOV_MAX/2 = 512 on Linux. +max_batch = 256 + +# Wire-level cap on a single framed message. +max_message_size = "64 MiB" + +# Bound on the per-peer mpsc queue. The writer task drains; the +# send_to_* path enqueues. +peer_queue_capacity = 256 + +# Interval between outbound reconnect attempts to peers with peer_id > self_id. +reconnect_period = "5 s" + +# Timeout for per-peer close drain (flush writer, tear down reader) +# before force-cancellation. +close_peer_timeout = "2 s" + +# Wall-clock bound on a single stream.shutdown() / ws.close() in the +# safe-shutdown sequence of the TLS-family transports. +close_grace = "2 s" + +# Wall-clock bound on a single connection's handshake phase. Threaded +# into compio::time::timeout(handshake_grace, ...) at each accept site +# (TCP-TLS rustls accept, WS HTTP-Upgrade, WSS combined TLS+WS, QUIC +# connecting.await + accept_bi.await) so a slowloris peer cannot pin +# per-conn channels + registry slot + spawned task indefinitely. +handshake_grace = "10 s" + +[extra.namespace] +max_streams = 4096 +max_topics = 4096 +max_partitions = 1_000_000 diff --git a/core/server-ng/server.http b/core/server-ng/server.http new file mode 100644 index 0000000000..b0be45ca82 --- /dev/null +++ b/core/server-ng/server.http @@ -0,0 +1,369 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +@url = http://localhost:3000 +@stream_id = 0 +@topic_id = 0 +@partition_id = 0 +@consumer_group_id = 1 +@consumer_id = 1 +@client_id = 1 +@partition_id_payload_base64 = AAAAAA== +@message_1_payload_base64 = aGVsbG8= +@message_2_payload_base64 = d29ybGQ= +@header_key_1_base64 = a2V5XzE= +@header_value_1_base64 = dmFsdWUgMQ== +@header_key_2_base64 = Kg== +@header_value_2_base64 = AAAA +@root_username = iggy +@root_password = iggy +@user1_username = user1 +@user1_password = secret +@access_token = secret +@root_id = 0 +@user1_id = 1 +@pat_name = dev_token +@pat_raw_token = secret + +### +GET {{url}}/ping + +### +POST {{url}}/users/login +Content-Type: application/json + +{ + "username": "{{root_username}}", + "password": "{{root_password}}" +} + +### +POST {{url}}/personal-access-tokens/login +Content-Type: application/json + +{ + "token": "{{pat_raw_token}}" +} + +### +GET {{url}}/stats +Authorization: Bearer {{access_token}} + +### +POST {{url}}/snapshot +Authorization: Bearer {{access_token}} +Content-Type: application/json + +{ + "compression": "Deflated", + "snapshot_types": ["All"] +} + +### +GET {{url}}/cluster/metadata +Authorization: Bearer {{access_token}} + +### +GET {{url}}/clients +Authorization: Bearer {{access_token}} + +### +GET {{url}}/clients/{{client_id}} +Authorization: Bearer {{access_token}} + +### +DELETE {{url}}/users/logout +Authorization: Bearer {{access_token}} + +### +POST {{url}}/users +Authorization: Bearer {{access_token}} +Content-Type: application/json + +{ + "username": "{{user1_username}}", + "password": "{{user1_password}}", + "status": "active", + "permissions": null +} + +### +GET {{url}}/users +Authorization: Bearer {{access_token}} + +### +GET {{url}}/users/{{user1_id}} +Authorization: Bearer {{access_token}} + +### +PUT {{url}}/users/{{user1_id}} +Authorization: Bearer {{access_token}} +Content-Type: application/json + +{ + "username": "{{user1_username}}", + "status": "active", + "permissions": null +} + +### +PUT {{url}}/users/{{user1_id}}/password +Authorization: Bearer {{access_token}} +Content-Type: application/json + +{ + "current_password": "{{user1_password}}", + "new_password": "secret1" +} + +### +PUT {{url}}/users/{{user1_id}}/permissions +Authorization: Bearer {{access_token}} +Content-Type: application/json + +{ + "permissions": { + "global": { + "manage_servers": false, + "read_servers": true, + "manage_users": true, + "read_users": true, + "manage_streams": false, + "read_streams": true, + "manage_topics": false, + "read_topics": true, + "poll_messages": true, + "send_messages": true + }, + "streams": { + "0": { + "manage_stream": false, + "read_stream": true, + "manage_topics": false, + "read_topics": true, + "poll_messages": true, + "send_messages": true, + "topics": { + "0": { + "manage_topic": false, + "read_topic": true, + "poll_messages": true, + "send_messages": true + } + } + } + } + } +} + + +### +DELETE {{url}}/users/{{user1_id}} +Authorization: Bearer {{access_token}} + +### +GET {{url}}/personal-access-tokens +Authorization: Bearer {{access_token}} + +### +POST {{url}}/personal-access-tokens +Authorization: Bearer {{access_token}} +Content-Type: application/json + +{ + "name": "{{pat_name}}", + "expiry": 1000 +} + +### +DELETE {{url}}/personal-access-tokens/{{pat_name}} +Authorization: Bearer {{access_token}} + +### +GET {{url}}/streams +Authorization: Bearer {{access_token}} + +### +GET {{url}}/streams/{{stream_id}} +Authorization: Bearer {{access_token}} + +### +POST {{url}}/streams +Authorization: Bearer {{access_token}} +Content-Type: application/json + +{ + "name": "stream1" +} + +### +PUT {{url}}/streams/{{stream_id}} +Authorization: Bearer {{access_token}} +Content-Type: application/json + +{ + "name": "stream1" +} + +### +DELETE {{url}}/streams/{{stream_id}} +Authorization: Bearer {{access_token}} + +### +DELETE {{url}}/streams/{{stream_id}}/purge +Authorization: Bearer {{access_token}} + +### +GET {{url}}/streams/{{stream_id}}/topics +Authorization: Bearer {{access_token}} + +### +GET {{url}}/streams/{{stream_id}}/topics/{{topic_id}} +Authorization: Bearer {{access_token}} + +### +POST {{url}}/streams/{{stream_id}}/topics +Authorization: Bearer {{access_token}} +Content-Type: application/json + +{ + "name": "topic1", + "partitions_count": 1, + "compression_algorithm": "none", + "max_topic_size": 0, + "message_expiry": 0 +} + +### +PUT {{url}}/streams/{{stream_id}}/topics/{{topic_id}} +Authorization: Bearer {{access_token}} +Content-Type: application/json + +{ + "name": "topic1", + "compression_algorithm": "none", + "max_topic_size": 0, + "message_expiry": 0 +} + +### +DELETE {{url}}/streams/{{stream_id}}/topics/{{topic_id}} +Authorization: Bearer {{access_token}} + +### +DELETE {{url}}/streams/{{stream_id}}/topics/{{topic_id}}/purge +Authorization: Bearer {{access_token}} + +### +POST {{url}}/streams/{{stream_id}}/topics/{{topic_id}}/partitions +Authorization: Bearer {{access_token}} +Content-Type: application/json + +{ + "partitions_count": 3 +} + +### +DELETE {{url}}/streams/{{stream_id}}/topics/{{topic_id}}/partitions?partitions_count=1 +Authorization: Bearer {{access_token}} + +### +### Delete segments +DELETE {{url}}/streams/{{stream_id}}/topics/{{topic_id}}/partitions/{{partition_id}}?segments_count=3 +Authorization: Bearer {{access_token}} + +### +POST {{url}}/streams/{{stream_id}}/topics/{{topic_id}}/messages +Authorization: Bearer {{access_token}} +Content-Type: application/json + +{ + "partitioning": { + "kind": "partition_id", + "value": "{{partition_id_payload_base64}}" + }, + "messages": [{ + "id": 0, + "payload": "{{message_1_payload_base64}}" + }, { + "id": 0, + "payload": "{{message_2_payload_base64}}", + "user_headers": [{ + "key": { + "kind": "string", + "value": "{{header_key_1_base64}}" + }, + "value": { + "kind": "string", + "value": "{{header_value_1_base64}}" + } + }, { + "key": { + "kind": "uint32", + "value": "{{header_key_2_base64}}" + }, + "value": { + "kind": "int32", + "value": "{{header_value_2_base64}}" + } + }] + }] +} + +### +GET {{url}}/streams/{{stream_id}}/topics/{{topic_id}}/messages?consumer_id={{consumer_id}}&partition_id={{partition_id}}&kind=offset&value=0&count=10&auto_commit=false +Authorization: Bearer {{access_token}} + +### +PUT {{url}}/streams/{{stream_id}}/topics/{{topic_id}}/consumer-offsets +Authorization: Bearer {{access_token}} +Content-Type: application/json + +{ + "consumer_id": {{consumer_id}}, + "partition_id": {{partition_id}}, + "offset": 1 +} + +### +GET {{url}}/streams/{{stream_id}}/topics/{{topic_id}}/consumer-offsets?consumer_id={{consumer_id}}&partition_id={{partition_id}} +Authorization: Bearer {{access_token}} + +### +DELETE {{url}}/streams/{{stream_id}}/topics/{{topic_id}}/consumer-offsets/{{consumer_id}}?partition_id={{partition_id}} +Authorization: Bearer {{access_token}} + +### +GET {{url}}/streams/{{stream_id}}/topics/{{topic_id}}/consumer-groups +Authorization: Bearer {{access_token}} + +### +GET {{url}}/streams/{{stream_id}}/topics/{{topic_id}}/consumer-groups/{{consumer_group_id}} +Authorization: Bearer {{access_token}} + +### +POST {{url}}/streams/{{stream_id}}/topics/{{topic_id}}/consumer-groups +Authorization: Bearer {{access_token}} +Content-Type: application/json + +{ + "name": "consumer_group_1" +} + +### +DELETE {{url}}/streams/{{stream_id}}/topics/{{topic_id}}/consumer-groups/{{consumer_group_id}} +Authorization: Bearer {{access_token}} diff --git a/foreign/csharp/Iggy_SDK/Vsr/EvictionReason.cs b/core/server-ng/src/args.rs similarity index 53% rename from foreign/csharp/Iggy_SDK/Vsr/EvictionReason.cs rename to core/server-ng/src/args.rs index e51deeea0f..8bff7717af 100644 --- a/foreign/csharp/Iggy_SDK/Vsr/EvictionReason.cs +++ b/core/server-ng/src/args.rs @@ -15,28 +15,20 @@ // specific language governing permissions and limitations // under the License. -namespace Apache.Iggy.Vsr; +use clap::Parser; -/// -/// Reason carried at byte 255 of an eviction frame. Session-terminal, never transient. -/// Discriminants are wire-pinned; a value outside this set decodes as . -/// -internal enum EvictionReason : byte -{ - Reserved = 0, - NoSession = 1, - ClientReleaseTooLow = 2, - ClientReleaseTooHigh = 3, - InvalidRequestOperation = 4, - InvalidRequestBody = 5, - InvalidRequestBodySize = 6, - SessionTooLow = 7, - SessionReleaseMismatch = 8, - InvalidCredentials = 9, - InvalidToken = 10, - UserInactive = 11, - SessionError = 12, - StaleClient = 13, - IncompatibleProtocol = 14, - MalformedLogin = 15 +#[derive(Parser, Debug)] +#[command( + author = "Apache Iggy (Incubating)", + version, + about = "Apache Iggy server-ng", + long_about = "Apache Iggy server-ng\n\nUse --replica-id together with a shared cluster config to run one binary per cluster node." +)] +pub struct Args { + /// Identifies this node within `cluster.nodes` by its replica ID. + /// + /// Required when `cluster.enabled = true`. The value must match exactly + /// one `cluster.nodes[*].replica_id` entry in the loaded configuration. + #[arg(long, verbatim_doc_comment)] + pub replica_id: Option, } diff --git a/core/server/src/auth.rs b/core/server-ng/src/auth.rs similarity index 97% rename from core/server/src/auth.rs rename to core/server-ng/src/auth.rs index a491eb9288..74a38444cc 100644 --- a/core/server/src/auth.rs +++ b/core/server-ng/src/auth.rs @@ -20,7 +20,7 @@ //! Verifies password + PAT credentials locally, then runs the consensus //! `Register` proposal on the metadata owner; terminal failures are //! surfaced as typed `Eviction` frames, transient ones as -//! `TransientNotAccepted` replay hints. +//! `TransientNotCommitted` replay hints. use crate::bootstrap::{ShellBus, ShellShard}; use crate::dispatch::{send_login_eviction, submit_register_on_owner}; @@ -29,7 +29,7 @@ use crate::responses::{build_login_register_reply, current_metadata_commit}; use crate::session_manager::{ClientSdkInfo, SessionManager}; use consensus::{MetadataHandle, build_result_rejection_reply}; use iggy_binary_protocol::PrepareHeader; -use iggy_binary_protocol::{ClientVersionInfo, EvictionReason, RoutedRequestHeader}; +use iggy_binary_protocol::{ClientVersionInfo, EvictionReason, RequestHeader}; use iggy_common::defaults::{ MAX_PASSWORD_LENGTH, MAX_USERNAME_LENGTH, MIN_PASSWORD_LENGTH, MIN_USERNAME_LENGTH, }; @@ -183,7 +183,7 @@ pub(crate) async fn complete_login_register( sessions: &Rc>, transport_client_id: u128, vsr_client_id: u128, - request_header: &RoutedRequestHeader, + request_header: &RequestHeader, user_id: u32, client_version: &ClientVersionInfo, ) -> Result<(), LoginRegisterError> @@ -291,7 +291,7 @@ where pub(crate) async fn surface_login_failure( shard: &Rc>, transport_client_id: u128, - request_header: &RoutedRequestHeader, + request_header: &RequestHeader, error: &LoginRegisterError, ) where B: ShellBus, @@ -310,7 +310,7 @@ pub(crate) async fn surface_login_failure( .await; } else { // Transient consensus failure (not-caught-up / not-primary / pipeline - // full): send the explicit `TransientNotAccepted` frame instead of + // full): send the explicit `TransientNotCommitted` frame instead of // staying silent, so the SDK replays the login immediately rather than // waiting out its read-timeout. Same contract as a transient metadata // request -- nothing committed, so the replayed Register is idempotent. @@ -318,7 +318,7 @@ pub(crate) async fn surface_login_failure( } } -/// Result-framed `TransientNotAccepted` Reply on a transient (non-terminal) +/// Result-framed `TransientNotCommitted` Reply on a transient (non-terminal) /// failed Register. The SDK decodes the nonzero result code and replays the /// same login on the same connection. Only call for transient errors -- see /// [`surface_login_failure`]. @@ -326,7 +326,7 @@ pub(crate) async fn surface_login_failure( async fn send_login_transient_reply( shard: &Rc>, transport_client_id: u128, - request_header: &RoutedRequestHeader, + request_header: &RequestHeader, ) where B: ShellBus, MJ: JournalHandle + 'static, diff --git a/core/server-ng/src/bootstrap.rs b/core/server-ng/src/bootstrap.rs new file mode 100644 index 0000000000..ad1671b201 --- /dev/null +++ b/core/server-ng/src/bootstrap.rs @@ -0,0 +1,4385 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::auth::warm_dummy_password_hash; +use crate::cluster_meta::ClusterRoster; +use crate::config_writer::write_current_config; +use crate::dispatch::{ + make_client_request_handler, make_deferred_client_request_handler, + make_deferred_replica_message_handler, make_list_clients_handler, make_metadata_submit_handler, + make_partition_read_handler, +}; +use crate::http; +use crate::partition_helpers::{ + build_partition_fresh, configure_consumer_offsets, ensure_initial_segment, + open_partition_superblock, restore_partition_view, validate_namespace_bounds, +}; +use crate::segment_recovery::{RecoveredSegment, load_persisted_segments}; +use crate::server_error::{ServerNgError, ShardJoinFailure, ShardJoinFailureKind}; +use crate::session_manager::SessionManager; +use configs::ng_sharding::{ + INBOX_CAPACITY_MAX, SHUTDOWN_DRAIN_TIMEOUT_MAX, SHUTDOWN_POLL_INTERVAL_MAX, +}; +use configs::server_ng::{NgSystemConfig, ServerNgConfig}; +use consensus::{ + ClientTable, LocalPipeline, MetadataHandle, PartitionsHandle, PipelineEntry, Sequencer, + VsrConsensus, +}; +// `try_send` / `try_recv` resolve through these traits on `MAsyncTx` / +// `MAsyncRx`; the metadata-handoff loops below depend on the +// non-blocking variants for cancel-safe shutdown polling. +use consensus::VsrState; +use crossfire::{AsyncRxTrait, AsyncTxTrait}; +use iggy_binary_protocol::{Operation, PrepareHeader}; +use iggy_common::defaults::{ + DEFAULT_ROOT_USERNAME, MAX_PASSWORD_LENGTH, MAX_USERNAME_LENGTH, MIN_PASSWORD_LENGTH, + MIN_USERNAME_LENGTH, +}; +use iggy_common::{Aes256GcmEncryptor, EncryptorKind, IggyByteSize, PartitionStats, variadic}; +use journal::prepare_journal::PrepareJournal; +use journal::superblock::{PingPongSuperblock, SuperblockStore}; +use journal::{Journal, JournalHandle}; +use message_bus::client_listener::{self, RequestHandler}; +use message_bus::installer; +use message_bus::installer::conn_info::{ClientConnMeta, ClientTransportKind}; +use message_bus::replica::auth::{self, ReplicaAuth}; +use message_bus::replica::handshake::{ReplicaHandshakeCtx, ReplicaTlsCtx}; +use message_bus::replica::io as replica_io; +use message_bus::replica::listener::{self as replica_listener, MessageHandler}; +use message_bus::transports::quic::server_config_with_cert; +use message_bus::transports::tls::{ + AcceptAnyServerCert, REPLICA_ALPN, TlsServerCredentials, install_default_crypto_provider, + load_ca_pem, load_pem, self_signed_for_loopback, +}; +use message_bus::{ + AcceptedClientFn, AcceptedQuicClientFn, AcceptedReplicaFn, AcceptedTlsClientFn, + AcceptedWsClientFn, AcceptedWssClientFn, ConnectionInstaller, DialedReplicaFn, IggyMessageBus, + MAX_INFLIGHT_REPLICA_HANDSHAKES, MessageBus, ReplicaOwnerTable, connector, +}; +use metadata::IggyMetadata; +use metadata::MuxStateMachine; +use metadata::ReplicaIdentity; +use metadata::impls::metadata::{IggySnapshot, StreamsFrontend}; +use metadata::impls::recovery::recover; +use metadata::stm::mux::WithFactory; +use metadata::stm::snapshot::Snapshot; +use metadata::stm::stream::{Partition, Streams}; +use metadata::stm::user::Users; +use partitions::{ + IggyIndexWriter, IggyPartition, IggyPartitions, MessagesWriter, PartitionsConfig, +}; +use rustls::pki_types::ServerName; +use server_common::Message; +use server_common::bootstrap::create_directories; +use server_common::crypto; +use server_common::executor::create_shard_executor; +use server_common::log::{Logging, LoggingSettings, TelemetrySettings}; +use server_common::sharding::{IggyNamespace, PartitionLocation, ShardId}; +use shard::builder::IggyShardBuilder; +use shard::metrics::{ShardMetrics, frame_drop_reason, frame_drop_variant}; +use shard::shards_table::{PapayaShardsTable, ShardsTable, calculate_shard_assignment}; +use shard::{ + CoordinatorConfig, IggyShard, LifecycleFrame, ListClientsHandler, MetadataSubmitHandler, + PartitionConsensusConfig, PartitionReadHandler, Receiver as ShardReceiver, ShardFrame, + ShardIdentity, TaggedSender, channel, shard_mesh_channels, +}; +use shard_allocator::{ShardAllocator, ShardInfo}; +use std::cell::RefCell; +use std::collections::HashMap; +use std::env; +use std::net::{IpAddr, SocketAddr}; +use std::path::{Path, PathBuf}; +use std::rc::{Rc, Weak}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::thread; +use std::time::{Duration, Instant}; +use tracing::{error, info, warn}; + +const SHARD_REPLICA_ID: u8 = 0; + +pub const IGGY_ROOT_USERNAME_ENV: &str = "IGGY_ROOT_USERNAME"; +pub const IGGY_ROOT_PASSWORD_ENV: &str = "IGGY_ROOT_PASSWORD"; + +type ServerNgMuxStateMachine = MuxStateMachine; + +/// Cross-thread bundle carrying one `ReadHandleFactory` per metadata +/// state. Shard 0 mints one after `recover()` and broadcasts a clone to +/// every peer shard; each peer rebuilds a reader-mode +/// [`ServerNgMuxStateMachine`] on its own runtime, skipping the WAL. +type ServerNgMetadataBundle = ::Bundle; + +pub(crate) type ServerNgMetadata = IggyMetadata< + VsrConsensus>, + PrepareJournal, + IggySnapshot, + ServerNgMuxStateMachine, +>; + +/// The shard type the dispatch layer is generic over. +/// +/// `B`/`MJ`/`S`/`SB` are free; the metadata state machine (`M`) and shards +/// table (`T`) are pinned, being identical in production and the simulator. +/// Production instantiates it as [`ServerNgShard`], defaulting `SB` to the +/// on-disk [`PingPongSuperblock`]; the simulator supplies its own +/// `B`/`MJ`/`S`/`SB`. +pub type ShellShard = + IggyShard; + +/// Late-bound self-reference the deferred dispatch handlers upgrade per frame. +pub type ShellShardHandle = + Rc>>>>; + +/// Bus bounds the dispatch/pump path needs (matches `run_message_pump`). +/// Blanket-impl'd, so it is only shorthand for the four underlying bounds. +pub trait ShellBus: MessageBus + ConnectionInstaller + Clone + 'static {} +impl ShellBus for B {} + +/// The five dispatch handlers a shard is built with, plus the +/// [`SessionManager`] the request-plane pair shares. +/// +/// Both production ([`build_shard_for_thread`]) and the simulator's shell +/// mode construct these through [`wire_shell_handlers`], so the request +/// plane is wired one way. The simulator's shell-off fast path uses +/// [`ShellHandlers::noop`] instead. +pub struct ShellHandlers { + pub on_replica_message: MessageHandler, + pub on_client_request: RequestHandler, + pub on_metadata_submit: MetadataSubmitHandler, + pub on_list_clients: ListClientsHandler, + pub on_partition_read: PartitionReadHandler, + /// Bound by the client-request handler, read by the get-clients + /// handler; the caller keeps it to reach locally-homed sessions. + pub sessions: Rc>, +} + +impl ShellHandlers { + /// Inert handlers for the shell-off fast path: every callback is a + /// no-op over an empty [`SessionManager`]. Behaviorally identical to + /// hand-written no-op closures, so a caller can keep one destructure + /// site across both toggle states. + #[must_use] + pub fn noop() -> Self { + Self { + on_replica_message: Rc::new(|_, _| {}), + on_client_request: Rc::new(|_, _| {}), + on_metadata_submit: Rc::new(|_| {}), + on_list_clients: Rc::new(|_| {}), + on_partition_read: Rc::new(|_, _, _| {}), + sessions: Rc::new(RefCell::new(SessionManager::new())), + } + } +} + +/// Build the deferred dispatch handlers for `shard_handle` against `bus`. +/// +/// They share one fresh [`SessionManager`]. The caller must set the weak +/// self-reference in `shard_handle` once the shard is built, so the +/// handlers can upgrade it per frame. +pub fn wire_shell_handlers( + bus: &B, + shard_handle: &ShellShardHandle, + system_config: Arc, + max_tokens_per_user: u32, +) -> ShellHandlers +where + B: ShellBus, + MJ: JournalHandle + 'static, + MJ::Target: Journal, Header = PrepareHeader>, + S: 'static, + SB: SuperblockStore + 'static, +{ + let sessions = Rc::new(RefCell::new(SessionManager::new())); + ShellHandlers { + on_replica_message: make_deferred_replica_message_handler(shard_handle), + on_client_request: make_deferred_client_request_handler( + bus, + shard_handle, + &sessions, + system_config, + max_tokens_per_user, + ), + on_metadata_submit: make_metadata_submit_handler(shard_handle), + on_list_clients: make_list_clients_handler(&sessions), + on_partition_read: make_partition_read_handler(shard_handle), + sessions, + } +} + +pub type ServerNgShard = ShellShard, PrepareJournal, IggySnapshot>; + +/// Result of a multi-shard bootstrap. +/// +/// Carries the cross-thread shutdown flag and one OS-thread `JoinHandle` +/// per shard. The caller flips the flag via [`Self::install_ctrlc_handler`] +/// and then drains every shard via [`Self::join_all`], bounded by +/// `join_timeout` (`system.sharding.shutdown_join_timeout`). +pub struct ShardHandles { + shutdown_flag: Arc, + shard_threads: Vec<(u16, thread::JoinHandle>)>, + join_timeout: Duration, +} + +impl ShardHandles { + /// Install a SIGINT/Ctrl-C handler that flips the shutdown flag on + /// the first signal. A second signal is logged but otherwise + /// ignored so an in-flight WAL fsync or replica drain runs to + /// completion. + /// + /// # Errors + /// + /// Returns the underlying `ctrlc::Error` if the handler cannot be + /// installed (typically because another handler already owns the + /// signal). + pub fn install_ctrlc_handler(&self) -> Result<(), ctrlc::Error> { + let flag = Arc::clone(&self.shutdown_flag); + ctrlc::set_handler(move || { + if flag.swap(true, Ordering::Relaxed) { + // Second Ctrl-C: leave the shutdown machinery to drain. + // Refusing to abort here keeps the WAL fsync / replica + // drain from being interrupted mid-frame. + warn!("second Ctrl-C ignored; server is already shutting down"); + } else { + info!("Ctrl-C received; signalling server shutdown"); + } + }) + } + + /// Drain every shard thread. This is the main thread's park for the + /// server's whole lifetime, so shards are awaited WITHOUT any time + /// bound while the server runs; the `shutdown_join_timeout` clock + /// only starts once the cross-thread shutdown flag flips (Ctrl-C or + /// a shard failure). Each shard's outcome is logged (`info` on clean + /// exit, `error` on Err, panic, or wedge). If any shard failed, + /// returns every failure together as + /// [`ServerNgError::ShardJoinFailures`] so the operator sees the + /// full set rather than just the first. + /// + /// A shard whose thread is still running when the post-shutdown + /// deadline passes is abandoned (its `JoinHandle` dropped, the OS + /// thread left to die with the process) and reported as + /// [`ShardJoinFailureKind::Wedged`]: a wedged pump or listener must + /// not block process exit forever. + /// + /// # Errors + /// + /// Returns [`ServerNgError::ShardJoinFailures`] if any shard + /// returned a `Result::Err`, panicked, or wedged past the deadline. + /// The variant carries every per-shard failure in shard-id order so + /// the caller does not need to read the trace log to discover + /// late-failing shards. + pub fn join_all(self) -> Result<(), ServerNgError> { + let mut failures: Vec = Vec::new(); + // Armed on the first poll that observes the shutdown flag, shared + // across all shards: one budget covers the whole drain, not one + // budget per shard. + let mut deadline: Option = None; + // Shards run thread-per-core with compio's blocking fallback pool + // disabled, so an io_uring opcode the kernel lacks aborts every shard + // with the same panic. Surface the actionable diagnostic once. + let mut io_uring_diagnostic_shown = false; + for (shard_id, handle) in self.shard_threads { + let Some(joined) = join_until_shutdown_deadline( + handle, + &self.shutdown_flag, + self.join_timeout, + &mut deadline, + ) else { + error!( + shard_id, + waited = ?self.join_timeout, + "shard thread still running at the shutdown join deadline; abandoning it" + ); + failures.push(ShardJoinFailure { + shard_id, + kind: ShardJoinFailureKind::Wedged { + waited: self.join_timeout, + }, + }); + continue; + }; + match joined { + Ok(Ok(())) => { + info!(shard_id, "shard thread exited cleanly"); + } + Ok(Err(error)) => { + error!(shard_id, error = %error, "shard thread returned error"); + failures.push(ShardJoinFailure { + shard_id, + kind: ShardJoinFailureKind::Error(Box::new(error)), + }); + } + Err(panic_payload) => { + let message = panic_payload_to_string(&*panic_payload); + error!(shard_id, message = %message, "shard thread panicked"); + if !io_uring_diagnostic_shown + && message + .contains(server_common::diagnostics::ASYNCIFY_POOL_DISABLED_PANIC_MSG) + { + server_common::diagnostics::print_incomplete_io_uring_ops_info(); + io_uring_diagnostic_shown = true; + } + failures.push(ShardJoinFailure { + shard_id, + kind: ShardJoinFailureKind::Panic { message }, + }); + } + } + } + if failures.is_empty() { + Ok(()) + } else { + Err(ServerNgError::ShardJoinFailures { failures }) + } + } +} + +/// Poll cadence for the bounded shard joins. Coarse enough to cost +/// nothing during a normal drain, fine enough that exit latency past +/// the last shard's return stays imperceptible. +const JOIN_POLL_INTERVAL: Duration = Duration::from_millis(25); + +/// Join `handle`, waiting indefinitely while the server runs. The +/// `join_timeout` clock starts only when `shutdown_flag` is observed set +/// (arming the caller-shared `deadline` once, so all shards drain under +/// ONE budget); a running server parked here for hours must never be +/// mistaken for a wedged shard. `None` means the thread was still +/// running at the post-shutdown deadline and the handle was dropped +/// (the OS thread keeps running detached; process exit reaps it). +/// `JoinHandle` has no timed join, so this polls `is_finished` at +/// [`JOIN_POLL_INTERVAL`]; the closing `join()` on a finished thread +/// returns immediately. +fn join_until_shutdown_deadline( + handle: thread::JoinHandle>, + shutdown_flag: &AtomicBool, + join_timeout: Duration, + deadline: &mut Option, +) -> Option>> { + while !handle.is_finished() { + if deadline.is_none() && shutdown_flag.load(Ordering::Relaxed) { + *deadline = Some(Instant::now() + join_timeout); + } + if let Some(deadline) = deadline + && Instant::now() >= *deadline + { + return None; + } + thread::sleep(JOIN_POLL_INTERVAL); + } + Some(handle.join()) +} + +/// Best-effort extraction of the panic message from a +/// `Box` returned by `JoinHandle::join`. Tries the two +/// payload shapes the standard library guarantees (`&'static str` and +/// `String`) and falls back to a placeholder so the panic still surfaces +/// in the error chain. +fn panic_payload_to_string(payload: &(dyn std::any::Any + Send)) -> String { + if let Some(s) = payload.downcast_ref::<&'static str>() { + return (*s).to_string(); + } + if let Some(s) = payload.downcast_ref::() { + return s.clone(); + } + "".to_string() +} + +/// Joins survivor shard threads after a partial-spawn failure, bounded +/// by the same `shutdown_join_timeout` budget as the normal exit path. +/// +/// Polls every survivor's `is_finished` in one loop instead of spawning +/// per-survivor joiner threads: the likely OS state on this path is +/// `pthread_create` EAGAIN (the parent spawn just failed with it), so +/// nothing here may create threads, and polling drains all survivors in +/// parallel anyway. A survivor still running at the deadline is +/// abandoned with an error log so the failed bootstrap can surface its +/// spawn error instead of hanging on a wedged shard. +fn join_partial_shard_survivors( + shard_threads: Vec<(u16, thread::JoinHandle>)>, + join_timeout: Duration, +) { + let deadline = Instant::now() + join_timeout; + let mut remaining = shard_threads; + loop { + let mut still_running = Vec::with_capacity(remaining.len()); + for (shard_id, survivor) in remaining { + if survivor.is_finished() { + let _ = survivor.join(); + info!(shard_id, "survivor shard thread drained"); + } else { + still_running.push((shard_id, survivor)); + } + } + remaining = still_running; + if remaining.is_empty() || Instant::now() >= deadline { + break; + } + thread::sleep(JOIN_POLL_INTERVAL); + } + for (shard_id, _survivor) in remaining { + error!( + shard_id, + waited = ?join_timeout, + "survivor shard thread still running at the shutdown join deadline; abandoning it" + ); + } +} + +/// Flips the cross-thread shutdown flag on `Drop` unless disarmed. +/// +/// A shard thread that exits via an error `?` or a panic unwind would +/// otherwise leave sibling shards parked forever on `bus.token().wait()`: +/// their watchdogs never observe the flag and the bus has no +/// `Drop`-triggered shutdown. Arming this for the whole thread body makes +/// every non-clean exit drive sibling-shard teardown. Disarmed only on a +/// clean `Ok(())`. +struct ShutdownOnDrop { + flag: Arc, + armed: bool, +} + +impl ShutdownOnDrop { + const fn new(flag: Arc) -> Self { + Self { flag, armed: true } + } + + const fn disarm(&mut self) { + self.armed = false; + } +} + +impl Drop for ShutdownOnDrop { + fn drop(&mut self) { + if self.armed { + self.flag.store(true, Ordering::Relaxed); + } + } +} + +/// Shard-local end of the metadata bundle handoff. +/// +/// Shard 0 owns the WAL writer and runs `recover()` to build the only +/// `WriteHandle`-bearing [`ServerNgMuxStateMachine`]. It then mints a +/// [`ServerNgMetadataBundle`] (a tuple of `Send + Sync` +/// `ReadHandleFactory`s) and pushes one clone per peer onto `bundle_tx`. +/// Every other shard receives the bundle and rebuilds a reader-mode +/// `MuxStateMachine` on its own runtime - no WAL access, no replay, no +/// `RecoverySync` two-phase fence. The old phase-2 WAL fence is gone +/// because peers no longer scan the WAL. They do still scan live shared +/// metadata to load their on-disk partitions, so a separate listener +/// fence is still required - see [`BootstrapBarrier`]. +/// +/// The channel is bounded to the peer count so shard 0's `send` never +/// blocks beyond a peer drain. A peer that dies before recv drops its +/// `bundle_rx`, so shard 0's `send` eventually sees a disconnected +/// channel; the cross-thread shutdown flag drives every waiter out of +/// its `recv` loop if shard 0 panics before broadcasting. +enum MetadataHandoff { + Owner { + bundle_tx: crossfire::MAsyncTx>, + }, + Waiter { + bundle_rx: crossfire::MAsyncRx>, + }, +} + +/// Reverse handshake to [`MetadataHandoff`]: gates shard 0's client +/// listeners until every peer has loaded its on-disk partitions. +/// +/// Peers build their owned-partition set from live shared metadata and +/// load each segment from disk in `build_shard_for_thread`. If shard 0 +/// opened listeners the instant `broadcast_metadata_bundle` returned +/// (peers have only *received* the bundle, not *loaded* partitions), a +/// client could create a partition before a peer's load scan finished. +/// That freshly committed partition would surface in the peer's scan +/// with no segment dir on disk yet, and `load_partition`'s `walk_dir` +/// would fail with `CannotReadPartitions`, aborting the whole node. A +/// partition created after boot must take the runtime reconciler path +/// (which creates its dir), never the bootstrap load path. +/// +/// Shard 0 (`Owner`) drains one signal per peer before binding +/// listeners; each peer (`Waiter`) sends one once its load completes. +/// The cross-thread shutdown flag drives both sides out of their poll +/// loop if any shard dies mid-boot. +enum BootstrapBarrier { + Owner { + ready_rx: crossfire::MAsyncRx>, + }, + Waiter { + ready_tx: crossfire::MAsyncTx>, + }, +} + +struct TcpTopology { + /// Domain-separation cluster id derived from `cluster.name`; threaded to + /// every consensus instance and the replica handshake so frames agree. + cluster_id: u128, + self_replica_id: u8, + replica_count: u8, + client_listen_addr: SocketAddr, + replica_listen_addr: Option, + ws_listen_addr: Option, + quic_listen_addr: Option, + http_listen_addr: Option, + tcp_tls_listen_addr: Option, + peers: Vec<(u8, SocketAddr)>, +} + +struct LocalClientAcceptFns { + tcp: AcceptedClientFn, + ws: AcceptedWsClientFn, + quic: AcceptedQuicClientFn, + tcp_tls: AcceptedTlsClientFn, + wss: AcceptedWssClientFn, +} + +#[derive(Default)] +struct BoundClientListeners { + tcp: Option, + tcp_tls: Option, + ws: Option, + quic: Option, +} + +/// Load config, prepare directories, and complete late logging init. +/// +/// # Errors +/// +/// Returns an error if config loading, directory preparation, or logging +/// setup fails. +pub async fn load_config(logging: &mut Logging) -> Result { + let config = ServerNgConfig::load() + .await + .map_err(ServerNgError::Config)?; + create_directories(&config.system).await.map_err(|source| { + error!( + system_path = %config.system.get_system_path(), + error = %source, + "failed to prepare server-ng directories" + ); + source + })?; + logging + .late_init( + config.system.get_system_path(), + &LoggingSettings::from(&config.system.logging), + &TelemetrySettings::from(&config.telemetry), + ) + .map_err(ServerNgError::Logging)?; + + Ok(config) +} + +/// Resolve the operator's `cpu_allocation` into concrete shard +/// assignments plus the checked `u16` shard count. +/// +/// Shard ids index `ReplicaOwnerTable` slots as `u16`. `OWNER_NONE` +/// (`u16::MAX`) is reserved as the empty-slot sentinel, so a server +/// configured with `u16::MAX` shards would mint a shard id that +/// collides with the sentinel and an owner-table lookup could never +/// tell that shard apart from an unowned slot. Reject at boot so the +/// invariant is held by the type system, not by hoping the operator +/// never configures 65535 cores worth of shards. +fn resolve_shard_assignments( + sharding: &configs::ng_sharding::ShardingConfig, +) -> Result<(Vec, u16), ServerNgError> { + let allocator = ShardAllocator::new(&sharding.cpu_allocation, sharding.pin_cores) + .map_err(ServerNgError::ShardAllocator)?; + let assignments = allocator + .to_shard_assignments() + .map_err(ServerNgError::ShardAllocator)?; + if assignments.is_empty() { + return Err(ServerNgError::ShardsCountZero); + } + match u16::try_from(assignments.len()) { + Ok(count) if count < message_bus::OWNER_NONE => Ok((assignments, count)), + _ => Err(ServerNgError::ShardsCountOverflow { + count: assignments.len(), + }), + } +} + +/// Re-validate the runtime sharding knobs that the per-shard runtime +/// consumes directly. Mirrors `ShardingConfig::validate` so a caller +/// that built the config without running it (e.g. tests, embedded +/// usage) cannot OOM at boot or wedge process exit with an out-of-range +/// value. +fn validate_sharding_runtime_knobs( + sharding: &configs::ng_sharding::ShardingConfig, +) -> Result<(), ServerNgError> { + let inbox_capacity = sharding.inbox_capacity; + if inbox_capacity == 0 || inbox_capacity > INBOX_CAPACITY_MAX { + return Err(ServerNgError::InvalidInboxCapacity { + value: inbox_capacity, + max: INBOX_CAPACITY_MAX, + }); + } + let drain_timeout = sharding.shutdown_drain_timeout.get_duration(); + if drain_timeout.is_zero() || drain_timeout > SHUTDOWN_DRAIN_TIMEOUT_MAX { + return Err(ServerNgError::InvalidShutdownDrainTimeout { + value: drain_timeout, + max: SHUTDOWN_DRAIN_TIMEOUT_MAX, + }); + } + let poll_interval = sharding.shutdown_poll_interval.get_duration(); + if poll_interval.is_zero() || poll_interval > SHUTDOWN_POLL_INTERVAL_MAX { + return Err(ServerNgError::InvalidShutdownPollInterval { + value: poll_interval, + max: SHUTDOWN_POLL_INTERVAL_MAX, + }); + } + // Ordering: a poll cadence coarser than the drain budget makes the + // cross-thread shutdown flag effectively unobservable during teardown. + if poll_interval > drain_timeout { + return Err(ServerNgError::ShutdownPollExceedsDrain { + poll: poll_interval, + drain: drain_timeout, + }); + } + Ok(()) +} + +/// Spawn the multi-shard `server-ng` runtime. +/// +/// Resolves shard count + CPU affinities from +/// `system.sharding.cpu_allocation`, builds canonical-ordered +/// `(senders, inboxes)` channels, and spawns one OS thread per shard. +/// +/// Each thread pins itself (`nix::sched::sched_setaffinity` on Linux via +/// [`ShardInfo::bind_cpu`]), binds memory to its NUMA node when +/// configured, builds a fresh `compio::runtime::Runtime` (one +/// `io_uring` instance per shard), and runs `shard_main` inside it. +/// +/// Returns [`ShardHandles`] containing the cross-thread shutdown flag +/// and the per-shard `JoinHandle`s. The caller (`main.rs`) installs a +/// `ctrlc` handler that flips the flag, then `.join()`s every handle. +/// +/// # Errors +/// +/// Returns an error if shard allocation fails, the inbox capacity is +/// invalid, or any OS thread fails to spawn. Per-shard recovery / +/// listener / consensus failures surface through the per-thread `Result` +/// the caller observes on `.join()`. +/// +/// # Panics +/// +/// Panics if [`shard_mesh_channels`] returns an inbox slot already +/// consumed - a bootstrap programming error that would only fire if this +/// function were called twice with the same inboxes. +#[allow(clippy::too_many_lines)] +pub fn bootstrap( + config: ServerNgConfig, + current_replica_id: Option, +) -> Result { + warm_dummy_password_hash(); + // The sync GetStats read path has no access to server config, so capture + // the data directory here for its disk-usage reporting. + crate::responses::init_stats_data_path(config.system.get_system_path().into()); + let (assignments, total_shards) = resolve_shard_assignments(&config.system.sharding)?; + let shards_count = assignments.len(); + + // Re-check the full valid range, not just the zero floor: a caller + // that built the config without running `ShardingConfig::validate` + // would otherwise OOM at boot allocating an oversized inbox channel, + // busy-loop every shutdown watchdog on a zero poll cadence, or wedge + // process exit on an unbounded drain budget. + let inbox_capacity = config.system.sharding.inbox_capacity; + validate_sharding_runtime_knobs(&config.system.sharding)?; + + let (senders, mut inboxes) = shard_mesh_channels(total_shards, inbox_capacity); + let shutdown_flag = Arc::new(AtomicBool::new(false)); + let config = Arc::new(config); + // One owner table per server process, Arc-cloned into every shard's bus so + // any shard's bus reads the same atomic slots that the owning + // shard's installer / disconnect path writes. + let owner_table = Arc::new(ReplicaOwnerTable::new()); + + // Single-shot bundle handoff (see `MetadataHandoff`): shard 0 sends + // one cloned `ServerNgMetadataBundle` per peer; each peer drains + // exactly one. Bounded to the peer count so shard 0's broadcast + // never blocks past a peer drain. A single-shard deployment (zero + // peers) still needs a non-zero capacity, so clamp up explicitly + // rather than relying on crossfire's internal cap=0 -> 1 promotion. + // If a peer dies before recv, shard 0's `send` eventually sees a + // disconnected channel; the cross-thread shutdown flag drives every + // waiter out of its recv loop if shard 0 panics before broadcasting. + let metadata_peers = shards_count.saturating_sub(1).max(1); + let (metadata_bundle_tx, metadata_bundle_rx) = + crossfire::mpmc::bounded_async::(metadata_peers); + + // Reverse barrier (see `BootstrapBarrier`): every peer sends one + // signal once it finishes loading its on-disk partitions; shard 0 + // drains them all before binding listeners. Bounded to the peer + // count so a sender never blocks (each peer sends exactly once). + let (ready_tx, ready_rx) = crossfire::mpmc::bounded_async::(metadata_peers); + + let mut shard_threads: Vec<(u16, thread::JoinHandle>)> = + Vec::with_capacity(shards_count); + // Shared metadata-group view: written by shard 0's publisher task, read by + // every shard's cluster-metadata roster so leader marking works off-shard. + let metadata_view = Arc::new(AtomicU64::new(crate::cluster_meta::METADATA_VIEW_UNKNOWN)); + for (idx, assignment) in assignments.into_iter().enumerate() { + #[allow(clippy::cast_possible_truncation)] + let shard_id = idx as u16; + let inbox = inboxes[idx] + .take() + .expect("shard_mesh_channels populates every inbox slot exactly once"); + let senders_for_shard = senders.clone(); + let config_for_shard = Arc::clone(&config); + let shutdown_flag_for_shard = Arc::clone(&shutdown_flag); + let owner_table_for_shard = Arc::clone(&owner_table); + let metadata_handoff_for_shard = if shard_id == 0 { + MetadataHandoff::Owner { + bundle_tx: metadata_bundle_tx.clone(), + } + } else { + MetadataHandoff::Waiter { + bundle_rx: metadata_bundle_rx.clone(), + } + }; + let barrier_for_shard = if shard_id == 0 { + BootstrapBarrier::Owner { + ready_rx: ready_rx.clone(), + } + } else { + BootstrapBarrier::Waiter { + ready_tx: ready_tx.clone(), + } + }; + + let metadata_view_for_shard = Arc::clone(&metadata_view); + let handle = match thread::Builder::new() + .name(format!("shard-{shard_id}")) + .spawn(move || -> Result<(), ServerNgError> { + run_shard_thread( + shard_id, + total_shards, + current_replica_id, + assignment, + senders_for_shard, + inbox, + config_for_shard, + shutdown_flag_for_shard, + metadata_handoff_for_shard, + barrier_for_shard, + owner_table_for_shard, + metadata_view_for_shard, + ) + }) { + Ok(handle) => handle, + Err(source) => { + // Signal every shard already spawned before propagating, so + // their watchdog loops drive `bus.shutdown(...)` and the + // process can exit instead of hanging on stuck OS threads. + shutdown_flag.store(true, Ordering::Relaxed); + // Drop bootstrap's own channel clones before joining + // survivors. Otherwise a peer waiting on `bundle_rx.recv` + // would never observe the sender side disconnecting and + // would hang until the shutdown watchdog kicks the bus. + drop(metadata_bundle_tx); + drop(metadata_bundle_rx); + drop(ready_tx); + drop(ready_rx); + join_partial_shard_survivors( + shard_threads, + config.system.sharding.shutdown_join_timeout.get_duration(), + ); + return Err(ServerNgError::ShardSpawnFailed { shard_id, source }); + } + }; + shard_threads.push((shard_id, handle)); + } + + // Drop bootstrap's own channel clones now that every shard owns its + // half. Keeping them on bootstrap's stack would deadlock a peer + // whose `bundle_rx.recv` only completes once every sender + // disconnects. + drop(metadata_bundle_tx); + drop(metadata_bundle_rx); + drop(ready_tx); + drop(ready_rx); + + info!( + shards_count, + "server-ng bootstrap dispatched; awaiting shard runtimes" + ); + + Ok(ShardHandles { + shutdown_flag, + shard_threads, + join_timeout: config.system.sharding.shutdown_join_timeout.get_duration(), + }) +} + +/// Per-shard OS thread entry. Pins CPU + memory, builds the compio +/// runtime, and `block_on`s `shard_main`. +#[allow(clippy::needless_pass_by_value, clippy::too_many_arguments)] +fn run_shard_thread( + shard_id: u16, + total_shards: u16, + replica_id: Option, + assignment: ShardInfo, + senders: Vec, + inbox: ShardReceiver, + config: Arc, + shutdown_flag: Arc, + metadata_handoff: MetadataHandoff, + barrier: BootstrapBarrier, + owner_table: Arc, + metadata_view: Arc, +) -> Result<(), ServerNgError> { + // Armed for the whole thread body: a post-spawn error `?` or a panic + // unwind here must flip `shutdown_flag` so sibling watchdogs drive + // their bus shutdown instead of parking forever on `bus.token().wait()`. + let mut shutdown_guard = ShutdownOnDrop::new(Arc::clone(&shutdown_flag)); + + assignment + .bind_cpu() + .map_err(|source| ServerNgError::CpuAffinityFailed { shard_id, source })?; + assignment + .bind_memory() + .map_err(|source| ServerNgError::MemoryAffinityFailed { shard_id, source })?; + + // `enrich_runtime_create_error` folds the io_uring remediation (raise + // `ulimit -l`, unblock seccomp, kernel-flag floor) into the error, so the + // guidance survives into the shard-join failure report instead of only + // stderr. Multi-shard boxes exhaust RLIMIT_MEMLOCK on per-shard rings + // before the bootstrap runtime does, so this path needs it most. + let runtime = create_shard_executor().map_err(|source| { + let source = server_common::diagnostics::enrich_runtime_create_error(source); + ServerNgError::ShardRuntimeCreateFailed { shard_id, source } + })?; + + let result = runtime.block_on(async move { + // `shard_main`'s future grows past clippy's `large_futures` cap + // (it ferries the metadata handoff, bus, builders, and inflight + // I/O in one state machine). Heap-pin it so the top-level + // `block_on` future stays small; one allocation per startup buys + // the stack budget back. + Box::pin(shard_main( + shard_id, + total_shards, + replica_id, + senders, + inbox, + &config, + shutdown_flag, + metadata_handoff, + barrier, + owner_table, + metadata_view, + )) + .await + }); + + if result.is_ok() { + shutdown_guard.disarm(); + } + result +} + +/// Per-shard async lifecycle. Builds the bus, recovers metadata, +/// constructs the `IggyShard` for this shard's slice of partitions, +/// wires listeners on shard 0, and runs the message pump until +/// shutdown. +#[allow(clippy::too_many_arguments, clippy::too_many_lines)] +async fn shard_main( + shard_id: u16, + total_shards: u16, + replica_id: Option, + senders: Vec, + inbox: ShardReceiver, + config: &ServerNgConfig, + shutdown_flag: Arc, + metadata_handoff: MetadataHandoff, + barrier: BootstrapBarrier, + owner_table: Arc, + metadata_view: Arc, +) -> Result<(), ServerNgError> { + let topology = resolve_tcp_topology(config, replica_id)?; + let bus = Rc::new(IggyMessageBus::with_config_and_owner_table( + shard_id, + config, + owner_table, + )); + // Every shard can own a delegated replica connection, so every + // shard's bus needs the handshake identity (the handshake itself + // runs on the owning shard, not on shard 0). + bus.set_replica_handshake_ctx(ReplicaHandshakeCtx { + cluster_id: topology.cluster_id, + self_id: topology.self_replica_id, + replica_count: topology.replica_count, + auth: load_replica_auth(config).map(Rc::new), + tls: load_replica_tls_ctx(config, &topology)?.map(Rc::new), + }); + + let drain_timeout = config.system.sharding.shutdown_drain_timeout.get_duration(); + let poll_interval = config.system.sharding.shutdown_poll_interval.get_duration(); + + let shutdown_flag_for_handoff = Arc::clone(&shutdown_flag); + spawn_shutdown_watchdog(Rc::clone(&bus), shutdown_flag, drain_timeout, poll_interval); + + // Metadata bootstrap is single-writer: shard 0 owns the WAL and the + // only `WriteHandle`-bearing `MuxStateMachine`. Peer shards receive + // a `ReadHandleFactory` bundle on the inter-thread channel and + // rebuild a reader-mode `MuxStateMachine` on their own runtime - no + // WAL access, no replay. Writes still funnel through shard 0's + // metadata VSR; per-commit `publish()` (in `WriteCell::apply`) + // bounds reader staleness to one op. + let data_dir = Path::new(&config.system.path); + let (mux_stm, owner_state) = match metadata_handoff { + MetadataHandoff::Owner { bundle_tx } => { + // Root is created locally at boot (never journaled), so replay + // must start from the same baseline or every WAL-created user + // shifts one slab id and root is lost after the first restart. + let recovered = recover::( + data_dir, + ReplicaIdentity { + cluster: topology.cluster_id, + replica_id: topology.self_replica_id, + replica_count: topology.replica_count, + }, + config.metadata.journal_slots, + config.metadata.clients_table_max, + |mux_stm| { + ensure_default_root_user(mux_stm); + }, + ) + .await + .map_err(ServerNgError::MetadataRecovery)?; + validate_cluster_root_bootstrap(config, &recovered.mux_stm)?; + ensure_default_root_user(&recovered.mux_stm); + // The factory bundle hands every peer a read handle over the + // same `Inner`, so `Arc` (and the parent + // `Arc`) is shared across all shards. Zero the + // snapshot totals here, once, before any peer can observe the + // bundle. Per-shard `load_partition` deltas in + // `build_shard_for_thread` then race only against other + // atomic adds, never against a concurrent `swap(0)` that + // would mistake an in-flight delta for the snapshot total + // and decrement the parent `StreamStats` by it. + let () = recovered.mux_stm.streams().read(|inner| { + for (_, stream) in &inner.items { + for (_, topic) in &stream.topics { + topic.stats.zero_out_all(); + } + } + }); + broadcast_metadata_bundle( + shard_id, + &bundle_tx, + recovered.mux_stm.factory_bundle(), + total_shards.saturating_sub(1), + &shutdown_flag_for_handoff, + poll_interval, + ) + .await?; + ( + recovered.mux_stm, + Some(RecoveredOwnerState { + journal: recovered.journal, + snapshot: recovered.snapshot, + last_applied_op: recovered.last_applied_op, + last_journaled_op: recovered.last_journaled_op, + client_table: recovered.client_table, + superblock: recovered.superblock, + recovered_state: recovered.recovered_state, + snapshot_checkpoint: recovered.snapshot_checkpoint, + }), + ) + } + MetadataHandoff::Waiter { bundle_rx } => { + let bundle = await_metadata_bundle( + shard_id, + &bundle_rx, + &shutdown_flag_for_handoff, + poll_interval, + ) + .await?; + (ServerNgMuxStateMachine::from_factory_bundle(bundle), None) + } + }; + + // Metadata consensus + journal + snapshot live only on shard 0. + // `IggyShard::tick_metadata` short-circuits when `consensus.is_none()`, + // so peer shards have no caller that reads `journal` or `snapshot`. + let ( + metadata_consensus, + journal_for_metadata, + snapshot_for_metadata, + superblock_for_metadata, + checkpoint_seed, + recovered_client_table, + ) = if let Some(owner) = owner_state { + // `recover()` already opened the superblock, read `recovered_state`, and + // verified the on-disk snapshot against its checkpoint pairing BEFORE decoding + // it. Reuse that superblock rather than re-opening it, which would fork the + // ping-pong sequence counter. Consensus recovers its true (view, log_view) + // from `recovered_state` instead of inferring a stale view from the WAL. + let consensus = restore_metadata_consensus(&owner, &topology, config, Rc::clone(&bus)); + let superblock = Rc::new(owner.superblock); + ( + Some(consensus), + Some(owner.journal), + owner.snapshot, + Some(superblock), + owner.snapshot_checkpoint, + Some(owner.client_table), + ) + } else { + (None, None, None, None, (0, 0), None) + }; + let metadata = ServerNgMetadata::new( + metadata_consensus, + journal_for_metadata, + snapshot_for_metadata, + superblock_for_metadata, + mux_stm, + Some(PathBuf::from(&config.system.path)), + ); + // Size the VSR client table before listeners bind and any client registers. + // Must precede the recovered-table install below: the setter rebuilds the + // table from scratch, so running it afterwards would drop every resumed + // session (and trip its empty-table assert). + metadata.set_clients_table_max(config.metadata.clients_table_max); + // Reinstall the sessions recovery restored from the checkpoint and the WAL + // suffix, so a rebooted node dedups retries and admits continuations from + // clients that kept their identity across the restart (IGGY-137). Recovery + // sized this table from the same config value, so the install preserves the + // configured cap. + if let Some(client_table) = recovered_client_table { + // Refusal (a client registered before this ran) keeps the live table + // and is logged by the callee; boot continues either way. + let _ = metadata.install_client_table(client_table); + } + // Seed the coordinator's last-checkpoint pairing so the first post-boot + // view-change superblock write records the real (checkpoint_op, checksum) + // instead of (0, 0). No-op on peer shards, which have no coordinator. + metadata.seed_checkpoint_ref(checkpoint_seed.0, checkpoint_seed.1); + // Shard 0's copy resolves the `ServerDefault` sentinels (max topic size and + // message expiry) at create admission; responses echo stored values verbatim. + metadata.set_default_max_topic_size(config.system.topic.max_size.as_bytes_u64()); + metadata.set_default_message_expiry(u64::from(config.system.topic.message_expiry)); + // Keep the forced-checkpoint margin >= the configured prepare-queue + // depth: ops already pipelined while a checkpoint runs append into that + // margin (config validation keeps journal_slots >= 4x this). + metadata.set_checkpoint_margin(config.metadata.checkpoint_margin()); + + let shard_metrics = ShardMetrics::for_shard(); + // Notifier install deferred until after tick handler wires below. + let senders_for_notifier = senders.clone(); + let metrics_for_notifier = shard_metrics.clone(); + // Heap-pin like `shard_main` above: the builder future carries the whole + // shard construction state machine and outgrew clippy's `large_futures` + // cap; one allocation per shard startup. + let (shard, sessions) = Box::pin(build_shard_for_thread( + shard_id, + total_shards, + config, + &topology, + metadata, + Rc::clone(&bus), + senders, + inbox, + shard_metrics, + Arc::clone(&metadata_view), + )) + .await?; + + // Shard 0 owns the metadata consensus; publish its view so every shard's + // cluster-metadata read (and the SDK's leader discovery) marks the live + // primary. Detached: dies with this shard's runtime at process exit. + if shard_id == 0 { + let publisher_shard = Rc::clone(&shard); + let publisher_view = Arc::clone(&metadata_view); + compio::runtime::spawn(async move { + loop { + if let Some(consensus) = publisher_shard.plane.metadata().consensus.as_ref() { + // While this replica declines its recovered view's + // primaryship, that view must not reach the roster: the + // delegated shards would compute a leader that never + // heartbeats. Publish "unknown" until the election + // resolves the role. + let published = if consensus.has_ceded_primaryship() + && consensus.primary_index(consensus.view()) == consensus.replica() + { + crate::cluster_meta::METADATA_VIEW_UNKNOWN + } else { + u64::from(consensus.view()) + }; + publisher_view.store(published, Ordering::Relaxed); + } + compio::time::sleep(std::time::Duration::from_millis(100)).await; + } + }) + .detach(); + } + + info!( + shard = shard_id, + partitions = shard.plane.partitions().len(), + "server-ng shard initialized" + ); + + // Re-check the cross-thread shutdown flag here, *before* spawning the + // message pump. A sibling shard may have failed in the window between + // the metadata broadcast and this point; gating before spawn keeps the + // bus' `background_tasks` vec empty on the shutdown path. Spawn-then- + // check would leave `bus.track_background(pump_handle)` registering a + // `JoinHandle` that only `bus.shutdown()` drains, but the watchdog + // driving `bus.shutdown()` is `.detach()`'d (see TODO at + // `spawn_shutdown_watchdog`) and may not be scheduled before this + // function returns `Ok(())` and the compio runtime drops, cancelling + // the pump mid-`write_vectored_all`. + // + // Without this gate shard 0 would also still open TCP/QUIC/WS + // listeners for a server that is already tearing down, briefly + // accepting connections that immediately get torn by the watchdog. + if shutdown_flag_for_handoff.load(Ordering::Relaxed) { + return Ok(()); + } + + // Tick handler must install before the notifier so early commits + // do not broadcast ticks whose handler slot is still `None`. + let (reconcile_wake_tx, reconcile_wake_rx) = channel::<()>(1); + let (reconcile_stop_tx, reconcile_stop_rx) = channel::<()>(1); + crate::partition_reconciler::install_tick_handler(&shard, reconcile_wake_tx); + + // Only shard 0 commits metadata. + if shard_id == 0 { + let notifier = make_metadata_commit_notifier(senders_for_notifier, metrics_for_notifier); + shard.plane.metadata().set_commit_notifier(Some(notifier)); + } else { + drop(senders_for_notifier); + drop(metrics_for_notifier); + } + + // The pump task also drives the consensus timer tick (heartbeats, prepare + // retransmit, view-change timeouts) as a select! arm, serialized with frame + // processing - see `run_message_pump`. + let (stop_tx, stop_rx) = channel(1); + let pump_shard = Rc::clone(&shard); + // Owned and awaited by shard_main at exit, NOT `track_background`: the + // background drain runs inside `bus.shutdown()`, which the Ctrl-C path + // never drives (the watchdog stands down when the token fires), so a + // tracked pump would be cancelled by runtime teardown mid final-flush + // and every graceful shutdown would silently drop the committed journal + // tail that had not hit a flush threshold yet. + let mut pump_handle = Some(compio::runtime::spawn(async move { + pump_shard.run_message_pump(stop_rx).await; + })); + + let reconciler_ctx = Rc::new(crate::partition_reconciler::ReconcilerCtx::new( + Rc::clone(&shard), + total_shards, + Rc::new(config.clone()), + topology.cluster_id, + topology.self_replica_id, + topology.replica_count, + )); + let reconcile_periodic = config + .system + .sharding + .reconcile_periodic_interval + .get_duration(); + let reconciler_handle = compio::runtime::spawn({ + let ctx = Rc::clone(&reconciler_ctx); + async move { + crate::partition_reconciler::run_reconciler( + ctx, + reconcile_wake_rx, + reconcile_stop_rx, + reconcile_periodic, + ) + .await; + } + }); + bus.track_background(reconciler_handle); + + // Per-shard heartbeat verifier: evicts connections that stop pinging, + // releasing their consumer-group membership. Gated on config so a + // deployment without heartbeats never reaps live sessions. + let heartbeat_stop_tx = if config.heartbeat.enabled { + let (hb_stop_tx, hb_stop_rx) = channel::<()>(1); + let hb_shard = Rc::clone(&shard); + let hb_sessions = Rc::clone(&sessions); + let hb_interval = config.heartbeat.interval.get_duration(); + let hb_handle = compio::runtime::spawn(async move { + crate::dispatch::run_heartbeat_verifier(hb_shard, hb_sessions, hb_interval, hb_stop_rx) + .await; + }); + bus.track_background(hb_handle); + Some(hb_stop_tx) + } else { + None + }; + // Expired-PAT cleaner: shard 0 only (it owns the metadata consensus + // group) and only when enabled. Each pass no-ops unless this node is + // the caught-up metadata primary, so the delete is proposed once and + // replicated to every replica. + let pat_cleaner_stop = if shard_id == 0 && config.personal_access_token.cleaner.enabled { + let (cleaner_stop_tx, cleaner_stop_rx) = channel(1); + let cleaner_shard = Rc::clone(&shard); + let interval = config.personal_access_token.cleaner.interval.get_duration(); + let cleaner_handle = compio::runtime::spawn(async move { + crate::personal_access_token_cleaner::run_pat_cleaner( + cleaner_shard, + cleaner_stop_rx, + interval, + ) + .await; + }); + bus.track_background(cleaner_handle); + Some(cleaner_stop_tx) + } else { + None + }; + + // Segment cleaner: runs on every shard (each replica trims its own log, + // primary and backup alike). Local and unreplicated; gated by the shared + // data-maintenance config. + let segment_cleaner_stop = if config.data_maintenance.messages.cleaner_enabled { + let (stop_tx, stop_rx) = channel(1); + let cleaner_shard = Rc::clone(&shard); + let interval = config.data_maintenance.messages.interval.get_duration(); + let cleaner_handle = compio::runtime::spawn(async move { + crate::segment_cleaner::run_segment_cleaner(cleaner_shard, stop_rx, interval).await; + }); + bus.track_background(cleaner_handle); + Some(stop_tx) + } else { + None + }; + + // Listener fence (see `BootstrapBarrier`). Peers still scan live + // shared metadata and load their on-disk partitions in + // `build_shard_for_thread`; the factory-bundle handoff only proves + // they *received* the bundle, not that they finished loading. Shard + // 0 must not accept client traffic until every peer's load scan is + // done, otherwise a partition created by the first client surfaces + // in a still-running scan with no segment dir on disk and aborts the + // node with `CannotReadPartitions`. By this point every shard has + // also spawned its pump + reconciler, so a partition created after + // the fence takes the runtime reconciler path on its owning shard. + match barrier { + BootstrapBarrier::Owner { ready_rx } => { + await_bootstrap_complete( + &ready_rx, + usize::from(total_shards.saturating_sub(1)), + &shutdown_flag_for_handoff, + poll_interval, + ) + .await?; + } + BootstrapBarrier::Waiter { ready_tx } => { + signal_bootstrap_complete( + shard_id, + &ready_tx, + &shutdown_flag_for_handoff, + poll_interval, + ) + .await?; + } + } + + // Listeners (replica + every client transport) bind on shard 0 only. + // Shard 0's coordinator round-robins inbound TCP/WS connections to + // peer shards via fd-transfer. QUIC and TCP-TLS clients terminate + // locally on shard 0 (their per-connection state is non-portable - + // see `LifecycleFrame::ClientWsConnectionSetup` rustdoc). + if shard_id == 0 { + let coord = shard + .coordinator() + .expect("shard 0 always has a coordinator attached by the builder"); + // Reseed the client-id minter above every recovered entry before any + // listener accepts. The counter is per process; the table it must not + // collide with was rebuilt from the previous boot's WAL. Keyed by view + // so a later promotion refolds the table (the minting path calls the + // same method, see `HttpInner::register_session_once`). + let boot_view = shard + .plane + .metadata() + .consensus + .as_ref() + .map_or(0, consensus::VsrConsensus::view); + coord.seed_client_sequence( + boot_view, + shard.plane.metadata().client_table.borrow().client_ids(), + ); + let on_client_request = make_client_request_handler( + &shard, + &sessions, + Arc::clone(&config.system), + config.personal_access_token.max_tokens_per_user, + ); + let (accepted_replica, dialed_replica) = + make_replica_delegation_fns(Rc::clone(&coord), &bus); + let accepted_client = make_shard_zero_client_accept_fns(coord, &bus, on_client_request); + + if let Err(error) = start_tcp_runtime( + &shard, + config, + &topology, + accepted_replica, + dialed_replica, + accepted_client, + ) + .await + { + let _ = stop_tx.try_send(()); + let _ = reconcile_stop_tx.try_send(()); + if let Some(tx) = &heartbeat_stop_tx { + let _ = tx.try_send(()); + } + if let Some(cleaner_stop_tx) = &pat_cleaner_stop { + let _ = cleaner_stop_tx.try_send(()); + } + if let Some(tx) = &segment_cleaner_stop { + let _ = tx.try_send(()); + } + await_pump_drain(pump_handle.take(), config, shard_id).await; + return Err(error); + } + } + + bus.token().wait().await; + let _ = stop_tx.try_send(()); + let _ = reconcile_stop_tx.try_send(()); + if let Some(tx) = &heartbeat_stop_tx { + let _ = tx.try_send(()); + } + if let Some(cleaner_stop_tx) = &pat_cleaner_stop { + let _ = cleaner_stop_tx.try_send(()); + } + if let Some(tx) = &segment_cleaner_stop { + let _ = tx.try_send(()); + } + + await_pump_drain(pump_handle.take(), config, shard_id).await; + + info!(shard = shard_id, "server-ng shard exited cleanly"); + Ok(()) +} + +/// Await the message pump's completion before the shard returns: its +/// post-loop work includes the final flush of every committed journal to +/// segment storage, and returning first drops the compio runtime, which +/// cancels that flush at its next await point. +async fn await_pump_drain( + pump_handle: Option>, + config: &ServerNgConfig, + shard_id: u16, +) { + let Some(pump_handle) = pump_handle else { + return; + }; + let drain_budget = config.system.sharding.shutdown_drain_timeout.get_duration(); + if compio::time::timeout(drain_budget, pump_handle) + .await + .is_err() + { + warn!( + shard = shard_id, + "message pump did not drain within the shutdown budget; \ + committed journal tail may not have flushed" + ); + } +} + +/// Block until shard 0 broadcasts the metadata factory bundle, or the +/// cross-thread shutdown flag flips. Polled in a `poll_interval` loop +/// so a shard 0 that panics before it broadcasts cannot strand peer +/// shards: the shutdown path flips the flag, every waiter observes it +/// on the next tick, and the server tears down instead of hanging. +/// +/// Uses `try_recv` + sleep rather than `timeout(recv())`. Crossfire 3.x +/// documents `recv()` as cancellation-safe (no leak/deadlock) but does +/// not guarantee atomicity for the dropped future's result; `try_recv` +/// keeps each tick fully synchronous and side-effect-free, so the +/// shutdown poll cadence cannot ambiguously consume a bundle. +async fn await_metadata_bundle( + shard_id: u16, + bundle_rx: &crossfire::MAsyncRx>, + shutdown_flag: &Arc, + poll_interval: Duration, +) -> Result { + loop { + match bundle_rx.try_recv() { + Ok(bundle) => return Ok(bundle), + Err(crossfire::TryRecvError::Disconnected) => { + return Err(ServerNgError::MetadataHandoffAborted { shard_id }); + } + Err(crossfire::TryRecvError::Empty) => { + if shutdown_flag.load(Ordering::Relaxed) { + return Err(ServerNgError::MetadataHandoffAborted { shard_id }); + } + compio::time::sleep(poll_interval).await; + } + } + } +} + +/// Push `peers` cloned bundles onto `bundle_tx`, polling each send in a +/// `poll_interval` loop so the cross-thread shutdown flag can interrupt +/// a stalled handoff. Symmetric to [`await_metadata_bundle`]: shutdown +/// observed mid-handshake aborts cleanly rather than stalling on a +/// `send` future that can no longer make progress. +/// +/// Uses `try_send` + sleep rather than `timeout(send())`. Crossfire 3.x +/// documents `send()` as cancellation-safe in the leak/deadlock sense +/// but explicitly warns the true result is unknown when `SendFuture` is +/// dropped on cancellation. For a retry loop that re-clones on every +/// tick that would risk publishing the same bundle twice, stuffing the +/// bounded channel past `peers` and stranding a follow-up `send`. +/// `try_send` returns the bundle back inside `TrySendError::Full`, so +/// the loop reuses it instead of re-cloning when the channel is full. +async fn broadcast_metadata_bundle( + shard_id: u16, + bundle_tx: &crossfire::MAsyncTx>, + bundle: ServerNgMetadataBundle, + peers: u16, + shutdown_flag: &Arc, + poll_interval: Duration, +) -> Result<(), ServerNgError> { + for _ in 0..peers { + let mut pending = bundle.clone(); + loop { + match bundle_tx.try_send(pending) { + Ok(()) => break, + Err(crossfire::TrySendError::Disconnected(_)) => { + // Every peer dropped its `bundle_rx` before recv. Shard + // 0 must not silently continue past handoff: it would + // bind listeners and commit consensus state for a + // cluster whose peers are gone. Propagate the abort so + // `shard_main` short-circuits before further side + // effects; `shutdown_flag` will flip via the normal + // teardown path. + return Err(ServerNgError::MetadataHandoffAborted { shard_id }); + } + Err(crossfire::TrySendError::Full(returned)) => { + if shutdown_flag.load(Ordering::Relaxed) { + return Err(ServerNgError::MetadataHandoffAborted { shard_id }); + } + pending = returned; + compio::time::sleep(poll_interval).await; + } + } + } + } + Ok(()) +} + +/// Peer side of [`BootstrapBarrier`]: tell shard 0 this shard finished +/// loading its on-disk partitions. Mirrors [`broadcast_metadata_bundle`]'s +/// `try_send`-or-shutdown poll loop so a sibling failure (which flips the +/// shutdown flag) drives this out instead of stranding it on a full +/// channel. The channel is sized to the peer count and each peer sends +/// exactly once, so `Full` is not expected; the branch only keeps the +/// loop interruptible. +async fn signal_bootstrap_complete( + shard_id: u16, + ready_tx: &crossfire::MAsyncTx>, + shutdown_flag: &Arc, + poll_interval: Duration, +) -> Result<(), ServerNgError> { + let mut pending = shard_id; + loop { + match ready_tx.try_send(pending) { + Ok(()) => return Ok(()), + Err(crossfire::TrySendError::Disconnected(_)) => { + // Shard 0 dropped its `ready_rx` before draining (it + // aborted before binding listeners). Propagate so this + // shard short-circuits; the shutdown flag flips via the + // normal teardown path. + return Err(ServerNgError::MetadataHandoffAborted { shard_id }); + } + Err(crossfire::TrySendError::Full(returned)) => { + if shutdown_flag.load(Ordering::Relaxed) { + return Err(ServerNgError::MetadataHandoffAborted { shard_id }); + } + pending = returned; + compio::time::sleep(poll_interval).await; + } + } + } +} + +/// Owner side of [`BootstrapBarrier`]: drain one ready signal per peer +/// before shard 0 binds listeners. Polls the shutdown flag so a peer that +/// dies mid-load (flipping the flag) aborts the wait instead of hanging on +/// a signal that will never arrive. A single shard (`peers == 0`) returns +/// immediately. +async fn await_bootstrap_complete( + ready_rx: &crossfire::MAsyncRx>, + peers: usize, + shutdown_flag: &Arc, + poll_interval: Duration, +) -> Result<(), ServerNgError> { + let mut remaining = peers; + while remaining > 0 { + match ready_rx.try_recv() { + Ok(_shard_id) => remaining -= 1, + Err(crossfire::TryRecvError::Disconnected) => { + return Err(ServerNgError::ShardBootstrapBarrierAborted { remaining }); + } + Err(crossfire::TryRecvError::Empty) => { + if shutdown_flag.load(Ordering::Relaxed) { + return Err(ServerNgError::ShardBootstrapBarrierAborted { remaining }); + } + compio::time::sleep(poll_interval).await; + } + } + } + Ok(()) +} + +/// Spawn a per-shard polling task that watches the cross-thread shutdown +/// flag and triggers this shard's bus shutdown on transition. The flag +/// is the only Send signal we have; the bus' shutdown machinery is +/// `!Send` (`Rc>` + per-shard `async_channel`), so it must be +/// triggered from within the runtime that owns the bus. +#[allow(clippy::needless_pass_by_value)] +fn spawn_shutdown_watchdog( + bus: Rc, + shutdown_flag: Arc, + drain_timeout: Duration, + poll_interval: Duration, +) { + let bus_for_task = Rc::clone(&bus); + let bus_token = bus.token(); + let watchdog = compio::runtime::spawn(async move { + loop { + if shutdown_flag.load(Ordering::Relaxed) { + break; + } + if bus_token.is_triggered() { + // Bus shutdown was driven from elsewhere (e.g. internal + // failure path). Watchdog has nothing left to do. + return; + } + compio::time::sleep(poll_interval).await; + } + let _ = bus_for_task.shutdown(drain_timeout).await; + }); + // TODO(hubcio): `.detach()` races bus shutdown: when `bus.token()` is + // triggered, `shard_main` returns and the runtime drops the watchdog + // mid-`bus.shutdown()`, truncating in-flight `ClientForwardFailed` + // replies (terminal per `SendError` docs). Cannot use + // `bus.track_background(watchdog)` here because the watchdog itself + // drives `bus.shutdown()`, and the bg-drain loop in `shutdown()` + // would re-enter awaiting the watchdog's own pending shutdown call + // (self-deadlock). Fix: extract a `core/task_registry` crate mirroring + // `core/server`'s task-tracking mechanism, share it between the bus + // and server-ng so background tasks can be reaped without coupling + // to the bus shutdown order. + watchdog.detach(); +} + +/// Copy the configured cluster roster plus this node's own client ports into +/// the shared [`ClusterRoster`] so the binary `GetClusterMetadata` read serves +/// the real topology. `self_*` back only the cluster-disabled self-synthesis +/// and carry the requested listener ports from the resolved topology, not the +/// bound ones (a `:0` wildcard is reported as 0). +fn build_cluster_roster( + config: &ServerNgConfig, + topology: &TcpTopology, + metadata_view: Arc, +) -> ClusterRoster { + ClusterRoster { + enabled: config.cluster.enabled, + name: config.cluster.name.clone(), + nodes: config + .cluster + .nodes + .iter() + .cloned() + .map(Into::into) + .collect(), + self_ip: topology.client_listen_addr.ip().to_string(), + self_ports: configs::ng_cluster::TransportPorts { + tcp: Some(topology.client_listen_addr.port()), + quic: topology.quic_listen_addr.map(|addr| addr.port()), + http: topology.http_listen_addr.map(|addr| addr.port()), + websocket: topology.ws_listen_addr.map(|addr| addr.port()), + tcp_replica: None, + }, + metadata_view, + } +} + +#[allow(clippy::too_many_arguments, clippy::too_many_lines)] +async fn build_shard_for_thread( + shard_id: u16, + total_shards: u16, + config: &ServerNgConfig, + topology: &TcpTopology, + metadata: ServerNgMetadata, + bus: Rc, + senders: Vec, + inbox: ShardReceiver, + metrics: ShardMetrics, + metadata_view: Arc, +) -> Result<(Rc, Rc>), ServerNgError> { + let shard_local_id = ShardId::new(shard_id); + let total_partitions = metadata.mux_stm.streams().read(|inner| { + inner + .items + .iter() + .map(|(_, stream)| { + stream + .topics + .iter() + .map(|(_, topic)| topic.partitions.len()) + .sum::() + }) + .sum::() + }); + + // IggyPartitions holds only the partitions owned by this shard + // (see the filter below at insert time), so the server-wide total + // is an N-fold overshoot. `ceil(total / shards) * 2` is a coarse + // upper bound that absorbs hash skew without paying the full + // multiplier. PapayaShardsTable below stays sized to the server-wide + // total because every shard routes every namespace. + let owned_partitions_capacity = total_partitions + .div_ceil(usize::from(total_shards).max(1)) + .saturating_mul(2); + // At-rest encryption: built once per shard from the shared config; the + // ingestion path encrypts on the primary and the poll reply decrypts. + // A bad key fails the boot rather than silently serving plaintext. + let encryptor = if config.system.encryption.enabled { + let aes = Aes256GcmEncryptor::from_base64_key(&config.system.encryption.key) + .map_err(|error| ServerNgError::Iggy(Box::new(error)))?; + Some(Arc::new(EncryptorKind::Aes256Gcm(aes))) + } else { + None + }; + let partitions = IggyPartitions::with_capacity( + shard_local_id, + PartitionsConfig { + messages_required_to_save: config.system.partition.messages_required_to_save, + size_of_messages_required_to_save: config + .system + .partition + .size_of_messages_required_to_save, + enforce_fsync: config.system.partition.enforce_fsync, + segment_size: config.system.segment.size, + encryptor, + }, + owned_partitions_capacity, + ); + let shards_table = PapayaShardsTable::with_capacity(total_partitions); + + // Stream-filter inside the `read()` closure: only partitions owned by + // this shard need the heavy (`Arc` + `Partition`) clones + // for the async `load_partition` below. Non-owning entries are pushed + // straight into `shards_table` here, so no Vec scales with the + // server-wide partition count. + let owned = metadata.mux_stm.streams().read(|inner| { + let mut owned = Vec::with_capacity(owned_partitions_capacity); + for (_, stream) in &inner.items { + for (topic_id, topic) in &stream.topics { + for partition in &topic.partitions { + let namespace = IggyNamespace::new(stream.id, topic_id, partition.id); + let owning_shard = + calculate_shard_assignment(&namespace, u32::from(total_shards)); + if owning_shard == shard_id { + // Shared per-partition stats from the registry: the + // same `Arc` backs every shard's `get_topic` reply. + let stats = inner.stats_registry.partition( + stream.id, + topic_id, + partition.id, + topic.stats.clone(), + ); + owned.push((stream.id, topic_id, stats, partition.clone())); + } else { + shards_table.insert( + namespace, + PartitionLocation::new( + ShardId::new(owning_shard), + partition.created_revision, + ), + ); + } + } + } + } + owned + }); + + // Snapshot totals were zeroed once on shard 0 before the factory + // bundle was broadcast (see `MetadataHandoff::Owner`). All shards + // here only add their per-partition deltas, so the shared + // `Arc` atomics race only against other atomic adds. + for (stream_id, topic_id, partition_stats, partition_metadata) in owned { + validate_namespace_bounds(config, stream_id, topic_id, partition_metadata.id)?; + let namespace = IggyNamespace::new(stream_id, topic_id, partition_metadata.id); + let partition = match load_partition( + config, + namespace, + Arc::clone(&partition_stats), + &partition_metadata, + topology.cluster_id, + topology.self_replica_id, + topology.replica_count, + Rc::clone(&bus), + ) + .await + { + Ok(partition) => partition, + // ONE damaged local chain must not take the node down. The shapes + // this refuses are exactly what a failed state-transfer quarantine + // leaves behind, so fence that group the same way the runtime path + // does -- move its segment files aside, keeping the superblock so it + // cannot re-enter view 0 -- and materialise it fresh. The ordinary + // rejoin path (repair, then state transfer on a refused floor) + // recovers its data from a peer. + Err(ServerNgError::PartitionChainRefused { dir, reason, .. }) => { + let partition_dir = dir.to_string_lossy().into_owned(); + error!( + stream_id, + topic_id, + partition_id = partition_metadata.id, + partition_dir, + %reason, + "refusing the recovered segment chain; fencing this partition and \ + rebuilding it empty for the rejoin path" + ); + match partitions::state_transfer::quarantine_segment_files(&partition_dir).await { + Ok(fenced_dir) => error!( + stream_id, + topic_id, + partition_id = partition_metadata.id, + fenced_dir, + "quarantined the refused segment files; they are kept for inspection" + ), + Err(error) => { + // NOT rebuilt: `build_partition_fresh` reaches + // `ensure_initial_segment`, which opens segment 0 with + // `file_exists = false` and TRUNCATES whatever the + // failed quarantine left behind. The likeliest failures + // (suffix cap exhausted, `create_dir_all`) move zero + // files, so rebuilding would destroy the oldest segment + // on the first attempt while the higher-offset survivors + // keep refusing every boot -- a loop that never + // terminates and eats the chain one segment at a time. + // Tombstone instead: the namespace stays unmaterialised + // and unrouted, the reconciler backs off, and an + // operator still has every byte. + error!( + stream_id, + topic_id, + partition_id = partition_metadata.id, + partition_dir, + %error, + "failed to quarantine the refused segment files; leaving this \ + partition tombstoned rather than rebuilding over them" + ); + partition_stats.zero_out_all(); + partitions.tombstone(namespace); + continue; + } + } + // The refused load already folded its segment counts in. + partition_stats.zero_out_all(); + build_partition_fresh( + config, + namespace, + partition_stats, + partition_metadata.created_revision, + topology.cluster_id, + topology.self_replica_id, + topology.replica_count, + Rc::clone(&bus), + ) + .await? + } + // An untrustworthy superblock fences ONE group, not the node. The + // segment files stay exactly where they are -- unlike a refused + // chain, the data on disk is not the thing in doubt -- so there is + // nothing to quarantine and nothing to rebuild: rebuilding fresh + // would hand this replica a view-0 identity while a record it + // cannot read says otherwise. Tombstoned, the namespace stays + // unmaterialised and unrouted, the reconciler backs off, and an + // operator has every byte plus a message naming the directory. + Err( + error @ (ServerNgError::PartitionSuperblockIo { .. } + | ServerNgError::PartitionSuperblockVersionUnknown { .. } + | ServerNgError::PartitionSuperblockUnverifiable { .. } + | ServerNgError::PartitionSuperblockUndecodable { .. } + | ServerNgError::PartitionSuperblockIdentityMismatch { .. }), + ) => { + error!( + stream_id, + topic_id, + partition_id = partition_metadata.id, + %error, + "cannot trust this partition's durable consensus state; tombstoning the \ + partition and continuing to boot the rest of the shard" + ); + partition_stats.zero_out_all(); + partitions.tombstone(namespace); + continue; + } + Err(error) => return Err(error), + }; + partitions.insert(namespace, partition); + shards_table.insert( + namespace, + PartitionLocation::new(ShardId::new(shard_id), partition_metadata.created_revision), + ); + } + + let shard_handle = Rc::new(RefCell::new(None)); + // Same wiring path as the simulator's shell mode: one per-shard + // SessionManager shared by the client-request handler (binds sessions) + // and the get_clients handler (reads them). It also carries this shard's + // cluster roster for the pre-auth GetClusterMetadata read. + let ShellHandlers { + on_replica_message, + on_client_request, + on_metadata_submit, + on_list_clients, + on_partition_read, + sessions, + } = wire_shell_handlers( + &bus, + &shard_handle, + Arc::clone(&config.system), + config.personal_access_token.max_tokens_per_user, + ); + sessions + .borrow_mut() + .set_cluster_roster(Rc::new(build_cluster_roster( + config, + topology, + metadata_view, + ))); + let shard_name = format!("server-ng-shard-{shard_id}"); + let built = IggyShardBuilder::new( + ShardIdentity::new(shard_id, shard_name), + Rc::clone(&bus), + on_replica_message, + on_client_request, + on_metadata_submit, + on_list_clients, + on_partition_read, + metadata, + partitions, + senders, + inbox, + shards_table, + PartitionConsensusConfig::new( + topology.cluster_id, + shard::ReplicaTopology::new(topology.self_replica_id, topology.replica_count), + Rc::clone(&bus), + ), + CoordinatorConfig::default(), + metrics, + ) + .build() + .map_err(ServerNgError::ShardConstruction)?; + + let shard = Rc::new(built.shard); + // Repair pacing is shared by both planes' repair loops, so it is a + // per-shard tunable set once here rather than per consensus group. + shard.set_repair_retry_ticks(repair_retry_ticks(config)); + shard.set_served_segment_cache_bytes_max( + config + .partition + .transfer_served_cache_bytes_max + .as_bytes_u64(), + ); + shard.set_partition_artifact_len_max( + config.partition.transfer_artifact_bytes_max.as_bytes_u64(), + ); + shard.set_repair_chunk_max(config.cluster.repair_chunk_max as u64); + // Bounds a served state-transfer chunk. A frame above the bus ceiling is + // rejected by the RECEIVING transport, which tears the replica connection + // down rather than dropping one message. + shard.set_bus_max_message_size( + usize::try_from(config.message_bus.max_message_size.as_bytes_u64()).unwrap_or(usize::MAX), + ); + *shard_handle.borrow_mut() = Some(Rc::downgrade(&shard)); + Ok((shard, sessions)) +} + +// Pin the configs-crate default literals (duplicated there to avoid a +// build-time edge onto the runtime crates) against the runtime constants, +// mirroring the message_bus IOV_MAX pin. A drift on either side fails this +// crate's build until both are reconciled. +const _: () = assert!( + configs::ng_metadata::DEFAULT_METADATA_PREPARE_QUEUE_DEPTH + == consensus::PIPELINE_PREPARE_QUEUE_MAX +); +const _: () = assert!( + configs::ng_metadata::DEFAULT_METADATA_JOURNAL_SLOTS + == journal::prepare_journal::DEFAULT_SLOT_COUNT +); +const _: () = assert!( + configs::ng_partition::DEFAULT_PARTITION_PREPARE_QUEUE_DEPTH + == consensus::PIPELINE_PREPARE_QUEUE_MAX +); +const _: () = assert!( + configs::ng_metadata::DEFAULT_METADATA_CLIENTS_TABLE_MAX == consensus::CLIENTS_TABLE_MAX +); +const _: () = + assert!(configs::ng_cluster::DEFAULT_VIEW_PROBE_ATTEMPTS_MAX == consensus::PROBE_ATTEMPTS_MAX); +const _: () = assert!( + configs::ng_partition::DEFAULT_EVICTED_RING_CAPACITY == partitions::EVICTED_RING_CAPACITY +); +const _: () = assert!( + configs::ng_partition::DEFAULT_EVICTED_RING_BYTES_MAX == partitions::EVICTED_RING_BYTES_MAX +); +const _: () = assert!( + configs::ng_partition::DEFAULT_TRANSFER_ARTIFACT_BYTES_MAX + == shard::PARTITION_ARTIFACT_LEN_DEFAULT +); +const _: () = assert!( + configs::ng_partition::DEFAULT_TRANSFER_SERVED_CACHE_BYTES_MAX + == shard::SERVED_SEGMENT_CACHE_BYTES_DEFAULT +); +const _: () = + assert!(configs::ng_cluster::DEFAULT_REPAIR_CHUNK_MAX as u64 == shard::REPAIR_CHUNK_MAX); +const _: () = assert!( + configs::ng_cluster::STATE_CHUNK_HEADER_LEN + == size_of::() as u64 +); +/// Convert a consensus-timer interval to whole ticks, floored at one tick so a +/// sub-tick value still fires and saturated on overflow. +fn duration_to_ticks(interval: Duration) -> u64 { + let ticks = interval.as_millis() / shard::CONSENSUS_TICK_INTERVAL.as_millis(); + u64::try_from(ticks.max(1)).unwrap_or(u64::MAX) +} + +/// `[cluster] heartbeat_timeout` in consensus ticks. Every consensus group +/// (metadata and per-partition planes alike) gets the same window: the failure +/// it guards against - a primary that stopped heartbeating - is host-level, not +/// per-plane. +pub(crate) fn cluster_heartbeat_ticks(config: &ServerNgConfig) -> u64 { + duration_to_ticks(config.cluster.heartbeat_timeout.get_duration()) +} + +/// Floor for the post-restart read-recovery deadline (see +/// [`recovery_barrier_deadline`]). At and below the 5s default heartbeat the +/// worst-case recovery is dominated by the heartbeat-independent term - the +/// `ViewChangeStatus` backstop plus election ceremony and suffix recommit, +/// empirically ~7s - so the scaled value must never fall under this or a +/// fast-heartbeat cluster would 503 legitimate reads mid-recovery. The backstop +/// is the configurable `[cluster] view_change_status_timeout`; raising it past +/// its 5s default is why `recovery_barrier_deadline` scales that knob in too +/// rather than leaning on this floor to cover it. +const RECOVERY_BARRIER_DEADLINE_FLOOR: Duration = Duration::from_secs(15); + +/// Safety factor applied to each scaled term of the recovery deadline: a slower +/// heartbeat stretches election and suffix recommit proportionally, and a wider +/// status backstop stretches the ceremony it bounds. 3x reproduces the +/// empirically chosen 15s margin at the shared 5s default (3 x 5s = 15s) and +/// holds that factor as either knob grows. +const RECOVERY_BARRIER_MULTIPLIER: u32 = 3; + +/// How long the post-restart read path waits for the recovered WAL suffix to +/// re-commit before failing loud (retryable 503): the largest of the fixed +/// floor, a `[cluster] heartbeat_timeout`-scaled window, and a +/// `[cluster] view_change_status_timeout`-scaled window. Both knobs feed it +/// because either, raised far past its default, stretches worst-case recovery +/// past the fixed floor; see `await_recovery_barrier` for the read-side wait. +pub(crate) fn recovery_barrier_deadline( + heartbeat: Duration, + view_change_status: Duration, +) -> Duration { + // saturating: neither timeout has a config ceiling, plain `*` panics + heartbeat + .saturating_mul(RECOVERY_BARRIER_MULTIPLIER) + .max(view_change_status.saturating_mul(RECOVERY_BARRIER_MULTIPLIER)) + .max(RECOVERY_BARRIER_DEADLINE_FLOOR) +} + +/// `[cluster] commit_broadcast_interval` in consensus ticks: how often the +/// primary broadcasts its commit point, the cluster's liveness feed. Applied +/// to every consensus group, matching `cluster_heartbeat_ticks`. +pub(crate) fn commit_broadcast_ticks(config: &ServerNgConfig) -> u64 { + duration_to_ticks(config.cluster.commit_broadcast_interval.get_duration()) +} + +/// `[cluster] prepare_retransmit_interval` in consensus ticks: how often the +/// primary retransmits un-acked prepares. Applied to every consensus group, +/// matching `cluster_heartbeat_ticks`. +pub(crate) fn prepare_retransmit_ticks(config: &ServerNgConfig) -> u64 { + duration_to_ticks(config.cluster.prepare_retransmit_interval.get_duration()) +} + +/// `[cluster] view_change_retransmit_interval` in consensus ticks: how often a +/// replica retransmits its `StartViewChange` / `DoViewChange` during a view +/// change. Applied to every consensus group, matching `cluster_heartbeat_ticks`. +pub(crate) fn view_change_retransmit_ticks(config: &ServerNgConfig) -> u64 { + duration_to_ticks( + config + .cluster + .view_change_retransmit_interval + .get_duration(), + ) +} + +/// `[cluster] view_change_status_timeout` in consensus ticks: the stalled +/// view-change backstop before escalating to a fresh election. Applied to every +/// consensus group, matching `cluster_heartbeat_ticks`. +pub(crate) fn view_change_status_ticks(config: &ServerNgConfig) -> u64 { + duration_to_ticks(config.cluster.view_change_status_timeout.get_duration()) +} + +/// `[cluster] request_start_view_retransmit_interval` in consensus ticks: how +/// often a recovering or view-change backup re-requests the current `StartView`. +/// Applied to every consensus group, matching `cluster_heartbeat_ticks`. +pub(crate) fn request_start_view_ticks(config: &ServerNgConfig) -> u64 { + duration_to_ticks( + config + .cluster + .request_start_view_retransmit_interval + .get_duration(), + ) +} + +/// `[cluster] repair_retry_interval` in consensus ticks: how long a stalled +/// journal-repair stream waits before re-requesting its window. Both planes' +/// repair loops share it, so it is applied once per shard (not per consensus +/// group). Clamped to `u32`, the width of the session idle-tick counter. +pub(crate) fn repair_retry_ticks(config: &ServerNgConfig) -> u32 { + u32::try_from(duration_to_ticks( + config.cluster.repair_retry_interval.get_duration(), + )) + .unwrap_or(u32::MAX) +} + +/// Shard 0's half of a metadata recovery: everything [`recover`] produced except the +/// state machine, which every shard receives through the factory bundle. +/// +/// Named rather than a positional tuple: the fields are same-typed `Option`s and +/// `(u64, u128)` pairs that a reorder would silently rebind, and one of them decides +/// what view the replica boots into. +struct RecoveredOwnerState { + journal: PrepareJournal, + snapshot: Option, + last_applied_op: Option, + last_journaled_op: Option, + client_table: ClientTable, + superblock: PingPongSuperblock, + recovered_state: Option, + snapshot_checkpoint: (u64, u128), +} + +/// Rebuild metadata consensus from what recovery read off this replica's own disk. +/// +/// Takes the recovery result, topology and config whole rather than the dozen-plus +/// scalars it needs from them: most were `u64` tick counts, where a misordered +/// argument type-checks and mistunes a timeout silently. +fn restore_metadata_consensus( + owner: &RecoveredOwnerState, + topology: &TcpTopology, + config: &ServerNgConfig, + bus: Rc, +) -> VsrConsensus> { + let journal = &owner.journal; + let replica_count = topology.replica_count; + let recovered_state = owner.recovered_state; + let snapshot_floor = owner + .snapshot + .as_ref() + .map_or(0, IggySnapshot::sequence_number); + let commit_watermark = owner.last_applied_op.unwrap_or(snapshot_floor); + let restored_op = owner.last_journaled_op.unwrap_or(snapshot_floor); + let recovery_deadline = recovery_barrier_deadline( + config.cluster.heartbeat_timeout.get_duration(), + config.cluster.view_change_status_timeout.get_duration(), + ); + let prepare_queue_depth = config.metadata.prepare_queue_depth; + + let mut consensus = VsrConsensus::new( + topology.cluster_id, + topology.self_replica_id, + replica_count, + server_common::sharding::METADATA_CONSENSUS_NAMESPACE, + bus, + // Request queue keeps the stock 2x ratio over the prepare queue + // (32 -> 64 at defaults): buffered requests are cheap relative to + // in-flight prepares and drain as prepares commit. + LocalPipeline::with_capacities(prepare_queue_depth, prepare_queue_depth * 2), + ); + consensus.set_normal_heartbeat_ticks(cluster_heartbeat_ticks(config)); + consensus.set_commit_message_ticks(commit_broadcast_ticks(config)); + consensus.set_prepare_ticks(prepare_retransmit_ticks(config)); + consensus.set_view_change_retransmit_ticks(view_change_retransmit_ticks(config)); + consensus.set_view_change_status_ticks(view_change_status_ticks(config)); + consensus.set_request_start_view_ticks(request_start_view_ticks(config)); + consensus.set_probe_attempts_max(config.cluster.view_probe_attempts_max); + // Fresh random incarnation each boot, so a StartView addressed to a previous + // incarnation still in flight is ignored (`handle_start_view` guard). `| 1` + // guarantees the non-zero the guard treats as set. The deterministic simulator + // overrides this with a seed-derived value bumped per restart. + consensus.set_incarnation(rand::random::() | 1); + + let last_header = journal + .last_op() + .and_then(|op| usize::try_from(op).ok()) + .and_then(|op| journal.header(op).map(|header| *header)); + // View and log_view come from the durable superblock when present. A present but + // unreadable superblock already refused boot in `recover()`, so reaching the + // `else` means it is genuinely absent: a fresh node, or one that took writes but + // never checkpointed or changed view. There, inferring the view from the last WAL + // prepare is safe, since the persist-before-send gate guarantees this replica + // never externalized a view beyond what a re-probe re-derives, and it re-probes + // as a backup below. log_view cannot be inferred and stays 0 until the next + // superblock write. + if let Some(state) = recovered_state { + consensus.set_view(state.view); + consensus.set_log_view(state.log_view); + consensus.mark_superblock_durable(state.view, state.log_view); + } else if let Some(header) = last_header { + consensus.set_view(header.view); + } + + // On a RESTART in a cluster, rejoin as a quorum-invisible backup and + // probe for the current view (`RequestStartView`): the view's primary + // answers with a `StartView`, the replica adopts it as a backup, and + // journal repair fills any WAL gap. A probing replica never resumes + // primaryship -- if this replica IS the current primary-by-index, its + // probe makes the backups elect past it. + // The probe re-broadcasts on its timeout, so it needs no live mesh at + // boot. A FRESH boot keeps the plain init: the cluster needs its view-0 + // primary to exist, and a single-replica cluster has no peer to ask. + // + // Prior life is EITHER a non-empty WAL or a recovered superblock. A view + // change persists without touching the WAL, so a replica that changed + // view before its first metadata write comes back with a non-zero view + // and an empty journal; gating on the WAL alone would `init()` it into + // `Status::Normal` as primary for a view the cluster may have moved past, + // with `ceded_primaryship` false and no probe to correct it. + if replica_count > 1 && (restored_op > 0 || recovered_state.is_some()) { + consensus.init_as_backup(); + consensus.begin_view_probe(); + // Restart in a cluster: replace snapshot-shaped metadata state + // (snapshot + client table) from the live primary the probe finds, + // then journal-repair the tail. If the probe exhausts instead -- + // full-cluster bootstrap, nobody live to fetch from -- the election + // fallback clears the stage and this local recovery stands. + consensus.begin_state_transfer_await(); + } else { + consensus.init(); + } + consensus.sequencer().set_sequence(restored_op); + // A SOLO replica's durable journal head IS its commit point: quorum is + // 1-of-1, so an entry commits the instant it is durable, and the acks + // the cluster ceremony below would wait on cannot topologically exist. + // The embedded watermark is structurally one op stale (the commit point + // is only ever written down inside the NEXT entry), so trusting it solo + // manufactures an "uncommitted" suffix that provably committed and + // wedges the recovery barrier forever. + let commit_watermark = if replica_count == 1 { + restored_op + } else { + commit_watermark + }; + // The commit point is restored from the WAL's embedded watermark (each + // journaled prepare carries the primary's commit at send time), NOT from + // the journal head: journaled does not imply committed, and claiming + // commit for the un-quorum'd tail both risks split-brain on a later view + // change and starves the tail of re-replication (it would live in no + // pipeline). The suffix `(commit_watermark, restored_op]` is re-pipelined + // below when this replica is the recovered view's primary. + // + // TODO(hubcio): the watermark is a lower bound (the last entry stamps + // the commit point as of its send). Persisting an explicit (view, + // commit_op) watermark on the commit path would tighten recovery and + // allow refusing boot on an excessive gap; a backup that recovered a + // LONGER tail than the cluster's primary still needs uncommitted-suffix + // truncation when conflicting ops arrive (message repair milestone). + consensus.restore_commit_state(commit_watermark, commit_watermark); + if let Some(header) = last_header { + consensus.set_last_prepare_checksum(header.checksum); + consensus.observe_prepare_timestamp(header.timestamp); + } + + // The WAL's tail past the watermark is prepared-but-not-provably-committed + // state. Until the cluster confirms it (re-pipelined below on a resumed + // primary; via StartView adoption + the local commit walk on a rejoined + // backup), serving reads would show pre-restart state that clients already + // saw acked -- gate them on the barrier regardless of role. If the suffix + // never re-commits cluster-wide, the read path fails loud with a retryable + // 503 once the paired deadline expires (`await_recovery_barrier`). + if commit_watermark < restored_op { + consensus.set_recovery_barrier(restored_op); + consensus.set_recovery_deadline(recovery_deadline); + } + + // Re-pipeline the prepared-but-uncommitted suffix so the primary's + // retransmit machinery re-replicates it and quorum can (re-)commit it. + // A backup's suffix stays journal-only: the primary's traffic either + // confirms it (re-forward + re-ack path) or supersedes it. + if consensus.is_primary() + && !consensus.has_ceded_primaryship() + && commit_watermark < restored_op + { + info!( + commit_watermark, + restored_op, "re-pipelining recovered uncommitted metadata suffix" + ); + let mut pipeline = consensus.pipeline().borrow_mut(); + #[allow(clippy::cast_possible_truncation)] + for op in (commit_watermark + 1)..=restored_op { + let Some(header) = journal.header(op as usize) else { + warn!( + op, + "recovered journal suffix has a gap; stopping re-pipeline" + ); + break; + }; + let mut entry = PipelineEntry::new(*header); + entry.add_ack(topology.self_replica_id); + pipeline.push(entry); + } + } + + consensus +} + +#[allow(clippy::too_many_arguments)] +async fn load_partition( + config: &ServerNgConfig, + namespace: IggyNamespace, + stats: Arc, + partition_metadata: &Partition, + cluster_id: u128, + self_replica_id: u8, + replica_count: u8, + bus: Rc, +) -> Result>, ServerNgError> { + let stream_id = namespace.stream_id(); + let topic_id = namespace.topic_id(); + let partition_id = namespace.partition_id(); + // Request queue holds 2x the prepare depth (buffered requests drain as + // prepares commit); depth is the per-partition `[partition]` knob. + let prepare_queue_depth = config.partition.prepare_queue_depth; + let mut consensus = VsrConsensus::new( + cluster_id, + self_replica_id, + replica_count, + namespace.inner(), + bus, + LocalPipeline::with_capacities(prepare_queue_depth, prepare_queue_depth * 2), + ); + consensus.set_normal_heartbeat_ticks(cluster_heartbeat_ticks(config)); + consensus.set_commit_message_ticks(commit_broadcast_ticks(config)); + consensus.set_prepare_ticks(prepare_retransmit_ticks(config)); + consensus.set_view_change_retransmit_ticks(view_change_retransmit_ticks(config)); + consensus.set_view_change_status_ticks(view_change_status_ticks(config)); + consensus.set_request_start_view_ticks(request_start_view_ticks(config)); + consensus.set_probe_attempts_max(config.cluster.view_probe_attempts_max); + + // (view, log_view) come from the group's durable superblock when present; + // a present but unverifiable record already refused boot inside + // `open_partition_superblock`. Restored BEFORE choosing how to join, so + // the backup probe below never advertises a view older than the recorded + // one. + let partition_dir = config + .system + .get_partition_path(stream_id, topic_id, partition_id); + let (superblock, recovered_state) = open_partition_superblock( + &partition_dir, + ReplicaIdentity { + cluster: cluster_id, + replica_id: self_replica_id, + replica_count, + }, + ) + .await?; + if let Some(state) = recovered_state.as_ref() { + restore_partition_view(&mut consensus, state); + } + + // A recovered partition lost its journal state with the process: the + // partition journal is in-memory and segments carry no op numbers, so + // this replica cannot know the group's (op, commit) even when the + // superblock restored its view. In a cluster it boots as a + // quorum-invisible backup and probes for the current view + // (`RequestStartView`): the view's primary answers with a `StartView`, + // journal repair fills the rejoin window, and the commit floor settles + // at the serving peer's retention point. The probe re-broadcasts on its + // timeout, so it needs no live mesh at boot. Single-replica groups + // have no peer to ask and keep the plain init. + if replica_count > 1 { + consensus.init_as_backup(); + consensus.begin_view_probe(); + } else { + consensus.init(); + } + + // No prepare-timestamp floor is restored here: the partition consensus + // journal is non-durable today, so there is no persisted head to observe + // (unlike `restore_metadata_consensus`, which observes its restored head). + // When PartitionJournal becomes durable (the milestone named in the + // multi-shard wiring commit body), observe the restored head and the max + // recovered message timestamp here, or an NTP rewind across a restart could + // regress persisted `base_timestamp`. + + let recovered_segments = + load_persisted_segments(config, stream_id, topic_id, partition_id, &stats) + .await + .map_err(|source| { + error!( + stream_id, + topic_id, + partition_id, + error = %source, + "failed to load partition log during server-ng bootstrap" + ); + source + })?; + + let mut partition = IggyPartition::new(stats.clone(), consensus); + partition.set_superblock(superblock, recovered_state.as_ref()); + // Recovered partitions honor the same config-surfaced ring ceilings as the + // fresh-create path (build_partition_fresh). Retention is already off for + // single-replica groups, so this only sizes the multi-replica ring. + partition.log.journal().inner.set_ring_caps( + config.partition.evicted_ring_capacity, + config.partition.evicted_ring_bytes_max.as_bytes_u64(), + ); + partition.set_partition_dir(partition_dir); + // Before the hydrate: the durable record is keyed by incarnation, so a + // `purge.gen` left behind by a previous life of this namespace reads 0. + partition.set_created_revision(partition_metadata.created_revision); + partition.hydrate_applied_purge_generation().await?; + hydrate_partition_log( + &mut partition, + config, + stream_id, + topic_id, + partition_id, + recovered_segments, + ) + .await?; + + let sized_end = partition + .log + .segments() + .iter() + .filter(|segment| segment.size > IggyByteSize::default()) + .map(|segment| segment.end_offset) + .max(); + // An empty chain whose segment is named for a nonzero offset is the + // shape a state-transfer install (or its converge) plants at the group + // frontier after the origin GC'd everything: the file name carries the + // frontier, and re-minting offsets from 0 here would fork this + // replica's batch stamps from the rest of the group after a restart. + let empty_frontier = partition + .log + .segments() + .iter() + .map(|segment| segment.start_offset) + .max() + .filter(|&start| sized_end.is_none() && start > 0); + let current_offset = sized_end.or_else(|| empty_frontier.map(|start| start - 1)); + partition.created_at = partition_metadata.created_at; + partition.recovered_durable_offset = sized_end; + // The OFFSET COUNTER is restored from that file name (above), but the + // `installed_frontier` CLAIM deliberately is not: the claim says "everything + // below me is represented here", and `converge_to_empty_after_failed_install` + // refuses to make it when staged segments were dropped -- yet a converge + // plants exactly the same empty `{frontier:020}.log` a legitimate empty + // install does, so boot provably cannot tell them apart. Re-deriving it here + // would hand the refused claim back: the repair floor stand-in would accept a + // commit floor over ops this replica holds zero bytes for, and the replica + // would pass the serve gate and offer that emptiness onward, making a peer + // unlink its own chain. Leaving it `None` costs one spurious full + // re-transfer on the legitimate empty-install restart; a false caught-up + // claim is not recoverable. A durable home for the frontier (the partition + // superblock already reserves a field) is what would settle it properly. + let counter = current_offset.unwrap_or(0); + partition.offset.store(counter, Ordering::Release); + partition.dirty_offset.store(counter, Ordering::Relaxed); + partition.should_increment_offset = current_offset.is_some(); + partition.stats.set_current_offset(counter); + // The durable frontier is a LOWER BOUND on top of what the segments proved: + // it is the only carrier left when the segments that named the frontier are + // gone (an all-GC'd origin's install, a crash inside the swap window), and + // taking the max means real recovered data always wins. + partition.restore_offset_frontier(recovered_state.as_ref()); + let current_offset = partition.offset.load(Ordering::Acquire); + + configure_consumer_offsets(&mut partition, config, namespace, current_offset)?; + ensure_initial_segment(&mut partition, config, stream_id, topic_id, partition_id).await?; + + Ok(partition) +} + +async fn hydrate_partition_log( + partition: &mut IggyPartition>, + config: &ServerNgConfig, + stream_id: usize, + topic_id: usize, + partition_id: usize, + recovered_segments: Vec, +) -> Result<(), ServerNgError> { + for RecoveredSegment { segment, storage } in recovered_segments { + partition + .log + .add_persisted_segment(segment, storage, None, None); + } + + if let Some(active_index) = partition.log.segments().len().checked_sub(1) { + let storage = &partition.log.storages()[active_index]; + if let ( + Some(messages_reader), + Some(index_reader), + Some(storage_messages_writer), + Some(storage_index_writer), + ) = ( + storage.messages_reader.as_ref(), + storage.index_reader.as_ref(), + storage.messages_writer.as_ref(), + storage.index_writer.as_ref(), + ) { + let index_path = index_reader.path(); + // Share the storage's size counters: the readers bound reads by + // these atomics, so a writer with a private counter persists bytes + // the readers never learn about. + let messages_size_counter = storage_messages_writer.size_counter(); + let index_size_counter = storage_index_writer.size_counter(); + partition.log.messages_writers_mut()[active_index] = Some(Rc::new( + MessagesWriter::new( + &messages_reader.path(), + messages_size_counter, + config.system.partition.enforce_fsync, + true, + ) + .await + .map_err(|source| { + error!( + stream_id, + topic_id, + partition_id, + path = %messages_reader.path(), + error = %source, + "failed to initialize persisted messages writer" + ); + source + })?, + )); + partition.log.index_writers_mut()[active_index] = Some(Rc::new( + IggyIndexWriter::new( + &index_path, + index_size_counter, + config.system.partition.enforce_fsync, + true, + ) + .await + .map_err(|source| { + error!( + stream_id, + topic_id, + partition_id, + path = %index_path, + error = %source, + "failed to initialize persisted sparse index writer" + ); + source + })?, + )); + } + } + + Ok(()) +} + +fn resolve_tcp_topology( + config: &ServerNgConfig, + current_replica_id: Option, +) -> Result { + let default_client_addr = parse_socket_addr("tcp.address", &config.tcp.address)?; + let default_ws_addr = resolve_optional_listener_addr( + config.websocket.enabled, + "websocket.address", + &config.websocket.address, + )?; + let default_quic_addr = + resolve_optional_listener_addr(config.quic.enabled, "quic.address", &config.quic.address)?; + let default_http_addr = + resolve_optional_listener_addr(config.http.enabled, "http.address", &config.http.address)?; + if !config.cluster.enabled { + if let Some(replica_id) = current_replica_id + && replica_id != SHARD_REPLICA_ID + { + return Err(ServerNgError::ReplicaIdRequiresCluster { + supplied: replica_id, + default: SHARD_REPLICA_ID, + }); + } + return Ok(TcpTopology { + cluster_id: auth::cluster_domain_id(&config.cluster.name), + // Keep parity with the current server binary and the integration + // harness: `--replica-id 0` may be passed unconditionally in + // single-node mode; any other id is rejected above so the WAL + // cannot commit under an identity that will later disagree with + // a cluster.nodes[] entry. + self_replica_id: SHARD_REPLICA_ID, + replica_count: 1, + client_listen_addr: default_client_addr, + replica_listen_addr: Some(SocketAddr::new(default_client_addr.ip(), 0)), + ws_listen_addr: default_ws_addr, + quic_listen_addr: default_quic_addr, + http_listen_addr: default_http_addr, + tcp_tls_listen_addr: config.tcp.tls.enabled.then_some(default_client_addr), + peers: Vec::new(), + }); + } + + let self_replica_id = current_replica_id.ok_or(ServerNgError::MissingReplicaId)?; + + let self_node = config + .cluster + .nodes + .iter() + .find(|node| node.replica_id == self_replica_id) + .ok_or(ServerNgError::ClusterNodeNotFound { + replica_id: self_replica_id, + })?; + let replica_count = u8::try_from(config.cluster.nodes.len()).map_err(|_| { + ServerNgError::ClusterReplicaCountTooLarge { + count: config.cluster.nodes.len(), + } + })?; + let ClusterClientAddrs { + client: client_listen_addr, + ws: ws_listen_addr, + quic: quic_listen_addr, + http: http_listen_addr, + } = resolve_cluster_client_addrs( + self_node, + default_client_addr, + default_ws_addr, + default_quic_addr, + default_http_addr, + )?; + let replica_port = self_node + .ports + .tcp_replica + .ok_or(ServerNgError::ClusterPortMissing { + transport: "tcp_replica", + replica_id: self_node.replica_id, + })?; + let replica_listen_addr = Some(socket_addr_from_parts( + "cluster.nodes[*].ports.tcp_replica", + &self_node.ip, + replica_port, + )?); + let peers = resolve_cluster_replica_peers(&config.cluster.nodes, self_replica_id)?; + + Ok(TcpTopology { + cluster_id: auth::cluster_domain_id(&config.cluster.name), + self_replica_id, + replica_count, + client_listen_addr, + replica_listen_addr, + ws_listen_addr, + quic_listen_addr, + http_listen_addr, + tcp_tls_listen_addr: config.tcp.tls.enabled.then_some(client_listen_addr), + peers, + }) +} + +fn resolve_optional_listener_addr( + enabled: bool, + context: &'static str, + address: &str, +) -> Result, ServerNgError> { + if enabled { + return Ok(Some(parse_socket_addr(context, address)?)); + } + Ok(None) +} + +/// Client-facing listener addresses resolved for this cluster node. Each port +/// comes from the node's roster entry; there is no fallback to the top-level +/// listener port, an enabled transport without a roster port refuses to boot. +/// Every transport keeps the bind interface from its own `address` config: the +/// roster ip is advertised, not bound. +struct ClusterClientAddrs { + client: SocketAddr, + ws: Option, + quic: Option, + http: Option, +} + +fn resolve_cluster_client_addrs( + self_node: &configs::ng_cluster::ClusterNodeConfig, + default_tcp_addr: SocketAddr, + default_ws_addr: Option, + default_quic_addr: Option, + default_http_addr: Option, +) -> Result { + let client_port = self_node + .ports + .tcp + .ok_or(ServerNgError::ClusterPortMissing { + transport: "tcp", + replica_id: self_node.replica_id, + })?; + let client = + merge_roster_port_with_bind_ip("tcp", &self_node.ip, default_tcp_addr, client_port); + let ws = resolve_cluster_optional_addr(self_node, "websocket", default_ws_addr, |ports| { + ports.websocket + })?; + let quic = + resolve_cluster_optional_addr(self_node, "quic", default_quic_addr, |ports| ports.quic)?; + let http = + resolve_cluster_optional_addr(self_node, "http", default_http_addr, |ports| ports.http)?; + Ok(ClusterClientAddrs { + client, + ws, + quic, + http, + }) +} + +fn resolve_cluster_optional_addr( + self_node: &configs::ng_cluster::ClusterNodeConfig, + transport: &'static str, + default_addr: Option, + port_selector: impl Fn(&configs::ng_cluster::TransportPorts) -> Option, +) -> Result, ServerNgError> { + let Some(default_addr) = default_addr else { + return Ok(None); + }; + // No fallback to the top-level port: two same-host nodes leaving the same + // transport port unset would race for one socket. Either the roster is + // explicit or the server refuses to boot. + let port = port_selector(&self_node.ports).ok_or(ServerNgError::ClusterPortMissing { + transport, + replica_id: self_node.replica_id, + })?; + Ok(Some(merge_roster_port_with_bind_ip( + transport, + &self_node.ip, + default_addr, + port, + ))) +} + +/// Combine the roster-supplied `port` with the bind interface the transport's +/// own `address` config asked for. +/// +/// The roster ip is what the cluster advertises (metadata, follower-to-primary +/// HTTP forwarding targets); the transport's own `address` decides the bind +/// interface. Merging keeps a loopback-only `127.0.0.1` private and a +/// `0.0.0.0` wide in cluster mode instead of silently rebinding to the roster +/// interface, which would strand every co-located dialer (sidecars, health +/// probes, on-host consumers) on `ECONNREFUSED`. +fn merge_roster_port_with_bind_ip( + transport: &'static str, + roster_ip: &str, + bind_addr: SocketAddr, + port: u16, +) -> SocketAddr { + let listen_addr = SocketAddr::new(bind_addr.ip(), port); + if roster_ip_unreachable_from_bind_addr(roster_ip, listen_addr) { + warn!( + "{transport} listener binds {listen_addr} but the roster advertises {roster_ip}:{port}; \ + peers and clients dialing the advertised endpoint may not reach this node" + ); + } + listen_addr +} + +/// Whether a dialer aiming at the advertised roster ip misses `listen_addr`. An +/// unspecified bind covers every interface, and a roster ip that parses as +/// neither IPv4 nor IPv6 (a DNS name, say) can resolve to the bound interface, +/// so both cases stay quiet. +fn roster_ip_unreachable_from_bind_addr(roster_ip: &str, listen_addr: SocketAddr) -> bool { + !listen_addr.ip().is_unspecified() + && roster_ip + .parse::() + .is_ok_and(|parsed| parsed != listen_addr.ip()) +} + +fn resolve_cluster_replica_peers( + nodes: &[configs::ng_cluster::ClusterNodeConfig], + self_replica_id: u8, +) -> Result, ServerNgError> { + let mut peers = Vec::with_capacity(nodes.len().saturating_sub(1)); + for node in nodes { + if node.replica_id == self_replica_id { + continue; + } + let replica_port = node + .ports + .tcp_replica + .ok_or(ServerNgError::ClusterPortMissing { + transport: "tcp_replica", + replica_id: node.replica_id, + })?; + peers.push(( + node.replica_id, + socket_addr_from_parts("cluster.nodes[*].ports.tcp_replica", &node.ip, replica_port)?, + )); + } + Ok(peers) +} + +async fn start_tcp_runtime( + shard: &Rc, + config: &ServerNgConfig, + topology: &TcpTopology, + accepted_replica: AcceptedReplicaFn, + dialed_replica: DialedReplicaFn, + accepted_clients: LocalClientAcceptFns, +) -> Result<(), ServerNgError> { + if config.tcp.enabled && !config.tcp.tls.enabled { + start_via_replica_io( + shard, + config, + topology, + accepted_replica, + dialed_replica, + accepted_clients, + ) + .await?; + } else { + start_manual_runtime( + shard, + config, + topology, + accepted_replica, + dialed_replica, + accepted_clients, + ) + .await?; + } + + // HTTP is served over TCP but sits outside the replica_io / manual client + // reactor, so it binds independently. Shard-0 gating comes from the sole + // caller of this function. + if let Some(http_addr) = topology.http_listen_addr { + let self_ports = configs::ng_cluster::TransportPorts { + tcp: config + .tcp + .enabled + .then(|| topology.client_listen_addr.port()), + quic: topology.quic_listen_addr.map(|addr| addr.port()), + websocket: topology.ws_listen_addr.map(|addr| addr.port()), + ..Default::default() + }; + http::start( + shard, + http_addr, + &config.http, + config.metadata.clients_table_max, + config.personal_access_token.max_tokens_per_user, + &config.cluster, + Arc::clone(&config.system), + self_ports, + ) + .await?; + } + + Ok(()) +} + +// ws/wss bindings intentionally mirror the transport names (same convention as +// `replica_io::start_on_shard_zero`). +#[allow(clippy::similar_names)] +async fn start_via_replica_io( + shard: &Rc, + config: &ServerNgConfig, + topology: &TcpTopology, + accepted_replica: AcceptedReplicaFn, + dialed_replica: DialedReplicaFn, + accepted_clients: LocalClientAcceptFns, +) -> Result<(), ServerNgError> { + let replica_addr = topology + .replica_listen_addr + .expect("topology must include replica listener address"); + let quic_credentials = topology + .quic_listen_addr + .is_some() + .then(|| load_quic_server_credentials(config)) + .transpose()?; + let tcp_tls_credentials = topology + .tcp_tls_listen_addr + .is_some() + .then(|| load_tcp_tls_server_credentials(config)) + .transpose()?; + // `websocket.tls.enabled` upgrades the websocket address to a WSS + // listener; the plain-WS listener must NOT also bind it (one port, one + // handshake kind -- a plain upgrade parser fed a TLS ClientHello rejects + // every connection with an httparse error). + let wss_enabled = config.websocket.tls.enabled; + let ws_listen_addr = (!wss_enabled).then_some(topology.ws_listen_addr).flatten(); + let wss_listen_addr = wss_enabled.then_some(topology.ws_listen_addr).flatten(); + let wss_credentials = wss_listen_addr + .is_some() + .then(|| load_wss_server_credentials(config)) + .transpose()?; + + let LocalClientAcceptFns { + tcp, + ws, + quic, + tcp_tls, + wss, + } = accepted_clients; + + let bound = replica_io::start_on_shard_zero( + &shard.bus, + replica_addr, + topology.client_listen_addr, + ws_listen_addr, + topology.quic_listen_addr, + quic_credentials, + topology.tcp_tls_listen_addr, + tcp_tls_credentials, + wss_listen_addr, + wss_credentials, + topology.self_replica_id, + topology.peers.clone(), + accepted_replica, + dialed_replica, + tcp, + ws_listen_addr.map(|_| ws), + topology.quic_listen_addr.map(|_| quic), + topology.tcp_tls_listen_addr.map(|_| tcp_tls), + wss_listen_addr.map(|_| wss), + shard.bus.config().reconnect_period, + ) + .await + .map_err(|source| { + error!( + replica_addr = %replica_addr, + client_addr = %topology.client_listen_addr, + error = %source, + "failed to start server-ng listeners via replica_io" + ); + source + })?; + let Some(bound) = bound else { + return Ok(()); + }; + + write_current_config( + config, + Some(topology.self_replica_id), + Some(bound.client), + config.cluster.enabled.then_some(bound.replica), + bound.tcp_tls, + bound.quic, + // The WSS listener occupies the configured websocket address slot. + bound.wss.or(bound.ws), + ) + .await?; + if config.cluster.enabled { + info!( + shard = shard.id, + replica = %bound.replica, + tcp = %bound.client, + tcp_tls = ?bound.tcp_tls, + ws = ?bound.ws, + quic = ?bound.quic, + "server-ng listeners started" + ); + } else { + info!( + shard = shard.id, + tcp = %bound.client, + tcp_tls = ?bound.tcp_tls, + ws = ?bound.ws, + quic = ?bound.quic, + "server-ng client listeners started" + ); + } + + Ok(()) +} + +async fn start_manual_runtime( + shard: &Rc, + config: &ServerNgConfig, + topology: &TcpTopology, + accepted_replica: AcceptedReplicaFn, + dialed_replica: DialedReplicaFn, + accepted_clients: LocalClientAcceptFns, +) -> Result<(), ServerNgError> { + let bound_replica = if config.cluster.enabled { + let replica_addr = topology + .replica_listen_addr + .expect("cluster-enabled topology must include replica listener address"); + let (replica_listener, bound_addr) = + replica_listener::bind(replica_addr) + .await + .map_err(|source| { + error!( + replica_addr = %replica_addr, + error = %source, + "failed to bind replica listener" + ); + source + })?; + let token = shard.bus.token(); + let replica_handle = compio::runtime::spawn(async move { + replica_listener::run(replica_listener, token, accepted_replica).await; + }); + shard.bus.track_background(replica_handle); + connector::start( + &shard.bus, + topology.self_replica_id, + topology.peers.clone(), + dialed_replica, + shard.bus.config().reconnect_period, + ) + .await; + Some(bound_addr) + } else { + None + }; + + let bound_clients = start_client_listeners(shard, config, topology, &accepted_clients).await?; + write_current_config( + config, + Some(topology.self_replica_id), + bound_clients.tcp, + bound_replica, + bound_clients.tcp_tls, + bound_clients.quic, + bound_clients.ws, + ) + .await?; + + if config.cluster.enabled { + info!( + shard = shard.id, + replica = ?bound_replica, + tcp = ?bound_clients.tcp, + tcp_tls = ?bound_clients.tcp_tls, + ws = ?bound_clients.ws, + quic = ?bound_clients.quic, + "server-ng listeners started" + ); + } else { + info!( + shard = shard.id, + tcp = ?bound_clients.tcp, + tcp_tls = ?bound_clients.tcp_tls, + ws = ?bound_clients.ws, + quic = ?bound_clients.quic, + "server-ng client listeners started" + ); + } + + Ok(()) +} + +fn ensure_default_root_user(mux_stm: &ServerNgMuxStateMachine) { + if !mux_stm.users().read(|users| users.items.is_empty()) { + return; + } + + let (username, password_hash) = create_root_credentials(); + mux_stm.users().ensure_root_user(&username, &password_hash); +} + +/// Resolve the root user credentials from `IGGY_ROOT_USERNAME` / +/// `IGGY_ROOT_PASSWORD`, falling back to the default username with a +/// generated password (printed to stdout, mirroring the legacy server). +/// +/// Returns `(username, password_hash)`; the plaintext password never +/// leaves this function. +fn create_root_credentials() -> (String, String) { + let mut username = env::var(IGGY_ROOT_USERNAME_ENV); + let mut password = env::var(IGGY_ROOT_PASSWORD_ENV); + assert_eq!( + username.is_ok(), + password.is_ok(), + "When providing the custom root user credentials, both username and password must be set." + ); + if username.is_ok() && password.is_ok() { + info!("Using the custom root user credentials."); + } else { + info!("Using the default root user credentials..."); + username = Ok(DEFAULT_ROOT_USERNAME.to_string()); + let generated_password = crypto::generate_secret(20..40); + println!("Generated root user password: {generated_password}"); + password = Ok(generated_password); + } + + let username = username.expect("Root username is not set."); + let password = password.expect("Root password is not set."); + assert!( + !username.is_empty() && !password.is_empty(), + "Root user credentials cannot be empty." + ); + assert!( + username.len() >= MIN_USERNAME_LENGTH, + "Root username is too short." + ); + assert!( + username.len() <= MAX_USERNAME_LENGTH, + "Root username is too long." + ); + assert!( + password.len() >= MIN_PASSWORD_LENGTH, + "Root password is too short." + ); + assert!( + password.len() <= MAX_PASSWORD_LENGTH, + "Root password is too long." + ); + + (username, crypto::hash_password(&password)) +} + +fn validate_cluster_root_bootstrap( + config: &ServerNgConfig, + mux_stm: &ServerNgMuxStateMachine, +) -> Result<(), ServerNgError> { + if !config.cluster.enabled || !mux_stm.users().read(|users| users.items.is_empty()) { + return Ok(()); + } + + if env::var(IGGY_ROOT_USERNAME_ENV).is_ok() && env::var(IGGY_ROOT_PASSWORD_ENV).is_ok() { + return Ok(()); + } + + Err(ServerNgError::ClusterRootCredentialsRequired { + username_env: IGGY_ROOT_USERNAME_ENV, + password_env: IGGY_ROOT_PASSWORD_ENV, + }) +} + +/// Replica delegation callbacks for shard 0's listener and connector. +/// +/// Inbound: acquire a slot in the shard-0-global in-flight handshake cap +/// (drop the connection when full), then blind-delegate the raw fd +/// through the coordinator's round-robin. The fd lands on the target +/// shard's inbox as a [`shard::LifecycleFrame::ReplicaInboundSetup`] +/// frame; the owning shard runs the acceptor handshake and acks the +/// slot back. A failed delegation releases the slot immediately. +/// +/// Outbound: delegate the dialed fd as +/// [`shard::LifecycleFrame::ReplicaOutboundSetup`] and mark the peer +/// dial-pending so the reconnect sweep skips it until the owning +/// shard's handshake outcome arrives (or the entry expires). +fn make_replica_delegation_fns( + coord: Rc, + bus: &Rc, +) -> (AcceptedReplicaFn, DialedReplicaFn) { + let inbound_bus = Rc::clone(bus); + let inbound_coord = Rc::clone(&coord); + let accepted: AcceptedReplicaFn = Rc::new(move |stream| { + let Some(slot) = inbound_bus.try_acquire_replica_handshake_slot() else { + warn!( + cap = MAX_INFLIGHT_REPLICA_HANDSHAKES, + "replica handshake in-flight cap reached; dropping inbound" + ); + return; + }; + match inbound_coord.delegate_replica_inbound(stream, slot) { + Ok(target) => { + info!(slot, target, "inbound replica connection delegated"); + } + Err(error) => { + inbound_bus.release_replica_handshake_slot(slot); + warn!( + error = ?error, + "delegate_replica_inbound failed; dropping inbound replica connection" + ); + } + } + }); + + let outbound_bus = Rc::clone(bus); + let dialed: DialedReplicaFn = + Rc::new( + move |stream, peer_id| match coord.delegate_replica_outbound(stream, peer_id) { + Ok(target) => { + outbound_bus.mark_dial_pending(peer_id); + info!(peer_id, target, "outbound replica connection delegated"); + } + Err(error) => { + warn!( + peer_id, + error = ?error, + "delegate_replica_outbound failed; dropping dialed replica connection" + ); + } + }, + ); + + (accepted, dialed) +} + +/// Shard-0 client accept callbacks. TCP and WS clients are delegated via +/// the coordinator (round-robin to peer shards); QUIC and TCP-TLS install +/// locally on shard 0 because their per-connection state is not portable +/// across shards (`compio_quic` endpoint binds one UDP socket; rustls TLS +/// state ties to the post-handshake reactor). +// ws/wss bindings intentionally mirror the transport names (same convention as +// `replica_io::start_on_shard_zero`). +#[allow(clippy::similar_names)] +fn make_shard_zero_client_accept_fns( + coord: Rc, + bus: &Rc, + on_request: RequestHandler, +) -> LocalClientAcceptFns { + let quic_bus = Rc::clone(bus); + let tcp_tls_bus = Rc::clone(bus); + let wss_bus = Rc::clone(bus); + let quic_request = on_request.clone(); + let wss_request = on_request.clone(); + let tcp_tls_request = on_request; + + let tcp_coord = Rc::clone(&coord); + let tcp = Rc::new(move |stream| match tcp_coord.delegate_client(stream) { + Ok(client_id) => info!(client_id, "TCP client delegated"), + Err(error) => warn!(error = ?error, "delegate_client failed; dropping TCP client"), + }); + + let ws_coord = Rc::clone(&coord); + let ws = Rc::new(move |stream| match ws_coord.delegate_ws_client(stream) { + Ok(client_id) => info!(client_id, "WS client delegated"), + Err(error) => warn!(error = ?error, "delegate_ws_client failed; dropping WS client"), + }); + + // QUIC and TCP-TLS terminate locally on shard 0 but mint their client + // ids through the coordinator's `client_seq`, the same counter the + // delegated TCP/WS path uses. A separate counter here would let a + // shard-0-local id collide with a delegated id that round-robined to + // shard 0 (both encode target shard 0) in shard 0's connection + // registry. + let quic_coord = Rc::clone(&coord); + let quic = Rc::new(move |accepted: message_bus::AcceptedQuicConn| { + let meta = mint_client_meta(&quic_coord, accepted.peer_addr(), ClientTransportKind::Quic); + installer::install_client_quic(&quic_bus, meta, accepted, quic_request.clone()); + }); + + let tcp_tls_coord = Rc::clone(&coord); + let tcp_tls = Rc::new(move |stream, tls_config| { + let Some(meta) = + client_meta_from_stream(&stream, &tcp_tls_coord, ClientTransportKind::TcpTls) + else { + return; + }; + installer::install_client_tcp_tls( + &tcp_tls_bus, + meta, + stream, + tls_config, + tcp_tls_request.clone(), + ); + }); + + // WSS terminates locally on shard 0 like TCP-TLS (rustls state is not + // serialisable across the delegate path), minting ids through the same + // coordinator counter. + let wss_coord = coord; + let wss = Rc::new(move |stream, tls_config| { + let Some(meta) = client_meta_from_stream(&stream, &wss_coord, ClientTransportKind::Wss) + else { + return; + }; + installer::install_client_wss(&wss_bus, meta, stream, tls_config, wss_request.clone()); + }); + + LocalClientAcceptFns { + tcp, + ws, + quic, + tcp_tls, + wss, + } +} + +fn client_meta_from_stream( + stream: &compio::net::TcpStream, + coord: &shard::coordinator::ShardZeroCoordinator, + transport: ClientTransportKind, +) -> Option { + let peer_addr = match stream.peer_addr() { + Ok(peer_addr) => peer_addr, + Err(error) => { + warn!(error = %error, "dropping accepted client with unknown peer address"); + return None; + } + }; + Some(mint_client_meta(coord, peer_addr, transport)) +} + +fn mint_client_meta( + coord: &shard::coordinator::ShardZeroCoordinator, + peer_addr: SocketAddr, + transport: ClientTransportKind, +) -> ClientConnMeta { + ClientConnMeta::new(coord.mint_shard_zero_client_id(), peer_addr, transport) +} + +async fn start_client_listeners( + shard: &Rc, + config: &ServerNgConfig, + topology: &TcpTopology, + accepted_clients: &LocalClientAcceptFns, +) -> Result { + let mut bound = BoundClientListeners::default(); + + if config.tcp.enabled && !config.tcp.tls.enabled { + let (listener, bound_addr) = client_listener::tcp::bind(topology.client_listen_addr) + .await + .map_err(|source| { + error!( + addr = %topology.client_listen_addr, + error = %source, + "failed to bind TCP client listener" + ); + source + })?; + let token = shard.bus.token(); + let accepted_client = accepted_clients.tcp.clone(); + let client_handle = compio::runtime::spawn(async move { + client_listener::tcp::run(listener, token, accepted_client).await; + }); + shard.bus.track_background(client_handle); + bound.tcp = Some(bound_addr); + } + + if let Some(ws_addr) = topology.ws_listen_addr { + bound.ws = Some(start_websocket_listener(shard, config, ws_addr, accepted_clients).await?); + } + + if let Some(quic_addr) = topology.quic_listen_addr { + install_default_crypto_provider(); + let credentials = load_quic_server_credentials(config)?; + let server_config = server_config_with_cert( + credentials.cert_chain, + credentials.key_der, + &shard.bus.config().quic, + ) + .map_err(|e| { + let source = + iggy_common::IggyError::IoError(format!("QUIC server config build failed: {e}")); + error!(addr = %quic_addr, error = %source, "failed to build QUIC server config"); + source + })?; + let (endpoint, bound_addr) = client_listener::quic::bind(quic_addr, server_config) + .map_err(|source| { + error!(addr = %quic_addr, error = %source, "failed to bind QUIC listener"); + source + })?; + let token = shard.bus.token(); + let handshake_grace = shard.bus.config().handshake_grace; + let accepted_quic = accepted_clients.quic.clone(); + let quic_handle = compio::runtime::spawn(async move { + client_listener::quic::run(endpoint, token, accepted_quic, handshake_grace).await; + }); + shard.bus.track_background(quic_handle); + bound.quic = Some(bound_addr); + } + + if config.tcp.enabled && config.tcp.tls.enabled { + let credentials = load_tcp_tls_server_credentials(config)?; + let (listener, tls_config, bound_addr) = + client_listener::tcp_tls::bind(topology.client_listen_addr, credentials).map_err( + |source| { + error!( + addr = %topology.client_listen_addr, + error = %source, + "failed to bind TCP TLS listener" + ); + source + }, + )?; + let token = shard.bus.token(); + let accepted_tls = accepted_clients.tcp_tls.clone(); + let tls_handle = compio::runtime::spawn(async move { + client_listener::tcp_tls::run(listener, tls_config, token, accepted_tls).await; + }); + shard.bus.track_background(tls_handle); + bound.tcp_tls = Some(bound_addr); + } + + Ok(bound) +} + +/// Build the replica auth context from cluster config. Returns `None` when the +/// cluster or replica auth is disabled, keeping the handshake in legacy mode. +/// Only the derived MAC keys are carried onward in [`ReplicaAuth`]; the raw +/// secrets (masked in config logs via `config_env(secret)`) are read here only +/// to derive them. A non-empty `previous_shared_secret` opens the verify-only +/// rotation acceptance window (see the [`ReplicaAuth`] rustdoc for the rolling +/// rotation procedure). `ClusterConfig::validate` guarantees a non-empty +/// secret whenever both `cluster.enabled` and `cluster.auth.enabled` are set +/// (validate early-returns `Ok` while `cluster.enabled` is false). +fn load_replica_auth(config: &ServerNgConfig) -> Option { + if !config.cluster.enabled || !config.cluster.auth.enabled { + return None; + } + let auth = ReplicaAuth::new(config.cluster.auth.shared_secret.as_bytes()); + let previous_shared_secret = &config.cluster.auth.previous_shared_secret; + if previous_shared_secret.is_empty() { + return Some(auth); + } + Some(auth.with_previous_secret(previous_shared_secret.as_bytes())) +} + +/// Build the replica TLS context from cluster config. Returns `None` when +/// the cluster or replica TLS is disabled. Every shard calls this once at +/// boot: CA mode re-reads the same PEM files per shard; self-signed mode +/// mints a per-shard throwaway certificate. Neither mode carries client +/// certificates, so TLS authenticates the acceptor only; peer +/// authentication comes from the PSK handshake (`ClusterConfig::validate` +/// enforces `cluster.auth.enabled` whenever `cluster.tls.enabled`). +/// +/// Both rustls configs are TLS 1.3 only with the [`REPLICA_ALPN`] +/// protocol pinned. The dialer's SNI / certificate-verify name for each +/// peer is the roster entry's `ip` field (a hostname or IP literal, the +/// same string the connector dials). +fn load_replica_tls_ctx( + config: &ServerNgConfig, + topology: &TcpTopology, +) -> Result, ServerNgError> { + let tls = &config.cluster.tls; + if !config.cluster.enabled || !tls.enabled { + return Ok(None); + } + install_default_crypto_provider(); + let credential_error = |source: std::io::Error| ServerNgError::ListenerCredentials { + transport: "cluster.tls", + source, + }; + + let credentials = if tls.self_signed { + let san = config + .cluster + .nodes + .iter() + .find(|node| node.replica_id == topology.self_replica_id) + .map(|node| node.ip.as_str()) + .ok_or_else(|| { + credential_error(std::io::Error::other(format!( + "replica id {} not present in cluster.nodes", + topology.self_replica_id + ))) + })?; + let (cert_chain, key_der) = server_common::generate_self_signed_certificate(san) + .map_err(|error| credential_error(std::io::Error::other(error.to_string())))?; + TlsServerCredentials { + cert_chain, + key_der, + } + } else { + load_pem(Path::new(&tls.cert_file), Path::new(&tls.key_file)).map_err(credential_error)? + }; + + let mut server = + rustls::ServerConfig::builder_with_protocol_versions(&[&rustls::version::TLS13]) + .with_no_client_auth() + .with_single_cert(credentials.cert_chain, credentials.key_der) + .map_err(|error| { + credential_error(std::io::Error::other(format!( + "replica TLS server config rejected credentials: {error}" + ))) + })?; + server.alpn_protocols = vec![REPLICA_ALPN.to_vec()]; + + let client_builder = + rustls::ClientConfig::builder_with_protocol_versions(&[&rustls::version::TLS13]); + let mut client = if tls.self_signed { + client_builder + .dangerous() + .with_custom_certificate_verifier(Arc::new(AcceptAnyServerCert)) + .with_no_client_auth() + } else { + let roots = load_ca_pem(Path::new(&tls.ca_file)).map_err(credential_error)?; + client_builder + .with_root_certificates(Arc::new(roots)) + .with_no_client_auth() + }; + client.alpn_protocols = vec![REPLICA_ALPN.to_vec()]; + + // Keyed by replica id, never by roster position: sparse ids (dynamic + // replica join) would make a positional lookup verify against another + // peer's SNI name. + let peer_names = config + .cluster + .nodes + .iter() + .map(|node| { + let name = ServerName::try_from(node.ip.clone()).map_err(|error| { + credential_error(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!( + "cluster node '{}' ip '{}' is not a valid TLS server name: {error}", + node.name, node.ip + ), + )) + })?; + Ok((node.replica_id, name)) + }) + .collect::, ServerNgError>>()?; + + Ok(Some(ReplicaTlsCtx { + server: Arc::new(server), + client: Arc::new(client), + peer_names, + })) +} + +fn load_tcp_tls_server_credentials( + config: &ServerNgConfig, +) -> Result { + let tls = &config.tcp.tls; + if tls.self_signed && !Path::new(&tls.cert_file).exists() { + return Ok(self_signed_for_loopback()); + } + + load_pem(Path::new(&tls.cert_file), Path::new(&tls.key_file)).map_err(|source| { + ServerNgError::ListenerCredentials { + transport: "tcp.tls", + source, + } + }) +} + +/// Bind the websocket client listener on `ws_addr`: WSS when +/// `websocket.tls.enabled` (the plain-WS accept loop must not also bind the +/// port -- a plain upgrade parser fed a TLS `ClientHello` rejects every +/// connection with an httparse error), plain WS otherwise. +async fn start_websocket_listener( + shard: &Rc, + config: &ServerNgConfig, + ws_addr: SocketAddr, + accepted_clients: &LocalClientAcceptFns, +) -> Result { + if config.websocket.tls.enabled { + let credentials = load_wss_server_credentials(config)?; + let (listener, tls_config, bound_addr) = client_listener::wss::bind(ws_addr, credentials) + .map_err(|source| { + error!(addr = %ws_addr, error = %source, "failed to bind WSS listener"); + source + })?; + let token = shard.bus.token(); + let accepted_wss = accepted_clients.wss.clone(); + let wss_handle = compio::runtime::spawn(async move { + client_listener::wss::run(listener, tls_config, token, accepted_wss).await; + }); + shard.bus.track_background(wss_handle); + Ok(bound_addr) + } else { + let (listener, bound_addr) = + client_listener::ws::bind(ws_addr).await.map_err(|source| { + error!(addr = %ws_addr, error = %source, "failed to bind websocket listener"); + source + })?; + let token = shard.bus.token(); + let accepted_ws = accepted_clients.ws.clone(); + let ws_handle = compio::runtime::spawn(async move { + client_listener::ws::run(listener, token, accepted_ws).await; + }); + shard.bus.track_background(ws_handle); + Ok(bound_addr) + } +} + +fn load_wss_server_credentials( + config: &ServerNgConfig, +) -> Result { + let tls = &config.websocket.tls; + if tls.self_signed && !Path::new(&tls.cert_file).exists() { + return Ok(self_signed_for_loopback()); + } + + load_pem(Path::new(&tls.cert_file), Path::new(&tls.key_file)).map_err(|source| { + ServerNgError::ListenerCredentials { + transport: "websocket.tls", + source, + } + }) +} + +fn load_quic_server_credentials( + config: &ServerNgConfig, +) -> Result { + let certificate = &config.quic.certificate; + if certificate.self_signed { + let (cert_chain, key_der) = server_common::generate_self_signed_certificate("localhost") + .map_err(|error| ServerNgError::ListenerCredentials { + transport: "quic", + source: std::io::Error::other(error.to_string()), + })?; + return Ok(replica_io::QuicServerCredentials { + cert_chain, + key_der, + }); + } + + let credentials = load_pem( + Path::new(&certificate.cert_file), + Path::new(&certificate.key_file), + ) + .map_err(|source| ServerNgError::ListenerCredentials { + transport: "quic", + source, + })?; + Ok(replica_io::QuicServerCredentials { + cert_chain: credentials.cert_chain, + key_der: credentials.key_der, + }) +} + +fn parse_socket_addr(context: &'static str, address: &str) -> Result { + address + .parse() + .map_err(|source| ServerNgError::SocketAddressParse { + context, + address: address.to_string(), + source, + }) +} + +fn socket_addr_from_parts( + context: &'static str, + host: &str, + port: u16, +) -> Result { + let ip = host + .parse::() + .map_err(|source| ServerNgError::SocketAddressParse { + context, + address: format!("{host}:{port}"), + source, + })?; + Ok(SocketAddr::new(ip, port)) +} + +/// Build the closure that broadcasts a +/// [`LifecycleFrame::MetadataCommitTick`] to every shard's inbox after a +/// partition-shaped metadata operation commits on shard 0. +/// +/// The receiver-side partition reconciliation loop listens for these +/// wake-ups; coalescing is intentional, so `Full` is recorded as a metric +/// and dropped (the periodic tick recovers). Installed via +/// [`metadata::IggyMetadata::set_commit_notifier`] on shard 0 only, the +/// sole writer of the metadata state machine. +fn make_metadata_commit_notifier( + senders: Vec, + metrics: ShardMetrics, +) -> metadata::CommitNotifier { + Rc::new(move |operation: Operation| { + if !operation_triggers_partition_reconcile(operation) { + return; + } + for sender in &senders { + let frame = ShardFrame::lifecycle(LifecycleFrame::MetadataCommitTick); + match sender.try_send(frame) { + Ok(()) => {} + Err(crossfire::TrySendError::Full(_)) => { + metrics.record_frame_drop( + frame_drop_variant::METADATA_COMMIT_TICK, + frame_drop_reason::FULL, + ); + } + Err(crossfire::TrySendError::Disconnected(_)) => { + metrics.record_frame_drop( + frame_drop_variant::METADATA_COMMIT_TICK, + frame_drop_reason::DISCONNECTED, + ); + } + } + } + }) +} + +/// Filter at the broadcast site, keeping unrelated ops off the SDK reply +/// path. Any new partition-shape op must be added here. +/// +/// The bare `CreateTopic` / `CreatePartitions` arms are unreachable: the +/// leader's prepare-builder in `IggyMetadata` rewrites both into their +/// `*WithAssignments` form, stamping each partition's `consensus_group_id` +/// before journaling, so a committed prepare only ever carries the +/// assignment-bearing variant. Kept as defense-in-depth against a future +/// commit path that emits a bare op. +/// +/// "Partition-shape" is not only the partition SET: the purge and truncate +/// ops leave the set intact but advance per-partition state (purge +/// generation, delete watermark) that only the reconciler enforces on disk. +/// Omitting them defers the on-disk effect to the periodic safety tick, +/// stretching a purge's client-visible tail to a full +/// `reconcile_periodic_interval`. `DeleteSegments` is absent by design: the +/// leader rewrites it into `TruncatePartition` before journaling, so no +/// commit ever carries it. +const fn operation_triggers_partition_reconcile(op: Operation) -> bool { + matches!( + op, + Operation::CreateTopic + | Operation::CreateTopicWithAssignments + | Operation::CreatePartitions + | Operation::CreatePartitionsWithAssignments + | Operation::DeleteTopic + | Operation::DeleteStream + | Operation::DeletePartitions + | Operation::PurgeStream + | Operation::PurgeTopic + | Operation::TruncatePartition + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn default_cluster_heartbeat_timeout_matches_consensus_constant() { + // The config default lives in core/server-ng/config.toml (a string, + // so no static assert can pin it); keep it in lockstep with the + // built-in the simulator and un-configured replicas run on. + let config_default = configs::ng_cluster::ClusterConfig::default() + .heartbeat_timeout + .get_duration() + .as_millis(); + let built_in = u128::from(consensus::TimeoutManager::NORMAL_HEARTBEAT_TICKS) + * shard::CONSENSUS_TICK_INTERVAL.as_millis(); + assert_eq!( + config_default, built_in, + "[cluster] heartbeat_timeout default drifted from \ + TimeoutManager::NORMAL_HEARTBEAT_TICKS" + ); + } + + #[test] + fn reconciler_driven_ops_broadcast_a_commit_tick() { + // These commit without touching the partition set, so nothing else + // signals the reconciler: `reconcile_partition_purges` and + // `reconcile_segment_truncations` are the only code that turns them + // into on-disk effect, and they run only when a pass runs. Dropping + // one from the filter silently downgrades it to the periodic tick. + for op in [ + Operation::PurgeStream, + Operation::PurgeTopic, + Operation::TruncatePartition, + ] { + assert!( + operation_triggers_partition_reconcile(op), + "{op:?} is enforced by the reconciler and must wake it on commit" + ); + } + assert!( + !operation_triggers_partition_reconcile(Operation::CreateUser), + "ops with no partition-shape effect must stay off the broadcast" + ); + } + + #[test] + fn recovery_barrier_deadline_holds_the_floor_for_small_heartbeats() { + // Below the 5s default the heartbeat-independent recovery term (~7s of + // ViewChangeStatus backstop plus ceremony) dominates, so the floor + // governs however small the heartbeat is; 3 x 5s lands exactly on it. + // A default-sized status backstop stays on the floor, not above it. + assert_eq!( + recovery_barrier_deadline(Duration::from_secs(1), Duration::from_secs(5)), + RECOVERY_BARRIER_DEADLINE_FLOOR + ); + assert_eq!( + recovery_barrier_deadline(Duration::from_secs(5), Duration::from_secs(5)), + RECOVERY_BARRIER_DEADLINE_FLOOR + ); + } + + #[test] + fn recovery_barrier_deadline_scales_past_the_floor_for_large_heartbeats() { + // Once 3 x heartbeat clears the floor the scaled window governs, so a + // slow-heartbeat cluster is not failed 503 before its longer recovery + // can finish. A default-sized status backstop stays under it. + assert_eq!( + recovery_barrier_deadline(Duration::from_secs(10), Duration::from_secs(5)), + Duration::from_secs(30) + ); + assert_eq!( + recovery_barrier_deadline(Duration::from_secs(15), Duration::from_secs(5)), + Duration::from_secs(45) + ); + } + + #[test] + fn recovery_barrier_deadline_scales_with_the_status_backstop() { + // A raised view-change status backstop stretches worst-case recovery + // even when the heartbeat stays fast, so the deadline must track it or + // post-restart reads 503 before a slow election settles. + assert_eq!( + recovery_barrier_deadline(Duration::from_secs(1), Duration::from_secs(10)), + Duration::from_secs(30) + ); + } + + #[test] + fn recovery_barrier_deadline_at_config_defaults_matches_the_floor() { + // Folding the status term in must not move the stock deadline: at the + // shared 5s defaults each scaled term lands exactly on the 15s floor, + // so an un-tuned cluster keeps its pre-existing recovery window. + let cluster = configs::ng_cluster::ClusterConfig::default(); + assert_eq!( + recovery_barrier_deadline( + cluster.heartbeat_timeout.get_duration(), + cluster.view_change_status_timeout.get_duration(), + ), + RECOVERY_BARRIER_DEADLINE_FLOOR + ); + } + + #[test] + fn recovery_barrier_deadline_saturates_instead_of_panicking() { + // Neither timeout has a config ceiling, so both multiplies must + // saturate rather than abort boot on an absurd parseable value. + assert_eq!( + recovery_barrier_deadline(Duration::MAX, Duration::from_secs(5)), + Duration::MAX + ); + assert_eq!( + recovery_barrier_deadline(Duration::from_secs(5), Duration::MAX), + Duration::MAX + ); + } + + #[test] + fn default_commit_broadcast_interval_matches_consensus_constant() { + // The config default lives in core/server-ng/config.toml (a string, + // so no static assert can pin it); keep it in lockstep with the + // built-in the simulator and un-configured replicas run on. + let config_default = configs::ng_cluster::ClusterConfig::default() + .commit_broadcast_interval + .get_duration() + .as_millis(); + let built_in = u128::from(consensus::TimeoutManager::COMMIT_MESSAGE_TICKS) + * shard::CONSENSUS_TICK_INTERVAL.as_millis(); + assert_eq!( + config_default, built_in, + "[cluster] commit_broadcast_interval default drifted from \ + TimeoutManager::COMMIT_MESSAGE_TICKS" + ); + } + + #[test] + fn default_prepare_retransmit_interval_matches_consensus_constant() { + // The config default lives in core/server-ng/config.toml (a string, + // so no static assert can pin it); keep it in lockstep with the + // built-in the simulator and un-configured replicas run on. + let config_default = configs::ng_cluster::ClusterConfig::default() + .prepare_retransmit_interval + .get_duration() + .as_millis(); + let built_in = u128::from(consensus::TimeoutManager::PREPARE_TICKS) + * shard::CONSENSUS_TICK_INTERVAL.as_millis(); + assert_eq!( + config_default, built_in, + "[cluster] prepare_retransmit_interval default drifted from \ + TimeoutManager::PREPARE_TICKS" + ); + } + + #[test] + fn default_partition_prepare_queue_depth_matches_consensus_constant() { + // The config default lives in core/server-ng/config.toml and flows + // through PartitionConfig::default(); keep the embedded value in + // lockstep with the pipeline depth LocalPipeline::new() (the simulator + // and tests) runs on, so a default deployment is byte-identical. + let config_default = configs::ng_partition::PartitionConfig::default().prepare_queue_depth; + assert_eq!( + config_default, + consensus::PIPELINE_PREPARE_QUEUE_MAX, + "[partition] prepare_queue_depth default drifted from \ + consensus::PIPELINE_PREPARE_QUEUE_MAX" + ); + } + + #[test] + fn default_view_change_retransmit_interval_matches_consensus_constant() { + // The config default lives in core/server-ng/config.toml (a string, so + // no static assert can pin it). One knob drives both view-change + // retransmit timers, which are equal by design, so pin it against both. + let config_default = configs::ng_cluster::ClusterConfig::default() + .view_change_retransmit_interval + .get_duration() + .as_millis(); + let start_view_change = + u128::from(consensus::TimeoutManager::START_VIEW_CHANGE_MESSAGE_TICKS) + * shard::CONSENSUS_TICK_INTERVAL.as_millis(); + let do_view_change = u128::from(consensus::TimeoutManager::DO_VIEW_CHANGE_MESSAGE_TICKS) + * shard::CONSENSUS_TICK_INTERVAL.as_millis(); + assert_eq!( + config_default, start_view_change, + "[cluster] view_change_retransmit_interval default drifted from \ + TimeoutManager::START_VIEW_CHANGE_MESSAGE_TICKS" + ); + assert_eq!( + config_default, do_view_change, + "[cluster] view_change_retransmit_interval default drifted from \ + TimeoutManager::DO_VIEW_CHANGE_MESSAGE_TICKS" + ); + } + + #[test] + fn default_view_change_status_timeout_matches_consensus_constant() { + // The config default lives in core/server-ng/config.toml (a string, so + // no static assert can pin it); keep it in lockstep with the built-in + // the simulator and un-configured replicas run on. + let config_default = configs::ng_cluster::ClusterConfig::default() + .view_change_status_timeout + .get_duration() + .as_millis(); + let built_in = u128::from(consensus::TimeoutManager::VIEW_CHANGE_STATUS_TICKS) + * shard::CONSENSUS_TICK_INTERVAL.as_millis(); + assert_eq!( + config_default, built_in, + "[cluster] view_change_status_timeout default drifted from \ + TimeoutManager::VIEW_CHANGE_STATUS_TICKS" + ); + } + + #[test] + fn default_request_start_view_retransmit_interval_matches_consensus_constant() { + // The config default lives in core/server-ng/config.toml (a string, so + // no static assert can pin it); keep it in lockstep with the built-in + // the simulator and un-configured replicas run on. + let config_default = configs::ng_cluster::ClusterConfig::default() + .request_start_view_retransmit_interval + .get_duration() + .as_millis(); + let built_in = u128::from(consensus::TimeoutManager::REQUEST_START_VIEW_MESSAGE_TICKS) + * shard::CONSENSUS_TICK_INTERVAL.as_millis(); + assert_eq!( + config_default, built_in, + "[cluster] request_start_view_retransmit_interval default drifted from \ + TimeoutManager::REQUEST_START_VIEW_MESSAGE_TICKS" + ); + } + + #[test] + fn default_view_probe_attempts_max_matches_consensus_constant() { + // Belt and suspenders with the static assert above: that pins the + // duplicated configs-crate literal, this pins the shipped config.toml + // value the simulator and un-configured replicas run on. + let config_default = configs::ng_cluster::ClusterConfig::default().view_probe_attempts_max; + assert_eq!( + config_default, + consensus::PROBE_ATTEMPTS_MAX, + "[cluster] view_probe_attempts_max default drifted from \ + consensus::PROBE_ATTEMPTS_MAX" + ); + } + + #[test] + fn default_repair_retry_interval_matches_partitions_constant() { + // The config default lives in core/server-ng/config.toml (a string, so + // no static assert can pin it); keep it in lockstep with the built-in + // the simulator and un-configured replicas run on. + let config_default = configs::ng_cluster::ClusterConfig::default() + .repair_retry_interval + .get_duration() + .as_millis(); + let built_in = + u128::from(partitions::REPAIR_RETRY_TICKS) * shard::CONSENSUS_TICK_INTERVAL.as_millis(); + assert_eq!( + config_default, built_in, + "[cluster] repair_retry_interval default drifted from \ + partitions::REPAIR_RETRY_TICKS" + ); + } + + #[test] + fn default_repair_chunk_max_matches_shard_constant() { + // Belt and suspenders with the static assert above: that pins the + // duplicated configs-crate literal, this pins the shipped config.toml + // value the simulator and un-configured replicas run on. + let config_default = configs::ng_cluster::ClusterConfig::default().repair_chunk_max; + assert_eq!( + config_default as u64, + shard::REPAIR_CHUNK_MAX, + "[cluster] repair_chunk_max default drifted from shard::REPAIR_CHUNK_MAX" + ); + } + + #[test] + fn default_evicted_ring_capacity_matches_partitions_constant() { + // Belt and suspenders with the static assert above; this pins the + // shipped config.toml value. + let config_default = + configs::ng_partition::PartitionConfig::default().evicted_ring_capacity; + assert_eq!( + config_default, + partitions::EVICTED_RING_CAPACITY, + "[partition] evicted_ring_capacity default drifted from \ + partitions::EVICTED_RING_CAPACITY" + ); + } + + #[test] + fn default_evicted_ring_bytes_max_matches_partitions_constant() { + // Belt and suspenders with the static assert above; this pins the + // shipped config.toml value. + let config_default = configs::ng_partition::PartitionConfig::default() + .evicted_ring_bytes_max + .as_bytes_u64(); + assert_eq!( + config_default, + partitions::EVICTED_RING_BYTES_MAX, + "[partition] evicted_ring_bytes_max default drifted from \ + partitions::EVICTED_RING_BYTES_MAX" + ); + } + + #[test] + fn shutdown_on_drop_armed_flips_flag() { + let flag = Arc::new(AtomicBool::new(false)); + drop(ShutdownOnDrop::new(Arc::clone(&flag))); + assert!( + flag.load(Ordering::Relaxed), + "an armed guard must flip the flag on drop (covers the error `?` \ + and panic-unwind exit paths of run_shard_thread)" + ); + } + + #[test] + fn shutdown_on_drop_disarmed_leaves_flag() { + let flag = Arc::new(AtomicBool::new(false)); + let mut guard = ShutdownOnDrop::new(Arc::clone(&flag)); + guard.disarm(); + drop(guard); + assert!( + !flag.load(Ordering::Relaxed), + "a disarmed guard must not flip the flag (clean `Ok(())` exit)" + ); + } + + const TEST_POLL_INTERVAL: Duration = Duration::from_millis(50); + + #[compio::test] + async fn broadcast_metadata_bundle_returns_immediately_with_no_peers() { + // Single-shard deployment: shard 0 has no peers to fan out to, + // so the handoff must complete without ever calling `send`. + let (bundle_tx, _bundle_rx) = crossfire::mpmc::bounded_async::(0); + let flag = Arc::new(AtomicBool::new(false)); + let mux = ServerNgMuxStateMachine::default(); + broadcast_metadata_bundle( + 0, + &bundle_tx, + mux.factory_bundle(), + 0, + &flag, + TEST_POLL_INTERVAL, + ) + .await + .expect("zero peers must not block shard 0"); + } + + #[compio::test] + async fn metadata_bundle_round_trips_through_channel() { + // End-to-end: shard 0 mints a bundle, a peer receives it on + // another runtime, and `from_factory_bundle` constructs a + // reader-mode mux that observes shard 0's writes via the same + // LeftRight pair. + let peers = 1u16; + let (bundle_tx, bundle_rx) = + crossfire::mpmc::bounded_async::(usize::from(peers)); + let flag = Arc::new(AtomicBool::new(false)); + + let owner = ServerNgMuxStateMachine::default(); + let bundle = owner.factory_bundle(); + broadcast_metadata_bundle(0, &bundle_tx, bundle, peers, &flag, TEST_POLL_INTERVAL) + .await + .expect("broadcast must succeed with one peer drained"); + + let received = await_metadata_bundle(1, &bundle_rx, &flag, TEST_POLL_INTERVAL) + .await + .expect("peer must receive the broadcast bundle"); + let _peer_mux = ServerNgMuxStateMachine::from_factory_bundle(received); + } + + #[compio::test] + async fn broadcast_metadata_bundle_aborts_when_peers_drop_rx() { + // Shard 0 drives handoff but every peer's `bundle_rx` was dropped + // before recv. Silently returning Ok would commit listener binds + // and consensus init for a cluster whose peers are gone; the + // broadcast must surface the disconnect so `shard_main` aborts. + let (bundle_tx, bundle_rx) = crossfire::mpmc::bounded_async::(0); + drop(bundle_rx); + let flag = Arc::new(AtomicBool::new(false)); + let mux = ServerNgMuxStateMachine::default(); + + let err = broadcast_metadata_bundle( + 0, + &bundle_tx, + mux.factory_bundle(), + 3, + &flag, + TEST_POLL_INTERVAL, + ) + .await + .expect_err("dropped rx must surface as MetadataHandoffAborted"); + assert!( + matches!(err, ServerNgError::MetadataHandoffAborted { shard_id: 0 }), + "expected MetadataHandoffAborted, got {err:?}" + ); + } + + #[compio::test] + async fn await_metadata_bundle_aborts_when_owner_drops_without_sending() { + let (bundle_tx, bundle_rx) = crossfire::mpmc::bounded_async::(1); + let flag = Arc::new(AtomicBool::new(false)); + + // Shard 0 dies before broadcasting; the peer must observe the + // disconnect and abort instead of hanging forever. + drop(bundle_tx); + + let err = await_metadata_bundle(1, &bundle_rx, &flag, TEST_POLL_INTERVAL) + .await + .expect_err("a peer whose owner never sends must abort"); + assert!( + matches!(err, ServerNgError::MetadataHandoffAborted { shard_id: 1 }), + "expected MetadataHandoffAborted, got {err:?}" + ); + } + + #[compio::test] + async fn await_metadata_bundle_aborts_on_shutdown_flag() { + // compio 0.19 `JoinHandle` yields `Result`; the + // `ResumeUnwind` impl re-raises a task panic and maps cancellation + // to `None`. + use compio::runtime::ResumeUnwind; + + let (_bundle_tx, bundle_rx) = crossfire::mpmc::bounded_async::(1); + let flag = Arc::new(AtomicBool::new(false)); + + let waiter = compio::runtime::spawn({ + let flag = Arc::clone(&flag); + async move { await_metadata_bundle(1, &bundle_rx, &flag, TEST_POLL_INTERVAL).await } + }); + + // Owner has not sent yet, but shutdown was requested; the peer + // must exit via the flag poll instead of hanging. + compio::time::sleep(TEST_POLL_INTERVAL / 2).await; + flag.store(true, Ordering::Relaxed); + + let err = waiter + .await + .resume_unwind() + .expect("waiter task was cancelled") + .expect_err("shutdown flag must abort the bundle wait"); + assert!( + matches!(err, ServerNgError::MetadataHandoffAborted { shard_id: 1 }), + "expected MetadataHandoffAborted on shutdown, got {err:?}" + ); + } + + #[compio::test] + async fn await_bootstrap_complete_returns_immediately_for_single_shard() { + // A single-shard server has no peers to wait on; the owner barrier + // must not block when `peers == 0`. + let (_ready_tx, ready_rx) = crossfire::mpmc::bounded_async::(1); + let flag = Arc::new(AtomicBool::new(false)); + await_bootstrap_complete(&ready_rx, 0, &flag, TEST_POLL_INTERVAL) + .await + .expect("single-shard server must not block on the barrier"); + } + + #[compio::test] + async fn await_bootstrap_complete_drains_every_peer_signal() { + // Two peers report load-complete; shard 0 drains both, then proceeds + // to bind listeners. + let (ready_tx, ready_rx) = crossfire::mpmc::bounded_async::(2); + let flag = Arc::new(AtomicBool::new(false)); + signal_bootstrap_complete(1, &ready_tx, &flag, TEST_POLL_INTERVAL) + .await + .expect("peer 1 must signal load-complete"); + signal_bootstrap_complete(2, &ready_tx, &flag, TEST_POLL_INTERVAL) + .await + .expect("peer 2 must signal load-complete"); + await_bootstrap_complete(&ready_rx, 2, &flag, TEST_POLL_INTERVAL) + .await + .expect("owner must drain both peer signals"); + } + + #[compio::test] + async fn await_bootstrap_complete_aborts_on_shutdown_flag() { + use compio::runtime::ResumeUnwind; + + // `_ready_tx` is held so the channel is not disconnected: the owner + // must exit via the shutdown flag, not a dropped sender. + let (_ready_tx, ready_rx) = crossfire::mpmc::bounded_async::(1); + let flag = Arc::new(AtomicBool::new(false)); + + let owner = compio::runtime::spawn({ + let flag = Arc::clone(&flag); + async move { await_bootstrap_complete(&ready_rx, 1, &flag, TEST_POLL_INTERVAL).await } + }); + + // The peer never signals, but a sibling failure flips the flag; the + // owner must abort instead of hanging before listeners. + compio::time::sleep(TEST_POLL_INTERVAL / 2).await; + flag.store(true, Ordering::Relaxed); + + let err = owner + .await + .resume_unwind() + .expect("owner task was cancelled") + .expect_err("shutdown flag must abort the barrier wait"); + assert!( + matches!( + err, + ServerNgError::ShardBootstrapBarrierAborted { remaining: 1 } + ), + "expected ShardBootstrapBarrierAborted, got {err:?}" + ); + } + + #[compio::test] + async fn signal_bootstrap_complete_aborts_when_owner_drops_rx() { + // Shard 0 aborted before draining and dropped its receiver; a peer's + // signal must surface the disconnect instead of stranding. + let (ready_tx, ready_rx) = crossfire::mpmc::bounded_async::(1); + let flag = Arc::new(AtomicBool::new(false)); + drop(ready_rx); + + let err = signal_bootstrap_complete(2, &ready_tx, &flag, TEST_POLL_INTERVAL) + .await + .expect_err("dropped rx must surface as an abort"); + assert!( + matches!(err, ServerNgError::MetadataHandoffAborted { shard_id: 2 }), + "expected MetadataHandoffAborted, got {err:?}" + ); + } + + fn cluster_node(ip: &str, http: Option) -> configs::ng_cluster::ClusterNodeConfig { + cluster_node_with_ports(ip, Some(18070), http) + } + + fn cluster_node_with_ports( + ip: &str, + tcp: Option, + http: Option, + ) -> configs::ng_cluster::ClusterNodeConfig { + configs::ng_cluster::ClusterNodeConfig { + name: "node".to_owned(), + ip: ip.to_owned(), + advertised_address: None, + advertised_addresses: Vec::new(), + replica_id: 0, + ports: configs::ng_cluster::TransportPorts { + tcp, + http, + ..Default::default() + }, + } + } + + fn addr(value: &str) -> SocketAddr { + value.parse().expect("valid socket address literal") + } + + #[test] + fn cluster_http_addr_takes_port_from_roster() { + // A byte-identical top-level [http].address is shared across nodes on + // one host; the per-node roster port is the only port source so each + // node binds a distinct HTTP socket. + let node = cluster_node("127.0.0.1", Some(18090)); + let addrs = resolve_cluster_client_addrs( + &node, + addr("127.0.0.1:8090"), + None, + None, + Some(addr("127.0.0.1:3000")), + ) + .expect("cluster address resolution must succeed"); + assert_eq!(addrs.http, Some(addr("127.0.0.1:18090"))); + } + + #[test] + fn cluster_http_addr_merges_config_ip_with_roster_port() { + // Docker/Helm bind `0.0.0.0` and probe loopback; the roster ip is + // only the advertised address. Cluster mode must keep the configured + // interface and take just the port from the roster. + let node = cluster_node("10.0.0.5", Some(18090)); + let addrs = resolve_cluster_client_addrs( + &node, + addr("0.0.0.0:8090"), + None, + None, + Some(addr("0.0.0.0:3000")), + ) + .expect("cluster address resolution must succeed"); + assert_eq!(addrs.http, Some(addr("0.0.0.0:18090"))); + } + + #[test] + fn cluster_http_addr_requires_roster_port_for_enabled_transport() { + // No fallback to the top-level port: a silent default could collide + // with another same-host node, so a missing roster port for an + // enabled transport must refuse to boot. + let node = cluster_node("10.0.0.5", None); + let result = resolve_cluster_client_addrs( + &node, + addr("127.0.0.1:8090"), + None, + None, + Some(addr("127.0.0.1:3000")), + ); + assert!(matches!( + result, + Err(ServerNgError::ClusterPortMissing { + transport: "http", + replica_id: 0, + }) + )); + } + + #[test] + fn cluster_http_addr_is_none_when_http_disabled() { + // http.enabled = false collapses default_http_addr to None; no roster + // port can revive a listener the operator turned off. + let node = cluster_node("127.0.0.1", Some(18090)); + let addrs = resolve_cluster_client_addrs(&node, addr("127.0.0.1:8090"), None, None, None) + .expect("cluster address resolution must succeed"); + assert_eq!(addrs.http, None); + } + + /// Regression: the shutdown-join deadline must arm at SHUTDOWN, not + /// at boot. The original bound measured from `join_all` entry, so any + /// healthy server outliving `shutdown_join_timeout` (30s default) was + /// abandoned as "wedged" and the process exited - every BDD run died + /// at t+30s while the test container was still compiling. + #[test] + fn join_waits_unbounded_while_the_server_runs() { + let shutdown_flag = AtomicBool::new(false); + // Thread outlives a deliberately tiny join budget; with the flag + // clear the budget must never even arm. + let handle = thread::spawn(|| -> Result<(), ServerNgError> { + thread::sleep(Duration::from_millis(300)); + Ok(()) + }); + let mut deadline = None; + let joined = join_until_shutdown_deadline( + handle, + &shutdown_flag, + Duration::from_millis(20), + &mut deadline, + ); + assert!( + matches!(joined, Some(Ok(Ok(())))), + "a running server must be awaited indefinitely, not abandoned as wedged" + ); + assert!( + deadline.is_none(), + "the join deadline must not arm before the shutdown flag flips" + ); + } + + #[test] + fn join_abandons_a_wedged_shard_after_the_shutdown_deadline() { + let shutdown_flag = AtomicBool::new(true); + // Never finishes: stands in for a wedged pump. The thread leaks + // into the test process, which exits right after. + let handle = thread::spawn(|| -> Result<(), ServerNgError> { + loop { + thread::sleep(Duration::from_secs(1)); + } + }); + let mut deadline = None; + let joined = join_until_shutdown_deadline( + handle, + &shutdown_flag, + Duration::from_millis(100), + &mut deadline, + ); + assert!( + joined.is_none(), + "a shard still running past the post-shutdown budget must be abandoned" + ); + assert!(deadline.is_some(), "the deadline arms once the flag is set"); + } + + #[test] + fn cluster_tcp_addr_takes_port_from_roster() { + // Same rule as the other transports: the roster owns the port so + // same-host nodes sharing one [tcp].address still bind distinct + // sockets. + let node = cluster_node("127.0.0.1", None); + let addrs = resolve_cluster_client_addrs(&node, addr("127.0.0.1:8090"), None, None, None) + .expect("cluster address resolution must succeed"); + assert_eq!(addrs.client, addr("127.0.0.1:18070")); + } + + #[test] + fn cluster_tcp_addr_merges_config_ip_with_roster_port() { + // The roster ip is advertised, not bound. Binding it directly would + // strand every co-located dialer (sidecars, health probes, on-host + // consumers) that reaches this node over loopback. + let node = cluster_node("10.0.0.5", None); + let addrs = resolve_cluster_client_addrs(&node, addr("0.0.0.0:8090"), None, None, None) + .expect("cluster address resolution must succeed"); + assert_eq!(addrs.client, addr("0.0.0.0:18070")); + } + + #[test] + fn cluster_tcp_addr_requires_roster_port() { + // tcp is always enabled in cluster mode, so a roster entry without a + // tcp port refuses to boot rather than falling back to [tcp].address. + let node = cluster_node_with_ports("10.0.0.5", None, None); + let result = resolve_cluster_client_addrs(&node, addr("127.0.0.1:8090"), None, None, None); + assert!(matches!( + result, + Err(ServerNgError::ClusterPortMissing { + transport: "tcp", + replica_id: 0, + }) + )); + } + + #[test] + fn cluster_tcp_addr_keeps_loopback_bind_and_warns_on_roster_mismatch() { + // A loopback [tcp].address under a routable roster ip is honoured + // as configured; remote peers cannot reach it, so the mismatch is + // warned about instead of silently rebinding. + let node = cluster_node("10.0.0.5", None); + let addrs = resolve_cluster_client_addrs(&node, addr("127.0.0.1:8090"), None, None, None) + .expect("cluster address resolution must succeed"); + assert_eq!(addrs.client, addr("127.0.0.1:18070")); + assert!(roster_ip_unreachable_from_bind_addr(&node.ip, addrs.client)); + } + + #[test] + fn roster_mismatch_warning_is_silent_for_wildcard_and_hostname_rosters() { + // A wildcard bind covers the roster interface, and a DNS roster entry + // can resolve to the bound one; neither is a misconfiguration. + assert!(!roster_ip_unreachable_from_bind_addr( + "10.0.0.5", + addr("0.0.0.0:18070") + )); + assert!(!roster_ip_unreachable_from_bind_addr( + "node-1.example.com", + addr("127.0.0.1:18070") + )); + assert!(!roster_ip_unreachable_from_bind_addr( + "10.0.0.5", + addr("10.0.0.5:18070") + )); + } +} diff --git a/core/server/src/cluster_meta.rs b/core/server-ng/src/cluster_meta.rs similarity index 98% rename from core/server/src/cluster_meta.rs rename to core/server-ng/src/cluster_meta.rs index 7372d3e597..10d17b83db 100644 --- a/core/server/src/cluster_meta.rs +++ b/core/server-ng/src/cluster_meta.rs @@ -28,7 +28,7 @@ //! leader, but the full roster is still returned). The self-synthesized single //! node is the cluster-disabled fallback, shared by both callers. -use configs::cluster::{ResolvedClusterNode, TransportPorts}; +use configs::ng_cluster::{ResolvedClusterNode, TransportPorts}; use iggy_common::{ ClusterMetadata, ClusterNode, ClusterNodeRole, ClusterNodeStatus, TransportEndpoints, }; @@ -181,7 +181,7 @@ fn ports_to_endpoints(ports: &TransportPorts) -> TransportEndpoints { mod tests { use super::*; - use configs::cluster::{AdvertisedAddressSelector, ClusterNodeConfig}; + use configs::ng_cluster::{AdvertisedAddressSelector, ClusterNodeConfig}; fn node_config(advertised_address: Option) -> ClusterNodeConfig { ClusterNodeConfig { diff --git a/core/server/src/config_writer.rs b/core/server-ng/src/config_writer.rs similarity index 87% rename from core/server/src/config_writer.rs rename to core/server-ng/src/config_writer.rs index 92d4fb2864..1d2602e82d 100644 --- a/core/server/src/config_writer.rs +++ b/core/server-ng/src/config_writer.rs @@ -15,10 +15,10 @@ // specific language governing permissions and limitations // under the License. -use crate::server_error::ServerError; +use crate::server_error::ServerNgError; use compio::fs::OpenOptions; use compio::io::AsyncWriteAtExt; -use configs::server::ServerConfig; +use configs::server_ng::ServerNgConfig; use std::net::SocketAddr; /// Write the runtime `current_config.toml` file with the effective bound ports. @@ -28,14 +28,14 @@ use std::net::SocketAddr; /// Returns an error if the config cannot be serialized or if the runtime /// config file cannot be written and synced. pub async fn write_current_config( - config: &ServerConfig, + config: &ServerNgConfig, current_replica_id: Option, bound_tcp: Option, bound_replica: Option, bound_tcp_tls: Option, bound_quic: Option, bound_websocket: Option, -) -> Result<(), ServerError> { +) -> Result<(), ServerNgError> { let mut current_config = config.clone(); if let Some(bound_client_tcp) = bound_tcp_tls.or(bound_tcp) { @@ -59,7 +59,7 @@ pub async fn write_current_config( .nodes .iter_mut() .find(|node| node.replica_id == replica_id) - .ok_or(ServerError::ClusterNodeNotFound { replica_id })?; + .ok_or(ServerNgError::ClusterNodeNotFound { replica_id })?; if let Some(bound_client_tcp) = bound_tcp_tls.or(bound_tcp) { node.ports.tcp = Some(bound_client_tcp.port()); } @@ -76,7 +76,8 @@ pub async fn write_current_config( let runtime_path = current_config.system.get_runtime_path(); let config_path = format!("{runtime_path}/current_config.toml"); - let content = toml::to_string(¤t_config).map_err(ServerError::CurrentConfigSerialize)?; + let content = + toml::to_string(¤t_config).map_err(ServerNgError::CurrentConfigSerialize)?; let mut file = OpenOptions::new() .write(true) @@ -84,7 +85,7 @@ pub async fn write_current_config( .truncate(true) .open(&config_path) .await - .map_err(|source| ServerError::CurrentConfigWrite { + .map_err(|source| ServerNgError::CurrentConfigWrite { path: config_path.clone(), source, })?; @@ -92,14 +93,14 @@ pub async fn write_current_config( file.write_all_at(content.into_bytes(), 0) .await .0 - .map_err(|source| ServerError::CurrentConfigWrite { + .map_err(|source| ServerNgError::CurrentConfigWrite { path: config_path.clone(), source, })?; file.sync_all() .await - .map_err(|source| ServerError::CurrentConfigWrite { + .map_err(|source| ServerNgError::CurrentConfigWrite { path: config_path, source, })?; diff --git a/core/server/src/consumer_group.rs b/core/server-ng/src/consumer_group.rs similarity index 97% rename from core/server/src/consumer_group.rs rename to core/server-ng/src/consumer_group.rs index fd93e35261..0347136444 100644 --- a/core/server/src/consumer_group.rs +++ b/core/server-ng/src/consumer_group.rs @@ -40,7 +40,7 @@ use iggy_binary_protocol::requests::consumer_offsets::{ DeleteConsumerOffset2Request, DeleteConsumerOffsetRequest, StoreConsumerOffset2Request, StoreConsumerOffsetRequest, }; -use iggy_binary_protocol::{KIND_CONSUMER_GROUP, Operation, RoutedRequestHeader, WireIdentifier}; +use iggy_binary_protocol::{KIND_CONSUMER_GROUP, Operation, RequestHeader, WireIdentifier}; use iggy_common::IggyError; use journal::superblock::SuperblockStore; use journal::{Journal, JournalHandle}; @@ -63,8 +63,8 @@ use std::rc::Rc; /// join. Every other operation passes through. pub(crate) async fn maybe_rewrite_consumer_group_request( shard: &Rc>, - request: Message, -) -> Result, IggyError> + request: Message, +) -> Result, IggyError> where B: ShellBus, MJ: JournalHandle + 'static, @@ -208,8 +208,8 @@ where #[allow(clippy::cast_possible_truncation)] pub(crate) fn maybe_rewrite_consumer_offset_request( shard: &Rc>, - request: Message, -) -> Result, IggyError> + request: Message, +) -> Result, IggyError> where B: ShellBus, MJ: JournalHandle + 'static, diff --git a/core/server/src/dispatch.rs b/core/server-ng/src/dispatch.rs similarity index 95% rename from core/server/src/dispatch.rs rename to core/server-ng/src/dispatch.rs index aa198b8d4a..9114d938de 100644 --- a/core/server/src/dispatch.rs +++ b/core/server-ng/src/dispatch.rs @@ -49,9 +49,9 @@ use crate::responses::{ use crate::session_manager::SessionManager; use crate::snapshot; use crate::users::maybe_rewrite_user_password_request; -use crate::wire::{request_body, usize_to_u32, verify_request_checksum}; +use crate::wire::{request_body, usize_to_u32}; use bytes::Bytes; -use configs::server::ServerSystemConfig; +use configs::server_ng::NgSystemConfig; use consensus::{ Consensus, EvictionContext, MetadataHandle, PartitionsHandle, build_eviction_message, build_incompatible_protocol_eviction_message, build_result_rejection_reply, @@ -59,9 +59,8 @@ use consensus::{ use iggy_binary_protocol::PrepareHeader; use iggy_binary_protocol::codes::{ GET_CLIENT_CODE, GET_CLIENTS_CODE, GET_CLUSTER_METADATA_CODE, GET_CONSUMER_OFFSET_CODE, - GET_ME_CODE, GET_PERSONAL_ACCESS_TOKENS_CODE, GET_SNAPSHOT_FILE_CODE, GET_STATS_CODE, - LOGIN_USER_CODE, LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE, PING_CODE, POLL_MESSAGES_CODE, - SYNC_CONSUMER_GROUP_CODE, + GET_ME_CODE, GET_PERSONAL_ACCESS_TOKENS_CODE, GET_SNAPSHOT_FILE_CODE, LOGIN_USER_CODE, + LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE, PING_CODE, POLL_MESSAGES_CODE, SYNC_CONSUMER_GROUP_CODE, }; use iggy_binary_protocol::primitives::consumer::WireConsumer; use iggy_binary_protocol::primitives::polling_strategy::WirePollingStrategy; @@ -86,7 +85,7 @@ use iggy_binary_protocol::responses::system::get_snapshot::GetSnapshotResponse; use iggy_binary_protocol::{ AckLevel, ClientVersionInfo, Command2, EvictionReason, GenericHeader, HEADER_SIZE, KIND_CONSUMER_GROUP, MAX_PARTITIONS_PER_REQUEST, Operation, ProtocolVersion, RequestHeader, - RoutedRequestHeader, WireDecode, WireEncode, WireIdentifier, is_protocol_compatible, + WireDecode, WireEncode, WireIdentifier, is_protocol_compatible, }; use iggy_common::{ IggyError, MaxTopicSize, PollingStrategy, SnapshotCompression, SystemSnapshotType, @@ -124,7 +123,7 @@ pub(crate) type ActiveClientRequests = Rc>>; pub(crate) fn make_client_request_handler( shard: &Rc>, sessions: &Rc>, - system_config: Arc, + system_config: Arc, max_tokens_per_user: u32, ) -> RequestHandler where @@ -402,7 +401,7 @@ fn submit_auto_commit( fn build_auto_commit_request( namespace: IggyNamespace, applied: &AutoCommitApplied, -) -> Result, IggyError> { +) -> Result, IggyError> { let request = StoreConsumerOffset2Request { consumer: WireConsumer { kind: applied.kind.as_code(), @@ -415,28 +414,26 @@ fn build_auto_commit_request( ack: AckLevel::Quorum, }; let body = request.to_bytes(); - let header_size = std::mem::size_of::(); + let header_size = std::mem::size_of::(); let total_size = header_size + body.len(); let size = u32::try_from(total_size).map_err(|_| IggyError::InvalidConfiguration)?; - let mut message = Message::::new(total_size); + let mut message = Message::::new(total_size); message.as_mut_slice()[header_size..].copy_from_slice(&body); - Ok( - message.transmute_header(|_, header: &mut RoutedRequestHeader| { - *header = RoutedRequestHeader { - command: Command2::Request, - operation: Operation::StoreConsumerOffset2, - size, - client: AUTO_COMMIT_CLIENT_ID, - // The partition plane is sessionless (no `ClientTable` dedup); a - // nonzero session + request just satisfy the wire header - // validation. - session: 1, - request: 1, - group: namespace.inner(), - ..Default::default() - }; - }), - ) + Ok(message.transmute_header(|_, header: &mut RequestHeader| { + *header = RequestHeader { + command: Command2::Request, + operation: Operation::StoreConsumerOffset2, + size, + client: AUTO_COMMIT_CLIENT_ID, + // The partition plane is sessionless (no `ClientTable` dedup); a + // nonzero session + request just satisfy the wire header + // validation. + session: 1, + request: 1, + namespace: namespace.inner(), + ..Default::default() + }; + })) } pub(crate) fn make_deferred_replica_message_handler( @@ -461,7 +458,7 @@ pub(crate) fn make_deferred_client_request_handler( bus: &B, shard_handle: &ShellShardHandle, sessions: &Rc>, - system_config: Arc, + system_config: Arc, max_tokens_per_user: u32, ) -> RequestHandler where @@ -571,7 +568,7 @@ where let _ = reply.try_send(commit); } shard::MetadataSubmit::ClientRequest { request, reply } => { - let committed = match request.try_into_typed::() { + let committed = match request.try_into_typed::() { Ok(typed) => shard .plane .metadata() @@ -654,7 +651,7 @@ where fn enqueue_client_request( shard: Rc>, sessions: Rc>, - system_config: Arc, + system_config: Arc, max_tokens_per_user: u32, queues: ClientRequestQueues, active: ActiveClientRequests, @@ -695,7 +692,7 @@ fn enqueue_client_request( async fn drain_client_requests( shard: Rc>, sessions: Rc>, - system_config: Arc, + system_config: Arc, max_tokens_per_user: u32, queues: ClientRequestQueues, active: ActiveClientRequests, @@ -780,7 +777,7 @@ pub(crate) const fn validate_partitions_change_count( /// `ServerDefault` is exempt from the size floor (it resolves against server /// config at admission, matching legacy); `Unlimited` passes numerically. pub(crate) fn validate_topic_bounds( - system_config: &ServerSystemConfig, + system_config: &NgSystemConfig, partitions_count: u32, max_topic_size: MaxTopicSize, ) -> Result<(), IggyError> { @@ -803,7 +800,7 @@ pub(crate) fn validate_topic_bounds( #[allow(clippy::future_not_send)] async fn send_pre_consensus_deny( shard: &Rc>, - header: &RoutedRequestHeader, + header: &RequestHeader, transport_client_id: u128, error: &IggyError, context: &'static str, @@ -841,7 +838,7 @@ async fn send_pre_consensus_deny( async fn handle_client_request( shard: &Rc>, sessions: &Rc>, - system_config: &Arc, + system_config: &Arc, max_tokens_per_user: u32, transport_client_id: u128, message: Message, @@ -863,42 +860,6 @@ async fn handle_client_request( return; } }; - // Promote to the server-internal routed shape at the boundary: the - // client wire carries no group (it is derived -- plane from `operation`, - // partition target from the payload), so it starts unset here and the - // resolution sites below stamp it before anything routes on it. - let request = request.into_routed(); - - // The last point that still sees the body the CLIENT sent; every rewrite below - // substitutes server-chosen bytes and carries the stamp through unchanged. - if let Err(error) = verify_request_checksum(&request) { - warn!( - transport_client_id, - operation = ?request.header().operation, - request = request.header().request, - "dropping client request whose body does not match its own checksum" - ); - let commit = current_metadata_commit(shard); - let reply = build_deny_reply( - request.header(), - transport_client_id, - 0, - commit, - error.as_code(), - ); - if let Err(send_error) = shard - .bus - .send_to_client(transport_client_id, reply.into_generic().into_frozen()) - .await - { - warn!( - transport_client_id, - error = %send_error, - "failed to send request-checksum deny reply" - ); - } - return; - } ensure_transport_connection(shard, sessions, transport_client_id); @@ -920,7 +881,7 @@ async fn handle_client_request( // MUST go through Register first, which binds the acting user the // per-op authz gates resolve. let nr_code = u32::from_le_bytes(request.header().reserved[..4].try_into().unwrap()); - // Legacy (pre-register) login codes. The server authenticates only via + // Legacy (pre-register) login codes. server-ng authenticates only via // the Register handshake (LOGIN_REGISTER / LOGIN_REGISTER_WITH_PAT, // Operation::Register); the vsr SDK funnels both logins there and never // emits these. Reject them uniformly with a typed MalformedLogin (the @@ -935,7 +896,7 @@ async fn handle_client_request( warn!( transport_client_id, code = nr_code, - "rejecting legacy login code; server requires the register handshake" + "rejecting legacy login code; server-ng requires the register handshake" ); send_login_eviction( shard, @@ -1021,10 +982,8 @@ async fn handle_client_request( return; } - let request = request.transmute_header(|header, new_header: &mut RoutedRequestHeader| { + let request = request.transmute_header(|header, new_header: &mut RequestHeader| { *new_header = header; - // Metadata-plane ops route by operation: stamp the sentinel group. - new_header.group = server_common::sharding::METADATA_GROUP; // `bound` is always Some here (unbound transports early-return above); // this sets the consensus client id + session for the replicated op. if let Some((bound_client_id, bound_session)) = bound { @@ -1173,7 +1132,7 @@ async fn handle_get_personal_access_tokens( shard: &Rc>, sessions: &Rc>, transport_client_id: u128, - request: &Message, + request: &Message, ) where B: ShellBus, MJ: JournalHandle + 'static, @@ -1200,7 +1159,7 @@ async fn handle_get_me( shard: &Rc>, sessions: &Rc>, transport_client_id: u128, - request: &Message, + request: &Message, ) where B: ShellBus, MJ: JournalHandle + 'static, @@ -1239,7 +1198,7 @@ async fn handle_get_me( #[allow(clippy::future_not_send)] pub(crate) async fn dispatch_partition_request( shard: &Rc>, - request: Message, + request: Message, vsr_client_id: u128, bound_session: u64, transport_client_id: u128, @@ -1360,9 +1319,9 @@ pub(crate) async fn dispatch_partition_request( return; } }; - let request = request.transmute_header(|header, new_header: &mut RoutedRequestHeader| { + let request = request.transmute_header(|header, new_header: &mut RequestHeader| { *new_header = header; - new_header.group = namespace; + new_header.namespace = namespace; new_header.client = transport_client_id; // Header validation requires `session > 0 && request > 0` for // non-register ops. The partition plane itself is sessionless @@ -1379,9 +1338,9 @@ pub(crate) async fn dispatch_partition_request( async fn handle_non_replicated_request( shard: &Rc>, sessions: &Rc>, - system_config: &Arc, + system_config: &Arc, transport_client_id: u128, - request: Message, + request: Message, ) where B: ShellBus, MJ: JournalHandle + 'static, @@ -1536,7 +1495,7 @@ async fn handle_default_non_replicated( shard: &Rc>, transport_client_id: u128, code: u32, - request: &Message, + request: &Message, user_id: Option, roster: &ClusterRoster, client_ip: Option, @@ -1554,13 +1513,6 @@ async fn handle_default_non_replicated( send_non_replicated_deny(shard, request, transport_client_id, error.as_code()).await; return; } - // Stats is the one default read with an async input: the cross-shard - // connected-client gather. Run it here so the shared builder stays sync. - let clients_count = if code == GET_STATS_CODE { - u32::try_from(shard.list_all_clients().await.len()).unwrap_or(u32::MAX) - } else { - 0 - }; match build_non_replicated_response( shard, code, @@ -1568,7 +1520,6 @@ async fn handle_default_non_replicated( user_id, roster, client_ip, - clients_count, ) { Ok(response) => { let commit = current_metadata_commit(shard); @@ -1614,9 +1565,9 @@ async fn handle_default_non_replicated( #[allow(clippy::future_not_send)] async fn handle_get_snapshot( shard: &Rc>, - system_config: &Arc, + system_config: &Arc, transport_client_id: u128, - request: &Message, + request: &Message, user_id: Option, ) where B: ShellBus, @@ -1696,7 +1647,7 @@ fn decode_get_snapshot( #[allow(clippy::future_not_send)] async fn send_non_replicated_bytes( shard: &Rc>, - request: &Message, + request: &Message, transport_client_id: u128, bytes: Bytes, label: &'static str, @@ -1889,7 +1840,7 @@ async fn evict_stale_client( async fn handle_poll_messages( shard: &Rc>, transport_client_id: u128, - request: &Message, + request: &Message, user_id: Option, ) where B: ShellBus, @@ -1961,19 +1912,14 @@ async fn handle_poll_messages( } } Err(error) => { - // A stream, topic, or partition id that does not resolve is a + // A partition id that does not exist in a resolvable topic is a // client addressing error and must surface as a typed rejection, // not an empty poll a consumer would read as end-of-partition. - if matches!( - error, - IggyError::PartitionNotFound(..) - | IggyError::StreamIdNotFound(_) - | IggyError::TopicIdNotFound(..) - ) { + if matches!(error, IggyError::PartitionNotFound(..)) { warn!( transport_client_id, error = %error, - "poll_messages rejected: target not found" + "poll_messages rejected: partition not found" ); send_non_replicated_deny(shard, request, transport_client_id, error.as_code()) .await; @@ -2010,7 +1956,7 @@ async fn handle_poll_messages( async fn handle_get_consumer_offset( shard: &Rc>, transport_client_id: u128, - request: &Message, + request: &Message, user_id: Option, ) where B: ShellBus, @@ -2097,7 +2043,7 @@ async fn handle_get_consumer_offset( async fn handle_sync_consumer_group( shard: &Rc>, transport_client_id: u128, - request: &Message, + request: &Message, ) where B: ShellBus, MJ: JournalHandle + 'static, @@ -2151,7 +2097,7 @@ async fn handle_sync_consumer_group( async fn send_empty_partition_reply( shard: &Rc>, transport_client_id: u128, - request_header: &RoutedRequestHeader, + request_header: &RequestHeader, ) where B: ShellBus, MJ: JournalHandle + 'static, @@ -2478,7 +2424,7 @@ async fn handle_delete_segments_request( shard: &Rc>, transport_client_id: u128, bound: Option<(u128, u64)>, - request: &Message, + request: &Message, ) where B: ShellBus, MJ: JournalHandle + 'static, @@ -2608,11 +2554,11 @@ async fn handle_delete_segments_request( #[allow(clippy::cast_possible_truncation)] pub(crate) async fn resolve_delete_segments_truncate( shard: &Rc>, - template: &RoutedRequestHeader, + template: &RequestHeader, client_id: u128, session: u64, body: &[u8], -) -> Result, IggyError> +) -> Result, IggyError> where B: ShellBus, MJ: JournalHandle + 'static, @@ -2773,7 +2719,7 @@ fn submit_disconnect_logout( #[allow(clippy::future_not_send)] pub(crate) async fn submit_client_request_on_owner( shard: &Rc>, - request: Message, + request: Message, ) -> Option> where B: ShellBus, @@ -2803,7 +2749,7 @@ async fn handle_logout_request( shard: &Rc>, sessions: &Rc>, transport_client_id: u128, - request: Message, + request: Message, ) where B: ShellBus, MJ: JournalHandle + 'static, @@ -2907,7 +2853,7 @@ async fn handle_login_register_request( shard: &Rc>, sessions: &Rc>, transport_client_id: u128, - request: Message, + request: Message, ) where B: ShellBus, MJ: JournalHandle + 'static, @@ -2958,7 +2904,6 @@ async fn handle_login_register_request( } let body_tail = &body[prefix_len..]; - let mut credentials_rejected = false; if let Ok((wire_request, _)) = LoginRegisterRequest::decode_after_prefix(version_info.clone(), body_tail) { @@ -2988,10 +2933,8 @@ async fn handle_login_register_request( Err(LoginRegisterError::InvalidCredentials) => { // Fall through to PAT attempt so a credential payload that // collides with a valid PAT payload shape still gets a - // chance. A password-shaped body rarely parses as a PAT - // body, so remember the rejection: the final fall-through - // must surface InvalidCredentials, not MalformedLogin. - credentials_rejected = true; + // chance; if PAT also rejects, the final fall-through emits + // the empty-reply failure path below. } Err(error) => { warn!(transport_client_id, error = %error, "login/register failed"); @@ -3039,21 +2982,6 @@ async fn handle_login_register_request( } } - if credentials_rejected { - warn!( - transport_client_id, - "rejecting register request: invalid credentials" - ); - send_login_eviction( - shard, - transport_client_id, - request.header().client, - EvictionReason::InvalidCredentials, - ) - .await; - return; - } - warn!( transport_client_id, "rejecting register request with unsupported payload shape" @@ -3272,16 +3200,16 @@ mod tests { session: u64, request: u64, body: &[u8], - ) -> Message { - let header_size = size_of::(); + ) -> Message { + let header_size = size_of::(); let total = header_size + body.len(); - let mut message = Message::::new(total); + let mut message = Message::::new(total); { let slice = message.as_mut_slice(); slice[header_size..total].copy_from_slice(body); let header = - bytemuck::checked::from_bytes_mut::(&mut slice[..header_size]); - *header = RoutedRequestHeader { + bytemuck::checked::from_bytes_mut::(&mut slice[..header_size]); + *header = RequestHeader { command: Command2::Request, operation, size: u32::try_from(total).expect("test request fits u32"), @@ -3289,7 +3217,7 @@ mod tests { session, request, user_id: 0, - group: server_common::sharding::METADATA_GROUP, + namespace: server_common::sharding::METADATA_CONSENSUS_NAMESPACE, ..Default::default() }; } @@ -3322,13 +3250,12 @@ mod tests { client, request, user_id: 0, - group: server_common::sharding::METADATA_GROUP, + checksum: 42, + namespace: server_common::sharding::METADATA_CONSENSUS_NAMESPACE, ..Default::default() }; } - // A real identity, not a placeholder: `on_replicate` recomputes it before the - // prepare reaches the WAL, so an arbitrary value reads as transit corruption. - consensus::seal_prepare_checksum(message) + message } /// Regression test for the production failure chain "CLI stream @@ -3373,7 +3300,7 @@ mod tests { 1, 0, 1, - server_common::sharding::METADATA_GROUP, + server_common::sharding::METADATA_CONSENSUS_NAMESPACE, bus.clone(), LocalPipeline::new(), ); @@ -3392,9 +3319,7 @@ mod tests { messages_required_to_save: 1, size_of_messages_required_to_save: iggy_common::IggyByteSize::from(1024_u64), enforce_fsync: false, - validate_checksum: true, segment_size: iggy_common::IggyByteSize::from(1_048_576_u64), - preallocate_segments: false, encryptor: None, }, ); @@ -3512,9 +3437,7 @@ mod tests { messages_required_to_save: 1, size_of_messages_required_to_save: iggy_common::IggyByteSize::from(1024_u64), enforce_fsync: false, - validate_checksum: true, segment_size: iggy_common::IggyByteSize::from(1_048_576_u64), - preallocate_segments: false, encryptor: None, }, ); @@ -3637,9 +3560,7 @@ mod tests { messages_required_to_save: 1, size_of_messages_required_to_save: iggy_common::IggyByteSize::from(1024_u64), enforce_fsync: false, - validate_checksum: true, segment_size: iggy_common::IggyByteSize::from(1_048_576_u64), - preallocate_segments: false, encryptor: None, }, ); @@ -3656,9 +3577,9 @@ mod tests { shard.plane.partitions().tombstone(namespace); let request = request_message(Operation::SendMessages, TRANSPORT, SESSION, 1, &[]) - .transmute_header(|header, new_header: &mut RoutedRequestHeader| { + .transmute_header(|header, new_header: &mut RequestHeader| { *new_header = header; - new_header.group = namespace.inner(); + new_header.namespace = namespace.inner(); }); shard.on_message(request.into_generic()).await; @@ -3701,9 +3622,7 @@ mod tests { messages_required_to_save: 1, size_of_messages_required_to_save: iggy_common::IggyByteSize::from(1024_u64), enforce_fsync: false, - validate_checksum: true, segment_size: iggy_common::IggyByteSize::from(1_048_576_u64), - preallocate_segments: false, encryptor: None, }, ); @@ -3732,9 +3651,9 @@ mod tests { let namespace = IggyNamespace::new(0, 0, 0); let request = request_message(Operation::SendMessages, TRANSPORT, SESSION, 1, &[]) - .transmute_header(|header, new_header: &mut RoutedRequestHeader| { + .transmute_header(|header, new_header: &mut RequestHeader| { *new_header = header; - new_header.group = namespace.inner(); + new_header.namespace = namespace.inner(); }); // Namespace neither materialised nor tombstoned: the frame parks. shard.on_message(request.into_generic()).await; @@ -3770,7 +3689,7 @@ mod tests { #[test] fn create_topic_bounds_deny_pre_consensus() { - let config = ServerSystemConfig::default(); + let config = NgSystemConfig::default(); let segment_size = config.segment.size.as_bytes_u64(); assert!(segment_size > 0, "default segment size must be nonzero"); diff --git a/core/server/src/dispatch/authz.rs b/core/server-ng/src/dispatch/authz.rs similarity index 98% rename from core/server/src/dispatch/authz.rs rename to core/server-ng/src/dispatch/authz.rs index afd8cc011c..08ed5cac12 100644 --- a/core/server/src/dispatch/authz.rs +++ b/core/server-ng/src/dispatch/authz.rs @@ -38,9 +38,7 @@ use iggy_binary_protocol::requests::consumer_groups::{ }; use iggy_binary_protocol::requests::streams::GetStreamRequest; use iggy_binary_protocol::requests::topics::{GetTopicRequest, GetTopicsRequest}; -use iggy_binary_protocol::{ - Operation, PrepareHeader, RoutedRequestHeader, WireDecode, WireIdentifier, -}; +use iggy_binary_protocol::{Operation, PrepareHeader, RequestHeader, WireDecode, WireIdentifier}; use iggy_common::IggyError; use journal::superblock::SuperblockStore; use journal::{Journal, JournalHandle}; @@ -141,7 +139,7 @@ where pub(super) async fn send_partition_deny_reply( shard: &Rc>, transport_client_id: u128, - request_header: &RoutedRequestHeader, + request_header: &RequestHeader, status: u32, ) where B: ShellBus, @@ -301,7 +299,7 @@ where #[allow(clippy::future_not_send)] pub(super) async fn send_non_replicated_deny( shard: &Rc>, - request: &Message, + request: &Message, transport_client_id: u128, status: u32, ) where diff --git a/core/server/src/http.rs b/core/server-ng/src/http.rs similarity index 98% rename from core/server/src/http.rs rename to core/server-ng/src/http.rs index d3e10ca58a..82b3fd997b 100644 --- a/core/server/src/http.rs +++ b/core/server-ng/src/http.rs @@ -55,16 +55,16 @@ use axum::middleware::{Next, from_fn, from_fn_with_state}; use axum::response::Response; use axum::routing::{delete, get, post, put}; use compio::net::TcpListener; -use configs::cluster::{ClusterConfig, TransportPorts, http_forwarding_key_material}; use configs::http::{HttpConfig, HttpCorsConfig}; -use configs::server::ServerSystemConfig; +use configs::ng_cluster::{ClusterConfig, TransportPorts, http_forwarding_key_material}; +use configs::server_ng::NgSystemConfig; use iggy_common::IggyError; use message_bus::client_listener; use send_wrapper::SendWrapper; use tower_http::cors::{AllowOrigin, CorsLayer}; use tracing::{error, info, warn}; -use crate::bootstrap::ServerShard; +use crate::bootstrap::ServerNgShard; use crate::cluster_meta::ClusterRoster; use crate::http::handlers::{ change_password, create_cg, create_partitions, create_pat, create_stream, create_topic, @@ -79,7 +79,7 @@ use crate::http::handlers::{ use crate::http::jwt::JwtManager; use crate::http::session::RegistrationBarrier; use crate::http::state::{HttpInner, HttpState, insert_view_header}; -use crate::server_error::ServerError; +use crate::server_error::ServerNgError; /// Bind the shard-0 HTTP listener and spawn the `cyper-axum` serve loop as a /// background task on shard 0's compio runtime. Serves HTTPS when @@ -91,20 +91,20 @@ use crate::server_error::ServerError; /// /// # Errors /// -/// Returns [`ServerError`] if the JWT manager cannot be built from +/// Returns [`ServerNgError`] if the JWT manager cannot be built from /// `http_config.jwt`, the `[http.cors]` config is invalid, the `[http.tls]` /// credentials cannot be loaded, or the listener cannot bind to `addr`. #[allow(clippy::too_many_arguments)] pub async fn start( - shard: &Rc, + shard: &Rc, addr: SocketAddr, http_config: &HttpConfig, clients_table_max: usize, max_tokens_per_user: u32, cluster: &ClusterConfig, - system_config: Arc, + system_config: Arc, self_ports: TransportPorts, -) -> Result<(), ServerError> { +) -> Result<(), ServerNgError> { // In cluster mode with no configured JWT secret the signing key derives // from the cluster PSK, so a bearer minted on any node verifies on every // node - the invariant follower-to-primary forwarding depends on. @@ -184,11 +184,11 @@ pub async fn start( shard.bus.token(), ); shard.bus.track_background(pump); - info!(address = %bound_addr, "server HTTPS listener started"); + info!(address = %bound_addr, "server-ng HTTPS listener started"); let handle = compio::runtime::spawn(tls::serve(connections, router, shard.bus.token())); shard.bus.track_background(handle); } else { - info!(address = %bound_addr, "server HTTP listener started"); + info!(address = %bound_addr, "server-ng HTTP listener started"); let shutdown = shard.bus.token(); let handle = compio::runtime::spawn(async move { if let Err(error) = cyper_axum::serve( @@ -198,7 +198,7 @@ pub async fn start( .with_graceful_shutdown(async move { shutdown.wait().await }) .await { - error!(%error, "server HTTP listener terminated with error"); + error!(%error, "server-ng HTTP listener terminated with error"); } }); shard.bus.track_background(handle); diff --git a/core/server/src/http/admission.rs b/core/server-ng/src/http/admission.rs similarity index 100% rename from core/server/src/http/admission.rs rename to core/server-ng/src/http/admission.rs diff --git a/core/server-ng/src/http/error.rs b/core/server-ng/src/http/error.rs new file mode 100644 index 0000000000..e5f9a18448 --- /dev/null +++ b/core/server-ng/src/http/error.rs @@ -0,0 +1,818 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! HTTP rejection types and hand-built error responses: the auth / write / +//! read / partition-write error enums, their `IntoResponse` renderings, the +//! `?consistency=` and `?ack=` query DTOs, and the primary-redirect helpers. + +use std::net::{IpAddr, SocketAddr}; + +use axum::Json; +use axum::http::header::{LOCATION, RETRY_AFTER}; +use axum::http::{HeaderValue, StatusCode}; +use axum::response::{IntoResponse, Response}; +use configs::ng_cluster::ResolvedClusterNode; +use iggy_binary_protocol::Operation; +use iggy_common::IggyError; +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use tracing::error; + +use crate::cluster_meta::ClusterRoster; + +#[derive(Debug, Error)] +pub(in crate::http) enum CustomError { + #[error(transparent)] + Error(#[from] IggyError), + #[error("Resource not found")] + ResourceNotFound, +} + +#[derive(Debug, Serialize)] +pub(in crate::http) struct ErrorResponse { + /// Two conventions by construction: the `IggyError` numeric code + /// (`IggyError::as_code`) when the error wraps one (via [`Self::from_error`]), + /// or the HTTP status code for the hand-built HTTP-layer errors (429/503/504 + /// and the 404 not-found fallback) that carry no underlying `IggyError`. + pub id: u32, + pub code: String, + pub reason: String, + pub field: Option, +} + +impl IntoResponse for CustomError { + fn into_response(self) -> Response { + match self { + Self::Error(error) => { + error!("There was an error: {error}"); + let status_code = match error { + IggyError::StreamIdNotFound(_) + | IggyError::TopicIdNotFound(_, _) + | IggyError::PartitionNotFound(_, _, _) + | IggyError::SegmentNotFound + | IggyError::ClientNotFound(_) + | IggyError::ConsumerGroupIdNotFound(_, _) + | IggyError::ConsumerGroupNameNotFound(_, _) + | IggyError::ConsumerGroupMemberNotFound(_, _, _) + | IggyError::ConsumerOffsetNotFound(_) + | IggyError::ResourceNotFound(_) => StatusCode::NOT_FOUND, + IggyError::Unauthenticated + | IggyError::AccessTokenMissing + | IggyError::InvalidAccessToken + | IggyError::InvalidPersonalAccessToken => StatusCode::UNAUTHORIZED, + IggyError::Unauthorized => StatusCode::FORBIDDEN, + // The pre-consensus retry frame: reaching this render + // means the write path's replay budget is exhausted and + // the op never committed - a transient server condition, + // retryable like the other cannot-commit-right-now 503s + // (see `service_unavailable`), never a caller error. + IggyError::TransientNotCommitted | IggyError::TransientNotAccepted => { + StatusCode::SERVICE_UNAVAILABLE + } + _ => StatusCode::BAD_REQUEST, + }; + let response = + (status_code, Json(ErrorResponse::from_error(&error))).into_response(); + // Transient 503s are retryable, so the advisory Retry-After hint + // rides along, matching the other transient 503 bodies + // (`service_unavailable`, `server_busy`). + if matches!( + error, + IggyError::TransientNotCommitted | IggyError::TransientNotAccepted + ) { + with_retry_after(response) + } else { + response + } + } + Self::ResourceNotFound => ( + StatusCode::NOT_FOUND, + Json(ErrorResponse { + id: 404, + code: "not_found".to_string(), + reason: "Resource not found".to_string(), + field: None, + }), + ) + .into_response(), + } + } +} + +impl ErrorResponse { + pub fn from_error(error: &IggyError) -> Self { + Self { + id: error.as_code(), + code: error.as_string().to_string(), + reason: error.to_string(), + field: match error { + IggyError::StreamIdNotFound(_) | IggyError::InvalidStreamId => { + Some("stream_id".to_string()) + } + IggyError::TopicIdNotFound(_, _) | IggyError::InvalidTopicId => { + Some("topic_id".to_string()) + } + IggyError::PartitionNotFound(_, _, _) => Some("partition_id".to_string()), + IggyError::SegmentNotFound => Some("segment_id".to_string()), + IggyError::ClientNotFound(_) => Some("client_id".to_string()), + IggyError::InvalidStreamName + | IggyError::StreamNameAlreadyExists(_) + | IggyError::InvalidTopicName + | IggyError::TopicNameAlreadyExists(_, _) + | IggyError::ConsumerGroupNameAlreadyExists(_, _) + | IggyError::PersonalAccessTokenAlreadyExists(_, _) => Some("name".to_string()), + IggyError::InvalidOffset(_) => Some("offset".to_string()), + IggyError::InvalidConsumerGroupId => Some("consumer_group_id".to_string()), + IggyError::UserAlreadyExists => Some("username".to_string()), + _ => None, + }, + } + } +} + +/// Rejection for protected routes. +/// +/// Two failure classes get two statuses: a missing, invalid, or expired +/// credential is the caller's fault (401, rendered as the JSON `ErrorResponse` +/// body every other route error uses), while a VSR session that cannot be +/// established right now is a transient server condition (503) and must never +/// masquerade as an auth failure. +pub(in crate::http) enum AuthError { + Unauthenticated(IggyError), + /// The Register provably never entered the consensus pipeline (not + /// primary, not caught up, or the prepare queue was full), so the request + /// is safe to re-issue anywhere. Rendered with the `TransientNotAccepted` + /// body so a forwarding follower recognizes it as retryable against a + /// re-resolved primary; a plain client sees the same retryable 503 either + /// way. + SessionNotAccepted, + SessionUnavailable, + /// The `client_id` this gateway minted already has a committed session + /// owned by a DIFFERENT user, so the Register was refused terminally. + /// + /// Distinct from [`Self::SessionUnavailable`] because the status code is + /// the whole point: 503 is about the most auto-retried status there is and + /// no foreign SDK special-cases it, so rendering a permanent, deterministic + /// refusal as 503 hands the caller's HTTP stack a retry loop it can never + /// escape. 409 says the id is taken and stops it. + SessionIdOwnedByAnotherUser, + /// The minted `client_id` already had a committed session for this SAME + /// user, so the Register rebound onto it instead of creating one. Internal + /// to the mint retry in `register_session` and never rendered: the caller + /// mints a different id. Present as a variant so the retry cannot confuse + /// it with a terminal cross-user refusal. + SessionIdTaken, +} + +impl From for AuthError { + fn from(error: IggyError) -> Self { + Self::Unauthenticated(error) + } +} + +impl IntoResponse for AuthError { + fn into_response(self) -> Response { + match self { + // Render 401 through the shared `IggyError -> CustomError` map so it + // carries the same JSON `ErrorResponse` body as every other ng error. + // The legacy server's protected-route 401 comes from a bare-status + // JWT middleware (empty body), so this is deliberately richer, not + // byte-identical to legacy. + Self::Unauthenticated(error) => CustomError::from(error).into_response(), + Self::SessionNotAccepted => { + CustomError::from(IggyError::TransientNotAccepted).into_response() + } + // A fresh session could not be established: the Register was + // canceled with its commit outcome unknown, or the session table + // is at its cap (half `[metadata] clients_table_max`) and refused + // the fresh registration. Transient server condition -> 503, + // retryable by the CLIENT only (a forwarder must not re-issue an + // unknown-outcome Register under this node's session budget on + // the caller's behalf). + // `SessionIdTaken` only escapes the mint retry when every attempt + // collided, which means the minter is wrong rather than unlucky -- + // same unknown-outcome answer as a canceled Register. + Self::SessionUnavailable | Self::SessionIdTaken => service_unavailable(), + // Terminal: retrying cannot change the answer, and admitting it + // would run this caller's replicated ops under the entry owner's + // authority. + Self::SessionIdOwnedByAnotherUser => ( + StatusCode::CONFLICT, + Json(ErrorResponse::from_error(&IggyError::InvalidClientId)), + ) + .into_response(), + } + } +} + +/// Rejection for an authenticated control-plane write (`POST /streams` and the +/// writes that follow it). +/// +/// Same two-class split as [`AuthError`], for the same reasons: a caller-side +/// validation failure or a committed business rejection (e.g. a duplicate +/// stream name) renders through the legacy `IggyError -> CustomError` map so +/// SDK error bodies stay byte-identical, while a write that cannot commit right +/// now is a transient server condition (503) and must never surface as a +/// business error or, worse, a 200 with a stale body. +pub(in crate::http) enum WriteError { + Rejected(IggyError), + /// The VSR session was evicted (its client slot was reclaimed cluster-side, + /// e.g. LRU-evicted from the full client table). Renders identically to a + /// terminal `Rejected` (401 -> re-authenticate), but is a distinct variant + /// so the submit path can drop the dead session entry and let the caller's + /// next request re-register cleanly instead of 401-looping on it. + Evicted(IggyError), + Unavailable, +} + +impl IntoResponse for WriteError { + fn into_response(self) -> Response { + match self { + Self::Rejected(error) | Self::Evicted(error) => { + CustomError::from(error).into_response() + } + Self::Unavailable => service_unavailable(), + } + } +} + +/// Rejection for a data-plane partition write (`POST .../messages` produce and +/// the `PUT`/`DELETE .../consumer-offsets` writes). +/// +/// Split differently from [`WriteError`] because the partition plane replies +/// carry no committed error code: a pre-dispatch gate failure is an +/// empty-bodied reply that names itself only in the header (see +/// [`classify_partition_reply`]), and an unanswered write is a distinct +/// outcome the caller must treat as unknown rather than failed. +#[derive(Debug)] +pub(in crate::http) enum PartitionWriteError { + /// Caller-side rejection (bad identifier, oversized batch, an authorization + /// denial), a typed pre-commit deny from the partition plane + /// (`ReplyHeader.status`), or a malformed reply frame, rendered through the + /// legacy `IggyError -> status` map for SDK-identical bodies. + Rejected(IggyError), + /// Backstop for a status-0 reply carrying `op` 0: an ack with no commit + /// number behind it, for a write that never reached the partition plane. + /// Routing failures name themselves through `ReplyHeader.status`, so this + /// shape is left to a peer that still answers a non-committing op this + /// way. Rendered as the legacy 404 body: the alternative is grading a + /// write that never happened as a success. + NotFound, + /// The in-process reply slot could not be installed. Transient server + /// condition -> the shared 503, retryable. + Unavailable, + /// This session is already at [`MAX_IN_FLIGHT_WRITES_PER_SESSION`] + /// awaited writes. 429: the caller's own concurrency is the problem, so + /// it must drain its outstanding writes before submitting more. + TooManyInFlight, + /// Shard 0 is already at [`MAX_IN_FLIGHT_WRITES_GLOBAL`] awaited writes + /// across all sessions. 503 with its own code (distinct from the shared + /// consensus-unavailable body) so an operator can tell admission shedding + /// from a consensus outage. + ServerBusy, + /// No committed reply within [`PARTITION_WRITE_REPLY_TIMEOUT`], or the + /// session's reply target was torn down mid-wait. 504: the commit may + /// still land (at-least-once), so this is a hard "outcome unknown", not a + /// failure the server may transparently retry. Carries the write's + /// operation so the 504 body names which write kind timed out. + Timeout(Operation), +} + +impl IntoResponse for PartitionWriteError { + fn into_response(self) -> Response { + match self { + Self::Rejected(error) => CustomError::from(error).into_response(), + Self::NotFound => CustomError::ResourceNotFound.into_response(), + Self::Unavailable => service_unavailable(), + Self::TooManyInFlight => too_many_in_flight_response(), + Self::ServerBusy => server_busy_response(), + Self::Timeout(operation) => partition_write_timeout_response(operation), + } + } +} + +/// 504 body for a partition write whose commit outcome is unknown, coded per +/// write kind so a caller can tell a produce timeout from an offset-write +/// timeout. Shaped like every other HTTP error (`ErrorResponse`) so clients +/// parse one error schema. +fn partition_write_timeout_response(operation: Operation) -> Response { + let (code, reason) = match operation { + Operation::SendMessages => ( + "produce_timeout", + "produce was not acknowledged in time; the write may still commit", + ), + _ => ( + "offset_write_timeout", + "consumer-offset write was not acknowledged in time; the write may still commit", + ), + }; + gateway_timeout_response(code, reason) +} + +/// Advisory `Retry-After` seconds for the shed / transient 429 and 503 +/// responses. One second: admission shedding, a briefly unavailable consensus +/// group, and a linearizable-read-on-follower all typically clear well within +/// it, and a small hint keeps a backing-off client responsive. +const RETRY_AFTER_SECONDS: u64 = 1; + +/// Attach the advisory [`RETRY_AFTER_SECONDS`] hint to a retryable 429/503. +pub(in crate::http) fn with_retry_after(mut response: Response) -> Response { + response + .headers_mut() + .insert(RETRY_AFTER, HeaderValue::from(RETRY_AFTER_SECONDS)); + response +} + +/// Render an `ErrorResponse` body for `status`, tagged with `code` / `reason` +/// and no field, so every hand-built HTTP error the routes return parses as the +/// one error schema clients already handle. +pub(in crate::http) fn error_response(status: StatusCode, code: &str, reason: &str) -> Response { + ( + status, + Json(ErrorResponse { + id: status.as_u16().into(), + code: code.to_owned(), + reason: reason.to_owned(), + field: None, + }), + ) + .into_response() +} + +/// Shared 504 rendering for an in-band request the partition plane did not +/// answer in time, shaped like every other HTTP error (`ErrorResponse`) so +/// clients parse one error schema. Consumed by the partition-write reply wait, +/// the partition reads ([`ReadError::Timeout`]), and the forward attempt bound. +pub(in crate::http) fn gateway_timeout_response(code: &str, reason: &str) -> Response { + error_response(StatusCode::GATEWAY_TIMEOUT, code, reason) +} + +/// The shared 503 body for a request that could not commit right now: no +/// caught-up primary, a full pipeline, or a view-change cancel. Retryable, and +/// rendered with the `CannotEstablishConnection` code the SDKs treat as a +/// connection-level retry rather than a terminal error. +fn service_unavailable() -> Response { + with_retry_after( + ( + StatusCode::SERVICE_UNAVAILABLE, + Json(ErrorResponse::from_error( + &IggyError::CannotEstablishConnection, + )), + ) + .into_response(), + ) +} + +/// 429 for a session at [`MAX_IN_FLIGHT_WRITES_PER_SESSION`] awaited partition +/// writes. Shaped like every other HTTP error (`ErrorResponse`) so clients +/// parse one error schema; the remedy is the caller's own: let outstanding +/// writes finish, then retry. +fn too_many_in_flight_response() -> Response { + with_retry_after(error_response( + StatusCode::TOO_MANY_REQUESTS, + "too_many_in_flight_writes", + "session reached its in-flight write cap; await outstanding writes and retry", + )) +} + +/// 503 for shard 0 at [`MAX_IN_FLIGHT_WRITES_GLOBAL`] awaited partition writes +/// across all sessions. A distinct `server_busy` code (unlike the shared +/// consensus-unavailable 503) so admission shedding is tellable from a +/// consensus outage; retry with backoff. +fn server_busy_response() -> Response { + with_retry_after(error_response( + StatusCode::SERVICE_UNAVAILABLE, + "server_busy", + "shard is at its in-flight write budget; retry with backoff", + )) +} + +/// Read consistency selected by the `?consistency=` query param. +/// +/// `serializable` (the default) serves from this node's local metadata STM: +/// correct and consensus-free, but may trail the primary by the replication +/// delay. `linearizable` demands the freshest committed state and is honored +/// only on the primary; a follower redirects (307) to the primary when its HTTP +/// address resolves from the roster, else fails closed to 503 (see +/// [`read_local`]). +#[derive(Clone, Copy, Default, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "lowercase")] +pub(in crate::http) enum Consistency { + #[default] + Serializable, + Linearizable, +} + +/// `?consistency=` query wrapper. An absent param defaults to +/// [`Consistency::Serializable`]; an unrecognized value is a 400 (axum `Query`). +#[derive(Default, Deserialize)] +pub(in crate::http) struct ConsistencyQuery { + #[serde(default)] + pub(in crate::http) consistency: Consistency, +} + +/// Produce acknowledgement selected by the `?ack=` query param. +/// +/// `replicated` (the default) answers 201 only after the partition group's +/// quorum commit. `none` is fire-and-forget: the request is validated, +/// dispatched, and answered 202 immediately; the commit still happens, but its +/// reply is shed at the bus (no reply slot is installed). +#[derive(Clone, Copy, Default, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "lowercase")] +pub(in crate::http) enum ProduceAck { + #[default] + Replicated, + None, +} + +/// `?ack=` query wrapper. An absent param defaults to +/// [`ProduceAck::Replicated`]; an unrecognized value is a 400 (axum `Query`). +#[derive(Default, Deserialize)] +pub(in crate::http) struct ProduceQuery { + #[serde(default)] + pub(in crate::http) ack: ProduceAck, +} + +/// Rejection for an authenticated read route (`GET /streams`, +/// `GET /streams/{id}`, and the reads that follow). +pub(in crate::http) enum ReadError { + /// Caller-side or STM rejection (bad identifier, unsupported op, or an + /// authorization denial) graded through the legacy `IggyError -> status` + /// map so SDK error bodies stay byte-identical. + Rejected(IggyError), + /// Requested entity is absent -> 404 with the legacy not-found body. + NotFound, + /// A linearizable read reached a follower and the primary's HTTP address was + /// not resolvable from the roster. Fail-closed 503, retryable against the + /// leader (see [`not_primary_response`]). + NotPrimary, + /// A linearizable read reached a follower and the current VSR primary's HTTP + /// address resolved: 307 to that address carrying the original path and + /// query, so the caller re-issues the read against the leader (see + /// [`primary_redirect_response`]). + RedirectToPrimary(String), + /// The post-restart read-recovery barrier expired with the recovered WAL + /// suffix still uncommitted: serving now could show state that rolls back + /// history a client already saw acked. Fail-closed 503 via the shared + /// [`service_unavailable`] body, retryable once the cluster re-commits the + /// suffix. + RecoveryIncomplete, + /// A partition read (poll / consumer-offset) got no reply from the owning + /// shard within the mesh budget. 504 like a produce timeout: the outcome is + /// unknown (the abandoned read may still be running), so the caller retries. + Timeout, +} + +impl IntoResponse for ReadError { + fn into_response(self) -> Response { + match self { + Self::Rejected(error) => CustomError::from(error).into_response(), + // Reuse the legacy 404 body so a missing stream renders exactly as + // the legacy server's `CustomError::ResourceNotFound` does. + Self::NotFound => CustomError::ResourceNotFound.into_response(), + Self::NotPrimary => not_primary_response(), + Self::RedirectToPrimary(location) => primary_redirect_response(&location), + Self::RecoveryIncomplete => service_unavailable(), + Self::Timeout => gateway_timeout_response( + "partition_read_timeout", + "the partition owner did not answer the read in time; retry", + ), + } + } +} + +/// The 503 fail-closed body for a linearizable read that reached a follower +/// whose primary HTTP address could not be resolved (absent consensus, a roster +/// with no node at the primary index, or a port-less node). The resolvable case +/// is a 307 via [`primary_redirect_response`] instead. Rendered as an +/// `ErrorResponse` so the body shape matches every other HTTP error; the caller +/// retries against the leader. +fn not_primary_response() -> Response { + with_retry_after(error_response( + StatusCode::SERVICE_UNAVAILABLE, + "not_primary", + "linearizable read requires the primary; retry against the leader", + )) +} + +/// 307 Temporary Redirect to the current VSR primary for a linearizable read +/// that reached a follower. `Location` is the primary's HTTP base plus the +/// original path and query, so the caller re-issues the identical read against +/// the leader. Dormant on a single node (always primary) and followed by no SDK +/// yet. A `Location` that is not a valid header value falls back to the 503. +fn primary_redirect_response(location: &str) -> Response { + HeaderValue::from_str(location).map_or_else( + |_| not_primary_response(), + |value| { + let mut response = StatusCode::TEMPORARY_REDIRECT.into_response(); + response.headers_mut().insert(LOCATION, value); + response + }, + ) +} + +/// Build the `Location` for a 307 redirect of a linearizable read to the VSR +/// primary: `://:`. The scheme is the +/// redirecting node's own listener scheme (uniform cluster HTTP config, same +/// assumption the forward hop makes). `client_ip` is the redirected client's +/// peer address, so the `Location` host comes from the primary's +/// per-client-network selectors when one matches. `None` when the primary does +/// not resolve from the roster, so the caller fails closed to a 503 rather +/// than pointing at an unreachable target. Pure (no consensus or axum +/// dependency) so the redirect target is unit-tested in isolation. +pub(in crate::http) fn primary_redirect_location( + roster: &ClusterRoster, + primary_index: u8, + scheme: &str, + path_and_query: &str, + client_ip: Option, +) -> Option { + let authority = primary_advertised_http_authority(roster, primary_index, client_ip)?; + Some(format!("{scheme}://{authority}{path_and_query}")) +} + +/// Resolve the VSR primary's HTTP socket from the static roster: the node +/// whose `replica_id` equals `primary_index`, its `ports.http`, and its +/// private roster `ip` (parsed once at roster build). Internal replica +/// forwarding uses this address; it must never route through +/// [`ResolvedClusterNode::advertised_for`], which picks client-facing hosts. +pub(in crate::http) fn primary_http_socket( + roster: &ClusterRoster, + primary_index: u8, +) -> Option { + let (node, http_port) = primary_node(roster, primary_index)?; + Some(SocketAddr::new(node.replica_ip()?, http_port)) +} + +/// Resolve the client-facing HTTP authority (`host:port`) for a redirect +/// through [`ResolvedClusterNode::advertised_for`]: a client-network selector +/// match first, then the catch-all advertised address, then the private +/// roster IP as the compatibility fallback. `AdvertisedAddress::authority` +/// brackets IPv6 hosts and passes hostnames through, so the redirect URL +/// stays valid. This is the fail-closed caller: a host that is neither a +/// valid IP nor a valid hostname yields `None` and the redirect becomes a +/// 503 rather than a `Location` pointing at an unparsable target (cluster +/// metadata makes the opposite choice and publishes such a host verbatim). +fn primary_advertised_http_authority( + roster: &ClusterRoster, + primary_index: u8, + client_ip: Option, +) -> Option { + let (node, http_port) = primary_node(roster, primary_index)?; + let address = node.advertised_for(client_ip)?; + Some(address.authority(http_port)) +} + +fn primary_node(roster: &ClusterRoster, primary_index: u8) -> Option<(&ResolvedClusterNode, u16)> { + let node = roster + .nodes + .iter() + .find(|node| node.config().replica_id == primary_index)?; + let http_port = node.config().ports.http?; + Some((node, http_port)) +} + +#[cfg(test)] +mod tests { + use super::*; + + use configs::ng_cluster::{ClusterNodeConfig, TransportPorts}; + + const READ_PATH: &str = "/streams?consistency=linearizable"; + fn node(replica_id: u8, ip: &str, http: Option) -> ClusterNodeConfig { + ClusterNodeConfig { + name: format!("node-{replica_id}"), + ip: ip.to_owned(), + advertised_address: None, + advertised_addresses: Vec::new(), + replica_id, + ports: TransportPorts { + tcp: None, + quic: None, + http, + websocket: None, + tcp_replica: None, + }, + } + } + + fn roster(nodes: Vec) -> ClusterRoster { + ClusterRoster { + enabled: true, + name: "test-cluster".to_owned(), + nodes: nodes.into_iter().map(Into::into).collect(), + self_ip: "127.0.0.1".to_owned(), + self_ports: TransportPorts::default(), + metadata_view: std::sync::Arc::new(std::sync::atomic::AtomicU64::new( + crate::cluster_meta::METADATA_VIEW_UNKNOWN, + )), + } + } + + #[test] + fn primary_redirect_location_targets_primary_http_addr_with_path_passthrough() { + let roster = roster(vec![ + node(0, "10.0.0.1", Some(8080)), + node(1, "10.0.0.2", Some(8090)), + ]); + assert_eq!( + primary_redirect_location(&roster, 1, "http", READ_PATH, None), + Some("http://10.0.0.2:8090/streams?consistency=linearizable".to_owned()) + ); + } + + #[test] + fn primary_redirect_location_uses_the_listener_scheme() { + let roster = roster(vec![node(0, "10.0.0.1", Some(8080))]); + assert_eq!( + primary_redirect_location(&roster, 0, "https", READ_PATH, None), + Some("https://10.0.0.1:8080/streams?consistency=linearizable".to_owned()) + ); + } + + #[test] + fn primary_redirect_location_is_none_when_no_node_matches_primary_index() { + let roster = roster(vec![node(0, "10.0.0.1", Some(8080))]); + assert_eq!( + primary_redirect_location(&roster, 2, "http", READ_PATH, None), + None + ); + } + + #[test] + fn primary_redirect_location_is_none_when_primary_has_no_http_port() { + let roster = roster(vec![node(0, "10.0.0.1", None)]); + assert_eq!( + primary_redirect_location(&roster, 0, "http", READ_PATH, None), + None + ); + } + + #[test] + fn primary_redirect_location_is_none_for_empty_roster() { + let roster = roster(Vec::new()); + assert_eq!( + primary_redirect_location(&roster, 0, "http", READ_PATH, None), + None + ); + } + + #[test] + fn primary_redirect_location_brackets_ipv6_host() { + let roster = roster(vec![node(0, "::1", Some(8080))]); + assert_eq!( + primary_redirect_location(&roster, 0, "http", READ_PATH, None), + Some("http://[::1]:8080/streams?consistency=linearizable".to_owned()) + ); + } + + #[test] + fn primary_redirect_location_uses_advertised_address() { + let mut primary = node(0, "10.0.0.1", Some(8080)); + primary.advertised_address = Some("2001:db8::1".to_owned()); + let roster = roster(vec![primary]); + + assert_eq!( + primary_redirect_location(&roster, 0, "https", READ_PATH, None), + Some("https://[2001:db8::1]:8080/streams?consistency=linearizable".to_owned()) + ); + } + + #[test] + fn primary_redirect_location_uses_advertised_hostname() { + let mut primary = node(0, "10.0.0.1", Some(8080)); + primary.advertised_address = Some("broker-1.example.com".to_owned()); + let roster = roster(vec![primary]); + + assert_eq!( + primary_redirect_location(&roster, 0, "https", READ_PATH, None), + Some("https://broker-1.example.com:8080/streams?consistency=linearizable".to_owned()) + ); + } + + #[test] + fn primary_redirect_location_uses_the_selector_address_for_a_matching_client() { + let mut primary = node(0, "10.0.0.1", Some(8080)); + primary.advertised_address = Some("203.0.113.1".to_owned()); + primary.advertised_addresses = vec![configs::ng_cluster::AdvertisedAddressSelector { + client_cidr: "10.0.0.0/16".to_owned(), + address: "10.0.0.1".to_owned(), + }]; + let roster = roster(vec![primary]); + + assert_eq!( + primary_redirect_location( + &roster, + 0, + "https", + READ_PATH, + Some("10.0.9.9".parse().unwrap()) + ), + Some("https://10.0.0.1:8080/streams?consistency=linearizable".to_owned()), + "an in-network client must be redirected to the selector address" + ); + assert_eq!( + primary_redirect_location( + &roster, + 0, + "https", + READ_PATH, + Some("198.51.100.7".parse().unwrap()) + ), + Some("https://203.0.113.1:8080/streams?consistency=linearizable".to_owned()), + "an out-of-network client must stay on the catch-all address" + ); + } + + #[test] + fn primary_http_socket_uses_private_roster_ip() { + let mut primary = node(0, "10.0.0.1", Some(8080)); + primary.advertised_address = Some("203.0.113.1".to_owned()); + let roster = roster(vec![primary]); + + assert_eq!( + primary_http_socket(&roster, 0), + Some("10.0.0.1:8080".parse().expect("valid socket address")) + ); + } + + #[test] + fn transient_not_committed_renders_503_with_retry_after() { + let response = CustomError::from(IggyError::TransientNotCommitted).into_response(); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert!(response.headers().contains_key(RETRY_AFTER)); + } + + #[test] + fn transient_not_accepted_renders_503_with_retry_after() { + let response = CustomError::from(IggyError::TransientNotAccepted).into_response(); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert!(response.headers().contains_key(RETRY_AFTER)); + } + + #[test] + fn business_error_renders_without_retry_after() { + let response = CustomError::from(IggyError::UserAlreadyExists).into_response(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + assert!(!response.headers().contains_key(RETRY_AFTER)); + } + + // The ownership refusal is permanent and deterministic. Rendering it as + // 503 would hand the caller's HTTP stack a retry loop it can never escape + // (no foreign SDK special-cases 503), so the status is load-bearing. + #[test] + fn owned_client_id_renders_as_terminal_conflict() { + let response = AuthError::SessionIdOwnedByAnotherUser.into_response(); + assert_eq!(response.status(), StatusCode::CONFLICT); + assert!( + response.headers().get(RETRY_AFTER).is_none(), + "a terminal refusal must not advertise a retry" + ); + } + + // Its siblings stay retryable, so the split is visible in one place. + #[test] + fn unknown_outcome_registers_stay_retryable() { + for error in [AuthError::SessionUnavailable, AuthError::SessionNotAccepted] { + let status = error.into_response().status(); + assert!( + status.is_server_error(), + "an unknown commit outcome must stay retryable, got {status}" + ); + } + } + + #[test] + fn recovery_incomplete_renders_retryable_503_like_not_primary() { + // Barrier expiry must render as the shared retryable 503: the same + // status and Retry-After hint as the not-primary 503, so an SDK treats + // it as a connection-level retry rather than a terminal error. + let recovery = ReadError::RecoveryIncomplete.into_response(); + assert_eq!(recovery.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!( + recovery.headers().get(RETRY_AFTER), + Some(&HeaderValue::from(RETRY_AFTER_SECONDS)) + ); + + let not_primary = ReadError::NotPrimary.into_response(); + assert_eq!(recovery.status(), not_primary.status()); + assert_eq!( + recovery.headers().get(RETRY_AFTER), + not_primary.headers().get(RETRY_AFTER) + ); + } +} diff --git a/core/server/src/http/extractor.rs b/core/server-ng/src/http/extractor.rs similarity index 100% rename from core/server/src/http/extractor.rs rename to core/server-ng/src/http/extractor.rs diff --git a/core/server/src/http/forward.rs b/core/server-ng/src/http/forward.rs similarity index 98% rename from core/server/src/http/forward.rs rename to core/server-ng/src/http/forward.rs index 5e4572f701..08f8608eba 100644 --- a/core/server/src/http/forward.rs +++ b/core/server-ng/src/http/forward.rs @@ -84,7 +84,7 @@ use crate::http::error::{ }; use crate::http::extractor::{bearer_token, resolve_credential}; use crate::http::state::{HttpInner, VIEW_HEADER}; -use crate::server_error::ServerError; +use crate::server_error::ServerNgError; /// Marker stamped on every forwarded request. Loop guard only: a node that is /// not primary and sees it answers the transient 503 instead of forwarding @@ -166,14 +166,14 @@ pub(in crate::http) struct ForwardState { /// /// # Errors /// -/// [`ServerError::ListenerCredentials`] when TLS is enabled but the PEM -/// files cannot be loaded, [`ServerError::HttpForwardClient`] when the +/// [`ServerNgError::ListenerCredentials`] when TLS is enabled but the PEM +/// files cannot be loaded, [`ServerNgError::HttpForwardClient`] when the /// outbound client cannot be built. pub(in crate::http) fn build_forward_state( tls: &HttpTlsConfig, body_limit: usize, active: bool, -) -> Result { +) -> Result { // Unconditional: cyper's rustls connector resolves the process default // provider even when the client only ever dials plain http. install_default_crypto_provider(); @@ -184,7 +184,7 @@ pub(in crate::http) fn build_forward_state( let (builder, scheme) = if tls.enabled { let credentials = load_pem(Path::new(&tls.cert_file), Path::new(&tls.key_file)).map_err(|source| { - ServerError::ListenerCredentials { + ServerNgError::ListenerCredentials { transport: "http.tls", source, } @@ -192,7 +192,7 @@ pub(in crate::http) fn build_forward_state( // load_pem guarantees a non-empty chain; this is the no-panic // path for the unreachable empty case. let pinned = credentials.cert_chain.into_iter().next().ok_or_else(|| { - ServerError::HttpForwardClient { + ServerNgError::HttpForwardClient { reason: "TLS certificate chain is empty".to_string(), } })?; @@ -218,7 +218,7 @@ pub(in crate::http) fn build_forward_state( }; let client = builder .build() - .map_err(|source| ServerError::HttpForwardClient { + .map_err(|source| ServerNgError::HttpForwardClient { reason: source.to_string(), })?; Ok(ForwardState { diff --git a/core/server/src/http/handlers.rs b/core/server-ng/src/http/handlers.rs similarity index 99% rename from core/server/src/http/handlers.rs rename to core/server-ng/src/http/handlers.rs index cc5d501c56..0dc160fe29 100644 --- a/core/server/src/http/handlers.rs +++ b/core/server-ng/src/http/handlers.rs @@ -218,7 +218,7 @@ pub(in crate::http) struct RefreshToken { /// `POST /users/refresh-token`: re-issue an access token from a still-valid one, /// answering the same `IdentityInfo` shape as login. /// -/// Stateless by design: server has no replicated revocation list (the P3 +/// Stateless by design: server-ng has no replicated revocation list (the P3 /// roadmap item), so refreshing cannot invalidate the presented token - it stays /// valid until its own `exp`, the same posture as logout ending a session /// without revoking its bearer. Per-node revocation would be false security in a @@ -691,7 +691,7 @@ pub(in crate::http) async fn get_client( /// `StreamDetails` JSON the legacy server returns. /// /// The accepted body is name-only (`{"name": ...}`), matching the legacy -/// request; server's wire `CreateStreamRequest` is likewise name-only and +/// request; server-ng's wire `CreateStreamRequest` is likewise name-only and /// auto-assigns the id, so there is no client-supplied stream id to honor. pub(in crate::http) async fn create_stream( State(state): State, diff --git a/core/server/src/http/jwks.rs b/core/server-ng/src/http/jwks.rs similarity index 100% rename from core/server/src/http/jwks.rs rename to core/server-ng/src/http/jwks.rs diff --git a/core/server/src/http/jwt.rs b/core/server-ng/src/http/jwt.rs similarity index 98% rename from core/server/src/http/jwt.rs rename to core/server-ng/src/http/jwt.rs index 29fae7e437..c5053e1470 100644 --- a/core/server/src/http/jwt.rs +++ b/core/server-ng/src/http/jwt.rs @@ -17,7 +17,7 @@ //! Minimal JWT issuer/verifier for the shard-0 HTTP listener. //! -//! Ported from the legacy server implementation's `JwtManager`, reduced +//! Ported from the legacy `server::http::jwt::jwt_manager::JwtManager`, reduced //! to the issue + verify half: no revoked-token persistence. Self-issued HS256 //! is the common path; with `[[http.jwt.trusted_issuers]]` configured, `decode` //! also verifies external RS256/EC tokens against the issuer's JWKS @@ -54,11 +54,7 @@ const GENERATED_SECRET_LEN: Range = 32..64; /// with this fallback a PSK compromise also yields the token-signing key, and /// rotating the PSK invalidates all bearers; operators who want the domains /// decoupled configure an explicit `http.jwt` secret, which always wins. -/// -/// Frozen string: it is KDF domain separation, not a label. Editing it derives -/// a different signing key and invalidates every bearer issued before the -/// change. -const JWT_KEY_CONTEXT: &str = "apache-iggy server http-jwt v1 psk->hs256-key"; +const JWT_KEY_CONTEXT: &str = "apache-iggy server-ng http-jwt v1 psk->hs256-key"; /// Expiry stamp used for a non-expiring token: far enough out to never trip /// `exp` validation, small enough to fit `u32`. Mirrors the legacy server. diff --git a/core/server-ng/src/http/metrics.rs b/core/server-ng/src/http/metrics.rs new file mode 100644 index 0000000000..bb18a8a9b9 --- /dev/null +++ b/core/server-ng/src/http/metrics.rs @@ -0,0 +1,297 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! The `[http.metrics]` scrape surface: the legacy-parity metric registry +//! (entity gauges plus the request counter), its public scrape handler, and +//! the config gate deciding whether the route is mounted. + +use axum::extract::State; +use configs::http::HttpMetricsConfig; +use consensus::MetadataHandle; +use iggy_common::IggyError; +use metadata::impls::metadata::StreamsFrontend; +use prometheus_client::encoding::text::encode; +use prometheus_client::metrics::counter::Counter; +use prometheus_client::metrics::gauge::Gauge; +use prometheus_client::registry::Registry; +use send_wrapper::SendWrapper; +use tracing::error; + +use crate::http::state::HttpState; + +/// The legacy server's metric set, registered under the same names and help +/// texts so existing dashboards and alerts keep working unchanged. +/// +/// Unlike the legacy server, the entity gauges are not counted at mutation +/// sites: [`get_metrics`] samples the live state on every scrape, so a gauge +/// can never drift from the state it describes. +pub(in crate::http) struct HttpMetrics { + registry: Registry, + http_requests: Counter, + streams: Gauge, + topics: Gauge, + partitions: Gauge, + segments: Gauge, + messages: Gauge, + users: Gauge, + clients: Gauge, +} + +impl HttpMetrics { + pub(in crate::http) fn init() -> Self { + let mut registry = Registry::default(); + let http_requests = Counter::default(); + let streams = Gauge::default(); + let topics = Gauge::default(); + let partitions = Gauge::default(); + let segments = Gauge::default(); + let messages = Gauge::default(); + let users = Gauge::default(); + let clients = Gauge::default(); + registry.register( + "http_requests", + "total count of http_requests", + http_requests.clone(), + ); + registry.register("streams", "total count of streams", streams.clone()); + registry.register("topics", "total count of topics", topics.clone()); + registry.register( + "partitions", + "total count of partitions", + partitions.clone(), + ); + registry.register("segments", "total count of segments", segments.clone()); + registry.register("messages", "total count of messages", messages.clone()); + registry.register("users", "total count of users", users.clone()); + registry.register("clients", "total count of clients", clients.clone()); + Self { + registry, + http_requests, + streams, + topics, + partitions, + segments, + messages, + users, + clients, + } + } + + /// Handle for the router's request-counting layer. The counter is + /// `Arc`-backed, so bumping the clone bumps the registered metric. + pub(in crate::http) fn request_counter(&self) -> Counter { + self.http_requests.clone() + } + + fn formatted_output(&self) -> String { + let mut buffer = String::new(); + if let Err(error) = encode(&mut buffer, &self.registry) { + error!(%error, "failed to encode metrics"); + } + buffer + } +} + +/// Resolve the configured scrape path: `None` when `[http.metrics]` is +/// disabled, so the route is never mounted and the endpoint answers 404. +/// +/// axum's `Router::route` panics on a path without a leading `/`, so an +/// enabled endpoint missing one is rejected as a configuration error before +/// the router is assembled. +/// +/// # Errors +/// +/// Returns [`IggyError::InvalidConfiguration`] when metrics are enabled and +/// the endpoint does not start with `/`. +pub(in crate::http) fn validated_endpoint( + config: &HttpMetricsConfig, +) -> Result, IggyError> { + if !config.enabled { + return Ok(None); + } + if !config.endpoint.starts_with('/') { + error!( + endpoint = %config.endpoint, + "invalid http.metrics.endpoint: the path must start with '/'" + ); + return Err(IggyError::InvalidConfiguration); + } + Ok(Some(config.endpoint.clone())) +} + +/// `GET `: the metric set in prometheus text +/// exposition. Public - reached without proving a credential, exactly like the +/// legacy endpoint. +/// +/// The entity gauges sample the same reads `/stats` serves: the metadata STM +/// stream and user maps plus the stats-registry rollups, whose partition-plane +/// increments are relaxed, so scraped values are approximate while writes are +/// in flight. The clients count scatter-gathers the per-shard session managers +/// exactly like `GET /clients` and turns partial when a shard misses the reply +/// deadline. +pub(in crate::http) async fn get_metrics(State(state): State) -> String { + let (streams_count, topics_count, partitions_count, segments_count, messages_count) = state + .shard + .plane + .metadata() + .mux_stm + .streams() + .read(|streams| { + let mut topics_count = 0u64; + let mut partitions_count = 0u64; + let mut segments_count = 0u64; + let mut messages_count = 0u64; + for (_, stream) in &streams.items { + topics_count = topics_count.saturating_add(stream.topics.len() as u64); + segments_count = segments_count + .saturating_add(u64::from(stream.stats.segments_count_inconsistent())); + messages_count = + messages_count.saturating_add(stream.stats.messages_count_inconsistent()); + for (_, topic) in &stream.topics { + partitions_count = + partitions_count.saturating_add(topic.partitions.len() as u64); + } + } + ( + streams.items.len() as u64, + topics_count, + partitions_count, + segments_count, + messages_count, + ) + }); + let users_count = state + .shard + .plane + .metadata() + .mux_stm + .users() + .read(|users| users.items.len() as u64); + let clients_count = SendWrapper::new(state.shard.list_all_clients()).await.len() as u64; + + let metrics = &state.metrics; + metrics.streams.set(gauge_value(streams_count)); + metrics.topics.set(gauge_value(topics_count)); + metrics.partitions.set(gauge_value(partitions_count)); + metrics.segments.set(gauge_value(segments_count)); + metrics.messages.set(gauge_value(messages_count)); + metrics.users.set(gauge_value(users_count)); + metrics.clients.set(gauge_value(clients_count)); + metrics.formatted_output() +} + +/// Clamp a count into the gauge's `i64` domain; only `messages` can pass +/// `i64::MAX` even in theory, the rest are bounded far below it. +fn gauge_value(count: u64) -> i64 { + i64::try_from(count).unwrap_or(i64::MAX) +} + +#[cfg(test)] +mod tests { + use super::*; + + const PARITY_METRIC_NAMES: [&str; 8] = [ + "http_requests", + "streams", + "topics", + "partitions", + "segments", + "messages", + "users", + "clients", + ]; + + fn metrics_config(enabled: bool, endpoint: &str) -> HttpMetricsConfig { + HttpMetricsConfig { + enabled, + endpoint: endpoint.to_owned(), + } + } + + #[test] + fn formatted_output_exposes_every_parity_metric() { + let metrics = HttpMetrics::init(); + let output = metrics.formatted_output(); + for name in PARITY_METRIC_NAMES { + assert!( + output.contains(&format!("# TYPE {name} ")), + "metric {name} missing from exposition:\n{output}" + ); + } + assert!( + output.ends_with("# EOF\n"), + "missing exposition trailer:\n{output}" + ); + } + + #[test] + fn scraped_values_land_in_the_exposition() { + let metrics = HttpMetrics::init(); + metrics.streams.set(1); + metrics.topics.set(2); + metrics.partitions.set(3); + metrics.segments.set(4); + metrics.messages.set(5); + metrics.users.set(6); + metrics.clients.set(7); + metrics.request_counter().inc(); + let output = metrics.formatted_output(); + for line in [ + "streams 1", + "topics 2", + "partitions 3", + "segments 4", + "messages 5", + "users 6", + "clients 7", + "http_requests_total 1", + ] { + assert!( + output.contains(&format!("\n{line}\n")), + "expected `{line}` in exposition:\n{output}" + ); + } + } + + #[test] + fn gauge_value_clamps_past_i64_range() { + assert_eq!(gauge_value(42), 42); + assert_eq!(gauge_value(u64::MAX), i64::MAX); + } + + #[test] + fn validated_endpoint_disabled_yields_none() { + assert!(matches!( + validated_endpoint(&metrics_config(false, "/metrics")), + Ok(None) + )); + } + + #[test] + fn validated_endpoint_returns_enabled_path() { + let endpoint = validated_endpoint(&metrics_config(true, "/metrics")).unwrap(); + assert_eq!(endpoint.as_deref(), Some("/metrics")); + } + + #[test] + fn validated_endpoint_rejects_missing_leading_slash() { + assert!(matches!( + validated_endpoint(&metrics_config(true, "metrics")), + Err(IggyError::InvalidConfiguration) + )); + } +} diff --git a/core/server/src/http/reads.rs b/core/server-ng/src/http/reads.rs similarity index 95% rename from core/server/src/http/reads.rs rename to core/server-ng/src/http/reads.rs index 9955242065..9246ad6109 100644 --- a/core/server/src/http/reads.rs +++ b/core/server-ng/src/http/reads.rs @@ -19,16 +19,14 @@ //! metadata-STM read entry, and the wire/domain identifier resolvers the read //! and data-plane routes ground their scopes through. -use crate::bootstrap::ServerShard; +use crate::bootstrap::ServerNgShard; use bytes::Bytes; use consensus::MetadataHandle; use iggy_binary_protocol::WireIdentifier; -use iggy_binary_protocol::codes::GET_STATS_CODE; use iggy_common::wire_conversions::identifier_to_wire; use iggy_common::{Identifier, IggyError}; use metadata::impls::metadata::StreamsFrontend; use metadata::permissioner::Permissioner; -use send_wrapper::SendWrapper; use std::rc::Rc; use crate::http::error::{Consistency, ReadError}; @@ -80,9 +78,8 @@ pub(in crate::http) fn authorize_read( /// a TCP read of the same entity return byte-identical bodies. /// /// Reads never touch consensus or a VSR session: `build_non_replicated_response` -/// is a pure STM read, with one exception - the stats read's cross-shard -/// connected-client gather, an async broadcast run here (under `SendWrapper`, -/// same as `/metrics`) before the sync builder. An absent entity surfaces as +/// is a pure STM read. It is synchronous, so this helper is too - no submit +/// await, no gate, no `SendWrapper`. An absent entity surfaces as /// [`NonReplicatedResponse::Empty`], mapped to 404 here because every REST read /// whose entity can be missing shares that not-found shape. pub(in crate::http) async fn read_local( @@ -95,12 +92,6 @@ pub(in crate::http) async fn read_local( ) -> Result { await_recovery_barrier(&state.shard).await?; authorize_read(state, identity, consistency, rule)?; - let clients_count = if code == GET_STATS_CODE { - u32::try_from(SendWrapper::new(state.shard.list_all_clients()).await.len()) - .unwrap_or(u32::MAX) - } else { - 0 - }; match build_non_replicated_response( &state.shard, code, @@ -108,7 +99,6 @@ pub(in crate::http) async fn read_local( Some(identity.user_id), &state.roster, identity.client_ip, - clients_count, ) .map_err(ReadError::Rejected)? { @@ -159,7 +149,7 @@ const fn barrier_state(barrier: u64, commit_min: u64, expired: bool) -> BarrierW /// state a client already saw acked; the caller retries against a converged /// cluster. pub(in crate::http) async fn await_recovery_barrier( - shard: &Rc, + shard: &Rc, ) -> Result<(), ReadError> { const POLL: std::time::Duration = std::time::Duration::from_millis(10); diff --git a/core/server/src/http/reply.rs b/core/server-ng/src/http/reply.rs similarity index 99% rename from core/server/src/http/reply.rs rename to core/server-ng/src/http/reply.rs index 9c3a59f889..d039c64c78 100644 --- a/core/server/src/http/reply.rs +++ b/core/server-ng/src/http/reply.rs @@ -105,7 +105,7 @@ pub(in crate::http) fn send_confirmations( Err(error) => { warn!( ?error, - "server HTTP: undecodable send_messages commit confirmation" + "server-ng HTTP: undecodable send_messages commit confirmation" ); None } @@ -315,7 +315,7 @@ mod tests { let commit = 9; let body = Bytes::from_static(b"body"); - // Reply builders, all funnelled through build_reply_with_body. + // server-ng reply builders, all funnelled through build_reply_with_body. for status in [ build_reply_with_body(header, 42, 7, commit, 0, |_| {}) .header() diff --git a/core/server/src/http/session.rs b/core/server-ng/src/http/session.rs similarity index 98% rename from core/server/src/http/session.rs rename to core/server-ng/src/http/session.rs index 2e104268d4..f6fd3df06a 100644 --- a/core/server/src/http/session.rs +++ b/core/server-ng/src/http/session.rs @@ -85,10 +85,10 @@ pub(in crate::http) struct HttpSession { pub(in crate::http) key: String, /// Shard-0 client id minted for this credential; its top 16 bits are 0, so /// it shares the shard-0 id space with TCP virtual clients without - /// colliding. Fills `RoutedRequestHeader.client` on every write. + /// colliding. Fills `RequestHeader.client` on every write. pub(in crate::http) client_id: u128, /// Cluster session number returned by the VSR `Register` commit. Fills - /// `RoutedRequestHeader.session` on every write. + /// `RequestHeader.session` on every write. pub(in crate::http) session: u64, /// User the credential authenticated as. Consumed by the write path for /// authorization. @@ -384,7 +384,7 @@ mod tests { // deployment untouched. #[test] fn runtime_cap_at_default_config_equals_pinned_default() { - let configured = configs::metadata::MetadataConfig::default().clients_table_max; + let configured = configs::ng_metadata::MetadataConfig::default().clients_table_max; assert_eq!(max_http_sessions(configured), DEFAULT_MAX_HTTP_SESSIONS); } diff --git a/core/server/src/http/state.rs b/core/server-ng/src/http/state.rs similarity index 97% rename from core/server/src/http/state.rs rename to core/server-ng/src/http/state.rs index 0c8a2cca29..7efb8bcc4c 100644 --- a/core/server/src/http/state.rs +++ b/core/server-ng/src/http/state.rs @@ -27,7 +27,7 @@ use std::sync::Arc; use axum::http::{HeaderName, HeaderValue}; use axum::response::Response; -use configs::server::ServerSystemConfig; +use configs::server_ng::NgSystemConfig; use consensus::{MetadataHandle, VsrConsensus}; use futures::channel::oneshot; use iggy_common::{ClusterMetadata, IggyTimestamp}; @@ -37,7 +37,7 @@ use send_wrapper::SendWrapper; use tokio::sync::Mutex; use tracing::warn; -use crate::bootstrap::ServerShard; +use crate::bootstrap::ServerNgShard; use crate::cluster_meta::ClusterRoster; use crate::dispatch::submit_register_on_owner; use crate::http::error::{AuthError, ReadError, primary_redirect_location}; @@ -69,12 +69,12 @@ pub(in crate::http) type HttpState = SendWrapper>; /// session table so every handler and the [`Authenticated`] extractor reach /// them through one axum `State`. pub(in crate::http) struct HttpInner { - pub(in crate::http) shard: Rc, + pub(in crate::http) shard: Rc, pub(in crate::http) jwt: JwtManager, /// Read-only server config for the snapshot collector (log directory + /// runtime config paths); the shard does not expose config on the read /// path. - pub(in crate::http) system_config: Arc, + pub(in crate::http) system_config: Arc, /// Per-credential VSR sessions keyed by JWT `jti` / PAT hash. `RefCell` is /// sound here - shard 0 is single-threaded and the `SendWrapper` state /// bridge tolerates the `!Sync` interior - but the guard must never be held @@ -256,7 +256,7 @@ impl HttpInner { { warn!( attempt, - "server HTTP: minted client id was already registered; re-minting" + "server-ng HTTP: minted client id was already registered; re-minting" ); } Err(error) => return Err(error), @@ -325,7 +325,7 @@ impl HttpInner { .await .map_err(|_| AuthError::SessionUnavailable)? .map_err(|error| { - warn!(?error, "server HTTP: VSR Register submit failed"); + warn!(?error, "server-ng HTTP: VSR Register submit failed"); match error { // The Register never entered the pipeline, so re-issuing // it anywhere is safe; the transient-not-accepted body @@ -361,7 +361,7 @@ impl HttpInner { client_id, user_id, watermark = bound.watermark, - "server HTTP: minted client id already had a committed session for this user" + "server-ng HTTP: minted client id already had a committed session for this user" ); return Err(AuthError::SessionIdTaken); } diff --git a/core/server/src/http/submit.rs b/core/server-ng/src/http/submit.rs similarity index 97% rename from core/server/src/http/submit.rs rename to core/server-ng/src/http/submit.rs index f9543cda8d..07c45b1824 100644 --- a/core/server/src/http/submit.rs +++ b/core/server-ng/src/http/submit.rs @@ -26,14 +26,14 @@ use bytes::Bytes; use consensus::MetadataHandle; use futures::channel::oneshot; use iggy_binary_protocol::consensus::Command2; -use iggy_binary_protocol::{GenericHeader, Operation, ReplyHeader, RoutedRequestHeader}; +use iggy_binary_protocol::{GenericHeader, Operation, ReplyHeader, RequestHeader}; use iggy_common::IggyError; use message_bus::BusMessage; use metadata::impls::metadata::StreamsFrontend; use server_common::Message; use tracing::warn; -use crate::bootstrap::ServerShard; +use crate::bootstrap::ServerNgShard; use crate::dispatch::{ dispatch_partition_request, resolve_delete_segments_truncate, submit_client_request_on_owner, submit_logout_on_owner, @@ -98,7 +98,7 @@ pub(in crate::http) async fn submit_committed( session: &Rc, operation: Operation, body: &[u8], -) -> Result<(RoutedRequestHeader, Message, Option), WriteError> { +) -> Result<(RequestHeader, Message, Option), WriteError> { // Control writes are authorized in-apply on the metadata STM: a denial // comes back as `Unauthorized` in the committed result section, which // `committed_payload` maps to a 403. No pre-submit gate here, so the @@ -178,12 +178,12 @@ pub(in crate::http) async fn submit_committed( /// that the replicated apply grades to `InvalidCredentials` (see /// `verify_and_rewrite_change_password`). async fn submit_gated( - shard: &Rc, + shard: &Rc, session: &HttpSession, operation: Operation, max_tokens_per_user: u32, body: &[u8], -) -> Result<(RoutedRequestHeader, Message, Option), WriteError> { +) -> Result<(RequestHeader, Message, Option), WriteError> { let mut next_request_id = session.gate.lock().await; // Burn the id at stamp time: every exit below (rewrite rejection, // unresolved delete-segments, unanswered submit, exhausted transient @@ -343,10 +343,10 @@ pub(in crate::http) async fn logout_session(state: &HttpInner, session: &Rc {} Ok(Err(error)) => warn!( ?error, - "server HTTP: VSR Logout submit failed; slot lingers until eviction" + "server-ng HTTP: VSR Logout submit failed; slot lingers until eviction" ), Err(_canceled) => warn!( - "server HTTP: VSR Logout task dropped before replying; slot lingers until eviction" + "server-ng HTTP: VSR Logout task dropped before replying; slot lingers until eviction" ), } state.forget_session(session); @@ -396,7 +396,7 @@ pub(in crate::http) async fn partition_write_replicated( warn!( ?error, ?operation, - "server HTTP: partition write reply slot install failed" + "server-ng HTTP: partition write reply slot install failed" ); PartitionWriteError::Unavailable })?; @@ -464,7 +464,7 @@ pub(in crate::http) async fn produce_unacked( /// Install this session's in-process reply target on first data-plane use. /// /// The registry key is the session's shard-0 client id - the same id stamped -/// into `RoutedRequestHeader.client` - so a partition reply routed through +/// into `RequestHeader.client` - so a partition reply routed through /// `send_to_client` lands on this entry and resolves the request-keyed slot. /// `None` from the registry means the key is already occupied; treat it as /// installed but leave the token unset so this session never tears down an diff --git a/core/server/src/http/tls.rs b/core/server-ng/src/http/tls.rs similarity index 92% rename from core/server/src/http/tls.rs rename to core/server-ng/src/http/tls.rs index a7b8510955..b4e0137a5a 100644 --- a/core/server/src/http/tls.rs +++ b/core/server-ng/src/http/tls.rs @@ -17,7 +17,7 @@ //! HTTPS for the shard-0 REST listener. //! -//! The server is thread-per-core `compio/io_uring`, so it cannot reuse the +//! server-ng is thread-per-core `compio/io_uring`, so it cannot reuse the //! legacy `axum-server` TLS acceptor. The plain-HTTP path uses //! `cyper_axum::serve`; the TLS path cannot, because that serve loop wraps //! its IO in a `compio` `Split` (one `BiLock`): hyper parks a pending read @@ -59,7 +59,7 @@ use tower_http::add_extension::AddExtension; use tracing::{debug, error}; use crate::http::ClientAddr; -use crate::server_error::ServerError; +use crate::server_error::ServerNgError; /// hyper's auto-builder serves whichever protocol the client selects via /// ALPN; advertise the same pair `axum-server` negotiates by default on the @@ -82,14 +82,14 @@ type Handshaken = (TlsStream, SocketAddr); /// /// # Errors /// -/// [`ServerError::ListenerCredentials`] with `transport: "http.tls"` if +/// [`ServerNgError::ListenerCredentials`] with `transport: "http.tls"` if /// the PEM files cannot be read or the certificate / key pair is rejected. pub fn load_http_tls_server_config( tls: &HttpTlsConfig, -) -> Result, ServerError> { +) -> Result, ServerNgError> { let credentials = load_pem(Path::new(&tls.cert_file), Path::new(&tls.key_file)).map_err(|source| { - ServerError::ListenerCredentials { + ServerNgError::ListenerCredentials { transport: "http.tls", source, } @@ -171,7 +171,7 @@ async fn serve_connection( futures::select! { result = conn.as_mut().fuse() => { if let Err(error) = result { - debug!(%peer, %error, "server HTTPS connection terminated with error"); + debug!(%peer, %error, "server-ng HTTPS connection terminated with error"); } break; } @@ -199,12 +199,12 @@ impl + 'static> Executor for LocalExecutor { /// the only HTTP-specific step is the ALPN advertisement. fn build_server_config( credentials: TlsServerCredentials, -) -> Result, ServerError> { +) -> Result, ServerNgError> { install_default_crypto_provider(); let mut config = rustls::ServerConfig::builder() .with_no_client_auth() .with_single_cert(credentials.cert_chain, credentials.key_der) - .map_err(|error| ServerError::ListenerCredentials { + .map_err(|error| ServerNgError::ListenerCredentials { transport: "http.tls", source: std::io::Error::other(format!( "http TLS server config rejected credentials: {error}" @@ -227,14 +227,14 @@ async fn accept_pump( loop { futures::select! { () = shutdown.wait().fuse() => { - debug!("server HTTPS accept pump shutting down"); + debug!("server-ng HTTPS accept pump shutting down"); break; } result = listener.accept().fuse() => match result { Ok((stream, peer)) => { spawn_handshake(&acceptor, &connections, handshake_grace, stream, peer); } - Err(error) => error!(%error, "server HTTPS accept failed"), + Err(error) => error!(%error, "server-ng HTTPS accept failed"), }, } } @@ -260,9 +260,9 @@ fn spawn_handshake( // shutdown, when the serve loop is already tearing down. let _ = connections.send((tls, peer)).await; } - Ok(Err(error)) => debug!(%peer, %error, "server HTTPS handshake failed"), + Ok(Err(error)) => debug!(%peer, %error, "server-ng HTTPS handshake failed"), Err(_elapsed) => { - debug!(%peer, grace = ?handshake_grace, "server HTTPS handshake timed out"); + debug!(%peer, grace = ?handshake_grace, "server-ng HTTPS handshake timed out"); } } }) diff --git a/core/server/src/http/wire.rs b/core/server-ng/src/http/wire.rs similarity index 97% rename from core/server/src/http/wire.rs rename to core/server-ng/src/http/wire.rs index c7026d09f5..e96866f80a 100644 --- a/core/server/src/http/wire.rs +++ b/core/server-ng/src/http/wire.rs @@ -16,7 +16,7 @@ // under the License. //! HTTP -> wire request mappers: produce/poll/consumer-offset encoders and -//! the control-plane [`Message`] builder shared by the write path. +//! the control-plane [`Message`] builder shared by the write path. use bytes::{Bytes, BytesMut}; use iggy_binary_protocol::consensus::{Command2, HEADER_SIZE}; @@ -28,7 +28,7 @@ use iggy_binary_protocol::requests::consumer_offsets::{ use iggy_binary_protocol::requests::messages::{ PollMessagesRequest, RawMessage, SendMessagesEncoder, }; -use iggy_binary_protocol::{AckLevel, Operation, RoutedRequestHeader}; +use iggy_binary_protocol::{AckLevel, Operation, RequestHeader}; use iggy_common::get_consumer_offset::GetConsumerOffset; use iggy_common::poll_messages::DEFAULT_PARTITION_ID; use iggy_common::store_consumer_offset::StoreConsumerOffset; @@ -179,7 +179,7 @@ pub(in crate::http) const fn resync_required_polled_messages() -> PolledMessages } } -/// Build a `Message` for a control-plane write by filling a zeroed +/// Build a `Message` for a control-plane write by filling a zeroed /// `#[repr(C)]` header, mirroring `wire::rewrite_request_body` and the partition /// reconciler's prepare builder. `body` is the already-encoded wire request, /// copied in after the header. @@ -189,14 +189,14 @@ pub(in crate::http) fn build_request_message( session_id: u64, request_id: u64, body: &[u8], -) -> Message { +) -> Message { let total = HEADER_SIZE + body.len(); - let mut message = Message::::new(total); + let mut message = Message::::new(total); message.as_mut_slice()[HEADER_SIZE..].copy_from_slice(body); - let header = bytemuck::checked::try_from_bytes_mut::( + let header = bytemuck::checked::try_from_bytes_mut::( &mut message.as_mut_slice()[..HEADER_SIZE], ) - .expect("zeroed bytes form a valid RoutedRequestHeader"); + .expect("zeroed bytes form a valid RequestHeader"); header.command = Command2::Request; header.operation = operation; header.client = client_id; diff --git a/core/server-ng/src/lib.rs b/core/server-ng/src/lib.rs new file mode 100644 index 0000000000..1b0f56b769 --- /dev/null +++ b/core/server-ng/src/lib.rs @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#![allow(clippy::future_not_send)] + +use iggy_common::SemanticVersion; + +pub const VERSION: &str = env!("CARGO_PKG_VERSION"); +pub const SEMANTIC_VERSION: SemanticVersion = SemanticVersion::parse_const(VERSION); + +pub mod auth; +pub mod bootstrap; +pub(crate) mod cluster_meta; +pub mod config_writer; +pub mod consumer_group; +pub mod dispatch; +pub(crate) mod http; +pub mod login_register; +pub(crate) mod offset_recovery; +pub mod partition_helpers; +pub mod partition_reconciler; +pub mod pat; +pub(crate) mod personal_access_token_cleaner; +pub mod responses; +pub(crate) mod segment_cleaner; +pub(crate) mod segment_recovery; +pub mod server_error; +pub mod session_manager; +pub(crate) mod snapshot; +pub mod users; +#[cfg(feature = "iggy-web")] +pub(crate) mod web; +pub mod wire; diff --git a/core/server/src/login_register.rs b/core/server-ng/src/login_register.rs similarity index 100% rename from core/server/src/login_register.rs rename to core/server-ng/src/login_register.rs diff --git a/core/server-ng/src/main.rs b/core/server-ng/src/main.rs new file mode 100644 index 0000000000..404a4496f2 --- /dev/null +++ b/core/server-ng/src/main.rs @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#![allow(clippy::future_not_send)] + +mod args; + +use args::Args; +use clap::Parser; +use server_ng::bootstrap::{bootstrap, load_config}; +use server_ng::server_error::ServerNgError; +use system_stats::capture_allowed_cpus; +use tracing::{error, info}; + +fn main() -> Result<(), ServerNgError> { + // Before shard threads pin themselves: a pinned capture sees one core. + capture_allowed_cpus(); + + let bootstrap_runtime = match server_common::create_shard_executor() { + Ok(rt) => rt, + Err(e) => { + let e = server_common::diagnostics::enrich_runtime_create_error(e); + panic!("Cannot create server-ng bootstrap executor: {e}"); + } + }; + + // Bootstrap on a temporary runtime: parse args, init logging, load + // config, init the memory pool. Then drop the runtime and spawn the + // per-shard runtimes - each shard thread builds its OWN + // `compio::runtime::Runtime` via `create_shard_executor`, pinned to + // its CPU. + let bootstrap_result: Result< + ( + configs::server_ng::ServerNgConfig, + Option, + server_common::log::Logging, + ), + ServerNgError, + > = bootstrap_runtime.block_on(async { + let args = Args::parse(); + if let Ok(env_path) = std::env::var("IGGY_ENV_PATH") { + let _ = dotenvy::from_path(&env_path); + } else { + let _ = dotenvy::dotenv(); + } + + let mut logging = server_common::log::Logging::new(server_ng::VERSION); + logging.early_init(); + server_common::print_build_info!(server_ng::VERSION); + + let config = load_config(&mut logging).await?; + server_common::MemoryPool::init_pool(&config.system.memory_pool.into_other()); + + Ok((config, args.replica_id, logging)) + }); + // `_logging` owns the tracing appender worker guards; it must outlive the + // shard threads or every log line after bootstrap is silently dropped. + let (config, replica_id, _logging) = bootstrap_result?; + drop(bootstrap_runtime); + + let shards = bootstrap(config, replica_id)?; + if let Err(error) = shards.install_ctrlc_handler() { + // Without a working SIGINT handler the server has no way to + // observe an operator Ctrl-C and the shutdown flag would never + // flip, leaving shard threads parked indefinitely. Fail fast + // rather than boot into an un-killable state. + error!(error = %error, "failed to install Ctrl-C handler; aborting boot"); + std::process::exit(1); + } + + info!("server-ng running; waiting on shard threads"); + shards.join_all()?; + info!("server-ng shutdown complete"); + Ok(()) +} diff --git a/core/server/src/offset_recovery.rs b/core/server-ng/src/offset_recovery.rs similarity index 73% rename from core/server/src/offset_recovery.rs rename to core/server-ng/src/offset_recovery.rs index afb60c8209..2e70291938 100644 --- a/core/server/src/offset_recovery.rs +++ b/core/server-ng/src/offset_recovery.rs @@ -15,19 +15,17 @@ // specific language governing permissions and limitations // under the License. -//! Server-owned consumer offset recovery. +//! server-ng-owned consumer offset recovery. //! //! Forked from `server::streaming::partitions::storage` (the legacy -//! `load_consumer_offsets` / `load_consumer_group_offsets`) so server +//! `load_consumer_offsets` / `load_consumer_group_offsets`) so server-ng //! owns the loaders for the offset files its own persistence path writes, -//! without depending on the legacy `server` crate. One file per consumer (numeric -//! file name = consumer id) holding a little-endian `u64` offset then a checksum over -//! it; see [`partitions::offset_storage`]. The legacy server stays compatible both -//! ways: it reads the first eight bytes and stops, and a file it wrote itself decodes -//! here as unchecksummed. +//! without depending on the legacy `server` crate. The on-disk format is +//! shared with the legacy server today: one file per consumer (numeric +//! file name = consumer id) holding a single little-endian `u64` offset. use iggy_common::{ConsumerGroupId, ConsumerKind, ConsumerOffset, IggyError}; -use partitions::offset_storage::{OffsetRecord, decode_offset_record}; +use std::io::Read; use std::sync::atomic::AtomicU64; use tracing::{error, trace, warn}; @@ -171,48 +169,23 @@ pub fn load_consumer_group_offsets( } fn read_offset_file(path: &str, offset_kind: &'static str) -> Option { - let bytes = match std::fs::read(path) { - Ok(bytes) => bytes, + let mut file = match std::fs::File::open(path) { + Ok(file) => file, Err(e) => { warn!( - "{COMPONENT} (error: {e}) - failed to read offset file, \ + "{COMPONENT} (error: {e}) - failed to open offset file, \ path: {path}, skipping." ); return None; } }; - match decode_offset_record(&bytes) { - OffsetRecord::Value { offset, .. } => Some(AtomicU64::new(offset)), - OffsetRecord::Torn => { - warn!( - "{COMPONENT} - failed to read {offset_kind} from file (truncated), \ - path: {path}, skipping." - ); - None - } - // Skipped rather than loaded: resuming from a cursor provably not the one - // written reads as ordinary redelivery or a gap, never as corruption. - // - // And unlinked, not just skipped: the offset map starts cold every boot, so a - // file left behind is re-read by the first auto-commit and trips the commit - // path again. - OffsetRecord::Corrupt { - offset, - expected, - found, - } => { - error!( - "{COMPONENT} - {offset_kind} file failed its checksum \ - (offset: {offset}, expected: {expected}, found: {found}), \ - path: {path}, removing it and resuming this consumer from the start." - ); - if let Err(e) = std::fs::remove_file(path) { - error!( - "{COMPONENT} (error: {e}) - could not remove the corrupt \ - {offset_kind} file, path: {path}; remove it manually." - ); - } - None - } + let mut offset = [0; 8]; + if let Err(e) = file.read_exact(&mut offset) { + warn!( + "{COMPONENT} (error: {e}) - failed to read {offset_kind} from file \ + (truncated or corrupt?), path: {path}, skipping." + ); + return None; } + Some(AtomicU64::new(u64::from_le_bytes(offset))) } diff --git a/core/server/src/partition_helpers.rs b/core/server-ng/src/partition_helpers.rs similarity index 94% rename from core/server/src/partition_helpers.rs rename to core/server-ng/src/partition_helpers.rs index 48ea83a791..a7a291eb76 100644 --- a/core/server/src/partition_helpers.rs +++ b/core/server-ng/src/partition_helpers.rs @@ -25,9 +25,9 @@ //! consumer-offset configuration, and initial-segment provisioning. use crate::offset_recovery::{load_consumer_group_offsets, load_consumer_offsets}; -use crate::server_error::ServerError; +use crate::server_error::ServerNgError; use compio::fs::create_dir_all; -use configs::server::ServerConfig; +use configs::server_ng::ServerNgConfig; use consensus::{LocalPipeline, VsrConsensus, VsrState}; use iggy_common::{ ConsumerGroupOffsets, ConsumerOffsets, IggyError, IggyTimestamp, PartitionStats, @@ -54,15 +54,15 @@ use tracing::{error, info, warn}; /// /// # Errors /// -/// Returns [`ServerError::RecoveredNamespaceOutOfBounds`] if any of +/// Returns [`ServerNgError::RecoveredNamespaceOutOfBounds`] if any of /// `stream_id`, `topic_id`, or `partition_id` exceed the configured /// maxima. pub const fn validate_namespace_bounds( - config: &ServerConfig, + config: &ServerNgConfig, stream_id: usize, topic_id: usize, partition_id: usize, -) -> Result<(), ServerError> { +) -> Result<(), ServerNgError> { let namespace = &config.extra.namespace; if stream_id < namespace.max_streams && topic_id < namespace.max_topics @@ -71,7 +71,7 @@ pub const fn validate_namespace_bounds( return Ok(()); } - Err(ServerError::RecoveredNamespaceOutOfBounds { + Err(ServerNgError::RecoveredNamespaceOutOfBounds { stream_id, topic_id, partition_id, @@ -96,7 +96,7 @@ pub async fn create_partition_file_hierarchy( stream_id: usize, topic_id: usize, partition_id: usize, - config: &ServerConfig, + config: &ServerNgConfig, ) -> Result<(), IggyError> { let partition_path = config .system @@ -174,15 +174,15 @@ pub async fn create_partition_file_hierarchy( /// /// # Errors /// -/// Returns [`ServerError::ConsumerOffsetsLoad`] when the on-disk files +/// Returns [`ServerNgError::ConsumerOffsetsLoad`] when the on-disk files /// exist but fail to decode. A stored offset ahead of `current_offset` is /// clamped (with a warning), not an error. pub fn configure_consumer_offsets( partition: &mut IggyPartition>, - config: &ServerConfig, + config: &ServerNgConfig, namespace: IggyNamespace, current_offset: u64, -) -> Result<(), ServerError> { +) -> Result<(), ServerNgError> { let stream_id = namespace.stream_id(); let topic_id = namespace.topic_id(); let partition_id = namespace.partition_id(); @@ -270,7 +270,7 @@ fn load_partition_consumer_offsets( stream_id: usize, topic_id: usize, partition_id: usize, -) -> Result, ServerError> { +) -> Result, ServerNgError> { if !Path::new(path).exists() { return Ok(Vec::new()); } @@ -281,7 +281,7 @@ fn load_partition_consumer_offsets( return Ok(Vec::new()); } - Err(ServerError::ConsumerOffsetsLoad { + Err(ServerNgError::ConsumerOffsetsLoad { consumer_kind, stream_id, topic_id, @@ -297,7 +297,7 @@ fn load_partition_consumer_group_offsets( stream_id: usize, topic_id: usize, partition_id: usize, -) -> Result, ServerError> { +) -> Result, ServerNgError> { if !Path::new(path).exists() { return Ok(Vec::new()); } @@ -308,7 +308,7 @@ fn load_partition_consumer_group_offsets( return Ok(Vec::new()); } - Err(ServerError::ConsumerOffsetsLoad { + Err(ServerNgError::ConsumerOffsetsLoad { consumer_kind: "consumer group", stream_id, topic_id, @@ -327,15 +327,15 @@ fn load_partition_consumer_group_offsets( /// /// # Errors /// -/// Returns [`ServerError`] on segment-storage creation failure or +/// Returns [`ServerNgError`] on segment-storage creation failure or /// writer initialisation failure. pub async fn ensure_initial_segment( partition: &mut IggyPartition>, - config: &ServerConfig, + config: &ServerNgConfig, stream_id: usize, topic_id: usize, partition_id: usize, -) -> Result<(), ServerError> { +) -> Result<(), ServerNgError> { if partition.log.has_segments() { return Ok(()); } @@ -401,11 +401,6 @@ pub async fn ensure_initial_segment( messages_size_counter, config.system.partition.enforce_fsync, false, - config - .system - .segment - .preallocate - .then_some(config.system.segment.size), ) .await .map_err(|source| { @@ -462,14 +457,14 @@ pub async fn ensure_initial_segment( /// /// # Errors /// -/// [`ServerError::PartitionSuperblockIo`] when the directory or a slot +/// [`ServerNgError::PartitionSuperblockIo`] when the directory or a slot /// cannot be read; the `VersionUnknown` / `Unverifiable` / `Undecodable` / /// `IdentityMismatch` variants when a record exists but cannot be trusted. pub(crate) async fn open_partition_superblock( partition_dir: &str, identity: ReplicaIdentity, -) -> Result<(Rc, Option), ServerError> { - let io_error = |source| ServerError::PartitionSuperblockIo { +) -> Result<(Rc, Option), ServerNgError> { + let io_error = |source| ServerNgError::PartitionSuperblockIo { dir: PathBuf::from(partition_dir), source, }; @@ -483,7 +478,7 @@ pub(crate) async fn open_partition_superblock( let recovered_state = match latest { SuperblockContents::Present(bytes) => { Some(VsrState::try_from(bytes.as_slice()).map_err(|source| { - ServerError::PartitionSuperblockUndecodable { + ServerNgError::PartitionSuperblockUndecodable { dir: PathBuf::from(partition_dir), source, } @@ -492,13 +487,13 @@ pub(crate) async fn open_partition_superblock( SuperblockContents::Unreadable { version: Some(version), } => { - return Err(ServerError::PartitionSuperblockVersionUnknown { + return Err(ServerNgError::PartitionSuperblockVersionUnknown { dir: PathBuf::from(partition_dir), version, }); } SuperblockContents::Unreadable { version: None } => { - return Err(ServerError::PartitionSuperblockUnverifiable { + return Err(ServerNgError::PartitionSuperblockUnverifiable { dir: PathBuf::from(partition_dir), }); } @@ -506,7 +501,7 @@ pub(crate) async fn open_partition_superblock( }; if let Some(state) = recovered_state.as_ref() { let mismatch = |field, expected: u128, found: u128| { - Err(ServerError::PartitionSuperblockIdentityMismatch { + Err(ServerNgError::PartitionSuperblockIdentityMismatch { dir: PathBuf::from(partition_dir), field, expected, @@ -547,7 +542,7 @@ pub(crate) fn restore_partition_view( // a replica that came back at view 0 is otherwise indistinguishable from one // that resumed correctly until it votes. info!( - namespace_raw = consensus.group(), + namespace_raw = consensus.namespace(), view = state.view, log_view = state.log_view, "restored partition view from its superblock" @@ -578,11 +573,11 @@ pub(crate) fn restore_partition_view( /// /// # Errors /// -/// Returns [`ServerError`] when bounds validation, directory creation, +/// Returns [`ServerNgError`] when bounds validation, directory creation, /// superblock recovery, or segment provisioning fails. #[allow(clippy::too_many_arguments)] pub async fn build_partition_fresh( - config: &ServerConfig, + config: &ServerNgConfig, namespace: IggyNamespace, stats: Arc, created_revision: u64, @@ -590,7 +585,7 @@ pub async fn build_partition_fresh( self_replica_id: u8, replica_count: u8, bus: Rc, -) -> Result>, ServerError> { +) -> Result>, ServerNgError> { let stream_id = namespace.stream_id(); let topic_id = namespace.topic_id(); let partition_id = namespace.partition_id(); @@ -699,6 +694,7 @@ pub async fn build_partition_fresh( partition.offset.store(0, Ordering::Release); partition.dirty_offset.store(0, Ordering::Relaxed); partition.should_increment_offset = false; + partition.stats.set_current_offset(0); debug_assert!( !partition.log.has_segments(), "fresh partition must not carry recovered segments" @@ -738,7 +734,7 @@ pub async fn delete_partitions_from_disk( stream_id: usize, topic_id: usize, partition_id: usize, - config: &ServerConfig, + config: &ServerNgConfig, ) -> Result<(), IggyError> { let partition_path = config .system @@ -879,7 +875,7 @@ mod tests { let refused = open_partition_superblock(&dir, test_identity()).await; match refused { - Err(ServerError::PartitionSuperblockIdentityMismatch { field, .. }) => { + Err(ServerNgError::PartitionSuperblockIdentityMismatch { field, .. }) => { assert_eq!(field, IdentityField::Cluster); } Err(other) => panic!("expected an identity mismatch, got {other}"), diff --git a/core/server/src/partition_reconciler.rs b/core/server-ng/src/partition_reconciler.rs similarity index 94% rename from core/server/src/partition_reconciler.rs rename to core/server-ng/src/partition_reconciler.rs index b08cf52290..63d6c2a5cb 100644 --- a/core/server/src/partition_reconciler.rs +++ b/core/server-ng/src/partition_reconciler.rs @@ -169,10 +169,10 @@ //! discriminator, like `checkpoint_id` on every prepare //! -- `PrepareHeader.reserved` has room, but it is a `#[repr(C)]` wire change. -use crate::bootstrap::ServerShard; +use crate::bootstrap::ServerNgShard; use crate::partition_helpers::{build_partition_fresh, delete_partitions_from_disk}; use ahash::{AHashMap, AHashSet}; -use configs::server::ServerConfig; +use configs::server_ng::ServerNgConfig; use consensus::{MetadataHandle, PartitionsHandle}; use futures::FutureExt; use iggy_common::{ConsumerGroupId, IggyTimestamp}; @@ -219,9 +219,9 @@ enum FailureCause { } pub struct ReconcilerCtx { - pub shard: Rc, + pub shard: Rc, pub total_shards: u16, - pub config: Rc, + pub config: Rc, pub cluster_id: u128, pub self_replica_id: u8, pub replica_count: u8, @@ -238,9 +238,9 @@ pub struct ReconcilerCtx { impl ReconcilerCtx { #[must_use] pub fn new( - shard: Rc, + shard: Rc, total_shards: u16, - config: Rc, + config: Rc, cluster_id: u128, self_replica_id: u8, replica_count: u8, @@ -413,10 +413,6 @@ struct PassCounters { /// tombstone and re-wakes us without bumping `Streams::revision`, so an /// armed skip would swallow that wake and strand the rebuild forever. deferred: usize, - /// Namespaces an earlier pass already built, whose `InsertOwned` the pump - /// has not applied yet. Counted so the pass does not arm the fast-skip - /// while work is in flight; applying it bumps no revision. - already_staged: usize, } impl PassCounters { @@ -432,7 +428,6 @@ impl PassCounters { + self.purges_staged + self.deferred + self.parked_reclaimed - + self.already_staged } } @@ -481,9 +476,9 @@ async fn reconcile_once(ctx: &ReconcilerCtx) -> bool { let target_set: AHashSet = target.iter().map(|(ns, _)| *ns).collect(); let mut counters = PassCounters::default(); - reconcile_additions(ctx, target, &mut counters).await; + let staged = reconcile_additions(ctx, target, &mut counters).await; reconcile_removals(ctx, &target_set, &mut counters).await; - reconcile_parked_frames(ctx, &mut counters); + reconcile_parked_frames(ctx, &staged, &mut counters); reconcile_consumer_group_offsets(ctx, &mut counters).await; reconcile_segment_truncations(ctx, &mut counters); reconcile_partition_purges(ctx, &mut counters); @@ -511,7 +506,6 @@ async fn reconcile_once(ctx: &ReconcilerCtx) -> bool { backoff_skipped = counters.backoff_skipped, stale = counters.stale, deferred = counters.deferred, - already_staged = counters.already_staged, parked_reclaimed = counters.parked_reclaimed, purges_staged = counters.purges_staged, trims_pending = counters.trims_pending, @@ -527,14 +521,19 @@ async fn reconcile_once(ctx: &ReconcilerCtx) -> bool { true } +/// Returns the namespaces whose `ReconcileOp::InsertOwned` this pass staged. The +/// pump applies the op on its own task, so they are not in `IggyPartitions` yet +/// and [`reconcile_parked_frames`] would read them as un-materialised, aging +/// their frames on the pass that built them. async fn reconcile_additions( ctx: &ReconcilerCtx, target: Vec<(IggyNamespace, u64)>, counters: &mut PassCounters, -) { +) -> AHashSet { let shard_id = ctx.shard.id; let partitions = ctx.shard.plane.partitions(); let total_shards = u32::from(ctx.total_shards); + let mut staged = AHashSet::new(); for (ns, epoch) in target { if partitions.contains(&ns) { @@ -584,7 +583,7 @@ async fn reconcile_additions( // means the local partition is a prior incarnation carrying // stale segments/offsets/log. Tear it down; the // post-ConfirmRemove wake rebuilds it fresh next pass. - if shards_table_has_epoch(ctx, ns, epoch) { + if ctx.shard.shards_table().epoch_for(ns) == Some(epoch) { continue; } trace!( @@ -600,15 +599,10 @@ async fn reconcile_additions( let owning_shard = calculate_shard_assignment(&ns, total_shards); if owning_shard != shard_id { - // Compare the epoch, not just presence: a delete + recreate - // recycles the slab keys, so the row survives with the DEAD - // incarnation's `created_revision`. A presence-only gate never - // refreshes it, and nothing else writes a non-owner's row. - // - // No mirror of the staged-`InsertOwned` guard below, deliberately: - // a lagging pump costs one duplicate `InsertRouted` per pass, and - // the apply is an idempotent row overwrite, while scanning the op - // queue per routed namespace would go quadratic. + // Compare the epoch, not just presence: a delete + recreate recycles + // the slab keys, so the row survives with the DEAD incarnation's + // `created_revision`. A presence-only gate never refreshes it, and + // nothing else writes a non-owner's row. if !shards_table_has_epoch(ctx, ns, epoch) { ctx.shard.enqueue_reconcile_op(ReconcileOp::InsertRouted { namespace: ns, @@ -620,17 +614,6 @@ async fn reconcile_additions( continue; } - // An earlier pass already built this one and the pump has not applied it - // yet, so the `contains` test above reads false for finished work. - // Rebuilding is not a wasted-effort question: the second build shares - // the namespace's `PartitionStats` with the queued sibling and re-opens - // segment 0 with `file_exists = false`, truncating the file that - // sibling is about to serve. - if ctx.shard.has_staged_insert_owned(ns) { - counters.already_staged += 1; - continue; - } - let now = Instant::now(); if ctx.is_backed_off(ns, FailureCause::Add, now) { counters.backoff_skipped += 1; @@ -665,6 +648,7 @@ async fn reconcile_additions( }); ctx.record_success(ns, FailureCause::Add); counters.materialised += 1; + staged.insert(ns); } Err(err) => { ctx.record_failure(ns, FailureCause::Add, now); @@ -680,6 +664,8 @@ async fn reconcile_additions( } } } + + staged } /// Retire parked frames the shard cannot serve, age the ones it might. @@ -719,15 +705,18 @@ async fn reconcile_additions( /// timeout and no committed op dies on a local-convergence signal. Residency /// only; see `ParkedFrame::passes`. /// -/// A namespace with a staged, unapplied `InsertOwned` is exempt: its partition -/// is on the way but reads as un-materialised here. The queue is asked per -/// parked namespace ([`shard::IggyShard::has_staged_insert_owned`]) rather than -/// carrying a set over from the additions pass, so the answer cannot go stale -/// across `reconcile_removals`' awaits; `parked` is empty on the steady path, -/// so the scan costs nothing there. The exemption spans passes, not just the -/// one that built the namespace, covering arbitrary pump lag; dropping it ages -/// frames on every commit-driven pass the pump falls behind. -fn reconcile_parked_frames(ctx: &ReconcilerCtx, counters: &mut PassCounters) { +/// `staged_this_pass` is exempt: its `InsertOwned` is queued but not applied, so +/// it reads as un-materialised here. Not a one-pass concession. +/// `reconcile_additions` has no cross-pass guard against a queued-but-unapplied +/// op (it tests `partitions.contains`, false the whole time it sits in the +/// queue), so it re-stages every pass until the pump drains. The exemption +/// therefore covers arbitrary pump lag; dropping it ages frames on every +/// commit-driven pass the pump falls behind. +fn reconcile_parked_frames( + ctx: &ReconcilerCtx, + staged_this_pass: &AHashSet, + counters: &mut PassCounters, +) { let parked = ctx.shard.parked_namespaces(); if parked.is_empty() { return; @@ -735,7 +724,7 @@ fn reconcile_parked_frames(ctx: &ReconcilerCtx, counters: &mut PassCounters) { let partitions = ctx.shard.plane.partitions(); let total_shards = u32::from(ctx.total_shards); for ns in parked { - if ctx.shard.has_staged_insert_owned(ns) { + if staged_this_pass.contains(&ns) { continue; } // Tombstoned namespaces are still in the map, so `contains` below reads @@ -1145,7 +1134,7 @@ fn reconcile_partition_purges(ctx: &ReconcilerCtx, counters: &mut PassCounters) } } -pub fn install_tick_handler(shard: &Rc, wake_tx: WakeTx) { +pub fn install_tick_handler(shard: &Rc, wake_tx: WakeTx) { let shard_id = shard.id; let handler = Rc::new(move || { if let Err(err) = wake_tx.try_send(()) { @@ -1158,10 +1147,9 @@ pub fn install_tick_handler(shard: &Rc, wake_tx: WakeTx) { #[cfg(test)] mod tests { use super::{ - FailureCause, FailureRecord, ReconcilerCtx, build_partition_fresh, - delete_partitions_from_disk, fetch_partition_stats, reconcile_once, + FailureCause, FailureRecord, ReconcilerCtx, delete_partitions_from_disk, reconcile_once, }; - use configs::server::{ServerConfig, ServerSystemConfig}; + use configs::server_ng::{NgSystemConfig, ServerNgConfig}; use consensus::{MetadataHandle, PartitionsHandle}; use iggy_binary_protocol::codec::WireEncode; use iggy_binary_protocol::primitives::identifier::WireName; @@ -1175,7 +1163,7 @@ mod tests { PurgeTopicRequest, }; use iggy_binary_protocol::{ - Command2, GenericHeader, Operation, PrepareHeader, ReplyHeader, RoutedRequestHeader, + Command2, GenericHeader, Operation, PrepareHeader, ReplyHeader, RequestHeader, WireIdentifier, }; use message_bus::IggyMessageBus; @@ -1193,7 +1181,6 @@ mod tests { use std::mem::size_of; use std::rc::Rc; use std::sync::Arc; - use std::sync::atomic::Ordering; use std::time::Instant; use tempfile::TempDir; @@ -1262,7 +1249,7 @@ mod tests { header.command = Command2::Prepare; header.size = u32::try_from(header_size).expect("prepare size fits u32"); header.operation = Operation::SendMessages; - header.group = namespace.inner(); + header.namespace = namespace.inner(); header.op = op; msg.into_generic() } @@ -1294,17 +1281,17 @@ mod tests { namespace: IggyNamespace, body_len: usize, ) -> Message { - let header_size = size_of::(); + let header_size = size_of::(); let total_size = header_size + body_len; - let mut msg = Message::::new(total_size); - let header = bytemuck::checked::try_from_bytes_mut::( + let mut msg = Message::::new(total_size); + let header = bytemuck::checked::try_from_bytes_mut::( &mut msg.as_mut_slice()[..header_size], ) - .expect("zeroed bytes form a valid RoutedRequestHeader"); + .expect("zeroed bytes form a valid RequestHeader"); header.command = Command2::Request; header.size = u32::try_from(total_size).expect("request size fits u32"); header.operation = Operation::SendMessages; - header.group = namespace.inner(); + header.namespace = namespace.inner(); // Header validation rejects a zero session / request on a non-register // op, and the park path runs after that validation. header.session = 1; @@ -1430,24 +1417,24 @@ mod tests { .expect("JoinConsumerGroup apply succeeds"); } - fn test_config(tmp: &TempDir) -> ServerConfig { - let mut cfg = ServerConfig::default(); - // `ServerSystemConfig` is not `Clone`, so `Arc::make_mut` is out; build a + fn test_config(tmp: &TempDir) -> ServerNgConfig { + let mut cfg = ServerNgConfig::default(); + // `NgSystemConfig` is not `Clone`, so `Arc::make_mut` is out; build a // fresh value via struct-update syntax and swap the Arc wholesale. // Only `path` differs from the default; every other field uses the // runtime's defaults. - let system = ServerSystemConfig { + let system = NgSystemConfig { path: tmp.path().to_string_lossy().into_owned(), - ..ServerSystemConfig::default() + ..NgSystemConfig::default() }; cfg.system = Arc::new(system); cfg } - /// Assemble a fully functional `ServerShard` for reconciler tests. + /// Assemble a fully functional `ServerNgShard` for reconciler tests. /// Uses `IggyShard::without_inbox` so no inter-shard pump runs; the /// reconciler can be driven directly by `reconcile_once`. - fn build_test_shard(shard_id: u16, config: &ServerConfig, mux: TestMux) -> Rc { + fn build_test_shard(shard_id: u16, config: &ServerNgConfig, mux: TestMux) -> Rc { let bus = Rc::new(IggyMessageBus::with_config(shard_id, config)); let metadata: IggyMetadata< consensus::VsrConsensus>, @@ -1461,9 +1448,7 @@ mod tests { messages_required_to_save: 1, size_of_messages_required_to_save: iggy_common::IggyByteSize::from(1024_u64), enforce_fsync: false, - validate_checksum: true, segment_size: config.system.segment.size, - preallocate_segments: false, encryptor: None, }, ); @@ -1494,7 +1479,7 @@ mod tests { /// in this shard's inbox and reading as success. fn build_test_shard_with_inbox( shard_id: u16, - config: &ServerConfig, + config: &ServerNgConfig, mux: TestMux, capacity: usize, ) -> (Rc, shard::Receiver) { @@ -1542,7 +1527,7 @@ mod tests { fn make_ctx( shard: Rc, total_shards: u16, - config: Rc, + config: Rc, ) -> Rc { Rc::new(ReconcilerCtx::new( shard, @@ -1610,149 +1595,61 @@ mod tests { ); } - /// The cross-pass guard: a pass must not rebuild a namespace an earlier pass - /// already built and left queued. Rebuilding is not merely wasted work -- - /// the second build shares the namespace's `PartitionStats` with the queued - /// sibling and re-opens segment 0 truncating -- so a pass has to recognise - /// the staged op, not just `partitions.contains`. + /// Regression (deferred-apply window): the reconciler stages + /// `ReconcileOp::InsertOwned` from a task separate from the pump that + /// applies it, so under a commit burst it can run a second pass before + /// the pump drains the first pass's staged ops. Both passes then + /// observe `!contains(ns)` and build the same namespace. The pump's + /// apply must be idempotent, else the second `insert` orphans the first + /// partition (leaked VSR group + writers) and inflates `len`. + /// `reconcile_pass` applies inline and cannot surface this, so here we + /// run two passes and only then drain once. #[compio::test] - async fn second_pass_does_not_rebuild_a_namespace_already_staged() { + async fn deferred_apply_window_does_not_duplicate_owned_partition() { let tmp = TempDir::new().expect("tempdir for system path"); let config = test_config(&tmp); let mux = TestMux::default(); seed_stream(&mux, 1, "stream-a"); - seed_topic(&mux, 2, 0, "topic-a", vec![assignment(0, 1)]); + seed_topic( + &mux, + 2, + 0, + "topic-a", + vec![assignment(0, 1), assignment(1, 2)], + ); let shard = build_test_shard(0, &config, mux); let ctx = make_ctx(Rc::clone(&shard), 1, Rc::new(config)); - let ns = IggyNamespace::new(0, 0, 0); + // Two passes with no pump drain in between: models the reconciler, + // woken by a second commit tick, running pass N+1 before the pump + // applies pass N's `InsertOwned`. Both passes see the namespaces as + // unmaterialised and stage a build for each, so the queue holds two + // `InsertOwned` per namespace when the pump finally drains. reconcile_once(&ctx).await; - assert!( - ctx.shard.has_staged_insert_owned(ns), - "the first pass must leave an unapplied InsertOwned to guard against" - ); - - // Second pass while that op is still queued: `partitions.contains(ns)` - // is false, so only the staged-op guard can stop the rebuild. reconcile_once(&ctx).await; ctx.shard.apply_reconcile_ops(); - assert_eq!( - shard.plane.partitions().len(), - 1, - "the namespace must materialise exactly once" - ); - - // Counting builds, not partitions: the pump discards the redundant - // `InsertOwned` either way, so `len` cannot tell one build from two. - // `ensure_initial_segment` plants exactly one segment per build and - // folds it into the namespace's shared stats, so this counter is the - // observable that separates them. - let stats = fetch_partition_stats(&ctx, ns).expect("materialised namespace has stats"); - assert_eq!( - stats.segments_count_inconsistent(), - 1, - "a second build ran: its initial segment was folded into the \ - namespace's shared stats on top of the live incarnation's" - ); - } - - /// The stats registry keys on the namespace, not the incarnation, so - /// `current_offset` moves on adoption only: the pump seeds it from the - /// incarnation it inserts, and a build that never becomes addressable - /// leaves it alone. Seeding from the build instead zeroed it under the live - /// incarnation, after which the partition plane's admission check read an - /// empty offset space and answered every `store_consumer_offset` above 0 - /// with `InvalidOffset` (error 4100) until the next send re-seeded it. - /// - /// Both incarnations are built and staged by hand: the adopted one so its - /// counter is non-zero BEFORE insertion (a reconciler build always adopts - /// at 0, where the publish is indistinguishable from a no-op), and the - /// redundant one because `has_staged_insert_owned` now stops a pass from - /// producing it. - #[compio::test] - async fn discarded_build_leaves_live_partition_offset_intact() { - const COMMITTED_OFFSET: u64 = 3; - const LIVE_EPOCH: u64 = 1; - let tmp = TempDir::new().expect("tempdir for system path"); - let config = test_config(&tmp); - let mux = TestMux::default(); - seed_stream(&mux, 1, "stream-a"); - seed_topic(&mux, 2, 0, "topic-a", vec![assignment(0, 1)]); - - let shard = build_test_shard(0, &config, mux); - let ctx = make_ctx(Rc::clone(&shard), 1, Rc::new(config.clone())); - let ns = IggyNamespace::new(0, 0, 0); - - let stats = fetch_partition_stats(&ctx, ns).expect("committed namespace has stats"); - let live = build_partition_fresh( - &config, - ns, - Arc::clone(&stats), - LIVE_EPOCH, - CLUSTER_ID, - 0, - 1, - Rc::clone(&ctx.shard.bus), - ) - .await - .expect("live build succeeds"); - // What a recovery leaves behind: an incarnation whose own counter is - // ahead of the zeroed shared stats until adoption publishes it. - live.offset.store(COMMITTED_OFFSET, Ordering::Release); - ctx.shard.enqueue_reconcile_op(ReconcileOp::InsertOwned { - namespace: ns, - partition: Box::new(live), - epoch: LIVE_EPOCH, - }); - ctx.shard.apply_reconcile_ops(); - - assert_eq!( - stats.current_offset(), - COMMITTED_OFFSET, - "adoption must publish the incarnation's offset into the shared stats" - ); - - let redundant = build_partition_fresh( - &config, - ns, - Arc::clone(&stats), - LIVE_EPOCH + 1, - CLUSTER_ID, - 0, - 1, - Rc::clone(&ctx.shard.bus), - ) - .await - .expect("redundant build succeeds over the live incarnation's path"); - ctx.shard.enqueue_reconcile_op(ReconcileOp::InsertOwned { - namespace: ns, - partition: Box::new(redundant), - epoch: LIVE_EPOCH + 1, - }); - ctx.shard.apply_reconcile_ops(); - - assert_eq!( - shard.plane.partitions().len(), - 1, - "the redundant build must be discarded, not adopted: a second insert \ - overwrites the ns -> idx entry and orphans the first partition, \ - leaking its VSR group and segment writers" - ); - // The epoch, not `shard_for`: on a single shard an adopt would write - // `ShardId::new(0)` too, but it would stamp the redundant op's epoch. - assert_eq!( - shard.shards_table().epoch_for(ns), - Some(LIVE_EPOCH), - "the discarded op must not rewrite the routing row" - ); + let partitions = shard.plane.partitions(); assert_eq!( - stats.current_offset(), - COMMITTED_OFFSET, - "a discarded build must not reset the live incarnation's current_offset" + partitions.len(), + 2, + "deferred-apply window must not duplicate partitions: \ + each namespace materialises exactly once" ); + for partition_id in 0..2 { + let ns = IggyNamespace::new(0, 0, partition_id); + assert!( + partitions.contains(&ns), + "namespace {ns:?} must be addressable exactly once" + ); + assert_eq!( + shard.shards_table().shard_for(ns), + Some(0), + "shards_table must point at the owning shard" + ); + } } /// Multi-shard scenario: only the partition whose hash maps to diff --git a/core/server/src/pat.rs b/core/server-ng/src/pat.rs similarity index 93% rename from core/server/src/pat.rs rename to core/server-ng/src/pat.rs index 26c8ea6db4..dc22f20004 100644 --- a/core/server/src/pat.rs +++ b/core/server-ng/src/pat.rs @@ -27,7 +27,7 @@ use iggy_binary_protocol::requests::personal_access_tokens::{ CreatePersonalAccessTokenRequest as WireCreatePersonalAccessTokenRequest, DeletePersonalAccessTokenRequest as WireDeletePersonalAccessTokenRequest, }; -use iggy_binary_protocol::{Operation, RoutedRequestHeader, WireDecode, WireEncode}; +use iggy_binary_protocol::{Operation, RequestHeader, WireDecode, WireEncode}; use iggy_common::IggyError; use metadata::stm::user::{ CreatePersonalAccessTokenRequest as ReplicatedCreatePersonalAccessTokenRequest, @@ -42,8 +42,8 @@ pub(crate) fn maybe_rewrite_pat_request( transport_client_id: u128, max_tokens_per_user: u32, pat_count_of: impl FnOnce(u32) -> usize, - request: Message, -) -> Result<(Message, Option), IggyError> { + request: Message, +) -> Result<(Message, Option), IggyError> { let user_id = match request.header().operation { Operation::CreatePersonalAccessToken | Operation::DeletePersonalAccessToken => sessions .borrow() @@ -69,8 +69,8 @@ pub(crate) fn rewrite_pat_request_for_user( user_id: u32, max_tokens_per_user: u32, pat_count_of: impl FnOnce(u32) -> usize, - request: Message, -) -> Result<(Message, Option), IggyError> { + request: Message, +) -> Result<(Message, Option), IggyError> { let body = request_body(&request); let mut raw_token = None; let rewritten = match request.header().operation { @@ -151,10 +151,10 @@ mod tests { const MAX_TOKENS: u32 = 3; const AT_LIMIT_TOKENS: [(&str, u8); 3] = [("one", b'a'), ("two", b'b'), ("three", b'c')]; - fn create_pat_request(name: &str) -> Message { - let header_len = std::mem::size_of::(); - let mut template = Message::::new(header_len); - let header = bytemuck::checked::try_from_bytes_mut::( + fn create_pat_request(name: &str) -> Message { + let header_len = std::mem::size_of::(); + let mut template = Message::::new(header_len); + let header = bytemuck::checked::try_from_bytes_mut::( &mut template.as_mut_slice()[..header_len], ) .expect("zeroed bytes are a valid request header"); @@ -266,9 +266,9 @@ mod tests { #[test] fn given_delete_op_when_at_limit_should_pass_the_gate() { - let header_len = std::mem::size_of::(); - let mut template = Message::::new(header_len); - let header = bytemuck::checked::try_from_bytes_mut::( + let header_len = std::mem::size_of::(); + let mut template = Message::::new(header_len); + let header = bytemuck::checked::try_from_bytes_mut::( &mut template.as_mut_slice()[..header_len], ) .expect("zeroed bytes are a valid request header"); diff --git a/core/server/src/personal_access_token_cleaner.rs b/core/server-ng/src/personal_access_token_cleaner.rs similarity index 96% rename from core/server/src/personal_access_token_cleaner.rs rename to core/server-ng/src/personal_access_token_cleaner.rs index a55093605a..9b150129d5 100644 --- a/core/server/src/personal_access_token_cleaner.rs +++ b/core/server-ng/src/personal_access_token_cleaner.rs @@ -22,7 +22,7 @@ //! proposes it once and every replica applies the commit. Backups never //! propose, so cleanup cannot race across the cluster. -use crate::bootstrap::ServerShard; +use crate::bootstrap::ServerNgShard; use consensus::MetadataHandle; use iggy_binary_protocol::WireName; use iggy_common::IggyTimestamp; @@ -54,7 +54,7 @@ enum Pass { /// Run the cleaner until `stop` fires. Wakes every `interval`; expiry is /// wall-clock driven, so no metadata-commit wake is needed. -pub async fn run_pat_cleaner(shard: Rc, stop: Receiver<()>, interval: Duration) { +pub async fn run_pat_cleaner(shard: Rc, stop: Receiver<()>, interval: Duration) { trace!( shard = shard.id, interval_ms = interval.as_millis(), @@ -81,7 +81,7 @@ pub async fn run_pat_cleaner(shard: Rc, stop: Receiver<()>, interva } /// Run one cleanup pass, deleting at most [`MAX_DELETIONS_PER_PASS`] tokens. -async fn clean_expired_tokens(shard: &Rc, stop: &Receiver<()>) -> Pass { +async fn clean_expired_tokens(shard: &Rc, stop: &Receiver<()>) -> Pass { let metadata = shard.plane.metadata(); if !metadata.is_caught_up_primary() { return Pass::Drained; diff --git a/core/server/src/responses.rs b/core/server-ng/src/responses.rs similarity index 97% rename from core/server/src/responses.rs rename to core/server-ng/src/responses.rs index 7adb4a46da..8d79bf648c 100644 --- a/core/server/src/responses.rs +++ b/core/server-ng/src/responses.rs @@ -76,7 +76,7 @@ use iggy_binary_protocol::responses::users::get_users::GetUsersResponse; use iggy_binary_protocol::responses::users::user_response::UserResponse; use iggy_binary_protocol::{ Command2, GenericHeader, IGGY_PROTOCOL_VERSION, KIND_CONSUMER_GROUP, Operation, ReplyHeader, - RoutedRequestHeader, WireDecode, WireEncode, WireIdentifier, WireName, WirePartitioning, + RequestHeader, WireDecode, WireEncode, WireIdentifier, WireName, WirePartitioning, }; use iggy_common::{EncryptorKind, Identifier, IggyError, IggyTimestamp}; use journal::superblock::SuperblockStore; @@ -156,7 +156,7 @@ where // No session record (shouldn't happen on an auth-gated // read). Report the connection id with the "no user" // sentinel + TCP default rather than impersonating root - // (user id 0 is a real user; server is 0-based). + // (user id 0 is a real user; server-ng is 0-based). #[allow(clippy::cast_possible_truncation)] ClientResponse { client_id: transport_client_id as u32, @@ -466,29 +466,18 @@ where if let Some(namespace) = streams.namespace_from_partition(stream_id, topic_id, partition_id) { return Ok(namespace); } - // Name the level that missed - partition, topic, or stream - with the - // legacy typed not-found, so a client can tell an addressing typo from an - // empty partition. Callers that shape their own reply (empty poll, group - // gather) treat every variant the same, so the split is reply-visible only - // where a caller denies typed. + // Tell a bad partition id apart from a bad stream/topic: the former is a + // typed not-found the client can act on, the latter keeps the generic + // rejection every caller already handles. if streams.topic_partition_ids(stream_id, topic_id).is_some() { - return Err(IggyError::PartitionNotFound( + Err(IggyError::PartitionNotFound( partition_id as usize, wire_identifier_for_display(topic_id), wire_identifier_for_display(stream_id), - )); + )) + } else { + Err(IggyError::InvalidIdentifier) } - Err(streams.read(|inner| { - let Some(resolved_stream) = resolve_stream_id(inner, stream_id) else { - return stream_not_found(stream_id); - }; - if resolve_topic_id(inner, resolved_stream, topic_id).is_none() { - return topic_not_found(stream_id, topic_id); - } - // Unreachable while `topic_partition_ids` misses only on stream/topic; - // kept as the safe generic rejection should that invariant drift. - IggyError::InvalidIdentifier - })) } /// Best-effort conversion for error payloads only: the wire reply carries just @@ -507,10 +496,7 @@ fn wire_identifier_for_display(id: &WireIdentifier) -> Identifier { /// stays with the per-transport gates that run before this builder. `client_ip` /// is the caller's transport-level peer address, used only by the /// cluster-metadata read to pick each node's advertised address; `None` -/// degrades to the catch-all address. `clients_count` is the cross-shard -/// connected-client total, used only by the stats read: it comes from the async -/// `ListClients` scatter-gather, which this sync builder cannot run, so both -/// transport callers gather it up front (0 for every other opcode). +/// degrades to the catch-all address. pub(crate) fn build_non_replicated_response( shard: &Rc>, code: u32, @@ -518,7 +504,6 @@ pub(crate) fn build_non_replicated_response( user_id: Option, roster: &ClusterRoster, client_ip: Option, - clients_count: u32, ) -> Result where B: ShellBus, @@ -532,7 +517,7 @@ where build_cluster_metadata_response(roster, shard, client_ip).to_bytes(), )), GET_STATS_CODE => Ok(NonReplicatedResponse::Bytes( - build_stats_response(shard, clients_count)?.to_bytes(), + build_stats_response(shard)?.to_bytes(), )), GET_STREAM_CODE => { let request = @@ -622,7 +607,7 @@ where NonReplicatedResponse::Bytes(GetConsumerGroupsResponse { groups }.to_bytes()) })) } - // The server has no on-demand flush primitive, so it denies honestly. + // server-ng has no on-demand flush primitive, so it denies honestly. // The non-replicated catch-all's empty-ok would otherwise attest a // durability guarantee the server never gave. FLUSH_UNSAVED_BUFFER_CODE => Err(IggyError::FeatureUnavailable), @@ -695,7 +680,6 @@ where fn build_stats_response( shard: &Rc>, - clients_count: u32, ) -> Result where B: ShellBus, @@ -782,7 +766,12 @@ where partitions_count, segments_count, messages_count, - clients_count, + // Connected clients are per-shard `SessionManager` state, aggregated + // across shards only by the async `ListClients` broadcast (see + // `get_clients`). This sync single-shard read can't gather it, and one + // shard's local count is a fraction of the total, so report 0 rather + // than a misleading partial. + clients_count: 0, consumer_groups_count, hostname: system.hostname, os_name: system.os_name, @@ -1253,7 +1242,7 @@ pub(crate) enum NonReplicatedResponse { impl NonReplicatedResponse { pub(crate) fn into_reply( self, - request_header: &RoutedRequestHeader, + request_header: &RequestHeader, client_id: u128, session: u64, commit: u64, @@ -1268,7 +1257,7 @@ impl NonReplicatedResponse { } pub(crate) fn build_empty_reply( - request_header: &RoutedRequestHeader, + request_header: &RequestHeader, client_id: u128, session: u64, commit: u64, @@ -1284,7 +1273,7 @@ pub(crate) fn build_empty_reply( /// reply, and only the partition primary's pre-pipeline deny pins it to 0, /// stamped through `consensus::build_deny_reply_from_request`. pub(crate) fn build_deny_reply( - request_header: &RoutedRequestHeader, + request_header: &RequestHeader, client_id: u128, session: u64, commit: u64, @@ -1304,7 +1293,7 @@ pub(crate) fn build_deny_reply( const SERVER_VERSION: &str = env!("CARGO_PKG_VERSION"); pub(crate) fn build_login_register_reply( - request_header: &RoutedRequestHeader, + request_header: &RequestHeader, client_id: u128, session: u64, commit: u64, @@ -1335,7 +1324,7 @@ pub(crate) fn build_login_register_reply( } pub(crate) fn build_reply_from_bytes( - request_header: &RoutedRequestHeader, + request_header: &RequestHeader, client_id: u128, session: u64, commit: u64, @@ -1358,7 +1347,7 @@ pub(crate) fn build_reply_from_bytes( /// (no token, a committed business rejection, or an eviction frame) the /// committed reply passes through unchanged. pub(crate) fn build_raw_pat_reply( - request_header: &RoutedRequestHeader, + request_header: &RequestHeader, committed: Message, raw_token: Option, ) -> Result, IggyError> { @@ -1419,7 +1408,7 @@ pub(crate) fn build_raw_pat_reply( } pub(crate) fn build_reply_with_body( - request_header: &RoutedRequestHeader, + request_header: &RequestHeader, client_id: u128, session: u64, commit: u64, @@ -1448,6 +1437,7 @@ pub(crate) fn build_reply_with_body( timestamp: request_header.timestamp, request: request_header.request, operation: request_header.operation, + namespace: request_header.namespace, ..Default::default() }; write_body(&mut reply.as_mut_slice()[header_len..total_size]); @@ -1639,10 +1629,10 @@ pub(crate) fn build_consumer_offset_body( mod tests { use super::*; - fn pat_request_header() -> RoutedRequestHeader { - let zeroed = [0u8; std::mem::size_of::()]; - let mut header = *bytemuck::checked::try_from_bytes::(&zeroed) - .expect("zeroed bytes form a valid RoutedRequestHeader"); + fn pat_request_header() -> RequestHeader { + let zeroed = [0u8; std::mem::size_of::()]; + let mut header = *bytemuck::checked::try_from_bytes::(&zeroed) + .expect("zeroed bytes form a valid RequestHeader"); header.command = Command2::Request; header.operation = Operation::CreatePersonalAccessToken; header.client = 42; diff --git a/core/server/src/segment_cleaner.rs b/core/server-ng/src/segment_cleaner.rs similarity index 97% rename from core/server/src/segment_cleaner.rs rename to core/server-ng/src/segment_cleaner.rs index 49bc338ff4..bffd016168 100644 --- a/core/server/src/segment_cleaner.rs +++ b/core/server-ng/src/segment_cleaner.rs @@ -26,7 +26,7 @@ //! with reads. This mirrors the legacy server's `MessagesCleaner` -> //! message-pump `CleanTopicMessages` path. -use crate::bootstrap::ServerShard; +use crate::bootstrap::ServerNgShard; use consensus::{MetadataHandle, PartitionsHandle}; use iggy_common::{IggyExpiry, IggyTimestamp, MaxTopicSize}; use metadata::impls::metadata::StreamsFrontend; @@ -38,7 +38,7 @@ use tracing::trace; /// Run the cleaner until `stop` fires. Wakes every `interval`; expiry and size /// are evaluated against wall-clock and resident bytes, so no metadata-commit /// wake is needed. -pub async fn run_segment_cleaner(shard: Rc, stop: Receiver<()>, interval: Duration) { +pub async fn run_segment_cleaner(shard: Rc, stop: Receiver<()>, interval: Duration) { trace!( shard = shard.id, interval_ms = interval.as_millis(), @@ -57,7 +57,7 @@ pub async fn run_segment_cleaner(shard: Rc, stop: Receiver<()>, int /// Stage a cleaner pass for every partition this shard owns whose topic has a /// retention policy. Reads config off-pump and hands the resolved decision to /// the pump; partitions with no policy are skipped without a frame. -fn stage_owned_partitions(shard: &Rc) { +fn stage_owned_partitions(shard: &Rc) { let now = IggyTimestamp::now(); let namespaces: Vec<_> = shard.plane.partitions().namespaces().copied().collect(); let streams = shard.plane.metadata().mux_stm.streams(); diff --git a/core/server/src/segment_recovery.rs b/core/server-ng/src/segment_recovery.rs similarity index 96% rename from core/server/src/segment_recovery.rs rename to core/server-ng/src/segment_recovery.rs index ecdeaa1709..ef9ba54839 100644 --- a/core/server/src/segment_recovery.rs +++ b/core/server-ng/src/segment_recovery.rs @@ -15,20 +15,20 @@ // specific language governing permissions and limitations // under the License. -//! Server-owned segment recovery. +//! server-ng-owned segment recovery. //! -//! Previously the bootstrap path borrowed `load_segments` from the legacy -//! server implementation to hydrate persisted segments. That loader +//! Previously the bootstrap path borrowed `server::bootstrap::load_segments` +//! from the legacy `server` crate to hydrate persisted segments. That loader //! reads the legacy 16-byte dense per-message index through -//! `server_common::IndexReader`, but the server persists a 24-byte sparse index +//! `server_common::IndexReader`, but server-ng persists a 24-byte sparse index //! (`partitions::IggyIndexWriter`: one entry per flush, absolute `offset`, //! `timestamp`, and batch-start `position`). Reading the 24-byte file with the //! 16-byte parser mis-strides it (the "Index data must be exactly 16 bytes" -//! recovery panic). This module is the server-owned loader, reading the same +//! recovery panic). This module is the server-ng-owned loader, reading the same //! 24-byte format its writer emits. -use crate::server_error::{PartitionChainRefusal, ServerError}; -use configs::server::ServerConfig; +use crate::server_error::{PartitionChainRefusal, ServerNgError}; +use configs::server_ng::ServerNgConfig; use iggy_common::{IggyByteSize, IggyError, PartitionStats}; use partitions::state_transfer::STAGING_SUFFIX; use partitions::{IggyIndexReader, Segment}; @@ -61,12 +61,12 @@ pub struct RecoveredSegment { /// read, or if a segment's index references a batch beyond the end of its /// messages file (torn write). pub async fn load_persisted_segments( - config: &ServerConfig, + config: &ServerNgConfig, stream_id: usize, topic_id: usize, partition_id: usize, stats: &PartitionStats, -) -> Result, ServerError> { +) -> Result, ServerNgError> { let partition_path = config .system .get_partition_path(stream_id, topic_id, partition_id); @@ -207,9 +207,9 @@ fn ensure_contiguous_chain( stream_id: usize, topic_id: usize, partition_id: usize, -) -> Result<(), ServerError> { +) -> Result<(), ServerNgError> { let refused = |reason| { - Err(ServerError::PartitionChainRefused { + Err(ServerNgError::PartitionChainRefused { dir: PathBuf::from(partition_path), stream_id, topic_id, @@ -264,7 +264,9 @@ fn ensure_contiguous_chain( /// Sweeps boot-time scratch (`.staging` spill, orphan `.index`) and returns the /// start offset parsed out of every remaining zero-padded `.log` file name. A /// missing directory means a never-persisted partition. -fn sweep_scratch_files_and_collect_offsets(partition_path: &str) -> Result, ServerError> { +fn sweep_scratch_files_and_collect_offsets( + partition_path: &str, +) -> Result, ServerNgError> { let entries = match fs::read_dir(partition_path) { Ok(entries) => entries, Err(source) if source.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), @@ -343,7 +345,7 @@ async fn recover_segment_bounds( stream_id: usize, topic_id: usize, partition_id: usize, -) -> Result, ServerError> { +) -> Result, ServerNgError> { let reader = IggyIndexReader::new(index_path).await.map_err(|source| { error!( stream_id, @@ -418,7 +420,7 @@ async fn recover_segment_bounds( position = extent; } if !walked_any { - return Err(ServerError::RecoveredSegmentSizeDivergence { + return Err(ServerNgError::RecoveredSegmentSizeDivergence { stream_id, topic_id, partition_id, diff --git a/core/server-ng/src/server_error.rs b/core/server-ng/src/server_error.rs new file mode 100644 index 0000000000..49581a190d --- /dev/null +++ b/core/server-ng/src/server_error.rs @@ -0,0 +1,391 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use consensus::VsrStateError; +use metadata::impls::recovery::RecoveryError; +use server_common::log::LogError; +use shard::ShardCtorError; +use shard_allocator::ShardingError; +use std::path::PathBuf; +use thiserror::Error; + +#[derive(Debug, Error)] +#[non_exhaustive] +pub enum ServerNgError { + #[error(transparent)] + Iggy(Box), + #[error("failed to load server-ng config")] + Config(#[source] configs::ConfigurationError), + #[error("failed to allocate shards from sharding.cpu_allocation")] + ShardAllocator(#[source] ShardingError), + #[error("failed to bind shard {shard_id} to its CPU set")] + CpuAffinityFailed { + shard_id: u16, + #[source] + source: ShardingError, + }, + #[error("failed to bind shard {shard_id} memory to its NUMA node")] + MemoryAffinityFailed { + shard_id: u16, + #[source] + source: ShardingError, + }, + #[error("failed to spawn OS thread for shard {shard_id}")] + ShardSpawnFailed { + shard_id: u16, + #[source] + source: std::io::Error, + }, + // `{source}` is deliberately part of the Display text: the shard-join + // failure report and `%error` log fields print Display only, and the + // source carries the io_uring remediation folded in by + // `server_common::diagnostics::enrich_runtime_create_error`. + #[error("failed to create io_uring runtime for shard {shard_id}: {source}")] + ShardRuntimeCreateFailed { + shard_id: u16, + #[source] + source: std::io::Error, + }, + #[error( + "shard allocator produced zero shards; server must run at least one \ + shard (check [system.sharding] cpu_allocation)" + )] + ShardsCountZero, + #[error( + "computed shards_count = {count} exceeds the maximum of {} shards per \ + server; shard ids must fit in u16 and stay below the OWNER_NONE \ + sentinel", + message_bus::OWNER_NONE - 1 + )] + ShardsCountOverflow { count: usize }, + #[error("system.sharding.inbox_capacity must be in 1..={max}; got {value}")] + InvalidInboxCapacity { value: usize, max: usize }, + #[error("system.sharding.shutdown_drain_timeout must be in (0, {max:?}]; got {value:?}")] + InvalidShutdownDrainTimeout { + value: std::time::Duration, + max: std::time::Duration, + }, + #[error("system.sharding.shutdown_poll_interval must be in (0, {max:?}]; got {value:?}")] + InvalidShutdownPollInterval { + value: std::time::Duration, + max: std::time::Duration, + }, + #[error( + "system.sharding.shutdown_poll_interval ({poll:?}) must be <= \ + shutdown_drain_timeout ({drain:?})" + )] + ShutdownPollExceedsDrain { + poll: std::time::Duration, + drain: std::time::Duration, + }, + #[error("failed to serialize current server-ng config")] + CurrentConfigSerialize(#[source] toml::ser::Error), + #[error("failed to write current server-ng config at {path}")] + CurrentConfigWrite { + path: String, + #[source] + source: std::io::Error, + }, + #[error("failed to initialize server-ng logging")] + Logging(#[source] LogError), + #[error("failed to recover metadata snapshot and journal")] + MetadataRecovery(#[source] RecoveryError), + #[error("failed to open partition superblock at {dir}")] + PartitionSuperblockIo { + dir: PathBuf, + #[source] + source: std::io::Error, + }, + // Quarantines the one partition rather than treating the group as fresh or + // reading through to a superseded view: mirrors the metadata plane's + // `RecoveryError::SuperblockUnreadable` policy, minus the boot refusal, + // because one unreadable partition directory must not strand every healthy + // group on the shard. + #[error( + "partition superblock at {dir} is present but its format version \ + {version} is unrecognized by this build (a downgrade, or a corrupt \ + version field)" + )] + PartitionSuperblockVersionUnknown { dir: PathBuf, version: u16 }, + #[error( + "partition superblock at {dir} is present but a copy holds bytes that \ + do not verify (bit-rot or a checksum failure), so its latest \ + generation cannot be established" + )] + PartitionSuperblockUnverifiable { dir: PathBuf }, + #[error( + "partition superblock at {dir} was checksum-clean but did not decode; \ + tombstoning this partition rather than inferring a stale view" + )] + PartitionSuperblockUndecodable { + dir: PathBuf, + #[source] + source: VsrStateError, + }, + #[error( + "partition superblock at {dir} belongs to a different {field}: expected \ + {expected}, found {found}; a copied or misplaced data directory, or the \ + cluster was resized without reconfiguration" + )] + PartitionSuperblockIdentityMismatch { + dir: PathBuf, + field: metadata::IdentityField, + expected: u128, + found: u128, + }, + // Per-partition, not fatal: the boot path fences this one group (quarantines + // its segment files and materialises it fresh) instead of taking the node + // down for one damaged local chain. The shapes it reports are exactly what a + // failed state-transfer quarantine leaves behind, and the rebuild recovers + // the data from a peer. + #[error( + "partition {stream_id}/{topic_id}/{partition_id} at {dir} recovered an \ + unusable segment chain: {reason}" + )] + PartitionChainRefused { + dir: PathBuf, + stream_id: usize, + topic_id: usize, + partition_id: usize, + reason: PartitionChainRefusal, + }, + #[error( + "shard {shard_id} aborted while waiting for shard-0 to broadcast the metadata \ + factory bundle; shard 0 dropped its sender (most likely it failed to recover)" + )] + MetadataHandoffAborted { shard_id: u16 }, + #[error( + "shard 0 aborted before binding listeners with {remaining} peer shard(s) still loading \ + their on-disk partitions; a peer most likely failed during bootstrap (shutdown flag set)" + )] + ShardBootstrapBarrierAborted { remaining: usize }, + #[error("failed to parse {context} socket address '{address}'")] + SocketAddressParse { + context: &'static str, + address: String, + #[source] + source: std::net::AddrParseError, + }, + #[error("cluster enabled but no node is configured for replica {replica_id}")] + ClusterNodeNotFound { replica_id: u8 }, + #[error("cluster node count {count} exceeds supported u8 replica count")] + ClusterReplicaCountTooLarge { count: usize }, + #[error("cluster mode requires --replica-id to identify the current node")] + MissingReplicaId, + #[error( + "--replica-id {supplied} was passed with cluster.enabled=false; the WAL would commit \ + under replica {default} which permanently fixes this node's identity. Either set \ + cluster.enabled=true with a matching nodes[] entry, or drop --replica-id" + )] + ReplicaIdRequiresCluster { supplied: u8, default: u8 }, + #[error( + "cluster node for replica {replica_id} is missing ports.{transport}; cluster mode \ + requires an explicit roster port for every enabled transport" + )] + ClusterPortMissing { + transport: &'static str, + replica_id: u8, + }, + #[error( + "cluster bootstrap with empty metadata requires both {username_env} and {password_env} to be set before server-ng can create the root user deterministically" + )] + ClusterRootCredentialsRequired { + username_env: &'static str, + password_env: &'static str, + }, + #[error( + "recovered segment for stream {stream_id}, topic {topic_id}, partition {partition_id} at start_offset {start_offset} has message/index divergence (messages_size={messages_size_bytes}, indexed_size={indexed_size_bytes}, end_offset={end_offset}); recovery aborted before opening listeners. Restore the partition from a healthy replica or snapshot, or move the segment aside for offline repair before restarting." + )] + RecoveredSegmentSizeDivergence { + stream_id: usize, + topic_id: usize, + partition_id: usize, + start_offset: u64, + end_offset: u64, + messages_size_bytes: u64, + indexed_size_bytes: u64, + }, + #[error( + "failed to load persisted {consumer_kind} offsets for stream {stream_id}, topic {topic_id}, partition {partition_id} from {path}" + )] + ConsumerOffsetsLoad { + consumer_kind: &'static str, + stream_id: usize, + topic_id: usize, + partition_id: usize, + path: String, + #[source] + source: Box, + }, + #[error( + "recovered namespace stream {stream_id}, topic {topic_id}, partition {partition_id} exceeds configured limits (max_streams={max_streams}, max_topics={max_topics}, max_partitions={max_partitions})" + )] + RecoveredNamespaceOutOfBounds { + stream_id: usize, + topic_id: usize, + partition_id: usize, + max_streams: usize, + max_topics: usize, + max_partitions: usize, + }, + #[error("failed to load {transport} listener credentials")] + ListenerCredentials { + transport: &'static str, + #[source] + source: std::io::Error, + }, + #[error("failed to build the HTTP forward client: {reason}")] + HttpForwardClient { reason: String }, + #[error("failed to construct IggyShard from bootstrap inputs")] + ShardConstruction(#[source] ShardCtorError), + #[error("{} shard thread(s) failed: {}", failures.len(), format_shard_failures(failures))] + ShardJoinFailures { failures: Vec }, +} + +/// Why a recovered segment chain cannot be served. +/// +/// Both shapes mean the same thing operationally -- the local files do not form +/// a chain this replica can serve -- but they are distinguished because they +/// point at different causes: an empty non-tail segment is a failed rebuild's +/// orphan pairing, a hole is a stray or half-unlinked file. +#[derive(Debug)] +pub enum PartitionChainRefusal { + EmptyNonTailSegment { + empty_start: u64, + next_start: u64, + }, + Hole { + previous_start: u64, + previous_end: u64, + next_start: u64, + }, +} + +impl std::fmt::Display for PartitionChainRefusal { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::EmptyNonTailSegment { + empty_start, + next_start, + } => write!( + f, + "segment {empty_start} is empty yet {next_start} follows it, so the \ + chain cannot be served past it" + ), + Self::Hole { + previous_start, + previous_end, + next_start, + } => write!( + f, + "segment {previous_start} ends at offset {previous_end} but the next \ + starts at {next_start}, leaving a hole" + ), + } + } +} + +/// Per-shard outcome captured by [`crate::bootstrap::ShardHandles::join_all`] +/// when a shard either returned `Err` or panicked. +/// +/// Bundled into [`ServerNgError::ShardJoinFailures`] so the operator sees +/// every failing shard rather than only the first one, which previously +/// lived in the trace log alone. +#[derive(Debug)] +pub struct ShardJoinFailure { + pub shard_id: u16, + pub kind: ShardJoinFailureKind, +} + +#[derive(Debug)] +pub enum ShardJoinFailureKind { + Error(Box), + Panic { + message: String, + }, + /// The shard thread never finished inside `shutdown_join_timeout` + /// and was abandoned so process exit is not blocked forever. + Wedged { + waited: std::time::Duration, + }, +} + +fn format_shard_failures(failures: &[ShardJoinFailure]) -> String { + use std::fmt::Write as _; + let mut out = String::new(); + for (idx, failure) in failures.iter().enumerate() { + if idx > 0 { + out.push_str("; "); + } + match &failure.kind { + ShardJoinFailureKind::Error(err) => { + let _ = write!(out, "shard {} -> {err}", failure.shard_id); + } + ShardJoinFailureKind::Panic { message } => { + let _ = write!(out, "shard {} panicked: {message}", failure.shard_id); + } + ShardJoinFailureKind::Wedged { waited } => { + let _ = write!( + out, + "shard {} wedged: thread still running after {waited:?}, abandoned", + failure.shard_id + ); + } + } + } + out +} + +impl From for ServerNgError { + fn from(source: iggy_common::IggyError) -> Self { + Self::Iggy(Box::new(source)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn shard_join_failures_display_aggregates_all_entries() { + let failures = vec![ + ShardJoinFailure { + shard_id: 0, + kind: ShardJoinFailureKind::Error(Box::new(ServerNgError::MissingReplicaId)), + }, + ShardJoinFailure { + shard_id: 2, + kind: ShardJoinFailureKind::Panic { + message: "boom".to_string(), + }, + }, + ]; + let rendered = ServerNgError::ShardJoinFailures { failures }.to_string(); + assert!( + rendered.starts_with("2 shard thread(s) failed:"), + "expected count prefix, got {rendered}" + ); + assert!( + rendered.contains("shard 0 ->"), + "shard 0 entry missing: {rendered}" + ); + assert!( + rendered.contains("shard 2 panicked: boom"), + "shard 2 panic entry missing: {rendered}" + ); + } +} diff --git a/core/server/src/session_manager.rs b/core/server-ng/src/session_manager.rs similarity index 99% rename from core/server/src/session_manager.rs rename to core/server-ng/src/session_manager.rs index 11fca2a5ba..9a35f6a1bd 100644 --- a/core/server/src/session_manager.rs +++ b/core/server-ng/src/session_manager.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -//! Transport-to-consensus session bridge for server. +//! Transport-to-consensus session bridge for server-ng. //! //! Maps ephemeral transport connections to durable consensus sessions. //! Each connection goes through: `connect → login → register → bound`. @@ -50,8 +50,8 @@ pub enum ConnectionState { Authenticated { user_id: u32 }, /// Register committed through consensus. Connection is bound to a /// `(client_id, session)` pair. Requests on this connection use - /// these values to populate `RoutedRequestHeader.client` and - /// `RoutedRequestHeader.session`. + /// these values to populate `RequestHeader.client` and + /// `RequestHeader.session`. Bound { user_id: u32, client_id: u128, @@ -84,7 +84,7 @@ pub struct Connection { /// Bridges transport connections to consensus sessions. /// /// NOT thread-safe: each shard owns one `SessionManager` on its -/// single-threaded compio runtime, the same way the rest of server +/// single-threaded compio runtime, the same way the rest of server-ng /// is structured. All mutators take `&mut self`; the type carries no /// internal locking. /// diff --git a/core/server/src/snapshot.rs b/core/server-ng/src/snapshot.rs similarity index 96% rename from core/server/src/snapshot.rs rename to core/server-ng/src/snapshot.rs index fca3b6e529..8761265ae8 100644 --- a/core/server/src/snapshot.rs +++ b/core/server-ng/src/snapshot.rs @@ -29,7 +29,7 @@ use std::time::Instant; use async_zip::base::write::ZipFileWriter; use async_zip::{Compression, ZipEntryBuilder}; -use configs::server::ServerSystemConfig; +use configs::server_ng::NgSystemConfig; use futures::channel::oneshot; use iggy_common::{IggyDuration, IggyError, SnapshotCompression, SystemSnapshotType}; use tracing::{error, info, warn}; @@ -55,7 +55,7 @@ static SNAPSHOT_IN_PROGRESS: AtomicBool = AtomicBool::new(false); /// [`SNAPSHOT_IN_PROGRESS`]); a concurrent request busy-rejects with /// [`IggyError::SnapshotFileCompletionFailed`]. pub async fn collect( - system_config: Arc, + system_config: Arc, compression: SnapshotCompression, snapshot_types: Vec, ) -> Result, IggyError> { @@ -128,7 +128,7 @@ impl Drop for SnapshotInProgressGuard { } fn collect_blocking( - system_config: &ServerSystemConfig, + system_config: &NgSystemConfig, compression: SnapshotCompression, snapshot_types: &[SystemSnapshotType], ) -> Result, IggyError> { @@ -156,7 +156,7 @@ fn collect_blocking( fn capture( snapshot_type: &SystemSnapshotType, - system_config: &ServerSystemConfig, + system_config: &NgSystemConfig, ) -> io::Result> { match snapshot_type { SystemSnapshotType::FilesystemOverview => { @@ -195,7 +195,7 @@ fn process_list() -> io::Result> { Ok(content) } -fn server_logs(system_config: &ServerSystemConfig) -> io::Result> { +fn server_logs(system_config: &NgSystemConfig) -> io::Result> { // Mirror the logger's path derivation (server_common `Logging::late_init`): // it canonicalizes the configured subdirectory before joining the system // path, so a relative `logging.path` that already exists resolves against the @@ -224,7 +224,7 @@ fn server_logs(system_config: &ServerSystemConfig) -> io::Result> { Ok(content) } -fn server_config(system_config: &ServerSystemConfig) -> io::Result> { +fn server_config(system_config: &NgSystemConfig) -> io::Result> { let config_path = PathBuf::from(system_config.get_runtime_path()).join("current_config.toml"); std::fs::read(config_path) } @@ -277,7 +277,7 @@ mod tests { // second collector thread) rather than piling up threads. let held = SnapshotInProgressGuard::acquire().expect("flag starts free"); let result = futures::executor::block_on(collect( - Arc::new(ServerSystemConfig::default()), + Arc::new(NgSystemConfig::default()), SnapshotCompression::Stored, vec![SystemSnapshotType::Test], )); diff --git a/core/server/src/snapshot/procdump.rs b/core/server-ng/src/snapshot/procdump.rs similarity index 100% rename from core/server/src/snapshot/procdump.rs rename to core/server-ng/src/snapshot/procdump.rs diff --git a/core/server/src/users.rs b/core/server-ng/src/users.rs similarity index 98% rename from core/server/src/users.rs rename to core/server-ng/src/users.rs index 802d092e34..94a5cb270c 100644 --- a/core/server/src/users.rs +++ b/core/server-ng/src/users.rs @@ -41,7 +41,7 @@ use bytes::Bytes; use consensus::MetadataHandle; use iggy_binary_protocol::codec::{WireDecode, WireEncode}; use iggy_binary_protocol::requests::users::{ChangePasswordRequest, CreateUserRequest}; -use iggy_binary_protocol::{Operation, PrepareHeader, RoutedRequestHeader}; +use iggy_binary_protocol::{Operation, PrepareHeader, RequestHeader}; use iggy_common::IggyError; use journal::superblock::SuperblockStore; use journal::{Journal, JournalHandle}; @@ -60,8 +60,8 @@ use std::rc::Rc; /// undecodable password body. pub(crate) fn maybe_rewrite_user_password_request( shard: &Rc>, - request: Message, -) -> Result, IggyError> + request: Message, +) -> Result, IggyError> where B: ShellBus, MJ: JournalHandle + 'static, diff --git a/core/server/src/web.rs b/core/server-ng/src/web.rs similarity index 100% rename from core/server/src/web.rs rename to core/server-ng/src/web.rs diff --git a/core/server-ng/src/wire.rs b/core/server-ng/src/wire.rs new file mode 100644 index 0000000000..e7a047089b --- /dev/null +++ b/core/server-ng/src/wire.rs @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Leaf wire helpers shared by the request-handling modules. +//! +//! Request-body slicing, the `usize -> u32` wire conversion, and the +//! transport-kind discriminant mapping. + +use bytes::Bytes; +use iggy_binary_protocol::RequestHeader; +use iggy_common::IggyError; +use message_bus::installer::conn_info::ClientTransportKind; +use server_common::Message; + +pub(crate) fn request_body(request: &Message) -> &[u8] { + &request.as_slice()[std::mem::size_of::()..request.header().size as usize] +} + +/// Map the transport kind to the legacy wire discriminant +/// (`1=TCP, 2=QUIC, 4=WebSocket`); TLS variants report their base +/// transport. `ClientTransportKind` is `#[non_exhaustive]`, so any other +/// (TCP, TCP-TLS, or a future) variant falls back to TCP. +pub(crate) const fn transport_kind_to_wire(kind: ClientTransportKind) -> u8 { + match kind { + ClientTransportKind::Quic => 2, + ClientTransportKind::Ws | ClientTransportKind::Wss => 4, + _ => 1, + } +} + +pub(crate) fn usize_to_u32(value: usize) -> Result { + u32::try_from(value).map_err(|_| IggyError::InvalidIdentifier) +} + +/// Rebuild a request message with `body` replacing the original payload, +/// preserving the header (and fixing `size`). Used by the primary-side +/// request rewrites that swap a secret-bearing wire body for the +/// hash-carrying replicated body before consensus. +pub(crate) fn rewrite_request_body( + request: &Message, + body: &Bytes, +) -> Result, IggyError> { + let total_size = std::mem::size_of::() + .checked_add(body.len()) + .ok_or(IggyError::InvalidConfiguration)?; + let size = u32::try_from(total_size).map_err(|_| IggyError::InvalidConfiguration)?; + let mut rewritten = Message::::new(total_size); + let header = bytemuck::checked::try_from_bytes_mut::( + &mut rewritten.as_mut_slice()[..std::mem::size_of::()], + ) + .expect("zeroed bytes are a valid request header"); + *header = *request.header(); + header.size = size; + rewritten.as_mut_slice()[std::mem::size_of::()..].copy_from_slice(body); + // TODO(vsr): the body changed but `request_checksum` / `checksum` / + // `checksum_body` were copied verbatim from the original header. Safe + // today because the SDK initializes `request_checksum` to 0 and the + // server does not validate it; the moment integrity checking lands, + // recompute these here (or zero them and re-sign in a follow-up step). + Ok(rewritten) +} diff --git a/core/server/tests/sdk_e2e.rs b/core/server-ng/tests/sdk_e2e.rs similarity index 91% rename from core/server/tests/sdk_e2e.rs rename to core/server-ng/tests/sdk_e2e.rs index 0e52e310e8..2cbd5d64cb 100644 --- a/core/server/tests/sdk_e2e.rs +++ b/core/server-ng/tests/sdk_e2e.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -//! Process-level boot smoke test for the `iggy-server` binary. +//! Process-level boot smoke test for the `iggy-server-ng` binary. //! //! Spawns the production binary against an isolated tempdir with every //! transport except TCP disabled and `IGGY_TCP_ADDRESS` bound to port 0, @@ -33,7 +33,7 @@ use std::time::{Duration, Instant}; use tempfile::TempDir; use toml::Value; -/// `iggy-server` bootstrap can take ~5s on cold caches before the TCP +/// `iggy-server-ng` bootstrap can take ~5s on cold caches before the TCP /// listener binds and `current_config.toml` is written; 30s is generous /// enough for slow CI runners without hanging the suite indefinitely. const STARTUP_TIMEOUT: Duration = Duration::from_secs(30); @@ -54,8 +54,8 @@ struct TestServer { impl TestServer { fn start() -> Self { let data_dir = TempDir::new().expect("tempdir for system.path"); - let mut cmd = Command::cargo_bin("iggy-server") - .expect("iggy-server binary must be built by the test runner"); + let mut cmd = Command::cargo_bin("iggy-server-ng") + .expect("iggy-server-ng binary must be built by the test runner"); cmd.env("IGGY_SYSTEM_PATH", data_dir.path()) // Ephemeral port; the actual bound port is read back from // `runtime/current_config.toml` after the listener binds. @@ -71,7 +71,7 @@ impl TestServer { let mut child = cmd .spawn() - .expect("iggy-server must spawn under cargo test"); + .expect("iggy-server-ng must spawn under cargo test"); let runtime_path = data_dir.path().join("runtime").join("current_config.toml"); let tcp_addr = match wait_for_bound_tcp(&runtime_path, STARTUP_TIMEOUT) { @@ -82,7 +82,7 @@ impl TestServer { // when `data_dir` goes out of scope at the panic. let _ = child.kill(); let _ = child.wait(); - panic!("server startup failed: {err}"); + panic!("server-ng startup failed: {err}"); } }; @@ -158,7 +158,7 @@ fn read_bound_tcp(config_path: &Path) -> Result { Ok(addr) } -/// Spawn `iggy-server` and verify the production binary bootstraps +/// Spawn `iggy-server-ng` and verify the production binary bootstraps /// cleanly. /// /// Specifically asserts: @@ -168,7 +168,7 @@ fn read_bound_tcp(config_path: &Path) -> Result { /// `current_config.toml`). /// * The bound port is `connect()`-able from this test process. #[tokio::test(flavor = "current_thread")] -async fn server_bootstraps_and_binds_ephemeral_tcp_port() { +async fn server_ng_bootstraps_and_binds_ephemeral_tcp_port() { let server = TestServer::start(); assert_ne!( server.tcp_addr.port(), @@ -180,5 +180,5 @@ async fn server_bootstraps_and_binds_ephemeral_tcp_port() { // `connect()`-able once; doing it again here is a sanity check that // the listener stays bound for the lifetime of the test harness. let _ = StdTcpStream::connect_timeout(&server.tcp_addr, Duration::from_secs(1)) - .expect("server TCP listener must accept connections post-bootstrap"); + .expect("server-ng TCP listener must accept connections post-bootstrap"); } diff --git a/core/server/Cargo.toml b/core/server/Cargo.toml index e36b0d1dd8..2c29e13b30 100644 --- a/core/server/Cargo.toml +++ b/core/server/Cargo.toml @@ -17,66 +17,13 @@ [package] name = "server" -version = "0.9.0-edge.2" +version = "0.8.2-edge.1" edition = "2024" license = "Apache-2.0" publish = false -[package.metadata.cargo-udeps.ignore] -normal = ["tracing-appender"] - [package.metadata.cargo-machete] -ignored = [ - "ahash", - "anyhow", - "argon2", - "async-channel", - "async_zip", - "axum", - "axum-server", - "bytes", - "chrono", - "ctrlc", - "cyper", - "cyper-axum", - "dashmap", - "err_trail", - "error_set", - "figlet-rs", - "hash32", - "human-repr", - "hwlocality", - "jsonwebtoken", - "left-right", - "mimalloc", - "mime_guess", - "nix", - "opentelemetry", - "opentelemetry-appender-tracing", - "opentelemetry-otlp", - "opentelemetry-semantic-conventions", - "opentelemetry_sdk", - "papaya", - "rand", - "ringbuffer", - "rmp-serde", - "rolling-file", - "rust-embed", - "rustls", - "rustls-pemfile", - "send_wrapper", - "serde", - "slab", - "socket2", - "strum", - "sysinfo", - "tempfile", - "tracing-appender", - "tracing-opentelemetry", - "ulid", - "uuid", - "vergen-git2", -] +ignored = ["vergen-git2"] [[bin]] name = "iggy-server" @@ -91,57 +38,40 @@ systemd = ["dep:sd-notify"] [dependencies] ahash = { workspace = true } -argon2 = { workspace = true } +anyhow = { workspace = true } async-channel = { workspace = true } async_zip = { workspace = true } axum = { workspace = true } axum-server = { workspace = true } -blake3 = { workspace = true } -bytemuck = { workspace = true } bytes = { workspace = true } chrono = { workspace = true } clap = { workspace = true } compio = { workspace = true } configs = { workspace = true } -consensus = { workspace = true } -crossfire = { workspace = true } ctrlc = { workspace = true } cyper = { workspace = true } cyper-axum = { workspace = true } -cyper-core = { workspace = true } dashmap = { workspace = true } dotenvy = { workspace = true } err_trail = { workspace = true } error_set = { workspace = true } figlet-rs = { workspace = true } +flume = { workspace = true } fs2 = { workspace = true } futures = { workspace = true } hash32 = { workspace = true } human-repr = { workspace = true } -hyper = { workspace = true } -hyper-util = { workspace = true } iggy_binary_protocol = { workspace = true } iggy_common = { workspace = true } -journal = { workspace = true } jsonwebtoken = { workspace = true } left-right = { workspace = true } -message_bus = { workspace = true } -metadata = { workspace = true } mimalloc = { workspace = true, optional = true } mime_guess = { workspace = true, optional = true } nix = { workspace = true } -opentelemetry = { workspace = true } -opentelemetry-appender-tracing = { workspace = true } -opentelemetry-otlp = { workspace = true } -opentelemetry-semantic-conventions = { workspace = true } -opentelemetry_sdk = { workspace = true } papaya = { workspace = true } -partitions = { workspace = true } prometheus-client = { workspace = true } -rand = { workspace = true } ringbuffer = { workspace = true } rmp-serde = { workspace = true } -rolling-file = { workspace = true } rust-embed = { workspace = true, optional = true } rustls = { workspace = true } rustls-pemfile = { workspace = true } @@ -151,7 +81,6 @@ send_wrapper = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } server_common = { workspace = true } -shard = { workspace = true } shard_allocator = { workspace = true } slab = { workspace = true } socket2 = { workspace = true } @@ -164,33 +93,8 @@ tokio = { workspace = true } toml = { workspace = true } tower-http = { workspace = true } tracing = { workspace = true } -tracing-appender = { workspace = true } -tracing-opentelemetry = { workspace = true } ulid = { workspace = true } uuid = { workspace = true } -[target.'cfg(not(target_env = "musl"))'.dependencies] -hwlocality = { workspace = true } - -[target.'cfg(target_env = "musl")'.dependencies] -hwlocality = { workspace = true, features = ["vendored"] } - [build-dependencies] vergen-git2 = { workspace = true } - -[dev-dependencies] -assert_cmd = { workspace = true } -bytemuck = { workspace = true } -# Reconciler unit tests assert on `ShardMetrics` snapshots and -# `IggyShard::parked_frame_count`, gated to test/simulator so they cannot grow -# production callers. `shard`'s own `cfg(test)` is false when compiled as our -# dependency, so the feature is how those accessors become visible. The resolver -# keeps a dev-dependency's features out of non-test targets, so a production -# build still links `shard` without `simulator`. -shard = { workspace = true, features = ["simulator"] } -tokio = { workspace = true, features = ["full", "test-util"] } - -[lints.clippy] -enum_glob_use = "deny" -pedantic = "deny" -nursery = "warn" diff --git a/core/server/Dockerfile b/core/server/Dockerfile index cbeaaa0a5c..b2dac2d582 100644 --- a/core/server/Dockerfile +++ b/core/server/Dockerfile @@ -29,7 +29,6 @@ ARG RUST_VERSION=1.97.1 ARG ALPINE_VERSION=3.23 - # ── from-source path ───────────────────────────────────────────────────────── FROM --platform=$BUILDPLATFORM lukemathwalker/cargo-chef:latest-rust-${RUST_VERSION}-alpine${ALPINE_VERSION} AS chef WORKDIR /app @@ -155,12 +154,12 @@ RUN apt-get update && apt-get install -y \ libudev-dev \ pkg-config \ && rm -rf /var/lib/apt/lists/* -COPY --from=prebuilt /out/iggy-server /usr/local/bin/iggy-server -COPY --from=prebuilt /out/iggy /usr/local/bin/iggy -COPY --from=license-gen /app/LICENSE-binary /usr/share/doc/iggy/LICENSE-binary +COPY --from=prebuilt /out/iggy-server /usr/local/bin/iggy-server +COPY --from=prebuilt /out/iggy /usr/local/bin/iggy +COPY --from=license-gen /app/LICENSE-binary /usr/share/doc/iggy/LICENSE-binary COPY LICENSE NOTICE /usr/share/doc/iggy/ RUN echo "═══════════════════════════════════════════════════════════════" && \ - echo " IGGY SERVER BUILD SUMMARY " && \ + echo " IGGY SERVER BUILD SUMMARY " && \ echo "═══════════════════════════════════════════════════════════════" && \ echo "Build Type: PREBUILT BINARIES" && \ echo "Platform: ${TARGETPLATFORM:-linux/amd64}" && \ @@ -190,7 +189,7 @@ COPY --from=builder /app/iggy /usr/local/bin/iggy COPY --from=builder /app/LICENSE-binary /usr/share/doc/iggy/LICENSE-binary COPY LICENSE NOTICE /usr/share/doc/iggy/ RUN echo "═══════════════════════════════════════════════════════════════" && \ - echo " IGGY SERVER BUILD SUMMARY " && \ + echo " IGGY SERVER BUILD SUMMARY " && \ echo "═══════════════════════════════════════════════════════════════" && \ echo "Build Type: FROM SOURCE" && \ echo "Platform: ${TARGETPLATFORM:-linux/amd64}" && \ diff --git a/core/server/README.md b/core/server/README.md index f0150b32eb..fd4f8cea89 100644 --- a/core/server/README.md +++ b/core/server/README.md @@ -1,46 +1,19 @@ # Apache Iggy Server -The core server component of Apache Iggy: a persistent, append-only log for message streaming. It runs thread-per-core and shared-nothing on `io_uring` (through `compio`), and commits every write through Viewstamped Replication (VSR), so the same binary serves a standalone node and a multi-node cluster. +This is the core server component of Apache Iggy. You can run it directly with `cargo run --bin iggy-server --release` or use the Docker image `apache/iggy:latest` (the `edge` tag is for the latest development version). -Clients connect over TCP (custom binary protocol), QUIC, WebSocket, or the HTTP REST API. - -## Running - -```sh -cargo run --bin iggy-server --release -``` - -The Docker image `apache/iggy:latest` ships the server together with the CLI; the `edge` tag tracks the latest development build. - -To run one node of a cluster, pass its replica ID from the `cluster.nodes` roster: - -```sh -cargo run --bin iggy-server --release -- --replica-id 0 -``` - -`--replica-id` is the only command line argument; everything else is configuration. - -## Configuration - -Settings are read from [config.toml](config.toml), resolved relative to the working directory. Set `IGGY_CONFIG_PATH` to load a different file. - -Any single value can be overridden with an `IGGY_`-prefixed environment variable that mirrors the TOML path: - -```sh -IGGY_TCP_ADDRESS=0.0.0.0:8090 IGGY_HTTP_ENABLED=false cargo run --bin iggy-server -``` - -Cluster membership, quorum and replica addressing live under `[cluster]`. +The configuration file is located at [core/server/config.toml](https://github.com/apache/iggy/blob/master/core/server/config.toml). You can customize the server settings by modifying this file or by using environment variables e.g. `IGGY_TCP_ADDRESS=0.0.0.0:8090`. ## Systemd integration -Build with the `systemd` feature to enable readiness and watchdog notifications: +Build with the `systemd` feature to enable systemd readiness and watchdog notifications: ```sh cargo build --bin iggy-server --release --features systemd ``` -The server sends `READY=1` only after every enabled transport is bound and accepting, so a unit ordered after it can dial as soon as it is notified. When the unit sets `WatchdogSec=`, the server pings `WATCHDOG=1` at half that interval. On shutdown it sends `STOPPING=1`, which stops a long drain from counting against the watchdog. +The server will notify systemd when it is ready and then periodically send +watchdog messages at half the configured `WatchdogSec` interval for the unit. ![Server](../../assets/server.png) diff --git a/core/server/config.toml b/core/server/config.toml index 418d4e9fd3..b1609e7a3e 100644 --- a/core/server/config.toml +++ b/core/server/config.toml @@ -20,8 +20,6 @@ # Maximum time a partition can remain in pending revocation before being force-transferred to the target member. rebalancing_timeout = "30s" # How often the periodic checker scans for timed-out pending revocations. -# TODO(hubcio): inert in the server, which paces the scan from -# system.sharding.reconcile_periodic_interval instead. Boot warns when set. rebalancing_check_interval = "5s" [data_maintenance.messages] @@ -36,18 +34,10 @@ interval = "1 m" # Determines if the HTTP server is active. # `true` enables the server, allowing it to handle HTTP requests. # `false` disables the server, preventing it from handling HTTP requests. -# In cluster mode, followers forward control-plane requests (streams, topics, -# users, ...) to the current primary when a cluster-wide JWT key exists (see -# http.jwt / cluster.auth below). -# TODO: forwarding does not cover the partition-plane APIs yet - message -# produce and consumer-offset writes are never forwarded and must reach the -# partition's primary node directly (message polls read locally on any node). enabled = true # Specifies the network address and port for the HTTP server. # The format is "HOST:PORT". For example, "127.0.0.1:3000" listens on localhost only on port 3000. -# In cluster mode the HOST still picks the bind interface, while the port -# comes from this node's cluster.nodes ports.http entry. address = "127.0.0.1:3000" # Maximum size of the request body in bytes. For security reasons, the default limit is 2 MB. @@ -75,9 +65,7 @@ enabled = true allowed_methods = ["GET", "POST", "PUT", "DELETE"] # Defines which origins are permitted to make cross-origin requests. -# An asterisk "*" as the first entry allows all origins (any entries after it -# are ignored); "*" in any other position fails the config. Specific domains -# can be listed to restrict access. +# An asterisk "*" allows all origins. Specific domains can be listed to restrict access. allowed_origins = ["*"] # Lists allowed headers that can be used in CORS requests. @@ -85,14 +73,11 @@ allowed_origins = ["*"] allowed_headers = ["content-type", "authorization"] # Headers that browsers are allowed to access in CORS responses. -# `iggy-view` carries the current VSR view number; exposing it lets browser -# clients read it on cross-origin responses. -exposed_headers = ["iggy-view"] +# An empty array means no additional headers are exposed to browsers. +exposed_headers = [""] # Determines if credentials like cookies or HTTP auth can be included in CORS requests. -# `true` allows credentials to be included, useful for authenticated sessions; -# it requires explicit (non-wildcard) allowed_origins, allowed_headers, and -# exposed_headers. +# `true` allows credentials to be included, useful for authenticated sessions. # `false` prevents credentials, enhancing privacy and security. allow_credentials = false @@ -130,10 +115,6 @@ not_before = "0 s" # Secret key for encoding JWTs. # If left empty, a secure random secret will be generated on each server start. -# In cluster mode a configured secret (identical on every node) makes bearers -# valid cluster-wide and activates follower-to-primary HTTP forwarding; with -# cluster.auth enabled the key is instead derived from the shared PSK. Without -# either, tokens are node-local and forwarding stays disabled. encoding_secret = "" # Secret key for decoding JWTs. @@ -199,12 +180,9 @@ enabled = true address = "127.0.0.1:8090" # Enable TCP socket migration across shards. -# TODO(hubcio): inert in the server, not implemented. Boot warns when set. socket_migration = true # Whether to use ipv4 or ipv6 -# TODO(hubcio): inert in the server, which takes the family from the -# tcp.address string. Boot warns when set. ipv6 = false # TLS configuration for the TCP server. @@ -226,8 +204,6 @@ cert_file = "core/certs/iggy_cert.pem" key_file = "core/certs/iggy_key.pem" # Configuration for the TCP socket -# TODO(hubcio): the whole section is inert in the server, which leaves the OS -# defaults in place. Boot warns when override_defaults is set. [tcp.socket] # Whether to overwrite the OS-default socket parameters override_defaults = false @@ -259,37 +235,26 @@ enabled = true # For example, "127.0.0.1:8080" binds to localhost on port 8080. address = "127.0.0.1:8080" -# Maximum number of simultaneous bidirectional streams per QUIC -# connection. The message bus opens exactly one bidi stream per peer -# (no multiplexing), so any value above 1 is wasted on extra -# preallocated quinn-proto state. -max_concurrent_bidi_streams = 1 +# Maximum number of simultaneous bidirectional streams in QUIC. +max_concurrent_bidi_streams = 10_000 -# Size of the buffer for sending datagrams in QUIC. Binary-aligned to -# match QuicTuning::default() (102_400 bytes) and the rest of this [quic] -# block, which uses MiB throughout (`send_window`, `receive_window`). -datagram_send_buffer_size = "100 KiB" +# Size of the buffer for sending datagrams in QUIC. +datagram_send_buffer_size = "100 KB" -# Initial Maximum Transmission Unit (MTU) for QUIC connections. Binary -# units for the same reason as `datagram_send_buffer_size`. -initial_mtu = "8 KiB" +# Initial Maximum Transmission Unit (MTU) for QUIC connections. +initial_mtu = "8 KB" -# Send-flow window per connection. Sized to fit a single max-size -# framed message (`message_bus.max_message_size`) without -# head-of-line wait. -send_window = "64 MiB" +# Size of the sending window in QUIC, controlling data flow. +send_window = "100 KB" -# Receive-flow window per connection. Symmetric with `send_window`. -receive_window = "64 MiB" +# Size of the receiving window in QUIC, controlling data flow. +receive_window = "100 KB" -# Interval for sending QUIC keep-alive PINGs. One third of -# `max_idle_timeout` so up to two consecutive losses fit before the -# idle timer closes the connection. Set to "0 s" to disable. -keep_alive_interval = "10 s" +# Interval for sending keep-alive messages in QUIC. +keep_alive_interval = "5 s" -# Maximum idle time before a QUIC connection is closed. Set to -# "0 s" to disable (not recommended). -max_idle_timeout = "30 s" +# Maximum idle time before a QUIC connection is closed. +max_idle_timeout = "10 s" # QUIC certificate configuration. [quic.certificate] @@ -305,8 +270,6 @@ cert_file = "core/certs/iggy_cert.pem" key_file = "core/certs/iggy_key.pem" # Configuration for the QUIC socket -# TODO(hubcio): the whole section is inert in the server, which leaves the OS -# defaults in place. Boot warns when override_defaults is set. [quic.socket] # Whether to override the OS-default socket parameters override_defaults = false @@ -330,12 +293,9 @@ enabled = true # Controls whether data saving is synchronous (enforce fsync) or asynchronous. # `true` for synchronous saving, ensuring data integrity at the cost of performance. # `false` for asynchronous saving, improving performance but with delayed data writing. -# TODO(hubcio): inert in the server, which only flushes on shutdown and has no -# periodic saver to configure. Boot warns when set. enforce_fsync = true # Interval for running the message saver. -# TODO(hubcio): inert in the server, see enforce_fsync above. Boot warns when set. interval = "30 s" # Personal access token configuration. @@ -387,8 +347,6 @@ endpoint = "http://localhost:7281/v1/traces" path = "local_data" # Backup configuration -# TODO(hubcio): backup is not supported; both paths below are -# inert. Boot warns when either is set. [system.backup] # Path for storing backup. path = "backup" @@ -398,9 +356,6 @@ path = "backup" # Subpath of the backup directory where converted segment data is stored after compatibility conversion. path = "compatibility" -# TODO(hubcio): the three tunables below are inert in the server, whose state -# writes do not go through the legacy retrying file layer. Boot warns when any -# is set. [system.state] # Determines whether to enforce file synchronization on state updates (boolean). # `true` ensures immediate writing of data to disk for durability. @@ -457,8 +412,6 @@ rotation_check_interval = "1 h" retention = "7 days" # Interval for printing system information to the log. -# TODO(hubcio): inert in the server, which has no sysinfo printer. Boot warns -# when set. sysinfo_print_interval = "10 s" # Encryption configuration @@ -475,15 +428,15 @@ key = "" # Compression configuration [system.compression] +# Reserved for future server-side compression support; use "none" today. +# For manual compression, store the algorithm in message headers. # Allows overriding the default compression algorithm per data segment (boolean). -# `true` permits different compression algorithms for individual segments. -# `false` means all data segments use the default compression algorithm. -# TODO(hubcio): inert in the server, where live compression is already per-topic -# from the request. Boot warns when set. +# `true` will permit different compression algorithms for individual segments. +# `false` will make all data segments use the default compression algorithm. allow_override = false # The default compression algorithm used for data storage (string). -# "none" indicates no compression, other values can specify different algorithms. +# "none" indicates no compression. Other values are reserved for future support. default_algorithm = "none" # Stream configuration @@ -527,12 +480,9 @@ path = "partitions" enforce_fsync = false # Enables checksum validation for data integrity (boolean). -# `true` re-hashes every batch a disk poll reads and fails the poll closed on a -# mismatch, so a segment damaged at rest is reported instead of served. -# `false` skips the re-hash and serves whatever decodes, which hands a consumer -# bytes provably not the ones written. Only turn it off with a corruption guard -# somewhere else in the stack. -validate_checksum = true +# `true` activates CRC checks when loading data, guarding against corruption. +# `false` skips these checks for faster loading at the risk of undetected corruption. +validate_checksum = false # The count threshold of buffered messages before triggering a save to disk. # Together with `size_of_messages_required_to_save` it defines the threshold. @@ -552,11 +502,8 @@ size_of_messages_required_to_save = "1 MiB" # Example: if `size` is set "1GiB", the actual segment size may be 1GiB + the size of remaining messages in received batch. # Maximum size is 1 GiB. Size has to be a multiple of 512 B. size = "1 GiB" -# Reserves segment space in advance when supported by the local filesystem. -preallocate = true # Configures whether expired segments are archived (boolean) or just deleted without archiving. -# Unsupported: setting this to `true` aborts boot. archive_expired = false # Controls whether to cache indexes (time and positional) for segment access. @@ -564,8 +511,6 @@ archive_expired = false # - "true" or "all": keeps indexes in memory, speeding up data retrieval at the cost of memory # - "open_segment": keeps indexes in memory only for the currently open segment # - "false" or "none": reads indexes from disk, which can conserve memory at the cost of access speed -# TODO(hubcio): inert in the server, which picks its own index residency. Boot -# warns when set. cache_indexes = "open_segment" # Message deduplication configuration @@ -573,7 +518,6 @@ cache_indexes = "open_segment" # Controls whether message deduplication is enabled (boolean). # `true` activates deduplication, ignoring messages with duplicate IDs. # `false` treats each message as unique, even if IDs are duplicated. -# Unsupported: setting this to `true` aborts boot. enabled = false # Maximum number of ID entries in the deduplication cache (u64). max_entries = 10000 @@ -583,7 +527,6 @@ expiry = "1 m" # Recovery configuration in case of lost data [system.recovery] # Controls whether streams/topics/partitions should be recreated if the expected data for existing state is missing (boolean). -# Unsupported: setting this to `true` aborts boot. recreate_missing_state = false # Memory pool configuration @@ -617,187 +560,26 @@ enabled = false # This prevents accidental cross-cluster communication. name = "iggy-cluster" -# Backup-side liveness window for a consensus plane's primary (duration). -# A replica that sees no primary traffic for this long starts a view change. -# Raise it on oversubscribed hosts where scheduling stalls fake primary -# death. Must be at least "2s" and at least 4x commit_broadcast_interval: the -# primary signals liveness through its commit broadcast, and the window must -# span several broadcasts so one delayed broadcast never trips an election. -heartbeat_timeout = "5s" - -# How often the primary broadcasts its commit point to every backup (duration). -# This is the cluster's liveness signal: each broadcast resets every backup's -# heartbeat_timeout window and carries the latest commit point forward. Must be -# nonzero and, with heartbeat_timeout, satisfy heartbeat_timeout >= 4x this -# value. Drives the consensus CommitMessage timer. -commit_broadcast_interval = "500ms" - -# How often the primary retransmits prepares that backups have not yet acked -# (duration). Lower values recover faster from a dropped prepare at the cost of -# more replica traffic; must be nonzero. Drives the consensus Prepare timer. -prepare_retransmit_interval = "250ms" - -# How often a replica retransmits its StartViewChange / DoViewChange while a -# view change is in progress (duration). Lower values converge a healthy -# election faster at the cost of more replica traffic; must be nonzero. Drives -# both consensus view-change retransmit timers. -view_change_retransmit_interval = "500ms" - -# Backstop for a stalled view change (duration): one that does not conclude -# within this window escalates to a fresh cluster-wide election. Must be nonzero -# and at least 4x view_change_retransmit_interval, so a few dropped view-change -# messages retransmit rather than prematurely escalate. -view_change_status_timeout = "5s" - -# How often a recovering or view-change backup re-requests the current view's -# StartView from its primary (duration); must be nonzero. Drives the consensus -# RequestStartView timer. -request_start_view_retransmit_interval = "1s" - -# How many consecutive unanswered RequestStartView probes a recovering replica -# tolerates before falling back to an election (integer). A full-cluster restart -# leaves nobody settled to answer, so the replica elects on its recovered log. -# Must be between 1 and 100. -view_probe_attempts_max = 5 - -# How long a stalled journal-repair stream waits before re-requesting its -# remaining window from the serving peer (duration). Repair frames are -# fire-and-forget over the lossy bus, so a session with no retry wedges forever -# on a single dropped frame. Paces both the metadata and partition repair loops; -# must be nonzero. -repair_retry_interval = "1s" - -# Prepares a peer serves per repair round before the requester walks to the next -# chunk (integer). Each frame rides the per-peer message-bus queue, so this must -# stay strictly below message_bus.peer_queue_capacity or a full round overruns -# the queue and drops frames. Must be > 0 and <= 1024. -repair_chunk_max = 128 - -# Replica-to-replica authentication (PSK + BLAKE3 keyed-MAC handshake). -[cluster.auth] -# When true, every replica peer must complete the authenticated handshake or be -# rejected, and shared_secret becomes mandatory. Off by default = legacy -# unauthenticated replica traffic. Enabling it is a coordinated-restart change. -# With http enabled and no http.jwt secrets configured, the PSK also becomes -# the JWT key source, making bearers valid cluster-wide and activating -# follower-to-primary HTTP forwarding. -enabled = false - -# Cluster-wide pre-shared key, >= 32 bytes of CSPRNG output, byte-identical on -# every node. Prefer the IGGY_CLUSTER_AUTH_SHARED_SECRET env var (masked in -# logs, never persisted) over storing it on disk. Ignored when enabled = false. -shared_secret = "" - -# Retiring pre-shared key, accepted for verification only during a rolling key -# rotation (this node keeps signing with shared_secret). Rotate in three rolls: -# 1) shared_secret = old + previous_shared_secret = new on every node, -# 2) shared_secret = new + previous_shared_secret = old on every node, -# 3) shared_secret = new alone. Leave empty outside a rotation. Same length -# floor and env-var preference as shared_secret -# (IGGY_CLUSTER_AUTH_PREVIOUS_SHARED_SECRET). -previous_shared_secret = "" - -# Replica-to-replica TLS for the consensus (tcp_replica) port. -[cluster.tls] -# When true every replica connection is wrapped in TLS 1.3 (ALPN -# "iggy-replica") before the replica handshake runs. Requires -# cluster.auth.enabled: TLS carries no client certificates, so it -# authenticates the acceptor only; the PSK handshake authenticates the -# peer, TLS supplies confidentiality. Off by default = plaintext replica -# traffic. Enabling it is a coordinated-restart change: a TLS dialer -# cannot talk to a plaintext acceptor or vice versa. -enabled = false - -# When true the node auto-generates a self-signed certificate at boot and -# the dialer accepts ANY peer certificate. When false (default), -# cert_file / key_file / ca_file are all required. -self_signed = false - -# PEM certificate chain presented by this node's acceptor side. -cert_file = "" - -# PEM private key matching cert_file. -key_file = "" - -# PEM trust anchor(s) the dialer verifies peer certificates against. -# Unused when self_signed = true. -ca_file = "" - -# Full roster of cluster members. Byte-identical on every node. The running -# node's identity is resolved at launch from the '--replica-id ' CLI -# flag, which selects the entry in this list that describes the current -# node. All other entries are remote peers. -# -# 'ip' is the node's roster address. Replica-to-replica traffic and -# follower-to-primary HTTP forwarding use it. It is not the bind interface for -# tcp/quic/http/websocket, which comes from each transport's own 'address' -# setting above; the roster supplies those transports their port only. A -# cluster spread across hosts therefore needs each transport's 'address' set to -# '0.0.0.0' or the routable NIC; the defaults below listen on loopback only, -# and a bind that cannot serve the advertised 'ip' is warned about at startup. -# -# Each node may also set 'advertised_address': the client-facing address -# handed out in cluster metadata and leader redirects. Set it when 'ip' is -# a private replica-network address unreachable by clients (Docker, -# Kubernetes, NAT). Accepts a literal IPv4/IPv6 address or a DNS hostname -# (RFC 1123: ASCII letters, digits, '-' and '.'; no port, no trailing dot). -# When unset, clients receive 'ip'. -# -# When different client networks need different addresses (a public -# 'advertised_address' would route in-VPC clients out through the public -# side), add per-network 'advertised_addresses' selectors: clients whose -# peer IP falls inside 'client_cidr' are handed 'address' instead of the -# catch-all. 'address' takes the same forms as 'advertised_address' -# (literal IP or RFC 1123 hostname, never a port - ports always come from -# 'ports'). At most 16 selectors per node; boot also rejects duplicate -# 'client_cidr' entries on one node (compared truncated, so '10.0.1.0/16' -# duplicates '10.0.0.0/16') and any two nodes advertising one host:port -# to overlapping client sets - reusing a host:port across nodes is legal -# only when no client would resolve both nodes to it. -# -# The longest matching prefix wins; clients matching no selector fall -# back to 'advertised_address', then 'ip'. Matching is per address -# family: '0.0.0.0/0' matches no IPv6 client and '::/0' matches no IPv4 -# client, so covering both families takes one selector per family (or the -# catch-all). IPv4-mapped IPv6 CIDRs ('::ffff:10.0.0.0/104') match like -# their IPv4 form only at prefix length 96 or longer; shorter ones match -# native IPv6 clients only. Matching sees the transport-level peer -# address, so clients behind a proxy or load balancer match the proxy's -# network, not their own. +# Full roster of cluster members. This list must be byte-identical on every +# node so operators can ship a single config.toml. The running node's +# identity is resolved at launch from the '--replica-id ' CLI flag, +# which selects the entry in this list that describes the current node. +# All other entries are treated as remote peers. # -# Every 'address' must be routable from inside its own 'client_cidr': -# leader-aware SDK clients redial whatever address metadata advertises, -# so a selector pointing at a host its own clients cannot reach strands -# them mid-redirect. Prefer literal IPs over hostnames - the SDKs differ -# in how they compare an advertised hostname against the address they -# dialed, and a mismatch costs a reconnect on every fresh connect. +# Requirements (enforced at startup): +# - replica_id values must be unique and strictly less than nodes.len(), +# - replica_id for this node must be set via `--replica-id ` command line argument +# - node names and IPs must be non-empty +# - (ip, port) pairs must be unique across the list # -# Note for rolling upgrades: older server binaries reject a TOML config -# containing 'advertised_addresses' but silently ignore the equivalent -# 'IGGY_CLUSTER_NODES_*_ADVERTISED_ADDRESSES_*' env vars; either way, -# upgrade every binary first, then add selectors. Mid-upgrade, an env-var -# roster would serve selector addresses from upgraded nodes and the -# catch-all from the rest. -# -# [[cluster.nodes]] -# name = "iggy-node-1" -# ip = "10.0.1.5" # replica plane + last-resort fallback -# advertised_address = "203.0.113.10" # catch-all for unmatched clients -# replica_id = 0 -# ports = { tcp = 8090, http = 3000, tcp_replica = 9090 } -# -# [[cluster.nodes.advertised_addresses]] -# client_cidr = "10.0.0.0/16" # in-VPC clients stay private -# address = "10.0.1.5" -# -# In cluster mode, 'ports' is the single source of listener ports: every -# enabled transport needs an explicit per-node port, otherwise the server -# refuses to start. +# Each field in 'ports' is optional. If omitted, the transport's primary +# port (tcp.address, quic.address, etc.) configured on the running node is +# used as the fallback when computing peer endpoints for cluster metadata. [[cluster.nodes]] name = "iggy-node-1" ip = "127.0.0.1" replica_id = 0 -ports = { tcp = 8090, quic = 8080, http = 3000, websocket = 8092, tcp_replica = 9090 } +ports = { tcp = 8090, quic = 8080, http = 3000, websocket = 8070, tcp_replica = 9090 } [[cluster.nodes]] name = "iggy-node-2" @@ -805,14 +587,12 @@ ip = "127.0.0.1" replica_id = 1 ports = { tcp = 8091, quic = 8081, http = 3001, websocket = 8093, tcp_replica = 9091 } -# Example additional node (commented out). tcp skips 8092-8094: those are the -# websocket ports of the three nodes, which collide once nodes share a host. +# Example additional node (commented out): # [[cluster.nodes]] # name = "iggy-node-3" # ip = "192.168.1.100" -# advertised_address = "iggy-node-3.example.com" # replica_id = 2 -# ports = { tcp = 8095, quic = 8082, http = 3002, websocket = 8094, tcp_replica = 9092 } +# ports = { tcp = 8092, http = 3002 } # Sharding configuration [system.sharding] @@ -825,7 +605,6 @@ ports = { tcp = 8091, quic = 8081, http = 3001, websocket = 8093, tcp_replica = # + "numa:auto": Use all available numa node, cores # + "numa:nodes=0,1;cores=4;no_ht=true": Use NUMA node 0 and 1, each nodes use 4 cores, and no hyperthreads cpu_allocation = "numa:auto" - # Whether shard threads are pinned to dedicated CPU cores (default: true). # Pinned cores are drawn from the process's allowed CPU set (affinity/cpuset # mask), so the server cooperates with systemd `AllowedCPUs=` and container @@ -835,202 +614,12 @@ cpu_allocation = "numa:auto" # process onto the same low-numbered cores. pin_cores = true -# Per-shard inter-shard inbox capacity. Bounded by design: consensus-frame -# drops recover via VSR retransmit, but cross-shard client-reply drops are -# terminal. Size for the worst-case sum of both: the consensus working set -# (~ the prepare queue depth of the planes the shard hosts - [metadata] on -# shard 0, [partition] elsewhere - times replica_count times directions) plus -# peak client-reply fan-out per shard. Both depths are tunable, so raising -# either raises the capacity needed here. -inbox_capacity = 1024 - -# Wall-clock budget for a single shard's bus drain on shutdown. Drives -# the per-shard watchdog and the parallel-join survivor path; sized -# larger than typical TCP RTT times in-flight write-batch so writers -# receive their full last `write_vectored_all` budget before the -# connection registry force-tears the bus. Slow-fsync hosts may need -# to extend this past the default. -shutdown_drain_timeout = "10 s" - -# Poll cadence for the cross-thread shutdown flag and for the -# metadata-handoff loops. Trades off Ctrl-C latency against idle wakeup -# cost; the default keeps shutdown observably prompt without measurable -# scheduler overhead. Must be less than or equal to shutdown_drain_timeout. -shutdown_poll_interval = "50 ms" - -# Hard wall-clock deadline for joining shard threads at process exit. A -# shard whose pump or listener wedges past this budget is abandoned with -# an error log instead of blocking exit forever. Must be at least -# shutdown_drain_timeout, or shards would be abandoned mid-drain. -shutdown_join_timeout = "30 s" - -# Safety-tick cadence for the partition reconciliation loop. The reconciler -# also wakes on every metadata commit from shard 0, so this only covers -# dropped wake-ups and the initial post-bootstrap convergence window. -reconcile_periodic_interval = "1 s" - -# WebSocket listener configuration. The frame-tuning knobs below are the -# live source for the server's WS / WSS plane; they are folded into a -# compio-ws WebSocketConfig once at bus construction. Each size knob is -# optional: commenting it out keeps the compio-ws (tungstenite) default -# noted next to it. A malformed size string fails config load. [websocket] enabled = true address = "127.0.0.1:8092" -# Target minimum size of the frame read buffer. compio-ws default: "128 KiB". -# read_buffer_size = "128 KiB" - -# Target buffer size for batched writes before flush. compio-ws -# default: "128 KiB". -# write_buffer_size = "128 KiB" - -# Hard ceiling on the write buffer; writes past it error instead of -# buffering, so it must exceed write_buffer_size by at least one message. -# compio-ws default: unlimited. -# max_write_buffer_size = "128 MiB" - -# Hard upper bound on a single inbound WebSocket message -# (post-fragment-reassembly). Must not exceed message_bus.max_message_size. -# compio-ws default: "64 MiB". -# max_message_size = "64 MiB" - -# Hard upper bound on a single inbound WebSocket frame -# (pre-fragment-reassembly). Must not exceed max_message_size. -# compio-ws default: "16 MiB". -# max_frame_size = "16 MiB" - -# Whether to accept unmasked frames from clients in violation of -# RFC 6455 client-to-server framing rules. Strict (false) by default. -accept_unmasked_frames = false - [websocket.tls] enabled = false self_signed = true cert_file = "core/certs/iggy_cert.pem" key_file = "core/certs/iggy_key.pem" - -# Metadata consensus plane tunables (shard 0's VSR replica: users, -# streams, topics, sessions). Size these together: a deeper prepare queue -# admits more concurrent in-flight metadata ops (e.g. login storms), and -# the journal must hold enough slots that a forced checkpoint (triggered -# when remaining slots fall to the checkpoint margin, which itself is -# max(64, prepare_queue_depth)) stays rare. Validation enforces -# journal_slots >= 4 * max(64, prepare_queue_depth). -[metadata] -# Depth of the metadata prepare queue: how many uncommitted metadata ops -# may be in flight at once. Submits beyond it are rejected with the -# transient "metadata prepare queue is full" and retried by the SDK. -# Capped at 127 by the view-change wire format: a DoViewChange describes the -# uncommitted suffix with one nack bit and one present bit per entry in a u128 each, -# so a deeper queue produces entries a view change can neither adopt nor prove dead. -prepare_queue_depth = 32 - -# Size of the metadata WAL's in-memory index, in slots (one committed but -# not-yet-snapshotted op per slot). Larger values buy more headroom -# between forced checkpoints at the cost of memory and bigger WAL -# rewrites per checkpoint. -journal_slots = 1024 - -# Slot count of the VSR client table: how many distinct clients (TCP/QUIC/WS -# virtual clients and HTTP sessions together) hold live session state at once. -# When full, the client whose last commit is oldest is evicted and its next -# request re-registers. The HTTP session cap tracks this at half, so raising -# it lifts both. Must be between 2 and 65536. -clients_table_max = 8192 - -# Per-partition consensus plane tunables. Unlike [metadata] (one shard-0 -# plane), a pipeline exists per partition, so raising this multiplies pinned -# request-buffer memory by the partition count. Keep it modest. -[partition] -# Depth of a partition's prepare queue: how many uncommitted produce / -# consumer-offset ops may be in flight at once for that partition. Submits past -# it spill into a request queue of twice this depth; once both are full the -# server drops the request without a reply and the client retries on its own -# request timeout. Must be > 0 and <= 127: the ceiling is the view-change wire, not -# memory. A DoViewChange describes the uncommitted suffix with one bit per op in a -# u128 bitset, and this depth bounds that suffix. -prepare_queue_depth = 32 - -# Entries the evicted ring retains per multi-replica partition for journal -# repair after a peer rejoins. Larger widens the window a restarting peer can be -# served from the ring before falling back to bulk sync, at the cost of pinned -# memory per partition. Must be > 0 and <= 65536. Single-replica partitions -# retain nothing regardless. -evicted_ring_capacity = 4096 - -# Byte ceiling for the evicted ring per partition; whichever ring cap (this or -# evicted_ring_capacity) trips first evicts. Bounds the ring memory a burst of -# large batches can pin. Must be > 0 and <= "256 MiB". -evicted_ring_bytes_max = "16 MiB" - -# Byte budget for segment payloads a SERVING shard keeps resident to answer -# state-transfer chunk requests. PER SHARD, and shard count defaults to core -# count, so the process-wide high-water is this times the core count on top of -# page cache -- keep that product in mind before raising it. The default is a -# FIXED 2176 MiB: two sealed segments at the SHIPPED system.segment.size of -# 1 GiB, each of which can close one whole message_bus.max_message_size past -# its target, which is why it is not 2 GiB. It does not track your segment -# size. How many groups this shard serves at once IS derived from yours: -# floor(this / max(partition.transfer_artifact_bytes_max, -# system.segment.size + 64 MiB)), minimum one. So raising either that knob or -# system.segment.size without raising this lowers concurrency and can take it -# to one, serialising rejoins, and nothing at boot warns about it. -# Below one segment a single rejoining node thrashes the cache by itself and -# every miss re-reads and re-hashes a whole segment to serve one 256 KiB chunk. -# Running under the budget costs re-reads, not failures. -# Must be > 0 and <= "64 GiB". -transfer_served_cache_bytes_max = "2176 MiB" - -# Alloc ceiling for ONE received state-transfer artifact, per shard. The -# receiver holds it resident through verify, walk and staging write, and up to -# four transfers run at once. MUST cover system.segment.size plus -# message_bus.max_message_size (a segment may close one whole batch past its -# cap): under that, a legal segment is refused, the whole manifest with it, and -# the partition livelocks re-requesting it from every peer. Boot validates the -# floor. Raising this above the floor for headroom also DIVIDES the serving -# concurrency derived from transfer_served_cache_bytes_max above, so raise that -# in step. Must be > 0 and <= "64 GiB". -transfer_artifact_bytes_max = "1088 MiB" - -# Message bus configuration. -# Tunables for the inter-shard / inter-replica internal bus that ships -# consensus traffic between replicas and SDK-client traffic between -# shards. These knobs are consensus-liveness-critical (max_batch gates -# throughput under backpressure). Defaults match -# core::message_bus::config::MessageBusConfig::default(). - -[message_bus] -# Maximum number of BusMessage entries coalesced into a single writev(2) -# call. Hard upper bound: IOV_MAX/2 = 512 on Linux. -max_batch = 256 - -# Wire-level cap on a single framed message. -max_message_size = "64 MiB" - -# Bound on the per-peer mpsc queue. The writer task drains; the -# send_to_* path enqueues. -peer_queue_capacity = 256 - -# Interval between outbound reconnect attempts to peers with peer_id > self_id. -reconnect_period = "5 s" - -# Timeout for per-peer close drain (flush writer, tear down reader) -# before force-cancellation. -close_peer_timeout = "2 s" - -# Wall-clock bound on a single stream.shutdown() / ws.close() in the -# safe-shutdown sequence of the TLS-family transports. -close_grace = "2 s" - -# Wall-clock bound on a single connection's handshake phase. Threaded -# into compio::time::timeout(handshake_grace, ...) at each accept site -# (TCP-TLS rustls accept, WS HTTP-Upgrade, WSS combined TLS+WS, QUIC -# connecting.await + accept_bi.await) so a slowloris peer cannot pin -# per-conn channels + registry slot + spawned task indefinitely. -handshake_grace = "10 s" - -[extra.namespace] -max_streams = 4096 -max_topics = 4096 -max_partitions = 1_000_000 diff --git a/core/server/server.http b/core/server/server.http index b0be45ca82..451bca8dd0 100644 --- a/core/server/server.http +++ b/core/server/server.http @@ -34,11 +34,14 @@ @user1_username = user1 @user1_password = secret @access_token = secret -@root_id = 0 -@user1_id = 1 +@root_id = 1 +@user1_id = 2 @pat_name = dev_token @pat_raw_token = secret +### +GET {{url}} + ### GET {{url}}/ping @@ -60,18 +63,11 @@ Content-Type: application/json } ### -GET {{url}}/stats -Authorization: Bearer {{access_token}} +GET {{url}}/metrics ### -POST {{url}}/snapshot +GET {{url}}/stats Authorization: Bearer {{access_token}} -Content-Type: application/json - -{ - "compression": "Deflated", - "snapshot_types": ["All"] -} ### GET {{url}}/cluster/metadata @@ -85,6 +81,14 @@ Authorization: Bearer {{access_token}} GET {{url}}/clients/{{client_id}} Authorization: Bearer {{access_token}} +### +POST {{url}}/users/refresh-token +Content-Type: application/json + +{ + "token": "{{access_token}}" +} + ### DELETE {{url}}/users/logout Authorization: Bearer {{access_token}} @@ -150,7 +154,7 @@ Content-Type: application/json "send_messages": true }, "streams": { - "0": { + "1": { "manage_stream": false, "read_stream": true, "manage_topics": false, @@ -158,7 +162,7 @@ Content-Type: application/json "poll_messages": true, "send_messages": true, "topics": { - "0": { + "1": { "manage_topic": false, "read_topic": true, "poll_messages": true, @@ -189,6 +193,9 @@ Content-Type: application/json "expiry": 1000 } +### + + ### DELETE {{url}}/personal-access-tokens/{{pat_name}} Authorization: Bearer {{access_token}} @@ -283,7 +290,7 @@ Authorization: Bearer {{access_token}} ### ### Delete segments -DELETE {{url}}/streams/{{stream_id}}/topics/{{topic_id}}/partitions/{{partition_id}}?segments_count=3 +DELETE {{url}}/streams/1/topics/1/partitions/1?segments_count=3 Authorization: Bearer {{access_token}} ### diff --git a/core/server/src/args.rs b/core/server/src/args.rs index 665a11c85c..ad63aed19c 100644 --- a/core/server/src/args.rs +++ b/core/server/src/args.rs @@ -22,11 +22,10 @@ use clap::Parser; author = "Apache Iggy (Incubating)", version, about = "Apache Iggy: Hyper-Efficient Message Streaming at Laser Speed", - long_about = r#"Apache Iggy (Incubating) - a persistent message streaming platform written in Rust + long_about = r#"Apache Iggy (Incubating) - A persistent message streaming platform written in Rust -Iggy stores every stream in a replicated log kept consistent by Viewstamped -Replication. One binary serves both the single-node and the clustered -deployment; the loaded configuration decides which one you get. +Apache Iggy is a high-performance message streaming platform that supports QUIC, TCP, and HTTP +transport protocols, capable of processing millions of messages per second with low latency. WEBSITE: https://iggy.apache.org @@ -38,89 +37,102 @@ DOCUMENTATION: https://iggy.apache.org/docs CONFIGURATION: - The server reads a TOML configuration file, by default 'core/server/config.toml' - resolved against the current working directory. Point IGGY_CONFIG_PATH at - another file to override it. + The server uses a TOML configuration file. By default, it looks for 'core/server/config.toml' + in the current working directory. You can override this with the IGGY_CONFIG_PATH environment + variable or use the --config-provider flag. Examples: - iggy-server # Default config file - IGGY_CONFIG_PATH=custom.toml iggy-server # Custom config file path + iggy-server # Uses default file provider (core/server/config.toml) + iggy-server --config-provider file # Explicitly use file provider + IGGY_CONFIG_PATH=custom.toml iggy-server # Use custom config file path ENVIRONMENT VARIABLES: - Any configuration value can be overridden with an IGGY_ prefixed variable; - underscores separate the nested keys (IGGY_TCP_ADDRESS sets [tcp] address). - A '.env' file in the working directory is loaded during startup, or the one - named by IGGY_ENV_PATH. + Any configuration value can be overridden using environment variables with the IGGY_ prefix. + Use underscores to separate nested configuration keys (e.g., IGGY_TCP_ADDRESS=127.0.0.1:8090). Common examples: - IGGY_SYSTEM_PATH=/data/iggy # Data directory - IGGY_TCP_ADDRESS=0.0.0.0:8090 # TCP listener address - IGGY_HTTP_ADDRESS=0.0.0.0:3000 # HTTP listener address - IGGY_SYSTEM_LOGGING_LEVEL=debug # Log level - IGGY_ROOT_USERNAME=iggy # Root user, set with the password - IGGY_ROOT_PASSWORD=secret # Root password, set with the username + IGGY_TCP_ADDRESS=0.0.0.0:8090 # Override TCP server address + IGGY_HTTP_ENABLED=true # Enable HTTP transport + IGGY_SYSTEM_PATH=/data/iggy # Set data storage path + IGGY_SYSTEM_LOGGING_LEVEL=debug # Set log level to debug TRANSPORT PROTOCOLS: - - TCP (binary protocol) (default: 127.0.0.1:8090) - - QUIC (default: 127.0.0.1:8080) - - WebSocket (default: 127.0.0.1:8092) - - HTTP (REST API) (default: 127.0.0.1:3000) + - TCP (binary protocol): High-performance, low-latency (default: 127.0.0.1:8090) + - QUIC: Modern UDP-based protocol with built-in encryption (default: 127.0.0.1:8080) + - HTTP: RESTful API for web integration (default: 127.0.0.1:3000, disabled by default) GETTING STARTED: - 1. Start the server: iggy-server --fresh --with-default-root-credentials - 2. Install the CLI: cargo install iggy-cli - 3. Create a stream: iggy stream create my-stream - 4. Create a topic: iggy topic create my-stream my-topic 1 none - 5. Send messages: echo "Hello, Iggy!" | iggy message send my-stream my-topic - -CLUSTER: - Every node runs the same configuration file with cluster.enabled = true and - is told apart only by --replica-id, which selects its own cluster.nodes entry: - - iggy-server --replica-id 0 + 1. Start the server: iggy-server + 2. Install CLI: cargo install iggy-cli + 3. Create a stream: iggy stream create my-stream + 4. Create a topic: iggy topic create my-stream my-topic 1 none + 5. Send messages: echo "Hello, Iggy!" | iggy message send my-stream my-topic For more information, visit: https://iggy.apache.org/docs/introduction/getting-started/"# )] -// These doc comments are rendered verbatim as `--help` output, so environment -// variable names and paths must stay unquoted rather than wear rustdoc backticks. -#[allow(clippy::doc_markdown)] pub struct Args { - /// Remove the system path before starting (WARNING: THIS WILL DELETE ALL DATA!) + /// Configuration provider type /// - /// Deletes the configured system data directory ('local_data' by default, - /// see IGGY_SYSTEM_PATH) before the server boots, so it starts on empty - /// state. Intended for clean development setups and testing. + /// Currently only 'file' provider is supported, which loads configuration from a TOML file. + /// The file path can be specified via IGGY_CONFIG_PATH environment variable. + #[arg(short, long, default_value = "file", verbatim_doc_comment)] + pub config_provider: String, + + /// Remove system path before starting (WARNING: THIS WILL DELETE ALL DATA!) /// - /// In cluster mode this wipes THIS replica only; it rejoins and refills by - /// state transfer from the others. Wiping a quorum at the same time destroys - /// committed data, and a service unit file carrying --fresh re-transfers the - /// whole dataset on every restart. + /// This flag will completely remove the system data directory (local_data by default) + /// before starting the server. Use this for clean development setups or testing. /// /// Examples: - /// iggy-server --fresh # Start with a fresh data directory - /// iggy-server -f # Short form + /// iggy-server --fresh # Start with fresh data directory + /// iggy-server -f # Short form #[arg(short, long, default_value_t = false, verbatim_doc_comment)] pub fresh: bool, /// Use default root credentials (INSECURE - FOR DEVELOPMENT ONLY!) /// - /// Sets IGGY_ROOT_USERNAME and IGGY_ROOT_PASSWORD to 'iggy' unless they are - /// already present in the environment, so the flag is equivalent to - /// exporting both by hand and the environment always takes precedence. + /// When this flag is set, the root user will be created with username 'iggy' + /// and password 'iggy' if it doesn't exist. If the root user already exists, + /// this flag has no effect. + /// + /// This flag is equivalent to setting IGGY_ROOT_USERNAME=iggy and IGGY_ROOT_PASSWORD=iggy, + /// but environment variables take precedence over this flag. /// - /// Only the first creation of the root user reads these values. On an - /// existing data directory the stored root user is recovered as it is and - /// the flag has no effect. + /// WARNING: This is insecure and should only be used for development and testing! /// /// Examples: - /// iggy-server --with-default-root-credentials # Root logs in as iggy/iggy + /// iggy-server --with-default-root-credentials # Use 'iggy/iggy' as root credentials #[arg(long, default_value_t = false, verbatim_doc_comment)] pub with_default_root_credentials: bool, + /// Run server as a follower node (FOR TESTING LEADER REDIRECTION) + /// + /// When this flag is set, the server will report itself as a follower node + /// in cluster metadata responses. This is useful for testing leader-aware + /// client connections and redirection logic. + /// + /// The server will return cluster metadata showing this server as a follower node. + /// + /// Examples: + /// iggy-server # Run as leader (default) + /// iggy-server --follower # Run as follower + /// IGGY_TCP_ADDRESS=127.0.0.1:8091 iggy-server --follower # Follower on port 8091 + #[arg(long, default_value_t = false, verbatim_doc_comment)] + pub follower: bool, + /// Identifies this node within `cluster.nodes` by its replica ID. /// /// Required when `cluster.enabled = true`. The value must match exactly - /// one `cluster.nodes[*].replica_id` entry in the loaded configuration. + /// one `cluster.nodes[*].replica_id` entry in the loaded configuration; + /// that entry describes the current node, and all other entries are + /// treated as remote peers. + /// + /// Supplying the identity on the command line lets operators ship a + /// single byte-identical `config.toml` to every node in the cluster + /// and differ only in this CLI flag. + /// + /// Examples: + /// iggy-server --replica-id 0 # This node is replica 0 #[arg(long, verbatim_doc_comment)] pub replica_id: Option, } diff --git a/core/server/src/binary/dispatch.rs b/core/server/src/binary/dispatch.rs new file mode 100644 index 0000000000..2225a1a48f --- /dev/null +++ b/core/server/src/binary/dispatch.rs @@ -0,0 +1,456 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::handlers; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::streaming::session::Session; +use bytes::BytesMut; +use iggy_binary_protocol::RequestFrame; +use iggy_binary_protocol::codec::WireDecode; +use iggy_binary_protocol::codes::*; +use iggy_binary_protocol::requests::consumer_groups::*; +use iggy_binary_protocol::requests::consumer_offsets::*; +use iggy_binary_protocol::requests::messages::*; +use iggy_binary_protocol::requests::partitions::*; +use iggy_binary_protocol::requests::personal_access_tokens::*; +use iggy_binary_protocol::requests::segments::*; +use iggy_binary_protocol::requests::streams::*; +use iggy_binary_protocol::requests::system::*; +use iggy_binary_protocol::requests::topics::*; +use iggy_binary_protocol::requests::users::*; +use iggy_common::{Consumer, ConsumerKind, Identifier, IggyError, PollingKind, PollingStrategy}; +use std::rc::Rc; +use tracing::{error, warn}; + +/// Result of handling a command. Most commands return `Finished`. +/// `SendMessages` may migrate the TCP connection to another shard. +pub enum HandlerResult { + Finished, + Migrated { to_shard: u16 }, +} + +/// Read the full payload from the sender into a buffer. +pub async fn read_payload(sender: &mut SenderKind, length: u32) -> Result { + if length > MAX_CONTROL_FRAME_PAYLOAD { + return Err(IggyError::InvalidCommand); + } + let mut buffer = BytesMut::with_capacity(length as usize); + // SAFETY: when length > 0, sender.read() fills exactly `length` bytes + // before returning Ok. On error the buffer is dropped without being read. + // When length == 0, set_len(0) is a no-op (no uninitialized bytes exposed). + unsafe { + buffer.set_len(length as usize); + } + if length > 0 { + let (result, buf) = sender.read(buffer).await; + result?; + buffer = buf; + } + Ok(buffer) +} + +fn decode(payload: &[u8]) -> Result { + use iggy_binary_protocol::error::WireError; + + let (val, consumed) = T::decode(payload).map_err(|e| { + warn!("wire decode error: {e}"); + match e { + WireError::PayloadTooLarge { .. } => IggyError::InvalidSizeBytes, + WireError::Validation(_) => IggyError::InvalidFormat, + _ => IggyError::InvalidCommand, + } + })?; + if consumed != payload.len() { + warn!( + "wire decode: {} trailing bytes (consumed {consumed}, payload {})", + payload.len() - consumed, + payload.len() + ); + } + Ok(val) +} + +/// Convert a `WireIdentifier` to the domain `Identifier`. +pub fn wire_id_to_identifier( + wire: &iggy_binary_protocol::WireIdentifier, +) -> Result { + match wire { + iggy_binary_protocol::WireIdentifier::Numeric(id) => Identifier::numeric(*id), + iggy_binary_protocol::WireIdentifier::String(name) => Identifier::named(name.as_str()), + } +} + +/// Convert a `WireConsumer` to the domain `Consumer`. +pub fn wire_consumer_to_consumer( + wire: &iggy_binary_protocol::WireConsumer, +) -> Result { + let id = wire_id_to_identifier(&wire.id)?; + let kind = ConsumerKind::from_code(wire.kind)?; + Ok(Consumer { kind, id }) +} + +/// Convert a `WirePollingStrategy` to the domain `PollingStrategy`. +pub fn wire_polling_to_strategy( + wire: &iggy_binary_protocol::WirePollingStrategy, +) -> Result { + Ok(PollingStrategy { + kind: PollingKind::from_code(wire.kind)?, + value: wire.value, + }) +} + +/// Maximum payload size for control-plane commands (non-SendMessages). +/// Prevents OOM from malicious clients sending `length = u32::MAX`. +/// SendMessages has its own size validation via `total_payload_size` checks. +pub const MAX_CONTROL_FRAME_PAYLOAD: u32 = 10 * 1024 * 1024; // 10 MB + +/// Dispatch a SendMessages command with staged socket reads (zero-copy path). +/// +/// Called by transport layers when the command code is `SEND_MESSAGES_CODE`. +/// The handler reads metadata, indexes, and messages directly from the socket +/// into separate `PooledBuffer`s for zero-copy partition append. +pub async fn dispatch_send_messages( + sender: &mut SenderKind, + payload_length: u32, + session: &Session, + shard: &Rc, +) -> Result { + handlers::messages::send_messages_handler::handle_send_messages( + sender, + payload_length, + session, + shard, + ) + .await +} + +/// Central command dispatch for a decoded request frame. +/// +/// Transport layers read the 8-byte header, validate via +/// `RequestFrame::payload_length()`, read the full payload, construct a +/// `RequestFrame::from_parts(code, frame.payload)`, and pass it here. +/// +/// SendMessages is handled separately via `dispatch_send_messages()`. +#[allow(clippy::too_many_lines)] +pub async fn dispatch( + frame: RequestFrame<'_>, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + match frame.code { + // System + PING_CODE => { + handlers::system::ping_handler::handle_ping(sender, session, shard).await + } + GET_STATS_CODE => { + handlers::system::get_stats_handler::handle_get_stats(sender, session, shard).await + } + GET_ME_CODE => { + handlers::system::get_me_handler::handle_get_me(sender, session, shard).await + } + GET_CLIENT_CODE => { + let req: GetClientRequest = decode(frame.payload)?; + handlers::system::get_client_handler::handle_get_client(req, sender, session, shard) + .await + } + GET_CLIENTS_CODE => { + handlers::system::get_clients_handler::handle_get_clients(sender, session, shard).await + } + GET_SNAPSHOT_FILE_CODE => { + let req: GetSnapshotRequest = decode(frame.payload)?; + handlers::system::get_snapshot_handler::handle_get_snapshot( + req, sender, session, shard, + ) + .await + } + GET_CLUSTER_METADATA_CODE => { + handlers::cluster::get_cluster_metadata_handler::handle_get_cluster_metadata( + sender, session, shard, + ) + .await + } + + // Streams + GET_STREAM_CODE => { + let req: GetStreamRequest = decode(frame.payload)?; + handlers::streams::get_stream_handler::handle_get_stream(req, sender, session, shard) + .await + } + GET_STREAMS_CODE => { + handlers::streams::get_streams_handler::handle_get_streams(sender, session, shard) + .await + } + CREATE_STREAM_CODE => { + let req: CreateStreamRequest = decode(frame.payload)?; + handlers::streams::create_stream_handler::handle_create_stream( + req, sender, session, shard, + ) + .await + } + DELETE_STREAM_CODE => { + let req: DeleteStreamRequest = decode(frame.payload)?; + handlers::streams::delete_stream_handler::handle_delete_stream( + req, sender, session, shard, + ) + .await + } + UPDATE_STREAM_CODE => { + let req: UpdateStreamRequest = decode(frame.payload)?; + handlers::streams::update_stream_handler::handle_update_stream( + req, sender, session, shard, + ) + .await + } + PURGE_STREAM_CODE => { + let req: PurgeStreamRequest = decode(frame.payload)?; + handlers::streams::purge_stream_handler::handle_purge_stream( + req, sender, session, shard, + ) + .await + } + + // Topics + GET_TOPIC_CODE => { + let req: GetTopicRequest = decode(frame.payload)?; + handlers::topics::get_topic_handler::handle_get_topic(req, sender, session, shard) + .await + } + GET_TOPICS_CODE => { + let req: GetTopicsRequest = decode(frame.payload)?; + handlers::topics::get_topics_handler::handle_get_topics(req, sender, session, shard) + .await + } + CREATE_TOPIC_CODE => { + let req: CreateTopicRequest = decode(frame.payload)?; + handlers::topics::create_topic_handler::handle_create_topic( + req, sender, session, shard, + ) + .await + } + DELETE_TOPIC_CODE => { + let req: DeleteTopicRequest = decode(frame.payload)?; + handlers::topics::delete_topic_handler::handle_delete_topic( + req, sender, session, shard, + ) + .await + } + UPDATE_TOPIC_CODE => { + let req: UpdateTopicRequest = decode(frame.payload)?; + handlers::topics::update_topic_handler::handle_update_topic( + req, sender, session, shard, + ) + .await + } + PURGE_TOPIC_CODE => { + let req: PurgeTopicRequest = decode(frame.payload)?; + handlers::topics::purge_topic_handler::handle_purge_topic( + req, sender, session, shard, + ) + .await + } + + // Partitions + CREATE_PARTITIONS_CODE => { + let req: CreatePartitionsRequest = decode(frame.payload)?; + handlers::partitions::create_partitions_handler::handle_create_partitions( + req, sender, session, shard, + ) + .await + } + DELETE_PARTITIONS_CODE => { + let req: DeletePartitionsRequest = decode(frame.payload)?; + handlers::partitions::delete_partitions_handler::handle_delete_partitions( + req, sender, session, shard, + ) + .await + } + + // Segments + DELETE_SEGMENTS_CODE => { + let req: DeleteSegmentsRequest = decode(frame.payload)?; + handlers::segments::delete_segments_handler::handle_delete_segments( + req, sender, session, shard, + ) + .await + } + + // Messages (PollMessages + FlushUnsavedBuffer; SendMessages handled above) + POLL_MESSAGES_CODE => { + let req: PollMessagesRequest = decode(frame.payload)?; + handlers::messages::poll_messages_handler::handle_poll_messages( + req, sender, session, shard, + ) + .await + } + FLUSH_UNSAVED_BUFFER_CODE => { + let req: FlushUnsavedBufferRequest = decode(frame.payload)?; + handlers::messages::flush_unsaved_buffer_handler::handle_flush_unsaved_buffer( + req, sender, session, shard, + ) + .await + } + + // Consumer Offsets + GET_CONSUMER_OFFSET_CODE => { + let req: GetConsumerOffsetRequest = decode(frame.payload)?; + handlers::consumer_offsets::get_consumer_offset_handler::handle_get_consumer_offset( + req, sender, session, shard, + ) + .await + } + STORE_CONSUMER_OFFSET_CODE => { + let req: StoreConsumerOffsetRequest = decode(frame.payload)?; + handlers::consumer_offsets::store_consumer_offset_handler::handle_store_consumer_offset( + req, sender, session, shard, + ) + .await + } + DELETE_CONSUMER_OFFSET_CODE => { + let req: DeleteConsumerOffsetRequest = decode(frame.payload)?; + handlers::consumer_offsets::delete_consumer_offset_handler::handle_delete_consumer_offset( + req, sender, session, shard, + ) + .await + } + + // Consumer Groups + GET_CONSUMER_GROUP_CODE => { + let req: GetConsumerGroupRequest = decode(frame.payload)?; + handlers::consumer_groups::get_consumer_group_handler::handle_get_consumer_group( + req, sender, session, shard, + ) + .await + } + GET_CONSUMER_GROUPS_CODE => { + let req: GetConsumerGroupsRequest = decode(frame.payload)?; + handlers::consumer_groups::get_consumer_groups_handler::handle_get_consumer_groups( + req, sender, session, shard, + ) + .await + } + CREATE_CONSUMER_GROUP_CODE => { + let req: CreateConsumerGroupRequest = decode(frame.payload)?; + handlers::consumer_groups::create_consumer_group_handler::handle_create_consumer_group( + req, sender, session, shard, + ) + .await + } + DELETE_CONSUMER_GROUP_CODE => { + let req: DeleteConsumerGroupRequest = decode(frame.payload)?; + handlers::consumer_groups::delete_consumer_group_handler::handle_delete_consumer_group( + req, sender, session, shard, + ) + .await + } + JOIN_CONSUMER_GROUP_CODE => { + let req: JoinConsumerGroupRequest = decode(frame.payload)?; + handlers::consumer_groups::join_consumer_group_handler::handle_join_consumer_group( + req, sender, session, shard, + ) + .await + } + LEAVE_CONSUMER_GROUP_CODE => { + let req: LeaveConsumerGroupRequest = decode(frame.payload)?; + handlers::consumer_groups::leave_consumer_group_handler::handle_leave_consumer_group( + req, sender, session, shard, + ) + .await + } + + // Users + GET_USER_CODE => { + let req: GetUserRequest = decode(frame.payload)?; + handlers::users::get_user_handler::handle_get_user(req, sender, session, shard).await + } + GET_USERS_CODE => { + handlers::users::get_users_handler::handle_get_users(sender, session, shard).await + } + CREATE_USER_CODE => { + let req: CreateUserRequest = decode(frame.payload)?; + handlers::users::create_user_handler::handle_create_user(req, sender, session, shard) + .await + } + DELETE_USER_CODE => { + let req: DeleteUserRequest = decode(frame.payload)?; + handlers::users::delete_user_handler::handle_delete_user(req, sender, session, shard) + .await + } + UPDATE_USER_CODE => { + let req: UpdateUserRequest = decode(frame.payload)?; + handlers::users::update_user_handler::handle_update_user(req, sender, session, shard) + .await + } + UPDATE_PERMISSIONS_CODE => { + let req: UpdatePermissionsRequest = decode(frame.payload)?; + handlers::users::update_permissions_handler::handle_update_permissions( + req, sender, session, shard, + ) + .await + } + CHANGE_PASSWORD_CODE => { + let req: ChangePasswordRequest = decode(frame.payload)?; + handlers::users::change_password_handler::handle_change_password( + req, sender, session, shard, + ) + .await + } + LOGIN_USER_CODE => { + let req: LoginUserRequest = decode(frame.payload)?; + handlers::users::login_user_handler::handle_login_user(req, sender, session, shard) + .await + } + LOGOUT_USER_CODE => { + handlers::users::logout_user_handler::handle_logout_user(sender, session, shard).await + } + + // Personal Access Tokens + GET_PERSONAL_ACCESS_TOKENS_CODE => { + handlers::personal_access_tokens::get_personal_access_tokens_handler::handle_get_personal_access_tokens( + sender, session, shard, + ) + .await + } + CREATE_PERSONAL_ACCESS_TOKEN_CODE => { + let req: CreatePersonalAccessTokenRequest = decode(frame.payload)?; + handlers::personal_access_tokens::create_personal_access_token_handler::handle_create_personal_access_token( + req, sender, session, shard, + ) + .await + } + DELETE_PERSONAL_ACCESS_TOKEN_CODE => { + let req: DeletePersonalAccessTokenRequest = decode(frame.payload)?; + handlers::personal_access_tokens::delete_personal_access_token_handler::handle_delete_personal_access_token( + req, sender, session, shard, + ) + .await + } + LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE => { + let req: LoginWithPersonalAccessTokenRequest = decode(frame.payload)?; + handlers::personal_access_tokens::login_with_personal_access_token_handler::handle_login_with_personal_access_token( + req, sender, session, shard, + ) + .await + } + + _ => { + error!("Unknown command code: {}", frame.code); + Err(IggyError::InvalidCommand) + } + } +} diff --git a/core/server/src/binary/handlers/cluster/get_cluster_metadata_handler.rs b/core/server/src/binary/handlers/cluster/get_cluster_metadata_handler.rs new file mode 100644 index 0000000000..7e32ae8247 --- /dev/null +++ b/core/server/src/binary/handlers/cluster/get_cluster_metadata_handler.rs @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::HandlerResult; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::streaming::session::Session; +use iggy_binary_protocol::codec::WireEncode; +use iggy_binary_protocol::responses::system::get_cluster_metadata::{ + ClusterMetadataResponse, ClusterNodeResponse, +}; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::{debug, instrument}; + +#[instrument(skip_all, name = "trace_get_cluster_metadata", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] +pub async fn handle_get_cluster_metadata( + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + debug!("session: {session}, command: get_cluster_metadata"); + shard.ensure_authenticated(session)?; + + let cluster_metadata = shard.get_cluster_metadata(); + + let response = ClusterMetadataResponse { + name: cluster_metadata.name, + nodes: cluster_metadata + .nodes + .into_iter() + .map(|node| ClusterNodeResponse { + name: node.name, + ip: node.ip, + tcp_port: node.endpoints.tcp, + quic_port: node.endpoints.quic, + http_port: node.endpoints.http, + websocket_port: node.endpoints.websocket, + role: node.role as u8, + status: node.status as u8, + }) + .collect(), + }; + sender.send_ok_response(&response.to_bytes()).await?; + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/cluster/mod.rs b/core/server/src/binary/handlers/cluster/mod.rs new file mode 100644 index 0000000000..46214f179c --- /dev/null +++ b/core/server/src/binary/handlers/cluster/mod.rs @@ -0,0 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod get_cluster_metadata_handler; diff --git a/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs new file mode 100644 index 0000000000..5fce66baaf --- /dev/null +++ b/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::HandlerResult; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; +use crate::streaming::session::Session; +use iggy_binary_protocol::WireName; +use iggy_binary_protocol::codec::WireEncode; +use iggy_binary_protocol::requests::consumer_groups::CreateConsumerGroupRequest; +use iggy_binary_protocol::responses::consumer_groups::ConsumerGroupResponse; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::{debug, instrument}; + +#[instrument(skip_all, name = "trace_create_consumer_group", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] +pub async fn handle_create_consumer_group( + req: CreateConsumerGroupRequest, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + debug!( + "session: {session}, command: create_consumer_group, stream_id: {:?}, topic_id: {:?}, name: {}", + req.stream_id, + req.topic_id, + req.name.as_str() + ); + shard.ensure_authenticated(session)?; + + let request = ShardRequest::control_plane(ShardRequestPayload::CreateConsumerGroupRequest { + user_id: session.get_user_id(), + command: req, + }); + + match shard.send_to_control_plane(request).await? { + ShardResponse::CreateConsumerGroupResponse(data) => { + let response = ConsumerGroupResponse { + id: data.id, + partitions_count: data.partitions_count, + members_count: 0, + name: WireName::new(data.name.as_ref()).map_err(|_| IggyError::InvalidCommand)?, + }; + sender.send_ok_response(&response.to_bytes()).await?; + } + ShardResponse::ErrorResponse(err) => return Err(err), + _ => unreachable!("Expected CreateConsumerGroupResponse"), + } + + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/consumer_groups/delete_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/delete_consumer_group_handler.rs new file mode 100644 index 0000000000..059fd87301 --- /dev/null +++ b/core/server/src/binary/handlers/consumer_groups/delete_consumer_group_handler.rs @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::HandlerResult; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; +use crate::streaming::session::Session; +use iggy_binary_protocol::requests::consumer_groups::DeleteConsumerGroupRequest; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::{debug, instrument}; + +#[instrument(skip_all, name = "trace_delete_consumer_group", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] +pub async fn handle_delete_consumer_group( + req: DeleteConsumerGroupRequest, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + debug!( + "session: {session}, command: delete_consumer_group, stream_id: {:?}, topic_id: {:?}, group_id: {:?}", + req.stream_id, req.topic_id, req.group_id + ); + shard.ensure_authenticated(session)?; + + let request = ShardRequest::control_plane(ShardRequestPayload::DeleteConsumerGroupRequest { + user_id: session.get_user_id(), + command: req, + }); + + match shard.send_to_control_plane(request).await? { + ShardResponse::DeleteConsumerGroupResponse => { + sender.send_empty_ok_response().await?; + } + ShardResponse::ErrorResponse(err) => return Err(err), + _ => unreachable!("Expected DeleteConsumerGroupResponse"), + } + + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/consumer_groups/get_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/get_consumer_group_handler.rs new file mode 100644 index 0000000000..00ff8f58e3 --- /dev/null +++ b/core/server/src/binary/handlers/consumer_groups/get_consumer_group_handler.rs @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::{HandlerResult, wire_id_to_identifier}; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::streaming::session::Session; +use iggy_binary_protocol::WireName; +use iggy_binary_protocol::codec::WireEncode; +use iggy_binary_protocol::requests::consumer_groups::GetConsumerGroupRequest; +use iggy_binary_protocol::responses::consumer_groups::{ + ConsumerGroupDetailsResponse, ConsumerGroupMemberResponse, ConsumerGroupResponse, +}; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::debug; + +pub async fn handle_get_consumer_group( + req: GetConsumerGroupRequest, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + let stream_id = wire_id_to_identifier(&req.stream_id)?; + let topic_id = wire_id_to_identifier(&req.topic_id)?; + let group_id = wire_id_to_identifier(&req.group_id)?; + debug!( + "session: {session}, command: get_consumer_group, stream_id: {stream_id}, topic_id: {topic_id}, group_id: {group_id}" + ); + shard.ensure_authenticated(session)?; + + let Some(consumer_group) = shard.metadata.query_consumer_group( + session.get_user_id(), + &stream_id, + &topic_id, + &group_id, + )? + else { + sender.send_empty_ok_response().await?; + return Ok(HandlerResult::Finished); + }; + + let members: Vec = consumer_group + .members + .iter() + .map(|(_, member)| ConsumerGroupMemberResponse { + id: member.id as u32, + partitions_count: member.partitions.len() as u32, + partitions: member.partitions.iter().map(|&p| p as u32).collect(), + }) + .collect(); + let response = ConsumerGroupDetailsResponse { + group: ConsumerGroupResponse { + id: consumer_group.id as u32, + partitions_count: consumer_group.partitions.len() as u32, + members_count: consumer_group.members.len() as u32, + name: WireName::new(consumer_group.name.as_ref()) + .map_err(|_| IggyError::InvalidCommand)?, + }, + members, + }; + sender.send_ok_response(&response.to_bytes()).await?; + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/consumer_groups/get_consumer_groups_handler.rs b/core/server/src/binary/handlers/consumer_groups/get_consumer_groups_handler.rs new file mode 100644 index 0000000000..5fec457f72 --- /dev/null +++ b/core/server/src/binary/handlers/consumer_groups/get_consumer_groups_handler.rs @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::{HandlerResult, wire_id_to_identifier}; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::streaming::session::Session; +use bytes::Bytes; +use iggy_binary_protocol::WireName; +use iggy_binary_protocol::codec::WireEncode; +use iggy_binary_protocol::requests::consumer_groups::GetConsumerGroupsRequest; +use iggy_binary_protocol::responses::consumer_groups::{ + ConsumerGroupResponse, GetConsumerGroupsResponse, +}; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::debug; + +pub async fn handle_get_consumer_groups( + req: GetConsumerGroupsRequest, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + let stream_id = wire_id_to_identifier(&req.stream_id)?; + let topic_id = wire_id_to_identifier(&req.topic_id)?; + debug!( + "session: {session}, command: get_consumer_groups, stream_id: {stream_id}, topic_id: {topic_id}" + ); + shard.ensure_authenticated(session)?; + + let Some(consumer_groups) = + shard + .metadata + .query_consumer_groups(session.get_user_id(), &stream_id, &topic_id)? + else { + sender.send_ok_response(&Bytes::new()).await?; + return Ok(HandlerResult::Finished); + }; + + let groups: Vec = consumer_groups + .iter() + .map(|cg| { + Ok(ConsumerGroupResponse { + id: cg.id as u32, + partitions_count: cg.partitions.len() as u32, + members_count: cg.members.len() as u32, + name: WireName::new(cg.name.as_ref()).map_err(|_| IggyError::InvalidCommand)?, + }) + }) + .collect::>()?; + let response = GetConsumerGroupsResponse { groups }; + sender.send_ok_response(&response.to_bytes()).await?; + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/consumer_groups/join_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/join_consumer_group_handler.rs new file mode 100644 index 0000000000..658952f342 --- /dev/null +++ b/core/server/src/binary/handlers/consumer_groups/join_consumer_group_handler.rs @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::HandlerResult; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; +use crate::streaming::session::Session; +use iggy_binary_protocol::requests::consumer_groups::JoinConsumerGroupRequest; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::{debug, instrument}; + +#[instrument(skip_all, name = "trace_join_consumer_group", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] +pub async fn handle_join_consumer_group( + req: JoinConsumerGroupRequest, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + debug!( + "session: {session}, command: join_consumer_group, stream_id: {:?}, topic_id: {:?}, group_id: {:?}", + req.stream_id, req.topic_id, req.group_id + ); + shard.ensure_authenticated(session)?; + + let request = ShardRequest::control_plane(ShardRequestPayload::JoinConsumerGroupRequest { + user_id: session.get_user_id(), + client_id: session.client_id, + command: req, + }); + + match shard.send_to_control_plane(request).await? { + ShardResponse::JoinConsumerGroupResponse => { + sender.send_empty_ok_response().await?; + } + ShardResponse::ErrorResponse(err) => return Err(err), + _ => unreachable!("Expected JoinConsumerGroupResponse"), + } + + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/consumer_groups/leave_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/leave_consumer_group_handler.rs new file mode 100644 index 0000000000..3caf382646 --- /dev/null +++ b/core/server/src/binary/handlers/consumer_groups/leave_consumer_group_handler.rs @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::HandlerResult; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; +use crate::streaming::session::Session; +use iggy_binary_protocol::requests::consumer_groups::LeaveConsumerGroupRequest; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::{debug, instrument}; + +#[instrument(skip_all, name = "trace_leave_consumer_group", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] +pub async fn handle_leave_consumer_group( + req: LeaveConsumerGroupRequest, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + debug!( + "session: {session}, command: leave_consumer_group, stream_id: {:?}, topic_id: {:?}, group_id: {:?}", + req.stream_id, req.topic_id, req.group_id + ); + shard.ensure_authenticated(session)?; + + let request = ShardRequest::control_plane(ShardRequestPayload::LeaveConsumerGroupRequest { + user_id: session.get_user_id(), + client_id: session.client_id, + command: req, + }); + + match shard.send_to_control_plane(request).await? { + ShardResponse::LeaveConsumerGroupResponse => { + sender.send_empty_ok_response().await?; + } + ShardResponse::ErrorResponse(err) => return Err(err), + _ => unreachable!("Expected LeaveConsumerGroupResponse"), + } + + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/consumer_groups/mod.rs b/core/server/src/binary/handlers/consumer_groups/mod.rs new file mode 100644 index 0000000000..dbefde591e --- /dev/null +++ b/core/server/src/binary/handlers/consumer_groups/mod.rs @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod create_consumer_group_handler; +pub mod delete_consumer_group_handler; +pub mod get_consumer_group_handler; +pub mod get_consumer_groups_handler; +pub mod join_consumer_group_handler; +pub mod leave_consumer_group_handler; + +pub const COMPONENT: &str = "CONSUMER_GROUP_HANDLER"; diff --git a/core/server/src/binary/handlers/consumer_offsets/delete_consumer_offset_handler.rs b/core/server/src/binary/handlers/consumer_offsets/delete_consumer_offset_handler.rs new file mode 100644 index 0000000000..8684567267 --- /dev/null +++ b/core/server/src/binary/handlers/consumer_offsets/delete_consumer_offset_handler.rs @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::{HandlerResult, wire_consumer_to_consumer, wire_id_to_identifier}; +use crate::binary::handlers::consumer_offsets::COMPONENT; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::streaming::session::Session; +use err_trail::ErrContext; +use iggy_binary_protocol::requests::consumer_offsets::DeleteConsumerOffsetRequest; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::debug; + +pub async fn handle_delete_consumer_offset( + req: DeleteConsumerOffsetRequest, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + let consumer = wire_consumer_to_consumer(&req.consumer)?; + let stream_id = wire_id_to_identifier(&req.stream_id)?; + let topic_id = wire_id_to_identifier(&req.topic_id)?; + debug!( + "session: {session}, command: delete_consumer_offset, stream_id: {stream_id}, topic_id: {topic_id}, partition_id: {:?}", + req.partition_id + ); + shard.ensure_authenticated(session)?; + let topic = shard.resolve_topic_for_delete_consumer_offset( + session.get_user_id(), + &stream_id, + &topic_id, + )?; + shard + .delete_consumer_offset(session.client_id, consumer, topic, req.partition_id) + .await + .error(|e: &IggyError| format!("{COMPONENT} (error: {e}) - failed to delete consumer offset for topic with ID: {} in stream with ID: {} partition ID: {:#?}, session: {}", + topic_id, stream_id, req.partition_id, session + ))?; + sender.send_empty_ok_response().await?; + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/consumer_offsets/get_consumer_offset_handler.rs b/core/server/src/binary/handlers/consumer_offsets/get_consumer_offset_handler.rs new file mode 100644 index 0000000000..7a8ff64047 --- /dev/null +++ b/core/server/src/binary/handlers/consumer_offsets/get_consumer_offset_handler.rs @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::{HandlerResult, wire_consumer_to_consumer, wire_id_to_identifier}; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::shard::transmission::message::ResolvedTopic; +use crate::streaming::session::Session; +use iggy_binary_protocol::codec::WireEncode; +use iggy_binary_protocol::requests::consumer_offsets::GetConsumerOffsetRequest; +use iggy_binary_protocol::responses::consumer_offsets::ConsumerOffsetResponse; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::debug; + +pub async fn handle_get_consumer_offset( + req: GetConsumerOffsetRequest, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + let consumer = wire_consumer_to_consumer(&req.consumer)?; + let stream_id = wire_id_to_identifier(&req.stream_id)?; + let topic_id = wire_id_to_identifier(&req.topic_id)?; + debug!( + "session: {session}, command: get_consumer_offset, stream_id: {stream_id}, topic_id: {topic_id}, partition_id: {:?}", + req.partition_id + ); + shard.ensure_authenticated(session)?; + + let Some(resolved) = + shard + .metadata + .resolve_for_consumer_offset(session.get_user_id(), &stream_id, &topic_id)? + else { + sender.send_empty_ok_response().await?; + return Ok(HandlerResult::Finished); + }; + + let topic = ResolvedTopic { + stream_id: resolved.stream_id, + topic_id: resolved.topic_id, + }; + + let Ok(offset) = shard + .get_consumer_offset(session.client_id, consumer, topic, req.partition_id) + .await + else { + sender.send_empty_ok_response().await?; + return Ok(HandlerResult::Finished); + }; + + let Some(offset) = offset else { + sender.send_empty_ok_response().await?; + return Ok(HandlerResult::Finished); + }; + + let response = ConsumerOffsetResponse { + partition_id: offset.partition_id, + current_offset: offset.current_offset, + stored_offset: offset.stored_offset, + }; + sender.send_ok_response(&response.to_bytes()).await?; + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/consumer_offsets/mod.rs b/core/server/src/binary/handlers/consumer_offsets/mod.rs new file mode 100644 index 0000000000..922fd459ab --- /dev/null +++ b/core/server/src/binary/handlers/consumer_offsets/mod.rs @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod delete_consumer_offset_handler; +pub mod get_consumer_offset_handler; +pub mod store_consumer_offset_handler; + +pub const COMPONENT: &str = "CONSUMER_OFFSET_HANDLER"; diff --git a/core/server/src/binary/handlers/consumer_offsets/store_consumer_offset_handler.rs b/core/server/src/binary/handlers/consumer_offsets/store_consumer_offset_handler.rs new file mode 100644 index 0000000000..bc2d265bc5 --- /dev/null +++ b/core/server/src/binary/handlers/consumer_offsets/store_consumer_offset_handler.rs @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::rc::Rc; + +use crate::binary::dispatch::{HandlerResult, wire_consumer_to_consumer, wire_id_to_identifier}; +use crate::binary::handlers::consumer_offsets::COMPONENT; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::streaming::session::Session; +use err_trail::ErrContext; +use iggy_binary_protocol::requests::consumer_offsets::StoreConsumerOffsetRequest; +use iggy_common::IggyError; +use tracing::debug; + +pub async fn handle_store_consumer_offset( + req: StoreConsumerOffsetRequest, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + let consumer = wire_consumer_to_consumer(&req.consumer)?; + let stream_id = wire_id_to_identifier(&req.stream_id)?; + let topic_id = wire_id_to_identifier(&req.topic_id)?; + debug!( + "session: {session}, command: store_consumer_offset, stream_id: {stream_id}, topic_id: {topic_id}, partition_id: {:?}, offset: {}", + req.partition_id, req.offset + ); + shard.ensure_authenticated(session)?; + let topic = shard.resolve_topic_for_store_consumer_offset( + session.get_user_id(), + &stream_id, + &topic_id, + )?; + shard + .store_consumer_offset( + session.client_id, + consumer, + topic, + req.partition_id, + req.offset, + ) + .await + .error(|e: &IggyError| format!("{COMPONENT} (error: {e}) - failed to store consumer offset for stream_id: {}, topic_id: {}, partition_id: {:?}, offset: {}, session: {}", + stream_id, topic_id, req.partition_id, req.offset, session + ))?; + sender.send_empty_ok_response().await?; + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/messages/flush_unsaved_buffer_handler.rs b/core/server/src/binary/handlers/messages/flush_unsaved_buffer_handler.rs new file mode 100644 index 0000000000..0d78ed907a --- /dev/null +++ b/core/server/src/binary/handlers/messages/flush_unsaved_buffer_handler.rs @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::{HandlerResult, wire_id_to_identifier}; +use crate::binary::handlers::messages::COMPONENT; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::shard::transmission::message::ResolvedPartition; +use crate::streaming::session::Session; +use err_trail::ErrContext; +use iggy_binary_protocol::requests::messages::FlushUnsavedBufferRequest; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::{debug, instrument}; + +#[instrument(skip_all, name = "trace_flush_unsaved_buffer", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id, iggy_partition_id = req.partition_id, iggy_fsync = req.fsync))] +pub async fn handle_flush_unsaved_buffer( + req: FlushUnsavedBufferRequest, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + let stream_id = wire_id_to_identifier(&req.stream_id)?; + let topic_id = wire_id_to_identifier(&req.topic_id)?; + let partition_id = req.partition_id; + let fsync = req.fsync; + debug!( + "session: {session}, command: flush_unsaved_buffer, stream_id: {stream_id}, topic_id: {topic_id}, partition_id: {partition_id}, fsync: {fsync}" + ); + shard.ensure_authenticated(session)?; + + let user_id = session.get_user_id(); + let topic = shard.resolve_topic(&stream_id, &topic_id)?; + let partition = ResolvedPartition { + stream_id: topic.stream_id, + topic_id: topic.topic_id, + partition_id: partition_id as usize, + }; + + shard + .flush_unsaved_buffer(user_id, partition, fsync) + .await + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to flush unsaved buffer for stream_id: {}, topic_id: {}, partition_id: {}, session: {}", + stream_id, topic_id, partition_id, session + ) + })?; + sender.send_empty_ok_response().await?; + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/messages/mod.rs b/core/server/src/binary/handlers/messages/mod.rs new file mode 100644 index 0000000000..64571728aa --- /dev/null +++ b/core/server/src/binary/handlers/messages/mod.rs @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod flush_unsaved_buffer_handler; +pub mod poll_messages_handler; +pub mod send_messages_handler; + +pub const COMPONENT: &str = "MESSAGE_HANDLER"; diff --git a/core/server/src/binary/handlers/messages/poll_messages_handler.rs b/core/server/src/binary/handlers/messages/poll_messages_handler.rs new file mode 100644 index 0000000000..6007020b7b --- /dev/null +++ b/core/server/src/binary/handlers/messages/poll_messages_handler.rs @@ -0,0 +1,87 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::{ + HandlerResult, wire_consumer_to_consumer, wire_id_to_identifier, wire_polling_to_strategy, +}; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::shard::system::messages::PollingArgs; +use crate::streaming::session::Session; +use iggy_binary_protocol::requests::messages::PollMessagesRequest; +use iggy_common::IggyError; +use server_common::PooledBuffer; +use std::rc::Rc; +use tracing::{debug, trace}; + +pub async fn handle_poll_messages( + req: PollMessagesRequest, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + let consumer = wire_consumer_to_consumer(&req.consumer)?; + let stream_id = wire_id_to_identifier(&req.stream_id)?; + let topic_id = wire_id_to_identifier(&req.topic_id)?; + let strategy = wire_polling_to_strategy(&req.strategy)?; + let partition_id = req.partition_id; + let count = req.count; + let auto_commit = req.auto_commit; + + debug!( + "session: {session}, command: poll_messages, stream_id: {stream_id}, topic_id: {topic_id}, partition_id: {partition_id:?}" + ); + shard.ensure_authenticated(session)?; + + let args = PollingArgs::new(strategy, count, auto_commit); + + let user_id = session.get_user_id(); + let client_id = session.client_id; + let topic = shard.resolve_topic_for_poll(user_id, &stream_id, &topic_id)?; + let (metadata, mut batch) = shard + .poll_messages(client_id, topic, consumer, partition_id, args) + .await?; + + let response_length = 4 + 8 + 4 + batch.size(); + let response_length_bytes = response_length.to_le_bytes(); + + let mut bufs = Vec::with_capacity(batch.containers_count() + 3); + let mut partition_id_buf = PooledBuffer::with_capacity(4); + let mut current_offset_buf = PooledBuffer::with_capacity(8); + let mut count_buf = PooledBuffer::with_capacity(4); + partition_id_buf.put_u32_le(metadata.partition_id); + current_offset_buf.put_u64_le(metadata.current_offset); + count_buf.put_u32_le(batch.count()); + + bufs.push(partition_id_buf); + bufs.push(current_offset_buf); + bufs.push(count_buf); + + batch.iter_mut().for_each(|m| { + bufs.push(m.take_messages()); + }); + trace!( + "Sending {} messages to client ({} bytes) to client", + batch.count(), + response_length + ); + + sender + .send_ok_response_vectored(&response_length_bytes, bufs) + .await?; + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/messages/send_messages_handler.rs b/core/server/src/binary/handlers/messages/send_messages_handler.rs new file mode 100644 index 0000000000..aabce4fd77 --- /dev/null +++ b/core/server/src/binary/handlers/messages/send_messages_handler.rs @@ -0,0 +1,203 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::HandlerResult; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::shard::transmission::message::{ResolvedPartition, ShardRequest, ShardRequestPayload}; +use crate::streaming::segments::{IggyIndexesMut, IggyMessagesBatchMut}; +use crate::streaming::session::Session; +use crate::streaming::topics; +use compio::buf::{IntoInner as _, IoBuf}; +use iggy_common::Identifier; +use iggy_common::Sizeable; +use iggy_common::{INDEX_SIZE, PartitioningKind}; +use iggy_common::{IggyError, Partitioning, Validatable}; +use server_common::PooledBuffer; +use server_common::sharding::IggyNamespace; +use std::rc::Rc; +use tracing::{debug, error, info, instrument}; + +#[instrument(skip_all, name = "trace_send_messages", fields( + iggy_user_id = session.get_user_id(), + iggy_client_id = session.client_id, +))] +pub async fn handle_send_messages( + sender: &mut SenderKind, + length: u32, + session: &Session, + shard: &Rc, +) -> Result { + shard.ensure_authenticated(session)?; + // `length` is the payload size (frame length minus the 4-byte code, + // already subtracted by the transport layer before calling dispatch). + let total_payload_size = length as usize; + let metadata_len_field_size = std::mem::size_of::(); + + let metadata_length_buffer = PooledBuffer::with_capacity(4); + let (result, metadata_len_buf) = sender.read(metadata_length_buffer.slice(0..4)).await; + let metadata_len_buf = metadata_len_buf.into_inner(); + result?; + let metadata_size = u32::from_le_bytes( + metadata_len_buf[..] + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); + if metadata_size as usize > total_payload_size { + return Err(IggyError::InvalidCommand); + } + + let metadata_buffer = PooledBuffer::with_capacity(metadata_size as usize); + let (result, metadata_buf) = sender + .read(metadata_buffer.slice(0..metadata_size as usize)) + .await; + result?; + let metadata_buf = metadata_buf.into_inner(); + + let mut element_size = 0; + + let stream_id = Identifier::from_raw_bytes(&metadata_buf)?; + element_size += stream_id.get_size_bytes().as_bytes_usize(); + + let topic_id = Identifier::from_raw_bytes( + metadata_buf + .get(element_size..) + .ok_or(IggyError::InvalidCommand)?, + )?; + element_size += topic_id.get_size_bytes().as_bytes_usize(); + + let partitioning = Partitioning::from_raw_bytes( + metadata_buf + .get(element_size..) + .ok_or(IggyError::InvalidCommand)?, + )?; + element_size += partitioning.get_size_bytes().as_bytes_usize(); + + let messages_count = u32::from_le_bytes( + metadata_buf + .get(element_size..element_size + 4) + .ok_or(IggyError::InvalidCommand)? + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); + let indexes_size = (messages_count as usize) + .checked_mul(INDEX_SIZE) + .ok_or(IggyError::InvalidCommand)?; + if indexes_size > total_payload_size { + return Err(IggyError::InvalidCommand); + } + + let indexes_buffer = PooledBuffer::with_capacity(indexes_size); + let (result, indexes_buffer) = sender.read(indexes_buffer.slice(0..indexes_size)).await; + result?; + let indexes_buffer = indexes_buffer.into_inner(); + + let messages_size = total_payload_size + .checked_sub(metadata_size as usize) + .and_then(|s| s.checked_sub(indexes_size)) + .and_then(|s| s.checked_sub(metadata_len_field_size)) + .ok_or(IggyError::InvalidCommand)?; + let messages_buffer = PooledBuffer::with_capacity(messages_size); + let (result, messages_buffer) = sender.read(messages_buffer.slice(0..messages_size)).await; + result?; + let messages_buffer = messages_buffer.into_inner(); + + let indexes = IggyIndexesMut::from_bytes(indexes_buffer, 0); + let batch = IggyMessagesBatchMut::from_indexes_and_messages(indexes, messages_buffer); + batch.validate()?; + + let topic = shard.resolve_topic_for_append(session.get_user_id(), &stream_id, &topic_id)?; + + let partition_id = match partitioning.kind { + PartitioningKind::Balanced => shard + .metadata + .get_next_partition_id(topic.stream_id, topic.topic_id) + .ok_or(IggyError::TopicIdNotFound( + stream_id.clone(), + topic_id.clone(), + ))?, + PartitioningKind::PartitionId => u32::from_le_bytes( + partitioning + .value + .get(..4) + .ok_or(IggyError::InvalidCommand)? + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ) as usize, + PartitioningKind::MessagesKey => { + let partitions_count = shard + .metadata + .partitions_count(topic.stream_id, topic.topic_id); + topics::helpers::calculate_partition_id_by_messages_key_hash( + partitions_count, + &partitioning.value, + ) + } + }; + + let namespace = IggyNamespace::new(topic.stream_id, topic.topic_id, partition_id); + let user_id = session.get_user_id(); + let unsupported_socket_transfer = matches!( + partitioning.kind, + PartitioningKind::Balanced | PartitioningKind::MessagesKey + ); + let enabled_socket_migration = shard.config.tcp.socket_migration; + + if enabled_socket_migration + && !(session.is_migrated() || unsupported_socket_transfer) + && let Some(target_shard) = shard.find_shard(&namespace) + && target_shard.id != shard.id + { + debug!( + "TCP wrong shared detected: migrating from_shard {}, to_shard {}", + shard.id, target_shard.id + ); + + if let Some(fd) = sender.take_and_migrate_tcp() { + let payload = ShardRequestPayload::SocketTransfer { + fd, + from_shard: shard.id, + client_id: session.client_id, + user_id, + address: session.ip_address, + initial_data: batch, + }; + + let request = ShardRequest::data_plane(namespace, payload); + + if let Err(e) = shard.send_to_data_plane(request).await { + error!("transfer socket to another shard failed, drop connection. {e:?}"); + return Ok(HandlerResult::Finished); + } + + info!("Sending socket transfer to shard {}", target_shard.id); + return Ok(HandlerResult::Migrated { + to_shard: target_shard.id, + }); + } + } + + let partition = ResolvedPartition { + stream_id: topic.stream_id, + topic_id: topic.topic_id, + partition_id, + }; + shard.append_messages(partition, batch).await?; + + sender.send_empty_ok_response().await?; + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/mod.rs b/core/server/src/binary/handlers/mod.rs new file mode 100644 index 0000000000..8a7a7beaf2 --- /dev/null +++ b/core/server/src/binary/handlers/mod.rs @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod cluster; +pub mod consumer_groups; +pub mod consumer_offsets; +pub mod messages; +pub mod partitions; +pub mod personal_access_tokens; +pub mod segments; +pub mod streams; +pub mod system; +pub mod topics; +pub mod users; diff --git a/core/server/src/binary/handlers/partitions/create_partitions_handler.rs b/core/server/src/binary/handlers/partitions/create_partitions_handler.rs new file mode 100644 index 0000000000..3e9d249dd9 --- /dev/null +++ b/core/server/src/binary/handlers/partitions/create_partitions_handler.rs @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::HandlerResult; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; +use crate::streaming::session::Session; +use iggy_binary_protocol::MAX_PARTITIONS_PER_REQUEST; +use iggy_binary_protocol::requests::partitions::CreatePartitionsRequest; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::{debug, instrument}; + +#[instrument(skip_all, name = "trace_create_partitions", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] +pub async fn handle_create_partitions( + req: CreatePartitionsRequest, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + debug!( + "session: {session}, command: create_partitions, stream_id: {:?}, topic_id: {:?}", + req.stream_id, req.topic_id + ); + shard.ensure_authenticated(session)?; + + if !(1..=MAX_PARTITIONS_PER_REQUEST).contains(&req.partitions_count) { + return Err(IggyError::TooManyPartitions); + } + + let request = ShardRequest::control_plane(ShardRequestPayload::CreatePartitionsRequest { + user_id: session.get_user_id(), + command: req, + }); + + match shard.send_to_control_plane(request).await? { + ShardResponse::CreatePartitionsResponse => { + sender.send_empty_ok_response().await?; + } + ShardResponse::ErrorResponse(err) => return Err(err), + _ => unreachable!("Expected CreatePartitionsResponse"), + } + + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/partitions/delete_partitions_handler.rs b/core/server/src/binary/handlers/partitions/delete_partitions_handler.rs new file mode 100644 index 0000000000..90d9836139 --- /dev/null +++ b/core/server/src/binary/handlers/partitions/delete_partitions_handler.rs @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::HandlerResult; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; +use crate::streaming::session::Session; +use iggy_binary_protocol::requests::partitions::DeletePartitionsRequest; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::{debug, instrument}; + +#[instrument(skip_all, name = "trace_delete_partitions", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] +pub async fn handle_delete_partitions( + req: DeletePartitionsRequest, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + debug!( + "session: {session}, command: delete_partitions, stream_id: {:?}, topic_id: {:?}", + req.stream_id, req.topic_id + ); + shard.ensure_authenticated(session)?; + + if req.partitions_count == 0 { + return Err(IggyError::TooManyPartitions); + } + + let request = ShardRequest::control_plane(ShardRequestPayload::DeletePartitionsRequest { + user_id: session.get_user_id(), + command: req, + }); + + match shard.send_to_control_plane(request).await? { + ShardResponse::DeletePartitionsResponse => { + sender.send_empty_ok_response().await?; + } + ShardResponse::ErrorResponse(err) => return Err(err), + _ => unreachable!("Expected DeletePartitionsResponse"), + } + + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/partitions/mod.rs b/core/server/src/binary/handlers/partitions/mod.rs new file mode 100644 index 0000000000..38e9d65215 --- /dev/null +++ b/core/server/src/binary/handlers/partitions/mod.rs @@ -0,0 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod create_partitions_handler; +pub mod delete_partitions_handler; + +pub const COMPONENT: &str = "PARTITIONS_HANDLER"; diff --git a/core/server/src/binary/handlers/personal_access_tokens/create_personal_access_token_handler.rs b/core/server/src/binary/handlers/personal_access_tokens/create_personal_access_token_handler.rs new file mode 100644 index 0000000000..9058f61908 --- /dev/null +++ b/core/server/src/binary/handlers/personal_access_tokens/create_personal_access_token_handler.rs @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::HandlerResult; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; +use crate::streaming::session::Session; +use iggy_binary_protocol::WireName; +use iggy_binary_protocol::codec::WireEncode; +use iggy_binary_protocol::requests::personal_access_tokens::CreatePersonalAccessTokenRequest; +use iggy_binary_protocol::responses::personal_access_tokens::RawPersonalAccessTokenResponse; +use iggy_common::IggyError; +use iggy_common::defaults::{ + MAX_PERSONAL_ACCESS_TOKEN_NAME_LENGTH, MIN_PERSONAL_ACCESS_TOKEN_NAME_LENGTH, +}; +use std::rc::Rc; +use tracing::{debug, instrument}; + +#[instrument(skip_all, name = "trace_create_personal_access_token", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] +pub async fn handle_create_personal_access_token( + req: CreatePersonalAccessTokenRequest, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + debug!( + "session: {session}, command: create_personal_access_token, name: {}", + req.name.as_str() + ); + shard.ensure_authenticated(session)?; + + let name_len = req.name.as_str().len(); + if !(MIN_PERSONAL_ACCESS_TOKEN_NAME_LENGTH..=MAX_PERSONAL_ACCESS_TOKEN_NAME_LENGTH) + .contains(&name_len) + { + return Err(IggyError::InvalidPersonalAccessTokenName); + } + + let request = + ShardRequest::control_plane(ShardRequestPayload::CreatePersonalAccessTokenRequest { + user_id: session.get_user_id(), + command: req, + }); + + match shard.send_to_control_plane(request).await? { + ShardResponse::CreatePersonalAccessTokenResponse(_, token) => { + let response = RawPersonalAccessTokenResponse { + token: WireName::new(token).map_err(|_| IggyError::InvalidCommand)?, + }; + sender.send_ok_response(&response.to_bytes()).await?; + } + ShardResponse::ErrorResponse(err) => return Err(err), + _ => unreachable!("Expected CreatePersonalAccessTokenResponse"), + } + + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/personal_access_tokens/delete_personal_access_token_handler.rs b/core/server/src/binary/handlers/personal_access_tokens/delete_personal_access_token_handler.rs new file mode 100644 index 0000000000..24f536e77a --- /dev/null +++ b/core/server/src/binary/handlers/personal_access_tokens/delete_personal_access_token_handler.rs @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::HandlerResult; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; +use crate::streaming::session::Session; +use iggy_binary_protocol::requests::personal_access_tokens::DeletePersonalAccessTokenRequest; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::{debug, instrument}; + +#[instrument(skip_all, name = "trace_delete_personal_access_token", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] +pub async fn handle_delete_personal_access_token( + req: DeletePersonalAccessTokenRequest, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + debug!( + "session: {session}, command: delete_personal_access_token, name: {}", + req.name.as_str() + ); + shard.ensure_authenticated(session)?; + + let request = + ShardRequest::control_plane(ShardRequestPayload::DeletePersonalAccessTokenRequest { + user_id: session.get_user_id(), + command: req, + }); + + match shard.send_to_control_plane(request).await? { + ShardResponse::DeletePersonalAccessTokenResponse => { + sender.send_empty_ok_response().await?; + } + ShardResponse::ErrorResponse(err) => return Err(err), + _ => unreachable!("Expected DeletePersonalAccessTokenResponse"), + } + + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/personal_access_tokens/get_personal_access_tokens_handler.rs b/core/server/src/binary/handlers/personal_access_tokens/get_personal_access_tokens_handler.rs new file mode 100644 index 0000000000..26af78fadb --- /dev/null +++ b/core/server/src/binary/handlers/personal_access_tokens/get_personal_access_tokens_handler.rs @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::HandlerResult; +use crate::binary::handlers::personal_access_tokens::COMPONENT; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::streaming::session::Session; +use err_trail::ErrContext; +use iggy_binary_protocol::WireName; +use iggy_binary_protocol::codec::WireEncode; +use iggy_binary_protocol::responses::personal_access_tokens::{ + GetPersonalAccessTokensResponse, PersonalAccessTokenResponse, +}; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::debug; + +pub async fn handle_get_personal_access_tokens( + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + debug!("session: {session}, command: get_personal_access_tokens"); + shard.ensure_authenticated(session)?; + let personal_access_tokens = shard + .get_personal_access_tokens(session.get_user_id()) + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to get personal access tokens for user: {}", + session.get_user_id() + ) + })?; + let tokens: Vec = personal_access_tokens + .iter() + .map(|pat| { + Ok(PersonalAccessTokenResponse { + name: WireName::new(pat.name.as_ref()).map_err(|_| IggyError::InvalidCommand)?, + expiry_at: pat.expiry_at.map_or(0, |e| e.as_micros()), + }) + }) + .collect::>()?; + let response = GetPersonalAccessTokensResponse { tokens }; + sender.send_ok_response(&response.to_bytes()).await?; + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/personal_access_tokens/login_with_personal_access_token_handler.rs b/core/server/src/binary/handlers/personal_access_tokens/login_with_personal_access_token_handler.rs new file mode 100644 index 0000000000..622410de7a --- /dev/null +++ b/core/server/src/binary/handlers/personal_access_tokens/login_with_personal_access_token_handler.rs @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::HandlerResult; +use crate::binary::handlers::personal_access_tokens::COMPONENT; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::streaming::session::Session; +use err_trail::ErrContext; +use iggy_binary_protocol::codec::WireEncode; +use iggy_binary_protocol::requests::personal_access_tokens::LoginWithPersonalAccessTokenRequest; +use iggy_binary_protocol::responses::users::IdentityResponse; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::{debug, instrument}; + +#[instrument(skip_all, name = "trace_login_with_personal_access_token", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] +pub async fn handle_login_with_personal_access_token( + req: LoginWithPersonalAccessTokenRequest, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + debug!("session: {session}, command: login_with_personal_access_token"); + let token = req.token.as_str(); + + let user = shard + .login_with_personal_access_token(token, Some(session)) + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to login with personal access token, session: {session}", + ) + })?; + let response = IdentityResponse { user_id: user.id }; + sender.send_ok_response(&response.to_bytes()).await?; + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/personal_access_tokens/mod.rs b/core/server/src/binary/handlers/personal_access_tokens/mod.rs new file mode 100644 index 0000000000..137de0531f --- /dev/null +++ b/core/server/src/binary/handlers/personal_access_tokens/mod.rs @@ -0,0 +1,23 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod create_personal_access_token_handler; +pub mod delete_personal_access_token_handler; +pub mod get_personal_access_tokens_handler; +pub mod login_with_personal_access_token_handler; + +pub const COMPONENT: &str = "PERSONAL_ACCESS_TOKEN_HANDLER"; diff --git a/core/server/src/binary/handlers/segments/delete_segments_handler.rs b/core/server/src/binary/handlers/segments/delete_segments_handler.rs new file mode 100644 index 0000000000..75b2b1c837 --- /dev/null +++ b/core/server/src/binary/handlers/segments/delete_segments_handler.rs @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::{HandlerResult, wire_id_to_identifier}; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; +use crate::streaming::session::Session; +use iggy_binary_protocol::requests::segments::DeleteSegmentsRequest; +use iggy_common::IggyError; +use server_common::sharding::IggyNamespace; +use std::rc::Rc; +use tracing::{debug, instrument}; + +#[instrument(skip_all, name = "trace_delete_segments", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] +pub async fn handle_delete_segments( + req: DeleteSegmentsRequest, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + let stream_id = wire_id_to_identifier(&req.stream_id)?; + let topic_id = wire_id_to_identifier(&req.topic_id)?; + debug!( + "session: {session}, command: delete_segments, stream_id: {stream_id}, topic_id: {topic_id}" + ); + shard.ensure_authenticated(session)?; + + let partition_id = req.partition_id as usize; + let segments_count = req.segments_count; + + let partition = shard.resolve_partition_for_delete_segments( + session.get_user_id(), + &stream_id, + &topic_id, + partition_id, + )?; + + let namespace = IggyNamespace::new( + partition.stream_id, + partition.topic_id, + partition.partition_id, + ); + let payload = ShardRequestPayload::DeleteSegments { segments_count }; + let request = ShardRequest::data_plane(namespace, payload); + + match shard.send_to_data_plane(request).await? { + ShardResponse::DeleteSegments { + deleted_segments, + deleted_messages, + } => { + shard.metrics.decrement_segments(deleted_segments as u32); + shard.metrics.decrement_messages(deleted_messages); + sender.send_empty_ok_response().await?; + } + ShardResponse::ErrorResponse(err) => return Err(err), + _ => unreachable!("Expected DeleteSegments"), + } + + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/segments/mod.rs b/core/server/src/binary/handlers/segments/mod.rs new file mode 100644 index 0000000000..ce45de03bb --- /dev/null +++ b/core/server/src/binary/handlers/segments/mod.rs @@ -0,0 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod delete_segments_handler; diff --git a/core/server/src/binary/handlers/streams/create_stream_handler.rs b/core/server/src/binary/handlers/streams/create_stream_handler.rs new file mode 100644 index 0000000000..c25f6f11d6 --- /dev/null +++ b/core/server/src/binary/handlers/streams/create_stream_handler.rs @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::HandlerResult; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; +use crate::streaming::session::Session; +use iggy_binary_protocol::WireName; +use iggy_binary_protocol::codec::WireEncode; +use iggy_binary_protocol::requests::streams::CreateStreamRequest; +use iggy_binary_protocol::responses::streams::StreamResponse; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::{debug, instrument}; + +#[instrument(skip_all, name = "trace_create_stream", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] +pub async fn handle_create_stream( + req: CreateStreamRequest, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + debug!( + "session: {session}, command: create_stream, name: {}", + req.name.as_str() + ); + shard.ensure_authenticated(session)?; + + let request = ShardRequest::control_plane(ShardRequestPayload::CreateStreamRequest { + user_id: session.get_user_id(), + command: req, + }); + + match shard.send_to_control_plane(request).await? { + ShardResponse::CreateStreamResponse(data) => { + let response = StreamResponse { + id: data.id, + created_at: data.created_at.into(), + topics_count: 0, + size_bytes: 0, + messages_count: 0, + name: WireName::new(data.name.as_ref()).map_err(|_| IggyError::InvalidCommand)?, + }; + sender.send_ok_response(&response.to_bytes()).await?; + } + ShardResponse::ErrorResponse(err) => return Err(err), + _ => unreachable!("Expected CreateStreamResponse"), + } + + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/streams/delete_stream_handler.rs b/core/server/src/binary/handlers/streams/delete_stream_handler.rs new file mode 100644 index 0000000000..967c042810 --- /dev/null +++ b/core/server/src/binary/handlers/streams/delete_stream_handler.rs @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::HandlerResult; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; +use crate::streaming::session::Session; +use iggy_binary_protocol::requests::streams::DeleteStreamRequest; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::{debug, instrument}; + +#[instrument(skip_all, name = "trace_delete_stream", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] +pub async fn handle_delete_stream( + req: DeleteStreamRequest, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + debug!( + "session: {session}, command: delete_stream, stream_id: {:?}", + req.stream_id + ); + shard.ensure_authenticated(session)?; + + let request = ShardRequest::control_plane(ShardRequestPayload::DeleteStreamRequest { + user_id: session.get_user_id(), + command: req, + }); + + match shard.send_to_control_plane(request).await? { + ShardResponse::DeleteStreamResponse => { + sender.send_empty_ok_response().await?; + } + ShardResponse::ErrorResponse(err) => return Err(err), + _ => unreachable!("Expected DeleteStreamResponse"), + } + + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/streams/get_stream_handler.rs b/core/server/src/binary/handlers/streams/get_stream_handler.rs new file mode 100644 index 0000000000..74551a8cb3 --- /dev/null +++ b/core/server/src/binary/handlers/streams/get_stream_handler.rs @@ -0,0 +1,121 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::{HandlerResult, wire_id_to_identifier}; +use crate::metadata::{StreamMeta, TopicMeta}; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::streaming::session::Session; +use iggy_binary_protocol::WireName; +use iggy_binary_protocol::codec::WireEncode; +use iggy_binary_protocol::requests::streams::GetStreamRequest; +use iggy_binary_protocol::responses::streams::StreamResponse; +use iggy_binary_protocol::responses::streams::get_stream::{GetStreamResponse, TopicHeader}; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::debug; + +pub async fn handle_get_stream( + req: GetStreamRequest, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + debug!( + "session: {session}, command: get_stream, stream_id: {:?}", + req.stream_id + ); + shard.ensure_authenticated(session)?; + + let stream_id = wire_id_to_identifier(&req.stream_id)?; + + let Some(stream) = shard + .metadata + .query_stream(session.get_user_id(), &stream_id)? + else { + sender.send_empty_ok_response().await?; + return Ok(HandlerResult::Finished); + }; + + let response = build_get_stream_response(&stream)?; + sender.send_ok_response(&response.to_bytes()).await?; + Ok(HandlerResult::Finished) +} + +pub(crate) fn compute_stream_stats(stream: &StreamMeta) -> (u64, u64) { + let mut size = 0u64; + let mut messages = 0u64; + for (_, topic) in stream.topics.iter() { + for partition in topic.partitions.iter() { + size += partition.stats.size_bytes_inconsistent(); + messages += partition.stats.messages_count_inconsistent(); + } + } + (size, messages) +} + +fn compute_topic_stats(topic: &TopicMeta) -> (u64, u64) { + let mut size = 0u64; + let mut messages = 0u64; + for partition in topic.partitions.iter() { + size += partition.stats.size_bytes_inconsistent(); + messages += partition.stats.messages_count_inconsistent(); + } + (size, messages) +} + +pub(crate) fn build_topic_header(topic: &TopicMeta) -> Result { + let (size, messages) = compute_topic_stats(topic); + Ok(TopicHeader { + id: topic.id as u32, + created_at: topic.created_at.into(), + partitions_count: topic.partitions.len() as u32, + message_expiry: topic.message_expiry.into(), + compression_algorithm: topic.compression_algorithm.as_code(), + max_topic_size: topic.max_topic_size.into(), + replication_factor: topic.replication_factor, + size_bytes: size, + messages_count: messages, + name: WireName::new(topic.name.as_ref()).map_err(|_| IggyError::InvalidCommand)?, + }) +} + +fn build_get_stream_response(stream: &StreamMeta) -> Result { + let mut topic_ids: Vec<_> = stream.topics.iter().map(|(k, _)| k).collect(); + topic_ids.sort_unstable(); + + let (total_size, total_messages) = compute_stream_stats(stream); + + let mut topics = Vec::with_capacity(topic_ids.len()); + for &topic_id in &topic_ids { + if let Some(topic) = stream.topics.get(topic_id) { + topics.push(build_topic_header(topic)?); + } + } + + Ok(GetStreamResponse { + stream: StreamResponse { + id: stream.id as u32, + created_at: stream.created_at.into(), + topics_count: topic_ids.len() as u32, + size_bytes: total_size, + messages_count: total_messages, + name: WireName::new(stream.name.as_ref()).map_err(|_| IggyError::InvalidCommand)?, + }, + topics, + }) +} diff --git a/core/server/src/binary/handlers/streams/get_streams_handler.rs b/core/server/src/binary/handlers/streams/get_streams_handler.rs new file mode 100644 index 0000000000..26d3264c09 --- /dev/null +++ b/core/server/src/binary/handlers/streams/get_streams_handler.rs @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::get_stream_handler::compute_stream_stats; +use crate::binary::dispatch::HandlerResult; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::streaming::session::Session; +use iggy_binary_protocol::WireName; +use iggy_binary_protocol::codec::WireEncode; +use iggy_binary_protocol::responses::streams::StreamResponse; +use iggy_binary_protocol::responses::streams::get_streams::GetStreamsResponse; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::debug; + +pub async fn handle_get_streams( + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + debug!("session: {session}, command: get_streams"); + shard.ensure_authenticated(session)?; + + let streams = shard.metadata.query_streams(session.get_user_id())?; + + let mut sorted: Vec<_> = streams.iter().collect(); + sorted.sort_by_key(|s| s.id); + + let mut wire_streams = Vec::with_capacity(sorted.len()); + for stream in sorted { + let (total_size, total_messages) = compute_stream_stats(stream); + wire_streams.push(StreamResponse { + id: stream.id as u32, + created_at: stream.created_at.into(), + topics_count: stream.topics.len() as u32, + size_bytes: total_size, + messages_count: total_messages, + name: WireName::new(stream.name.as_ref()).map_err(|_| IggyError::InvalidCommand)?, + }); + } + + let response = GetStreamsResponse { + streams: wire_streams, + }; + sender.send_ok_response(&response.to_bytes()).await?; + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/streams/mod.rs b/core/server/src/binary/handlers/streams/mod.rs new file mode 100644 index 0000000000..9fd1a45d47 --- /dev/null +++ b/core/server/src/binary/handlers/streams/mod.rs @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod create_stream_handler; +pub mod delete_stream_handler; +pub mod get_stream_handler; +pub mod get_streams_handler; +pub mod purge_stream_handler; +pub mod update_stream_handler; + +pub const COMPONENT: &str = "STREAM_HANDLER"; diff --git a/core/server/src/binary/handlers/streams/purge_stream_handler.rs b/core/server/src/binary/handlers/streams/purge_stream_handler.rs new file mode 100644 index 0000000000..7806bc8250 --- /dev/null +++ b/core/server/src/binary/handlers/streams/purge_stream_handler.rs @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::HandlerResult; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; +use crate::streaming::session::Session; +use iggy_binary_protocol::requests::streams::PurgeStreamRequest; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::{debug, instrument}; + +#[instrument(skip_all, name = "trace_purge_stream", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] +pub async fn handle_purge_stream( + req: PurgeStreamRequest, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + debug!( + "session: {session}, command: purge_stream, stream_id: {:?}", + req.stream_id + ); + shard.ensure_authenticated(session)?; + + let request = ShardRequest::control_plane(ShardRequestPayload::PurgeStreamRequest { + user_id: session.get_user_id(), + command: req, + }); + + match shard.send_to_control_plane(request).await? { + ShardResponse::PurgeStreamResponse => { + sender.send_empty_ok_response().await?; + } + ShardResponse::ErrorResponse(err) => return Err(err), + _ => unreachable!("Expected PurgeStreamResponse"), + } + + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/streams/update_stream_handler.rs b/core/server/src/binary/handlers/streams/update_stream_handler.rs new file mode 100644 index 0000000000..193d9d85aa --- /dev/null +++ b/core/server/src/binary/handlers/streams/update_stream_handler.rs @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::HandlerResult; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; +use crate::streaming::session::Session; +use iggy_binary_protocol::requests::streams::UpdateStreamRequest; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::{debug, instrument}; + +#[instrument(skip_all, name = "trace_update_stream", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] +pub async fn handle_update_stream( + req: UpdateStreamRequest, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + debug!( + "session: {session}, command: update_stream, stream_id: {:?}", + req.stream_id + ); + shard.ensure_authenticated(session)?; + + let request = ShardRequest::control_plane(ShardRequestPayload::UpdateStreamRequest { + user_id: session.get_user_id(), + command: req, + }); + + match shard.send_to_control_plane(request).await? { + ShardResponse::UpdateStreamResponse => { + sender.send_empty_ok_response().await?; + } + ShardResponse::ErrorResponse(err) => return Err(err), + _ => unreachable!("Expected UpdateStreamResponse"), + } + + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/system/get_client_handler.rs b/core/server/src/binary/handlers/system/get_client_handler.rs new file mode 100644 index 0000000000..a0456dd9b9 --- /dev/null +++ b/core/server/src/binary/handlers/system/get_client_handler.rs @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::get_me_handler::build_client_details_response; +use crate::binary::dispatch::HandlerResult; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::streaming::session::Session; +use iggy_binary_protocol::codec::WireEncode; +use iggy_binary_protocol::requests::system::GetClientRequest; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::debug; + +pub async fn handle_get_client( + req: GetClientRequest, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + debug!( + "session: {session}, command: get_client, client_id: {}", + req.client_id + ); + shard.ensure_authenticated(session)?; + shard.metadata.perm_get_client(session.get_user_id())?; + + if req.client_id == 0 { + return Err(IggyError::InvalidClientId); + } + + let Some(client) = shard.get_client(req.client_id) else { + sender.send_empty_ok_response().await?; + return Ok(HandlerResult::Finished); + }; + + let response = build_client_details_response(&client); + sender.send_ok_response(&response.to_bytes()).await?; + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/system/get_clients_handler.rs b/core/server/src/binary/handlers/system/get_clients_handler.rs new file mode 100644 index 0000000000..1743012a02 --- /dev/null +++ b/core/server/src/binary/handlers/system/get_clients_handler.rs @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::get_me_handler::build_client_response; +use crate::binary::dispatch::HandlerResult; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::streaming::session::Session; +use iggy_binary_protocol::codec::WireEncode; +use iggy_binary_protocol::responses::clients::GetClientsResponse; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::debug; + +pub async fn handle_get_clients( + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + debug!("session: {session}, command: get_clients"); + shard.ensure_authenticated(session)?; + shard.metadata.perm_get_clients(session.get_user_id())?; + + let clients = shard.get_clients(); + let response = GetClientsResponse { + clients: clients.iter().map(build_client_response).collect(), + }; + sender.send_ok_response(&response.to_bytes()).await?; + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/system/get_me_handler.rs b/core/server/src/binary/handlers/system/get_me_handler.rs new file mode 100644 index 0000000000..44e332cf7d --- /dev/null +++ b/core/server/src/binary/handlers/system/get_me_handler.rs @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::HandlerResult; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::streaming::clients::client_manager::Client; +use crate::streaming::session::Session; +use iggy_binary_protocol::codec::WireEncode; +use iggy_binary_protocol::responses::clients::{ + ClientDetailsResponse, ClientResponse, ConsumerGroupInfoResponse, +}; +use iggy_common::IggyError; +use std::rc::Rc; + +pub async fn handle_get_me( + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + shard.ensure_authenticated(session)?; + let Some(client) = shard.get_client(session.client_id) else { + return Err(IggyError::ClientNotFound(session.client_id)); + }; + + let response = build_client_details_response(&client); + sender.send_ok_response(&response.to_bytes()).await?; + Ok(HandlerResult::Finished) +} + +pub(crate) fn build_client_details_response(client: &Client) -> ClientDetailsResponse { + ClientDetailsResponse { + client: build_client_response(client), + consumer_groups: client + .consumer_groups + .iter() + .map(|cg| ConsumerGroupInfoResponse { + stream_id: cg.stream_id, + topic_id: cg.topic_id, + group_id: cg.group_id, + }) + .collect(), + } +} + +pub(crate) fn build_client_response(client: &Client) -> ClientResponse { + ClientResponse { + client_id: client.session.client_id, + user_id: client.user_id.unwrap_or(u32::MAX), + transport: transport_to_u8(&client.transport), + address: client.session.ip_address.to_string(), + consumer_groups_count: client.consumer_groups.len() as u32, + } +} + +pub(crate) fn transport_to_u8(transport: &iggy_common::TransportProtocol) -> u8 { + match transport { + iggy_common::TransportProtocol::Tcp => 1, + iggy_common::TransportProtocol::Quic => 2, + iggy_common::TransportProtocol::Http => 3, + iggy_common::TransportProtocol::WebSocket => 4, + } +} diff --git a/core/server/src/binary/handlers/system/get_snapshot_handler.rs b/core/server/src/binary/handlers/system/get_snapshot_handler.rs new file mode 100644 index 0000000000..132613f6a5 --- /dev/null +++ b/core/server/src/binary/handlers/system/get_snapshot_handler.rs @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::HandlerResult; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::streaming::session::Session; +use bytes::Bytes; +use iggy_binary_protocol::requests::system::GetSnapshotRequest; +use iggy_common::{IggyError, SnapshotCompression, SystemSnapshotType}; +use std::rc::Rc; +use tracing::debug; + +pub async fn handle_get_snapshot( + req: GetSnapshotRequest, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + debug!("session: {session}, command: get_snapshot"); + shard.ensure_authenticated(session)?; + shard.metadata.perm_get_snapshot(session.get_user_id())?; + + let compression = SnapshotCompression::from_code(req.compression)?; + let snapshot_types: Vec = req + .snapshot_types + .iter() + .map(|&code| SystemSnapshotType::from_code(code)) + .collect::>()?; + + if snapshot_types.contains(&SystemSnapshotType::All) && snapshot_types.len() > 1 { + return Err(IggyError::InvalidCommand); + } + + let snapshot = shard.get_snapshot(compression, &snapshot_types).await?; + let bytes = Bytes::copy_from_slice(&snapshot.0); + sender.send_ok_response(&bytes).await?; + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/system/get_stats_handler.rs b/core/server/src/binary/handlers/system/get_stats_handler.rs new file mode 100644 index 0000000000..96ffca68a7 --- /dev/null +++ b/core/server/src/binary/handlers/system/get_stats_handler.rs @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::HandlerResult; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; +use crate::streaming::session::Session; +use iggy_binary_protocol::codec::WireEncode; +use iggy_binary_protocol::responses::system::get_stats::{CacheMetricEntry, StatsResponse}; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::debug; + +pub async fn handle_get_stats( + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + debug!("session: {session}, command: get_stats"); + shard.ensure_authenticated(session)?; + shard.metadata.perm_get_stats(session.get_user_id())?; + + let request = ShardRequest::control_plane(ShardRequestPayload::GetStats { + user_id: session.get_user_id(), + }); + + match shard.send_to_control_plane(request).await? { + ShardResponse::GetStatsResponse(stats) => { + let response = StatsResponse { + process_id: stats.process_id, + cpu_usage: stats.cpu_usage, + total_cpu_usage: stats.total_cpu_usage, + memory_usage: stats.memory_usage.as_bytes_u64(), + total_memory: stats.total_memory.as_bytes_u64(), + available_memory: stats.available_memory.as_bytes_u64(), + run_time: stats.run_time.into(), + start_time: stats.start_time.into(), + read_bytes: stats.read_bytes.as_bytes_u64(), + written_bytes: stats.written_bytes.as_bytes_u64(), + messages_size_bytes: stats.messages_size_bytes.as_bytes_u64(), + streams_count: stats.streams_count, + topics_count: stats.topics_count, + partitions_count: stats.partitions_count, + segments_count: stats.segments_count, + messages_count: stats.messages_count, + clients_count: stats.clients_count, + consumer_groups_count: stats.consumer_groups_count, + hostname: stats.hostname, + os_name: stats.os_name, + os_version: stats.os_version, + kernel_version: stats.kernel_version, + iggy_server_version: stats.iggy_server_version, + iggy_server_semver: stats.iggy_server_semver, + cache_metrics: stats + .cache_metrics + .iter() + .map(|(key, metrics)| CacheMetricEntry { + stream_id: key.stream_id, + topic_id: key.topic_id, + partition_id: key.partition_id, + hits: metrics.hits, + misses: metrics.misses, + hit_ratio: metrics.hit_ratio, + }) + .collect(), + threads_count: stats.threads_count, + free_disk_space: stats.free_disk_space.as_bytes_u64(), + total_disk_space: stats.total_disk_space.as_bytes_u64(), + }; + sender.send_ok_response(&response.to_bytes()).await?; + } + ShardResponse::ErrorResponse(err) => return Err(err), + _ => unreachable!("Expected GetStatsResponse"), + } + + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/system/mod.rs b/core/server/src/binary/handlers/system/mod.rs new file mode 100644 index 0000000000..065260f2d2 --- /dev/null +++ b/core/server/src/binary/handlers/system/mod.rs @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod get_client_handler; +pub mod get_clients_handler; +pub mod get_me_handler; +pub mod get_snapshot_handler; +pub mod get_stats_handler; +pub mod ping_handler; + +pub const COMPONENT: &str = "SYSTEM_HANDLER"; diff --git a/core/server/src/binary/handlers/system/ping_handler.rs b/core/server/src/binary/handlers/system/ping_handler.rs new file mode 100644 index 0000000000..93d5e0ae58 --- /dev/null +++ b/core/server/src/binary/handlers/system/ping_handler.rs @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::HandlerResult; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::streaming::session::Session; +use iggy_common::IggyError; +use iggy_common::IggyTimestamp; +use std::rc::Rc; +use tracing::debug; + +pub async fn handle_ping( + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + debug!("session: {session}, command: ping"); + if let Some(mut client) = shard.client_manager.try_get_client_mut(session.client_id) { + let now = IggyTimestamp::now(); + client.last_heartbeat = now; + debug!("Updated last heartbeat to: {now} for session: {session}"); + } + + sender.send_empty_ok_response().await?; + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/topics/create_topic_handler.rs b/core/server/src/binary/handlers/topics/create_topic_handler.rs new file mode 100644 index 0000000000..402fb02ba5 --- /dev/null +++ b/core/server/src/binary/handlers/topics/create_topic_handler.rs @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::HandlerResult; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; +use crate::streaming::session::Session; +use bytes::BytesMut; +use iggy_binary_protocol::MAX_PARTITIONS_PER_REQUEST; +use iggy_binary_protocol::WireName; +use iggy_binary_protocol::codec::WireEncode; +use iggy_binary_protocol::requests::topics::CreateTopicRequest; +use iggy_binary_protocol::responses::streams::get_stream::TopicHeader; +use iggy_binary_protocol::responses::topics::get_topic::PartitionResponse; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::{debug, instrument}; + +#[instrument(skip_all, name = "trace_create_topic", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] +pub async fn handle_create_topic( + req: CreateTopicRequest, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + debug!( + "session: {session}, command: create_topic, stream_id: {:?}, name: {}", + req.stream_id, + req.name.as_str() + ); + shard.ensure_authenticated(session)?; + + if req.partitions_count > MAX_PARTITIONS_PER_REQUEST { + return Err(IggyError::TooManyPartitions); + } + + let request = ShardRequest::control_plane(ShardRequestPayload::CreateTopicRequest { + user_id: session.get_user_id(), + command: req, + }); + + match shard.send_to_control_plane(request).await? { + ShardResponse::CreateTopicResponse(data) => { + let header = TopicHeader { + id: data.id, + created_at: data.created_at.into(), + partitions_count: data.partitions.len() as u32, + message_expiry: data.message_expiry.into(), + compression_algorithm: data.compression_algorithm.as_code(), + max_topic_size: data.max_topic_size.into(), + replication_factor: data.replication_factor, + size_bytes: 0, + messages_count: 0, + name: WireName::new(data.name.as_ref()).map_err(|_| IggyError::InvalidCommand)?, + }; + let partitions: Vec = data + .partitions + .iter() + .map(|p| PartitionResponse { + id: p.id as u32, + created_at: p.created_at.into(), + segments_count: 0, + current_offset: 0, + size_bytes: 0, + messages_count: 0, + }) + .collect(); + + let mut buf = BytesMut::with_capacity( + header.encoded_size() + + partitions + .iter() + .map(WireEncode::encoded_size) + .sum::(), + ); + header.encode(&mut buf); + for partition in &partitions { + partition.encode(&mut buf); + } + sender.send_ok_response(&buf.freeze()).await?; + } + ShardResponse::ErrorResponse(err) => return Err(err), + _ => unreachable!("Expected CreateTopicResponse"), + } + + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/topics/delete_topic_handler.rs b/core/server/src/binary/handlers/topics/delete_topic_handler.rs new file mode 100644 index 0000000000..d66dfbf010 --- /dev/null +++ b/core/server/src/binary/handlers/topics/delete_topic_handler.rs @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::HandlerResult; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; +use crate::streaming::session::Session; +use iggy_binary_protocol::requests::topics::DeleteTopicRequest; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::{debug, instrument}; + +#[instrument(skip_all, name = "trace_delete_topic", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] +pub async fn handle_delete_topic( + req: DeleteTopicRequest, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + debug!( + "session: {session}, command: delete_topic, stream_id: {:?}, topic_id: {:?}", + req.stream_id, req.topic_id + ); + shard.ensure_authenticated(session)?; + + let request = ShardRequest::control_plane(ShardRequestPayload::DeleteTopicRequest { + user_id: session.get_user_id(), + command: req, + }); + + match shard.send_to_control_plane(request).await? { + ShardResponse::DeleteTopicResponse => { + sender.send_empty_ok_response().await?; + } + ShardResponse::ErrorResponse(err) => return Err(err), + _ => unreachable!("Expected DeleteTopicResponse"), + } + + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/topics/get_topic_handler.rs b/core/server/src/binary/handlers/topics/get_topic_handler.rs new file mode 100644 index 0000000000..7632d25065 --- /dev/null +++ b/core/server/src/binary/handlers/topics/get_topic_handler.rs @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::{HandlerResult, wire_id_to_identifier}; +use crate::binary::handlers::streams::get_stream_handler::build_topic_header; +use crate::metadata::TopicMeta; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::streaming::session::Session; +use iggy_binary_protocol::codec::WireEncode; +use iggy_binary_protocol::requests::topics::GetTopicRequest; +use iggy_binary_protocol::responses::topics::get_topic::{GetTopicResponse, PartitionResponse}; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::debug; + +pub async fn handle_get_topic( + req: GetTopicRequest, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + let stream_id = wire_id_to_identifier(&req.stream_id)?; + let topic_id = wire_id_to_identifier(&req.topic_id)?; + debug!("session: {session}, command: get_topic, stream_id: {stream_id}, topic_id: {topic_id}"); + shard.ensure_authenticated(session)?; + + let Some(topic) = shard + .metadata + .query_topic(session.get_user_id(), &stream_id, &topic_id)? + else { + sender.send_empty_ok_response().await?; + return Ok(HandlerResult::Finished); + }; + + let response = build_get_topic_response(&topic)?; + sender.send_ok_response(&response.to_bytes()).await?; + Ok(HandlerResult::Finished) +} + +fn build_get_topic_response(topic: &TopicMeta) -> Result { + let header = build_topic_header(topic)?; + + let partitions: Vec = topic + .partitions + .iter() + .enumerate() + .map(|(partition_id, partition)| PartitionResponse { + id: partition_id as u32, + created_at: partition.created_at.into(), + segments_count: partition.stats.segments_count_inconsistent(), + current_offset: partition.stats.current_offset(), + size_bytes: partition.stats.size_bytes_inconsistent(), + messages_count: partition.stats.messages_count_inconsistent(), + }) + .collect(); + + Ok(GetTopicResponse { + topic: header, + partitions, + }) +} diff --git a/core/server/src/binary/handlers/topics/get_topics_handler.rs b/core/server/src/binary/handlers/topics/get_topics_handler.rs new file mode 100644 index 0000000000..32434ef485 --- /dev/null +++ b/core/server/src/binary/handlers/topics/get_topics_handler.rs @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::{HandlerResult, wire_id_to_identifier}; +use crate::binary::handlers::streams::get_stream_handler::build_topic_header; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::streaming::session::Session; +use iggy_binary_protocol::codec::WireEncode; +use iggy_binary_protocol::requests::topics::GetTopicsRequest; +use iggy_binary_protocol::responses::topics::get_topics::GetTopicsResponse; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::debug; + +pub async fn handle_get_topics( + req: GetTopicsRequest, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + let stream_id = wire_id_to_identifier(&req.stream_id)?; + debug!("session: {session}, command: get_topics, stream_id: {stream_id}"); + shard.ensure_authenticated(session)?; + + let Some(topics) = shard + .metadata + .query_topics(session.get_user_id(), &stream_id)? + else { + sender.send_empty_ok_response().await?; + return Ok(HandlerResult::Finished); + }; + + let mut sorted: Vec<_> = topics.iter().collect(); + sorted.sort_by_key(|t| t.id); + + let mut wire_topics = Vec::with_capacity(sorted.len()); + for topic in sorted { + wire_topics.push(build_topic_header(topic)?); + } + + let response = GetTopicsResponse { + topics: wire_topics, + }; + sender.send_ok_response(&response.to_bytes()).await?; + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/topics/mod.rs b/core/server/src/binary/handlers/topics/mod.rs new file mode 100644 index 0000000000..023a96e103 --- /dev/null +++ b/core/server/src/binary/handlers/topics/mod.rs @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod create_topic_handler; +pub mod delete_topic_handler; +pub mod get_topic_handler; +pub mod get_topics_handler; +pub mod purge_topic_handler; +pub mod update_topic_handler; + +pub const COMPONENT: &str = "TOPIC_HANDLER"; diff --git a/core/server/src/binary/handlers/topics/purge_topic_handler.rs b/core/server/src/binary/handlers/topics/purge_topic_handler.rs new file mode 100644 index 0000000000..8b2171e272 --- /dev/null +++ b/core/server/src/binary/handlers/topics/purge_topic_handler.rs @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::HandlerResult; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; +use crate::streaming::session::Session; +use iggy_binary_protocol::requests::topics::PurgeTopicRequest; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::{debug, instrument}; + +#[instrument(skip_all, name = "trace_purge_topic", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] +pub async fn handle_purge_topic( + req: PurgeTopicRequest, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + debug!( + "session: {session}, command: purge_topic, stream_id: {:?}, topic_id: {:?}", + req.stream_id, req.topic_id + ); + shard.ensure_authenticated(session)?; + + let request = ShardRequest::control_plane(ShardRequestPayload::PurgeTopicRequest { + user_id: session.get_user_id(), + command: req, + }); + + match shard.send_to_control_plane(request).await? { + ShardResponse::PurgeTopicResponse => { + sender.send_empty_ok_response().await?; + } + ShardResponse::ErrorResponse(err) => return Err(err), + _ => unreachable!("Expected PurgeTopicResponse"), + } + + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/topics/update_topic_handler.rs b/core/server/src/binary/handlers/topics/update_topic_handler.rs new file mode 100644 index 0000000000..4559fe3469 --- /dev/null +++ b/core/server/src/binary/handlers/topics/update_topic_handler.rs @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::HandlerResult; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; +use crate::streaming::session::Session; +use iggy_binary_protocol::requests::topics::UpdateTopicRequest; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::{debug, instrument}; + +#[instrument(skip_all, name = "trace_update_topic", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] +pub async fn handle_update_topic( + req: UpdateTopicRequest, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + debug!( + "session: {session}, command: update_topic, stream_id: {:?}, topic_id: {:?}", + req.stream_id, req.topic_id + ); + shard.ensure_authenticated(session)?; + + let request = ShardRequest::control_plane(ShardRequestPayload::UpdateTopicRequest { + user_id: session.get_user_id(), + command: req, + }); + + match shard.send_to_control_plane(request).await? { + ShardResponse::UpdateTopicResponse => { + sender.send_empty_ok_response().await?; + } + ShardResponse::ErrorResponse(err) => return Err(err), + _ => unreachable!("Expected UpdateTopicResponse"), + } + + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/users/change_password_handler.rs b/core/server/src/binary/handlers/users/change_password_handler.rs new file mode 100644 index 0000000000..c2799f6a27 --- /dev/null +++ b/core/server/src/binary/handlers/users/change_password_handler.rs @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::HandlerResult; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; +use crate::streaming::session::Session; +use iggy_binary_protocol::requests::users::ChangePasswordRequest; +use iggy_common::IggyError; +use iggy_common::defaults::{MAX_PASSWORD_LENGTH, MIN_PASSWORD_LENGTH}; +use std::rc::Rc; +use tracing::{debug, instrument}; + +#[instrument(skip_all, name = "trace_change_password", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] +pub async fn handle_change_password( + req: ChangePasswordRequest, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + debug!( + "session: {session}, command: change_password, user_id: {:?}", + req.user_id + ); + shard.ensure_authenticated(session)?; + + let current_len = req.current_password.len(); + if !(MIN_PASSWORD_LENGTH..=MAX_PASSWORD_LENGTH).contains(¤t_len) { + return Err(IggyError::InvalidPassword); + } + let new_len = req.new_password.len(); + if !(MIN_PASSWORD_LENGTH..=MAX_PASSWORD_LENGTH).contains(&new_len) { + return Err(IggyError::InvalidPassword); + } + + let request = ShardRequest::control_plane(ShardRequestPayload::ChangePasswordRequest { + user_id: session.get_user_id(), + command: req, + }); + + match shard.send_to_control_plane(request).await? { + ShardResponse::ChangePasswordResponse => { + sender.send_empty_ok_response().await?; + } + ShardResponse::ErrorResponse(err) => return Err(err), + _ => unreachable!("Expected ChangePasswordResponse"), + } + + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/users/create_user_handler.rs b/core/server/src/binary/handlers/users/create_user_handler.rs new file mode 100644 index 0000000000..d76ef0504e --- /dev/null +++ b/core/server/src/binary/handlers/users/create_user_handler.rs @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::HandlerResult; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; +use crate::streaming::session::Session; +use iggy_binary_protocol::WireName; +use iggy_binary_protocol::codec::WireEncode; +use iggy_binary_protocol::requests::users::CreateUserRequest; +use iggy_binary_protocol::responses::users::{UserDetailsResponse, UserResponse}; +use iggy_common::IggyError; +use iggy_common::defaults::{ + MAX_PASSWORD_LENGTH, MAX_USERNAME_LENGTH, MIN_PASSWORD_LENGTH, MIN_USERNAME_LENGTH, +}; +use iggy_common::wire_conversions::permissions_to_wire; +use std::rc::Rc; +use tracing::{debug, instrument}; + +#[instrument(skip_all, name = "trace_create_user", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] +pub async fn handle_create_user( + req: CreateUserRequest, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + debug!( + "session: {session}, command: create_user, username: {}", + req.username.as_str() + ); + shard.ensure_authenticated(session)?; + shard.metadata.perm_create_user(session.get_user_id())?; + + let username_len = req.username.as_str().len(); + if !(MIN_USERNAME_LENGTH..=MAX_USERNAME_LENGTH).contains(&username_len) { + return Err(IggyError::InvalidUsername); + } + let password_len = req.password.len(); + if !(MIN_PASSWORD_LENGTH..=MAX_PASSWORD_LENGTH).contains(&password_len) { + return Err(IggyError::InvalidPassword); + } + + let request = ShardRequest::control_plane(ShardRequestPayload::CreateUserRequest { + user_id: session.get_user_id(), + command: req, + }); + + match shard.send_to_control_plane(request).await? { + ShardResponse::CreateUserResponse(user) => { + let response = UserDetailsResponse { + user: UserResponse { + id: user.id, + created_at: user.created_at.as_micros(), + status: user.status.as_code(), + username: WireName::new(&user.username) + .map_err(|_| IggyError::InvalidCommand)?, + }, + permissions: user.permissions.as_ref().map(permissions_to_wire), + }; + sender.send_ok_response(&response.to_bytes()).await?; + } + ShardResponse::ErrorResponse(err) => return Err(err), + _ => unreachable!("Expected CreateUserResponse"), + } + + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/users/delete_user_handler.rs b/core/server/src/binary/handlers/users/delete_user_handler.rs new file mode 100644 index 0000000000..e7dce1c9d7 --- /dev/null +++ b/core/server/src/binary/handlers/users/delete_user_handler.rs @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::HandlerResult; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; +use crate::streaming::session::Session; +use iggy_binary_protocol::requests::users::DeleteUserRequest; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::{debug, instrument}; + +#[instrument(skip_all, name = "trace_delete_user", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] +pub async fn handle_delete_user( + req: DeleteUserRequest, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + debug!( + "session: {session}, command: delete_user, user_id: {:?}", + req.user_id + ); + shard.ensure_authenticated(session)?; + shard.metadata.perm_delete_user(session.get_user_id())?; + + let request = ShardRequest::control_plane(ShardRequestPayload::DeleteUserRequest { + user_id: session.get_user_id(), + command: req, + }); + + match shard.send_to_control_plane(request).await? { + ShardResponse::DeleteUserResponse(_) => { + sender.send_empty_ok_response().await?; + } + ShardResponse::ErrorResponse(err) => return Err(err), + _ => unreachable!("Expected DeleteUserResponse"), + } + + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/users/get_user_handler.rs b/core/server/src/binary/handlers/users/get_user_handler.rs new file mode 100644 index 0000000000..8e243d93ad --- /dev/null +++ b/core/server/src/binary/handlers/users/get_user_handler.rs @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::{HandlerResult, wire_id_to_identifier}; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::streaming::session::Session; +use iggy_binary_protocol::WireName; +use iggy_binary_protocol::codec::WireEncode; +use iggy_binary_protocol::requests::users::GetUserRequest; +use iggy_binary_protocol::responses::users::{UserDetailsResponse, UserResponse}; +use iggy_common::IggyError; +use iggy_common::wire_conversions::permissions_to_wire; +use std::rc::Rc; +use tracing::debug; + +pub async fn handle_get_user( + req: GetUserRequest, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + debug!( + "session: {session}, command: get_user, user_id: {:?}", + req.user_id + ); + shard.ensure_authenticated(session)?; + + let user_id = wire_id_to_identifier(&req.user_id)?; + + let Some(user) = shard.metadata.query_user(session.get_user_id(), &user_id)? else { + sender.send_empty_ok_response().await?; + return Ok(HandlerResult::Finished); + }; + + let response = UserDetailsResponse { + user: UserResponse { + id: user.id, + created_at: user.created_at.as_micros(), + status: user.status.as_code(), + username: WireName::new(user.username.as_ref()) + .map_err(|_| IggyError::InvalidCommand)?, + }, + permissions: user.permissions.as_deref().map(permissions_to_wire), + }; + sender.send_ok_response(&response.to_bytes()).await?; + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/users/get_users_handler.rs b/core/server/src/binary/handlers/users/get_users_handler.rs new file mode 100644 index 0000000000..99fa31372f --- /dev/null +++ b/core/server/src/binary/handlers/users/get_users_handler.rs @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::HandlerResult; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::streaming::session::Session; +use iggy_binary_protocol::WireName; +use iggy_binary_protocol::codec::WireEncode; +use iggy_binary_protocol::responses::users::{GetUsersResponse, UserResponse}; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::debug; + +pub async fn handle_get_users( + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + debug!("session: {session}, command: get_users"); + shard.ensure_authenticated(session)?; + + let users = shard.metadata.query_users(session.get_user_id())?; + let wire_users: Vec = users + .iter() + .map(|u| { + Ok(UserResponse { + id: u.id, + created_at: u.created_at.as_micros(), + status: u.status.as_code(), + username: WireName::new(u.username.as_ref()) + .map_err(|_| IggyError::InvalidCommand)?, + }) + }) + .collect::>()?; + let response = GetUsersResponse { users: wire_users }; + sender.send_ok_response(&response.to_bytes()).await?; + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/users/login_user_handler.rs b/core/server/src/binary/handlers/users/login_user_handler.rs new file mode 100644 index 0000000000..22d0943f84 --- /dev/null +++ b/core/server/src/binary/handlers/users/login_user_handler.rs @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::HandlerResult; +use crate::binary::handlers::users::COMPONENT; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::streaming::session::Session; +use err_trail::ErrContext; +use iggy_binary_protocol::codec::WireEncode; +use iggy_binary_protocol::requests::users::LoginUserRequest; +use iggy_binary_protocol::responses::users::IdentityResponse; +use iggy_common::IggyError; +use iggy_common::defaults::{ + MAX_PASSWORD_LENGTH, MAX_USERNAME_LENGTH, MIN_PASSWORD_LENGTH, MIN_USERNAME_LENGTH, +}; +use std::rc::Rc; +use tracing::{debug, info, instrument, warn}; + +#[instrument(skip_all, name = "trace_login_user", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] +pub async fn handle_login_user( + req: LoginUserRequest, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + if shard.is_shutting_down() { + warn!("Rejecting login request during shutdown"); + return Err(IggyError::Disconnected); + } + + let username = req.username.as_str(); + let username_len = username.len(); + if !(MIN_USERNAME_LENGTH..=MAX_USERNAME_LENGTH).contains(&username_len) { + return Err(IggyError::InvalidUsername); + } + let password_len = req.password.len(); + if !(MIN_PASSWORD_LENGTH..=MAX_PASSWORD_LENGTH).contains(&password_len) { + return Err(IggyError::InvalidPassword); + } + + debug!("session: {session}, command: login_user, username: {username}"); + + info!("Logging in user: {username} ..."); + let user = shard + .login_user(username, &req.password, Some(session)) + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to login user with name: {username}, session: {session}", + ) + })?; + info!("Logged in user: {username} with ID: {}.", user.id); + + let response = IdentityResponse { user_id: user.id }; + sender.send_ok_response(&response.to_bytes()).await?; + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/users/logout_user_handler.rs b/core/server/src/binary/handlers/users/logout_user_handler.rs new file mode 100644 index 0000000000..8113f7bc7d --- /dev/null +++ b/core/server/src/binary/handlers/users/logout_user_handler.rs @@ -0,0 +1,44 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::HandlerResult; +use crate::binary::handlers::users::COMPONENT; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::streaming::session::Session; +use err_trail::ErrContext; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::{debug, info, instrument}; + +#[instrument(skip_all, name = "trace_logout_user", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] +pub async fn handle_logout_user( + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + debug!("session: {session}, command: logout_user"); + shard.ensure_authenticated(session)?; + info!("Logging out user with ID: {}...", session.get_user_id()); + shard.logout_user(session).error(|e: &IggyError| { + format!("{COMPONENT} (error: {e}) - failed to logout user, session: {session}") + })?; + info!("Logged out user with ID: {}.", session.get_user_id()); + session.clear_user_id(); + sender.send_empty_ok_response().await?; + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/users/mod.rs b/core/server/src/binary/handlers/users/mod.rs new file mode 100644 index 0000000000..df933d49a2 --- /dev/null +++ b/core/server/src/binary/handlers/users/mod.rs @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod change_password_handler; +pub mod create_user_handler; +pub mod delete_user_handler; +pub mod get_user_handler; +pub mod get_users_handler; +pub mod login_user_handler; +pub mod logout_user_handler; +pub mod update_permissions_handler; +pub mod update_user_handler; + +pub const COMPONENT: &str = "USER_HANDLER"; diff --git a/core/server/src/binary/handlers/users/update_permissions_handler.rs b/core/server/src/binary/handlers/users/update_permissions_handler.rs new file mode 100644 index 0000000000..74426704ef --- /dev/null +++ b/core/server/src/binary/handlers/users/update_permissions_handler.rs @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::HandlerResult; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; +use crate::streaming::session::Session; +use iggy_binary_protocol::requests::users::UpdatePermissionsRequest; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::{debug, instrument}; + +#[instrument(skip_all, name = "trace_update_permissions", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] +pub async fn handle_update_permissions( + req: UpdatePermissionsRequest, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + debug!( + "session: {session}, command: update_permissions, user_id: {:?}", + req.user_id + ); + shard.ensure_authenticated(session)?; + shard + .metadata + .perm_update_permissions(session.get_user_id())?; + + let request = ShardRequest::control_plane(ShardRequestPayload::UpdatePermissionsRequest { + user_id: session.get_user_id(), + command: req, + }); + + match shard.send_to_control_plane(request).await? { + ShardResponse::UpdatePermissionsResponse => { + sender.send_empty_ok_response().await?; + } + ShardResponse::ErrorResponse(err) => return Err(err), + _ => unreachable!("Expected UpdatePermissionsResponse"), + } + + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/handlers/users/update_user_handler.rs b/core/server/src/binary/handlers/users/update_user_handler.rs new file mode 100644 index 0000000000..96d655fbe7 --- /dev/null +++ b/core/server/src/binary/handlers/users/update_user_handler.rs @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::HandlerResult; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; +use crate::streaming::session::Session; +use iggy_binary_protocol::requests::users::UpdateUserRequest; +use iggy_common::IggyError; +use iggy_common::defaults::{MAX_USERNAME_LENGTH, MIN_USERNAME_LENGTH}; +use std::rc::Rc; +use tracing::{debug, instrument}; + +#[instrument(skip_all, name = "trace_update_user", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] +pub async fn handle_update_user( + req: UpdateUserRequest, + sender: &mut SenderKind, + session: &Session, + shard: &Rc, +) -> Result { + debug!( + "session: {session}, command: update_user, user_id: {:?}", + req.user_id + ); + shard.ensure_authenticated(session)?; + shard.metadata.perm_update_user(session.get_user_id())?; + + if let Some(ref username) = req.username { + let username_len = username.as_str().len(); + if !(MIN_USERNAME_LENGTH..=MAX_USERNAME_LENGTH).contains(&username_len) { + return Err(IggyError::InvalidUsername); + } + } + + let request = ShardRequest::control_plane(ShardRequestPayload::UpdateUserRequest { + user_id: session.get_user_id(), + command: req, + }); + + match shard.send_to_control_plane(request).await? { + ShardResponse::UpdateUserResponse(_) => { + sender.send_empty_ok_response().await?; + } + ShardResponse::ErrorResponse(err) => return Err(err), + _ => unreachable!("Expected UpdateUserResponse"), + } + + Ok(HandlerResult::Finished) +} diff --git a/core/server/src/binary/mod.rs b/core/server/src/binary/mod.rs new file mode 100644 index 0000000000..8f9f22b20b --- /dev/null +++ b/core/server/src/binary/mod.rs @@ -0,0 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod dispatch; +pub mod handlers; + +pub const COMPONENT: &str = "BINARY"; diff --git a/core/server/src/bootstrap.rs b/core/server/src/bootstrap.rs index 15e290532f..eb42c0539e 100644 --- a/core/server/src/bootstrap.rs +++ b/core/server/src/bootstrap.rs @@ -15,4597 +15,602 @@ // specific language governing permissions and limitations // under the License. -use crate::auth::warm_dummy_password_hash; -use crate::cluster_meta::ClusterRoster; -use crate::config_writer::write_current_config; -use crate::dispatch::{ - make_client_request_handler, make_deferred_client_request_handler, - make_deferred_replica_message_handler, make_list_clients_handler, make_metadata_submit_handler, - make_partition_read_handler, -}; -use crate::http; -use crate::partition_helpers::{ - build_partition_fresh, configure_consumer_offsets, ensure_initial_segment, - open_partition_superblock, restore_partition_view, validate_namespace_bounds, -}; -use crate::segment_recovery::{RecoveredSegment, load_persisted_segments}; -use crate::server_error::{ServerError, ShardJoinFailure, ShardJoinFailureKind}; -use crate::session_manager::SessionManager; -use configs::server::{ServerConfig, ServerSystemConfig}; -use configs::sharding::{ - INBOX_CAPACITY_MAX, SHUTDOWN_DRAIN_TIMEOUT_MAX, SHUTDOWN_POLL_INTERVAL_MAX, -}; -use consensus::{ - ClientTable, LocalPipeline, MetadataHandle, PartitionsHandle, PipelineEntry, Sequencer, - VsrConsensus, -}; -// `try_send` / `try_recv` resolve through these traits on `MAsyncTx` / -// `MAsyncRx`; the metadata-handoff loops below depend on the -// non-blocking variants for cancel-safe shutdown polling. -use consensus::VsrState; -use crossfire::{AsyncRxTrait, AsyncTxTrait}; -use iggy_binary_protocol::{Operation, PrepareHeader}; -use iggy_common::defaults::{ - DEFAULT_ROOT_PASSWORD, DEFAULT_ROOT_USERNAME, MAX_PASSWORD_LENGTH, MAX_USERNAME_LENGTH, - MIN_PASSWORD_LENGTH, MIN_USERNAME_LENGTH, -}; -use iggy_common::{Aes256GcmEncryptor, EncryptorKind, IggyByteSize, PartitionStats, variadic}; -use journal::prepare_journal::PrepareJournal; -use journal::superblock::{PingPongSuperblock, SuperblockStore}; -use journal::{Journal, JournalHandle}; -use message_bus::client_listener::{self, RequestHandler}; -use message_bus::installer; -use message_bus::installer::conn_info::{ClientConnMeta, ClientTransportKind}; -use message_bus::replica::auth::{self, ReplicaAuth}; -use message_bus::replica::handshake::{ReplicaHandshakeCtx, ReplicaTlsCtx}; -use message_bus::replica::io as replica_io; -use message_bus::replica::listener::{self as replica_listener, MessageHandler}; -use message_bus::transports::quic::server_config_with_cert; -use message_bus::transports::tls::{ - AcceptAnyServerCert, REPLICA_ALPN, TlsServerCredentials, install_default_crypto_provider, - load_ca_pem, load_pem, self_signed_for_loopback, -}; -use message_bus::{ - AcceptedClientFn, AcceptedQuicClientFn, AcceptedReplicaFn, AcceptedTlsClientFn, - AcceptedWsClientFn, AcceptedWssClientFn, ConnectionInstaller, DialedReplicaFn, IggyMessageBus, - MAX_INFLIGHT_REPLICA_HANDSHAKES, MessageBus, ReplicaOwnerTable, connector, -}; -use metadata::IggyMetadata; -use metadata::MuxStateMachine; -use metadata::ReplicaIdentity; -use metadata::impls::metadata::{IggySnapshot, StreamsFrontend}; -use metadata::impls::recovery::recover; -use metadata::stm::mux::WithFactory; -use metadata::stm::snapshot::Snapshot; -use metadata::stm::stream::{Partition, Streams}; -use metadata::stm::user::Users; -use partitions::{ - IggyIndexWriter, IggyPartition, IggyPartitions, MessagesWriter, PartitionsConfig, +use crate::{ + IGGY_ROOT_PASSWORD_ENV, IGGY_ROOT_USERNAME_ENV, + compat::index_rebuilding::index_rebuilder::IndexRebuilder, + configs::{ + cache_indexes::CacheIndexesConfig, + server::ServerConfig, + system::{INDEX_EXTENSION, LOG_EXTENSION, SystemConfig}, + }, + io::fs_utils::{self, DirEntry}, + metadata::{ConsumerGroupMeta, MetadataWriter, PartitionMeta, StreamMeta, TopicMeta, UserMeta}, + server_error::ServerError, + shard::{ + system::info::SystemInfo, + transmission::{ + connector::{ShardConnector, StopSender}, + frame::ShardFrame, + }, + }, + state::system::{StreamState, TopicState, UserState}, + streaming::{ + partitions::{ + consumer_group_offsets::ConsumerGroupOffsets, consumer_offsets::ConsumerOffsets, + journal::MemoryMessageJournal, log::SegmentedLog, + }, + persistence::persister::{FilePersister, FileWithSyncPersister, PersisterKind}, + segments::{Segment, storage::Storage}, + stats::{PartitionStats, StreamStats, TopicStats}, + storage::SystemStorage, + users::user::User, + utils::crypto, + }, }; -use rustls::pki_types::ServerName; -use server_common::Message; -use server_common::bootstrap::create_directories; -use server_common::crypto; -use server_common::executor::create_shard_executor; -use server_common::fs_utils::remove_dir_all; -use server_common::log::{Logging, LoggingSettings, TelemetrySettings}; -use server_common::sharding::{IggyNamespace, PartitionLocation, ShardId}; -use shard::builder::IggyShardBuilder; -use shard::metrics::{ShardMetrics, frame_drop_reason, frame_drop_variant}; -use shard::shards_table::{PapayaShardsTable, ShardsTable, calculate_shard_assignment}; -use shard::{ - CoordinatorConfig, IggyShard, LifecycleFrame, ListClientsHandler, MetadataSubmitHandler, - PartitionConsensusConfig, PartitionReadHandler, Receiver as ShardReceiver, ShardFrame, - ShardIdentity, TaggedSender, channel, shard_mesh_channels, +use err_trail::ErrContext; +use iggy_common::SemanticVersion; +use iggy_common::{ + IggyByteSize, IggyError, PersonalAccessToken, + defaults::{ + DEFAULT_ROOT_USERNAME, MAX_PASSWORD_LENGTH, MAX_USERNAME_LENGTH, MIN_PASSWORD_LENGTH, + MIN_USERNAME_LENGTH, + }, }; -use shard_allocator::{ShardAllocator, ShardInfo}; -use std::cell::RefCell; -use std::collections::HashMap; -use std::env; -use std::net::{IpAddr, SocketAddr}; -use std::path::{Path, PathBuf}; -use std::rc::{Rc, Weak}; -use std::sync::Arc; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::thread; -use std::time::{Duration, Instant}; -use tracing::{error, info, warn}; - -const SHARD_REPLICA_ID: u8 = 0; - -pub const IGGY_ROOT_USERNAME_ENV: &str = "IGGY_ROOT_USERNAME"; -pub const IGGY_ROOT_PASSWORD_ENV: &str = "IGGY_ROOT_PASSWORD"; - -type ServerMuxStateMachine = MuxStateMachine; - -/// Cross-thread bundle carrying one `ReadHandleFactory` per metadata -/// state. Shard 0 mints one after `recover()` and broadcasts a clone to -/// every peer shard; each peer rebuilds a reader-mode -/// [`ServerMuxStateMachine`] on its own runtime, skipping the WAL. -type ServerMetadataBundle = ::Bundle; - -pub(crate) type ServerMetadata = IggyMetadata< - VsrConsensus>, - PrepareJournal, - IggySnapshot, - ServerMuxStateMachine, ->; - -/// The shard type the dispatch layer is generic over. -/// -/// `B`/`MJ`/`S`/`SB` are free; the metadata state machine (`M`) and shards -/// table (`T`) are pinned, being identical in production and the simulator. -/// Production instantiates it as [`ServerShard`], defaulting `SB` to the -/// on-disk [`PingPongSuperblock`]; the simulator supplies its own -/// `B`/`MJ`/`S`/`SB`. -pub type ShellShard = - IggyShard; - -/// Late-bound self-reference the deferred dispatch handlers upgrade per frame. -pub type ShellShardHandle = - Rc>>>>; - -/// Bus bounds the dispatch/pump path needs (matches `run_message_pump`). -/// Blanket-impl'd, so it is only shorthand for the four underlying bounds. -pub trait ShellBus: MessageBus + ConnectionInstaller + Clone + 'static {} -impl ShellBus for B {} - -/// The five dispatch handlers a shard is built with, plus the -/// [`SessionManager`] the request-plane pair shares. -/// -/// Both production ([`build_shard_for_thread`]) and the simulator's shell -/// mode construct these through [`wire_shell_handlers`], so the request -/// plane is wired one way. The simulator's shell-off fast path uses -/// [`ShellHandlers::noop`] instead. -pub struct ShellHandlers { - pub on_replica_message: MessageHandler, - pub on_client_request: RequestHandler, - pub on_metadata_submit: MetadataSubmitHandler, - pub on_list_clients: ListClientsHandler, - pub on_partition_read: PartitionReadHandler, - /// Bound by the client-request handler, read by the get-clients - /// handler; the caller keeps it to reach locally-homed sessions. - pub sessions: Rc>, -} +use shard_allocator::ShardInfo; +use slab::Slab; +use std::{env, sync::Arc}; +use tracing::{info, warn}; + +pub fn create_shard_connections( + shard_assignment: &[ShardInfo], +) -> (Vec>, Vec<(u16, StopSender)>) { + // Create connectors with sequential IDs (0, 1, 2, ...) regardless of CPU core numbers + let connectors: Vec> = shard_assignment + .iter() + .enumerate() + .map(|(idx, _assignment)| { + // let cpu_id = assignment.cpu_set.iter().next().unwrap_or(&idx); + ShardConnector::new(idx as u16) + }) + .collect(); -impl ShellHandlers { - /// Inert handlers for the shell-off fast path: every callback is a - /// no-op over an empty [`SessionManager`]. Behaviorally identical to - /// hand-written no-op closures, so a caller can keep one destructure - /// site across both toggle states. - #[must_use] - pub fn noop() -> Self { - Self { - on_replica_message: Rc::new(|_, _| {}), - on_client_request: Rc::new(|_, _| {}), - on_metadata_submit: Rc::new(|_| {}), - on_list_clients: Rc::new(|_| {}), - on_partition_read: Rc::new(|_, _, _| {}), - sessions: Rc::new(RefCell::new(SessionManager::new())), - } - } -} + let shutdown_handles = connectors + .iter() + .map(|conn| (conn.id, conn.stop_sender.clone())) + .collect(); -/// Build the deferred dispatch handlers for `shard_handle` against `bus`. -/// -/// They share one fresh [`SessionManager`]. The caller must set the weak -/// self-reference in `shard_handle` once the shard is built, so the -/// handlers can upgrade it per frame. -pub fn wire_shell_handlers( - bus: &B, - shard_handle: &ShellShardHandle, - system_config: Arc, - max_tokens_per_user: u32, -) -> ShellHandlers -where - B: ShellBus, - MJ: JournalHandle + 'static, - MJ::Target: Journal, Header = PrepareHeader>, - S: 'static, - SB: SuperblockStore + 'static, -{ - let sessions = Rc::new(RefCell::new(SessionManager::new())); - ShellHandlers { - on_replica_message: make_deferred_replica_message_handler(shard_handle), - on_client_request: make_deferred_client_request_handler( - bus, - shard_handle, - &sessions, - system_config, - max_tokens_per_user, - ), - on_metadata_submit: make_metadata_submit_handler(shard_handle), - on_list_clients: make_list_clients_handler(&sessions), - on_partition_read: make_partition_read_handler(shard_handle), - sessions, - } + (connectors, shutdown_handles) } -pub type ServerShard = ShellShard, PrepareJournal, IggySnapshot>; - -/// Result of a multi-shard bootstrap. -/// -/// Carries the cross-thread shutdown flag and one OS-thread `JoinHandle` -/// per shard. The caller flips the flag via [`Self::install_ctrlc_handler`] -/// and then drains every shard via [`Self::join_all`], bounded by -/// `join_timeout` (`system.sharding.shutdown_join_timeout`). -pub struct ShardHandles { - shutdown_flag: Arc, - shard_threads: Vec<(u16, thread::JoinHandle>)>, - join_timeout: Duration, +pub async fn load_config() -> Result { + let config = ServerConfig::load().await?; + Ok(config) } -impl ShardHandles { - /// Install a SIGINT/Ctrl-C handler that flips the shutdown flag on - /// the first signal. A second signal is logged but otherwise - /// ignored so an in-flight WAL fsync or replica drain runs to - /// completion. - /// - /// # Errors - /// - /// Returns the underlying `ctrlc::Error` if the handler cannot be - /// installed (typically because another handler already owns the - /// signal). - pub fn install_ctrlc_handler(&self) -> Result<(), ctrlc::Error> { - let flag = Arc::clone(&self.shutdown_flag); - ctrlc::set_handler(move || { - if flag.swap(true, Ordering::Relaxed) { - // Second Ctrl-C: leave the shutdown machinery to drain. - // Refusing to abort here keeps the WAL fsync / replica - // drain from being interrupted mid-frame. - warn!("second Ctrl-C ignored; server is already shutting down"); - } else { - info!("Ctrl-C received; signalling server shutdown"); - } - }) - } +pub fn create_root_user() -> User { + let mut username = env::var(IGGY_ROOT_USERNAME_ENV); + let mut password = env::var(IGGY_ROOT_PASSWORD_ENV); + assert_eq!( + username.is_ok(), + password.is_ok(), + "When providing the custom root user credentials, both username and password must be set." + ); + if username.is_ok() && password.is_ok() { + info!("Using the custom root user credentials."); + } else { + info!("Using the default root user credentials..."); + username = Ok(DEFAULT_ROOT_USERNAME.to_string()); + let generated_password = crypto::generate_secret(20..40); + println!("Generated root user password: {generated_password}"); + password = Ok(generated_password); + } + + let username = username.expect("Root username is not set."); + let password = password.expect("Root password is not set."); + assert!( + !username.is_empty() && !password.is_empty(), + "Root user credentials cannot be empty." + ); + assert!( + username.len() >= MIN_USERNAME_LENGTH, + "Root username is too short." + ); + assert!( + username.len() <= MAX_USERNAME_LENGTH, + "Root username is too long." + ); + assert!( + password.len() >= MIN_PASSWORD_LENGTH, + "Root password is too short." + ); + assert!( + password.len() <= MAX_PASSWORD_LENGTH, + "Root password is too long." + ); - /// Drain every shard thread. This is the main thread's park for the - /// server's whole lifetime, so shards are awaited WITHOUT any time - /// bound while the server runs; the `shutdown_join_timeout` clock - /// only starts once the cross-thread shutdown flag flips (Ctrl-C or - /// a shard failure). Each shard's outcome is logged (`info` on clean - /// exit, `error` on Err, panic, or wedge). If any shard failed, - /// returns every failure together as - /// [`ServerError::ShardJoinFailures`] so the operator sees the - /// full set rather than just the first. - /// - /// A shard whose thread is still running when the post-shutdown - /// deadline passes is abandoned (its `JoinHandle` dropped, the OS - /// thread left to die with the process) and reported as - /// [`ShardJoinFailureKind::Wedged`]: a wedged pump or listener must - /// not block process exit forever. - /// - /// # Errors - /// - /// Returns [`ServerError::ShardJoinFailures`] if any shard - /// returned a `Result::Err`, panicked, or wedged past the deadline. - /// The variant carries every per-shard failure in shard-id order so - /// the caller does not need to read the trace log to discover - /// late-failing shards. - pub fn join_all(self) -> Result<(), ServerError> { - let mut failures: Vec = Vec::new(); - // Armed on the first poll that observes the shutdown flag, shared - // across all shards: one budget covers the whole drain, not one - // budget per shard. - let mut deadline: Option = None; - // Shards run thread-per-core with compio's blocking fallback pool - // disabled, so an io_uring opcode the kernel lacks aborts every shard - // with the same panic. Surface the actionable diagnostic once. - let mut io_uring_diagnostic_shown = false; - for (shard_id, handle) in self.shard_threads { - let Some(joined) = join_until_shutdown_deadline( - handle, - &self.shutdown_flag, - self.join_timeout, - &mut deadline, - ) else { - error!( - shard_id, - waited = ?self.join_timeout, - "shard thread still running at the shutdown join deadline; abandoning it" - ); - failures.push(ShardJoinFailure { - shard_id, - kind: ShardJoinFailureKind::Wedged { - waited: self.join_timeout, - }, - }); - continue; - }; - match joined { - Ok(Ok(())) => { - info!(shard_id, "shard thread exited cleanly"); - } - Ok(Err(error)) => { - error!(shard_id, error = %error, "shard thread returned error"); - failures.push(ShardJoinFailure { - shard_id, - kind: ShardJoinFailureKind::Error(Box::new(error)), - }); - } - Err(panic_payload) => { - let message = panic_payload_to_string(&*panic_payload); - error!(shard_id, message = %message, "shard thread panicked"); - if !io_uring_diagnostic_shown - && message - .contains(server_common::diagnostics::ASYNCIFY_POOL_DISABLED_PANIC_MSG) - { - server_common::diagnostics::print_incomplete_io_uring_ops_info(); - io_uring_diagnostic_shown = true; - } - failures.push(ShardJoinFailure { - shard_id, - kind: ShardJoinFailureKind::Panic { message }, - }); - } - } - } - if failures.is_empty() { - Ok(()) - } else { - Err(ServerError::ShardJoinFailures { failures }) - } - } + User::root(&username, &password) } -/// Poll cadence for the bounded shard joins. Coarse enough to cost -/// nothing during a normal drain, fine enough that exit latency past -/// the last shard's return stays imperceptible. -const JOIN_POLL_INTERVAL: Duration = Duration::from_millis(25); +pub use server_common::{create_directories, create_shard_executor}; -/// Join `handle`, waiting indefinitely while the server runs. The -/// `join_timeout` clock starts only when `shutdown_flag` is observed set -/// (arming the caller-shared `deadline` once, so all shards drain under -/// ONE budget); a running server parked here for hours must never be -/// mistaken for a wedged shard. `None` means the thread was still -/// running at the post-shutdown deadline and the handle was dropped -/// (the OS thread keeps running detached; process exit reaps it). -/// `JoinHandle` has no timed join, so this polls `is_finished` at -/// [`JOIN_POLL_INTERVAL`]; the closing `join()` on a finished thread -/// returns immediately. -fn join_until_shutdown_deadline( - handle: thread::JoinHandle>, - shutdown_flag: &AtomicBool, - join_timeout: Duration, - deadline: &mut Option, -) -> Option>> { - while !handle.is_finished() { - if deadline.is_none() && shutdown_flag.load(Ordering::Relaxed) { - *deadline = Some(Instant::now() + join_timeout); - } - if let Some(deadline) = deadline - && Instant::now() >= *deadline - { - return None; - } - thread::sleep(JOIN_POLL_INTERVAL); +pub fn resolve_persister(enforce_fsync: bool) -> Arc { + match enforce_fsync { + true => Arc::new(PersisterKind::FileWithSync(FileWithSyncPersister)), + false => Arc::new(PersisterKind::File(FilePersister)), } - Some(handle.join()) } -/// Best-effort extraction of the panic message from a -/// `Box` returned by `JoinHandle::join`. Tries the two -/// payload shapes the standard library guarantees (`&'static str` and -/// `String`) and falls back to a placeholder so the panic still surfaces -/// in the error chain. -fn panic_payload_to_string(payload: &(dyn std::any::Any + Send)) -> String { - if let Some(s) = payload.downcast_ref::<&'static str>() { - return (*s).to_string(); - } - if let Some(s) = payload.downcast_ref::() { - return s.clone(); - } - "".to_string() +pub async fn update_system_info( + storage: &SystemStorage, + system_info: &mut SystemInfo, + version: &SemanticVersion, +) -> Result<(), IggyError> { + system_info.update_version(version); + storage.info.save(system_info).await?; + Ok(()) } -/// Joins survivor shard threads after a partial-spawn failure, bounded -/// by the same `shutdown_join_timeout` budget as the normal exit path. -/// -/// Polls every survivor's `is_finished` in one loop instead of spawning -/// per-survivor joiner threads: the likely OS state on this path is -/// `pthread_create` EAGAIN (the parent spawn just failed with it), so -/// nothing here may create threads, and polling drains all survivors in -/// parallel anyway. A survivor still running at the deadline is -/// abandoned with an error log so the failed bootstrap can surface its -/// spawn error instead of hanging on a wedged shard. -fn join_partial_shard_survivors( - shard_threads: Vec<(u16, thread::JoinHandle>)>, - join_timeout: Duration, -) { - let deadline = Instant::now() + join_timeout; - let mut remaining = shard_threads; - loop { - let mut still_running = Vec::with_capacity(remaining.len()); - for (shard_id, survivor) in remaining { - if survivor.is_finished() { - let _ = survivor.join(); - info!(shard_id, "survivor shard thread drained"); - } else { - still_running.push((shard_id, survivor)); - } - } - remaining = still_running; - if remaining.is_empty() || Instant::now() >= deadline { - break; +async fn collect_log_files(partition_path: &str) -> Result, IggyError> { + let dir_entries = fs_utils::walk_dir(&partition_path) + .await + .map_err(|_| IggyError::CannotReadPartitions)?; + let mut log_files = Vec::new(); + for entry in dir_entries { + if entry.is_dir { + continue; } - thread::sleep(JOIN_POLL_INTERVAL); - } - for (shard_id, _survivor) in remaining { - error!( - shard_id, - waited = ?join_timeout, - "survivor shard thread still running at the shutdown join deadline; abandoning it" - ); - } -} - -/// Flips the cross-thread shutdown flag on `Drop` unless disarmed. -/// -/// A shard thread that exits via an error `?` or a panic unwind would -/// otherwise leave sibling shards parked forever on `bus.token().wait()`: -/// their watchdogs never observe the flag and the bus has no -/// `Drop`-triggered shutdown. Arming this for the whole thread body makes -/// every non-clean exit drive sibling-shard teardown. Disarmed only on a -/// clean `Ok(())`. -struct ShutdownOnDrop { - flag: Arc, - armed: bool, -} - -impl ShutdownOnDrop { - const fn new(flag: Arc) -> Self { - Self { flag, armed: true } - } - - const fn disarm(&mut self) { - self.armed = false; - } -} -impl Drop for ShutdownOnDrop { - fn drop(&mut self) { - if self.armed { - self.flag.store(true, Ordering::Relaxed); + let extension = entry.path.extension(); + if extension.is_none() || extension.unwrap() != LOG_EXTENSION { + continue; } - } -} - -/// Shard-local end of the metadata bundle handoff. -/// -/// Shard 0 owns the WAL writer and runs `recover()` to build the only -/// `WriteHandle`-bearing [`ServerMuxStateMachine`]. It then mints a -/// [`ServerMetadataBundle`] (a tuple of `Send + Sync` -/// `ReadHandleFactory`s) and pushes one clone per peer onto `bundle_tx`. -/// Every other shard receives the bundle and rebuilds a reader-mode -/// `MuxStateMachine` on its own runtime - no WAL access, no replay, no -/// `RecoverySync` two-phase fence. The old phase-2 WAL fence is gone -/// because peers no longer scan the WAL. They do still scan live shared -/// metadata to load their on-disk partitions, so a separate listener -/// fence is still required - see [`BootstrapBarrier`]. -/// -/// The channel is bounded to the peer count so shard 0's `send` never -/// blocks beyond a peer drain. A peer that dies before recv drops its -/// `bundle_rx`, so shard 0's `send` eventually sees a disconnected -/// channel; the cross-thread shutdown flag drives every waiter out of -/// its `recv` loop if shard 0 panics before broadcasting. -enum MetadataHandoff { - Owner { - bundle_tx: crossfire::MAsyncTx>, - }, - Waiter { - bundle_rx: crossfire::MAsyncRx>, - }, -} - -/// Reverse handshake to [`MetadataHandoff`]: gates shard 0's client -/// listeners until every peer has loaded its on-disk partitions. -/// -/// Peers build their owned-partition set from live shared metadata and -/// load each segment from disk in `build_shard_for_thread`. If shard 0 -/// opened listeners the instant `broadcast_metadata_bundle` returned -/// (peers have only *received* the bundle, not *loaded* partitions), a -/// client could create a partition before a peer's load scan finished. -/// That freshly committed partition would surface in the peer's scan -/// with no segment dir on disk yet, and `load_partition`'s `walk_dir` -/// would fail with `CannotReadPartitions`, aborting the whole node. A -/// partition created after boot must take the runtime reconciler path -/// (which creates its dir), never the bootstrap load path. -/// -/// Shard 0 (`Owner`) drains one signal per peer before binding -/// listeners; each peer (`Waiter`) sends one once its load completes. -/// The cross-thread shutdown flag drives both sides out of their poll -/// loop if any shard dies mid-boot. -enum BootstrapBarrier { - Owner { - ready_rx: crossfire::MAsyncRx>, - }, - Waiter { - ready_tx: crossfire::MAsyncTx>, - }, -} - -struct TcpTopology { - /// Domain-separation cluster id derived from `cluster.name`; threaded to - /// every consensus instance and the replica handshake so frames agree. - cluster_id: u128, - self_replica_id: u8, - replica_count: u8, - client_listen_addr: SocketAddr, - replica_listen_addr: Option, - ws_listen_addr: Option, - quic_listen_addr: Option, - http_listen_addr: Option, - tcp_tls_listen_addr: Option, - peers: Vec<(u8, SocketAddr)>, -} - -struct LocalClientAcceptFns { - tcp: AcceptedClientFn, - ws: AcceptedWsClientFn, - quic: AcceptedQuicClientFn, - tcp_tls: AcceptedTlsClientFn, - wss: AcceptedWssClientFn, -} - -#[derive(Default)] -struct BoundClientListeners { - tcp: Option, - tcp_tls: Option, - ws: Option, - quic: Option, -} - -/// Load the server configuration from the active config provider. -/// -/// # Errors -/// -/// Returns an error if the configuration cannot be read or parsed. -pub async fn load_config() -> Result { - ServerConfig::load().await.map_err(ServerError::Config) -} -/// Prepare the on-disk layout the server boots from and complete late -/// logging init. -/// -/// `fresh` wipes the system path first: `late_init` opens a rolling -/// appender under `{system_path}/logs` and `create_directories` -/// materialises exactly what the wipe is meant to remove, so both have to -/// run after it. -/// -/// # Errors -/// -/// Returns an error if the wipe, directory preparation, or logging setup -/// fails. -pub async fn prepare_runtime_dirs( - config: &ServerConfig, - logging: &mut Logging, - fresh: bool, -) -> Result<(), ServerError> { - if fresh { - wipe_system_path(config).await?; + log_files.push(entry); } - create_directories(&config.system).await.map_err(|source| { - error!( - system_path = %config.system.get_system_path(), - error = %source, - "failed to prepare server directories" - ); - source - })?; - logging - .late_init( - config.system.get_system_path(), - &LoggingSettings::from(&config.system.logging), - &TelemetrySettings::from(&config.telemetry), - ) - .map_err(ServerError::Logging)?; - Ok(()) + Ok(log_files) } -/// Delete the configured system path so the server boots on empty state. -async fn wipe_system_path(config: &ServerConfig) -> Result<(), ServerError> { - let path = config.system.get_system_path(); - // `system.path` is relative by default and IGGY_SYSTEM_PATH-overridable, - // so report what is actually about to be deleted, not what was configured. - let resolved = std::path::absolute(&path).unwrap_or_else(|_| PathBuf::from(&path)); +pub async fn load_segments( + config: &SystemConfig, + stream_id: usize, + topic_id: usize, + partition_id: usize, + partition_path: String, + stats: Arc, +) -> Result, IggyError> { + let mut log_files = collect_log_files(&partition_path).await?; + log_files.sort_by(|a, b| a.path.file_name().cmp(&b.path.file_name())); + let mut log = SegmentedLog::new(MemoryMessageJournal::empty()); + for entry in log_files { + let log_file_name = entry + .path + .file_stem() + .unwrap() + .to_string_lossy() + .to_string(); + + let start_offset = log_file_name.parse::().unwrap(); + + let messages_file_path = format!("{}/{}.{}", partition_path, log_file_name, LOG_EXTENSION); + let index_file_path = format!("{}/{}.{}", partition_path, log_file_name, INDEX_EXTENSION); + + async fn try_exists(path: &str) -> Result { + match compio::fs::metadata(path).await { + Ok(_) => Ok(true), + Err(err) => match err.kind() { + std::io::ErrorKind::NotFound => Ok(false), + _ => Err(err), + }, + } + } - if config.cluster.enabled { - warn!( - path = %resolved.display(), - "--fresh wipes only this replica, which then refills from the cluster by \ - state transfer; wiping a quorum at once destroys committed data, and a \ - service unit file carrying --fresh re-transfers everything on every restart" + let index_path_exists = try_exists(&index_file_path).await.unwrap(); + let index_cache_enabled = matches!( + config.segment.cache_indexes, + CacheIndexesConfig::All | CacheIndexesConfig::OpenSegment ); - } - - if !Path::new(&path).exists() { - info!(path = %resolved.display(), "--fresh: system path does not exist, nothing to remove"); - return Ok(()); - } - - warn!(path = %resolved.display(), "--fresh: removing the system path, ALL local data will be deleted"); - // A half-removed directory is worse than no removal at all: the surviving - // superblock and snapshot no longer pair up, and boot would report the - // leftovers as a durability violation rather than as a failed wipe. - remove_dir_all(&path) - .await - .map_err(|source| ServerError::FreshWipeFailed { - path: resolved, - source, - }) -} - -/// Resolve the operator's `cpu_allocation` into concrete shard -/// assignments plus the checked `u16` shard count. -/// -/// Shard ids index `ReplicaOwnerTable` slots as `u16`. `OWNER_NONE` -/// (`u16::MAX`) is reserved as the empty-slot sentinel, so a server -/// configured with `u16::MAX` shards would mint a shard id that -/// collides with the sentinel and an owner-table lookup could never -/// tell that shard apart from an unowned slot. Reject at boot so the -/// invariant is held by the type system, not by hoping the operator -/// never configures 65535 cores worth of shards. -fn resolve_shard_assignments( - sharding: &configs::sharding::ShardingConfig, -) -> Result<(Vec, u16), ServerError> { - let allocator = ShardAllocator::new(&sharding.cpu_allocation, sharding.pin_cores) - .map_err(ServerError::ShardAllocator)?; - let assignments = allocator - .to_shard_assignments() - .map_err(ServerError::ShardAllocator)?; - if assignments.is_empty() { - return Err(ServerError::ShardsCountZero); - } - match u16::try_from(assignments.len()) { - Ok(count) if count < message_bus::OWNER_NONE => Ok((assignments, count)), - _ => Err(ServerError::ShardsCountOverflow { - count: assignments.len(), - }), - } -} - -/// Re-validate the runtime sharding knobs that the per-shard runtime -/// consumes directly. Mirrors `ShardingConfig::validate` so a caller -/// that built the config without running it (e.g. tests, embedded -/// usage) cannot OOM at boot or wedge process exit with an out-of-range -/// value. -fn validate_sharding_runtime_knobs( - sharding: &configs::sharding::ShardingConfig, -) -> Result<(), ServerError> { - let inbox_capacity = sharding.inbox_capacity; - if inbox_capacity == 0 || inbox_capacity > INBOX_CAPACITY_MAX { - return Err(ServerError::InvalidInboxCapacity { - value: inbox_capacity, - max: INBOX_CAPACITY_MAX, - }); - } - let drain_timeout = sharding.shutdown_drain_timeout.get_duration(); - if drain_timeout.is_zero() || drain_timeout > SHUTDOWN_DRAIN_TIMEOUT_MAX { - return Err(ServerError::InvalidShutdownDrainTimeout { - value: drain_timeout, - max: SHUTDOWN_DRAIN_TIMEOUT_MAX, - }); - } - let poll_interval = sharding.shutdown_poll_interval.get_duration(); - if poll_interval.is_zero() || poll_interval > SHUTDOWN_POLL_INTERVAL_MAX { - return Err(ServerError::InvalidShutdownPollInterval { - value: poll_interval, - max: SHUTDOWN_POLL_INTERVAL_MAX, - }); - } - // Ordering: a poll cadence coarser than the drain budget makes the - // cross-thread shutdown flag effectively unobservable during teardown. - if poll_interval > drain_timeout { - return Err(ServerError::ShutdownPollExceedsDrain { - poll: poll_interval, - drain: drain_timeout, - }); - } - Ok(()) -} -/// Spawn the multi-shard `server` runtime. -/// -/// Resolves shard count + CPU affinities from -/// `system.sharding.cpu_allocation`, builds canonical-ordered -/// `(senders, inboxes)` channels, and spawns one OS thread per shard. -/// -/// Each thread pins itself (`nix::sched::sched_setaffinity` on Linux via -/// [`ShardInfo::bind_cpu`]), binds memory to its NUMA node when -/// configured, builds a fresh `compio::runtime::Runtime` (one -/// `io_uring` instance per shard), and runs `shard_main` inside it. -/// -/// Returns [`ShardHandles`] containing the cross-thread shutdown flag -/// and the per-shard `JoinHandle`s. The caller (`main.rs`) installs a -/// `ctrlc` handler that flips the flag, then `.join()`s every handle. -/// -/// # Errors -/// -/// Returns an error if shard allocation fails, the inbox capacity is -/// invalid, or any OS thread fails to spawn. Per-shard recovery / -/// listener / consensus failures surface through the per-thread `Result` -/// the caller observes on `.join()`. -/// -/// # Panics -/// -/// Panics if [`shard_mesh_channels`] returns an inbox slot already -/// consumed - a bootstrap programming error that would only fire if this -/// function were called twice with the same inboxes. -#[allow(clippy::too_many_lines)] -pub fn bootstrap( - config: ServerConfig, - current_replica_id: Option, -) -> Result { - validate_root_credentials_env(&config)?; - warm_dummy_password_hash(); - // The sync GetStats read path has no access to server config, so capture - // the data directory here for its disk-usage reporting. - crate::responses::init_stats_data_path(config.system.get_system_path().into()); - let (assignments, total_shards) = resolve_shard_assignments(&config.system.sharding)?; - let shards_count = assignments.len(); + if index_cache_enabled && !index_path_exists { + warn!( + "Index at path {} does not exist, rebuilding it based on {}...", + index_file_path, messages_file_path + ); + let now = std::time::Instant::now(); + let index_rebuilder = IndexRebuilder::new( + messages_file_path.clone(), + index_file_path.clone(), + start_offset, + ); + index_rebuilder.rebuild().await.unwrap_or_else(|e| { + panic!( + "Failed to rebuild index for partition with ID: {} for stream with ID: {} and topic with ID: {}. Error: {e}", + partition_id, stream_id, topic_id, + ) + }); + info!( + "Rebuilding index for path {} finished, it took {} ms", + index_file_path, + now.elapsed().as_millis() + ); + } - // Re-check the full valid range, not just the zero floor: a caller - // that built the config without running `ShardingConfig::validate` - // would otherwise OOM at boot allocating an oversized inbox channel, - // busy-loop every shutdown watchdog on a zero poll cadence, or wedge - // process exit on an unbounded drain budget. - let inbox_capacity = config.system.sharding.inbox_capacity; - validate_sharding_runtime_knobs(&config.system.sharding)?; + let messages_metadata = compio::fs::metadata(&messages_file_path) + .await + .map_err(|_| IggyError::CannotReadPartitions)?; + let messages_size = messages_metadata.len() as u32; - let (senders, mut inboxes) = shard_mesh_channels(total_shards, inbox_capacity); - let shutdown_flag = Arc::new(AtomicBool::new(false)); - let config = Arc::new(config); - // One owner table per server process, Arc-cloned into every shard's bus so - // any shard's bus reads the same atomic slots that the owning - // shard's installer / disconnect path writes. - let owner_table = Arc::new(ReplicaOwnerTable::new()); + let index_size = match compio::fs::metadata(&index_file_path).await { + Ok(metadata) => metadata.len() as u32, + Err(_) => 0, // Default to 0 if index file doesn't exist + }; - // Single-shot bundle handoff (see `MetadataHandoff`): shard 0 sends - // one cloned `ServerMetadataBundle` per peer; each peer drains - // exactly one. Bounded to the peer count so shard 0's broadcast - // never blocks past a peer drain. A single-shard deployment (zero - // peers) still needs a non-zero capacity, so clamp up explicitly - // rather than relying on crossfire's internal cap=0 -> 1 promotion. - // If a peer dies before recv, shard 0's `send` eventually sees a - // disconnected channel; the cross-thread shutdown flag drives every - // waiter out of its recv loop if shard 0 panics before broadcasting. - let metadata_peers = shards_count.saturating_sub(1).max(1); - let (metadata_bundle_tx, metadata_bundle_rx) = - crossfire::mpmc::bounded_async::(metadata_peers); + let storage = Storage::new( + &messages_file_path, + &index_file_path, + messages_size as u64, + index_size as u64, + config.partition.enforce_fsync, + config.partition.enforce_fsync, + true, + ) + .await?; - // Reverse barrier (see `BootstrapBarrier`): every peer sends one - // signal once it finishes loading its on-disk partitions; shard 0 - // drains them all before binding listeners. Bounded to the peer - // count so a sender never blocks (each peer sends exactly once). - let (ready_tx, ready_rx) = crossfire::mpmc::bounded_async::(metadata_peers); + let loaded_indexes = { + storage. + index_reader + .as_ref() + .unwrap() + .load_all_indexes_from_disk() + .await + .error(|e: &IggyError| format!("Failed to load indexes during startup for stream ID: {}, topic ID: {}, partition_id: {}, {e}", stream_id, topic_id, partition_id)) + .map_err(|_| IggyError::CannotReadFile)? + }; - let mut shard_threads: Vec<(u16, thread::JoinHandle>)> = - Vec::with_capacity(shards_count); - // Shared metadata-group view: written by shard 0's publisher task, read by - // every shard's cluster-metadata roster so leader marking works off-shard. - let metadata_view = Arc::new(AtomicU64::new(crate::cluster_meta::METADATA_VIEW_UNKNOWN)); - for (idx, assignment) in assignments.into_iter().enumerate() { - #[allow(clippy::cast_possible_truncation)] - let shard_id = idx as u16; - let inbox = inboxes[idx] - .take() - .expect("shard_mesh_channels populates every inbox slot exactly once"); - let senders_for_shard = senders.clone(); - let config_for_shard = Arc::clone(&config); - let shutdown_flag_for_shard = Arc::clone(&shutdown_flag); - let owner_table_for_shard = Arc::clone(&owner_table); - let metadata_handoff_for_shard = if shard_id == 0 { - MetadataHandoff::Owner { - bundle_tx: metadata_bundle_tx.clone(), - } + let end_offset = if loaded_indexes.count() == 0 { + start_offset } else { - MetadataHandoff::Waiter { - bundle_rx: metadata_bundle_rx.clone(), - } + let last_index_offset = loaded_indexes.last().unwrap().offset() as u64; + start_offset + last_index_offset }; - let barrier_for_shard = if shard_id == 0 { - BootstrapBarrier::Owner { - ready_rx: ready_rx.clone(), - } + + let (start_timestamp, end_timestamp) = if loaded_indexes.count() == 0 { + (0, 0) } else { - BootstrapBarrier::Waiter { - ready_tx: ready_tx.clone(), - } + ( + loaded_indexes.get(0).unwrap().timestamp(), + loaded_indexes.last().unwrap().timestamp(), + ) }; - let metadata_view_for_shard = Arc::clone(&metadata_view); - let handle = match thread::Builder::new() - .name(format!("shard-{shard_id}")) - .spawn(move || -> Result<(), ServerError> { - run_shard_thread( - shard_id, - total_shards, - current_replica_id, - assignment, - senders_for_shard, - inbox, - config_for_shard, - shutdown_flag_for_shard, - metadata_handoff_for_shard, - barrier_for_shard, - owner_table_for_shard, - metadata_view_for_shard, - ) - }) { - Ok(handle) => handle, - Err(source) => { - // Signal every shard already spawned before propagating, so - // their watchdog loops drive `bus.shutdown(...)` and the - // process can exit instead of hanging on stuck OS threads. - shutdown_flag.store(true, Ordering::Relaxed); - // Drop bootstrap's own channel clones before joining - // survivors. Otherwise a peer waiting on `bundle_rx.recv` - // would never observe the sender side disconnecting and - // would hang until the shutdown watchdog kicks the bus. - drop(metadata_bundle_tx); - drop(metadata_bundle_rx); - drop(ready_tx); - drop(ready_rx); - join_partial_shard_survivors( - shard_threads, - config.system.sharding.shutdown_join_timeout.get_duration(), + let mut segment = Segment::new(start_offset, config.segment.size); + + segment.start_timestamp = start_timestamp; + segment.end_timestamp = end_timestamp; + segment.end_offset = end_offset; + segment.size = IggyByteSize::from(messages_size as u64); + // At segment load, set the current position to the size of the segment (No data is buffered yet). + segment.current_position = segment.size.as_bytes_u32(); + segment.sealed = true; // Persisted segments are assumed to be sealed + + if config.partition.validate_checksum { + info!( + "Validating checksum for segment at offset {} in stream ID: {}, topic ID: {}, partition ID: {}", + start_offset, stream_id, topic_id, partition_id + ); + let messages_count = loaded_indexes.count() as u32; + if messages_count > 0 { + const BATCH_COUNT: u32 = 10000; + let mut current_relative_offset = 0u32; + let mut processed_count = 0u32; + + while processed_count < messages_count { + let remaining_count = messages_count - processed_count; + let batch_count = std::cmp::min(BATCH_COUNT, remaining_count); + let batch_indexes = loaded_indexes + .slice_by_offset(current_relative_offset, batch_count) + .unwrap(); + + let messages_reader = storage.messages_reader.as_ref().unwrap(); + match messages_reader.load_messages_from_disk(batch_indexes).await { + Ok(messages_batch) => { + if let Err(e) = messages_batch.validate_checksums() { + return Err(IggyError::CannotReadPartitions).error(|_: &IggyError| { + format!( + "Failed to validate message checksum for segment at offset {} in stream ID: {}, topic ID: {}, partition ID: {}, error: {}", + start_offset, stream_id, topic_id, partition_id, e + ) + }); + } + processed_count += messages_batch.count(); + current_relative_offset += batch_count; + } + Err(e) => { + return Err(e).error(|_: &IggyError| { + format!( + "Failed to load messages from disk for checksum validation at offset {} in stream ID: {}, topic ID: {}, partition ID: {}", + start_offset, stream_id, topic_id, partition_id + ) + }); + } + } + } + info!( + "Checksum validation completed for segment at offset {}", + start_offset ); - return Err(ServerError::ShardSpawnFailed { shard_id, source }); } - }; - shard_threads.push((shard_id, handle)); - } - - // Drop bootstrap's own channel clones now that every shard owns its - // half. Keeping them on bootstrap's stack would deadlock a peer - // whose `bundle_rx.recv` only completes once every sender - // disconnects. - drop(metadata_bundle_tx); - drop(metadata_bundle_rx); - drop(ready_tx); - drop(ready_rx); + } - info!( - shards_count, - "server bootstrap dispatched; awaiting shard runtimes" - ); + log.add_persisted_segment(segment, storage); - Ok(ShardHandles { - shutdown_flag, - shard_threads, - join_timeout: config.system.sharding.shutdown_join_timeout.get_duration(), - }) -} + stats.increment_segments_count(1); -/// Per-shard OS thread entry. Pins CPU + memory, builds the compio -/// runtime, and `block_on`s `shard_main`. -#[allow(clippy::needless_pass_by_value, clippy::too_many_arguments)] -fn run_shard_thread( - shard_id: u16, - total_shards: u16, - replica_id: Option, - assignment: ShardInfo, - senders: Vec, - inbox: ShardReceiver, - config: Arc, - shutdown_flag: Arc, - metadata_handoff: MetadataHandoff, - barrier: BootstrapBarrier, - owner_table: Arc, - metadata_view: Arc, -) -> Result<(), ServerError> { - // Armed for the whole thread body: a post-spawn error `?` or a panic - // unwind here must flip `shutdown_flag` so sibling watchdogs drive - // their bus shutdown instead of parking forever on `bus.token().wait()`. - let mut shutdown_guard = ShutdownOnDrop::new(Arc::clone(&shutdown_flag)); + stats.increment_size_bytes(messages_size as u64); - assignment - .bind_cpu() - .map_err(|source| ServerError::CpuAffinityFailed { shard_id, source })?; - assignment - .bind_memory() - .map_err(|source| ServerError::MemoryAffinityFailed { shard_id, source })?; + let messages_count = if end_offset > start_offset { + (end_offset - start_offset + 1) as u64 + } else if messages_size > 0 { + loaded_indexes.count() as u64 + } else { + 0 + }; - // `enrich_runtime_create_error` folds the io_uring remediation (raise - // `ulimit -l`, unblock seccomp, kernel-flag floor) into the error, so the - // guidance survives into the shard-join failure report instead of only - // stderr. Multi-shard boxes exhaust RLIMIT_MEMLOCK on per-shard rings - // before the bootstrap runtime does, so this path needs it most. - let runtime = create_shard_executor().map_err(|source| { - let source = server_common::diagnostics::enrich_runtime_create_error(source); - ServerError::ShardRuntimeCreateFailed { shard_id, source } - })?; + if messages_count > 0 { + stats.increment_messages_count(messages_count); + } - let result = runtime.block_on(async move { - // `shard_main`'s future grows past clippy's `large_futures` cap - // (it ferries the metadata handoff, bus, builders, and inflight - // I/O in one state machine). Heap-pin it so the top-level - // `block_on` future stays small; one allocation per startup buys - // the stack budget back. - Box::pin(shard_main( - shard_id, - total_shards, - replica_id, - senders, - inbox, - &config, - shutdown_flag, - metadata_handoff, - barrier, - owner_table, - metadata_view, - )) - .await - }); + let should_cache_indexes = match config.segment.cache_indexes { + CacheIndexesConfig::All => true, + CacheIndexesConfig::OpenSegment => false, + CacheIndexesConfig::None => false, + }; - if result.is_ok() { - shutdown_guard.disarm(); + if should_cache_indexes { + let segment_index = log.segments().len() - 1; + log.set_segment_indexes(segment_index, loaded_indexes); + } } - result -} - -/// Per-shard async lifecycle. Builds the bus, recovers metadata, -/// constructs the `IggyShard` for this shard's slice of partitions, -/// wires listeners on shard 0, and runs the message pump until -/// shutdown. -#[allow(clippy::too_many_arguments, clippy::too_many_lines)] -async fn shard_main( - shard_id: u16, - total_shards: u16, - replica_id: Option, - senders: Vec, - inbox: ShardReceiver, - config: &ServerConfig, - shutdown_flag: Arc, - metadata_handoff: MetadataHandoff, - barrier: BootstrapBarrier, - owner_table: Arc, - metadata_view: Arc, -) -> Result<(), ServerError> { - let topology = resolve_tcp_topology(config, replica_id)?; - let bus = Rc::new(IggyMessageBus::with_config_and_owner_table( - shard_id, - config, - owner_table, - )); - // Every shard can own a delegated replica connection, so every - // shard's bus needs the handshake identity (the handshake itself - // runs on the owning shard, not on shard 0). - bus.set_replica_handshake_ctx(ReplicaHandshakeCtx { - cluster_id: topology.cluster_id, - self_id: topology.self_replica_id, - replica_count: topology.replica_count, - auth: load_replica_auth(config).map(Rc::new), - tls: load_replica_tls_ctx(config, &topology)?.map(Rc::new), - }); - let drain_timeout = config.system.sharding.shutdown_drain_timeout.get_duration(); - let poll_interval = config.system.sharding.shutdown_poll_interval.get_duration(); - - let shutdown_flag_for_handoff = Arc::clone(&shutdown_flag); - spawn_shutdown_watchdog(Rc::clone(&bus), shutdown_flag, drain_timeout, poll_interval); + // The last segment is the active one and must remain unsealed for writes + if log.has_segments() { + log.segments_mut().last_mut().unwrap().sealed = false; + } - // Metadata bootstrap is single-writer: shard 0 owns the WAL and the - // only `WriteHandle`-bearing `MuxStateMachine`. Peer shards receive - // a `ReadHandleFactory` bundle on the inter-thread channel and - // rebuild a reader-mode `MuxStateMachine` on their own runtime - no - // WAL access, no replay. Writes still funnel through shard 0's - // metadata VSR; per-commit `publish()` (in `WriteCell::apply`) - // bounds reader staleness to one op. - let data_dir = Path::new(&config.system.path); - let (mux_stm, owner_state) = match metadata_handoff { - MetadataHandoff::Owner { bundle_tx } => { - // Root is created locally at boot (never journaled), so replay - // must start from the same baseline or every WAL-created user - // shifts one slab id and root is lost after the first restart. - let recovered = recover::( - data_dir, - ReplicaIdentity { - cluster: topology.cluster_id, - replica_id: topology.self_replica_id, - replica_count: topology.replica_count, - }, - config.metadata.journal_slots, - config.metadata.clients_table_max, - |mux_stm| { - ensure_default_root_user(mux_stm); - }, - ) - .await - .map_err(ServerError::MetadataRecovery)?; - ensure_default_root_user(&recovered.mux_stm); - // The factory bundle hands every peer a read handle over the - // same `Inner`, so `Arc` (and the parent - // `Arc`) is shared across all shards. Zero the - // snapshot totals here, once, before any peer can observe the - // bundle. Per-shard `load_partition` deltas in - // `build_shard_for_thread` then race only against other - // atomic adds, never against a concurrent `swap(0)` that - // would mistake an in-flight delta for the snapshot total - // and decrement the parent `StreamStats` by it. - let () = recovered.mux_stm.streams().read(|inner| { - for (_, stream) in &inner.items { - for (_, topic) in &stream.topics { - topic.stats.zero_out_all(); + if matches!( + config.segment.cache_indexes, + CacheIndexesConfig::OpenSegment + ) && log.has_segments() + { + let segments_count = log.segments().len(); + if segments_count > 0 { + let last_storage = log.storages().last().unwrap(); + match last_storage.index_reader.as_ref() { + Some(index_reader) => { + if let Ok(loaded_indexes) = index_reader.load_all_indexes_from_disk().await { + log.set_segment_indexes(segments_count - 1, loaded_indexes); } } - }); - broadcast_metadata_bundle( - shard_id, - &bundle_tx, - recovered.mux_stm.factory_bundle(), - total_shards.saturating_sub(1), - &shutdown_flag_for_handoff, - poll_interval, - ) - .await?; - ( - recovered.mux_stm, - Some(RecoveredOwnerState { - journal: recovered.journal, - snapshot: recovered.snapshot, - last_applied_op: recovered.last_applied_op, - last_journaled_op: recovered.last_journaled_op, - client_table: recovered.client_table, - superblock: recovered.superblock, - recovered_state: recovered.recovered_state, - snapshot_checkpoint: recovered.snapshot_checkpoint, - }), - ) - } - MetadataHandoff::Waiter { bundle_rx } => { - let bundle = await_metadata_bundle( - shard_id, - &bundle_rx, - &shutdown_flag_for_handoff, - poll_interval, - ) - .await?; - (ServerMuxStateMachine::from_factory_bundle(bundle), None) - } - }; - - // Metadata consensus + journal + snapshot live only on shard 0. - // `IggyShard::tick_metadata` short-circuits when `consensus.is_none()`, - // so peer shards have no caller that reads `journal` or `snapshot`. - let ( - metadata_consensus, - journal_for_metadata, - snapshot_for_metadata, - superblock_for_metadata, - checkpoint_seed, - recovered_client_table, - ) = if let Some(owner) = owner_state { - // `recover()` already opened the superblock, read `recovered_state`, and - // verified the on-disk snapshot against its checkpoint pairing BEFORE decoding - // it. Reuse that superblock rather than re-opening it, which would fork the - // ping-pong sequence counter. Consensus recovers its true (view, log_view) - // from `recovered_state` instead of inferring a stale view from the WAL. - let consensus = restore_metadata_consensus(&owner, &topology, config, Rc::clone(&bus)); - let superblock = Rc::new(owner.superblock); - ( - Some(consensus), - Some(owner.journal), - owner.snapshot, - Some(superblock), - owner.snapshot_checkpoint, - Some(owner.client_table), - ) - } else { - (None, None, None, None, (0, 0), None) - }; - let metadata = ServerMetadata::new( - metadata_consensus, - journal_for_metadata, - snapshot_for_metadata, - superblock_for_metadata, - mux_stm, - Some(PathBuf::from(&config.system.path)), - ); - // Size the VSR client table before listeners bind and any client registers. - // Must precede the recovered-table install below: the setter rebuilds the - // table from scratch, so running it afterwards would drop every resumed - // session (and trip its empty-table assert). - metadata.set_clients_table_max(config.metadata.clients_table_max); - // Reinstall the sessions recovery restored from the checkpoint and the WAL - // suffix, so a rebooted node dedups retries and admits continuations from - // clients that kept their identity across the restart (IGGY-137). Recovery - // sized this table from the same config value, so the install preserves the - // configured cap. - if let Some(client_table) = recovered_client_table { - // Refusal (a client registered before this ran) keeps the live table - // and is logged by the callee; boot continues either way. - let _ = metadata.install_client_table(client_table); - } - // Seed the coordinator's last-checkpoint pairing so the first post-boot - // view-change superblock write records the real (checkpoint_op, checksum) - // instead of (0, 0). No-op on peer shards, which have no coordinator. - metadata.seed_checkpoint_ref(checkpoint_seed.0, checkpoint_seed.1); - // Shard 0's copy resolves the `ServerDefault` sentinels (max topic size and - // message expiry) at create admission; responses echo stored values verbatim. - metadata.set_default_max_topic_size(config.system.topic.max_size.as_bytes_u64()); - metadata.set_default_message_expiry(u64::from(config.system.topic.message_expiry)); - // Keep the forced-checkpoint margin >= the configured prepare-queue - // depth: ops already pipelined while a checkpoint runs append into that - // margin (config validation keeps journal_slots >= 4x this). - metadata.set_checkpoint_margin(config.metadata.checkpoint_margin()); - - let shard_metrics = ShardMetrics::for_shard(); - // Notifier install deferred until after tick handler wires below. - let senders_for_notifier = senders.clone(); - let metrics_for_notifier = shard_metrics.clone(); - // Heap-pin like `shard_main` above: the builder future carries the whole - // shard construction state machine and outgrew clippy's `large_futures` - // cap; one allocation per shard startup. - let (shard, sessions) = Box::pin(build_shard_for_thread( - shard_id, - total_shards, - config, - &topology, - metadata, - Rc::clone(&bus), - senders, - inbox, - shard_metrics, - Arc::clone(&metadata_view), - )) - .await?; - - // Shard 0 owns the metadata consensus; publish its view so every shard's - // cluster-metadata read (and the SDK's leader discovery) marks the live - // primary. Detached: dies with this shard's runtime at process exit. - if shard_id == 0 { - let publisher_shard = Rc::clone(&shard); - let publisher_view = Arc::clone(&metadata_view); - compio::runtime::spawn(async move { - loop { - if let Some(consensus) = publisher_shard.plane.metadata().consensus.as_ref() { - // While this replica declines its recovered view's - // primaryship, that view must not reach the roster: the - // delegated shards would compute a leader that never - // heartbeats. Publish "unknown" until the election - // resolves the role. - let published = if consensus.has_ceded_primaryship() - && consensus.primary_index(consensus.view()) == consensus.replica() - { - crate::cluster_meta::METADATA_VIEW_UNKNOWN - } else { - u64::from(consensus.view()) - }; - publisher_view.store(published, Ordering::Relaxed); + None => { + warn!("Index reader not available for last segment in OpenSegment mode"); } - compio::time::sleep(std::time::Duration::from_millis(100)).await; } - }) - .detach(); - } - - info!( - shard = shard_id, - partitions = shard.plane.partitions().len(), - "server shard initialized" - ); - - // Re-check the cross-thread shutdown flag here, *before* spawning the - // message pump. A sibling shard may have failed in the window between - // the metadata broadcast and this point; gating before spawn keeps the - // bus' `background_tasks` vec empty on the shutdown path. Spawn-then- - // check would leave `bus.track_background(pump_handle)` registering a - // `JoinHandle` that only `bus.shutdown()` drains, but the watchdog - // driving `bus.shutdown()` is `.detach()`'d (see TODO at - // `spawn_shutdown_watchdog`) and may not be scheduled before this - // function returns `Ok(())` and the compio runtime drops, cancelling - // the pump mid-`write_vectored_all`. - // - // Without this gate shard 0 would also still open TCP/QUIC/WS - // listeners for a server that is already tearing down, briefly - // accepting connections that immediately get torn by the watchdog. - if shutdown_flag_for_handoff.load(Ordering::Relaxed) { - return Ok(()); - } - - // Tick handler must install before the notifier so early commits - // do not broadcast ticks whose handler slot is still `None`. - let (reconcile_wake_tx, reconcile_wake_rx) = channel::<()>(1); - let (reconcile_stop_tx, reconcile_stop_rx) = channel::<()>(1); - crate::partition_reconciler::install_tick_handler(&shard, reconcile_wake_tx); - - // Only shard 0 commits metadata. - if shard_id == 0 { - let notifier = make_metadata_commit_notifier(senders_for_notifier, metrics_for_notifier); - shard.plane.metadata().set_commit_notifier(Some(notifier)); - } else { - drop(senders_for_notifier); - drop(metrics_for_notifier); - } - - // The pump task also drives the consensus timer tick (heartbeats, prepare - // retransmit, view-change timeouts) as a select! arm, serialized with frame - // processing - see `run_message_pump`. - let (stop_tx, stop_rx) = channel(1); - let pump_shard = Rc::clone(&shard); - // Owned and awaited by shard_main at exit, NOT `track_background`: the - // background drain runs inside `bus.shutdown()`, which the Ctrl-C path - // never drives (the watchdog stands down when the token fires), so a - // tracked pump would be cancelled by runtime teardown mid final-flush - // and every graceful shutdown would silently drop the committed journal - // tail that had not hit a flush threshold yet. - let mut pump_handle = Some(compio::runtime::spawn(async move { - pump_shard.run_message_pump(stop_rx).await; - })); - - let reconciler_ctx = Rc::new(crate::partition_reconciler::ReconcilerCtx::new( - Rc::clone(&shard), - total_shards, - Rc::new(config.clone()), - topology.cluster_id, - topology.self_replica_id, - topology.replica_count, - )); - let reconcile_periodic = config - .system - .sharding - .reconcile_periodic_interval - .get_duration(); - let reconciler_handle = compio::runtime::spawn({ - let ctx = Rc::clone(&reconciler_ctx); - async move { - crate::partition_reconciler::run_reconciler( - ctx, - reconcile_wake_rx, - reconcile_stop_rx, - reconcile_periodic, - ) - .await; } - }); - bus.track_background(reconciler_handle); - - // Per-shard heartbeat verifier: evicts connections that stop pinging, - // releasing their consumer-group membership. Gated on config so a - // deployment without heartbeats never reaps live sessions. - let heartbeat_stop_tx = if config.heartbeat.enabled { - let (hb_stop_tx, hb_stop_rx) = channel::<()>(1); - let hb_shard = Rc::clone(&shard); - let hb_sessions = Rc::clone(&sessions); - let hb_interval = config.heartbeat.interval.get_duration(); - let hb_handle = compio::runtime::spawn(async move { - crate::dispatch::run_heartbeat_verifier(hb_shard, hb_sessions, hb_interval, hb_stop_rx) - .await; - }); - bus.track_background(hb_handle); - Some(hb_stop_tx) - } else { - None - }; - // Expired-PAT cleaner: shard 0 only (it owns the metadata consensus - // group) and only when enabled. Each pass no-ops unless this node is - // the caught-up metadata primary, so the delete is proposed once and - // replicated to every replica. - let pat_cleaner_stop = if shard_id == 0 && config.personal_access_token.cleaner.enabled { - let (cleaner_stop_tx, cleaner_stop_rx) = channel(1); - let cleaner_shard = Rc::clone(&shard); - let interval = config.personal_access_token.cleaner.interval.get_duration(); - let cleaner_handle = compio::runtime::spawn(async move { - crate::personal_access_token_cleaner::run_pat_cleaner( - cleaner_shard, - cleaner_stop_rx, - interval, - ) - .await; - }); - bus.track_background(cleaner_handle); - Some(cleaner_stop_tx) - } else { - None - }; - - // Segment cleaner: runs on every shard (each replica trims its own log, - // primary and backup alike). Local and unreplicated; gated by the shared - // data-maintenance config. - let segment_cleaner_stop = if config.data_maintenance.messages.cleaner_enabled { - let (stop_tx, stop_rx) = channel(1); - let cleaner_shard = Rc::clone(&shard); - let interval = config.data_maintenance.messages.interval.get_duration(); - let cleaner_handle = compio::runtime::spawn(async move { - crate::segment_cleaner::run_segment_cleaner(cleaner_shard, stop_rx, interval).await; - }); - bus.track_background(cleaner_handle); - Some(stop_tx) - } else { - None - }; - - // One keep-alive per process, so shard 0 owns it. Started before the - // listeners bind: systemd counts `WatchdogSec=` from unit start, not from - // `READY=1`, so a slow recovery must not look like a hang. - #[cfg(feature = "systemd")] - if shard_id == 0 { - crate::systemd::spawn_watchdog(&bus); } - // Listener fence (see `BootstrapBarrier`). Peers still scan live - // shared metadata and load their on-disk partitions in - // `build_shard_for_thread`; the factory-bundle handoff only proves - // they *received* the bundle, not that they finished loading. Shard - // 0 must not accept client traffic until every peer's load scan is - // done, otherwise a partition created by the first client surfaces - // in a still-running scan with no segment dir on disk and aborts the - // node with `CannotReadPartitions`. By this point every shard has - // also spawned its pump + reconciler, so a partition created after - // the fence takes the runtime reconciler path on its owning shard. - match barrier { - BootstrapBarrier::Owner { ready_rx } => { - await_bootstrap_complete( - &ready_rx, - usize::from(total_shards.saturating_sub(1)), - &shutdown_flag_for_handoff, - poll_interval, - ) - .await?; - } - BootstrapBarrier::Waiter { ready_tx } => { - signal_bootstrap_complete( - shard_id, - &ready_tx, - &shutdown_flag_for_handoff, - poll_interval, - ) - .await?; + Ok(log) +} + +/// Builds `InnerMetadata` from persisted user and stream state. +pub fn build_inner_metadata( + users_state: impl IntoIterator, + streams_state: impl IntoIterator, +) -> crate::metadata::InnerMetadata { + use crate::metadata::InnerMetadata; + use std::sync::atomic::AtomicUsize; + + let mut user_entries = Vec::new(); + let mut user_index = ahash::AHashMap::default(); + let mut personal_access_tokens: ahash::AHashMap< + u32, + ahash::AHashMap, PersonalAccessToken>, + > = ahash::AHashMap::default(); + let mut users_count = 0; + let mut pats_count = 0; + + for UserState { + id, + username, + password_hash, + status, + created_at, + permissions, + personal_access_tokens: user_pats, + } in users_state + { + let username_arc: Arc = Arc::from(username.as_str()); + let user_meta = UserMeta { + id, + username: username_arc.clone(), + password_hash: Arc::from(password_hash.as_str()), + status, + permissions: permissions.map(Arc::new), + created_at, + }; + user_entries.push((id as usize, user_meta)); + user_index.insert(username_arc, id); + + if !user_pats.is_empty() { + let user_pat_map: ahash::AHashMap, PersonalAccessToken> = user_pats + .into_values() + .map(|token| { + let pat = PersonalAccessToken::raw( + id, + &token.name, + &token.token_hash, + token.expiry_at, + ); + (Arc::from(token.token_hash.as_str()), pat) + }) + .collect(); + pats_count += user_pat_map.len(); + personal_access_tokens.insert(id, user_pat_map); } - } - // Listeners (replica + every client transport) bind on shard 0 only. - // Shard 0's coordinator round-robins inbound TCP/WS connections to - // peer shards via fd-transfer. QUIC and TCP-TLS clients terminate - // locally on shard 0 (their per-connection state is non-portable - - // see `LifecycleFrame::ClientWsConnectionSetup` rustdoc). - if shard_id == 0 { - let coord = shard - .coordinator() - .expect("shard 0 always has a coordinator attached by the builder"); - // Reseed the client-id minter above every recovered entry before any - // listener accepts. The counter is per process; the table it must not - // collide with was rebuilt from the previous boot's WAL. Keyed by view - // so a later promotion refolds the table (the minting path calls the - // same method, see `HttpInner::register_session_once`). - let boot_view = shard - .plane - .metadata() - .consensus - .as_ref() - .map_or(0, consensus::VsrConsensus::view); - coord.seed_client_sequence( - boot_view, - shard.plane.metadata().client_table.borrow().client_ids(), - ); - let on_client_request = make_client_request_handler( - &shard, - &sessions, - Arc::clone(&config.system), - config.personal_access_token.max_tokens_per_user, - ); - let (accepted_replica, dialed_replica) = - make_replica_delegation_fns(Rc::clone(&coord), &bus); - let accepted_client = make_shard_zero_client_accept_fns(coord, &bus, on_client_request); + users_count += 1; + } + info!( + "Building metadata: {} users, {} personal access tokens", + users_count, pats_count + ); - if let Err(error) = start_tcp_runtime( - &shard, - config, - &topology, - accepted_replica, - dialed_replica, - accepted_client, - ) - .await + let mut stream_entries = Vec::new(); + let mut stream_index = ahash::AHashMap::default(); + let mut streams_count = 0; + let mut topics_count = 0; + let mut partitions_count = 0; + let mut consumer_groups_count = 0; + + for StreamState { + name, + created_at, + id, + topics, + } in streams_state + { + info!( + "Building stream with ID: {}, name: {} metadata...", + id, name + ); + let stream_id = id as usize; + let stream_name: Arc = Arc::from(name.as_str()); + + let stream_stats = Arc::new(StreamStats::default()); + + let mut topic_entries = Vec::new(); + let mut topic_index = ahash::AHashMap::default(); + + for TopicState { + id, + name, + created_at, + compression_algorithm, + message_expiry, + max_topic_size, + replication_factor, + consumer_groups, + partitions, + } in topics.into_values() { - let _ = stop_tx.try_send(()); - let _ = reconcile_stop_tx.try_send(()); - if let Some(tx) = &heartbeat_stop_tx { - let _ = tx.try_send(()); - } - if let Some(cleaner_stop_tx) = &pat_cleaner_stop { - let _ = cleaner_stop_tx.try_send(()); + info!("Building topic with ID: {}, name: {} metadata...", id, name); + let topic_id = id as usize; + let topic_name: Arc = Arc::from(name.as_str()); + + let topic_stats = Arc::new(TopicStats::new(stream_stats.clone())); + + let mut partition_entries = Vec::new(); + let mut partition_ids = Vec::new(); + + for partition_state in partitions.into_values() { + let partition_id = partition_state.id as usize; + partition_ids.push(partition_id); + + let partition_stats = Arc::new(PartitionStats::new(topic_stats.clone())); + let partition_meta = PartitionMeta { + id: partition_id, + created_at: partition_state.created_at, + revision_id: 0, + stats: partition_stats, + consumer_offsets: Arc::new(ConsumerOffsets::with_capacity(0)), + consumer_group_offsets: Arc::new(ConsumerGroupOffsets::with_capacity(0)), + last_polled_offsets: Arc::new(papaya::HashMap::new()), + }; + partition_entries.push((partition_id, partition_meta)); + partitions_count += 1; } - if let Some(tx) = &segment_cleaner_stop { - let _ = tx.try_send(()); - } - await_pump_drain(pump_handle.take(), config, shard_id).await; - return Err(error); - } - // Every enabled client transport is bound and accepting by here, so - // this is the first point at which a unit ordered after us may dial. - #[cfg(feature = "systemd")] - crate::systemd::notify_ready(); - } - - bus.token().wait().await; - #[cfg(feature = "systemd")] - if shard_id == 0 { - crate::systemd::notify_stopping(); - } - let _ = stop_tx.try_send(()); - let _ = reconcile_stop_tx.try_send(()); - if let Some(tx) = &heartbeat_stop_tx { - let _ = tx.try_send(()); - } - if let Some(cleaner_stop_tx) = &pat_cleaner_stop { - let _ = cleaner_stop_tx.try_send(()); - } - if let Some(tx) = &segment_cleaner_stop { - let _ = tx.try_send(()); - } + partition_ids.sort_unstable(); - await_pump_drain(pump_handle.take(), config, shard_id).await; + let mut cg_entries = Vec::new(); + let mut cg_index = ahash::AHashMap::default(); - info!(shard = shard_id, "server shard exited cleanly"); - Ok(()) -} + for cg_state in consumer_groups.into_values() { + info!( + "Building consumer group with ID: {}, name: {} for topic with ID: {} metadata...", + cg_state.id, cg_state.name, topic_id + ); + let group_id = cg_state.id as usize; + let group_name: Arc = Arc::from(cg_state.name.as_str()); + let cg_meta = ConsumerGroupMeta { + id: group_id, + name: group_name.clone(), + partitions: partition_ids.clone(), + members: Slab::new(), + }; + cg_entries.push((group_id, cg_meta)); + cg_index.insert(group_name, group_id); + consumer_groups_count += 1; + } -/// Await the message pump's completion before the shard returns: its -/// post-loop work includes the final flush of every committed journal to -/// segment storage, and returning first drops the compio runtime, which -/// cancels that flush at its next await point. -async fn await_pump_drain( - pump_handle: Option>, - config: &ServerConfig, - shard_id: u16, -) { - let Some(pump_handle) = pump_handle else { - return; - }; - let drain_budget = config.system.sharding.shutdown_drain_timeout.get_duration(); - if compio::time::timeout(drain_budget, pump_handle) - .await - .is_err() - { - warn!( - shard = shard_id, - "message pump did not drain within the shutdown budget; \ - committed journal tail may not have flushed" - ); + let topic_meta = TopicMeta { + id: topic_id, + name: topic_name.clone(), + created_at, + message_expiry, + compression_algorithm, + max_topic_size, + replication_factor: replication_factor.unwrap_or(1), + stats: topic_stats, + partitions: partition_entries.into_iter().map(|(_, p)| p).collect(), + consumer_groups: cg_entries.into_iter().collect(), + consumer_group_index: cg_index, + round_robin_counter: Arc::new(AtomicUsize::new(0)), + }; + topic_entries.push((topic_id, topic_meta)); + topic_index.insert(topic_name, topic_id); + topics_count += 1; + } + + let stream_meta = StreamMeta { + id: stream_id, + name: stream_name.clone(), + created_at, + stats: stream_stats, + topics: topic_entries.into_iter().collect(), + topic_index, + }; + stream_entries.push((stream_id, stream_meta)); + stream_index.insert(stream_name, stream_id); + streams_count += 1; } -} -/// Block until shard 0 broadcasts the metadata factory bundle, or the -/// cross-thread shutdown flag flips. Polled in a `poll_interval` loop -/// so a shard 0 that panics before it broadcasts cannot strand peer -/// shards: the shutdown path flips the flag, every waiter observes it -/// on the next tick, and the server tears down instead of hanging. -/// -/// Uses `try_recv` + sleep rather than `timeout(recv())`. Crossfire 3.x -/// documents `recv()` as cancellation-safe (no leak/deadlock) but does -/// not guarantee atomicity for the dropped future's result; `try_recv` -/// keeps each tick fully synchronous and side-effect-free, so the -/// shutdown poll cadence cannot ambiguously consume a bundle. -async fn await_metadata_bundle( - shard_id: u16, - bundle_rx: &crossfire::MAsyncRx>, - shutdown_flag: &Arc, - poll_interval: Duration, -) -> Result { - loop { - match bundle_rx.try_recv() { - Ok(bundle) => return Ok(bundle), - Err(crossfire::TryRecvError::Disconnected) => { - return Err(ServerError::MetadataHandoffAborted { shard_id }); - } - Err(crossfire::TryRecvError::Empty) => { - if shutdown_flag.load(Ordering::Relaxed) { - return Err(ServerError::MetadataHandoffAborted { shard_id }); - } - compio::time::sleep(poll_interval).await; - } - } - } -} + info!( + "Built metadata: {} streams, {} topics, {} partitions, {} consumer groups", + streams_count, topics_count, partitions_count, consumer_groups_count + ); -/// Push `peers` cloned bundles onto `bundle_tx`, polling each send in a -/// `poll_interval` loop so the cross-thread shutdown flag can interrupt -/// a stalled handoff. Symmetric to [`await_metadata_bundle`]: shutdown -/// observed mid-handshake aborts cleanly rather than stalling on a -/// `send` future that can no longer make progress. -/// -/// Uses `try_send` + sleep rather than `timeout(send())`. Crossfire 3.x -/// documents `send()` as cancellation-safe in the leak/deadlock sense -/// but explicitly warns the true result is unknown when `SendFuture` is -/// dropped on cancellation. For a retry loop that re-clones on every -/// tick that would risk publishing the same bundle twice, stuffing the -/// bounded channel past `peers` and stranding a follow-up `send`. -/// `try_send` returns the bundle back inside `TrySendError::Full`, so -/// the loop reuses it instead of re-cloning when the channel is full. -async fn broadcast_metadata_bundle( - shard_id: u16, - bundle_tx: &crossfire::MAsyncTx>, - bundle: ServerMetadataBundle, - peers: u16, - shutdown_flag: &Arc, - poll_interval: Duration, -) -> Result<(), ServerError> { - for _ in 0..peers { - let mut pending = bundle.clone(); - loop { - match bundle_tx.try_send(pending) { - Ok(()) => break, - Err(crossfire::TrySendError::Disconnected(_)) => { - // Every peer dropped its `bundle_rx` before recv. Shard - // 0 must not silently continue past handoff: it would - // bind listeners and commit consensus state for a - // cluster whose peers are gone. Propagate the abort so - // `shard_main` short-circuits before further side - // effects; `shutdown_flag` will flip via the normal - // teardown path. - return Err(ServerError::MetadataHandoffAborted { shard_id }); - } - Err(crossfire::TrySendError::Full(returned)) => { - if shutdown_flag.load(Ordering::Relaxed) { - return Err(ServerError::MetadataHandoffAborted { shard_id }); - } - pending = returned; - compio::time::sleep(poll_interval).await; - } - } - } - } - Ok(()) -} - -/// Peer side of [`BootstrapBarrier`]: tell shard 0 this shard finished -/// loading its on-disk partitions. Mirrors [`broadcast_metadata_bundle`]'s -/// `try_send`-or-shutdown poll loop so a sibling failure (which flips the -/// shutdown flag) drives this out instead of stranding it on a full -/// channel. The channel is sized to the peer count and each peer sends -/// exactly once, so `Full` is not expected; the branch only keeps the -/// loop interruptible. -async fn signal_bootstrap_complete( - shard_id: u16, - ready_tx: &crossfire::MAsyncTx>, - shutdown_flag: &Arc, - poll_interval: Duration, -) -> Result<(), ServerError> { - let mut pending = shard_id; - loop { - match ready_tx.try_send(pending) { - Ok(()) => return Ok(()), - Err(crossfire::TrySendError::Disconnected(_)) => { - // Shard 0 dropped its `ready_rx` before draining (it - // aborted before binding listeners). Propagate so this - // shard short-circuits; the shutdown flag flips via the - // normal teardown path. - return Err(ServerError::MetadataHandoffAborted { shard_id }); - } - Err(crossfire::TrySendError::Full(returned)) => { - if shutdown_flag.load(Ordering::Relaxed) { - return Err(ServerError::MetadataHandoffAborted { shard_id }); - } - pending = returned; - compio::time::sleep(poll_interval).await; - } - } - } -} - -/// Owner side of [`BootstrapBarrier`]: drain one ready signal per peer -/// before shard 0 binds listeners. Polls the shutdown flag so a peer that -/// dies mid-load (flipping the flag) aborts the wait instead of hanging on -/// a signal that will never arrive. A single shard (`peers == 0`) returns -/// immediately. -async fn await_bootstrap_complete( - ready_rx: &crossfire::MAsyncRx>, - peers: usize, - shutdown_flag: &Arc, - poll_interval: Duration, -) -> Result<(), ServerError> { - let mut remaining = peers; - while remaining > 0 { - match ready_rx.try_recv() { - Ok(_shard_id) => remaining -= 1, - Err(crossfire::TryRecvError::Disconnected) => { - return Err(ServerError::ShardBootstrapBarrierAborted { remaining }); - } - Err(crossfire::TryRecvError::Empty) => { - if shutdown_flag.load(Ordering::Relaxed) { - return Err(ServerError::ShardBootstrapBarrierAborted { remaining }); - } - compio::time::sleep(poll_interval).await; - } - } - } - Ok(()) -} - -/// Spawn a per-shard polling task that watches the cross-thread shutdown -/// flag and triggers this shard's bus shutdown on transition. The flag -/// is the only Send signal we have; the bus' shutdown machinery is -/// `!Send` (`Rc>` + per-shard `async_channel`), so it must be -/// triggered from within the runtime that owns the bus. -#[allow(clippy::needless_pass_by_value)] -fn spawn_shutdown_watchdog( - bus: Rc, - shutdown_flag: Arc, - drain_timeout: Duration, - poll_interval: Duration, + InnerMetadata { + streams: stream_entries.into_iter().collect(), + users: user_entries.into_iter().collect(), + stream_index, + user_index, + personal_access_tokens, + users_global_permissions: Default::default(), + users_stream_permissions: Default::default(), + users_can_poll_all_streams: Default::default(), + users_can_send_all_streams: Default::default(), + users_can_poll_stream: Default::default(), + users_can_send_stream: Default::default(), + } +} + +/// Loads all metadata from persisted state into metadata writer. +/// Single atomic initialization via `Initialize` operation. +pub fn load_metadata( + users_state: impl IntoIterator, + streams_state: impl IntoIterator, + writer: &mut MetadataWriter, ) { - let bus_for_task = Rc::clone(&bus); - let bus_token = bus.token(); - let watchdog = compio::runtime::spawn(async move { - loop { - if shutdown_flag.load(Ordering::Relaxed) { - break; - } - if bus_token.is_triggered() { - // Bus shutdown was driven from elsewhere (e.g. internal - // failure path). Watchdog has nothing left to do. - return; - } - compio::time::sleep(poll_interval).await; - } - let _ = bus_for_task.shutdown(drain_timeout).await; - }); - // TODO(hubcio): `.detach()` races bus shutdown: when `bus.token()` is - // triggered, `shard_main` returns and the runtime drops the watchdog - // mid-`bus.shutdown()`, truncating in-flight `ClientForwardFailed` - // replies (terminal per `SendError` docs). Cannot use - // `bus.track_background(watchdog)` here because the watchdog itself - // drives `bus.shutdown()`, and the bg-drain loop in `shutdown()` - // would re-enter awaiting the watchdog's own pending shutdown call - // (self-deadlock). Fix: extract a `core/task_registry` crate mirroring - // `core/server`'s task-tracking mechanism, share it between the bus - // and server so background tasks can be reaped without coupling - // to the bus shutdown order. - watchdog.detach(); -} - -/// Copy the configured cluster roster plus this node's own client ports into -/// the shared [`ClusterRoster`] so the binary `GetClusterMetadata` read serves -/// the real topology. `self_*` back only the cluster-disabled self-synthesis -/// and carry the requested listener ports from the resolved topology, not the -/// bound ones (a `:0` wildcard is reported as 0). -fn build_cluster_roster( - config: &ServerConfig, - topology: &TcpTopology, - metadata_view: Arc, -) -> ClusterRoster { - ClusterRoster { - enabled: config.cluster.enabled, - name: config.cluster.name.clone(), - nodes: config - .cluster - .nodes - .iter() - .cloned() - .map(Into::into) - .collect(), - self_ip: topology.client_listen_addr.ip().to_string(), - self_ports: configs::cluster::TransportPorts { - tcp: Some(topology.client_listen_addr.port()), - quic: topology.quic_listen_addr.map(|addr| addr.port()), - http: topology.http_listen_addr.map(|addr| addr.port()), - websocket: topology.ws_listen_addr.map(|addr| addr.port()), - tcp_replica: None, - }, - metadata_view, - } -} - -#[allow(clippy::too_many_arguments, clippy::too_many_lines)] -async fn build_shard_for_thread( - shard_id: u16, - total_shards: u16, - config: &ServerConfig, - topology: &TcpTopology, - metadata: ServerMetadata, - bus: Rc, - senders: Vec, - inbox: ShardReceiver, - metrics: ShardMetrics, - metadata_view: Arc, -) -> Result<(Rc, Rc>), ServerError> { - let shard_local_id = ShardId::new(shard_id); - let total_partitions = metadata.mux_stm.streams().read(|inner| { - inner - .items - .iter() - .map(|(_, stream)| { - stream - .topics - .iter() - .map(|(_, topic)| topic.partitions.len()) - .sum::() - }) - .sum::() - }); - - // IggyPartitions holds only the partitions owned by this shard - // (see the filter below at insert time), so the server-wide total - // is an N-fold overshoot. `ceil(total / shards) * 2` is a coarse - // upper bound that absorbs hash skew without paying the full - // multiplier. PapayaShardsTable below stays sized to the server-wide - // total because every shard routes every namespace. - let owned_partitions_capacity = total_partitions - .div_ceil(usize::from(total_shards).max(1)) - .saturating_mul(2); - // At-rest encryption: built once per shard from the shared config; the - // ingestion path encrypts on the primary and the poll reply decrypts. - // A bad key fails the boot rather than silently serving plaintext. - let encryptor = if config.system.encryption.enabled { - let aes = Aes256GcmEncryptor::from_base64_key(&config.system.encryption.key) - .map_err(|error| ServerError::Iggy(Box::new(error)))?; - Some(Arc::new(EncryptorKind::Aes256Gcm(aes))) - } else { - None - }; - let partitions = IggyPartitions::with_capacity( - shard_local_id, - PartitionsConfig { - messages_required_to_save: config.system.partition.messages_required_to_save, - size_of_messages_required_to_save: config - .system - .partition - .size_of_messages_required_to_save, - enforce_fsync: config.system.partition.enforce_fsync, - validate_checksum: config.system.partition.validate_checksum, - segment_size: config.system.segment.size, - preallocate_segments: config.system.segment.preallocate, - encryptor, - }, - owned_partitions_capacity, - ); - let shards_table = PapayaShardsTable::with_capacity(total_partitions); - - // Stream-filter inside the `read()` closure: only partitions owned by - // this shard need the heavy (`Arc` + `Partition`) clones - // for the async `load_partition` below. Non-owning entries are pushed - // straight into `shards_table` here, so no Vec scales with the - // server-wide partition count. - let owned = metadata.mux_stm.streams().read(|inner| { - let mut owned = Vec::with_capacity(owned_partitions_capacity); - for (_, stream) in &inner.items { - for (topic_id, topic) in &stream.topics { - for partition in &topic.partitions { - let namespace = IggyNamespace::new(stream.id, topic_id, partition.id); - let owning_shard = - calculate_shard_assignment(&namespace, u32::from(total_shards)); - if owning_shard == shard_id { - // Shared per-partition stats from the registry: the - // same `Arc` backs every shard's `get_topic` reply. - let stats = inner.stats_registry.partition( - stream.id, - topic_id, - partition.id, - topic.stats.clone(), - ); - owned.push((stream.id, topic_id, stats, partition.clone())); - } else { - shards_table.insert( - namespace, - PartitionLocation::new( - ShardId::new(owning_shard), - partition.created_revision, - ), - ); - } - } - } - } - owned - }); - - // Snapshot totals were zeroed once on shard 0 before the factory - // bundle was broadcast (see `MetadataHandoff::Owner`). All shards - // here only add their per-partition deltas, so the shared - // `Arc` atomics race only against other atomic adds. - for (stream_id, topic_id, partition_stats, partition_metadata) in owned { - validate_namespace_bounds(config, stream_id, topic_id, partition_metadata.id)?; - let namespace = IggyNamespace::new(stream_id, topic_id, partition_metadata.id); - let partition = match load_partition( - config, - namespace, - Arc::clone(&partition_stats), - &partition_metadata, - topology.cluster_id, - topology.self_replica_id, - topology.replica_count, - Rc::clone(&bus), - ) - .await - { - Ok(partition) => partition, - // ONE damaged local chain must not take the node down. The shapes - // this refuses are exactly what a failed state-transfer quarantine - // leaves behind, so fence that group the same way the runtime path - // does -- move its segment files aside, keeping the superblock so it - // cannot re-enter view 0 -- and materialise it fresh. The ordinary - // rejoin path (repair, then state transfer on a refused floor) - // recovers its data from a peer. - Err(ServerError::PartitionChainRefused { dir, reason, .. }) => { - let partition_dir = dir.to_string_lossy().into_owned(); - error!( - stream_id, - topic_id, - partition_id = partition_metadata.id, - partition_dir, - %reason, - "refusing the recovered segment chain; fencing this partition and \ - rebuilding it empty for the rejoin path" - ); - match partitions::state_transfer::quarantine_segment_files(&partition_dir).await { - Ok(fenced_dir) => error!( - stream_id, - topic_id, - partition_id = partition_metadata.id, - fenced_dir, - "quarantined the refused segment files; they are kept for inspection" - ), - Err(error) => { - // NOT rebuilt: `build_partition_fresh` reaches - // `ensure_initial_segment`, which opens segment 0 with - // `file_exists = false` and TRUNCATES whatever the - // failed quarantine left behind. The likeliest failures - // (suffix cap exhausted, `create_dir_all`) move zero - // files, so rebuilding would destroy the oldest segment - // on the first attempt while the higher-offset survivors - // keep refusing every boot -- a loop that never - // terminates and eats the chain one segment at a time. - // Tombstone instead: the namespace stays unmaterialised - // and unrouted, the reconciler backs off, and an - // operator still has every byte. - error!( - stream_id, - topic_id, - partition_id = partition_metadata.id, - partition_dir, - %error, - "failed to quarantine the refused segment files; leaving this \ - partition tombstoned rather than rebuilding over them" - ); - partition_stats.zero_out_all(); - partitions.tombstone(namespace); - continue; - } - } - // The refused load already folded its segment counts in. - partition_stats.zero_out_all(); - build_partition_fresh( - config, - namespace, - partition_stats, - partition_metadata.created_revision, - topology.cluster_id, - topology.self_replica_id, - topology.replica_count, - Rc::clone(&bus), - ) - .await? - } - // An untrustworthy superblock fences ONE group, not the node. The - // segment files stay exactly where they are -- unlike a refused - // chain, the data on disk is not the thing in doubt -- so there is - // nothing to quarantine and nothing to rebuild: rebuilding fresh - // would hand this replica a view-0 identity while a record it - // cannot read says otherwise. Tombstoned, the namespace stays - // unmaterialised and unrouted, the reconciler backs off, and an - // operator has every byte plus a message naming the directory. - Err( - error @ (ServerError::PartitionSuperblockIo { .. } - | ServerError::PartitionSuperblockVersionUnknown { .. } - | ServerError::PartitionSuperblockUnverifiable { .. } - | ServerError::PartitionSuperblockUndecodable { .. } - | ServerError::PartitionSuperblockIdentityMismatch { .. }), - ) => { - error!( - stream_id, - topic_id, - partition_id = partition_metadata.id, - %error, - "cannot trust this partition's durable consensus state; tombstoning the \ - partition and continuing to boot the rest of the shard" - ); - partition_stats.zero_out_all(); - partitions.tombstone(namespace); - continue; - } - Err(error) => return Err(error), - }; - partitions.insert(namespace, partition); - shards_table.insert( - namespace, - PartitionLocation::new(ShardId::new(shard_id), partition_metadata.created_revision), - ); - } - - let shard_handle = Rc::new(RefCell::new(None)); - // Same wiring path as the simulator's shell mode: one per-shard - // SessionManager shared by the client-request handler (binds sessions) - // and the get_clients handler (reads them). It also carries this shard's - // cluster roster for the pre-auth GetClusterMetadata read. - let ShellHandlers { - on_replica_message, - on_client_request, - on_metadata_submit, - on_list_clients, - on_partition_read, - sessions, - } = wire_shell_handlers( - &bus, - &shard_handle, - Arc::clone(&config.system), - config.personal_access_token.max_tokens_per_user, - ); - sessions - .borrow_mut() - .set_cluster_roster(Rc::new(build_cluster_roster( - config, - topology, - metadata_view, - ))); - let shard_name = format!("server-shard-{shard_id}"); - let built = IggyShardBuilder::new( - ShardIdentity::new(shard_id, shard_name), - Rc::clone(&bus), - on_replica_message, - on_client_request, - on_metadata_submit, - on_list_clients, - on_partition_read, - metadata, - partitions, - senders, - inbox, - shards_table, - PartitionConsensusConfig::new( - topology.cluster_id, - shard::ReplicaTopology::new(topology.self_replica_id, topology.replica_count), - Rc::clone(&bus), - ), - CoordinatorConfig::default(), - metrics, - ) - .build() - .map_err(ServerError::ShardConstruction)?; - - let shard = Rc::new(built.shard); - // Repair pacing is shared by both planes' repair loops, so it is a - // per-shard tunable set once here rather than per consensus group. - shard.set_repair_retry_ticks(repair_retry_ticks(config)); - shard.set_served_segment_cache_bytes_max( - config - .partition - .transfer_served_cache_bytes_max - .as_bytes_u64(), - ); - shard.set_partition_artifact_len_max( - config.partition.transfer_artifact_bytes_max.as_bytes_u64(), - ); - shard.set_repair_chunk_max(config.cluster.repair_chunk_max as u64); - // Bounds a served state-transfer chunk. A frame above the bus ceiling is - // rejected by the RECEIVING transport, which tears the replica connection - // down rather than dropping one message. - shard.set_bus_max_message_size( - usize::try_from(config.message_bus.max_message_size.as_bytes_u64()).unwrap_or(usize::MAX), - ); - *shard_handle.borrow_mut() = Some(Rc::downgrade(&shard)); - Ok((shard, sessions)) -} - -// Pin the configs-crate default literals (duplicated there to avoid a -// build-time edge onto the runtime crates) against the runtime constants, -// mirroring the message_bus IOV_MAX pin. A drift on either side fails this -// crate's build until both are reconciled. -const _: () = assert!( - configs::metadata::DEFAULT_METADATA_PREPARE_QUEUE_DEPTH - == consensus::PIPELINE_PREPARE_QUEUE_MAX -); -const _: () = assert!( - configs::metadata::DEFAULT_METADATA_JOURNAL_SLOTS - == journal::prepare_journal::DEFAULT_SLOT_COUNT -); -const _: () = assert!( - configs::partition::DEFAULT_PARTITION_PREPARE_QUEUE_DEPTH - == consensus::PIPELINE_PREPARE_QUEUE_MAX -); -const _: () = - assert!(configs::metadata::DEFAULT_METADATA_CLIENTS_TABLE_MAX == consensus::CLIENTS_TABLE_MAX); -const _: () = - assert!(configs::cluster::DEFAULT_VIEW_PROBE_ATTEMPTS_MAX == consensus::PROBE_ATTEMPTS_MAX); -const _: () = - assert!(configs::partition::DEFAULT_EVICTED_RING_CAPACITY == partitions::EVICTED_RING_CAPACITY); -const _: () = assert!( - configs::partition::DEFAULT_EVICTED_RING_BYTES_MAX == partitions::EVICTED_RING_BYTES_MAX -); -const _: () = assert!( - configs::partition::DEFAULT_TRANSFER_ARTIFACT_BYTES_MAX - == shard::PARTITION_ARTIFACT_LEN_DEFAULT -); -const _: () = assert!( - configs::partition::DEFAULT_TRANSFER_SERVED_CACHE_BYTES_MAX - == shard::SERVED_SEGMENT_CACHE_BYTES_DEFAULT -); -const _: () = assert!(configs::cluster::DEFAULT_REPAIR_CHUNK_MAX as u64 == shard::REPAIR_CHUNK_MAX); -const _: () = assert!( - configs::cluster::STATE_CHUNK_HEADER_LEN - == size_of::() as u64 -); -// Both prepare-queue ceilings are pinned by the view-change wire, not by memory: a -// `DoViewChange` carries the sender's suffix spanning `commit..=op` with one nack -// bit and one present bit per entry, each bitset a single `u128`. The depth bounds -// `op - commit`, so a depth at or above `DVC_HEADERS_MAX` produces entries the new -// primary can neither adopt nor prove dead. Strictly less than, because the head op -// needs the reserved slot. -const _: () = - assert!(configs::metadata::MAX_METADATA_PREPARE_QUEUE_DEPTH < consensus::DVC_HEADERS_MAX); -const _: () = - assert!(configs::partition::MAX_PARTITION_PREPARE_QUEUE_DEPTH < consensus::DVC_HEADERS_MAX); -// `DVC_HEADERS_MAX` is a bare literal in both the wire crate, which sizes the -// bitsets, and the consensus crate, which cannot depend on it the other way around. -// Same u128, so a drift lets one side address entries the other cannot. -const _: () = - assert!(consensus::DVC_HEADERS_MAX == iggy_binary_protocol::consensus::DVC_HEADERS_MAX); -const _: () = assert!(consensus::DVC_HEADERS_MAX == u128::BITS as usize); -/// Convert a consensus-timer interval to whole ticks, floored at one tick so a -/// sub-tick value still fires and saturated on overflow. -fn duration_to_ticks(interval: Duration) -> u64 { - let ticks = interval.as_millis() / shard::CONSENSUS_TICK_INTERVAL.as_millis(); - u64::try_from(ticks.max(1)).unwrap_or(u64::MAX) -} - -/// `[cluster] heartbeat_timeout` in consensus ticks. Every consensus group -/// (metadata and per-partition planes alike) gets the same window: the failure -/// it guards against - a primary that stopped heartbeating - is host-level, not -/// per-plane. -pub(crate) fn cluster_heartbeat_ticks(config: &ServerConfig) -> u64 { - duration_to_ticks(config.cluster.heartbeat_timeout.get_duration()) -} - -/// Floor for the post-restart read-recovery deadline (see -/// [`recovery_barrier_deadline`]). At and below the 5s default heartbeat the -/// worst-case recovery is dominated by the heartbeat-independent term - the -/// `ViewChangeStatus` backstop plus election ceremony and suffix recommit, -/// empirically ~7s - so the scaled value must never fall under this or a -/// fast-heartbeat cluster would 503 legitimate reads mid-recovery. The backstop -/// is the configurable `[cluster] view_change_status_timeout`; raising it past -/// its 5s default is why `recovery_barrier_deadline` scales that knob in too -/// rather than leaning on this floor to cover it. -const RECOVERY_BARRIER_DEADLINE_FLOOR: Duration = Duration::from_secs(15); - -/// Safety factor applied to each scaled term of the recovery deadline: a slower -/// heartbeat stretches election and suffix recommit proportionally, and a wider -/// status backstop stretches the ceremony it bounds. 3x reproduces the -/// empirically chosen 15s margin at the shared 5s default (3 x 5s = 15s) and -/// holds that factor as either knob grows. -const RECOVERY_BARRIER_MULTIPLIER: u32 = 3; - -/// How long the post-restart read path waits for the recovered WAL suffix to -/// re-commit before failing loud (retryable 503): the largest of the fixed -/// floor, a `[cluster] heartbeat_timeout`-scaled window, and a -/// `[cluster] view_change_status_timeout`-scaled window. Both knobs feed it -/// because either, raised far past its default, stretches worst-case recovery -/// past the fixed floor; see `await_recovery_barrier` for the read-side wait. -pub(crate) fn recovery_barrier_deadline( - heartbeat: Duration, - view_change_status: Duration, -) -> Duration { - // saturating: neither timeout has a config ceiling, plain `*` panics - heartbeat - .saturating_mul(RECOVERY_BARRIER_MULTIPLIER) - .max(view_change_status.saturating_mul(RECOVERY_BARRIER_MULTIPLIER)) - .max(RECOVERY_BARRIER_DEADLINE_FLOOR) -} - -/// `[cluster] commit_broadcast_interval` in consensus ticks: how often the -/// primary broadcasts its commit point, the cluster's liveness feed. Applied -/// to every consensus group, matching `cluster_heartbeat_ticks`. -pub(crate) fn commit_broadcast_ticks(config: &ServerConfig) -> u64 { - duration_to_ticks(config.cluster.commit_broadcast_interval.get_duration()) -} - -/// `[cluster] prepare_retransmit_interval` in consensus ticks: how often the -/// primary retransmits un-acked prepares. Applied to every consensus group, -/// matching `cluster_heartbeat_ticks`. -pub(crate) fn prepare_retransmit_ticks(config: &ServerConfig) -> u64 { - duration_to_ticks(config.cluster.prepare_retransmit_interval.get_duration()) -} - -/// `[cluster] view_change_retransmit_interval` in consensus ticks: how often a -/// replica retransmits its `StartViewChange` / `DoViewChange` during a view -/// change. Applied to every consensus group, matching `cluster_heartbeat_ticks`. -pub(crate) fn view_change_retransmit_ticks(config: &ServerConfig) -> u64 { - duration_to_ticks( - config - .cluster - .view_change_retransmit_interval - .get_duration(), - ) -} - -/// `[cluster] view_change_status_timeout` in consensus ticks: the stalled -/// view-change backstop before escalating to a fresh election. Applied to every -/// consensus group, matching `cluster_heartbeat_ticks`. -pub(crate) fn view_change_status_ticks(config: &ServerConfig) -> u64 { - duration_to_ticks(config.cluster.view_change_status_timeout.get_duration()) -} - -/// `[cluster] request_start_view_retransmit_interval` in consensus ticks: how -/// often a recovering or view-change backup re-requests the current `StartView`. -/// Applied to every consensus group, matching `cluster_heartbeat_ticks`. -pub(crate) fn request_start_view_ticks(config: &ServerConfig) -> u64 { - duration_to_ticks( - config - .cluster - .request_start_view_retransmit_interval - .get_duration(), - ) -} - -/// `[cluster] repair_retry_interval` in consensus ticks: how long a stalled -/// journal-repair stream waits before re-requesting its window. Both planes' -/// repair loops share it, so it is applied once per shard (not per consensus -/// group). Clamped to `u32`, the width of the session idle-tick counter. -pub(crate) fn repair_retry_ticks(config: &ServerConfig) -> u32 { - u32::try_from(duration_to_ticks( - config.cluster.repair_retry_interval.get_duration(), - )) - .unwrap_or(u32::MAX) -} - -/// Shard 0's half of a metadata recovery: everything [`recover`] produced except the -/// state machine, which every shard receives through the factory bundle. -/// -/// Named rather than a positional tuple: the fields are same-typed `Option`s and -/// `(u64, u128)` pairs that a reorder would silently rebind, and one of them decides -/// what view the replica boots into. -struct RecoveredOwnerState { - journal: PrepareJournal, - snapshot: Option, - last_applied_op: Option, - last_journaled_op: Option, - client_table: ClientTable, - superblock: PingPongSuperblock, - recovered_state: Option, - snapshot_checkpoint: (u64, u128), -} - -/// Rebuild metadata consensus from what recovery read off this replica's own disk. -/// -/// Takes the recovery result, topology and config whole rather than the dozen-plus -/// scalars it needs from them: most were `u64` tick counts, where a misordered -/// argument type-checks and mistunes a timeout silently. -fn restore_metadata_consensus( - owner: &RecoveredOwnerState, - topology: &TcpTopology, - config: &ServerConfig, - bus: Rc, -) -> VsrConsensus> { - let journal = &owner.journal; - let replica_count = topology.replica_count; - let recovered_state = owner.recovered_state; - let snapshot_floor = owner - .snapshot - .as_ref() - .map_or(0, IggySnapshot::sequence_number); - let commit_watermark = owner.last_applied_op.unwrap_or(snapshot_floor); - let restored_op = owner.last_journaled_op.unwrap_or(snapshot_floor); - let recovery_deadline = recovery_barrier_deadline( - config.cluster.heartbeat_timeout.get_duration(), - config.cluster.view_change_status_timeout.get_duration(), - ); - let prepare_queue_depth = config.metadata.prepare_queue_depth; - - let mut consensus = VsrConsensus::new( - topology.cluster_id, - topology.self_replica_id, - replica_count, - server_common::sharding::METADATA_GROUP, - bus, - // Request queue keeps the stock 2x ratio over the prepare queue - // (32 -> 64 at defaults): buffered requests are cheap relative to - // in-flight prepares and drain as prepares commit. - LocalPipeline::with_capacities(prepare_queue_depth, prepare_queue_depth * 2), - ); - consensus.set_normal_heartbeat_ticks(cluster_heartbeat_ticks(config)); - consensus.set_commit_message_ticks(commit_broadcast_ticks(config)); - consensus.set_prepare_ticks(prepare_retransmit_ticks(config)); - consensus.set_view_change_retransmit_ticks(view_change_retransmit_ticks(config)); - consensus.set_view_change_status_ticks(view_change_status_ticks(config)); - consensus.set_request_start_view_ticks(request_start_view_ticks(config)); - consensus.set_probe_attempts_max(config.cluster.view_probe_attempts_max); - // Fresh random incarnation each boot, so a StartView addressed to a previous - // incarnation still in flight is ignored (`handle_start_view` guard). `| 1` - // guarantees the non-zero the guard treats as set. The deterministic simulator - // overrides this with a seed-derived value bumped per restart. - consensus.set_incarnation(rand::random::() | 1); - - let last_header = journal - .last_op() - .and_then(|op| usize::try_from(op).ok()) - .and_then(|op| journal.header(op).map(|header| *header)); - // View and log_view come from the durable superblock when present. A present but - // unreadable superblock already refused boot in `recover()`, so reaching the - // `else` means it is genuinely absent: a fresh node, or one that took writes but - // never checkpointed or changed view. There, inferring the view from the last WAL - // prepare is safe, since the persist-before-send gate guarantees this replica - // never externalized a view beyond what a re-probe re-derives, and it re-probes - // as a backup below. log_view cannot be inferred and stays 0 until the next - // superblock write. - if let Some(state) = recovered_state { - consensus.set_view(state.view); - consensus.set_log_view(state.log_view); - consensus.mark_superblock_durable(state.view, state.log_view); - } else if let Some(header) = last_header { - consensus.set_view(header.view); - } - - // On a RESTART in a cluster, rejoin as a quorum-invisible backup and - // probe for the current view (`RequestStartView`): the view's primary - // answers with a `StartView`, the replica adopts it as a backup, and - // journal repair fills any WAL gap. A probing replica never resumes - // primaryship -- if this replica IS the current primary-by-index, its - // probe makes the backups elect past it. - // The probe re-broadcasts on its timeout, so it needs no live mesh at - // boot. A FRESH boot keeps the plain init: the cluster needs its view-0 - // primary to exist, and a single-replica cluster has no peer to ask. - // - // Prior life is EITHER a non-empty WAL or a recovered superblock. A view - // change persists without touching the WAL, so a replica that changed - // view before its first metadata write comes back with a non-zero view - // and an empty journal; gating on the WAL alone would `init()` it into - // `Status::Normal` as primary for a view the cluster may have moved past, - // with `ceded_primaryship` false and no probe to correct it. - if replica_count > 1 && (restored_op > 0 || recovered_state.is_some()) { - consensus.init_as_backup(); - consensus.begin_view_probe(); - // Restart in a cluster: replace snapshot-shaped metadata state - // (snapshot + client table) from the live primary the probe finds, - // then journal-repair the tail. If the probe exhausts instead -- - // full-cluster bootstrap, nobody live to fetch from -- the election - // fallback clears the stage and this local recovery stands. - consensus.begin_state_transfer_await(); - } else { - consensus.init(); - } - consensus.sequencer().set_sequence(restored_op); - // A SOLO replica's durable journal head IS its commit point: quorum is - // 1-of-1, so an entry commits the instant it is durable, and the acks - // the cluster ceremony below would wait on cannot topologically exist. - // The embedded watermark is structurally one op stale (the commit point - // is only ever written down inside the NEXT entry), so trusting it solo - // manufactures an "uncommitted" suffix that provably committed and - // wedges the recovery barrier forever. - let commit_watermark = if replica_count == 1 { - restored_op - } else { - commit_watermark - }; - // The commit point is restored from the WAL's embedded watermark (each - // journaled prepare carries the primary's commit at send time), NOT from - // the journal head: journaled does not imply committed, and claiming - // commit for the un-quorum'd tail both risks split-brain on a later view - // change and starves the tail of re-replication (it would live in no - // pipeline). The suffix `(commit_watermark, restored_op]` is re-pipelined - // below when this replica is the recovered view's primary. - // - // TODO(hubcio): the watermark is a lower bound (the last entry stamps - // the commit point as of its send). Persisting an explicit (view, - // commit_op) watermark on the commit path would tighten recovery and - // allow refusing boot on an excessive gap; a backup that recovered a - // LONGER tail than the cluster's primary still needs uncommitted-suffix - // truncation when conflicting ops arrive (message repair milestone). - consensus.restore_commit_state(commit_watermark, commit_watermark); - if let Some(header) = last_header { - consensus.set_last_prepare_checksum(header.checksum); - consensus.observe_prepare_timestamp(header.timestamp); - } - - // The WAL's tail past the watermark is prepared-but-not-provably-committed - // state. Until the cluster confirms it (re-pipelined below on a resumed - // primary; via StartView adoption + the local commit walk on a rejoined - // backup), serving reads would show pre-restart state that clients already - // saw acked -- gate them on the barrier regardless of role. If the suffix - // never re-commits cluster-wide, the read path fails loud with a retryable - // 503 once the paired deadline expires (`await_recovery_barrier`). - if commit_watermark < restored_op { - consensus.set_recovery_barrier(restored_op); - consensus.set_recovery_deadline(recovery_deadline); - } - - // Re-pipeline the prepared-but-uncommitted suffix so the primary's - // retransmit machinery re-replicates it and quorum can (re-)commit it. - // A backup's suffix stays journal-only: the primary's traffic either - // confirms it (re-forward + re-ack path) or supersedes it. - if consensus.is_primary() - && !consensus.has_ceded_primaryship() - && commit_watermark < restored_op - { - info!( - commit_watermark, - restored_op, "re-pipelining recovered uncommitted metadata suffix" - ); - let mut pipeline = consensus.pipeline().borrow_mut(); - #[allow(clippy::cast_possible_truncation)] - for op in (commit_watermark + 1)..=restored_op { - let Some(header) = journal.header(op as usize) else { - warn!( - op, - "recovered journal suffix has a gap; stopping re-pipeline" - ); - break; - }; - let mut entry = PipelineEntry::new(*header); - entry.add_ack(topology.self_replica_id); - pipeline.push(entry); - } - } - - consensus -} - -#[allow(clippy::too_many_arguments)] -async fn load_partition( - config: &ServerConfig, - namespace: IggyNamespace, - stats: Arc, - partition_metadata: &Partition, - cluster_id: u128, - self_replica_id: u8, - replica_count: u8, - bus: Rc, -) -> Result>, ServerError> { - let stream_id = namespace.stream_id(); - let topic_id = namespace.topic_id(); - let partition_id = namespace.partition_id(); - // Request queue holds 2x the prepare depth (buffered requests drain as - // prepares commit); depth is the per-partition `[partition]` knob. - let prepare_queue_depth = config.partition.prepare_queue_depth; - let mut consensus = VsrConsensus::new( - cluster_id, - self_replica_id, - replica_count, - namespace.inner(), - bus, - LocalPipeline::with_capacities(prepare_queue_depth, prepare_queue_depth * 2), - ); - consensus.set_normal_heartbeat_ticks(cluster_heartbeat_ticks(config)); - consensus.set_commit_message_ticks(commit_broadcast_ticks(config)); - consensus.set_prepare_ticks(prepare_retransmit_ticks(config)); - consensus.set_view_change_retransmit_ticks(view_change_retransmit_ticks(config)); - consensus.set_view_change_status_ticks(view_change_status_ticks(config)); - consensus.set_request_start_view_ticks(request_start_view_ticks(config)); - consensus.set_probe_attempts_max(config.cluster.view_probe_attempts_max); - - // (view, log_view) come from the group's durable superblock when present; - // a present but unverifiable record already refused boot inside - // `open_partition_superblock`. Restored BEFORE choosing how to join, so - // the backup probe below never advertises a view older than the recorded - // one. - let partition_dir = config - .system - .get_partition_path(stream_id, topic_id, partition_id); - let (superblock, recovered_state) = open_partition_superblock( - &partition_dir, - ReplicaIdentity { - cluster: cluster_id, - replica_id: self_replica_id, - replica_count, - }, - ) - .await?; - if let Some(state) = recovered_state.as_ref() { - restore_partition_view(&mut consensus, state); - } - - // A recovered partition lost its journal state with the process: the - // partition journal is in-memory and segments carry no op numbers, so - // this replica cannot know the group's (op, commit) even when the - // superblock restored its view. In a cluster it boots as a - // quorum-invisible backup and probes for the current view - // (`RequestStartView`): the view's primary answers with a `StartView`, - // journal repair fills the rejoin window, and the commit floor settles - // at the serving peer's retention point. The probe re-broadcasts on its - // timeout, so it needs no live mesh at boot. Single-replica groups - // have no peer to ask and keep the plain init. - if replica_count > 1 { - consensus.init_as_backup(); - consensus.begin_view_probe(); - } else { - consensus.init(); - } - - // No prepare-timestamp floor is restored here: the partition consensus - // journal is non-durable today, so there is no persisted head to observe - // (unlike `restore_metadata_consensus`, which observes its restored head). - // When PartitionJournal becomes durable (the milestone named in the - // multi-shard wiring commit body), observe the restored head and the max - // recovered message timestamp here, or an NTP rewind across a restart could - // regress persisted `base_timestamp`. - - let recovered_segments = - load_persisted_segments(config, stream_id, topic_id, partition_id, &stats) - .await - .map_err(|source| { - error!( - stream_id, - topic_id, - partition_id, - error = %source, - "failed to load partition log during server bootstrap" - ); - source - })?; - - let mut partition = IggyPartition::new(stats.clone(), consensus); - partition.set_superblock(superblock, recovered_state.as_ref()); - // Recovered partitions honor the same config-surfaced ring ceilings as the - // fresh-create path (build_partition_fresh). Retention is already off for - // single-replica groups, so this only sizes the multi-replica ring. - partition.log.journal().inner.set_ring_caps( - config.partition.evicted_ring_capacity, - config.partition.evicted_ring_bytes_max.as_bytes_u64(), - ); - partition.set_partition_dir(partition_dir); - // Before the hydrate: the durable record is keyed by incarnation, so a - // `purge.gen` left behind by a previous life of this namespace reads 0. - partition.set_created_revision(partition_metadata.created_revision); - partition.hydrate_applied_purge_generation().await?; - hydrate_partition_log( - &mut partition, - config, - stream_id, - topic_id, - partition_id, - recovered_segments, - ) - .await?; - - let sized_end = partition - .log - .segments() - .iter() - .filter(|segment| segment.size > IggyByteSize::default()) - .map(|segment| segment.end_offset) - .max(); - // An empty chain whose segment is named for a nonzero offset is the - // shape a state-transfer install (or its converge) plants at the group - // frontier after the origin GC'd everything: the file name carries the - // frontier, and re-minting offsets from 0 here would fork this - // replica's batch stamps from the rest of the group after a restart. - let empty_frontier = partition - .log - .segments() - .iter() - .map(|segment| segment.start_offset) - .max() - .filter(|&start| sized_end.is_none() && start > 0); - let current_offset = sized_end.or_else(|| empty_frontier.map(|start| start - 1)); - partition.created_at = partition_metadata.created_at; - partition.recovered_durable_offset = sized_end; - // The OFFSET COUNTER is restored from that file name (above), but the - // `installed_frontier` CLAIM deliberately is not: the claim says "everything - // below me is represented here", and `converge_to_empty_after_failed_install` - // refuses to make it when staged segments were dropped -- yet a converge - // plants exactly the same empty `{frontier:020}.log` a legitimate empty - // install does, so boot provably cannot tell them apart. Re-deriving it here - // would hand the refused claim back: the repair floor stand-in would accept a - // commit floor over ops this replica holds zero bytes for, and the replica - // would pass the serve gate and offer that emptiness onward, making a peer - // unlink its own chain. Leaving it `None` costs one spurious full - // re-transfer on the legitimate empty-install restart; a false caught-up - // claim is not recoverable. A durable home for the frontier (the partition - // superblock already reserves a field) is what would settle it properly. - let counter = current_offset.unwrap_or(0); - partition.offset.store(counter, Ordering::Release); - partition.dirty_offset.store(counter, Ordering::Relaxed); - partition.should_increment_offset = current_offset.is_some(); - // The durable frontier is a LOWER BOUND on top of what the segments proved: - // it is the only carrier left when the segments that named the frontier are - // gone (an all-GC'd origin's install, a crash inside the swap window), and - // taking the max means real recovered data always wins. - partition.restore_offset_frontier(recovered_state.as_ref()); - let current_offset = partition.offset.load(Ordering::Acquire); - - configure_consumer_offsets(&mut partition, config, namespace, current_offset)?; - ensure_initial_segment(&mut partition, config, stream_id, topic_id, partition_id).await?; - - Ok(partition) -} - -async fn hydrate_partition_log( - partition: &mut IggyPartition>, - config: &ServerConfig, - stream_id: usize, - topic_id: usize, - partition_id: usize, - recovered_segments: Vec, -) -> Result<(), ServerError> { - for RecoveredSegment { segment, storage } in recovered_segments { - partition - .log - .add_persisted_segment(segment, storage, None, None); - } - - if let Some(active_index) = partition.log.segments().len().checked_sub(1) { - let storage = &partition.log.storages()[active_index]; - if let ( - Some(messages_reader), - Some(index_reader), - Some(storage_messages_writer), - Some(storage_index_writer), - ) = ( - storage.messages_reader.as_ref(), - storage.index_reader.as_ref(), - storage.messages_writer.as_ref(), - storage.index_writer.as_ref(), - ) { - let index_path = index_reader.path(); - // Share the storage's size counters: the readers bound reads by - // these atomics, so a writer with a private counter persists bytes - // the readers never learn about. - let messages_size_counter = storage_messages_writer.size_counter(); - let index_size_counter = storage_index_writer.size_counter(); - partition.log.messages_writers_mut()[active_index] = Some(Rc::new( - MessagesWriter::new( - &messages_reader.path(), - messages_size_counter, - config.system.partition.enforce_fsync, - true, - config - .system - .segment - .preallocate - .then_some(config.system.segment.size), - ) - .await - .map_err(|source| { - error!( - stream_id, - topic_id, - partition_id, - path = %messages_reader.path(), - error = %source, - "failed to initialize persisted messages writer" - ); - source - })?, - )); - partition.log.index_writers_mut()[active_index] = Some(Rc::new( - IggyIndexWriter::new( - &index_path, - index_size_counter, - config.system.partition.enforce_fsync, - true, - ) - .await - .map_err(|source| { - error!( - stream_id, - topic_id, - partition_id, - path = %index_path, - error = %source, - "failed to initialize persisted sparse index writer" - ); - source - })?, - )); - } - } - - Ok(()) -} - -fn resolve_tcp_topology( - config: &ServerConfig, - current_replica_id: Option, -) -> Result { - let default_client_addr = parse_socket_addr("tcp.address", &config.tcp.address)?; - let default_ws_addr = resolve_optional_listener_addr( - config.websocket.enabled, - "websocket.address", - &config.websocket.address, - )?; - let default_quic_addr = - resolve_optional_listener_addr(config.quic.enabled, "quic.address", &config.quic.address)?; - let default_http_addr = - resolve_optional_listener_addr(config.http.enabled, "http.address", &config.http.address)?; - if !config.cluster.enabled { - if let Some(replica_id) = current_replica_id - && replica_id != SHARD_REPLICA_ID - { - return Err(ServerError::ReplicaIdRequiresCluster { - supplied: replica_id, - default: SHARD_REPLICA_ID, - }); - } - return Ok(TcpTopology { - cluster_id: auth::cluster_domain_id(&config.cluster.name), - // Keep parity with the current server binary and the integration - // harness: `--replica-id 0` may be passed unconditionally in - // single-node mode; any other id is rejected above so the WAL - // cannot commit under an identity that will later disagree with - // a cluster.nodes[] entry. - self_replica_id: SHARD_REPLICA_ID, - replica_count: 1, - client_listen_addr: default_client_addr, - replica_listen_addr: Some(SocketAddr::new(default_client_addr.ip(), 0)), - ws_listen_addr: default_ws_addr, - quic_listen_addr: default_quic_addr, - http_listen_addr: default_http_addr, - tcp_tls_listen_addr: config.tcp.tls.enabled.then_some(default_client_addr), - peers: Vec::new(), - }); - } - - let self_replica_id = current_replica_id.ok_or(ServerError::MissingReplicaId)?; - - let self_node = config - .cluster - .nodes - .iter() - .find(|node| node.replica_id == self_replica_id) - .ok_or(ServerError::ClusterNodeNotFound { - replica_id: self_replica_id, - })?; - let replica_count = u8::try_from(config.cluster.nodes.len()).map_err(|_| { - ServerError::ClusterReplicaCountTooLarge { - count: config.cluster.nodes.len(), - } - })?; - let ClusterClientAddrs { - client: client_listen_addr, - ws: ws_listen_addr, - quic: quic_listen_addr, - http: http_listen_addr, - } = resolve_cluster_client_addrs( - self_node, - default_client_addr, - default_ws_addr, - default_quic_addr, - default_http_addr, - )?; - let replica_port = self_node - .ports - .tcp_replica - .ok_or(ServerError::ClusterPortMissing { - transport: "tcp_replica", - replica_id: self_node.replica_id, - })?; - let replica_listen_addr = Some(socket_addr_from_parts( - "cluster.nodes[*].ports.tcp_replica", - &self_node.ip, - replica_port, - )?); - let peers = resolve_cluster_replica_peers(&config.cluster.nodes, self_replica_id)?; - - Ok(TcpTopology { - cluster_id: auth::cluster_domain_id(&config.cluster.name), - self_replica_id, - replica_count, - client_listen_addr, - replica_listen_addr, - ws_listen_addr, - quic_listen_addr, - http_listen_addr, - tcp_tls_listen_addr: config.tcp.tls.enabled.then_some(client_listen_addr), - peers, - }) -} - -fn resolve_optional_listener_addr( - enabled: bool, - context: &'static str, - address: &str, -) -> Result, ServerError> { - if enabled { - return Ok(Some(parse_socket_addr(context, address)?)); - } - Ok(None) -} - -/// Client-facing listener addresses resolved for this cluster node. Each port -/// comes from the node's roster entry; there is no fallback to the top-level -/// listener port, an enabled transport without a roster port refuses to boot. -/// Every transport keeps the bind interface from its own `address` config: the -/// roster ip is advertised, not bound. -struct ClusterClientAddrs { - client: SocketAddr, - ws: Option, - quic: Option, - http: Option, -} - -fn resolve_cluster_client_addrs( - self_node: &configs::cluster::ClusterNodeConfig, - default_tcp_addr: SocketAddr, - default_ws_addr: Option, - default_quic_addr: Option, - default_http_addr: Option, -) -> Result { - let client_port = self_node.ports.tcp.ok_or(ServerError::ClusterPortMissing { - transport: "tcp", - replica_id: self_node.replica_id, - })?; - let client = - merge_roster_port_with_bind_ip("tcp", &self_node.ip, default_tcp_addr, client_port); - let ws = resolve_cluster_optional_addr(self_node, "websocket", default_ws_addr, |ports| { - ports.websocket - })?; - let quic = - resolve_cluster_optional_addr(self_node, "quic", default_quic_addr, |ports| ports.quic)?; - let http = - resolve_cluster_optional_addr(self_node, "http", default_http_addr, |ports| ports.http)?; - Ok(ClusterClientAddrs { - client, - ws, - quic, - http, - }) -} - -fn resolve_cluster_optional_addr( - self_node: &configs::cluster::ClusterNodeConfig, - transport: &'static str, - default_addr: Option, - port_selector: impl Fn(&configs::cluster::TransportPorts) -> Option, -) -> Result, ServerError> { - let Some(default_addr) = default_addr else { - return Ok(None); - }; - // No fallback to the top-level port: two same-host nodes leaving the same - // transport port unset would race for one socket. Either the roster is - // explicit or the server refuses to boot. - let port = port_selector(&self_node.ports).ok_or(ServerError::ClusterPortMissing { - transport, - replica_id: self_node.replica_id, - })?; - Ok(Some(merge_roster_port_with_bind_ip( - transport, - &self_node.ip, - default_addr, - port, - ))) -} - -/// Combine the roster-supplied `port` with the bind interface the transport's -/// own `address` config asked for. -/// -/// The roster ip is what the cluster advertises (metadata, follower-to-primary -/// HTTP forwarding targets); the transport's own `address` decides the bind -/// interface. Merging keeps a loopback-only `127.0.0.1` private and a -/// `0.0.0.0` wide in cluster mode instead of silently rebinding to the roster -/// interface, which would strand every co-located dialer (sidecars, health -/// probes, on-host consumers) on `ECONNREFUSED`. -fn merge_roster_port_with_bind_ip( - transport: &'static str, - roster_ip: &str, - bind_addr: SocketAddr, - port: u16, -) -> SocketAddr { - let listen_addr = SocketAddr::new(bind_addr.ip(), port); - if roster_ip_unreachable_from_bind_addr(roster_ip, listen_addr) { - warn!( - "{transport} listener binds {listen_addr} but the roster advertises {roster_ip}:{port}; \ - peers and clients dialing the advertised endpoint may not reach this node" - ); - } - listen_addr -} - -/// Whether a dialer aiming at the advertised roster ip misses `listen_addr`. An -/// unspecified bind covers every interface, and a roster ip that parses as -/// neither IPv4 nor IPv6 (a DNS name, say) can resolve to the bound interface, -/// so both cases stay quiet. -fn roster_ip_unreachable_from_bind_addr(roster_ip: &str, listen_addr: SocketAddr) -> bool { - !listen_addr.ip().is_unspecified() - && roster_ip - .parse::() - .is_ok_and(|parsed| parsed != listen_addr.ip()) -} - -fn resolve_cluster_replica_peers( - nodes: &[configs::cluster::ClusterNodeConfig], - self_replica_id: u8, -) -> Result, ServerError> { - let mut peers = Vec::with_capacity(nodes.len().saturating_sub(1)); - for node in nodes { - if node.replica_id == self_replica_id { - continue; - } - let replica_port = node - .ports - .tcp_replica - .ok_or(ServerError::ClusterPortMissing { - transport: "tcp_replica", - replica_id: node.replica_id, - })?; - peers.push(( - node.replica_id, - socket_addr_from_parts("cluster.nodes[*].ports.tcp_replica", &node.ip, replica_port)?, - )); - } - Ok(peers) -} - -async fn start_tcp_runtime( - shard: &Rc, - config: &ServerConfig, - topology: &TcpTopology, - accepted_replica: AcceptedReplicaFn, - dialed_replica: DialedReplicaFn, - accepted_clients: LocalClientAcceptFns, -) -> Result<(), ServerError> { - if config.tcp.enabled && !config.tcp.tls.enabled { - start_via_replica_io( - shard, - config, - topology, - accepted_replica, - dialed_replica, - accepted_clients, - ) - .await?; - } else { - start_manual_runtime( - shard, - config, - topology, - accepted_replica, - dialed_replica, - accepted_clients, - ) - .await?; - } - - // HTTP is served over TCP but sits outside the replica_io / manual client - // reactor, so it binds independently. Shard-0 gating comes from the sole - // caller of this function. - if let Some(http_addr) = topology.http_listen_addr { - let self_ports = configs::cluster::TransportPorts { - tcp: config - .tcp - .enabled - .then(|| topology.client_listen_addr.port()), - quic: topology.quic_listen_addr.map(|addr| addr.port()), - websocket: topology.ws_listen_addr.map(|addr| addr.port()), - ..Default::default() - }; - http::start( - shard, - http_addr, - &config.http, - config.metadata.clients_table_max, - config.personal_access_token.max_tokens_per_user, - &config.cluster, - Arc::clone(&config.system), - self_ports, - ) - .await?; - } - - Ok(()) -} - -// ws/wss bindings intentionally mirror the transport names (same convention as -// `replica_io::start_on_shard_zero`). -#[allow(clippy::similar_names)] -async fn start_via_replica_io( - shard: &Rc, - config: &ServerConfig, - topology: &TcpTopology, - accepted_replica: AcceptedReplicaFn, - dialed_replica: DialedReplicaFn, - accepted_clients: LocalClientAcceptFns, -) -> Result<(), ServerError> { - let replica_addr = topology - .replica_listen_addr - .expect("topology must include replica listener address"); - let quic_credentials = topology - .quic_listen_addr - .is_some() - .then(|| load_quic_server_credentials(config)) - .transpose()?; - let tcp_tls_credentials = topology - .tcp_tls_listen_addr - .is_some() - .then(|| load_tcp_tls_server_credentials(config)) - .transpose()?; - // `websocket.tls.enabled` upgrades the websocket address to a WSS - // listener; the plain-WS listener must NOT also bind it (one port, one - // handshake kind -- a plain upgrade parser fed a TLS ClientHello rejects - // every connection with an httparse error). - let wss_enabled = config.websocket.tls.enabled; - let ws_listen_addr = (!wss_enabled).then_some(topology.ws_listen_addr).flatten(); - let wss_listen_addr = wss_enabled.then_some(topology.ws_listen_addr).flatten(); - let wss_credentials = wss_listen_addr - .is_some() - .then(|| load_wss_server_credentials(config)) - .transpose()?; - - let LocalClientAcceptFns { - tcp, - ws, - quic, - tcp_tls, - wss, - } = accepted_clients; - - let bound = replica_io::start_on_shard_zero( - &shard.bus, - replica_addr, - topology.client_listen_addr, - ws_listen_addr, - topology.quic_listen_addr, - quic_credentials, - topology.tcp_tls_listen_addr, - tcp_tls_credentials, - wss_listen_addr, - wss_credentials, - topology.self_replica_id, - topology.peers.clone(), - accepted_replica, - dialed_replica, - tcp, - ws_listen_addr.map(|_| ws), - topology.quic_listen_addr.map(|_| quic), - topology.tcp_tls_listen_addr.map(|_| tcp_tls), - wss_listen_addr.map(|_| wss), - shard.bus.config().reconnect_period, - ) - .await - .map_err(|source| { - error!( - replica_addr = %replica_addr, - client_addr = %topology.client_listen_addr, - error = %source, - "failed to start server listeners via replica_io" - ); - source - })?; - let Some(bound) = bound else { - return Ok(()); - }; - - write_current_config( - config, - Some(topology.self_replica_id), - Some(bound.client), - config.cluster.enabled.then_some(bound.replica), - bound.tcp_tls, - bound.quic, - // The WSS listener occupies the configured websocket address slot. - bound.wss.or(bound.ws), - ) - .await?; - if config.cluster.enabled { - info!( - shard = shard.id, - replica = %bound.replica, - tcp = %bound.client, - tcp_tls = ?bound.tcp_tls, - ws = ?bound.ws, - quic = ?bound.quic, - "server listeners started" - ); - } else { - info!( - shard = shard.id, - tcp = %bound.client, - tcp_tls = ?bound.tcp_tls, - ws = ?bound.ws, - quic = ?bound.quic, - "server client listeners started" - ); - } - - Ok(()) -} - -async fn start_manual_runtime( - shard: &Rc, - config: &ServerConfig, - topology: &TcpTopology, - accepted_replica: AcceptedReplicaFn, - dialed_replica: DialedReplicaFn, - accepted_clients: LocalClientAcceptFns, -) -> Result<(), ServerError> { - let bound_replica = if config.cluster.enabled { - let replica_addr = topology - .replica_listen_addr - .expect("cluster-enabled topology must include replica listener address"); - let (replica_listener, bound_addr) = - replica_listener::bind(replica_addr) - .await - .map_err(|source| { - error!( - replica_addr = %replica_addr, - error = %source, - "failed to bind replica listener" - ); - source - })?; - let token = shard.bus.token(); - let replica_handle = compio::runtime::spawn(async move { - replica_listener::run(replica_listener, token, accepted_replica).await; - }); - shard.bus.track_background(replica_handle); - connector::start( - &shard.bus, - topology.self_replica_id, - topology.peers.clone(), - dialed_replica, - shard.bus.config().reconnect_period, - ) - .await; - Some(bound_addr) - } else { - None - }; - - let bound_clients = start_client_listeners(shard, config, topology, &accepted_clients).await?; - write_current_config( - config, - Some(topology.self_replica_id), - bound_clients.tcp, - bound_replica, - bound_clients.tcp_tls, - bound_clients.quic, - bound_clients.ws, - ) - .await?; - - if config.cluster.enabled { - info!( - shard = shard.id, - replica = ?bound_replica, - tcp = ?bound_clients.tcp, - tcp_tls = ?bound_clients.tcp_tls, - ws = ?bound_clients.ws, - quic = ?bound_clients.quic, - "server listeners started" - ); - } else { - info!( - shard = shard.id, - tcp = ?bound_clients.tcp, - tcp_tls = ?bound_clients.tcp_tls, - ws = ?bound_clients.ws, - quic = ?bound_clients.quic, - "server client listeners started" - ); - } - - Ok(()) -} - -fn ensure_default_root_user(mux_stm: &ServerMuxStateMachine) { - if !mux_stm.users().read(|users| users.items.is_empty()) { - return; - } - - let (username, password_hash) = create_root_credentials(); - mux_stm.users().ensure_root_user(&username, &password_hash); -} - -/// Apply `--with-default-root-credentials`. -/// -/// Fills in whichever of [`IGGY_ROOT_USERNAME_ENV`] / -/// [`IGGY_ROOT_PASSWORD_ENV`] the operator did not export, so the flag is -/// exactly the sugar for setting both by hand and the environment keeps -/// winning over it. -/// -/// # Safety -/// -/// Mutates the process environment, so the caller must still be -/// single-threaded. -pub unsafe fn apply_default_root_credentials(enabled: bool) { - if !enabled { - return; - } - - let username_set = env::var(IGGY_ROOT_USERNAME_ENV).is_ok(); - let password_set = env::var(IGGY_ROOT_PASSWORD_ENV).is_ok(); - if username_set && password_set { - warn!( - "--with-default-root-credentials ignored: {IGGY_ROOT_USERNAME_ENV} and \ - {IGGY_ROOT_PASSWORD_ENV} are already set" - ); - return; - } - - // SAFETY: single-threaded caller, per this function's contract. - unsafe { - if !username_set { - env::set_var(IGGY_ROOT_USERNAME_ENV, DEFAULT_ROOT_USERNAME); - } - if !password_set { - env::set_var(IGGY_ROOT_PASSWORD_ENV, DEFAULT_ROOT_PASSWORD); - } - } - warn!( - "--with-default-root-credentials: a newly created root user will use the \ - well-known development credentials; INSECURE outside development" - ); -} - -/// Resolve the root user credentials from `IGGY_ROOT_USERNAME` / -/// `IGGY_ROOT_PASSWORD`, falling back to the default username with a -/// generated password. -/// -/// Returns `(username, password_hash)`; the plaintext password never -/// leaves this function. -fn create_root_credentials() -> (String, String) { - if let Some((username, password)) = root_credentials_from_env() { - info!("Using the custom root user credentials."); - return (username, crypto::hash_password(&password)); - } - - info!("Using the default root user credentials..."); - let password = crypto::generate_secret(20..40); - // Through tracing, not stdout: this is the only time the operator can read - // the password, so it has to reach the log file too. - warn!("Generated root user password: {password}"); - ( - DEFAULT_ROOT_USERNAME.to_string(), - crypto::hash_password(&password), - ) -} - -/// The credentials the operator supplied, `None` when neither variable is -/// set. A half-set pair never reaches here: [`validate_root_credentials`] -/// rejects it at boot. -fn root_credentials_from_env() -> Option<(String, String)> { - match ( - env::var(IGGY_ROOT_USERNAME_ENV), - env::var(IGGY_ROOT_PASSWORD_ENV), - ) { - (Ok(username), Ok(password)) => Some((username, password)), - _ => None, - } -} - -/// Reject root-credential misconfiguration before any shard thread exists. -/// -/// Shard 0 seeds the root user from inside `recover`'s baseline closure, -/// which cannot fail, so every operator-facing check has to run here or it -/// would have to panic a shard thread instead. -fn validate_root_credentials_env(config: &ServerConfig) -> Result<(), ServerError> { - // `recover` creates the metadata directory, so its absence is what tells a - // first cluster boot (root must come out identical on every replica, hence - // explicit credentials) apart from a restart that recovers the root user it - // already stored. `--fresh` has already wiped by this point, so a wiped - // replica is correctly treated as a first boot. - let fresh_cluster = config.cluster.enabled - && !Path::new(&config.system.path) - .join(metadata::impls::METADATA_DIR) - .exists(); - - validate_root_credentials( - fresh_cluster, - env::var(IGGY_ROOT_USERNAME_ENV).ok().as_deref(), - env::var(IGGY_ROOT_PASSWORD_ENV).ok().as_deref(), - ) -} - -fn validate_root_credentials( - explicit_required: bool, - username: Option<&str>, - password: Option<&str>, -) -> Result<(), ServerError> { - match (username, password) { - (Some(username), Some(password)) => { - validate_credential_length( - IGGY_ROOT_USERNAME_ENV, - username, - MIN_USERNAME_LENGTH, - MAX_USERNAME_LENGTH, - )?; - validate_credential_length( - IGGY_ROOT_PASSWORD_ENV, - password, - MIN_PASSWORD_LENGTH, - MAX_PASSWORD_LENGTH, - ) - } - (Some(_), None) => Err(ServerError::RootCredentialsIncomplete { - provided_env: IGGY_ROOT_USERNAME_ENV, - missing_env: IGGY_ROOT_PASSWORD_ENV, - }), - (None, Some(_)) => Err(ServerError::RootCredentialsIncomplete { - provided_env: IGGY_ROOT_PASSWORD_ENV, - missing_env: IGGY_ROOT_USERNAME_ENV, - }), - (None, None) if explicit_required => Err(ServerError::ClusterRootCredentialsRequired { - username_env: IGGY_ROOT_USERNAME_ENV, - password_env: IGGY_ROOT_PASSWORD_ENV, - }), - (None, None) => Ok(()), - } -} - -fn validate_credential_length( - env_name: &'static str, - value: &str, - min: usize, - max: usize, -) -> Result<(), ServerError> { - if (min..=max).contains(&value.len()) { - Ok(()) - } else { - Err(ServerError::RootCredentialLength { - env_name, - length: value.len(), - min, - max, - }) - } -} - -/// Replica delegation callbacks for shard 0's listener and connector. -/// -/// Inbound: acquire a slot in the shard-0-global in-flight handshake cap -/// (drop the connection when full), then blind-delegate the raw fd -/// through the coordinator's round-robin. The fd lands on the target -/// shard's inbox as a [`shard::LifecycleFrame::ReplicaInboundSetup`] -/// frame; the owning shard runs the acceptor handshake and acks the -/// slot back. A failed delegation releases the slot immediately. -/// -/// Outbound: delegate the dialed fd as -/// [`shard::LifecycleFrame::ReplicaOutboundSetup`] and mark the peer -/// dial-pending so the reconnect sweep skips it until the owning -/// shard's handshake outcome arrives (or the entry expires). -fn make_replica_delegation_fns( - coord: Rc, - bus: &Rc, -) -> (AcceptedReplicaFn, DialedReplicaFn) { - let inbound_bus = Rc::clone(bus); - let inbound_coord = Rc::clone(&coord); - let accepted: AcceptedReplicaFn = Rc::new(move |stream| { - let Some(slot) = inbound_bus.try_acquire_replica_handshake_slot() else { - warn!( - cap = MAX_INFLIGHT_REPLICA_HANDSHAKES, - "replica handshake in-flight cap reached; dropping inbound" - ); - return; - }; - match inbound_coord.delegate_replica_inbound(stream, slot) { - Ok(target) => { - info!(slot, target, "inbound replica connection delegated"); - } - Err(error) => { - inbound_bus.release_replica_handshake_slot(slot); - warn!( - error = ?error, - "delegate_replica_inbound failed; dropping inbound replica connection" - ); - } - } - }); - - let outbound_bus = Rc::clone(bus); - let dialed: DialedReplicaFn = - Rc::new( - move |stream, peer_id| match coord.delegate_replica_outbound(stream, peer_id) { - Ok(target) => { - outbound_bus.mark_dial_pending(peer_id); - info!(peer_id, target, "outbound replica connection delegated"); - } - Err(error) => { - warn!( - peer_id, - error = ?error, - "delegate_replica_outbound failed; dropping dialed replica connection" - ); - } - }, - ); - - (accepted, dialed) -} - -/// Shard-0 client accept callbacks. TCP and WS clients are delegated via -/// the coordinator (round-robin to peer shards); QUIC and TCP-TLS install -/// locally on shard 0 because their per-connection state is not portable -/// across shards (`compio_quic` endpoint binds one UDP socket; rustls TLS -/// state ties to the post-handshake reactor). -// ws/wss bindings intentionally mirror the transport names (same convention as -// `replica_io::start_on_shard_zero`). -#[allow(clippy::similar_names)] -fn make_shard_zero_client_accept_fns( - coord: Rc, - bus: &Rc, - on_request: RequestHandler, -) -> LocalClientAcceptFns { - let quic_bus = Rc::clone(bus); - let tcp_tls_bus = Rc::clone(bus); - let wss_bus = Rc::clone(bus); - let quic_request = on_request.clone(); - let wss_request = on_request.clone(); - let tcp_tls_request = on_request; - - let tcp_coord = Rc::clone(&coord); - let tcp = Rc::new(move |stream| match tcp_coord.delegate_client(stream) { - Ok(client_id) => info!(client_id, "TCP client delegated"), - Err(error) => warn!(error = ?error, "delegate_client failed; dropping TCP client"), - }); - - let ws_coord = Rc::clone(&coord); - let ws = Rc::new(move |stream| match ws_coord.delegate_ws_client(stream) { - Ok(client_id) => info!(client_id, "WS client delegated"), - Err(error) => warn!(error = ?error, "delegate_ws_client failed; dropping WS client"), - }); - - // QUIC and TCP-TLS terminate locally on shard 0 but mint their client - // ids through the coordinator's `client_seq`, the same counter the - // delegated TCP/WS path uses. A separate counter here would let a - // shard-0-local id collide with a delegated id that round-robined to - // shard 0 (both encode target shard 0) in shard 0's connection - // registry. - let quic_coord = Rc::clone(&coord); - let quic = Rc::new(move |accepted: message_bus::AcceptedQuicConn| { - let meta = mint_client_meta(&quic_coord, accepted.peer_addr(), ClientTransportKind::Quic); - installer::install_client_quic(&quic_bus, meta, accepted, quic_request.clone()); - }); - - let tcp_tls_coord = Rc::clone(&coord); - let tcp_tls = Rc::new(move |stream, tls_config| { - let Some(meta) = - client_meta_from_stream(&stream, &tcp_tls_coord, ClientTransportKind::TcpTls) - else { - return; - }; - installer::install_client_tcp_tls( - &tcp_tls_bus, - meta, - stream, - tls_config, - tcp_tls_request.clone(), - ); - }); - - // WSS terminates locally on shard 0 like TCP-TLS (rustls state is not - // serialisable across the delegate path), minting ids through the same - // coordinator counter. - let wss_coord = coord; - let wss = Rc::new(move |stream, tls_config| { - let Some(meta) = client_meta_from_stream(&stream, &wss_coord, ClientTransportKind::Wss) - else { - return; - }; - installer::install_client_wss(&wss_bus, meta, stream, tls_config, wss_request.clone()); - }); - - LocalClientAcceptFns { - tcp, - ws, - quic, - tcp_tls, - wss, - } -} - -fn client_meta_from_stream( - stream: &compio::net::TcpStream, - coord: &shard::coordinator::ShardZeroCoordinator, - transport: ClientTransportKind, -) -> Option { - let peer_addr = match stream.peer_addr() { - Ok(peer_addr) => peer_addr, - Err(error) => { - warn!(error = %error, "dropping accepted client with unknown peer address"); - return None; - } - }; - Some(mint_client_meta(coord, peer_addr, transport)) -} - -fn mint_client_meta( - coord: &shard::coordinator::ShardZeroCoordinator, - peer_addr: SocketAddr, - transport: ClientTransportKind, -) -> ClientConnMeta { - ClientConnMeta::new(coord.mint_shard_zero_client_id(), peer_addr, transport) -} - -async fn start_client_listeners( - shard: &Rc, - config: &ServerConfig, - topology: &TcpTopology, - accepted_clients: &LocalClientAcceptFns, -) -> Result { - let mut bound = BoundClientListeners::default(); - - if config.tcp.enabled && !config.tcp.tls.enabled { - let (listener, bound_addr) = client_listener::tcp::bind(topology.client_listen_addr) - .await - .map_err(|source| { - error!( - addr = %topology.client_listen_addr, - error = %source, - "failed to bind TCP client listener" - ); - source - })?; - let token = shard.bus.token(); - let accepted_client = accepted_clients.tcp.clone(); - let client_handle = compio::runtime::spawn(async move { - client_listener::tcp::run(listener, token, accepted_client).await; - }); - shard.bus.track_background(client_handle); - bound.tcp = Some(bound_addr); - } - - if let Some(ws_addr) = topology.ws_listen_addr { - bound.ws = Some(start_websocket_listener(shard, config, ws_addr, accepted_clients).await?); - } - - if let Some(quic_addr) = topology.quic_listen_addr { - install_default_crypto_provider(); - let credentials = load_quic_server_credentials(config)?; - let server_config = server_config_with_cert( - credentials.cert_chain, - credentials.key_der, - &shard.bus.config().quic, - ) - .map_err(|e| { - let source = - iggy_common::IggyError::IoError(format!("QUIC server config build failed: {e}")); - error!(addr = %quic_addr, error = %source, "failed to build QUIC server config"); - source - })?; - let (endpoint, bound_addr) = client_listener::quic::bind(quic_addr, server_config) - .map_err(|source| { - error!(addr = %quic_addr, error = %source, "failed to bind QUIC listener"); - source - })?; - let token = shard.bus.token(); - let handshake_grace = shard.bus.config().handshake_grace; - let accepted_quic = accepted_clients.quic.clone(); - let quic_handle = compio::runtime::spawn(async move { - client_listener::quic::run(endpoint, token, accepted_quic, handshake_grace).await; - }); - shard.bus.track_background(quic_handle); - bound.quic = Some(bound_addr); - } - - if config.tcp.enabled && config.tcp.tls.enabled { - let credentials = load_tcp_tls_server_credentials(config)?; - let (listener, tls_config, bound_addr) = - client_listener::tcp_tls::bind(topology.client_listen_addr, credentials).map_err( - |source| { - error!( - addr = %topology.client_listen_addr, - error = %source, - "failed to bind TCP TLS listener" - ); - source - }, - )?; - let token = shard.bus.token(); - let accepted_tls = accepted_clients.tcp_tls.clone(); - let tls_handle = compio::runtime::spawn(async move { - client_listener::tcp_tls::run(listener, tls_config, token, accepted_tls).await; - }); - shard.bus.track_background(tls_handle); - bound.tcp_tls = Some(bound_addr); - } - - Ok(bound) -} - -/// Build the replica auth context from cluster config. Returns `None` when the -/// cluster or replica auth is disabled, keeping the handshake in legacy mode. -/// Only the derived MAC keys are carried onward in [`ReplicaAuth`]; the raw -/// secrets (masked in config logs via `config_env(secret)`) are read here only -/// to derive them. A non-empty `previous_shared_secret` opens the verify-only -/// rotation acceptance window (see the [`ReplicaAuth`] rustdoc for the rolling -/// rotation procedure). `ClusterConfig::validate` guarantees a non-empty -/// secret whenever both `cluster.enabled` and `cluster.auth.enabled` are set -/// (validate early-returns `Ok` while `cluster.enabled` is false). -fn load_replica_auth(config: &ServerConfig) -> Option { - if !config.cluster.enabled || !config.cluster.auth.enabled { - return None; - } - let auth = ReplicaAuth::new(config.cluster.auth.shared_secret.as_bytes()); - let previous_shared_secret = &config.cluster.auth.previous_shared_secret; - if previous_shared_secret.is_empty() { - return Some(auth); - } - Some(auth.with_previous_secret(previous_shared_secret.as_bytes())) -} - -/// Build the replica TLS context from cluster config. Returns `None` when -/// the cluster or replica TLS is disabled. Every shard calls this once at -/// boot: CA mode re-reads the same PEM files per shard; self-signed mode -/// mints a per-shard throwaway certificate. Neither mode carries client -/// certificates, so TLS authenticates the acceptor only; peer -/// authentication comes from the PSK handshake (`ClusterConfig::validate` -/// enforces `cluster.auth.enabled` whenever `cluster.tls.enabled`). -/// -/// Both rustls configs are TLS 1.3 only with the [`REPLICA_ALPN`] -/// protocol pinned. The dialer's SNI / certificate-verify name for each -/// peer is the roster entry's `ip` field (a hostname or IP literal, the -/// same string the connector dials). -fn load_replica_tls_ctx( - config: &ServerConfig, - topology: &TcpTopology, -) -> Result, ServerError> { - let tls = &config.cluster.tls; - if !config.cluster.enabled || !tls.enabled { - return Ok(None); - } - install_default_crypto_provider(); - let credential_error = |source: std::io::Error| ServerError::ListenerCredentials { - transport: "cluster.tls", - source, - }; - - let credentials = if tls.self_signed { - let san = config - .cluster - .nodes - .iter() - .find(|node| node.replica_id == topology.self_replica_id) - .map(|node| node.ip.as_str()) - .ok_or_else(|| { - credential_error(std::io::Error::other(format!( - "replica id {} not present in cluster.nodes", - topology.self_replica_id - ))) - })?; - let (cert_chain, key_der) = server_common::generate_self_signed_certificate(san) - .map_err(|error| credential_error(std::io::Error::other(error.to_string())))?; - TlsServerCredentials { - cert_chain, - key_der, - } - } else { - load_pem(Path::new(&tls.cert_file), Path::new(&tls.key_file)).map_err(credential_error)? - }; - - let mut server = - rustls::ServerConfig::builder_with_protocol_versions(&[&rustls::version::TLS13]) - .with_no_client_auth() - .with_single_cert(credentials.cert_chain, credentials.key_der) - .map_err(|error| { - credential_error(std::io::Error::other(format!( - "replica TLS server config rejected credentials: {error}" - ))) - })?; - server.alpn_protocols = vec![REPLICA_ALPN.to_vec()]; - - let client_builder = - rustls::ClientConfig::builder_with_protocol_versions(&[&rustls::version::TLS13]); - let mut client = if tls.self_signed { - client_builder - .dangerous() - .with_custom_certificate_verifier(Arc::new(AcceptAnyServerCert)) - .with_no_client_auth() - } else { - let roots = load_ca_pem(Path::new(&tls.ca_file)).map_err(credential_error)?; - client_builder - .with_root_certificates(Arc::new(roots)) - .with_no_client_auth() - }; - client.alpn_protocols = vec![REPLICA_ALPN.to_vec()]; - - // Keyed by replica id, never by roster position: sparse ids (dynamic - // replica join) would make a positional lookup verify against another - // peer's SNI name. - let peer_names = config - .cluster - .nodes - .iter() - .map(|node| { - let name = ServerName::try_from(node.ip.clone()).map_err(|error| { - credential_error(std::io::Error::new( - std::io::ErrorKind::InvalidInput, - format!( - "cluster node '{}' ip '{}' is not a valid TLS server name: {error}", - node.name, node.ip - ), - )) - })?; - Ok((node.replica_id, name)) - }) - .collect::, ServerError>>()?; - - Ok(Some(ReplicaTlsCtx { - server: Arc::new(server), - client: Arc::new(client), - peer_names, - })) -} - -fn load_tcp_tls_server_credentials( - config: &ServerConfig, -) -> Result { - let tls = &config.tcp.tls; - if tls.self_signed && !Path::new(&tls.cert_file).exists() { - return Ok(self_signed_for_loopback()); - } - - load_pem(Path::new(&tls.cert_file), Path::new(&tls.key_file)).map_err(|source| { - ServerError::ListenerCredentials { - transport: "tcp.tls", - source, - } - }) -} - -/// Bind the websocket client listener on `ws_addr`: WSS when -/// `websocket.tls.enabled` (the plain-WS accept loop must not also bind the -/// port -- a plain upgrade parser fed a TLS `ClientHello` rejects every -/// connection with an httparse error), plain WS otherwise. -async fn start_websocket_listener( - shard: &Rc, - config: &ServerConfig, - ws_addr: SocketAddr, - accepted_clients: &LocalClientAcceptFns, -) -> Result { - if config.websocket.tls.enabled { - let credentials = load_wss_server_credentials(config)?; - let (listener, tls_config, bound_addr) = client_listener::wss::bind(ws_addr, credentials) - .map_err(|source| { - error!(addr = %ws_addr, error = %source, "failed to bind WSS listener"); - source - })?; - let token = shard.bus.token(); - let accepted_wss = accepted_clients.wss.clone(); - let wss_handle = compio::runtime::spawn(async move { - client_listener::wss::run(listener, tls_config, token, accepted_wss).await; - }); - shard.bus.track_background(wss_handle); - Ok(bound_addr) - } else { - let (listener, bound_addr) = - client_listener::ws::bind(ws_addr).await.map_err(|source| { - error!(addr = %ws_addr, error = %source, "failed to bind websocket listener"); - source - })?; - let token = shard.bus.token(); - let accepted_ws = accepted_clients.ws.clone(); - let ws_handle = compio::runtime::spawn(async move { - client_listener::ws::run(listener, token, accepted_ws).await; - }); - shard.bus.track_background(ws_handle); - Ok(bound_addr) - } -} - -fn load_wss_server_credentials(config: &ServerConfig) -> Result { - let tls = &config.websocket.tls; - if tls.self_signed && !Path::new(&tls.cert_file).exists() { - return Ok(self_signed_for_loopback()); - } - - load_pem(Path::new(&tls.cert_file), Path::new(&tls.key_file)).map_err(|source| { - ServerError::ListenerCredentials { - transport: "websocket.tls", - source, - } - }) -} - -fn load_quic_server_credentials( - config: &ServerConfig, -) -> Result { - let certificate = &config.quic.certificate; - if certificate.self_signed { - let (cert_chain, key_der) = server_common::generate_self_signed_certificate("localhost") - .map_err(|error| ServerError::ListenerCredentials { - transport: "quic", - source: std::io::Error::other(error.to_string()), - })?; - return Ok(replica_io::QuicServerCredentials { - cert_chain, - key_der, - }); - } - - let credentials = load_pem( - Path::new(&certificate.cert_file), - Path::new(&certificate.key_file), - ) - .map_err(|source| ServerError::ListenerCredentials { - transport: "quic", - source, - })?; - Ok(replica_io::QuicServerCredentials { - cert_chain: credentials.cert_chain, - key_der: credentials.key_der, - }) -} - -fn parse_socket_addr(context: &'static str, address: &str) -> Result { - address - .parse() - .map_err(|source| ServerError::SocketAddressParse { - context, - address: address.to_string(), - source, - }) -} - -fn socket_addr_from_parts( - context: &'static str, - host: &str, - port: u16, -) -> Result { - let ip = host - .parse::() - .map_err(|source| ServerError::SocketAddressParse { - context, - address: format!("{host}:{port}"), - source, - })?; - Ok(SocketAddr::new(ip, port)) -} - -/// Build the closure that broadcasts a -/// [`LifecycleFrame::MetadataCommitTick`] to every shard's inbox after a -/// partition-shaped metadata operation commits on shard 0. -/// -/// The receiver-side partition reconciliation loop listens for these -/// wake-ups; coalescing is intentional, so `Full` is recorded as a metric -/// and dropped (the periodic tick recovers). Installed via -/// [`metadata::IggyMetadata::set_commit_notifier`] on shard 0 only, the -/// sole writer of the metadata state machine. -fn make_metadata_commit_notifier( - senders: Vec, - metrics: ShardMetrics, -) -> metadata::CommitNotifier { - Rc::new(move |operation: Operation| { - if !operation_triggers_partition_reconcile(operation) { - return; - } - for sender in &senders { - let frame = ShardFrame::lifecycle(LifecycleFrame::MetadataCommitTick); - match sender.try_send(frame) { - Ok(()) => {} - Err(crossfire::TrySendError::Full(_)) => { - metrics.record_frame_drop( - frame_drop_variant::METADATA_COMMIT_TICK, - frame_drop_reason::FULL, - ); - } - Err(crossfire::TrySendError::Disconnected(_)) => { - metrics.record_frame_drop( - frame_drop_variant::METADATA_COMMIT_TICK, - frame_drop_reason::DISCONNECTED, - ); - } - } - } - }) -} - -/// Filter at the broadcast site, keeping unrelated ops off the SDK reply -/// path. Any new partition-shape op must be added here. -/// -/// The bare `CreateTopic` / `CreatePartitions` arms are unreachable: the -/// leader's prepare-builder in `IggyMetadata` rewrites both into their -/// `*WithAssignments` form, stamping each partition's `consensus_group_id` -/// before journaling, so a committed prepare only ever carries the -/// assignment-bearing variant. Kept as defense-in-depth against a future -/// commit path that emits a bare op. -/// -/// "Partition-shape" is not only the partition SET: the purge and truncate -/// ops leave the set intact but advance per-partition state (purge -/// generation, delete watermark) that only the reconciler enforces on disk. -/// Omitting them defers the on-disk effect to the periodic safety tick, -/// stretching a purge's client-visible tail to a full -/// `reconcile_periodic_interval`. `DeleteSegments` is absent by design: the -/// leader rewrites it into `TruncatePartition` before journaling, so no -/// commit ever carries it. -const fn operation_triggers_partition_reconcile(op: Operation) -> bool { - matches!( - op, - Operation::CreateTopic - | Operation::CreateTopicWithAssignments - | Operation::CreatePartitions - | Operation::CreatePartitionsWithAssignments - | Operation::DeleteTopic - | Operation::DeleteStream - | Operation::DeletePartitions - | Operation::PurgeStream - | Operation::PurgeTopic - | Operation::TruncatePartition - ) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn fresh_cluster_bootstrap_requires_explicit_root_credentials() { - assert!(matches!( - validate_root_credentials(true, None, None), - Err(ServerError::ClusterRootCredentialsRequired { - username_env: IGGY_ROOT_USERNAME_ENV, - password_env: IGGY_ROOT_PASSWORD_ENV, - }) - )); - validate_root_credentials(true, Some("root"), Some("secret")) - .expect("both credentials supplied must satisfy the fresh-cluster guard"); - } - - #[test] - fn single_node_bootstrap_generates_root_credentials_when_unset() { - validate_root_credentials(false, None, None) - .expect("a single node mints its own root password"); - } - - #[test] - fn half_set_root_credentials_are_rejected_in_both_directions() { - assert!(matches!( - validate_root_credentials(false, Some("root"), None), - Err(ServerError::RootCredentialsIncomplete { - provided_env: IGGY_ROOT_USERNAME_ENV, - missing_env: IGGY_ROOT_PASSWORD_ENV, - }) - )); - assert!(matches!( - validate_root_credentials(false, None, Some("secret")), - Err(ServerError::RootCredentialsIncomplete { - provided_env: IGGY_ROOT_PASSWORD_ENV, - missing_env: IGGY_ROOT_USERNAME_ENV, - }) - )); - } - - #[test] - fn out_of_range_root_credentials_are_rejected() { - assert!(matches!( - validate_root_credentials(false, Some(""), Some("secret")), - Err(ServerError::RootCredentialLength { - env_name: IGGY_ROOT_USERNAME_ENV, - length: 0, - .. - }) - )); - let too_long = "x".repeat(MAX_PASSWORD_LENGTH + 1); - assert!(matches!( - validate_root_credentials(false, Some("root"), Some(&too_long)), - Err(ServerError::RootCredentialLength { - env_name: IGGY_ROOT_PASSWORD_ENV, - .. - }) - )); - } - - #[test] - fn default_cluster_heartbeat_timeout_matches_consensus_constant() { - // The config default lives in core/server/config.toml (a string, - // so no static assert can pin it); keep it in lockstep with the - // built-in the simulator and un-configured replicas run on. - let config_default = configs::cluster::ClusterConfig::default() - .heartbeat_timeout - .get_duration() - .as_millis(); - let built_in = u128::from(consensus::TimeoutManager::NORMAL_HEARTBEAT_TICKS) - * shard::CONSENSUS_TICK_INTERVAL.as_millis(); - assert_eq!( - config_default, built_in, - "[cluster] heartbeat_timeout default drifted from \ - TimeoutManager::NORMAL_HEARTBEAT_TICKS" - ); - } - - #[test] - fn reconciler_driven_ops_broadcast_a_commit_tick() { - // These commit without touching the partition set, so nothing else - // signals the reconciler: `reconcile_partition_purges` and - // `reconcile_segment_truncations` are the only code that turns them - // into on-disk effect, and they run only when a pass runs. Dropping - // one from the filter silently downgrades it to the periodic tick. - for op in [ - Operation::PurgeStream, - Operation::PurgeTopic, - Operation::TruncatePartition, - ] { - assert!( - operation_triggers_partition_reconcile(op), - "{op:?} is enforced by the reconciler and must wake it on commit" - ); - } - assert!( - !operation_triggers_partition_reconcile(Operation::CreateUser), - "ops with no partition-shape effect must stay off the broadcast" - ); - } - - #[test] - fn recovery_barrier_deadline_holds_the_floor_for_small_heartbeats() { - // Below the 5s default the heartbeat-independent recovery term (~7s of - // ViewChangeStatus backstop plus ceremony) dominates, so the floor - // governs however small the heartbeat is; 3 x 5s lands exactly on it. - // A default-sized status backstop stays on the floor, not above it. - assert_eq!( - recovery_barrier_deadline(Duration::from_secs(1), Duration::from_secs(5)), - RECOVERY_BARRIER_DEADLINE_FLOOR - ); - assert_eq!( - recovery_barrier_deadline(Duration::from_secs(5), Duration::from_secs(5)), - RECOVERY_BARRIER_DEADLINE_FLOOR - ); - } - - #[test] - fn recovery_barrier_deadline_scales_past_the_floor_for_large_heartbeats() { - // Once 3 x heartbeat clears the floor the scaled window governs, so a - // slow-heartbeat cluster is not failed 503 before its longer recovery - // can finish. A default-sized status backstop stays under it. - assert_eq!( - recovery_barrier_deadline(Duration::from_secs(10), Duration::from_secs(5)), - Duration::from_secs(30) - ); - assert_eq!( - recovery_barrier_deadline(Duration::from_secs(15), Duration::from_secs(5)), - Duration::from_secs(45) - ); - } - - #[test] - fn recovery_barrier_deadline_scales_with_the_status_backstop() { - // A raised view-change status backstop stretches worst-case recovery - // even when the heartbeat stays fast, so the deadline must track it or - // post-restart reads 503 before a slow election settles. - assert_eq!( - recovery_barrier_deadline(Duration::from_secs(1), Duration::from_secs(10)), - Duration::from_secs(30) - ); - } - - #[test] - fn recovery_barrier_deadline_at_config_defaults_matches_the_floor() { - // Folding the status term in must not move the stock deadline: at the - // shared 5s defaults each scaled term lands exactly on the 15s floor, - // so an un-tuned cluster keeps its pre-existing recovery window. - let cluster = configs::cluster::ClusterConfig::default(); - assert_eq!( - recovery_barrier_deadline( - cluster.heartbeat_timeout.get_duration(), - cluster.view_change_status_timeout.get_duration(), - ), - RECOVERY_BARRIER_DEADLINE_FLOOR - ); - } - - #[test] - fn recovery_barrier_deadline_saturates_instead_of_panicking() { - // Neither timeout has a config ceiling, so both multiplies must - // saturate rather than abort boot on an absurd parseable value. - assert_eq!( - recovery_barrier_deadline(Duration::MAX, Duration::from_secs(5)), - Duration::MAX - ); - assert_eq!( - recovery_barrier_deadline(Duration::from_secs(5), Duration::MAX), - Duration::MAX - ); - } - - #[test] - fn default_commit_broadcast_interval_matches_consensus_constant() { - // The config default lives in core/server/config.toml (a string, - // so no static assert can pin it); keep it in lockstep with the - // built-in the simulator and un-configured replicas run on. - let config_default = configs::cluster::ClusterConfig::default() - .commit_broadcast_interval - .get_duration() - .as_millis(); - let built_in = u128::from(consensus::TimeoutManager::COMMIT_MESSAGE_TICKS) - * shard::CONSENSUS_TICK_INTERVAL.as_millis(); - assert_eq!( - config_default, built_in, - "[cluster] commit_broadcast_interval default drifted from \ - TimeoutManager::COMMIT_MESSAGE_TICKS" - ); - } - - #[test] - fn default_prepare_retransmit_interval_matches_consensus_constant() { - // The config default lives in core/server/config.toml (a string, - // so no static assert can pin it); keep it in lockstep with the - // built-in the simulator and un-configured replicas run on. - let config_default = configs::cluster::ClusterConfig::default() - .prepare_retransmit_interval - .get_duration() - .as_millis(); - let built_in = u128::from(consensus::TimeoutManager::PREPARE_TICKS) - * shard::CONSENSUS_TICK_INTERVAL.as_millis(); - assert_eq!( - config_default, built_in, - "[cluster] prepare_retransmit_interval default drifted from \ - TimeoutManager::PREPARE_TICKS" - ); - } - - #[test] - fn default_partition_prepare_queue_depth_matches_consensus_constant() { - // The config default lives in core/server/config.toml and flows - // through PartitionConfig::default(); keep the embedded value in - // lockstep with the pipeline depth LocalPipeline::new() (the simulator - // and tests) runs on, so a default deployment is byte-identical. - let config_default = configs::partition::PartitionConfig::default().prepare_queue_depth; - assert_eq!( - config_default, - consensus::PIPELINE_PREPARE_QUEUE_MAX, - "[partition] prepare_queue_depth default drifted from \ - consensus::PIPELINE_PREPARE_QUEUE_MAX" - ); - } - - #[test] - fn default_view_change_retransmit_interval_matches_consensus_constant() { - // The config default lives in core/server/config.toml (a string, so - // no static assert can pin it). One knob drives both view-change - // retransmit timers, which are equal by design, so pin it against both. - let config_default = configs::cluster::ClusterConfig::default() - .view_change_retransmit_interval - .get_duration() - .as_millis(); - let start_view_change = - u128::from(consensus::TimeoutManager::START_VIEW_CHANGE_MESSAGE_TICKS) - * shard::CONSENSUS_TICK_INTERVAL.as_millis(); - let do_view_change = u128::from(consensus::TimeoutManager::DO_VIEW_CHANGE_MESSAGE_TICKS) - * shard::CONSENSUS_TICK_INTERVAL.as_millis(); - assert_eq!( - config_default, start_view_change, - "[cluster] view_change_retransmit_interval default drifted from \ - TimeoutManager::START_VIEW_CHANGE_MESSAGE_TICKS" - ); - assert_eq!( - config_default, do_view_change, - "[cluster] view_change_retransmit_interval default drifted from \ - TimeoutManager::DO_VIEW_CHANGE_MESSAGE_TICKS" - ); - } - - #[test] - fn default_view_change_status_timeout_matches_consensus_constant() { - // The config default lives in core/server/config.toml (a string, so - // no static assert can pin it); keep it in lockstep with the built-in - // the simulator and un-configured replicas run on. - let config_default = configs::cluster::ClusterConfig::default() - .view_change_status_timeout - .get_duration() - .as_millis(); - let built_in = u128::from(consensus::TimeoutManager::VIEW_CHANGE_STATUS_TICKS) - * shard::CONSENSUS_TICK_INTERVAL.as_millis(); - assert_eq!( - config_default, built_in, - "[cluster] view_change_status_timeout default drifted from \ - TimeoutManager::VIEW_CHANGE_STATUS_TICKS" - ); - } - - #[test] - fn default_request_start_view_retransmit_interval_matches_consensus_constant() { - // The config default lives in core/server/config.toml (a string, so - // no static assert can pin it); keep it in lockstep with the built-in - // the simulator and un-configured replicas run on. - let config_default = configs::cluster::ClusterConfig::default() - .request_start_view_retransmit_interval - .get_duration() - .as_millis(); - let built_in = u128::from(consensus::TimeoutManager::REQUEST_START_VIEW_MESSAGE_TICKS) - * shard::CONSENSUS_TICK_INTERVAL.as_millis(); - assert_eq!( - config_default, built_in, - "[cluster] request_start_view_retransmit_interval default drifted from \ - TimeoutManager::REQUEST_START_VIEW_MESSAGE_TICKS" - ); - } - - #[test] - fn default_view_probe_attempts_max_matches_consensus_constant() { - // Belt and suspenders with the static assert above: that pins the - // duplicated configs-crate literal, this pins the shipped config.toml - // value the simulator and un-configured replicas run on. - let config_default = configs::cluster::ClusterConfig::default().view_probe_attempts_max; - assert_eq!( - config_default, - consensus::PROBE_ATTEMPTS_MAX, - "[cluster] view_probe_attempts_max default drifted from \ - consensus::PROBE_ATTEMPTS_MAX" - ); - } - - #[test] - fn default_repair_retry_interval_matches_partitions_constant() { - // The config default lives in core/server/config.toml (a string, so - // no static assert can pin it); keep it in lockstep with the built-in - // the simulator and un-configured replicas run on. - let config_default = configs::cluster::ClusterConfig::default() - .repair_retry_interval - .get_duration() - .as_millis(); - let built_in = - u128::from(partitions::REPAIR_RETRY_TICKS) * shard::CONSENSUS_TICK_INTERVAL.as_millis(); - assert_eq!( - config_default, built_in, - "[cluster] repair_retry_interval default drifted from \ - partitions::REPAIR_RETRY_TICKS" - ); - } - - #[test] - fn default_repair_chunk_max_matches_shard_constant() { - // Belt and suspenders with the static assert above: that pins the - // duplicated configs-crate literal, this pins the shipped config.toml - // value the simulator and un-configured replicas run on. - let config_default = configs::cluster::ClusterConfig::default().repair_chunk_max; - assert_eq!( - config_default as u64, - shard::REPAIR_CHUNK_MAX, - "[cluster] repair_chunk_max default drifted from shard::REPAIR_CHUNK_MAX" - ); - } - - #[test] - fn default_evicted_ring_capacity_matches_partitions_constant() { - // Belt and suspenders with the static assert above; this pins the - // shipped config.toml value. - let config_default = configs::partition::PartitionConfig::default().evicted_ring_capacity; - assert_eq!( - config_default, - partitions::EVICTED_RING_CAPACITY, - "[partition] evicted_ring_capacity default drifted from \ - partitions::EVICTED_RING_CAPACITY" - ); - } - - #[test] - fn default_evicted_ring_bytes_max_matches_partitions_constant() { - // Belt and suspenders with the static assert above; this pins the - // shipped config.toml value. - let config_default = configs::partition::PartitionConfig::default() - .evicted_ring_bytes_max - .as_bytes_u64(); - assert_eq!( - config_default, - partitions::EVICTED_RING_BYTES_MAX, - "[partition] evicted_ring_bytes_max default drifted from \ - partitions::EVICTED_RING_BYTES_MAX" - ); - } - - #[test] - fn shutdown_on_drop_armed_flips_flag() { - let flag = Arc::new(AtomicBool::new(false)); - drop(ShutdownOnDrop::new(Arc::clone(&flag))); - assert!( - flag.load(Ordering::Relaxed), - "an armed guard must flip the flag on drop (covers the error `?` \ - and panic-unwind exit paths of run_shard_thread)" - ); - } - - #[test] - fn shutdown_on_drop_disarmed_leaves_flag() { - let flag = Arc::new(AtomicBool::new(false)); - let mut guard = ShutdownOnDrop::new(Arc::clone(&flag)); - guard.disarm(); - drop(guard); - assert!( - !flag.load(Ordering::Relaxed), - "a disarmed guard must not flip the flag (clean `Ok(())` exit)" - ); - } - - const TEST_POLL_INTERVAL: Duration = Duration::from_millis(50); - - #[compio::test] - async fn broadcast_metadata_bundle_returns_immediately_with_no_peers() { - // Single-shard deployment: shard 0 has no peers to fan out to, - // so the handoff must complete without ever calling `send`. - let (bundle_tx, _bundle_rx) = crossfire::mpmc::bounded_async::(0); - let flag = Arc::new(AtomicBool::new(false)); - let mux = ServerMuxStateMachine::default(); - broadcast_metadata_bundle( - 0, - &bundle_tx, - mux.factory_bundle(), - 0, - &flag, - TEST_POLL_INTERVAL, - ) - .await - .expect("zero peers must not block shard 0"); - } - - #[compio::test] - async fn metadata_bundle_round_trips_through_channel() { - // End-to-end: shard 0 mints a bundle, a peer receives it on - // another runtime, and `from_factory_bundle` constructs a - // reader-mode mux that observes shard 0's writes via the same - // LeftRight pair. - let peers = 1u16; - let (bundle_tx, bundle_rx) = - crossfire::mpmc::bounded_async::(usize::from(peers)); - let flag = Arc::new(AtomicBool::new(false)); - - let owner = ServerMuxStateMachine::default(); - let bundle = owner.factory_bundle(); - broadcast_metadata_bundle(0, &bundle_tx, bundle, peers, &flag, TEST_POLL_INTERVAL) - .await - .expect("broadcast must succeed with one peer drained"); - - let received = await_metadata_bundle(1, &bundle_rx, &flag, TEST_POLL_INTERVAL) - .await - .expect("peer must receive the broadcast bundle"); - let _peer_mux = ServerMuxStateMachine::from_factory_bundle(received); - } - - #[compio::test] - async fn broadcast_metadata_bundle_aborts_when_peers_drop_rx() { - // Shard 0 drives handoff but every peer's `bundle_rx` was dropped - // before recv. Silently returning Ok would commit listener binds - // and consensus init for a cluster whose peers are gone; the - // broadcast must surface the disconnect so `shard_main` aborts. - let (bundle_tx, bundle_rx) = crossfire::mpmc::bounded_async::(0); - drop(bundle_rx); - let flag = Arc::new(AtomicBool::new(false)); - let mux = ServerMuxStateMachine::default(); - - let err = broadcast_metadata_bundle( - 0, - &bundle_tx, - mux.factory_bundle(), - 3, - &flag, - TEST_POLL_INTERVAL, - ) - .await - .expect_err("dropped rx must surface as MetadataHandoffAborted"); - assert!( - matches!(err, ServerError::MetadataHandoffAborted { shard_id: 0 }), - "expected MetadataHandoffAborted, got {err:?}" - ); - } - - #[compio::test] - async fn await_metadata_bundle_aborts_when_owner_drops_without_sending() { - let (bundle_tx, bundle_rx) = crossfire::mpmc::bounded_async::(1); - let flag = Arc::new(AtomicBool::new(false)); - - // Shard 0 dies before broadcasting; the peer must observe the - // disconnect and abort instead of hanging forever. - drop(bundle_tx); - - let err = await_metadata_bundle(1, &bundle_rx, &flag, TEST_POLL_INTERVAL) - .await - .expect_err("a peer whose owner never sends must abort"); - assert!( - matches!(err, ServerError::MetadataHandoffAborted { shard_id: 1 }), - "expected MetadataHandoffAborted, got {err:?}" - ); - } - - #[compio::test] - async fn await_metadata_bundle_aborts_on_shutdown_flag() { - // compio 0.19 `JoinHandle` yields `Result`; the - // `ResumeUnwind` impl re-raises a task panic and maps cancellation - // to `None`. - use compio::runtime::ResumeUnwind; - - let (_bundle_tx, bundle_rx) = crossfire::mpmc::bounded_async::(1); - let flag = Arc::new(AtomicBool::new(false)); - - let waiter = compio::runtime::spawn({ - let flag = Arc::clone(&flag); - async move { await_metadata_bundle(1, &bundle_rx, &flag, TEST_POLL_INTERVAL).await } - }); - - // Owner has not sent yet, but shutdown was requested; the peer - // must exit via the flag poll instead of hanging. - compio::time::sleep(TEST_POLL_INTERVAL / 2).await; - flag.store(true, Ordering::Relaxed); - - let err = waiter - .await - .resume_unwind() - .expect("waiter task was cancelled") - .expect_err("shutdown flag must abort the bundle wait"); - assert!( - matches!(err, ServerError::MetadataHandoffAborted { shard_id: 1 }), - "expected MetadataHandoffAborted on shutdown, got {err:?}" - ); - } - - #[compio::test] - async fn await_bootstrap_complete_returns_immediately_for_single_shard() { - // A single-shard server has no peers to wait on; the owner barrier - // must not block when `peers == 0`. - let (_ready_tx, ready_rx) = crossfire::mpmc::bounded_async::(1); - let flag = Arc::new(AtomicBool::new(false)); - await_bootstrap_complete(&ready_rx, 0, &flag, TEST_POLL_INTERVAL) - .await - .expect("single-shard server must not block on the barrier"); - } - - #[compio::test] - async fn await_bootstrap_complete_drains_every_peer_signal() { - // Two peers report load-complete; shard 0 drains both, then proceeds - // to bind listeners. - let (ready_tx, ready_rx) = crossfire::mpmc::bounded_async::(2); - let flag = Arc::new(AtomicBool::new(false)); - signal_bootstrap_complete(1, &ready_tx, &flag, TEST_POLL_INTERVAL) - .await - .expect("peer 1 must signal load-complete"); - signal_bootstrap_complete(2, &ready_tx, &flag, TEST_POLL_INTERVAL) - .await - .expect("peer 2 must signal load-complete"); - await_bootstrap_complete(&ready_rx, 2, &flag, TEST_POLL_INTERVAL) - .await - .expect("owner must drain both peer signals"); - } - - #[compio::test] - async fn await_bootstrap_complete_aborts_on_shutdown_flag() { - use compio::runtime::ResumeUnwind; - - // `_ready_tx` is held so the channel is not disconnected: the owner - // must exit via the shutdown flag, not a dropped sender. - let (_ready_tx, ready_rx) = crossfire::mpmc::bounded_async::(1); - let flag = Arc::new(AtomicBool::new(false)); - - let owner = compio::runtime::spawn({ - let flag = Arc::clone(&flag); - async move { await_bootstrap_complete(&ready_rx, 1, &flag, TEST_POLL_INTERVAL).await } - }); - - // The peer never signals, but a sibling failure flips the flag; the - // owner must abort instead of hanging before listeners. - compio::time::sleep(TEST_POLL_INTERVAL / 2).await; - flag.store(true, Ordering::Relaxed); - - let err = owner - .await - .resume_unwind() - .expect("owner task was cancelled") - .expect_err("shutdown flag must abort the barrier wait"); - assert!( - matches!( - err, - ServerError::ShardBootstrapBarrierAborted { remaining: 1 } - ), - "expected ShardBootstrapBarrierAborted, got {err:?}" - ); - } - - #[compio::test] - async fn signal_bootstrap_complete_aborts_when_owner_drops_rx() { - // Shard 0 aborted before draining and dropped its receiver; a peer's - // signal must surface the disconnect instead of stranding. - let (ready_tx, ready_rx) = crossfire::mpmc::bounded_async::(1); - let flag = Arc::new(AtomicBool::new(false)); - drop(ready_rx); - - let err = signal_bootstrap_complete(2, &ready_tx, &flag, TEST_POLL_INTERVAL) - .await - .expect_err("dropped rx must surface as an abort"); - assert!( - matches!(err, ServerError::MetadataHandoffAborted { shard_id: 2 }), - "expected MetadataHandoffAborted, got {err:?}" - ); - } - - fn cluster_node(ip: &str, http: Option) -> configs::cluster::ClusterNodeConfig { - cluster_node_with_ports(ip, Some(18070), http) - } - - fn cluster_node_with_ports( - ip: &str, - tcp: Option, - http: Option, - ) -> configs::cluster::ClusterNodeConfig { - configs::cluster::ClusterNodeConfig { - name: "node".to_owned(), - ip: ip.to_owned(), - advertised_address: None, - advertised_addresses: Vec::new(), - replica_id: 0, - ports: configs::cluster::TransportPorts { - tcp, - http, - ..Default::default() - }, - } - } - - fn addr(value: &str) -> SocketAddr { - value.parse().expect("valid socket address literal") - } - - #[test] - fn cluster_http_addr_takes_port_from_roster() { - // A byte-identical top-level [http].address is shared across nodes on - // one host; the per-node roster port is the only port source so each - // node binds a distinct HTTP socket. - let node = cluster_node("127.0.0.1", Some(18090)); - let addrs = resolve_cluster_client_addrs( - &node, - addr("127.0.0.1:8090"), - None, - None, - Some(addr("127.0.0.1:3000")), - ) - .expect("cluster address resolution must succeed"); - assert_eq!(addrs.http, Some(addr("127.0.0.1:18090"))); - } - - #[test] - fn cluster_http_addr_merges_config_ip_with_roster_port() { - // Docker/Helm bind `0.0.0.0` and probe loopback; the roster ip is - // only the advertised address. Cluster mode must keep the configured - // interface and take just the port from the roster. - let node = cluster_node("10.0.0.5", Some(18090)); - let addrs = resolve_cluster_client_addrs( - &node, - addr("0.0.0.0:8090"), - None, - None, - Some(addr("0.0.0.0:3000")), - ) - .expect("cluster address resolution must succeed"); - assert_eq!(addrs.http, Some(addr("0.0.0.0:18090"))); - } - - #[test] - fn cluster_http_addr_requires_roster_port_for_enabled_transport() { - // No fallback to the top-level port: a silent default could collide - // with another same-host node, so a missing roster port for an - // enabled transport must refuse to boot. - let node = cluster_node("10.0.0.5", None); - let result = resolve_cluster_client_addrs( - &node, - addr("127.0.0.1:8090"), - None, - None, - Some(addr("127.0.0.1:3000")), - ); - assert!(matches!( - result, - Err(ServerError::ClusterPortMissing { - transport: "http", - replica_id: 0, - }) - )); - } - - #[test] - fn cluster_http_addr_is_none_when_http_disabled() { - // http.enabled = false collapses default_http_addr to None; no roster - // port can revive a listener the operator turned off. - let node = cluster_node("127.0.0.1", Some(18090)); - let addrs = resolve_cluster_client_addrs(&node, addr("127.0.0.1:8090"), None, None, None) - .expect("cluster address resolution must succeed"); - assert_eq!(addrs.http, None); - } - - /// Regression: the shutdown-join deadline must arm at SHUTDOWN, not - /// at boot. The original bound measured from `join_all` entry, so any - /// healthy server outliving `shutdown_join_timeout` (30s default) was - /// abandoned as "wedged" and the process exited - every BDD run died - /// at t+30s while the test container was still compiling. - #[test] - fn join_waits_unbounded_while_the_server_runs() { - let shutdown_flag = AtomicBool::new(false); - // Thread outlives a deliberately tiny join budget; with the flag - // clear the budget must never even arm. - let handle = thread::spawn(|| -> Result<(), ServerError> { - thread::sleep(Duration::from_millis(300)); - Ok(()) - }); - let mut deadline = None; - let joined = join_until_shutdown_deadline( - handle, - &shutdown_flag, - Duration::from_millis(20), - &mut deadline, - ); - assert!( - matches!(joined, Some(Ok(Ok(())))), - "a running server must be awaited indefinitely, not abandoned as wedged" - ); - assert!( - deadline.is_none(), - "the join deadline must not arm before the shutdown flag flips" - ); - } - - #[test] - fn join_abandons_a_wedged_shard_after_the_shutdown_deadline() { - let shutdown_flag = AtomicBool::new(true); - // Never finishes: stands in for a wedged pump. The thread leaks - // into the test process, which exits right after. - let handle = thread::spawn(|| -> Result<(), ServerError> { - loop { - thread::sleep(Duration::from_secs(1)); - } - }); - let mut deadline = None; - let joined = join_until_shutdown_deadline( - handle, - &shutdown_flag, - Duration::from_millis(100), - &mut deadline, - ); - assert!( - joined.is_none(), - "a shard still running past the post-shutdown budget must be abandoned" - ); - assert!(deadline.is_some(), "the deadline arms once the flag is set"); - } - - #[test] - fn cluster_tcp_addr_takes_port_from_roster() { - // Same rule as the other transports: the roster owns the port so - // same-host nodes sharing one [tcp].address still bind distinct - // sockets. - let node = cluster_node("127.0.0.1", None); - let addrs = resolve_cluster_client_addrs(&node, addr("127.0.0.1:8090"), None, None, None) - .expect("cluster address resolution must succeed"); - assert_eq!(addrs.client, addr("127.0.0.1:18070")); - } - - #[test] - fn cluster_tcp_addr_merges_config_ip_with_roster_port() { - // The roster ip is advertised, not bound. Binding it directly would - // strand every co-located dialer (sidecars, health probes, on-host - // consumers) that reaches this node over loopback. - let node = cluster_node("10.0.0.5", None); - let addrs = resolve_cluster_client_addrs(&node, addr("0.0.0.0:8090"), None, None, None) - .expect("cluster address resolution must succeed"); - assert_eq!(addrs.client, addr("0.0.0.0:18070")); - } - - #[test] - fn cluster_tcp_addr_requires_roster_port() { - // tcp is always enabled in cluster mode, so a roster entry without a - // tcp port refuses to boot rather than falling back to [tcp].address. - let node = cluster_node_with_ports("10.0.0.5", None, None); - let result = resolve_cluster_client_addrs(&node, addr("127.0.0.1:8090"), None, None, None); - assert!(matches!( - result, - Err(ServerError::ClusterPortMissing { - transport: "tcp", - replica_id: 0, - }) - )); - } - - #[test] - fn cluster_tcp_addr_keeps_loopback_bind_and_warns_on_roster_mismatch() { - // A loopback [tcp].address under a routable roster ip is honoured - // as configured; remote peers cannot reach it, so the mismatch is - // warned about instead of silently rebinding. - let node = cluster_node("10.0.0.5", None); - let addrs = resolve_cluster_client_addrs(&node, addr("127.0.0.1:8090"), None, None, None) - .expect("cluster address resolution must succeed"); - assert_eq!(addrs.client, addr("127.0.0.1:18070")); - assert!(roster_ip_unreachable_from_bind_addr(&node.ip, addrs.client)); - } - - #[test] - fn roster_mismatch_warning_is_silent_for_wildcard_and_hostname_rosters() { - // A wildcard bind covers the roster interface, and a DNS roster entry - // can resolve to the bound one; neither is a misconfiguration. - assert!(!roster_ip_unreachable_from_bind_addr( - "10.0.0.5", - addr("0.0.0.0:18070") - )); - assert!(!roster_ip_unreachable_from_bind_addr( - "node-1.example.com", - addr("127.0.0.1:18070") - )); - assert!(!roster_ip_unreachable_from_bind_addr( - "10.0.0.5", - addr("10.0.0.5:18070") - )); - } + let inner = build_inner_metadata(users_state, streams_state); + writer.initialize(inner); } diff --git a/core/server/src/compat/index_rebuilding/index_rebuilder.rs b/core/server/src/compat/index_rebuilding/index_rebuilder.rs new file mode 100644 index 0000000000..c36c53b63f --- /dev/null +++ b/core/server/src/compat/index_rebuilding/index_rebuilder.rs @@ -0,0 +1,118 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::server_error::CompatError; +use crate::streaming::utils::file; +use compio::{ + fs::File, + io::{AsyncBufRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, BufReader, BufWriter}, +}; +use iggy_common::{IGGY_MESSAGE_HEADER_SIZE, IggyMessageHeader}; + +pub struct IndexRebuilder { + pub messages_file_path: String, + pub index_path: String, + pub start_offset: u64, +} + +impl IndexRebuilder { + pub fn new(messages_file_path: String, index_path: String, start_offset: u64) -> Self { + Self { + messages_file_path, + index_path, + start_offset, + } + } + + async fn read_message_header( + reader: &mut BufReader>, + ) -> Result { + let buf = [0u8; IGGY_MESSAGE_HEADER_SIZE]; + let (result, buf) = reader.read_exact(Box::new(buf)).await.into(); + result?; + IggyMessageHeader::from_raw_bytes(&*buf) + .map_err(|_| std::io::Error::from(std::io::ErrorKind::InvalidData)) + } + + async fn write_index_entry( + writer: &mut BufWriter>, + header: &IggyMessageHeader, + position: usize, + start_offset: u64, + ) -> Result<(), CompatError> { + // Write offset (4 bytes) - base_offset + last_offset_delta - start_offset + let offset = start_offset - header.offset; + debug_assert!(offset <= u32::MAX as u64); + let (result, _) = writer + .write_all(Box::new(offset.to_le_bytes())) + .await + .into(); + result?; + + // Write position (4 bytes) + let (result, _) = writer + .write_all(Box::new(position.to_le_bytes())) + .await + .into(); + result?; + + // Write timestamp (8 bytes) + let (result, _) = writer + .write_all(Box::new(header.timestamp.to_le_bytes())) + .await + .into(); + result?; + + Ok(()) + } + + pub async fn rebuild(&self) -> Result<(), CompatError> { + let read_cursor = std::io::Cursor::new(file::open(&self.messages_file_path).await?); + let write_cursor = std::io::Cursor::new(file::overwrite(&self.index_path).await?); + let mut reader = BufReader::new(read_cursor); + let mut writer = BufWriter::new(write_cursor); + let mut position = 0; + let mut next_position; + + loop { + match Self::read_message_header(&mut reader).await { + Ok(header) => { + next_position = position + + IGGY_MESSAGE_HEADER_SIZE + + header.payload_length as usize + + header.user_headers_length as usize; + + Self::write_index_entry(&mut writer, &header, position, self.start_offset) + .await?; + + // Skip message payload and headers + reader.consume( + header.payload_length as usize + header.user_headers_length as usize, + ); + + // Update position for next iteration + position = next_position; + } + Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => break, + Err(e) => return Err(e.into()), + } + } + + writer.flush().await?; + Ok(()) + } +} diff --git a/core/server/src/compat/index_rebuilding/mod.rs b/core/server/src/compat/index_rebuilding/mod.rs new file mode 100644 index 0000000000..6bc85fd6a3 --- /dev/null +++ b/core/server/src/compat/index_rebuilding/mod.rs @@ -0,0 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod index_rebuilder; diff --git a/core/server/src/compat/mod.rs b/core/server/src/compat/mod.rs new file mode 100644 index 0000000000..acfe367fbc --- /dev/null +++ b/core/server/src/compat/mod.rs @@ -0,0 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod index_rebuilding; diff --git a/core/server/src/configs.rs b/core/server/src/configs.rs new file mode 100644 index 0000000000..8f5c7821e1 --- /dev/null +++ b/core/server/src/configs.rs @@ -0,0 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub use configs::{ + COMPONENT, cache_indexes, cluster, defaults, displays, http, quic, server, sharding, system, + tcp, validators, websocket, +}; diff --git a/core/server/src/diagnostics.rs b/core/server/src/diagnostics.rs new file mode 100644 index 0000000000..d33d55a82e --- /dev/null +++ b/core/server/src/diagnostics.rs @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub use server_common::diagnostics::ASYNCIFY_POOL_DISABLED_PANIC_MSG; +pub use server_common::diagnostics::print_incomplete_io_uring_ops_info; +pub use server_common::diagnostics::print_invalid_io_uring_args_info; +pub use server_common::diagnostics::print_io_uring_permission_info; +pub use server_common::diagnostics::print_locked_memory_limit_info; diff --git a/core/server/src/http/consumer_groups.rs b/core/server/src/http/consumer_groups.rs new file mode 100644 index 0000000000..9ef910a81d --- /dev/null +++ b/core/server/src/http/consumer_groups.rs @@ -0,0 +1,176 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::http::error::CustomError; +use crate::http::jwt::json_web_token::Identity; +use crate::http::mapper; +use crate::http::shared::AppState; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; +use axum::debug_handler; +use axum::extract::{Path, State}; +use axum::http::StatusCode; +use axum::routing::get; +use axum::{Extension, Json, Router}; +use iggy_binary_protocol::WireName; +use iggy_binary_protocol::requests::consumer_groups::{ + CreateConsumerGroupRequest as WireCreateConsumerGroup, + DeleteConsumerGroupRequest as WireDeleteConsumerGroup, +}; +use iggy_common::Identifier; +use iggy_common::Validatable; +use iggy_common::create_consumer_group::CreateConsumerGroup; +use iggy_common::wire_conversions::identifier_to_wire; +use iggy_common::{ConsumerGroup, ConsumerGroupDetails, IggyError}; +use std::sync::Arc; +use tracing::instrument; + +pub fn router(state: Arc) -> Router { + Router::new() + .route( + "/streams/{stream_id}/topics/{topic_id}/consumer-groups", + get(get_consumer_groups).post(create_consumer_group), + ) + .route( + "/streams/{stream_id}/topics/{topic_id}/consumer-groups/{group_id}", + get(get_consumer_group).delete(delete_consumer_group), + ) + .with_state(state) +} + +async fn get_consumer_group( + State(state): State>, + Extension(identity): Extension, + Path((stream_id, topic_id, group_id)): Path<(String, String, String)>, +) -> Result, CustomError> { + let identifier_stream_id = Identifier::from_str_value(&stream_id)?; + let identifier_topic_id = Identifier::from_str_value(&topic_id)?; + let identifier_group_id = Identifier::from_str_value(&group_id)?; + + let shard = state.shard.shard(); + let group = shard.resolve_consumer_group( + &identifier_stream_id, + &identifier_topic_id, + &identifier_group_id, + )?; + + shard + .metadata + .perm_get_consumer_group(identity.user_id, group.stream_id, group.topic_id)?; + + let cg_meta = shard + .metadata + .get_consumer_group(group.stream_id, group.topic_id, group.group_id) + .ok_or(CustomError::ResourceNotFound)?; + + let consumer_group = mapper::map_consumer_group_details_from_metadata(&cg_meta); + + Ok(Json(consumer_group)) +} + +async fn get_consumer_groups( + State(state): State>, + Extension(identity): Extension, + Path((stream_id, topic_id)): Path<(String, String)>, +) -> Result>, CustomError> { + let identifier_stream_id = Identifier::from_str_value(&stream_id)?; + let identifier_topic_id = Identifier::from_str_value(&topic_id)?; + + let shard = state.shard.shard(); + let topic = shard.resolve_topic(&identifier_stream_id, &identifier_topic_id)?; + + shard + .metadata + .perm_get_consumer_groups(identity.user_id, topic.stream_id, topic.topic_id)?; + + let topic_meta = shard + .metadata + .get_topic(topic.stream_id, topic.topic_id) + .ok_or(CustomError::ResourceNotFound)?; + let consumer_groups = mapper::map_consumer_groups_from_metadata(&topic_meta); + + Ok(Json(consumer_groups)) +} + +#[debug_handler] +#[instrument(skip_all, name = "trace_create_consumer_group", fields(iggy_user_id = identity.user_id, iggy_stream_id = stream_id, iggy_topic_id = topic_id))] +async fn create_consumer_group( + State(state): State>, + Extension(identity): Extension, + Path((stream_id, topic_id)): Path<(String, String)>, + Json(mut command): Json, +) -> Result<(StatusCode, Json), CustomError> { + command.stream_id = Identifier::from_str_value(&stream_id)?; + command.topic_id = Identifier::from_str_value(&topic_id)?; + command.validate()?; + + let shard = state.shard.shard(); + let topic = shard.resolve_topic(&command.stream_id, &command.topic_id)?; + + let wire_command = WireCreateConsumerGroup { + stream_id: identifier_to_wire(&command.stream_id)?, + topic_id: identifier_to_wire(&command.topic_id)?, + name: WireName::new(&command.name).map_err(|_| IggyError::InvalidConsumerGroupName)?, + }; + let request = ShardRequest::control_plane(ShardRequestPayload::CreateConsumerGroupRequest { + user_id: identity.user_id, + command: wire_command, + }); + + match state.shard.send_to_control_plane(request).await? { + ShardResponse::CreateConsumerGroupResponse(data) => { + let cg_meta = state + .shard + .shard() + .metadata + .get_consumer_group(topic.stream_id, topic.topic_id, data.id as usize) + .expect("Consumer group must exist after creation"); + let consumer_group_details = mapper::map_consumer_group_details_from_metadata(&cg_meta); + Ok((StatusCode::CREATED, Json(consumer_group_details))) + } + ShardResponse::ErrorResponse(err) => Err(err.into()), + _ => unreachable!("Expected CreateConsumerGroupResponse"), + } +} + +#[debug_handler] +#[instrument(skip_all, name = "trace_delete_consumer_group", fields(iggy_user_id = identity.user_id, iggy_stream_id = stream_id, iggy_topic_id = topic_id, iggy_group_id = group_id))] +async fn delete_consumer_group( + State(state): State>, + Extension(identity): Extension, + Path((stream_id, topic_id, group_id)): Path<(String, String, String)>, +) -> Result { + let stream_id = Identifier::from_str_value(&stream_id)?; + let topic_id = Identifier::from_str_value(&topic_id)?; + let group_id = Identifier::from_str_value(&group_id)?; + + let wire_command = WireDeleteConsumerGroup { + stream_id: identifier_to_wire(&stream_id)?, + topic_id: identifier_to_wire(&topic_id)?, + group_id: identifier_to_wire(&group_id)?, + }; + let request = ShardRequest::control_plane(ShardRequestPayload::DeleteConsumerGroupRequest { + user_id: identity.user_id, + command: wire_command, + }); + + match state.shard.send_to_control_plane(request).await? { + ShardResponse::DeleteConsumerGroupResponse => Ok(StatusCode::NO_CONTENT), + ShardResponse::ErrorResponse(err) => Err(err.into()), + _ => unreachable!("Expected DeleteConsumerGroupResponse"), + } +} diff --git a/core/server/src/http/consumer_offsets.rs b/core/server/src/http/consumer_offsets.rs new file mode 100644 index 0000000000..1d856a7311 --- /dev/null +++ b/core/server/src/http/consumer_offsets.rs @@ -0,0 +1,150 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::http::COMPONENT; +use crate::http::error::CustomError; +use crate::http::jwt::json_web_token::Identity; +use crate::http::shared::AppState; +use axum::debug_handler; +use axum::extract::{Path, Query, State}; +use axum::http::StatusCode; +use axum::routing::{delete, get}; +use axum::{Extension, Json, Router}; +use err_trail::ErrContext; +use iggy_common::Consumer; +use iggy_common::ConsumerOffsetInfo; +use iggy_common::Identifier; +use iggy_common::IggyError; +use iggy_common::delete_consumer_offset::DeleteConsumerOffset; +use iggy_common::get_consumer_offset::GetConsumerOffset; +use iggy_common::store_consumer_offset::StoreConsumerOffset; +use std::sync::Arc; + +pub fn router(state: Arc) -> Router { + Router::new() + .route( + "/streams/{stream_id}/topics/{topic_id}/consumer-offsets", + get(get_consumer_offset).put(store_consumer_offset), + ) + .route( + "/streams/{stream_id}/topics/{topic_id}/consumer-offsets/{consumer_id}", + delete(delete_consumer_offset), + ) + .with_state(state) +} + +#[debug_handler] +async fn get_consumer_offset( + State(state): State>, + Extension(identity): Extension, + Path((stream_id, topic_id)): Path<(String, String)>, + query: Query, +) -> Result, CustomError> { + let stream_id = Identifier::from_str_value(&stream_id)?; + let topic_id = Identifier::from_str_value(&topic_id)?; + + let shard = state.shard.shard(); + let topic = shard.resolve_topic(&stream_id, &topic_id)?; + shard + .metadata + .perm_get_consumer_offset(identity.user_id, topic.stream_id, topic.topic_id)?; + + let consumer = Consumer::new(query.consumer.id.clone()); + let Ok(offset) = state + .shard + .get_consumer_offset( + 0, // HTTP uses client_id 0 as it doesn't have persistent sessions + consumer, + &stream_id, + &topic_id, + query.partition_id, + ) + .await + else { + return Err(CustomError::ResourceNotFound); + }; + + let Some(offset) = offset else { + return Err(CustomError::ResourceNotFound); + }; + + Ok(Json(offset)) +} + +#[debug_handler] +async fn store_consumer_offset( + State(state): State>, + Extension(identity): Extension, + Path((stream_id, topic_id)): Path<(String, String)>, + Json(body): Json, +) -> Result { + let stream_id = Identifier::from_str_value(&stream_id)?; + let topic_id = Identifier::from_str_value(&topic_id)?; + + let shard = state.shard.shard(); + let topic = shard.resolve_topic(&stream_id, &topic_id)?; + shard + .metadata + .perm_store_consumer_offset(identity.user_id, topic.stream_id, topic.topic_id)?; + + let consumer = Consumer::new(body.consumer.id); + state.shard + .store_consumer_offset( + 0, // HTTP uses client_id 0 as it doesn't have persistent sessions + consumer, + &stream_id, + &topic_id, + body.partition_id, + body.offset, + ) + .await + .error(|e: &IggyError| format!("{COMPONENT} (error: {e}) - failed to store consumer offset, stream ID: {stream_id}, topic ID: {topic_id}, partition ID: {:?}", body.partition_id))?; + Ok(StatusCode::NO_CONTENT) +} + +#[debug_handler] +async fn delete_consumer_offset( + State(state): State>, + Extension(identity): Extension, + Path((stream_id, topic_id, consumer_id)): Path<(String, String, String)>, + query: Query, +) -> Result { + let stream_id_ident = Identifier::from_str_value(&stream_id)?; + let topic_id_ident = Identifier::from_str_value(&topic_id)?; + + let shard = state.shard.shard(); + let topic = shard.resolve_topic(&stream_id_ident, &topic_id_ident)?; + shard.metadata.perm_delete_consumer_offset( + identity.user_id, + topic.stream_id, + topic.topic_id, + )?; + + let consumer = Consumer::new(consumer_id.try_into()?); + state + .shard + .delete_consumer_offset( + 0, // HTTP uses client_id 0 as it doesn't have persistent sessions + consumer, + &stream_id_ident, + &topic_id_ident, + query.partition_id, + ) + .await + .error(|e: &IggyError| format!("{COMPONENT} (error: {e}) - failed to delete consumer offset, stream ID: {}, topic ID: {}, partition ID: {:?}", stream_id, topic_id, query.partition_id))?; + Ok(StatusCode::NO_CONTENT) +} diff --git a/core/server/src/http/diagnostics.rs b/core/server/src/http/diagnostics.rs new file mode 100644 index 0000000000..56566cc99a --- /dev/null +++ b/core/server/src/http/diagnostics.rs @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::http::http_server::CompioSocketAddr; +use crate::http::shared::RequestDetails; +use crate::streaming::utils::random_id; +use axum::body::Body; +use axum::{ + extract::ConnectInfo, + http::{Request, StatusCode}, + middleware::Next, + response::Response, +}; +use std::time::Instant; +use tracing::{debug, error}; + +pub async fn request_diagnostics( + ConnectInfo(ip_address): ConnectInfo, + mut request: Request, + next: Next, +) -> Result { + let request_id = random_id::get_ulid(); + let path_and_query = request + .uri() + .path_and_query() + .map(|p| p.as_str()) + .unwrap_or("/"); + let ip_address = ip_address.0; + debug!( + "Processing a request {} {} with ID: {request_id} from client with IP address: {ip_address}...", + request.method(), + path_and_query, + ); + request.extensions_mut().insert(RequestDetails { + request_id, + ip_address, + }); + let now = Instant::now(); + let result = Ok(next.run(request).await); + if let Ok(response) = &result { + let status = response.status(); + if status != StatusCode::NOT_FOUND && status >= StatusCode::BAD_REQUEST { + error!( + "Returning an invalid status code: {status}, IP address: {ip_address}, request ID: {request_id}" + ); + } + } + let elapsed = now.elapsed(); + debug!( + "Processed a request with ID: {request_id} from client with IP address: {ip_address} in {} ms.", + elapsed.as_millis() + ); + result +} diff --git a/core/server/src/http/error.rs b/core/server/src/http/error.rs index 45ef05315c..1c422c0a0d 100644 --- a/core/server/src/http/error.rs +++ b/core/server/src/http/error.rs @@ -15,27 +15,16 @@ // specific language governing permissions and limitations // under the License. -//! HTTP rejection types and hand-built error responses: the auth / write / -//! read / partition-write error enums, their `IntoResponse` renderings, the -//! `?consistency=` and `?ack=` query DTOs, and the primary-redirect helpers. - -use std::net::{IpAddr, SocketAddr}; - use axum::Json; -use axum::http::header::{LOCATION, RETRY_AFTER}; -use axum::http::{HeaderValue, StatusCode}; +use axum::http::StatusCode; use axum::response::{IntoResponse, Response}; -use configs::cluster::ResolvedClusterNode; -use iggy_binary_protocol::Operation; use iggy_common::IggyError; -use serde::{Deserialize, Serialize}; +use serde::Serialize; use thiserror::Error; use tracing::error; -use crate::cluster_meta::ClusterRoster; - #[derive(Debug, Error)] -pub(in crate::http) enum CustomError { +pub enum CustomError { #[error(transparent)] Error(#[from] IggyError), #[error("Resource not found")] @@ -43,11 +32,7 @@ pub(in crate::http) enum CustomError { } #[derive(Debug, Serialize)] -pub(in crate::http) struct ErrorResponse { - /// Two conventions by construction: the `IggyError` numeric code - /// (`IggyError::as_code`) when the error wraps one (via [`Self::from_error`]), - /// or the HTTP status code for the hand-built HTTP-layer errors (429/503/504 - /// and the 404 not-found fallback) that carry no underlying `IggyError`. +pub struct ErrorResponse { pub id: u32, pub code: String, pub reason: String, @@ -57,49 +42,29 @@ pub(in crate::http) struct ErrorResponse { impl IntoResponse for CustomError { fn into_response(self) -> Response { match self { - Self::Error(error) => { + CustomError::Error(error) => { error!("There was an error: {error}"); let status_code = match error { - IggyError::StreamIdNotFound(_) - | IggyError::TopicIdNotFound(_, _) - | IggyError::PartitionNotFound(_, _, _) - | IggyError::SegmentNotFound - | IggyError::ClientNotFound(_) - | IggyError::ConsumerGroupIdNotFound(_, _) - | IggyError::ConsumerGroupNameNotFound(_, _) - | IggyError::ConsumerGroupMemberNotFound(_, _, _) - | IggyError::ConsumerOffsetNotFound(_) - | IggyError::ResourceNotFound(_) => StatusCode::NOT_FOUND, - IggyError::Unauthenticated - | IggyError::AccessTokenMissing - | IggyError::InvalidAccessToken - | IggyError::InvalidPersonalAccessToken => StatusCode::UNAUTHORIZED, + IggyError::StreamIdNotFound(_) => StatusCode::NOT_FOUND, + IggyError::TopicIdNotFound(_, _) => StatusCode::NOT_FOUND, + IggyError::PartitionNotFound(_, _, _) => StatusCode::NOT_FOUND, + IggyError::SegmentNotFound => StatusCode::NOT_FOUND, + IggyError::ClientNotFound(_) => StatusCode::NOT_FOUND, + IggyError::ConsumerGroupIdNotFound(_, _) => StatusCode::NOT_FOUND, + IggyError::ConsumerGroupNameNotFound(_, _) => StatusCode::NOT_FOUND, + IggyError::ConsumerGroupMemberNotFound(_, _, _) => StatusCode::NOT_FOUND, + IggyError::ConsumerOffsetNotFound(_) => StatusCode::NOT_FOUND, + IggyError::ResourceNotFound(_) => StatusCode::NOT_FOUND, + IggyError::Unauthenticated => StatusCode::UNAUTHORIZED, + IggyError::AccessTokenMissing => StatusCode::UNAUTHORIZED, + IggyError::InvalidAccessToken => StatusCode::UNAUTHORIZED, + IggyError::InvalidPersonalAccessToken => StatusCode::UNAUTHORIZED, IggyError::Unauthorized => StatusCode::FORBIDDEN, - // The pre-consensus retry frame: reaching this render - // means the write path's replay budget is exhausted and - // the op never committed - a transient server condition, - // retryable like the other cannot-commit-right-now 503s - // (see `service_unavailable`), never a caller error. - IggyError::TransientNotCommitted | IggyError::TransientNotAccepted => { - StatusCode::SERVICE_UNAVAILABLE - } _ => StatusCode::BAD_REQUEST, }; - let response = - (status_code, Json(ErrorResponse::from_error(&error))).into_response(); - // Transient 503s are retryable, so the advisory Retry-After hint - // rides along, matching the other transient 503 bodies - // (`service_unavailable`, `server_busy`). - if matches!( - error, - IggyError::TransientNotCommitted | IggyError::TransientNotAccepted - ) { - with_retry_after(response) - } else { - response - } + (status_code, Json(ErrorResponse::from_error(error))) } - Self::ResourceNotFound => ( + CustomError::ResourceNotFound => ( StatusCode::NOT_FOUND, Json(ErrorResponse { id: 404, @@ -107,712 +72,37 @@ impl IntoResponse for CustomError { reason: "Resource not found".to_string(), field: None, }), - ) - .into_response(), + ), } + .into_response() } } impl ErrorResponse { - pub fn from_error(error: &IggyError) -> Self { - Self { + pub fn from_error(error: IggyError) -> Self { + ErrorResponse { id: error.as_code(), code: error.as_string().to_string(), reason: error.to_string(), field: match error { - IggyError::StreamIdNotFound(_) | IggyError::InvalidStreamId => { - Some("stream_id".to_string()) - } - IggyError::TopicIdNotFound(_, _) | IggyError::InvalidTopicId => { - Some("topic_id".to_string()) - } + IggyError::StreamIdNotFound(_) => Some("stream_id".to_string()), + IggyError::TopicIdNotFound(_, _) => Some("topic_id".to_string()), IggyError::PartitionNotFound(_, _, _) => Some("partition_id".to_string()), IggyError::SegmentNotFound => Some("segment_id".to_string()), IggyError::ClientNotFound(_) => Some("client_id".to_string()), - IggyError::InvalidStreamName - | IggyError::StreamNameAlreadyExists(_) - | IggyError::InvalidTopicName - | IggyError::TopicNameAlreadyExists(_, _) - | IggyError::ConsumerGroupNameAlreadyExists(_, _) - | IggyError::PersonalAccessTokenAlreadyExists(_, _) => Some("name".to_string()), + IggyError::InvalidStreamName => Some("name".to_string()), + IggyError::StreamNameAlreadyExists(_) => Some("name".to_string()), + IggyError::InvalidTopicName => Some("name".to_string()), + IggyError::TopicNameAlreadyExists(_, _) => Some("name".to_string()), + IggyError::InvalidStreamId => Some("stream_id".to_string()), + IggyError::InvalidTopicId => Some("topic_id".to_string()), IggyError::InvalidOffset(_) => Some("offset".to_string()), IggyError::InvalidConsumerGroupId => Some("consumer_group_id".to_string()), + IggyError::ConsumerGroupNameAlreadyExists(_, _) => Some("name".to_string()), IggyError::UserAlreadyExists => Some("username".to_string()), + IggyError::PersonalAccessTokenAlreadyExists(_, _) => Some("name".to_string()), _ => None, }, } } } - -/// Rejection for protected routes. -/// -/// Two failure classes get two statuses: a missing, invalid, or expired -/// credential is the caller's fault (401, rendered as the JSON `ErrorResponse` -/// body every other route error uses), while a VSR session that cannot be -/// established right now is a transient server condition (503) and must never -/// masquerade as an auth failure. -pub(in crate::http) enum AuthError { - Unauthenticated(IggyError), - /// The Register provably never entered the consensus pipeline (not - /// primary, not caught up, or the prepare queue was full), so the request - /// is safe to re-issue anywhere. Rendered with the `TransientNotAccepted` - /// body so a forwarding follower recognizes it as retryable against a - /// re-resolved primary; a plain client sees the same retryable 503 either - /// way. - SessionNotAccepted, - SessionUnavailable, - /// The `client_id` this gateway minted already has a committed session - /// owned by a DIFFERENT user, so the Register was refused terminally. - /// - /// Distinct from [`Self::SessionUnavailable`] because the status code is - /// the whole point: 503 is about the most auto-retried status there is and - /// no foreign SDK special-cases it, so rendering a permanent, deterministic - /// refusal as 503 hands the caller's HTTP stack a retry loop it can never - /// escape. 409 says the id is taken and stops it. - SessionIdOwnedByAnotherUser, - /// The minted `client_id` already had a committed session for this SAME - /// user, so the Register rebound onto it instead of creating one. Internal - /// to the mint retry in `register_session` and never rendered: the caller - /// mints a different id. Present as a variant so the retry cannot confuse - /// it with a terminal cross-user refusal. - SessionIdTaken, -} - -impl From for AuthError { - fn from(error: IggyError) -> Self { - Self::Unauthenticated(error) - } -} - -impl IntoResponse for AuthError { - fn into_response(self) -> Response { - match self { - // Render 401 through the shared `IggyError -> CustomError` map so it - // carries the same JSON `ErrorResponse` body as every other ng error. - // The legacy server's protected-route 401 comes from a bare-status - // JWT middleware (empty body), so this is deliberately richer, not - // byte-identical to legacy. - Self::Unauthenticated(error) => CustomError::from(error).into_response(), - Self::SessionNotAccepted => { - CustomError::from(IggyError::TransientNotAccepted).into_response() - } - // A fresh session could not be established: the Register was - // canceled with its commit outcome unknown, or the session table - // is at its cap (half `[metadata] clients_table_max`) and refused - // the fresh registration. Transient server condition -> 503, - // retryable by the CLIENT only (a forwarder must not re-issue an - // unknown-outcome Register under this node's session budget on - // the caller's behalf). - // `SessionIdTaken` only escapes the mint retry when every attempt - // collided, which means the minter is wrong rather than unlucky -- - // same unknown-outcome answer as a canceled Register. - Self::SessionUnavailable | Self::SessionIdTaken => service_unavailable(), - // Terminal: retrying cannot change the answer, and admitting it - // would run this caller's replicated ops under the entry owner's - // authority. - Self::SessionIdOwnedByAnotherUser => ( - StatusCode::CONFLICT, - Json(ErrorResponse::from_error(&IggyError::InvalidClientId)), - ) - .into_response(), - } - } -} - -/// Rejection for an authenticated control-plane write (`POST /streams` and the -/// writes that follow it). -/// -/// Same two-class split as [`AuthError`], for the same reasons: a caller-side -/// validation failure or a committed business rejection (e.g. a duplicate -/// stream name) renders through the legacy `IggyError -> CustomError` map so -/// SDK error bodies stay byte-identical, while a write that cannot commit right -/// now is a transient server condition (503) and must never surface as a -/// business error or, worse, a 200 with a stale body. -pub(in crate::http) enum WriteError { - Rejected(IggyError), - /// The VSR session was evicted (its client slot was reclaimed cluster-side, - /// e.g. LRU-evicted from the full client table). Renders identically to a - /// terminal `Rejected` (401 -> re-authenticate), but is a distinct variant - /// so the submit path can drop the dead session entry and let the caller's - /// next request re-register cleanly instead of 401-looping on it. - Evicted(IggyError), - Unavailable, -} - -impl IntoResponse for WriteError { - fn into_response(self) -> Response { - match self { - Self::Rejected(error) | Self::Evicted(error) => { - CustomError::from(error).into_response() - } - Self::Unavailable => service_unavailable(), - } - } -} - -/// Rejection for a data-plane partition write (`POST .../messages` produce and -/// the `PUT`/`DELETE .../consumer-offsets` writes). -/// -/// Split differently from [`WriteError`] because the partition plane replies -/// carry no committed error code: a pre-dispatch gate failure is an -/// empty-bodied reply that names itself only in the header (see -/// [`classify_partition_reply`]), and an unanswered write is a distinct -/// outcome the caller must treat as unknown rather than failed. -#[derive(Debug)] -pub(in crate::http) enum PartitionWriteError { - /// Caller-side rejection (bad identifier, oversized batch, an authorization - /// denial), a typed pre-commit deny from the partition plane - /// (`ReplyHeader.status`), or a malformed reply frame, rendered through the - /// legacy `IggyError -> status` map for SDK-identical bodies. - Rejected(IggyError), - /// Backstop for a status-0 reply carrying `op` 0: an ack with no commit - /// number behind it, for a write that never reached the partition plane. - /// Routing failures name themselves through `ReplyHeader.status`, so this - /// shape is left to a peer that still answers a non-committing op this - /// way. Rendered as the legacy 404 body: the alternative is grading a - /// write that never happened as a success. - NotFound, - /// The in-process reply slot could not be installed. Transient server - /// condition -> the shared 503, retryable. - Unavailable, - /// This session is already at [`MAX_IN_FLIGHT_WRITES_PER_SESSION`] - /// awaited writes. 429: the caller's own concurrency is the problem, so - /// it must drain its outstanding writes before submitting more. - TooManyInFlight, - /// Shard 0 is already at [`MAX_IN_FLIGHT_WRITES_GLOBAL`] awaited writes - /// across all sessions. 503 with its own code (distinct from the shared - /// consensus-unavailable body) so an operator can tell admission shedding - /// from a consensus outage. - ServerBusy, - /// No committed reply within [`PARTITION_WRITE_REPLY_TIMEOUT`], or the - /// session's reply target was torn down mid-wait. 504: the commit may - /// still land (at-least-once), so this is a hard "outcome unknown", not a - /// failure the server may transparently retry. Carries the write's - /// operation so the 504 body names which write kind timed out. - Timeout(Operation), -} - -impl IntoResponse for PartitionWriteError { - fn into_response(self) -> Response { - match self { - Self::Rejected(error) => CustomError::from(error).into_response(), - Self::NotFound => CustomError::ResourceNotFound.into_response(), - Self::Unavailable => service_unavailable(), - Self::TooManyInFlight => too_many_in_flight_response(), - Self::ServerBusy => server_busy_response(), - Self::Timeout(operation) => partition_write_timeout_response(operation), - } - } -} - -/// 504 body for a partition write whose commit outcome is unknown, coded per -/// write kind so a caller can tell a produce timeout from an offset-write -/// timeout. Shaped like every other HTTP error (`ErrorResponse`) so clients -/// parse one error schema. -fn partition_write_timeout_response(operation: Operation) -> Response { - let (code, reason) = match operation { - Operation::SendMessages => ( - "produce_timeout", - "produce was not acknowledged in time; the write may still commit", - ), - _ => ( - "offset_write_timeout", - "consumer-offset write was not acknowledged in time; the write may still commit", - ), - }; - gateway_timeout_response(code, reason) -} - -/// Advisory `Retry-After` seconds for the shed / transient 429 and 503 -/// responses. One second: admission shedding, a briefly unavailable consensus -/// group, and a linearizable-read-on-follower all typically clear well within -/// it, and a small hint keeps a backing-off client responsive. -const RETRY_AFTER_SECONDS: u64 = 1; - -/// Attach the advisory [`RETRY_AFTER_SECONDS`] hint to a retryable 429/503. -pub(in crate::http) fn with_retry_after(mut response: Response) -> Response { - response - .headers_mut() - .insert(RETRY_AFTER, HeaderValue::from(RETRY_AFTER_SECONDS)); - response -} - -/// Render an `ErrorResponse` body for `status`, tagged with `code` / `reason` -/// and no field, so every hand-built HTTP error the routes return parses as the -/// one error schema clients already handle. -pub(in crate::http) fn error_response(status: StatusCode, code: &str, reason: &str) -> Response { - ( - status, - Json(ErrorResponse { - id: status.as_u16().into(), - code: code.to_owned(), - reason: reason.to_owned(), - field: None, - }), - ) - .into_response() -} - -/// Shared 504 rendering for an in-band request the partition plane did not -/// answer in time, shaped like every other HTTP error (`ErrorResponse`) so -/// clients parse one error schema. Consumed by the partition-write reply wait, -/// the partition reads ([`ReadError::Timeout`]), and the forward attempt bound. -pub(in crate::http) fn gateway_timeout_response(code: &str, reason: &str) -> Response { - error_response(StatusCode::GATEWAY_TIMEOUT, code, reason) -} - -/// The shared 503 body for a request that could not commit right now: no -/// caught-up primary, a full pipeline, or a view-change cancel. Retryable, and -/// rendered with the `CannotEstablishConnection` code the SDKs treat as a -/// connection-level retry rather than a terminal error. -fn service_unavailable() -> Response { - with_retry_after( - ( - StatusCode::SERVICE_UNAVAILABLE, - Json(ErrorResponse::from_error( - &IggyError::CannotEstablishConnection, - )), - ) - .into_response(), - ) -} - -/// 429 for a session at [`MAX_IN_FLIGHT_WRITES_PER_SESSION`] awaited partition -/// writes. Shaped like every other HTTP error (`ErrorResponse`) so clients -/// parse one error schema; the remedy is the caller's own: let outstanding -/// writes finish, then retry. -fn too_many_in_flight_response() -> Response { - with_retry_after(error_response( - StatusCode::TOO_MANY_REQUESTS, - "too_many_in_flight_writes", - "session reached its in-flight write cap; await outstanding writes and retry", - )) -} - -/// 503 for shard 0 at [`MAX_IN_FLIGHT_WRITES_GLOBAL`] awaited partition writes -/// across all sessions. A distinct `server_busy` code (unlike the shared -/// consensus-unavailable 503) so admission shedding is tellable from a -/// consensus outage; retry with backoff. -fn server_busy_response() -> Response { - with_retry_after(error_response( - StatusCode::SERVICE_UNAVAILABLE, - "server_busy", - "shard is at its in-flight write budget; retry with backoff", - )) -} - -/// Read consistency selected by the `?consistency=` query param. -/// -/// `serializable` (the default) serves from this node's local metadata STM: -/// correct and consensus-free, but may trail the primary by the replication -/// delay. `linearizable` demands the freshest committed state and is honored -/// only on the primary; a follower redirects (307) to the primary when its HTTP -/// address resolves from the roster, else fails closed to 503 (see -/// [`read_local`]). -#[derive(Clone, Copy, Default, PartialEq, Eq, Deserialize)] -#[serde(rename_all = "lowercase")] -pub(in crate::http) enum Consistency { - #[default] - Serializable, - Linearizable, -} - -/// `?consistency=` query wrapper. An absent param defaults to -/// [`Consistency::Serializable`]; an unrecognized value is a 400 (axum `Query`). -#[derive(Default, Deserialize)] -pub(in crate::http) struct ConsistencyQuery { - #[serde(default)] - pub(in crate::http) consistency: Consistency, -} - -/// Produce acknowledgement selected by the `?ack=` query param. -/// -/// `replicated` (the default) answers 201 only after the partition group's -/// quorum commit. `none` is fire-and-forget: the request is validated, -/// dispatched, and answered 202 immediately; the commit still happens, but its -/// reply is shed at the bus (no reply slot is installed). -#[derive(Clone, Copy, Default, PartialEq, Eq, Deserialize)] -#[serde(rename_all = "lowercase")] -pub(in crate::http) enum ProduceAck { - #[default] - Replicated, - None, -} - -/// `?ack=` query wrapper. An absent param defaults to -/// [`ProduceAck::Replicated`]; an unrecognized value is a 400 (axum `Query`). -#[derive(Default, Deserialize)] -pub(in crate::http) struct ProduceQuery { - #[serde(default)] - pub(in crate::http) ack: ProduceAck, -} - -/// Rejection for an authenticated read route (`GET /streams`, -/// `GET /streams/{id}`, and the reads that follow). -pub(in crate::http) enum ReadError { - /// Caller-side or STM rejection (bad identifier, unsupported op, or an - /// authorization denial) graded through the legacy `IggyError -> status` - /// map so SDK error bodies stay byte-identical. - Rejected(IggyError), - /// Requested entity is absent -> 404 with the legacy not-found body. - NotFound, - /// A linearizable read reached a follower and the primary's HTTP address was - /// not resolvable from the roster. Fail-closed 503, retryable against the - /// leader (see [`not_primary_response`]). - NotPrimary, - /// A linearizable read reached a follower and the current VSR primary's HTTP - /// address resolved: 307 to that address carrying the original path and - /// query, so the caller re-issues the read against the leader (see - /// [`primary_redirect_response`]). - RedirectToPrimary(String), - /// The post-restart read-recovery barrier expired with the recovered WAL - /// suffix still uncommitted: serving now could show state that rolls back - /// history a client already saw acked. Fail-closed 503 via the shared - /// [`service_unavailable`] body, retryable once the cluster re-commits the - /// suffix. - RecoveryIncomplete, - /// A partition read (poll / consumer-offset) got no reply from the owning - /// shard within the mesh budget. 504 like a produce timeout: the outcome is - /// unknown (the abandoned read may still be running), so the caller retries. - Timeout, -} - -impl IntoResponse for ReadError { - fn into_response(self) -> Response { - match self { - Self::Rejected(error) => CustomError::from(error).into_response(), - // Reuse the legacy 404 body so a missing stream renders exactly as - // the legacy server's `CustomError::ResourceNotFound` does. - Self::NotFound => CustomError::ResourceNotFound.into_response(), - Self::NotPrimary => not_primary_response(), - Self::RedirectToPrimary(location) => primary_redirect_response(&location), - Self::RecoveryIncomplete => service_unavailable(), - Self::Timeout => gateway_timeout_response( - "partition_read_timeout", - "the partition owner did not answer the read in time; retry", - ), - } - } -} - -/// The 503 fail-closed body for a linearizable read that reached a follower -/// whose primary HTTP address could not be resolved (absent consensus, a roster -/// with no node at the primary index, or a port-less node). The resolvable case -/// is a 307 via [`primary_redirect_response`] instead. Rendered as an -/// `ErrorResponse` so the body shape matches every other HTTP error; the caller -/// retries against the leader. -fn not_primary_response() -> Response { - with_retry_after(error_response( - StatusCode::SERVICE_UNAVAILABLE, - "not_primary", - "linearizable read requires the primary; retry against the leader", - )) -} - -/// 307 Temporary Redirect to the current VSR primary for a linearizable read -/// that reached a follower. `Location` is the primary's HTTP base plus the -/// original path and query, so the caller re-issues the identical read against -/// the leader. Dormant on a single node (always primary) and followed by no SDK -/// yet. A `Location` that is not a valid header value falls back to the 503. -fn primary_redirect_response(location: &str) -> Response { - HeaderValue::from_str(location).map_or_else( - |_| not_primary_response(), - |value| { - let mut response = StatusCode::TEMPORARY_REDIRECT.into_response(); - response.headers_mut().insert(LOCATION, value); - response - }, - ) -} - -/// Build the `Location` for a 307 redirect of a linearizable read to the VSR -/// primary: `://:`. The scheme is the -/// redirecting node's own listener scheme (uniform cluster HTTP config, same -/// assumption the forward hop makes). `client_ip` is the redirected client's -/// peer address, so the `Location` host comes from the primary's -/// per-client-network selectors when one matches. `None` when the primary does -/// not resolve from the roster, so the caller fails closed to a 503 rather -/// than pointing at an unreachable target. Pure (no consensus or axum -/// dependency) so the redirect target is unit-tested in isolation. -pub(in crate::http) fn primary_redirect_location( - roster: &ClusterRoster, - primary_index: u8, - scheme: &str, - path_and_query: &str, - client_ip: Option, -) -> Option { - let authority = primary_advertised_http_authority(roster, primary_index, client_ip)?; - Some(format!("{scheme}://{authority}{path_and_query}")) -} - -/// Resolve the VSR primary's HTTP socket from the static roster: the node -/// whose `replica_id` equals `primary_index`, its `ports.http`, and its -/// private roster `ip` (parsed once at roster build). Internal replica -/// forwarding uses this address; it must never route through -/// [`ResolvedClusterNode::advertised_for`], which picks client-facing hosts. -pub(in crate::http) fn primary_http_socket( - roster: &ClusterRoster, - primary_index: u8, -) -> Option { - let (node, http_port) = primary_node(roster, primary_index)?; - Some(SocketAddr::new(node.replica_ip()?, http_port)) -} - -/// Resolve the client-facing HTTP authority (`host:port`) for a redirect -/// through [`ResolvedClusterNode::advertised_for`]: a client-network selector -/// match first, then the catch-all advertised address, then the private -/// roster IP as the compatibility fallback. `AdvertisedAddress::authority` -/// brackets IPv6 hosts and passes hostnames through, so the redirect URL -/// stays valid. This is the fail-closed caller: a host that is neither a -/// valid IP nor a valid hostname yields `None` and the redirect becomes a -/// 503 rather than a `Location` pointing at an unparsable target (cluster -/// metadata makes the opposite choice and publishes such a host verbatim). -fn primary_advertised_http_authority( - roster: &ClusterRoster, - primary_index: u8, - client_ip: Option, -) -> Option { - let (node, http_port) = primary_node(roster, primary_index)?; - let address = node.advertised_for(client_ip)?; - Some(address.authority(http_port)) -} - -fn primary_node(roster: &ClusterRoster, primary_index: u8) -> Option<(&ResolvedClusterNode, u16)> { - let node = roster - .nodes - .iter() - .find(|node| node.config().replica_id == primary_index)?; - let http_port = node.config().ports.http?; - Some((node, http_port)) -} - -#[cfg(test)] -mod tests { - use super::*; - - use configs::cluster::{ClusterNodeConfig, TransportPorts}; - - const READ_PATH: &str = "/streams?consistency=linearizable"; - fn node(replica_id: u8, ip: &str, http: Option) -> ClusterNodeConfig { - ClusterNodeConfig { - name: format!("node-{replica_id}"), - ip: ip.to_owned(), - advertised_address: None, - advertised_addresses: Vec::new(), - replica_id, - ports: TransportPorts { - tcp: None, - quic: None, - http, - websocket: None, - tcp_replica: None, - }, - } - } - - fn roster(nodes: Vec) -> ClusterRoster { - ClusterRoster { - enabled: true, - name: "test-cluster".to_owned(), - nodes: nodes.into_iter().map(Into::into).collect(), - self_ip: "127.0.0.1".to_owned(), - self_ports: TransportPorts::default(), - metadata_view: std::sync::Arc::new(std::sync::atomic::AtomicU64::new( - crate::cluster_meta::METADATA_VIEW_UNKNOWN, - )), - } - } - - #[test] - fn primary_redirect_location_targets_primary_http_addr_with_path_passthrough() { - let roster = roster(vec![ - node(0, "10.0.0.1", Some(8080)), - node(1, "10.0.0.2", Some(8090)), - ]); - assert_eq!( - primary_redirect_location(&roster, 1, "http", READ_PATH, None), - Some("http://10.0.0.2:8090/streams?consistency=linearizable".to_owned()) - ); - } - - #[test] - fn primary_redirect_location_uses_the_listener_scheme() { - let roster = roster(vec![node(0, "10.0.0.1", Some(8080))]); - assert_eq!( - primary_redirect_location(&roster, 0, "https", READ_PATH, None), - Some("https://10.0.0.1:8080/streams?consistency=linearizable".to_owned()) - ); - } - - #[test] - fn primary_redirect_location_is_none_when_no_node_matches_primary_index() { - let roster = roster(vec![node(0, "10.0.0.1", Some(8080))]); - assert_eq!( - primary_redirect_location(&roster, 2, "http", READ_PATH, None), - None - ); - } - - #[test] - fn primary_redirect_location_is_none_when_primary_has_no_http_port() { - let roster = roster(vec![node(0, "10.0.0.1", None)]); - assert_eq!( - primary_redirect_location(&roster, 0, "http", READ_PATH, None), - None - ); - } - - #[test] - fn primary_redirect_location_is_none_for_empty_roster() { - let roster = roster(Vec::new()); - assert_eq!( - primary_redirect_location(&roster, 0, "http", READ_PATH, None), - None - ); - } - - #[test] - fn primary_redirect_location_brackets_ipv6_host() { - let roster = roster(vec![node(0, "::1", Some(8080))]); - assert_eq!( - primary_redirect_location(&roster, 0, "http", READ_PATH, None), - Some("http://[::1]:8080/streams?consistency=linearizable".to_owned()) - ); - } - - #[test] - fn primary_redirect_location_uses_advertised_address() { - let mut primary = node(0, "10.0.0.1", Some(8080)); - primary.advertised_address = Some("2001:db8::1".to_owned()); - let roster = roster(vec![primary]); - - assert_eq!( - primary_redirect_location(&roster, 0, "https", READ_PATH, None), - Some("https://[2001:db8::1]:8080/streams?consistency=linearizable".to_owned()) - ); - } - - #[test] - fn primary_redirect_location_uses_advertised_hostname() { - let mut primary = node(0, "10.0.0.1", Some(8080)); - primary.advertised_address = Some("broker-1.example.com".to_owned()); - let roster = roster(vec![primary]); - - assert_eq!( - primary_redirect_location(&roster, 0, "https", READ_PATH, None), - Some("https://broker-1.example.com:8080/streams?consistency=linearizable".to_owned()) - ); - } - - #[test] - fn primary_redirect_location_uses_the_selector_address_for_a_matching_client() { - let mut primary = node(0, "10.0.0.1", Some(8080)); - primary.advertised_address = Some("203.0.113.1".to_owned()); - primary.advertised_addresses = vec![configs::cluster::AdvertisedAddressSelector { - client_cidr: "10.0.0.0/16".to_owned(), - address: "10.0.0.1".to_owned(), - }]; - let roster = roster(vec![primary]); - - assert_eq!( - primary_redirect_location( - &roster, - 0, - "https", - READ_PATH, - Some("10.0.9.9".parse().unwrap()) - ), - Some("https://10.0.0.1:8080/streams?consistency=linearizable".to_owned()), - "an in-network client must be redirected to the selector address" - ); - assert_eq!( - primary_redirect_location( - &roster, - 0, - "https", - READ_PATH, - Some("198.51.100.7".parse().unwrap()) - ), - Some("https://203.0.113.1:8080/streams?consistency=linearizable".to_owned()), - "an out-of-network client must stay on the catch-all address" - ); - } - - #[test] - fn primary_http_socket_uses_private_roster_ip() { - let mut primary = node(0, "10.0.0.1", Some(8080)); - primary.advertised_address = Some("203.0.113.1".to_owned()); - let roster = roster(vec![primary]); - - assert_eq!( - primary_http_socket(&roster, 0), - Some("10.0.0.1:8080".parse().expect("valid socket address")) - ); - } - - #[test] - fn transient_not_committed_renders_503_with_retry_after() { - let response = CustomError::from(IggyError::TransientNotCommitted).into_response(); - assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); - assert!(response.headers().contains_key(RETRY_AFTER)); - } - - #[test] - fn transient_not_accepted_renders_503_with_retry_after() { - let response = CustomError::from(IggyError::TransientNotAccepted).into_response(); - assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); - assert!(response.headers().contains_key(RETRY_AFTER)); - } - - #[test] - fn business_error_renders_without_retry_after() { - let response = CustomError::from(IggyError::UserAlreadyExists).into_response(); - assert_eq!(response.status(), StatusCode::BAD_REQUEST); - assert!(!response.headers().contains_key(RETRY_AFTER)); - } - - // The ownership refusal is permanent and deterministic. Rendering it as - // 503 would hand the caller's HTTP stack a retry loop it can never escape - // (no foreign SDK special-cases 503), so the status is load-bearing. - #[test] - fn owned_client_id_renders_as_terminal_conflict() { - let response = AuthError::SessionIdOwnedByAnotherUser.into_response(); - assert_eq!(response.status(), StatusCode::CONFLICT); - assert!( - response.headers().get(RETRY_AFTER).is_none(), - "a terminal refusal must not advertise a retry" - ); - } - - // Its siblings stay retryable, so the split is visible in one place. - #[test] - fn unknown_outcome_registers_stay_retryable() { - for error in [AuthError::SessionUnavailable, AuthError::SessionNotAccepted] { - let status = error.into_response().status(); - assert!( - status.is_server_error(), - "an unknown commit outcome must stay retryable, got {status}" - ); - } - } - - #[test] - fn recovery_incomplete_renders_retryable_503_like_not_primary() { - // Barrier expiry must render as the shared retryable 503: the same - // status and Retry-After hint as the not-primary 503, so an SDK treats - // it as a connection-level retry rather than a terminal error. - let recovery = ReadError::RecoveryIncomplete.into_response(); - assert_eq!(recovery.status(), StatusCode::SERVICE_UNAVAILABLE); - assert_eq!( - recovery.headers().get(RETRY_AFTER), - Some(&HeaderValue::from(RETRY_AFTER_SECONDS)) - ); - - let not_primary = ReadError::NotPrimary.into_response(); - assert_eq!(recovery.status(), not_primary.status()); - assert_eq!( - recovery.headers().get(RETRY_AFTER), - not_primary.headers().get(RETRY_AFTER) - ); - } -} diff --git a/core/server/src/http/http_server.rs b/core/server/src/http/http_server.rs new file mode 100644 index 0000000000..a085b75441 --- /dev/null +++ b/core/server/src/http/http_server.rs @@ -0,0 +1,399 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::configs::http::{HttpConfig, HttpCorsConfig}; +use crate::http::diagnostics::request_diagnostics; +use crate::http::http_shard_wrapper::HttpSafeShard; +use crate::http::jwt::jwt_manager::JwtManager; +use crate::http::jwt::middleware::jwt_auth; +use crate::http::metrics::metrics; +use crate::http::shared::AppState; +use crate::http::*; +use crate::shard::IggyShard; +use crate::shard::task_registry::ShutdownToken; +use crate::shard::tasks::periodic::spawn_jwt_token_cleaner; +use crate::shard::transmission::event::ShardEvent; +use crate::streaming::persistence::persister::PersisterKind; +use crate::streaming::utils::crypto; +use axum::extract::DefaultBodyLimit; +use axum::extract::connect_info::Connected; +use axum::http::Method; +use axum::{Router, middleware}; +use axum_server::tls_rustls::RustlsConfig; +use compio::net::TcpListener; +use err_trail::ErrContext; +use iggy_common::IggyError; +use iggy_common::TransportProtocol; +use socket2::{Domain, Protocol, Socket, Type}; +use std::net::SocketAddr; +use std::path::PathBuf; +use std::rc::Rc; +use std::sync::Arc; +use tower_http::cors::{AllowOrigin, CorsLayer}; +use tracing::{error, info, warn}; + +#[derive(Debug, Clone, Copy)] +pub struct CompioSocketAddr(pub SocketAddr); + +impl From for CompioSocketAddr { + fn from(addr: SocketAddr) -> Self { + CompioSocketAddr(addr) + } +} + +impl From for SocketAddr { + fn from(addr: CompioSocketAddr) -> Self { + addr.0 + } +} + +impl<'a> Connected> for CompioSocketAddr { + fn connect_info(target: cyper_axum::IncomingStream<'a, TcpListener>) -> Self { + let addr = *target.remote_addr(); + CompioSocketAddr(addr) + } +} + +/// Starts the HTTP API server. +/// Returns the address the server is listening on. +pub async fn start_http_server( + config: HttpConfig, + persister: Arc, + shard: Rc, + shutdown: ShutdownToken, +) -> Result<(), IggyError> { + if shard.id != 0 { + info!( + "HTTP server disabled for shard {} (only runs on shard 0)", + shard.id + ); + panic!("HTTP server only runs on shard 0"); + } + + let api_name = if config.tls.enabled { + "HTTP API (TLS)" + } else { + "HTTP API" + }; + + let app_state = build_app_state(&config, persister, shard.clone()).await; + let mut app = Router::new() + .merge(system::router(app_state.clone(), &config.metrics)) + .merge(personal_access_tokens::router(app_state.clone())) + .merge(users::router(app_state.clone())) + .merge(streams::router(app_state.clone())) + .merge(topics::router(app_state.clone())) + .merge(consumer_groups::router(app_state.clone())) + .merge(consumer_offsets::router(app_state.clone())) + .merge(partitions::router(app_state.clone())) + .merge(segments::router(app_state.clone())) + .merge(messages::router(app_state.clone())) + .layer(DefaultBodyLimit::max( + config.max_request_size.as_bytes_u64() as usize, + )) + .layer(middleware::from_fn_with_state(app_state.clone(), jwt_auth)); + + if config.cors.enabled { + app = app.layer(configure_cors(config.cors)?); + } + + if config.metrics.enabled { + app = app.layer(middleware::from_fn_with_state(app_state.clone(), metrics)); + } + + spawn_jwt_token_cleaner(shard.clone(), app_state.clone()); + + app = app.layer(middleware::from_fn(request_diagnostics)); + + #[cfg(feature = "iggy-web")] + if config.web_ui { + app = app.merge(web::router()); + info!("Web UI enabled at /ui"); + } + + #[cfg(not(feature = "iggy-web"))] + if config.web_ui { + tracing::warn!( + "Web UI is enabled in configuration (http.web_ui = true) but the server \ + was not compiled with 'iggy-web' feature. The Web UI will not be available. \ + To enable it, rebuild the server with: cargo build --features iggy-web" + ); + } + + if !config.tls.enabled { + let bind_addr: SocketAddr = config + .address + .parse() + .unwrap_or_else(|_| panic!("Failed to parse HTTP address {}", config.address)); + let listener = crate::tcp::bind_reuseport_listener(bind_addr, true, None) + .await + .unwrap_or_else(|_| panic!("Failed to bind to HTTP address {}", config.address)); + let address = listener + .local_addr() + .expect("Failed to get local address for HTTP server"); + info!("Started {api_name} on: {address}"); + + // Notify shard about the bound address + let event = ShardEvent::AddressBound { + protocol: TransportProtocol::Http, + address, + }; + + crate::shard::handlers::handle_event(&shard, event) + .await + .ok(); + + let service = app.into_make_service_with_connect_info::(); + + let shutdown_token = shutdown.clone(); + let result = cyper_axum::serve(listener, service) + .with_graceful_shutdown(async move { shutdown_token.wait().await }) + .await; + + match result { + Ok(()) => { + info!("{api_name} shut down gracefully"); + Ok(()) + } + Err(error) => { + error!("{api_name} server error: {}", error); + Err(IggyError::CannotBindToSocket(format!("HTTP: {}", error))) + } + } + } else { + let tls_config = RustlsConfig::from_pem_file( + PathBuf::from(config.tls.cert_file), + PathBuf::from(config.tls.key_file), + ) + .await + .unwrap(); + + let addr: SocketAddr = config + .address + .parse() + .unwrap_or_else(|e| panic!("Invalid HTTPS address '{}': {e}", config.address)); + let domain = if addr.is_ipv6() { + Domain::IPV6 + } else { + Domain::IPV4 + }; + let socket = Socket::new(domain, Type::STREAM, Some(Protocol::TCP)) + .unwrap_or_else(|e| panic!("Failed to create HTTPS socket: {e}")); + socket + .set_reuse_address(true) + .unwrap_or_else(|e| panic!("Failed to set SO_REUSEADDR: {e}")); + #[cfg(unix)] + socket + .set_reuse_port(true) + .unwrap_or_else(|e| panic!("Failed to set SO_REUSEPORT: {e}")); + socket + .bind(&addr.into()) + .unwrap_or_else(|e| panic!("Failed to bind to HTTPS address {}: {e}", config.address)); + socket + .listen(128) + .unwrap_or_else(|e| panic!("Failed to listen on HTTPS: {e}")); + let listener: std::net::TcpListener = socket.into(); + listener + .set_nonblocking(true) + .expect("Failed to set TLS listener to non-blocking"); + let address = listener + .local_addr() + .expect("Failed to get local address for HTTPS / TLS server"); + + info!("Started {api_name} on: {address}"); + + // Notify shard about the bound address + use crate::shard::transmission::event::ShardEvent; + use iggy_common::TransportProtocol; + let event = ShardEvent::AddressBound { + protocol: TransportProtocol::Http, + address, + }; + + crate::shard::handlers::handle_event(&shard, event) + .await + .ok(); + + let service = app.into_make_service_with_connect_info::(); + let handle = axum_server::Handle::new(); + let shutdown_handle = handle.clone(); + let api_name_for_task = api_name; + shard + .task_registry + .oneshot("http_shutdown_listener") + .critical(false) + .run(move |shutdown: ShutdownToken| async move { + shutdown.wait().await; + info!("Initiating graceful shutdown for {api_name_for_task}"); + shutdown_handle.graceful_shutdown(None); + Ok(()) + }) + .spawn(); + + let server = axum_server::from_tcp_rustls(listener, tls_config) + .map_err(|err| IggyError::HttpError(err.to_string()))? + .handle(handle); + match server.serve(service).await { + Ok(()) => { + info!("{api_name} shut down gracefully"); + Ok(()) + } + Err(error) => { + error!("Failed to start {api_name} server, error: {}", error); + Err(IggyError::CannotBindToSocket(format!("HTTPS: {}", error))) + } + } + } +} + +async fn build_app_state( + config: &HttpConfig, + persister: Arc, + shard: Rc, +) -> Arc { + let tokens_path; + { + tokens_path = shard.config.system.get_state_tokens_path(); + } + + let mut jwt_config = config.jwt.clone(); + let encoding_empty = jwt_config.encoding_secret.is_empty(); + let decoding_empty = jwt_config.decoding_secret.is_empty(); + match (encoding_empty, decoding_empty) { + (true, true) => { + let secret = crypto::generate_secret(32..64); + let redacted: String = secret.chars().take(3).collect(); + warn!( + "JWT encoding and decoding secrets are not configured - generated a random secret: {redacted}***. JWT tokens will be invalidated on server restart. Set 'encoding_secret' and 'decoding_secret' in the config to use persistent secrets." + ); + jwt_config.encoding_secret = secret.clone(); + jwt_config.decoding_secret = secret; + } + (true, false) => { + warn!( + "JWT encoding secret is not configured but decoding secret is set - using decoding secret for both. Set 'encoding_secret' in the config to avoid this warning." + ); + jwt_config.encoding_secret = jwt_config.decoding_secret.clone(); + } + (false, true) => { + warn!( + "JWT decoding secret is not configured but encoding secret is set - using encoding secret for both. Set 'decoding_secret' in the config to avoid this warning." + ); + jwt_config.decoding_secret = jwt_config.encoding_secret.clone(); + } + (false, false) => { + if jwt_config.encoding_secret != jwt_config.decoding_secret + && jwt_config.algorithm.starts_with("HS") + { + warn!( + "JWT encoding and decoding secrets are different but algorithm is {} (HMAC) - both secrets must be identical for symmetric algorithms.", + jwt_config.algorithm + ); + } + } + } + + let jwt_manager = match JwtManager::from_config(persister, &tokens_path, &jwt_config) { + Ok(manager) => manager, + Err(error) => panic!("Failed to initialize JWT manager: {error}"), + }; + if let Err(error) = jwt_manager.load_revoked_tokens().await { + panic!("Failed to load revoked access tokens: {error}"); + } + + Arc::new(AppState { + jwt_manager, + shard: HttpSafeShard::new(shard), + }) +} + +fn configure_cors(config: HttpCorsConfig) -> Result { + let allowed_origins = match config.allowed_origins { + ref origins if origins.is_empty() => AllowOrigin::default(), + ref origins if origins.first().unwrap() == "*" => AllowOrigin::any(), + origins => { + let parsed: Result, _> = origins + .iter() + .filter(|s| !s.trim().is_empty()) + .map(|s| { + s.parse() + .error(|e: &axum::http::header::InvalidHeaderValue| { + format!("Invalid CORS origin '{s}': {e}") + }) + .map_err(|_| IggyError::InvalidConfiguration) + }) + .collect(); + AllowOrigin::list(parsed?) + } + }; + + let allowed_headers: Result, _> = config + .allowed_headers + .iter() + .filter(|s| !s.trim().is_empty()) + .map(|s| { + s.parse() + .error(|e: &axum::http::header::InvalidHeaderName| { + format!("Invalid CORS header '{s}': {e}") + }) + .map_err(|_| IggyError::InvalidConfiguration) + }) + .collect(); + let allowed_headers = allowed_headers?; + + let exposed_headers: Result, _> = config + .exposed_headers + .iter() + .filter(|s| !s.trim().is_empty()) + .map(|s| { + s.parse() + .error(|e: &axum::http::header::InvalidHeaderName| { + format!("Invalid CORS exposed header '{s}': {e}") + }) + .map_err(|_| IggyError::InvalidConfiguration) + }) + .collect(); + let exposed_headers = exposed_headers?; + + let allowed_methods: Result, _> = config + .allowed_methods + .iter() + .filter(|s| !s.trim().is_empty()) + .map(|s| match s.to_uppercase().as_str() { + "GET" => Ok(Method::GET), + "POST" => Ok(Method::POST), + "PUT" => Ok(Method::PUT), + "DELETE" => Ok(Method::DELETE), + "HEAD" => Ok(Method::HEAD), + "OPTIONS" => Ok(Method::OPTIONS), + "CONNECT" => Ok(Method::CONNECT), + "PATCH" => Ok(Method::PATCH), + "TRACE" => Ok(Method::TRACE), + _ => Err(IggyError::InvalidConfiguration) + .error(|_: &IggyError| format!("Invalid HTTP method in CORS config: '{s}'")), + }) + .collect(); + let allowed_methods = allowed_methods?; + + Ok(CorsLayer::new() + .allow_methods(allowed_methods) + .allow_origin(allowed_origins) + .allow_headers(allowed_headers) + .expose_headers(exposed_headers) + .allow_credentials(config.allow_credentials) + .allow_private_network(config.allow_private_network)) +} diff --git a/core/server/src/http/http_shard_wrapper.rs b/core/server/src/http/http_shard_wrapper.rs new file mode 100644 index 0000000000..9952e4de78 --- /dev/null +++ b/core/server/src/http/http_shard_wrapper.rs @@ -0,0 +1,232 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::rc::Rc; + +use iggy_common::{ + Consumer, ConsumerOffsetInfo, Identifier, IggyError, Partitioning, PartitioningKind, +}; +use send_wrapper::SendWrapper; + +use crate::shard::system::messages::PollingArgs; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::ShardRequest; +use crate::streaming::segments::{IggyMessagesBatchMut, IggyMessagesBatchSet}; +use crate::streaming::topics; +use crate::streaming::users::user::User; +use crate::{shard::IggyShard, streaming::session::Session}; +use iggy_common::IggyPollMetadata; + +/// Wrapper around IggyShard for HTTP handlers. +/// +/// Provides three categories of access: +/// 1. Control-plane mutations via `send_to_control_plane()` (routed through message pump) +/// 2. Read-only metadata access via `shard()` (direct, same-thread safe) +/// 3. Data-plane operations (poll/append messages, consumer offsets) +/// +/// # Safety +/// This wrapper is safe because: +/// 1. HTTP server runs on shard 0's single thread (compio model) +/// 2. All operations are confined to that thread +/// 3. The underlying IggyShard is never accessed from multiple threads +pub struct HttpSafeShard { + inner: Rc, +} + +// Safety: HttpSafeShard is only used in HTTP handlers on shard 0's thread. +// All operations are confined to the thread that created the IggyShard instance. +// The underlying IggyShard contains RefCell and Rc types that are not thread-safe, +// but they are never accessed across threads in the HTTP server context with +// compio's single-threaded model. +unsafe impl Send for HttpSafeShard {} +unsafe impl Sync for HttpSafeShard {} + +impl HttpSafeShard { + pub fn new(shard: Rc) -> Self { + Self { inner: shard } + } + + /// Direct access to shard for read-only operations and auth. + pub fn shard(&self) -> &IggyShard { + &self.inner + } + + /// Route control-plane mutations through the message pump. + pub async fn send_to_control_plane( + &self, + request: ShardRequest, + ) -> Result { + let future = SendWrapper::new(self.inner.send_to_control_plane(request)); + future.await + } + + // === Data-plane operations (message polling/appending) === + + pub async fn get_consumer_offset( + &self, + client_id: u32, + consumer: Consumer, + stream_id: &Identifier, + topic_id: &Identifier, + partition_id: Option, + ) -> Result, IggyError> { + let topic = self.shard().resolve_topic(stream_id, topic_id)?; + let future = SendWrapper::new(self.shard().get_consumer_offset( + client_id, + consumer, + topic, + partition_id, + )); + future.await + } + + pub async fn store_consumer_offset( + &self, + client_id: u32, + consumer: Consumer, + stream_id: &Identifier, + topic_id: &Identifier, + partition_id: Option, + offset: u64, + ) -> Result<(), IggyError> { + let topic = self.shard().resolve_topic(stream_id, topic_id)?; + let future = SendWrapper::new(self.shard().store_consumer_offset( + client_id, + consumer, + topic, + partition_id, + offset, + )); + let _result = future.await?; + Ok(()) + } + + pub async fn delete_consumer_offset( + &self, + client_id: u32, + consumer: Consumer, + stream_id: &Identifier, + topic_id: &Identifier, + partition_id: Option, + ) -> Result<(), IggyError> { + let topic = self.shard().resolve_topic(stream_id, topic_id)?; + let future = SendWrapper::new(self.shard().delete_consumer_offset( + client_id, + consumer, + topic, + partition_id, + )); + let _result = future.await?; + Ok(()) + } + + #[allow(clippy::too_many_arguments)] + pub async fn poll_messages( + &self, + client_id: u32, + user_id: u32, + stream_id: Identifier, + topic_id: Identifier, + consumer: Consumer, + maybe_partition_id: Option, + args: PollingArgs, + ) -> Result<(IggyPollMetadata, IggyMessagesBatchSet), IggyError> { + let topic = self + .shard() + .resolve_topic_for_poll(user_id, &stream_id, &topic_id)?; + let future = SendWrapper::new(self.shard().poll_messages( + client_id, + topic, + consumer.clone(), + maybe_partition_id, + args, + )); + + future.await + } + + pub async fn append_messages( + &self, + user_id: u32, + stream_id: Identifier, + topic_id: Identifier, + partitioning: &Partitioning, + batch: IggyMessagesBatchMut, + ) -> Result<(), IggyError> { + use crate::shard::transmission::message::ResolvedPartition; + + let topic = self + .shard() + .resolve_topic_for_append(user_id, &stream_id, &topic_id)?; + let partition_id = match partitioning.kind { + PartitioningKind::Balanced => self + .shard() + .metadata + .get_next_partition_id(topic.stream_id, topic.topic_id) + .ok_or(IggyError::TopicIdNotFound(stream_id, topic_id))?, + PartitioningKind::PartitionId => u32::from_le_bytes( + partitioning + .value + .get(..4) + .ok_or(IggyError::InvalidCommand)? + .try_into() + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ) as usize, + PartitioningKind::MessagesKey => { + let partitions_count = self + .shard() + .metadata + .partitions_count(topic.stream_id, topic.topic_id); + topics::helpers::calculate_partition_id_by_messages_key_hash( + partitions_count, + &partitioning.value, + ) + } + }; + + let partition = ResolvedPartition { + stream_id: topic.stream_id, + topic_id: topic.topic_id, + partition_id, + }; + + let future = SendWrapper::new(self.shard().append_messages(partition, batch)); + future.await + } + + pub fn login_user( + &self, + username: &str, + password: &str, + session: Option<&Session>, + ) -> Result { + self.shard().login_user(username, password, session) + } + + pub fn logout_user(&self, session: &Session) -> Result<(), IggyError> { + self.shard().logout_user(session) + } + + pub fn login_with_personal_access_token( + &self, + token: &str, + session: Option<&Session>, + ) -> Result { + self.shard() + .login_with_personal_access_token(token, session) + } +} diff --git a/core/server/src/http/jwt/json_web_token.rs b/core/server/src/http/jwt/json_web_token.rs new file mode 100644 index 0000000000..c8c95ecf72 --- /dev/null +++ b/core/server/src/http/jwt/json_web_token.rs @@ -0,0 +1,155 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use iggy_common::UserId; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use std::net::SocketAddr; +use std::{fmt, fmt::Display}; + +#[derive(Debug, Clone)] +pub struct Identity { + pub token_id: String, + pub token_expiry: u64, + pub user_id: UserId, + pub ip_address: SocketAddr, +} + +#[derive(Debug, Clone)] +pub enum Audience { + Single(String), + Multiple(Vec), +} + +impl Audience { + pub fn contains(&self, audience: &str) -> bool { + match self { + Audience::Single(aud) => aud == audience, + Audience::Multiple(auds) => auds.iter().any(|a| a == audience), + } + } +} + +impl Display for Audience { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Audience::Single(aud) => f.write_str(aud), + Audience::Multiple(auds) => f.write_str(&auds.join(",")), + } + } +} + +impl From for Audience { + fn from(aud: String) -> Self { + Audience::Single(aud) + } +} + +impl From<&str> for Audience { + fn from(aud: &str) -> Self { + Audience::Single(aud.to_string()) + } +} + +impl From> for Audience { + fn from(auds: Vec) -> Self { + if auds.len() == 1 { + Audience::Single(auds.into_iter().next().unwrap()) + } else { + Audience::Multiple(auds) + } + } +} + +impl Serialize for Audience { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + match self { + Audience::Single(aud) => serializer.serialize_str(aud), + Audience::Multiple(auds) => auds.serialize(serializer), + } + } +} + +impl<'de> Deserialize<'de> for Audience { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct AudienceVisitor; + + impl<'de> serde::de::Visitor<'de> for AudienceVisitor { + type Value = Audience; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a string or an array of strings") + } + + fn visit_str(self, value: &str) -> Result + where + E: serde::de::Error, + { + Ok(Audience::Single(value.to_string())) + } + + fn visit_string(self, value: String) -> Result + where + E: serde::de::Error, + { + Ok(Audience::Single(value)) + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let mut auds = Vec::new(); + while let Some(aud) = seq.next_element::()? { + auds.push(aud); + } + Ok(Audience::Multiple(auds)) + } + } + + deserializer.deserialize_any(AudienceVisitor) + } +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct JwtClaims { + pub jti: String, + pub iss: String, + pub aud: Audience, + pub sub: String, + pub iat: u64, + pub exp: u64, + pub nbf: u64, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct RevokedAccessToken { + pub id: String, + pub expiry: u64, +} + +#[derive(Debug)] +pub struct GeneratedToken { + pub user_id: UserId, + pub access_token: String, + pub access_token_expiry: u64, +} diff --git a/core/server/src/http/jwt/jwks.rs b/core/server/src/http/jwt/jwks.rs new file mode 100644 index 0000000000..0932e5ed44 --- /dev/null +++ b/core/server/src/http/jwt/jwks.rs @@ -0,0 +1,357 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::hash::Hash; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use dashmap::DashMap; +use iggy_common::IggyError; +use jsonwebtoken::DecodingKey; +use serde::Deserialize; +use strum::{Display, EnumString}; +use tokio::sync::Mutex; + +/// Minimum wall-clock gap between outbound JWKS fetches for one trusted issuer. +/// Inside this window a cache miss is authoritative: the issuer's key set was +/// just read, so an absent `kid` is genuinely absent and no fetch is issued. +/// This bounds a pre-auth caller replaying unknown `kid`s to at most one +/// outbound request per issuer per window; the trade is that a freshly rotated +/// key is only discoverable once the window elapses. +const JWKS_REFRESH_MIN_INTERVAL: Duration = Duration::from_secs(10); + +thread_local! { + // cyper 0.9's `Client` is `!Send`/`!Sync` (`Rc`-backed) and `new()` + // now returns a `Result`, so it can no longer be a global `OnceLock`. + // compio is thread-per-core; keep one client per thread. The `Rc` + // inner makes cloning cheap, so callers take an owned handle. + static HTTP_CLIENT: cyper::Client = + cyper::Client::new().expect("failed to build cyper HTTP client for JWKS"); +} + +fn get_http_client() -> cyper::Client { + HTTP_CLIENT.with(cyper::Client::clone) +} + +/// JWK key type enumeration +#[derive(Debug, Clone, Copy, Display, EnumString, Deserialize, PartialEq, Eq)] +#[strum(serialize_all = "UPPERCASE")] +#[serde(rename_all = "UPPERCASE")] +enum JwkKeyType { + /// RSA key type + #[strum(serialize = "RSA")] + Rsa, + /// EC (Elliptic Curve) key type + #[strum(serialize = "EC")] + Ec, +} + +/// EC curve type enumeration +#[derive(Debug, Clone, Copy, Display, EnumString, Deserialize, PartialEq, Eq)] +#[strum(serialize_all = "UPPERCASE")] +#[serde(rename_all = "UPPERCASE")] +enum EcCurve { + /// P-256 curve + #[strum(serialize = "P-256")] + P256, + /// P-384 curve + #[strum(serialize = "P-384")] + P384, + /// P-521 curve + #[strum(serialize = "P-521")] + P521, +} + +#[derive(Debug, Deserialize)] +struct Jwk { + kty: JwkKeyType, + kid: Option, + n: Option, + e: Option, + x: Option, + y: Option, + crv: Option, +} + +#[derive(Debug, Deserialize)] +struct JwkSet { + keys: Vec, +} + +#[derive(Debug, Clone, Hash, Eq, PartialEq)] +struct CacheKey { + issuer: String, + kid: String, +} + +#[derive(Debug, Clone)] +pub struct JwksClient { + cache: DashMap, + /// Per-issuer single-flight and rate-limit guard. The async mutex serialises + /// refresh attempts for one issuer so a burst of concurrent misses collapses + /// onto a single fetch; the inner instant is when that issuer was last + /// fetched and gates [`JWKS_REFRESH_MIN_INTERVAL`]. Keyed only by issuer + /// (operator-configured, bounded), never by the attacker-controlled `kid`. + refresh_guards: DashMap>>>, +} + +impl Default for JwksClient { + fn default() -> Self { + Self { + cache: DashMap::new(), + refresh_guards: DashMap::new(), + } + } +} + +impl JwksClient { + /// Resolve the decoding key for `{issuer, kid}`, fetching and caching the + /// issuer's JWKS on a cache miss. + /// + /// A cache miss is served under a per-issuer guard: concurrent misses fetch + /// once, and within [`JWKS_REFRESH_MIN_INTERVAL`] of the last fetch a miss is + /// treated as a known-absent `kid` and rejected without an outbound request. + /// This keeps an unauthenticated caller replaying unknown `kid`s from + /// amplifying into unbounded fetches against the issuer's JWKS endpoint. + // The per-issuer guard is deliberately held across the JWKS fetch: that hold + // is what serialises concurrent misses onto a single outbound request. Drop- + // tightening would release it before the await and defeat the single-flight. + #[allow(clippy::significant_drop_tightening)] + pub async fn get_key(&self, issuer: &str, jwks_url: &str, kid: &str) -> Option { + let cache_key = CacheKey { + issuer: issuer.to_string(), + kid: kid.to_string(), + }; + + // Positive-cache fast path: no lock, no fetch. + if let Some(key) = self.cache.get(&cache_key) { + return Some(key.clone()); + } + + // Take the per-issuer guard so concurrent misses serialise onto one + // fetch. Clone the Arc out and drop the DashMap entry lock before the + // await, so no shard lock is held across the network I/O. + let entry = self.refresh_guards.entry(issuer.to_string()).or_default(); + let guard = entry.value().clone(); + drop(entry); + let mut last_fetch = guard.lock().await; + + // A prior holder of the guard may have populated our kid while we waited. + if let Some(key) = self.cache.get(&cache_key) { + return Some(key.clone()); + } + + // Inside the refresh window the last fetch's key set still stands, so a + // miss here means the kid is genuinely absent: reject without touching + // the network. One per-issuer timestamp both negative-caches unknown kids + // and rate-limits outbound fetches, with no attacker-keyed state. + if let Some(fetched_at) = *last_fetch + && fetched_at.elapsed() < JWKS_REFRESH_MIN_INTERVAL + { + return None; + } + + // Stale or first contact: fetch. Record the attempt up front so a failing + // issuer is rate-limited too, not re-hit on every miss. + *last_fetch = Some(Instant::now()); + if self.refresh_keys(issuer, jwks_url).await.is_err() { + return None; + } + + self.cache.get(&cache_key).map(|entry| entry.clone()) + } + + async fn refresh_keys(&self, issuer: &str, jwks_url: &str) -> Result<(), IggyError> { + // The cyper client is `!Send` since 0.9; callers reached from axum + // middleware wrap this future in `SendWrapper` (see + // `http::jwt::middleware::jwt_auth`), so it's free to await cyper + // directly here. + let client = get_http_client(); + let request = client + .get(jwks_url) + .map_err(|e| IggyError::CannotFetchJwks(format!("Failed to build request: {}", e)))? + .build(); + let response = client + .execute(request) + .await + .map_err(|e| IggyError::CannotFetchJwks(format!("HTTP request failed: {}", e)))?; + + let body = response.text().await.map_err(|e| { + IggyError::CannotFetchJwks(format!("Failed to read response body: {}", e)) + })?; + + let jwks: JwkSet = serde_json::from_str(&body) + .map_err(|e| IggyError::CannotFetchJwks(format!("Failed to parse JWKS: {}", e)))?; + + // Collect all current kids from the JWKS response + let current_kids: std::collections::HashSet = + jwks.keys.iter().filter_map(|key| key.kid.clone()).collect(); + + // Remove cached keys for this issuer that are no longer in the JWKS response + // Security fix: Clean up revoked/rotated keys to prevent accepting tokens signed with old keys + let keys_to_remove: Vec = self + .cache + .iter() + .filter(|entry| { + entry.key().issuer == issuer && !current_kids.contains(&entry.key().kid) + }) + .map(|entry| entry.key().clone()) + .collect(); + + for key in keys_to_remove { + self.cache.remove(&key); + } + + for key in jwks.keys { + if let Some(kid) = key.kid { + let decoding_key: DecodingKey = match key.kty { + JwkKeyType::Rsa => { + if let (Some(n), Some(e)) = (key.n.as_deref(), key.e.as_deref()) { + DecodingKey::from_rsa_components(n, e).map_err(|e| { + IggyError::CannotFetchJwks(format!("Invalid RSA key: {}", e)) + })? + } else { + continue; + } + } + JwkKeyType::Ec => { + if let (Some(x), Some(y), Some(crv_str)) = + (key.x.as_deref(), key.y.as_deref(), key.crv.as_deref()) + { + if let Ok(_curve) = crv_str.parse::() { + DecodingKey::from_ec_components(x, y).map_err(|e| { + IggyError::CannotFetchJwks(format!("Invalid EC key: {}", e)) + })? + } else { + continue; + } + } else { + continue; + } + } + }; + + let cache_key = CacheKey { + issuer: issuer.to_string(), + kid, + }; + self.cache.insert(cache_key, decoding_key); + } + } + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use jsonwebtoken::DecodingKey; + + const TEST_ISSUER: &str = "https://test-issuer.com"; + const TEST_KID: &str = "test-key"; + + fn create_test_decoding_key() -> DecodingKey { + // Use HMAC secret to create a simple test DecodingKey + // Note: This is only for testing cache logic, not a real RSA/EC key + DecodingKey::from_secret(b"test-secret-key-for-cache-testing-only") + } + + #[test] + fn test_cache_key_equality() { + let key1 = CacheKey { + issuer: TEST_ISSUER.to_string(), + kid: TEST_KID.to_string(), + }; + let key2 = CacheKey { + issuer: TEST_ISSUER.to_string(), + kid: TEST_KID.to_string(), + }; + assert_eq!(key1, key2); + } + + #[test] + fn test_cache_key_different_issuer() { + let key1 = CacheKey { + issuer: "issuer1".to_string(), + kid: TEST_KID.to_string(), + }; + let key2 = CacheKey { + issuer: "issuer2".to_string(), + kid: TEST_KID.to_string(), + }; + assert_ne!(key1, key2); + } + + #[test] + fn test_cache_key_different_kid() { + let key1 = CacheKey { + issuer: TEST_ISSUER.to_string(), + kid: "kid1".to_string(), + }; + let key2 = CacheKey { + issuer: TEST_ISSUER.to_string(), + kid: "kid2".to_string(), + }; + assert_ne!(key1, key2); + } + + #[test] + fn test_jwks_client_default() { + let client = JwksClient::default(); + assert!(client.cache.is_empty()); + } + + #[test] + fn test_cache_insert_and_get() { + let client = JwksClient::default(); + let cache_key = CacheKey { + issuer: TEST_ISSUER.to_string(), + kid: TEST_KID.to_string(), + }; + let decoding_key = create_test_decoding_key(); + + client.cache.insert(cache_key.clone(), decoding_key.clone()); + + let cached = client.cache.get(&cache_key); + assert!(cached.is_some()); + } + + #[test] + fn test_cache_multiple_keys() { + let client = JwksClient::default(); + + let key1 = CacheKey { + issuer: "issuer1".to_string(), + kid: "kid1".to_string(), + }; + let key2 = CacheKey { + issuer: "issuer2".to_string(), + kid: "kid2".to_string(), + }; + + let decoding_key1 = create_test_decoding_key(); + let decoding_key2 = create_test_decoding_key(); + + client.cache.insert(key1.clone(), decoding_key1); + client.cache.insert(key2.clone(), decoding_key2); + + assert_eq!(client.cache.len(), 2); + assert!(client.cache.get(&key1).is_some()); + assert!(client.cache.get(&key2).is_some()); + } +} diff --git a/core/server/src/http/jwt/jwt_manager.rs b/core/server/src/http/jwt/jwt_manager.rs new file mode 100644 index 0000000000..2c40416be9 --- /dev/null +++ b/core/server/src/http/jwt/jwt_manager.rs @@ -0,0 +1,457 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::configs::http::{HttpJwtConfig, TrustedIssuerConfig}; +use crate::http::jwt::COMPONENT; +use crate::http::jwt::json_web_token::{Audience, GeneratedToken, JwtClaims, RevokedAccessToken}; +use crate::http::jwt::jwks::JwksClient; +use crate::http::jwt::storage::TokenStorage; +use crate::streaming::persistence::persister::PersisterKind; +use ahash::AHashMap; +use err_trail::ErrContext; +use iggy_common::IggyDuration; +use iggy_common::IggyError; +use iggy_common::IggyExpiry; +use iggy_common::IggyTimestamp; +use iggy_common::UserId; +use iggy_common::locking::IggyRwLock; +use iggy_common::locking::IggyRwLockFn; +use jsonwebtoken::{Algorithm, DecodingKey, EncodingKey, Header, TokenData, Validation, encode}; +use std::collections::HashMap; +use std::sync::Arc; +use tracing::{debug, error, info}; + +pub struct IssuerOptions { + pub issuer: String, + pub audience: String, + pub access_token_expiry: IggyExpiry, + pub not_before: IggyDuration, + pub key: EncodingKey, + pub algorithm: Algorithm, +} + +pub struct ValidatorOptions { + pub valid_audiences: Vec, + pub valid_issuers: Vec, + pub clock_skew: IggyDuration, + pub key: DecodingKey, +} + +pub struct JwtManager { + issuer: IssuerOptions, + validator: ValidatorOptions, + tokens_storage: TokenStorage, + revoked_tokens: IggyRwLock>, + validations: AHashMap, + jwks_client: JwksClient, + trusted_issuer: HashMap, +} + +impl JwtManager { + pub fn new( + persister: Arc, + path: &str, + issuer: IssuerOptions, + validator: ValidatorOptions, + ) -> Result { + let validation = JwtManager::create_validation( + issuer.algorithm, + &validator.valid_issuers, + &validator.valid_audiences, + validator.clock_skew, + ); + + Ok(Self { + validations: vec![(issuer.algorithm, validation)].into_iter().collect(), + issuer, + validator, + tokens_storage: TokenStorage::new(persister, path), + revoked_tokens: IggyRwLock::new(AHashMap::new()), + jwks_client: JwksClient::default(), + trusted_issuer: HashMap::new(), + }) + } + + pub fn from_config( + persister: Arc, + path: &str, + config: &HttpJwtConfig, + ) -> Result { + let algorithm = config.get_algorithm()?; + let issuer = IssuerOptions { + issuer: config.issuer.clone(), + audience: config.audience.clone(), + access_token_expiry: config.access_token_expiry, + not_before: config.not_before, + key: config.get_encoding_key().error(|e: &IggyError| { + format!("{COMPONENT} (error: {e}) - failed to get encoding key") + })?, + algorithm, + }; + let validator = ValidatorOptions { + valid_audiences: config.valid_audiences.clone(), + valid_issuers: config.valid_issuers.clone(), + clock_skew: config.clock_skew, + key: config.get_decoding_key().error(|e: &IggyError| { + format!("{COMPONENT} (error: {e}) - failed to get decoding key") + })?, + }; + let mut manager = JwtManager::new(persister, path, issuer, validator)?; + + if let Some(trusted_issuers) = config.trusted_issuers.as_ref() { + for issuer_config in trusted_issuers { + let normalized_issuer = normalize_issuer_url(&issuer_config.issuer); + manager + .trusted_issuer + .insert(normalized_issuer, issuer_config.clone()); + } + } + + Ok(manager) + } + + fn create_validation( + algorithm: Algorithm, + issuers: &[String], + audiences: &[String], + clock_skew: IggyDuration, + ) -> Validation { + let mut validator = Validation::new(algorithm); + validator.set_issuer(issuers); + validator.set_audience(audiences); + validator.leeway = clock_skew.as_secs() as u64; + validator + } + + pub async fn load_revoked_tokens(&self) -> Result<(), IggyError> { + let revoked_tokens = self.tokens_storage.load_all_revoked_access_tokens().await?; + let mut tokens = self.revoked_tokens.write().await; + for token in revoked_tokens { + tokens.insert(token.id, token.expiry); + } + Ok(()) + } + + pub async fn delete_expired_revoked_tokens(&self, now: u64) -> Result<(), IggyError> { + let mut tokens_to_delete = Vec::new(); + let revoked_tokens = self.revoked_tokens.read().await; + for (id, expiry) in revoked_tokens.iter() { + if expiry <= &now { + tokens_to_delete.push(id.to_string()); + } + } + drop(revoked_tokens); + + debug!( + "Found {} expired revoked access tokens to delete.", + tokens_to_delete.len() + ); + if tokens_to_delete.is_empty() { + return Ok(()); + } + + debug!( + "Deleting {} expired revoked access tokens...", + tokens_to_delete.len() + ); + self.tokens_storage + .delete_revoked_access_tokens(&tokens_to_delete) + .await + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to delete revoked access tokens, IDs {tokens_to_delete:?}" + ) + })?; + let mut revoked_tokens = self.revoked_tokens.write().await; + for id in tokens_to_delete { + revoked_tokens.remove(&id); + info!("Deleted expired revoked access token with ID: {id}") + } + Ok(()) + } + + pub fn generate(&self, user_id: UserId) -> Result { + let header = Header::new(self.issuer.algorithm); + let now = IggyTimestamp::now().to_secs(); + let iat = now; + let exp = iat + + (match self.issuer.access_token_expiry { + IggyExpiry::NeverExpire => 1_000_000_000, + IggyExpiry::ServerDefault => 0, // This is not a case, as the server default is not allowed here + IggyExpiry::ExpireDuration(duration) => duration.as_secs(), + }) as u64; + let nbf = iat + self.issuer.not_before.as_secs() as u64; + let claims = JwtClaims { + jti: uuid::Uuid::now_v7().to_string(), + sub: user_id.to_string(), + aud: Audience::from(self.issuer.audience.clone()), + iss: self.issuer.issuer.to_string(), + iat, + exp, + nbf, + }; + + let access_token = encode::(&header, &claims, &self.issuer.key); + if let Err(e) = access_token { + error!("Cannot generate JWT token. Error: {e}"); + return Err(IggyError::CannotGenerateJwt); + } + + Ok(GeneratedToken { + user_id, + access_token: access_token.unwrap(), + access_token_expiry: exp, + }) + } + + // The access token can be refreshed only once and if it is not expired + pub async fn refresh_token(&self, token: &str) -> Result { + if token.is_empty() { + return Err(IggyError::InvalidAccessToken); + } + + let token_header = + jsonwebtoken::decode_header(token).map_err(|_| IggyError::InvalidAccessToken)?; + let jwt_claims = self.decode(token, token_header.alg).await?; + + // Security fix: Reject A2A tokens from external trusted issuers + // A2A tokens should not be refreshable - they have their own lifecycle + let normalized_iss = normalize_issuer_url(&jwt_claims.claims.iss); + if self.trusted_issuer.contains_key(&normalized_iss) { + error!( + "Cannot refresh A2A token from external issuer: {}", + jwt_claims.claims.iss + ); + return Err(IggyError::InvalidAccessToken); + } + + let id = jwt_claims.claims.jti; + let expiry = jwt_claims.claims.exp; + if self + .revoked_tokens + .write() + .await + .insert(id.clone(), expiry) + .is_some() + { + return Err(IggyError::InvalidAccessToken); + } + + self.tokens_storage + .save_revoked_access_token(&RevokedAccessToken { + id: id.clone(), + expiry, + }) + .await + .error(|e: &IggyError| { + format!("{COMPONENT} (error: {e}) - failed to save revoked access token: {id}") + })?; + let user_id = jwt_claims + .claims + .sub + .parse::() + .map_err(|_| IggyError::InvalidAccessToken)?; + self.generate(user_id) + } + + pub async fn decode( + &self, + token: &str, + algorithm: Algorithm, + ) -> Result, IggyError> { + let validation = self.validations.get(&algorithm); + let kid = jsonwebtoken::decode_header(token).ok().and_then(|h| h.kid); + + // try to decode using JWKS if it's a trusted issuer + let insecure = match jsonwebtoken::dangerous::insecure_decode::(token) { + Ok(claims) => claims, + Err(_) => { + error!("Failed to decode JWT insecurely"); + return self.decode_with_fallback(token, validation, algorithm); + } + }; + + let normalized_iss = normalize_issuer_url(&insecure.claims.iss); + let config = match self.trusted_issuer.get(&normalized_iss) { + Some(config) => config, + None => { + debug!("No trusted issuer found for: {}", insecure.claims.iss); + return self.decode_with_fallback(token, validation, algorithm); + } + }; + + if config.user_id == 0 { + error!( + "A2A token cannot map to root user (user_id = 0) for issuer: {}", + config.issuer + ); + return Err(IggyError::Unauthenticated); + } + + let kid_str = match kid.as_deref() { + Some(kid) => kid, + None => { + error!("No kid found in JWT header"); + return self.decode_with_fallback(token, validation, algorithm); + } + }; + + let decoding_key = match self + .jwks_client + .get_key(&config.issuer, &config.jwks_url, kid_str) + .await + { + Some(key) => key, + None => { + error!("Failed to get decoding key from JWKS for kid: {}", kid_str); + return self.decode_with_fallback(token, validation, algorithm); + } + }; + let mut validation = Validation::new(algorithm); + validation.set_issuer(std::slice::from_ref(&config.issuer)); + validation.set_audience(std::slice::from_ref(&config.audience)); + + let mut result = jsonwebtoken::decode::(token, &decoding_key, &validation) + .map_err(|e| { + error!("Failed to decode JWT: {}", e); + IggyError::Unauthenticated + })?; + + result.claims.sub = config.user_id.to_string(); + + Ok(result) + } + + /// fallback to standard JWT validation if JWKS validation fails + fn decode_with_fallback( + &self, + token: &str, + validation: Option<&Validation>, + algorithm: Algorithm, + ) -> Result, IggyError> { + let validation = validation.ok_or_else(|| { + IggyError::InvalidJwtAlgorithm(Self::map_algorithm_to_string(algorithm)) + })?; + + jsonwebtoken::decode::(token, &self.validator.key, validation) + .map_err(|_| IggyError::Unauthenticated) + } + + fn map_algorithm_to_string(algorithm: Algorithm) -> String { + match algorithm { + Algorithm::HS256 => "HS256", + Algorithm::HS384 => "HS384", + Algorithm::HS512 => "HS512", + Algorithm::RS256 => "RS256", + Algorithm::RS384 => "RS384", + Algorithm::RS512 => "RS512", + _ => "Unknown", + } + .to_string() + } + + pub async fn revoke_token(&self, token_id: &str, expiry: u64) -> Result<(), IggyError> { + let mut revoked_tokens = self.revoked_tokens.write().await; + revoked_tokens.insert(token_id.to_string(), expiry); + self.tokens_storage + .save_revoked_access_token(&RevokedAccessToken { + id: token_id.to_string(), + expiry, + }) + .await + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to save revoked access token: {token_id}" + ) + })?; + info!("Revoked access token with ID: {token_id}"); + Ok(()) + } + + pub async fn is_token_revoked(&self, token_id: &str) -> bool { + let revoked_tokens = self.revoked_tokens.read().await; + revoked_tokens.contains_key(token_id) + } +} + +/// Normalize issuer URL by lowercasing scheme and host, preserving path case +/// +/// Example: "HTTPS://Example.COM/PATH" -> "https://example.com/PATH" +fn normalize_issuer_url(url: &str) -> String { + match url.split_once("://") { + Some((scheme, rest)) => { + let scheme = scheme.to_lowercase(); + // Find end of host (first '/' or end of string) + let (host, path) = match rest.find('/') { + Some(idx) => rest.split_at(idx), + None => (rest, ""), + }; + format!("{}://{}{}", scheme, host.to_lowercase(), path) + } + None => url.trim_end_matches('/').to_lowercase(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_normalize_issuer_url_basic() { + assert_eq!( + normalize_issuer_url("HTTPS://Example.COM/PATH"), + "https://example.com/PATH" + ); + } + + #[test] + fn test_normalize_issuer_url_no_path() { + assert_eq!( + normalize_issuer_url("HTTPS://Example.COM"), + "https://example.com" + ); + } + + #[test] + fn test_normalize_issuer_url_no_scheme() { + assert_eq!(normalize_issuer_url("Example.COM"), "example.com"); + } + + #[test] + fn test_normalize_issuer_url_trailing_slash() { + assert_eq!( + normalize_issuer_url("HTTPS://Example.COM/"), + "https://example.com/" + ); + } + + #[test] + fn test_normalize_issuer_url_preserves_path_case() { + assert_eq!( + normalize_issuer_url("https://EXAMPLE.com/MyPath/SubPath"), + "https://example.com/MyPath/SubPath" + ); + } + + #[test] + fn test_normalize_issuer_url_already_normalized() { + assert_eq!( + normalize_issuer_url("https://example.com/path"), + "https://example.com/path" + ); + } +} diff --git a/core/server/src/http/jwt/middleware.rs b/core/server/src/http/jwt/middleware.rs new file mode 100644 index 0000000000..0ec31ccdad --- /dev/null +++ b/core/server/src/http/jwt/middleware.rs @@ -0,0 +1,105 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::http::jwt::json_web_token::Identity; +use crate::http::shared::{AppState, RequestDetails}; +use axum::body::Body; +use axum::{ + extract::State, + http::{Request, StatusCode}, + middleware::Next, + response::Response, +}; +use err_trail::ErrContext; +use send_wrapper::SendWrapper; +use std::sync::Arc; + +const COMPONENT: &str = "JWT_MIDDLEWARE"; +const AUTHORIZATION: &str = "authorization"; +const BEARER: &str = "Bearer "; +const UNAUTHORIZED: StatusCode = StatusCode::UNAUTHORIZED; + +const PUBLIC_PATHS: &[&str] = &[ + "/", + "/metrics", + "/ping", + "/users/login", + "/users/refresh-token", + "/personal-access-tokens/login", + "/ui", +]; + +pub async fn jwt_auth( + State(state): State>, + mut request: Request, + next: Next, +) -> Result { + if PUBLIC_PATHS.contains(&request.uri().path()) { + return Ok(next.run(request).await); + } + + let bearer = request + .headers() + .get(AUTHORIZATION) + .ok_or(UNAUTHORIZED) + .error(|e: &StatusCode| { + format!("{COMPONENT} (error: {e}) - missing or inaccessible Authorization header") + })? + .to_str() + .error(|e: &axum::http::header::ToStrError| { + format!("{COMPONENT} (error: {e}) - invalid authorization header format") + }) + .map_err(|_| UNAUTHORIZED)?; + + if !bearer.starts_with(BEARER) { + return Err(StatusCode::UNAUTHORIZED); + } + + let jwt_token = &bearer[BEARER.len()..]; + let token_header = jsonwebtoken::decode_header(jwt_token) + .error(|e: &jsonwebtoken::errors::Error| { + format!("{COMPONENT} (error: {e}) - failed to decode JWT header") + }) + .map_err(|_| UNAUTHORIZED)?; + // `decode` may fetch JWKS via the cyper client, which is `!Send` since + // cyper 0.9 (`Rc`-backed). axum requires this middleware's future to be + // `Send`, so wrap the `!Send` sub-futures in `SendWrapper` -- the same + // pattern the rest of the HTTP layer uses for compio shard ops. + // Sound under compio's thread-per-core model: the future is never + // polled from another thread. + let jwt_claims = SendWrapper::new(state.jwt_manager.decode(jwt_token, token_header.alg)) + .await + .map_err(|_| UNAUTHORIZED)?; + if SendWrapper::new(state.jwt_manager.is_token_revoked(&jwt_claims.claims.jti)).await { + return Err(StatusCode::UNAUTHORIZED); + } + + let request_details = request.extensions().get::().unwrap(); + let user_id = jwt_claims + .claims + .sub + .parse::() + .map_err(|_| UNAUTHORIZED)?; + let identity = Identity { + token_id: jwt_claims.claims.jti, + token_expiry: jwt_claims.claims.exp, + user_id, + ip_address: request_details.ip_address, + }; + request.extensions_mut().insert(identity); + Ok(next.run(request).await) +} diff --git a/core/server/src/http/jwt/mod.rs b/core/server/src/http/jwt/mod.rs new file mode 100644 index 0000000000..e63d2165fd --- /dev/null +++ b/core/server/src/http/jwt/mod.rs @@ -0,0 +1,24 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod json_web_token; +pub mod jwks; +pub mod jwt_manager; +pub mod middleware; +pub mod storage; + +pub const COMPONENT: &str = "HTTP_JWT"; diff --git a/core/server/src/http/jwt/storage.rs b/core/server/src/http/jwt/storage.rs new file mode 100644 index 0000000000..c56c7509a3 --- /dev/null +++ b/core/server/src/http/jwt/storage.rs @@ -0,0 +1,149 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::http::jwt::COMPONENT; +use crate::{ + http::jwt::json_web_token::RevokedAccessToken, streaming::persistence::persister::PersisterKind, +}; +use ahash::AHashMap; +use anyhow::Context; +use err_trail::ErrContext; +use iggy_common::IggyError; +use std::sync::Arc; +use tracing::{error, info}; + +#[derive(Debug)] +pub struct TokenStorage { + persister: Arc, + path: String, +} + +impl TokenStorage { + pub fn new(persister: Arc, path: &str) -> Self { + Self { + persister, + path: path.to_owned(), + } + } + + pub async fn load_all_revoked_access_tokens( + &self, + ) -> Result, IggyError> { + // Check if file exists by trying to get metadata (equivalent to original file open check) + let file_size = match compio::fs::metadata(&self.path).await { + Err(_) => { + info!("No revoked access tokens found to load."); + return Ok(vec![]); + } + Ok(metadata) => metadata.len() as usize, + }; + + info!("Loading revoked access tokens from: {}", self.path); + + let buffer = compio::fs::read(&self.path) + .await + .error(|e: &std::io::Error| { + format!( + "{COMPONENT} (error: {e}) - failed to read file into buffer, path: {}", + self.path + ) + }) + .map_err(|e| { + error!("Cannot open revoked access tokens file: {e}"); + IggyError::CannotReadFile + })?; + + if buffer.len() != file_size { + error!( + "File size mismatch: expected {file_size}, got {}", + buffer.len() + ); + return Err(IggyError::CannotReadFile); + } + + let tokens: AHashMap = rmp_serde::from_slice(&buffer) + .with_context(|| "Failed to deserialize revoked access tokens") + .map_err(|_| IggyError::CannotDeserializeResource)?; + + let tokens = tokens + .into_iter() + .map(|(id, expiry)| RevokedAccessToken { id, expiry }) + .collect::>(); + + info!("Loaded {} revoked access tokens", tokens.len()); + Ok(tokens) + } + + pub async fn save_revoked_access_token( + &self, + token: &RevokedAccessToken, + ) -> Result<(), IggyError> { + let tokens = self.load_all_revoked_access_tokens().await?; + let mut map = tokens + .into_iter() + .map(|token| (token.id, token.expiry)) + .collect::>(); + map.insert(token.id.to_owned(), token.expiry); + let bytes = rmp_serde::to_vec(&map) + .with_context(|| "Failed to serialize revoked access tokens") + .map_err(|_| IggyError::CannotSerializeResource)?; + self.persister + .overwrite(&self.path, bytes) + .await + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to overwrite file, path: {}", + self.path + ) + })?; + Ok(()) + } + + pub async fn delete_revoked_access_tokens(&self, id: &[String]) -> Result<(), IggyError> { + let tokens = self + .load_all_revoked_access_tokens() + .await + .error(|e: &IggyError| { + format!("{COMPONENT} (error: {e}) - failed to load revoked access tokens") + })?; + if tokens.is_empty() { + return Ok(()); + } + + let mut map = tokens + .into_iter() + .map(|token| (token.id, token.expiry)) + .collect::>(); + for id in id { + map.remove(id); + } + + let bytes = rmp_serde::to_vec(&map) + .with_context(|| "Failed to serialize revoked access tokens") + .map_err(|_| IggyError::CannotSerializeResource)?; + self.persister + .overwrite(&self.path, bytes) + .await + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to overwrite file, path: {}", + self.path + ) + })?; + Ok(()) + } +} diff --git a/core/server/src/http/mapper.rs b/core/server/src/http/mapper.rs new file mode 100644 index 0000000000..eaf031a6a8 --- /dev/null +++ b/core/server/src/http/mapper.rs @@ -0,0 +1,329 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::http::jwt::json_web_token::GeneratedToken; +use crate::metadata::{ConsumerGroupMeta, InnerMetadata, PartitionMeta, StreamMeta, TopicMeta}; +use crate::streaming::clients::client_manager::Client; +use crate::streaming::users::user::User; +use iggy_common::PersonalAccessToken; +use iggy_common::{ConsumerGroupDetails, ConsumerGroupInfo, ConsumerGroupMember, IggyByteSize}; +use iggy_common::{IdentityInfo, PersonalAccessTokenInfo, TokenInfo, TopicDetails}; +use iggy_common::{UserInfo, UserInfoDetails}; + +pub fn map_user(user: &User) -> UserInfoDetails { + UserInfoDetails { + id: user.id, + username: user.username.clone(), + created_at: user.created_at, + status: user.status, + permissions: user.permissions.clone(), + } +} + +pub fn map_users(users: &[&User]) -> Vec { + let mut users_data = Vec::with_capacity(users.len()); + for user in users { + let user = UserInfo { + id: user.id, + username: user.username.clone(), + created_at: user.created_at, + status: user.status, + }; + users_data.push(user); + } + users_data.sort_by_key(|u| u.id); + users_data +} + +pub fn map_personal_access_tokens( + personal_access_tokens: &[PersonalAccessToken], +) -> Vec { + let mut personal_access_tokens_data = Vec::with_capacity(personal_access_tokens.len()); + for personal_access_token in personal_access_tokens { + let personal_access_token = PersonalAccessTokenInfo { + name: (*personal_access_token.name).to_owned(), + expiry_at: personal_access_token.expiry_at, + }; + personal_access_tokens_data.push(personal_access_token); + } + personal_access_tokens_data.sort_by(|a, b| a.name.cmp(&b.name)); + personal_access_tokens_data +} + +pub fn map_client(client: &Client) -> iggy_common::ClientInfoDetails { + iggy_common::ClientInfoDetails { + client_id: client.session.client_id, + user_id: client.user_id, + transport: client.transport.to_string(), + address: client.session.ip_address.to_string(), + consumer_groups_count: client.consumer_groups.len() as u32, + consumer_groups: client + .consumer_groups + .iter() + .map(|consumer_group| ConsumerGroupInfo { + stream_id: consumer_group.stream_id, + topic_id: consumer_group.topic_id, + group_id: consumer_group.group_id, + }) + .collect(), + } +} + +pub fn map_clients(clients: &[Client]) -> Vec { + let mut all_clients = Vec::new(); + for client in clients { + let client = iggy_common::ClientInfo { + client_id: client.session.client_id, + user_id: client.user_id, + transport: client.transport.to_string(), + address: client.session.ip_address.to_string(), + consumer_groups_count: client.consumer_groups.len() as u32, + }; + all_clients.push(client); + } + + all_clients.sort_by_key(|c| c.client_id); + all_clients +} + +pub fn map_generated_access_token_to_identity_info(token: GeneratedToken) -> IdentityInfo { + IdentityInfo { + user_id: token.user_id, + access_token: Some(TokenInfo { + token: token.access_token, + expiry: token.access_token_expiry, + }), + } +} + +/// Map a stream from SharedMetadata to StreamDetails (with topics) +pub fn map_stream_details_from_metadata(stream_meta: &StreamMeta) -> iggy_common::StreamDetails { + // Get topic IDs sorted + let mut topic_ids: Vec<_> = stream_meta.topics.iter().map(|(k, _)| k).collect(); + topic_ids.sort_unstable(); + + // Map topics + let mut topics = Vec::with_capacity(topic_ids.len()); + for topic_id in topic_ids { + if let Some(topic_meta) = stream_meta.topics.get(topic_id) { + topics.push(map_topic_from_metadata(topic_meta)); + } + } + + // Aggregate stats + let (total_size, total_messages) = aggregate_stream_stats(stream_meta); + + iggy_common::StreamDetails { + id: stream_meta.id as u32, + created_at: stream_meta.created_at, + name: stream_meta.name.to_string(), + topics_count: topics.len() as u32, + size: IggyByteSize::from(total_size), + messages_count: total_messages, + topics, + } +} + +/// Map a stream from SharedMetadata to Stream (without topics) +pub fn map_stream_from_metadata(stream_meta: &StreamMeta) -> iggy_common::Stream { + let (total_size, total_messages) = aggregate_stream_stats(stream_meta); + + iggy_common::Stream { + id: stream_meta.id as u32, + created_at: stream_meta.created_at, + name: stream_meta.name.to_string(), + topics_count: stream_meta.topics.len() as u32, + size: IggyByteSize::from(total_size), + messages_count: total_messages, + } +} + +/// Map all streams from SharedMetadata +pub fn map_streams_from_metadata(metadata: &InnerMetadata) -> Vec { + let mut stream_ids: Vec<_> = metadata.streams.iter().map(|(k, _)| k).collect(); + stream_ids.sort_unstable(); + + let mut streams = Vec::with_capacity(stream_ids.len()); + for stream_id in stream_ids { + if let Some(stream_meta) = metadata.streams.get(stream_id) { + streams.push(map_stream_from_metadata(stream_meta)); + } + } + streams +} + +/// Map a topic from SharedMetadata to Topic (without partitions) +pub fn map_topic_from_metadata(topic_meta: &TopicMeta) -> iggy_common::Topic { + let (total_size, total_messages) = aggregate_topic_stats(topic_meta); + + iggy_common::Topic { + id: topic_meta.id as u32, + created_at: topic_meta.created_at, + name: topic_meta.name.to_string(), + size: IggyByteSize::from(total_size), + partitions_count: topic_meta.partitions.len() as u32, + messages_count: total_messages, + message_expiry: topic_meta.message_expiry, + compression_algorithm: topic_meta.compression_algorithm, + max_topic_size: topic_meta.max_topic_size, + replication_factor: topic_meta.replication_factor, + } +} + +/// Map all topics for a stream from SharedMetadata +pub fn map_topics_from_metadata(stream_meta: &StreamMeta) -> Vec { + let mut topic_ids: Vec<_> = stream_meta.topics.iter().map(|(k, _)| k).collect(); + topic_ids.sort_unstable(); + + let mut topics = Vec::with_capacity(topic_ids.len()); + for topic_id in topic_ids { + if let Some(topic_meta) = stream_meta.topics.get(topic_id) { + topics.push(map_topic_from_metadata(topic_meta)); + } + } + topics +} + +/// Map a topic from SharedMetadata to TopicDetails (with partitions) +pub fn map_topic_details_from_metadata(topic_meta: &TopicMeta) -> TopicDetails { + // Get partition IDs sorted + let mut partition_ids: Vec<_> = topic_meta + .partitions + .iter() + .enumerate() + .map(|(k, _)| k) + .collect(); + partition_ids.sort_unstable(); + + // Map partitions + let mut partitions = Vec::with_capacity(partition_ids.len()); + for partition_id in partition_ids { + if let Some(partition_meta) = topic_meta.partitions.get(partition_id) { + partitions.push(map_partition_from_metadata(partition_meta)); + } + } + + // Aggregate stats + let (total_size, total_messages) = aggregate_topic_stats(topic_meta); + + TopicDetails { + id: topic_meta.id as u32, + created_at: topic_meta.created_at, + name: topic_meta.name.to_string(), + size: IggyByteSize::from(total_size), + messages_count: total_messages, + partitions_count: partitions.len() as u32, + partitions, + message_expiry: topic_meta.message_expiry, + compression_algorithm: topic_meta.compression_algorithm, + max_topic_size: topic_meta.max_topic_size, + replication_factor: topic_meta.replication_factor, + } +} + +/// Map a partition from SharedMetadata +pub fn map_partition_from_metadata(partition_meta: &PartitionMeta) -> iggy_common::Partition { + let stats = &partition_meta.stats; + let segments_count = stats.segments_count_inconsistent(); + let size_bytes = stats.size_bytes_inconsistent(); + let messages_count = stats.messages_count_inconsistent(); + let current_offset = stats.current_offset(); + + iggy_common::Partition { + id: partition_meta.id as u32, + created_at: partition_meta.created_at, + segments_count, + current_offset, + size: IggyByteSize::from(size_bytes), + messages_count, + } +} + +/// Map a consumer group from SharedMetadata +pub fn map_consumer_group_from_metadata(cg_meta: &ConsumerGroupMeta) -> iggy_common::ConsumerGroup { + iggy_common::ConsumerGroup { + id: cg_meta.id as u32, + name: cg_meta.name.to_string(), + partitions_count: cg_meta.partitions.len() as u32, + members_count: cg_meta.members.len() as u32, + } +} + +/// Map a consumer group to ConsumerGroupDetails from SharedMetadata +pub fn map_consumer_group_details_from_metadata( + cg_meta: &ConsumerGroupMeta, +) -> ConsumerGroupDetails { + let members: Vec = cg_meta + .members + .iter() + .map(|(_, member)| ConsumerGroupMember { + id: member.id as u32, + partitions_count: member.partitions.len() as u32, + partitions: member.partitions.iter().map(|&p| p as u32).collect(), + }) + .collect(); + + ConsumerGroupDetails { + id: cg_meta.id as u32, + name: cg_meta.name.to_string(), + partitions_count: cg_meta.partitions.len() as u32, + members_count: members.len() as u32, + members, + } +} + +/// Map all consumer groups for a topic from SharedMetadata +pub fn map_consumer_groups_from_metadata( + topic_meta: &TopicMeta, +) -> Vec { + let mut group_ids: Vec<_> = topic_meta.consumer_groups.iter().map(|(k, _)| k).collect(); + group_ids.sort_unstable(); + + let mut groups = Vec::with_capacity(group_ids.len()); + for group_id in group_ids { + if let Some(cg_meta) = topic_meta.consumer_groups.get(group_id) { + groups.push(map_consumer_group_from_metadata(cg_meta)); + } + } + groups +} + +fn aggregate_stream_stats(stream_meta: &StreamMeta) -> (u64, u64) { + let mut total_size = 0u64; + let mut total_messages = 0u64; + + for (_, topic_meta) in stream_meta.topics.iter() { + for partition_meta in topic_meta.partitions.iter() { + total_size += partition_meta.stats.size_bytes_inconsistent(); + total_messages += partition_meta.stats.messages_count_inconsistent(); + } + } + + (total_size, total_messages) +} + +fn aggregate_topic_stats(topic_meta: &TopicMeta) -> (u64, u64) { + let mut total_size = 0u64; + let mut total_messages = 0u64; + + for partition_meta in topic_meta.partitions.iter() { + total_size += partition_meta.stats.size_bytes_inconsistent(); + total_messages += partition_meta.stats.messages_count_inconsistent(); + } + + (total_size, total_messages) +} diff --git a/core/server/src/http/messages.rs b/core/server/src/http/messages.rs new file mode 100644 index 0000000000..b51df4aef1 --- /dev/null +++ b/core/server/src/http/messages.rs @@ -0,0 +1,164 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::http::COMPONENT; +use crate::http::error::CustomError; +use crate::http::jwt::json_web_token::Identity; +use crate::http::shared::AppState; +use crate::shard::system::messages::PollingArgs; +use crate::shard::transmission::message::ResolvedPartition; +use crate::streaming::segments::IggyMessagesBatchMut; +use crate::streaming::session::Session; +use axum::extract::{Path, Query, State}; +use axum::http::StatusCode; +use axum::routing::get; +use axum::{Extension, Json, Router, debug_handler}; +use err_trail::ErrContext; +use iggy_common::Identifier; +use iggy_common::Validatable; +use iggy_common::{Consumer, PollMessages, SendMessages}; +use iggy_common::{IggyError, IggyMessagesBatch, PolledMessages}; +use send_wrapper::SendWrapper; +use server_common::{IggyIndexesMut, PooledBuffer}; +use std::sync::Arc; +use tracing::instrument; + +pub fn router(state: Arc) -> Router { + Router::new() + .route( + "/streams/{stream_id}/topics/{topic_id}/messages", + get(poll_messages).post(send_messages), + ) + .route( + "/streams/{stream_id}/topics/{topic_id}/messages/flush/{partition_id}/{fsync}", + get(flush_unsaved_buffer), + ) + .with_state(state) +} + +#[debug_handler] +async fn poll_messages( + State(state): State>, + Extension(identity): Extension, + Path((stream_id, topic_id)): Path<(String, String)>, + mut query: Query, +) -> Result, CustomError> { + query.stream_id = Identifier::from_str_value(&stream_id)?; + query.topic_id = Identifier::from_str_value(&topic_id)?; + query.validate()?; + + let consumer = Consumer::new(query.0.consumer.id); + + let session = Session::stateless(identity.user_id, identity.ip_address); + + let poll_future = SendWrapper::new(state.shard.poll_messages( + session.client_id, + session.get_user_id(), + query.0.stream_id, + query.0.topic_id, + consumer, + query.0.partition_id, + PollingArgs::new(query.0.strategy, query.0.count, query.0.auto_commit), + )); + + let (metadata, messages) = poll_future + .await + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to poll messages, stream ID: {}, topic ID: {}, partition ID: {:?}", + stream_id, topic_id, query.0.partition_id + ) + })?; + let polled_messages = messages.into_polled_messages(metadata); + + Ok(Json(polled_messages)) +} + +#[debug_handler] +async fn send_messages( + State(state): State>, + Extension(identity): Extension, + Path((stream_id, topic_id)): Path<(String, String)>, + Json(mut command): Json, +) -> Result { + command.stream_id = Identifier::from_str_value(&stream_id)?; + command.topic_id = Identifier::from_str_value(&topic_id)?; + command.partitioning.length = command.partitioning.value.len() as u8; + command.validate()?; + + let batch = make_mutable(command.batch); + let command_stream_id = command.stream_id; + let command_topic_id = command.topic_id; + let partitioning = command.partitioning; + + let append_future = SendWrapper::new(state.shard.append_messages( + identity.user_id, + command_stream_id, + command_topic_id, + &partitioning, + batch, + )); + + append_future + .await + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to append messages, stream ID: {stream_id}, topic ID: {topic_id}" + ) + })?; + + Ok(StatusCode::CREATED) +} + +#[debug_handler] +#[instrument(skip_all, name = "trace_flush_unsaved_buffer", fields(iggy_user_id = identity.user_id, iggy_stream_id = stream_id, iggy_topic_id = topic_id, iggy_partition_id = partition_id, iggy_fsync = fsync))] +async fn flush_unsaved_buffer( + State(state): State>, + Extension(identity): Extension, + Path((stream_id, topic_id, partition_id, fsync)): Path<(String, String, u32, bool)>, +) -> Result { + let stream_id_ident = Identifier::from_str_value(&stream_id)?; + let topic_id_ident = Identifier::from_str_value(&topic_id)?; + let partition_id = partition_id as usize; + + let shard = state.shard.shard(); + let topic = shard.resolve_topic(&stream_id_ident, &topic_id_ident)?; + let partition = ResolvedPartition { + stream_id: topic.stream_id, + topic_id: topic.topic_id, + partition_id, + }; + + let flush_future = + SendWrapper::new(shard.flush_unsaved_buffer(identity.user_id, partition, fsync)); + flush_future.await?; + Ok(StatusCode::OK) +} + +fn make_mutable(batch: IggyMessagesBatch) -> IggyMessagesBatchMut { + let (_, indexes, messages) = batch.decompose(); + let (base_position, indexes_buffer) = indexes.decompose(); + + let mut indexes_buffer_mut = PooledBuffer::with_capacity(indexes_buffer.len()); + indexes_buffer_mut.extend_from_slice(&indexes_buffer); + let indexes_mut = IggyIndexesMut::from_bytes(indexes_buffer_mut, base_position); + + let mut messages_buffer_mut = PooledBuffer::with_capacity(messages.len()); + messages_buffer_mut.extend_from_slice(&messages); + + IggyMessagesBatchMut::from_indexes_and_messages(indexes_mut, messages_buffer_mut) +} diff --git a/core/server/src/http/metrics.rs b/core/server/src/http/metrics.rs index bb18a8a9b9..2866e5c09a 100644 --- a/core/server/src/http/metrics.rs +++ b/core/server/src/http/metrics.rs @@ -15,283 +15,21 @@ // specific language governing permissions and limitations // under the License. -//! The `[http.metrics]` scrape surface: the legacy-parity metric registry -//! (entity gauges plus the request counter), its public scrape handler, and -//! the config gate deciding whether the route is mounted. - -use axum::extract::State; -use configs::http::HttpMetricsConfig; -use consensus::MetadataHandle; -use iggy_common::IggyError; -use metadata::impls::metadata::StreamsFrontend; -use prometheus_client::encoding::text::encode; -use prometheus_client::metrics::counter::Counter; -use prometheus_client::metrics::gauge::Gauge; -use prometheus_client::registry::Registry; -use send_wrapper::SendWrapper; -use tracing::error; - -use crate::http::state::HttpState; - -/// The legacy server's metric set, registered under the same names and help -/// texts so existing dashboards and alerts keep working unchanged. -/// -/// Unlike the legacy server, the entity gauges are not counted at mutation -/// sites: [`get_metrics`] samples the live state on every scrape, so a gauge -/// can never drift from the state it describes. -pub(in crate::http) struct HttpMetrics { - registry: Registry, - http_requests: Counter, - streams: Gauge, - topics: Gauge, - partitions: Gauge, - segments: Gauge, - messages: Gauge, - users: Gauge, - clients: Gauge, -} - -impl HttpMetrics { - pub(in crate::http) fn init() -> Self { - let mut registry = Registry::default(); - let http_requests = Counter::default(); - let streams = Gauge::default(); - let topics = Gauge::default(); - let partitions = Gauge::default(); - let segments = Gauge::default(); - let messages = Gauge::default(); - let users = Gauge::default(); - let clients = Gauge::default(); - registry.register( - "http_requests", - "total count of http_requests", - http_requests.clone(), - ); - registry.register("streams", "total count of streams", streams.clone()); - registry.register("topics", "total count of topics", topics.clone()); - registry.register( - "partitions", - "total count of partitions", - partitions.clone(), - ); - registry.register("segments", "total count of segments", segments.clone()); - registry.register("messages", "total count of messages", messages.clone()); - registry.register("users", "total count of users", users.clone()); - registry.register("clients", "total count of clients", clients.clone()); - Self { - registry, - http_requests, - streams, - topics, - partitions, - segments, - messages, - users, - clients, - } - } - - /// Handle for the router's request-counting layer. The counter is - /// `Arc`-backed, so bumping the clone bumps the registered metric. - pub(in crate::http) fn request_counter(&self) -> Counter { - self.http_requests.clone() - } - - fn formatted_output(&self) -> String { - let mut buffer = String::new(); - if let Err(error) = encode(&mut buffer, &self.registry) { - error!(%error, "failed to encode metrics"); - } - buffer - } -} - -/// Resolve the configured scrape path: `None` when `[http.metrics]` is -/// disabled, so the route is never mounted and the endpoint answers 404. -/// -/// axum's `Router::route` panics on a path without a leading `/`, so an -/// enabled endpoint missing one is rejected as a configuration error before -/// the router is assembled. -/// -/// # Errors -/// -/// Returns [`IggyError::InvalidConfiguration`] when metrics are enabled and -/// the endpoint does not start with `/`. -pub(in crate::http) fn validated_endpoint( - config: &HttpMetricsConfig, -) -> Result, IggyError> { - if !config.enabled { - return Ok(None); - } - if !config.endpoint.starts_with('/') { - error!( - endpoint = %config.endpoint, - "invalid http.metrics.endpoint: the path must start with '/'" - ); - return Err(IggyError::InvalidConfiguration); - } - Ok(Some(config.endpoint.clone())) -} - -/// `GET `: the metric set in prometheus text -/// exposition. Public - reached without proving a credential, exactly like the -/// legacy endpoint. -/// -/// The entity gauges sample the same reads `/stats` serves: the metadata STM -/// stream and user maps plus the stats-registry rollups, whose partition-plane -/// increments are relaxed, so scraped values are approximate while writes are -/// in flight. The clients count scatter-gathers the per-shard session managers -/// exactly like `GET /clients` and turns partial when a shard misses the reply -/// deadline. -pub(in crate::http) async fn get_metrics(State(state): State) -> String { - let (streams_count, topics_count, partitions_count, segments_count, messages_count) = state - .shard - .plane - .metadata() - .mux_stm - .streams() - .read(|streams| { - let mut topics_count = 0u64; - let mut partitions_count = 0u64; - let mut segments_count = 0u64; - let mut messages_count = 0u64; - for (_, stream) in &streams.items { - topics_count = topics_count.saturating_add(stream.topics.len() as u64); - segments_count = segments_count - .saturating_add(u64::from(stream.stats.segments_count_inconsistent())); - messages_count = - messages_count.saturating_add(stream.stats.messages_count_inconsistent()); - for (_, topic) in &stream.topics { - partitions_count = - partitions_count.saturating_add(topic.partitions.len() as u64); - } - } - ( - streams.items.len() as u64, - topics_count, - partitions_count, - segments_count, - messages_count, - ) - }); - let users_count = state - .shard - .plane - .metadata() - .mux_stm - .users() - .read(|users| users.items.len() as u64); - let clients_count = SendWrapper::new(state.shard.list_all_clients()).await.len() as u64; - - let metrics = &state.metrics; - metrics.streams.set(gauge_value(streams_count)); - metrics.topics.set(gauge_value(topics_count)); - metrics.partitions.set(gauge_value(partitions_count)); - metrics.segments.set(gauge_value(segments_count)); - metrics.messages.set(gauge_value(messages_count)); - metrics.users.set(gauge_value(users_count)); - metrics.clients.set(gauge_value(clients_count)); - metrics.formatted_output() -} - -/// Clamp a count into the gauge's `i64` domain; only `messages` can pass -/// `i64::MAX` even in theory, the rest are bounded far below it. -fn gauge_value(count: u64) -> i64 { - i64::try_from(count).unwrap_or(i64::MAX) -} - -#[cfg(test)] -mod tests { - use super::*; - - const PARITY_METRIC_NAMES: [&str; 8] = [ - "http_requests", - "streams", - "topics", - "partitions", - "segments", - "messages", - "users", - "clients", - ]; - - fn metrics_config(enabled: bool, endpoint: &str) -> HttpMetricsConfig { - HttpMetricsConfig { - enabled, - endpoint: endpoint.to_owned(), - } - } - - #[test] - fn formatted_output_exposes_every_parity_metric() { - let metrics = HttpMetrics::init(); - let output = metrics.formatted_output(); - for name in PARITY_METRIC_NAMES { - assert!( - output.contains(&format!("# TYPE {name} ")), - "metric {name} missing from exposition:\n{output}" - ); - } - assert!( - output.ends_with("# EOF\n"), - "missing exposition trailer:\n{output}" - ); - } - - #[test] - fn scraped_values_land_in_the_exposition() { - let metrics = HttpMetrics::init(); - metrics.streams.set(1); - metrics.topics.set(2); - metrics.partitions.set(3); - metrics.segments.set(4); - metrics.messages.set(5); - metrics.users.set(6); - metrics.clients.set(7); - metrics.request_counter().inc(); - let output = metrics.formatted_output(); - for line in [ - "streams 1", - "topics 2", - "partitions 3", - "segments 4", - "messages 5", - "users 6", - "clients 7", - "http_requests_total 1", - ] { - assert!( - output.contains(&format!("\n{line}\n")), - "expected `{line}` in exposition:\n{output}" - ); - } - } - - #[test] - fn gauge_value_clamps_past_i64_range() { - assert_eq!(gauge_value(42), 42); - assert_eq!(gauge_value(u64::MAX), i64::MAX); - } - - #[test] - fn validated_endpoint_disabled_yields_none() { - assert!(matches!( - validated_endpoint(&metrics_config(false, "/metrics")), - Ok(None) - )); - } - - #[test] - fn validated_endpoint_returns_enabled_path() { - let endpoint = validated_endpoint(&metrics_config(true, "/metrics")).unwrap(); - assert_eq!(endpoint.as_deref(), Some("/metrics")); - } - - #[test] - fn validated_endpoint_rejects_missing_leading_slash() { - assert!(matches!( - validated_endpoint(&metrics_config(true, "metrics")), - Err(IggyError::InvalidConfiguration) - )); - } +use crate::http::shared::AppState; +use axum::body::Body; +use axum::{ + extract::State, + http::{Request, StatusCode}, + middleware::Next, + response::Response, +}; +use std::sync::Arc; + +pub async fn metrics( + State(state): State>, + request: Request, + next: Next, +) -> Result { + state.shard.shard().metrics.increment_http_requests(); + Ok(next.run(request).await) } diff --git a/core/server/src/http/mod.rs b/core/server/src/http/mod.rs new file mode 100644 index 0000000000..ad3186a412 --- /dev/null +++ b/core/server/src/http/mod.rs @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod diagnostics; +pub mod error; +pub mod http_server; +mod http_shard_wrapper; +pub mod jwt; +mod mapper; +pub mod metrics; +pub mod shared; + +pub mod consumer_groups; +pub mod consumer_offsets; +pub mod messages; +pub mod partitions; +pub mod personal_access_tokens; +pub mod segments; +pub mod streams; +pub mod system; +pub mod topics; +pub mod users; +#[cfg(feature = "iggy-web")] +pub mod web; + +pub const COMPONENT: &str = "HTTP"; diff --git a/core/server/src/http/partitions.rs b/core/server/src/http/partitions.rs new file mode 100644 index 0000000000..46dd3853f8 --- /dev/null +++ b/core/server/src/http/partitions.rs @@ -0,0 +1,104 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::http::error::CustomError; +use crate::http::jwt::json_web_token::Identity; +use crate::http::shared::AppState; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; +use axum::extract::{Path, Query, State}; +use axum::http::StatusCode; +use axum::routing::post; +use axum::{Extension, Json, Router, debug_handler}; +use iggy_binary_protocol::requests::partitions::{ + CreatePartitionsRequest as WireCreatePartitions, + DeletePartitionsRequest as WireDeletePartitions, +}; +use iggy_common::Identifier; +use iggy_common::Validatable; +use iggy_common::create_partitions::CreatePartitions; +use iggy_common::delete_partitions::DeletePartitions; +use iggy_common::wire_conversions::identifier_to_wire; +use std::sync::Arc; +use tracing::instrument; + +pub fn router(state: Arc) -> Router { + Router::new() + .route( + "/streams/{stream_id}/topics/{topic_id}/partitions", + post(create_partitions).delete(delete_partitions), + ) + .with_state(state) +} + +#[debug_handler] +#[instrument(skip_all, name = "trace_create_partitions", fields(iggy_user_id = identity.user_id, iggy_stream_id = stream_id, iggy_topic_id = topic_id))] +async fn create_partitions( + State(state): State>, + Extension(identity): Extension, + Path((stream_id, topic_id)): Path<(String, String)>, + Json(mut command): Json, +) -> Result { + command.stream_id = Identifier::from_str_value(&stream_id)?; + command.topic_id = Identifier::from_str_value(&topic_id)?; + command.validate()?; + + let wire_command = WireCreatePartitions { + stream_id: identifier_to_wire(&command.stream_id)?, + topic_id: identifier_to_wire(&command.topic_id)?, + partitions_count: command.partitions_count, + }; + let request = ShardRequest::control_plane(ShardRequestPayload::CreatePartitionsRequest { + user_id: identity.user_id, + command: wire_command, + }); + + match state.shard.send_to_control_plane(request).await? { + ShardResponse::CreatePartitionsResponse => Ok(StatusCode::CREATED), + ShardResponse::ErrorResponse(err) => Err(err.into()), + _ => unreachable!("Expected CreatePartitionsResponse"), + } +} + +#[debug_handler] +#[instrument(skip_all, name = "trace_delete_partitions", fields(iggy_user_id = identity.user_id, iggy_stream_id = stream_id, iggy_topic_id = topic_id))] +async fn delete_partitions( + State(state): State>, + Extension(identity): Extension, + Path((stream_id, topic_id)): Path<(String, String)>, + mut query: Query, +) -> Result { + query.stream_id = Identifier::from_str_value(&stream_id)?; + query.topic_id = Identifier::from_str_value(&topic_id)?; + query.validate()?; + + let wire_command = WireDeletePartitions { + stream_id: identifier_to_wire(&query.stream_id)?, + topic_id: identifier_to_wire(&query.topic_id)?, + partitions_count: query.partitions_count, + }; + let request = ShardRequest::control_plane(ShardRequestPayload::DeletePartitionsRequest { + user_id: identity.user_id, + command: wire_command, + }); + + match state.shard.send_to_control_plane(request).await? { + ShardResponse::DeletePartitionsResponse => Ok(StatusCode::NO_CONTENT), + ShardResponse::ErrorResponse(err) => Err(err.into()), + _ => unreachable!("Expected DeletePartitionsResponse"), + } +} diff --git a/core/server/src/http/personal_access_tokens.rs b/core/server/src/http/personal_access_tokens.rs new file mode 100644 index 0000000000..bb8d2a8d41 --- /dev/null +++ b/core/server/src/http/personal_access_tokens.rs @@ -0,0 +1,148 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::http::COMPONENT; +use crate::http::error::CustomError; +use crate::http::jwt::json_web_token::Identity; +use crate::http::mapper; +use crate::http::mapper::map_generated_access_token_to_identity_info; +use crate::http::shared::AppState; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; +use axum::extract::{Path, State}; +use axum::http::StatusCode; +use axum::routing::{delete, get, post}; +use axum::{Extension, Json, Router, debug_handler}; +use err_trail::ErrContext; +use iggy_binary_protocol::WireName; +use iggy_binary_protocol::requests::personal_access_tokens::{ + CreatePersonalAccessTokenRequest as WireCreatePat, + DeletePersonalAccessTokenRequest as WireDeletePat, +}; +use iggy_common::IdentityInfo; +use iggy_common::PersonalAccessTokenInfo; +use iggy_common::Validatable; +use iggy_common::create_personal_access_token::CreatePersonalAccessToken; +use iggy_common::login_with_personal_access_token::LoginWithPersonalAccessToken; +use iggy_common::{IggyError, RawPersonalAccessToken}; +use secrecy::ExposeSecret; +use std::sync::Arc; +use tracing::instrument; + +pub fn router(state: Arc) -> Router { + Router::new() + .route( + "/personal-access-tokens", + get(get_personal_access_tokens).post(create_personal_access_token), + ) + .route( + "/personal-access-tokens/{name}", + delete(delete_personal_access_token), + ) + .route( + "/personal-access-tokens/login", + post(login_with_personal_access_token), + ) + .with_state(state) +} + +#[debug_handler] +async fn get_personal_access_tokens( + State(state): State>, + Extension(identity): Extension, +) -> Result>, CustomError> { + let personal_access_tokens = state + .shard + .shard() + .get_personal_access_tokens(identity.user_id) + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to get personal access tokens, user ID: {}", + identity.user_id + ) + })?; + let personal_access_tokens = mapper::map_personal_access_tokens(&personal_access_tokens); + Ok(Json(personal_access_tokens)) +} + +#[debug_handler] +#[instrument(skip_all, name = "trace_create_personal_access_token", fields(iggy_user_id = identity.user_id))] +async fn create_personal_access_token( + State(state): State>, + Extension(identity): Extension, + Json(command): Json, +) -> Result, CustomError> { + command.validate()?; + + let wire_command = WireCreatePat { + name: WireName::new(&command.name) + .map_err(|_| IggyError::InvalidPersonalAccessTokenName)?, + expiry: command.expiry.into(), + }; + let request = + ShardRequest::control_plane(ShardRequestPayload::CreatePersonalAccessTokenRequest { + user_id: identity.user_id, + command: wire_command, + }); + + match state.shard.send_to_control_plane(request).await? { + ShardResponse::CreatePersonalAccessTokenResponse(_, token) => { + Ok(Json(RawPersonalAccessToken { token })) + } + ShardResponse::ErrorResponse(err) => Err(err.into()), + _ => unreachable!("Expected CreatePersonalAccessTokenResponse"), + } +} + +#[debug_handler] +#[instrument(skip_all, name = "trace_delete_personal_access_token", fields(iggy_user_id = identity.user_id))] +async fn delete_personal_access_token( + State(state): State>, + Extension(identity): Extension, + Path(name): Path, +) -> Result { + let wire_command = WireDeletePat { + name: WireName::new(&name).map_err(|_| IggyError::InvalidPersonalAccessTokenName)?, + }; + let request = + ShardRequest::control_plane(ShardRequestPayload::DeletePersonalAccessTokenRequest { + user_id: identity.user_id, + command: wire_command, + }); + + match state.shard.send_to_control_plane(request).await? { + ShardResponse::DeletePersonalAccessTokenResponse => Ok(StatusCode::NO_CONTENT), + ShardResponse::ErrorResponse(err) => Err(err.into()), + _ => unreachable!("Expected DeletePersonalAccessTokenResponse"), + } +} + +#[instrument(skip_all, name = "trace_login_with_personal_access_token")] +async fn login_with_personal_access_token( + State(state): State>, + Json(command): Json, +) -> Result, CustomError> { + let user = state + .shard + .shard() + .login_with_personal_access_token(command.token.expose_secret(), None) + .error(|e: &IggyError| { + format!("{COMPONENT} (error: {e}) - failed to login with personal access token") + })?; + let tokens = state.jwt_manager.generate(user.id)?; + Ok(Json(map_generated_access_token_to_identity_info(tokens))) +} diff --git a/core/server/src/http/segments.rs b/core/server/src/http/segments.rs new file mode 100644 index 0000000000..e83be5de26 --- /dev/null +++ b/core/server/src/http/segments.rs @@ -0,0 +1,120 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::http::COMPONENT; +use crate::http::error::CustomError; +use crate::http::jwt::json_web_token::Identity; +use crate::http::shared::AppState; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; +use axum::extract::{Path, Query, State}; +use axum::http::StatusCode; +use axum::routing::delete; +use axum::{Extension, Router, debug_handler}; +use err_trail::ErrContext; +use iggy_binary_protocol::requests::segments::DeleteSegmentsRequest; +use iggy_common::Identifier; +use iggy_common::Validatable; +use iggy_common::delete_segments::DeleteSegments; +use iggy_common::wire_conversions::identifier_to_wire; +use send_wrapper::SendWrapper; +use server_common::sharding::IggyNamespace; +use std::sync::Arc; +use tracing::instrument; + +pub fn router(state: Arc) -> Router { + Router::new() + .route( + "/streams/{stream_id}/topics/{topic_id}/partitions/{partition_id}", + delete(delete_segments), + ) + .with_state(state) +} + +#[debug_handler] +#[instrument(skip_all, name = "trace_delete_segments", fields(iggy_user_id = identity.user_id, iggy_stream_id = stream_id, iggy_topic_id = topic_id))] +async fn delete_segments( + State(state): State>, + Extension(identity): Extension, + Path((stream_id, topic_id, partition_id)): Path<(String, String, u32)>, + mut query: Query, +) -> Result { + query.stream_id = Identifier::from_str_value(&stream_id)?; + query.topic_id = Identifier::from_str_value(&topic_id)?; + query.partition_id = partition_id; + query.validate()?; + let segments_count = query.segments_count; + + let partition = state.shard.shard().resolve_partition_for_delete_segments( + identity.user_id, + &query.stream_id, + &query.topic_id, + partition_id as usize, + )?; + + let namespace = IggyNamespace::new( + partition.stream_id, + partition.topic_id, + partition.partition_id, + ); + let request = ShardRequest::data_plane( + namespace, + ShardRequestPayload::DeleteSegments { segments_count }, + ); + + let delete_future = SendWrapper::new(state.shard.shard().send_to_data_plane(request)); + match delete_future.await? { + ShardResponse::DeleteSegments { + deleted_segments, + deleted_messages, + } => { + state + .shard + .shard() + .metrics + .decrement_segments(deleted_segments as u32); + state + .shard + .shard() + .metrics + .decrement_messages(deleted_messages); + } + ShardResponse::ErrorResponse(err) => return Err(err.into()), + _ => unreachable!("Expected DeleteSegments"), + } + + let wire_command = DeleteSegmentsRequest { + stream_id: identifier_to_wire(&query.stream_id)?, + topic_id: identifier_to_wire(&query.topic_id)?, + partition_id: query.partition_id, + segments_count, + }; + let entry_command = crate::state::command::EntryCommand::DeleteSegments(wire_command); + let state_future = SendWrapper::new( + state + .shard + .shard() + .state + .apply(identity.user_id, &entry_command), + ); + state_future.await.error(|e: &iggy_common::IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to apply delete segments, stream ID: {stream_id}, topic ID: {topic_id}" + ) + })?; + Ok(StatusCode::NO_CONTENT) +} diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/RedirectionClusterFixture.cs b/core/server/src/http/shared.rs similarity index 65% rename from foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/RedirectionClusterFixture.cs rename to core/server/src/http/shared.rs index ddcc2dcf6a..faab1b8cb6 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/RedirectionClusterFixture.cs +++ b/core/server/src/http/shared.rs @@ -15,15 +15,19 @@ // specific language governing permissions and limitations // under the License. -namespace Apache.Iggy.Tests.Integrations.Fixtures; +use super::http_shard_wrapper::HttpSafeShard; +use crate::http::jwt::jwt_manager::JwtManager; +use std::net::SocketAddr; +use ulid::Ulid; -/// -/// A dedicated three-node cluster for the redirection tests, pinned regardless of the -/// IGGY_TEST_CLUSTER_NODES knob: dialing a follower needs real replicas to redirect between. -/// -public class RedirectionClusterFixture : IggyServerFixture -{ - protected override string ClusterName => "redirection"; +pub struct AppState { + pub jwt_manager: JwtManager, + pub shard: HttpSafeShard, +} - protected override int NodeCount => 3; +#[derive(Debug, Copy, Clone)] +pub struct RequestDetails { + #[allow(dead_code)] + pub request_id: Ulid, + pub ip_address: SocketAddr, } diff --git a/core/server/src/http/streams.rs b/core/server/src/http/streams.rs new file mode 100644 index 0000000000..742575144c --- /dev/null +++ b/core/server/src/http/streams.rs @@ -0,0 +1,200 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::http::error::CustomError; +use crate::http::jwt::json_web_token::Identity; +use crate::http::shared::AppState; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; +use axum::extract::{Path, State}; +use axum::http::StatusCode; +use axum::routing::{delete, get}; +use axum::{Extension, Json, Router, debug_handler}; +use iggy_binary_protocol::WireName; +use iggy_binary_protocol::requests::streams::{ + CreateStreamRequest as WireCreateStream, DeleteStreamRequest as WireDeleteStream, + PurgeStreamRequest as WirePurgeStream, UpdateStreamRequest as WireUpdateStream, +}; +use iggy_common::Identifier; +use iggy_common::Validatable; +use iggy_common::create_stream::CreateStream; +use iggy_common::update_stream::UpdateStream; +use iggy_common::wire_conversions::identifier_to_wire; +use iggy_common::{IggyError, Stream, StreamDetails}; +use std::sync::Arc; +use tracing::instrument; + +pub fn router(state: Arc) -> Router { + Router::new() + .route("/streams", get(get_streams).post(create_stream)) + .route( + "/streams/{stream_id}", + get(get_stream).put(update_stream).delete(delete_stream), + ) + .route("/streams/{stream_id}/purge", delete(purge_stream)) + .with_state(state) +} + +#[debug_handler] +async fn get_stream( + State(state): State>, + Extension(identity): Extension, + Path(stream_id): Path, +) -> Result, CustomError> { + let stream_id = Identifier::from_str_value(&stream_id)?; + + let shard = state.shard.shard(); + let numeric_stream_id = shard + .metadata + .get_stream_id(&stream_id) + .ok_or(CustomError::ResourceNotFound)?; + + shard + .metadata + .perm_get_stream(identity.user_id, numeric_stream_id)?; + + let stream_meta = shard + .metadata + .get_stream(numeric_stream_id) + .ok_or(CustomError::ResourceNotFound)?; + + let stream_details = crate::http::mapper::map_stream_details_from_metadata(&stream_meta); + + Ok(Json(stream_details)) +} + +#[debug_handler] +async fn get_streams( + State(state): State>, + Extension(identity): Extension, +) -> Result>, CustomError> { + let shard = state.shard.shard(); + + shard.metadata.perm_get_streams(identity.user_id)?; + + let streams = shard + .metadata + .with_metadata(crate::http::mapper::map_streams_from_metadata); + + Ok(Json(streams)) +} + +#[debug_handler] +#[instrument(skip_all, name = "trace_create_stream", fields(iggy_user_id = identity.user_id))] +async fn create_stream( + State(state): State>, + Extension(identity): Extension, + Json(command): Json, +) -> Result, CustomError> { + command.validate()?; + + let wire_command = WireCreateStream { + name: WireName::new(&command.name).map_err(|_| IggyError::InvalidStreamName)?, + }; + let request = ShardRequest::control_plane(ShardRequestPayload::CreateStreamRequest { + user_id: identity.user_id, + command: wire_command, + }); + + match state.shard.send_to_control_plane(request).await? { + ShardResponse::CreateStreamResponse(data) => { + let stream_meta = state + .shard + .shard() + .metadata + .get_stream(data.id as usize) + .expect("Stream must exist after creation"); + let response = crate::http::mapper::map_stream_details_from_metadata(&stream_meta); + Ok(Json(response)) + } + ShardResponse::ErrorResponse(err) => Err(err.into()), + _ => unreachable!("Expected CreateStreamResponse"), + } +} + +#[debug_handler] +#[instrument(skip_all, name = "trace_update_stream", fields(iggy_user_id = identity.user_id, iggy_stream_id = stream_id))] +async fn update_stream( + State(state): State>, + Extension(identity): Extension, + Path(stream_id): Path, + Json(mut command): Json, +) -> Result { + command.stream_id = Identifier::from_str_value(&stream_id)?; + command.validate()?; + + let wire_command = WireUpdateStream { + stream_id: identifier_to_wire(&command.stream_id)?, + name: WireName::new(&command.name).map_err(|_| IggyError::InvalidStreamName)?, + }; + let request = ShardRequest::control_plane(ShardRequestPayload::UpdateStreamRequest { + user_id: identity.user_id, + command: wire_command, + }); + + match state.shard.send_to_control_plane(request).await? { + ShardResponse::UpdateStreamResponse => Ok(StatusCode::NO_CONTENT), + ShardResponse::ErrorResponse(err) => Err(err.into()), + _ => unreachable!("Expected UpdateStreamResponse"), + } +} + +#[debug_handler] +#[instrument(skip_all, name = "trace_delete_stream", fields(iggy_user_id = identity.user_id, iggy_stream_id = stream_id))] +async fn delete_stream( + State(state): State>, + Extension(identity): Extension, + Path(stream_id): Path, +) -> Result { + let stream_id = Identifier::from_str_value(&stream_id)?; + + let request = ShardRequest::control_plane(ShardRequestPayload::DeleteStreamRequest { + user_id: identity.user_id, + command: WireDeleteStream { + stream_id: identifier_to_wire(&stream_id)?, + }, + }); + + match state.shard.send_to_control_plane(request).await? { + ShardResponse::DeleteStreamResponse => Ok(StatusCode::NO_CONTENT), + ShardResponse::ErrorResponse(err) => Err(err.into()), + _ => unreachable!("Expected DeleteStreamResponse"), + } +} + +#[debug_handler] +#[instrument(skip_all, name = "trace_purge_stream", fields(iggy_user_id = identity.user_id, iggy_stream_id = stream_id))] +async fn purge_stream( + State(state): State>, + Extension(identity): Extension, + Path(stream_id): Path, +) -> Result { + let stream_id = Identifier::from_str_value(&stream_id)?; + + let request = ShardRequest::control_plane(ShardRequestPayload::PurgeStreamRequest { + user_id: identity.user_id, + command: WirePurgeStream { + stream_id: identifier_to_wire(&stream_id)?, + }, + }); + + match state.shard.send_to_control_plane(request).await? { + ShardResponse::PurgeStreamResponse => Ok(StatusCode::NO_CONTENT), + ShardResponse::ErrorResponse(err) => Err(err.into()), + _ => unreachable!("Expected PurgeStreamResponse"), + } +} diff --git a/core/server/src/http/system.rs b/core/server/src/http/system.rs new file mode 100644 index 0000000000..7d8e26b941 --- /dev/null +++ b/core/server/src/http/system.rs @@ -0,0 +1,168 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::configs::http::HttpMetricsConfig; +use crate::http::COMPONENT; +use crate::http::error::CustomError; +use crate::http::jwt::json_web_token::Identity; +use crate::http::mapper; +use crate::http::shared::AppState; +use axum::body::Body; +use axum::extract::{Path, State}; +use axum::http::{HeaderMap, header}; +use axum::response::IntoResponse; +use axum::routing::{get, post}; +use axum::{Extension, Json, Router, debug_handler}; +use bytes::Bytes; +use chrono::Local; +use err_trail::ErrContext; +use iggy_common::Stats; +use iggy_common::get_snapshot::GetSnapshot; +use iggy_common::{ClientInfo, ClientInfoDetails, ClusterMetadata, IggyError, SystemSnapshotType}; +use send_wrapper::SendWrapper; +use std::sync::Arc; +use tracing::error; + +const NAME: &str = "Iggy API"; +const PONG: &str = "pong"; + +pub fn router(state: Arc, metrics_config: &HttpMetricsConfig) -> Router { + let mut router = Router::new() + .route("/", get(|| async { NAME })) + .route("/ping", get(|| async { PONG })) + .route("/stats", get(get_stats)) + .route("/cluster/metadata", get(get_cluster_metadata)) + .route("/clients", get(get_clients)) + .route("/clients/{client_id}", get(get_client)) + .route("/snapshot", post(get_snapshot)); + if metrics_config.enabled { + router = router.route(&metrics_config.endpoint, get(get_metrics)); + } + + router.with_state(state) +} + +#[debug_handler] +async fn get_metrics(State(state): State>) -> Result { + let metrics_formatted_output = state.shard.shard().metrics.get_formatted_output(); + Ok(metrics_formatted_output) +} + +#[debug_handler] +async fn get_stats( + State(state): State>, + Extension(identity): Extension, +) -> Result, CustomError> { + state + .shard + .shard() + .metadata + .perm_get_stats(identity.user_id)?; + let stats_future = SendWrapper::new(state.shard.shard().get_stats()); + let stats = stats_future.await.error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to get stats, user ID: {}", + identity.user_id + ) + })?; + Ok(Json(stats)) +} + +async fn get_cluster_metadata( + State(state): State>, + Extension(identity): Extension, +) -> Result, CustomError> { + let _ = identity; // authenticated via middleware + let cluster_metadata = state.shard.shard().get_cluster_metadata(); + Ok(Json(cluster_metadata)) +} + +async fn get_client( + State(state): State>, + Extension(identity): Extension, + Path(client_id): Path, +) -> Result, CustomError> { + state + .shard + .shard() + .metadata + .perm_get_client(identity.user_id)?; + let Some(client) = state.shard.shard().get_client(client_id) else { + return Err(CustomError::ResourceNotFound); + }; + + let client = mapper::map_client(&client); + Ok(Json(client)) +} + +#[debug_handler] +async fn get_clients( + State(state): State>, + Extension(identity): Extension, +) -> Result>, CustomError> { + state + .shard + .shard() + .metadata + .perm_get_clients(identity.user_id)?; + let clients = state.shard.shard().get_clients(); + let clients = mapper::map_clients(&clients); + Ok(Json(clients)) +} + +#[debug_handler] +async fn get_snapshot( + State(state): State>, + Extension(identity): Extension, + Json(command): Json, +) -> Result { + state + .shard + .shard() + .metadata + .perm_get_snapshot(identity.user_id)?; + if command.snapshot_types.contains(&SystemSnapshotType::All) && command.snapshot_types.len() > 1 + { + error!("When using 'All' snapshot type, no other types can be specified"); + return Err(IggyError::InvalidCommand.into()); + } + + let snapshot_future = SendWrapper::new( + state + .shard + .shard() + .get_snapshot(command.compression, &command.snapshot_types), + ); + + let snapshot = snapshot_future + .await + .error(|e: &IggyError| format!("{COMPONENT} (error: {e}) - failed to get snapshot"))?; + + let zip_data = Bytes::from(snapshot.0); + let filename = format!("iggy_snapshot_{}.zip", Local::now().format("%Y%m%d_%H%M%S")); + + let mut headers = HeaderMap::new(); + headers.insert( + header::CONTENT_TYPE, + header::HeaderValue::from_static("application/zip"), + ); + headers.insert( + header::CONTENT_DISPOSITION, + header::HeaderValue::from_str(&format!("attachment; filename=\"{filename}\"")).unwrap(), + ); + Ok((headers, Body::from(zip_data))) +} diff --git a/core/server/src/http/topics.rs b/core/server/src/http/topics.rs new file mode 100644 index 0000000000..c1b12d5a94 --- /dev/null +++ b/core/server/src/http/topics.rs @@ -0,0 +1,263 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::http::COMPONENT; +use crate::http::error::CustomError; +use crate::http::jwt::json_web_token::Identity; +use crate::http::shared::AppState; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; +use axum::extract::{Path, State}; +use axum::http::StatusCode; +use axum::routing::{delete, get}; +use axum::{Extension, Json, Router, debug_handler}; +use err_trail::ErrContext; +use iggy_binary_protocol::WireName; +use iggy_binary_protocol::requests::topics::{ + CreateTopicRequest as WireCreateTopic, DeleteTopicRequest as WireDeleteTopic, + PurgeTopicRequest as WirePurgeTopic, UpdateTopicRequest as WireUpdateTopic, +}; +use iggy_common::Identifier; +use iggy_common::Validatable; +use iggy_common::create_topic::CreateTopic; +use iggy_common::update_topic::UpdateTopic; +use iggy_common::wire_conversions::identifier_to_wire; +use iggy_common::{IggyError, Topic, TopicDetails}; +use std::sync::Arc; +use tracing::instrument; + +pub fn router(state: Arc) -> Router { + Router::new() + .route( + "/streams/{stream_id}/topics", + get(get_topics).post(create_topic), + ) + .route( + "/streams/{stream_id}/topics/{topic_id}", + get(get_topic).put(update_topic).delete(delete_topic), + ) + .route( + "/streams/{stream_id}/topics/{topic_id}/purge", + delete(purge_topic), + ) + .with_state(state) +} + +#[debug_handler] +async fn get_topic( + State(state): State>, + Extension(identity): Extension, + Path((stream_id, topic_id)): Path<(String, String)>, +) -> Result, CustomError> { + let identity_stream_id = Identifier::from_str_value(&stream_id)?; + let identity_topic_id = Identifier::from_str_value(&topic_id)?; + + let shard = state.shard.shard(); + + let numeric_stream_id = shard + .metadata + .get_stream_id(&identity_stream_id) + .ok_or(CustomError::ResourceNotFound)?; + + let numeric_topic_id = shard + .metadata + .get_topic_id(numeric_stream_id, &identity_topic_id) + .ok_or(CustomError::ResourceNotFound)?; + + shard + .metadata + .perm_get_topic(identity.user_id, numeric_stream_id, numeric_topic_id) + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - permission denied to get topic with ID: {topic_id} in stream with ID: {stream_id} for user with ID: {}", + identity.user_id, + ) + })?; + + let topic_meta = shard + .metadata + .get_topic(numeric_stream_id, numeric_topic_id) + .ok_or(CustomError::ResourceNotFound)?; + + let topic_details = crate::http::mapper::map_topic_details_from_metadata(&topic_meta); + + Ok(Json(topic_details)) +} + +#[debug_handler] +async fn get_topics( + State(state): State>, + Extension(identity): Extension, + Path(stream_id): Path, +) -> Result>, CustomError> { + let stream_id_ident = Identifier::from_str_value(&stream_id)?; + let shard = state.shard.shard(); + + let numeric_stream_id = shard + .metadata + .get_stream_id(&stream_id_ident) + .ok_or(CustomError::ResourceNotFound)?; + + shard + .metadata + .perm_get_topics(identity.user_id, numeric_stream_id) + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - permission denied to get topics in stream with ID: {stream_id} for user with ID: {}", + identity.user_id, + ) + })?; + + let stream_meta = shard + .metadata + .get_stream(numeric_stream_id) + .ok_or(CustomError::ResourceNotFound)?; + let topics = crate::http::mapper::map_topics_from_metadata(&stream_meta); + + Ok(Json(topics)) +} + +#[debug_handler] +#[instrument(skip_all, name = "trace_create_topic", fields(iggy_user_id = identity.user_id, iggy_stream_id = stream_id))] +async fn create_topic( + State(state): State>, + Extension(identity): Extension, + Path(stream_id): Path, + Json(mut command): Json, +) -> Result, CustomError> { + command.stream_id = Identifier::from_str_value(&stream_id)?; + command.validate()?; + + let numeric_stream_id = state + .shard + .shard() + .metadata + .get_stream_id(&command.stream_id) + .ok_or(CustomError::ResourceNotFound)?; + + let wire_command = WireCreateTopic { + stream_id: identifier_to_wire(&command.stream_id)?, + partitions_count: command.partitions_count, + compression_algorithm: command.compression_algorithm.as_code(), + message_expiry: command.message_expiry.into(), + max_topic_size: command.max_topic_size.into(), + replication_factor: command.replication_factor.unwrap_or(0), + name: WireName::new(&command.name).map_err(|_| IggyError::InvalidTopicName)?, + }; + let request = ShardRequest::control_plane(ShardRequestPayload::CreateTopicRequest { + user_id: identity.user_id, + command: wire_command, + }); + + match state.shard.send_to_control_plane(request).await? { + ShardResponse::CreateTopicResponse(data) => { + let topic_meta = state + .shard + .shard() + .metadata + .get_topic(numeric_stream_id, data.id as usize) + .expect("Topic must exist after creation"); + let response = crate::http::mapper::map_topic_details_from_metadata(&topic_meta); + Ok(Json(response)) + } + ShardResponse::ErrorResponse(err) => Err(err.into()), + _ => unreachable!("Expected CreateTopicResponse"), + } +} + +#[debug_handler] +#[instrument(skip_all, name = "trace_update_topic", fields(iggy_user_id = identity.user_id, iggy_stream_id = stream_id, iggy_topic_id = topic_id))] +async fn update_topic( + State(state): State>, + Extension(identity): Extension, + Path((stream_id, topic_id)): Path<(String, String)>, + Json(mut command): Json, +) -> Result { + command.stream_id = Identifier::from_str_value(&stream_id)?; + command.topic_id = Identifier::from_str_value(&topic_id)?; + command.validate()?; + + let wire_command = WireUpdateTopic { + stream_id: identifier_to_wire(&command.stream_id)?, + topic_id: identifier_to_wire(&command.topic_id)?, + compression_algorithm: command.compression_algorithm.as_code(), + message_expiry: command.message_expiry.into(), + max_topic_size: command.max_topic_size.into(), + replication_factor: command.replication_factor.unwrap_or(0), + name: WireName::new(&command.name).map_err(|_| IggyError::InvalidTopicName)?, + }; + let request = ShardRequest::control_plane(ShardRequestPayload::UpdateTopicRequest { + user_id: identity.user_id, + command: wire_command, + }); + + match state.shard.send_to_control_plane(request).await? { + ShardResponse::UpdateTopicResponse => Ok(StatusCode::NO_CONTENT), + ShardResponse::ErrorResponse(err) => Err(err.into()), + _ => unreachable!("Expected UpdateTopicResponse"), + } +} + +#[debug_handler] +#[instrument(skip_all, name = "trace_delete_topic", fields(iggy_user_id = identity.user_id, iggy_stream_id = stream_id, iggy_topic_id = topic_id))] +async fn delete_topic( + State(state): State>, + Extension(identity): Extension, + Path((stream_id, topic_id)): Path<(String, String)>, +) -> Result { + let stream_id = Identifier::from_str_value(&stream_id)?; + let topic_id = Identifier::from_str_value(&topic_id)?; + + let request = ShardRequest::control_plane(ShardRequestPayload::DeleteTopicRequest { + user_id: identity.user_id, + command: WireDeleteTopic { + stream_id: identifier_to_wire(&stream_id)?, + topic_id: identifier_to_wire(&topic_id)?, + }, + }); + + match state.shard.send_to_control_plane(request).await? { + ShardResponse::DeleteTopicResponse => Ok(StatusCode::NO_CONTENT), + ShardResponse::ErrorResponse(err) => Err(err.into()), + _ => unreachable!("Expected DeleteTopicResponse"), + } +} + +#[debug_handler] +#[instrument(skip_all, name = "trace_purge_topic", fields(iggy_user_id = identity.user_id, iggy_stream_id = stream_id, iggy_topic_id = topic_id))] +async fn purge_topic( + State(state): State>, + Extension(identity): Extension, + Path((stream_id, topic_id)): Path<(String, String)>, +) -> Result { + let stream_id = Identifier::from_str_value(&stream_id)?; + let topic_id = Identifier::from_str_value(&topic_id)?; + + let request = ShardRequest::control_plane(ShardRequestPayload::PurgeTopicRequest { + user_id: identity.user_id, + command: WirePurgeTopic { + stream_id: identifier_to_wire(&stream_id)?, + topic_id: identifier_to_wire(&topic_id)?, + }, + }); + + match state.shard.send_to_control_plane(request).await? { + ShardResponse::PurgeTopicResponse => Ok(StatusCode::NO_CONTENT), + ShardResponse::ErrorResponse(err) => Err(err.into()), + _ => unreachable!("Expected PurgeTopicResponse"), + } +} diff --git a/core/server/src/http/users.rs b/core/server/src/http/users.rs new file mode 100644 index 0000000000..1fc6da7567 --- /dev/null +++ b/core/server/src/http/users.rs @@ -0,0 +1,331 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::http::COMPONENT; +use crate::http::error::CustomError; +use crate::http::jwt::json_web_token::Identity; +use crate::http::mapper; +use crate::http::mapper::map_generated_access_token_to_identity_info; +use crate::http::shared::AppState; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; +use crate::streaming::session::Session; +use crate::streaming::users::user::User; +use ::iggy_common::change_password::ChangePassword; +use ::iggy_common::create_user::CreateUser; +use ::iggy_common::update_permissions::UpdatePermissions; +use ::iggy_common::update_user::UpdateUser; +use axum::extract::{Path, State}; +use axum::http::StatusCode; +use axum::routing::{delete, get, post, put}; +use axum::{Extension, Json, Router, debug_handler}; +use err_trail::ErrContext; +use iggy_binary_protocol::WireName; +use iggy_binary_protocol::requests::users::{ + ChangePasswordRequest as WireChangePassword, CreateUserRequest as WireCreateUser, + DeleteUserRequest as WireDeleteUser, UpdatePermissionsRequest as WireUpdatePermissions, + UpdateUserRequest as WireUpdateUser, +}; +use iggy_common::Identifier; +use iggy_common::IdentityInfo; +use iggy_common::Validatable; +use iggy_common::login_user::LoginUser; +use iggy_common::wire_conversions::{identifier_to_wire, permissions_to_wire}; +use iggy_common::{IggyError, UserInfo, UserInfoDetails}; +use secrecy::ExposeSecret; +use send_wrapper::SendWrapper; +use serde::Deserialize; +use std::sync::Arc; +use tracing::instrument; + +pub fn router(state: Arc) -> Router { + Router::new() + .route("/users", get(get_users).post(create_user)) + .route( + "/users/{user_id}", + get(get_user).put(update_user).delete(delete_user), + ) + .route("/users/{user_id}/permissions", put(update_permissions)) + .route("/users/{user_id}/password", put(change_password)) + .route("/users/login", post(login_user)) + .route("/users/logout", delete(logout_user)) + .route("/users/refresh-token", post(refresh_token)) + .with_state(state) +} + +#[debug_handler] +async fn get_user( + State(state): State>, + Extension(identity): Extension, + Path(user_id): Path, +) -> Result, CustomError> { + let identifier_user_id = Identifier::from_str_value(&user_id)?; + let Ok(user) = state.shard.shard().find_user(&identifier_user_id) else { + return Err(CustomError::ResourceNotFound); + }; + let Some(user) = user else { + return Err(CustomError::ResourceNotFound); + }; + + if user.id != identity.user_id { + state + .shard + .shard() + .metadata + .perm_get_user(identity.user_id)?; + } + + let user = mapper::map_user(&user); + Ok(Json(user)) +} + +#[debug_handler] +async fn get_users( + State(state): State>, + Extension(identity): Extension, +) -> Result>, CustomError> { + state + .shard + .shard() + .metadata + .perm_get_users(identity.user_id)?; + + let users = state.shard.shard().get_users(); + let user_refs: Vec<&User> = users.iter().collect(); + let users = mapper::map_users(&user_refs); + Ok(Json(users)) +} + +#[debug_handler] +#[instrument(skip_all, name = "trace_create_user", fields(iggy_user_id = identity.user_id))] +async fn create_user( + State(state): State>, + Extension(identity): Extension, + Json(command): Json, +) -> Result, CustomError> { + command.validate()?; + + let wire_command = WireCreateUser { + username: WireName::new(&command.username).map_err(|_| IggyError::InvalidUsername)?, + password: command.password.expose_secret().to_string(), + status: command.status.as_code(), + permissions: command.permissions.as_ref().map(permissions_to_wire), + }; + let request = ShardRequest::control_plane(ShardRequestPayload::CreateUserRequest { + user_id: identity.user_id, + command: wire_command, + }); + + match state.shard.send_to_control_plane(request).await? { + ShardResponse::CreateUserResponse(user) => { + let response = mapper::map_user(&user); + Ok(Json(response)) + } + ShardResponse::ErrorResponse(err) => Err(err.into()), + _ => unreachable!("Expected CreateUserResponse"), + } +} + +#[debug_handler] +#[instrument(skip_all, name = "trace_update_user", fields(iggy_user_id = identity.user_id, iggy_updated_user_id = user_id))] +async fn update_user( + State(state): State>, + Extension(identity): Extension, + Path(user_id): Path, + Json(mut command): Json, +) -> Result { + command.user_id = Identifier::from_str_value(&user_id)?; + command.validate()?; + + let wire_command = WireUpdateUser { + user_id: identifier_to_wire(&command.user_id)?, + username: command + .username + .as_deref() + .map(WireName::new) + .transpose() + .map_err(|_| IggyError::InvalidUsername)?, + status: command.status.map(|s| s.as_code()), + }; + let request = ShardRequest::control_plane(ShardRequestPayload::UpdateUserRequest { + user_id: identity.user_id, + command: wire_command, + }); + + match state.shard.send_to_control_plane(request).await? { + ShardResponse::UpdateUserResponse(_) => Ok(StatusCode::NO_CONTENT), + ShardResponse::ErrorResponse(err) => Err(err.into()), + _ => unreachable!("Expected UpdateUserResponse"), + } +} + +#[debug_handler] +#[instrument(skip_all, name = "trace_update_permissions", fields(iggy_user_id = identity.user_id, iggy_updated_user_id = user_id))] +async fn update_permissions( + State(state): State>, + Extension(identity): Extension, + Path(user_id): Path, + Json(mut command): Json, +) -> Result { + command.user_id = Identifier::from_str_value(&user_id)?; + command.validate()?; + + let wire_command = WireUpdatePermissions { + user_id: identifier_to_wire(&command.user_id)?, + permissions: command.permissions.as_ref().map(permissions_to_wire), + }; + let request = ShardRequest::control_plane(ShardRequestPayload::UpdatePermissionsRequest { + user_id: identity.user_id, + command: wire_command, + }); + + match state.shard.send_to_control_plane(request).await? { + ShardResponse::UpdatePermissionsResponse => Ok(StatusCode::NO_CONTENT), + ShardResponse::ErrorResponse(err) => Err(err.into()), + _ => unreachable!("Expected UpdatePermissionsResponse"), + } +} + +#[debug_handler] +#[instrument(skip_all, name = "trace_change_password", fields(iggy_user_id = identity.user_id, iggy_updated_user_id = user_id))] +async fn change_password( + State(state): State>, + Extension(identity): Extension, + Path(user_id): Path, + Json(mut command): Json, +) -> Result { + command.user_id = Identifier::from_str_value(&user_id)?; + command.validate()?; + + let wire_command = WireChangePassword { + user_id: identifier_to_wire(&command.user_id)?, + current_password: command.current_password.expose_secret().to_string(), + new_password: command.new_password.expose_secret().to_string(), + }; + let request = ShardRequest::control_plane(ShardRequestPayload::ChangePasswordRequest { + user_id: identity.user_id, + command: wire_command, + }); + + match state.shard.send_to_control_plane(request).await? { + ShardResponse::ChangePasswordResponse => Ok(StatusCode::NO_CONTENT), + ShardResponse::ErrorResponse(err) => Err(err.into()), + _ => unreachable!("Expected ChangePasswordResponse"), + } +} + +#[debug_handler] +#[instrument(skip_all, name = "trace_delete_user", fields(iggy_user_id = identity.user_id, iggy_deleted_user_id = user_id))] +async fn delete_user( + State(state): State>, + Extension(identity): Extension, + Path(user_id): Path, +) -> Result { + let user_id = Identifier::from_str_value(&user_id)?; + + let wire_command = WireDeleteUser { + user_id: identifier_to_wire(&user_id)?, + }; + let request = ShardRequest::control_plane(ShardRequestPayload::DeleteUserRequest { + user_id: identity.user_id, + command: wire_command, + }); + + match state.shard.send_to_control_plane(request).await? { + ShardResponse::DeleteUserResponse(_) => Ok(StatusCode::NO_CONTENT), + ShardResponse::ErrorResponse(err) => Err(err.into()), + _ => unreachable!("Expected DeleteUserResponse"), + } +} + +#[debug_handler] +#[instrument(skip_all, name = "trace_login_user")] +async fn login_user( + State(state): State>, + Json(command): Json, +) -> Result, CustomError> { + let user = state + .shard + .shard() + .login_user(&command.username, command.password.expose_secret(), None) + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to login, username: {}", + command.username + ) + })?; + let tokens = state.jwt_manager.generate(user.id)?; + Ok(Json(map_generated_access_token_to_identity_info(tokens))) +} + +#[debug_handler] +#[instrument(skip_all, name = "trace_logout_user", fields(iggy_user_id = identity.user_id))] +async fn logout_user( + State(state): State>, + Extension(identity): Extension, +) -> Result { + let session = Session::stateless(identity.user_id, identity.ip_address); + state + .shard + .shard() + .logout_user(&session) + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to logout, user ID: {}", + identity.user_id + ) + })?; + + { + let revoke_token_future = SendWrapper::new( + state + .jwt_manager + .revoke_token(&identity.token_id, identity.token_expiry), + ); + + revoke_token_future.await.error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to revoke token, user ID: {}", + identity.user_id + ) + })?; + } + + Ok(StatusCode::NO_CONTENT) +} + +#[debug_handler] +async fn refresh_token( + State(state): State>, + Json(command): Json, +) -> Result, CustomError> { + let token = { + let refresh_token_future = + SendWrapper::new(state.jwt_manager.refresh_token(&command.token)); + + refresh_token_future + .await + .error(|e: &IggyError| format!("{COMPONENT} (error: {e}) - failed to refresh token"))? + }; + + Ok(Json(map_generated_access_token_to_identity_info(token))) +} + +#[derive(Debug, Deserialize)] +struct RefreshToken { + token: String, +} diff --git a/core/server/src/http/web.rs b/core/server/src/http/web.rs new file mode 100644 index 0000000000..76e740e701 --- /dev/null +++ b/core/server/src/http/web.rs @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use axum::Router; +use axum::body::Body; +use axum::extract::Path; +use axum::http::{Response, StatusCode, header}; +use axum::response::IntoResponse; +use axum::routing::get; +use rust_embed::{Embed, EmbeddedFile}; + +#[derive(Embed)] +#[folder = "../../web/build/static/"] +#[allow_missing = true] +struct WebAssets; + +impl WebAssets { + fn get_file(path: &str) -> Option { + ::get(path) + } +} + +pub fn router() -> Router { + Router::new() + .route("/ui/{*wildcard}", get(serve_web_asset)) + .route("/ui", get(serve_index)) + .route("/ui/", get(serve_index)) +} + +async fn serve_index() -> impl IntoResponse { + serve_file("index.html") +} + +async fn serve_web_asset(Path(wildcard): Path) -> impl IntoResponse { + if let Some(response) = try_serve_file(&wildcard) { + return response; + } + + if !wildcard.contains('.') { + return serve_file("index.html"); + } + + Response::builder() + .status(StatusCode::NOT_FOUND) + .body(Body::from("Not Found")) + .unwrap() +} + +fn try_serve_file(path: &str) -> Option> { + let asset = WebAssets::get_file(path)?; + let mime = mime_guess::from_path(path).first_or_octet_stream(); + + Some( + Response::builder() + .status(StatusCode::OK) + .header(header::CONTENT_TYPE, mime.as_ref()) + .body(Body::from(asset.data.into_owned())) + .unwrap(), + ) +} + +fn serve_file(path: &str) -> Response { + try_serve_file(path).unwrap_or_else(|| { + Response::builder() + .status(StatusCode::NOT_FOUND) + .body(Body::from("Not Found")) + .unwrap() + }) +} diff --git a/core/server/src/io/mod.rs b/core/server/src/io/mod.rs new file mode 100644 index 0000000000..4efe7ebd13 --- /dev/null +++ b/core/server/src/io/mod.rs @@ -0,0 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub use server_common::fs_utils; + +pub mod storage; diff --git a/core/server/src/io/storage.rs b/core/server/src/io/storage.rs new file mode 100644 index 0000000000..c065e50081 --- /dev/null +++ b/core/server/src/io/storage.rs @@ -0,0 +1,171 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::future::Future; + +use compio::{ + buf::{IoBuf, IoBufMut}, + io::{AsyncReadAtExt, AsyncWriteAtExt}, +}; + +pub trait Storage { + fn read_exact_at( + &self, + buf: B, + pos: u64, + ) -> impl Future>; + fn write_all_at( + &self, + buf: B, + pos: u64, + ) -> impl Future>; +} + +pub struct OpenOpts { + keep_fd: bool, + path: String, +} + +impl OpenOpts { + pub fn ephemeral(path: String) -> Self { + Self { + keep_fd: false, + path, + } + } + + pub fn permanent(path: String) -> Self { + Self { + keep_fd: true, + path, + } + } +} + +pub enum StorageImpl { + Block(BlockStorage), +} + +impl StorageImpl { + pub fn read_exact_at( + &self, + buf: B, + pos: u64, + ) -> impl Future> { + match self { + StorageImpl::Block(storage) => storage.read_exact_at(buf, pos), + } + } + + pub fn write_all_at( + &mut self, + buf: B, + pos: u64, + ) -> impl Future> { + match self { + StorageImpl::Block(storage) => storage.write_all_at(buf, pos), + } + } +} + +pub struct BlockStorage { + file: Option, + path: Option, +} + +impl BlockStorage { + pub async fn xd(&self) { + let file = self.file.as_ref().unwrap(); + let buf = Vec::new(); + (&*file).write_all_at(buf, 0).await.unwrap(); + } +} + +impl BlockStorage { + pub async fn new(opts: OpenOpts) -> Result { + let path = opts.path; + let keep_fd = opts.keep_fd; + let file = if keep_fd { + let file = compio::fs::OpenOptions::new() + .create(true) + .read(true) + .write(true) + .open(&path) + .await?; + Some(file) + } else { + None + }; + let path = if file.is_some() { None } else { Some(path) }; + Ok(Self { file, path }) + } +} + +impl Storage for BlockStorage { + async fn read_exact_at(&self, buf: B, pos: u64) -> Result { + let (result, buf) = match &self.file { + Some(file) => file.read_exact_at(buf, pos).await.into(), + None => { + let path = self.path.as_ref().unwrap(); + let file = compio::fs::File::open(path).await?; + file.read_exact_at(buf, pos).await.into() + } + }; + result?; + Ok(buf) + } + + async fn write_all_at(&self, buf: B, pos: u64) -> Result { + let (result, buf) = match self.file { + Some(ref file) => (&*file).write_all_at(buf, pos).await.into(), + None => { + let path = self.path.as_ref().unwrap(); + let mut file = compio::fs::OpenOptions::new() + .create(true) + .write(true) + .open(path) + .await?; + file.write_all_at(buf, pos).await.into() + } + }; + result?; + Ok(buf) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn open_opts_ephemeral_creates_correct_config() { + let path = "path/to/file".to_string(); + let opts = OpenOpts::ephemeral(path.clone()); + + assert!(!opts.keep_fd); + assert_eq!(opts.path, path); + } + + #[test] + fn open_opts_permanent_creates_correct_config() { + let path = "path/to/file".to_string(); + let opts = OpenOpts::permanent(path.clone()); + + assert!(opts.keep_fd); + assert_eq!(opts.path, path); + } +} diff --git a/core/server/src/lib.rs b/core/server/src/lib.rs index 5ad252b5d5..feddf428d6 100644 --- a/core/server/src/lib.rs +++ b/core/server/src/lib.rs @@ -15,35 +15,39 @@ // specific language governing permissions and limitations // under the License. -#![allow(clippy::future_not_send)] +#[cfg(not(feature = "disable-mimalloc"))] +use mimalloc::MiMalloc; use iggy_common::SemanticVersion; -pub const VERSION: &str = env!("CARGO_PKG_VERSION"); -pub const SEMANTIC_VERSION: SemanticVersion = SemanticVersion::parse_const(VERSION); +#[cfg(not(feature = "disable-mimalloc"))] +#[global_allocator] +static GLOBAL: MiMalloc = MiMalloc; -pub mod auth; +#[cfg(windows)] +compile_error!("iggy-server doesn't support windows."); + +pub mod args; +pub mod binary; pub mod bootstrap; -pub(crate) mod cluster_meta; -pub mod config_writer; -pub mod consumer_group; -pub mod dispatch; -pub(crate) mod http; -pub mod login_register; -pub(crate) mod offset_recovery; -pub mod partition_helpers; -pub mod partition_reconciler; -pub mod pat; -pub(crate) mod personal_access_token_cleaner; -pub mod responses; -pub(crate) mod segment_cleaner; -pub(crate) mod segment_recovery; +pub(crate) mod compat; +pub mod configs; +pub mod diagnostics; +pub mod http; +pub mod io; +pub mod metadata; +pub mod quic; +pub mod sender; pub mod server_error; -pub mod session_manager; -pub(crate) mod snapshot; -#[cfg(feature = "systemd")] -pub mod systemd; -pub mod users; -#[cfg(feature = "iggy-web")] -pub(crate) mod web; -pub mod wire; +pub mod shard; +pub mod state; +pub mod streaming; +pub mod tcp; +pub mod websocket; + +pub use server_common::log; + +pub const VERSION: &str = env!("CARGO_PKG_VERSION"); +pub const SEMANTIC_VERSION: SemanticVersion = SemanticVersion::parse_const(VERSION); +pub const IGGY_ROOT_USERNAME_ENV: &str = "IGGY_ROOT_USERNAME"; +pub const IGGY_ROOT_PASSWORD_ENV: &str = "IGGY_ROOT_PASSWORD"; diff --git a/core/server/src/main.rs b/core/server/src/main.rs index f6a8e63e36..1f0ac18d90 100644 --- a/core/server/src/main.rs +++ b/core/server/src/main.rs @@ -15,86 +15,549 @@ // specific language governing permissions and limitations // under the License. -#![allow(clippy::future_not_send)] - -mod args; - -use args::Args; +use anyhow::Result; use clap::Parser; -use configs::server::ServerConfig; +use dashmap::DashMap; +use dotenvy::dotenv; +use err_trail::ErrContext; +use figlet_rs::FIGlet; +use iggy_common::SemanticVersion; +use iggy_common::{Aes256GcmEncryptor, EncryptorKind, IggyError}; +use server::SEMANTIC_VERSION; +use server::args::Args; use server::bootstrap::{ - apply_default_root_credentials, bootstrap, load_config, prepare_runtime_dirs, + create_directories, create_shard_connections, create_shard_executor, load_config, + load_metadata, resolve_persister, update_system_info, }; +use server::diagnostics::{ASYNCIFY_POOL_DISABLED_PANIC_MSG, print_incomplete_io_uring_ops_info}; +use server::io::fs_utils; +use server::log::logger::Logging; +use server::metadata::{Metadata, create_metadata_handles}; use server::server_error::ServerError; -use server_common::log::Logging; +use server::shard::system::info::SystemInfo; +use server::shard::{IggyShard, calculate_shard_assignment}; +use server::state::file::FileState; +use server::state::system::SystemState; +use server::streaming::clients::client_manager::{Client, ClientManager}; +use server::streaming::diagnostics::metrics::Metrics; +use server::streaming::storage::SystemStorage; +use server::streaming::utils::ptr::EternalPtr; +use server_common::MemoryPool; +use server_common::log::{LoggingSettings, TelemetrySettings}; +use server_common::sharding::{IggyNamespace, PartitionLocation, ShardId}; +use shard_allocator::ShardAllocator; +use std::panic::AssertUnwindSafe; +use std::rc::Rc; +use std::str::FromStr; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; +use std::sync::mpsc; +use std::thread::JoinHandle; use system_stats::capture_allowed_cpus; -use tracing::{error, info}; +use tracing::{error, info, instrument, warn}; -fn main() -> Result<(), ServerError> { - // This prelude must stay ahead of the first thread the process ever - // spawns: `--with-default-root-credentials` writes to the environment, - // and `set_var` is only sound while single-threaded. `early_init` just - // installs the tracing registry (buffered until `late_init`) and starts - // no worker of its own, so it can run here and make the warnings below - // visible. `create_shard_executor` also reads its capacity knob from the - // environment, which is why the `.env` load has to precede it. - let args = Args::parse(); - // `logging` owns the tracing appender worker guards; it must outlive the - // shard threads or every log line after bootstrap is silently dropped. - let mut logging = Logging::new(server::VERSION); - logging.early_init(); - server_common::print_build_info!(server::VERSION); - if let Ok(env_path) = std::env::var("IGGY_ENV_PATH") { - let _ = dotenvy::from_path(&env_path); - } else { - let _ = dotenvy::dotenv(); +const COMPONENT: &str = "MAIN"; +const SHARDS_TABLE_CAPACITY: usize = 16384; + +static SHUTDOWN_START_TIME: AtomicU64 = AtomicU64::new(0); +static SHUTDOWN_INITIATED: AtomicBool = AtomicBool::new(false); +// Separate latch from the ring-setup one inside +// `enrich_runtime_create_error`: a shard that fails ring setup (e.g. partial +// ENOMEM under a tight RLIMIT_MEMLOCK) must not consume the latch and +// suppress the unsupported-opcode diagnostic from a sibling shard that did +// start. Setup vs runtime io_uring failures can co-occur across shards. +static SHARD_RUNTIME_DIAGNOSTIC: std::sync::Once = std::sync::Once::new(); + +enum ShardExitStatus { + Success, + Error(String), + Panic(String), +} + +fn initiate_shutdown( + reason: &str, + shutdown_handles: &[(u16, server::shard::transmission::connector::StopSender)], +) { + if SHUTDOWN_INITIATED.swap(true, Ordering::SeqCst) { + return; + } + + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0); + SHUTDOWN_START_TIME.store(now, Ordering::SeqCst); + + info!("{reason}, initiating graceful shutdown..."); + + for (shard_id, stop_sender) in shutdown_handles { + if let Err(e) = stop_sender.try_send(()) { + error!("Failed to send shutdown signal to shard {shard_id}: {e}"); + } } - // SAFETY: no thread has been spawned yet, see the comment above. - unsafe { apply_default_root_credentials(args.with_default_root_credentials) }; +} + +fn extract_panic_message(payload: Box) -> String { + payload + .downcast_ref::<&str>() + .map(|s| s.to_string()) + .or_else(|| payload.downcast_ref::().cloned()) + .unwrap_or_else(|| "unknown panic".to_string()) +} +fn print_ascii_art(text: &str) { + let standard_font = FIGlet::standard().unwrap(); + let figure = standard_font.convert(text); + println!("{}", figure.unwrap()); +} + +#[instrument(skip_all, name = "trace_start_server")] +fn main() -> Result<(), ServerError> { // Before shard threads pin themselves: a pinned capture sees one core. capture_allowed_cpus(); - let bootstrap_runtime = match server_common::create_shard_executor() { + let rt = match compio::runtime::Runtime::new() { Ok(rt) => rt, Err(e) => { let e = server_common::diagnostics::enrich_runtime_create_error(e); - panic!("Cannot create server bootstrap executor: {e}"); + panic!("Cannot create runtime: {e}"); } }; + rt.block_on(async move { + if let Ok(env_path) = std::env::var("IGGY_ENV_PATH") { + if dotenvy::from_path(&env_path).is_ok() { + println!("Loaded environment variables from path: {env_path}"); + } + } else if let Ok(path) = dotenv() { + println!( + "Loaded environment variables from .env file at path: {}", + path.display() + ); + } + let args = Args::parse(); + print_ascii_art("Iggy Server"); - // Bootstrap on a temporary runtime: load config, prepare the data - // directory, init the memory pool. Then drop the runtime and spawn the - // per-shard runtimes - each shard thread builds its OWN - // `compio::runtime::Runtime` via `create_shard_executor`, pinned to - // its CPU. - let bootstrap_result: Result = bootstrap_runtime.block_on(async { - let config = load_config().await?; - prepare_runtime_dirs(&config, &mut logging, args.fresh).await?; - server_common::MemoryPool::init_pool(&config.system.memory_pool.into_other()); - - Ok(config) - }); - let config = bootstrap_result?; - drop(bootstrap_runtime); - - let shards = bootstrap(config, args.replica_id)?; - if let Err(error) = shards.install_ctrlc_handler() { - // Without a working SIGINT handler the server has no way to - // observe an operator Ctrl-C and the shutdown flag would never - // flip, leaving shard threads parked indefinitely. Fail fast - // rather than boot into an un-killable state. - error!(error = %error, "failed to install Ctrl-C handler; aborting boot"); - std::process::exit(1); - } + let is_follower = args.follower; + let replica_id = args.replica_id; - info!("server running; waiting on shard threads"); - let joined = shards.join_all(); - #[cfg(feature = "systemd")] - if let Err(error) = &joined { - server::systemd::notify_shutdown_failure(error); - } - joined?; - info!("server shutdown complete"); - Ok(()) + // FIRST DISCRETE LOADING STEP. + // Initialize early logging before config parsing so we can log during bootstrap. + let mut logging = Logging::new(server::VERSION); + logging.early_init(); + server_common::print_build_info!(server::VERSION); + + // SECOND DISCRETE LOADING STEP. + // Load config and create directories. + // Remove `local_data` directory if run with `--fresh` flag. + let config = load_config().await.error(|e: &ServerError| { + format!("{COMPONENT} (error: {e}) - failed to load config during bootstrap") + })?; + if args.fresh { + let system_path = config.system.get_system_path(); + if compio::fs::metadata(&system_path).await.is_ok() { + warn!( + "Removing system path at: {} because `--fresh` flag was set", + system_path + ); + if let Err(e) = fs_utils::remove_dir_all(&system_path).await { + warn!("Failed to remove system path at {system_path}: {e}"); + } + } + } + + // THIRD DISCRETE LOADING STEP. + // Create directories. + create_directories(&config.system).await?; + + // FOURTH DISCRETE LOADING STEP. + // Complete logging setup with config (file output, telemetry). + // From this point on, logs are persisted to file and telemetry is active. + logging.late_init( + config.system.get_system_path(), + &LoggingSettings::from(&config.system.logging), + &TelemetrySettings::from(&config.telemetry), + )?; + + if is_follower { + info!("Server is running in FOLLOWER mode for testing leader redirection"); + } + + if args.with_default_root_credentials { + let username_set = std::env::var("IGGY_ROOT_USERNAME").is_ok(); + let password_set = std::env::var("IGGY_ROOT_PASSWORD").is_ok(); + + if !username_set || !password_set { + if !username_set { + unsafe { + std::env::set_var("IGGY_ROOT_USERNAME", "iggy"); + } + } + if !password_set { + unsafe { + std::env::set_var("IGGY_ROOT_PASSWORD", "iggy"); + } + } + info!( + "Using default root credentials (username: iggy, password: iggy) - FOR DEVELOPMENT ONLY! \ + If root user already exists, existing credentials will be reused." + ); + } else { + warn!( + "--with-default-root-credentials flag is ignored because root credentials are already set via environment variables" + ); + } + } + + // FIFTH DISCRETE LOADING STEP. + MemoryPool::init_pool(&config.system.memory_pool.into_other()); + + // SIXTH DISCRETE LOADING STEP. + let partition_persister = resolve_persister(config.system.partition.enforce_fsync); + let storage = SystemStorage::new(config.system.clone(), partition_persister); + + // SEVENTH DISCRETE LOADING STEP. + let current_version = SEMANTIC_VERSION; + info!("Current semantic version: {:?}", current_version); + + let mut system_info; + let load_system_info = storage.info.load().await; + match load_system_info { + Ok(info) => { + system_info = info; + } + Err(e) => { + if let IggyError::ResourceNotFound(_) = e { + info!("System info not found, creating..."); + system_info = SystemInfo::default(); + update_system_info(&storage, &mut system_info, ¤t_version).await?; + } else { + panic!("Failed to load system info from disk. {e}"); + } + } + } + info!("Loaded {system_info}."); + let loaded_version = SemanticVersion::from_str(&system_info.version.version)?; + if current_version.is_equal_to(&loaded_version) { + info!("System version {current_version} is up to date."); + } else if current_version.is_greater_than(&loaded_version) { + info!( + "System version {current_version} is greater than {loaded_version}, checking the available migrations..." + ); + update_system_info(&storage, &mut system_info, ¤t_version).await?; + } else { + info!( + "System version {current_version} is lower than {loaded_version}, possible downgrade." + ); + update_system_info(&storage, &mut system_info, ¤t_version).await?; + } + + // EIGHTH DISCRETE LOADING STEP. + info!( + "Server-side encryption is {}.", + match config.system.encryption.enabled { + true => "enabled", + false => "disabled", + } + ); + let encryptor: Option = match config.system.encryption.enabled { + true => Some(EncryptorKind::Aes256Gcm( + Aes256GcmEncryptor::from_base64_key(&config.system.encryption.key).unwrap(), + )), + false => None, + }; + + // TENTH DISCRETE LOADING STEP. + let state_persister = resolve_persister(config.system.state.enforce_fsync); + let state_current_index = Arc::new(AtomicU64::new(0)); + let state_entries_count = Arc::new(AtomicU64::new(0)); + let state_current_leader = Arc::new(AtomicU32::new(0)); + let state_term = Arc::new(AtomicU64::new(0)); + let state = FileState::new( + &config.system.get_state_messages_file_path(), + ¤t_version, + state_persister, + encryptor.clone(), + state_current_index.clone(), + state_entries_count.clone(), + state_current_leader.clone(), + state_term.clone(), + ); + let state = SystemState::load(state).await?; + let (streams_state, users_state) = state.decompose(); + + // Create left-right handles for metadata + let (mut metadata_writer, metadata_reader) = create_metadata_handles(); + + // Create shared metadata reader - each shard will get a clone + let metadata = Metadata::new(metadata_reader); + + // Load initial metadata using the writer + load_metadata( + users_state.into_values(), + streams_state.into_values(), + &mut metadata_writer, + ); + + // ELEVENTH DISCRETE LOADING STEP. + let shard_allocator = ShardAllocator::new( + &config.system.sharding.cpu_allocation, + config.system.sharding.pin_cores, + )?; + let shard_assignment = shard_allocator.to_shard_assignments()?; + + #[cfg(feature = "disable-mimalloc")] + warn!("Using default system allocator because code was build with `disable-mimalloc` feature"); + #[cfg(not(feature = "disable-mimalloc"))] + info!("Using mimalloc allocator"); + + // DISCRETE STEP. + // Increment the metrics. + let metrics = Metrics::init(); + + // TWELFTH DISCRETE LOADING STEP. + info!( + "Enable TCP socket migration across shards: {}.", + config.tcp.socket_migration + ); + + info!("Starting {} shard(s)", shard_assignment.len()); + let (connections, shutdown_handles) = create_shard_connections(&shard_assignment); + let shards_count = shard_assignment.len(); + let mut handles: Vec> = Vec::with_capacity(shards_count); + + // Channel for shard completion notifications + let (shard_done_tx, shard_done_rx) = mpsc::channel::<(u16, ShardExitStatus)>(); + + // TODO: Persist the shards table and load it from the disk, so it does not have to be + // THIRTEENTH DISCRETE LOADING STEP. + // Shared resources bootstrap. + let shards_table = Box::new(DashMap::with_capacity(SHARDS_TABLE_CAPACITY)); + let shards_table = Box::leak(shards_table); + let shards_table: EternalPtr> = shards_table.into(); + + let client_manager = Box::new(DashMap::new()); + let client_manager = Box::leak(client_manager); + let client_manager: EternalPtr> = client_manager.into(); + let client_manager = ClientManager::new(client_manager); + + // Populate shards_table from SharedMetadata partitions (hierarchical traversal) + metadata.with_metadata(|metadata| { + for (stream_id, stream_meta) in metadata.streams.iter() { + for (topic_id, topic_meta) in stream_meta.topics.iter() { + for (partition_id, _partition_meta) in topic_meta.partitions.iter().enumerate() { + let ns = IggyNamespace::new(stream_id, topic_id, partition_id); + let shard_id = ShardId::new(calculate_shard_assignment( + &ns, + shard_assignment.len() as u32, + )); + // epoch is reconciler-only; unused by legacy server. + let location = PartitionLocation::new(shard_id, 0); + shards_table.insert(ns, location); + } + } + } + }); + + // Wrap metadata_writer in Option so we can take it for shard 0 + let mut metadata_writer_opt = Some(metadata_writer); + + for (id, assignment) in shard_assignment + .into_iter() + .enumerate() + .map(|(idx, assignment)| (idx as u16, assignment)) + { + let shards_table = shards_table.clone(); + let connections = connections.clone(); + let config = config.clone(); + let encryptor = encryptor.clone(); + let metrics = metrics.clone(); + let current_version = current_version.clone(); + let state_persister = resolve_persister(config.system.state.enforce_fsync); + let state = FileState::new( + &config.system.get_state_messages_file_path(), + ¤t_version, + state_persister, + encryptor.clone(), + state_current_index.clone(), + state_entries_count.clone(), + state_current_leader.clone(), + state_term.clone(), + ); + let client_manager = client_manager.clone(); + let shard_metadata = metadata.clone(); + + // Take metadata_writer for shard 0 only + let shard_metadata_writer = if id == 0 { + metadata_writer_opt.take() + } else { + None + }; + + let shard_done_tx = shard_done_tx.clone(); + let handle = std::thread::Builder::new() + .name(format!("shard-{id}")) + .spawn(move || { + let result = std::panic::catch_unwind(AssertUnwindSafe(|| { + if let Err(e) = assignment.bind_cpu() { + error!("Failed to bind cpu: {e:?}"); + } + + if let Err(e) = assignment.bind_memory() { + error!("Failed to bind memory: {e:?}"); + } + + let rt = match create_shard_executor() { + Ok(rt) => rt, + Err(e) => { + // Prints the verbose remediation once across + // all shard threads; the panic message itself + // carries the one-line fix. + let e = + server_common::diagnostics::enrich_runtime_create_error(e); + panic!("Cannot create shard-{id} executor: {e}"); + } + }; + rt.block_on(async move { + let mut builder = IggyShard::builder(); + builder = builder + .id(id) + .state(state) + .shards_table(shards_table) + .connections(connections) + .clients_manager(client_manager) + .config(config) + .encryptor(encryptor) + .version(current_version) + .metrics(metrics) + .is_follower(is_follower) + .current_replica_id(replica_id) + .metadata(shard_metadata); + + if let Some(writer) = shard_metadata_writer { + builder = builder.metadata_writer(writer); + } + + let shard = builder.build(); + + let shard = Rc::new(shard); + + if let Err(e) = shard.run().await { + error!("Failed to run shard-{id}: {e}"); + return Err(e.to_string()); + } + info!("Shard {id} run completed"); + + Ok(()) + }) + })); + + let status = match result { + Ok(Ok(())) => ShardExitStatus::Success, + Ok(Err(msg)) => ShardExitStatus::Error(msg), + Err(panic_payload) => { + ShardExitStatus::Panic(extract_panic_message(panic_payload)) + } + }; + + let _ = shard_done_tx.send((id, status)); + }) + .unwrap_or_else(|e| panic!("Failed to spawn thread for shard-{id}: {e}")); + handles.push(handle); + } + + drop(shard_done_tx); + + let shutdown_handles_for_signal = shutdown_handles.clone(); + ctrlc::set_handler(move || { + initiate_shutdown( + "Received shutdown signal (SIGTERM/SIGINT)", + &shutdown_handles_for_signal, + ); + }) + .expect("Error setting Ctrl-C handler"); + + info!("Iggy server is running. Press Ctrl+C or send SIGTERM to shutdown."); + + let mut completed_shards = 0usize; + let mut failure_message: Option = None; + + while completed_shards < shards_count { + match shard_done_rx.recv() { + Ok((shard_id, status)) => { + completed_shards += 1; + + match status { + ShardExitStatus::Success => { + info!("Shard {shard_id} exited successfully"); + } + ShardExitStatus::Error(msg) => { + error!("Shard {shard_id} exited with error: {msg}"); + if failure_message.is_none() { + failure_message = + Some(format!("Shard {shard_id} exited with error: {msg}")); + } + initiate_shutdown( + &format!("Shard {shard_id} exited with error"), + &shutdown_handles, + ); + } + ShardExitStatus::Panic(msg) => { + error!("Shard {shard_id} panicked: {msg}"); + if msg.contains(ASYNCIFY_POOL_DISABLED_PANIC_MSG) { + SHARD_RUNTIME_DIAGNOSTIC + .call_once(print_incomplete_io_uring_ops_info); + } + if failure_message.is_none() { + failure_message = + Some(format!("Shard {shard_id} panicked: {msg}")); + } + initiate_shutdown( + &format!("Shard {shard_id} panicked"), + &shutdown_handles, + ); + } + } + } + Err(_) => { + error!("Shard completion channel closed unexpectedly"); + break; + } + } + } + + for (idx, handle) in handles.into_iter().enumerate() { + if let Err(e) = handle.join() { + warn!("Shard {idx} thread join returned panic: {e:?}"); + } + } + + let shutdown_duration_msg = { + let start_time = SHUTDOWN_START_TIME.load(Ordering::SeqCst); + if start_time > 0 { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0); + let elapsed = now.saturating_sub(start_time); + format!(" (shutdown took {} ms)", elapsed) + } else { + String::new() + } + }; + + if let Some(msg) = failure_message { + error!( + "Server shutting down due to shard failure.{}", + shutdown_duration_msg + ); + return Err(ServerError::ShardFailure { message: msg }); + } + + info!( + "All shards have shut down. Iggy server is exiting.{}", + shutdown_duration_msg + ); + + Ok(()) + }) } diff --git a/core/server/src/metadata/absorb.rs b/core/server/src/metadata/absorb.rs new file mode 100644 index 0000000000..a90734f2aa --- /dev/null +++ b/core/server/src/metadata/absorb.rs @@ -0,0 +1,500 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::metadata::ConsumerGroupMemberMeta; +use crate::metadata::inner::InnerMetadata; +use crate::metadata::ops::MetadataOp; +use crate::metadata::{StreamId, UserId}; +use crate::streaming::polling_consumer::ConsumerGroupId; +use iggy_common::Permissions; +use left_right::Absorb; +use std::sync::atomic::Ordering; + +impl Absorb for InnerMetadata { + fn absorb_first(&mut self, op: &mut MetadataOp, other: &Self) { + apply_op(self, op, true, other); + } + + fn absorb_second(&mut self, op: MetadataOp, other: &Self) { + apply_op(self, &op, false, other); + } + + fn sync_with(&mut self, first: &Self) { + *self = first.clone(); + } +} + +fn apply_op( + metadata: &mut InnerMetadata, + op: &MetadataOp, + populate_ids: bool, + _reader_copy: &InnerMetadata, +) { + match op { + MetadataOp::Initialize(initial) => { + *metadata = (**initial).clone(); + rebuild_all_permission_indexes(metadata); + } + + MetadataOp::AddStream { meta, assigned_id } => { + let entry = metadata.streams.vacant_entry(); + let id = entry.key(); + if populate_ids { + assigned_id.store(id, Ordering::Release); + } + let mut meta = meta.clone(); + meta.id = id; + let name = meta.name.clone(); + entry.insert(meta); + metadata.stream_index.insert(name, id); + } + + MetadataOp::UpdateStream { id, new_name } => { + if let Some(stream) = metadata.streams.get_mut(*id) { + let old_name = stream.name.clone(); + stream.name = new_name.clone(); + metadata.stream_index.remove(&old_name); + metadata.stream_index.insert(new_name.clone(), *id); + } + } + + MetadataOp::DeleteStream { id } => { + if metadata.streams.contains(*id) { + let stream = metadata.streams.remove(*id); + metadata.stream_index.remove(&stream.name); + clear_stream_permission_indexes(metadata, *id); + } + } + + MetadataOp::AddTopic { + stream_id, + meta, + assigned_id, + } => { + if let Some(stream) = metadata.streams.get_mut(*stream_id) { + let entry = stream.topics.vacant_entry(); + let id = entry.key(); + if populate_ids { + assigned_id.store(id, Ordering::Release); + } + let mut meta = meta.clone(); + meta.id = id; + let name = meta.name.clone(); + entry.insert(meta); + stream.topic_index.insert(name, id); + } + } + + MetadataOp::UpdateTopic { + stream_id, + topic_id, + new_name, + message_expiry, + compression_algorithm, + max_topic_size, + replication_factor, + } => { + if let Some(stream) = metadata.streams.get_mut(*stream_id) + && let Some(topic) = stream.topics.get_mut(*topic_id) + { + let old_name = topic.name.clone(); + + topic.name = new_name.clone(); + topic.message_expiry = *message_expiry; + topic.compression_algorithm = *compression_algorithm; + topic.max_topic_size = *max_topic_size; + topic.replication_factor = *replication_factor; + + if old_name != *new_name { + stream.topic_index.remove(&old_name); + stream.topic_index.insert(new_name.clone(), *topic_id); + } + } + } + + MetadataOp::DeleteTopic { + stream_id, + topic_id, + } => { + if let Some(stream) = metadata.streams.get_mut(*stream_id) + && stream.topics.contains(*topic_id) + { + let topic = stream.topics.remove(*topic_id); + stream.topic_index.remove(&topic.name); + } + } + + MetadataOp::AddPartitions { + stream_id, + topic_id, + partitions, + revision_id, + } => { + if partitions.is_empty() { + return; + } + if let Some(stream) = metadata.streams.get_mut(*stream_id) + && let Some(topic) = stream.topics.get_mut(*topic_id) + { + for meta in partitions { + let mut meta = meta.clone(); + meta.id = topic.partitions.len(); + meta.revision_id = *revision_id; + topic.partitions.push(meta); + } + } + } + + MetadataOp::DeletePartitions { + stream_id, + topic_id, + count, + } => { + if *count == 0 { + return; + } + if let Some(stream) = metadata.streams.get_mut(*stream_id) + && let Some(topic) = stream.topics.get_mut(*topic_id) + { + let new_len = topic.partitions.len().saturating_sub(*count as usize); + topic.partitions.truncate(new_len); + } + } + + MetadataOp::AddUser { meta, assigned_id } => { + let entry = metadata.users.vacant_entry(); + let id = entry.key(); + if populate_ids { + assigned_id.store(id, Ordering::Release); + } + let mut meta = meta.clone(); + meta.id = id as u32; + let username = meta.username.clone(); + let permissions = meta.permissions.clone(); + entry.insert(meta); + metadata.user_index.insert(username, id as u32); + update_permission_indexes(metadata, id as u32, permissions.as_deref()); + } + + MetadataOp::UpdateUserMeta { id, meta } => { + let user_id = *id as usize; + if let Some(old_user) = metadata.users.get(user_id) + && old_user.username != meta.username + { + metadata.user_index.remove(&old_user.username); + metadata.user_index.insert(meta.username.clone(), *id); + } + if metadata.users.contains(user_id) { + let permissions = meta.permissions.clone(); + metadata.users[user_id] = meta.clone(); + update_permission_indexes(metadata, *id, permissions.as_deref()); + } + } + + MetadataOp::DeleteUser { id } => { + let user_id = *id as usize; + if metadata.users.contains(user_id) { + let user = metadata.users.remove(user_id); + metadata.user_index.remove(&user.username); + } + metadata.personal_access_tokens.remove(id); + clear_permission_indexes(metadata, *id); + } + + MetadataOp::AddPersonalAccessToken { user_id, pat } => { + metadata + .personal_access_tokens + .entry(*user_id) + .or_default() + .insert(pat.token.clone(), pat.clone()); + } + + MetadataOp::DeletePersonalAccessToken { + user_id, + token_hash, + } => { + if let Some(user_pats) = metadata.personal_access_tokens.get_mut(user_id) { + user_pats.remove(token_hash); + } + } + + MetadataOp::AddConsumerGroup { + stream_id, + topic_id, + meta, + assigned_id, + } => { + if let Some(stream) = metadata.streams.get_mut(*stream_id) + && let Some(topic) = stream.topics.get_mut(*topic_id) + { + let entry = topic.consumer_groups.vacant_entry(); + let id = entry.key(); + if populate_ids { + assigned_id.store(id, Ordering::Release); + } + let mut meta = meta.clone(); + meta.id = id; + let name = meta.name.clone(); + entry.insert(meta); + topic.consumer_group_index.insert(name, id); + } + } + + MetadataOp::DeleteConsumerGroup { + stream_id, + topic_id, + group_id, + } => { + if let Some(stream) = metadata.streams.get_mut(*stream_id) + && let Some(topic) = stream.topics.get_mut(*topic_id) + && topic.consumer_groups.contains(*group_id) + { + let group = topic.consumer_groups.remove(*group_id); + topic.consumer_group_index.remove(&group.name); + } + } + + MetadataOp::JoinConsumerGroup { + stream_id, + topic_id, + group_id, + client_id, + member_id, + valid_client_ids, + completable_revocations, + } => { + if let Some(stream) = metadata.streams.get_mut(*stream_id) + && let Some(topic) = stream.topics.get_mut(*topic_id) + && let Some(group) = topic.consumer_groups.get_mut(*group_id) + { + if let Some(valid_ids) = valid_client_ids { + let stale_members: Vec = group + .members + .iter() + .filter(|(_, m)| !valid_ids.contains(&m.client_id)) + .map(|(slot_id, _)| slot_id) + .collect(); + + for slot_id in stale_members { + group.members.remove(slot_id); + } + } + + let next_id = group + .members + .iter() + .map(|(_, m)| m.id) + .max() + .map(|m| m + 1) + .unwrap_or(0); + + if populate_ids { + member_id.store(next_id, Ordering::Release); + } + + if !group.members.iter().any(|(_, m)| m.client_id == *client_id) { + let new_member = ConsumerGroupMemberMeta::new(next_id, *client_id); + group.members.insert(new_member); + group.rebalance_cooperative(); + } + + if populate_ids { + let cg_id = ConsumerGroupId(*group_id); + let found = group.find_completable_revocations(&topic.partitions, cg_id); + if !found.is_empty() { + *completable_revocations.lock().unwrap() = found; + } + } + } + } + + MetadataOp::LeaveConsumerGroup { + stream_id, + topic_id, + group_id, + client_id, + removed_member_id, + } => { + if let Some(stream) = metadata.streams.get_mut(*stream_id) + && let Some(topic) = stream.topics.get_mut(*topic_id) + && let Some(group) = topic.consumer_groups.get_mut(*group_id) + { + let member_to_remove: Option = group + .members + .iter() + .find(|(_, m)| m.client_id == *client_id) + .map(|(id, _)| id); + + if let Some(member_id) = member_to_remove { + if populate_ids { + removed_member_id.store(member_id, Ordering::Release); + } + // Partitions owned by the leaving member + let leaving_partitions: Vec = group + .members + .get(member_id) + .map(|m| m.partitions.clone()) + .unwrap_or_default(); + + group.members.remove(member_id); + group.rebalance_members(); + + // Clear polled offsets only for the leaving member's partitions + let consumer_group_id = ConsumerGroupId(*group_id); + for partition_id in leaving_partitions { + if let Some(partition) = topic.partitions.get(partition_id) { + let guard = partition.last_polled_offsets.pin(); + guard.remove(&consumer_group_id); + } + } + } + } + } + + MetadataOp::RebalanceConsumerGroupsForTopic { + stream_id, + topic_id, + partitions_count, + } => { + if let Some(stream) = metadata.streams.get_mut(*stream_id) + && let Some(topic) = stream.topics.get_mut(*topic_id) + { + let partition_ids: Vec = (0..*partitions_count as usize).collect(); + let group_ids: Vec<_> = topic.consumer_groups.iter().map(|(id, _)| id).collect(); + + for group_id in group_ids { + if let Some(group) = topic.consumer_groups.get_mut(group_id) { + group.partitions = partition_ids.clone(); + group.rebalance_members(); + + let consumer_group_id = ConsumerGroupId(group_id); + for partition in topic.partitions.iter() { + let guard = partition.last_polled_offsets.pin(); + guard.remove(&consumer_group_id); + } + } + } + } + } + + MetadataOp::CompletePartitionRevocation { + stream_id, + topic_id, + group_id, + member_slab_id, + member_id, + partition_id, + timed_out: _, + } => { + if let Some(stream) = metadata.streams.get_mut(*stream_id) + && let Some(topic) = stream.topics.get_mut(*topic_id) + && let Some(group) = topic.consumer_groups.get_mut(*group_id) + { + // Pre-validated by maybe_complete_pending_revocation before dispatch. + group.complete_revocation(*member_slab_id, *member_id, *partition_id); + } + } + } +} + +fn clear_permission_indexes(metadata: &mut InnerMetadata, user_id: UserId) { + metadata.users_global_permissions.remove(&user_id); + metadata.users_can_poll_all_streams.remove(&user_id); + metadata.users_can_send_all_streams.remove(&user_id); + metadata + .users_stream_permissions + .retain(|(uid, _), _| *uid != user_id); + metadata + .users_can_poll_stream + .retain(|(uid, _)| *uid != user_id); + metadata + .users_can_send_stream + .retain(|(uid, _)| *uid != user_id); +} + +fn clear_stream_permission_indexes(metadata: &mut InnerMetadata, stream_id: StreamId) { + metadata + .users_stream_permissions + .retain(|(_, sid), _| *sid != stream_id); + metadata + .users_can_poll_stream + .retain(|(_, sid)| *sid != stream_id); + metadata + .users_can_send_stream + .retain(|(_, sid)| *sid != stream_id); +} + +fn update_permission_indexes( + metadata: &mut InnerMetadata, + user_id: UserId, + permissions: Option<&Permissions>, +) { + clear_permission_indexes(metadata, user_id); + + let Some(permissions) = permissions else { + return; + }; + + if permissions.global.poll_messages { + metadata.users_can_poll_all_streams.insert(user_id); + } + + if permissions.global.send_messages { + metadata.users_can_send_all_streams.insert(user_id); + } + + metadata + .users_global_permissions + .insert(user_id, permissions.global.clone()); + + let Some(streams) = &permissions.streams else { + return; + }; + + for (stream_id, stream_perm) in streams { + if stream_perm.poll_messages { + metadata.users_can_poll_stream.insert((user_id, *stream_id)); + } + + if stream_perm.send_messages { + metadata.users_can_send_stream.insert((user_id, *stream_id)); + } + + metadata + .users_stream_permissions + .insert((user_id, *stream_id), stream_perm.clone()); + } +} + +fn rebuild_all_permission_indexes(metadata: &mut InnerMetadata) { + metadata.users_global_permissions.clear(); + metadata.users_stream_permissions.clear(); + metadata.users_can_poll_all_streams.clear(); + metadata.users_can_send_all_streams.clear(); + metadata.users_can_poll_stream.clear(); + metadata.users_can_send_stream.clear(); + + let user_permissions: Vec<_> = metadata + .users + .iter() + .map(|(_, user)| (user.id, user.permissions.clone())) + .collect(); + + for (user_id, permissions) in user_permissions { + update_permission_indexes(metadata, user_id, permissions.as_deref()); + } +} diff --git a/core/server/src/metadata/consumer_group.rs b/core/server/src/metadata/consumer_group.rs new file mode 100644 index 0000000000..c619939c6f --- /dev/null +++ b/core/server/src/metadata/consumer_group.rs @@ -0,0 +1,315 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::metadata::consumer_group_member::{ + CompletableRevocation, ConsumerGroupMemberMeta, PendingRevocation, +}; +use crate::metadata::partition::PartitionMeta; +use crate::metadata::{ConsumerGroupId, PartitionId}; +use crate::streaming::polling_consumer::ConsumerGroupId as CgId; +use iggy_common::IggyTimestamp; +use slab::Slab; +use std::collections::HashSet; +use std::sync::Arc; +use std::sync::atomic::Ordering; +use tracing::warn; + +#[derive(Clone, Debug)] +pub struct ConsumerGroupMeta { + pub id: ConsumerGroupId, + pub name: Arc, + pub partitions: Vec, + pub members: Slab, +} + +impl ConsumerGroupMeta { + /// Full rebalance: clear all assignments and redistribute round-robin. + /// Used when a member leaves or partition count changes. + pub fn rebalance_members(&mut self) { + let partition_count = self.partitions.len(); + let member_count = self.members.len(); + + if member_count == 0 || partition_count == 0 { + return; + } + + // Clear all member partitions and pending revocations + let member_ids: Vec = self.members.iter().map(|(id, _)| id).collect(); + for &member_id in &member_ids { + if let Some(member) = self.members.get_mut(member_id) { + member.partitions.clear(); + member.pending_revocations.clear(); + } + } + + // Rebuild assignments (round-robin) + for (i, &partition_id) in self.partitions.iter().enumerate() { + let member_idx = i % member_count; + if let Some(&member_id) = member_ids.get(member_idx) + && let Some(member) = self.members.get_mut(member_id) + { + member.partitions.push(partition_id); + } + } + } + + /// Cooperative rebalance: assign unassigned partitions to idle members and + /// mark excess partitions on over-assigned members as pending revocation. + pub fn rebalance_cooperative(&mut self) { + let member_count = self.members.len(); + if member_count == 0 || self.partitions.is_empty() { + return; + } + + // Find which partitions are already assigned + let mut assigned: HashSet = HashSet::new(); + for (_, member) in self.members.iter() { + for &pid in &member.partitions { + assigned.insert(pid); + } + } + + // Step 1: Assign unassigned partitions to idle members + let unassigned: Vec = self + .partitions + .iter() + .copied() + .filter(|pid| !assigned.contains(pid)) + .collect(); + + if !unassigned.is_empty() { + let idle_member_ids: Vec = self + .members + .iter() + .filter(|(_, m)| m.partitions.is_empty()) + .map(|(id, _)| id) + .collect(); + + if !idle_member_ids.is_empty() { + for (i, partition_id) in unassigned.into_iter().enumerate() { + let member_idx = i % idle_member_ids.len(); + if let Some(member) = self.members.get_mut(idle_member_ids[member_idx]) { + member.partitions.push(partition_id); + } + } + } + } + + // Step 2: Mark excess partitions as pending revocation + let partition_count = self.partitions.len(); + let fair_share = partition_count / member_count; + let remainder = partition_count % member_count; + + // Collect members already targeted by a pending revocation + let revocation_targets: HashSet = self + .members + .iter() + .flat_map(|(_, m)| { + m.pending_revocations + .iter() + .map(|revocation| revocation.target_slab_id) + }) + .collect(); + + // Collect idle members (no partitions and not already a revocation target) + let idle_slab_ids: Vec = self + .members + .iter() + .filter(|(id, m)| m.partitions.is_empty() && !revocation_targets.contains(id)) + .map(|(id, _)| id) + .collect(); + + if idle_slab_ids.is_empty() { + return; + } + + // Two-pass distribution: we first collect ALL excess partitions, then distribute + // them round-robin. This ensures even distribution when one member holds many + // partitions and multiple idle members join. Without this, the first idle member + // would receive all excess partitions while others starve. + // + // Example: 16 partitions held by 1 member, 15 idle members join + // Single-pass: member1 gets 15, members 2-15 get 0-1 each (unbalanced) + // Two-pass: each of 16 members gets exactly 1 partition + + // Pass 1: Collect excess partitions from over-assigned members + let member_ids: Vec = self.members.iter().map(|(id, _)| id).collect(); + let mut members_with_remainder = remainder; + let mut all_excess: Vec<(PartitionId, usize)> = Vec::new(); + + for &mid in &member_ids { + let Some(member) = self.members.get(mid) else { + continue; + }; + + let pending: HashSet = member + .pending_revocations + .iter() + .map(|revocation| revocation.partition_id) + .collect(); + let effective_count = member + .partitions + .iter() + .filter(|p| !pending.contains(p)) + .count(); + + let max_allowed = if members_with_remainder > 0 { + fair_share + 1 + } else { + fair_share + }; + + if effective_count <= max_allowed { + if effective_count > fair_share { + members_with_remainder = members_with_remainder.saturating_sub(1); + } + continue; + } + + let excess_count = effective_count - max_allowed; + if members_with_remainder > 0 && effective_count > fair_share { + members_with_remainder = members_with_remainder.saturating_sub(1); + } + + let revocable: Vec = member + .partitions + .iter() + .rev() + .filter(|p| !pending.contains(p)) + .copied() + .collect(); + + for partition_id in revocable.into_iter().take(excess_count) { + all_excess.push((partition_id, mid)); + } + } + + // Pass 2: Distribute collected partitions round-robin across idle members + let now = IggyTimestamp::now().as_micros(); + for (i, (partition_id, source_mid)) in all_excess.into_iter().enumerate() { + // Modulo ensures partitions cycle through idle members evenly + let idle_id = idle_slab_ids[i % idle_slab_ids.len()]; + let target_member_id = self + .members + .get(idle_id) + .map(|m| m.id) + .unwrap_or(usize::MAX); + if let Some(member) = self.members.get_mut(source_mid) { + member.pending_revocations.push(PendingRevocation { + partition_id, + target_slab_id: idle_id, + target_member_id, + created_at_micros: now, + }); + } + } + } + + /// Find revocations completable immediately (never polled or already committed). + pub fn find_completable_revocations( + &self, + partitions: &[PartitionMeta], + cg_id: CgId, + ) -> Vec { + let mut result = Vec::new(); + for (slab_id, member) in self.members.iter() { + for revocation in &member.pending_revocations { + let partition = match partitions.get(revocation.partition_id) { + Some(p) => p, + None => continue, + }; + let last_polled = { + let guard = partition.last_polled_offsets.pin(); + guard.get(&cg_id).map(|v| v.load(Ordering::Acquire)) + }; + let can_complete = match last_polled { + None => true, + Some(polled) => { + let offsets_guard = partition.consumer_group_offsets.pin(); + offsets_guard + .get(&cg_id) + .map(|offset| offset.offset.load(Ordering::Acquire)) + .is_some_and(|committed| committed >= polled) + } + }; + if can_complete { + result.push(CompletableRevocation { + slab_id, + member_id: member.id, + partition_id: revocation.partition_id, + }); + } + } + } + result + } + + /// Complete a pending revocation, moving the partition to the target member. + pub fn complete_revocation( + &mut self, + member_slab_id: usize, + member_id: usize, + partition_id: PartitionId, + ) -> bool { + let target_info = if let Some(member) = self.members.get_mut(member_slab_id) { + if member.id != member_id { + warn!( + "Revocation rejected: member ID mismatch (slab={member_slab_id}, expected={member_id}, actual={})", + member.id + ); + return false; + } + let pos = member + .pending_revocations + .iter() + .position(|revocation| revocation.partition_id == partition_id); + if let Some(pos) = pos { + let removed = member.pending_revocations.remove(pos); + member.partitions.retain(|&p| p != partition_id); + Some((removed.target_slab_id, removed.target_member_id)) + } else { + warn!( + "Revocation rejected: no pending revocation for partition={partition_id} on slab={member_slab_id}" + ); + None + } + } else { + warn!("Revocation rejected: source slab={member_slab_id} not found"); + None + }; + + if let Some((target_slab, expected_target_id)) = target_info { + if let Some(target_member) = self.members.get_mut(target_slab) { + if target_member.id != expected_target_id { + warn!( + "Revocation target slab={target_slab} reused (expected={expected_target_id}, actual={}), full rebalance", + target_member.id + ); + self.rebalance_members(); + return true; + } + target_member.partitions.push(partition_id); + return true; + } + warn!("Revocation target slab={target_slab} gone, full rebalance"); + self.rebalance_members(); + return true; + } + + false + } +} diff --git a/core/server/src/metadata/consumer_group_member.rs b/core/server/src/metadata/consumer_group_member.rs new file mode 100644 index 0000000000..7605de0cc7 --- /dev/null +++ b/core/server/src/metadata/consumer_group_member.rs @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::metadata::{ClientId, ConsumerGroupMemberId, PartitionId}; +use std::sync::Arc; +use std::sync::atomic::AtomicUsize; + +/// A partition pending cooperative revocation from this member to a target member. +#[derive(Clone, Debug)] +pub struct PendingRevocation { + pub partition_id: PartitionId, + pub target_slab_id: usize, + pub target_member_id: usize, + pub created_at_micros: u64, +} + +/// A revocation that can be completed immediately (never polled or already committed). +#[derive(Clone, Debug)] +pub struct CompletableRevocation { + pub slab_id: usize, + pub member_id: usize, + pub partition_id: PartitionId, +} + +#[derive(Clone, Debug)] +pub struct ConsumerGroupMemberMeta { + pub id: ConsumerGroupMemberId, + pub client_id: ClientId, + pub partitions: Vec, + pub partition_index: Arc, + pub pending_revocations: Vec, +} + +impl ConsumerGroupMemberMeta { + pub fn new(id: ConsumerGroupMemberId, client_id: ClientId) -> Self { + Self { + id, + client_id, + partitions: Vec::new(), + partition_index: Arc::new(AtomicUsize::new(0)), + pending_revocations: Vec::new(), + } + } +} diff --git a/core/server/src/metadata/inner.rs b/core/server/src/metadata/inner.rs new file mode 100644 index 0000000000..4bcc165f54 --- /dev/null +++ b/core/server/src/metadata/inner.rs @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::metadata::{StreamId, StreamMeta, UserId, UserMeta}; +use ahash::{AHashMap, AHashSet}; +use iggy_common::{GlobalPermissions, PersonalAccessToken, StreamPermissions}; +use slab::Slab; +use std::sync::Arc; + +#[derive(Clone, Default)] +pub struct InnerMetadata { + /// Streams indexed by StreamId (slab-assigned) + pub streams: Slab, + + /// Users indexed by UserId (slab-assigned) + pub users: Slab, + + /// Forward indexes (name → ID) + pub stream_index: AHashMap, StreamId>, + pub user_index: AHashMap, UserId>, + + /// user_id -> (token_hash -> PAT) + pub personal_access_tokens: AHashMap, PersonalAccessToken>>, + + // Permission indexes (auto-maintained by absorb) + pub users_global_permissions: AHashMap, + pub users_stream_permissions: AHashMap<(UserId, StreamId), StreamPermissions>, + + // Hot-path optimizations for message send/poll + pub users_can_poll_all_streams: AHashSet, + pub users_can_send_all_streams: AHashSet, + pub users_can_poll_stream: AHashSet<(UserId, StreamId)>, + pub users_can_send_stream: AHashSet<(UserId, StreamId)>, +} + +impl InnerMetadata { + pub fn new() -> Self { + Self::default() + } +} diff --git a/core/server/src/metadata/mod.rs b/core/server/src/metadata/mod.rs new file mode 100644 index 0000000000..6e557c9ddc --- /dev/null +++ b/core/server/src/metadata/mod.rs @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Shared metadata module providing a single source of truth for all shards. +//! +//! This module provides a `LeftRight`-based approach where all shards read from +//! a shared snapshot, and only shard 0 can write. +//! +//! # Architecture +//! +//! - `InnerMetadata` (inner.rs): Immutable snapshot with all metadata +//! - `Metadata` (reader.rs): Thread-safe read handle for querying metadata +//! - Entity types: `StreamMeta`, `TopicMeta`, `PartitionMeta`, `UserMeta`, `ConsumerGroupMeta` +//! - Consumer offsets are stored in `PartitionMeta` for cross-shard visibility + +mod absorb; +mod consumer_group; +mod consumer_group_member; +mod inner; +pub mod ops; +mod partition; +mod reader; +mod stream; +mod topic; +mod user; +mod writer; + +pub use consumer_group::ConsumerGroupMeta; +pub use consumer_group_member::ConsumerGroupMemberMeta; +pub use inner::InnerMetadata; +pub use ops::MetadataOp; +pub use partition::PartitionMeta; +pub use reader::{Metadata, PartitionInitInfo}; +pub(crate) use reader::{ + resolve_consumer_group_id_inner, resolve_stream_id_inner, resolve_topic_id_inner, +}; +pub use stream::StreamMeta; +pub use topic::TopicMeta; +pub use user::UserMeta; +pub use writer::MetadataWriter; + +pub type MetadataReadHandle = left_right::ReadHandle; +pub type StreamId = usize; +pub type TopicId = usize; +pub type PartitionId = usize; +pub type UserId = u32; +pub type ClientId = u32; +pub type ConsumerGroupId = usize; +pub type ConsumerGroupMemberId = usize; +pub type ConsumerGroupKey = (StreamId, TopicId, ConsumerGroupId); + +pub fn create_metadata_handles() -> (MetadataWriter, MetadataReadHandle) { + let (write_handle, read_handle) = left_right::new::(); + let mut writer = MetadataWriter::new(write_handle); + writer.publish(); + (writer, read_handle) +} diff --git a/core/server/src/metadata/ops.rs b/core/server/src/metadata/ops.rs new file mode 100644 index 0000000000..a6d80c2376 --- /dev/null +++ b/core/server/src/metadata/ops.rs @@ -0,0 +1,134 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::metadata::consumer_group_member::CompletableRevocation; +use crate::metadata::inner::InnerMetadata; +use crate::metadata::{ + ConsumerGroupId, ConsumerGroupMeta, PartitionId, PartitionMeta, StreamId, StreamMeta, TopicId, + TopicMeta, UserId, UserMeta, +}; +use iggy_common::{CompressionAlgorithm, IggyExpiry, MaxTopicSize, PersonalAccessToken}; +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::atomic::AtomicUsize; + +#[derive(Clone)] +pub enum MetadataOp { + Initialize(Box), + + AddStream { + meta: StreamMeta, + assigned_id: Arc, + }, + UpdateStream { + id: StreamId, + new_name: Arc, + }, + DeleteStream { + id: StreamId, + }, + AddTopic { + stream_id: StreamId, + meta: TopicMeta, + assigned_id: Arc, + }, + UpdateTopic { + stream_id: StreamId, + topic_id: TopicId, + new_name: Arc, + message_expiry: IggyExpiry, + compression_algorithm: CompressionAlgorithm, + max_topic_size: MaxTopicSize, + replication_factor: u8, + }, + DeleteTopic { + stream_id: StreamId, + topic_id: TopicId, + }, + AddPartitions { + stream_id: StreamId, + topic_id: TopicId, + partitions: Vec, + revision_id: u64, + }, + DeletePartitions { + stream_id: StreamId, + topic_id: TopicId, + count: u32, + }, + AddUser { + meta: UserMeta, + assigned_id: Arc, + }, + UpdateUserMeta { + id: UserId, + meta: UserMeta, + }, + DeleteUser { + id: UserId, + }, + + AddPersonalAccessToken { + user_id: UserId, + pat: PersonalAccessToken, + }, + DeletePersonalAccessToken { + user_id: UserId, + token_hash: Arc, + }, + AddConsumerGroup { + stream_id: StreamId, + topic_id: TopicId, + meta: ConsumerGroupMeta, + assigned_id: Arc, + }, + DeleteConsumerGroup { + stream_id: StreamId, + topic_id: TopicId, + group_id: ConsumerGroupId, + }, + JoinConsumerGroup { + stream_id: StreamId, + topic_id: TopicId, + group_id: ConsumerGroupId, + client_id: u32, + member_id: Arc, + valid_client_ids: Option>, + completable_revocations: Arc>>, + }, + LeaveConsumerGroup { + stream_id: StreamId, + topic_id: TopicId, + group_id: ConsumerGroupId, + client_id: u32, + removed_member_id: Arc, + }, + RebalanceConsumerGroupsForTopic { + stream_id: StreamId, + topic_id: TopicId, + partitions_count: u32, + }, + CompletePartitionRevocation { + stream_id: StreamId, + topic_id: TopicId, + group_id: ConsumerGroupId, + member_slab_id: usize, + member_id: usize, + partition_id: PartitionId, + timed_out: bool, + }, +} diff --git a/core/server/src/metadata/partition.rs b/core/server/src/metadata/partition.rs new file mode 100644 index 0000000000..97cb9c7934 --- /dev/null +++ b/core/server/src/metadata/partition.rs @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::metadata::PartitionId; +use crate::streaming::partitions::consumer_group_offsets::ConsumerGroupOffsets; +use crate::streaming::partitions::consumer_offsets::ConsumerOffsets; +use crate::streaming::polling_consumer::ConsumerGroupId; +use crate::streaming::stats::PartitionStats; +use iggy_common::IggyTimestamp; +use std::sync::Arc; +use std::sync::atomic::AtomicU64; + +#[derive(Clone, Debug)] +pub struct PartitionMeta { + pub id: PartitionId, + pub created_at: IggyTimestamp, + /// Monotonically increasing version to detect stale local_partitions entries. + /// Set to the Metadata version when the partition was created. + pub revision_id: u64, + pub stats: Arc, + pub consumer_offsets: Arc, + pub consumer_group_offsets: Arc, + /// Last offset polled by each consumer group from this partition. + pub last_polled_offsets: Arc>>, +} diff --git a/core/server/src/metadata/reader.rs b/core/server/src/metadata/reader.rs new file mode 100644 index 0000000000..fa284777a1 --- /dev/null +++ b/core/server/src/metadata/reader.rs @@ -0,0 +1,1920 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::metadata::{ + ConsumerGroupId, ConsumerGroupMeta, InnerMetadata, MetadataReadHandle, PartitionId, + PartitionMeta, StreamId, StreamMeta, TopicId, TopicMeta, UserId, UserMeta, +}; +use crate::shard::transmission::message::{ResolvedPartition, ResolvedTopic}; +use crate::streaming::partitions::consumer_group_offsets::ConsumerGroupOffsets; +use crate::streaming::partitions::consumer_offsets::ConsumerOffsets; +use crate::streaming::polling_consumer::PollingConsumer; +use crate::streaming::stats::{PartitionStats, StreamStats, TopicStats}; +use iggy_common::{ + IdKind, Identifier, IggyError, IggyExpiry, IggyTimestamp, MaxTopicSize, PersonalAccessToken, +}; +use left_right::ReadGuard; +use server_common::sharding::IggyNamespace; +use std::sync::Arc; +use std::sync::atomic::Ordering; + +/// Thread-safe wrapper for GlobalMetadata using left-right for lock-free reads. +/// Uses hierarchical structure: streams contain topics, topics contain partitions and consumer groups. +/// All mutations go through MetadataWriter (shard 0 only). +/// +/// Each shard should own its own `Metadata` instance (cloned from a common source). +/// The underlying data is shared via left-right's internal mechanism. +#[derive(Clone)] +pub struct Metadata { + inner: MetadataReadHandle, +} + +impl Metadata { + pub fn new(reader: MetadataReadHandle) -> Self { + Self { inner: reader } + } + + #[inline] + pub(super) fn load(&self) -> ReadGuard<'_, InnerMetadata> { + self.inner + .enter() + .expect("metadata not initialized - writer must publish before reads") + } + + pub fn get_stream_id(&self, identifier: &Identifier) -> Option { + let metadata = self.load(); + match identifier.kind { + IdKind::Numeric => { + let stream_id = identifier.get_u32_value().ok()? as StreamId; + if metadata.streams.get(stream_id).is_some() { + Some(stream_id) + } else { + None + } + } + IdKind::String => { + let name = identifier.get_cow_str_value().ok()?; + metadata.stream_index.get(name.as_ref()).copied() + } + } + } + + pub fn stream_name_exists(&self, name: &str) -> bool { + self.load().stream_index.contains_key(name) + } + + pub fn get_topic_id(&self, stream_id: StreamId, identifier: &Identifier) -> Option { + let metadata = self.load(); + let stream = metadata.streams.get(stream_id)?; + + match identifier.kind { + IdKind::Numeric => { + let topic_id = identifier.get_u32_value().ok()? as TopicId; + if stream.topics.get(topic_id).is_some() { + Some(topic_id) + } else { + None + } + } + IdKind::String => { + let name = identifier.get_cow_str_value().ok()?; + stream.topic_index.get(&Arc::from(name.as_ref())).copied() + } + } + } + + pub fn get_user_id(&self, identifier: &Identifier) -> Option { + let metadata = self.load(); + match identifier.kind { + IdKind::Numeric => Some(identifier.get_u32_value().ok()? as UserId), + IdKind::String => { + let name = identifier.get_cow_str_value().ok()?; + metadata.user_index.get(name.as_ref()).copied() + } + } + } + + pub fn get_consumer_group_id( + &self, + stream_id: StreamId, + topic_id: TopicId, + identifier: &Identifier, + ) -> Option { + let metadata = self.load(); + let stream = metadata.streams.get(stream_id)?; + let topic = stream.topics.get(topic_id)?; + + match identifier.kind { + IdKind::Numeric => { + let group_id = identifier.get_u32_value().ok()? as ConsumerGroupId; + if topic.consumer_groups.get(group_id).is_some() { + Some(group_id) + } else { + None + } + } + IdKind::String => { + let name = identifier.get_cow_str_value().ok()?; + topic + .consumer_group_index + .get(&Arc::from(name.as_ref())) + .copied() + } + } + } + + pub fn stream_exists(&self, id: StreamId) -> bool { + self.load().streams.get(id).is_some() + } + + pub fn topic_exists(&self, stream_id: StreamId, topic_id: TopicId) -> bool { + self.load() + .streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .is_some() + } + + pub fn partition_exists( + &self, + stream_id: StreamId, + topic_id: TopicId, + partition_id: PartitionId, + ) -> bool { + self.load() + .streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .and_then(|t| t.partitions.get(partition_id)) + .is_some() + } + + pub fn user_exists(&self, id: UserId) -> bool { + self.load().users.get(id as usize).is_some() + } + + pub fn consumer_group_exists( + &self, + stream_id: StreamId, + topic_id: TopicId, + group_id: ConsumerGroupId, + ) -> bool { + self.load() + .streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .and_then(|t| t.consumer_groups.get(group_id)) + .is_some() + } + + pub fn consumer_group_exists_by_name( + &self, + stream_id: StreamId, + topic_id: TopicId, + name: &str, + ) -> bool { + self.load() + .streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .map(|t| t.consumer_group_index.contains_key(name)) + .unwrap_or(false) + } + + pub fn streams_count(&self) -> usize { + self.load().streams.len() + } + + pub fn next_stream_id(&self) -> usize { + self.load().streams.vacant_key() + } + + pub fn topics_count(&self, stream_id: StreamId) -> usize { + self.load() + .streams + .get(stream_id) + .map(|s| s.topics.len()) + .unwrap_or(0) + } + + pub fn next_topic_id(&self, stream_id: StreamId) -> Option { + self.load() + .streams + .get(stream_id) + .map(|s| s.topics.vacant_key()) + } + + pub fn partitions_count(&self, stream_id: StreamId, topic_id: TopicId) -> usize { + self.load() + .streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .map(|t| t.partitions.len()) + .unwrap_or(0) + } + + pub fn get_partitions_count(&self, stream_id: StreamId, topic_id: TopicId) -> Option { + self.load() + .streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .map(|t| t.partitions.len()) + } + + pub fn get_next_partition_id(&self, stream_id: StreamId, topic_id: TopicId) -> Option { + let metadata = self.load(); + let topic = metadata.streams.get(stream_id)?.topics.get(topic_id)?; + let partitions_count = topic.partitions.len(); + + if partitions_count == 0 { + return None; + } + + let counter = &topic.round_robin_counter; + let current = counter + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |c| { + Some((c + 1) % partitions_count) + }) + .unwrap(); + Some(current % partitions_count) + } + + /// Resolve consumer group partition under a single metadata read guard. + pub fn resolve_consumer_group_partition( + &self, + stream_id: StreamId, + topic_id: TopicId, + group_identifier: &Identifier, + client_id: u32, + explicit_partition_id: Option, + calculate_partition_id: bool, + ) -> Result, IggyError> { + let metadata = self.load(); + + // Step 1: Resolve group ID + let group_id = { + let stream = metadata.streams.get(stream_id).ok_or_else(|| { + IggyError::ConsumerGroupIdNotFound( + group_identifier.clone(), + Identifier::numeric(topic_id as u32).unwrap(), + ) + })?; + let topic = stream.topics.get(topic_id).ok_or_else(|| { + IggyError::ConsumerGroupIdNotFound( + group_identifier.clone(), + Identifier::numeric(topic_id as u32).unwrap(), + ) + })?; + match group_identifier.kind { + IdKind::Numeric => { + let gid = group_identifier.get_u32_value().map_err(|_| { + IggyError::ConsumerGroupIdNotFound( + group_identifier.clone(), + Identifier::numeric(topic_id as u32).unwrap(), + ) + })? as ConsumerGroupId; + if topic.consumer_groups.get(gid).is_none() { + return Err(IggyError::ConsumerGroupIdNotFound( + group_identifier.clone(), + Identifier::numeric(topic_id as u32).unwrap(), + )); + } + gid + } + IdKind::String => { + let name = group_identifier.get_cow_str_value().map_err(|_| { + IggyError::ConsumerGroupIdNotFound( + group_identifier.clone(), + Identifier::numeric(topic_id as u32).unwrap(), + ) + })?; + *topic + .consumer_group_index + .get(name.as_ref()) + .ok_or_else(|| { + IggyError::ConsumerGroupIdNotFound( + group_identifier.clone(), + Identifier::numeric(topic_id as u32).unwrap(), + ) + })? + } + } + }; + + // Step 2: Find member by client_id (same read guard, same metadata snapshot) + let group = metadata + .streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .and_then(|t| t.consumer_groups.get(group_id)) + .ok_or_else(|| { + IggyError::ConsumerGroupIdNotFound( + group_identifier.clone(), + Identifier::numeric(topic_id as u32).unwrap(), + ) + })?; + + let (member_slab_id, member) = group + .members + .iter() + .find(|(_, m)| m.client_id == client_id) + .ok_or_else(|| { + IggyError::ConsumerGroupMemberNotFound( + client_id, + group_identifier.clone(), + Identifier::numeric(topic_id as u32).unwrap(), + ) + })?; + + // Step 3a: If explicit partition_id provided, validate member owns it + if let Some(pid) = explicit_partition_id { + let pid_usize = pid as usize; + if !member.partitions.contains(&pid_usize) { + // Member doesn't own this partition — check if it's pending revocation + let is_pending = member + .pending_revocations + .iter() + .any(|revocation| revocation.partition_id == pid_usize); + if !is_pending { + return Ok(None); + } + } + return Ok(Some(( + PollingConsumer::consumer_group(group_id, member_slab_id), + pid_usize, + ))); + } + + // Step 3b: Round-robin partition selection (same snapshot, no race) + if member.pending_revocations.is_empty() { + // Fast path + let partitions = &member.partitions; + let count = partitions.len(); + if count == 0 { + return Ok(None); + } + let counter = &member.partition_index; + let partition_id = if calculate_partition_id { + let current = counter + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |c| { + Some((c + 1) % count) + }) + .unwrap(); + partitions[current % count] + } else { + let current = counter.load(Ordering::Relaxed); + partitions[current % count] + }; + return Ok(Some(( + PollingConsumer::consumer_group(group_id, member_slab_id), + partition_id, + ))); + } + + // Slow path: skip revoked partitions + let effective_count = member.partitions.len() - member.pending_revocations.len(); + if effective_count == 0 { + return Ok(None); + } + + let counter = &member.partition_index; + let idx = if calculate_partition_id { + counter + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |c| { + Some((c + 1) % effective_count) + }) + .unwrap() + % effective_count + } else { + counter.load(Ordering::Relaxed) % effective_count + }; + + let mut seen = 0; + for &pid in &member.partitions { + let is_revoked = member + .pending_revocations + .iter() + .any(|revocation| revocation.partition_id == pid); + if is_revoked { + continue; + } + if seen == idx { + return Ok(Some(( + PollingConsumer::consumer_group(group_id, member_slab_id), + pid, + ))); + } + seen += 1; + } + + Ok(None) + } + + /// Record the last offset returned to a CG member during poll. + pub fn record_polled_offset( + &self, + stream_id: StreamId, + topic_id: TopicId, + group_id: ConsumerGroupId, + partition_id: PartitionId, + offset: u64, + ) { + use crate::streaming::polling_consumer::ConsumerGroupId as CgIdNewtype; + + let metadata = self.load(); + if let Some(partition) = metadata + .streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .and_then(|t| t.partitions.get(partition_id)) + { + let key = CgIdNewtype(group_id); + let guard = partition.last_polled_offsets.pin(); + match guard.get(&key) { + Some(existing) => { + existing.store(offset, Ordering::Release); + } + None => { + guard.insert(key, Arc::new(std::sync::atomic::AtomicU64::new(offset))); + } + } + } + } + + pub fn users_count(&self) -> usize { + self.load().users.len() + } + + pub fn username_exists(&self, username: &str) -> bool { + self.load().user_index.contains_key(username) + } + + pub fn consumer_groups_count(&self, stream_id: StreamId, topic_id: TopicId) -> usize { + self.load() + .streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .map(|t| t.consumer_groups.len()) + .unwrap_or(0) + } + + pub fn get_stream_stats(&self, id: StreamId) -> Option> { + self.load().streams.get(id).map(|s| s.stats.clone()) + } + + pub fn get_topic_stats( + &self, + stream_id: StreamId, + topic_id: TopicId, + ) -> Option> { + self.load() + .streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .map(|t| t.stats.clone()) + } + + pub fn get_partition_stats(&self, ns: &IggyNamespace) -> Option> { + self.load() + .streams + .get(ns.stream_id()) + .and_then(|s| s.topics.get(ns.topic_id())) + .and_then(|t| t.partitions.get(ns.partition_id())) + .map(|p| p.stats.clone()) + } + + pub fn get_partition_stats_by_ids( + &self, + stream_id: StreamId, + topic_id: TopicId, + partition_id: PartitionId, + ) -> Option> { + self.load() + .streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .and_then(|t| t.partitions.get(partition_id)) + .map(|p| p.stats.clone()) + } + + pub fn get_partition_consumer_offsets( + &self, + stream_id: StreamId, + topic_id: TopicId, + partition_id: PartitionId, + ) -> Option> { + self.load() + .streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .and_then(|t| t.partitions.get(partition_id)) + .map(|p| p.consumer_offsets.clone()) + } + + pub fn get_partition_consumer_group_offsets( + &self, + stream_id: StreamId, + topic_id: TopicId, + partition_id: PartitionId, + ) -> Option> { + self.load() + .streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .and_then(|t| t.partitions.get(partition_id)) + .map(|p| p.consumer_group_offsets.clone()) + } + + pub fn get_user(&self, id: UserId) -> Option { + self.load().users.get(id as usize).cloned() + } + + pub fn get_all_users(&self) -> Vec { + self.load().users.iter().map(|(_, u)| u.clone()).collect() + } + + pub fn get_stream(&self, id: StreamId) -> Option { + self.load().streams.get(id).cloned() + } + + pub fn get_topic(&self, stream_id: StreamId, topic_id: TopicId) -> Option { + self.load() + .streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id).cloned()) + } + + pub fn get_partition( + &self, + stream_id: StreamId, + topic_id: TopicId, + partition_id: PartitionId, + ) -> Option { + self.load() + .streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .and_then(|t| t.partitions.get(partition_id).cloned()) + } + + pub fn get_consumer_group( + &self, + stream_id: StreamId, + topic_id: TopicId, + group_id: ConsumerGroupId, + ) -> Option { + self.load() + .streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .and_then(|t| t.consumer_groups.get(group_id).cloned()) + } + + pub fn get_user_personal_access_tokens(&self, user_id: UserId) -> Vec { + self.load() + .personal_access_tokens + .get(&user_id) + .map(|pats| pats.values().cloned().collect()) + .unwrap_or_default() + } + + pub fn get_personal_access_token_by_hash( + &self, + token_hash: &str, + ) -> Option { + let token_hash_arc: Arc = Arc::from(token_hash); + let metadata = self.load(); + for user_pats in metadata.personal_access_tokens.values() { + if let Some(pat) = user_pats.get(&token_hash_arc) { + return Some(pat.clone()); + } + } + None + } + + pub fn user_pat_count(&self, user_id: UserId) -> usize { + self.load() + .personal_access_tokens + .get(&user_id) + .map(|pats| pats.len()) + .unwrap_or(0) + } + + pub fn user_has_pat_with_name(&self, user_id: UserId, name: &str) -> bool { + self.load() + .personal_access_tokens + .get(&user_id) + .map(|pats| pats.values().any(|pat| &*pat.name == name)) + .unwrap_or(false) + } + + pub fn find_pat_token_hash_by_name(&self, user_id: UserId, name: &str) -> Option> { + self.load() + .personal_access_tokens + .get(&user_id) + .and_then(|pats| { + pats.iter() + .find(|(_, pat)| &*pat.name == name) + .map(|(hash, _)| hash.clone()) + }) + } + + pub fn is_consumer_group_member( + &self, + stream_id: StreamId, + topic_id: TopicId, + group_id: ConsumerGroupId, + client_id: u32, + ) -> bool { + let metadata = self.load(); + metadata + .streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .and_then(|t| t.consumer_groups.get(group_id)) + .map(|g| g.members.iter().any(|(_, m)| m.client_id == client_id)) + .unwrap_or(false) + } + + /// Execute a closure with read access to the metadata snapshot. + /// This is the safe way to perform complex read operations that need + /// atomic access to multiple metadata fields. + /// + /// The closure receives an immutable reference to the metadata and must + /// return owned data (not references). This ensures the ReadGuard is + /// dropped before any async operations can occur. + #[inline] + pub fn with_metadata(&self, f: F) -> R + where + F: FnOnce(&InnerMetadata) -> R, + { + let guard = self.load(); + f(&guard) + } + + /// Get all partition IDs for a topic, sorted. + pub fn get_partition_ids(&self, stream_id: StreamId, topic_id: TopicId) -> Vec { + self.with_metadata(|m| { + m.streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .map(|t| { + let mut ids: Vec<_> = t.partitions.iter().enumerate().map(|(k, _)| k).collect(); + ids.sort_unstable(); + ids + }) + .unwrap_or_default() + }) + } + + /// Get all topic IDs for a stream, sorted. + pub fn get_topic_ids(&self, stream_id: StreamId) -> Vec { + self.with_metadata(|m| { + m.streams + .get(stream_id) + .map(|s| { + let mut ids: Vec<_> = s.topics.iter().map(|(k, _)| k).collect(); + ids.sort_unstable(); + ids + }) + .unwrap_or_default() + }) + } + + /// Get all stream IDs, sorted. + pub fn get_stream_ids(&self) -> Vec { + self.with_metadata(|m| { + let mut ids: Vec<_> = m.streams.iter().map(|(k, _)| k).collect(); + ids.sort_unstable(); + ids + }) + } + + /// Get all namespaces (stream/topic/partition combinations). + pub fn get_all_namespaces(&self) -> Vec { + self.with_metadata(|m| { + let mut namespaces = Vec::new(); + for (stream_id, stream) in m.streams.iter() { + for (topic_id, topic) in stream.topics.iter() { + for (partition_id, _) in topic.partitions.iter().enumerate() { + namespaces.push(IggyNamespace::new(stream_id, topic_id, partition_id)); + } + } + } + namespaces + }) + } + + /// Get topic configuration (message_expiry, max_topic_size). + pub fn get_topic_config( + &self, + stream_id: StreamId, + topic_id: TopicId, + ) -> Option<(IggyExpiry, MaxTopicSize)> { + self.with_metadata(|m| { + m.streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .map(|t| (t.message_expiry, t.max_topic_size)) + }) + } + + /// Get partition initialization info needed for LocalPartition setup. + pub fn get_partition_init_info( + &self, + stream_id: StreamId, + topic_id: TopicId, + partition_id: PartitionId, + ) -> Option { + self.with_metadata(|m| { + m.streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .and_then(|t| t.partitions.get(partition_id)) + .map(|p| PartitionInitInfo { + created_at: p.created_at, + revision_id: p.revision_id, + stats: p.stats.clone(), + consumer_offsets: p.consumer_offsets.clone(), + consumer_group_offsets: p.consumer_group_offsets.clone(), + }) + }) + } + + /// Get consumer group member ID for a client. + pub fn get_consumer_group_member_id( + &self, + stream_id: StreamId, + topic_id: TopicId, + group_id: ConsumerGroupId, + client_id: u32, + ) -> Option { + self.with_metadata(|m| { + m.streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .and_then(|t| t.consumer_groups.get(group_id)) + .and_then(|g| { + g.members + .iter() + .find(|(_, member)| member.client_id == client_id) + .map(|(id, _)| id) + }) + }) + } + + /// Get all consumer groups for a topic. + pub fn get_all_consumer_groups( + &self, + stream_id: StreamId, + topic_id: TopicId, + ) -> Vec { + self.with_metadata(|m| { + m.streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .map(|t| t.consumer_groups.iter().map(|(_, cg)| cg.clone()).collect()) + .unwrap_or_default() + }) + } + + /// Inheritance: manage_streams → read_streams → read_topics → poll_messages + pub fn perm_poll_messages( + &self, + user_id: u32, + stream_id: StreamId, + topic_id: TopicId, + ) -> Result<(), IggyError> { + let metadata = self.load(); + + if metadata.users_can_poll_all_streams.contains(&user_id) { + return Ok(()); + } + + if let Some(global) = metadata.users_global_permissions.get(&user_id) + && (global.read_topics + || global.manage_topics + || global.read_streams + || global.manage_streams) + { + return Ok(()); + } + + if metadata + .users_can_poll_stream + .contains(&(user_id, stream_id)) + { + return Ok(()); + } + + let Some(stream_permissions) = metadata.users_stream_permissions.get(&(user_id, stream_id)) + else { + return Err(IggyError::Unauthorized); + }; + + if stream_permissions.manage_stream || stream_permissions.read_stream { + return Ok(()); + } + + if stream_permissions.manage_topics || stream_permissions.read_topics { + return Ok(()); + } + + if stream_permissions.poll_messages { + return Ok(()); + } + + if let Some(topics) = &stream_permissions.topics + && let Some(topic_permissions) = topics.get(&topic_id) + && (topic_permissions.manage_topic + || topic_permissions.read_topic + || topic_permissions.poll_messages) + { + return Ok(()); + } + + Err(IggyError::Unauthorized) + } + + /// Inheritance: manage_streams → manage_topics → send_messages + pub fn perm_append_messages( + &self, + user_id: u32, + stream_id: StreamId, + topic_id: TopicId, + ) -> Result<(), IggyError> { + let metadata = self.load(); + + if metadata.users_can_send_all_streams.contains(&user_id) { + return Ok(()); + } + + if let Some(global) = metadata.users_global_permissions.get(&user_id) + && (global.manage_streams || global.manage_topics) + { + return Ok(()); + } + + if metadata + .users_can_send_stream + .contains(&(user_id, stream_id)) + { + return Ok(()); + } + + let Some(stream_permissions) = metadata.users_stream_permissions.get(&(user_id, stream_id)) + else { + return Err(IggyError::Unauthorized); + }; + + if stream_permissions.manage_stream || stream_permissions.manage_topics { + return Ok(()); + } + + if stream_permissions.send_messages { + return Ok(()); + } + + if let Some(topics) = &stream_permissions.topics + && let Some(topic_permissions) = topics.get(&topic_id) + && (topic_permissions.manage_topic || topic_permissions.send_messages) + { + return Ok(()); + } + + Err(IggyError::Unauthorized) + } + + pub fn perm_get_stream(&self, user_id: u32, stream_id: StreamId) -> Result<(), IggyError> { + let metadata = self.load(); + + if let Some(global_permissions) = metadata.users_global_permissions.get(&user_id) + && (global_permissions.manage_streams || global_permissions.read_streams) + { + return Ok(()); + } + + if let Some(stream_permissions) = + metadata.users_stream_permissions.get(&(user_id, stream_id)) + && (stream_permissions.manage_stream || stream_permissions.read_stream) + { + return Ok(()); + } + + Err(IggyError::Unauthorized) + } + + pub fn perm_get_streams(&self, user_id: u32) -> Result<(), IggyError> { + let metadata = self.load(); + + if let Some(global_permissions) = metadata.users_global_permissions.get(&user_id) + && (global_permissions.manage_streams || global_permissions.read_streams) + { + return Ok(()); + } + + Err(IggyError::Unauthorized) + } + + pub fn perm_create_stream(&self, user_id: u32) -> Result<(), IggyError> { + let metadata = self.load(); + + if let Some(global_permissions) = metadata.users_global_permissions.get(&user_id) + && global_permissions.manage_streams + { + return Ok(()); + } + + Err(IggyError::Unauthorized) + } + + pub fn perm_update_stream(&self, user_id: u32, stream_id: StreamId) -> Result<(), IggyError> { + self.perm_manage_stream(user_id, stream_id) + } + + pub fn perm_delete_stream(&self, user_id: u32, stream_id: StreamId) -> Result<(), IggyError> { + self.perm_manage_stream(user_id, stream_id) + } + + pub fn perm_purge_stream(&self, user_id: u32, stream_id: StreamId) -> Result<(), IggyError> { + self.perm_manage_stream(user_id, stream_id) + } + + fn perm_manage_stream(&self, user_id: u32, stream_id: StreamId) -> Result<(), IggyError> { + let metadata = self.load(); + + if let Some(global_permissions) = metadata.users_global_permissions.get(&user_id) + && global_permissions.manage_streams + { + return Ok(()); + } + + if let Some(stream_permissions) = + metadata.users_stream_permissions.get(&(user_id, stream_id)) + && stream_permissions.manage_stream + { + return Ok(()); + } + + Err(IggyError::Unauthorized) + } + + /// Inheritance: manage_streams → read_streams → read_topics + pub fn perm_get_topic( + &self, + user_id: u32, + stream_id: StreamId, + topic_id: TopicId, + ) -> Result<(), IggyError> { + let metadata = self.load(); + + if let Some(global) = metadata.users_global_permissions.get(&user_id) + && (global.read_streams + || global.manage_streams + || global.manage_topics + || global.read_topics) + { + return Ok(()); + } + + if let Some(stream_permissions) = + metadata.users_stream_permissions.get(&(user_id, stream_id)) + { + if stream_permissions.manage_stream + || stream_permissions.read_stream + || stream_permissions.manage_topics + || stream_permissions.read_topics + { + return Ok(()); + } + + if let Some(topics) = &stream_permissions.topics + && let Some(topic_permissions) = topics.get(&topic_id) + && (topic_permissions.manage_topic || topic_permissions.read_topic) + { + return Ok(()); + } + } + + Err(IggyError::Unauthorized) + } + + pub fn perm_get_topics(&self, user_id: u32, stream_id: StreamId) -> Result<(), IggyError> { + let metadata = self.load(); + + if let Some(global) = metadata.users_global_permissions.get(&user_id) + && (global.read_streams + || global.manage_streams + || global.manage_topics + || global.read_topics) + { + return Ok(()); + } + + if let Some(stream_permissions) = + metadata.users_stream_permissions.get(&(user_id, stream_id)) + && (stream_permissions.manage_stream + || stream_permissions.read_stream + || stream_permissions.manage_topics + || stream_permissions.read_topics) + { + return Ok(()); + } + + Err(IggyError::Unauthorized) + } + + /// Inheritance: manage_streams → manage_topics + pub fn perm_create_topic(&self, user_id: u32, stream_id: StreamId) -> Result<(), IggyError> { + let metadata = self.load(); + + if let Some(global) = metadata.users_global_permissions.get(&user_id) + && (global.manage_streams || global.manage_topics) + { + return Ok(()); + } + + if let Some(stream_permissions) = + metadata.users_stream_permissions.get(&(user_id, stream_id)) + && (stream_permissions.manage_stream || stream_permissions.manage_topics) + { + return Ok(()); + } + + Err(IggyError::Unauthorized) + } + + pub fn perm_update_topic( + &self, + user_id: u32, + stream_id: StreamId, + topic_id: TopicId, + ) -> Result<(), IggyError> { + self.perm_manage_topic(user_id, stream_id, topic_id) + } + + pub fn perm_delete_topic( + &self, + user_id: u32, + stream_id: StreamId, + topic_id: TopicId, + ) -> Result<(), IggyError> { + self.perm_manage_topic(user_id, stream_id, topic_id) + } + + pub fn perm_purge_topic( + &self, + user_id: u32, + stream_id: StreamId, + topic_id: TopicId, + ) -> Result<(), IggyError> { + self.perm_manage_topic(user_id, stream_id, topic_id) + } + + /// Inheritance: manage_streams → manage_topics + fn perm_manage_topic( + &self, + user_id: u32, + stream_id: StreamId, + topic_id: TopicId, + ) -> Result<(), IggyError> { + let metadata = self.load(); + + if let Some(global) = metadata.users_global_permissions.get(&user_id) + && (global.manage_streams || global.manage_topics) + { + return Ok(()); + } + + if let Some(stream_permissions) = + metadata.users_stream_permissions.get(&(user_id, stream_id)) + { + if stream_permissions.manage_stream || stream_permissions.manage_topics { + return Ok(()); + } + + if let Some(topics) = &stream_permissions.topics + && let Some(topic_permissions) = topics.get(&topic_id) + && topic_permissions.manage_topic + { + return Ok(()); + } + } + + Err(IggyError::Unauthorized) + } + + pub fn perm_create_partitions( + &self, + user_id: u32, + stream_id: StreamId, + topic_id: TopicId, + ) -> Result<(), IggyError> { + self.perm_update_topic(user_id, stream_id, topic_id) + } + + pub fn perm_delete_partitions( + &self, + user_id: u32, + stream_id: StreamId, + topic_id: TopicId, + ) -> Result<(), IggyError> { + self.perm_update_topic(user_id, stream_id, topic_id) + } + + pub fn perm_delete_segments( + &self, + user_id: u32, + stream_id: StreamId, + topic_id: TopicId, + ) -> Result<(), IggyError> { + self.perm_update_topic(user_id, stream_id, topic_id) + } + + pub fn perm_create_consumer_group( + &self, + user_id: u32, + stream_id: StreamId, + topic_id: TopicId, + ) -> Result<(), IggyError> { + self.perm_get_topic(user_id, stream_id, topic_id) + } + + pub fn perm_delete_consumer_group( + &self, + user_id: u32, + stream_id: StreamId, + topic_id: TopicId, + ) -> Result<(), IggyError> { + self.perm_get_topic(user_id, stream_id, topic_id) + } + + pub fn perm_get_consumer_group( + &self, + user_id: u32, + stream_id: StreamId, + topic_id: TopicId, + ) -> Result<(), IggyError> { + self.perm_get_topic(user_id, stream_id, topic_id) + } + + pub fn perm_get_consumer_groups( + &self, + user_id: u32, + stream_id: StreamId, + topic_id: TopicId, + ) -> Result<(), IggyError> { + self.perm_get_topic(user_id, stream_id, topic_id) + } + + pub fn perm_join_consumer_group( + &self, + user_id: u32, + stream_id: StreamId, + topic_id: TopicId, + ) -> Result<(), IggyError> { + self.perm_get_topic(user_id, stream_id, topic_id) + } + + pub fn perm_leave_consumer_group( + &self, + user_id: u32, + stream_id: StreamId, + topic_id: TopicId, + ) -> Result<(), IggyError> { + self.perm_get_topic(user_id, stream_id, topic_id) + } + + pub fn perm_get_consumer_offset( + &self, + user_id: u32, + stream_id: StreamId, + topic_id: TopicId, + ) -> Result<(), IggyError> { + self.perm_poll_messages(user_id, stream_id, topic_id) + } + + pub fn perm_store_consumer_offset( + &self, + user_id: u32, + stream_id: StreamId, + topic_id: TopicId, + ) -> Result<(), IggyError> { + self.perm_poll_messages(user_id, stream_id, topic_id) + } + + pub fn perm_delete_consumer_offset( + &self, + user_id: u32, + stream_id: StreamId, + topic_id: TopicId, + ) -> Result<(), IggyError> { + self.perm_poll_messages(user_id, stream_id, topic_id) + } + + pub fn perm_get_user(&self, user_id: u32) -> Result<(), IggyError> { + self.perm_read_users(user_id) + } + + pub fn perm_get_users(&self, user_id: u32) -> Result<(), IggyError> { + self.perm_read_users(user_id) + } + + pub fn perm_create_user(&self, user_id: u32) -> Result<(), IggyError> { + self.perm_manage_users(user_id) + } + + pub fn perm_delete_user(&self, user_id: u32) -> Result<(), IggyError> { + self.perm_manage_users(user_id) + } + + pub fn perm_update_user(&self, user_id: u32) -> Result<(), IggyError> { + self.perm_manage_users(user_id) + } + + pub fn perm_update_permissions(&self, user_id: u32) -> Result<(), IggyError> { + self.perm_manage_users(user_id) + } + + pub fn perm_change_password(&self, user_id: u32) -> Result<(), IggyError> { + self.perm_manage_users(user_id) + } + + fn perm_manage_users(&self, user_id: u32) -> Result<(), IggyError> { + let metadata = self.load(); + + if let Some(global_permissions) = metadata.users_global_permissions.get(&user_id) + && global_permissions.manage_users + { + return Ok(()); + } + + Err(IggyError::Unauthorized) + } + + fn perm_read_users(&self, user_id: u32) -> Result<(), IggyError> { + let metadata = self.load(); + + if let Some(global_permissions) = metadata.users_global_permissions.get(&user_id) + && (global_permissions.manage_users || global_permissions.read_users) + { + return Ok(()); + } + + Err(IggyError::Unauthorized) + } + + pub fn perm_get_stats(&self, user_id: u32) -> Result<(), IggyError> { + self.perm_get_server_info(user_id) + } + + pub fn perm_get_clients(&self, user_id: u32) -> Result<(), IggyError> { + self.perm_get_server_info(user_id) + } + + pub fn perm_get_client(&self, user_id: u32) -> Result<(), IggyError> { + self.perm_get_server_info(user_id) + } + + pub fn perm_get_snapshot(&self, user_id: u32) -> Result<(), IggyError> { + self.perm_get_server_info(user_id) + } + + fn perm_get_server_info(&self, user_id: u32) -> Result<(), IggyError> { + let metadata = self.load(); + + if let Some(global_permissions) = metadata.users_global_permissions.get(&user_id) + && (global_permissions.manage_servers || global_permissions.read_servers) + { + return Ok(()); + } + + Err(IggyError::Unauthorized) + } + + /// Atomically resolve, authorize, and return stream metadata. + pub fn query_stream( + &self, + user_id: u32, + stream_id: &Identifier, + ) -> Result, IggyError> { + self.with_metadata(|m| { + let sid = match resolve_stream_id_inner(m, stream_id) { + Some(s) => s, + None => return Ok(None), + }; + perm_get_stream_inner(m, user_id, sid)?; + Ok(m.streams.get(sid).cloned()) + }) + } + + /// Atomically authorize and return all streams. + pub fn query_streams(&self, user_id: u32) -> Result, IggyError> { + self.with_metadata(|m| { + perm_get_streams_inner(m, user_id)?; + Ok(m.streams.iter().map(|(_, s)| s.clone()).collect()) + }) + } + + /// Atomically resolve, authorize, and return topic metadata. + pub fn query_topic( + &self, + user_id: u32, + stream_id: &Identifier, + topic_id: &Identifier, + ) -> Result, IggyError> { + self.with_metadata(|m| { + let sid = match resolve_stream_id_inner(m, stream_id) { + Some(s) => s, + None => return Ok(None), + }; + let tid = match resolve_topic_id_inner(m, sid, topic_id) { + Some(id) => id, + None => return Ok(None), + }; + perm_get_topic_inner(m, user_id, sid, tid)?; + Ok(m.streams.get(sid).and_then(|s| s.topics.get(tid).cloned())) + }) + } + + /// Atomically resolve, authorize, and return all topics for a stream. + pub fn query_topics( + &self, + user_id: u32, + stream_id: &Identifier, + ) -> Result>, IggyError> { + self.with_metadata(|m| { + let sid = match resolve_stream_id_inner(m, stream_id) { + Some(s) => s, + None => return Ok(None), + }; + perm_get_topics_inner(m, user_id, sid)?; + Ok(m.streams + .get(sid) + .map(|s| s.topics.iter().map(|(_, t)| t.clone()).collect())) + }) + } + + /// Atomically resolve, authorize, and return consumer group metadata. + pub fn query_consumer_group( + &self, + user_id: u32, + stream_id: &Identifier, + topic_id: &Identifier, + group_id: &Identifier, + ) -> Result, IggyError> { + self.with_metadata(|m| { + let sid = match resolve_stream_id_inner(m, stream_id) { + Some(s) => s, + None => return Ok(None), + }; + let tid = match resolve_topic_id_inner(m, sid, topic_id) { + Some(id) => id, + None => return Ok(None), + }; + let gid = match resolve_consumer_group_id_inner(m, sid, tid, group_id) { + Some(id) => id, + None => return Ok(None), + }; + perm_get_consumer_group_inner(m, user_id, sid, tid)?; + Ok(m.streams + .get(sid) + .and_then(|s| s.topics.get(tid)) + .and_then(|t| t.consumer_groups.get(gid).cloned())) + }) + } + + /// Atomically resolve, authorize, and return all consumer groups for a topic. + pub fn query_consumer_groups( + &self, + user_id: u32, + stream_id: &Identifier, + topic_id: &Identifier, + ) -> Result>, IggyError> { + self.with_metadata(|m| { + let sid = match resolve_stream_id_inner(m, stream_id) { + Some(s) => s, + None => return Ok(None), + }; + let tid = match resolve_topic_id_inner(m, sid, topic_id) { + Some(id) => id, + None => return Ok(None), + }; + perm_get_consumer_group_inner(m, user_id, sid, tid)?; + Ok(m.streams + .get(sid) + .and_then(|s| s.topics.get(tid)) + .map(|t| t.consumer_groups.iter().map(|(_, cg)| cg.clone()).collect())) + }) + } + + /// Atomically resolve, authorize, and return user metadata. + /// Permission check skipped when requesting own data. + pub fn query_user( + &self, + requesting_user_id: u32, + target_user_id: &Identifier, + ) -> Result, IggyError> { + self.with_metadata(|m| { + let uid = match resolve_user_id_inner(m, target_user_id) { + Some(id) => id, + None => return Ok(None), + }; + if uid != requesting_user_id { + perm_get_user_inner(m, requesting_user_id)?; + } + Ok(m.users.get(uid as usize).cloned()) + }) + } + + /// Atomically authorize and return all users. + pub fn query_users(&self, user_id: u32) -> Result, IggyError> { + self.with_metadata(|m| { + perm_get_users_inner(m, user_id)?; + Ok(m.users.iter().map(|(_, u)| u.clone()).collect()) + }) + } + + /// Atomically resolve topic and check permission for consumer offset query. + /// Returns resolved topic for use with get_consumer_offset. + pub fn resolve_for_consumer_offset( + &self, + user_id: u32, + stream_id: &Identifier, + topic_id: &Identifier, + ) -> Result, IggyError> { + self.with_metadata(|m| { + let sid = match resolve_stream_id_inner(m, stream_id) { + Some(s) => s, + None => return Ok(None), + }; + let tid = match resolve_topic_id_inner(m, sid, topic_id) { + Some(id) => id, + None => return Ok(None), + }; + perm_get_consumer_offset_inner(m, user_id, sid, tid)?; + Ok(Some(ResolvedTopic { + stream_id: sid, + topic_id: tid, + })) + }) + } + + /// Atomically resolve topic and check append permission. + pub fn resolve_for_append( + &self, + user_id: u32, + stream_id: &Identifier, + topic_id: &Identifier, + ) -> Result { + self.with_metadata(|m| { + let sid = resolve_stream_id_inner(m, stream_id) + .ok_or_else(|| IggyError::StreamIdNotFound(stream_id.clone()))?; + let tid = resolve_topic_id_inner(m, sid, topic_id) + .ok_or_else(|| IggyError::TopicIdNotFound(stream_id.clone(), topic_id.clone()))?; + perm_append_messages_inner(m, user_id, sid, tid)?; + Ok(ResolvedTopic { + stream_id: sid, + topic_id: tid, + }) + }) + } + + /// Atomically resolve topic and check poll permission. + pub fn resolve_for_poll( + &self, + user_id: u32, + stream_id: &Identifier, + topic_id: &Identifier, + ) -> Result { + self.with_metadata(|m| { + let sid = resolve_stream_id_inner(m, stream_id) + .ok_or_else(|| IggyError::StreamIdNotFound(stream_id.clone()))?; + let tid = resolve_topic_id_inner(m, sid, topic_id) + .ok_or_else(|| IggyError::TopicIdNotFound(stream_id.clone(), topic_id.clone()))?; + perm_poll_messages_inner(m, user_id, sid, tid)?; + Ok(ResolvedTopic { + stream_id: sid, + topic_id: tid, + }) + }) + } + + /// Atomically resolve topic and check store consumer offset permission. + pub fn resolve_for_store_consumer_offset( + &self, + user_id: u32, + stream_id: &Identifier, + topic_id: &Identifier, + ) -> Result { + self.with_metadata(|m| { + let sid = resolve_stream_id_inner(m, stream_id) + .ok_or_else(|| IggyError::StreamIdNotFound(stream_id.clone()))?; + let tid = resolve_topic_id_inner(m, sid, topic_id) + .ok_or_else(|| IggyError::TopicIdNotFound(stream_id.clone(), topic_id.clone()))?; + perm_get_consumer_offset_inner(m, user_id, sid, tid)?; + Ok(ResolvedTopic { + stream_id: sid, + topic_id: tid, + }) + }) + } + + /// Atomically resolve topic and check delete consumer offset permission. + pub fn resolve_for_delete_consumer_offset( + &self, + user_id: u32, + stream_id: &Identifier, + topic_id: &Identifier, + ) -> Result { + self.with_metadata(|m| { + let sid = resolve_stream_id_inner(m, stream_id) + .ok_or_else(|| IggyError::StreamIdNotFound(stream_id.clone()))?; + let tid = resolve_topic_id_inner(m, sid, topic_id) + .ok_or_else(|| IggyError::TopicIdNotFound(stream_id.clone(), topic_id.clone()))?; + perm_get_consumer_offset_inner(m, user_id, sid, tid)?; + Ok(ResolvedTopic { + stream_id: sid, + topic_id: tid, + }) + }) + } + + /// Atomically resolve partition and check delete segments permission. + pub fn resolve_for_delete_segments( + &self, + user_id: u32, + stream_id: &Identifier, + topic_id: &Identifier, + partition_id: PartitionId, + ) -> Result { + self.with_metadata(|m| { + let sid = resolve_stream_id_inner(m, stream_id) + .ok_or_else(|| IggyError::StreamIdNotFound(stream_id.clone()))?; + let tid = resolve_topic_id_inner(m, sid, topic_id) + .ok_or_else(|| IggyError::TopicIdNotFound(stream_id.clone(), topic_id.clone()))?; + let exists = m + .streams + .get(sid) + .and_then(|s| s.topics.get(tid)) + .and_then(|t| t.partitions.get(partition_id)) + .is_some(); + if !exists { + return Err(IggyError::PartitionNotFound( + partition_id, + topic_id.clone(), + stream_id.clone(), + )); + } + perm_manage_topic_inner(m, user_id, sid, tid)?; + Ok(ResolvedPartition { + stream_id: sid, + topic_id: tid, + partition_id, + }) + }) + } +} + +/// Information needed to initialize a LocalPartition. +#[derive(Clone, Debug)] +pub struct PartitionInitInfo { + pub created_at: IggyTimestamp, + pub revision_id: u64, + pub stats: Arc, + pub consumer_offsets: Arc, + pub consumer_group_offsets: Arc, +} + +pub(crate) fn resolve_stream_id_inner( + m: &InnerMetadata, + stream_id: &Identifier, +) -> Option { + match stream_id.kind { + IdKind::Numeric => { + let sid = stream_id.get_u32_value().ok()? as StreamId; + if m.streams.get(sid).is_some() { + Some(sid) + } else { + None + } + } + IdKind::String => { + let name = stream_id.get_cow_str_value().ok()?; + m.stream_index.get(name.as_ref()).copied() + } + } +} + +pub(crate) fn resolve_topic_id_inner( + m: &InnerMetadata, + stream_id: StreamId, + topic_id: &Identifier, +) -> Option { + let stream = m.streams.get(stream_id)?; + match topic_id.kind { + IdKind::Numeric => { + let tid = topic_id.get_u32_value().ok()? as TopicId; + if stream.topics.get(tid).is_some() { + Some(tid) + } else { + None + } + } + IdKind::String => { + let name = topic_id.get_cow_str_value().ok()?; + stream.topic_index.get(&Arc::from(name.as_ref())).copied() + } + } +} + +pub(crate) fn resolve_consumer_group_id_inner( + m: &InnerMetadata, + stream_id: StreamId, + topic_id: TopicId, + group_id: &Identifier, +) -> Option { + let stream = m.streams.get(stream_id)?; + let topic = stream.topics.get(topic_id)?; + match group_id.kind { + IdKind::Numeric => { + let gid = group_id.get_u32_value().ok()? as ConsumerGroupId; + if topic.consumer_groups.get(gid).is_some() { + Some(gid) + } else { + None + } + } + IdKind::String => { + let name = group_id.get_cow_str_value().ok()?; + topic + .consumer_group_index + .get(&Arc::from(name.as_ref())) + .copied() + } + } +} + +fn resolve_user_id_inner(m: &InnerMetadata, user_id: &Identifier) -> Option { + match user_id.kind { + IdKind::Numeric => Some(user_id.get_u32_value().ok()?), + IdKind::String => { + let name = user_id.get_cow_str_value().ok()?; + m.user_index.get(name.as_ref()).copied() + } + } +} + +fn perm_get_stream_inner( + m: &InnerMetadata, + user_id: u32, + stream_id: StreamId, +) -> Result<(), IggyError> { + if let Some(global) = m.users_global_permissions.get(&user_id) + && (global.manage_streams || global.read_streams) + { + return Ok(()); + } + if let Some(stream_perm) = m.users_stream_permissions.get(&(user_id, stream_id)) + && (stream_perm.manage_stream || stream_perm.read_stream) + { + return Ok(()); + } + Err(IggyError::Unauthorized) +} + +fn perm_get_streams_inner(m: &InnerMetadata, user_id: u32) -> Result<(), IggyError> { + if let Some(global) = m.users_global_permissions.get(&user_id) + && (global.manage_streams || global.read_streams) + { + return Ok(()); + } + Err(IggyError::Unauthorized) +} + +/// Inheritance: manage_streams -> manage_topics -> manage_topic +fn perm_manage_topic_inner( + m: &InnerMetadata, + user_id: u32, + stream_id: StreamId, + topic_id: TopicId, +) -> Result<(), IggyError> { + if let Some(global) = m.users_global_permissions.get(&user_id) + && (global.manage_streams || global.manage_topics) + { + return Ok(()); + } + + if let Some(stream_permissions) = m.users_stream_permissions.get(&(user_id, stream_id)) { + if stream_permissions.manage_stream || stream_permissions.manage_topics { + return Ok(()); + } + + if let Some(topics) = &stream_permissions.topics + && let Some(topic_permissions) = topics.get(&topic_id) + && topic_permissions.manage_topic + { + return Ok(()); + } + } + + Err(IggyError::Unauthorized) +} + +fn perm_get_topic_inner( + m: &InnerMetadata, + user_id: u32, + stream_id: StreamId, + topic_id: TopicId, +) -> Result<(), IggyError> { + if let Some(global) = m.users_global_permissions.get(&user_id) + && (global.read_streams + || global.manage_streams + || global.manage_topics + || global.read_topics) + { + return Ok(()); + } + + if let Some(stream_permissions) = m.users_stream_permissions.get(&(user_id, stream_id)) { + if stream_permissions.manage_stream + || stream_permissions.read_stream + || stream_permissions.manage_topics + || stream_permissions.read_topics + { + return Ok(()); + } + + if let Some(topics) = &stream_permissions.topics + && let Some(topic_permissions) = topics.get(&topic_id) + && (topic_permissions.manage_topic || topic_permissions.read_topic) + { + return Ok(()); + } + } + + Err(IggyError::Unauthorized) +} + +fn perm_get_topics_inner( + m: &InnerMetadata, + user_id: u32, + stream_id: StreamId, +) -> Result<(), IggyError> { + if let Some(global) = m.users_global_permissions.get(&user_id) + && (global.read_streams + || global.manage_streams + || global.manage_topics + || global.read_topics) + { + return Ok(()); + } + + if let Some(stream_permissions) = m.users_stream_permissions.get(&(user_id, stream_id)) + && (stream_permissions.manage_stream + || stream_permissions.read_stream + || stream_permissions.manage_topics + || stream_permissions.read_topics) + { + return Ok(()); + } + + Err(IggyError::Unauthorized) +} + +fn perm_get_consumer_group_inner( + m: &InnerMetadata, + user_id: u32, + stream_id: StreamId, + topic_id: TopicId, +) -> Result<(), IggyError> { + perm_get_topic_inner(m, user_id, stream_id, topic_id) +} + +fn perm_get_user_inner(m: &InnerMetadata, user_id: u32) -> Result<(), IggyError> { + if let Some(global) = m.users_global_permissions.get(&user_id) + && (global.manage_users || global.read_users) + { + return Ok(()); + } + Err(IggyError::Unauthorized) +} + +fn perm_get_users_inner(m: &InnerMetadata, user_id: u32) -> Result<(), IggyError> { + perm_get_user_inner(m, user_id) +} + +fn perm_get_consumer_offset_inner( + m: &InnerMetadata, + user_id: u32, + stream_id: StreamId, + topic_id: TopicId, +) -> Result<(), IggyError> { + if m.users_can_poll_all_streams.contains(&user_id) { + return Ok(()); + } + + if let Some(global) = m.users_global_permissions.get(&user_id) + && (global.read_topics + || global.manage_topics + || global.read_streams + || global.manage_streams) + { + return Ok(()); + } + + if m.users_can_poll_stream.contains(&(user_id, stream_id)) { + return Ok(()); + } + + let Some(stream_permissions) = m.users_stream_permissions.get(&(user_id, stream_id)) else { + return Err(IggyError::Unauthorized); + }; + + if stream_permissions.manage_stream || stream_permissions.read_stream { + return Ok(()); + } + + if stream_permissions.manage_topics || stream_permissions.read_topics { + return Ok(()); + } + + if stream_permissions.poll_messages { + return Ok(()); + } + + if let Some(topics) = &stream_permissions.topics + && let Some(topic_permissions) = topics.get(&topic_id) + && (topic_permissions.manage_topic + || topic_permissions.read_topic + || topic_permissions.poll_messages) + { + return Ok(()); + } + + Err(IggyError::Unauthorized) +} + +fn perm_poll_messages_inner( + m: &InnerMetadata, + user_id: u32, + stream_id: StreamId, + topic_id: TopicId, +) -> Result<(), IggyError> { + perm_get_consumer_offset_inner(m, user_id, stream_id, topic_id) +} + +fn perm_append_messages_inner( + m: &InnerMetadata, + user_id: u32, + stream_id: StreamId, + topic_id: TopicId, +) -> Result<(), IggyError> { + if m.users_can_send_all_streams.contains(&user_id) { + return Ok(()); + } + + if let Some(global) = m.users_global_permissions.get(&user_id) + && (global.manage_streams || global.manage_topics) + { + return Ok(()); + } + + if m.users_can_send_stream.contains(&(user_id, stream_id)) { + return Ok(()); + } + + let Some(stream_permissions) = m.users_stream_permissions.get(&(user_id, stream_id)) else { + return Err(IggyError::Unauthorized); + }; + + if stream_permissions.manage_stream + || stream_permissions.manage_topics + || stream_permissions.send_messages + { + return Ok(()); + } + + if let Some(topics) = &stream_permissions.topics + && let Some(topic_permissions) = topics.get(&topic_id) + && (topic_permissions.manage_topic || topic_permissions.send_messages) + { + return Ok(()); + } + + Err(IggyError::Unauthorized) +} diff --git a/core/server/src/metadata/stream.rs b/core/server/src/metadata/stream.rs new file mode 100644 index 0000000000..02aee459dd --- /dev/null +++ b/core/server/src/metadata/stream.rs @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::metadata::topic::TopicMeta; +use crate::metadata::{StreamId, TopicId}; +use crate::streaming::stats::StreamStats; +use ahash::AHashMap; +use iggy_common::IggyTimestamp; +use slab::Slab; +use std::sync::Arc; + +/// Stream metadata stored in the shared snapshot. +#[derive(Clone, Debug)] +pub struct StreamMeta { + pub id: StreamId, + pub name: Arc, + pub created_at: IggyTimestamp, + pub stats: Arc, + pub topics: Slab, + pub topic_index: AHashMap, TopicId>, +} + +impl StreamMeta { + pub fn new(id: StreamId, name: Arc, created_at: IggyTimestamp) -> Self { + Self { + id, + name, + created_at, + stats: Arc::new(StreamStats::default()), + topics: Slab::new(), + topic_index: AHashMap::default(), + } + } + + pub fn with_stats( + id: StreamId, + name: Arc, + created_at: IggyTimestamp, + stats: Arc, + ) -> Self { + Self { + id, + name, + created_at, + stats, + topics: Slab::new(), + topic_index: AHashMap::default(), + } + } +} diff --git a/core/server/src/metadata/topic.rs b/core/server/src/metadata/topic.rs new file mode 100644 index 0000000000..ba0a88b3da --- /dev/null +++ b/core/server/src/metadata/topic.rs @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::metadata::consumer_group::ConsumerGroupMeta; +use crate::metadata::partition::PartitionMeta; +use crate::metadata::{ConsumerGroupId, TopicId}; +use crate::streaming::stats::TopicStats; +use ahash::AHashMap; +use iggy_common::{CompressionAlgorithm, IggyExpiry, IggyTimestamp, MaxTopicSize}; +use slab::Slab; +use std::sync::Arc; +use std::sync::atomic::AtomicUsize; + +/// Topic metadata stored in the shared snapshot. +#[derive(Clone, Debug)] +pub struct TopicMeta { + pub id: TopicId, + pub name: Arc, + pub created_at: IggyTimestamp, + pub message_expiry: IggyExpiry, + pub compression_algorithm: CompressionAlgorithm, + pub max_topic_size: MaxTopicSize, + pub replication_factor: u8, + pub stats: Arc, + pub partitions: Vec, + pub consumer_groups: Slab, + pub consumer_group_index: AHashMap, ConsumerGroupId>, + pub round_robin_counter: Arc, +} + +impl TopicMeta { + #[allow(clippy::too_many_arguments)] + pub fn with_stats( + id: TopicId, + name: Arc, + created_at: IggyTimestamp, + message_expiry: IggyExpiry, + compression_algorithm: CompressionAlgorithm, + max_topic_size: MaxTopicSize, + replication_factor: u8, + stats: Arc, + ) -> Self { + Self { + id, + name, + created_at, + message_expiry, + compression_algorithm, + max_topic_size, + replication_factor, + stats, + partitions: Vec::new(), + consumer_groups: Slab::new(), + consumer_group_index: AHashMap::default(), + round_robin_counter: Arc::new(AtomicUsize::new(0)), + } + } +} diff --git a/foreign/csharp/Iggy_SDK/Vsr/Command2.cs b/core/server/src/metadata/user.rs similarity index 69% rename from foreign/csharp/Iggy_SDK/Vsr/Command2.cs rename to core/server/src/metadata/user.rs index c8f7a765d7..c82d49a441 100644 --- a/foreign/csharp/Iggy_SDK/Vsr/Command2.cs +++ b/core/server/src/metadata/user.rs @@ -15,16 +15,16 @@ // specific language governing permissions and limitations // under the License. -namespace Apache.Iggy.Vsr; +use crate::metadata::UserId; +use iggy_common::{IggyTimestamp, Permissions, UserStatus}; +use std::sync::Arc; -/// -/// VSR frame discriminant, byte 60 of every consensus header. Only the frames a client emits or -/// receives are named; every other discriminant decodes as . -/// -internal enum Command2 : byte -{ - Reserved = 0, - Request = 5, - Reply = 8, - Eviction = 13 +#[derive(Clone, Debug)] +pub struct UserMeta { + pub id: UserId, + pub username: Arc, + pub password_hash: Arc, + pub status: UserStatus, + pub permissions: Option>, + pub created_at: IggyTimestamp, } diff --git a/core/server/src/metadata/writer.rs b/core/server/src/metadata/writer.rs new file mode 100644 index 0000000000..4712679c82 --- /dev/null +++ b/core/server/src/metadata/writer.rs @@ -0,0 +1,617 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::metadata::consumer_group_member::CompletableRevocation; +use crate::metadata::inner::InnerMetadata; +use crate::metadata::ops::MetadataOp; +use crate::metadata::reader::Metadata; +use crate::metadata::{ + ConsumerGroupId, ConsumerGroupMeta, PartitionId, PartitionMeta, StreamId, StreamMeta, TopicId, + TopicMeta, UserId, UserMeta, +}; +use crate::streaming::partitions::consumer_group_offsets::ConsumerGroupOffsets; +use crate::streaming::partitions::consumer_offsets::ConsumerOffsets; +use crate::streaming::stats::{PartitionStats, StreamStats, TopicStats}; +use iggy_common::{ + CompressionAlgorithm, Identifier, IggyError, IggyExpiry, IggyTimestamp, MaxTopicSize, + Permissions, PersonalAccessToken, UserStatus, +}; +use left_right::WriteHandle; +use slab::Slab; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; + +pub struct MetadataWriter { + inner: WriteHandle, + revision: u64, +} + +impl MetadataWriter { + pub fn new(handle: WriteHandle) -> Self { + Self { + inner: handle, + revision: 0, + } + } + + fn next_revision(&mut self) -> u64 { + self.revision += 1; + self.revision + } + + pub fn append(&mut self, op: MetadataOp) { + self.inner.append(op); + } + + pub fn publish(&mut self) { + self.inner.publish(); + } + + pub fn initialize(&mut self, initial: InnerMetadata) { + self.append(MetadataOp::Initialize(Box::new(initial))); + self.publish(); + } + + pub fn add_stream(&mut self, meta: StreamMeta) -> StreamId { + let assigned_id = Arc::new(AtomicUsize::new(usize::MAX)); + self.append(MetadataOp::AddStream { + meta, + assigned_id: assigned_id.clone(), + }); + self.publish(); + let id = assigned_id.load(Ordering::Acquire); + debug_assert_ne!(id, usize::MAX, "add_stream should always succeed"); + id + } + + pub fn update_stream(&mut self, id: StreamId, new_name: Arc) { + self.append(MetadataOp::UpdateStream { id, new_name }); + self.publish(); + } + + pub fn delete_stream(&mut self, id: StreamId) { + self.append(MetadataOp::DeleteStream { id }); + self.publish(); + } + + pub fn add_topic(&mut self, stream_id: StreamId, meta: TopicMeta) -> Option { + let assigned_id = Arc::new(AtomicUsize::new(usize::MAX)); + self.append(MetadataOp::AddTopic { + stream_id, + meta, + assigned_id: assigned_id.clone(), + }); + self.publish(); + let id = assigned_id.load(Ordering::Acquire); + if id == usize::MAX { None } else { Some(id) } + } + + #[allow(clippy::too_many_arguments)] + pub fn update_topic( + &mut self, + stream_id: StreamId, + topic_id: TopicId, + new_name: Arc, + message_expiry: IggyExpiry, + compression_algorithm: CompressionAlgorithm, + max_topic_size: MaxTopicSize, + replication_factor: u8, + ) { + self.append(MetadataOp::UpdateTopic { + stream_id, + topic_id, + new_name, + message_expiry, + compression_algorithm, + max_topic_size, + replication_factor, + }); + self.publish(); + } + + pub fn delete_topic(&mut self, stream_id: StreamId, topic_id: TopicId) { + self.append(MetadataOp::DeleteTopic { + stream_id, + topic_id, + }); + self.publish(); + } + + /// Add partitions to a topic. Returns the assigned partition IDs (sequential from current count). + pub fn add_partitions( + &mut self, + reader: &Metadata, + stream_id: StreamId, + topic_id: TopicId, + partitions: Vec, + ) -> Vec { + if partitions.is_empty() { + return Vec::new(); + } + + let count_before = reader + .get_partitions_count(stream_id, topic_id) + .expect("stream and topic must exist when adding partitions"); + let count = partitions.len(); + + let revision_id = self.next_revision(); + self.append(MetadataOp::AddPartitions { + stream_id, + topic_id, + partitions, + revision_id, + }); + self.publish(); + + (count_before..count_before + count).collect() + } + + /// Delete partitions from the end of a topic. + pub fn delete_partitions(&mut self, stream_id: StreamId, topic_id: TopicId, count: u32) { + if count == 0 { + return; + } + self.append(MetadataOp::DeletePartitions { + stream_id, + topic_id, + count, + }); + self.publish(); + } + + pub fn add_user(&mut self, meta: UserMeta) -> UserId { + let assigned_id = Arc::new(AtomicUsize::new(usize::MAX)); + self.append(MetadataOp::AddUser { + meta, + assigned_id: assigned_id.clone(), + }); + self.publish(); + let id = assigned_id.load(Ordering::Acquire); + debug_assert_ne!(id, usize::MAX, "add_user should always succeed"); + id as UserId + } + + pub fn update_user_meta(&mut self, id: UserId, meta: UserMeta) { + self.append(MetadataOp::UpdateUserMeta { id, meta }); + self.publish(); + } + + pub fn delete_user(&mut self, id: UserId) { + self.append(MetadataOp::DeleteUser { id }); + self.publish(); + } + + pub fn add_personal_access_token(&mut self, user_id: UserId, pat: PersonalAccessToken) { + self.append(MetadataOp::AddPersonalAccessToken { user_id, pat }); + self.publish(); + } + + pub fn delete_personal_access_token(&mut self, user_id: UserId, token_hash: Arc) { + self.append(MetadataOp::DeletePersonalAccessToken { + user_id, + token_hash, + }); + self.publish(); + } + + pub fn add_consumer_group( + &mut self, + stream_id: StreamId, + topic_id: TopicId, + meta: ConsumerGroupMeta, + ) -> Option { + let assigned_id = Arc::new(AtomicUsize::new(usize::MAX)); + self.append(MetadataOp::AddConsumerGroup { + stream_id, + topic_id, + meta, + assigned_id: assigned_id.clone(), + }); + self.publish(); + let id = assigned_id.load(Ordering::Acquire); + if id == usize::MAX { None } else { Some(id) } + } + + pub fn delete_consumer_group( + &mut self, + stream_id: StreamId, + topic_id: TopicId, + group_id: ConsumerGroupId, + ) { + self.append(MetadataOp::DeleteConsumerGroup { + stream_id, + topic_id, + group_id, + }); + self.publish(); + } + + pub fn join_consumer_group( + &mut self, + stream_id: StreamId, + topic_id: TopicId, + group_id: ConsumerGroupId, + client_id: u32, + valid_client_ids: Option>, + ) -> (Option, Vec) { + let member_id = Arc::new(AtomicUsize::new(usize::MAX)); + let completable = Arc::new(Mutex::new(Vec::new())); + self.append(MetadataOp::JoinConsumerGroup { + stream_id, + topic_id, + group_id, + client_id, + member_id: member_id.clone(), + valid_client_ids, + completable_revocations: completable.clone(), + }); + self.publish(); + let id = member_id.load(Ordering::Acquire); + let revocations = match Arc::try_unwrap(completable) { + Ok(mutex) => mutex.into_inner().unwrap(), + Err(arc) => std::mem::take(&mut *arc.lock().unwrap()), + }; + (if id == usize::MAX { None } else { Some(id) }, revocations) + } + + pub fn leave_consumer_group( + &mut self, + stream_id: StreamId, + topic_id: TopicId, + group_id: ConsumerGroupId, + client_id: u32, + ) -> Option { + let removed_member_id = Arc::new(AtomicUsize::new(usize::MAX)); + self.append(MetadataOp::LeaveConsumerGroup { + stream_id, + topic_id, + group_id, + client_id, + removed_member_id: removed_member_id.clone(), + }); + self.publish(); + let id = removed_member_id.load(Ordering::Acquire); + if id == usize::MAX { None } else { Some(id) } + } + + pub fn rebalance_consumer_groups_for_topic( + &mut self, + stream_id: StreamId, + topic_id: TopicId, + partitions_count: u32, + ) { + self.append(MetadataOp::RebalanceConsumerGroupsForTopic { + stream_id, + topic_id, + partitions_count, + }); + self.publish(); + } + + #[allow(clippy::too_many_arguments)] + pub fn complete_partition_revocation( + &mut self, + stream_id: StreamId, + topic_id: TopicId, + group_id: ConsumerGroupId, + member_slab_id: usize, + member_id: usize, + partition_id: PartitionId, + timed_out: bool, + ) { + self.append(MetadataOp::CompletePartitionRevocation { + stream_id, + topic_id, + group_id, + member_slab_id, + member_id, + partition_id, + timed_out, + }); + self.publish(); + } + + // High-level registration methods with validation + + pub fn create_stream( + &mut self, + reader: &Metadata, + name: Arc, + created_at: IggyTimestamp, + ) -> Result<(StreamId, Arc), IggyError> { + if reader.stream_name_exists(&name) { + return Err(IggyError::StreamNameAlreadyExists(name.to_string())); + } + + let stats = Arc::new(StreamStats::default()); + let meta = StreamMeta::with_stats(0, name, created_at, stats.clone()); + let id = self.add_stream(meta); + Ok((id, stats)) + } + + pub fn try_update_stream( + &mut self, + reader: &Metadata, + id: StreamId, + new_name: Arc, + ) -> Result<(), IggyError> { + let guard = reader.load(); + let Some(stream) = guard.streams.get(id) else { + return Err(IggyError::StreamIdNotFound( + Identifier::numeric(id as u32).unwrap(), + )); + }; + + if stream.name == new_name { + return Ok(()); + } + + if let Some(&existing_id) = guard.stream_index.get(&new_name) + && existing_id != id + { + return Err(IggyError::StreamNameAlreadyExists(new_name.to_string())); + } + drop(guard); + + self.update_stream(id, new_name); + Ok(()) + } + + #[allow(clippy::too_many_arguments)] + pub fn create_topic( + &mut self, + reader: &Metadata, + stream_id: StreamId, + name: Arc, + created_at: IggyTimestamp, + message_expiry: IggyExpiry, + compression_algorithm: CompressionAlgorithm, + max_topic_size: MaxTopicSize, + replication_factor: u8, + ) -> Result<(TopicId, Arc), IggyError> { + let parent_stats = reader.get_stream_stats(stream_id).ok_or_else(|| { + IggyError::StreamIdNotFound(Identifier::numeric(stream_id as u32).unwrap()) + })?; + + let guard = reader.load(); + let Some(stream) = guard.streams.get(stream_id) else { + return Err(IggyError::StreamIdNotFound( + Identifier::numeric(stream_id as u32).unwrap(), + )); + }; + + if stream.topic_index.contains_key(&name) { + return Err(IggyError::TopicNameAlreadyExists( + name.to_string(), + Identifier::numeric(stream_id as u32).unwrap(), + )); + } + drop(guard); + + let stats = Arc::new(TopicStats::new(parent_stats)); + let meta = TopicMeta { + id: 0, + name, + created_at, + message_expiry, + compression_algorithm, + max_topic_size, + replication_factor, + stats: stats.clone(), + partitions: Vec::new(), + consumer_groups: Slab::new(), + consumer_group_index: ahash::AHashMap::default(), + round_robin_counter: Arc::new(AtomicUsize::new(0)), + }; + + // change to create_topic + let id = self.add_topic(stream_id, meta).ok_or_else(|| { + IggyError::StreamIdNotFound(Identifier::numeric(stream_id as u32).unwrap()) + })?; + Ok((id, stats)) + } + + #[allow(clippy::too_many_arguments)] + pub fn try_update_topic( + &mut self, + reader: &Metadata, + stream_id: StreamId, + topic_id: TopicId, + new_name: Arc, + message_expiry: IggyExpiry, + compression_algorithm: CompressionAlgorithm, + max_topic_size: MaxTopicSize, + replication_factor: u8, + ) -> Result<(), IggyError> { + let guard = reader.load(); + let Some(stream) = guard.streams.get(stream_id) else { + return Err(IggyError::StreamIdNotFound( + Identifier::numeric(stream_id as u32).unwrap(), + )); + }; + + let Some(topic) = stream.topics.get(topic_id) else { + return Err(IggyError::TopicIdNotFound( + Identifier::numeric(topic_id as u32).unwrap(), + Identifier::numeric(stream_id as u32).unwrap(), + )); + }; + + if topic.name != new_name + && let Some(&existing_id) = stream.topic_index.get(&new_name) + && existing_id != topic_id + { + return Err(IggyError::TopicNameAlreadyExists( + new_name.to_string(), + Identifier::numeric(stream_id as u32).unwrap(), + )); + } + drop(guard); + + self.update_topic( + stream_id, + topic_id, + new_name, + message_expiry, + compression_algorithm, + max_topic_size, + replication_factor, + ); + Ok(()) + } + + pub fn register_partitions( + &mut self, + reader: &Metadata, + stream_id: StreamId, + topic_id: TopicId, + count: usize, + created_at: IggyTimestamp, + ) -> Vec<(PartitionId, Arc)> { + if count == 0 { + return Vec::new(); + } + + let parent_stats = reader + .get_topic_stats(stream_id, topic_id) + .expect("Parent topic stats must exist before registering partitions"); + + let mut metas = Vec::with_capacity(count); + let mut stats_list = Vec::with_capacity(count); + + for _ in 0..count { + let stats = Arc::new(PartitionStats::new(parent_stats.clone())); + metas.push(PartitionMeta { + id: 0, + created_at, + revision_id: 0, + stats: stats.clone(), + consumer_offsets: Arc::new(ConsumerOffsets::with_capacity(0)), + consumer_group_offsets: Arc::new(ConsumerGroupOffsets::with_capacity(0)), + last_polled_offsets: Arc::new(papaya::HashMap::new()), + }); + stats_list.push(stats); + } + + let ids = self.add_partitions(reader, stream_id, topic_id, metas); + ids.into_iter().zip(stats_list).collect() + } + + pub fn create_user( + &mut self, + reader: &Metadata, + username: Arc, + password_hash: Arc, + status: UserStatus, + permissions: Option>, + max_users: usize, + ) -> Result { + if reader.username_exists(&username) { + return Err(IggyError::UserAlreadyExists); + } + + if reader.users_count() >= max_users { + return Err(IggyError::UsersLimitReached); + } + + let meta = UserMeta { + id: 0, + username, + password_hash, + status, + permissions, + created_at: IggyTimestamp::now(), + }; + let id = self.add_user(meta); + Ok(id) + } + + pub fn update_user( + &mut self, + reader: &Metadata, + id: UserId, + username: Option>, + status: Option, + ) -> Result { + let Some(mut meta) = reader.get_user(id) else { + return Err(IggyError::ResourceNotFound(format!("user:{id}"))); + }; + + if let Some(new_username) = username { + if meta.username != new_username && reader.username_exists(&new_username) { + return Err(IggyError::UserAlreadyExists); + } + meta.username = new_username; + } + + if let Some(new_status) = status { + meta.status = new_status; + } + + let updated = meta.clone(); + self.update_user_meta(id, meta); + Ok(updated) + } + + pub fn create_consumer_group( + &mut self, + reader: &Metadata, + stream_id: StreamId, + topic_id: TopicId, + name: Arc, + partitions_count: u32, + ) -> Result { + let guard = reader.load(); + let Some(stream) = guard.streams.get(stream_id) else { + return Err(IggyError::StreamIdNotFound( + Identifier::numeric(stream_id as u32).unwrap(), + )); + }; + + let Some(topic) = stream.topics.get(topic_id) else { + return Err(IggyError::TopicIdNotFound( + Identifier::numeric(topic_id as u32).unwrap(), + Identifier::numeric(stream_id as u32).unwrap(), + )); + }; + + if topic.consumer_group_index.contains_key(&name) { + return Err(IggyError::ConsumerGroupNameAlreadyExists( + name.to_string(), + Identifier::numeric(topic_id as u32).unwrap(), + )); + } + drop(guard); + + let meta = ConsumerGroupMeta { + id: 0, + name, + partitions: (0..partitions_count as usize).collect(), + members: Slab::new(), + }; + + let id = self + .add_consumer_group(stream_id, topic_id, meta) + .ok_or_else(|| { + IggyError::TopicIdNotFound( + Identifier::numeric(topic_id as u32).unwrap(), + Identifier::numeric(stream_id as u32).unwrap(), + ) + })?; + Ok(id) + } +} diff --git a/core/server/src/quic/listener.rs b/core/server/src/quic/listener.rs new file mode 100644 index 0000000000..6cd3d50d65 --- /dev/null +++ b/core/server/src/quic/listener.rs @@ -0,0 +1,239 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::{self, HandlerResult, MAX_CONTROL_FRAME_PAYLOAD}; +use crate::sender::SenderKind; +use crate::server_error::ConnectionError; +use crate::shard::IggyShard; +use crate::shard::task_registry::ShutdownToken; +use crate::streaming::session::Session; +use anyhow::anyhow; +use compio::io::AsyncReadExt; +use compio::quic::{Connection, Endpoint, RecvStream, SendStream}; +use futures::FutureExt; +use iggy_binary_protocol::RequestFrame; +use iggy_binary_protocol::codes::{GET_CLUSTER_METADATA_CODE, SEND_MESSAGES_CODE, command_name}; +use iggy_common::{IggyError, TransportProtocol}; +use std::rc::Rc; +use tracing::{debug, error, info, trace, warn}; + +pub async fn start( + endpoint: Endpoint, + shard: Rc, + shutdown: ShutdownToken, +) -> Result<(), IggyError> { + loop { + let accept_future = endpoint.wait_incoming(); + + futures::select! { + _ = shutdown.wait().fuse() => { + debug!( "QUIC listener received shutdown signal, no longer accepting connections"); + break; + } + incoming_conn = accept_future.fuse() => { + match incoming_conn { + Some(incoming_conn) => { + let remote_addr = incoming_conn.remote_address(); + info!("Received incoming QUIC connection from {}", remote_addr); + + if shard.is_shutting_down() { + info!( "Rejecting new QUIC connection from {} during shutdown", remote_addr); + continue; + } + + trace!("Incoming connection from client: {}", remote_addr); + let shard_for_conn = shard.clone(); + + shard.task_registry.spawn_connection(async move { + trace!("Accepting connection from {}", remote_addr); + match incoming_conn.await { + Ok(connection) => { + trace!("Connection established from {}", remote_addr); + if let Err(error) = handle_connection(connection, shard_for_conn).await { + error!("QUIC connection from {} has failed: {error}", remote_addr); + } + } + Err(error) => { + error!( + "Error when accepting incoming connection from {}: {:?}", + remote_addr, error + ); + } + } + }); + } + None => { + info!("QUIC endpoint closed for shard {}", shard.id); + break; + } + } + } + } + } + Ok(()) +} + +async fn handle_connection( + connection: Connection, + shard: Rc, +) -> Result<(), ConnectionError> { + let address = connection.remote_address(); + info!("Client has connected: {address}"); + let session = Rc::new(shard.add_client(&address, TransportProtocol::Quic)); + + let client_id = session.client_id; + debug!( + "Added {} client with session: {} for IP address: {}", + TransportProtocol::Quic, + session, + address + ); + + let conn_stop_receiver = shard.task_registry.add_connection(client_id); + + loop { + let shard = shard.clone(); + futures::select! { + // Check for shutdown signal + _ = conn_stop_receiver.recv().fuse() => { + info!("QUIC connection {} shutting down gracefully", client_id); + break; + } + // Accept new connection + stream_result = accept_stream(&connection, shard.clone(), client_id).fuse() => { + match stream_result? { + Some(stream) => { + let shard_clone = shard.clone(); + let session_rc = session.clone(); + + shard.task_registry.spawn_connection(async move { + if let Err(err) = handle_stream(stream, shard_clone, &session_rc).await { + error!("Error when handling QUIC stream: {:?}", err) + } + }); + } + None => break, // Connection closed + } + } + } + } + + shard.delete_client(client_id).await; + shard.task_registry.remove_connection(&client_id); + info!("QUIC connection {} closed", client_id); + Ok(()) +} + +type BiStream = (SendStream, RecvStream); + +async fn accept_stream( + connection: &Connection, + _shard: Rc, + _client_id: u32, +) -> Result, ConnectionError> { + match connection.accept_bi().await { + Err(compio::quic::ConnectionError::ApplicationClosed { .. }) => { + info!("Connection closed"); + Ok(None) + } + Err(error) => { + error!("Error when accepting QUIC connection: {:?}", error); + Err(error.into()) + } + Ok(stream) => Ok(Some(stream)), + } +} + +async fn handle_stream( + stream: BiStream, + shard: Rc, + session: &Session, +) -> anyhow::Result<()> { + let (send_stream, mut recv_stream) = stream; + + let header_buf = [0u8; RequestFrame::HEADER_SIZE]; + let compio::BufResult(result, header_buf) = recv_stream.read_exact(header_buf).await; + result?; + + let length = u32::from_le_bytes(header_buf[0..4].try_into().unwrap()); + let code = u32::from_le_bytes(header_buf[4..8].try_into().unwrap()); + + let cmd_name = command_name(code).unwrap_or("unknown"); + trace!("Received a QUIC request, length: {length}, code: {code} ({cmd_name})"); + + let payload_length = RequestFrame::payload_length(length) + .map_err(|_| anyhow!("Invalid frame length: {length}"))?; + + let mut sender = SenderKind::get_quic_sender(send_stream, recv_stream); + + let result = if code == SEND_MESSAGES_CODE { + dispatch::dispatch_send_messages(&mut sender, payload_length, session, &shard).await + } else { + if payload_length > MAX_CONTROL_FRAME_PAYLOAD { + sender + .send_error_response(IggyError::InvalidCommand) + .await?; + return Ok(()); + } + let payload = dispatch::read_payload(&mut sender, payload_length).await?; + let frame = RequestFrame::from_parts(code, &payload); + dispatch::dispatch(frame, &mut sender, session, &shard).await + }; + + match result { + Ok(HandlerResult::Finished) => { + trace!( + "Command was handled successfully, session: {:?}. QUIC response was sent.", + session + ); + Ok(()) + } + Ok(HandlerResult::Migrated { to_shard }) => { + warn!("Unexpected migration on QUIC: to_shard {to_shard}, session: {session:?}"); + Ok(()) + } + Err(e) => { + // Special handling for GetClusterMetadata when clustering is disabled + if code == GET_CLUSTER_METADATA_CODE && matches!(e, IggyError::FeatureUnavailable) { + debug!( + "GetClusterMetadata command not available (clustering disabled), session: {:?}.", + session + ); + sender.send_error_response(e).await?; + trace!("QUIC error response was sent."); + Ok(()) + } else { + error!( + "Command was not handled successfully, session: {:?}, error: {e}.", + session + ); + // Only return a connection-terminating error for client not found or stale + if matches!(e, IggyError::ClientNotFound(_) | IggyError::StaleClient) { + sender.send_error_response(e.clone()).await?; + trace!("QUIC error response was sent."); + error!("Session will be deleted."); + Err(anyhow!("Client invalid: {e}")) + } else { + // For all other errors, send response and continue the connection + sender.send_error_response(e).await?; + trace!("QUIC error response was sent."); + Ok(()) + } + } + } + } +} diff --git a/core/server/src/quic/mod.rs b/core/server/src/quic/mod.rs new file mode 100644 index 0000000000..cc600f01c3 --- /dev/null +++ b/core/server/src/quic/mod.rs @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod listener; +pub mod quic_server; +pub mod quic_socket; + +pub const COMPONENT: &str = "QUIC"; diff --git a/core/server/src/quic/quic_server.rs b/core/server/src/quic/quic_server.rs new file mode 100644 index 0000000000..3effc7cea6 --- /dev/null +++ b/core/server/src/quic/quic_server.rs @@ -0,0 +1,222 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::configs::quic::QuicConfig; +use crate::quic::{COMPONENT, listener, quic_socket}; +use crate::server_error::QuicError; +use crate::shard::IggyShard; +use crate::shard::task_registry::ShutdownToken; +use crate::shard::transmission::event::ShardEvent; +use anyhow::Result; +use compio::quic::{ + Endpoint, EndpointConfig, IdleTimeout, ServerBuilder, ServerConfig, TransportConfig, VarInt, +}; +use err_trail::ErrContext; +use rustls::crypto::ring::default_provider; +use rustls::pki_types::{CertificateDer, PrivateKeyDer}; +use std::fs::File; +use std::io::BufReader; +use std::net::SocketAddr; +use std::rc::Rc; +use std::sync::Arc; +use tracing::info; +use tracing::{error, trace, warn}; + +/// Starts the QUIC server. +/// Returns the address the server is listening on. +pub async fn spawn_quic_server( + shard: Rc, + shutdown: ShutdownToken, +) -> Result<(), iggy_common::IggyError> { + // Ensure rustls crypto provider is installed (thread-safe, idempotent) + if rustls::crypto::CryptoProvider::get_default().is_none() { + if let Err(e) = default_provider().install_default() { + warn!( + "Failed to install rustls crypto provider: {:?}. This may be normal if another thread installed it first.", + e + ); + } else { + trace!("Rustls crypto provider installed successfully"); + } + } else { + trace!("Rustls crypto provider already installed"); + } + + let config = shard.config.quic.clone(); + let mut addr: SocketAddr = config.address.parse().map_err(|e| { + error!("Failed to parse QUIC address '{}': {}", config.address, e); + iggy_common::IggyError::QuicError + })?; + + if shard.id != 0 && addr.port() == 0 { + info!("Waiting for QUIC address from shard 0..."); + loop { + if let Some(bound_addr) = shard.quic_bound_address.get() { + addr = bound_addr; + info!("Received QUIC address: {}", addr); + break; + } + compio::time::sleep(std::time::Duration::from_millis(50)).await; + } + } + + info!( + "Initializing Iggy QUIC server on shard {} for address {}", + shard.id, addr + ); + + let server_config = configure_quic(&config).map_err(|e| { + error!("Failed to configure QUIC server: {:?}", e); + iggy_common::IggyError::QuicError + })?; + trace!("Building UDP socket for QUIC endpoint on {}", addr); + + let socket = quic_socket::build(&addr, &config.socket); + socket.bind(&addr.into()).map_err(|e| { + error!("Failed to bind socket: {}", e); + iggy_common::IggyError::CannotBindToSocket(addr.to_string()) + })?; + socket.set_nonblocking(true).map_err(|e| { + error!("Failed to set nonblocking: {}", e); + iggy_common::IggyError::QuicError + })?; + + let std_socket: std::net::UdpSocket = socket.into(); + let socket = compio::net::UdpSocket::from_std(std_socket).map_err(|e| { + error!("Failed to convert std socket to compio socket: {:?}", e); + iggy_common::IggyError::QuicError + })?; + trace!("Creating QUIC endpoint with server config"); + + let endpoint = Endpoint::new(socket, EndpointConfig::default(), Some(server_config), None) + .map_err(|e| { + error!("Failed to create QUIC endpoint: {:?}", e); + iggy_common::IggyError::QuicError + })?; + + let actual_addr = endpoint.local_addr().map_err(|e| { + error!("Failed to get local address: {e}"); + iggy_common::IggyError::CannotBindToSocket(addr.to_string()) + })?; + + info!("Iggy QUIC server has started on: {:?}", actual_addr); + + if shard.id == 0 { + // Store bound address locally + shard.quic_bound_address.set(Some(actual_addr)); + + if addr.port() == 0 { + // Notify config writer on shard 0 + let _ = shard.config_writer_notify.try_send(()); + + // Broadcast to other shards for SO_REUSEPORT binding + let event = ShardEvent::AddressBound { + protocol: iggy_common::TransportProtocol::Quic, + address: actual_addr, + }; + shard.broadcast_event_to_all_shards(event).await?; + } + } else { + shard.quic_bound_address.set(Some(actual_addr)); + } + + listener::start(endpoint, shard, shutdown).await +} + +fn configure_quic(config: &QuicConfig) -> Result { + let (certificates, private_key) = match config.certificate.self_signed { + true => generate_self_signed_cert()?, + false => load_certificates(&config.certificate.cert_file, &config.certificate.key_file)?, + }; + + let builder = ServerBuilder::new_with_single_cert(certificates, private_key) + .error(|e: &rustls::Error| { + format!("{COMPONENT} (error: {e}) - failed to create QUIC server builder") + }) + .map_err(|_| QuicError::ConfigCreationError)?; + let mut transport = TransportConfig::default(); + transport.initial_mtu(config.initial_mtu.as_bytes_u64() as u16); + transport.send_window(config.send_window.as_bytes_u64()); + transport.receive_window( + VarInt::try_from(config.receive_window.as_bytes_u64()).map_err(|e| { + error!("{COMPONENT} (error: {e}) - invalid receive window"); + QuicError::TransportConfigError + })?, + ); + transport.datagram_send_buffer_size(config.datagram_send_buffer_size.as_bytes_u64() as usize); + transport.max_concurrent_bidi_streams( + VarInt::try_from(config.max_concurrent_bidi_streams).map_err(|e| { + error!("{COMPONENT} (error: {e}) - invalid bidi stream limit"); + QuicError::TransportConfigError + })?, + ); + + if !config.keep_alive_interval.is_zero() { + transport.keep_alive_interval(Some(config.keep_alive_interval.get_duration())); + } + if !config.max_idle_timeout.is_zero() { + let max_idle_timeout = IdleTimeout::try_from(config.max_idle_timeout.get_duration()) + .map_err(|e| { + error!("{COMPONENT} (error: {e}) - invalid idle timeout"); + QuicError::TransportConfigError + })?; + transport.max_idle_timeout(Some(max_idle_timeout)); + } + + let mut server_config = builder.build(); + server_config.transport_config(Arc::new(transport)); + Ok(server_config) +} + +fn generate_self_signed_cert<'a>() -> Result<(Vec>, PrivateKeyDer<'a>), QuicError> +{ + server_common::generate_self_signed_certificate("localhost").map_err(|e| { + error!("{COMPONENT} (error: {e}) - failed to generate self-signed certificate"); + QuicError::CertGenerationError + }) +} + +fn load_certificates( + cert_file: &str, + key_file: &str, +) -> Result<(Vec>, PrivateKeyDer<'static>), QuicError> { + let mut cert_chain_reader = BufReader::new( + File::open(cert_file) + .error(|e: &std::io::Error| { + format!("{COMPONENT} (error: {e}) - failed to open cert file: {cert_file}") + }) + .map_err(|_| QuicError::CertLoadError)?, + ); + let certs = rustls_pemfile::certs(&mut cert_chain_reader) + .map(|x| CertificateDer::from(x.unwrap().to_vec())) + .collect(); + let mut key_reader = BufReader::new( + File::open(key_file) + .error(|e: &std::io::Error| { + format!("{COMPONENT} (error: {e}) - failed to open key file: {key_file}") + }) + .map_err(|_| QuicError::CertLoadError)?, + ); + let mut keys = rustls_pemfile::rsa_private_keys(&mut key_reader) + .filter(|key| key.is_ok()) + .map(|key| PrivateKeyDer::try_from(key.unwrap().secret_pkcs1_der().to_vec())) + .collect::, _>>() + .error(|e: &&str| format!("{COMPONENT} (error: {e}) - failed to parse private key")) + .map_err(|_| QuicError::CertLoadError)?; + let key = keys.remove(0); + Ok((certs, key)) +} diff --git a/core/server/src/quic/quic_socket.rs b/core/server/src/quic/quic_socket.rs new file mode 100644 index 0000000000..91acc54a1f --- /dev/null +++ b/core/server/src/quic/quic_socket.rs @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use socket2::{Domain, Protocol, Socket, Type}; +use std::net::SocketAddr; +use std::num::TryFromIntError; + +use crate::configs::quic::QuicSocketConfig; + +/// Build a UDP socket for the given address and configure the options that are +/// required by the server +pub fn build(addr: &SocketAddr, config: &QuicSocketConfig) -> Socket { + // Choose the correct address family based on the target address + let socket = Socket::new(Domain::for_address(*addr), Type::DGRAM, Some(Protocol::UDP)) + .expect("Unable to create a UDP socket"); + + // Allow multiple sockets (shards) to bind to the same address + socket + .set_reuse_address(true) + .expect("Unable to set SO_REUSEADDR on socket"); + + // SO_REUSEPORT is only available on Unix-like systems + #[cfg(unix)] + socket + .set_reuse_port(true) + .expect("Unable to set SO_REUSEPORT on socket"); + + // Configure socket buffer sizes and keepalive if override is enabled + if config.override_defaults { + config + .recv_buffer_size + .as_bytes_u64() + .try_into() + .map_err(|e: TryFromIntError| std::io::Error::other(e.to_string())) + .and_then(|size| socket.set_recv_buffer_size(size)) + .expect("Unable to set SO_RCVBUF on socket"); + + config + .send_buffer_size + .as_bytes_u64() + .try_into() + .map_err(|e: TryFromIntError| std::io::Error::other(e.to_string())) + .and_then(|size| socket.set_send_buffer_size(size)) + .expect("Unable to set SO_SNDBUF on socket"); + + socket + .set_keepalive(config.keepalive) + .expect("Unable to set SO_KEEPALIVE on socket"); + } + + socket +} diff --git a/core/server/src/sender/mod.rs b/core/server/src/sender/mod.rs new file mode 100644 index 0000000000..c6a0b4acaa --- /dev/null +++ b/core/server/src/sender/mod.rs @@ -0,0 +1,257 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod quic_sender; +mod tcp_sender; +mod tcp_tls_sender; +mod websocket_sender; +mod websocket_tls_sender; + +pub use quic_sender::QuicSender; +pub use tcp_sender::TcpSender; +pub use tcp_tls_sender::TcpTlsSender; +pub use websocket_sender::WebSocketSender; +pub use websocket_tls_sender::WebSocketTlsSender; + +use compio::BufResult; +use compio::buf::IoBufMut; +use compio::io::{AsyncReadExt, AsyncWriteExt}; +use compio::net::TcpStream; +use compio::quic::{RecvStream, SendStream}; +use compio::tls::TlsStream; +use iggy_common::IggyError; +use server_common::PooledBuffer; +use std::future::Future; +#[cfg(unix)] +use std::os::fd::{AsFd, OwnedFd}; +use tracing::debug; +#[cfg(unix)] +use tracing::error; + +macro_rules! forward_async_methods { + ( + $( + async fn $method_name:ident + $(<$($generic:ident $(: $bound:path)?),+>)? + ( + &mut self $(, $arg:ident : $arg_ty:ty )* + ) -> $ret:ty ; + )* + ) => { + $( + pub async fn $method_name + $(<$($generic $(: $bound)?),+>)? + (&mut self, $( $arg: $arg_ty ),* ) -> $ret { + match self { + Self::Tcp(d) => d.$method_name$(::<$($generic),+>)?($( $arg ),*).await, + Self::TcpTls(s) => s.$method_name$(::<$($generic),+>)?($( $arg ),*).await, + Self::Quic(s) => s.$method_name$(::<$($generic),+>)?($( $arg ),*).await, + Self::WebSocket(s) => s.$method_name$(::<$($generic),+>)?($( $arg ),*).await, + Self::WebSocketTls(s) => s.$method_name$(::<$($generic),+>)?($( $arg ),*).await, + } + } + )* + } +} + +pub trait Sender { + fn read(&mut self, buffer: B) -> impl Future, B)>; + fn send_empty_ok_response(&mut self) -> impl Future>; + fn send_ok_response(&mut self, payload: &[u8]) -> impl Future>; + fn send_ok_response_vectored( + &mut self, + length: &[u8], + slices: Vec, + ) -> impl Future>; + fn send_error_response( + &mut self, + error: IggyError, + ) -> impl Future>; + fn shutdown(&mut self) -> impl Future>; +} + +#[allow(clippy::large_enum_variant)] +#[derive(Debug)] +pub enum SenderKind { + Tcp(TcpSender), + TcpTls(TcpTlsSender), + Quic(QuicSender), + WebSocket(WebSocketSender), + WebSocketTls(WebSocketTlsSender), +} + +impl SenderKind { + pub fn get_tcp_sender(stream: TcpStream) -> Self { + Self::Tcp(TcpSender { + stream: Some(stream), + }) + } + + pub fn get_tcp_tls_sender(stream: TlsStream) -> Self { + Self::TcpTls(TcpTlsSender { stream }) + } + + pub fn get_quic_sender(send_stream: SendStream, recv_stream: RecvStream) -> Self { + Self::Quic(QuicSender { + send: send_stream, + recv: recv_stream, + }) + } + + pub fn get_websocket_sender(stream: WebSocketSender) -> Self { + Self::WebSocket(stream) + } + + pub fn get_websocket_tls_sender(stream: WebSocketTlsSender) -> Self { + Self::WebSocketTls(stream) + } + + #[cfg(unix)] + pub fn take_and_migrate_tcp(&mut self) -> Option { + match self { + SenderKind::Tcp(tcp_sender) => { + let stream = tcp_sender.stream.take()?; + let poll_fd = stream.into_poll_fd().ok()?; + + let raw_fd = poll_fd.as_fd(); + let Ok(owned_fd) = nix::unistd::dup(raw_fd) else { + // TODO(tungtose): recover tcp stream? + error!("Failed to dup fd"); + return None; + }; + + Some(owned_fd) + } + // TODO(tungtose): support TCP TLS + _ => None, + } + } + + forward_async_methods! { + async fn read(&mut self, buffer: B) -> (Result<(), IggyError>, B); + async fn send_empty_ok_response(&mut self) -> Result<(), IggyError>; + async fn send_ok_response(&mut self, payload: &[u8]) -> Result<(), IggyError>; + async fn send_ok_response_vectored(&mut self, length: &[u8], slices: Vec) -> Result<(), IggyError>; + async fn send_error_response(&mut self, error: IggyError) -> Result<(), IggyError>; + async fn shutdown(&mut self) -> Result<(), IggyError>; + } +} + +const STATUS_OK: &[u8] = &[0; 4]; + +pub(crate) async fn read(stream: &mut T, buffer: B) -> (Result<(), IggyError>, B) +where + T: AsyncReadExt + AsyncWriteExt + Unpin, + B: IoBufMut, +{ + let BufResult(result, buffer) = stream.read_exact(buffer).await; + match (result, buffer) { + (Ok(_), buffer) => (Ok(()), buffer), + (Err(e), buffer) => { + if e.kind() == std::io::ErrorKind::UnexpectedEof { + (Err(IggyError::ConnectionClosed), buffer) + } else { + (Err(IggyError::TcpError), buffer) + } + } + } +} + +pub(crate) async fn send_empty_ok_response(stream: &mut T) -> Result<(), IggyError> +where + T: AsyncReadExt + AsyncWriteExt + Unpin, +{ + send_ok_response(stream, &[]).await +} + +pub(crate) async fn send_ok_response(stream: &mut T, payload: &[u8]) -> Result<(), IggyError> +where + T: AsyncReadExt + AsyncWriteExt + Unpin, +{ + send_response(stream, STATUS_OK, payload).await +} + +pub(crate) async fn send_ok_response_vectored( + stream: &mut T, + length: &[u8], + slices: Vec, +) -> Result<(), IggyError> +where + T: AsyncReadExt + AsyncWriteExt + Unpin, +{ + send_response_vectored(stream, STATUS_OK, length, slices).await +} + +pub(crate) async fn send_error_response( + stream: &mut T, + error: IggyError, +) -> Result<(), IggyError> +where + T: AsyncReadExt + AsyncWriteExt + Unpin, +{ + send_response(stream, &error.as_code().to_le_bytes(), &[]).await +} + +pub(crate) async fn send_response( + stream: &mut T, + status: &[u8], + payload: &[u8], +) -> Result<(), IggyError> +where + T: AsyncReadExt + AsyncWriteExt + Unpin, +{ + debug!( + "Sending response of len: {} with status: {:?}...", + payload.len(), + status + ); + let length = (payload.len() as u32).to_le_bytes(); + stream + .write_all([status, &length, payload].concat()) + .await + .0 + .map_err(|_| IggyError::TcpError)?; + debug!("Sent response with status: {:?}", status); + Ok(()) +} + +pub(crate) async fn send_response_vectored( + stream: &mut T, + status: &[u8], + length: &[u8], + mut slices: Vec, +) -> Result<(), IggyError> +where + T: AsyncReadExt + AsyncWriteExt + Unpin, +{ + let resp_status = u32::from_le_bytes(status.try_into().unwrap()); + debug!( + "Sending vectored response of len: {} with status: {:?}...", + slices.len(), + resp_status + ); + let status = PooledBuffer::from(status); + let length = PooledBuffer::from(length); + slices.splice(0..0, [status, length]); + stream + .write_vectored_all(slices) + .await + .0 + .map_err(|_| IggyError::TcpError)?; + debug!("Sent response with status: {:?}", resp_status); + Ok(()) +} diff --git a/core/server/src/sender/quic_sender.rs b/core/server/src/sender/quic_sender.rs new file mode 100644 index 0000000000..2759e3b34d --- /dev/null +++ b/core/server/src/sender/quic_sender.rs @@ -0,0 +1,140 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::Sender; +use compio::BufResult; +use compio::buf::IoBufMut; +use compio::io::{AsyncReadExt, AsyncWriteExt}; +use compio::quic::{ClosedStream, RecvStream, SendStream}; +use err_trail::ErrContext; +use iggy_common::IggyError; +use server_common::PooledBuffer; +use tracing::{debug, error}; + +const COMPONENT: &str = "QUIC"; +const STATUS_OK: &[u8] = &[0; 4]; + +#[derive(Debug)] +pub struct QuicSender { + pub(crate) send: SendStream, + pub(crate) recv: RecvStream, +} + +impl Sender for QuicSender { + /// Reads data from the QUIC stream directly into the buffer. + async fn read(&mut self, buffer: B) -> (Result<(), IggyError>, B) { + let BufResult(result, buffer) = + ::read_exact(&mut self.recv, buffer).await; + match (result, buffer) { + (Ok(_), buffer) => (Ok(()), buffer), + (Err(error), buffer) => { + error!("Failed to read from the stream: {:?}", error); + (Err(IggyError::QuicError), buffer) + } + } + } + + async fn send_empty_ok_response(&mut self) -> Result<(), IggyError> { + self.send_ok_response(&[]).await + } + + async fn send_ok_response(&mut self, payload: &[u8]) -> Result<(), IggyError> { + self.send_response(STATUS_OK, payload).await + } + + async fn send_error_response(&mut self, error: IggyError) -> Result<(), IggyError> { + self.send_response(&error.as_code().to_le_bytes(), &[]) + .await + } + + async fn shutdown(&mut self) -> Result<(), IggyError> { + Ok(()) + } + + async fn send_ok_response_vectored( + &mut self, + length: &[u8], + slices: Vec, + ) -> Result<(), IggyError> { + debug!("Sending vectored response with status: {:?}...", STATUS_OK); + + let headers = [STATUS_OK, length].concat(); + let BufResult(result, _) = self.send.write_all(headers).await; + result + .error(|e: &std::io::Error| { + format!("{COMPONENT} (error: {e}) - failed to write headers to stream") + }) + .map_err(|_| IggyError::QuicError)?; + + let mut total_bytes_written = 0; + + for slice in slices { + let slice_len = slice.len(); + if slice_len > 0 { + let BufResult(result, _) = self.send.write_all(slice).await; + result + .error(|e: &std::io::Error| { + format!("{COMPONENT} (error: {e}) - failed to write slice to stream") + }) + .map_err(|_| IggyError::QuicError)?; + + total_bytes_written += slice_len; + } + } + + debug!( + "Sent vectored response: {} bytes of payload", + total_bytes_written + ); + + self.send + .finish() + .error(|e: &ClosedStream| { + format!("{COMPONENT} (error: {e}) - failed to finish send stream") + }) + .map_err(|_| IggyError::QuicError)?; + + debug!("Sent vectored response with status: {:?}", STATUS_OK); + Ok(()) + } +} + +impl QuicSender { + async fn send_response(&mut self, status: &[u8], payload: &[u8]) -> Result<(), IggyError> { + debug!( + "Sending response of len: {} with status: {:?}...", + payload.len(), + status + ); + let length = (payload.len() as u32).to_le_bytes(); + let data = [status, &length, payload].concat(); + let BufResult(result, _) = self.send.write_all(data).await; + result + .error(|e: &std::io::Error| { + format!("{COMPONENT} (error: {e}) - failed to write buffer to the stream") + }) + .map_err(|_| IggyError::QuicError)?; + self.send + .finish() + .error(|e: &ClosedStream| { + format!("{COMPONENT} (error: {e}) - failed to finish send stream") + }) + .map_err(|_| IggyError::QuicError)?; + debug!("Sent response with status: {:?}", status); + Ok(()) + } +} diff --git a/core/server/src/sender/tcp_sender.rs b/core/server/src/sender/tcp_sender.rs new file mode 100644 index 0000000000..15dba86da3 --- /dev/null +++ b/core/server/src/sender/tcp_sender.rs @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::Sender; +use compio::buf::IoBufMut; +use compio::io::AsyncWrite; +use compio::net::TcpStream; +use err_trail::ErrContext; +use iggy_common::IggyError; +use server_common::PooledBuffer; + +const COMPONENT: &str = "TCP"; + +#[derive(Debug)] +pub struct TcpSender { + pub(crate) stream: Option, +} + +impl Sender for TcpSender { + async fn read(&mut self, buffer: B) -> (Result<(), IggyError>, B) { + match self.stream.as_mut() { + Some(stream) => super::read(stream, buffer).await, + None => (Err(IggyError::ConnectionClosed), buffer), + } + } + + async fn send_empty_ok_response(&mut self) -> Result<(), IggyError> { + match self.stream.as_mut() { + Some(stream) => super::send_empty_ok_response(stream).await, + None => Err(IggyError::ConnectionClosed), + } + } + + async fn send_ok_response(&mut self, payload: &[u8]) -> Result<(), IggyError> { + match self.stream.as_mut() { + Some(stream) => super::send_ok_response(stream, payload).await, + None => Err(IggyError::ConnectionClosed), + } + } + + async fn send_error_response(&mut self, error: IggyError) -> Result<(), IggyError> { + match self.stream.as_mut() { + Some(stream) => super::send_error_response(stream, error).await, + None => Err(IggyError::ConnectionClosed), + } + } + + async fn shutdown(&mut self) -> Result<(), IggyError> { + match self.stream.as_mut() { + Some(stream) => stream + .shutdown() + .await + .error(|e: &std::io::Error| { + format!("{COMPONENT} (error: {e}) - failed to shutdown TCP stream") + }) + .map_err(|e| IggyError::IoError(e.to_string())), + + None => Err(IggyError::ConnectionClosed), + } + } + + async fn send_ok_response_vectored( + &mut self, + length: &[u8], + slices: Vec, + ) -> Result<(), IggyError> { + if self.stream.is_none() { + tracing::error!("Tried to send but stream is None!"); + } + match self.stream.as_mut() { + Some(stream) => super::send_ok_response_vectored(stream, length, slices).await, + + None => Err(IggyError::ConnectionClosed), + } + } +} diff --git a/core/server/src/sender/tcp_tls_sender.rs b/core/server/src/sender/tcp_tls_sender.rs new file mode 100644 index 0000000000..346aa46d2a --- /dev/null +++ b/core/server/src/sender/tcp_tls_sender.rs @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::Sender; +use compio::buf::IoBufMut; +use compio::io::AsyncWrite; +use compio::net::TcpStream; +use compio::tls::TlsStream; +use err_trail::ErrContext; +use iggy_common::IggyError; +use server_common::PooledBuffer; + +const COMPONENT: &str = "TCP"; + +#[derive(Debug)] +pub struct TcpTlsSender { + pub(crate) stream: TlsStream, +} + +impl Sender for TcpTlsSender { + async fn read(&mut self, buffer: B) -> (Result<(), IggyError>, B) { + super::read(&mut self.stream, buffer).await + } + + async fn send_empty_ok_response(&mut self) -> Result<(), IggyError> { + super::send_empty_ok_response(&mut self.stream).await?; + self.stream + .flush() + .await + .error(|e: &std::io::Error| { + format!("failed to flush TCP stream after sending response: {e}") + }) + .map_err(|_| IggyError::TcpError) + } + + async fn send_ok_response(&mut self, payload: &[u8]) -> Result<(), IggyError> { + super::send_ok_response(&mut self.stream, payload).await?; + self.stream + .flush() + .await + .error(|e: &std::io::Error| { + format!("failed to flush TCP stream after sending response: {e}") + }) + .map_err(|_| IggyError::TcpError) + } + + async fn send_error_response(&mut self, error: IggyError) -> Result<(), IggyError> { + super::send_error_response(&mut self.stream, error).await?; + self.stream + .flush() + .await + .error(|e: &std::io::Error| { + format!("failed to flush TCP stream after sending response: {e}") + }) + .map_err(|_| IggyError::TcpError) + } + + async fn shutdown(&mut self) -> Result<(), IggyError> { + self.stream + .shutdown() + .await + .error(|e: &std::io::Error| { + format!("{COMPONENT} (error: {e}) - failed to shutdown TCP TLS stream") + }) + .map_err(|e| IggyError::IoError(e.to_string())) + } + + async fn send_ok_response_vectored( + &mut self, + length: &[u8], + slices: Vec, + ) -> Result<(), IggyError> { + super::send_ok_response_vectored(&mut self.stream, length, slices).await?; + self.stream + .flush() + .await + .error(|e: &std::io::Error| { + format!("failed to flush TCP stream after sending response: {e}") + }) + .map_err(|_| IggyError::TcpError) + } +} diff --git a/core/server/src/sender/websocket_sender.rs b/core/server/src/sender/websocket_sender.rs new file mode 100644 index 0000000000..9e92a72f05 --- /dev/null +++ b/core/server/src/sender/websocket_sender.rs @@ -0,0 +1,206 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::Sender; +use bytes::{BufMut, BytesMut}; +use compio::buf::IoBufMut; +use compio::net::TcpStream; +use compio::ws::WebSocketStream; +use compio::ws::tungstenite::{Error as TungsteniteError, Message}; +use iggy_common::IggyError; +use server_common::PooledBuffer; +use std::ptr; +use tracing::{debug, warn}; + +const READ_BUFFER_CAPACITY: usize = 8192; +const WRITE_BUFFER_CAPACITY: usize = 8192; +const STATUS_OK: &[u8] = &[0; 4]; + +pub struct WebSocketSender { + pub(crate) stream: WebSocketStream, + pub(crate) read_buffer: BytesMut, + pub(crate) write_buffer: BytesMut, +} + +impl WebSocketSender { + pub fn new(stream: WebSocketStream) -> Self { + Self { + stream, + read_buffer: BytesMut::with_capacity(READ_BUFFER_CAPACITY), + write_buffer: BytesMut::with_capacity(WRITE_BUFFER_CAPACITY), + } + } + + async fn flush_write_buffer(&mut self) -> Result<(), IggyError> { + if self.write_buffer.is_empty() { + return Ok(()); + } + let data = self.write_buffer.split().freeze(); + debug!("WebSocket sending data: {:?}", data.to_vec()); + + self.stream.send(Message::Binary(data)).await.map_err(|e| { + debug!("WebSocket send error: {:?}", e); + match e { + TungsteniteError::ConnectionClosed | TungsteniteError::AlreadyClosed => { + IggyError::ConnectionClosed + } + TungsteniteError::Io(ref io_err) + if io_err.kind() == std::io::ErrorKind::BrokenPipe => + { + warn!("Broken pipe detected (client closed connection)"); + IggyError::ConnectionClosed + } + _ => IggyError::TcpError, + } + }) + } +} + +impl std::fmt::Debug for WebSocketSender { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("WebSocketSender").finish() + } +} + +impl Sender for WebSocketSender { + async fn read(&mut self, mut buffer: B) -> (Result<(), IggyError>, B) { + let required_len = buffer.buf_capacity(); + if required_len == 0 { + return (Ok(()), buffer); + } + + while self.read_buffer.len() < required_len { + match self.stream.read().await { + Ok(Message::Binary(data)) => { + self.read_buffer.extend_from_slice(&data); + } + Ok(Message::Close(_)) => { + return (Err(IggyError::ConnectionClosed), buffer); + } + Ok(Message::Ping(data)) => { + if self.stream.send(Message::Pong(data)).await.is_err() { + return (Err(IggyError::ConnectionClosed), buffer); + } + } + Ok(_) => { /* Ignore other message types */ } + Err(_) => { + return (Err(IggyError::ConnectionClosed), buffer); + } + } + } + + let data_to_copy = self.read_buffer.split_to(required_len); + + unsafe { + ptr::copy_nonoverlapping( + data_to_copy.as_ptr(), + buffer.buf_mut_ptr().cast::(), + required_len, + ); + buffer.set_len(required_len); + } + + (Ok(()), buffer) + } + + async fn send_empty_ok_response(&mut self) -> Result<(), IggyError> { + self.send_ok_response(&[]).await + } + + async fn send_ok_response(&mut self, payload: &[u8]) -> Result<(), IggyError> { + debug!( + "Sending WebSocket response with status: OK, payload length: {}", + payload.len() + ); + + let length = (payload.len() as u32).to_le_bytes(); + let total_size = STATUS_OK.len() + length.len() + payload.len(); + + if self.write_buffer.len() + total_size > self.write_buffer.capacity() { + self.flush_write_buffer().await?; + } + + self.write_buffer.put_slice(STATUS_OK); + self.write_buffer.put_slice(&length); + self.write_buffer.put_slice(payload); + + self.flush_write_buffer().await + } + + async fn send_error_response(&mut self, error: IggyError) -> Result<(), IggyError> { + let status = &error.as_code().to_le_bytes(); + debug!("Sending WebSocket error response with status: {:?}", status); + let length = 0u32.to_le_bytes(); + let total_size = status.len() + length.len(); + + if self.write_buffer.len() + total_size > self.write_buffer.capacity() { + self.flush_write_buffer().await?; + } + self.write_buffer.put_slice(status); + self.write_buffer.put_slice(&length); + self.flush_write_buffer().await + } + + async fn shutdown(&mut self) -> Result<(), IggyError> { + self.flush_write_buffer().await?; + + match self.stream.close(None).await { + Ok(_) => Ok(()), + Err(e) => match e { + TungsteniteError::ConnectionClosed | TungsteniteError::AlreadyClosed => { + debug!("WebSocket connection already closed: {}", e); + Ok(()) + } + _ => Err(IggyError::CannotCloseWebSocketConnection(format!("{}", e))), + }, + } + } + + async fn send_ok_response_vectored( + &mut self, + length: &[u8], + slices: Vec, + ) -> Result<(), IggyError> { + self.flush_write_buffer().await?; + + let total_payload_size = slices.iter().map(|s| s.len()).sum::(); + let total_size = STATUS_OK.len() + length.len() + total_payload_size; + + let mut response_bytes = BytesMut::with_capacity(total_size); + response_bytes.put_slice(STATUS_OK); + response_bytes.put_slice(length); + for slice in slices { + response_bytes.put_slice(&slice); + } + + self.stream + .send(Message::Binary(response_bytes.freeze())) + .await + .map_err(|e| match e { + TungsteniteError::ConnectionClosed | TungsteniteError::AlreadyClosed => { + IggyError::ConnectionClosed + } + TungsteniteError::Io(ref io_err) + if io_err.kind() == std::io::ErrorKind::BrokenPipe => + { + warn!("Broken pipe in vectored send - client closed connection"); + IggyError::ConnectionClosed + } + _ => IggyError::TcpError, + }) + } +} diff --git a/core/server/src/sender/websocket_tls_sender.rs b/core/server/src/sender/websocket_tls_sender.rs new file mode 100644 index 0000000000..8a25854274 --- /dev/null +++ b/core/server/src/sender/websocket_tls_sender.rs @@ -0,0 +1,185 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::Sender; +use bytes::{BufMut, BytesMut}; +use compio::buf::IoBufMut; +use compio::net::TcpStream; +use compio::ws::WebSocketStream; +use compio::ws::tungstenite::{Error as TungsteniteError, Message}; +use iggy_common::IggyError; +use server_common::PooledBuffer; +use std::ptr; +use tracing::debug; + +const READ_BUFFER_CAPACITY: usize = 8192; +const WRITE_BUFFER_CAPACITY: usize = 8192; +const STATUS_OK: &[u8] = &[0; 4]; + +pub struct WebSocketTlsSender { + pub(crate) stream: WebSocketStream, + pub(crate) read_buffer: BytesMut, + pub(crate) write_buffer: BytesMut, +} + +impl WebSocketTlsSender { + pub fn new(stream: WebSocketStream) -> Self { + Self { + stream, + read_buffer: BytesMut::with_capacity(READ_BUFFER_CAPACITY), + write_buffer: BytesMut::with_capacity(WRITE_BUFFER_CAPACITY), + } + } + + async fn flush_write_buffer(&mut self) -> Result<(), IggyError> { + if self.write_buffer.is_empty() { + return Ok(()); + } + let data = self.write_buffer.split().freeze(); + self.stream + .send(Message::Binary(data)) + .await + .map_err(|_| IggyError::TcpError) + } +} + +impl std::fmt::Debug for WebSocketTlsSender { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("WebSocketTlsSender").finish() + } +} + +impl Sender for WebSocketTlsSender { + async fn read(&mut self, mut buffer: B) -> (Result<(), IggyError>, B) { + let required_len = buffer.buf_capacity(); + if required_len == 0 { + return (Ok(()), buffer); + } + + while self.read_buffer.len() < required_len { + match self.stream.read().await { + Ok(Message::Binary(data)) => { + self.read_buffer.extend_from_slice(&data); + } + Ok(Message::Close(_)) => { + return (Err(IggyError::ConnectionClosed), buffer); + } + Ok(Message::Ping(data)) => { + if self.stream.send(Message::Pong(data)).await.is_err() { + return (Err(IggyError::ConnectionClosed), buffer); + } + } + Ok(_) => { /* Ignore other message types */ } + Err(_) => { + return (Err(IggyError::ConnectionClosed), buffer); + } + } + } + + let data_to_copy = self.read_buffer.split_to(required_len); + + unsafe { + ptr::copy_nonoverlapping( + data_to_copy.as_ptr(), + buffer.buf_mut_ptr().cast::(), + required_len, + ); + buffer.set_len(required_len); + } + + (Ok(()), buffer) + } + + async fn send_empty_ok_response(&mut self) -> Result<(), IggyError> { + self.send_ok_response(&[]).await + } + + async fn send_ok_response(&mut self, payload: &[u8]) -> Result<(), IggyError> { + debug!( + "Sending WebSocket TLS response with status: OK, payload length: {}", + payload.len() + ); + + let length = (payload.len() as u32).to_le_bytes(); + let total_size = STATUS_OK.len() + length.len() + payload.len(); + + if self.write_buffer.len() + total_size > self.write_buffer.capacity() { + self.flush_write_buffer().await?; + } + + self.write_buffer.put_slice(STATUS_OK); + self.write_buffer.put_slice(&length); + self.write_buffer.put_slice(payload); + + self.flush_write_buffer().await + } + + async fn send_error_response(&mut self, error: IggyError) -> Result<(), IggyError> { + let status = &error.as_code().to_le_bytes(); + debug!( + "Sending WebSocket TLS error response with status: {:?}", + status + ); + let length = 0u32.to_le_bytes(); + let total_size = status.len() + length.len(); + + if self.write_buffer.len() + total_size > self.write_buffer.capacity() { + self.flush_write_buffer().await?; + } + self.write_buffer.put_slice(status); + self.write_buffer.put_slice(&length); + self.flush_write_buffer().await + } + + async fn shutdown(&mut self) -> Result<(), IggyError> { + self.flush_write_buffer().await?; + + match self.stream.close(None).await { + Ok(_) => Ok(()), + Err(e) => match e { + TungsteniteError::ConnectionClosed | TungsteniteError::AlreadyClosed => { + debug!("WebSocket TLS connection already closed: {}", e); + Ok(()) + } + _ => Err(IggyError::CannotCloseWebSocketConnection(format!("{}", e))), + }, + } + } + + async fn send_ok_response_vectored( + &mut self, + length: &[u8], + slices: Vec, + ) -> Result<(), IggyError> { + self.flush_write_buffer().await?; + + let total_payload_size = slices.iter().map(|s| s.len()).sum::(); + let total_size = STATUS_OK.len() + length.len() + total_payload_size; + + let mut response_bytes = BytesMut::with_capacity(total_size); + response_bytes.put_slice(STATUS_OK); + response_bytes.put_slice(length); + for slice in slices { + response_bytes.put_slice(&slice); + } + + self.stream + .send(Message::Binary(response_bytes.freeze())) + .await + .map_err(|_| IggyError::TcpError) + } +} diff --git a/core/server/src/server_error.rs b/core/server/src/server_error.rs index 85281932c0..94d20d4c56 100644 --- a/core/server/src/server_error.rs +++ b/core/server/src/server_error.rs @@ -15,398 +15,87 @@ // specific language governing permissions and limitations // under the License. -use consensus::VsrStateError; -use metadata::impls::recovery::RecoveryError; -use server_common::log::LogError; -use shard::ShardCtorError; -use shard_allocator::ShardingError; -use std::path::PathBuf; -use thiserror::Error; +use compio::quic::{ConnectionError as QuicConnectionError, ReadError, WriteError}; +use error_set::error_set; +use std::array::TryFromSliceError; +use std::io; -#[derive(Debug, Error)] -#[non_exhaustive] -pub enum ServerError { - #[error(transparent)] - Iggy(Box), - #[error("failed to load server config")] - Config(#[source] configs::ConfigurationError), - #[error("failed to allocate shards from sharding.cpu_allocation")] - ShardAllocator(#[source] ShardingError), - #[error("failed to bind shard {shard_id} to its CPU set")] - CpuAffinityFailed { - shard_id: u16, - #[source] - source: ShardingError, - }, - #[error("failed to bind shard {shard_id} memory to its NUMA node")] - MemoryAffinityFailed { - shard_id: u16, - #[source] - source: ShardingError, - }, - #[error("failed to spawn OS thread for shard {shard_id}")] - ShardSpawnFailed { - shard_id: u16, - #[source] - source: std::io::Error, - }, - // `{source}` is deliberately part of the Display text: the shard-join - // failure report and `%error` log fields print Display only, and the - // source carries the io_uring remediation folded in by - // `server_common::diagnostics::enrich_runtime_create_error`. - #[error("failed to create io_uring runtime for shard {shard_id}: {source}")] - ShardRuntimeCreateFailed { - shard_id: u16, - #[source] - source: std::io::Error, - }, - #[error( - "shard allocator produced zero shards; server must run at least one \ - shard (check [system.sharding] cpu_allocation)" - )] - ShardsCountZero, - #[error( - "computed shards_count = {count} exceeds the maximum of {} shards per \ - server; shard ids must fit in u16 and stay below the OWNER_NONE \ - sentinel", - message_bus::OWNER_NONE - 1 - )] - ShardsCountOverflow { count: usize }, - #[error("system.sharding.inbox_capacity must be in 1..={max}; got {value}")] - InvalidInboxCapacity { value: usize, max: usize }, - #[error("system.sharding.shutdown_drain_timeout must be in (0, {max:?}]; got {value:?}")] - InvalidShutdownDrainTimeout { - value: std::time::Duration, - max: std::time::Duration, - }, - #[error("system.sharding.shutdown_poll_interval must be in (0, {max:?}]; got {value:?}")] - InvalidShutdownPollInterval { - value: std::time::Duration, - max: std::time::Duration, - }, - #[error( - "system.sharding.shutdown_poll_interval ({poll:?}) must be <= \ - shutdown_drain_timeout ({drain:?})" - )] - ShutdownPollExceedsDrain { - poll: std::time::Duration, - drain: std::time::Duration, - }, - #[error("failed to serialize current server config")] - CurrentConfigSerialize(#[source] toml::ser::Error), - #[error("failed to write current server config at {path}")] - CurrentConfigWrite { - path: String, - #[source] - source: std::io::Error, - }, - #[error("failed to initialize server logging")] - Logging(#[source] LogError), - #[error("failed to recover metadata snapshot and journal")] - MetadataRecovery(#[source] RecoveryError), - #[error("failed to open partition superblock at {dir}")] - PartitionSuperblockIo { - dir: PathBuf, - #[source] - source: std::io::Error, - }, - // Quarantines the one partition rather than treating the group as fresh or - // reading through to a superseded view: mirrors the metadata plane's - // `RecoveryError::SuperblockUnreadable` policy, minus the boot refusal, - // because one unreadable partition directory must not strand every healthy - // group on the shard. - #[error( - "partition superblock at {dir} is present but its format version \ - {version} is unrecognized by this build (a downgrade, or a corrupt \ - version field)" - )] - PartitionSuperblockVersionUnknown { dir: PathBuf, version: u16 }, - #[error( - "partition superblock at {dir} is present but a copy holds bytes that \ - do not verify (bit-rot or a checksum failure), so its latest \ - generation cannot be established" - )] - PartitionSuperblockUnverifiable { dir: PathBuf }, - #[error( - "partition superblock at {dir} was checksum-clean but did not decode; \ - tombstoning this partition rather than inferring a stale view" - )] - PartitionSuperblockUndecodable { - dir: PathBuf, - #[source] - source: VsrStateError, - }, - #[error( - "partition superblock at {dir} belongs to a different {field}: expected \ - {expected}, found {found}; a copied or misplaced data directory, or the \ - cluster was resized without reconfiguration" - )] - PartitionSuperblockIdentityMismatch { - dir: PathBuf, - field: metadata::IdentityField, - expected: u128, - found: u128, - }, - // Per-partition, not fatal: the boot path fences this one group (quarantines - // its segment files and materialises it fresh) instead of taking the node - // down for one damaged local chain. The shapes it reports are exactly what a - // failed state-transfer quarantine leaves behind, and the rebuild recovers - // the data from a peer. - #[error( - "partition {stream_id}/{topic_id}/{partition_id} at {dir} recovered an \ - unusable segment chain: {reason}" - )] - PartitionChainRefused { - dir: PathBuf, - stream_id: usize, - topic_id: usize, - partition_id: usize, - reason: PartitionChainRefusal, - }, - #[error( - "shard {shard_id} aborted while waiting for shard-0 to broadcast the metadata \ - factory bundle; shard 0 dropped its sender (most likely it failed to recover)" - )] - MetadataHandoffAborted { shard_id: u16 }, - #[error( - "shard 0 aborted before binding listeners with {remaining} peer shard(s) still loading \ - their on-disk partitions; a peer most likely failed during bootstrap (shutdown flag set)" - )] - ShardBootstrapBarrierAborted { remaining: usize }, - #[error("failed to parse {context} socket address '{address}'")] - SocketAddressParse { - context: &'static str, - address: String, - #[source] - source: std::net::AddrParseError, - }, - #[error("cluster enabled but no node is configured for replica {replica_id}")] - ClusterNodeNotFound { replica_id: u8 }, - #[error("cluster node count {count} exceeds supported u8 replica count")] - ClusterReplicaCountTooLarge { count: usize }, - #[error("cluster mode requires --replica-id to identify the current node")] - MissingReplicaId, - #[error( - "--replica-id {supplied} was passed with cluster.enabled=false; the WAL would commit \ - under replica {default} which permanently fixes this node's identity. Either set \ - cluster.enabled=true with a matching nodes[] entry, or drop --replica-id" - )] - ReplicaIdRequiresCluster { supplied: u8, default: u8 }, - #[error( - "cluster node for replica {replica_id} is missing ports.{transport}; cluster mode \ - requires an explicit roster port for every enabled transport" - )] - ClusterPortMissing { - transport: &'static str, - replica_id: u8, - }, - #[error( - "cluster bootstrap with empty metadata requires both {username_env} and {password_env} to be set before server can create the root user deterministically" - )] - ClusterRootCredentialsRequired { - username_env: &'static str, - password_env: &'static str, - }, - #[error( - "{provided_env} is set but {missing_env} is not; the root user credentials must be \ - provided as a pair" - )] - RootCredentialsIncomplete { - provided_env: &'static str, - missing_env: &'static str, - }, - #[error("{env_name} must be {min}..={max} characters long; got {length}")] - RootCredentialLength { - env_name: &'static str, - length: usize, - min: usize, - max: usize, - }, - #[error("--fresh could not remove the system path at {path}")] - FreshWipeFailed { - path: PathBuf, - #[source] - source: std::io::Error, - }, - #[error( - "recovered segment for stream {stream_id}, topic {topic_id}, partition {partition_id} at start_offset {start_offset} has message/index divergence (messages_size={messages_size_bytes}, indexed_size={indexed_size_bytes}, end_offset={end_offset}); recovery aborted before opening listeners. Restore the partition from a healthy replica or snapshot, or move the segment aside for offline repair before restarting." - )] - RecoveredSegmentSizeDivergence { - stream_id: usize, - topic_id: usize, - partition_id: usize, - start_offset: u64, - end_offset: u64, - messages_size_bytes: u64, - indexed_size_bytes: u64, - }, - #[error( - "failed to load persisted {consumer_kind} offsets for stream {stream_id}, topic {topic_id}, partition {partition_id} from {path}" - )] - ConsumerOffsetsLoad { - consumer_kind: &'static str, - stream_id: usize, - topic_id: usize, - partition_id: usize, - path: String, - #[source] - source: Box, - }, - #[error( - "recovered namespace stream {stream_id}, topic {topic_id}, partition {partition_id} exceeds configured limits (max_streams={max_streams}, max_topics={max_topics}, max_partitions={max_partitions})" - )] - RecoveredNamespaceOutOfBounds { - stream_id: usize, - topic_id: usize, - partition_id: usize, - max_streams: usize, - max_topics: usize, - max_partitions: usize, - }, - #[error("failed to load {transport} listener credentials")] - ListenerCredentials { - transport: &'static str, - #[source] - source: std::io::Error, - }, - #[error("failed to build the HTTP forward client: {reason}")] - HttpForwardClient { reason: String }, - #[error("failed to construct IggyShard from bootstrap inputs")] - ShardConstruction(#[source] ShardCtorError), - #[error("{} shard thread(s) failed: {}", failures.len(), format_shard_failures(failures))] - ShardJoinFailures { failures: Vec }, -} +error_set!( + ServerError := NumaError || ConfigurationError || ArchiverError || ConnectionError || LogError || CompatError || QuicError || ShardError -/// Why a recovered segment chain cannot be served. -/// -/// Both shapes mean the same thing operationally -- the local files do not form -/// a chain this replica can serve -- but they are distinguished because they -/// point at different causes: an empty non-tail segment is a failed rebuild's -/// orphan pairing, a hole is a stray or half-unlinked file. -#[derive(Debug)] -pub enum PartitionChainRefusal { - EmptyNonTailSegment { - empty_start: u64, - next_start: u64, - }, - Hole { - previous_start: u64, - previous_end: u64, - next_start: u64, - }, -} + IoError := { + #[display("IO error")] + IoError(io::Error), -impl std::fmt::Display for PartitionChainRefusal { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::EmptyNonTailSegment { - empty_start, - next_start, - } => write!( - f, - "segment {empty_start} is empty yet {next_start} follows it, so the \ - chain cannot be served past it" - ), - Self::Hole { - previous_start, - previous_end, - next_start, - } => write!( - f, - "segment {previous_start} ends at offset {previous_end} but the next \ - starts at {next_start}, leaving a hole" - ), - } + #[display("Write error")] + WriteError(WriteError), + + #[display("Read error")] + ReadToEndError(ReadError) + } + + NumaError := { + #[display("{0}")] + Sharding(shard_allocator::ShardingError), } -} -/// Per-shard outcome captured by [`crate::bootstrap::ShardHandles::join_all`] -/// when a shard either returned `Err` or panicked. -/// -/// Bundled into [`ServerError::ShardJoinFailures`] so the operator sees -/// every failing shard rather than only the first one, which previously -/// lived in the trace log alone. -#[derive(Debug)] -pub struct ShardJoinFailure { - pub shard_id: u16, - pub kind: ShardJoinFailureKind, -} + ConfigurationError := { + ConfigurationError(configs::ConfigurationError), + } + + ArchiverError := { + #[display("File to archive not found: {}", file_path)] + FileToArchiveNotFound { file_path: String }, + + #[display("Cannot initialize S3 archiver")] + CannotInitializeS3Archiver, -#[derive(Debug)] -pub enum ShardJoinFailureKind { - Error(Box), - Panic { - message: String, - }, - /// The shard thread never finished inside `shutdown_join_timeout` - /// and was abandoned so process exit is not blocked forever. - Wedged { - waited: std::time::Duration, - }, -} + #[display("Invalid S3 credentials")] + InvalidS3Credentials, -fn format_shard_failures(failures: &[ShardJoinFailure]) -> String { - use std::fmt::Write as _; - let mut out = String::new(); - for (idx, failure) in failures.iter().enumerate() { - if idx > 0 { - out.push_str("; "); - } - match &failure.kind { - ShardJoinFailureKind::Error(err) => { - let _ = write!(out, "shard {} -> {err}", failure.shard_id); - } - ShardJoinFailureKind::Panic { message } => { - let _ = write!(out, "shard {} panicked: {message}", failure.shard_id); - } - ShardJoinFailureKind::Wedged { waited } => { - let _ = write!( - out, - "shard {} wedged: thread still running after {waited:?}, abandoned", - failure.shard_id - ); - } - } + #[display("HTTP request error: {0}")] + CyperError(cyper::Error), + + #[display("Cannot archive file: {}", file_path)] + CannotArchiveFile { file_path: String }, + } || IoError + + ConnectionError := { + #[display("Connection error")] + QuicConnectionError(QuicConnectionError), + } || IoError || CommonError + + LogError := { + #[display("{0}")] + Logging(server_common::log::LogError), } - out -} -impl From for ServerError { - fn from(source: iggy_common::IggyError) -> Self { - Self::Iggy(Box::new(source)) + CompatError := { + #[display("Index migration error")] + IndexMigrationError, + } || IoError || CommonError + + CommonError := { + #[display("Try from slice error")] + TryFromSliceError(TryFromSliceError), + + #[display("SDK error")] + SdkError(iggy_common::IggyError), } -} -#[cfg(test)] -mod tests { - use super::*; + QuicError := { + #[display("Cert load error")] + CertLoadError, + #[display("Cert generation error")] + CertGenerationError, + #[display("Config creation error")] + ConfigCreationError, + #[display("Transport config error")] + TransportConfigError, + } - #[test] - fn shard_join_failures_display_aggregates_all_entries() { - let failures = vec![ - ShardJoinFailure { - shard_id: 0, - kind: ShardJoinFailureKind::Error(Box::new(ServerError::MissingReplicaId)), - }, - ShardJoinFailure { - shard_id: 2, - kind: ShardJoinFailureKind::Panic { - message: "boom".to_string(), - }, - }, - ]; - let rendered = ServerError::ShardJoinFailures { failures }.to_string(); - assert!( - rendered.starts_with("2 shard thread(s) failed:"), - "expected count prefix, got {rendered}" - ); - assert!( - rendered.contains("shard 0 ->"), - "shard 0 entry missing: {rendered}" - ); - assert!( - rendered.contains("shard 2 panicked: boom"), - "shard 2 panic entry missing: {rendered}" - ); + ShardError := { + #[display("Shard failed: {}", message)] + ShardFailure { message: String }, } -} +); diff --git a/core/server/src/shard/builder.rs b/core/server/src/shard/builder.rs new file mode 100644 index 0000000000..817276484e --- /dev/null +++ b/core/server/src/shard/builder.rs @@ -0,0 +1,197 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::{ + IggyShard, TaskRegistry, transmission::connector::ShardConnector, + transmission::frame::ShardFrame, +}; +use crate::metadata::{Metadata, MetadataWriter}; +use crate::streaming::partitions::local_partitions::LocalPartitions; +use crate::{ + configs::server::ServerConfig, + state::file::FileState, + streaming::{ + clients::client_manager::ClientManager, diagnostics::metrics::Metrics, + utils::ptr::EternalPtr, + }, +}; +use ahash::AHashSet; +use dashmap::DashMap; +use iggy_common::EncryptorKind; +use iggy_common::SemanticVersion; +use server_common::sharding::{IggyNamespace, PartitionLocation}; +use std::{ + cell::{Cell, RefCell}, + rc::Rc, + sync::atomic::AtomicBool, +}; + +#[derive(Default)] +pub struct IggyShardBuilder { + id: Option, + shards_table: Option>>, + state: Option, + client_manager: Option, + connections: Option>>, + config: Option, + encryptor: Option, + version: Option, + metrics: Option, + is_follower: bool, + /// Runtime-supplied replica identity. Resolved from the `--replica-id` + /// CLI flag and matched against `config.cluster.nodes[*].replica_id` at + /// startup. `None` when cluster mode is disabled. + current_replica_id: Option, + metadata: Option, + metadata_writer: Option, +} + +impl IggyShardBuilder { + pub fn id(mut self, id: u16) -> Self { + self.id = Some(id); + self + } + + pub fn connections(mut self, connections: Vec>) -> Self { + self.connections = Some(connections); + self + } + + pub fn config(mut self, config: ServerConfig) -> Self { + self.config = Some(config); + self + } + + pub fn shards_table( + mut self, + shards_table: EternalPtr>, + ) -> Self { + self.shards_table = Some(shards_table); + self + } + + pub fn clients_manager(mut self, client_manager: ClientManager) -> Self { + self.client_manager = Some(client_manager); + self + } + + pub fn encryptor(mut self, encryptor: Option) -> Self { + self.encryptor = encryptor; + self + } + + pub fn version(mut self, version: SemanticVersion) -> Self { + self.version = Some(version); + self + } + + pub fn state(mut self, state: FileState) -> Self { + self.state = Some(state); + self + } + + pub fn metrics(mut self, metrics: Metrics) -> Self { + self.metrics = Some(metrics); + self + } + + pub fn is_follower(mut self, is_follower: bool) -> Self { + self.is_follower = is_follower; + self + } + + pub fn current_replica_id(mut self, current_replica_id: Option) -> Self { + self.current_replica_id = current_replica_id; + self + } + + pub fn metadata(mut self, metadata: Metadata) -> Self { + self.metadata = Some(metadata); + self + } + + pub fn metadata_writer(mut self, metadata_writer: MetadataWriter) -> Self { + self.metadata_writer = Some(metadata_writer); + self + } + + // TODO: Too much happens in there, some of those bootstrapping logic should be moved outside. + pub fn build(self) -> IggyShard { + let id = self.id.unwrap(); + let shards_table = self.shards_table.unwrap(); + let state = self.state.unwrap(); + let config = self.config.unwrap(); + let connections = self.connections.unwrap(); + let encryptor = self.encryptor; + let client_manager = self.client_manager.unwrap(); + let version = self.version.unwrap(); + let metadata = self.metadata.expect("metadata is required"); + let (stop_receiver, frame_receiver) = connections + .iter() + .filter(|c| c.id == id) + .map(|c| (c.stop_receiver.clone(), c.receiver.clone())) + .next() + .expect("Failed to find connection with the specified ID"); + + // Collect all stop_senders for broadcasting shutdown to all shards + let all_stop_senders: Vec<_> = connections.iter().map(|c| c.stop_sender.clone()).collect(); + let shards = connections; + + // Initialize metrics + let metrics = self.metrics.unwrap_or_else(Metrics::init); + + // Create TaskRegistry with all stop_senders for critical task failures + let task_registry = Rc::new(TaskRegistry::new(id, all_stop_senders)); + + // Create notification channel for config writer + let (config_writer_notify, config_writer_receiver) = async_channel::bounded(1); + + // Trigger initial check in case servers bind before task starts + let _ = config_writer_notify.try_send(()); + + // Create per-shard stores (wrapped in RefCell for interior mutability) + let local_partitions = RefCell::new(LocalPartitions::new()); + + IggyShard { + id, + shards, + shards_table, + metadata, + metadata_writer: self.metadata_writer.map(RefCell::new), + local_partitions, + pending_partition_inits: RefCell::new(AHashSet::new()), + encryptor, + config, + _version: version, + state, + stop_receiver, + messages_receiver: Cell::new(Some(frame_receiver)), + metrics, + is_follower: self.is_follower, + current_replica_id: self.current_replica_id, + is_shutting_down: AtomicBool::new(false), + tcp_bound_address: Cell::new(None), + quic_bound_address: Cell::new(None), + websocket_bound_address: Cell::new(None), + http_bound_address: Cell::new(None), + config_writer_notify, + config_writer_receiver, + task_registry, + client_manager, + } + } +} diff --git a/core/server/src/shard/communication.rs b/core/server/src/shard/communication.rs new file mode 100644 index 0000000000..a67014110a --- /dev/null +++ b/core/server/src/shard/communication.rs @@ -0,0 +1,198 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::shard::{ + BROADCAST_TIMEOUT, COMPONENT, IggyShard, + transmission::{ + connector::ShardConnector, + event::ShardEvent, + frame::{ShardFrame, ShardResponse}, + message::{ShardMessage, ShardRequest}, + }, +}; +use futures::future::join_all; +use hash32::{Hasher, Murmur3Hasher}; +use iggy_common::{Identifier, IggyError}; +use server_common::sharding::{IggyNamespace, PartitionLocation}; +use std::hash::Hasher as _; +use tracing::{error, info, warn}; + +impl IggyShard { + /// Sends a control-plane request to shard 0's message pump. + pub async fn send_to_control_plane( + &self, + request: ShardRequest, + ) -> Result { + let shard0 = &self.shards[0]; + shard0 + .send_request(ShardMessage::Request(request)) + .await + .map_err(|err| { + error!( + "{COMPONENT} - failed to send control-plane request to shard 0, error: {err}" + ); + err + }) + } + + /// Sends a data-plane request to the shard owning the partition. + pub async fn send_to_data_plane( + &self, + request: ShardRequest, + ) -> Result { + let ns = request + .routing + .as_ref() + .expect("data-plane request requires namespace"); + let shard = self + .find_shard(ns) + .ok_or_else(|| self.namespace_not_found_error(ns))?; + shard + .send_request(ShardMessage::Request(request)) + .await + .map_err(|err| { + error!( + "{COMPONENT} - failed to send data-plane request to shard {}, error: {err}", + shard.id + ); + err + }) + } + + /// Converts a missing namespace in shards_table to the appropriate entity-not-found error. + fn namespace_not_found_error(&self, ns: &IggyNamespace) -> IggyError { + let stream_id = + Identifier::numeric(ns.stream_id() as u32).expect("numeric identifier is always valid"); + let topic_id = + Identifier::numeric(ns.topic_id() as u32).expect("numeric identifier is always valid"); + + if self.metadata.get_stream_id(&stream_id).is_none() { + return IggyError::StreamIdNotFound(stream_id); + } + + if self + .metadata + .get_topic_id(ns.stream_id(), &topic_id) + .is_none() + { + return IggyError::TopicIdNotFound(stream_id, topic_id); + } + + IggyError::PartitionNotFound(ns.partition_id(), topic_id, stream_id) + } + + pub async fn broadcast_event_to_all_shards(&self, event: ShardEvent) -> Result<(), IggyError> { + if self.is_shutting_down() { + info!("Skipping broadcast during shutdown for event: {}", event); + return Ok(()); + } + + let event_type = event.to_string(); + let futures = self + .shards + .iter() + .filter(|s| s.id != self.id) + .map(|shard| { + let event = event.clone(); + let conn = shard.clone(); + let shard_id = shard.id; + let event_type = event_type.clone(); + + async move { + let (sender, receiver) = async_channel::bounded(1); + conn.send(ShardFrame::new(ShardMessage::Event(event), Some(sender))); + + match compio::time::timeout(BROADCAST_TIMEOUT, receiver.recv()).await { + Ok(Ok(_)) => Ok(()), + Ok(Err(e)) => { + warn!( + "Broadcast to shard {} failed for event {}: channel error: {}", + shard_id, event_type, e + ); + Err(()) + } + Err(e) => { + warn!( + "Broadcast to shard {} failed for event {}: timeout waiting for response after {:?}, elapsed: {:?}", + shard_id, event_type, + BROADCAST_TIMEOUT, + e + ); + Err(()) + } + } + } + }) + .collect::>(); + + if futures.is_empty() { + return Ok(()); + } + + let results = join_all(futures).await; + let has_failures = results.iter().any(|r| r.is_err()); + + if has_failures { + Err(IggyError::ShardCommunicationError) + } else { + Ok(()) + } + } + + pub fn find_shard(&self, namespace: &IggyNamespace) -> Option<&ShardConnector> { + self.shards_table.get(namespace).map(|location| { + self.shards + .iter() + .find(|shard| shard.id == *location.shard_id) + .expect("Shard not found in the shards table.") + }) + } + + pub fn remove_shard_table_record(&self, namespace: &IggyNamespace) -> PartitionLocation { + self.shards_table + .remove(namespace) + .map(|(_, location)| location) + .expect("remove_shard_table_record: namespace not found") + } + + pub fn insert_shard_table_record(&self, ns: IggyNamespace, location: PartitionLocation) { + self.shards_table.insert(ns, location); + } + + pub fn get_current_shard_namespaces(&self) -> Vec { + self.shards_table + .iter() + .filter_map(|entry| { + let (ns, location) = entry.pair(); + if *location.shard_id == self.id { + Some(*ns) + } else { + None + } + }) + .collect() + } +} + +// Utility function for shard assignment calculation +pub fn calculate_shard_assignment(ns: &IggyNamespace, upperbound: u32) -> u16 { + let mut hasher = Murmur3Hasher::default(); + hasher.write_u64(ns.inner()); + let hash = hasher.finish32(); + // Murmur3 has problems with weak lower bits for small integer inputs, so we use bits from the middle. + ((hash >> 16) % upperbound) as u16 +} diff --git a/core/server/src/shard/execution.rs b/core/server/src/shard/execution.rs new file mode 100644 index 0000000000..14664aa9b7 --- /dev/null +++ b/core/server/src/shard/execution.rs @@ -0,0 +1,732 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::wire_id_to_identifier; +use crate::streaming::users::user::User; +use crate::streaming::utils::crypto; +use crate::{ + shard::{ + IggyShard, + transmission::{ + event::ShardEvent, + frame::{ConsumerGroupResponseData, StreamResponseData, TopicResponseData}, + message::ResolvedTopic, + }, + }, + state::{ + command::EntryCommand, + models::{ + CreateConsumerGroupWithId, CreatePersonalAccessTokenWithHash, CreateStreamWithId, + CreateTopicWithId, CreateUserWithId, + }, + }, + streaming::polling_consumer::ConsumerGroupId, +}; +use iggy_binary_protocol::requests::{ + consumer_groups::*, partitions::*, personal_access_tokens::*, streams::*, topics::*, users::*, +}; +use iggy_common::wire_conversions::wire_permissions_to_permissions; +use iggy_common::{ + CompressionAlgorithm, Identifier, IggyError, IggyExpiry, MaxTopicSize, PersonalAccessToken, + UserStatus, +}; +use secrecy::{ExposeSecret, SecretString}; + +pub async fn execute_create_stream( + shard: &IggyShard, + user_id: u32, + wire: CreateStreamRequest, +) -> Result { + shard.metadata.perm_create_stream(user_id)?; + + let stream_id = shard.create_stream(wire.name.to_string()).await?; + + let response_data = shard.metadata.with_metadata(|m| { + let stream = m + .streams + .get(stream_id) + .expect("stream missing from metadata after creation"); + StreamResponseData { + id: stream_id as u32, + name: stream.name.clone(), + created_at: stream.created_at, + } + }); + + shard + .state + .apply( + user_id, + &EntryCommand::CreateStream(CreateStreamWithId { + stream_id: stream_id as u32, + command: wire, + }), + ) + .await?; + + Ok(response_data) +} + +pub async fn execute_update_stream( + shard: &IggyShard, + user_id: u32, + wire: UpdateStreamRequest, +) -> Result<(), IggyError> { + let stream_id = wire_id_to_identifier(&wire.stream_id)?; + let stream = shard.resolve_stream(&stream_id)?; + shard.metadata.perm_update_stream(user_id, stream.id())?; + + shard.update_stream(stream, wire.name.to_string())?; + + shard + .state + .apply(user_id, &EntryCommand::UpdateStream(wire)) + .await?; + + Ok(()) +} + +pub async fn execute_delete_stream( + shard: &IggyShard, + user_id: u32, + wire: DeleteStreamRequest, +) -> Result<(), IggyError> { + let stream_id = wire_id_to_identifier(&wire.stream_id)?; + let stream = shard.resolve_stream(&stream_id)?; + shard.metadata.perm_delete_stream(user_id, stream.id())?; + + // Capture all topic/partition info BEFORE deletion for broadcast + let topics_with_partitions: Vec<(usize, Vec)> = shard + .metadata + .get_topic_ids(stream.id()) + .into_iter() + .map(|topic_id| { + let partition_ids = shard.metadata.get_partition_ids(stream.id(), topic_id); + (topic_id, partition_ids) + }) + .collect(); + + shard.delete_stream(stream).await?; + + shard + .state + .apply(user_id, &EntryCommand::DeleteStream(wire)) + .await?; + + // Broadcast DeletedPartitions to all shards for each topic's partitions (best-effort) + for (topic_id, partition_ids) in topics_with_partitions { + if partition_ids.is_empty() { + continue; + } + let event = ShardEvent::DeletedPartitions { + stream_id: Identifier::numeric(stream.id() as u32) + .expect("numeric identifier is always valid"), + topic_id: Identifier::numeric(topic_id as u32) + .expect("numeric identifier is always valid"), + partitions_count: partition_ids.len() as u32, + partition_ids, + }; + if let Err(e) = shard.broadcast_event_to_all_shards(event).await { + tracing::warn!("Broadcast failed: {e}. Shards will sync on restart."); + } + } + + Ok(()) +} + +pub async fn execute_purge_stream( + shard: &IggyShard, + user_id: u32, + wire: PurgeStreamRequest, +) -> Result<(), IggyError> { + let stream_id = wire_id_to_identifier(&wire.stream_id)?; + let stream = shard.resolve_stream(&stream_id)?; + shard.metadata.perm_purge_stream(user_id, stream.id())?; + + shard.purge_stream(stream).await?; + shard.purge_stream_local(stream).await?; + + shard + .state + .apply(user_id, &EntryCommand::PurgeStream(wire)) + .await?; + + let event = ShardEvent::PurgedStream { + stream_id: Identifier::numeric(stream.id() as u32) + .expect("numeric identifier is always valid"), + }; + if let Err(e) = shard.broadcast_event_to_all_shards(event).await { + tracing::warn!("Broadcast failed: {e}. Shards will sync on restart."); + } + + Ok(()) +} + +pub async fn execute_create_topic( + shard: &IggyShard, + user_id: u32, + wire: CreateTopicRequest, +) -> Result { + let stream_id = wire_id_to_identifier(&wire.stream_id)?; + let compression = CompressionAlgorithm::from_code(wire.compression_algorithm)?; + let message_expiry = IggyExpiry::from(wire.message_expiry); + let max_topic_size = MaxTopicSize::from(wire.max_topic_size); + let replication_factor = if wire.replication_factor == 0 { + None + } else { + Some(wire.replication_factor) + }; + + let stream = shard.resolve_stream(&stream_id)?; + shard.metadata.perm_create_topic(user_id, stream.id())?; + + let topic_id = shard + .create_topic( + stream, + wire.name.to_string(), + message_expiry, + compression, + max_topic_size, + replication_factor, + ) + .await?; + + let resolved_topic = ResolvedTopic { + stream_id: stream.id(), + topic_id, + }; + let partition_infos = shard + .create_partitions(resolved_topic, wire.partitions_count) + .await?; + + let response_data = shard.metadata.with_metadata(|m| { + let topic = m + .streams + .get(stream.id()) + .and_then(|s| s.topics.get(topic_id)) + .expect("topic missing from metadata after creation"); + TopicResponseData { + id: topic_id as u32, + name: topic.name.clone(), + created_at: topic.created_at, + partitions: partition_infos.clone(), + message_expiry: topic.message_expiry, + compression_algorithm: topic.compression_algorithm, + max_topic_size: topic.max_topic_size, + replication_factor: topic.replication_factor, + } + }); + + shard + .state + .apply( + user_id, + &EntryCommand::CreateTopic(CreateTopicWithId { + topic_id: topic_id as u32, + command: wire, + }), + ) + .await?; + + let event = ShardEvent::CreatedPartitions { + stream_id: Identifier::numeric(stream.id() as u32) + .expect("numeric identifier is always valid"), + topic_id: Identifier::numeric(topic_id as u32).expect("numeric identifier is always valid"), + partitions: partition_infos, + }; + if let Err(e) = shard.broadcast_event_to_all_shards(event).await { + tracing::warn!("Broadcast failed: {e}. Shards will sync on restart."); + } + + Ok(response_data) +} + +pub async fn execute_update_topic( + shard: &IggyShard, + user_id: u32, + wire: UpdateTopicRequest, +) -> Result<(), IggyError> { + let stream_id = wire_id_to_identifier(&wire.stream_id)?; + let topic_id = wire_id_to_identifier(&wire.topic_id)?; + let compression = CompressionAlgorithm::from_code(wire.compression_algorithm)?; + let message_expiry = IggyExpiry::from(wire.message_expiry); + let max_topic_size = MaxTopicSize::from(wire.max_topic_size); + let replication_factor = if wire.replication_factor == 0 { + None + } else { + Some(wire.replication_factor) + }; + + let topic = shard.resolve_topic(&stream_id, &topic_id)?; + shard + .metadata + .perm_update_topic(user_id, topic.stream_id, topic.topic_id)?; + + shard.update_topic( + topic, + wire.name.to_string(), + message_expiry, + compression, + max_topic_size, + replication_factor, + )?; + + shard + .state + .apply(user_id, &EntryCommand::UpdateTopic(wire)) + .await?; + + Ok(()) +} + +pub async fn execute_delete_topic( + shard: &IggyShard, + user_id: u32, + wire: DeleteTopicRequest, +) -> Result<(), IggyError> { + let stream_id = wire_id_to_identifier(&wire.stream_id)?; + let topic_id = wire_id_to_identifier(&wire.topic_id)?; + let topic = shard.resolve_topic(&stream_id, &topic_id)?; + shard + .metadata + .perm_delete_topic(user_id, topic.stream_id, topic.topic_id)?; + + // Capture partition_ids BEFORE deletion for broadcast + let partition_ids = shard + .metadata + .get_partition_ids(topic.stream_id, topic.topic_id); + + shard.delete_topic(topic).await?; + + shard + .state + .apply(user_id, &EntryCommand::DeleteTopic(wire)) + .await?; + + // Broadcast to all shards to clean up their local_partitions entries (best-effort) + let event = ShardEvent::DeletedPartitions { + stream_id: Identifier::numeric(topic.stream_id as u32) + .expect("numeric identifier is always valid"), + topic_id: Identifier::numeric(topic.topic_id as u32) + .expect("numeric identifier is always valid"), + partitions_count: partition_ids.len() as u32, + partition_ids, + }; + if let Err(e) = shard.broadcast_event_to_all_shards(event).await { + tracing::warn!("Broadcast failed: {e}. Shards will sync on restart."); + } + + Ok(()) +} + +pub async fn execute_purge_topic( + shard: &IggyShard, + user_id: u32, + wire: PurgeTopicRequest, +) -> Result<(), IggyError> { + let stream_id = wire_id_to_identifier(&wire.stream_id)?; + let topic_id = wire_id_to_identifier(&wire.topic_id)?; + let topic = shard.resolve_topic(&stream_id, &topic_id)?; + shard + .metadata + .perm_purge_topic(user_id, topic.stream_id, topic.topic_id)?; + + shard.purge_topic(topic).await?; + shard.purge_topic_local(topic).await?; + + shard + .state + .apply(user_id, &EntryCommand::PurgeTopic(wire)) + .await?; + + let event = ShardEvent::PurgedTopic { + stream_id: Identifier::numeric(topic.stream_id as u32) + .expect("numeric identifier is always valid"), + topic_id: Identifier::numeric(topic.topic_id as u32) + .expect("numeric identifier is always valid"), + }; + if let Err(e) = shard.broadcast_event_to_all_shards(event).await { + tracing::warn!("Broadcast failed: {e}. Shards will sync on restart."); + } + + Ok(()) +} + +pub async fn execute_create_partitions( + shard: &IggyShard, + user_id: u32, + wire: CreatePartitionsRequest, +) -> Result<(), IggyError> { + let stream_id = wire_id_to_identifier(&wire.stream_id)?; + let topic_id = wire_id_to_identifier(&wire.topic_id)?; + let topic = shard.resolve_topic(&stream_id, &topic_id)?; + shard + .metadata + .perm_create_partitions(user_id, topic.stream_id, topic.topic_id)?; + + let partition_infos = shard + .create_partitions(topic, wire.partitions_count) + .await?; + let total_partition_count = shard + .metadata + .partitions_count(topic.stream_id, topic.topic_id) as u32; + shard.writer().rebalance_consumer_groups_for_topic( + topic.stream_id, + topic.topic_id, + total_partition_count, + ); + + shard + .state + .apply(user_id, &EntryCommand::CreatePartitions(wire)) + .await?; + + let event = ShardEvent::CreatedPartitions { + stream_id: Identifier::numeric(topic.stream_id as u32) + .expect("numeric identifier is always valid"), + topic_id: Identifier::numeric(topic.topic_id as u32) + .expect("numeric identifier is always valid"), + partitions: partition_infos, + }; + if let Err(e) = shard.broadcast_event_to_all_shards(event).await { + tracing::warn!("Broadcast failed: {e}. Shards will sync on restart."); + } + + Ok(()) +} + +pub async fn execute_delete_partitions( + shard: &IggyShard, + user_id: u32, + wire: DeletePartitionsRequest, +) -> Result<(), IggyError> { + let stream_id = wire_id_to_identifier(&wire.stream_id)?; + let topic_id = wire_id_to_identifier(&wire.topic_id)?; + let topic = shard.resolve_topic(&stream_id, &topic_id)?; + shard + .metadata + .perm_delete_partitions(user_id, topic.stream_id, topic.topic_id)?; + + let deleted_partition_ids = shard + .delete_partitions(topic, wire.partitions_count) + .await?; + + let remaining_partition_count = shard + .metadata + .partitions_count(topic.stream_id, topic.topic_id) + as u32; + shard.writer().rebalance_consumer_groups_for_topic( + topic.stream_id, + topic.topic_id, + remaining_partition_count, + ); + + shard + .state + .apply(user_id, &EntryCommand::DeletePartitions(wire)) + .await?; + + let event = ShardEvent::DeletedPartitions { + stream_id: Identifier::numeric(topic.stream_id as u32) + .expect("numeric identifier is always valid"), + topic_id: Identifier::numeric(topic.topic_id as u32) + .expect("numeric identifier is always valid"), + partitions_count: deleted_partition_ids.len() as u32, + partition_ids: deleted_partition_ids.clone(), + }; + if let Err(e) = shard.broadcast_event_to_all_shards(event).await { + tracing::warn!("Broadcast failed: {e}. Shards will sync on restart."); + } + + Ok(()) +} + +pub async fn execute_create_consumer_group( + shard: &IggyShard, + user_id: u32, + wire: CreateConsumerGroupRequest, +) -> Result { + let stream_id = wire_id_to_identifier(&wire.stream_id)?; + let topic_id = wire_id_to_identifier(&wire.topic_id)?; + let topic = shard.resolve_topic(&stream_id, &topic_id)?; + shard + .metadata + .perm_create_consumer_group(user_id, topic.stream_id, topic.topic_id)?; + + let group_id = shard.create_consumer_group(topic, wire.name.to_string())?; + + let response_data = shard + .metadata + .get_consumer_group(topic.stream_id, topic.topic_id, group_id) + .map(|cg| ConsumerGroupResponseData { + id: group_id as u32, + name: cg.name.clone(), + partitions_count: cg.partitions.len() as u32, + }) + .expect("consumer group missing from metadata after creation"); + + shard + .state + .apply( + user_id, + &EntryCommand::CreateConsumerGroup(CreateConsumerGroupWithId { + group_id: group_id as u32, + command: wire, + }), + ) + .await?; + + Ok(response_data) +} + +pub async fn execute_delete_consumer_group( + shard: &IggyShard, + user_id: u32, + wire: DeleteConsumerGroupRequest, +) -> Result<(), IggyError> { + let stream_id = wire_id_to_identifier(&wire.stream_id)?; + let topic_id = wire_id_to_identifier(&wire.topic_id)?; + let group_id = wire_id_to_identifier(&wire.group_id)?; + let group = shard.resolve_consumer_group(&stream_id, &topic_id, &group_id)?; + shard + .metadata + .perm_delete_consumer_group(user_id, group.stream_id, group.topic_id)?; + + let deleted = shard.delete_consumer_group(group)?; + + let cg_id = ConsumerGroupId(deleted.group_id); + shard + .delete_consumer_group_offsets( + cg_id, + group.stream_id, + group.topic_id, + &deleted.partition_ids, + ) + .await?; + + shard + .state + .apply(user_id, &EntryCommand::DeleteConsumerGroup(wire)) + .await?; + + Ok(()) +} + +pub fn execute_join_consumer_group( + shard: &IggyShard, + user_id: u32, + client_id: u32, + wire: JoinConsumerGroupRequest, +) -> Result<(), IggyError> { + let stream_id = wire_id_to_identifier(&wire.stream_id)?; + let topic_id = wire_id_to_identifier(&wire.topic_id)?; + let group_id = wire_id_to_identifier(&wire.group_id)?; + let group = shard.resolve_consumer_group(&stream_id, &topic_id, &group_id)?; + shard + .metadata + .perm_join_consumer_group(user_id, group.stream_id, group.topic_id)?; + + shard.join_consumer_group(client_id, group)?; + + Ok(()) +} + +pub fn execute_leave_consumer_group( + shard: &IggyShard, + user_id: u32, + client_id: u32, + wire: LeaveConsumerGroupRequest, +) -> Result<(), IggyError> { + let stream_id = wire_id_to_identifier(&wire.stream_id)?; + let topic_id = wire_id_to_identifier(&wire.topic_id)?; + let group_id = wire_id_to_identifier(&wire.group_id)?; + let group = shard.resolve_consumer_group(&stream_id, &topic_id, &group_id)?; + shard + .metadata + .perm_leave_consumer_group(user_id, group.stream_id, group.topic_id)?; + + shard.leave_consumer_group(client_id, group)?; + + Ok(()) +} + +pub async fn execute_create_user( + shard: &IggyShard, + user_id: u32, + wire: CreateUserRequest, +) -> Result { + shard.metadata.perm_create_user(user_id)?; + + let username = wire.username.to_string(); + let password = SecretString::from(wire.password.clone()); + let status = UserStatus::from_code(wire.status)?; + let permissions = wire + .permissions + .as_ref() + .map(wire_permissions_to_permissions); + + let user = shard.create_user(&username, password.expose_secret(), status, permissions)?; + + // Hash the password before persisting to WAL + let mut wal_wire = wire; + wal_wire.password = crypto::hash_password(password.expose_secret()); + + shard + .state + .apply( + user_id, + &EntryCommand::CreateUser(CreateUserWithId { + user_id: user.id, + command: wal_wire, + }), + ) + .await?; + + Ok(user) +} + +pub async fn execute_delete_user( + shard: &IggyShard, + user_id: u32, + wire: DeleteUserRequest, +) -> Result { + shard.metadata.perm_delete_user(user_id)?; + + let target_id = wire_id_to_identifier(&wire.user_id)?; + let user = shard.delete_user(&target_id)?; + + shard + .state + .apply(user_id, &EntryCommand::DeleteUser(wire)) + .await?; + + Ok(user) +} + +pub async fn execute_update_user( + shard: &IggyShard, + user_id: u32, + wire: UpdateUserRequest, +) -> Result { + shard.metadata.perm_update_user(user_id)?; + + let target_id = wire_id_to_identifier(&wire.user_id)?; + let username = wire.username.as_ref().map(|n| n.to_string()); + let status = wire.status.map(UserStatus::from_code).transpose()?; + let user = shard.update_user(&target_id, username, status)?; + + shard + .state + .apply(user_id, &EntryCommand::UpdateUser(wire)) + .await?; + + Ok(user) +} + +pub async fn execute_change_password( + shard: &IggyShard, + user_id: u32, + wire: ChangePasswordRequest, +) -> Result<(), IggyError> { + let target_id = wire_id_to_identifier(&wire.user_id)?; + let target_user = shard.get_user(&target_id)?; + if target_user.id != user_id { + shard.metadata.perm_change_password(user_id)?; + } + + shard.change_password(&target_id, &wire.current_password, &wire.new_password)?; + + // Clear current password and hash new password before persisting to WAL + let wal_wire = ChangePasswordRequest { + user_id: wire.user_id, + current_password: String::new(), + new_password: crypto::hash_password(&wire.new_password), + }; + + shard + .state + .apply(user_id, &EntryCommand::ChangePassword(wal_wire)) + .await?; + + Ok(()) +} + +pub async fn execute_update_permissions( + shard: &IggyShard, + user_id: u32, + wire: UpdatePermissionsRequest, +) -> Result<(), IggyError> { + shard.metadata.perm_update_permissions(user_id)?; + + let target_id = wire_id_to_identifier(&wire.user_id)?; + let target_user = shard.get_user(&target_id)?; + if target_user.is_root() { + return Err(IggyError::CannotChangePermissions(target_user.id)); + } + + let permissions = wire + .permissions + .as_ref() + .map(wire_permissions_to_permissions); + shard.update_permissions(&target_id, permissions)?; + + shard + .state + .apply(user_id, &EntryCommand::UpdatePermissions(wire)) + .await?; + + Ok(()) +} + +pub async fn execute_create_personal_access_token( + shard: &IggyShard, + user_id: u32, + wire: CreatePersonalAccessTokenRequest, +) -> Result<(PersonalAccessToken, String), IggyError> { + let name = wire.name.to_string(); + let expiry = IggyExpiry::from(wire.expiry); + let (personal_access_token, token) = + shard.create_personal_access_token(user_id, &name, expiry)?; + + shard + .state + .apply( + user_id, + &EntryCommand::CreatePersonalAccessToken(CreatePersonalAccessTokenWithHash { + hash: personal_access_token.token.to_string(), + command: wire, + }), + ) + .await?; + + Ok((personal_access_token, token)) +} + +pub async fn execute_delete_personal_access_token( + shard: &IggyShard, + user_id: u32, + wire: DeletePersonalAccessTokenRequest, +) -> Result<(), IggyError> { + shard.delete_personal_access_token(user_id, wire.name.as_str())?; + + shard + .state + .apply(user_id, &EntryCommand::DeletePersonalAccessToken(wire)) + .await?; + + Ok(()) +} diff --git a/core/server/src/shard/handlers.rs b/core/server/src/shard/handlers.rs new file mode 100644 index 0000000000..44cc1c0cc2 --- /dev/null +++ b/core/server/src/shard/handlers.rs @@ -0,0 +1,614 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::*; +use crate::sender::SenderKind; +use crate::{ + shard::{ + IggyShard, execution, + transmission::{ + event::ShardEvent, + frame::ShardResponse, + message::{ShardMessage, ShardRequest, ShardRequestPayload}, + }, + }, + tcp::{ + connection_handler::{ConnectionAction, handle_connection, handle_error}, + tcp_listener::cleanup_connection, + }, +}; +use compio::net::TcpStream; +use iggy_common::{IggyError, TransportProtocol}; +use nix::sys::stat::SFlag; +use server_common::sharding::IggyNamespace; +use std::os::fd::{FromRawFd, IntoRawFd}; +use tracing::info; + +pub(super) async fn handle_shard_message( + shard: &Rc, + message: ShardMessage, +) -> Option { + match message { + ShardMessage::Request(request) => match handle_request(shard, request).await { + Ok(response) => Some(response), + Err(err) => Some(ShardResponse::ErrorResponse(err)), + }, + ShardMessage::Event(event) => match handle_event(shard, event).await { + Ok(_) => Some(ShardResponse::Event), + Err(err) => Some(ShardResponse::ErrorResponse(err)), + }, + } +} + +async fn handle_request( + shard: &Rc, + request: ShardRequest, +) -> Result { + // Data-plane operations extract namespace from routing + let namespace = request.routing; + match request.payload { + ShardRequestPayload::SendMessages { batch } => { + let batch = shard.maybe_encrypt_messages(batch)?; + let messages_count = batch.count(); + + let namespace = namespace.expect("SendMessages requires routing namespace"); + + shard.ensure_partition(&namespace).await?; + + shard + .append_messages_to_local_partition(&namespace, batch, &shard.config.system) + .await?; + + shard.metrics.increment_messages(messages_count as u64); + Ok(ShardResponse::SendMessages) + } + ShardRequestPayload::PollMessages { args, consumer } => { + let namespace = namespace.expect("PollMessages requires routing namespace"); + + if args.count == 0 { + let current_offset = shard + .local_partitions + .borrow() + .get(&namespace) + .map(|p| p.offset.load(std::sync::atomic::Ordering::Relaxed)) + .unwrap_or(0); + return Ok(ShardResponse::PollMessages(( + iggy_common::IggyPollMetadata::new( + namespace.partition_id() as u32, + current_offset, + ), + crate::streaming::segments::IggyMessagesBatchSet::empty(), + ))); + } + + let auto_commit = args.auto_commit; + + shard.ensure_partition(&namespace).await?; + + let (poll_metadata, batches) = shard + .poll_messages_from_local_partition(&namespace, consumer, args) + .await?; + + if auto_commit && !batches.is_empty() { + let offset = batches + .last_offset() + .expect("Batch set should have at least one batch"); + shard + .auto_commit_consumer_offset_from_local_partition(&namespace, consumer, offset) + .await?; + } + Ok(ShardResponse::PollMessages((poll_metadata, batches))) + } + ShardRequestPayload::FlushUnsavedBuffer { fsync } => { + let ns = namespace.expect("FlushUnsavedBuffer requires routing namespace"); + let flushed_count = shard + .flush_unsaved_buffer_from_local_partitions(&ns, fsync) + .await?; + Ok(ShardResponse::FlushUnsavedBuffer { flushed_count }) + } + ShardRequestPayload::DeleteSegments { segments_count } => { + let ns = namespace.expect("DeleteSegments requires routing namespace"); + let (deleted_segments, deleted_messages) = shard + .delete_oldest_segments( + ns.stream_id(), + ns.topic_id(), + ns.partition_id(), + segments_count, + ) + .await?; + Ok(ShardResponse::DeleteSegments { + deleted_segments, + deleted_messages, + }) + } + ShardRequestPayload::CleanTopicMessages { + stream_id, + topic_id, + partition_ids, + } => { + let (deleted_segments, deleted_messages) = shard + .clean_topic_messages(stream_id, topic_id, &partition_ids) + .await?; + Ok(ShardResponse::CleanTopicMessages { + deleted_segments, + deleted_messages, + }) + } + ShardRequestPayload::CreatePartitionsRequest { user_id, command } => { + assert_eq!( + shard.id, 0, + "CreatePartitionsRequest should only be handled by shard0" + ); + + execution::execute_create_partitions(shard, user_id, command).await?; + Ok(ShardResponse::CreatePartitionsResponse) + } + ShardRequestPayload::DeletePartitionsRequest { user_id, command } => { + assert_eq!( + shard.id, 0, + "DeletePartitionsRequest should only be handled by shard0" + ); + + execution::execute_delete_partitions(shard, user_id, command).await?; + Ok(ShardResponse::DeletePartitionsResponse) + } + ShardRequestPayload::CreateStreamRequest { user_id, command } => { + assert_eq!( + shard.id, 0, + "CreateStreamRequest should only be handled by shard0" + ); + + let result = execution::execute_create_stream(shard, user_id, command).await?; + Ok(ShardResponse::CreateStreamResponse(result)) + } + ShardRequestPayload::CreateTopicRequest { user_id, command } => { + assert_eq!( + shard.id, 0, + "CreateTopicRequest should only be handled by shard0" + ); + + let result = execution::execute_create_topic(shard, user_id, command).await?; + Ok(ShardResponse::CreateTopicResponse(result)) + } + ShardRequestPayload::UpdateTopicRequest { user_id, command } => { + assert_eq!( + shard.id, 0, + "UpdateTopicRequest should only be handled by shard0" + ); + + execution::execute_update_topic(shard, user_id, command).await?; + Ok(ShardResponse::UpdateTopicResponse) + } + ShardRequestPayload::DeleteTopicRequest { user_id, command } => { + assert_eq!( + shard.id, 0, + "DeleteTopicRequest should only be handled by shard0" + ); + + execution::execute_delete_topic(shard, user_id, command).await?; + Ok(ShardResponse::DeleteTopicResponse) + } + ShardRequestPayload::CreateUserRequest { user_id, command } => { + assert_eq!( + shard.id, 0, + "CreateUserRequest should only be handled by shard0" + ); + let user = execution::execute_create_user(shard, user_id, command).await?; + Ok(ShardResponse::CreateUserResponse(user)) + } + ShardRequestPayload::GetStats { .. } => { + assert_eq!(shard.id, 0, "GetStats should only be handled by shard0"); + let stats = shard.get_stats().await?; + Ok(ShardResponse::GetStatsResponse(stats)) + } + ShardRequestPayload::DeleteUserRequest { user_id, command } => { + assert_eq!( + shard.id, 0, + "DeleteUserRequest should only be handled by shard0" + ); + let user = execution::execute_delete_user(shard, user_id, command).await?; + Ok(ShardResponse::DeleteUserResponse(user)) + } + ShardRequestPayload::UpdateStreamRequest { user_id, command } => { + assert_eq!( + shard.id, 0, + "UpdateStreamRequest should only be handled by shard0" + ); + + execution::execute_update_stream(shard, user_id, command).await?; + Ok(ShardResponse::UpdateStreamResponse) + } + ShardRequestPayload::DeleteStreamRequest { user_id, command } => { + assert_eq!( + shard.id, 0, + "DeleteStreamRequest should only be handled by shard0" + ); + + execution::execute_delete_stream(shard, user_id, command).await?; + Ok(ShardResponse::DeleteStreamResponse) + } + ShardRequestPayload::UpdatePermissionsRequest { user_id, command } => { + assert_eq!( + shard.id, 0, + "UpdatePermissionsRequest should only be handled by shard0" + ); + execution::execute_update_permissions(shard, user_id, command).await?; + Ok(ShardResponse::UpdatePermissionsResponse) + } + ShardRequestPayload::ChangePasswordRequest { user_id, command } => { + assert_eq!( + shard.id, 0, + "ChangePasswordRequest should only be handled by shard0" + ); + execution::execute_change_password(shard, user_id, command).await?; + Ok(ShardResponse::ChangePasswordResponse) + } + ShardRequestPayload::UpdateUserRequest { user_id, command } => { + assert_eq!( + shard.id, 0, + "UpdateUserRequest should only be handled by shard0" + ); + let user = execution::execute_update_user(shard, user_id, command).await?; + Ok(ShardResponse::UpdateUserResponse(user)) + } + ShardRequestPayload::CreateConsumerGroupRequest { user_id, command } => { + assert_eq!( + shard.id, 0, + "CreateConsumerGroupRequest should only be handled by shard0" + ); + + let result = execution::execute_create_consumer_group(shard, user_id, command).await?; + Ok(ShardResponse::CreateConsumerGroupResponse(result)) + } + ShardRequestPayload::JoinConsumerGroupRequest { + user_id, + client_id, + command, + } => { + assert_eq!( + shard.id, 0, + "JoinConsumerGroupRequest should only be handled by shard0" + ); + + execution::execute_join_consumer_group(shard, user_id, client_id, command)?; + Ok(ShardResponse::JoinConsumerGroupResponse) + } + ShardRequestPayload::LeaveConsumerGroupRequest { + user_id, + client_id, + command, + } => { + assert_eq!( + shard.id, 0, + "LeaveConsumerGroupRequest should only be handled by shard0" + ); + + execution::execute_leave_consumer_group(shard, user_id, client_id, command)?; + Ok(ShardResponse::LeaveConsumerGroupResponse) + } + ShardRequestPayload::DeleteConsumerGroupRequest { user_id, command } => { + assert_eq!( + shard.id, 0, + "DeleteConsumerGroupRequest should only be handled by shard0" + ); + + execution::execute_delete_consumer_group(shard, user_id, command).await?; + Ok(ShardResponse::DeleteConsumerGroupResponse) + } + ShardRequestPayload::CreatePersonalAccessTokenRequest { user_id, command } => { + assert_eq!( + shard.id, 0, + "CreatePersonalAccessTokenRequest should only be handled by shard0" + ); + + let (personal_access_token, token) = + execution::execute_create_personal_access_token(shard, user_id, command).await?; + + Ok(ShardResponse::CreatePersonalAccessTokenResponse( + personal_access_token, + token, + )) + } + ShardRequestPayload::DeletePersonalAccessTokenRequest { user_id, command } => { + assert_eq!( + shard.id, 0, + "DeletePersonalAccessTokenRequest should only be handled by shard0" + ); + + execution::execute_delete_personal_access_token(shard, user_id, command).await?; + + Ok(ShardResponse::DeletePersonalAccessTokenResponse) + } + ShardRequestPayload::LeaveConsumerGroupMetadataOnly { + stream_id, + topic_id, + group_id, + client_id, + } => { + assert_eq!( + shard.id, 0, + "LeaveConsumerGroupMetadataOnly should only be handled by shard0" + ); + + shard + .writer() + .leave_consumer_group(stream_id, topic_id, group_id, client_id); + + Ok(ShardResponse::LeaveConsumerGroupMetadataOnlyResponse) + } + ShardRequestPayload::CompletePartitionRevocation { + stream_id, + topic_id, + group_id, + member_slab_id, + member_id, + partition_id, + timed_out, + } => { + assert_eq!( + shard.id, 0, + "CompletePartitionRevocation should only be handled by shard0" + ); + + shard.writer().complete_partition_revocation( + stream_id, + topic_id, + group_id, + member_slab_id, + member_id, + partition_id, + timed_out, + ); + + Ok(ShardResponse::CompletePartitionRevocationResponse) + } + ShardRequestPayload::SocketTransfer { + fd, + from_shard, + client_id, + user_id, + address, + initial_data, + } => { + info!( + "Received socket transfer msg, fd: {fd:?}, from_shard: {from_shard}, address: {address}" + ); + + // Safety: The fd already != 1. + let stat = nix::sys::stat::fstat(&fd) + .map_err(|e| IggyError::IoError(format!("Invalid fd: {}", e)))?; + + if !SFlag::from_bits_truncate(stat.st_mode).contains(SFlag::S_IFSOCK) { + return Err(IggyError::IoError(format!("fd {:?} is not a socket", fd))); + } + + // restore TcpStream from fd + let tcp_stream = unsafe { TcpStream::from_raw_fd(fd.into_raw_fd()) }; + let session = shard.add_client(&address, TransportProtocol::Tcp); + session.set_user_id(user_id); + session.set_migrated(); + + let mut sender = SenderKind::get_tcp_sender(tcp_stream); + let conn_stop_receiver = shard.task_registry.add_connection(session.client_id); + let shard_for_conn = shard.clone(); + let registry = shard.task_registry.clone(); + let registry_clone = registry.clone(); + + let batch = shard.maybe_encrypt_messages(initial_data)?; + let messages_count = batch.count(); + + let ns = namespace.expect("SocketTransfer requires routing namespace"); + shard.ensure_partition(&ns).await?; + + shard + .append_messages_to_local_partition(&ns, batch, &shard.config.system) + .await?; + + shard.metrics.increment_messages(messages_count as u64); + + sender.send_empty_ok_response().await?; + + registry.spawn_connection(async move { + match handle_connection(&session, &mut sender, &shard_for_conn, conn_stop_receiver) + .await + { + Ok(ConnectionAction::Migrated { to_shard }) => { + info!("Migrated to shard {to_shard}, ignore cleanup connection"); + } + Ok(ConnectionAction::Finished) => { + cleanup_connection( + &mut sender, + client_id, + address, + ®istry_clone, + &shard_for_conn, + ) + .await; + } + Err(err) => { + handle_error(err); + cleanup_connection( + &mut sender, + client_id, + address, + ®istry_clone, + &shard_for_conn, + ) + .await; + } + } + }); + + Ok(ShardResponse::SocketTransferResponse) + } + ShardRequestPayload::PurgeStreamRequest { user_id, command } => { + assert_eq!( + shard.id, 0, + "PurgeStreamRequest should only be handled by shard0" + ); + + execution::execute_purge_stream(shard, user_id, command).await?; + Ok(ShardResponse::PurgeStreamResponse) + } + ShardRequestPayload::PurgeTopicRequest { user_id, command } => { + assert_eq!( + shard.id, 0, + "PurgeTopicRequest should only be handled by shard0" + ); + + execution::execute_purge_topic(shard, user_id, command).await?; + Ok(ShardResponse::PurgeTopicResponse) + } + } +} + +pub async fn handle_event(shard: &Rc, event: ShardEvent) -> Result<(), IggyError> { + match event { + ShardEvent::DeletedPartitions { + stream_id, + topic_id, + partitions_count: _, + partition_ids, + } => { + // SharedMetadata was already updated by the request handler before broadcasting. + // Here we only need to clean up local local_partitions entries on all shards. + // + // For DeleteTopic, the topic is already removed from metadata, so we extract + // numeric IDs directly from the Identifier (which must be numeric in that case). + // For DeletePartitions, the topic still exists, so metadata lookup works. + let numeric_stream_id = stream_id + .get_u32_value() + .map(|v| v as usize) + .unwrap_or_else(|_| shard.metadata.get_stream_id(&stream_id).unwrap_or_default()); + let numeric_topic_id = + topic_id + .get_u32_value() + .map(|v| v as usize) + .unwrap_or_else(|_| { + shard + .metadata + .get_topic_id(numeric_stream_id, &topic_id) + .unwrap_or_default() + }); + let mut partitions = shard.local_partitions.borrow_mut(); + for partition_id in partition_ids { + let ns = IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_id); + partitions.remove(&ns); + } + Ok(()) + } + ShardEvent::PurgedStream { stream_id } => { + let stream = shard.resolve_stream(&stream_id)?; + shard.purge_stream_local(stream).await?; + Ok(()) + } + ShardEvent::PurgedTopic { + stream_id, + topic_id, + } => { + let topic = shard.resolve_topic(&stream_id, &topic_id)?; + shard.purge_topic_local(topic).await?; + Ok(()) + } + ShardEvent::AddressBound { protocol, address } => { + info!( + "Received AddressBound event for {:?} with address: {}", + protocol, address + ); + match protocol { + TransportProtocol::Tcp => { + shard.tcp_bound_address.set(Some(address)); + let _ = shard.config_writer_notify.try_send(()); + } + TransportProtocol::Quic => { + shard.quic_bound_address.set(Some(address)); + let _ = shard.config_writer_notify.try_send(()); + } + TransportProtocol::Http => { + shard.http_bound_address.set(Some(address)); + let _ = shard.config_writer_notify.try_send(()); + } + TransportProtocol::WebSocket => { + shard.websocket_bound_address.set(Some(address)); + let _ = shard.config_writer_notify.try_send(()); + } + } + Ok(()) + } + ShardEvent::CreatedPartitions { + stream_id, + topic_id, + partitions, + } => { + let numeric_stream_id = match shard.metadata.get_stream_id(&stream_id) { + Some(id) => id, + None => { + tracing::warn!( + "CreatedPartitions: stream {:?} not found in SharedMetadata", + stream_id + ); + return Ok(()); + } + }; + let numeric_topic_id = match shard.metadata.get_topic_id(numeric_stream_id, &topic_id) { + Some(id) => id, + None => { + tracing::warn!( + "CreatedPartitions: topic {:?} not found in SharedMetadata for stream {}", + topic_id, + numeric_stream_id + ); + return Ok(()); + } + }; + + let shards_count = shard.get_available_shards_count(); + for partition_info in partitions { + let ns = IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_info.id); + let owner_shard_id = crate::shard::calculate_shard_assignment(&ns, shards_count); + + if shard.id == owner_shard_id as u16 { + shard.ensure_partition(&ns).await?; + } + } + Ok(()) + } + ShardEvent::FlushUnsavedBuffer { + stream_id, + topic_id, + partition_id, + fsync, + } => { + let numeric_stream_id = match shard.metadata.get_stream_id(&stream_id) { + Some(id) => id, + None => return Ok(()), + }; + let numeric_topic_id = match shard.metadata.get_topic_id(numeric_stream_id, &topic_id) { + Some(id) => id, + None => return Ok(()), + }; + + let ns = IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_id); + if shard.local_partitions.borrow().get(&ns).is_some() { + shard + .flush_unsaved_buffer_from_local_partitions(&ns, fsync) + .await?; + } + Ok(()) + } + } +} diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs new file mode 100644 index 0000000000..03f0bad1b1 --- /dev/null +++ b/core/server/src/shard/mod.rs @@ -0,0 +1,482 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use self::tasks::{continuous, periodic}; +use crate::{ + bootstrap::load_segments, + configs::server::ServerConfig, + metadata::{Metadata, MetadataWriter}, + shard::{task_registry::TaskRegistry, transmission::frame::ShardFrame}, + state::file::FileState, + streaming::{ + clients::client_manager::ClientManager, + diagnostics::metrics::Metrics, + partitions::{local_partition::LocalPartition, local_partitions::LocalPartitions}, + session::Session, + utils::ptr::EternalPtr, + }, +}; +use ahash::AHashSet; +use builder::IggyShardBuilder; +use dashmap::DashMap; +use iggy_common::SemanticVersion; +use iggy_common::{EncryptorKind, IggyByteSize, IggyError}; +use server_common::sharding::{IggyNamespace, PartitionLocation}; +use std::{ + cell::{Cell, RefCell}, + net::SocketAddr, + rc::Rc, + sync::{ + Arc, + atomic::{AtomicBool, AtomicU64, Ordering}, + }, + time::{Duration, Instant}, +}; +use tracing::{debug, error, info, instrument, warn}; +use transmission::connector::{Receiver, ShardConnector, StopReceiver}; + +pub mod builder; +pub mod execution; +pub mod handlers; +pub mod system; +pub mod task_registry; +pub mod tasks; +pub mod transmission; + +#[cfg(feature = "systemd")] +pub mod systemd; + +mod communication; + +pub use communication::calculate_shard_assignment; + +pub const COMPONENT: &str = "SHARD"; +pub const SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(10); +pub const BROADCAST_TIMEOUT: Duration = Duration::from_secs(20); + +pub struct IggyShard { + pub id: u16, + shards: Vec>, + _version: SemanticVersion, + + pub(crate) metadata: Metadata, + pub(crate) metadata_writer: Option>, + pub(crate) local_partitions: RefCell, + pub(crate) pending_partition_inits: RefCell>, + + pub(crate) shards_table: EternalPtr>, + pub(crate) state: FileState, + + pub(crate) encryptor: Option, + pub(crate) config: ServerConfig, + pub(crate) client_manager: ClientManager, + pub(crate) metrics: Metrics, + pub(crate) is_follower: bool, + /// Index into `config.cluster.nodes` that describes this running node. + /// `Some` only when cluster mode is enabled; validated at bootstrap to + /// match exactly one entry in the nodes list. + pub(crate) current_replica_id: Option, + pub messages_receiver: Cell>>, + pub(crate) stop_receiver: StopReceiver, + pub(crate) is_shutting_down: AtomicBool, + pub(crate) tcp_bound_address: Cell>, + pub(crate) quic_bound_address: Cell>, + pub(crate) websocket_bound_address: Cell>, + pub(crate) http_bound_address: Cell>, + pub(crate) config_writer_notify: async_channel::Sender<()>, + config_writer_receiver: async_channel::Receiver<()>, + pub(crate) task_registry: Rc, +} + +impl IggyShard { + pub fn builder() -> IggyShardBuilder { + Default::default() + } + + pub fn writer(&self) -> std::cell::RefMut<'_, crate::metadata::MetadataWriter> { + self.metadata_writer + .as_ref() + .expect("MetadataWriter only available on shard 0") + .borrow_mut() + } + + pub async fn init(&self) -> Result<(), IggyError> { + self.load_segments().await?; + let _ = self.load_users().await; + Ok(()) + } + + fn init_tasks(self: &Rc) { + continuous::spawn_message_pump(self.clone()); + + // Spawn config writer task on shard 0 if we need to wait for bound addresses + if self.id == 0 + && (self.config.tcp.enabled + || self.config.quic.enabled + || self.config.http.enabled + || self.config.websocket.enabled) + { + tasks::oneshot::spawn_config_writer_task(self); + } + + if self.config.tcp.enabled { + continuous::spawn_tcp_server(self.clone()); + } + + if self.config.http.enabled && self.id == 0 { + continuous::spawn_http_server(self.clone()); + } + + // JWT token cleaner task is spawned inside HTTP server because it needs `AppState`. + + // TODO(hubcio): QUIC doesn't properly work on all shards, especially tests `concurrent` and `system_scenario`. + // it's probably related to Endpoint not Cloned between shards, but all shards are creating its own instance. + // This way packet CID is invalid. (crypto-related stuff) + if self.config.quic.enabled && self.id == 0 { + continuous::spawn_quic_server(self.clone()); + } + if self.config.websocket.enabled { + continuous::spawn_websocket_server(self.clone()); + } + + if self.config.message_saver.enabled { + periodic::spawn_message_saver(self.clone()); + } + + if self.config.data_maintenance.messages.cleaner_enabled { + periodic::spawn_message_cleaner(self.clone()); + } + + if self.config.heartbeat.enabled { + periodic::spawn_heartbeat_verifier(self.clone()); + } + + if self.config.personal_access_token.cleaner.enabled { + periodic::spawn_personal_access_token_cleaner(self.clone()); + } + + if self.id == 0 { + periodic::spawn_revocation_timeout_checker(self.clone()); + } + + if !self.config.system.logging.sysinfo_print_interval.is_zero() && self.id == 0 { + periodic::spawn_sysinfo_printer(self.clone()); + } + + #[cfg(feature = "systemd")] + if self.id == 0 { + periodic::spawn_systemd_watchdog(self.clone()); + } + } + + pub async fn run(self: &Rc) -> Result<(), IggyError> { + let now: Instant = Instant::now(); + + info!("Starting..."); + self.init().await?; + + // TODO: Fixme + //self.assert_init(); + + self.init_tasks(); + let (shutdown_complete_tx, shutdown_complete_rx) = async_channel::bounded(1); + let stop_receiver = self.get_stop_receiver(); + let shard_for_shutdown = self.clone(); + + // Spawn shutdown handler + compio::runtime::spawn(async move { + let _ = stop_receiver.recv().await; + #[cfg(feature = "systemd")] + if shard_for_shutdown.id == 0 { + systemd::notify_stopping(); + } + let drained = shard_for_shutdown.trigger_shutdown().await; + #[cfg(feature = "systemd")] + if shard_for_shutdown.id == 0 && !drained { + warn!("Graceful shutdown timed out; some tasks did not drain in time"); + systemd::notify_status("graceful shutdown timed out"); + } + #[cfg(not(feature = "systemd"))] + let _ = drained; + let _ = shutdown_complete_tx.send(()).await; + }) + .detach(); + + let elapsed = now.elapsed(); + info!("Initialized in {} ms.", elapsed.as_millis()); + + shutdown_complete_rx.recv().await.ok(); + Ok(()) + } + + async fn load_segments(&self) -> Result<(), IggyError> { + for shard_entry in self.shards_table.iter() { + let (namespace, location) = shard_entry.pair(); + + if *location.shard_id == self.id { + let stream_id = namespace.stream_id(); + let topic_id: usize = namespace.topic_id(); + let partition_id = namespace.partition_id(); + + info!( + "Loading segments for stream: {}, topic: {}, partition: {}", + stream_id, topic_id, partition_id + ); + + let partition_path = + self.config + .system + .get_partition_path(stream_id, topic_id, partition_id); + + let init_info = self + .metadata + .get_partition_init_info(stream_id, topic_id, partition_id) + .expect("Partition must exist in SharedMetadata"); + let created_at = init_info.created_at; + let stats = init_info.stats; + + use crate::streaming::partitions::helpers::create_message_deduplicator; + use crate::streaming::partitions::storage::{ + load_consumer_group_offsets, load_consumer_offsets, + }; + + let consumer_offset_path = + self.config + .system + .get_consumer_offsets_path(stream_id, topic_id, partition_id); + let consumer_group_offsets_path = self + .config + .system + .get_consumer_group_offsets_path(stream_id, topic_id, partition_id); + + // Reuse metadata's Arcs so both metadata and local_partitions + // reference the same allocation — writes via store_consumer_offset + // (metadata path) are visible to delete_oldest_segments (local path). + let consumer_offsets = init_info.consumer_offsets; + let consumer_group_offsets = init_info.consumer_group_offsets; + + { + let guard = consumer_offsets.pin(); + for co in load_consumer_offsets(&consumer_offset_path).unwrap_or_default() { + guard.insert(co.consumer_id as usize, co); + } + } + + { + let guard = consumer_group_offsets.pin(); + for (cg_id, co) in load_consumer_group_offsets(&consumer_group_offsets_path) + .unwrap_or_default() + { + guard.insert(cg_id, co); + } + } + + let message_deduplicator = + create_message_deduplicator(&self.config.system).map(Arc::new); + + match load_segments( + &self.config.system, + stream_id, + topic_id, + partition_id, + partition_path, + stats.clone(), + ) + .await + { + Ok(mut loaded_log) => { + if !loaded_log.has_segments() { + info!( + "No segments found on disk for partition ID: {} for topic ID: {} for stream ID: {}, creating initial segment", + partition_id, topic_id, stream_id + ); + let segment = crate::streaming::segments::Segment::new( + 0, + self.config.system.segment.size, + ); + let storage = + crate::streaming::segments::storage::create_segment_storage( + &self.config.system, + stream_id, + topic_id, + partition_id, + 0, + 0, + 0, + ) + .await?; + loaded_log.add_persisted_segment(segment, storage); + stats.increment_segments_count(1); + } + + // Use the max end_offset across segments that have data, + // not just the active segment. Handles the edge case where + // the active segment is empty (rotated right before shutdown). + let current_offset = loaded_log + .segments() + .iter() + .filter(|s| s.size > IggyByteSize::default()) + .map(|s| s.end_offset) + .max() + .unwrap_or(0); + stats.set_current_offset(current_offset); + + // Check if ANY segment has data. Cannot use current_offset > 0 + // because a single message at offset 0 yields current_offset = 0 + // yet must still increment on the next append. + let should_increment_offset = loaded_log + .segments() + .iter() + .any(|s| s.size > IggyByteSize::default()); + + // After a crash (OOM, SIGKILL), auto_commit may have persisted + // a consumer offset beyond what was flushed to disk. Clamp to + // the partition's actual offset to prevent permanent empty polls. + { + let guard = consumer_offsets.pin(); + for entry in guard.iter() { + let stored = entry.1.offset.load(Ordering::Relaxed); + if stored > current_offset { + warn!( + "Consumer {} offset {} ahead of partition offset {} \ + for stream {}, topic {}, partition {} - clamping \ + (crash recovery)", + entry.0, + stored, + current_offset, + stream_id, + topic_id, + partition_id + ); + entry.1.offset.store(current_offset, Ordering::Relaxed); + } + } + } + { + let guard = consumer_group_offsets.pin(); + for entry in guard.iter() { + let stored = entry.1.offset.load(Ordering::Relaxed); + if stored > current_offset { + warn!( + "Consumer group {:?} offset {} ahead of partition \ + offset {} for stream {}, topic {}, partition {} - \ + clamping (crash recovery)", + entry.0, + stored, + current_offset, + stream_id, + topic_id, + partition_id + ); + entry.1.offset.store(current_offset, Ordering::Relaxed); + } + } + } + + // Initialize journal base_offset so the three-tier + // routing in ops.rs computes correct in_memory_floor. + // Without this, journal defaults to base_offset=0 which + // causes disk reads to be skipped after restart. + if should_increment_offset { + use crate::streaming::partitions::journal::{Inner, Journal}; + loaded_log.journal_mut().init(Inner { + base_offset: current_offset + 1, + ..Default::default() + }); + } + + let revision_id = init_info.revision_id; + + let partition = LocalPartition::with_log( + loaded_log, + stats, + Arc::new(AtomicU64::new(current_offset)), + consumer_offsets, + consumer_group_offsets, + message_deduplicator, + created_at, + revision_id, + should_increment_offset, + ); + + self.local_partitions + .borrow_mut() + .insert(*namespace, partition); + + info!( + "Successfully loaded segments for stream: {}, topic: {}, partition: {}", + stream_id, topic_id, partition_id + ); + } + Err(e) => { + error!( + "Failed to load segments for stream: {}, topic: {}, partition: {}: {}", + stream_id, topic_id, partition_id, e + ); + return Err(e); + } + } + } + } + + Ok(()) + } + + async fn load_users(&self) -> Result<(), IggyError> { + let users_count = self.metadata.users_count(); + self.metrics.increment_users(users_count as u32); + info!("Initialized {} user(s).", users_count); + Ok(()) + } + + pub fn assert_init(&self) -> Result<(), IggyError> { + Ok(()) + } + + pub fn is_shutting_down(&self) -> bool { + self.is_shutting_down.load(Ordering::Relaxed) + } + + pub fn get_stop_receiver(&self) -> StopReceiver { + self.stop_receiver.clone() + } + + #[instrument(skip_all, name = "trace_shutdown")] + pub async fn trigger_shutdown(&self) -> bool { + self.is_shutting_down.store(true, Ordering::SeqCst); + debug!("Shard {} shutdown state set", self.id); + self.task_registry.graceful_shutdown(SHUTDOWN_TIMEOUT).await + } + + pub fn get_available_shards_count(&self) -> u32 { + self.shards.len() as u32 + } + + pub fn ensure_authenticated(&self, session: &Session) -> Result<(), IggyError> { + if !session.is_active() { + error!("{COMPONENT} - session is inactive, session: {session}"); + return Err(IggyError::StaleClient); + } + + if session.is_authenticated() { + Ok(()) + } else { + error!("{COMPONENT} - unauthenticated access attempt, session: {session}"); + Err(IggyError::Unauthenticated) + } + } +} diff --git a/core/server/src/shard/system/clients.rs b/core/server/src/shard/system/clients.rs new file mode 100644 index 0000000000..91cfccfadd --- /dev/null +++ b/core/server/src/shard/system/clients.rs @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::shard::IggyShard; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; +use crate::streaming::clients::client_manager::Client; +use crate::streaming::session::Session; +use iggy_common::TransportProtocol; +use std::net::SocketAddr; +use tracing::{error, info, warn}; + +impl IggyShard { + pub fn add_client(&self, address: &SocketAddr, transport: TransportProtocol) -> Session { + let session = self.client_manager.add_client(address, transport); + self.metrics.increment_clients(1); + session + } + + pub async fn delete_client(&self, client_id: u32) { + let consumer_groups: Vec<(u32, u32, u32)>; + + { + let client = self.client_manager.try_get_client(client_id); + if client.is_none() { + error!("Client with ID: {client_id} was not found in the client manager.",); + return; + } + + self.metrics.decrement_clients(1); + let client = client.unwrap(); + consumer_groups = client + .consumer_groups + .iter() + .map(|c| (c.stream_id, c.topic_id, c.group_id)) + .collect(); + + info!( + "Deleted {} client with ID: {} for IP address: {}", + client.transport, client.session.client_id, client.session.ip_address + ); + } + + for (stream_id, topic_id, consumer_group_id) in consumer_groups.into_iter() { + let request = + ShardRequest::control_plane(ShardRequestPayload::LeaveConsumerGroupMetadataOnly { + stream_id: stream_id as usize, + topic_id: topic_id as usize, + group_id: consumer_group_id as usize, + client_id, + }); + + match self.send_to_control_plane(request).await { + Ok(ShardResponse::LeaveConsumerGroupMetadataOnlyResponse) => {} + Ok(ShardResponse::ErrorResponse(err)) => { + warn!( + "Failed to leave consumer group {consumer_group_id} for client {client_id} during cleanup: {err}" + ); + } + Ok(_) => {} + Err(err) => { + warn!( + "Failed to send leave consumer group request for client {client_id} during cleanup: {err}" + ); + } + } + } + self.client_manager.delete_client(client_id); + } + + pub fn get_client(&self, client_id: u32) -> Option { + self.client_manager.try_get_client(client_id) + } + + pub fn get_clients(&self) -> Vec { + self.client_manager.get_clients() + } +} diff --git a/core/server/src/shard/system/cluster.rs b/core/server/src/shard/system/cluster.rs new file mode 100644 index 0000000000..e4e2688795 --- /dev/null +++ b/core/server/src/shard/system/cluster.rs @@ -0,0 +1,196 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::shard::IggyShard; +use crate::streaming::utils::address::{extract_ip, extract_port}; +use iggy_common::{ + ClusterMetadata, ClusterNode, ClusterNodeRole, ClusterNodeStatus, TransportEndpoints, +}; +use tracing::trace; + +impl IggyShard { + pub fn get_cluster_metadata(&self) -> ClusterMetadata { + let mut nodes = Vec::new(); + + if !self.config.cluster.enabled { + // Single-node: report ourselves as the sole leader; identity + // fields fall back to transport addresses since no cluster list + // is configured. + let current_endpoints = self.get_actual_bound_ports().unwrap_or_else(|| { + TransportEndpoints::new( + extract_port(&self.config.tcp.address), + extract_port(&self.config.quic.address), + extract_port(&self.config.http.address), + extract_port(&self.config.websocket.address), + ) + }); + + nodes.push(ClusterNode { + name: "iggy-node".to_string(), + ip: extract_ip(&self.config.tcp.address), + endpoints: current_endpoints, + role: ClusterNodeRole::Leader, + status: ClusterNodeStatus::Healthy, + }); + + return ClusterMetadata { + name: "single-node".to_string(), + nodes, + }; + } + + // Cluster mode: resolve the current node from the roster by the + // runtime-supplied replica_id. Bootstrap already validated that + // exactly one entry matches; missing `current_replica_id` here + // would indicate a bootstrap bug, so fall back to a safe default + // rather than panicking on the hot metadata path. + let current_id = self.current_replica_id; + let current_node = current_id + .and_then(|id| { + self.config + .cluster + .nodes + .iter() + .find(|node| node.replica_id == id) + }) + .or_else(|| self.config.cluster.nodes.first()); + + let Some(current_node) = current_node else { + // Nodes list is empty even though cluster.enabled=true. Validator + // refuses this state at startup; treat defensively if reached. + return ClusterMetadata { + name: self.config.cluster.name.clone(), + nodes, + }; + }; + + // Use the actual bound ports for the current node so tests that bind + // to port 0 still report the OS-assigned port over the wire. + let current_endpoints = self.get_actual_bound_ports().unwrap_or_else(|| { + TransportEndpoints::new( + current_node + .ports + .tcp + .unwrap_or_else(|| extract_port(&self.config.tcp.address)), + current_node + .ports + .quic + .unwrap_or_else(|| extract_port(&self.config.quic.address)), + current_node + .ports + .http + .unwrap_or_else(|| extract_port(&self.config.http.address)), + current_node + .ports + .websocket + .unwrap_or_else(|| extract_port(&self.config.websocket.address)), + ) + }); + + nodes.push(ClusterNode { + name: current_node.name.clone(), + ip: current_node.ip.clone(), + endpoints: current_endpoints, + role: if self.is_follower { + ClusterNodeRole::Follower + } else { + ClusterNodeRole::Leader + }, + status: ClusterNodeStatus::Healthy, + }); + + for peer in self + .config + .cluster + .nodes + .iter() + .filter(|node| node.replica_id != current_node.replica_id) + { + let endpoints = TransportEndpoints::new( + peer.ports + .tcp + .unwrap_or_else(|| extract_port(&self.config.tcp.address)), + peer.ports + .quic + .unwrap_or_else(|| extract_port(&self.config.quic.address)), + peer.ports + .http + .unwrap_or_else(|| extract_port(&self.config.http.address)), + peer.ports + .websocket + .unwrap_or_else(|| extract_port(&self.config.websocket.address)), + ); + + nodes.push(ClusterNode { + name: peer.name.clone(), + ip: peer.ip.clone(), + endpoints, + role: if self.is_follower { + ClusterNodeRole::Leader + } else { + ClusterNodeRole::Follower + }, + status: ClusterNodeStatus::Healthy, + }); + } + + ClusterMetadata { + name: self.config.cluster.name.clone(), + nodes, + } + } + + /// Get actual bound ports from the shard's bound addresses + /// This is needed when server binds to port 0 (OS-assigned port) + fn get_actual_bound_ports(&self) -> Option { + let tcp_port = self + .tcp_bound_address + .get() + .map(|addr| addr.port()) + .unwrap_or_else(|| extract_port(&self.config.tcp.address)); + + let quic_port = self + .quic_bound_address + .get() + .map(|addr| addr.port()) + .unwrap_or_else(|| extract_port(&self.config.quic.address)); + + let http_port = self + .http_bound_address + .get() + .map(|addr| addr.port()) + .unwrap_or_else(|| extract_port(&self.config.http.address)); + + let websocket_port = self + .websocket_bound_address + .get() + .map(|addr| addr.port()) + .unwrap_or_else(|| extract_port(&self.config.websocket.address)); + + trace!( + "Using actual bound ports - TCP: {}, QUIC: {}, HTTP: {}, WebSocket: {}", + tcp_port, quic_port, http_port, websocket_port + ); + + Some(TransportEndpoints::new( + tcp_port, + quic_port, + http_port, + websocket_port, + )) + } +} diff --git a/core/server/src/shard/system/consumer_groups.rs b/core/server/src/shard/system/consumer_groups.rs new file mode 100644 index 0000000000..7c3b446c19 --- /dev/null +++ b/core/server/src/shard/system/consumer_groups.rs @@ -0,0 +1,205 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::COMPONENT; +use crate::shard::IggyShard; +use crate::shard::transmission::message::{ResolvedConsumerGroup, ResolvedTopic}; +use err_trail::ErrContext; +use iggy_common::Identifier; +use iggy_common::IggyError; +use std::sync::Arc; + +pub struct DeletedConsumerGroup { + pub group_id: usize, + pub partition_ids: Vec, +} + +impl IggyShard { + pub fn create_consumer_group( + &self, + topic: ResolvedTopic, + name: String, + ) -> Result { + let stream = topic.stream_id; + let topic_id = topic.topic_id; + + let partitions_count = self.metadata.partitions_count(stream, topic_id) as u32; + + let id = self + .writer() + .create_consumer_group( + &self.metadata, + stream, + topic_id, + Arc::from(name.as_str()), + partitions_count, + ) + .map_err(|e| { + if let IggyError::ConsumerGroupNameAlreadyExists(_, _) = &e { + IggyError::ConsumerGroupNameAlreadyExists( + name.clone(), + Identifier::numeric(topic_id as u32).unwrap(), + ) + } else { + e + } + })?; + + Ok(id) + } + + pub fn delete_consumer_group( + &self, + group: ResolvedConsumerGroup, + ) -> Result { + let stream = group.stream_id; + let topic = group.topic_id; + let group_id = group.group_id; + + let partition_ids = self + .metadata + .get_consumer_group(stream, topic, group_id) + .map(|cg| cg.partitions.clone()) + .unwrap_or_default(); + + self.client_manager + .delete_consumer_group(stream, topic, group_id); + + self.writer().delete_consumer_group(stream, topic, group_id); + + Ok(DeletedConsumerGroup { + group_id, + partition_ids, + }) + } + + /// Join runs on shard 0 (control plane), single-threaded — no concurrent joins. + pub fn join_consumer_group( + &self, + client_id: u32, + group: ResolvedConsumerGroup, + ) -> Result<(), IggyError> { + let valid_client_ids: Vec = self + .client_manager + .get_clients() + .iter() + .map(|c| c.session.client_id) + .collect(); + + let (_, completable) = self.writer().join_consumer_group( + group.stream_id, + group.topic_id, + group.group_id, + client_id, + Some(valid_client_ids), + ); + + for revocation in completable { + self.writer().complete_partition_revocation( + group.stream_id, + group.topic_id, + group.group_id, + revocation.slab_id, + revocation.member_id, + revocation.partition_id, + false, + ); + } + + if let Some(cg) = + self.metadata + .get_consumer_group(group.stream_id, group.topic_id, group.group_id) + && let Some((_, member)) = cg.members.iter().find(|(_, m)| m.client_id == client_id) + && member.partitions.is_empty() + && !cg.partitions.is_empty() + { + let current_valid_ids: Vec = self + .client_manager + .get_clients() + .iter() + .map(|c| c.session.client_id) + .collect(); + + let potentially_stale: Vec = cg + .members + .iter() + .filter(|(_, m)| { + !m.partitions.is_empty() && !current_valid_ids.contains(&m.client_id) + }) + .map(|(_, m)| m.client_id) + .collect(); + + if !potentially_stale.is_empty() { + tracing::info!( + "join_consumer_group: new member {client_id} has no partitions, found stale members: {potentially_stale:?}, forcing leave" + ); + + for stale_client_id in potentially_stale { + let _ = self.writer().leave_consumer_group( + group.stream_id, + group.topic_id, + group.group_id, + stale_client_id, + ); + } + } + } + + self.client_manager + .join_consumer_group(client_id, group.stream_id, group.topic_id, group.group_id) + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to make client join consumer group for client ID: {}", + client_id + ) + })?; + + Ok(()) + } + + pub fn leave_consumer_group( + &self, + client_id: u32, + group: ResolvedConsumerGroup, + ) -> Result<(), IggyError> { + let member_id = self.writer().leave_consumer_group( + group.stream_id, + group.topic_id, + group.group_id, + client_id, + ); + + if member_id.is_none() { + return Err(IggyError::ConsumerGroupMemberNotFound( + client_id, + Identifier::numeric(group.group_id as u32).unwrap(), + Identifier::numeric(group.topic_id as u32).unwrap(), + )); + } + + self.client_manager + .leave_consumer_group(client_id, group.stream_id, group.topic_id, group.group_id) + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to make client leave consumer group for client ID: {}", + client_id + ) + })?; + + Ok(()) + } +} diff --git a/core/server/src/shard/system/consumer_offsets.rs b/core/server/src/shard/system/consumer_offsets.rs new file mode 100644 index 0000000000..76234ab070 --- /dev/null +++ b/core/server/src/shard/system/consumer_offsets.rs @@ -0,0 +1,489 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::COMPONENT; +use crate::{ + shard::IggyShard, + shard::transmission::message::{ResolvedTopic, ShardRequest, ShardRequestPayload}, + streaming::{ + partitions::consumer_offset::ConsumerOffset, + polling_consumer::{ConsumerGroupId, PollingConsumer}, + }, +}; +use err_trail::ErrContext; +use iggy_common::{Consumer, ConsumerKind, ConsumerOffsetInfo, Identifier, IggyError}; +use server_common::sharding::IggyNamespace; +use std::sync::atomic::Ordering; + +impl IggyShard { + pub async fn store_consumer_offset( + &self, + client_id: u32, + consumer: Consumer, + topic: ResolvedTopic, + partition_id: Option, + offset: u64, + ) -> Result<(PollingConsumer, usize), IggyError> { + let Some((polling_consumer, partition_id)) = self.resolve_consumer_with_partition_id( + topic, + &consumer, + client_id, + partition_id, + false, + )? + else { + return Err(IggyError::NotResolvedConsumer(consumer.id)); + }; + + self.validate_partition_offset(topic.stream_id, topic.topic_id, partition_id, offset)?; + + self.store_consumer_offset_base( + topic.stream_id, + topic.topic_id, + &polling_consumer, + partition_id, + offset, + ); + self.persist_consumer_offset_to_disk( + topic.stream_id, + topic.topic_id, + &polling_consumer, + partition_id, + ) + .await?; + + self.maybe_complete_pending_revocation( + &polling_consumer, + topic.stream_id, + topic.topic_id, + partition_id, + ) + .await; + + Ok((polling_consumer, partition_id)) + } + + pub async fn get_consumer_offset( + &self, + client_id: u32, + consumer: Consumer, + topic: ResolvedTopic, + partition_id: Option, + ) -> Result, IggyError> { + let (polling_consumer, partition_id) = match consumer.kind { + ConsumerKind::Consumer => { + let Some((polling_consumer, partition_id)) = self + .resolve_consumer_with_partition_id( + topic, + &consumer, + client_id, + partition_id, + false, + )? + else { + return Err(IggyError::NotResolvedConsumer(consumer.id.clone())); + }; + (polling_consumer, partition_id) + } + ConsumerKind::ConsumerGroup => { + // Reading offsets doesn't require group membership — offsets are stored + // per consumer group (not per member), so any client can query the + // group's progress. Only store_consumer_offset enforces membership. + let cg_id = self + .metadata + .get_consumer_group_id(topic.stream_id, topic.topic_id, &consumer.id) + .ok_or_else(|| { + IggyError::ConsumerGroupIdNotFound( + consumer.id.clone(), + Identifier::numeric(topic.topic_id as u32).unwrap(), + ) + })?; + let partition_id = partition_id.unwrap_or(0) as usize; + (PollingConsumer::consumer_group(cg_id, 0), partition_id) + } + }; + + if !self + .metadata + .partition_exists(topic.stream_id, topic.topic_id, partition_id) + { + return Err(IggyError::PartitionNotFound( + partition_id, + Identifier::numeric(topic.topic_id as u32).expect("valid topic id"), + Identifier::numeric(topic.stream_id as u32).expect("valid stream id"), + )); + } + + let ns = IggyNamespace::new(topic.stream_id, topic.topic_id, partition_id); + let partition_current_offset = self + .metadata + .get_partition_stats(&ns) + .map(|s| s.current_offset()) + .unwrap_or(0); + + let offset = match polling_consumer { + PollingConsumer::Consumer(id, _) => { + let offsets = self.metadata.get_partition_consumer_offsets( + topic.stream_id, + topic.topic_id, + partition_id, + ); + offsets.and_then(|co| { + let guard = co.pin(); + guard.get(&id).map(|item| ConsumerOffsetInfo { + partition_id: partition_id as u32, + current_offset: partition_current_offset, + stored_offset: item.offset.load(Ordering::Relaxed), + }) + }) + } + PollingConsumer::ConsumerGroup(consumer_group_id, _) => { + let offsets = self.metadata.get_partition_consumer_group_offsets( + topic.stream_id, + topic.topic_id, + partition_id, + ); + offsets.and_then(|co| { + let guard = co.pin(); + guard + .get(&consumer_group_id) + .map(|item| ConsumerOffsetInfo { + partition_id: partition_id as u32, + current_offset: partition_current_offset, + stored_offset: item.offset.load(Ordering::Relaxed), + }) + }) + } + }; + Ok(offset) + } + + pub async fn delete_consumer_offset( + &self, + client_id: u32, + consumer: Consumer, + topic: ResolvedTopic, + partition_id: Option, + ) -> Result<(PollingConsumer, usize), IggyError> { + let Some((polling_consumer, partition_id)) = self.resolve_consumer_with_partition_id( + topic, + &consumer, + client_id, + partition_id, + false, + )? + else { + return Err(IggyError::NotResolvedConsumer(consumer.id)); + }; + + if !self + .metadata + .partition_exists(topic.stream_id, topic.topic_id, partition_id) + { + return Err(IggyError::PartitionNotFound( + partition_id, + Identifier::numeric(topic.topic_id as u32).expect("valid topic id"), + Identifier::numeric(topic.stream_id as u32).expect("valid stream id"), + )); + } + + let path = self.delete_consumer_offset_base( + topic.stream_id, + topic.topic_id, + &polling_consumer, + partition_id, + )?; + self.delete_consumer_offset_from_disk(&path).await?; + Ok((polling_consumer, partition_id)) + } + + pub async fn delete_consumer_group_offsets( + &self, + cg_id: ConsumerGroupId, + stream_id: usize, + topic_id: usize, + partition_ids: &[usize], + ) -> Result<(), IggyError> { + for &partition_id in partition_ids { + if !self + .metadata + .partition_exists(stream_id, topic_id, partition_id) + { + tracing::trace!( + "{COMPONENT} - partition {partition_id} not found in stream {stream_id}/topic {topic_id}, skipping offset cleanup for consumer group {}", + cg_id.0 + ); + continue; + } + + let offsets = self.metadata.get_partition_consumer_group_offsets( + stream_id, + topic_id, + partition_id, + ); + + let Some(offsets) = offsets else { + continue; + }; + + let path = offsets.pin().remove(&cg_id).map(|item| item.path.clone()); + + if let Some(path) = path { + self.delete_consumer_offset_from_disk(&path) + .await + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to delete consumer group offset file for group with ID: {} in partition {} of topic with ID: {} and stream with ID: {}", + cg_id, partition_id, topic_id, stream_id + ) + })?; + } + } + + Ok(()) + } + + fn store_consumer_offset_base( + &self, + stream_id: usize, + topic_id: usize, + polling_consumer: &PollingConsumer, + partition_id: usize, + offset: u64, + ) { + match polling_consumer { + PollingConsumer::Consumer(id, _) => { + let Some(offsets) = + self.metadata + .get_partition_consumer_offsets(stream_id, topic_id, partition_id) + else { + return; + }; + + let guard = offsets.pin(); + let entry = guard.get_or_insert_with(*id, || { + let dir_path = self.config.system.get_consumer_offsets_path( + stream_id, + topic_id, + partition_id, + ); + let path = format!("{}/{}", dir_path, id); + ConsumerOffset::new(ConsumerKind::Consumer, *id as u32, offset, path) + }); + entry.offset.store(offset, Ordering::Release); + } + PollingConsumer::ConsumerGroup(cg_id, _) => { + let Some(offsets) = self.metadata.get_partition_consumer_group_offsets( + stream_id, + topic_id, + partition_id, + ) else { + return; + }; + + let guard = offsets.pin(); + let entry = guard.get_or_insert_with(*cg_id, || { + let dir_path = self.config.system.get_consumer_group_offsets_path( + stream_id, + topic_id, + partition_id, + ); + let path = format!("{}/{}", dir_path, cg_id.0); + ConsumerOffset::new(ConsumerKind::ConsumerGroup, cg_id.0 as u32, offset, path) + }); + entry.offset.store(offset, Ordering::Release); + } + } + } + + fn delete_consumer_offset_base( + &self, + stream_id: usize, + topic_id: usize, + polling_consumer: &PollingConsumer, + partition_id: usize, + ) -> Result { + match polling_consumer { + PollingConsumer::Consumer(id, _) => { + let offsets = self + .metadata + .get_partition_consumer_offsets(stream_id, topic_id, partition_id) + .ok_or_else(|| IggyError::ConsumerOffsetNotFound(*id))?; + + let guard = offsets.pin(); + let offset = guard + .remove(id) + .ok_or_else(|| IggyError::ConsumerOffsetNotFound(*id))?; + Ok(offset.path.clone()) + } + PollingConsumer::ConsumerGroup(cg_id, _) => { + let offsets = self + .metadata + .get_partition_consumer_group_offsets(stream_id, topic_id, partition_id) + .ok_or_else(|| IggyError::ConsumerOffsetNotFound(cg_id.0))?; + + let guard = offsets.pin(); + let offset = guard + .remove(cg_id) + .ok_or_else(|| IggyError::ConsumerOffsetNotFound(cg_id.0))?; + Ok(offset.path.clone()) + } + } + } + + async fn persist_consumer_offset_to_disk( + &self, + stream_id: usize, + topic_id: usize, + polling_consumer: &PollingConsumer, + partition_id: usize, + ) -> Result<(), IggyError> { + use crate::streaming::partitions::storage::persist_offset; + + let (offset_value, path) = match polling_consumer { + PollingConsumer::Consumer(id, _) => { + let offsets = self + .metadata + .get_partition_consumer_offsets(stream_id, topic_id, partition_id) + .ok_or_else(|| IggyError::ConsumerOffsetNotFound(*id))?; + + let guard = offsets.pin(); + let item = guard + .get(id) + .ok_or_else(|| IggyError::ConsumerOffsetNotFound(*id))?; + (item.offset.load(Ordering::Relaxed), item.path.clone()) + } + PollingConsumer::ConsumerGroup(cg_id, _) => { + let offsets = self + .metadata + .get_partition_consumer_group_offsets(stream_id, topic_id, partition_id) + .ok_or_else(|| IggyError::ConsumerOffsetNotFound(cg_id.0))?; + + let guard = offsets.pin(); + let item = guard + .get(cg_id) + .ok_or_else(|| IggyError::ConsumerOffsetNotFound(cg_id.0))?; + (item.offset.load(Ordering::Relaxed), item.path.clone()) + } + }; + persist_offset(&path, offset_value).await + } + + pub async fn delete_consumer_offset_from_disk(&self, path: &str) -> Result<(), IggyError> { + crate::streaming::partitions::storage::delete_persisted_offset(path).await + } + + /// Enumerates and deletes all consumer/group offset files for a partition from disk. + /// Uses filesystem paths from config rather than in-memory state (which may already be cleared). + pub async fn delete_all_consumer_offset_files( + &self, + stream_id: usize, + topic_id: usize, + partition_id: usize, + ) -> Result<(), IggyError> { + let consumers_path = + self.config + .system + .get_consumer_offsets_path(stream_id, topic_id, partition_id); + let groups_path = + self.config + .system + .get_consumer_group_offsets_path(stream_id, topic_id, partition_id); + + Self::delete_all_files_in_dir(&consumers_path).await?; + Self::delete_all_files_in_dir(&groups_path).await?; + Ok(()) + } + + /// Complete a pending partition revocation if this offset commit satisfies it. + pub(crate) async fn maybe_complete_pending_revocation( + &self, + polling_consumer: &PollingConsumer, + stream_id: usize, + topic_id: usize, + partition_id: usize, + ) { + let PollingConsumer::ConsumerGroup(group_id, member_id) = polling_consumer else { + return; + }; + + let completion_info = self.metadata.with_metadata(|m| { + let topic = m.streams.get(stream_id)?.topics.get(topic_id)?; + let group = topic.consumer_groups.get(group_id.0)?; + let member = group.members.get(member_id.0)?; + if !member + .pending_revocations + .iter() + .any(|revocation| revocation.partition_id == partition_id) + { + return None; + } + let partition = topic.partitions.get(partition_id)?; + let last_polled = { + let guard = partition.last_polled_offsets.pin(); + guard.get(group_id).map(|v| v.load(Ordering::Acquire)) + }; + let can_complete = match last_polled { + None => true, + Some(polled) => { + let guard = partition.consumer_group_offsets.pin(); + guard + .get(group_id) + .map(|co| co.offset.load(Ordering::Acquire)) + .is_some_and(|c| c >= polled) + } + }; + if can_complete { Some(member.id) } else { None } + }); + + if let Some(logical_member_id) = completion_info { + let request = + ShardRequest::control_plane(ShardRequestPayload::CompletePartitionRevocation { + stream_id, + topic_id, + group_id: group_id.0, + member_slab_id: member_id.0, + member_id: logical_member_id, + partition_id, + timed_out: false, + }); + let _ = self.send_to_control_plane(request).await; + } + } + + async fn delete_all_files_in_dir(dir: &str) -> Result<(), IggyError> { + let entries = match std::fs::read_dir(dir) { + Ok(entries) => entries, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(e) => { + return Err(IggyError::IoError(format!( + "Failed to read directory {dir}: {e}" + ))); + } + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_file() { + crate::streaming::partitions::storage::delete_persisted_offset( + &path.to_string_lossy(), + ) + .await?; + } + } + Ok(()) + } +} diff --git a/core/server/src/shard/system/info.rs b/core/server/src/shard/system/info.rs new file mode 100644 index 0000000000..4d7ba7e7bc --- /dev/null +++ b/core/server/src/shard/system/info.rs @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use iggy_common::SemanticVersion; +use serde::{Deserialize, Serialize}; +use std::collections::hash_map::DefaultHasher; +use std::fmt::Display; +use std::hash::{Hash, Hasher}; + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct SystemInfo { + pub version: Version, + pub migrations: Vec, +} + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct Version { + pub version: String, + pub hash: String, +} + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct Migration { + pub id: u32, + pub name: String, + pub hash: String, + pub applied_at: u64, +} + +impl SystemInfo { + pub fn update_version(&mut self, version: &SemanticVersion) { + self.version.version = version.to_string(); + let mut hasher = DefaultHasher::new(); + self.version.hash.hash(&mut hasher); + self.version.hash = hasher.finish().to_string(); + } +} + +impl Hash for SystemInfo { + fn hash(&self, state: &mut H) { + self.version.version.hash(state); + for migration in &self.migrations { + migration.hash(state); + } + } +} + +impl Hash for Migration { + fn hash(&self, state: &mut H) { + self.id.hash(state); + } +} + +impl Display for Version { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "version: {}", self.version) + } +} + +impl Display for SystemInfo { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "system info, {}", self.version) + } +} diff --git a/core/server/src/shard/system/messages.rs b/core/server/src/shard/system/messages.rs new file mode 100644 index 0000000000..8770ac905c --- /dev/null +++ b/core/server/src/shard/system/messages.rs @@ -0,0 +1,695 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::COMPONENT; +use crate::shard::IggyShard; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ + ResolvedPartition, ResolvedTopic, ShardRequest, ShardRequestPayload, +}; +use crate::streaming::partitions::journal::Journal; +use crate::streaming::polling_consumer::PollingConsumer; +use crate::streaming::segments::{IggyIndexesMut, IggyMessagesBatchMut, IggyMessagesBatchSet}; +use err_trail::ErrContext; +use iggy_common::IggyPollMetadata; +use iggy_common::{ + Consumer, EncryptorKind, IGGY_MESSAGE_HEADER_SIZE, Identifier, IggyError, PollingStrategy, +}; +use server_common::PooledBuffer; +use server_common::sharding::IggyNamespace; +use std::sync::atomic::Ordering; +use tracing::error; + +impl IggyShard { + /// Appends messages to partition. Permission must be checked by caller via + /// `resolve_topic_for_append()` before calling this method. + pub async fn append_messages( + &self, + partition: ResolvedPartition, + batch: IggyMessagesBatchMut, + ) -> Result<(), IggyError> { + if batch.count() == 0 { + return Ok(()); + } + + let namespace = IggyNamespace::new( + partition.stream_id, + partition.topic_id, + partition.partition_id, + ); + + let payload = ShardRequestPayload::SendMessages { batch }; + let request = ShardRequest::data_plane(namespace, payload); + + match self.send_to_data_plane(request).await? { + ShardResponse::SendMessages => Ok(()), + ShardResponse::ErrorResponse(err) => Err(err), + _ => unreachable!("Expected SendMessages response"), + } + } + + /// Polls messages from partition. Permission must be checked by caller via + /// `resolve_topic_for_poll()` before calling this method. + pub async fn poll_messages( + &self, + client_id: u32, + topic: ResolvedTopic, + consumer: Consumer, + maybe_partition_id: Option, + args: PollingArgs, + ) -> Result<(IggyPollMetadata, IggyMessagesBatchSet), IggyError> { + let Some((consumer, partition_id)) = self.resolve_consumer_with_partition_id( + topic, + &consumer, + client_id, + maybe_partition_id, + true, + )? + else { + return Ok((IggyPollMetadata::new(0, 0), IggyMessagesBatchSet::empty())); + }; + + let namespace = IggyNamespace::new(topic.stream_id, topic.topic_id, partition_id); + + let payload = ShardRequestPayload::PollMessages { consumer, args }; + let request = ShardRequest::data_plane(namespace, payload); + + let (metadata, batch) = match self.send_to_data_plane(request).await? { + ShardResponse::PollMessages(result) => result, + ShardResponse::ErrorResponse(err) => return Err(err), + _ => unreachable!("Expected PollMessages response"), + }; + + let batch = if let Some(encryptor) = &self.encryptor { + self.decrypt_messages(batch, encryptor).await? + } else { + batch + }; + + // Track last offset sent to CG member for cooperative rebalance. + if let PollingConsumer::ConsumerGroup(group_id, _) = &consumer + && let Some(last_offset) = batch.last_offset() + { + self.metadata.record_polled_offset( + topic.stream_id, + topic.topic_id, + group_id.0, + partition_id, + last_offset, + ); + } + + Ok((metadata, batch)) + } + + pub async fn flush_unsaved_buffer( + &self, + user_id: u32, + partition: ResolvedPartition, + fsync: bool, + ) -> Result<(), IggyError> { + self.metadata + .perm_append_messages(user_id, partition.stream_id, partition.topic_id) + .error(|e: &IggyError| { + format!("{COMPONENT} (error: {e}) - permission denied to flush unsaved buffer for user {} on stream ID: {}, topic ID: {}", user_id, partition.stream_id as u32, partition.topic_id as u32) + })?; + + let namespace = IggyNamespace::new( + partition.stream_id, + partition.topic_id, + partition.partition_id, + ); + let payload = ShardRequestPayload::FlushUnsavedBuffer { fsync }; + let request = ShardRequest::data_plane(namespace, payload); + + match self.send_to_data_plane(request).await? { + ShardResponse::FlushUnsavedBuffer { .. } => Ok(()), + ShardResponse::ErrorResponse(err) => Err(err), + _ => unreachable!("Expected FlushUnsavedBuffer response"), + } + } + + /// Flushes unsaved messages from the partition store to disk. + /// Returns the number of messages saved. + pub(crate) async fn flush_unsaved_buffer_from_local_partitions( + &self, + namespace: &IggyNamespace, + fsync: bool, + ) -> Result { + let frozen_batches = { + let mut partitions = self.local_partitions.borrow_mut(); + let Some(partition) = partitions.get_mut(namespace) else { + return Ok(0); + }; + if !partition.log.has_segments() || partition.log.journal().is_empty() { + return Ok(0); + } + let batches = partition.log.journal_mut().commit(); + partition.log.ensure_indexes(); + batches.append_indexes_to(partition.log.active_indexes_mut().unwrap()); + + let frozen: Vec<_> = batches + .into_inner() + .into_iter() + .map(|mut b| b.freeze()) + .collect(); + partition.log.set_in_flight(frozen.clone()); + frozen + }; + + let saved_count = self + .persist_frozen_batches_to_disk(namespace, frozen_batches) + .await?; + + if fsync { + self.fsync_all_messages_from_local_partitions(namespace) + .await?; + } + + Ok(saved_count) + } + + pub(crate) async fn fsync_all_messages_from_local_partitions( + &self, + namespace: &IggyNamespace, + ) -> Result<(), IggyError> { + let storage = { + let partitions = self.local_partitions.borrow(); + let Some(partition) = partitions.get(namespace) else { + return Ok(()); + }; + if !partition.log.has_segments() { + return Ok(()); + } + partition.log.active_storage().clone() + }; + + if storage.messages_writer.is_none() || storage.index_writer.is_none() { + return Ok(()); + } + + if let Some(ref messages_writer) = storage.messages_writer + && let Err(e) = messages_writer.fsync().await + { + tracing::error!( + "Failed to fsync messages writer for partition {:?}: {}", + namespace, + e + ); + return Err(e); + } + + if let Some(ref index_writer) = storage.index_writer + && let Err(e) = index_writer.fsync().await + { + tracing::error!( + "Failed to fsync index writer for partition {:?}: {}", + namespace, + e + ); + return Err(e); + } + + Ok(()) + } + + pub(crate) async fn auto_commit_consumer_offset_from_local_partition( + &self, + namespace: &IggyNamespace, + consumer: PollingConsumer, + offset: u64, + ) -> Result<(), IggyError> { + let (offset_value, path) = { + let partitions = self.local_partitions.borrow(); + let partition = partitions.get(namespace).ok_or_else(|| { + IggyError::PartitionNotFound( + namespace.partition_id(), + Identifier::numeric(namespace.topic_id() as u32).unwrap(), + Identifier::numeric(namespace.stream_id() as u32).unwrap(), + ) + })?; + + match consumer { + PollingConsumer::Consumer(consumer_id, _) => { + tracing::trace!( + "Auto-committing offset {} for consumer {} on partition {:?}", + offset, + consumer_id, + namespace + ); + let hdl = partition.consumer_offsets.pin(); + let item = hdl.get_or_insert( + consumer_id, + crate::streaming::partitions::consumer_offset::ConsumerOffset::default_for_consumer( + consumer_id as u32, + &self.config.system.get_consumer_offsets_path( + namespace.stream_id(), + namespace.topic_id(), + namespace.partition_id(), + ), + ), + ); + item.offset.store(offset, Ordering::Release); + (item.offset.load(Ordering::Relaxed), item.path.clone()) + } + PollingConsumer::ConsumerGroup(consumer_group_id, _) => { + tracing::trace!( + "Auto-committing offset {} for consumer group {} on partition {:?}", + offset, + consumer_group_id.0, + namespace + ); + let hdl = partition.consumer_group_offsets.pin(); + let item = hdl.get_or_insert( + consumer_group_id, + crate::streaming::partitions::consumer_offset::ConsumerOffset::default_for_consumer_group( + consumer_group_id, + &self.config.system.get_consumer_group_offsets_path( + namespace.stream_id(), + namespace.topic_id(), + namespace.partition_id(), + ), + ), + ); + item.offset.store(offset, Ordering::Release); + (item.offset.load(Ordering::Relaxed), item.path.clone()) + } + } + }; + + crate::streaming::partitions::storage::persist_offset(&path, offset_value).await?; + + self.maybe_complete_pending_revocation( + &consumer, + namespace.stream_id(), + namespace.topic_id(), + namespace.partition_id(), + ) + .await; + + Ok(()) + } + + /// Appends a batch to the active segment, flushing to disk and rotating if needed. + /// + /// Safety: called exclusively from the message pump — segment indices captured before + /// internal `.await` points (prepare_for_persistence, persist, rotate) remain valid + /// because no other handler can modify the segment vec while this frame is in progress. + pub(crate) async fn append_messages_to_local_partition( + &self, + namespace: &IggyNamespace, + mut batch: IggyMessagesBatchMut, + config: &crate::configs::system::SystemConfig, + ) -> Result<(), IggyError> { + let ( + current_offset, + current_position, + segment_start_offset, + segment_index, + message_deduplicator, + ) = { + let partitions = self.local_partitions.borrow(); + let partition = partitions + .get(namespace) + .expect("local_partitions: partition must exist"); + + let current_offset = if partition.should_increment_offset { + partition.offset.load(Ordering::Relaxed) + 1 + } else { + 0 + }; + + let segment = partition.log.active_segment(); + let segment_index = partition.log.segments().len() - 1; + + ( + current_offset, + segment.current_position, + segment.start_offset, + segment_index, + partition.message_deduplicator.clone(), + ) + }; + + batch + .prepare_for_persistence( + segment_start_offset, + current_offset, + current_position, + message_deduplicator.as_ref(), + ) + .await; + + let (journal_messages_count, journal_size, is_full) = { + let mut partitions = self.local_partitions.borrow_mut(); + let partition = partitions + .get_mut(namespace) + .expect("local_partitions: partition must exist"); + + let segment = &mut partition.log.segments_mut()[segment_index]; + + if segment.start_timestamp == 0 { + segment.start_timestamp = batch.first_timestamp().unwrap(); + } + + let batch_messages_size = batch.size(); + let batch_messages_count = batch.count(); + + partition + .stats + .increment_size_bytes(batch_messages_size as u64); + partition + .stats + .increment_messages_count(batch_messages_count as u64); + + segment.end_timestamp = batch.last_timestamp().unwrap(); + segment.end_offset = batch.last_offset().unwrap(); + + let (journal_messages_count, journal_size) = + partition.log.journal_mut().append(batch)?; + + let last_offset = if batch_messages_count == 0 { + current_offset + } else { + current_offset + batch_messages_count as u64 - 1 + }; + + if partition.should_increment_offset { + partition.offset.store(last_offset, Ordering::Relaxed); + } else { + partition.should_increment_offset = true; + partition.offset.store(last_offset, Ordering::Relaxed); + } + partition.stats.set_current_offset(last_offset); + partition.log.segments_mut()[segment_index].current_position += batch_messages_size; + + let is_full = partition.log.segments()[segment_index].is_full(); + + (journal_messages_count, journal_size, is_full) + }; + + let unsaved_messages_count_exceeded = + journal_messages_count >= config.partition.messages_required_to_save; + let unsaved_messages_size_exceeded = journal_size + >= config + .partition + .size_of_messages_required_to_save + .as_bytes_u64() as u32; + + if is_full || unsaved_messages_count_exceeded || unsaved_messages_size_exceeded { + let frozen_batches = { + let mut partitions = self.local_partitions.borrow_mut(); + let partition = partitions + .get_mut(namespace) + .expect("local_partitions: partition must exist"); + let batches = partition.log.journal_mut().commit(); + partition.log.ensure_indexes(); + batches.append_indexes_to(partition.log.active_indexes_mut().unwrap()); + + let frozen: Vec<_> = batches + .into_inner() + .into_iter() + .map(|mut b| b.freeze()) + .collect(); + partition.log.set_in_flight(frozen.clone()); + frozen + }; + + self.persist_frozen_batches_to_disk(namespace, frozen_batches) + .await?; + + if is_full { + self.rotate_segment_in_local_partitions(namespace).await?; + } + } + + Ok(()) + } + + /// Persists already-frozen batches to disk. Caller must have set in_flight buffer. + async fn persist_frozen_batches_to_disk( + &self, + namespace: &IggyNamespace, + frozen_batches: Vec, + ) -> Result { + let batch_count: u32 = frozen_batches.iter().map(|b| b.count()).sum(); + + if batch_count == 0 { + return Ok(0); + } + + let (messages_writer, index_writer) = { + let partitions = self.local_partitions.borrow(); + let partition = partitions + .get(namespace) + .expect("local_partitions: partition must exist"); + + if !partition.log.has_segments() { + return Ok(0); + } + + let messages_writer = partition + .log + .active_storage() + .messages_writer + .as_ref() + .expect("Messages writer not initialized") + .clone(); + let index_writer = partition + .log + .active_storage() + .index_writer + .as_ref() + .expect("Index writer not initialized") + .clone(); + (messages_writer, index_writer) + }; + + let saved = messages_writer + .as_ref() + .save_frozen_batches(&frozen_batches) + .await?; + + let unsaved_indexes_slice = { + let partitions = self.local_partitions.borrow(); + let partition = partitions + .get(namespace) + .expect("local_partitions: partition must exist"); + let segment_index = partition.log.segments().len() - 1; + partition.log.indexes()[segment_index] + .as_ref() + .expect("indexes must exist for segment being persisted") + .unsaved_slice() + }; + + index_writer + .as_ref() + .save_indexes(unsaved_indexes_slice) + .await?; + + tracing::trace!( + "Persisted {} messages on disk for partition: {:?}, total bytes written: {}.", + batch_count, + namespace, + saved + ); + + { + let mut partitions = self.local_partitions.borrow_mut(); + let partition = partitions + .get_mut(namespace) + .expect("local_partitions: partition must exist"); + + let segment_index = partition.log.segments().len() - 1; + let indexes = partition.log.indexes_mut()[segment_index] + .as_mut() + .expect("indexes must exist for segment being persisted"); + indexes.mark_saved(); + + let segment = &mut partition.log.segments_mut()[segment_index]; + segment.size = + iggy_common::IggyByteSize::from(segment.size.as_bytes_u64() + saved.as_bytes_u64()); + + partition.log.clear_in_flight(); + } + + Ok(batch_count) + } + + pub(crate) async fn poll_messages_from_local_partition( + &self, + namespace: &IggyNamespace, + consumer: crate::streaming::polling_consumer::PollingConsumer, + args: PollingArgs, + ) -> Result<(IggyPollMetadata, IggyMessagesBatchSet), IggyError> { + crate::streaming::partitions::ops::poll_messages( + &self.local_partitions, + namespace, + consumer, + args, + ) + .await + } + + async fn decrypt_messages( + &self, + batches: IggyMessagesBatchSet, + encryptor: &EncryptorKind, + ) -> Result { + let mut decrypted_batches = Vec::with_capacity(batches.containers_count()); + for batch in batches.iter() { + let mut indexes = IggyIndexesMut::with_capacity(batch.count() as usize, 0); + let mut decrypted_messages = PooledBuffer::with_capacity(batch.size() as usize); + let mut position = 0; + + for message in batch.iter() { + let mut header = message.header().to_header(); + let offset = header.offset; + let payload = encryptor.decrypt(message.payload()); + match payload { + Ok(payload) => { + // Update the header with the decrypted payload length + header.payload_length = payload.len() as u32; + + // Decrypt user headers if present + let decrypted_user_headers = if let Some(user_headers) = + message.user_headers() + { + match encryptor.decrypt(user_headers) { + Ok(decrypted) => { + header.user_headers_length = decrypted.len() as u32; + Some(decrypted) + } + Err(error) => { + error!( + "Cannot decrypt the message user headers at offset: {offset}. Error: {error}" + ); + continue; + } + } + } else { + None + }; + + decrypted_messages.extend_from_slice(&header.to_bytes()); + decrypted_messages.extend_from_slice(&payload); + if let Some(ref user_headers) = decrypted_user_headers { + decrypted_messages.extend_from_slice(user_headers); + } + position += IGGY_MESSAGE_HEADER_SIZE + + payload.len() + + header.user_headers_length as usize; + indexes.insert(0, position as u32, 0); + } + Err(error) => { + error!("Cannot decrypt the message at offset: {offset}. Error: {error}",); + continue; + } + } + } + let decrypted_batch = + IggyMessagesBatchMut::from_indexes_and_messages(indexes, decrypted_messages); + decrypted_batches.push(decrypted_batch); + } + + Ok(IggyMessagesBatchSet::from_vec(decrypted_batches)) + } + + pub fn maybe_encrypt_messages( + &self, + batch: IggyMessagesBatchMut, + ) -> Result { + let encryptor = match self.encryptor.as_ref() { + Some(encryptor) => encryptor, + None => return Ok(batch), + }; + let mut encrypted_messages = PooledBuffer::with_capacity(batch.size() as usize * 2); + let mut indexes = IggyIndexesMut::with_capacity(batch.count() as usize, 0); + let mut position = 0; + + for message in batch.iter() { + let header = message.header().to_header(); + let offset = header.offset; + let payload_bytes = message.payload(); + let user_headers_bytes = message.user_headers(); + + let encrypted_payload = encryptor.encrypt(payload_bytes); + match encrypted_payload { + Ok(encrypted_payload) => { + let mut updated_header = header; + updated_header.payload_length = encrypted_payload.len() as u32; + + // Encrypt user headers if present + let encrypted_user_headers = if let Some(user_headers_bytes) = + user_headers_bytes + { + match encryptor.encrypt(user_headers_bytes) { + Ok(encrypted) => { + updated_header.user_headers_length = encrypted.len() as u32; + Some(encrypted) + } + Err(error) => { + error!( + "Cannot encrypt the message user headers at offset: {offset}. Error: {error}" + ); + continue; + } + } + } else { + None + }; + + encrypted_messages.extend_from_slice(&updated_header.to_bytes()); + encrypted_messages.extend_from_slice(&encrypted_payload); + if let Some(ref encrypted_user_headers) = encrypted_user_headers { + encrypted_messages.extend_from_slice(encrypted_user_headers); + } + position += IGGY_MESSAGE_HEADER_SIZE + + encrypted_payload.len() + + updated_header.user_headers_length as usize; + indexes.insert(0, position as u32, 0); + } + Err(error) => { + error!("Cannot encrypt the message at offset: {offset}. Error: {error}",); + continue; + } + } + } + + Ok(IggyMessagesBatchMut::from_indexes_and_messages( + indexes, + encrypted_messages, + )) + } +} + +#[derive(Debug)] +pub struct PollingArgs { + pub strategy: PollingStrategy, + pub count: u32, + pub auto_commit: bool, +} + +impl PollingArgs { + pub fn new(strategy: PollingStrategy, count: u32, auto_commit: bool) -> Self { + Self { + strategy, + count, + auto_commit, + } + } +} diff --git a/core/server/src/shard/system/mod.rs b/core/server/src/shard/system/mod.rs new file mode 100644 index 0000000000..f326ea08e4 --- /dev/null +++ b/core/server/src/shard/system/mod.rs @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod clients; +pub mod cluster; +pub mod consumer_groups; +pub mod consumer_offsets; +pub mod info; +pub mod messages; +pub mod partitions; +pub mod personal_access_tokens; +pub mod segments; +pub mod snapshot; +pub mod stats; +pub mod storage; +pub mod streams; +pub mod topics; +pub mod users; +pub mod utils; + +pub const COMPONENT: &str = "SHARD_SYSTEM"; diff --git a/core/server/src/shard/system/partitions.rs b/core/server/src/shard/system/partitions.rs new file mode 100644 index 0000000000..519601bc1e --- /dev/null +++ b/core/server/src/shard/system/partitions.rs @@ -0,0 +1,443 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::metadata::PartitionMeta; +use crate::shard::IggyShard; +use crate::shard::calculate_shard_assignment; +use crate::shard::transmission::event::PartitionInfo; +use crate::shard::transmission::message::ResolvedTopic; +use crate::streaming::partitions::consumer_group_offsets::ConsumerGroupOffsets; +use crate::streaming::partitions::consumer_offsets::ConsumerOffsets; +use crate::streaming::partitions::local_partition::LocalPartition; +use crate::streaming::partitions::storage::create_partition_file_hierarchy; +use crate::streaming::partitions::storage::delete_partitions_from_disk; +use crate::streaming::segments::Segment; +use crate::streaming::segments::storage::create_segment_storage; +use crate::streaming::stats::PartitionStats; +use iggy_common::Identifier; +use iggy_common::IggyError; +use iggy_common::IggyTimestamp; +use server_common::sharding::IggyNamespace; +use server_common::sharding::{PartitionLocation, ShardId}; +use std::sync::Arc; +use std::time::Duration; +use tracing::{info, warn}; + +const PARTITION_INIT_BASE_INTERVAL: Duration = Duration::from_micros(100); +const PARTITION_INIT_MAX_INTERVAL: Duration = Duration::from_millis(50); +const PARTITION_INIT_TIMEOUT: Duration = Duration::from_secs(5); + +impl IggyShard { + pub async fn create_partitions( + &self, + topic: ResolvedTopic, + partitions_count: u32, + ) -> Result, IggyError> { + let stream = topic.stream_id; + let topic_id = topic.topic_id; + + let created_at = IggyTimestamp::now(); + let shards_count = self.get_available_shards_count(); + + let parent_stats = self + .metadata + .get_topic_stats(stream, topic_id) + .expect("Parent topic stats must exist"); + + let count_before = self + .metadata + .get_partitions_count(stream, topic_id) + .unwrap_or(0); + let partition_ids: Vec = + (count_before..count_before + partitions_count as usize).collect(); + let partition_infos: Vec = partition_ids + .iter() + .map(|&id| PartitionInfo { id, created_at }) + .collect(); + + for info in &partition_infos { + create_partition_file_hierarchy(stream, topic_id, info.id, &self.config.system).await?; + } + + let metas: Vec = (0..partitions_count) + .map(|_| PartitionMeta { + id: 0, + created_at, + revision_id: 0, + stats: Arc::new(PartitionStats::new(parent_stats.clone())), + consumer_offsets: Arc::new(ConsumerOffsets::with_capacity(0)), + consumer_group_offsets: Arc::new(ConsumerGroupOffsets::with_capacity(0)), + last_polled_offsets: Arc::new(papaya::HashMap::new()), + }) + .collect(); + + let assigned_ids = self + .writer() + .add_partitions(&self.metadata, stream, topic_id, metas); + debug_assert_eq!( + assigned_ids, partition_ids, + "Partition IDs mismatch: expected {:?}, got {:?}", + partition_ids, assigned_ids + ); + + self.metrics.increment_partitions(partitions_count); + self.metrics.increment_segments(partitions_count); + + for info in &partition_infos { + let partition_id = info.id; + let ns = IggyNamespace::new(stream, topic_id, partition_id); + let shard_id = ShardId::new(calculate_shard_assignment(&ns, shards_count)); + let is_current_shard = self.id == *shard_id; + // epoch is reconciler-only; unused by legacy server. + let location = PartitionLocation::new(shard_id, 0); + self.insert_shard_table_record(ns, location); + + if is_current_shard { + self.ensure_partition(&ns).await?; + } + } + Ok(partition_infos) + } + + /// Ensures partition is initialized in local_partitions. Idempotent. + /// Returns error if partition doesn't exist in metadata. + /// If another task is already initializing the partition, waits for it to complete. + pub async fn ensure_partition(&self, ns: &IggyNamespace) -> Result<(), IggyError> { + use std::time::Instant; + + let deadline = Instant::now() + PARTITION_INIT_TIMEOUT; + let mut backoff = PARTITION_INIT_BASE_INTERVAL; + + loop { + // partition_needs_init handles both fresh and stale entries: + // - Returns None if fresh entry exists (same revision_id) + // - Removes stale entry and returns Some if revision_id differs + // - Returns Some if no entry exists + let Some(created_at) = self.partition_needs_init(ns)? else { + return Ok(()); + }; + + if Instant::now() >= deadline { + warn!( + "Partition initialization timed out after {:?} for stream: {}, topic: {}, partition: {}", + PARTITION_INIT_TIMEOUT, + ns.stream_id(), + ns.topic_id(), + ns.partition_id() + ); + return Err(IggyError::TaskTimeout); + } + + let is_pending = self.pending_partition_inits.borrow().contains(ns); + if is_pending { + compio::time::sleep(backoff).await; + backoff = (backoff * 2).min(PARTITION_INIT_MAX_INTERVAL); + continue; + } + + // Double-check after potential yield - another task may have claimed it + { + let mut pending = self.pending_partition_inits.borrow_mut(); + if pending.contains(ns) { + continue; + } + pending.insert(*ns); + } + + let result = self.init_partition_inner(ns, created_at).await; + self.pending_partition_inits.borrow_mut().remove(ns); + return result; + } + } + + /// Returns `Ok(Some(timestamp))` if partition needs initialization, + /// `Ok(None)` if already initialized, or `Err` if partition doesn't exist in metadata. + fn partition_needs_init(&self, ns: &IggyNamespace) -> Result, IggyError> { + let init_info = + self.metadata + .get_partition_init_info(ns.stream_id(), ns.topic_id(), ns.partition_id()); + + let revision_id = init_info.as_ref().map(|m| m.revision_id); + + let needs_init = { + let partitions = self.local_partitions.borrow(); + match (partitions.get(ns), revision_id) { + (Some(data), Some(rev)) if data.revision_id == rev => false, + (Some(_), _) => { + drop(partitions); + self.local_partitions.borrow_mut().remove(ns); + true + } + (None, _) => true, + } + }; + + if needs_init { + let created_at = init_info.map(|m| m.created_at).ok_or_else(|| { + IggyError::PartitionNotFound( + ns.partition_id(), + Identifier::numeric(ns.topic_id() as u32).unwrap(), + Identifier::numeric(ns.stream_id() as u32).unwrap(), + ) + })?; + Ok(Some(created_at)) + } else { + Ok(None) + } + } + + async fn init_partition_inner( + &self, + ns: &IggyNamespace, + created_at: IggyTimestamp, + ) -> Result<(), IggyError> { + let stream_id = ns.stream_id(); + let topic_id = ns.topic_id(); + let partition_id = ns.partition_id(); + + info!( + "Initializing partition in local_partitions: partition ID: {} for topic ID: {} for stream ID: {}", + partition_id, topic_id, stream_id + ); + + let stats = self + .metadata + .get_partition_stats_by_ids(stream_id, topic_id, partition_id) + .expect("Partition stats must exist in SharedMetadata"); + + let partition_path = + self.config + .system + .get_partition_path(stream_id, topic_id, partition_id); + + let mut loaded_log = crate::bootstrap::load_segments( + &self.config.system, + stream_id, + topic_id, + partition_id, + partition_path, + stats.clone(), + ) + .await?; + + if !loaded_log.has_segments() { + info!( + "No segments found on disk for partition ID: {} for topic ID: {} for stream ID: {}, creating initial segment", + partition_id, topic_id, stream_id + ); + + let start_offset = 0; + let segment = Segment::new(start_offset, self.config.system.segment.size); + + let storage = create_segment_storage( + &self.config.system, + stream_id, + topic_id, + partition_id, + 0, + 0, + start_offset, + ) + .await?; + + loaded_log.add_persisted_segment(segment, storage); + stats.increment_segments_count(1); + } + + // Use the max end_offset across segments that have data, not just + // the active segment. Mirrors the fix in shard/mod.rs bootstrap. + let current_offset = loaded_log + .segments() + .iter() + .filter(|s| s.size > iggy_common::IggyByteSize::default()) + .map(|s| s.end_offset) + .max() + .unwrap_or(0); + + let should_increment_offset = loaded_log + .segments() + .iter() + .any(|s| s.size > iggy_common::IggyByteSize::default()); + + // Initialize journal base_offset so three-tier routing works correctly. + if should_increment_offset { + use crate::streaming::partitions::journal::{Inner, Journal}; + loaded_log.journal_mut().init(Inner { + base_offset: current_offset + 1, + ..Default::default() + }); + } + + let (revision_id, consumer_offsets, consumer_group_offsets) = self + .metadata + .get_partition_init_info(stream_id, topic_id, partition_id) + .map(|info| { + ( + info.revision_id, + info.consumer_offsets, + info.consumer_group_offsets, + ) + }) + .unwrap_or_else(|| { + ( + 0, + Arc::new(ConsumerOffsets::with_capacity(0)), + Arc::new(ConsumerGroupOffsets::with_capacity(0)), + ) + }); + + // Clamp consumer offsets that are ahead of partition offset (crash recovery). + { + let guard = consumer_offsets.pin(); + for entry in guard.iter() { + let stored = entry.1.offset.load(std::sync::atomic::Ordering::Relaxed); + if stored > current_offset { + tracing::warn!( + "Consumer {} offset {} ahead of partition offset {} \ + for stream {}, topic {}, partition {} - clamping \ + (lazy init recovery)", + entry.0, + stored, + current_offset, + stream_id, + topic_id, + partition_id + ); + entry + .1 + .offset + .store(current_offset, std::sync::atomic::Ordering::Relaxed); + } + } + } + { + let guard = consumer_group_offsets.pin(); + for entry in guard.iter() { + let stored = entry.1.offset.load(std::sync::atomic::Ordering::Relaxed); + if stored > current_offset { + tracing::warn!( + "Consumer group {:?} offset {} ahead of partition \ + offset {} for stream {}, topic {}, partition {} - \ + clamping (lazy init recovery)", + entry.0, + stored, + current_offset, + stream_id, + topic_id, + partition_id + ); + entry + .1 + .offset + .store(current_offset, std::sync::atomic::Ordering::Relaxed); + } + } + } + + let partition = LocalPartition::with_log( + loaded_log, + stats, + std::sync::Arc::new(std::sync::atomic::AtomicU64::new(current_offset)), + consumer_offsets, + consumer_group_offsets, + None, + created_at, + revision_id, + should_increment_offset, + ); + + self.local_partitions.borrow_mut().insert(*ns, partition); + + info!( + "Initialized partition in local_partitions: partition ID: {} for topic ID: {} for stream ID: {} with offset: {}", + partition_id, topic_id, stream_id, current_offset + ); + + Ok(()) + } + + pub async fn delete_partitions( + &self, + topic: ResolvedTopic, + partitions_count: u32, + ) -> Result, IggyError> { + let stream = topic.stream_id; + let topic_id = topic.topic_id; + + self.validate_partitions_count(topic, partitions_count)?; + + let all_partition_ids = self.metadata.get_partition_ids(stream, topic_id); + + let partitions_to_delete: Vec = all_partition_ids + .into_iter() + .rev() + .take(partitions_count as usize) + .collect(); + + let topic_stats = self.metadata.get_topic_stats(stream, topic_id); + + let mut total_messages_count: u64 = 0; + let mut total_segments_count: u32 = 0; + let mut total_size_bytes: u64 = 0; + + for partition_id in &partitions_to_delete { + if let Some(stats) = + self.metadata + .get_partition_stats_by_ids(stream, topic_id, *partition_id) + { + total_segments_count += stats.segments_count_inconsistent(); + total_messages_count += stats.messages_count_inconsistent(); + total_size_bytes += stats.size_bytes_inconsistent(); + } + } + + self.writer() + .delete_partitions(stream, topic_id, partitions_to_delete.len() as u32); + + for partition_id in &partitions_to_delete { + let ns = IggyNamespace::new(stream, topic_id, *partition_id); + self.remove_shard_table_record(&ns); + self.local_partitions.borrow_mut().remove(&ns); + } + + for partition_id in &partitions_to_delete { + self.delete_partition_dir(stream, topic_id, *partition_id) + .await?; + } + + self.metrics + .decrement_partitions(partitions_to_delete.len() as u32); + self.metrics.decrement_segments(total_segments_count); + + if let Some(parent) = topic_stats { + parent.decrement_messages_count(total_messages_count); + parent.decrement_size_bytes(total_size_bytes); + parent.decrement_segments_count(total_segments_count); + } + + Ok(partitions_to_delete) + } + + async fn delete_partition_dir( + &self, + stream_id: usize, + topic_id: usize, + partition_id: usize, + ) -> Result<(), IggyError> { + delete_partitions_from_disk(stream_id, topic_id, partition_id, &self.config.system).await + } +} diff --git a/core/server/src/shard/system/personal_access_tokens.rs b/core/server/src/shard/system/personal_access_tokens.rs new file mode 100644 index 0000000000..8c69a8ad3b --- /dev/null +++ b/core/server/src/shard/system/personal_access_tokens.rs @@ -0,0 +1,149 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::COMPONENT; +use crate::shard::IggyShard; +use crate::streaming::session::Session; +use crate::streaming::users::user::User; +use err_trail::ErrContext; +use iggy_common::IggyError; +use iggy_common::IggyExpiry; +use iggy_common::IggyTimestamp; +use iggy_common::PersonalAccessToken; +use tracing::{error, info}; + +impl IggyShard { + pub fn get_personal_access_tokens( + &self, + user_id: u32, + ) -> Result, IggyError> { + let _ = self.get_user(&user_id.try_into()?).error(|e: &IggyError| { + format!("{COMPONENT} (error: {e}) - failed to get user with id: {user_id}") + })?; + + info!("Loading personal access tokens for user with ID: {user_id}...",); + + let personal_access_tokens = self.metadata.get_user_personal_access_tokens(user_id); + + info!( + "Loaded {} personal access tokens for user with ID: {user_id}.", + personal_access_tokens.len(), + ); + Ok(personal_access_tokens) + } + + pub fn create_personal_access_token( + &self, + user_id: u32, + name: &str, + expiry: IggyExpiry, + ) -> Result<(PersonalAccessToken, String), IggyError> { + let _ = self.get_user(&user_id.try_into()?).error(|e: &IggyError| { + format!("{COMPONENT} (error: {e}) - failed to get user with id: {user_id}") + })?; + + let max_token_per_user = self.config.personal_access_token.max_tokens_per_user; + let current_count = self.metadata.user_pat_count(user_id); + if current_count as u32 >= max_token_per_user { + error!( + "User with ID: {user_id} has reached the maximum number of personal access tokens: {max_token_per_user}.", + ); + return Err(IggyError::PersonalAccessTokensLimitReached( + user_id, + max_token_per_user, + )); + } + + let (personal_access_token, token) = + PersonalAccessToken::new(user_id, name, IggyTimestamp::now(), expiry); + + let pat_name = personal_access_token.name.clone(); + if self.metadata.user_has_pat_with_name(user_id, &pat_name) { + error!("Personal access token: {pat_name} for user with ID: {user_id} already exists."); + return Err(IggyError::PersonalAccessTokenAlreadyExists( + pat_name.to_string(), + user_id, + )); + } + + self.writer() + .add_personal_access_token(user_id, personal_access_token.clone()); + info!("Created personal access token: {pat_name} for user with ID: {user_id}."); + + Ok((personal_access_token, token)) + } + + pub fn delete_personal_access_token(&self, user_id: u32, name: &str) -> Result<(), IggyError> { + let token_hash = + self.metadata + .find_pat_token_hash_by_name(user_id, name) + .ok_or_else(|| { + error!( + "Personal access token: {name} for user with ID: {user_id} does not exist.", + ); + IggyError::ResourceNotFound(name.to_owned()) + })?; + + info!("Deleting personal access token: {name} for user with ID: {user_id}..."); + self.writer() + .delete_personal_access_token(user_id, token_hash); + info!("Deleted personal access token: {name} for user with ID: {user_id}."); + Ok(()) + } + + pub fn login_with_personal_access_token( + &self, + token: &str, + session: Option<&Session>, + ) -> Result { + let token_hash = PersonalAccessToken::hash_token(token); + + let personal_access_token = self + .metadata + .get_personal_access_token_by_hash(&token_hash) + .ok_or_else(|| { + let redacted_token = if token.len() > 4 { + format!("{}****", &token[..4]) + } else { + "****".to_string() + }; + error!("Personal access token: {redacted_token} does not exist."); + IggyError::ResourceNotFound(token.to_owned()) + })?; + + if personal_access_token.is_expired(IggyTimestamp::now()) { + error!( + "Personal access token: {} for user with ID: {} has expired.", + personal_access_token.name, personal_access_token.user_id + ); + return Err(IggyError::PersonalAccessTokenExpired( + (*personal_access_token.name).to_owned(), + personal_access_token.user_id, + )); + } + + let user = self + .get_user(&personal_access_token.user_id.try_into()?) + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to get user with id: {}", + personal_access_token.user_id + ) + })?; + self.login_user_with_credentials(&user.username, None, session) + } +} diff --git a/core/server/src/shard/system/segments.rs b/core/server/src/shard/system/segments.rs new file mode 100644 index 0000000000..fb48846a76 --- /dev/null +++ b/core/server/src/shard/system/segments.rs @@ -0,0 +1,586 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::configs::cache_indexes::CacheIndexesConfig; +use crate::shard::IggyShard; +use crate::streaming::segments::Segment; +use iggy_common::{ConsumerKind, IggyError, IggyExpiry, IggyTimestamp, MaxTopicSize}; +use server_common::sharding::IggyNamespace; + +impl IggyShard { + /// Performs all cleanup for a topic's partitions: time-based expiry then size-based trimming. + /// + /// Runs entirely inside the message pump's serialized loop — reads partition state and + /// deletes segments atomically with no TOCTOU window. + pub(crate) async fn clean_topic_messages( + &self, + stream_id: usize, + topic_id: usize, + partition_ids: &[usize], + ) -> Result<(u64, u64), IggyError> { + let (expiry, max_topic_size) = self + .metadata + .get_topic_config(stream_id, topic_id) + .unwrap_or(( + self.config.system.topic.message_expiry, + MaxTopicSize::Unlimited, + )); + + let mut total_segments = 0u64; + let mut total_messages = 0u64; + + // Phase 1: time-based expiry + if !matches!(expiry, IggyExpiry::NeverExpire) { + let now = IggyTimestamp::now(); + for &partition_id in partition_ids { + let (s, m) = self + .delete_expired_segments_for_partition( + stream_id, + topic_id, + partition_id, + now, + expiry, + ) + .await?; + total_segments += s; + total_messages += m; + } + } + + // Phase 2: size-based trimming + if !matches!(max_topic_size, MaxTopicSize::Unlimited) { + let max_bytes = max_topic_size.as_bytes_u64(); + let threshold = max_bytes * 9 / 10; + + loop { + let current_size = self + .metadata + .with_metadata(|m| { + m.streams + .get(stream_id) + .and_then(|s| s.topics.get(topic_id)) + .map(|t| t.stats.size_bytes_inconsistent()) + }) + .unwrap_or(0); + + if current_size < threshold { + break; + } + + let Some((target_partition_id, target_offset)) = + self.find_oldest_sealed_segment(stream_id, topic_id, partition_ids) + else { + break; + }; + + let (s, m) = self + .remove_segment_by_offset( + stream_id, + topic_id, + target_partition_id, + target_offset, + ) + .await?; + if s == 0 { + break; + } + total_segments += s; + total_messages += m; + } + } + + Ok((total_segments, total_messages)) + } + + /// Deletes all expired sealed segments from a single partition. + async fn delete_expired_segments_for_partition( + &self, + stream_id: usize, + topic_id: usize, + partition_id: usize, + now: IggyTimestamp, + expiry: IggyExpiry, + ) -> Result<(u64, u64), IggyError> { + let ns = IggyNamespace::new(stream_id, topic_id, partition_id); + + let expired_offsets: Vec = { + let partitions = self.local_partitions.borrow(); + let Some(partition) = partitions.get(&ns) else { + return Ok((0, 0)); + }; + + let min_committed = Self::min_committed_offset( + &partition.consumer_offsets, + &partition.consumer_group_offsets, + ); + + let segments = partition.log.segments(); + let last_idx = segments.len().saturating_sub(1); + let mut offsets = Vec::new(); + + for (idx, seg) in segments.iter().enumerate() { + if idx == last_idx || !seg.is_expired(now, expiry) { + continue; + } + if let Some((barrier, kind, id)) = &min_committed + && seg.end_offset > *barrier + { + tracing::warn!( + "Segment [{}..{}] blocked from expiry-based deletion \ + by {kind} (ID: {id}) at offset {barrier} \ + in partition {partition_id} (stream: {stream_id}, topic: {topic_id})", + seg.start_offset, + seg.end_offset, + ); + continue; + } + offsets.push(seg.start_offset); + } + + offsets + }; + + let mut total_segments = 0u64; + let mut total_messages = 0u64; + for offset in expired_offsets { + let (s, m) = self + .remove_segment_by_offset(stream_id, topic_id, partition_id, offset) + .await?; + total_segments += s; + total_messages += m; + } + Ok((total_segments, total_messages)) + } + + /// Finds the oldest sealed segment across the given partitions, comparing by timestamp. + /// Returns `(partition_id, start_offset)` or `None` if no deletable segments exist. + fn find_oldest_sealed_segment( + &self, + stream_id: usize, + topic_id: usize, + partition_ids: &[usize], + ) -> Option<(usize, u64)> { + let partitions = self.local_partitions.borrow(); + let mut oldest: Option<(usize, u64, u64)> = None; + + for &partition_id in partition_ids { + let ns = IggyNamespace::new(stream_id, topic_id, partition_id); + let Some(partition) = partitions.get(&ns) else { + continue; + }; + + let segments = partition.log.segments(); + if segments.len() <= 1 { + continue; + } + + let first = &segments[0]; + if !first.sealed { + continue; + } + + let min_committed = Self::min_committed_offset( + &partition.consumer_offsets, + &partition.consumer_group_offsets, + ); + if let Some((barrier, kind, id)) = &min_committed + && first.end_offset > *barrier + { + tracing::warn!( + "Segment [{}..{}] blocked from size-based deletion \ + by {kind} (ID: {id}) at offset {barrier} \ + in partition {partition_id} (stream: {stream_id}, topic: {topic_id})", + first.start_offset, + first.end_offset, + ); + continue; + } + + match &oldest { + None => oldest = Some((partition_id, first.start_offset, first.start_timestamp)), + Some((_, _, ts)) if first.start_timestamp < *ts => { + oldest = Some((partition_id, first.start_offset, first.start_timestamp)); + } + _ => {} + } + } + + oldest.map(|(pid, offset, _)| (pid, offset)) + } + + /// Removes a single segment identified by its start_offset from the given partition. + /// Skips if the segment no longer exists or is the active (last) segment. + async fn remove_segment_by_offset( + &self, + stream_id: usize, + topic_id: usize, + partition_id: usize, + start_offset: u64, + ) -> Result<(u64, u64), IggyError> { + let ns = IggyNamespace::new(stream_id, topic_id, partition_id); + + let removed = { + let mut partitions = self.local_partitions.borrow_mut(); + let Some(partition) = partitions.get_mut(&ns) else { + return Ok((0, 0)); + }; + + let log = &mut partition.log; + let last_idx = log.segments().len().saturating_sub(1); + + let Some(idx) = log + .segments() + .iter() + .position(|s| s.start_offset == start_offset) + else { + return Ok((0, 0)); + }; + + if idx == last_idx { + tracing::warn!( + "Refusing to delete active segment (start_offset: {start_offset}) \ + for partition ID: {partition_id}" + ); + return Ok((0, 0)); + } + + let segment = log.segments_mut().remove(idx); + let storage = log.storages_mut().remove(idx); + log.indexes_mut().remove(idx); + + Some((segment, storage, partition.stats.clone())) + }; + + let Some((segment, mut storage, stats)) = removed else { + return Ok((0, 0)); + }; + + let segment_size = segment.size.as_bytes_u64(); + let end_offset = segment.end_offset; + let messages_in_segment = if start_offset == end_offset { + 0 + } else { + (end_offset - start_offset) + 1 + }; + + let _ = storage.shutdown(); + let (messages_path, index_path) = storage.segment_and_index_paths(); + + if let Some(path) = messages_path + && let Err(e) = compio::fs::remove_file(&path).await + { + tracing::error!("Failed to delete messages file {}: {}", path, e); + } + + if let Some(path) = index_path + && let Err(e) = compio::fs::remove_file(&path).await + { + tracing::error!("Failed to delete index file {}: {}", path, e); + } + + stats.decrement_size_bytes(segment_size); + stats.decrement_segments_count(1); + stats.decrement_messages_count(messages_in_segment); + + tracing::info!( + "Deleted segment (start: {}, end: {}, size: {}, messages: {}) from partition {}", + start_offset, + end_offset, + segment_size, + messages_in_segment, + partition_id + ); + + Ok((1, messages_in_segment)) + } + + /// Deletes the N oldest **sealed** segments from a partition, preserving the active segment + /// and partition offset. Reuses `remove_segment_by_offset` - same logic as the message cleaner. + /// + /// Segments containing unconsumed messages are protected by a barrier: deletion is skipped + /// when `end_offset > min_committed_offset` and a warning is logged identifying the + /// blocking consumer. If no consumers exist, there is no barrier. + pub(crate) async fn delete_oldest_segments( + &self, + stream_id: usize, + topic_id: usize, + partition_id: usize, + segments_count: u32, + ) -> Result<(u64, u64), IggyError> { + let ns = IggyNamespace::new(stream_id, topic_id, partition_id); + + let sealed_offsets: Vec = { + let partitions = self.local_partitions.borrow(); + let Some(partition) = partitions.get(&ns) else { + return Ok((0, 0)); + }; + + let min_committed = Self::min_committed_offset( + &partition.consumer_offsets, + &partition.consumer_group_offsets, + ); + + let segments = partition.log.segments(); + let last_idx = segments.len().saturating_sub(1); + let mut offsets = Vec::new(); + let mut collected = 0u32; + + for (idx, seg) in segments.iter().enumerate() { + if collected >= segments_count { + break; + } + if idx == last_idx || !seg.sealed { + continue; + } + if let Some((barrier, kind, id)) = &min_committed + && seg.end_offset > *barrier + { + tracing::warn!( + "Segment [{}..{}] blocked from size-based deletion \ + by {kind} (ID: {id}) at offset {barrier} \ + in partition {partition_id} (stream: {stream_id}, topic: {topic_id})", + seg.start_offset, + seg.end_offset, + ); + continue; + } + offsets.push(seg.start_offset); + collected += 1; + } + + offsets + }; + + let mut total_segments = 0u64; + let mut total_messages = 0u64; + for offset in sealed_offsets { + let (s, m) = self + .remove_segment_by_offset(stream_id, topic_id, partition_id, offset) + .await?; + total_segments += s; + total_messages += m; + } + Ok((total_segments, total_messages)) + } + + /// Returns the minimum committed offset across all consumers and consumer groups, + /// along with the identity of the consumer holding it. Returns `None` if no + /// consumers exist (no barrier). + fn min_committed_offset( + consumer_offsets: &crate::streaming::partitions::consumer_offsets::ConsumerOffsets, + consumer_group_offsets: &crate::streaming::partitions::consumer_group_offsets::ConsumerGroupOffsets, + ) -> Option<(u64, ConsumerKind, u32)> { + let co_guard = consumer_offsets.pin(); + let cg_guard = consumer_group_offsets.pin(); + let consumers = co_guard.iter().map(|(_, co)| { + ( + co.offset.load(std::sync::atomic::Ordering::Relaxed), + co.kind, + co.consumer_id, + ) + }); + let groups = cg_guard.iter().map(|(_, co)| { + ( + co.offset.load(std::sync::atomic::Ordering::Relaxed), + co.kind, + co.consumer_id, + ) + }); + consumers.chain(groups).min_by_key(|(offset, _, _)| *offset) + } + + /// Drains all segments, deletes their files, and re-initializes the partition log at offset 0. + /// Used exclusively by `purge_topic_inner` — a destructive full reset. + pub(crate) async fn purge_all_segments( + &self, + stream_id: usize, + topic_id: usize, + partition_id: usize, + ) -> Result<(), IggyError> { + let namespace = IggyNamespace::new(stream_id, topic_id, partition_id); + + // Drain segments from local_partitions + let (segments, storages, stats) = { + let mut partitions = self.local_partitions.borrow_mut(); + let partition = partitions + .get_mut(&namespace) + .expect("purge_all_segments: partition must exist in local_partitions"); + + let upperbound = partition.log.segments().len(); + let segments = partition + .log + .segments_mut() + .drain(..upperbound) + .collect::>(); + let storages = partition + .log + .storages_mut() + .drain(..upperbound) + .collect::>(); + let _ = partition + .log + .indexes_mut() + .drain(..upperbound) + .collect::>(); + (segments, storages, partition.stats.clone()) + }; + + for (mut storage, segment) in storages.into_iter().zip(segments) { + let (msg_writer, index_writer) = storage.shutdown(); + let start_offset = segment.start_offset; + + let log_path = if let Some(msg_writer) = msg_writer { + let path = msg_writer.path(); + drop(msg_writer); + path + } else { + self.config.system.get_messages_file_path( + stream_id, + topic_id, + partition_id, + start_offset, + ) + }; + drop(index_writer); + + let index_path = + self.config + .system + .get_index_path(stream_id, topic_id, partition_id, start_offset); + + for path in [&log_path, &index_path] { + match compio::fs::remove_file(path).await { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + tracing::debug!("File already gone at path: {path}"); + } + Err(e) => { + tracing::error!("Failed to delete file at path: {path}, err: {e}"); + return Err(IggyError::CannotDeleteFile); + } + } + } + } + + self.init_log_in_local_partitions(&namespace).await?; + stats.increment_segments_count(1); + Ok(()) + } + + /// Creates a fresh segment at offset 0 after all segments have been drained. + /// + /// The log is momentarily empty between `delete_segments`' drain and this call, which is + /// safe because the message pump serializes all handlers — no concurrent operation can + /// observe the empty state. + async fn init_log_in_local_partitions( + &self, + namespace: &IggyNamespace, + ) -> Result<(), IggyError> { + use crate::streaming::segments::storage::create_segment_storage; + + let start_offset = 0; + let segment = Segment::new(start_offset, self.config.system.segment.size); + + let storage = create_segment_storage( + &self.config.system, + namespace.stream_id(), + namespace.topic_id(), + namespace.partition_id(), + 0, // messages_size + 0, // indexes_size + start_offset, + ) + .await?; + + let mut partitions = self.local_partitions.borrow_mut(); + if let Some(partition) = partitions.get_mut(namespace) { + partition.log.add_persisted_segment(segment, storage); + // Reset offset when starting fresh with a new segment at offset 0 + partition + .offset + .store(start_offset, std::sync::atomic::Ordering::SeqCst); + partition.should_increment_offset = false; + } + Ok(()) + } + + /// Rotate to a new segment when the current segment is full. + /// The new segment starts at the next offset after the current segment's end. + /// Seals the old segment so it becomes eligible for expiry-based cleanup. + /// + /// Safety: called exclusively from the message pump (via append handler) — the captured + /// `old_segment_index` remains valid across the `create_segment_storage` await because + /// no other handler can modify the segment vec while this frame is in progress. + pub(crate) async fn rotate_segment_in_local_partitions( + &self, + namespace: &IggyNamespace, + ) -> Result<(), IggyError> { + use crate::streaming::segments::storage::create_segment_storage; + + let (start_offset, old_segment_index) = { + let mut partitions = self.local_partitions.borrow_mut(); + let partition = partitions + .get_mut(namespace) + .expect("rotate_segment: partition must exist"); + let old_segment_index = partition.log.segments().len() - 1; + let active_segment = partition.log.active_segment_mut(); + active_segment.sealed = true; + (active_segment.end_offset + 1, old_segment_index) + }; + + let segment = Segment::new(start_offset, self.config.system.segment.size); + + let storage = create_segment_storage( + &self.config.system, + namespace.stream_id(), + namespace.topic_id(), + namespace.partition_id(), + 0, // messages_size + 0, // indexes_size + start_offset, + ) + .await?; + + let mut partitions = self.local_partitions.borrow_mut(); + if let Some(partition) = partitions.get_mut(namespace) { + // Clear old segment's indexes if cache_indexes is not set to All. + // This prevents memory accumulation from keeping index buffers for sealed segments. + if !matches!( + self.config.system.segment.cache_indexes, + CacheIndexesConfig::All + ) { + partition.log.indexes_mut()[old_segment_index] = None; + } + + // Close writers for the sealed segment - they're never needed after sealing. + // This releases file handles and associated kernel/io_uring resources. + let old_storage = &mut partition.log.storages_mut()[old_segment_index]; + let _ = old_storage.shutdown(); + + partition.log.add_persisted_segment(segment, storage); + partition.stats.increment_segments_count(1); + tracing::info!( + "Rotated to new segment at offset {} for partition {} (stream {}, topic {})", + start_offset, + namespace.partition_id(), + namespace.stream_id(), + namespace.topic_id() + ); + } + Ok(()) + } +} diff --git a/core/server/src/shard/system/snapshot/mod.rs b/core/server/src/shard/system/snapshot/mod.rs new file mode 100644 index 0000000000..e3c802f939 --- /dev/null +++ b/core/server/src/shard/system/snapshot/mod.rs @@ -0,0 +1,257 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod procdump; + +use crate::configs::system::SystemConfig; +use crate::shard::IggyShard; +use async_zip::base::write::ZipFileWriter; +use async_zip::{Compression, ZipEntryBuilder}; +use compio::fs::OpenOptions; +use compio::io::AsyncWriteAtExt; +use iggy_common::{IggyDuration, IggyError, Snapshot, SnapshotCompression, SystemSnapshotType}; +use std::path::PathBuf; +use std::time::Instant; +use tempfile::NamedTempFile; +use tracing::{error, info}; + +// NOTE(hubcio): compio has a `process` module, but it currently blocks the executor when the runtime +// has thread_pool_limit(0) configured (which we do on non-macOS platforms in bootstrap.rs). +// To use compio::process::Command, we need to either: +// 1. Enable thread pool by removing/increasing thread_pool_limit(0) +// 2. Use std::process::Command with compio::runtime::spawn_blocking (requires thread pool) +// 3. Find alternative approach that doesn't rely on thread pool +// See: https://compio.rs/docs/compio/process and bootstrap::create_shard_executor +use std::process::Command; + +impl IggyShard { + pub async fn get_snapshot( + &self, + compression: SnapshotCompression, + snapshot_types: &Vec, + ) -> Result { + let snapshot_types = if snapshot_types.contains(&SystemSnapshotType::All) { + if snapshot_types.len() > 1 { + error!("When using 'All' snapshot type, no other types can be specified"); + return Err(IggyError::InvalidCommand); + } + &SystemSnapshotType::all_snapshot_types() + } else { + snapshot_types + }; + + let mut zip_writer = ZipFileWriter::new(Vec::new()); + let compression = match compression { + SnapshotCompression::Stored => Compression::Stored, + SnapshotCompression::Deflated => Compression::Deflate, + SnapshotCompression::Bzip2 => Compression::Bz, + SnapshotCompression::Lzma => Compression::Lzma, + SnapshotCompression::Xz => Compression::Xz, + SnapshotCompression::Zstd => Compression::Zstd, + }; + + info!("Executing snapshot commands: {:?}", snapshot_types); + let now = Instant::now(); + + for snapshot_type in snapshot_types { + info!("Processing snapshot type: {:?}", snapshot_type); + match get_command_result(snapshot_type, &self.config.system).await { + Ok(temp_file) => { + info!( + "Got temp file for {:?}: {}", + snapshot_type, + temp_file.path().display() + ); + let filename = format!("{snapshot_type}.txt"); + let entry = ZipEntryBuilder::new(filename.clone().into(), compression); + + // Read file using compio fs + let content = match compio::fs::read(temp_file.path()).await { + Ok(data) => data, + Err(e) => { + error!("Failed to read temporary file: {}", e); + continue; + } + }; + + info!( + "Read {} bytes from temp file for {}", + content.len(), + filename + ); + + if let Err(e) = zip_writer.write_entry_whole(entry, &content).await { + error!("Failed to write to snapshot file: {}", e); + continue; + } + info!("Wrote entry {} to zip file", filename); + } + Err(e) => { + error!( + "Failed to execute command for snapshot type {:?}: {}", + snapshot_type, e + ); + continue; + } + } + } + + info!( + "Snapshot commands {:?} finished in {}", + snapshot_types, + IggyDuration::new(now.elapsed()) + ); + + let zip_data = zip_writer + .close() + .await + .map_err(|_| IggyError::SnapshotFileCompletionFailed)?; + + info!("Final zip size: {} bytes", zip_data.len()); + Ok(Snapshot::new(zip_data)) + } +} + +async fn write_command_output_to_temp_file( + command: &mut Command, +) -> Result { + let output = command.output()?; + + info!( + "Command output: {} bytes, stderr: {}", + output.stdout.len(), + String::from_utf8_lossy(&output.stderr) + ); + + let temp_file = NamedTempFile::new()?; + + // Use compio to write the file - create/truncate to ensure clean write + let mut file = OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(temp_file.path()) + .await?; + + // Write the command output - compio takes ownership of the buffer + let stdout = output.stdout; + let (result, _buf) = file.write_all_at(stdout, 0).await.into(); + result?; + + file.sync_all().await?; + + info!( + "Wrote {} bytes to temp file: {}", + _buf.len(), + temp_file.path().display() + ); + + Ok(temp_file) +} + +async fn get_filesystem_overview() -> Result { + write_command_output_to_temp_file(Command::new("ls").args(["-la", "/tmp", "/proc"])).await +} + +async fn get_process_info() -> Result { + let temp_file = NamedTempFile::new()?; + let mut file = OpenOptions::new() + .create(true) + .write(true) + .open(temp_file.path()) + .await?; + + let mut position = 0; + let ps_output = Command::new("ps").arg("aux").output()?; + let (result, written) = file + .write_all_at(b"=== Process List (ps aux) ===\n", 0) + .await + .into(); + result?; + position += written.len() as u64; + + let (result, written) = file.write_all_at(ps_output.stdout, position).await.into(); + result?; + position += written.len() as u64; + + let (result, written) = file.write_all_at(b"\n\n", position).await.into(); + result?; + position += written.len() as u64; + + let (result, written) = file + .write_all_at(b"=== Detailed Process Information ===\n", position) + .await + .into(); + result?; + position += written.len() as u64; + + let proc_info = procdump::get_proc_info().await?; + let bytes = proc_info.as_bytes().to_owned(); + let (result, _) = file.write_all_at(bytes, position).await.into(); + result?; + file.sync_all().await?; + + Ok(temp_file) +} + +async fn get_resource_usage() -> Result { + write_command_output_to_temp_file(Command::new("top").args(["-H", "-b", "-n", "1"])).await +} + +async fn get_test_snapshot() -> Result { + write_command_output_to_temp_file(Command::new("echo").arg("test")).await +} + +async fn get_server_logs(config: &SystemConfig) -> Result { + let base_directory = PathBuf::from(config.get_system_path()); + let logs_subdirectory = PathBuf::from(&config.logging.path); + let logs_path = base_directory.join(logs_subdirectory); + + let list_and_cat = format!( + r#"ls -tr "{logs}" | xargs -I {{}} cat "{logs}/{{}}" "#, + logs = logs_path.display() + ); + + write_command_output_to_temp_file(Command::new("sh").args(["-c", &list_and_cat])).await +} + +async fn get_server_config(config: &SystemConfig) -> Result { + let base_directory = PathBuf::from(config.get_system_path()); + let config_path = base_directory.join("runtime").join("current_config.toml"); + + write_command_output_to_temp_file(Command::new("cat").arg(config_path)).await +} + +async fn get_command_result( + snapshot_type: &SystemSnapshotType, + config: &SystemConfig, +) -> Result { + match snapshot_type { + SystemSnapshotType::FilesystemOverview => get_filesystem_overview().await, + SystemSnapshotType::ProcessList => get_process_info().await, + SystemSnapshotType::ResourceUsage => get_resource_usage().await, + SystemSnapshotType::Test => get_test_snapshot().await, + SystemSnapshotType::ServerLogs => get_server_logs(config).await, + SystemSnapshotType::ServerConfig => get_server_config(config).await, + SystemSnapshotType::All => { + // This should not be reached (we filter out `All`` at the call site) + unreachable!( + "SystemSnapshotType::All should be handled before calling get_command_result()." + ) + } + } +} diff --git a/core/server/src/shard/system/snapshot/procdump.rs b/core/server/src/shard/system/snapshot/procdump.rs new file mode 100644 index 0000000000..7c6f5b6183 --- /dev/null +++ b/core/server/src/shard/system/snapshot/procdump.rs @@ -0,0 +1,213 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/// Parse the contents of a /proc/[pid]/task/[tid]/stat file into a human-readable format +fn parse_stat(contents: &str) -> String { + let fields: Vec<&str> = contents.split_whitespace().collect(); + if fields.len() < 52 { + return format!("Invalid stat format: {contents}"); + } + + let cmd_start = contents.find('(').unwrap_or(0); + let cmd_end = contents.rfind(')').unwrap_or(contents.len()); + let comm = &contents[cmd_start + 1..cmd_end]; + + let mut result = String::new(); + result.push_str(&format!("PID: {}\n", fields[0])); + result.push_str(&format!("Command: {comm}\n")); + result.push_str(&format!( + "State: {} ({})\n", + fields[2], + match fields[2] { + "R" => "Running", + "S" => "Sleeping (interruptible)", + "D" => "Waiting in uninterruptible disk sleep", + "Z" => "Zombie", + "T" => "Stopped", + "t" => "Tracing stop", + "W" => "Paging", + "X" | "x" => "Dead", + "K" => "Wakekill", + "P" => "Parked", + _ => "Unknown", + } + )); + result.push_str(&format!("Parent PID: {}\n", fields[3])); + result.push_str(&format!("Process Group: {}\n", fields[4])); + result.push_str(&format!("Session ID: {}\n", fields[5])); + result.push_str(&format!("TTY: {}\n", fields[6])); + result.push_str(&format!("Foreground Process Group: {}\n", fields[7])); + result.push_str(&format!("Kernel Flags: {}\n", fields[8])); + result.push_str(&format!("Minor Faults: {}\n", fields[9])); + result.push_str(&format!("Children Minor Faults: {}\n", fields[10])); + result.push_str(&format!("Major Faults: {}\n", fields[11])); + result.push_str(&format!("Children Major Faults: {}\n", fields[12])); + result.push_str(&format!("User Mode Time: {} ticks\n", fields[13])); + result.push_str(&format!("System Mode Time: {} ticks\n", fields[14])); + result.push_str(&format!("Children User Mode Time: {} ticks\n", fields[15])); + result.push_str(&format!( + "Children System Mode Time: {} ticks\n", + fields[16] + )); + result.push_str(&format!("Priority: {}\n", fields[17])); + result.push_str(&format!("Nice Value: {}\n", fields[18])); + result.push_str(&format!("Number of Threads: {}\n", fields[19])); + result.push_str(&format!("Real-time Priority: {}\n", fields[39])); + result.push_str(&format!( + "Policy: {} ({})\n", + fields[40], + match fields[40] { + "0" => "SCHED_NORMAL/OTHER", + "1" => "SCHED_FIFO", + "2" => "SCHED_RR", + "3" => "SCHED_BATCH", + "5" => "SCHED_IDLE", + "6" => "SCHED_DEADLINE", + _ => "Unknown", + } + )); + result.push_str(&format!( + "Aggregated Block I/O Delays: {} ticks\n", + fields[41] + )); + result.push_str(&format!("Guest Time: {} ticks\n", fields[42])); + result.push_str(&format!("Children Guest Time: {} ticks\n", fields[43])); + + result +} + +/// Get detailed information about the system's processes and related /proc data +pub async fn get_proc_info() -> Result { + let static_proc_files = vec![ + "/proc/uptime", + "/proc/cpuinfo", + "/proc/stat", + "/proc/meminfo", + "/proc/interrupts", + "/proc/softirqs", + "/proc/latency", + "/proc/buddyinfo", + "/proc/slabinfo", + "/proc/vmstat", + "/proc/loadavg", + "/proc/cmdline", + "/proc/version", + "/proc/net/sockstat", + "/proc/net/snmp", + "/proc/net/netlink", + "/proc/net/netstat", + "/proc/net/dev", + "/proc/net/packet", + "/proc/net/tcp", + "/proc/net/tcp6", + "/proc/net/udp", + "/proc/net/udp6", + "/proc/net/raw", + "/proc/net/raw6", + "/proc/net/icmp", + "/proc/net/icmp6", + "/proc/net/udplite", + "/proc/net/udplite6", + "/proc/net/unix", + "/proc/net/softnet_stat", + "/proc/tty/drivers", + "/proc/sys/kernel/pid_max", + "/proc/sys/kernel/random/boot_id", + "/proc/mounts", + "/proc/modules", + ]; + + let mut result = String::new(); + + async fn dump_file(result: &mut String, path: &str) -> Result<(), std::io::Error> { + match std::fs::read_to_string(path) { + Ok(contents) => { + result.push_str(&format!("=== {path} ===\n")); + + if path.ends_with("/stat") && path.contains("/task/") { + result.push_str(&parse_stat(&contents)); + } else { + result.push_str(&contents); + } + + result.push_str("\n\n"); + } + Err(e) => { + if let Ok(metadata) = std::fs::metadata(path) { + if metadata.is_dir() && path.ends_with("/fd") { + result.push_str(&format!("=== {path} (directory) ===\n")); + if let Ok(mut rd) = std::fs::read_dir(path) { + while let Some(Ok(entry)) = rd.next() { + let fd_path = entry.path(); + match std::fs::read_link(&fd_path) { + Ok(link) => { + result.push_str(&format!( + "{} -> {}\n", + fd_path.display(), + link.display() + )); + } + Err(_) => { + result.push_str(&format!( + "{} (unreadable symlink)\n", + fd_path.display() + )); + } + } + } + } + result.push('\n'); + } else { + result.push_str(&format!("=== {path} ERROR: {e} ===\n\n")); + } + } else { + result.push_str(&format!("=== {path} ERROR: {e} ===\n\n")); + } + } + } + Ok(()) + } + + for path in &static_proc_files { + dump_file(&mut result, path).await?; + } + + let mut proc_dir = std::fs::read_dir("/proc")?; + while let Some(Ok(entry)) = proc_dir.next() { + let file_type = entry.file_type()?; + if file_type.is_dir() + && let Ok(pid) = entry.file_name().to_string_lossy().parse::() + { + let pid_paths = vec![ + format!("/proc/{}/cmdline", pid), + format!("/proc/{}/statm", pid), + format!("/proc/{}/cgroup", pid), + format!("/proc/{}/task/{}/stat", pid, pid), + format!("/proc/{}/task/{}/status", pid, pid), + format!("/proc/{}/task/{}/wchan", pid, pid), + format!("/proc/{}/task/{}/syscall", pid, pid), + format!("/proc/{}/task/{}/fd", pid, pid), + ]; + + for p in pid_paths { + dump_file(&mut result, &p).await?; + } + } + } + + Ok(result) +} diff --git a/core/server/src/shard/system/stats.rs b/core/server/src/shard/system/stats.rs new file mode 100644 index 0000000000..a1d0114cf0 --- /dev/null +++ b/core/server/src/shard/system/stats.rs @@ -0,0 +1,120 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::shard::IggyShard; +use crate::{SEMANTIC_VERSION, VERSION}; +use iggy_common::{IggyDuration, IggyError, Stats}; +use std::cell::RefCell; +use sysinfo::System as SysinfoSystem; +use system_stats::SystemProbe; + +thread_local! { + static SYSINFO: RefCell> = const { RefCell::new(None) }; +} + +impl IggyShard { + pub async fn get_stats(&self) -> Result { + assert_eq!(self.id, 0, "GetStats should only be called on shard0"); + + let probe = SYSINFO.with_borrow_mut(|slot| { + let sys = slot.get_or_insert_with(SysinfoSystem::new); + SystemProbe::capture(sys) + }); + + let clients_count = self.client_manager.get_clients().len() as u32; + let hostname = sysinfo::System::host_name().unwrap_or("unknown_hostname".to_string()); + let os_name = sysinfo::System::name().unwrap_or("unknown_os_name".to_string()); + let os_version = + sysinfo::System::long_os_version().unwrap_or("unknown_os_version".to_string()); + let kernel_version = + sysinfo::System::kernel_version().unwrap_or("unknown_kernel_version".to_string()); + + let mut stats = Stats { + process_id: probe.process_id, + cpu_usage: probe.cpu_usage, + total_cpu_usage: probe.total_cpu_usage, + memory_usage: probe.memory_usage.into(), + total_memory: probe.total_memory.into(), + available_memory: probe.available_memory.into(), + run_time: IggyDuration::new_from_secs(probe.run_time_secs), + start_time: IggyDuration::new_from_secs(probe.start_time_secs) + .as_micros() + .into(), + read_bytes: probe.read_bytes.into(), + written_bytes: probe.written_bytes.into(), + threads_count: probe.threads_count, + clients_count, + hostname, + os_name, + os_version, + kernel_version, + iggy_server_version: VERSION.to_owned(), + iggy_server_semver: SEMANTIC_VERSION.get_numeric_version().ok(), + ..Default::default() + }; + + let (streams_count, topics_count, partitions_count, consumer_groups_count, stream_ids) = + self.metadata.with_metadata(|m| { + let mut topics = 0u32; + let mut partitions = 0u32; + let mut cg = 0u32; + let ids: Vec<_> = m.streams.iter().map(|(k, _)| k).collect(); + for (_, stream) in m.streams.iter() { + topics += stream.topics.len() as u32; + for (_, topic) in stream.topics.iter() { + partitions += topic.partitions.len() as u32; + cg += topic.consumer_groups.len() as u32; + } + } + (m.streams.len() as u32, topics, partitions, cg, ids) + }); + + stats.streams_count = streams_count; + stats.topics_count = topics_count; + stats.partitions_count = partitions_count; + stats.consumer_groups_count = consumer_groups_count; + + for stream_id in stream_ids { + if let Some(stream_stat) = self.metadata.get_stream_stats(stream_id) { + stats.messages_count += stream_stat.messages_count_inconsistent(); + stats.segments_count += stream_stat.segments_count_inconsistent(); + stats.messages_size_bytes += stream_stat.size_bytes_inconsistent().into(); + } + } + + match fs2::available_space(&self.config.system.path) { + Ok(space) => stats.free_disk_space = space.into(), + Err(err) => { + tracing::warn!( + "Failed to get available disk space for '{}': {err}", + self.config.system.path + ); + } + } + match fs2::total_space(&self.config.system.path) { + Ok(space) => stats.total_disk_space = space.into(), + Err(err) => { + tracing::warn!( + "Failed to get total disk space for '{}': {err}", + self.config.system.path + ); + } + } + + Ok(stats) + } +} diff --git a/core/server/src/shard/system/storage.rs b/core/server/src/shard/system/storage.rs new file mode 100644 index 0000000000..eb736a83cd --- /dev/null +++ b/core/server/src/shard/system/storage.rs @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::COMPONENT; +use crate::shard::system::info::SystemInfo; +use crate::streaming::persistence::persister::PersisterKind; +use crate::streaming::utils::file; +use anyhow::Context; +use compio::buf::IoBuf; +use compio::io::AsyncReadAtExt; +use err_trail::ErrContext; +use iggy_common::IggyError; +use server_common::PooledBuffer; +use std::sync::Arc; +use tracing::info; + +#[derive(Debug)] +pub struct FileSystemInfoStorage { + persister: Arc, + path: String, +} + +impl FileSystemInfoStorage { + pub fn new(path: String, persister: Arc) -> Self { + Self { path, persister } + } + + pub async fn load(&self) -> Result { + let file = file::open(&self.path).await; + if file.is_err() { + return Err(IggyError::ResourceNotFound(self.path.to_owned())); + } + + let file = file.unwrap(); + let file_size = file + .metadata() + .await + .error(|e: &std::io::Error| { + format!( + "{COMPONENT} (error: {e}) - failed to retrieve metadata for file at path: {}", + self.path + ) + }) + .map_err(|_| IggyError::CannotReadFileMetadata)? + .len() as usize; + + let file = file::open(&self.path) + .await + .map_err(|_| IggyError::CannotReadFile)?; + let buffer = PooledBuffer::with_capacity(file_size); + let (result, buffer) = file + .read_exact_at(buffer.slice(0..file_size), 0) + .await + .into(); + result + .error(|e: &std::io::Error| { + format!( + "{COMPONENT} Failed to read system info from file at path: {} (error: {e})", + self.path + ) + }) + .map_err(|_| IggyError::CannotReadFile)?; + let system_info = rmp_serde::from_slice(&buffer) + .with_context(|| "Failed to deserialize system info") + .map_err(|_| IggyError::CannotDeserializeResource)?; + Ok(system_info) + } + + pub async fn save(&self, system_info: &SystemInfo) -> Result<(), IggyError> { + let data = rmp_serde::to_vec(system_info) + .with_context(|| "Failed to serialize system info") + .map_err(|_| IggyError::CannotSerializeResource)?; + self.persister + .overwrite(&self.path, data) + .await + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to overwrite file at path: {}", + self.path + ) + })?; + info!("Saved system info, {system_info}"); + Ok(()) + } +} diff --git a/core/server/src/shard/system/streams.rs b/core/server/src/shard/system/streams.rs new file mode 100644 index 0000000000..ef80c7a1d7 --- /dev/null +++ b/core/server/src/shard/system/streams.rs @@ -0,0 +1,179 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::metadata::StreamMeta; +use crate::shard::IggyShard; +use crate::shard::transmission::message::{ResolvedStream, ResolvedTopic}; +use crate::streaming::streams::storage::{create_stream_file_hierarchy, delete_stream_directory}; +use iggy_common::{IggyError, IggyTimestamp}; +use server_common::sharding::IggyNamespace; +use std::sync::Arc; + +/// Info returned when a stream is deleted - contains what callers need for logging/events. +pub struct DeletedStreamInfo { + pub id: usize, + pub name: String, +} + +impl IggyShard { + pub async fn create_stream(&self, name: String) -> Result { + let name_arc = Arc::from(name.as_str()); + if self.metadata.stream_name_exists(&name_arc) { + return Err(IggyError::StreamNameAlreadyExists(name)); + } + + let stream_id = self.metadata.next_stream_id(); + create_stream_file_hierarchy(stream_id, &self.config.system).await?; + + let created_at = IggyTimestamp::now(); + let stats = Arc::new(crate::streaming::stats::StreamStats::default()); + let meta = StreamMeta::with_stats(0, name_arc, created_at, stats); + let assigned_id = self.writer().add_stream(meta); + debug_assert_eq!( + assigned_id, stream_id, + "Stream ID mismatch: expected {stream_id}, got {assigned_id}" + ); + + self.metrics.increment_streams(1); + Ok(stream_id) + } + + pub fn update_stream(&self, stream: ResolvedStream, name: String) -> Result<(), IggyError> { + self.writer() + .try_update_stream(&self.metadata, stream.id(), Arc::from(name.as_str())) + } + + pub async fn delete_stream( + &self, + stream: ResolvedStream, + ) -> Result { + let stream_id = stream.id(); + + let (topics_with_partitions, stream_name, stats, topics_count, partitions_count) = + self.metadata.with_metadata(|m| { + let stream_meta = m + .streams + .get(stream_id) + .expect("Stream metadata must exist"); + let twp: Vec<_> = stream_meta + .topics + .iter() + .map(|(topic_id, topic)| { + let partition_ids: Vec = (0..topic.partitions.len()).collect(); + (topic_id, partition_ids) + }) + .collect(); + let partitions_count: usize = stream_meta + .topics + .iter() + .map(|(_, t)| t.partitions.len()) + .sum(); + ( + twp, + stream_meta.name.to_string(), + stream_meta.stats.clone(), + stream_meta.topics.len(), + partitions_count, + ) + }); + + { + let namespaces: Vec<_> = topics_with_partitions + .iter() + .flat_map(|(topic_id, partition_ids)| { + partition_ids + .iter() + .map(|&partition_id| IggyNamespace::new(stream_id, *topic_id, partition_id)) + }) + .collect(); + let mut partitions = self.local_partitions.borrow_mut(); + for ns in namespaces { + partitions.remove(&ns); + } + } + + self.metrics.decrement_streams(1); + self.metrics.decrement_topics(topics_count as u32); + self.metrics.decrement_partitions(partitions_count as u32); + self.metrics + .decrement_messages(stats.messages_count_inconsistent()); + self.metrics + .decrement_segments(stats.segments_count_inconsistent()); + + self.writer().delete_stream(stream_id); + + let stream_info = DeletedStreamInfo { + id: stream_id, + name: stream_name, + }; + + self.client_manager + .delete_consumer_groups_for_stream(stream_id); + + let namespaces_to_remove: Vec<_> = self + .shards_table + .iter() + .filter_map(|entry| { + let (ns, _) = entry.pair(); + if ns.stream_id() == stream_id { + Some(*ns) + } else { + None + } + }) + .collect(); + + for ns in namespaces_to_remove { + self.remove_shard_table_record(&ns); + } + + delete_stream_directory(stream_id, &topics_with_partitions, &self.config.system).await?; + Ok(stream_info) + } + + /// Clears in-memory state for all topics in a stream. + pub async fn purge_stream(&self, stream: ResolvedStream) -> Result<(), IggyError> { + let stream_id = stream.id(); + let topic_ids = self.metadata.get_topic_ids(stream_id); + + for topic_id in topic_ids { + let topic = ResolvedTopic { + stream_id, + topic_id, + }; + self.purge_topic(topic).await?; + } + + Ok(()) + } + + /// Disk cleanup for local partitions across all topics in a stream. + pub(crate) async fn purge_stream_local(&self, stream: ResolvedStream) -> Result<(), IggyError> { + let stream_id = stream.id(); + let topic_ids = self.metadata.get_topic_ids(stream_id); + + for topic_id in topic_ids { + let topic = ResolvedTopic { + stream_id, + topic_id, + }; + self.purge_topic_local(topic).await?; + } + + Ok(()) + } +} diff --git a/core/server/src/shard/system/topics.rs b/core/server/src/shard/system/topics.rs new file mode 100644 index 0000000000..a8d99ed0e4 --- /dev/null +++ b/core/server/src/shard/system/topics.rs @@ -0,0 +1,250 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::metadata::TopicMeta; +use crate::shard::IggyShard; +use crate::shard::transmission::message::{ResolvedStream, ResolvedTopic}; +use crate::streaming::topics::storage::{create_topic_file_hierarchy, delete_topic_directory}; +use iggy_common::{ + CompressionAlgorithm, Identifier, IggyError, IggyExpiry, IggyTimestamp, MaxTopicSize, +}; +use server_common::sharding::IggyNamespace; +use std::sync::Arc; + +/// Info returned when a topic is deleted - contains what callers need for logging/events. +pub struct DeletedTopicInfo { + pub id: usize, + pub name: String, + pub stream_id: usize, +} + +impl IggyShard { + #[allow(clippy::too_many_arguments)] + pub async fn create_topic( + &self, + stream: ResolvedStream, + name: String, + message_expiry: IggyExpiry, + compression: CompressionAlgorithm, + max_topic_size: MaxTopicSize, + replication_factor: Option, + ) -> Result { + let stream_id = stream.0; + + let config = &self.config.system; + let message_expiry = config.resolve_message_expiry(message_expiry); + let max_topic_size = config.resolve_max_topic_size(max_topic_size)?; + + let name_arc = Arc::from(name.as_str()); + let parent_stats = self.metadata.get_stream_stats(stream_id).ok_or_else(|| { + IggyError::StreamIdNotFound(Identifier::numeric(stream_id as u32).unwrap()) + })?; + + let name_exists = self.metadata.with_metadata(|m| { + m.streams + .get(stream_id) + .map(|s| s.topic_index.contains_key(&name_arc)) + .unwrap_or(false) + }); + if name_exists { + return Err(IggyError::TopicNameAlreadyExists( + name, + Identifier::numeric(stream_id as u32).unwrap(), + )); + } + + let topic_id = self.metadata.next_topic_id(stream_id).ok_or_else(|| { + IggyError::StreamIdNotFound(Identifier::numeric(stream_id as u32).unwrap()) + })?; + create_topic_file_hierarchy(stream_id, topic_id, &self.config.system).await?; + + let created_at = IggyTimestamp::now(); + let stats = Arc::new(crate::streaming::stats::TopicStats::new(parent_stats)); + let topic_meta = TopicMeta { + id: 0, + name: name_arc, + created_at, + message_expiry, + compression_algorithm: compression, + max_topic_size, + replication_factor: replication_factor.unwrap_or(1), + stats, + partitions: Vec::new(), + consumer_groups: slab::Slab::new(), + consumer_group_index: ahash::AHashMap::default(), + round_robin_counter: Arc::new(std::sync::atomic::AtomicUsize::new(0)), + }; + let assigned_id = self + .writer() + .add_topic(stream_id, topic_meta) + .ok_or_else(|| { + IggyError::StreamIdNotFound(Identifier::numeric(stream_id as u32).unwrap()) + })?; + debug_assert_eq!( + assigned_id, topic_id, + "Topic ID mismatch: expected {topic_id}, got {assigned_id}" + ); + + self.metrics.increment_topics(1); + Ok(topic_id) + } + + #[allow(clippy::too_many_arguments)] + pub fn update_topic( + &self, + topic: ResolvedTopic, + name: String, + message_expiry: IggyExpiry, + compression_algorithm: CompressionAlgorithm, + max_topic_size: MaxTopicSize, + replication_factor: Option, + ) -> Result<(), IggyError> { + self.writer().try_update_topic( + &self.metadata, + topic.stream_id, + topic.topic_id, + Arc::from(name.as_str()), + message_expiry, + compression_algorithm, + max_topic_size, + replication_factor.unwrap_or(1), + ) + } + + pub async fn delete_topic(&self, topic: ResolvedTopic) -> Result { + let stream = topic.stream_id; + let topic_id = topic.topic_id; + + let (partition_ids, topic_name, messages_count, size_bytes, segments_count, parent_stats) = + self.metadata.with_metadata(|m| { + let stream_meta = m.streams.get(stream).expect("Stream metadata must exist"); + let topic_meta = stream_meta + .topics + .get(topic_id) + .expect("Topic metadata must exist"); + let pids: Vec = (0..topic_meta.partitions.len()).collect(); + ( + pids, + topic_meta.name.to_string(), + topic_meta.stats.messages_count_inconsistent(), + topic_meta.stats.size_bytes_inconsistent(), + topic_meta.stats.segments_count_inconsistent(), + topic_meta.stats.parent().clone(), + ) + }); + + { + let mut partitions = self.local_partitions.borrow_mut(); + for &partition_id in &partition_ids { + let ns = IggyNamespace::new(stream, topic_id, partition_id); + partitions.remove(&ns); + } + } + + self.writer().delete_topic(stream, topic_id); + + let topic_info = DeletedTopicInfo { + id: topic_id, + name: topic_name, + stream_id: stream, + }; + + self.client_manager + .delete_consumer_groups_for_topic(stream, topic_id); + + let namespaces_to_remove: Vec<_> = self + .shards_table + .iter() + .filter_map(|entry| { + let (ns, _) = entry.pair(); + if ns.stream_id() == stream && ns.topic_id() == topic_id { + Some(*ns) + } else { + None + } + }) + .collect(); + + for ns in namespaces_to_remove { + self.remove_shard_table_record(&ns); + } + + delete_topic_directory(stream, topic_id, &partition_ids, &self.config.system).await?; + + parent_stats.decrement_messages_count(messages_count); + parent_stats.decrement_size_bytes(size_bytes); + parent_stats.decrement_segments_count(segments_count); + self.metrics.decrement_topics(1); + Ok(topic_info) + } + + /// Clears in-memory state for a topic: consumer offsets and stats. + /// Called on the control plane before broadcasting to other shards. + pub async fn purge_topic(&self, topic: ResolvedTopic) -> Result<(), IggyError> { + let stream = topic.stream_id; + let topic_id = topic.topic_id; + let partition_ids = self.metadata.get_partition_ids(stream, topic_id); + + for &partition_id in &partition_ids { + if let Some(offsets) = + self.metadata + .get_partition_consumer_offsets(stream, topic_id, partition_id) + { + offsets.pin().clear(); + } + if let Some(offsets) = + self.metadata + .get_partition_consumer_group_offsets(stream, topic_id, partition_id) + { + offsets.pin().clear(); + } + } + + // Zero partition stats — propagation handles topic and stream counters. + // Topic stats must NOT be zeroed separately to avoid double-decrementing the stream. + for &partition_id in &partition_ids { + let ns = IggyNamespace::new(stream, topic_id, partition_id); + if let Some(partition_stats) = self.metadata.get_partition_stats(&ns) { + partition_stats.zero_out_all(); + } + } + + Ok(()) + } + + /// Disk cleanup for local partitions: deletes consumer offset files and purges segments. + /// Called on each shard (including shard 0) after in-memory state is cleared. + pub(crate) async fn purge_topic_local(&self, topic: ResolvedTopic) -> Result<(), IggyError> { + let stream = topic.stream_id; + let topic_id = topic.topic_id; + let partition_ids = self.metadata.get_partition_ids(stream, topic_id); + + for &partition_id in &partition_ids { + let ns = IggyNamespace::new(stream, topic_id, partition_id); + if !self.local_partitions.borrow().contains(&ns) { + continue; + } + + self.delete_all_consumer_offset_files(stream, topic_id, partition_id) + .await?; + self.purge_all_segments(stream, topic_id, partition_id) + .await?; + } + + Ok(()) + } +} diff --git a/core/server/src/shard/system/users.rs b/core/server/src/shard/system/users.rs new file mode 100644 index 0000000000..69a9d66e85 --- /dev/null +++ b/core/server/src/shard/system/users.rs @@ -0,0 +1,301 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::COMPONENT; +use crate::metadata::UserMeta; +use crate::shard::IggyShard; +use crate::streaming::session::Session; +use crate::streaming::users::user::User; +use crate::streaming::utils::crypto; +use dashmap::DashMap; +use err_trail::ErrContext; +use iggy_common::Identifier; +use iggy_common::IggyError; +use iggy_common::Permissions; +use iggy_common::UserStatus; +use std::sync::Arc; +use tracing::{error, warn}; + +const MAX_USERS: usize = u32::MAX as usize; + +impl IggyShard { + fn user_from_meta(&self, meta: &UserMeta) -> User { + let pats = self.metadata.get_user_personal_access_tokens(meta.id); + let pat_map = DashMap::new(); + for pat in pats { + pat_map.insert(pat.token.clone(), pat); + } + User { + id: meta.id, + status: meta.status, + username: meta.username.to_string(), + password: meta.password_hash.to_string(), + created_at: meta.created_at, + permissions: meta.permissions.as_ref().map(|p| (**p).clone()), + personal_access_tokens: pat_map, + } + } + + fn get_user_from_metadata(&self, identifier: &Identifier) -> Result, IggyError> { + let user_id = match self.metadata.get_user_id(identifier) { + Some(id) => id, + None => return Ok(None), + }; + + Ok(self + .metadata + .get_user(user_id) + .map(|meta| self.user_from_meta(&meta))) + } + + pub fn find_user(&self, user_id: &Identifier) -> Result, IggyError> { + self.try_get_user(user_id) + } + + pub fn get_user(&self, user_id: &Identifier) -> Result { + self.try_get_user(user_id)? + .ok_or(IggyError::ResourceNotFound(user_id.to_string())) + } + + pub fn try_get_user(&self, user_id: &Identifier) -> Result, IggyError> { + self.get_user_from_metadata(user_id) + } + + pub fn get_users(&self) -> Vec { + self.metadata + .get_all_users() + .iter() + .map(|meta| self.user_from_meta(meta)) + .collect() + } + + pub fn create_user( + &self, + username: &str, + password: &str, + status: UserStatus, + permissions: Option, + ) -> Result { + let password_hash = crypto::hash_password(password); + + let user_id = self + .writer() + .create_user( + &self.metadata, + Arc::from(username), + Arc::from(password_hash.as_str()), + status, + permissions.map(Arc::new), + MAX_USERS, + ) + .inspect_err(|e| match e { + IggyError::UserAlreadyExists => error!("User: {username} already exists."), + IggyError::UsersLimitReached => error!("Available users limit reached."), + _ => {} + })?; + + self.metrics.increment_users(1); + + self.get_user(&user_id.try_into()?).error(|e: &IggyError| { + format!("{COMPONENT} (error: {e}) - failed to get user with id: {user_id}") + }) + } + + pub fn delete_user(&self, user_id: &Identifier) -> Result { + let user = self.get_user(user_id).error(|e: &IggyError| { + format!("{COMPONENT} (error: {e}) - failed to get user with id: {user_id}") + })?; + + if user.is_root() { + error!("Cannot delete the root user."); + return Err(IggyError::CannotDeleteUser(user.id)); + } + + let user_u32_id = user.id; + + self.client_manager + .delete_clients_for_user(user_u32_id) + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to delete clients for user with ID: {user_u32_id}" + ) + })?; + self.metrics.decrement_users(1); + + self.writer().delete_user(user_u32_id); + + Ok(user) + } + + pub fn update_user( + &self, + user_id: &Identifier, + username: Option, + status: Option, + ) -> Result { + let user = self.get_user(user_id)?; + let numeric_user_id = user.id; + + let updated_meta = self.writer().update_user( + &self.metadata, + numeric_user_id, + username.map(|u| Arc::from(u.as_str())), + status, + )?; + + Ok(self.user_from_meta(&updated_meta)) + } + + pub fn update_permissions( + &self, + user_id: &Identifier, + permissions: Option, + ) -> Result<(), IggyError> { + let user: User = self.get_user(user_id).error(|e: &IggyError| { + format!("{COMPONENT} (error: {e}) - failed to get user with id: {user_id}") + })?; + + let current_meta = self + .metadata + .get_user(user.id) + .ok_or_else(|| IggyError::ResourceNotFound(user_id.to_string()))?; + + let updated_meta = UserMeta { + id: current_meta.id, + username: current_meta.username, + password_hash: current_meta.password_hash, + status: current_meta.status, + permissions: permissions.map(Arc::new), + created_at: current_meta.created_at, + }; + + self.writer().update_user_meta(user.id, updated_meta); + + Ok(()) + } + + pub fn change_password( + &self, + user_id: &Identifier, + current_password: &str, + new_password: &str, + ) -> Result<(), IggyError> { + let user = self.get_user(user_id).error(|e: &IggyError| { + format!( + "{COMPONENT} change password (error: {e}) - failed to get user with id: {user_id}" + ) + })?; + + if !crypto::verify_password(current_password, &user.password) { + error!( + "Invalid current password for user: {} with ID: {user_id}.", + user.username + ); + return Err(IggyError::InvalidCredentials); + } + + let current_meta = self + .metadata + .get_user(user.id) + .ok_or_else(|| IggyError::ResourceNotFound(user_id.to_string()))?; + + let new_password_hash = crypto::hash_password(new_password); + let updated_meta = UserMeta { + id: current_meta.id, + username: current_meta.username, + password_hash: Arc::from(new_password_hash.as_str()), + status: current_meta.status, + permissions: current_meta.permissions, + created_at: current_meta.created_at, + }; + + self.writer().update_user_meta(user.id, updated_meta); + + Ok(()) + } + + pub fn login_user( + &self, + username: &str, + password: &str, + session: Option<&Session>, + ) -> Result { + self.login_user_with_credentials(username, Some(password), session) + } + + pub fn login_user_with_credentials( + &self, + username: &str, + password: Option<&str>, + session: Option<&Session>, + ) -> Result { + let user = match self.get_user(&username.try_into()?) { + Ok(user) => user, + Err(_) => { + error!("Cannot login user: {username} (not found)."); + return Err(IggyError::InvalidCredentials); + } + }; + + if !user.is_active() { + warn!("User: {username} with ID: {} is inactive.", user.id); + return Err(IggyError::UserInactive); + } + + if let Some(password) = password + && !crypto::verify_password(password, &user.password) + { + warn!( + "Invalid password for user: {username} with ID: {}.", + user.id + ); + return Err(IggyError::InvalidCredentials); + } + + if session.is_none() { + return Ok(user); + } + + let session = session.unwrap(); + if session.is_authenticated() { + warn!( + "User: {} with ID: {} was already authenticated, removing the previous session...", + user.username, + session.get_user_id() + ); + self.logout_user(session)?; + } + session.set_user_id(user.id); + self.client_manager + .set_user_id(session.client_id, user.id) + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to set user_id to client, client ID: {}, user ID: {}", + session.client_id, user.id + ) + })?; + Ok(user) + } + + pub fn logout_user(&self, session: &Session) -> Result<(), IggyError> { + let client_id = session.client_id; + if client_id > 0 { + self.client_manager.clear_user_id(client_id)?; + } + Ok(()) + } +} diff --git a/core/server/src/shard/system/utils.rs b/core/server/src/shard/system/utils.rs new file mode 100644 index 0000000000..fc5f37a5f7 --- /dev/null +++ b/core/server/src/shard/system/utils.rs @@ -0,0 +1,255 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::{ + metadata::{resolve_consumer_group_id_inner, resolve_stream_id_inner, resolve_topic_id_inner}, + shard::{ + IggyShard, + transmission::message::{ + ResolvedConsumerGroup, ResolvedPartition, ResolvedStream, ResolvedTopic, + }, + }, + streaming::polling_consumer::PollingConsumer, +}; +use iggy_common::{Consumer, ConsumerKind, Identifier, IggyError}; + +impl IggyShard { + /// Resolves stream identifier to typed `ResolvedStream`. + pub fn resolve_stream(&self, stream_id: &Identifier) -> Result { + self.metadata + .with_metadata(|m| resolve_stream_inner(m, stream_id)) + } + + /// Resolves topic from identifiers. Returns StreamIdNotFound if stream doesn't exist, + /// TopicIdNotFound if topic doesn't exist. + pub fn resolve_topic( + &self, + stream_id: &Identifier, + topic_id: &Identifier, + ) -> Result { + self.metadata.with_metadata(|m| { + let stream = resolve_stream_inner(m, stream_id)?; + let id = resolve_topic_id_inner(m, stream.0, topic_id) + .ok_or_else(|| IggyError::TopicIdNotFound(stream_id.clone(), topic_id.clone()))?; + Ok(ResolvedTopic { + stream_id: stream.0, + topic_id: id, + }) + }) + } + + /// Resolves partition from identifiers. Returns appropriate error at each level. + pub fn resolve_partition( + &self, + stream_id: &Identifier, + topic_id: &Identifier, + partition_id: usize, + ) -> Result { + self.metadata.with_metadata(|m| { + let stream = resolve_stream_inner(m, stream_id)?; + let tid = resolve_topic_id_inner(m, stream.0, topic_id) + .ok_or_else(|| IggyError::TopicIdNotFound(stream_id.clone(), topic_id.clone()))?; + + let exists = m + .streams + .get(stream.0) + .and_then(|s| s.topics.get(tid)) + .and_then(|t| t.partitions.get(partition_id)) + .is_some(); + + if !exists { + return Err(IggyError::PartitionNotFound( + partition_id, + topic_id.clone(), + stream_id.clone(), + )); + } + + Ok(ResolvedPartition { + stream_id: stream.0, + topic_id: tid, + partition_id, + }) + }) + } + + /// Resolves consumer group from identifiers. Returns appropriate error at each level. + pub fn resolve_consumer_group( + &self, + stream_id: &Identifier, + topic_id: &Identifier, + group_id: &Identifier, + ) -> Result { + self.metadata.with_metadata(|m| { + let stream = resolve_stream_inner(m, stream_id)?; + let tid = resolve_topic_id_inner(m, stream.0, topic_id) + .ok_or_else(|| IggyError::TopicIdNotFound(stream_id.clone(), topic_id.clone()))?; + + let gid = + resolve_consumer_group_id_inner(m, stream.0, tid, group_id).ok_or_else(|| { + IggyError::ConsumerGroupIdNotFound(group_id.clone(), topic_id.clone()) + })?; + + Ok(ResolvedConsumerGroup { + stream_id: stream.0, + topic_id: tid, + group_id: gid, + }) + }) + } + + /// Validates that partitions_count does not exceed actual partition count. + pub fn validate_partitions_count( + &self, + topic: ResolvedTopic, + partitions_count: u32, + ) -> Result<(), IggyError> { + let actual = self + .metadata + .partitions_count(topic.stream_id, topic.topic_id); + if partitions_count > actual as u32 { + return Err(IggyError::InvalidPartitionsCount); + } + Ok(()) + } + + /// Validates that consumer_offset does not exceed actual partition offset. + pub fn validate_partition_offset( + &self, + stream_id: usize, + topic_id: usize, + partition_id: usize, + consumer_offset: u64, + ) -> Result<(), IggyError> { + let partition_stats = self + .metadata + .get_partition_stats_by_ids(stream_id, topic_id, partition_id) + .ok_or(IggyError::PartitionNotFound( + partition_id, + Identifier::numeric(topic_id as u32).expect("numeric identifier is always valid"), + Identifier::numeric(stream_id as u32).expect("numeric identifier is always valid"), + ))?; + + // Also rejects storing any offset if the partition is completely empty (i.e., has never contained any messages). + if (partition_stats.messages_count_inconsistent() == 0 + && partition_stats.current_offset() == 0) + || consumer_offset > partition_stats.current_offset() + { + return Err(IggyError::InvalidOffset(consumer_offset)); + } + Ok(()) + } + + /// Resolves consumer with partition ID for polling/offset operations. + /// For consumer groups, all lookups happen under a single metadata read guard. + pub fn resolve_consumer_with_partition_id( + &self, + topic: ResolvedTopic, + consumer: &Consumer, + client_id: u32, + partition_id: Option, + calculate_partition_id: bool, + ) -> Result, IggyError> { + match consumer.kind { + ConsumerKind::Consumer => { + let partition_id = partition_id.unwrap_or(0); + Ok(Some(( + PollingConsumer::consumer(&consumer.id, partition_id as usize), + partition_id as usize, + ))) + } + ConsumerKind::ConsumerGroup => { + if self.client_manager.try_get_client(client_id).is_none() { + return Err(IggyError::StaleClient); + } + + self.metadata.resolve_consumer_group_partition( + topic.stream_id, + topic.topic_id, + &consumer.id, + client_id, + partition_id, + calculate_partition_id, + ) + } + } + } + + /// Resolves topic and verifies user has append permission atomically. + pub fn resolve_topic_for_append( + &self, + user_id: u32, + stream_id: &Identifier, + topic_id: &Identifier, + ) -> Result { + self.metadata + .resolve_for_append(user_id, stream_id, topic_id) + } + + /// Resolves topic and verifies user has poll permission atomically. + pub fn resolve_topic_for_poll( + &self, + user_id: u32, + stream_id: &Identifier, + topic_id: &Identifier, + ) -> Result { + self.metadata.resolve_for_poll(user_id, stream_id, topic_id) + } + + /// Resolves topic and verifies user has permission to store consumer offset atomically. + pub fn resolve_topic_for_store_consumer_offset( + &self, + user_id: u32, + stream_id: &Identifier, + topic_id: &Identifier, + ) -> Result { + self.metadata + .resolve_for_store_consumer_offset(user_id, stream_id, topic_id) + } + + /// Resolves topic and verifies user has permission to delete consumer offset atomically. + pub fn resolve_topic_for_delete_consumer_offset( + &self, + user_id: u32, + stream_id: &Identifier, + topic_id: &Identifier, + ) -> Result { + self.metadata + .resolve_for_delete_consumer_offset(user_id, stream_id, topic_id) + } + + /// Resolves partition and verifies user has permission to delete segments atomically. + pub fn resolve_partition_for_delete_segments( + &self, + user_id: u32, + stream_id: &Identifier, + topic_id: &Identifier, + partition_id: usize, + ) -> Result { + self.metadata + .resolve_for_delete_segments(user_id, stream_id, topic_id, partition_id) + } +} + +fn resolve_stream_inner( + m: &crate::metadata::InnerMetadata, + stream_id: &Identifier, +) -> Result { + resolve_stream_id_inner(m, stream_id) + .map(ResolvedStream) + .ok_or_else(|| IggyError::StreamIdNotFound(stream_id.clone())) +} diff --git a/core/server/src/shard/systemd.rs b/core/server/src/shard/systemd.rs new file mode 100644 index 0000000000..a10c2fbba5 --- /dev/null +++ b/core/server/src/shard/systemd.rs @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Thin wrappers around `sd_notify` so every systemd interaction on the server +//! side lives in one place (mirrors `core/ai/mcp/src/systemd.rs`). + +use tracing::warn; + +/// Tell systemd the service has finished start-up (`READY=1`). +pub fn notify_ready() { + if let Err(e) = sd_notify::notify(&[sd_notify::NotifyState::Ready]) { + warn!("Failed to send systemd READY=1 notification: {e}"); + } +} + +/// Tell systemd the service has begun shutting down (`STOPPING=1`). +pub fn notify_stopping() { + let _ = sd_notify::notify(&[sd_notify::NotifyState::Stopping]); +} + +/// Surface a non-fatal shutdown problem in `systemctl status` / journald. +pub fn notify_status(status: &str) { + let _ = sd_notify::notify(&[sd_notify::NotifyState::Status(status)]); +} + +/// Send a single watchdog keep-alive ping (`WATCHDOG=1`). +pub fn ping_watchdog() { + if let Err(e) = sd_notify::notify(&[sd_notify::NotifyState::Watchdog]) { + warn!("Failed to send systemd watchdog ping: {e}"); + } +} diff --git a/core/server/src/shard/task_registry/builders.rs b/core/server/src/shard/task_registry/builders.rs new file mode 100644 index 0000000000..ac61d27ff6 --- /dev/null +++ b/core/server/src/shard/task_registry/builders.rs @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod continuous; +pub mod oneshot; +pub mod periodic; + +use super::registry::TaskRegistry; + +// Marker type for when no shutdown callback is provided +pub struct NoShutdown; + +impl TaskRegistry { + pub fn periodic(&self, name: &'static str) -> periodic::PeriodicBuilder<'_, (), NoShutdown> { + periodic::PeriodicBuilder::new(self, name) + } + + pub fn continuous( + &self, + name: &'static str, + ) -> continuous::ContinuousBuilder<'_, (), NoShutdown> { + continuous::ContinuousBuilder::new(self, name) + } + + pub fn oneshot(&self, name: &'static str) -> oneshot::OneShotBuilder<'_, (), NoShutdown> { + oneshot::OneShotBuilder::new(self, name) + } +} diff --git a/core/server/src/shard/task_registry/builders/continuous.rs b/core/server/src/shard/task_registry/builders/continuous.rs new file mode 100644 index 0000000000..4f4aad0a2e --- /dev/null +++ b/core/server/src/shard/task_registry/builders/continuous.rs @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::NoShutdown; +use crate::shard::task_registry::ShutdownToken; +use crate::shard::task_registry::registry::TaskRegistry; +use iggy_common::IggyError; +use std::ops::AsyncFnOnce; + +pub struct ContinuousBuilder<'a, Task, OnShutdown = NoShutdown> { + reg: &'a TaskRegistry, + name: &'static str, + critical: bool, + run_fn: Option, + on_shutdown: Option, +} + +impl<'a> ContinuousBuilder<'a, (), NoShutdown> { + pub fn new(reg: &'a TaskRegistry, name: &'static str) -> Self { + Self { + reg, + name, + critical: false, + run_fn: None, + on_shutdown: None, + } + } +} + +impl<'a, Task, OnShutdown> ContinuousBuilder<'a, Task, OnShutdown> { + pub fn critical(mut self, c: bool) -> Self { + self.critical = c; + self + } + + pub fn on_shutdown( + self, + f: NewShutdown, + ) -> ContinuousBuilder<'a, Task, NewShutdown> + where + NewShutdown: AsyncFnOnce(Result<(), IggyError>) + 'static, + { + ContinuousBuilder { + reg: self.reg, + name: self.name, + critical: self.critical, + run_fn: self.run_fn, + on_shutdown: Some(f), + } + } +} + +impl<'a, OnShutdown> ContinuousBuilder<'a, (), OnShutdown> { + pub fn run(self, f: NewTask) -> ContinuousBuilder<'a, NewTask, OnShutdown> + where + NewTask: AsyncFnOnce(ShutdownToken) -> Result<(), IggyError> + 'static, + { + ContinuousBuilder { + reg: self.reg, + name: self.name, + critical: self.critical, + run_fn: Some(f), + on_shutdown: self.on_shutdown, + } + } +} + +impl<'a, Task> ContinuousBuilder<'a, Task, NoShutdown> +where + Task: AsyncFnOnce(ShutdownToken) -> Result<(), IggyError> + 'static, +{ + pub fn spawn(self) { + if let Some(f) = self.run_fn { + self.reg + .spawn_continuous_closure(self.name, self.critical, f, Some(|_| async {})); + } else { + panic!("run() must be called before spawn()"); + } + } +} + +impl<'a, Task, OnShutdown> ContinuousBuilder<'a, Task, OnShutdown> +where + Task: AsyncFnOnce(ShutdownToken) -> Result<(), IggyError> + 'static, + OnShutdown: AsyncFnOnce(Result<(), IggyError>) + 'static, +{ + pub fn spawn(self) { + if let Some(f) = self.run_fn { + self.reg + .spawn_continuous_closure(self.name, self.critical, f, self.on_shutdown); + } else { + panic!("run() must be called before spawn()"); + } + } +} diff --git a/core/server/src/shard/task_registry/builders/oneshot.rs b/core/server/src/shard/task_registry/builders/oneshot.rs new file mode 100644 index 0000000000..abf0b727c5 --- /dev/null +++ b/core/server/src/shard/task_registry/builders/oneshot.rs @@ -0,0 +1,120 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::NoShutdown; +use crate::shard::task_registry::ShutdownToken; +use crate::shard::task_registry::registry::TaskRegistry; +use iggy_common::IggyError; +use std::ops::AsyncFnOnce; +use std::time::Duration; + +pub struct OneShotBuilder<'a, Task, OnShutdown = NoShutdown> { + reg: &'a TaskRegistry, + name: &'static str, + critical: bool, + timeout: Option, + run_fn: Option, + on_shutdown: Option, +} + +impl<'a> OneShotBuilder<'a, (), NoShutdown> { + pub fn new(reg: &'a TaskRegistry, name: &'static str) -> Self { + Self { + reg, + name, + critical: false, + timeout: None, + run_fn: None, + on_shutdown: None, + } + } +} + +impl<'a, Task, OnShutdown> OneShotBuilder<'a, Task, OnShutdown> { + pub fn critical(mut self, c: bool) -> Self { + self.critical = c; + self + } + + pub fn timeout(mut self, d: Duration) -> Self { + self.timeout = Some(d); + self + } + + pub fn on_shutdown(self, f: NewShutdown) -> OneShotBuilder<'a, Task, NewShutdown> + where + NewShutdown: AsyncFnOnce(Result<(), IggyError>) + 'static, + { + OneShotBuilder { + reg: self.reg, + name: self.name, + critical: self.critical, + timeout: self.timeout, + run_fn: self.run_fn, + on_shutdown: Some(f), + } + } +} + +impl<'a, OnShutdown> OneShotBuilder<'a, (), OnShutdown> { + pub fn run(self, f: NewTask) -> OneShotBuilder<'a, NewTask, OnShutdown> + where + NewTask: AsyncFnOnce(ShutdownToken) -> Result<(), IggyError> + 'static, + { + OneShotBuilder { + reg: self.reg, + name: self.name, + critical: self.critical, + timeout: self.timeout, + run_fn: Some(f), + on_shutdown: self.on_shutdown, + } + } +} + +impl<'a, Task> OneShotBuilder<'a, Task, NoShutdown> +where + Task: AsyncFnOnce(ShutdownToken) -> Result<(), IggyError> + 'static, +{ + pub fn spawn(self) { + let run_fn = self.run_fn.expect("run() must be called before spawn()"); + self.reg.spawn_oneshot_closure( + self.name, + self.critical, + self.timeout, + run_fn, + Some(|_| async {}), + ); + } +} + +impl<'a, Task, OnShutdown> OneShotBuilder<'a, Task, OnShutdown> +where + Task: AsyncFnOnce(ShutdownToken) -> Result<(), IggyError> + 'static, + OnShutdown: AsyncFnOnce(Result<(), IggyError>) + 'static, +{ + pub fn spawn(self) { + let run_fn = self.run_fn.expect("run() must be called before spawn()"); + self.reg.spawn_oneshot_closure( + self.name, + self.critical, + self.timeout, + run_fn, + self.on_shutdown, + ); + } +} diff --git a/core/server/src/shard/task_registry/builders/periodic.rs b/core/server/src/shard/task_registry/builders/periodic.rs new file mode 100644 index 0000000000..e91c4b8107 --- /dev/null +++ b/core/server/src/shard/task_registry/builders/periodic.rs @@ -0,0 +1,135 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::NoShutdown; +use crate::shard::task_registry::ShutdownToken; +use crate::shard::task_registry::registry::TaskRegistry; +use iggy_common::IggyError; +use std::ops::{AsyncFn, AsyncFnOnce}; +use std::time::Duration; + +pub struct PeriodicBuilder<'a, Tick, OnShutdown = NoShutdown> { + reg: &'a TaskRegistry, + name: &'static str, + critical: bool, + period: Option, + last_on_shutdown: bool, + tick_fn: Option, + on_shutdown: Option, +} + +impl<'a> PeriodicBuilder<'a, (), NoShutdown> { + pub fn new(reg: &'a TaskRegistry, name: &'static str) -> Self { + Self { + reg, + name, + critical: false, + period: None, + last_on_shutdown: false, + tick_fn: None, + on_shutdown: None, + } + } +} + +impl<'a, Tick, OnShutdown> PeriodicBuilder<'a, Tick, OnShutdown> { + pub fn every(mut self, d: Duration) -> Self { + self.period = Some(d); + self + } + + pub fn critical(mut self, c: bool) -> Self { + self.critical = c; + self + } + + pub fn last_tick_on_shutdown(mut self, v: bool) -> Self { + self.last_on_shutdown = v; + self + } + + pub fn on_shutdown(self, f: NewShutdown) -> PeriodicBuilder<'a, Tick, NewShutdown> + where + NewShutdown: AsyncFnOnce(Result<(), IggyError>) + 'static, + { + PeriodicBuilder { + reg: self.reg, + name: self.name, + critical: self.critical, + period: self.period, + last_on_shutdown: self.last_on_shutdown, + tick_fn: self.tick_fn, + on_shutdown: Some(f), + } + } +} + +impl<'a> PeriodicBuilder<'a, ()> { + pub fn tick(self, f: NewTick) -> PeriodicBuilder<'a, NewTick> + where + NewTick: AsyncFn(ShutdownToken) -> Result<(), IggyError> + 'static, + { + PeriodicBuilder { + reg: self.reg, + name: self.name, + critical: self.critical, + period: self.period, + last_on_shutdown: self.last_on_shutdown, + tick_fn: Some(f), + on_shutdown: self.on_shutdown, + } + } +} + +impl<'a, Tick> PeriodicBuilder<'a, Tick, NoShutdown> +where + Tick: AsyncFn(ShutdownToken) -> Result<(), IggyError> + 'static, +{ + pub fn spawn(self) { + let period = self.period.expect("period required - use .every()"); + let tick_fn = self.tick_fn.expect("tick function required - use .tick()"); + + self.reg.spawn_periodic_closure( + self.name, + period, + self.critical, + self.last_on_shutdown, + tick_fn, + Some(|_| async {}), + ); + } +} + +impl<'a, Tick, OnShutdown> PeriodicBuilder<'a, Tick, OnShutdown> +where + Tick: AsyncFn(ShutdownToken) -> Result<(), IggyError> + 'static, + OnShutdown: AsyncFnOnce(Result<(), IggyError>) + 'static, +{ + pub fn spawn(self) { + let period = self.period.expect("period required - use .every()"); + let tick_fn = self.tick_fn.expect("tick function required - use .tick()"); + + self.reg.spawn_periodic_closure( + self.name, + period, + self.critical, + self.last_on_shutdown, + tick_fn, + self.on_shutdown, + ); + } +} diff --git a/core/server/src/shard/task_registry/mod.rs b/core/server/src/shard/task_registry/mod.rs new file mode 100644 index 0000000000..8020c34af1 --- /dev/null +++ b/core/server/src/shard/task_registry/mod.rs @@ -0,0 +1,23 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod builders; +pub mod registry; +pub mod shutdown; + +pub use registry::TaskRegistry; +pub use shutdown::{Shutdown, ShutdownToken}; diff --git a/core/server/src/shard/task_registry/registry.rs b/core/server/src/shard/task_registry/registry.rs new file mode 100644 index 0000000000..531cdcbdcb --- /dev/null +++ b/core/server/src/shard/task_registry/registry.rs @@ -0,0 +1,732 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::shutdown::{Shutdown, ShutdownToken}; +use crate::shard::transmission::connector::StopSender; +use compio::runtime::JoinHandle; +use futures::FutureExt; +use futures::future::join_all; +use iggy_common::IggyError; +use std::cell::RefCell; +use std::collections::HashMap; +use std::future::Future; +use std::ops::{AsyncFn, AsyncFnOnce}; +use std::panic::AssertUnwindSafe; +use std::time::{Duration, Instant}; +use tracing::{debug, error, trace, warn}; + +#[derive(Debug)] +enum Kind { + Continuous, + Periodic, + OneShot, +} + +#[derive(Debug)] +struct TaskHandle { + name: String, + kind: Kind, + handle: JoinHandle>, + critical: bool, +} + +pub struct TaskRegistry { + shard_id: u16, + shutdown: Shutdown, + shutdown_token: ShutdownToken, + all_stop_senders: Vec, + long_running: RefCell>, + oneshots: RefCell>, + connections: RefCell>>, + shutting_down: RefCell, +} + +impl TaskRegistry { + pub fn new(shard_id: u16, all_stop_senders: Vec) -> Self { + let (s, t) = Shutdown::new(); + Self { + shard_id, + shutdown: s, + shutdown_token: t, + all_stop_senders, + long_running: RefCell::new(vec![]), + oneshots: RefCell::new(vec![]), + connections: RefCell::new(HashMap::new()), + shutting_down: RefCell::new(false), + } + } + + pub fn shutdown_token(&self) -> ShutdownToken { + self.shutdown_token.clone() + } + + pub(crate) fn spawn_continuous_closure( + &self, + name: &'static str, + critical: bool, + f: Task, + on_shutdown: Option, + ) where + Task: AsyncFnOnce(ShutdownToken) -> Result<(), IggyError> + 'static, + OnShutdown: AsyncFnOnce(Result<(), IggyError>) + 'static, + { + if *self.shutting_down.borrow() { + warn!( + "Attempted to spawn continuous task '{}' during shutdown", + name + ); + return; + } + + let shutdown = self.shutdown_token.clone(); + let shard_id = self.shard_id; + let all_stop_senders = self.all_stop_senders.clone(); + + let handle = compio::runtime::spawn(async move { + trace!("continuous '{}' starting on shard {}", name, shard_id); + + let fut = AssertUnwindSafe(f(shutdown)).catch_unwind(); + let result = fut.await; + + let (r, should_trigger_shutdown) = match result { + Ok(r) => { + match &r { + Ok(()) => debug!("continuous '{}' completed on shard {}", name, shard_id), + Err(e) => { + error!("continuous '{}' failed on shard {}: {}", name, shard_id, e); + } + } + // Trigger shutdown for critical task errors + let trigger = critical && r.is_err(); + (r, trigger) + } + Err(panic_payload) => { + let panic_msg = panic_payload + .downcast_ref::<&str>() + .map(|s| s.to_string()) + .or_else(|| panic_payload.downcast_ref::().cloned()) + .unwrap_or_else(|| "unknown panic".to_string()); + error!( + "continuous '{}' panicked on shard {}: {}", + name, shard_id, panic_msg + ); + // Trigger shutdown for critical task panics + (Err(IggyError::Error), critical) + } + }; + + // Execute on_shutdown callback if provided + if let Some(shutdown_fn) = on_shutdown { + trace!("continuous '{}' executing on_shutdown callback", name); + shutdown_fn(r.clone()).await; + } + + // Trigger shutdown for ALL shards when critical task fails + if should_trigger_shutdown { + error!( + "Critical task '{}' failed on shard {}, triggering shutdown for all shards", + name, shard_id + ); + for stop_sender in &all_stop_senders { + let _ = stop_sender.try_send(()); + } + } + + r + }); + + self.long_running.borrow_mut().push(TaskHandle { + name: name.into(), + kind: Kind::Continuous, + handle, + critical, + }); + } + + pub(crate) fn spawn_periodic_closure( + &self, + name: &'static str, + period: Duration, + critical: bool, + last_on_shutdown: bool, + tick_fn: Tick, + on_shutdown: Option, + ) where + Tick: AsyncFn(ShutdownToken) -> Result<(), IggyError> + 'static, + OnShutdown: AsyncFnOnce(Result<(), IggyError>) + 'static, + { + if *self.shutting_down.borrow() { + warn!( + "Attempted to spawn periodic task '{}' during shutdown", + name + ); + return; + } + + let shutdown = self.shutdown_token.clone(); + let shutdown_for_task = self.shutdown_token.clone(); + let shard_id = self.shard_id; + let all_stop_senders = self.all_stop_senders.clone(); + + let handle = compio::runtime::spawn(async move { + trace!( + "periodic '{}' every {:?} on shard {}", + name, period, shard_id + ); + + loop { + if !shutdown.sleep_or_shutdown(period).await { + break; + } + + let fut = AssertUnwindSafe(tick_fn(shutdown_for_task.clone())).catch_unwind(); + match fut.await { + Ok(Ok(())) => {} + Ok(Err(e)) => { + error!( + "periodic '{}' tick failed on shard {}: {}", + name, shard_id, e + ); + } + Err(panic_payload) => { + let panic_msg = panic_payload + .downcast_ref::<&str>() + .map(|s| s.to_string()) + .or_else(|| panic_payload.downcast_ref::().cloned()) + .unwrap_or_else(|| "unknown panic".to_string()); + error!( + "periodic '{}' tick panicked on shard {}: {}", + name, shard_id, panic_msg + ); + if critical { + error!( + "Critical periodic task '{}' panicked on shard {}, triggering shutdown", + name, shard_id + ); + for stop_sender in &all_stop_senders { + let _ = stop_sender.try_send(()); + } + return Err(IggyError::Error); + } + } + } + } + + if last_on_shutdown { + const FINAL_TICK_TIMEOUT: Duration = Duration::from_secs(5); + trace!( + "periodic '{}' executing final tick on shutdown (timeout: {:?})", + name, FINAL_TICK_TIMEOUT + ); + + let fut = tick_fn(shutdown_for_task); + match compio::time::timeout(FINAL_TICK_TIMEOUT, fut).await { + Ok(Ok(())) => trace!("periodic '{}' final tick completed", name), + Ok(Err(e)) => error!("periodic '{}' final tick failed: {}", name, e), + Err(_) => error!( + "periodic '{}' final tick timed out after {:?}", + name, FINAL_TICK_TIMEOUT + ), + } + } + + let result = Ok(()); + + if let Some(on_shutdown) = on_shutdown { + on_shutdown(result.clone()).await; + } + + result + }); + + self.long_running.borrow_mut().push(TaskHandle { + name: name.into(), + kind: Kind::Periodic, + handle, + critical, + }); + } + + pub(crate) fn spawn_oneshot_closure( + &self, + name: &'static str, + critical: bool, + timeout: Option, + f: Task, + on_shutdown: Option, + ) where + Task: AsyncFnOnce(ShutdownToken) -> Result<(), IggyError> + 'static, + OnShutdown: AsyncFnOnce(Result<(), IggyError>) + 'static, + { + if *self.shutting_down.borrow() { + warn!("Attempted to spawn oneshot task '{}' during shutdown", name); + return; + } + + let shutdown = self.shutdown_token.clone(); + let shard_id = self.shard_id; + let all_stop_senders = self.all_stop_senders.clone(); + + let handle = compio::runtime::spawn(async move { + trace!("oneshot '{}' starting on shard {}", name, shard_id); + + let fut = if let Some(d) = timeout { + let inner_fut = AssertUnwindSafe(f(shutdown)).catch_unwind(); + match compio::time::timeout(d, inner_fut).await { + Ok(Ok(r)) => Ok(r), + Ok(Err(panic_payload)) => Err(panic_payload), + Err(_) => Ok(Err(IggyError::TaskTimeout)), + } + } else { + AssertUnwindSafe(f(shutdown)).catch_unwind().await + }; + + let r = match fut { + Ok(r) => { + match &r { + Ok(()) => trace!("oneshot '{}' completed on shard {}", name, shard_id), + Err(e) => { + error!("oneshot '{}' failed on shard {}: {}", name, shard_id, e); + if critical { + error!( + "Critical oneshot task '{}' failed on shard {}, triggering shutdown", + name, shard_id + ); + for stop_sender in &all_stop_senders { + let _ = stop_sender.try_send(()); + } + } + } + } + r + } + Err(panic_payload) => { + let panic_msg = panic_payload + .downcast_ref::<&str>() + .map(|s| s.to_string()) + .or_else(|| panic_payload.downcast_ref::().cloned()) + .unwrap_or_else(|| "unknown panic".to_string()); + error!( + "oneshot '{}' panicked on shard {}: {}", + name, shard_id, panic_msg + ); + if critical { + error!( + "Critical oneshot task '{}' panicked on shard {}, triggering shutdown", + name, shard_id + ); + for stop_sender in &all_stop_senders { + let _ = stop_sender.try_send(()); + } + } + Err(IggyError::Error) + } + }; + + if let Some(on_shutdown) = on_shutdown { + on_shutdown(r.clone()).await; + } + + r + }); + + self.oneshots.borrow_mut().push(TaskHandle { + name: name.into(), + kind: Kind::OneShot, + handle, + critical, + }); + } + + pub async fn graceful_shutdown(&self, timeout: Duration) -> bool { + let start = Instant::now(); + *self.shutting_down.borrow_mut() = true; + self.shutdown_connections(); + self.shutdown.trigger(); + + // First shutdown long-running tasks (continuous and periodic) + let long = self.long_running.take(); + let long_ok = if !long.is_empty() { + debug!( + "Shutting down {} long-running task(s) on shard {}", + long.len(), + self.shard_id + ); + self.await_with_timeout(long, timeout).await + } else { + true + }; + + // Calculate remaining time for oneshots + let elapsed = start.elapsed(); + let remaining = timeout.saturating_sub(elapsed); + + // Then shutdown oneshot tasks with remaining time + let ones = self.oneshots.take(); + let ones_ok = if !ones.is_empty() { + if remaining.is_zero() { + warn!( + "No time remaining for {} oneshot task(s) on shard {}, they will be cancelled", + ones.len(), + self.shard_id + ); + false + } else { + debug!( + "Shutting down {} oneshot task(s) on shard {} with {:?} remaining", + ones.len(), + self.shard_id, + remaining + ); + self.await_with_timeout(ones, remaining).await + } + } else { + true + }; + + let total_elapsed = start.elapsed(); + if long_ok && ones_ok { + debug!( + "Graceful shutdown completed successfully on shard {} in {:?}", + self.shard_id, total_elapsed + ); + } else { + warn!( + "Graceful shutdown completed with failures on shard {} in {:?}", + self.shard_id, total_elapsed + ); + } + + long_ok && ones_ok + } + + async fn await_with_timeout(&self, tasks: Vec, timeout: Duration) -> bool { + if tasks.is_empty() { + return true; + } + let results = join_all(tasks.into_iter().map(|t| async move { + match compio::time::timeout(timeout, t.handle).await { + Ok(Ok(Ok(()))) => true, + Ok(Ok(Err(e))) => { + error!("task '{}' of kind {:?} failed: {}", t.name, t.kind, e); + !t.critical + } + Ok(Err(_)) => { + error!("task '{}' of kind {:?} panicked", t.name, t.kind); + !t.critical + } + Err(_) => { + error!( + "task '{}' of kind {:?} timed out after {:?}", + t.name, t.kind, timeout + ); + !t.critical + } + } + })) + .await; + + results.into_iter().all(|x| x) + } + + #[cfg(test)] + async fn await_all(&self, tasks: Vec) -> bool { + if tasks.is_empty() { + return true; + } + let results = join_all(tasks.into_iter().map(|t| async move { + match t.handle.await { + Ok(Ok(())) => true, + Ok(Err(e)) => { + error!("task '{}' failed: {}", t.name, e); + !t.critical + } + Err(_) => { + error!("task '{}' panicked", t.name); + !t.critical + } + } + })) + .await; + results.into_iter().all(|x| x) + } + + pub fn add_connection(&self, client_id: u32) -> async_channel::Receiver<()> { + let (tx, rx) = async_channel::bounded(1); + self.connections.borrow_mut().insert(client_id, tx); + rx + } + + pub fn remove_connection(&self, client_id: &u32) { + self.connections.borrow_mut().remove(client_id); + } + + fn shutdown_connections(&self) { + // Close all connection channels to signal shutdown + // We use close() instead of send_blocking() to avoid potential blocking + for tx in self.connections.borrow().values() { + tx.close(); + } + } + + /// Spawn a connection handler that doesn't need to be tracked for shutdown. + /// These handlers have their own shutdown mechanism via connection channels. + /// If the handler panics, shutdown is triggered for all shards. + pub fn spawn_connection(&self, future: F) + where + F: Future + 'static, + { + let shard_id = self.shard_id; + let all_stop_senders = self.all_stop_senders.clone(); + + compio::runtime::spawn(async move { + let fut = AssertUnwindSafe(future).catch_unwind(); + if let Err(panic_payload) = fut.await { + let panic_msg = panic_payload + .downcast_ref::<&str>() + .map(|s| s.to_string()) + .or_else(|| panic_payload.downcast_ref::().cloned()) + .unwrap_or_else(|| "unknown panic".to_string()); + + error!( + "Connection handler panicked on shard {}: {}, triggering shutdown", + shard_id, panic_msg + ); + + for stop_sender in &all_stop_senders { + let _ = stop_sender.try_send(()); + } + } + }) + .detach(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn create_test_registry(shard_id: u16) -> TaskRegistry { + let (stop_sender, _stop_receiver) = async_channel::bounded(1); + TaskRegistry::new(shard_id, vec![stop_sender]) + } + + #[compio::test] + async fn test_oneshot_completion_detection() { + let registry = create_test_registry(1); + + // Spawn a failing non-critical task + registry + .oneshot("failing_non_critical") + .run(|_shutdown| async { Err(IggyError::Error) }) + .spawn(); + + // Spawn a successful task + registry + .oneshot("successful") + .run(|_shutdown| async { Ok(()) }) + .spawn(); + + // Wait for all tasks + let all_ok = registry.await_all(registry.oneshots.take()).await; + + // Should return true because the failing task is not critical + assert!(all_ok); + } + + #[compio::test] + async fn test_oneshot_critical_failure() { + let registry = create_test_registry(1); + + // Spawn a failing critical task + registry + .oneshot("failing_critical") + .critical(true) + .run(|_shutdown| async { Err(IggyError::Error) }) + .spawn(); + + // Wait for all tasks + let all_ok = registry.await_all(registry.oneshots.take()).await; + + // Should return false because the failing task is critical + assert!(!all_ok); + } + + #[compio::test] + async fn test_shutdown_prevents_spawning() { + let registry = create_test_registry(1); + + // Trigger shutdown + *registry.shutting_down.borrow_mut() = true; + + let initial_count = registry.oneshots.borrow().len(); + + // Try to spawn after shutdown + registry + .oneshot("should_not_spawn") + .run(|_shutdown| async { Ok(()) }) + .spawn(); + + // Task should not be added + assert_eq!(registry.oneshots.borrow().len(), initial_count); + } + + #[compio::test] + async fn test_timeout_error() { + let registry = create_test_registry(1); + + // Create a task that will timeout + let handle = compio::runtime::spawn(async move { + compio::time::sleep(Duration::from_secs(10)).await; + Ok(()) + }); + + let task_handle = TaskHandle { + name: "timeout_test".to_string(), + kind: Kind::OneShot, + handle, + critical: false, + }; + + let tasks = vec![task_handle]; + let all_ok = registry + .await_with_timeout(tasks, Duration::from_millis(50)) + .await; + + // Should return true because the task is not critical + assert!(all_ok); + } + + #[compio::test] + async fn test_composite_timeout() { + let registry = create_test_registry(1); + + // Create a long-running task that takes 100ms + let long_handle = compio::runtime::spawn(async move { + compio::time::sleep(Duration::from_millis(100)).await; + Ok(()) + }); + + registry.long_running.borrow_mut().push(TaskHandle { + name: "long_task".to_string(), + kind: Kind::Continuous, + handle: long_handle, + critical: false, + }); + + // Create a oneshot that would succeed quickly + let oneshot_handle = compio::runtime::spawn(async move { + compio::time::sleep(Duration::from_millis(10)).await; + Ok(()) + }); + + registry.oneshots.borrow_mut().push(TaskHandle { + name: "quick_oneshot".to_string(), + kind: Kind::OneShot, + handle: oneshot_handle, + critical: false, + }); + + // Give total timeout of 150ms + // Long-running should complete in ~100ms + // Oneshot should have ~50ms remaining, which is enough + let all_ok = registry.graceful_shutdown(Duration::from_millis(150)).await; + assert!(all_ok); + } + + #[compio::test] + async fn test_composite_timeout_insufficient() { + let registry = create_test_registry(1); + + // Create a long-running task that takes 50ms + let long_handle = compio::runtime::spawn(async move { + compio::time::sleep(Duration::from_millis(50)).await; + Ok(()) + }); + + registry.long_running.borrow_mut().push(TaskHandle { + name: "long_task".to_string(), + kind: Kind::Continuous, + handle: long_handle, + critical: false, + }); + + // Create a oneshot that would take 100ms (much longer) + let oneshot_handle = compio::runtime::spawn(async move { + compio::time::sleep(Duration::from_millis(100)).await; + Ok(()) + }); + + registry.oneshots.borrow_mut().push(TaskHandle { + name: "slow_oneshot".to_string(), + kind: Kind::OneShot, + handle: oneshot_handle, + critical: true, // Make it critical so failure is detected + }); + + // Give total timeout of 60ms + // Long-running should complete in ~50ms + // Oneshot would need 100ms but only has ~10ms, so it should definitely fail + let all_ok = registry.graceful_shutdown(Duration::from_millis(60)).await; + assert!(!all_ok); // Should fail because critical oneshot times out + } + + #[compio::test] + async fn test_periodic_last_tick_timeout() { + // This test verifies that periodic tasks with last_tick_on_shutdown + // don't hang shutdown if the final tick takes too long + let registry = create_test_registry(1); + + // Create a handle that simulates a periodic task whose final tick will hang + let handle = compio::runtime::spawn(async move { + // Simulate the periodic task loop that already exited + // Now simulate the last_tick_on_shutdown logic with a hanging tick + const FINAL_TICK_TIMEOUT: Duration = Duration::from_millis(100); + let fut = async { + // This would hang for 500ms without timeout + compio::time::sleep(Duration::from_millis(500)).await; + Ok::<(), IggyError>(()) + }; + + match compio::time::timeout(FINAL_TICK_TIMEOUT, fut).await { + Ok(Ok(())) => {} + Ok(Err(_)) => {} + Err(_) => { + // Timeout occurred as expected + } + } + Ok(()) + }); + + registry.long_running.borrow_mut().push(TaskHandle { + name: "periodic_with_slow_final".to_string(), + kind: Kind::Periodic, + handle, + critical: false, + }); + + // Shutdown should complete in ~100ms (the FINAL_TICK_TIMEOUT), not 500ms + let start = std::time::Instant::now(); + let all_ok = registry.graceful_shutdown(Duration::from_secs(1)).await; + let elapsed = start.elapsed(); + + // Should complete in about 100ms due to the timeout, not hang for 500ms + assert!(elapsed >= Duration::from_millis(80)); // At least 80ms + assert!(elapsed < Duration::from_millis(200)); // But less than 200ms (not the full 500ms) + assert!(all_ok); + } +} diff --git a/core/server/src/shard/task_registry/shutdown.rs b/core/server/src/shard/task_registry/shutdown.rs new file mode 100644 index 0000000000..09eab0a3ab --- /dev/null +++ b/core/server/src/shard/task_registry/shutdown.rs @@ -0,0 +1,232 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use async_channel::{Receiver, Sender, bounded}; +use futures::FutureExt; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; +use tracing::trace; + +/// Coordinates graceful shutdown across multiple tasks +#[derive(Clone)] +pub struct Shutdown { + sender: Sender<()>, + is_triggered: Arc, +} + +impl Shutdown { + pub fn new() -> (Self, ShutdownToken) { + let (sender, receiver) = bounded(1); + let is_triggered = Arc::new(AtomicBool::new(false)); + + let shutdown = Self { + sender, + is_triggered: is_triggered.clone(), + }; + + let token = ShutdownToken { + receiver, + is_triggered, + }; + + (shutdown, token) + } + + pub fn trigger(&self) { + if self.is_triggered.swap(true, Ordering::SeqCst) { + return; + } + + trace!("Triggering shutdown signal"); + let _ = self.sender.close(); + } + + pub fn is_triggered(&self) -> bool { + self.is_triggered.load(Ordering::Relaxed) + } +} + +/// Token held by tasks to receive shutdown signals +#[derive(Clone)] +pub struct ShutdownToken { + receiver: Receiver<()>, + is_triggered: Arc, +} + +impl ShutdownToken { + /// Wait for shutdown signal + pub async fn wait(&self) { + let _ = self.receiver.recv().await; + } + + /// Check if shutdown has been triggered (non-blocking) + pub fn is_triggered(&self) -> bool { + self.is_triggered.load(Ordering::Relaxed) + } + + /// Sleep for the specified duration or until shutdown is triggered + /// Returns true if the full duration elapsed, false if shutdown was triggered + pub async fn sleep_or_shutdown(&self, duration: Duration) -> bool { + futures::select! { + _ = self.wait().fuse() => false, + _ = compio::time::sleep(duration).fuse() => !self.is_triggered(), + } + } + + /// Creates a scoped shutdown pair (child `Shutdown`, combined `ShutdownToken`). + /// + /// This is a bit complicated, but it needs to be this way to avoid deadlocks. + /// + /// The returned token fires when EITHER the parent or the child is triggered, + /// while a child trigger does NOT propagate back to the parent. + /// Internally spawns a tiny forwarder to merge both signals into one channel, + /// so callers can await a single `wait()` and use fast `is_triggered()` checks + /// without writing `select!` at every call site. + /// Use when a subtree needs cancelation that respects parent cancelation, + /// but can also be canceled locally. + pub fn child(&self) -> (Shutdown, ShutdownToken) { + let (child_shutdown, child_token) = Shutdown::new(); + let parent_receiver = self.receiver.clone(); + let child_receiver = child_token.receiver.clone(); + + let (combined_sender, combined_receiver) = bounded(1); + let combined_is_triggered = Arc::new(AtomicBool::new(false)); + + let parent_triggered = self.is_triggered.clone(); + let child_triggered = child_token.is_triggered.clone(); + let combined_flag_for_task = combined_is_triggered.clone(); + + compio::runtime::spawn(async move { + futures::select! { + _ = parent_receiver.recv().fuse() => { + trace!("Child token triggered by parent shutdown"); + }, + _ = child_receiver.recv().fuse() => { + trace!("Child token triggered by child shutdown"); + }, + } + + if parent_triggered.load(Ordering::Relaxed) || child_triggered.load(Ordering::Relaxed) { + combined_flag_for_task.store(true, Ordering::SeqCst); + } + + let _ = combined_sender.close(); + }) + .detach(); + + let combined_token = ShutdownToken { + receiver: combined_receiver, + is_triggered: combined_is_triggered, + }; + + (child_shutdown, combined_token) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[compio::test] + async fn test_shutdown_trigger() { + let (shutdown, token) = Shutdown::new(); + + assert!(!token.is_triggered()); + + shutdown.trigger(); + + assert!(token.is_triggered()); + + token.wait().await; + } + + #[compio::test] + async fn test_sleep_or_shutdown_completes() { + let (_shutdown, token) = Shutdown::new(); + + let completed = token.sleep_or_shutdown(Duration::from_millis(10)).await; + assert!(completed); + } + + #[compio::test] + async fn test_sleep_or_shutdown_interrupted() { + let (shutdown, token) = Shutdown::new(); + + // Trigger shutdown after a short delay + let shutdown_clone = shutdown.clone(); + compio::runtime::spawn(async move { + compio::time::sleep(Duration::from_millis(10)).await; + shutdown_clone.trigger(); + }) + .detach(); + + // Should be interrupted + let completed = token.sleep_or_shutdown(Duration::from_secs(10)).await; + assert!(!completed); + } + + #[compio::test] + async fn test_child_token_parent_trigger() { + let (parent_shutdown, parent_token) = Shutdown::new(); + let (_child_shutdown, combined_token) = parent_token.child(); + + assert!(!combined_token.is_triggered()); + + // Trigger parent shutdown + parent_shutdown.trigger(); + + // Combined token should be triggered + combined_token.wait().await; + assert!(combined_token.is_triggered()); + } + + #[compio::test] + async fn test_child_token_child_trigger() { + let (_parent_shutdown, parent_token) = Shutdown::new(); + let (child_shutdown, combined_token) = parent_token.child(); + + assert!(!combined_token.is_triggered()); + + // Trigger child shutdown + child_shutdown.trigger(); + + // Combined token should be triggered + combined_token.wait().await; + assert!(combined_token.is_triggered()); + } + + #[compio::test] + async fn test_child_token_no_polling_overhead() { + let (_parent_shutdown, parent_token) = Shutdown::new(); + let (_child_shutdown, combined_token) = parent_token.child(); + + // Test that we can create many child tokens without performance issues + let start = std::time::Instant::now(); + for _ in 0..100 { + let _ = combined_token.child(); + } + let elapsed = start.elapsed(); + + // Should complete very quickly since there's no polling + assert!( + elapsed.as_millis() < 100, + "Creating child tokens took too long: {:?}", + elapsed + ); + } +} diff --git a/core/server/src/shard/tasks/continuous/http_server.rs b/core/server/src/shard/tasks/continuous/http_server.rs new file mode 100644 index 0000000000..2b88639905 --- /dev/null +++ b/core/server/src/shard/tasks/continuous/http_server.rs @@ -0,0 +1,40 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::bootstrap::resolve_persister; +use crate::http::http_server::start_http_server; +use crate::shard::IggyShard; +use crate::shard::task_registry::ShutdownToken; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::info; + +pub fn spawn_http_server(shard: Rc) { + let shard_clone = shard.clone(); + shard + .task_registry + .continuous("http_server") + .critical(true) + .run(move |shutdown| http_server(shard_clone, shutdown)) + .spawn(); +} + +async fn http_server(shard: Rc, shutdown: ShutdownToken) -> Result<(), IggyError> { + info!("Starting HTTP server on shard: {}", shard.id); + let persister = resolve_persister(shard.config.system.partition.enforce_fsync); + start_http_server(shard.config.http.clone(), persister, shard, shutdown).await +} diff --git a/core/server/src/shard/tasks/continuous/message_pump.rs b/core/server/src/shard/tasks/continuous/message_pump.rs new file mode 100644 index 0000000000..2837ef626d --- /dev/null +++ b/core/server/src/shard/tasks/continuous/message_pump.rs @@ -0,0 +1,135 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::shard::task_registry::ShutdownToken; +use crate::shard::transmission::frame::ShardFrame; +use crate::shard::{IggyShard, handlers::handle_shard_message}; +use futures::FutureExt; +use std::rc::Rc; +use tracing::{debug, error, info}; + +pub fn spawn_message_pump(shard: Rc) { + let shard_clone = shard.clone(); + shard + .task_registry + .continuous("message_pump") + .critical(true) + .run(move |shutdown| message_pump(shard_clone, shutdown)) + .spawn(); +} + +/// Single serialization point for all partition mutations on this shard. +/// +/// Every operation that mutates `local_partitions` — appends, segment rotation, flush, +/// segment deletion — is dispatched exclusively through this pump. The loop awaits each +/// `process_frame` to completion before dequeuing the next message, so handlers never +/// interleave even across internal `.await` points (disk I/O, fsync). +/// +/// Periodic tasks (message_saver, message_cleaner) run as separate futures on the same +/// compio thread but **cannot** mutate partitions directly. They read partition metadata +/// via `borrow()` and enqueue mutation requests back into this pump's channel. Those +/// requests block on a response that is only sent after the current frame completes, +/// guaranteeing strict ordering. +/// +/// This invariant replaces per-partition write locks and eliminates TOCTOU races between +/// concurrent handlers. All `pub(crate)` mutation methods on `IggyShard` (e.g. +/// `append_messages_to_local_partition`, `delete_expired_segments`, +/// `rotate_segment_in_local_partitions`) assume they are called from within this pump. +async fn message_pump( + shard: Rc, + shutdown: ShutdownToken, +) -> Result<(), iggy_common::IggyError> { + let Some(messages_receiver) = shard.messages_receiver.take() else { + info!("Message receiver already taken; pump not started"); + return Ok(()); + }; + + info!("Starting message passing task"); + + let receiver = messages_receiver.inner; + + loop { + futures::select! { + _ = shutdown.wait().fuse() => { + debug!("Message pump shutting down"); + break; + } + frame = receiver.recv_async().fuse() => { + match frame { + Ok(frame) => process_frame(&shard, frame).await, + Err(_) => { + debug!("Message receiver closed; exiting pump"); + break; + } + } + } + } + } + + // Drain remaining frames before flushing — any in-flight appends must + // complete so their data lands in the journal before we flush to disk. + while let Ok(frame) = receiver.try_recv() { + process_frame(&shard, frame).await; + } + + flush_and_fsync_all_partitions(&shard).await; + + Ok(()) +} + +async fn process_frame(shard: &Rc, frame: ShardFrame) { + let ShardFrame { + message, + response_sender, + } = frame; + if let (Some(response), Some(tx)) = + (handle_shard_message(shard, message).await, response_sender) + { + let _ = tx.send(response).await; + } +} + +/// Final flush + fsync of all local partitions. Runs inside the pump after +/// the main loop exits, so no other pump frame can interleave. +async fn flush_and_fsync_all_partitions(shard: &Rc) { + let namespaces = shard.get_current_shard_namespaces(); + if namespaces.is_empty() { + return; + } + + let mut flushed = 0u32; + for ns in &namespaces { + match shard + .flush_unsaved_buffer_from_local_partitions(ns, false) + .await + { + Ok(saved) if saved > 0 => flushed += 1, + Ok(_) => {} + Err(e) => error!("Shutdown flush failed for partition {:?}: {}", ns, e), + } + } + if flushed > 0 { + info!("Shutdown: flushed {flushed} partitions."); + } + + for ns in &namespaces { + if let Err(e) = shard.fsync_all_messages_from_local_partitions(ns).await { + error!("Shutdown fsync failed for partition {:?}: {}", ns, e); + } + } + info!("Shutdown: fsync complete for all partitions."); +} diff --git a/core/server/src/shard/tasks/continuous/mod.rs b/core/server/src/shard/tasks/continuous/mod.rs new file mode 100644 index 0000000000..421d77be7e --- /dev/null +++ b/core/server/src/shard/tasks/continuous/mod.rs @@ -0,0 +1,28 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod http_server; +mod message_pump; +mod quic_server; +mod tcp_server; +mod websocket_server; + +pub use http_server::spawn_http_server; +pub use message_pump::spawn_message_pump; +pub use quic_server::spawn_quic_server; +pub use tcp_server::spawn_tcp_server; +pub use websocket_server::spawn_websocket_server; diff --git a/core/server/src/shard/tasks/continuous/quic_server.rs b/core/server/src/shard/tasks/continuous/quic_server.rs new file mode 100644 index 0000000000..93af50455e --- /dev/null +++ b/core/server/src/shard/tasks/continuous/quic_server.rs @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::quic::quic_server; +use crate::shard::IggyShard; +use crate::shard::task_registry::ShutdownToken; +use iggy_common::IggyError; +use std::rc::Rc; + +pub fn spawn_quic_server(shard: Rc) { + let shard_clone = shard.clone(); + shard + .task_registry + .continuous("quic_server") + .critical(true) + .run(move |shutdown| quic_server_task(shard_clone, shutdown)) + .spawn(); +} + +async fn quic_server_task(shard: Rc, shutdown: ShutdownToken) -> Result<(), IggyError> { + quic_server::spawn_quic_server(shard, shutdown).await +} diff --git a/core/server/src/shard/tasks/continuous/tcp_server.rs b/core/server/src/shard/tasks/continuous/tcp_server.rs new file mode 100644 index 0000000000..7a4348f94b --- /dev/null +++ b/core/server/src/shard/tasks/continuous/tcp_server.rs @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::shard::IggyShard; +use crate::shard::task_registry::ShutdownToken; +use crate::tcp::tcp_server; +use iggy_common::IggyError; +use std::rc::Rc; + +pub fn spawn_tcp_server(shard: Rc) { + let shard_clone = shard.clone(); + shard + .task_registry + .continuous("tcp_server") + .critical(true) + .run(move |shutdown| tcp_server_task(shard_clone, shutdown)) + .spawn(); +} + +async fn tcp_server_task(shard: Rc, shutdown: ShutdownToken) -> Result<(), IggyError> { + tcp_server::spawn_tcp_server(shard, shutdown).await +} diff --git a/core/server/src/shard/tasks/continuous/websocket_server.rs b/core/server/src/shard/tasks/continuous/websocket_server.rs new file mode 100644 index 0000000000..2219534861 --- /dev/null +++ b/core/server/src/shard/tasks/continuous/websocket_server.rs @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::shard::IggyShard; +use crate::shard::task_registry::ShutdownToken; +use iggy_common::IggyError; +use std::rc::Rc; + +pub fn spawn_websocket_server(shard: Rc) { + let shard_clone = shard.clone(); + shard + .task_registry + .continuous("websocket_server") + .critical(true) + .run(move |shutdown| websocket_server_task(shard_clone, shutdown)) + .spawn(); +} + +async fn websocket_server_task( + shard: Rc, + shutdown: ShutdownToken, +) -> Result<(), IggyError> { + crate::websocket::websocket_server::spawn_websocket_server(shard, shutdown).await +} diff --git a/core/server/src/shard/tasks/mod.rs b/core/server/src/shard/tasks/mod.rs new file mode 100644 index 0000000000..7c5f23ba8e --- /dev/null +++ b/core/server/src/shard/tasks/mod.rs @@ -0,0 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod continuous; +pub mod oneshot; +pub mod periodic; diff --git a/core/server/src/shard/tasks/oneshot/config_writer.rs b/core/server/src/shard/tasks/oneshot/config_writer.rs new file mode 100644 index 0000000000..c48a7c8ef2 --- /dev/null +++ b/core/server/src/shard/tasks/oneshot/config_writer.rs @@ -0,0 +1,142 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::shard::IggyShard; +use crate::shard::task_registry::ShutdownToken; +use compio::io::AsyncWriteAtExt; +use err_trail::ErrContext; +use futures::FutureExt; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::{info, warn}; + +pub fn spawn_config_writer_task(shard: &Rc) { + let shard_clone = shard.clone(); + shard + .task_registry + .oneshot("config_writer") + .critical(false) + .run(move |shutdown_token| async move { write_config(shard_clone, shutdown_token).await }) + .spawn(); +} + +async fn write_config( + shard: Rc, + shutdown_token: ShutdownToken, +) -> Result<(), IggyError> { + let shard_clone = shard.clone(); + let tcp_enabled = shard.config.tcp.enabled; + let quic_enabled = shard.config.quic.enabled; + let http_enabled = shard.config.http.enabled; + let websocket_enabled = shard.config.websocket.enabled; + + let notify_receiver = shard_clone.config_writer_receiver.clone(); + + // Wait for notifications until all servers have bound, or shutdown is triggered + loop { + futures::select! { + _ = shutdown_token.wait().fuse() => { + warn!("config_writer: shutdown triggered before all servers bound, skipping config write"); + return Ok(()); + } + result = notify_receiver.recv().fuse() => { + if result.is_err() { + return Err(IggyError::CannotWriteToFile).error( + |_: &IggyError| "config_writer: notification channel closed before all servers bound", + ); + } + } + } + + let tcp_ready = !tcp_enabled || shard_clone.tcp_bound_address.get().is_some(); + let quic_ready = !quic_enabled || shard_clone.quic_bound_address.get().is_some(); + let http_ready = !http_enabled || shard_clone.http_bound_address.get().is_some(); + let websocket_ready = + !websocket_enabled || shard_clone.websocket_bound_address.get().is_some(); + + if tcp_ready && quic_ready && http_ready && websocket_ready { + break; + } + } + + #[cfg(feature = "systemd")] + crate::shard::systemd::notify_ready(); + + let mut current_config = shard_clone.config.clone(); + + let tcp_addr = shard_clone.tcp_bound_address.get(); + let quic_addr = shard_clone.quic_bound_address.get(); + let http_addr = shard_clone.http_bound_address.get(); + let websocket_addr = shard_clone.websocket_bound_address.get(); + + info!( + "Config writer: TCP addr = {:?}, QUIC addr = {:?}, HTTP addr = {:?}, WebSocket addr = {:?}", + tcp_addr, quic_addr, http_addr, websocket_addr + ); + + if let Some(tcp_addr) = tcp_addr { + current_config.tcp.address = tcp_addr.to_string(); + } + + if let Some(quic_addr) = quic_addr { + current_config.quic.address = quic_addr.to_string(); + } + + if let Some(http_addr) = http_addr { + current_config.http.address = http_addr.to_string(); + } + + if let Some(websocket_addr) = websocket_addr { + current_config.websocket.address = websocket_addr.to_string(); + } + + let runtime_path = current_config.system.get_runtime_path(); + let config_path = format!("{runtime_path}/current_config.toml"); + let content = toml::to_string(¤t_config) + .map_err(|_| IggyError::CannotWriteToFile) + .error(|_: &IggyError| "config_writer: cannot serialize current_config")?; + + let mut file = compio::fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(&config_path) + .await + .map_err(|_| IggyError::CannotWriteToFile) + .error(|_: &IggyError| { + format!("config_writer: failed to open current config at {config_path}") + })?; + + file.write_all_at(content.into_bytes(), 0) + .await + .0 + .map_err(|_| IggyError::CannotWriteToFile) + .error(|_: &IggyError| { + format!("config_writer: failed to write current config to {config_path}") + })?; + + file.sync_all() + .await + .map_err(|_| IggyError::CannotWriteToFile) + .error(|_: &IggyError| { + format!("config_writer: failed to fsync current config to {config_path}") + })?; + + info!("Current config written and synced to: {config_path} with all bound addresses",); + + Ok(()) +} diff --git a/core/server/src/shard/tasks/oneshot/mod.rs b/core/server/src/shard/tasks/oneshot/mod.rs new file mode 100644 index 0000000000..3bc0d71138 --- /dev/null +++ b/core/server/src/shard/tasks/oneshot/mod.rs @@ -0,0 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod config_writer; + +pub use config_writer::spawn_config_writer_task; diff --git a/core/server/src/shard/tasks/periodic/heartbeat_verifier.rs b/core/server/src/shard/tasks/periodic/heartbeat_verifier.rs new file mode 100644 index 0000000000..f2c1c16848 --- /dev/null +++ b/core/server/src/shard/tasks/periodic/heartbeat_verifier.rs @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::shard::IggyShard; +use iggy_common::{IggyDuration, IggyError, IggyTimestamp}; +use std::rc::Rc; +use tracing::{debug, info, trace, warn}; + +const MAX_THRESHOLD: f64 = 1.2; + +pub fn spawn_heartbeat_verifier(shard: Rc) { + let period = shard.config.heartbeat.interval.get_duration(); + let interval = iggy_common::IggyDuration::from(period); + let max_interval = + iggy_common::IggyDuration::from((MAX_THRESHOLD * interval.as_micros() as f64) as u64); + info!( + "Heartbeats will be verified every: {}. Max allowed interval: {}.", + interval, max_interval + ); + let shard_clone = shard.clone(); + shard + .task_registry + .periodic("verify_heartbeats") + .every(period) + .tick(move |_shutdown| verify_heartbeats(shard_clone.clone())) + .spawn(); +} + +async fn verify_heartbeats(shard: Rc) -> Result<(), IggyError> { + trace!("Verifying heartbeats..."); + + // Get the period from config to compute max_interval + let period = shard.config.heartbeat.interval.get_duration(); + let interval = IggyDuration::from(period); + let max_interval = IggyDuration::from((MAX_THRESHOLD * interval.as_micros() as f64) as u64); + + let clients = shard.client_manager.get_clients(); + + let now = IggyTimestamp::now(); + let heartbeat_to = IggyTimestamp::from(now.as_micros() - max_interval.as_micros()); + debug!("Verifying heartbeats at: {now}, max allowed timestamp: {heartbeat_to}"); + + let mut stale_clients = Vec::new(); + for client in clients { + if client.last_heartbeat.as_micros() < heartbeat_to.as_micros() { + warn!( + "Stale client session: {}, last heartbeat at: {}, max allowed timestamp: {heartbeat_to}", + client.session, client.last_heartbeat, + ); + client.session.set_stale(); + stale_clients.push(client.session.client_id); + } else { + debug!( + "Valid heartbeat at: {} for client session: {}, max allowed timestamp: {heartbeat_to}", + client.last_heartbeat, client.session, + ); + } + } + + if stale_clients.is_empty() { + return Ok(()); + } + + let count = stale_clients.len(); + + for client_id in stale_clients { + shard.delete_client(client_id).await; + } + info!("Removed {count} stale clients."); + + Ok(()) +} diff --git a/core/server/src/shard/tasks/periodic/jwt_token_cleaner.rs b/core/server/src/shard/tasks/periodic/jwt_token_cleaner.rs new file mode 100644 index 0000000000..bd9943612d --- /dev/null +++ b/core/server/src/shard/tasks/periodic/jwt_token_cleaner.rs @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::http::shared::AppState; +use crate::shard::IggyShard; +use iggy_common::{IggyError, IggyTimestamp}; +use std::rc::Rc; +use std::sync::Arc; +use std::time::Duration; +use tracing::{error, info, trace}; + +const JWT_TOKENS_CLEANER_PERIOD: Duration = Duration::from_secs(300); + +pub fn spawn_jwt_token_cleaner(shard: Rc, app_state: Arc) { + info!( + "JWT token cleaner is enabled, expired revoked tokens will be deleted every: {} seconds.", + JWT_TOKENS_CLEANER_PERIOD.as_secs() + ); + shard + .task_registry + .periodic("clear_jwt_tokens") + .every(JWT_TOKENS_CLEANER_PERIOD) + .tick(move |_shutdown| clear_jwt_tokens(app_state.clone())) + .spawn(); +} + +async fn clear_jwt_tokens(app_state: Arc) -> Result<(), IggyError> { + trace!("Checking for expired revoked JWT tokens..."); + + let now = IggyTimestamp::now().to_secs(); + + match app_state + .jwt_manager + .delete_expired_revoked_tokens(now) + .await + { + Ok(()) => { + trace!("Successfully cleaned up expired revoked JWT tokens"); + } + Err(err) => { + error!("Failed to delete expired revoked JWT tokens: {}", err); + } + } + + Ok(()) +} diff --git a/core/server/src/shard/tasks/periodic/message_cleaner.rs b/core/server/src/shard/tasks/periodic/message_cleaner.rs new file mode 100644 index 0000000000..a4bac188be --- /dev/null +++ b/core/server/src/shard/tasks/periodic/message_cleaner.rs @@ -0,0 +1,120 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::shard::IggyShard; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; +use iggy_common::IggyError; +use server_common::sharding::IggyNamespace; +use std::rc::Rc; +use tracing::{error, info, trace}; + +pub fn spawn_message_cleaner(shard: Rc) { + if !shard.config.data_maintenance.messages.cleaner_enabled { + info!("Message cleaner is disabled."); + return; + } + + let period = shard + .config + .data_maintenance + .messages + .interval + .get_duration(); + info!( + "Message cleaner is enabled, expired segments will be automatically deleted every: {:?}", + period + ); + let shard_clone = shard.clone(); + shard + .task_registry + .periodic("clean_messages") + .every(period) + .tick(move |_shutdown| clean_messages(shard_clone.clone())) + .spawn(); +} + +/// Groups namespaces by topic and sends a single `CleanTopicMessages` per topic to the pump. +/// All segment inspection and deletion happens inside the pump handler — no TOCTOU. +async fn clean_messages(shard: Rc) -> Result<(), IggyError> { + trace!("Cleaning expired messages..."); + + let namespaces = shard.get_current_shard_namespaces(); + + let mut topics: std::collections::HashMap<(usize, usize), Vec> = + std::collections::HashMap::new(); + + for ns in namespaces { + topics + .entry((ns.stream_id(), ns.topic_id())) + .or_default() + .push(ns.partition_id()); + } + + let mut total_deleted_segments = 0u64; + let mut total_deleted_messages = 0u64; + + for ((stream_id, topic_id), partition_ids) in topics { + let ns = IggyNamespace::new(stream_id, topic_id, partition_ids[0]); + let payload = ShardRequestPayload::CleanTopicMessages { + stream_id, + topic_id, + partition_ids, + }; + let request = ShardRequest::data_plane(ns, payload); + + match shard.send_to_data_plane(request).await { + Ok(ShardResponse::CleanTopicMessages { + deleted_segments, + deleted_messages, + }) => { + if deleted_segments > 0 { + info!( + "Deleted {} segments and {} messages for stream {}, topic {}", + deleted_segments, deleted_messages, stream_id, topic_id + ); + shard.metrics.decrement_segments(deleted_segments as u32); + shard.metrics.decrement_messages(deleted_messages); + total_deleted_segments += deleted_segments; + total_deleted_messages += deleted_messages; + } + } + Ok(ShardResponse::ErrorResponse(err)) => { + error!( + "Failed to clean messages for stream {}, topic {}: {}", + stream_id, topic_id, err + ); + } + Ok(_) => unreachable!("Expected CleanTopicMessages response"), + Err(err) => { + error!( + "Failed to send CleanTopicMessages for stream {}, topic {}: {}", + stream_id, topic_id, err + ); + } + } + } + + if total_deleted_segments > 0 { + info!( + "Total cleaned: {} segments and {} messages", + total_deleted_segments, total_deleted_messages + ); + } + + Ok(()) +} diff --git a/core/server/src/shard/tasks/periodic/message_saver.rs b/core/server/src/shard/tasks/periodic/message_saver.rs new file mode 100644 index 0000000000..f1e973378c --- /dev/null +++ b/core/server/src/shard/tasks/periodic/message_saver.rs @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::shard::IggyShard; +use crate::shard::transmission::frame::ShardResponse; +use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::{error, info, trace}; + +pub fn spawn_message_saver(shard: Rc) { + let period = shard.config.message_saver.interval.get_duration(); + let enforce_fsync = shard.config.message_saver.enforce_fsync; + info!( + "Message saver is enabled, buffered messages will be automatically saved every: {:?}, enforce fsync: {enforce_fsync}.", + period + ); + let shard_clone = shard.clone(); + shard + .task_registry + .periodic("save_messages") + .every(period) + // No last_tick_on_shutdown — the pump handles final flush + fsync + // during its own shutdown (see message_pump.rs). + .tick(move |_shutdown| save_messages(shard_clone.clone())) + .spawn(); +} + +async fn save_messages(shard: Rc) -> Result<(), IggyError> { + trace!("Saving buffered messages..."); + + let namespaces = shard.get_current_shard_namespaces(); + let mut partitions_flushed = 0u32; + + for ns in namespaces { + let payload = ShardRequestPayload::FlushUnsavedBuffer { fsync: false }; + let request = ShardRequest::data_plane(ns, payload); + match shard.send_to_data_plane(request).await { + Ok(ShardResponse::FlushUnsavedBuffer { flushed_count }) if flushed_count > 0 => { + partitions_flushed += 1; + } + Ok(ShardResponse::FlushUnsavedBuffer { .. }) => {} + Ok(ShardResponse::ErrorResponse(err)) => { + error!("Failed to save messages for partition {:?}: {}", ns, err); + } + Err(err) => { + error!("Failed to save messages for partition {:?}: {}", ns, err); + } + _ => {} + } + } + + if partitions_flushed > 0 { + info!("Flushed {partitions_flushed} partitions."); + } + Ok(()) +} diff --git a/core/server/src/shard/tasks/periodic/mod.rs b/core/server/src/shard/tasks/periodic/mod.rs new file mode 100644 index 0000000000..55799697a3 --- /dev/null +++ b/core/server/src/shard/tasks/periodic/mod.rs @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod heartbeat_verifier; +mod jwt_token_cleaner; +mod message_cleaner; +mod message_saver; +mod personal_access_token_cleaner; +mod revocation_timeout; +mod sysinfo_printer; +#[cfg(feature = "systemd")] +mod systemd_watchdog; + +pub use heartbeat_verifier::spawn_heartbeat_verifier; +pub use jwt_token_cleaner::spawn_jwt_token_cleaner; +pub use message_cleaner::spawn_message_cleaner; +pub use message_saver::spawn_message_saver; +pub use personal_access_token_cleaner::spawn_personal_access_token_cleaner; +pub use revocation_timeout::spawn_revocation_timeout_checker; +pub use sysinfo_printer::spawn_sysinfo_printer; +#[cfg(feature = "systemd")] +pub use systemd_watchdog::spawn_systemd_watchdog; diff --git a/core/server/src/shard/tasks/periodic/personal_access_token_cleaner.rs b/core/server/src/shard/tasks/periodic/personal_access_token_cleaner.rs new file mode 100644 index 0000000000..e258929e0e --- /dev/null +++ b/core/server/src/shard/tasks/periodic/personal_access_token_cleaner.rs @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::shard::IggyShard; +use iggy_common::{IggyError, IggyTimestamp}; +use std::rc::Rc; +use tracing::{info, trace}; + +pub fn spawn_personal_access_token_cleaner(shard: Rc) { + if shard.id != 0 { + return; + } + let period = shard + .config + .personal_access_token + .cleaner + .interval + .get_duration(); + info!( + "Personal access token cleaner is enabled, expired tokens will be deleted every: {:?}.", + period + ); + let shard_clone = shard.clone(); + shard + .task_registry + .periodic("clear_personal_access_tokens") + .every(period) + .tick(move |_shutdown| clear_personal_access_tokens(shard_clone.clone())) + .spawn(); +} + +async fn clear_personal_access_tokens(shard: Rc) -> Result<(), IggyError> { + trace!("Checking for expired personal access tokens..."); + + let now = IggyTimestamp::now(); + let mut total_removed = 0; + + let user_ids: Vec = shard + .metadata + .get_all_users() + .iter() + .map(|u| u.id) + .collect(); + + for user_id in user_ids { + let pats = shard.metadata.get_user_personal_access_tokens(user_id); + + let expired_tokens: Vec<_> = pats + .iter() + .filter(|pat| pat.is_expired(now)) + .map(|pat| (pat.name.clone(), pat.token.clone())) + .collect(); + + for (name, token_hash) in expired_tokens { + shard + .writer() + .delete_personal_access_token(user_id, token_hash); + info!( + "Removed expired personal access token '{}' for user ID {}", + name, user_id + ); + total_removed += 1; + } + } + + if total_removed > 0 { + info!("Removed {total_removed} expired personal access tokens"); + } + + Ok(()) +} diff --git a/core/server/src/shard/tasks/periodic/revocation_timeout.rs b/core/server/src/shard/tasks/periodic/revocation_timeout.rs new file mode 100644 index 0000000000..69fa251b1e --- /dev/null +++ b/core/server/src/shard/tasks/periodic/revocation_timeout.rs @@ -0,0 +1,105 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::shard::IggyShard; +use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; +use iggy_common::{IggyError, IggyTimestamp}; +use std::rc::Rc; +use tracing::{info, trace, warn}; + +pub fn spawn_revocation_timeout_checker(shard: Rc) { + let interval = shard.config.consumer_group.rebalancing_check_interval; + let timeout = shard.config.consumer_group.rebalancing_timeout; + info!( + "Pending partition revocations will be checked every: {}. Timeout: {}.", + interval, timeout + ); + let shard_clone = shard.clone(); + shard + .task_registry + .periodic("check_revocation_timeouts") + .every(interval.get_duration()) + .tick(move |_shutdown| check_revocation_timeouts(shard_clone.clone())) + .spawn(); +} + +async fn check_revocation_timeouts(shard: Rc) -> Result<(), IggyError> { + trace!("Checking pending revocation timeouts..."); + + let now = IggyTimestamp::now().as_micros(); + let timeout_micros = shard.config.consumer_group.rebalancing_timeout.as_micros(); + + let timed_out = shard.metadata.with_metadata(|metadata| { + metadata + .streams + .iter() + .flat_map(|(stream_id, stream)| { + stream.topics.iter().flat_map(move |(topic_id, topic)| { + topic.consumer_groups.iter().flat_map(move |(_, group)| { + let group_id = group.id; + group.members.iter().flat_map(move |(slab_id, member)| { + let member_id = member.id; + member + .pending_revocations + .iter() + .filter(move |revocation| { + now.saturating_sub(revocation.created_at_micros) + >= timeout_micros + }) + .map(move |revocation| { + ( + stream_id, + topic_id, + group_id, + slab_id, + member_id, + revocation.partition_id, + ) + }) + }) + }) + }) + }) + .collect::>() + }); + + if timed_out.is_empty() { + return Ok(()); + } + + let count = timed_out.len(); + for (stream_id, topic_id, group_id, member_slab_id, member_id, partition_id) in timed_out { + warn!( + "Force-completing timed out revocation: stream={stream_id}, topic={topic_id}, group={group_id}, \ + member_slab={member_slab_id}, partition={partition_id}", + ); + let request = + ShardRequest::control_plane(ShardRequestPayload::CompletePartitionRevocation { + stream_id, + topic_id, + group_id, + member_slab_id, + member_id, + partition_id, + timed_out: true, + }); + let _ = shard.send_to_control_plane(request).await; + } + info!("Force-completed {count} timed out partition revocations."); + + Ok(()) +} diff --git a/core/server/src/shard/tasks/periodic/sysinfo_printer.rs b/core/server/src/shard/tasks/periodic/sysinfo_printer.rs new file mode 100644 index 0000000000..c70f78e694 --- /dev/null +++ b/core/server/src/shard/tasks/periodic/sysinfo_printer.rs @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::shard::IggyShard; +use human_repr::HumanCount; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::{error, info, trace}; + +pub fn spawn_sysinfo_printer(shard: Rc) { + let period = shard + .config + .system + .logging + .sysinfo_print_interval + .get_duration(); + info!( + "System info logger is enabled, OS info will be printed every: {:?}", + period + ); + let shard_clone = shard.clone(); + shard + .task_registry + .periodic("print_sysinfo") + .every(period) + .tick(move |_shutdown| print_sysinfo(shard_clone.clone())) + .spawn(); +} + +fn get_open_file_descriptors() -> Option { + #[cfg(target_os = "linux")] + { + let pid = std::process::id(); + let fd_path = format!("/proc/{}/fd", pid); + if let Ok(entries) = std::fs::read_dir(&fd_path) { + return Some(entries.count()); + } + } + None +} + +async fn print_sysinfo(shard: Rc) -> Result<(), IggyError> { + trace!("Printing system information..."); + + let stats = match shard.get_stats().await { + Ok(stats) => stats, + Err(e) => { + error!("Failed to get system information. Error: {e}"); + return Ok(()); + } + }; + + let free_memory_percent = (stats.available_memory.as_bytes_u64() as f64 + / stats.total_memory.as_bytes_u64() as f64) + * 100f64; + + let threads_info = if stats.threads_count > 0 { + format!(", Threads: {}", stats.threads_count) + } else { + String::new() + }; + + let open_files_info = if let Some(open_files) = get_open_file_descriptors() { + format!(", OpenFDs: {}", open_files) + } else { + String::new() + }; + + info!( + "CPU: {:.2}%/{:.2}% (IggyUsage/Total), Mem: {:.2}%/{}/{}/{} (Free/IggyUsage/TotalUsed/Total), Disk: {}/{} (Free/Total), IggyUsage: {}, Clients: {}, Messages: {}, Read: {}, Written: {}{}{}", + stats.cpu_usage, + stats.total_cpu_usage, + free_memory_percent, + stats.memory_usage, + stats.total_memory - stats.available_memory, + stats.total_memory, + stats.free_disk_space, + stats.total_disk_space, + stats.messages_size_bytes, + stats.clients_count.human_count_bare().to_string(), + stats.messages_count.human_count_bare().to_string(), + stats.read_bytes, + stats.written_bytes, + threads_info, + open_files_info, + ); + + Ok(()) +} diff --git a/core/server/src/shard/tasks/periodic/systemd_watchdog.rs b/core/server/src/shard/tasks/periodic/systemd_watchdog.rs new file mode 100644 index 0000000000..29aea68da7 --- /dev/null +++ b/core/server/src/shard/tasks/periodic/systemd_watchdog.rs @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::shard::IggyShard; +use crate::shard::systemd; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::info; + +pub fn spawn_systemd_watchdog(shard: Rc) { + let Some(timeout) = sd_notify::watchdog_enabled() else { + return; + }; + + let interval = timeout / 2; + info!( + "Systemd watchdog enabled, pinging every {}s (timeout: {}s).", + interval.as_secs(), + timeout.as_secs() + ); + + shard + .task_registry + .periodic("systemd_watchdog") + .every(interval) + .tick(move |_shutdown| ping_watchdog()) + .spawn(); +} + +async fn ping_watchdog() -> Result<(), IggyError> { + systemd::ping_watchdog(); + Ok(()) +} diff --git a/core/server/src/shard/transmission/connector.rs b/core/server/src/shard/transmission/connector.rs new file mode 100644 index 0000000000..309d9fe0bd --- /dev/null +++ b/core/server/src/shard/transmission/connector.rs @@ -0,0 +1,106 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::{ + frame::{ShardFrame, ShardResponse}, + message::ShardMessage, +}; +use iggy_common::IggyError; +use tracing::error; + +pub type StopSender = async_channel::Sender<()>; +pub type StopReceiver = async_channel::Receiver<()>; + +/// Inter-shard communication channel +pub struct ShardConnector { + pub id: u16, + pub sender: flume::Sender, + pub receiver: Receiver, + pub stop_receiver: StopReceiver, + pub stop_sender: StopSender, +} + +impl Clone for ShardConnector { + fn clone(&self) -> Self { + Self { + id: self.id, + sender: self.sender.clone(), + receiver: self.receiver.clone(), + stop_receiver: self.stop_receiver.clone(), + stop_sender: self.stop_sender.clone(), + } + } +} + +impl ShardConnector { + /// Creates a new shard connector with unbounded capacity. + pub fn new(id: u16) -> Self { + let (sender, receiver) = flume::unbounded(); + let (stop_sender, stop_receiver) = async_channel::bounded(1); + Self { + id, + sender, + receiver: Receiver::new(receiver), + stop_receiver, + stop_sender, + } + } + + /// Sends a message to this shard. + /// + /// For unbounded channels, this operation is infallible and never blocks. + pub fn send(&self, data: T) { + let _ = self.sender.send(data); + } +} + +impl ShardConnector { + /// Sends a request and waits for a response. + /// This implements the request-response pattern for inter-shard communication. + pub async fn send_request(&self, message: ShardMessage) -> Result { + let (sender, receiver) = async_channel::bounded(1); + // Note: sender needs to be passed to ShardFrame to keep the channel open + self.send(ShardFrame::new(message, Some(sender))); + + receiver.recv().await.map_err(|err| { + error!("Failed to receive response from shard {}: {err}", self.id); + IggyError::ShardCommunicationError + }) + } +} + +/// Wrapper around flume's Receiver that provides Clone capability. +/// +/// This wraps the flume receiver to allow cloning while still providing +/// access to the underlying receiver for direct use. +pub struct Receiver { + pub inner: flume::Receiver, +} + +impl Receiver { + fn new(receiver: flume::Receiver) -> Self { + Self { inner: receiver } + } +} + +impl Clone for Receiver { + fn clone(&self) -> Self { + Self { + inner: self.inner.clone(), + } + } +} diff --git a/core/server/src/shard/transmission/event.rs b/core/server/src/shard/transmission/event.rs new file mode 100644 index 0000000000..b243e26dd2 --- /dev/null +++ b/core/server/src/shard/transmission/event.rs @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use iggy_common::{Identifier, IggyTimestamp, TransportProtocol}; +use std::net::SocketAddr; +use strum::Display; + +/// Minimal partition info for event broadcasting (no slab dependency) +#[derive(Debug, Clone)] +pub struct PartitionInfo { + pub id: usize, + pub created_at: IggyTimestamp, +} + +/// Events that require broadcasting between shards. +/// +/// Note: Metadata events (CreatedStream, DeletedStream, CreatedTopic, DeletedTopic, +/// UpdatedStream, UpdatedTopic, CreatedConsumerGroup, DeletedConsumerGroup) are NOT +/// broadcast because SharedMetadata is already visible to all shards via LeftRight. +/// Only events that require per-shard local actions are broadcast. +#[derive(Debug, Clone, Display)] +#[strum(serialize_all = "PascalCase")] +pub enum ShardEvent { + /// Flush unsaved buffer to disk for a specific partition + FlushUnsavedBuffer { + stream_id: Identifier, + topic_id: Identifier, + partition_id: usize, + fsync: bool, + }, + /// Purge all messages, consumer groups and consumer group offsets from a topic + PurgedTopic { + stream_id: Identifier, + topic_id: Identifier, + }, + /// Purges all topics in a stream + PurgedStream { stream_id: Identifier }, + /// New partitions created (requires per-shard log initialization) + CreatedPartitions { + stream_id: Identifier, + topic_id: Identifier, + partitions: Vec, + }, + /// Partitions deleted (requires per-shard log cleanup) + DeletedPartitions { + stream_id: Identifier, + topic_id: Identifier, + partitions_count: u32, + partition_ids: Vec, + }, + /// Transport address bound (for config file writing) + AddressBound { + protocol: TransportProtocol, + address: SocketAddr, + }, +} diff --git a/core/server/src/shard/transmission/frame.rs b/core/server/src/shard/transmission/frame.rs new file mode 100644 index 0000000000..fb77565deb --- /dev/null +++ b/core/server/src/shard/transmission/frame.rs @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::{ + shard::transmission::message::ShardMessage, + streaming::{segments::IggyMessagesBatchSet, users::user::User}, +}; +use async_channel::Sender; +use iggy_common::{ + CompressionAlgorithm, IggyError, IggyExpiry, IggyPollMetadata, IggyTimestamp, MaxTopicSize, + PersonalAccessToken, Stats, +}; +use std::sync::Arc; + +/// Data needed to construct a stream creation response. +#[derive(Debug)] +pub struct StreamResponseData { + pub id: u32, + pub name: Arc, + pub created_at: IggyTimestamp, +} + +/// Data needed to construct a topic creation response. +#[derive(Debug)] +pub struct TopicResponseData { + pub id: u32, + pub name: Arc, + pub created_at: IggyTimestamp, + pub partitions: Vec, + pub message_expiry: IggyExpiry, + pub compression_algorithm: CompressionAlgorithm, + pub max_topic_size: MaxTopicSize, + pub replication_factor: u8, +} + +/// Data needed to construct a consumer group creation response. +#[derive(Debug)] +pub struct ConsumerGroupResponseData { + pub id: u32, + pub name: Arc, + pub partitions_count: u32, +} + +// TODO: make nice types in common module so that each command has respective *Response struct, i.e. CreateStream -> CreateStreamResponse +#[derive(Debug)] +pub enum ShardResponse { + PollMessages((IggyPollMetadata, IggyMessagesBatchSet)), + SendMessages, + FlushUnsavedBuffer { + flushed_count: u32, + }, + DeleteSegments { + deleted_segments: u64, + deleted_messages: u64, + }, + CleanTopicMessages { + deleted_segments: u64, + deleted_messages: u64, + }, + Event, + CreateStreamResponse(StreamResponseData), + DeleteStreamResponse, + CreateTopicResponse(TopicResponseData), + UpdateTopicResponse, + DeleteTopicResponse, + CreateUserResponse(User), + DeleteUserResponse(User), + GetStatsResponse(Stats), + CreatePartitionsResponse, + DeletePartitionsResponse, + UpdateStreamResponse, + SocketTransferResponse, + UpdatePermissionsResponse, + ChangePasswordResponse, + UpdateUserResponse(User), + CreateConsumerGroupResponse(ConsumerGroupResponseData), + JoinConsumerGroupResponse, + LeaveConsumerGroupResponse, + DeleteConsumerGroupResponse, + CreatePersonalAccessTokenResponse(PersonalAccessToken, String), + DeletePersonalAccessTokenResponse, + LeaveConsumerGroupMetadataOnlyResponse, + CompletePartitionRevocationResponse, + PurgeStreamResponse, + PurgeTopicResponse, + ErrorResponse(IggyError), +} + +#[derive(Debug)] +pub struct ShardFrame { + pub message: ShardMessage, + pub response_sender: Option>, +} + +impl ShardFrame { + pub fn new(message: ShardMessage, response_sender: Option>) -> Self { + Self { + message, + response_sender, + } + } +} diff --git a/core/server/src/shard/transmission/message.rs b/core/server/src/shard/transmission/message.rs new file mode 100644 index 0000000000..f1acf9777f --- /dev/null +++ b/core/server/src/shard/transmission/message.rs @@ -0,0 +1,254 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::{ + shard::{system::messages::PollingArgs, transmission::event::ShardEvent}, + streaming::{polling_consumer::PollingConsumer, segments::IggyMessagesBatchMut}, +}; +use iggy_binary_protocol::requests::{ + consumer_groups::*, partitions::*, personal_access_tokens::*, streams::*, topics::*, users::*, +}; +use server_common::sharding::IggyNamespace; + +use std::{net::SocketAddr, os::fd::OwnedFd}; + +/// Resolved stream ID. Contains only the numeric ID - `Identifier` stays at handler boundary. +#[derive(Debug, Clone, Copy)] +pub struct ResolvedStream(pub usize); + +impl ResolvedStream { + pub fn id(self) -> usize { + self.0 + } +} + +/// Resolved topic with parent stream context. +#[derive(Debug, Clone, Copy)] +pub struct ResolvedTopic { + pub stream_id: usize, + pub topic_id: usize, +} + +/// Resolved partition with full context. +#[derive(Debug, Clone, Copy)] +pub struct ResolvedPartition { + pub stream_id: usize, + pub topic_id: usize, + pub partition_id: usize, +} + +/// Resolved consumer group with full context. +#[derive(Debug, Clone, Copy)] +pub struct ResolvedConsumerGroup { + pub stream_id: usize, + pub topic_id: usize, + pub group_id: usize, +} + +#[allow(clippy::large_enum_variant)] +#[derive(Debug)] +pub enum ShardMessage { + Request(ShardRequest), + Event(ShardEvent), +} + +/// Routing envelope determining which shard handles the request. +#[derive(Debug)] +pub struct ShardRequest { + /// None = shard 0 (control-plane), Some = partition owner (data-plane) + pub routing: Option, + pub payload: ShardRequestPayload, +} + +impl ShardRequest { + /// Control-plane operations always route to shard 0 + pub fn control_plane(payload: ShardRequestPayload) -> Self { + Self { + routing: None, + payload, + } + } + + /// Data-plane operations route by partition namespace + pub fn data_plane(namespace: IggyNamespace, payload: ShardRequestPayload) -> Self { + Self { + routing: Some(namespace), + payload, + } + } +} + +#[derive(Debug)] +pub enum ShardRequestPayload { + // Data-plane operations: namespace provided via ShardRequest + SendMessages { + batch: IggyMessagesBatchMut, + }, + PollMessages { + consumer: PollingConsumer, + args: PollingArgs, + }, + FlushUnsavedBuffer { + fsync: bool, + }, + DeleteSegments { + segments_count: u32, + }, + CleanTopicMessages { + stream_id: usize, + topic_id: usize, + partition_ids: Vec, + }, + SocketTransfer { + fd: OwnedFd, + from_shard: u16, + client_id: u32, + user_id: u32, + address: SocketAddr, + initial_data: IggyMessagesBatchMut, + }, + + // Control-plane: stream operations + CreateStreamRequest { + user_id: u32, + command: CreateStreamRequest, + }, + UpdateStreamRequest { + user_id: u32, + command: UpdateStreamRequest, + }, + DeleteStreamRequest { + user_id: u32, + command: DeleteStreamRequest, + }, + PurgeStreamRequest { + user_id: u32, + command: PurgeStreamRequest, + }, + + // Control-plane: topic operations + CreateTopicRequest { + user_id: u32, + command: CreateTopicRequest, + }, + UpdateTopicRequest { + user_id: u32, + command: UpdateTopicRequest, + }, + DeleteTopicRequest { + user_id: u32, + command: DeleteTopicRequest, + }, + PurgeTopicRequest { + user_id: u32, + command: PurgeTopicRequest, + }, + + // Control-plane: partition operations + CreatePartitionsRequest { + user_id: u32, + command: CreatePartitionsRequest, + }, + DeletePartitionsRequest { + user_id: u32, + command: DeletePartitionsRequest, + }, + + // Control-plane: user operations + CreateUserRequest { + user_id: u32, + command: CreateUserRequest, + }, + UpdateUserRequest { + user_id: u32, + command: UpdateUserRequest, + }, + DeleteUserRequest { + user_id: u32, + command: DeleteUserRequest, + }, + UpdatePermissionsRequest { + user_id: u32, + command: UpdatePermissionsRequest, + }, + ChangePasswordRequest { + user_id: u32, + command: ChangePasswordRequest, + }, + + // Control-plane: consumer group operations + CreateConsumerGroupRequest { + user_id: u32, + command: CreateConsumerGroupRequest, + }, + DeleteConsumerGroupRequest { + user_id: u32, + command: DeleteConsumerGroupRequest, + }, + JoinConsumerGroupRequest { + user_id: u32, + client_id: u32, + command: JoinConsumerGroupRequest, + }, + LeaveConsumerGroupRequest { + user_id: u32, + client_id: u32, + command: LeaveConsumerGroupRequest, + }, + LeaveConsumerGroupMetadataOnly { + stream_id: usize, + topic_id: usize, + group_id: usize, + client_id: u32, + }, + CompletePartitionRevocation { + stream_id: usize, + topic_id: usize, + group_id: usize, + member_slab_id: usize, + member_id: usize, + partition_id: usize, + timed_out: bool, + }, + + // Control-plane: PAT operations + CreatePersonalAccessTokenRequest { + user_id: u32, + command: CreatePersonalAccessTokenRequest, + }, + DeletePersonalAccessTokenRequest { + user_id: u32, + command: DeletePersonalAccessTokenRequest, + }, + + // Control-plane: stats + GetStats { + user_id: u32, + }, +} + +impl From for ShardMessage { + fn from(request: ShardRequest) -> Self { + ShardMessage::Request(request) + } +} + +impl From for ShardMessage { + fn from(event: ShardEvent) -> Self { + ShardMessage::Event(event) + } +} diff --git a/core/server/src/shard/transmission/mod.rs b/core/server/src/shard/transmission/mod.rs new file mode 100644 index 0000000000..cc326cb27d --- /dev/null +++ b/core/server/src/shard/transmission/mod.rs @@ -0,0 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod connector; +pub mod event; +pub mod frame; +pub mod message; diff --git a/core/server/src/state/command.rs b/core/server/src/state/command.rs new file mode 100644 index 0000000000..9138df06d7 --- /dev/null +++ b/core/server/src/state/command.rs @@ -0,0 +1,268 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::state::models::{ + CreateConsumerGroupWithId, CreatePersonalAccessTokenWithHash, CreateStreamWithId, + CreateTopicWithId, CreateUserWithId, +}; +use bytes::{BufMut, BytesMut}; +use iggy_binary_protocol::codes::{ + CHANGE_PASSWORD_CODE, CREATE_CONSUMER_GROUP_CODE, CREATE_PARTITIONS_CODE, + CREATE_PERSONAL_ACCESS_TOKEN_CODE, CREATE_STREAM_CODE, CREATE_TOPIC_CODE, CREATE_USER_CODE, + DELETE_CONSUMER_GROUP_CODE, DELETE_PARTITIONS_CODE, DELETE_PERSONAL_ACCESS_TOKEN_CODE, + DELETE_SEGMENTS_CODE, DELETE_STREAM_CODE, DELETE_TOPIC_CODE, DELETE_USER_CODE, + PURGE_STREAM_CODE, PURGE_TOPIC_CODE, UPDATE_PERMISSIONS_CODE, UPDATE_STREAM_CODE, + UPDATE_TOPIC_CODE, UPDATE_USER_CODE, +}; +use iggy_binary_protocol::requests::{ + consumer_groups::DeleteConsumerGroupRequest, + partitions::{CreatePartitionsRequest, DeletePartitionsRequest}, + personal_access_tokens::DeletePersonalAccessTokenRequest, + segments::DeleteSegmentsRequest, + streams::{DeleteStreamRequest, PurgeStreamRequest, UpdateStreamRequest}, + topics::{DeleteTopicRequest, PurgeTopicRequest, UpdateTopicRequest}, + users::{ + ChangePasswordRequest, DeleteUserRequest, UpdatePermissionsRequest, UpdateUserRequest, + }, +}; +use iggy_binary_protocol::{WireDecode, WireEncode, WireError}; +use std::fmt::{Display, Formatter}; + +#[derive(Debug)] +pub enum EntryCommand { + CreateStream(CreateStreamWithId), + UpdateStream(UpdateStreamRequest), + DeleteStream(DeleteStreamRequest), + PurgeStream(PurgeStreamRequest), + CreateTopic(CreateTopicWithId), + UpdateTopic(UpdateTopicRequest), + DeleteTopic(DeleteTopicRequest), + PurgeTopic(PurgeTopicRequest), + CreatePartitions(CreatePartitionsRequest), + DeletePartitions(DeletePartitionsRequest), + DeleteSegments(DeleteSegmentsRequest), + CreateConsumerGroup(CreateConsumerGroupWithId), + DeleteConsumerGroup(DeleteConsumerGroupRequest), + CreateUser(CreateUserWithId), + UpdateUser(UpdateUserRequest), + DeleteUser(DeleteUserRequest), + ChangePassword(ChangePasswordRequest), + UpdatePermissions(UpdatePermissionsRequest), + CreatePersonalAccessToken(CreatePersonalAccessTokenWithHash), + DeletePersonalAccessToken(DeletePersonalAccessTokenRequest), +} + +impl WireEncode for EntryCommand { + fn encoded_size(&self) -> usize { + let inner_size = match self { + EntryCommand::CreateStream(cmd) => cmd.encoded_size(), + EntryCommand::UpdateStream(cmd) => cmd.encoded_size(), + EntryCommand::DeleteStream(cmd) => cmd.encoded_size(), + EntryCommand::PurgeStream(cmd) => cmd.encoded_size(), + EntryCommand::CreateTopic(cmd) => cmd.encoded_size(), + EntryCommand::UpdateTopic(cmd) => cmd.encoded_size(), + EntryCommand::DeleteTopic(cmd) => cmd.encoded_size(), + EntryCommand::PurgeTopic(cmd) => cmd.encoded_size(), + EntryCommand::CreatePartitions(cmd) => cmd.encoded_size(), + EntryCommand::DeletePartitions(cmd) => cmd.encoded_size(), + EntryCommand::DeleteSegments(cmd) => cmd.encoded_size(), + EntryCommand::CreateConsumerGroup(cmd) => cmd.encoded_size(), + EntryCommand::DeleteConsumerGroup(cmd) => cmd.encoded_size(), + EntryCommand::CreateUser(cmd) => cmd.encoded_size(), + EntryCommand::UpdateUser(cmd) => cmd.encoded_size(), + EntryCommand::DeleteUser(cmd) => cmd.encoded_size(), + EntryCommand::ChangePassword(cmd) => cmd.encoded_size(), + EntryCommand::UpdatePermissions(cmd) => cmd.encoded_size(), + EntryCommand::CreatePersonalAccessToken(cmd) => cmd.encoded_size(), + EntryCommand::DeletePersonalAccessToken(cmd) => cmd.encoded_size(), + }; + 4 + 4 + inner_size + } + + fn encode(&self, buf: &mut BytesMut) { + let (code, inner_size) = match self { + EntryCommand::CreateStream(cmd) => (CREATE_STREAM_CODE, cmd.encoded_size()), + EntryCommand::UpdateStream(cmd) => (UPDATE_STREAM_CODE, cmd.encoded_size()), + EntryCommand::DeleteStream(cmd) => (DELETE_STREAM_CODE, cmd.encoded_size()), + EntryCommand::PurgeStream(cmd) => (PURGE_STREAM_CODE, cmd.encoded_size()), + EntryCommand::CreateTopic(cmd) => (CREATE_TOPIC_CODE, cmd.encoded_size()), + EntryCommand::UpdateTopic(cmd) => (UPDATE_TOPIC_CODE, cmd.encoded_size()), + EntryCommand::DeleteTopic(cmd) => (DELETE_TOPIC_CODE, cmd.encoded_size()), + EntryCommand::PurgeTopic(cmd) => (PURGE_TOPIC_CODE, cmd.encoded_size()), + EntryCommand::CreatePartitions(cmd) => (CREATE_PARTITIONS_CODE, cmd.encoded_size()), + EntryCommand::DeletePartitions(cmd) => (DELETE_PARTITIONS_CODE, cmd.encoded_size()), + EntryCommand::DeleteSegments(cmd) => (DELETE_SEGMENTS_CODE, cmd.encoded_size()), + EntryCommand::CreateConsumerGroup(cmd) => { + (CREATE_CONSUMER_GROUP_CODE, cmd.encoded_size()) + } + EntryCommand::DeleteConsumerGroup(cmd) => { + (DELETE_CONSUMER_GROUP_CODE, cmd.encoded_size()) + } + EntryCommand::CreateUser(cmd) => (CREATE_USER_CODE, cmd.encoded_size()), + EntryCommand::UpdateUser(cmd) => (UPDATE_USER_CODE, cmd.encoded_size()), + EntryCommand::DeleteUser(cmd) => (DELETE_USER_CODE, cmd.encoded_size()), + EntryCommand::ChangePassword(cmd) => (CHANGE_PASSWORD_CODE, cmd.encoded_size()), + EntryCommand::UpdatePermissions(cmd) => (UPDATE_PERMISSIONS_CODE, cmd.encoded_size()), + EntryCommand::CreatePersonalAccessToken(cmd) => { + (CREATE_PERSONAL_ACCESS_TOKEN_CODE, cmd.encoded_size()) + } + EntryCommand::DeletePersonalAccessToken(cmd) => { + (DELETE_PERSONAL_ACCESS_TOKEN_CODE, cmd.encoded_size()) + } + }; + buf.put_u32_le(code); + buf.put_u32_le(inner_size as u32); + match self { + EntryCommand::CreateStream(cmd) => cmd.encode(buf), + EntryCommand::UpdateStream(cmd) => cmd.encode(buf), + EntryCommand::DeleteStream(cmd) => cmd.encode(buf), + EntryCommand::PurgeStream(cmd) => cmd.encode(buf), + EntryCommand::CreateTopic(cmd) => cmd.encode(buf), + EntryCommand::UpdateTopic(cmd) => cmd.encode(buf), + EntryCommand::DeleteTopic(cmd) => cmd.encode(buf), + EntryCommand::PurgeTopic(cmd) => cmd.encode(buf), + EntryCommand::CreatePartitions(cmd) => cmd.encode(buf), + EntryCommand::DeletePartitions(cmd) => cmd.encode(buf), + EntryCommand::DeleteSegments(cmd) => cmd.encode(buf), + EntryCommand::CreateConsumerGroup(cmd) => cmd.encode(buf), + EntryCommand::DeleteConsumerGroup(cmd) => cmd.encode(buf), + EntryCommand::CreateUser(cmd) => cmd.encode(buf), + EntryCommand::UpdateUser(cmd) => cmd.encode(buf), + EntryCommand::DeleteUser(cmd) => cmd.encode(buf), + EntryCommand::ChangePassword(cmd) => cmd.encode(buf), + EntryCommand::UpdatePermissions(cmd) => cmd.encode(buf), + EntryCommand::CreatePersonalAccessToken(cmd) => cmd.encode(buf), + EntryCommand::DeletePersonalAccessToken(cmd) => cmd.encode(buf), + } + } +} + +impl WireDecode for EntryCommand { + fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> { + if buf.len() < 8 { + return Err(WireError::UnexpectedEof { + offset: 0, + need: 8, + have: buf.len(), + }); + } + let code = u32::from_le_bytes(buf[0..4].try_into().unwrap()); + let length = u32::from_le_bytes(buf[4..8].try_into().unwrap()) as usize; + if buf.len() < 8 + length { + return Err(WireError::UnexpectedEof { + offset: 8, + need: length, + have: buf.len() - 8, + }); + } + let payload = &buf[8..8 + length]; + let consumed = 8 + length; + let cmd = match code { + CREATE_STREAM_CODE => { + EntryCommand::CreateStream(CreateStreamWithId::decode_from(payload)?) + } + UPDATE_STREAM_CODE => { + EntryCommand::UpdateStream(UpdateStreamRequest::decode_from(payload)?) + } + DELETE_STREAM_CODE => { + EntryCommand::DeleteStream(DeleteStreamRequest::decode_from(payload)?) + } + PURGE_STREAM_CODE => { + EntryCommand::PurgeStream(PurgeStreamRequest::decode_from(payload)?) + } + CREATE_TOPIC_CODE => { + EntryCommand::CreateTopic(CreateTopicWithId::decode_from(payload)?) + } + UPDATE_TOPIC_CODE => { + EntryCommand::UpdateTopic(UpdateTopicRequest::decode_from(payload)?) + } + DELETE_TOPIC_CODE => { + EntryCommand::DeleteTopic(DeleteTopicRequest::decode_from(payload)?) + } + PURGE_TOPIC_CODE => EntryCommand::PurgeTopic(PurgeTopicRequest::decode_from(payload)?), + CREATE_PARTITIONS_CODE => { + EntryCommand::CreatePartitions(CreatePartitionsRequest::decode_from(payload)?) + } + DELETE_PARTITIONS_CODE => { + EntryCommand::DeletePartitions(DeletePartitionsRequest::decode_from(payload)?) + } + DELETE_SEGMENTS_CODE => { + EntryCommand::DeleteSegments(DeleteSegmentsRequest::decode_from(payload)?) + } + CREATE_CONSUMER_GROUP_CODE => { + EntryCommand::CreateConsumerGroup(CreateConsumerGroupWithId::decode_from(payload)?) + } + DELETE_CONSUMER_GROUP_CODE => { + EntryCommand::DeleteConsumerGroup(DeleteConsumerGroupRequest::decode_from(payload)?) + } + CREATE_USER_CODE => EntryCommand::CreateUser(CreateUserWithId::decode_from(payload)?), + UPDATE_USER_CODE => EntryCommand::UpdateUser(UpdateUserRequest::decode_from(payload)?), + DELETE_USER_CODE => EntryCommand::DeleteUser(DeleteUserRequest::decode_from(payload)?), + CHANGE_PASSWORD_CODE => { + EntryCommand::ChangePassword(ChangePasswordRequest::decode_from(payload)?) + } + UPDATE_PERMISSIONS_CODE => { + EntryCommand::UpdatePermissions(UpdatePermissionsRequest::decode_from(payload)?) + } + CREATE_PERSONAL_ACCESS_TOKEN_CODE => EntryCommand::CreatePersonalAccessToken( + CreatePersonalAccessTokenWithHash::decode_from(payload)?, + ), + DELETE_PERSONAL_ACCESS_TOKEN_CODE => EntryCommand::DeletePersonalAccessToken( + DeletePersonalAccessTokenRequest::decode_from(payload)?, + ), + _ => return Err(WireError::UnknownCommand(code)), + }; + Ok((cmd, consumed)) + } +} + +impl Display for EntryCommand { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + EntryCommand::CreateStream(command) => write!(f, "CreateStream({command})"), + EntryCommand::UpdateStream(command) => write!(f, "UpdateStream({command:?})"), + EntryCommand::DeleteStream(command) => write!(f, "DeleteStream({command:?})"), + EntryCommand::PurgeStream(command) => write!(f, "PurgeStream({command:?})"), + EntryCommand::CreateTopic(command) => write!(f, "CreateTopic({command})"), + EntryCommand::UpdateTopic(command) => write!(f, "UpdateTopic({command:?})"), + EntryCommand::DeleteTopic(command) => write!(f, "DeleteTopic({command:?})"), + EntryCommand::PurgeTopic(command) => write!(f, "PurgeTopic({command:?})"), + EntryCommand::CreatePartitions(command) => write!(f, "CreatePartitions({command:?})"), + EntryCommand::DeletePartitions(command) => write!(f, "DeletePartitions({command:?})"), + EntryCommand::DeleteSegments(command) => write!(f, "DeleteSegments({command:?})"), + EntryCommand::CreateConsumerGroup(command) => { + write!(f, "CreateConsumerGroup({command})") + } + EntryCommand::DeleteConsumerGroup(command) => { + write!(f, "DeleteConsumerGroup({command:?})") + } + EntryCommand::CreateUser(command) => write!(f, "CreateUser({command})"), + EntryCommand::UpdateUser(command) => write!(f, "UpdateUser({command:?})"), + EntryCommand::DeleteUser(command) => write!(f, "DeleteUser({command:?})"), + EntryCommand::ChangePassword(command) => write!(f, "ChangePassword({command:?})"), + EntryCommand::UpdatePermissions(command) => { + write!(f, "UpdatePermissions({command:?})") + } + EntryCommand::CreatePersonalAccessToken(command) => { + write!(f, "CreatePersonalAccessToken({command})") + } + EntryCommand::DeletePersonalAccessToken(command) => { + write!(f, "DeletePersonalAccessToken({command:?})") + } + } + } +} diff --git a/core/server/src/state/entry.rs b/core/server/src/state/entry.rs new file mode 100644 index 0000000000..c4905a6dbe --- /dev/null +++ b/core/server/src/state/entry.rs @@ -0,0 +1,150 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::state::command::EntryCommand; +use bytes::{BufMut, Bytes, BytesMut}; +use iggy_binary_protocol::{WireDecode, WireEncode}; +use iggy_common::IggyError; +use iggy_common::IggyTimestamp; +use iggy_common::calculate_checksum; +use std::fmt::{Display, Formatter}; + +/// State entry in the log +/// - `index` - Index (operation number) of the entry in the log +/// - `term` - Election term (view number) for replication +/// - `leader_id` - Leader ID for replication +/// - `version` - Server version based on semver as number e.g. 1.234.567 -> 1234567 +/// - `flags` - Reserved for future use +/// - `timestamp` - Timestamp when the command was issued +/// - `user_id` - User ID of the user who issued the command +/// - `checksum` - Checksum of the entry +/// - `code` - Command code +/// - `command` - Payload of the command +/// - `context` - Optional context e.g. used to enrich the payload with additional data +#[derive(Debug)] +pub struct StateEntry { + pub index: u64, + pub term: u64, + pub leader_id: u32, + pub version: u32, + pub flags: u64, + pub timestamp: IggyTimestamp, + pub user_id: u32, + pub checksum: u64, + pub context: Bytes, + pub command: Bytes, +} + +impl StateEntry { + #[allow(clippy::too_many_arguments)] + pub fn new( + index: u64, + term: u64, + leader_id: u32, + version: u32, + flags: u64, + timestamp: IggyTimestamp, + user_id: u32, + checksum: u64, + context: Bytes, + command: Bytes, + ) -> Self { + Self { + index, + term, + leader_id, + version, + flags, + timestamp, + user_id, + checksum, + context, + command, + } + } + + pub fn command(&self) -> Result { + EntryCommand::decode_from(&self.command).map_err(|e| { + tracing::warn!("wire decode error during WAL replay: {e}"); + IggyError::InvalidCommand + }) + } + + #[allow(clippy::too_many_arguments)] + pub fn calculate_checksum( + index: u64, + term: u64, + leader_id: u32, + version: u32, + flags: u64, + timestamp: IggyTimestamp, + user_id: u32, + context: &Bytes, + command: &Bytes, + ) -> u64 { + let mut bytes = + BytesMut::with_capacity(8 + 8 + 4 + 4 + 8 + 8 + 4 + 4 + context.len() + command.len()); + bytes.put_u64_le(index); + bytes.put_u64_le(term); + bytes.put_u32_le(leader_id); + bytes.put_u32_le(version); + bytes.put_u64_le(flags); + bytes.put_u64_le(timestamp.into()); + bytes.put_u32_le(user_id); + bytes.put_u32_le(context.len() as u32); + bytes.put_slice(context); + bytes.extend(command); + calculate_checksum(&bytes.freeze()) + } +} + +impl Display for StateEntry { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "StateEntry {{ index: {}, term: {}, leader ID: {}, version: {}, flags: {}, timestamp: {}, user ID: {}, checksum: {} }}", + self.index, + self.term, + self.leader_id, + self.version, + self.flags, + self.timestamp, + self.user_id, + self.checksum, + ) + } +} + +impl WireEncode for StateEntry { + fn encoded_size(&self) -> usize { + 8 + 8 + 4 + 4 + 8 + 8 + 4 + 8 + 4 + self.context.len() + self.command.len() + } + + fn encode(&self, buf: &mut BytesMut) { + buf.put_u64_le(self.index); + buf.put_u64_le(self.term); + buf.put_u32_le(self.leader_id); + buf.put_u32_le(self.version); + buf.put_u64_le(self.flags); + buf.put_u64_le(self.timestamp.into()); + buf.put_u32_le(self.user_id); + buf.put_u64_le(self.checksum); + buf.put_u32_le(self.context.len() as u32); + buf.put_slice(&self.context); + buf.extend_from_slice(&self.command); + } +} diff --git a/core/server/src/state/file.rs b/core/server/src/state/file.rs new file mode 100644 index 0000000000..ff710f415d --- /dev/null +++ b/core/server/src/state/file.rs @@ -0,0 +1,378 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::state::command::EntryCommand; +use crate::state::{COMPONENT, StateEntry}; +use crate::streaming::persistence::persister::PersisterKind; +use crate::streaming::utils::file; +use bytes::{Buf, BufMut, Bytes, BytesMut}; +use compio::io::AsyncReadExt; +use err_trail::ErrContext; +use iggy_binary_protocol::{WireDecode, WireEncode}; +use iggy_common::EncryptorKind; +use iggy_common::IggyByteSize; +use iggy_common::IggyError; +use iggy_common::IggyTimestamp; +use iggy_common::SemanticVersion; +use std::fmt::Debug; +use std::path::Path; +use std::sync::Arc; +use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +use tracing::{debug, error, info}; + +pub const BUF_CURSOR_CAPACITY_BYTES: usize = 512 * 1000; +const FILE_STATE_PARSE_ERROR: &str = "STATE - failed to parse file state"; + +#[derive(Debug)] +pub struct FileState { + current_index: Arc, + entries_count: Arc, + current_leader: Arc, + term: Arc, + version: u32, + path: String, + persister: Arc, + encryptor: Option, +} + +impl FileState { + #[allow(clippy::too_many_arguments)] + pub fn new( + path: &str, + version: &SemanticVersion, + persister: Arc, + encryptor: Option, + current_index: Arc, + entries_count: Arc, + current_leader: Arc, + term: Arc, + ) -> Self { + Self { + current_index, + entries_count, + current_leader, + term, + path: path.into(), + persister, + encryptor, + version: version.get_numeric_version().expect("Invalid version"), + } + } + + pub fn current_index(&self) -> u64 { + self.current_index.load(Ordering::SeqCst) + } + + pub fn entries_count(&self) -> u64 { + self.entries_count.load(Ordering::SeqCst) + } + + pub fn term(&self) -> u64 { + self.term.load(Ordering::SeqCst) + } + + pub async fn init(&self) -> Result, IggyError> { + assert!(Path::new(&self.path).exists()); + + let entries = self + .load_entries() + .await + .error(|e: &IggyError| format!("{COMPONENT} (error: {e}) - failed to load entries"))?; + let entries_count = entries.len() as u64; + self.entries_count.store(entries_count, Ordering::SeqCst); + if entries_count == 0 { + self.current_index.store(0, Ordering::SeqCst); + } else { + let last_index = entries[entries_count as usize - 1].index; + self.current_index.store(last_index, Ordering::SeqCst); + } + + Ok(entries) + } + + pub async fn load_entries(&self) -> Result, IggyError> { + if !Path::new(&self.path).exists() { + return Err(IggyError::StateFileNotFound); + } + + let file = file::open(&self.path) + .await + .error(|e: &std::io::Error| { + format!( + "{COMPONENT} (error: {e}) - failed to open state file, path: {}", + self.path + ) + }) + .map_err(|_| IggyError::CannotReadFile)?; + let file_size = file + .metadata() + .await + .error(|e: &std::io::Error| { + format!( + "{COMPONENT} (error: {e}) - failed to load state file metadata, path: {}", + self.path + ) + }) + .map_err(|_| IggyError::CannotReadFileMetadata)? + .len(); + + if file_size == 0 { + info!("State file is empty"); + return Ok(Vec::new()); + } + + info!( + "Loading state, file size: {}", + IggyByteSize::from(file_size).as_human_string() + ); + let mut entries = Vec::new(); + let mut total_size: u64 = 0; + let mut cursor = std::io::Cursor::new(file); + let mut current_index = 0; + let mut entries_count = 0; + loop { + let index = cursor + .read_u64_le() + .await + .error(|e: &std::io::Error| format!("{FILE_STATE_PARSE_ERROR} index. {e}")) + .map_err(|_| IggyError::InvalidNumberEncoding)?; + total_size += 8; + // Greater than one, because one of the entries after a fresh reboot is the default root user. + if entries_count > 1 && index != current_index + 1 { + error!( + "State file is corrupted, expected index: {}, got: {}", + current_index + 1, + index + ); + return Err(IggyError::StateFileCorrupted); + } + + current_index = index; + entries_count += 1; + let term = cursor + .read_u64_le() + .await + .error(|e: &std::io::Error| format!("{FILE_STATE_PARSE_ERROR} term. {e}")) + .map_err(|_| IggyError::InvalidNumberEncoding)?; + total_size += 8; + let leader_id = cursor + .read_u32_le() + .await + .error(|e: &std::io::Error| format!("{FILE_STATE_PARSE_ERROR} leader_id. {e}")) + .map_err(|_| IggyError::InvalidNumberEncoding)?; + total_size += 4; + let version = cursor + .read_u32_le() + .await + .error(|e: &std::io::Error| format!("{FILE_STATE_PARSE_ERROR} version. {e}")) + .map_err(|_| IggyError::InvalidNumberEncoding)?; + total_size += 4; + let flags = cursor + .read_u64_le() + .await + .error(|e: &std::io::Error| format!("{FILE_STATE_PARSE_ERROR} flags. {e}")) + .map_err(|_| IggyError::InvalidNumberEncoding)?; + total_size += 8; + let timestamp = IggyTimestamp::from( + cursor + .read_u64_le() + .await + .error(|e: &std::io::Error| format!("{FILE_STATE_PARSE_ERROR} timestamp. {e}")) + .map_err(|_| IggyError::InvalidNumberEncoding)?, + ); + total_size += 8; + let user_id = cursor + .read_u32_le() + .await + .error(|e: &std::io::Error| format!("{FILE_STATE_PARSE_ERROR} user_id. {e}")) + .map_err(|_| IggyError::InvalidNumberEncoding)?; + total_size += 4; + let checksum = cursor + .read_u64_le() + .await + .error(|e: &std::io::Error| format!("{FILE_STATE_PARSE_ERROR} checksum. {e}")) + .map_err(|_| IggyError::InvalidNumberEncoding)?; + total_size += 8; + let context_length = cursor + .read_u32_le() + .await + .error(|e: &std::io::Error| { + format!("{FILE_STATE_PARSE_ERROR} context context_length. {e}") + }) + .map_err(|_| IggyError::InvalidNumberEncoding)? + as usize; + total_size += 4; + let mut context = BytesMut::with_capacity(context_length); + context.put_bytes(0, context_length); + let (result, context) = cursor.read_exact(context).await.into(); + + result + .error(|e: &std::io::Error| format!("{FILE_STATE_PARSE_ERROR} code. {e}")) + .map_err(|_| IggyError::CannotReadFile)?; + let context = context.freeze(); + total_size += context_length as u64; + let code = cursor + .read_u32_le() + .await + .error(|e: &std::io::Error| format!("{FILE_STATE_PARSE_ERROR} code. {e}")) + .map_err(|_| IggyError::InvalidNumberEncoding)?; + total_size += 4; + let mut command_length = cursor + .read_u32_le() + .await + .error(|e: &std::io::Error| format!("{FILE_STATE_PARSE_ERROR} command_length. {e}")) + .map_err(|_| IggyError::InvalidNumberEncoding)? + as usize; + total_size += 4; + let mut command = BytesMut::with_capacity(command_length); + command.put_bytes(0, command_length); + let (result, command) = cursor.read_exact(command).await.into(); + result + .error(|e: &std::io::Error| format!("{FILE_STATE_PARSE_ERROR} command. {e}")) + .map_err(|_| IggyError::CannotReadFile)?; + total_size += command_length as u64; + let command_payload; + if let Some(encryptor) = &self.encryptor { + debug!("Decrypting state entry with index: {index}"); + command_payload = Bytes::from(encryptor.decrypt(&command.freeze())?); + command_length = command_payload.len(); + } else { + command_payload = command.freeze(); + } + + let mut entry_command = BytesMut::with_capacity(4 + 4 + command_length); + entry_command.put_u32_le(code); + entry_command.put_u32_le(command_length as u32); + entry_command.extend(command_payload); + let command = entry_command.freeze(); + EntryCommand::decode_from(&command) + .map_err(|e| { + tracing::warn!("wire decode error during WAL replay: {e}"); + IggyError::InvalidCommand + }) + .error(|e: &IggyError| { + format!("{COMPONENT} (error: {e}) - failed to parse entry command from bytes") + })?; + let calculated_checksum = StateEntry::calculate_checksum( + index, term, leader_id, version, flags, timestamp, user_id, &context, &command, + ); + let entry = StateEntry::new( + index, + term, + leader_id, + version, + flags, + timestamp, + user_id, + calculated_checksum, + context, + command, + ); + debug!("Read state entry: {entry}"); + if entry.checksum != checksum { + return Err(IggyError::InvalidStateEntryChecksum( + entry.checksum, + checksum, + entry.index, + )); + } + + entries.push(entry); + if total_size == file_size { + break; + } + } + + info!("Loaded {entries_count} state entries, current index: {current_index}"); + Ok(entries) + } + + pub async fn apply(&self, user_id: u32, command: &EntryCommand) -> Result<(), IggyError> { + debug!("Applying state entry with command: {command}, user ID: {user_id}"); + let timestamp = IggyTimestamp::now(); + let index = if self.entries_count.load(Ordering::SeqCst) == 0 { + 0 + } else { + self.current_index.fetch_add(1, Ordering::SeqCst) + 1 + }; + let term = self.term.load(Ordering::SeqCst); + let current_leader = self.current_leader.load(Ordering::SeqCst); + let version = self.version; + let flags = 0; + let context = Bytes::new(); + let mut command = command.to_bytes(); + let checksum = StateEntry::calculate_checksum( + index, + term, + current_leader, + version, + flags, + timestamp, + user_id, + &context, + &command, + ); + + if let Some(encryptor) = &self.encryptor { + debug!("Encrypting state entry command with index: {index}"); + let command_code = command.slice(0..4).get_u32_le(); + let mut command_length = command.slice(4..8).get_u32_le() as usize; + let command_payload = command.slice(8..8 + command_length); + let encrypted_command_payload = encryptor + .encrypt(&command_payload) + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to encrypt state entry command, index: {index}" + ) + })?; + command_length = encrypted_command_payload.len(); + let mut command_bytes = BytesMut::with_capacity(4 + 4 + command_length); + command_bytes.put_u32_le(command_code); + command_bytes.put_u32_le(command_length as u32); + command_bytes.extend(encrypted_command_payload); + command = command_bytes.freeze(); + } + + let entry = StateEntry::new( + index, + term, + current_leader, + version, + flags, + timestamp, + user_id, + checksum, + context, + command, + ); + let bytes = entry.to_bytes(); + let len = bytes.len(); + self.entries_count.fetch_add(1, Ordering::SeqCst); + self.persister + .append(&self.path, bytes) + .await + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to append state entry data to file, path: {}, data size: {}", + self.path, + len + ) + })?; + debug!("Applied state entry: {entry}"); + Ok(()) + } +} diff --git a/core/configs/src/common/mod.rs b/core/server/src/state/mod.rs similarity index 70% rename from core/configs/src/common/mod.rs rename to core/server/src/state/mod.rs index b14356b0a9..bb34bd1082 100644 --- a/core/configs/src/common/mod.rs +++ b/core/server/src/state/mod.rs @@ -15,16 +15,13 @@ // specific language governing permissions and limitations // under the License. -//! Config vocabulary shared across the crate: the generic -//! [`system::SystemConfig`], the HTTP section, and the top-level sections -//! that [`crate::server_config::server::ServerConfig`] composes. - -pub mod cache_indexes; -pub mod defaults; -pub mod displays; -pub mod http; -pub mod server; +pub mod command; +pub mod entry; +pub mod file; +pub mod models; pub mod system; -pub mod validators; -pub const COMPONENT: &str = "CONFIG"; +pub const COMPONENT: &str = "STATE"; + +pub use command::EntryCommand; +pub use entry::StateEntry; diff --git a/core/server/src/state/models.rs b/core/server/src/state/models.rs new file mode 100644 index 0000000000..15cd357789 --- /dev/null +++ b/core/server/src/state/models.rs @@ -0,0 +1,334 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use bytes::{BufMut, BytesMut}; +use iggy_binary_protocol::requests::{ + consumer_groups::CreateConsumerGroupRequest, + personal_access_tokens::CreatePersonalAccessTokenRequest, streams::CreateStreamRequest, + topics::CreateTopicRequest, users::CreateUserRequest, +}; +use iggy_binary_protocol::{WireDecode, WireEncode}; +use std::fmt; +use std::fmt::{Display, Formatter}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CreateStreamWithId { + pub stream_id: u32, + pub command: CreateStreamRequest, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CreateTopicWithId { + pub topic_id: u32, + pub command: CreateTopicRequest, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CreateConsumerGroupWithId { + pub group_id: u32, + pub command: CreateConsumerGroupRequest, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CreateUserWithId { + pub user_id: u32, + pub command: CreateUserRequest, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CreatePersonalAccessTokenWithHash { + pub hash: String, + pub command: CreatePersonalAccessTokenRequest, +} + +impl Display for CreateStreamWithId { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!( + f, + "CreateStreamWithId {{ name: {}, stream_id: {} }}", + self.command.name, self.stream_id + ) + } +} + +impl Display for CreateTopicWithId { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!( + f, + "CreateTopicWithId {{ name: {}, topic_id: {} }}", + self.command.name, self.topic_id + ) + } +} + +impl Display for CreateConsumerGroupWithId { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!( + f, + "CreateConsumerGroupWithId {{ name: {}, group_id: {} }}", + self.command.name, self.group_id + ) + } +} + +impl Display for CreateUserWithId { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!( + f, + "CreateUserWithId {{ username: {}, user_id: {} }}", + self.command.username, self.user_id + ) + } +} + +impl Display for CreatePersonalAccessTokenWithHash { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!( + f, + "CreatePersonalAccessTokenWithHash {{ name: {}, hash: [REDACTED] }}", + self.command.name, + ) + } +} + +// Wire format for WithId wrappers: id:u32_le | inner_length:u32_le | inner_bytes + +impl WireEncode for CreateStreamWithId { + fn encoded_size(&self) -> usize { + 4 + 4 + self.command.encoded_size() + } + + fn encode(&self, buf: &mut BytesMut) { + buf.put_u32_le(self.stream_id); + buf.put_u32_le(self.command.encoded_size() as u32); + self.command.encode(buf); + } +} + +impl WireDecode for CreateStreamWithId { + fn decode(buf: &[u8]) -> Result<(Self, usize), iggy_binary_protocol::WireError> { + if buf.len() < 8 { + return Err(iggy_binary_protocol::WireError::UnexpectedEof { + offset: 0, + need: 8, + have: buf.len(), + }); + } + let stream_id = u32::from_le_bytes(buf[0..4].try_into().unwrap()); + let command_length = u32::from_le_bytes(buf[4..8].try_into().unwrap()) as usize; + let total = 8usize.checked_add(command_length).ok_or( + iggy_binary_protocol::WireError::UnexpectedEof { + offset: 4, + need: command_length, + have: buf.len() - 8, + }, + )?; + if buf.len() < total { + return Err(iggy_binary_protocol::WireError::UnexpectedEof { + offset: 8, + need: command_length, + have: buf.len() - 8, + }); + } + let (command, _) = CreateStreamRequest::decode(&buf[8..total])?; + Ok((Self { stream_id, command }, total)) + } +} + +impl WireEncode for CreateTopicWithId { + fn encoded_size(&self) -> usize { + 4 + 4 + self.command.encoded_size() + } + + fn encode(&self, buf: &mut BytesMut) { + buf.put_u32_le(self.topic_id); + buf.put_u32_le(self.command.encoded_size() as u32); + self.command.encode(buf); + } +} + +impl WireDecode for CreateTopicWithId { + fn decode(buf: &[u8]) -> Result<(Self, usize), iggy_binary_protocol::WireError> { + if buf.len() < 8 { + return Err(iggy_binary_protocol::WireError::UnexpectedEof { + offset: 0, + need: 8, + have: buf.len(), + }); + } + let topic_id = u32::from_le_bytes(buf[0..4].try_into().unwrap()); + let command_length = u32::from_le_bytes(buf[4..8].try_into().unwrap()) as usize; + let total = 8usize.checked_add(command_length).ok_or( + iggy_binary_protocol::WireError::UnexpectedEof { + offset: 4, + need: command_length, + have: buf.len() - 8, + }, + )?; + if buf.len() < total { + return Err(iggy_binary_protocol::WireError::UnexpectedEof { + offset: 8, + need: command_length, + have: buf.len() - 8, + }); + } + let (command, _) = CreateTopicRequest::decode(&buf[8..total])?; + Ok((Self { topic_id, command }, total)) + } +} + +impl WireEncode for CreateConsumerGroupWithId { + fn encoded_size(&self) -> usize { + 4 + 4 + self.command.encoded_size() + } + + fn encode(&self, buf: &mut BytesMut) { + buf.put_u32_le(self.group_id); + buf.put_u32_le(self.command.encoded_size() as u32); + self.command.encode(buf); + } +} + +impl WireDecode for CreateConsumerGroupWithId { + fn decode(buf: &[u8]) -> Result<(Self, usize), iggy_binary_protocol::WireError> { + if buf.len() < 8 { + return Err(iggy_binary_protocol::WireError::UnexpectedEof { + offset: 0, + need: 8, + have: buf.len(), + }); + } + let group_id = u32::from_le_bytes(buf[0..4].try_into().unwrap()); + let command_length = u32::from_le_bytes(buf[4..8].try_into().unwrap()) as usize; + let total = 8usize.checked_add(command_length).ok_or( + iggy_binary_protocol::WireError::UnexpectedEof { + offset: 4, + need: command_length, + have: buf.len() - 8, + }, + )?; + if buf.len() < total { + return Err(iggy_binary_protocol::WireError::UnexpectedEof { + offset: 8, + need: command_length, + have: buf.len() - 8, + }); + } + let (command, _) = CreateConsumerGroupRequest::decode(&buf[8..total])?; + Ok((Self { group_id, command }, total)) + } +} + +impl WireEncode for CreateUserWithId { + fn encoded_size(&self) -> usize { + 4 + 4 + self.command.encoded_size() + } + + fn encode(&self, buf: &mut BytesMut) { + buf.put_u32_le(self.user_id); + buf.put_u32_le(self.command.encoded_size() as u32); + self.command.encode(buf); + } +} + +impl WireDecode for CreateUserWithId { + fn decode(buf: &[u8]) -> Result<(Self, usize), iggy_binary_protocol::WireError> { + if buf.len() < 8 { + return Err(iggy_binary_protocol::WireError::UnexpectedEof { + offset: 0, + need: 8, + have: buf.len(), + }); + } + let user_id = u32::from_le_bytes(buf[0..4].try_into().unwrap()); + let command_length = u32::from_le_bytes(buf[4..8].try_into().unwrap()) as usize; + let total = 8usize.checked_add(command_length).ok_or( + iggy_binary_protocol::WireError::UnexpectedEof { + offset: 4, + need: command_length, + have: buf.len() - 8, + }, + )?; + if buf.len() < total { + return Err(iggy_binary_protocol::WireError::UnexpectedEof { + offset: 8, + need: command_length, + have: buf.len() - 8, + }); + } + let (command, _) = CreateUserRequest::decode(&buf[8..total])?; + Ok((Self { user_id, command }, total)) + } +} + +impl WireEncode for CreatePersonalAccessTokenWithHash { + fn encoded_size(&self) -> usize { + 4 + self.hash.len() + 4 + self.command.encoded_size() + } + + fn encode(&self, buf: &mut BytesMut) { + buf.put_u32_le(self.hash.len() as u32); + buf.put_slice(self.hash.as_bytes()); + buf.put_u32_le(self.command.encoded_size() as u32); + self.command.encode(buf); + } +} + +impl WireDecode for CreatePersonalAccessTokenWithHash { + fn decode(buf: &[u8]) -> Result<(Self, usize), iggy_binary_protocol::WireError> { + if buf.len() < 4 { + return Err(iggy_binary_protocol::WireError::UnexpectedEof { + offset: 0, + need: 4, + have: buf.len(), + }); + } + let hash_length = u32::from_le_bytes(buf[0..4].try_into().unwrap()) as usize; + let mut pos = 4; + if buf.len() < pos + hash_length { + return Err(iggy_binary_protocol::WireError::UnexpectedEof { + offset: pos, + need: hash_length, + have: buf.len() - pos, + }); + } + let hash = std::str::from_utf8(&buf[pos..pos + hash_length]) + .map_err(|_| iggy_binary_protocol::WireError::InvalidUtf8 { offset: pos })? + .to_string(); + pos += hash_length; + if buf.len() < pos + 4 { + return Err(iggy_binary_protocol::WireError::UnexpectedEof { + offset: pos, + need: 4, + have: buf.len() - pos, + }); + } + let command_length = u32::from_le_bytes(buf[pos..pos + 4].try_into().unwrap()) as usize; + pos += 4; + if buf.len() < pos + command_length { + return Err(iggy_binary_protocol::WireError::UnexpectedEof { + offset: pos, + need: command_length, + have: buf.len() - pos, + }); + } + let (command, _) = + CreatePersonalAccessTokenRequest::decode(&buf[pos..pos + command_length])?; + pos += command_length; + Ok((Self { hash, command }, pos)) + } +} diff --git a/core/server/src/state/system.rs b/core/server/src/state/system.rs new file mode 100644 index 0000000000..76a7b86174 --- /dev/null +++ b/core/server/src/state/system.rs @@ -0,0 +1,633 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::bootstrap::create_root_user; +use crate::state::file::FileState; +use crate::state::models::CreateUserWithId; +use crate::state::{COMPONENT, EntryCommand, StateEntry}; +use ahash::AHashMap; +use err_trail::ErrContext; +use iggy_binary_protocol::requests::users::CreateUserRequest; +use iggy_binary_protocol::{WireIdentifier, WireName}; +use iggy_common::CompressionAlgorithm; +use iggy_common::IggyError; +use iggy_common::IggyExpiry; +use iggy_common::IggyTimestamp; +use iggy_common::MaxTopicSize; +use iggy_common::PersonalAccessToken; +use iggy_common::defaults::DEFAULT_ROOT_USER_ID; +use iggy_common::wire_conversions::{permissions_to_wire, wire_permissions_to_permissions}; +use iggy_common::{Permissions, UserStatus}; +use std::collections::BTreeMap; +use std::fmt::Display; +use tracing::{debug, error, info}; + +#[derive(Debug, Clone)] +pub struct SystemState { + pub streams: BTreeMap, + pub users: AHashMap, +} + +impl SystemState { + pub fn decompose(self) -> (BTreeMap, AHashMap) { + (self.streams, self.users) + } +} + +#[derive(Debug, Clone)] +pub struct StreamState { + pub id: u32, + pub name: String, + pub created_at: IggyTimestamp, + pub topics: BTreeMap, +} + +#[derive(Debug, Clone)] +pub struct TopicState { + pub id: u32, + pub name: String, + pub partitions: BTreeMap, + pub consumer_groups: BTreeMap, + pub compression_algorithm: CompressionAlgorithm, + pub message_expiry: IggyExpiry, + pub max_topic_size: MaxTopicSize, + pub replication_factor: Option, + pub created_at: IggyTimestamp, +} + +#[derive(Debug, Clone)] +pub struct PartitionState { + pub id: u32, + pub created_at: IggyTimestamp, +} + +// TODO: consider converting token_hash to SecretString (requires updating the full hash flow across crates) +#[derive(Clone)] +pub struct PersonalAccessTokenState { + pub name: String, + pub token_hash: String, + pub expiry_at: Option, +} + +impl std::fmt::Debug for PersonalAccessTokenState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PersonalAccessTokenState") + .field("name", &self.name) + .field("token_hash", &"[REDACTED]") + .field("expiry_at", &self.expiry_at) + .finish() + } +} + +// TODO: consider converting password_hash to SecretString (requires updating the full hash flow across crates) +#[derive(Clone)] +pub struct UserState { + pub id: u32, + pub username: String, + pub password_hash: String, + pub status: UserStatus, + pub created_at: IggyTimestamp, + pub permissions: Option, + pub personal_access_tokens: AHashMap, +} + +impl std::fmt::Debug for UserState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("UserState") + .field("id", &self.id) + .field("username", &self.username) + .field("password_hash", &"[REDACTED]") + .field("status", &self.status) + .field("created_at", &self.created_at) + .field("permissions", &self.permissions) + .field("personal_access_tokens", &self.personal_access_tokens) + .finish() + } +} + +#[derive(Debug, Clone)] +pub struct ConsumerGroupState { + pub id: u32, + pub name: String, +} + +impl SystemState { + pub async fn load(state: FileState) -> Result { + let mut state_entries = state.init().await.error(|e: &IggyError| { + format!("{COMPONENT} (error: {e}) - failed to initialize state entries") + })?; + + // Create root user if does not exist. + let root_exists = state_entries + .iter() + .any(|entry| { + entry + .command() + .map(|command| matches!(command, EntryCommand::CreateUser(payload) if payload.user_id == DEFAULT_ROOT_USER_ID)) + .unwrap_or_else(|err| { + error!("Failed to check if root user exists: {err}"); + false + }) + }); + + if !root_exists { + info!("No users found, creating the root user..."); + let root = create_root_user(); + let command = CreateUserRequest { + username: WireName::new(root.username.clone()) + .expect("root username must be valid"), + password: root.password.clone(), + status: root.status.as_code(), + permissions: root.permissions.as_ref().map(permissions_to_wire), + }; + state + .apply(0, &EntryCommand::CreateUser(CreateUserWithId { + user_id: root.id, + command + })) + .await + .error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to apply create user command, username: {}", + root.username + ) + })?; + state_entries = state.init().await.error(|e: &IggyError| { + format!("{COMPONENT} (error: {e}) - failed to initialize state entries") + })?; + } + + let system_state = Self::init(state_entries).await.error(|e: &IggyError| { + format!("{COMPONENT} (error: {e}) - failed to initialize system state") + })?; + Ok(system_state) + } + + pub async fn init(entries: Vec) -> Result { + let mut streams = BTreeMap::new(); + let mut users = AHashMap::new(); + for entry in entries { + debug!("Processing state entry: {entry}",); + match entry.command().error(|e: &IggyError| { + format!( + "{COMPONENT} (error: {e}) - failed to retrieve state entry command: {entry}" + ) + })? { + EntryCommand::CreateStream(command) => { + info!("Creating stream: {command:?}"); + let stream_id = command.stream_id; + let stream = StreamState { + id: stream_id, + name: command.command.name.to_string(), + topics: BTreeMap::new(), + created_at: entry.timestamp, + }; + streams.insert(stream.id, stream); + } + EntryCommand::UpdateStream(command) => { + let stream_id = find_stream_id(&streams, &command.stream_id); + let stream = streams + .get_mut(&stream_id) + .unwrap_or_else(|| panic!("{}", format!("Stream: {stream_id} not found"))); + stream.name = command.name.to_string(); + } + EntryCommand::DeleteStream(command) => { + let stream_id = find_stream_id(&streams, &command.stream_id); + streams.remove(&stream_id); + } + EntryCommand::PurgeStream(command) => { + let stream_id = find_stream_id(&streams, &command.stream_id); + streams + .get(&stream_id) + .unwrap_or_else(|| panic!("{}", format!("Stream: {stream_id} not found"))); + } + EntryCommand::CreateTopic(command) => { + let stream_id = find_stream_id(&streams, &command.command.stream_id); + let stream = streams + .get_mut(&stream_id) + .unwrap_or_else(|| panic!("{}", format!("Stream: {stream_id} not found"))); + let topic_id = command.topic_id; + let wire = command.command; + let topic = TopicState { + id: topic_id, + name: wire.name.to_string(), + consumer_groups: BTreeMap::new(), + compression_algorithm: CompressionAlgorithm::from_code( + wire.compression_algorithm, + )?, + message_expiry: IggyExpiry::from(wire.message_expiry), + max_topic_size: MaxTopicSize::from(wire.max_topic_size), + replication_factor: if wire.replication_factor == 0 { + None + } else { + Some(wire.replication_factor) + }, + created_at: entry.timestamp, + partitions: if wire.partitions_count > 0 { + let mut partitions = BTreeMap::new(); + for i in 0..wire.partitions_count { + partitions.insert( + i, + PartitionState { + id: i, + created_at: entry.timestamp, + }, + ); + } + partitions + } else { + BTreeMap::new() + }, + }; + stream.topics.insert(topic.id, topic); + } + EntryCommand::UpdateTopic(command) => { + let stream_id = find_stream_id(&streams, &command.stream_id); + let stream = streams + .get_mut(&stream_id) + .unwrap_or_else(|| panic!("{}", format!("Stream: {stream_id} not found"))); + let topic_id = find_topic_id(&stream.topics, &command.topic_id); + let topic = stream + .topics + .get_mut(&topic_id) + .unwrap_or_else(|| panic!("{}", format!("Topic: {topic_id} not found"))); + topic.name = command.name.to_string(); + topic.compression_algorithm = + CompressionAlgorithm::from_code(command.compression_algorithm)?; + topic.message_expiry = IggyExpiry::from(command.message_expiry); + topic.max_topic_size = MaxTopicSize::from(command.max_topic_size); + topic.replication_factor = if command.replication_factor == 0 { + None + } else { + Some(command.replication_factor) + }; + } + EntryCommand::DeleteTopic(command) => { + let stream_id = find_stream_id(&streams, &command.stream_id); + let stream = streams + .get_mut(&stream_id) + .unwrap_or_else(|| panic!("{}", format!("Stream: {stream_id} not found"))); + let topic_id = find_topic_id(&stream.topics, &command.topic_id); + stream.topics.remove(&topic_id); + } + EntryCommand::PurgeTopic(command) => { + let stream_id = find_stream_id(&streams, &command.stream_id); + let stream = streams + .get(&stream_id) + .unwrap_or_else(|| panic!("{}", format!("Stream: {stream_id} not found"))); + let topic_id = find_topic_id(&stream.topics, &command.topic_id); + stream + .topics + .get(&topic_id) + .unwrap_or_else(|| panic!("{}", format!("Topic: {topic_id} not found"))); + } + EntryCommand::CreatePartitions(command) => { + let stream_id = find_stream_id(&streams, &command.stream_id); + let stream = streams + .get_mut(&stream_id) + .unwrap_or_else(|| panic!("{}", format!("Stream: {stream_id} not found"))); + let topic_id = find_topic_id(&stream.topics, &command.topic_id); + let topic = stream + .topics + .get_mut(&topic_id) + .unwrap_or_else(|| panic!("{}", format!("Topic: {topic_id} not found"))); + let last_partition_id = if topic.partitions.is_empty() { + 0 + } else { + topic + .partitions + .values() + .map(|p| p.id) + .max() + .unwrap_or_else(|| panic!("No partition found")) + }; + for i in 1..=command.partitions_count { + topic.partitions.insert( + last_partition_id + i, + PartitionState { + id: last_partition_id + i, + created_at: entry.timestamp, + }, + ); + } + } + EntryCommand::DeletePartitions(command) => { + let stream_id = find_stream_id(&streams, &command.stream_id); + let stream = streams + .get_mut(&stream_id) + .unwrap_or_else(|| panic!("{}", format!("Stream: {stream_id} not found"))); + let topic_id = find_topic_id(&stream.topics, &command.topic_id); + let topic = stream + .topics + .get_mut(&topic_id) + .unwrap_or_else(|| panic!("{}", format!("Topic: {topic_id} not found"))); + if topic.partitions.is_empty() { + continue; + } + + let last_partition_id = topic + .partitions + .values() + .map(|p| p.id) + .max() + .unwrap_or_else(|| panic!("No partition found")); + for i in 0..command.partitions_count { + topic.partitions.remove(&(last_partition_id - i)); + } + } + EntryCommand::DeleteSegments(command) => { + let stream_id = find_stream_id(&streams, &command.stream_id); + let stream = streams + .get_mut(&stream_id) + .unwrap_or_else(|| panic!("{}", format!("Stream: {stream_id} not found"))); + let topic_id = find_topic_id(&stream.topics, &command.topic_id); + let topic = stream + .topics + .get_mut(&topic_id) + .unwrap_or_else(|| panic!("{}", format!("Topic: {topic_id} not found"))); + if topic.partitions.is_empty() { + continue; + } + + let partition_id = command.partition_id; + + let _partition = + topic + .partitions + .get(&command.partition_id) + .unwrap_or_else(|| { + panic!("{}", format!("Partition {partition_id} not found.")) + }); + } + EntryCommand::CreateConsumerGroup(command) => { + let consumer_group_id = command.group_id; + let wire = command.command; + let stream_id = find_stream_id(&streams, &wire.stream_id); + let stream = streams + .get_mut(&stream_id) + .unwrap_or_else(|| panic!("{}", format!("Stream: {stream_id} not found"))); + let topic_id = find_topic_id(&stream.topics, &wire.topic_id); + let topic = stream + .topics + .get_mut(&topic_id) + .unwrap_or_else(|| panic!("{}", format!("Topic: {topic_id} not found"))); + let consumer_group = ConsumerGroupState { + id: consumer_group_id, + name: wire.name.to_string(), + }; + topic + .consumer_groups + .insert(consumer_group.id, consumer_group); + } + EntryCommand::DeleteConsumerGroup(command) => { + let stream_id = find_stream_id(&streams, &command.stream_id); + let stream = streams + .get_mut(&stream_id) + .unwrap_or_else(|| panic!("{}", format!("Stream: {stream_id} not found"))); + let topic_id = find_topic_id(&stream.topics, &command.topic_id); + let topic = stream + .topics + .get_mut(&topic_id) + .unwrap_or_else(|| panic!("{}", format!("Topic: {topic_id} not found"))); + let consumer_group_id = + find_consumer_group_id(&topic.consumer_groups, &command.group_id); + topic.consumer_groups.remove(&consumer_group_id); + } + EntryCommand::CreateUser(command) => { + let user_id = command.user_id; + let wire = command.command; + let user = UserState { + id: user_id, + username: wire.username.to_string(), + password_hash: wire.password, // already hashed at write time + status: UserStatus::from_code(wire.status)?, + created_at: entry.timestamp, + permissions: wire + .permissions + .as_ref() + .map(wire_permissions_to_permissions), + personal_access_tokens: AHashMap::new(), + }; + users.insert(user.id, user); + } + EntryCommand::UpdateUser(command) => { + let user_id = find_user_id(&users, &command.user_id); + let user = users + .get_mut(&user_id) + .unwrap_or_else(|| panic!("{}", format!("User: {user_id} not found"))); + if let Some(username) = &command.username { + user.username = username.to_string(); + } + if let Some(status) = command.status { + user.status = UserStatus::from_code(status)?; + } + } + EntryCommand::DeleteUser(command) => { + let user_id = find_user_id(&users, &command.user_id); + users.remove(&user_id); + } + EntryCommand::ChangePassword(command) => { + let user_id = find_user_id(&users, &command.user_id); + let user = users + .get_mut(&user_id) + .unwrap_or_else(|| panic!("{}", format!("User: {user_id} not found"))); + user.password_hash = command.new_password; // already hashed at write time + } + EntryCommand::UpdatePermissions(command) => { + let user_id = find_user_id(&users, &command.user_id); + let user = users + .get_mut(&user_id) + .unwrap_or_else(|| panic!("{}", format!("User: {user_id} not found"))); + user.permissions = command + .permissions + .as_ref() + .map(wire_permissions_to_permissions); + } + EntryCommand::CreatePersonalAccessToken(command) => { + let token_hash = command.hash; + let user_id = find_user_id(&users, &WireIdentifier::numeric(entry.user_id)); + let user = users + .get_mut(&user_id) + .unwrap_or_else(|| panic!("{}", format!("User: {user_id} not found"))); + let expiry_at = PersonalAccessToken::calculate_expiry_at( + entry.timestamp, + IggyExpiry::from(command.command.expiry), + ); + if let Some(expiry_at) = expiry_at + && expiry_at.as_micros() <= IggyTimestamp::now().as_micros() + { + debug!("Personal access token: {token_hash} has already expired."); + continue; + } + + let name = command.command.name.to_string(); + user.personal_access_tokens.insert( + name.clone(), + PersonalAccessTokenState { + name, + token_hash, + expiry_at, + }, + ); + } + EntryCommand::DeletePersonalAccessToken(command) => { + let user_id = find_user_id(&users, &WireIdentifier::numeric(entry.user_id)); + let user = users + .get_mut(&user_id) + .unwrap_or_else(|| panic!("{}", format!("User: {user_id} not found"))); + user.personal_access_tokens.remove(command.name.as_str()); + } + } + } + + let state = SystemState { streams, users }; + debug!("+++ State +++"); + debug!("{state}"); + debug!("+++ State +++"); + Ok(state) + } +} + +fn find_stream_id(streams: &BTreeMap, stream_id: &WireIdentifier) -> u32 { + match stream_id { + WireIdentifier::Numeric(id) => *id, + WireIdentifier::String(name) => { + let name = name.as_str(); + let stream = streams + .values() + .find(|s| s.name == name) + .unwrap_or_else(|| panic!("Stream: {name} not found")); + stream.id + } + } +} + +fn find_topic_id(topics: &BTreeMap, topic_id: &WireIdentifier) -> u32 { + match topic_id { + WireIdentifier::Numeric(id) => *id, + WireIdentifier::String(name) => { + let name = name.as_str(); + let topic = topics + .values() + .find(|s| s.name == name) + .unwrap_or_else(|| panic!("Topic: {name} not found")); + topic.id + } + } +} + +fn find_consumer_group_id( + groups: &BTreeMap, + group_id: &WireIdentifier, +) -> u32 { + match group_id { + WireIdentifier::Numeric(id) => *id, + WireIdentifier::String(name) => { + let name = name.as_str(); + let group = groups + .values() + .find(|s| s.name == name) + .unwrap_or_else(|| panic!("Consumer group: {name} not found")); + group.id + } + } +} + +fn find_user_id(users: &AHashMap, user_id: &WireIdentifier) -> u32 { + match user_id { + WireIdentifier::Numeric(id) => *id, + WireIdentifier::String(name) => { + let name = name.as_str(); + let user = users + .values() + .find(|s| s.username == name) + .unwrap_or_else(|| panic!("User: {name} not found")); + user.id + } + } +} + +impl Display for SystemState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Streams:")?; + for stream in self.streams.iter() { + write!(f, "\n================\n")?; + write!(f, "{}", stream.1)?; + } + write!(f, "Users:")?; + for user in self.users.iter() { + write!(f, "\n================\n")?; + write!(f, "{}", user.1)?; + } + Ok(()) + } +} + +impl Display for ConsumerGroupState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "ConsumerGroup -> ID: {}, Name: {}", self.id, self.name) + } +} + +impl Display for UserState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let permissions = if let Some(permissions) = &self.permissions { + permissions.to_string() + } else { + "no_permissions".to_string() + }; + write!( + f, + "User -> ID: {}, Username: {}, Status: {}, Permissions: {}", + self.id, self.username, self.status, permissions + ) + } +} + +impl Display for StreamState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Stream -> ID: {}, Name: {}", self.id, self.name,)?; + for topic in self.topics.iter() { + write!(f, "\n {}", topic.1)?; + } + Ok(()) + } +} + +impl Display for TopicState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Topic -> ID: {}, Name: {}", self.id, self.name,)?; + for partition in self.partitions.iter() { + write!(f, "\n {}", partition.1)?; + } + write!(f, "\nConsumer Groups:")?; + for consumer_group in self.consumer_groups.iter() { + write!(f, "\n {}", consumer_group.1)?; + } + Ok(()) + } +} + +impl Display for PartitionState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "Partition -> ID: {}, Created At: {}", + self.id, self.created_at + ) + } +} diff --git a/core/server/src/streaming/clients/client_manager.rs b/core/server/src/streaming/clients/client_manager.rs new file mode 100644 index 0000000000..f8ba859378 --- /dev/null +++ b/core/server/src/streaming/clients/client_manager.rs @@ -0,0 +1,233 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::streaming::session::Session; +use crate::streaming::utils::ptr::EternalPtr; +use dashmap::DashMap; +use iggy_common::IggyTimestamp; +use iggy_common::TransportProtocol; +use iggy_common::UserId; +use iggy_common::{IggyError, calculate_32}; +use std::net::SocketAddr; + +pub struct ClientManager { + clients: EternalPtr>, +} + +impl ClientManager { + pub fn new(clients: EternalPtr>) -> Self { + Self { clients } + } +} + +impl Clone for ClientManager { + fn clone(&self) -> Self { + Self { + clients: self.clients.clone(), + } + } +} + +#[derive(Debug, Clone)] +pub struct Client { + pub user_id: Option, + pub session: Session, + pub transport: TransportProtocol, + pub consumer_groups: Vec, + pub last_heartbeat: IggyTimestamp, +} + +#[derive(Debug, Clone)] +pub struct ConsumerGroup { + pub stream_id: u32, + pub topic_id: u32, + pub group_id: u32, +} + +impl ClientManager { + pub fn add_client(&self, address: &SocketAddr, transport: TransportProtocol) -> Session { + let client_id = calculate_32(address.to_string().as_bytes()); + let session = Session::from_client_id(client_id, *address); + let client = Client { + user_id: None, + session: session.clone(), + transport, + consumer_groups: Vec::new(), + last_heartbeat: IggyTimestamp::now(), + }; + self.clients.insert(client_id, client); + session + } + + pub fn set_user_id(&self, client_id: u32, user_id: UserId) -> Result<(), IggyError> { + self.clients + .get_mut(&client_id) + .ok_or(IggyError::ClientNotFound(client_id))? + .user_id = Some(user_id); + Ok(()) + } + + pub fn clear_user_id(&self, client_id: u32) -> Result<(), IggyError> { + self.clients + .get_mut(&client_id) + .ok_or(IggyError::ClientNotFound(client_id))? + .user_id = None; + Ok(()) + } + + pub fn try_get_client(&self, client_id: u32) -> Option { + self.clients.get(&client_id).map(|c| c.clone()) + } + + pub fn try_get_client_mut( + &'_ self, + client_id: u32, + ) -> Option> { + self.clients.get_mut(&client_id) + } + + pub fn get_clients(&self) -> Vec { + self.clients + .iter() + .map(|entry| entry.value().clone()) + .collect() + } + + pub fn delete_clients_for_user(&self, user_id: UserId) -> Result<(), IggyError> { + let clients_to_remove: Vec = self + .clients + .iter() + .filter(|entry| entry.value().user_id == Some(user_id)) + .map(|entry| *entry.key()) + .collect(); + + for client_id in clients_to_remove { + self.clients.remove(&client_id); + } + Ok(()) + } + + pub fn delete_client(&self, client_id: u32) -> Option { + self.clients.remove(&client_id).map(|(_, client)| client) + } + + pub fn get_client_count(&self) -> usize { + self.clients.len() + } + + pub fn heartbeat(&mut self, client_id: u32) -> Result<(), IggyError> { + let mut client = self + .clients + .get_mut(&client_id) + .ok_or(IggyError::StaleClient)?; + client.last_heartbeat = IggyTimestamp::now(); + Ok(()) + } + + pub fn join_consumer_group( + &self, + client_id: u32, + stream_id: usize, + topic_id: usize, + group_id: usize, + ) -> Result<(), IggyError> { + let stream_id = stream_id as u32; + let topic_id = topic_id as u32; + let group_id = group_id as u32; + + let mut client = self + .clients + .get_mut(&client_id) + .ok_or(IggyError::StaleClient)?; + + if client.consumer_groups.iter().any(|consumer_group| { + consumer_group.group_id == group_id + && consumer_group.topic_id == topic_id + && consumer_group.stream_id == stream_id + }) { + return Ok(()); + } + + client.consumer_groups.push(ConsumerGroup { + stream_id, + topic_id, + group_id, + }); + Ok(()) + } + + pub fn leave_consumer_group( + &self, + client_id: u32, + stream_id: usize, + topic_id: usize, + consumer_group_id: usize, + ) -> Result<(), IggyError> { + let stream_id = stream_id as u32; + let topic_id = topic_id as u32; + let consumer_group_id = consumer_group_id as u32; + + let mut client = self + .clients + .get_mut(&client_id) + .ok_or(IggyError::StaleClient)?; + + if let Some(index) = client.consumer_groups.iter().position(|consumer_group| { + consumer_group.stream_id == stream_id + && consumer_group.topic_id == topic_id + && consumer_group.group_id == consumer_group_id + }) { + client.consumer_groups.remove(index); + } + Ok(()) + } + + pub fn delete_consumer_group(&self, stream_id: usize, topic_id: usize, group_id: usize) { + let stream_id = stream_id as u32; + let topic_id = topic_id as u32; + let group_id = group_id as u32; + + for mut client in self.clients.iter_mut() { + client.consumer_groups.retain(|consumer_group| { + !(consumer_group.stream_id == stream_id + && consumer_group.topic_id == topic_id + && consumer_group.group_id == group_id) + }); + } + } + + pub fn delete_consumer_groups_for_stream(&self, stream_id: usize) { + let stream_id = stream_id as u32; + + for mut client in self.clients.iter_mut() { + client + .consumer_groups + .retain(|consumer_group| consumer_group.stream_id != stream_id); + } + } + + pub fn delete_consumer_groups_for_topic(&self, stream_id: usize, topic_id: usize) { + let stream_id = stream_id as u32; + let topic_id = topic_id as u32; + + for mut client in self.clients.iter_mut() { + client.consumer_groups.retain(|consumer_group| { + !(consumer_group.stream_id == stream_id && consumer_group.topic_id == topic_id) + }); + } + } +} diff --git a/core/server/src/streaming/clients/mod.rs b/core/server/src/streaming/clients/mod.rs new file mode 100644 index 0000000000..3048ad3603 --- /dev/null +++ b/core/server/src/streaming/clients/mod.rs @@ -0,0 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod client_manager; diff --git a/core/server/src/streaming/deduplication/mod.rs b/core/server/src/streaming/deduplication/mod.rs new file mode 100644 index 0000000000..bd18c67f31 --- /dev/null +++ b/core/server/src/streaming/deduplication/mod.rs @@ -0,0 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub use server_common::MessageDeduplicator; diff --git a/core/server/src/streaming/diagnostics/metrics.rs b/core/server/src/streaming/diagnostics/metrics.rs new file mode 100644 index 0000000000..a0a1dca062 --- /dev/null +++ b/core/server/src/streaming/diagnostics/metrics.rs @@ -0,0 +1,149 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use prometheus_client::encoding::text::encode; +use prometheus_client::metrics::counter::Counter; +use prometheus_client::metrics::gauge::Gauge; +use prometheus_client::registry::Registry; +use std::sync::Arc; +use tracing::error; + +#[derive(Debug, Clone)] +pub struct Metrics { + registry: Arc, + http_requests: Counter, + streams: Gauge, + topics: Gauge, + partitions: Gauge, + segments: Gauge, + messages: Gauge, + users: Gauge, + clients: Gauge, +} + +impl Metrics { + pub fn init() -> Self { + let mut registry = Registry::default(); + + let http_requests = Counter::default(); + let streams = Gauge::default(); + let topics = Gauge::default(); + let partitions = Gauge::default(); + let segments = Gauge::default(); + let messages = Gauge::default(); + let users = Gauge::default(); + let clients = Gauge::default(); + + registry.register( + "http_requests", + "total count of http_requests", + http_requests.clone(), + ); + registry.register("streams", "total count of streams", streams.clone()); + registry.register("topics", "total count of topics", topics.clone()); + registry.register( + "partitions", + "total count of partitions", + partitions.clone(), + ); + registry.register("segments", "total count of segments", segments.clone()); + registry.register("messages", "total count of messages", messages.clone()); + registry.register("users", "total count of users", users.clone()); + registry.register("clients", "total count of clients", clients.clone()); + + let registry = registry.into(); + Self { + registry, + http_requests, + streams, + topics, + partitions, + segments, + messages, + users, + clients, + } + } + + pub fn get_formatted_output(&self) -> String { + let mut buffer = String::new(); + if let Err(err) = encode(&mut buffer, &self.registry) { + error!("Failed to encode metrics: {}", err); + } + buffer + } + + pub fn increment_http_requests(&self) { + self.http_requests.inc(); + } + + pub fn increment_streams(&self, count: u32) { + self.streams.inc_by(count as i64); + } + + pub fn decrement_streams(&self, count: u32) { + self.streams.dec_by(count as i64); + } + + pub fn increment_topics(&self, count: u32) { + self.topics.inc_by(count as i64); + } + + pub fn decrement_topics(&self, count: u32) { + self.topics.dec_by(count as i64); + } + + pub fn increment_partitions(&self, count: u32) { + self.partitions.inc_by(count as i64); + } + + pub fn decrement_partitions(&self, count: u32) { + self.partitions.dec_by(count as i64); + } + + pub fn increment_segments(&self, count: u32) { + self.segments.inc_by(count as i64); + } + + pub fn decrement_segments(&self, count: u32) { + self.segments.dec_by(count as i64); + } + + pub fn increment_messages(&self, count: u64) { + self.messages.inc_by(count as i64); + } + + pub fn decrement_messages(&self, count: u64) { + self.messages.dec_by(count as i64); + } + + pub fn increment_users(&self, count: u32) { + self.users.inc_by(count as i64); + } + + pub fn decrement_users(&self, count: u32) { + self.users.dec_by(count as i64); + } + + pub fn increment_clients(&self, count: u32) { + self.clients.inc_by(count as i64); + } + + pub fn decrement_clients(&self, count: u32) { + self.clients.dec_by(count as i64); + } +} diff --git a/core/server/src/streaming/diagnostics/mod.rs b/core/server/src/streaming/diagnostics/mod.rs new file mode 100644 index 0000000000..054bb9088b --- /dev/null +++ b/core/server/src/streaming/diagnostics/mod.rs @@ -0,0 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod metrics; diff --git a/core/server/src/streaming/mod.rs b/core/server/src/streaming/mod.rs new file mode 100644 index 0000000000..d370e9c2b8 --- /dev/null +++ b/core/server/src/streaming/mod.rs @@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod clients; +pub mod deduplication; +pub mod diagnostics; +pub mod partitions; +pub mod persistence; +pub mod polling_consumer; +pub mod segments; +pub mod session; +pub mod stats; +pub mod storage; +pub mod streams; +pub mod topics; +pub mod users; +pub mod utils; diff --git a/core/server/src/streaming/partitions/consumer_group_offsets.rs b/core/server/src/streaming/partitions/consumer_group_offsets.rs new file mode 100644 index 0000000000..623d397a5c --- /dev/null +++ b/core/server/src/streaming/partitions/consumer_group_offsets.rs @@ -0,0 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub use iggy_common::ConsumerGroupOffsets; diff --git a/core/server/src/streaming/partitions/consumer_offset.rs b/core/server/src/streaming/partitions/consumer_offset.rs new file mode 100644 index 0000000000..9cff9043b9 --- /dev/null +++ b/core/server/src/streaming/partitions/consumer_offset.rs @@ -0,0 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub use iggy_common::ConsumerOffset; diff --git a/core/server/src/streaming/partitions/consumer_offsets.rs b/core/server/src/streaming/partitions/consumer_offsets.rs new file mode 100644 index 0000000000..4dba7dd28e --- /dev/null +++ b/core/server/src/streaming/partitions/consumer_offsets.rs @@ -0,0 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub use iggy_common::ConsumerOffsets; diff --git a/core/server/src/streaming/partitions/helpers.rs b/core/server/src/streaming/partitions/helpers.rs new file mode 100644 index 0000000000..89f4e73c10 --- /dev/null +++ b/core/server/src/streaming/partitions/helpers.rs @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::{configs::system::SystemConfig, streaming::deduplication::MessageDeduplicator}; + +pub fn create_message_deduplicator(config: &SystemConfig) -> Option { + if !config.message_deduplication.enabled { + return None; + } + let max_entries = if config.message_deduplication.max_entries > 0 { + Some(config.message_deduplication.max_entries) + } else { + None + }; + let expiry = if !config.message_deduplication.expiry.is_zero() { + Some(config.message_deduplication.expiry) + } else { + None + }; + + Some(MessageDeduplicator::new(max_entries, expiry)) +} diff --git a/core/server/src/streaming/partitions/in_flight.rs b/core/server/src/streaming/partitions/in_flight.rs new file mode 100644 index 0000000000..ca726df8f3 --- /dev/null +++ b/core/server/src/streaming/partitions/in_flight.rs @@ -0,0 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub use server_common::IggyMessagesBatchSetInFlight; diff --git a/core/server/src/streaming/partitions/journal.rs b/core/server/src/streaming/partitions/journal.rs new file mode 100644 index 0000000000..87bd9cfcb5 --- /dev/null +++ b/core/server/src/streaming/partitions/journal.rs @@ -0,0 +1,212 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::streaming::segments::{IggyMessagesBatchMut, IggyMessagesBatchSet}; +use iggy_common::{IggyByteSize, IggyError}; +use std::fmt::Debug; + +#[derive(Default, Debug)] +pub struct Inner { + /// Base offset for the next journal epoch. After commit(), set to + /// current_offset + 1. Used in `append()`: `current_offset = base_offset + + /// messages_count - 1`. + pub base_offset: u64, + pub current_offset: u64, + pub first_timestamp: u64, + pub end_timestamp: u64, + pub messages_count: u32, + pub size: IggyByteSize, +} + +#[derive(Debug)] +pub struct MemoryMessageJournal { + batches: IggyMessagesBatchSet, + inner: Inner, +} + +impl MemoryMessageJournal { + /// Create an empty journal for a fresh partition (no existing data). + pub fn empty() -> Self { + Self { + batches: IggyMessagesBatchSet::default(), + inner: Inner::default(), + } + } + + /// Create an empty journal positioned at the given offset. Used after + /// bootstrap when the partition already has data on disk up to some offset. + pub fn at_offset(base_offset: u64) -> Self { + Self { + batches: IggyMessagesBatchSet::default(), + inner: Inner { + base_offset, + ..Default::default() + }, + } + } +} + +impl Journal for MemoryMessageJournal { + type Container = IggyMessagesBatchSet; + type Entry = IggyMessagesBatchMut; + type Inner = Inner; + type AppendResult = Result<(u32, u32), IggyError>; + + fn append(&mut self, entry: Self::Entry) -> Self::AppendResult { + let batch_messages_count = entry.count(); + tracing::trace!( + "Coalescing batch with base_offset: {}, current_offset: {}, self.messages_count: {}, batch.count: {}", + self.inner.base_offset, + self.inner.current_offset, + self.inner.messages_count, + batch_messages_count + ); + + // Defense-in-depth: on first append after empty/default state, correct + // base_offset from the batch's actual first offset. Mirrors the existing + // first_timestamp initialization pattern below. Catches code paths that + // create a journal without calling init(). + if self.inner.messages_count == 0 + && let Some(first_offset) = entry.first_offset() + { + // Allow disagreement when either side is 0 (fresh partition or + // reset after purge). Only flag when both are non-zero and differ. + debug_assert!( + self.inner.base_offset == 0 + || first_offset == 0 + || self.inner.base_offset == first_offset, + "journal base_offset ({}) disagrees with batch first_offset ({})", + self.inner.base_offset, + first_offset + ); + self.inner.base_offset = first_offset; + } + + let batch_size = entry.size(); + let first_timestamp = entry.first_timestamp().unwrap(); + let last_timestamp = entry.last_timestamp().unwrap(); + self.batches.add_batch(entry); + + if self.inner.first_timestamp == 0 { + self.inner.first_timestamp = first_timestamp; + } + self.inner.end_timestamp = last_timestamp; + self.inner.messages_count += batch_messages_count; + self.inner.current_offset = self.inner.base_offset + self.inner.messages_count as u64 - 1; + self.inner.size = IggyByteSize::from(self.inner.size.as_bytes_u64() + batch_size as u64); + + Ok((self.inner.messages_count, self.inner.size.as_bytes_u32())) + } + + async fn flush(&self) -> Result<(), IggyError> { + Ok(()) + } + + fn init(&mut self, inner: Self::Inner) { + self.inner = inner + } + + fn get(&self, filter: impl FnOnce(&Self::Container) -> U) -> U { + filter(&self.batches) + } + + fn commit(&mut self) -> Self::Container { + self.inner.base_offset = self.inner.current_offset + 1; + self.inner.first_timestamp = 0; + self.inner.end_timestamp = 0; + self.inner.size = IggyByteSize::default(); + self.inner.messages_count = 0; + std::mem::take(&mut self.batches) + } + + fn is_empty(&self) -> bool { + self.batches.is_empty() + } + + fn inner(&self) -> &Self::Inner { + &self.inner + } + + fn first_offset(&self) -> Option { + if self.is_empty() { + None + } else { + Some(self.inner.base_offset) + } + } + + fn last_offset(&self) -> Option { + if self.is_empty() { + None + } else { + Some(self.inner.current_offset) + } + } + + fn first_timestamp(&self) -> Option { + if self.is_empty() || self.inner.first_timestamp == 0 { + None + } else { + Some(self.inner.first_timestamp) + } + } + + fn last_timestamp(&self) -> Option { + if self.is_empty() || self.inner.end_timestamp == 0 { + None + } else { + Some(self.inner.end_timestamp) + } + } +} + +pub trait Journal { + type Container; + type Entry; + type Inner; + type AppendResult; + + fn init(&mut self, inner: Self::Inner); + + fn append(&mut self, entry: Self::Entry) -> Self::AppendResult; + + fn get(&self, filter: impl FnOnce(&Self::Container) -> U) -> U; + + fn commit(&mut self) -> Self::Container; + + fn is_empty(&self) -> bool; + + fn inner(&self) -> &Self::Inner; + + /// First offset of data in the journal, or None if empty. + fn first_offset(&self) -> Option; + + /// Last offset of data in the journal, or None if empty. + fn last_offset(&self) -> Option; + + /// Timestamp of first message in journal, or None if empty. + fn first_timestamp(&self) -> Option; + + /// Timestamp of last message in journal, or None if empty. + fn last_timestamp(&self) -> Option; + + // `flush` is only useful in case of an journal that has disk backed WAL. + // This could be merged together with `append`, but not doing this for two reasons. + // 1. In case of the `Journal` being used as part of structure that utilizes interior mutability, async with borrow_mut is not possible. + // 2. Having it as separate function allows for more optimal usage patterns, e.g. batching multiple appends before flushing. + fn flush(&self) -> impl Future>; +} diff --git a/core/server/src/streaming/partitions/local_partition.rs b/core/server/src/streaming/partitions/local_partition.rs new file mode 100644 index 0000000000..de49720efa --- /dev/null +++ b/core/server/src/streaming/partitions/local_partition.rs @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Per-shard partition data. +//! +//! Each shard runs on a single-threaded compio runtime, so per-shard data +//! needs NO synchronization. + +use super::{ + consumer_group_offsets::ConsumerGroupOffsets, consumer_offsets::ConsumerOffsets, + journal::MemoryMessageJournal, log::SegmentedLog, +}; +use crate::streaming::{deduplication::MessageDeduplicator, stats::PartitionStats}; +use iggy_common::IggyTimestamp; +use std::sync::{Arc, atomic::AtomicU64}; + +/// Per-shard partition data - mutable, single-threaded access. +#[derive(Debug)] +pub struct LocalPartition { + pub log: SegmentedLog, + pub offset: Arc, + pub consumer_offsets: Arc, + pub consumer_group_offsets: Arc, + pub message_deduplicator: Option>, + pub stats: Arc, + pub created_at: IggyTimestamp, + pub revision_id: u64, + pub should_increment_offset: bool, +} + +impl LocalPartition { + /// Create new partition data with default log. + #[allow(clippy::too_many_arguments)] + pub fn new( + stats: Arc, + offset: Arc, + consumer_offsets: Arc, + consumer_group_offsets: Arc, + message_deduplicator: Option>, + created_at: IggyTimestamp, + revision_id: u64, + should_increment_offset: bool, + ) -> Self { + Self { + log: SegmentedLog::new( + crate::streaming::partitions::journal::MemoryMessageJournal::empty(), + ), + offset, + consumer_offsets, + consumer_group_offsets, + message_deduplicator, + stats, + created_at, + revision_id, + should_increment_offset, + } + } + + /// Create partition data with existing log (e.g., loaded from disk). + #[allow(clippy::too_many_arguments)] + pub fn with_log( + log: SegmentedLog, + stats: Arc, + offset: Arc, + consumer_offsets: Arc, + consumer_group_offsets: Arc, + message_deduplicator: Option>, + created_at: IggyTimestamp, + revision_id: u64, + should_increment_offset: bool, + ) -> Self { + Self { + log, + offset, + consumer_offsets, + consumer_group_offsets, + message_deduplicator, + stats, + created_at, + revision_id, + should_increment_offset, + } + } +} diff --git a/core/server/src/streaming/partitions/local_partitions.rs b/core/server/src/streaming/partitions/local_partitions.rs new file mode 100644 index 0000000000..1b3073fcbe --- /dev/null +++ b/core/server/src/streaming/partitions/local_partitions.rs @@ -0,0 +1,213 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Per-shard partition storage. +//! +//! Single-threaded (compio runtime) - NO synchronization needed! + +use super::local_partition::LocalPartition; +use server_common::sharding::IggyNamespace; +use std::collections::HashMap; + +/// Per-shard partition storage. +/// Single-threaded (compio runtime) - NO synchronization needed! +#[derive(Debug, Default)] +pub struct LocalPartitions { + partitions: HashMap, +} + +impl LocalPartitions { + pub fn new() -> Self { + Self { + partitions: HashMap::new(), + } + } + + pub fn with_capacity(capacity: usize) -> Self { + Self { + partitions: HashMap::with_capacity(capacity), + } + } + + #[inline] + pub fn get(&self, ns: &IggyNamespace) -> Option<&LocalPartition> { + self.partitions.get(ns) + } + + #[inline] + pub fn get_mut(&mut self, ns: &IggyNamespace) -> Option<&mut LocalPartition> { + self.partitions.get_mut(ns) + } + + #[inline] + pub fn insert(&mut self, ns: IggyNamespace, data: LocalPartition) { + self.partitions.insert(ns, data); + } + + #[inline] + pub fn remove(&mut self, ns: &IggyNamespace) -> Option { + self.partitions.remove(ns) + } + + #[inline] + pub fn contains(&self, ns: &IggyNamespace) -> bool { + self.partitions.contains_key(ns) + } + + #[inline] + pub fn len(&self) -> usize { + self.partitions.len() + } + + #[inline] + pub fn is_empty(&self) -> bool { + self.partitions.is_empty() + } + + /// Iterate over all namespaces owned by this shard. + pub fn namespaces(&self) -> impl Iterator { + self.partitions.keys() + } + + /// Iterate over all partition data. + pub fn iter(&self) -> impl Iterator { + self.partitions.iter() + } + + /// Iterate over all partition data mutably. + pub fn iter_mut(&mut self) -> impl Iterator { + self.partitions.iter_mut() + } + + /// Remove multiple partitions at once. + pub fn remove_many(&mut self, namespaces: &[IggyNamespace]) -> Vec { + namespaces + .iter() + .filter_map(|ns| self.partitions.remove(ns)) + .collect() + } + + /// Get partition data, initializing if not present. + pub fn get_or_init(&mut self, ns: IggyNamespace, init: F) -> &mut LocalPartition + where + F: FnOnce() -> LocalPartition, + { + self.partitions.entry(ns).or_insert_with(init) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::streaming::{ + partitions::{ + consumer_group_offsets::ConsumerGroupOffsets, consumer_offsets::ConsumerOffsets, + }, + stats::{PartitionStats, StreamStats, TopicStats}, + }; + use iggy_common::IggyTimestamp; + use std::sync::{Arc, atomic::AtomicU64}; + + fn create_test_partition() -> LocalPartition { + let stream_stats = Arc::new(StreamStats::default()); + let topic_stats = Arc::new(TopicStats::new(stream_stats)); + let partition_stats = Arc::new(PartitionStats::new(topic_stats)); + + LocalPartition::new( + partition_stats, + Arc::new(AtomicU64::new(0)), + Arc::new(ConsumerOffsets::with_capacity(10)), + Arc::new(ConsumerGroupOffsets::with_capacity(10)), + None, + IggyTimestamp::now(), + 1, + true, + ) + } + + #[test] + fn test_basic_operations() { + let mut partitions = LocalPartitions::new(); + let ns = IggyNamespace::new(1, 1, 0); + + assert!(!partitions.contains(&ns)); + assert!(partitions.is_empty()); + + partitions.insert(ns, create_test_partition()); + + assert!(partitions.contains(&ns)); + assert_eq!(partitions.len(), 1); + assert!(partitions.get(&ns).is_some()); + assert!(partitions.get_mut(&ns).is_some()); + + let removed = partitions.remove(&ns); + assert!(removed.is_some()); + assert!(!partitions.contains(&ns)); + assert!(partitions.is_empty()); + } + + #[test] + fn test_iteration() { + let mut partitions = LocalPartitions::new(); + let ns1 = IggyNamespace::new(1, 1, 0); + let ns2 = IggyNamespace::new(1, 1, 1); + let ns3 = IggyNamespace::new(1, 2, 0); + + partitions.insert(ns1, create_test_partition()); + partitions.insert(ns2, create_test_partition()); + partitions.insert(ns3, create_test_partition()); + + let namespaces: Vec<_> = partitions.namespaces().collect(); + assert_eq!(namespaces.len(), 3); + + let pairs: Vec<_> = partitions.iter().collect(); + assert_eq!(pairs.len(), 3); + } + + #[test] + fn test_remove_many() { + let mut partitions = LocalPartitions::new(); + let ns1 = IggyNamespace::new(1, 1, 0); + let ns2 = IggyNamespace::new(1, 1, 1); + let ns3 = IggyNamespace::new(1, 2, 0); + + partitions.insert(ns1, create_test_partition()); + partitions.insert(ns2, create_test_partition()); + partitions.insert(ns3, create_test_partition()); + + let removed = partitions.remove_many(&[ns1, ns2]); + assert_eq!(removed.len(), 2); + assert!(!partitions.contains(&ns1)); + assert!(!partitions.contains(&ns2)); + assert!(partitions.contains(&ns3)); + } + + #[test] + fn test_get_or_init() { + let mut partitions = LocalPartitions::new(); + let ns = IggyNamespace::new(1, 1, 0); + + assert!(!partitions.contains(&ns)); + + let _ = partitions.get_or_init(ns, create_test_partition); + assert!(partitions.contains(&ns)); + + // Second call should not reinitialize + let data = partitions.get_or_init(ns, || panic!("Should not be called")); + assert!(data.should_increment_offset); + } +} diff --git a/core/server/src/streaming/partitions/log.rs b/core/server/src/streaming/partitions/log.rs new file mode 100644 index 0000000000..16318553e5 --- /dev/null +++ b/core/server/src/streaming/partitions/log.rs @@ -0,0 +1,207 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::streaming::{ + partitions::{in_flight::IggyMessagesBatchSetInFlight, journal::Journal}, + segments::{IggyIndexesMut, Segment, storage::Storage}, +}; +use iggy_common::{INDEX_SIZE, IggyMessagesBatch}; +use ringbuffer::AllocRingBuffer; +use std::fmt::Debug; + +const SEGMENTS_CAPACITY: usize = 1024; +const ACCESS_MAP_CAPACITY: usize = 8; +const SIZE_16MB: usize = 16 * 1024 * 1024; + +#[derive(Debug)] +pub struct SegmentedLog +where + J: Journal + Debug, +{ + journal: J, + // Ring buffer tracking recently accessed segment indices for cleanup optimization. + // A background task uses this to identify and close file descriptors for unused segments. + _access_map: AllocRingBuffer, + _cache: (), + segments: Vec, + indexes: Vec>, + storage: Vec, + in_flight: IggyMessagesBatchSetInFlight, +} + +impl Default for SegmentedLog +where + J: Journal + Debug + Default, +{ + fn default() -> Self { + Self { + journal: J::default(), + _access_map: AllocRingBuffer::with_capacity_power_of_2(ACCESS_MAP_CAPACITY), + _cache: (), + segments: Vec::with_capacity(SEGMENTS_CAPACITY), + storage: Vec::with_capacity(SEGMENTS_CAPACITY), + indexes: Vec::with_capacity(SEGMENTS_CAPACITY), + in_flight: IggyMessagesBatchSetInFlight::default(), + } + } +} + +impl SegmentedLog +where + J: Journal + Debug, +{ + pub fn new(journal: J) -> Self { + Self { + journal, + _access_map: AllocRingBuffer::with_capacity_power_of_2(ACCESS_MAP_CAPACITY), + _cache: (), + segments: Vec::with_capacity(SEGMENTS_CAPACITY), + storage: Vec::with_capacity(SEGMENTS_CAPACITY), + indexes: Vec::with_capacity(SEGMENTS_CAPACITY), + in_flight: IggyMessagesBatchSetInFlight::default(), + } + } + + pub fn has_segments(&self) -> bool { + !self.segments.is_empty() + } + + pub fn segments(&self) -> &Vec { + &self.segments + } + + pub fn segments_mut(&mut self) -> &mut Vec { + &mut self.segments + } + + pub fn storages_mut(&mut self) -> &mut Vec { + &mut self.storage + } + + pub fn storages(&self) -> &Vec { + &self.storage + } + + pub fn active_segment(&self) -> &Segment { + self.segments + .last() + .expect("active segment called on empty log") + } + + pub fn active_segment_mut(&mut self) -> &mut Segment { + self.segments + .last_mut() + .expect("active segment called on empty log") + } + + pub fn active_storage(&self) -> &Storage { + self.storage + .last() + .expect("active storage called on empty log") + } + + pub fn active_storage_mut(&mut self) -> &mut Storage { + self.storage + .last_mut() + .expect("active storage called on empty log") + } + + pub fn indexes(&self) -> &Vec> { + &self.indexes + } + + pub fn indexes_mut(&mut self) -> &mut Vec> { + &mut self.indexes + } + + pub fn active_indexes(&self) -> Option<&IggyIndexesMut> { + self.indexes + .last() + .expect("active indexes called on empty log") + .as_ref() + } + + pub fn active_indexes_mut(&mut self) -> Option<&mut IggyIndexesMut> { + self.indexes + .last_mut() + .expect("active indexes called on empty log") + .as_mut() + } + + pub fn clear_active_indexes(&mut self) { + let indexes = self + .indexes + .last_mut() + .expect("active indexes called on empty log"); + *indexes = None; + } + + pub fn ensure_indexes(&mut self) { + let indexes = self + .indexes + .last_mut() + .expect("active indexes called on empty log"); + if indexes.is_none() { + let capacity = SIZE_16MB / INDEX_SIZE; + *indexes = Some(IggyIndexesMut::with_capacity(capacity, 0)); + } + } + + pub fn add_persisted_segment(&mut self, segment: Segment, storage: Storage) { + self.segments.push(segment); + self.storage.push(storage); + self.indexes.push(None); + } + + pub fn set_segment_indexes(&mut self, segment_index: usize, indexes: IggyIndexesMut) { + if let Some(segment_indexes) = self.indexes.get_mut(segment_index) { + *segment_indexes = Some(indexes); + } + } + + pub fn in_flight(&self) -> &IggyMessagesBatchSetInFlight { + &self.in_flight + } + + pub fn in_flight_mut(&mut self) -> &mut IggyMessagesBatchSetInFlight { + &mut self.in_flight + } + + pub fn set_in_flight(&mut self, batches: Vec) { + self.in_flight.set(batches); + } + + pub fn clear_in_flight(&mut self) { + self.in_flight.clear(); + } +} + +impl SegmentedLog +where + J: Journal + Debug, +{ + pub fn journal_mut(&mut self) -> &mut J { + &mut self.journal + } + + pub fn journal(&self) -> &J { + &self.journal + } +} + +impl Log for SegmentedLog where J: Journal + Debug {} +pub trait Log {} diff --git a/core/server/src/streaming/partitions/mod.rs b/core/server/src/streaming/partitions/mod.rs new file mode 100644 index 0000000000..110edaa1a9 --- /dev/null +++ b/core/server/src/streaming/partitions/mod.rs @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod consumer_group_offsets; +pub mod consumer_offset; +pub mod consumer_offsets; +pub mod helpers; +pub mod in_flight; +pub mod journal; +pub mod local_partition; +pub mod local_partitions; +pub mod log; +pub mod ops; +#[cfg(test)] +mod ops_tests; +pub mod segments; +pub mod storage; + +pub const COMPONENT: &str = "STREAMING_PARTITIONS"; diff --git a/core/server/src/streaming/partitions/ops.rs b/core/server/src/streaming/partitions/ops.rs new file mode 100644 index 0000000000..e2d206b413 --- /dev/null +++ b/core/server/src/streaming/partitions/ops.rs @@ -0,0 +1,730 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Shared partition operations that can be used by both production code and tests. +//! +//! This module provides the core logic for polling and loading messages from partitions, +//! avoiding code duplication between `IggyShard` and test harnesses. +//! +//! # Safety invariants +//! +//! The snapshot-then-read pattern in [`get_messages_by_offset`] and +//! [`poll_messages_by_timestamp`] is safe only under **single-threaded shard +//! execution** (compio runtime). Between the metadata snapshot and the actual +//! reads, no other shard request can mutate the partition state because the +//! message pump processes one request at a time. +//! +//! The poll + auto_commit sequence in the handler (`handlers.rs`) is likewise +//! non-atomic but safe for the same reason. +//! +//! If the architecture ever moves to multi-threaded shard processing or adds +//! compaction/message deletion, these invariants must be re-evaluated. + +use super::journal::Journal; +use super::local_partitions::LocalPartitions; +use crate::shard::system::messages::PollingArgs; +use crate::streaming::polling_consumer::PollingConsumer; +use crate::streaming::segments::IggyMessagesBatchSet; +use iggy_common::IggyPollMetadata; +use iggy_common::{IggyError, PollingKind}; +use server_common::sharding::IggyNamespace; +use std::cell::RefCell; +use std::sync::atomic::Ordering; + +/// Poll messages from a partition partitions. +/// +/// This is the core polling logic shared between production code and tests. +pub async fn poll_messages( + local_partitions: &RefCell, + namespace: &IggyNamespace, + consumer: PollingConsumer, + args: PollingArgs, +) -> Result<(IggyPollMetadata, IggyMessagesBatchSet), IggyError> { + let partition_id = namespace.partition_id(); + let count = args.count; + let strategy = args.strategy; + let value = strategy.value; + + // Handle timestamp polling separately - it has different logic + if strategy.kind == PollingKind::Timestamp { + return poll_messages_by_timestamp(local_partitions, namespace, value, count).await; + } + + // Phase 1: Extract metadata and determine start offset + let (metadata, start_offset) = { + let store = local_partitions.borrow(); + let partition = store + .get(namespace) + .expect("local_partitions: partition must exist for poll"); + + let current_offset = partition.offset.load(Ordering::Relaxed); + let metadata = IggyPollMetadata::new(partition_id as u32, current_offset); + + let start_offset = match strategy.kind { + PollingKind::Offset => { + let offset = value; + if offset > current_offset { + return Ok((metadata, IggyMessagesBatchSet::empty())); + } + offset + } + PollingKind::First => partition + .log + .segments() + .first() + .map(|segment| segment.start_offset) + .unwrap_or(0), + PollingKind::Last => { + let mut requested_count = count as u64; + if requested_count > current_offset + 1 { + requested_count = current_offset + 1; + } + 1 + current_offset - requested_count + } + PollingKind::Next => { + let stored_offset = match consumer { + PollingConsumer::Consumer(id, _) => partition + .consumer_offsets + .pin() + .get(&id) + .map(|item| item.offset.load(Ordering::Relaxed)), + PollingConsumer::ConsumerGroup(cg_id, _) => partition + .consumer_group_offsets + .pin() + .get(&cg_id) + .map(|item| item.offset.load(Ordering::Relaxed)), + }; + match stored_offset { + Some(offset) => offset + 1, + None => partition + .log + .segments() + .first() + .map(|segment| segment.start_offset) + .unwrap_or(0), + } + } + PollingKind::Timestamp => unreachable!("Timestamp handled above"), + }; + + if start_offset > current_offset || count == 0 { + return Ok((metadata, IggyMessagesBatchSet::empty())); + } + + (metadata, start_offset) + }; + + // Phase 2: Get messages using hybrid disk+journal logic + let batches = get_messages_by_offset(local_partitions, namespace, start_offset, count).await?; + Ok((metadata, batches)) +} + +/// Get messages by offset, handling the hybrid disk+journal case. +pub async fn get_messages_by_offset( + local_partitions: &RefCell, + namespace: &IggyNamespace, + start_offset: u64, + count: u32, +) -> Result { + if count == 0 { + return Ok(IggyMessagesBatchSet::empty()); + } + + // Snapshot journal and in-flight metadata for routing decisions. + let (journal_first_offset, in_flight_empty, in_flight_first, in_flight_last) = { + let store = local_partitions.borrow(); + let partition = store + .get(namespace) + .expect("local_partitions: partition must exist for poll"); + + let journal = partition.log.journal(); + let in_flight = partition.log.in_flight(); + ( + journal.first_offset(), + in_flight.is_empty(), + in_flight.first_offset(), + in_flight.last_offset(), + ) + }; + + // Lookup ordered by ascending offset: disk -> in-flight -> journal. + // + // Offsets are sequential: disk < in-flight < journal. A request may span + // multiple tiers, so we advance `current` through each sequentially. + // See issue #2715. + + let mut combined = IggyMessagesBatchSet::empty(); + let mut remaining = count; + let mut current = start_offset; + + // Lowest in-memory tier boundary (if any). Disk handles offsets below this. + let in_memory_floor = if !in_flight_empty { + in_flight_first + } else { + journal_first_offset.unwrap_or(u64::MAX) + }; + + // Disk (pre-tier): offsets below the lowest in-memory tier. + if remaining > 0 && current < in_memory_floor { + let disk_count = + ((in_memory_floor.min(current + remaining as u64) - current) as u32).min(remaining); + let disk_messages = + load_messages_from_disk(local_partitions, namespace, current, disk_count).await?; + let loaded = disk_messages.count(); + if loaded > 0 { + current += loaded as u64; + remaining = remaining.saturating_sub(loaded); + combined.add_batch_set(disk_messages); + } + } + + // In-flight: committed data being persisted to disk. + if remaining > 0 && !in_flight_empty && current >= in_flight_first && current <= in_flight_last + { + let in_flight_count = ((in_flight_last - current + 1) as u32).min(remaining); + let in_flight_batches = { + let store = local_partitions.borrow(); + let partition = store + .get(namespace) + .expect("local_partitions: partition must exist for poll"); + partition + .log + .in_flight() + .get_by_offset(current, in_flight_count) + .to_vec() + }; + if !in_flight_batches.is_empty() { + let mut result = IggyMessagesBatchSet::empty(); + result.add_immutable_batches(&in_flight_batches); + let sliced = result.get_by_offset(current, in_flight_count); + let loaded = sliced.count(); + if loaded > 0 { + current += loaded as u64; + remaining = remaining.saturating_sub(loaded); + combined.add_batch_set(sliced); + } + } + } + + // Journal: may hold data from recent appends. + if remaining > 0 + && let Some(jfo) = journal_first_offset + && current >= jfo + { + let journal_messages = { + let store = local_partitions.borrow(); + let partition = store + .get(namespace) + .expect("local_partitions: partition must exist for poll"); + partition + .log + .journal() + .get(|batches| batches.get_by_offset(current, remaining)) + }; + if !journal_messages.is_empty() { + combined.add_batch_set(journal_messages); + } + } + + Ok(combined) +} + +/// Poll messages by timestamp. +async fn poll_messages_by_timestamp( + local_partitions: &RefCell, + namespace: &IggyNamespace, + timestamp: u64, + count: u32, +) -> Result<(IggyPollMetadata, IggyMessagesBatchSet), IggyError> { + let partition_id = namespace.partition_id(); + + // Snapshot metadata from journal and in-flight for routing decisions. + let ( + metadata, + journal_first_ts, + journal_last_ts, + in_flight_empty, + in_flight_first_ts, + in_flight_last_ts, + ) = { + let store = local_partitions.borrow(); + let partition = store + .get(namespace) + .expect("local_partitions: partition must exist for poll"); + + let current_offset = partition.offset.load(Ordering::Relaxed); + let metadata = IggyPollMetadata::new(partition_id as u32, current_offset); + + let journal = partition.log.journal(); + + let in_flight = partition.log.in_flight(); + let (ife, ifts, ilts) = if in_flight.is_empty() { + (true, 0u64, 0u64) + } else { + let first_ts = in_flight + .batches() + .first() + .and_then(|b| b.first_timestamp()) + .unwrap_or(0); + let last_ts = in_flight + .batches() + .last() + .and_then(|b| b.last_timestamp()) + .unwrap_or(0); + (false, first_ts, last_ts) + }; + + ( + metadata, + journal.first_timestamp(), + journal.last_timestamp(), + ife, + ifts, + ilts, + ) + }; + + if count == 0 { + return Ok((metadata, IggyMessagesBatchSet::empty())); + } + + // Three-tier timestamp lookup: disk -> in-flight -> journal. + // Same structure as offset-based polling (see issue #2715). + + let mut combined = IggyMessagesBatchSet::empty(); + let mut remaining = count; + + // Phase 1: Disk - timestamps before in-flight range. + let disk_upper_ts = if !in_flight_empty { + in_flight_first_ts + } else { + journal_first_ts.unwrap_or(u64::MAX) + }; + + if timestamp < disk_upper_ts && remaining > 0 { + let disk_messages = + load_messages_from_disk_by_timestamp(local_partitions, namespace, timestamp, remaining) + .await?; + let loaded = disk_messages.count(); + if loaded > 0 { + remaining = remaining.saturating_sub(loaded); + combined.add_batch_set(disk_messages); + } + } + + // Phase 2: In-flight - committed data being persisted. + if remaining > 0 && !in_flight_empty && timestamp <= in_flight_last_ts { + let in_flight_batches = { + let store = local_partitions.borrow(); + let partition = store + .get(namespace) + .expect("local_partitions: partition must exist for poll"); + partition.log.in_flight().batches().to_vec() + }; + if !in_flight_batches.is_empty() { + let mut batch_set = IggyMessagesBatchSet::empty(); + batch_set.add_immutable_batches(&in_flight_batches); + let filtered = batch_set.get_by_timestamp(timestamp, remaining); + let loaded = filtered.count(); + if loaded > 0 { + remaining = remaining.saturating_sub(loaded); + combined.add_batch_set(filtered); + } + } + } + + // Phase 3: Journal - newest appends (post-commit). + if remaining > 0 + && let Some(jlts) = journal_last_ts + && timestamp <= jlts + { + let journal_messages = { + let store = local_partitions.borrow(); + let partition = store + .get(namespace) + .expect("local_partitions: partition must exist for poll"); + partition + .log + .journal() + .get(|batches| batches.get_by_timestamp(timestamp, remaining)) + }; + if !journal_messages.is_empty() { + combined.add_batch_set(journal_messages); + } + } + + Ok((metadata, combined)) +} + +/// Load messages from disk by offset. +pub async fn load_messages_from_disk( + local_partitions: &RefCell, + namespace: &IggyNamespace, + start_offset: u64, + count: u32, +) -> Result { + if count == 0 { + return Ok(IggyMessagesBatchSet::empty()); + } + + // Get segment range containing the requested offset + let segment_range = { + let store = local_partitions.borrow(); + let partition = store + .get(namespace) + .expect("local_partitions: partition must exist"); + + let segments = partition.log.segments(); + if segments.is_empty() { + return Ok(IggyMessagesBatchSet::empty()); + } + + let start = segments + .iter() + .rposition(|segment| segment.start_offset <= start_offset) + .unwrap_or(0); + let end = segments.len(); + start..end + }; + + let mut remaining_count = count; + let mut batches = IggyMessagesBatchSet::empty(); + let mut current_offset = start_offset; + + for idx in segment_range { + if remaining_count == 0 { + break; + } + + let (segment_start_offset, segment_end_offset) = { + let store = local_partitions.borrow(); + let partition = store + .get(namespace) + .expect("local_partitions: partition must exist"); + + let segment = &partition.log.segments()[idx]; + (segment.start_offset, segment.end_offset) + }; + + let offset = if current_offset < segment_start_offset { + segment_start_offset + } else { + current_offset + }; + + let mut end_offset = offset + (remaining_count - 1) as u64; + if end_offset > segment_end_offset { + end_offset = segment_end_offset; + } + + let messages = load_segment_messages( + local_partitions, + namespace, + idx, + offset, + end_offset, + remaining_count, + segment_start_offset, + ) + .await?; + + let loaded_count = messages.count(); + if loaded_count > 0 { + batches.add_batch_set(messages); + remaining_count = remaining_count.saturating_sub(loaded_count); + current_offset = end_offset + 1; + } else { + break; + } + } + + Ok(batches) +} + +/// Load messages from a specific segment. +async fn load_segment_messages( + local_partitions: &RefCell, + namespace: &IggyNamespace, + idx: usize, + start_offset: u64, + end_offset: u64, + count: u32, + segment_start_offset: u64, +) -> Result { + let relative_start_offset = (start_offset - segment_start_offset) as u32; + + // Check journal for this segment's data (handles callers outside get_messages_by_offset). + let journal_data = { + let store = local_partitions.borrow(); + let partition = store + .get(namespace) + .expect("local_partitions: partition must exist"); + + let journal = partition.log.journal(); + + if let (Some(jfo), Some(jlo)) = (journal.first_offset(), journal.last_offset()) + && start_offset >= jfo + && end_offset <= jlo + { + Some(journal.get(|batches| batches.get_by_offset(start_offset, count))) + } else { + None + } + }; + + if let Some(batches) = journal_data { + return Ok(batches); + } + + // Load from disk + let (index_reader, messages_reader, indexes) = { + let store = local_partitions.borrow(); + let partition = store + .get(namespace) + .expect("local_partitions: partition must exist"); + + let storages = partition.log.storages(); + if idx >= storages.len() { + return Ok(IggyMessagesBatchSet::empty()); + } + + let index_reader = storages[idx] + .index_reader + .as_ref() + .expect("Index reader not initialized") + .clone(); + let messages_reader = storages[idx] + .messages_reader + .as_ref() + .expect("Messages reader not initialized") + .clone(); + let indexes_vec = partition.log.indexes(); + let indexes = indexes_vec + .get(idx) + .and_then(|opt| opt.as_ref()) + .map(|indexes| { + indexes + .slice_by_offset(relative_start_offset, count) + .unwrap_or_default() + }); + (index_reader, messages_reader, indexes) + }; + + let indexes_to_read = if let Some(indexes) = indexes { + if !indexes.is_empty() { + Some(indexes) + } else { + index_reader + .as_ref() + .load_from_disk_by_offset(relative_start_offset, count) + .await? + } + } else { + index_reader + .as_ref() + .load_from_disk_by_offset(relative_start_offset, count) + .await? + }; + + if indexes_to_read.is_none() { + return Ok(IggyMessagesBatchSet::empty()); + } + + let indexes_to_read = indexes_to_read.unwrap(); + let batch = messages_reader + .as_ref() + .load_messages_from_disk(indexes_to_read) + .await?; + + batch.validate_checksums_and_offsets(start_offset)?; + + Ok(IggyMessagesBatchSet::from(batch)) +} + +/// Load messages from disk by timestamp. +async fn load_messages_from_disk_by_timestamp( + local_partitions: &RefCell, + namespace: &IggyNamespace, + timestamp: u64, + count: u32, +) -> Result { + if count == 0 { + return Ok(IggyMessagesBatchSet::empty()); + } + + // Find segment range that might contain messages >= timestamp + let segment_range = { + let store = local_partitions.borrow(); + let partition = store + .get(namespace) + .expect("local_partitions: partition must exist"); + + let segments = partition.log.segments(); + if segments.is_empty() { + return Ok(IggyMessagesBatchSet::empty()); + } + + let start = segments + .iter() + .position(|segment| segment.end_timestamp >= timestamp) + .unwrap_or(segments.len()); + + if start >= segments.len() { + return Ok(IggyMessagesBatchSet::empty()); + } + + start..segments.len() + }; + + let mut remaining_count = count; + let mut batches = IggyMessagesBatchSet::empty(); + + for idx in segment_range { + if remaining_count == 0 { + break; + } + + let segment_end_timestamp = { + let store = local_partitions.borrow(); + let partition = store + .get(namespace) + .expect("local_partitions: partition must exist"); + partition.log.segments()[idx].end_timestamp + }; + + if segment_end_timestamp < timestamp { + continue; + } + + let messages = load_segment_messages_by_timestamp( + local_partitions, + namespace, + idx, + timestamp, + remaining_count, + ) + .await?; + + let messages_count = messages.count(); + if messages_count == 0 { + continue; + } + + remaining_count = remaining_count.saturating_sub(messages_count); + batches.add_batch_set(messages); + } + + Ok(batches) +} + +/// Load messages from a specific segment by timestamp. +async fn load_segment_messages_by_timestamp( + local_partitions: &RefCell, + namespace: &IggyNamespace, + idx: usize, + timestamp: u64, + count: u32, +) -> Result { + if count == 0 { + return Ok(IggyMessagesBatchSet::empty()); + } + + // Check journal first + let journal_data = { + let store = local_partitions.borrow(); + let partition = store + .get(namespace) + .expect("local_partitions: partition must exist"); + + let journal = partition.log.journal(); + + if let (Some(jfts), Some(jlts)) = (journal.first_timestamp(), journal.last_timestamp()) + && timestamp >= jfts + && timestamp <= jlts + { + Some(journal.get(|batches| batches.get_by_timestamp(timestamp, count))) + } else { + None + } + }; + + if let Some(batches) = journal_data { + return Ok(batches); + } + + // Load from disk + let (index_reader, messages_reader, indexes) = { + let store = local_partitions.borrow(); + let partition = store + .get(namespace) + .expect("local_partitions: partition must exist"); + + let storages = partition.log.storages(); + if idx >= storages.len() { + return Ok(IggyMessagesBatchSet::empty()); + } + + let index_reader = storages[idx] + .index_reader + .as_ref() + .expect("Index reader not initialized") + .clone(); + let messages_reader = storages[idx] + .messages_reader + .as_ref() + .expect("Messages reader not initialized") + .clone(); + let indexes_vec = partition.log.indexes(); + let indexes = indexes_vec + .get(idx) + .and_then(|opt| opt.as_ref()) + .map(|indexes| { + indexes + .slice_by_timestamp(timestamp, count) + .unwrap_or_default() + }); + (index_reader, messages_reader, indexes) + }; + + let indexes_to_read = if let Some(indexes) = indexes { + if !indexes.is_empty() { + Some(indexes) + } else { + index_reader + .as_ref() + .load_from_disk_by_timestamp(timestamp, count) + .await? + } + } else { + index_reader + .as_ref() + .load_from_disk_by_timestamp(timestamp, count) + .await? + }; + + if indexes_to_read.is_none() { + return Ok(IggyMessagesBatchSet::empty()); + } + + let indexes_to_read = indexes_to_read.unwrap(); + let batch = messages_reader + .as_ref() + .load_messages_from_disk(indexes_to_read) + .await?; + + Ok(IggyMessagesBatchSet::from(batch)) +} diff --git a/core/server/src/streaming/partitions/ops_tests.rs b/core/server/src/streaming/partitions/ops_tests.rs new file mode 100644 index 0000000000..7769666f4f --- /dev/null +++ b/core/server/src/streaming/partitions/ops_tests.rs @@ -0,0 +1,358 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Tests for the in-flight buffer visibility gap fix (issue #2715). +//! +//! These tests set up "State C" directly in memory: +//! - in-flight holds offsets [0..N-1] (committed journal, not yet on disk) +//! - journal holds offsets [N..N+M-1] (new appends after commit) +//! - no actual disk data +//! +//! Before the fix, Cases 1-3 in get_messages_by_offset never checked +//! in-flight, causing the consumer to miss committed data and either +//! get empty results or skip directly to journal offsets. + +#[cfg(test)] +mod tests { + use crate::streaming::partitions::consumer_group_offsets::ConsumerGroupOffsets; + use crate::streaming::partitions::consumer_offsets::ConsumerOffsets; + use crate::streaming::partitions::journal::{Inner, Journal}; + use crate::streaming::partitions::local_partition::LocalPartition; + use crate::streaming::partitions::local_partitions::LocalPartitions; + use crate::streaming::partitions::ops; + use crate::streaming::polling_consumer::PollingConsumer; + use crate::streaming::stats::{PartitionStats, StreamStats, TopicStats}; + use iggy_common::{IggyByteSize, IggyMessage, PollingStrategy, Sizeable}; + use server_common::sharding::IggyNamespace; + use server_common::{MemoryPool, MemoryPoolConfigOther}; + use std::cell::RefCell; + use std::sync::Arc; + use std::sync::atomic::AtomicU64; + + fn init_memory_pool() { + static INIT: std::sync::Once = std::sync::Once::new(); + INIT.call_once(|| { + let config = MemoryPoolConfigOther { + enabled: false, + size: IggyByteSize::from(64 * 1024 * 1024u64), + bucket_capacity: 256, + }; + MemoryPool::init_pool(&config); + }); + } + + fn create_test_partition(current_offset: u64) -> LocalPartition { + let stream_stats = Arc::new(StreamStats::default()); + let topic_stats = Arc::new(TopicStats::new(stream_stats)); + let partition_stats = Arc::new(PartitionStats::new(topic_stats)); + + LocalPartition::new( + partition_stats, + Arc::new(AtomicU64::new(current_offset)), + Arc::new(ConsumerOffsets::with_capacity(10)), + Arc::new(ConsumerGroupOffsets::with_capacity(10)), + None, + iggy_common::IggyTimestamp::now(), + 1, + true, + ) + } + + fn create_batch(count: u32) -> server_common::IggyMessagesBatchMut { + let messages: Vec = (0..count) + .map(|_| { + IggyMessage::builder() + .payload(bytes::Bytes::from("test-payload")) + .build() + .unwrap() + }) + .collect(); + + let messages_size: u32 = messages + .iter() + .map(|m| m.get_size_bytes().as_bytes_u32()) + .sum(); + server_common::IggyMessagesBatchMut::from_messages(&messages, messages_size) + } + + /// Sets up "State C": in-flight holds committed data, journal holds new + /// appends that arrived after commit but before persist completes. + /// + /// Layout: + /// segment metadata: [0..journal_end] (no actual disk data) + /// in-flight: [0..in_flight_count-1] + /// journal: [in_flight_count..in_flight_count+journal_count-1] + /// partition.offset: in_flight_count + journal_count - 1 + async fn setup_state_c( + in_flight_count: u32, + journal_count: u32, + ) -> (RefCell, IggyNamespace) { + init_memory_pool(); + let ns = IggyNamespace::new(1, 1, 0); + + let in_flight_end = in_flight_count as u64 - 1; + let journal_base = in_flight_end + 1; + let journal_end = journal_base + journal_count as u64 - 1; + + let mut partition = create_test_partition(journal_end); + + let segment = iggy_common::Segment::new(0, IggyByteSize::from(1_073_741_824u64)); + let storage = server_common::SegmentStorage::default(); + partition.log.add_persisted_segment(segment, storage); + + let seg = &mut partition.log.segments_mut()[0]; + seg.end_offset = journal_end; + seg.start_timestamp = 1; + seg.end_timestamp = 2; + + let mut in_flight_batch = create_batch(in_flight_count); + in_flight_batch.prepare_for_persistence(0, 0, 0, None).await; + let in_flight_size = in_flight_batch.size(); + partition.log.set_in_flight(vec![in_flight_batch.freeze()]); + + let journal_inner = Inner { + base_offset: journal_base, + current_offset: 0, + first_timestamp: 0, + end_timestamp: 0, + messages_count: 0, + size: IggyByteSize::default(), + }; + partition.log.journal_mut().init(journal_inner); + + let mut journal_batch = create_batch(journal_count); + journal_batch + .prepare_for_persistence(0, journal_base, in_flight_size, None) + .await; + partition.log.journal_mut().append(journal_batch).unwrap(); + + let mut store = LocalPartitions::new(); + store.insert(ns, partition); + (RefCell::new(store), ns) + } + + // ----------------------------------------------------------------------- + // Issue #2715: In-flight buffer must be reachable when journal is non-empty + // ----------------------------------------------------------------------- + + #[compio::test] + async fn in_flight_reachable_when_journal_non_empty() { + let (store, ns) = setup_state_c(10, 5).await; + let batches = ops::get_messages_by_offset(&store, &ns, 0, 5) + .await + .unwrap(); + assert_eq!(batches.count(), 5); + assert_eq!(batches.first_offset(), Some(0)); + } + + #[compio::test] + async fn spanning_in_flight_and_journal_returns_all_in_order() { + let (store, ns) = setup_state_c(10, 5).await; + let batches = ops::get_messages_by_offset(&store, &ns, 0, 15) + .await + .unwrap(); + assert_eq!(batches.count(), 15); + assert_eq!(batches.first_offset(), Some(0)); + } + + #[compio::test] + async fn polling_next_starts_from_in_flight_not_journal() { + let (store, ns) = setup_state_c(10, 5).await; + let consumer = PollingConsumer::Consumer(1, 0); + let args = + crate::shard::system::messages::PollingArgs::new(PollingStrategy::next(), 15, false); + let (metadata, batches) = ops::poll_messages(&store, &ns, consumer, args) + .await + .unwrap(); + assert_eq!(batches.first_offset(), Some(0)); + assert!(metadata.current_offset >= 14); + } + + #[compio::test] + async fn single_message_at_in_flight_journal_boundary() { + let (store, ns) = setup_state_c(10, 5).await; + let batches = ops::get_messages_by_offset(&store, &ns, 9, 1) + .await + .unwrap(); + assert_eq!(batches.count(), 1); + assert_eq!(batches.first_offset(), Some(9)); + } + + #[compio::test] + async fn single_message_from_in_flight_at_offset_zero() { + let (store, ns) = setup_state_c(10, 5).await; + let batches = ops::get_messages_by_offset(&store, &ns, 0, 1) + .await + .unwrap(); + assert_eq!(batches.count(), 1); + assert_eq!(batches.first_offset(), Some(0)); + } + + // ----------------------------------------------------------------------- + // Existing correct behavior must still work + // ----------------------------------------------------------------------- + + #[compio::test] + async fn in_flight_reachable_when_journal_empty() { + init_memory_pool(); + let ns = IggyNamespace::new(1, 1, 0); + let mut partition = create_test_partition(9); + + let segment = iggy_common::Segment::new(0, IggyByteSize::from(1_073_741_824u64)); + partition + .log + .add_persisted_segment(segment, server_common::SegmentStorage::default()); + let seg = &mut partition.log.segments_mut()[0]; + seg.end_offset = 9; + seg.start_timestamp = 1; + seg.end_timestamp = 2; + + let mut batch = create_batch(10); + batch.prepare_for_persistence(0, 0, 0, None).await; + partition.log.set_in_flight(vec![batch.freeze()]); + + let mut store = LocalPartitions::new(); + store.insert(ns, partition); + let store = RefCell::new(store); + + let batches = ops::get_messages_by_offset(&store, &ns, 0, 10) + .await + .unwrap(); + assert_eq!(batches.count(), 10); + } + + #[compio::test] + async fn journal_reachable_when_in_flight_empty() { + init_memory_pool(); + let ns = IggyNamespace::new(1, 1, 0); + let mut partition = create_test_partition(9); + + let segment = iggy_common::Segment::new(0, IggyByteSize::from(1_073_741_824u64)); + partition + .log + .add_persisted_segment(segment, server_common::SegmentStorage::default()); + let seg = &mut partition.log.segments_mut()[0]; + seg.end_offset = 9; + seg.start_timestamp = 1; + seg.end_timestamp = 2; + + partition.log.journal_mut().init(Inner { + base_offset: 0, + current_offset: 0, + first_timestamp: 0, + end_timestamp: 0, + messages_count: 0, + size: IggyByteSize::default(), + }); + + let mut batch = create_batch(10); + batch.prepare_for_persistence(0, 0, 0, None).await; + partition.log.journal_mut().append(batch).unwrap(); + + let mut store = LocalPartitions::new(); + store.insert(ns, partition); + let store = RefCell::new(store); + + let batches = ops::get_messages_by_offset(&store, &ns, 0, 10) + .await + .unwrap(); + assert_eq!(batches.count(), 10); + } + + #[compio::test] + async fn journal_single_message_at_specific_offset() { + let (store, ns) = setup_state_c(10, 5).await; + let batches = ops::get_messages_by_offset(&store, &ns, 12, 1) + .await + .unwrap(); + assert_eq!(batches.count(), 1); + assert_eq!(batches.first_offset(), Some(12)); + } + + // ----------------------------------------------------------------------- + // Bug reproduction: journal base_offset=0 after restart causes offset skip + // ----------------------------------------------------------------------- + + /// Verifies that journal self-heals base_offset on first append. + /// Without self-healing, a journal created via Default would have + /// base_offset=0, causing incorrect offset calculations. + #[compio::test] + async fn journal_self_heals_base_offset_on_first_append() { + init_memory_pool(); + + let mut journal = crate::streaming::partitions::journal::MemoryMessageJournal::empty(); + assert_eq!(journal.inner().base_offset, 0); + + let mut batch = create_batch(5); + batch.prepare_for_persistence(0, 100, 0, None).await; + journal.append(batch).unwrap(); + + assert_eq!( + journal.inner().base_offset, + 100, + "Journal should self-heal base_offset from batch's first offset" + ); + assert_eq!( + journal.inner().current_offset, + 104, + "current_offset should be base_offset + messages_count - 1" + ); + } + + /// Verifies that slice_by_offset returns None when start_offset is below + /// the batch's range, instead of clamping to index 0 (the old bug). + #[compio::test] + async fn slice_by_offset_rejects_offset_below_range() { + init_memory_pool(); + + let mut batch = create_batch(10); + batch.prepare_for_persistence(0, 100, 0, None).await; + + let result = batch.slice_by_offset(95, 10); + + assert!( + result.is_none(), + "slice_by_offset should return None when start_offset(95) < first_offset(100), \ + got {} messages at offset {:?}", + result.as_ref().map(|r| r.count()).unwrap_or(0), + result.as_ref().and_then(|r| r.first_offset()) + ); + } + + /// After proper journal initialization, polling across the in-flight/journal + /// boundary returns contiguous messages with no gaps. + #[compio::test] + async fn post_restart_poll_with_correct_journal_init_no_skip() { + let (store, ns) = setup_state_c(100, 10).await; + + let batches = ops::get_messages_by_offset(&store, &ns, 95, 10) + .await + .unwrap(); + + assert_eq!(batches.count(), 10, "should return exactly 10 messages"); + assert_eq!( + batches.first_offset(), + Some(95), + "first message should be at requested offset 95" + ); + assert_eq!( + batches.last_offset(), + Some(104), + "last message should be at offset 104 (contiguous)" + ); + } +} diff --git a/core/server/src/streaming/partitions/segments.rs b/core/server/src/streaming/partitions/segments.rs new file mode 100644 index 0000000000..8e40174abe --- /dev/null +++ b/core/server/src/streaming/partitions/segments.rs @@ -0,0 +1,123 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub struct DeletedSegment { + pub end_offset: u64, + pub messages_count: u32, +} + +/* +impl Partition { + pub fn get_segments_count(&self) -> u32 { + self.segments.len() as u32 + } + + pub fn get_segments(&self) -> &Vec { + &self.segments + } + + pub fn get_segments_mut(&mut self) -> &mut Vec { + &mut self.segments + } + + pub fn get_segment(&self, start_offset: u64) -> Option<&Segment> { + self.segments + .iter() + .find(|s| s.start_offset() == start_offset) + } + + pub fn get_segment_mut(&mut self, start_offset: u64) -> Option<&mut Segment> { + self.segments + .iter_mut() + .find(|s| s.start_offset() == start_offset) + } + + pub async fn get_expired_segments_start_offsets(&self, now: IggyTimestamp) -> Vec { + let mut expired_segments = Vec::new(); + for segment in &self.segments { + if segment.is_expired(now).await { + expired_segments.push(segment.start_offset()); + } + } + + expired_segments.sort(); + expired_segments + } + + pub async fn add_persisted_segment(&mut self, start_offset: u64) -> Result<(), IggyError> { + info!( + "Creating the new segment for partition with ID: {}, stream with ID: {}, topic with ID: {}...", + self.partition_id, self.stream_id, self.topic_id + ); + let mut new_segment = Segment::create( + self.stream_id, + self.topic_id, + self.partition_id, + start_offset, + self.config.clone(), + self.message_expiry, + self.size_of_parent_stream.clone(), + self.size_of_parent_topic.clone(), + self.size_bytes.clone(), + self.messages_count_of_parent_stream.clone(), + self.messages_count_of_parent_topic.clone(), + self.messages_count.clone(), + true, + ); + new_segment.open().await.error(|e: &IggyError| { + format!("{COMPONENT} (error: {e}) - failed to persist new segment: {new_segment}",) + })?; + self.segments.push(new_segment); + self.segments_count_of_parent_stream + .fetch_add(1, Ordering::SeqCst); + self.segments.sort_by_key(|a| a.start_offset()); + Ok(()) + } + + pub async fn delete_segment(&mut self, start_offset: u64) -> Result { + let deleted_segment; + { + let segment = self.get_segment_mut(start_offset); + if segment.is_none() { + return Err(IggyError::SegmentNotFound); + } + + let segment = segment.unwrap(); + segment.delete().await.error(|e: &IggyError| { + format!("{COMPONENT} (error: {e}) - failed to delete segment: {segment}",) + })?; + + deleted_segment = DeletedSegment { + end_offset: segment.end_offset(), + messages_count: segment.get_messages_count(), + }; + } + + self.segments_count_of_parent_stream + .fetch_sub(1, Ordering::SeqCst); + + self.segments.retain(|s| s.start_offset() != start_offset); + self.segments.sort_by_key(|a| a.start_offset()); + info!( + "Segment with start offset: {} has been deleted from partition with ID: {}, stream with ID: {}, topic with ID: {}", + start_offset, self.partition_id, self.stream_id, self.topic_id + ); + Ok(deleted_segment) + } +} + +*/ diff --git a/core/server/src/streaming/partitions/storage.rs b/core/server/src/streaming/partitions/storage.rs new file mode 100644 index 0000000000..e641b3ec31 --- /dev/null +++ b/core/server/src/streaming/partitions/storage.rs @@ -0,0 +1,346 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::COMPONENT; +use crate::{ + configs::system::SystemConfig, io::fs_utils::remove_dir_all, + streaming::partitions::consumer_offset::ConsumerOffset, + streaming::polling_consumer::ConsumerGroupId, +}; +use compio::{ + fs::{self, OpenOptions, create_dir_all}, + io::AsyncWriteAtExt, +}; +use iggy_common::{ConsumerKind, IggyError}; +use std::{io::Read, path::Path, sync::atomic::AtomicU64}; +use tracing::{error, trace, warn}; + +pub async fn create_partition_file_hierarchy( + stream_id: usize, + topic_id: usize, + partition_id: usize, + config: &SystemConfig, +) -> Result<(), IggyError> { + let partition_path = config.get_partition_path(stream_id, topic_id, partition_id); + tracing::info!( + "Saving partition with ID: {} for stream with ID: {} and topic with ID: {}...", + partition_id, + stream_id, + topic_id + ); + if !Path::new(&partition_path).exists() && create_dir_all(&partition_path).await.is_err() { + return Err(IggyError::CannotCreatePartitionDirectory( + partition_id, + stream_id, + topic_id, + )); + } + + let offset_path = config.get_offsets_path(stream_id, topic_id, partition_id); + if !Path::new(&offset_path).exists() && create_dir_all(&offset_path).await.is_err() { + tracing::error!( + "Failed to create offsets directory for partition with ID: {} for stream with ID: {} and topic with ID: {}.", + partition_id, + stream_id, + topic_id + ); + return Err(IggyError::CannotCreatePartition( + partition_id, + stream_id, + topic_id, + )); + } + + let consumer_offset_path = config.get_consumer_offsets_path(stream_id, topic_id, partition_id); + if !Path::new(&consumer_offset_path).exists() + && create_dir_all(&consumer_offset_path).await.is_err() + { + tracing::error!( + "Failed to create consumer offsets directory for partition with ID: {} for stream with ID: {} and topic with ID: {}.", + partition_id, + stream_id, + topic_id + ); + return Err(IggyError::CannotCreatePartition( + partition_id, + stream_id, + topic_id, + )); + } + + let consumer_group_offsets_path = + config.get_consumer_group_offsets_path(stream_id, topic_id, partition_id); + if !Path::new(&consumer_group_offsets_path).exists() + && create_dir_all(&consumer_group_offsets_path).await.is_err() + { + tracing::error!( + "Failed to create consumer group offsets directory for partition with ID: {} for stream with ID: {} and topic with ID: {}.", + partition_id, + stream_id, + topic_id + ); + return Err(IggyError::CannotCreatePartition( + partition_id, + stream_id, + topic_id, + )); + } + + tracing::info!( + "Saved partition with start ID: {} for stream with ID: {} and topic with ID: {}, path: {}.", + partition_id, + stream_id, + topic_id, + partition_path + ); + + Ok(()) +} + +pub async fn delete_partitions_from_disk( + stream_id: usize, + topic_id: usize, + partition_id: usize, + config: &SystemConfig, +) -> Result<(), IggyError> { + let partition_path = config.get_partition_path(stream_id, topic_id, partition_id); + remove_dir_all(&partition_path).await.map_err(|_| { + IggyError::CannotDeletePartitionDirectory(stream_id, topic_id, partition_id) + })?; + tracing::info!( + "Deleted partition files for partition with ID: {} stream with ID: {} and topic with ID: {}.", + partition_id, + stream_id, + topic_id + ); + Ok(()) +} + +pub async fn delete_persisted_offset(path: &str) -> Result<(), IggyError> { + if !Path::new(path).exists() { + tracing::trace!("Consumer offset file does not exist: {path}."); + return Ok(()); + } + + if fs::remove_file(path).await.is_err() { + tracing::error!("Cannot delete consumer offset file: {path}."); + return Err(IggyError::CannotDeleteConsumerOffsetFile(path.to_owned())); + } + Ok(()) +} + +pub async fn persist_offset(path: &str, offset: u64) -> Result<(), IggyError> { + let mut file = OpenOptions::new() + .write(true) + .create(true) + .open(path) + .await + .map_err(|_| IggyError::CannotOpenConsumerOffsetsFile(path.to_owned()))?; + let buf = offset.to_le_bytes(); + file.write_all_at(buf, 0) + .await + .0 + .map_err(|_| IggyError::CannotWriteToFile)?; + tracing::trace!("Stored consumer offset value: {}, path: {}", offset, path); + Ok(()) +} + +pub fn load_consumer_offsets(path: &str) -> Result, IggyError> { + trace!("Loading consumer offsets from path: {path}..."); + let dir_entries = std::fs::read_dir(path); + if dir_entries.is_err() { + return Err(IggyError::CannotReadConsumerOffsets(path.to_owned())); + } + + let mut consumer_offsets = Vec::new(); + let dir_entries = dir_entries.unwrap(); + for dir_entry in dir_entries { + let dir_entry = match dir_entry { + Ok(entry) => entry, + Err(e) => { + warn!( + "Failed to read directory entry in consumer offsets path: {path}, \ + error: {e}, skipping." + ); + continue; + } + }; + + let metadata = match dir_entry.metadata() { + Ok(m) => m, + Err(e) => { + warn!( + "Failed to read metadata for entry in consumer offsets path: {path}, \ + error: {e}, skipping." + ); + continue; + } + }; + + if metadata.is_dir() { + continue; + } + + let name = dir_entry.file_name().to_string_lossy().to_string(); + let consumer_id = match name.parse::() { + Ok(id) => id, + Err(_) => { + warn!( + "Unexpected non-numeric consumer offset file: '{}', skipping.", + name + ); + continue; + } + }; + + let path = dir_entry.path(); + let path = path.to_str(); + if path.is_none() { + error!("Invalid consumer ID path for file with name: '{}'.", name); + continue; + } + + let path = path.unwrap().to_string(); + let file = match std::fs::File::open(&path) { + Ok(f) => f, + Err(e) => { + warn!( + "{COMPONENT} (error: {e}) - failed to open offset file, \ + path: {path}, skipping." + ); + continue; + } + }; + let mut cursor = std::io::Cursor::new(file); + let mut offset = [0; 8]; + if let Err(e) = cursor.get_mut().read_exact(&mut offset) { + warn!( + "{COMPONENT} (error: {e}) - failed to read consumer offset from file \ + (truncated or corrupt?), path: {path}, skipping." + ); + continue; + } + let offset = AtomicU64::new(u64::from_le_bytes(offset)); + + consumer_offsets.push(ConsumerOffset { + kind: ConsumerKind::Consumer, + consumer_id, + offset, + path, + }); + } + + consumer_offsets.sort_by_key(|o| o.consumer_id); + Ok(consumer_offsets) +} + +pub fn load_consumer_group_offsets( + path: &str, +) -> Result, IggyError> { + trace!("Loading consumer group offsets from path: {path}..."); + let dir_entries = std::fs::read_dir(path); + if dir_entries.is_err() { + return Err(IggyError::CannotReadConsumerOffsets(path.to_owned())); + } + + let mut consumer_group_offsets = Vec::new(); + let dir_entries = dir_entries.unwrap(); + for dir_entry in dir_entries { + let dir_entry = match dir_entry { + Ok(entry) => entry, + Err(e) => { + warn!( + "Failed to read directory entry in consumer group offsets path: {path}, \ + error: {e}, skipping." + ); + continue; + } + }; + + let metadata = match dir_entry.metadata() { + Ok(m) => m, + Err(e) => { + warn!( + "Failed to read metadata for entry in consumer group offsets path: {path}, \ + error: {e}, skipping." + ); + continue; + } + }; + + if metadata.is_dir() { + continue; + } + + let name = dir_entry.file_name().to_string_lossy().to_string(); + + let consumer_group_id = match name.parse::() { + Ok(id) => id, + Err(_) => { + warn!( + "Unexpected non-numeric consumer group offset file: '{}', skipping.", + name + ); + continue; + } + }; + let consumer_group_id = ConsumerGroupId(consumer_group_id as usize); + + let path = dir_entry.path(); + let path = path.to_str(); + if path.is_none() { + error!( + "Invalid consumer group offset path for file with name: '{}'.", + name + ); + continue; + } + + let path = path.unwrap().to_string(); + let file = match std::fs::File::open(&path) { + Ok(f) => f, + Err(e) => { + warn!( + "{COMPONENT} (error: {e}) - failed to open offset file, \ + path: {path}, skipping." + ); + continue; + } + }; + let mut cursor = std::io::Cursor::new(file); + let mut offset = [0; 8]; + if let Err(e) = cursor.get_mut().read_exact(&mut offset) { + warn!( + "{COMPONENT} (error: {e}) - failed to read consumer group offset from file \ + (truncated or corrupt?), path: {path}, skipping." + ); + continue; + } + let offset = AtomicU64::new(u64::from_le_bytes(offset)); + + let consumer_offset = ConsumerOffset { + kind: ConsumerKind::ConsumerGroup, + consumer_id: consumer_group_id.0 as u32, + offset, + path, + }; + + consumer_group_offsets.push((consumer_group_id, consumer_offset)); + } + + Ok(consumer_group_offsets) +} diff --git a/core/server/src/streaming/persistence/mod.rs b/core/server/src/streaming/persistence/mod.rs new file mode 100644 index 0000000000..23f1d21799 --- /dev/null +++ b/core/server/src/streaming/persistence/mod.rs @@ -0,0 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod persister; + +pub const COMPONENT: &str = "STREAMING_PERSISTENCE"; diff --git a/core/server/src/streaming/persistence/persister.rs b/core/server/src/streaming/persistence/persister.rs new file mode 100644 index 0000000000..466b68c944 --- /dev/null +++ b/core/server/src/streaming/persistence/persister.rs @@ -0,0 +1,166 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::streaming::persistence::COMPONENT; +use crate::streaming::utils::file; +use compio::buf::IoBuf; +use compio::fs::remove_file; +use compio::io::AsyncWriteAtExt; +use err_trail::ErrContext; +use iggy_common::IggyError; +use std::fmt::Debug; + +#[derive(Debug)] +pub enum PersisterKind { + File(FilePersister), + FileWithSync(FileWithSyncPersister), +} + +impl PersisterKind { + pub async fn append(&self, path: &str, bytes: B) -> Result<(), IggyError> { + match self { + PersisterKind::File(p) => p.append(path, bytes).await, + PersisterKind::FileWithSync(p) => p.append(path, bytes).await, + } + } + + pub async fn overwrite(&self, path: &str, bytes: B) -> Result<(), IggyError> { + match self { + PersisterKind::File(p) => p.overwrite(path, bytes).await, + PersisterKind::FileWithSync(p) => p.overwrite(path, bytes).await, + } + } + + pub async fn delete(&self, path: &str) -> Result<(), IggyError> { + match self { + PersisterKind::File(p) => p.delete(path).await, + PersisterKind::FileWithSync(p) => p.delete(path).await, + } + } +} + +#[derive(Debug)] +pub struct FilePersister; + +impl FilePersister { + pub async fn append(&self, path: &str, bytes: B) -> Result<(), IggyError> { + let (mut file, position) = file::append(path) + .await + .error(|e: &std::io::Error| { + format!("{COMPONENT} (error: {e}) - failed to append to file: {path}") + }) + .map_err(|_| IggyError::CannotAppendToFile)?; + file.write_all_at(bytes, position) + .await + .0 + .error(|e: &std::io::Error| { + format!("{COMPONENT} (error: {e}) - failed to write data to file: {path}") + }) + .map_err(|_| IggyError::CannotWriteToFile)?; + Ok(()) + } + + pub async fn overwrite(&self, path: &str, bytes: B) -> Result<(), IggyError> { + let mut file = file::overwrite(path) + .await + .error(|e: &std::io::Error| { + format!("{COMPONENT} (error: {e}) - failed to overwrite file: {path}") + }) + .map_err(|_| IggyError::CannotOverwriteFile)?; + let position = 0; + file.write_all_at(bytes, position) + .await + .0 + .error(|e: &std::io::Error| { + format!("{COMPONENT} (error: {e}) - failed to write data to file: {path}") + }) + .map_err(|_| IggyError::CannotWriteToFile)?; + Ok(()) + } + + pub async fn delete(&self, path: &str) -> Result<(), IggyError> { + remove_file(path) + .await + .error(|e: &std::io::Error| { + format!("{COMPONENT} (error: {e}) - failed to delete file: {path}") + }) + .map_err(|_| IggyError::CannotDeleteFile)?; + Ok(()) + } +} + +#[derive(Debug)] +pub struct FileWithSyncPersister; + +impl FileWithSyncPersister { + pub async fn append(&self, path: &str, bytes: B) -> Result<(), IggyError> { + let (mut file, position) = file::append(path) + .await + .error(|e: &std::io::Error| { + format!("{COMPONENT} (error: {e}) - failed to append to file: {path}") + }) + .map_err(|_| IggyError::CannotAppendToFile)?; + file.write_all_at(bytes, position) + .await + .0 + .error(|e: &std::io::Error| { + format!("{COMPONENT} (error: {e}) - failed to write data to file: {path}") + }) + .map_err(|_| IggyError::CannotWriteToFile)?; + file.sync_all() + .await + .error(|e: &std::io::Error| { + format!("{COMPONENT} (error: {e}) - failed to sync file after appending: {path}") + }) + .map_err(|_| IggyError::CannotSyncFile)?; + Ok(()) + } + + pub async fn overwrite(&self, path: &str, bytes: B) -> Result<(), IggyError> { + let mut file = file::overwrite(path) + .await + .error(|e: &std::io::Error| { + format!("{COMPONENT} (error: {e}) - failed to overwrite file: {path}") + }) + .map_err(|_| IggyError::CannotOverwriteFile)?; + let position = 0; + file.write_all_at(bytes, position) + .await + .0 + .error(|e: &std::io::Error| { + format!("{COMPONENT} (error: {e}) - failed to write data to file: {path}") + }) + .map_err(|_| IggyError::CannotWriteToFile)?; + file.sync_all() + .await + .error(|e: &std::io::Error| { + format!("{COMPONENT} (error: {e}) - failed to sync file after overwriting: {path}") + }) + .map_err(|_| IggyError::CannotSyncFile)?; + Ok(()) + } + + pub async fn delete(&self, path: &str) -> Result<(), IggyError> { + remove_file(path) + .await + .error(|e: &std::io::Error| { + format!("{COMPONENT} (error: {e}) - failed to delete file: {path}") + }) + .map_err(|_| IggyError::CannotDeleteFile)?; + Ok(()) + } +} diff --git a/core/server/src/streaming/polling_consumer.rs b/core/server/src/streaming/polling_consumer.rs new file mode 100644 index 0000000000..02f657f288 --- /dev/null +++ b/core/server/src/streaming/polling_consumer.rs @@ -0,0 +1,129 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub use iggy_common::ConsumerGroupId; +use iggy_common::{IdKind, Identifier, calculate_32}; +use std::fmt::{Display, Formatter}; + +#[derive(Debug, PartialEq, Eq, Hash, Copy, Clone)] +pub struct MemberId(pub usize); + +impl Display for MemberId { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +#[derive(Debug, PartialEq, Copy, Clone)] +pub enum PollingConsumer { + Consumer(usize, usize), // Consumer ID + Partition ID + ConsumerGroup(ConsumerGroupId, MemberId), // Consumer Group ID + Member ID +} + +impl PollingConsumer { + pub fn consumer(consumer_id: &Identifier, partition_id: usize) -> Self { + PollingConsumer::Consumer(Self::resolve_consumer_id(consumer_id), partition_id) + } + + pub fn consumer_group(consumer_group_id: usize, member_id: usize) -> Self { + PollingConsumer::ConsumerGroup(ConsumerGroupId(consumer_group_id), MemberId(member_id)) + } + + pub fn resolve_consumer_id(identifier: &Identifier) -> usize { + match identifier.kind { + IdKind::Numeric => identifier.get_u32_value().unwrap() as usize, + IdKind::String => calculate_32(&identifier.value) as usize, + } + } +} + +impl Display for PollingConsumer { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + PollingConsumer::Consumer(consumer_id, partition_id) => write!( + f, + "consumer ID: {consumer_id}, partition ID: {partition_id}" + ), + PollingConsumer::ConsumerGroup(consumer_group_id, member_id) => { + write!( + f, + "consumer group ID: {consumer_group_id}, member ID: {member_id}" + ) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use iggy_common::Consumer; + + #[test] + fn given_consumer_with_numeric_id_polling_consumer_should_be_created() { + let consumer_id_value = 1; + let partition_id = 3; + let consumer_id = Identifier::numeric(consumer_id_value).unwrap(); + let consumer = Consumer::new(consumer_id); + let polling_consumer = PollingConsumer::consumer(&consumer.id, partition_id); + + assert_eq!( + polling_consumer, + PollingConsumer::Consumer(consumer_id_value as usize, partition_id) + ); + } + + #[test] + fn given_consumer_with_named_id_polling_consumer_should_be_created() { + let consumer_name = "consumer"; + let partition_id = 3; + let consumer_id = Identifier::named(consumer_name).unwrap(); + let consumer = Consumer::new(consumer_id); + + let resolved_consumer_id = PollingConsumer::resolve_consumer_id(&consumer.id); + let polling_consumer = PollingConsumer::consumer(&consumer.id, partition_id); + + assert_eq!( + polling_consumer, + PollingConsumer::Consumer(resolved_consumer_id, partition_id) + ); + } + + #[test] + fn given_consumer_group_with_numeric_id_polling_consumer_group_should_be_created() { + let group_id = 1; + let client_id = 2; + let polling_consumer = PollingConsumer::consumer_group(group_id, client_id); + + match polling_consumer { + PollingConsumer::ConsumerGroup(consumer_group_id, member_id) => { + assert_eq!(consumer_group_id, ConsumerGroupId(group_id)); + assert_eq!(member_id, MemberId(client_id)); + } + _ => panic!("Expected ConsumerGroup"), + } + } + + #[test] + fn given_distinct_named_ids_unique_polling_consumer_ids_should_be_created() { + let name1 = Identifier::named("consumer1").unwrap(); + let name2 = Identifier::named("consumer2").unwrap(); + let id1 = PollingConsumer::resolve_consumer_id(&name1); + let id2 = PollingConsumer::resolve_consumer_id(&name2); + assert_ne!(id1, id2); + } +} diff --git a/core/server/src/streaming/segments/indexes/index_reader.rs b/core/server/src/streaming/segments/indexes/index_reader.rs new file mode 100644 index 0000000000..3decdb46e7 --- /dev/null +++ b/core/server/src/streaming/segments/indexes/index_reader.rs @@ -0,0 +1,19 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#[allow(unused_imports)] +pub use server_common::IndexReader; diff --git a/core/server/src/streaming/segments/indexes/index_writer.rs b/core/server/src/streaming/segments/indexes/index_writer.rs new file mode 100644 index 0000000000..c77bada32e --- /dev/null +++ b/core/server/src/streaming/segments/indexes/index_writer.rs @@ -0,0 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub use server_common::IndexWriter; diff --git a/core/server/src/streaming/segments/indexes/mod.rs b/core/server/src/streaming/segments/indexes/mod.rs new file mode 100644 index 0000000000..d6bfd8ffaf --- /dev/null +++ b/core/server/src/streaming/segments/indexes/mod.rs @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod index_reader; +mod index_writer; + +pub use index_writer::IndexWriter; +pub use server_common::IggyIndexesMut; diff --git a/core/server/src/streaming/segments/memory_journal.rs b/core/server/src/streaming/segments/memory_journal.rs new file mode 100644 index 0000000000..5cd17fb5a6 --- /dev/null +++ b/core/server/src/streaming/segments/memory_journal.rs @@ -0,0 +1,17 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + diff --git a/core/server/src/streaming/segments/messages/messages_reader.rs b/core/server/src/streaming/segments/messages/messages_reader.rs new file mode 100644 index 0000000000..3398351b8d --- /dev/null +++ b/core/server/src/streaming/segments/messages/messages_reader.rs @@ -0,0 +1,19 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#[allow(unused_imports)] +pub use server_common::MessagesReader; diff --git a/core/server/src/streaming/segments/messages/messages_writer.rs b/core/server/src/streaming/segments/messages/messages_writer.rs new file mode 100644 index 0000000000..b9911b201c --- /dev/null +++ b/core/server/src/streaming/segments/messages/messages_writer.rs @@ -0,0 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub use server_common::MessagesWriter; diff --git a/core/server/src/streaming/segments/messages/mod.rs b/core/server/src/streaming/segments/messages/mod.rs new file mode 100644 index 0000000000..fbd279ec7c --- /dev/null +++ b/core/server/src/streaming/segments/messages/mod.rs @@ -0,0 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod messages_reader; +mod messages_writer; + +pub use messages_writer::MessagesWriter; diff --git a/core/server/src/streaming/segments/mod.rs b/core/server/src/streaming/segments/mod.rs new file mode 100644 index 0000000000..df4bd3c53e --- /dev/null +++ b/core/server/src/streaming/segments/mod.rs @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod indexes; +mod messages; +mod segment; +mod types; + +pub mod storage; + +pub use indexes::IggyIndexesMut; +pub use indexes::IndexWriter; +pub use messages::MessagesWriter; +pub use segment::Segment; +pub use types::IggyMessageHeaderViewMut; +pub use types::IggyMessageViewMut; +pub use types::IggyMessagesBatchMut; +pub use types::IggyMessagesBatchSet; + +pub use crate::configs::validators::SEGMENT_MAX_SIZE_BYTES; diff --git a/core/server/src/streaming/segments/segment.rs b/core/server/src/streaming/segments/segment.rs new file mode 100644 index 0000000000..f2938d3541 --- /dev/null +++ b/core/server/src/streaming/segments/segment.rs @@ -0,0 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub use iggy_common::Segment; diff --git a/core/server/src/streaming/segments/storage.rs b/core/server/src/streaming/segments/storage.rs new file mode 100644 index 0000000000..8d0212ce39 --- /dev/null +++ b/core/server/src/streaming/segments/storage.rs @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub use server_common::SegmentStorage as Storage; + +use crate::configs::system::SystemConfig; +use iggy_common::IggyError; + +/// Creates a new storage for the specified partition with the given start offset +pub async fn create_segment_storage( + config: &SystemConfig, + stream_id: usize, + topic_id: usize, + partition_id: usize, + messages_size: u64, + indexes_size: u64, + start_offset: u64, +) -> Result { + let messages_path = + config.get_messages_file_path(stream_id, topic_id, partition_id, start_offset); + let index_path = config.get_index_path(stream_id, topic_id, partition_id, start_offset); + let log_fsync = config.partition.enforce_fsync; + let index_fsync = config.partition.enforce_fsync; + let file_exists = false; + + Storage::new( + &messages_path, + &index_path, + messages_size, + indexes_size, + log_fsync, + index_fsync, + file_exists, + ) + .await +} diff --git a/core/server/src/streaming/segments/types/mod.rs b/core/server/src/streaming/segments/types/mod.rs new file mode 100644 index 0000000000..a822fd3fc5 --- /dev/null +++ b/core/server/src/streaming/segments/types/mod.rs @@ -0,0 +1,19 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub use iggy_common::{IggyMessageHeaderViewMut, IggyMessageViewMut}; +pub use server_common::{IggyMessagesBatchMut, IggyMessagesBatchSet}; diff --git a/core/server/src/streaming/session.rs b/core/server/src/streaming/session.rs new file mode 100644 index 0000000000..57984663fe --- /dev/null +++ b/core/server/src/streaming/session.rs @@ -0,0 +1,105 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use iggy_common::UserId; +use std::cell::Cell; +use std::fmt::Display; +use std::net::SocketAddr; + +// This might be extended with more fields in the future e.g. custom name, permissions etc. +#[derive(Debug, Clone)] +pub struct Session { + pub client_id: u32, + user_id: Cell, + active: Cell, + pub ip_address: SocketAddr, + pub migrated: Cell, +} + +impl Session { + pub fn new(client_id: u32, user_id: UserId, ip_address: SocketAddr) -> Self { + Self { + client_id, + user_id: Cell::new(user_id), + active: Cell::new(true), + migrated: Cell::new(false), + ip_address, + } + } + + pub fn stateless(user_id: UserId, ip_address: SocketAddr) -> Self { + Self::new(0, user_id, ip_address) + } + + pub fn from_client_id(client_id: u32, ip_address: SocketAddr) -> Self { + Self::new(client_id, u32::MAX, ip_address) + } + + pub fn get_user_id(&self) -> UserId { + self.user_id.get() + } + + pub fn set_user_id(&self, user_id: UserId) { + self.user_id.set(user_id); + } + + pub fn set_stale(&self) { + self.active.set(false); + } + + /// Returns true if this session has been migrated to another shard. + /// + /// Prevents socket ping-ponging between shards. Subsequent wrong-shard requests use message forwarding instead + pub fn is_migrated(&self) -> bool { + self.migrated.get() + } + + pub fn set_migrated(&self) { + self.migrated.set(true) + } + + pub fn clear_user_id(&self) { + self.set_user_id(u32::MAX); + } + + pub fn is_active(&self) -> bool { + self.active.get() + } + + pub fn is_authenticated(&self) -> bool { + self.get_user_id() != u32::MAX + } +} + +impl Display for Session { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let user_id = self.get_user_id(); + if user_id != u32::MAX { + write!( + f, + "client ID: {}, user ID: {}, IP address: {}", + self.client_id, user_id, self.ip_address + ) + } else { + write!( + f, + "client ID: {}, IP address: {}", + self.client_id, self.ip_address + ) + } + } +} diff --git a/core/server/src/streaming/stats/mod.rs b/core/server/src/streaming/stats/mod.rs new file mode 100644 index 0000000000..a745b9bb3a --- /dev/null +++ b/core/server/src/streaming/stats/mod.rs @@ -0,0 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub use iggy_common::{PartitionStats, StreamStats, TopicStats}; diff --git a/core/server/src/streaming/storage.rs b/core/server/src/streaming/storage.rs new file mode 100644 index 0000000000..9c91b52dbc --- /dev/null +++ b/core/server/src/streaming/storage.rs @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::persistence::persister::PersisterKind; +use crate::configs::system::SystemConfig; +use crate::shard::system::storage::FileSystemInfoStorage; +use std::sync::Arc; + +#[derive(Debug, Clone)] +pub struct SystemStorage { + pub info: Arc, + pub persister: Arc, +} + +impl SystemStorage { + pub fn new(config: Arc, persister: Arc) -> Self { + Self { + info: Arc::new(FileSystemInfoStorage::new( + config.get_state_info_path(), + persister.clone(), + )), + persister, + } + } +} diff --git a/core/server/src/streaming/streams/mod.rs b/core/server/src/streaming/streams/mod.rs new file mode 100644 index 0000000000..976caf7a52 --- /dev/null +++ b/core/server/src/streaming/streams/mod.rs @@ -0,0 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod storage; + +pub const COMPONENT: &str = "STREAMING_STREAMS"; diff --git a/core/server/src/streaming/streams/storage.rs b/core/server/src/streaming/streams/storage.rs new file mode 100644 index 0000000000..5068c1d1e2 --- /dev/null +++ b/core/server/src/streaming/streams/storage.rs @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::{configs::system::SystemConfig, io::fs_utils::remove_dir_all}; +use compio::fs::create_dir_all; +use iggy_common::IggyError; +use std::path::Path; + +pub async fn create_stream_file_hierarchy( + id: usize, + config: &SystemConfig, +) -> Result<(), IggyError> { + let path = config.get_stream_path(id); + + if !Path::new(&path).exists() && create_dir_all(&path).await.is_err() { + return Err(IggyError::CannotCreateStreamDirectory( + id as u32, + path.clone(), + )); + } + + tracing::info!("Saved stream with ID: {}.", id); + Ok(()) +} + +/// Delete stream directory using only IDs. +/// Does not require slab access - works with SharedMetadata. +/// topics_with_partitions: Vec<(topic_id, Vec)> +pub async fn delete_stream_directory( + stream_id: usize, + topics_with_partitions: &[(usize, Vec)], + config: &SystemConfig, +) -> Result<(), IggyError> { + use crate::streaming::topics::storage::delete_topic_directory; + + let stream_path = config.get_stream_path(stream_id); + if !Path::new(&stream_path).exists() { + return Err(IggyError::StreamDirectoryNotFound(stream_path)); + } + + // Delete all topics + for (topic_id, partition_ids) in topics_with_partitions { + delete_topic_directory(stream_id, *topic_id, partition_ids, config).await?; + } + + remove_dir_all(&stream_path) + .await + .map_err(|_| IggyError::CannotDeleteStreamDirectory(stream_id as u32))?; + tracing::info!("Deleted stream files for stream with ID: {}.", stream_id); + Ok(()) +} diff --git a/core/server/src/streaming/topics/helpers.rs b/core/server/src/streaming/topics/helpers.rs new file mode 100644 index 0000000000..119222b427 --- /dev/null +++ b/core/server/src/streaming/topics/helpers.rs @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use iggy_common::calculate_32; + +pub fn calculate_partition_id_by_messages_key_hash( + upperbound: usize, + messages_key: &[u8], +) -> usize { + let messages_key_hash = calculate_32(messages_key) as usize; + let partition_id = messages_key_hash % upperbound; + tracing::trace!( + "Calculated partition ID: {} for messages key: {:?}, hash: {}", + partition_id, + messages_key, + messages_key_hash + ); + partition_id +} diff --git a/core/server/src/streaming/topics/mod.rs b/core/server/src/streaming/topics/mod.rs new file mode 100644 index 0000000000..da72843cce --- /dev/null +++ b/core/server/src/streaming/topics/mod.rs @@ -0,0 +1,21 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod helpers; +pub mod storage; + +pub const COMPONENT: &str = "STREAMING_TOPICS"; diff --git a/core/server/src/streaming/topics/storage.rs b/core/server/src/streaming/topics/storage.rs new file mode 100644 index 0000000000..10774d9a5e --- /dev/null +++ b/core/server/src/streaming/topics/storage.rs @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use compio::fs::create_dir_all; +use iggy_common::IggyError; +use std::path::Path; + +use crate::{ + configs::system::SystemConfig, io::fs_utils::remove_dir_all, + streaming::partitions::storage::delete_partitions_from_disk, +}; + +pub async fn create_topic_file_hierarchy( + stream_id: usize, + topic_id: usize, + config: &SystemConfig, +) -> Result<(), IggyError> { + let topic_path = config.get_topic_path(stream_id, topic_id); + let partitions_path = config.get_partitions_path(stream_id, topic_id); + if !Path::new(&topic_path).exists() && create_dir_all(&topic_path).await.is_err() { + return Err(IggyError::CannotCreateTopicDirectory( + topic_id, stream_id, topic_path, + )); + } + tracing::info!( + "Saved topic with ID: {}. for stream with ID: {}", + topic_id, + stream_id + ); + + if !Path::new(&partitions_path).exists() && create_dir_all(&partitions_path).await.is_err() { + return Err(IggyError::CannotCreatePartitionsDirectory( + stream_id, topic_id, + )); + } + Ok(()) +} + +/// Delete topic directory and all partition subdirectories using only IDs. +/// Does not require slab access - works with SharedMetadata. +pub async fn delete_topic_directory( + stream_id: usize, + topic_id: usize, + partition_ids: &[usize], + config: &SystemConfig, +) -> Result<(), IggyError> { + let topic_path = config.get_topic_path(stream_id, topic_id); + if !Path::new(&topic_path).exists() { + return Err(IggyError::TopicDirectoryNotFound(topic_path)); + } + + // Delete partition directories + for &partition_id in partition_ids { + delete_partitions_from_disk(stream_id, topic_id, partition_id, config).await?; + } + + // Delete the topic directory itself + remove_dir_all(&topic_path).await.map_err(|_| { + IggyError::CannotDeleteTopicDirectory(topic_id as u32, stream_id as u32, topic_path) + })?; + + tracing::info!( + "Deleted topic files for topic with ID: {} in stream with ID: {}.", + topic_id, + stream_id + ); + Ok(()) +} diff --git a/core/server/src/streaming/users/mod.rs b/core/server/src/streaming/users/mod.rs new file mode 100644 index 0000000000..36e32eca12 --- /dev/null +++ b/core/server/src/streaming/users/mod.rs @@ -0,0 +1,18 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod user; diff --git a/core/server/src/streaming/users/user.rs b/core/server/src/streaming/users/user.rs new file mode 100644 index 0000000000..334370dd40 --- /dev/null +++ b/core/server/src/streaming/users/user.rs @@ -0,0 +1,137 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::streaming::utils::crypto; +use dashmap::DashMap; +use iggy_common::IggyTimestamp; +use iggy_common::PersonalAccessToken; +use iggy_common::UserStatus; +use iggy_common::defaults::*; +use iggy_common::{Permissions, UserId}; +use std::sync::Arc; + +#[derive(Debug, Clone)] +pub struct User { + pub id: UserId, + pub status: UserStatus, + pub username: String, + pub password: String, + pub created_at: IggyTimestamp, + pub permissions: Option, + pub personal_access_tokens: DashMap, PersonalAccessToken>, +} + +impl Default for User { + fn default() -> Self { + Self { + id: 0, + status: UserStatus::Active, + username: "user".to_string(), + password: "secret".to_string(), + created_at: IggyTimestamp::now(), + permissions: None, + personal_access_tokens: DashMap::new(), + } + } +} + +impl User { + pub fn empty(id: UserId) -> Self { + Self { + id, + ..Default::default() + } + } + + pub fn new( + id: u32, + username: &str, + password: &str, + status: UserStatus, + permissions: Option, + ) -> Self { + Self::with_password( + id, + username, + crypto::hash_password(password), + status, + permissions, + ) + } + + pub fn with_password( + id: u32, + username: &str, + password: String, + status: UserStatus, + permissions: Option, + ) -> Self { + Self { + id, + username: username.into(), + password, + created_at: IggyTimestamp::now(), + status, + permissions, + personal_access_tokens: DashMap::new(), + } + } + + pub fn root(username: &str, password: &str) -> Self { + Self::new( + DEFAULT_ROOT_USER_ID, + username, + password, + UserStatus::Active, + Some(Permissions::root()), + ) + } + + pub fn is_root(&self) -> bool { + self.id == DEFAULT_ROOT_USER_ID + } + + pub fn is_active(&self) -> bool { + self.status == UserStatus::Active + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn given_root_user_data_and_credentials_should_be_valid() { + let user = User::root(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD); + assert_eq!(user.id, DEFAULT_ROOT_USER_ID); + assert_eq!(user.username, DEFAULT_ROOT_USERNAME); + assert_ne!(user.password, DEFAULT_ROOT_PASSWORD); + assert!(crypto::verify_password( + DEFAULT_ROOT_PASSWORD, + &user.password + )); + assert_eq!(user.status, UserStatus::Active); + assert!(user.created_at.as_micros() > 0); + } + + #[test] + fn should_be_created_given_specific_status() { + let status = UserStatus::Inactive; + let user = User::new(1, "test", "test", status, None); + assert_eq!(user.status, status); + } +} diff --git a/core/server/src/streaming/utils/address.rs b/core/server/src/streaming/utils/address.rs new file mode 100644 index 0000000000..32bf83f3ac --- /dev/null +++ b/core/server/src/streaming/utils/address.rs @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/// Extracts IP from an address string like "127.0.0.1:8090" or "[::1]:8090" +pub fn extract_ip(address: &str) -> String { + if let Some(colon_pos) = address.rfind(':') { + // Handle IPv6 addresses like [::1]:8090 + if address.starts_with('[') + && let Some(bracket_pos) = address.rfind(']') + { + return address[1..bracket_pos].to_string(); + } + // Handle IPv4 addresses like 127.0.0.1:8090 + return address[..colon_pos].to_string(); + } + address.to_string() +} + +/// Extracts port from an address string like "127.0.0.1:8090" +pub fn extract_port(address: &str) -> u16 { + if let Some(colon_pos) = address.rfind(':') + && let Ok(port) = address[colon_pos + 1..].parse::() + { + return port; + } + 0 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_extract_ip_ipv4() { + assert_eq!(extract_ip("127.0.0.1:8090"), "127.0.0.1"); + assert_eq!(extract_ip("192.168.1.100:3000"), "192.168.1.100"); + } + + #[test] + fn test_extract_ip_ipv6() { + assert_eq!(extract_ip("[::1]:8090"), "::1"); + assert_eq!(extract_ip("[2001:db8::1]:443"), "2001:db8::1"); + } + + #[test] + fn test_extract_ip_no_port() { + assert_eq!(extract_ip("127.0.0.1"), "127.0.0.1"); + } + + #[test] + fn test_extract_port() { + assert_eq!(extract_port("127.0.0.1:8090"), 8090); + assert_eq!(extract_port("192.168.1.100:3000"), 3000); + assert_eq!(extract_port("[::1]:8090"), 8090); + } + + #[test] + fn test_extract_port_no_port() { + assert_eq!(extract_port("127.0.0.1"), 0); + } +} diff --git a/core/server/src/streaming/utils/file.rs b/core/server/src/streaming/utils/file.rs new file mode 100644 index 0000000000..75d833d39a --- /dev/null +++ b/core/server/src/streaming/utils/file.rs @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use compio::fs::{File, OpenOptions, remove_file}; +use std::path::Path; + +pub async fn open(path: &str) -> Result { + OpenOptions::new().read(true).open(path).await +} + +pub async fn append(path: &str) -> Result<(File, u64), std::io::Error> { + let file = OpenOptions::new() + .create(true) + .write(true) + .open(path) + .await?; + let position = file.metadata().await?.len(); + Ok((file, position)) +} + +pub async fn overwrite(path: &str) -> Result { + OpenOptions::new() + .create(true) + .write(true) + .truncate(false) + .open(path) + .await +} + +pub async fn remove(path: &str) -> Result<(), std::io::Error> { + remove_file(path).await +} + +pub async fn rename(old_path: &str, new_path: &str) -> Result<(), std::io::Error> { + compio::fs::rename(Path::new(old_path), Path::new(new_path)).await +} + +pub async fn exists(path: &str) -> Result { + std::fs::exists(path) +} diff --git a/core/server/src/streaming/utils/mod.rs b/core/server/src/streaming/utils/mod.rs new file mode 100644 index 0000000000..b03232126f --- /dev/null +++ b/core/server/src/streaming/utils/mod.rs @@ -0,0 +1,22 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod address; +pub mod file; +pub mod ptr; +pub use iggy_common::random_id; +pub use server_common::crypto; diff --git a/core/server/src/streaming/utils/ptr.rs b/core/server/src/streaming/utils/ptr.rs new file mode 100644 index 0000000000..a98977049b --- /dev/null +++ b/core/server/src/streaming/utils/ptr.rs @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::{ops::Deref, ptr::NonNull}; + +// Wrapper around an imutable pointer to a 'static value, that can be cloned and sent across threads. +pub struct EternalPtr { + ptr: NonNull, + _marker: std::marker::PhantomData<&'static T>, +} + +impl From> for EternalPtr { + fn from(value: NonNull) -> Self { + Self { + ptr: value, + _marker: std::marker::PhantomData, + } + } +} + +impl From<&'static T> for EternalPtr { + fn from(value: &'static T) -> Self { + Self { + ptr: value.into(), + _marker: std::marker::PhantomData, + } + } +} + +impl From<&'static mut T> for EternalPtr { + fn from(value: &'static mut T) -> Self { + Self { + ptr: value.into(), + _marker: std::marker::PhantomData, + } + } +} + +impl Clone for EternalPtr { + fn clone(&self) -> Self { + Self { + ptr: self.ptr, + _marker: std::marker::PhantomData, + } + } +} + +impl Deref for EternalPtr { + type Target = T; + + fn deref(&self) -> &Self::Target { + unsafe { self.ptr.as_ref() } + } +} + +unsafe impl Send for EternalPtr {} diff --git a/core/server/src/systemd.rs b/core/server/src/systemd.rs deleted file mode 100644 index 5ef2ba80be..0000000000 --- a/core/server/src/systemd.rs +++ /dev/null @@ -1,80 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Thin wrappers around `sd_notify` so every systemd interaction on the -//! server side lives in one place (mirrors `core/ai/mcp/src/systemd.rs`). - -use crate::server_error::{ServerError, ShardJoinFailureKind}; -use message_bus::{IggyMessageBus, ShutdownToken}; -use std::time::Duration; -use tracing::{info, warn}; - -/// Tell systemd the service has finished start-up (`READY=1`). -pub fn notify_ready() { - if let Err(error) = sd_notify::notify(&[sd_notify::NotifyState::Ready]) { - warn!("Failed to send systemd READY=1 notification: {error}"); - } -} - -/// Tell systemd the service has begun shutting down (`STOPPING=1`), which -/// also stops the watchdog timer from counting against a long drain. -pub fn notify_stopping() { - let _ = sd_notify::notify(&[sd_notify::NotifyState::Stopping]); -} - -/// Surface a dirty shutdown in `systemctl status` / journald. -pub fn notify_shutdown_failure(error: &ServerError) { - let wedged = matches!( - error, - ServerError::ShardJoinFailures { failures } - if failures - .iter() - .any(|failure| matches!(failure.kind, ShardJoinFailureKind::Wedged { .. })) - ); - let status = if wedged { - "graceful shutdown timed out" - } else { - "shard threads failed during shutdown" - }; - let _ = sd_notify::notify(&[sd_notify::NotifyState::Status(status)]); -} - -/// Start the `WATCHDOG=1` keep-alive. Does nothing unless the unit set -/// `WatchdogSec=`. Tracked on the bus so `bus.shutdown()` reaps the task. -pub fn spawn_watchdog(bus: &IggyMessageBus) { - let Some(timeout) = sd_notify::watchdog_enabled() else { - return; - }; - - let interval = timeout / 2; - info!( - "Systemd watchdog enabled, pinging every {}s (timeout: {}s).", - interval.as_secs(), - timeout.as_secs() - ); - - let handle = compio::runtime::spawn(run_watchdog(bus.token(), interval)); - bus.track_background(handle); -} - -async fn run_watchdog(token: ShutdownToken, interval: Duration) { - while token.sleep_or_shutdown(interval).await { - if let Err(error) = sd_notify::notify(&[sd_notify::NotifyState::Watchdog]) { - warn!("Failed to send systemd watchdog ping: {error}"); - } - } -} diff --git a/core/server/src/tcp/connection_handler.rs b/core/server/src/tcp/connection_handler.rs new file mode 100644 index 0000000000..d48012258d --- /dev/null +++ b/core/server/src/tcp/connection_handler.rs @@ -0,0 +1,184 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::{self, HandlerResult, MAX_CONTROL_FRAME_PAYLOAD}; +use crate::sender::SenderKind; +use crate::server_error::ConnectionError; +use crate::shard::IggyShard; +use crate::streaming::session::Session; +use async_channel::Receiver; +use bytes::BytesMut; +use futures::FutureExt; +use iggy_binary_protocol::RequestFrame; +use iggy_binary_protocol::codes::{GET_CLUSTER_METADATA_CODE, SEND_MESSAGES_CODE, command_name}; +use iggy_common::IggyError; +use std::io::ErrorKind; +use std::rc::Rc; +use tracing::{debug, error, info}; + +/// Connection lifecycle action after command handling. +pub enum ConnectionAction { + /// Continue handling connection on current shard. + Finished, + + /// Connection migrated to another shard, exit without cleanup. + Migrated { to_shard: u16 }, +} + +pub(crate) async fn handle_connection( + session: &Session, + sender: &mut SenderKind, + shard: &Rc, + stop_receiver: Receiver<()>, +) -> Result { + let mut header_buffer = BytesMut::with_capacity(RequestFrame::HEADER_SIZE); + loop { + let read_future = sender.read(header_buffer); + // TODO(hubcio): this futures::select! call is translated to epoll_wait syscall for every + // message, which adds around 100 us median latency. We could instead just call sender.shutdown() + // if some atomic bool is set, since this is all happenng within single thread. + let (_, mut header_buf) = futures::select! { + _ = stop_receiver.recv().fuse() => { + info!("Connection stop signal received for session: {}", session); + let _ = sender.send_error_response(IggyError::Disconnected).await; + return Ok(ConnectionAction::Finished); + } + result = read_future.fuse() => { + match result { + (Ok(_), buf) => (Ok::<(), IggyError>(()), buf), + (Err(error), buf) => { + header_buffer = buf; + if error.as_code() == IggyError::ConnectionClosed.as_code() { + return Err(ConnectionError::from(error)); + } else { + error!("got error: {:?}", error); + sender.send_error_response(error).await?; + continue; + } + } + } + } + }; + + let length = u32::from_le_bytes(header_buf[0..4].try_into().unwrap()); + let code = u32::from_le_bytes(header_buf[4..8].try_into().unwrap()); + header_buf.clear(); + header_buffer = header_buf; + + let cmd_name = command_name(code).unwrap_or("unknown"); + debug!("Received a TCP request, length: {length}, code: {code} ({cmd_name})"); + + let payload_length = match RequestFrame::payload_length(length) { + Ok(len) => len, + Err(_) => { + sender + .send_error_response(IggyError::InvalidCommand) + .await?; + continue; + } + }; + + let result = if code == SEND_MESSAGES_CODE { + dispatch::dispatch_send_messages(sender, payload_length, session, shard).await + } else { + if payload_length > MAX_CONTROL_FRAME_PAYLOAD { + sender + .send_error_response(IggyError::InvalidCommand) + .await?; + continue; + } + let payload = dispatch::read_payload(sender, payload_length).await?; + let frame = RequestFrame::from_parts(code, &payload); + dispatch::dispatch(frame, sender, session, shard).await + }; + + match result { + Ok(handler_result) => match handler_result { + HandlerResult::Finished => { + debug!( + "Command {code} ({cmd_name}) was handled successfully, session: {session}. TCP response was sent." + ); + } + HandlerResult::Migrated { to_shard } => { + info!( + "Command {code} ({cmd_name}) was transferred to shard {to_shard}, session: {session}." + ); + + return Ok(ConnectionAction::Migrated { to_shard }); + } + }, + Err(error) => { + if code == GET_CLUSTER_METADATA_CODE + && matches!(error, IggyError::FeatureUnavailable) + { + debug!( + "GetClusterMetadata command not available (clustering disabled), session: {session}." + ); + sender.send_error_response(error).await?; + debug!("TCP error response was sent to: {session}."); + } else { + error!( + "Command with code {code} ({cmd_name}) was not handled successfully, session: {session}, error: {error}." + ); + + if matches!(error, IggyError::ClientNotFound(_) | IggyError::StaleClient) { + sender.send_error_response(error.clone()).await?; + debug!("TCP error response was sent to: {session}."); + error!("Session: {session} will be deleted."); + return Err(ConnectionError::from(error)); + } else { + sender.send_error_response(error).await?; + debug!("TCP error response was sent to: {session}."); + } + } + } + } + } +} + +pub(crate) fn handle_error(error: ConnectionError) { + match error { + ConnectionError::IoError(e) => match e.kind() { + ErrorKind::UnexpectedEof => { + info!("Connection has been closed."); + } + ErrorKind::ConnectionAborted => { + info!("Connection has been aborted."); + } + ErrorKind::ConnectionRefused => { + info!("Connection has been refused."); + } + ErrorKind::ConnectionReset => { + info!("Connection has been reset."); + } + _ => { + error!("Connection has failed: {e}"); + } + }, + ConnectionError::SdkError(sdk_error) => match sdk_error { + IggyError::ConnectionClosed => { + debug!("Client closed connection."); + } + _ => { + error!("Failure in internal SDK call: {sdk_error}"); + } + }, + _ => { + error!("Connection has failed: {error}"); + } + } +} diff --git a/core/server/src/tcp/mod.rs b/core/server/src/tcp/mod.rs new file mode 100644 index 0000000000..e5f9d42cba --- /dev/null +++ b/core/server/src/tcp/mod.rs @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod connection_handler; +pub mod tcp_listener; +pub mod tcp_server; +pub mod tcp_socket; +pub mod tcp_tls_listener; + +pub const COMPONENT: &str = "TCP"; + +/// Bind a `TcpListener` via the compio 0.19 `TcpSocket` builder. +/// +/// compio 0.19 removed `TcpListener::bind_with_options(addr, SocketOpts)`; +/// this is the shared replacement for every legacy listener bind. +/// `SO_REUSEPORT` is always set (thread-per-core: many sockets bind the +/// same addr+port); `reuseaddr` is opt-in. When `tuning` is `Some` and +/// `override_defaults` is set, the socket buffers / keepalive / nodelay +/// are applied (a zero linger maps to `set_zero_linger`; a non-zero +/// linger has no compio-0.19 successor and is dropped). +pub(crate) async fn bind_reuseport_listener( + addr: std::net::SocketAddr, + reuseaddr: bool, + tuning: Option<&crate::configs::tcp::TcpSocketConfig>, +) -> std::io::Result { + use compio::net::TcpSocket; + + let socket = match addr { + std::net::SocketAddr::V4(_) => TcpSocket::new_v4().await?, + std::net::SocketAddr::V6(_) => TcpSocket::new_v6().await?, + }; + socket.set_reuseport(true)?; + if reuseaddr { + socket.set_reuseaddr(true)?; + } + if let Some(config) = tuning + && config.override_defaults + { + let recv_buffer_size = config + .recv_buffer_size + .as_bytes_u64() + .try_into() + .expect("Failed to parse recv_buffer_size for TCP socket"); + let send_buffer_size = config + .send_buffer_size + .as_bytes_u64() + .try_into() + .expect("Failed to parse send_buffer_size for TCP socket"); + socket.set_recv_buffer_size(recv_buffer_size)?; + socket.set_send_buffer_size(send_buffer_size)?; + socket.set_keepalive(config.keepalive)?; + if config.linger.get_duration().is_zero() { + socket.set_zero_linger()?; + } + socket.set_nodelay(config.nodelay)?; + } + socket.bind(addr).await?; + socket.listen(1024).await +} diff --git a/core/server/src/tcp/tcp_listener.rs b/core/server/src/tcp/tcp_listener.rs new file mode 100644 index 0000000000..3f53e4e640 --- /dev/null +++ b/core/server/src/tcp/tcp_listener.rs @@ -0,0 +1,171 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::configs::tcp::TcpSocketConfig; + +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::shard::task_registry::{ShutdownToken, TaskRegistry}; +use crate::shard::transmission::event::ShardEvent; +use crate::tcp::connection_handler::{ConnectionAction, handle_connection, handle_error}; +use compio::net::TcpListener; +use err_trail::ErrContext; +use futures::FutureExt; +use iggy_common::{IggyError, TransportProtocol}; +use std::net::SocketAddr; +use std::rc::Rc; +use std::time::Duration; +use tracing::{debug, error, info}; + +async fn create_listener( + addr: SocketAddr, + config: &TcpSocketConfig, +) -> Result { + crate::tcp::bind_reuseport_listener(addr, false, Some(config)).await +} + +pub async fn start( + server_name: &'static str, + mut addr: SocketAddr, + config: &TcpSocketConfig, + shard: Rc, + shutdown: ShutdownToken, +) -> Result<(), IggyError> { + if shard.id != 0 && addr.port() == 0 { + info!("Waiting for TCP address from shard 0..."); + loop { + if let Some(bound_addr) = shard.tcp_bound_address.get() { + addr = bound_addr; + info!("Received TCP address: {}", addr); + break; + } + compio::time::sleep(Duration::from_millis(50)).await; + } + } + + let listener = create_listener(addr, config) + .await + .map_err(|_| IggyError::CannotBindToSocket(addr.to_string())) + .error(|err: &IggyError| { + format!("Failed to bind {server_name} server to address: {addr}, {err}") + })?; + let actual_addr = listener.local_addr().map_err(|e| { + error!("Failed to get local address: {}", e); + IggyError::CannotBindToSocket(addr.to_string()) + })?; + info!("{} server has started on: {:?}", server_name, actual_addr); + + if shard.id == 0 { + // Store bound address locally + shard.tcp_bound_address.set(Some(actual_addr)); + + if addr.port() == 0 { + // Notify config writer on shard 0 + let _ = shard.config_writer_notify.try_send(()); + + // Broadcast to other shards for SO_REUSEPORT binding + let event = ShardEvent::AddressBound { + protocol: TransportProtocol::Tcp, + address: actual_addr, + }; + shard.broadcast_event_to_all_shards(event).await?; + } + } + + accept_loop(server_name, listener, shard, shutdown).await +} + +async fn accept_loop( + server_name: &'static str, + listener: TcpListener, + shard: Rc, + shutdown: ShutdownToken, +) -> Result<(), IggyError> { + loop { + let shard = shard.clone(); + let accept_future = listener.accept(); + futures::select! { + _ = shutdown.wait().fuse() => { + debug!("{} received shutdown signal, no longer accepting connections", server_name); + break; + } + result = accept_future.fuse() => { + match result { + Ok((stream, address)) => { + if shard.is_shutting_down() { + info!("Rejecting new connection from {} during shutdown", address); + continue; + } + let shard_clone = shard.clone(); + info!("Accepted new TCP connection: {}", address); + let transport = TransportProtocol::Tcp; + let session = shard_clone.add_client(&address, transport); + info!("Added {} client with session: {} for IP address: {}", transport, session, address); + + let client_id = session.client_id; + info!("Created new session: {}", session); + let mut sender = SenderKind::get_tcp_sender(stream); + + let conn_stop_receiver = shard.task_registry.add_connection(client_id); + + let shard_for_conn = shard_clone.clone(); + let registry = shard.task_registry.clone(); + let registry_clone = registry.clone(); + registry.spawn_connection(async move { + match handle_connection(&session, &mut sender, &shard_for_conn, conn_stop_receiver).await { + Ok(ConnectionAction::Migrated { to_shard }) => { + info!("Migrated to shard {to_shard}, ignore cleanup connection"); + } + Ok(ConnectionAction::Finished) => { + cleanup_connection(&mut sender, client_id, address, ®istry_clone, &shard_for_conn).await; + } + Err(err) => { + handle_error(err); + cleanup_connection(&mut sender, client_id, address, ®istry_clone, &shard_for_conn).await; + }, + } + }); + } + Err(error) => error!("Unable to accept TCP socket. {}", error), + } + } + } + } + Ok(()) +} + +pub async fn cleanup_connection( + sender: &mut SenderKind, + client_id: u32, + address: SocketAddr, + registry: &Rc, + shard: &IggyShard, +) { + registry.remove_connection(&client_id); + shard.delete_client(client_id).await; + if let Err(error) = sender.shutdown().await { + error!( + "Failed to shutdown for client {}, address {}: {}", + client_id, address, error + ); + } else { + info!( + "Successfully closed for client {}, address {}", + client_id, address + ); + } +} diff --git a/core/server/src/tcp/tcp_server.rs b/core/server/src/tcp/tcp_server.rs new file mode 100644 index 0000000000..3f9bcdb2ef --- /dev/null +++ b/core/server/src/tcp/tcp_server.rs @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::shard::IggyShard; +use crate::shard::task_registry::ShutdownToken; +use crate::tcp::{tcp_listener, tcp_tls_listener}; +use iggy_common::IggyError; +use std::net::SocketAddr; +use std::rc::Rc; +use tracing::info; + +/// Starts the TCP server. +pub async fn spawn_tcp_server( + shard: Rc, + shutdown: ShutdownToken, +) -> Result<(), IggyError> { + let server_name = if shard.config.tcp.tls.enabled { + "Iggy TCP TLS" + } else { + "Iggy TCP" + }; + let socket_config = &shard.config.tcp.socket; + let addr: SocketAddr = shard + .config + .tcp + .address + .parse() + .expect("Failed to parse TCP address"); + info!("Initializing {} server...", server_name); + + match shard.config.tcp.tls.enabled { + true => { + tcp_tls_listener::start(server_name, addr, socket_config, shard.clone(), shutdown) + .await? + } + false => { + tcp_listener::start(server_name, addr, socket_config, shard.clone(), shutdown).await? + } + }; + + Ok(()) +} diff --git a/core/server/src/tcp/tcp_socket.rs b/core/server/src/tcp/tcp_socket.rs new file mode 100644 index 0000000000..6946e0c86f --- /dev/null +++ b/core/server/src/tcp/tcp_socket.rs @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use socket2::{Domain, Protocol, Socket, Type}; +use std::num::TryFromIntError; + +use crate::configs::tcp::TcpSocketConfig; + +pub fn build(ipv6: bool, config: &TcpSocketConfig) -> Socket { + let socket = if ipv6 { + Socket::new(Domain::IPV6, Type::STREAM, Some(Protocol::TCP)) + .expect("Unable to create an ipv6 socket") + } else { + Socket::new(Domain::IPV4, Type::STREAM, Some(Protocol::TCP)) + .expect("Unable to create an ipv4 socket") + }; + + // Required by the thread-per-core model... + // We create bunch of sockets on different threads, that bind to exactly the same address and port. + socket + .set_reuse_address(true) + .expect("Unable to set SO_REUSEADDR on socket"); + socket + .set_reuse_port(true) + .expect("Unable to set SO_REUSEPORT on socket"); + + if config.override_defaults { + config + .recv_buffer_size + .as_bytes_u64() + .try_into() + .map_err(|e: TryFromIntError| std::io::Error::other(e.to_string())) + .and_then(|size| socket.set_recv_buffer_size(size)) + .expect("Unable to set SO_RCVBUF on socket"); + config + .send_buffer_size + .as_bytes_u64() + .try_into() + .map_err(|e: TryFromIntError| std::io::Error::other(e.to_string())) + .and_then(|size| socket.set_send_buffer_size(size)) + .expect("Unable to set SO_SNDBUF on socket"); + socket + .set_keepalive(config.keepalive) + .expect("Unable to set SO_KEEPALIVE on socket"); + socket + .set_tcp_nodelay(config.nodelay) + .expect("Unable to set TCP_NODELAY on socket"); + socket + .set_linger(Some(config.linger.get_duration())) + .expect("Unable to set SO_LINGER on socket"); + } + + socket +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use iggy_common::{IggyByteSize, IggyDuration}; + + use super::*; + + #[test] + fn given_override_defaults_socket_should_be_configured() { + let buffer_size = 425984; + let linger_dur = Duration::new(1, 0); + let config = TcpSocketConfig { + override_defaults: true, + recv_buffer_size: IggyByteSize::from(buffer_size), + send_buffer_size: IggyByteSize::from(buffer_size), + keepalive: true, + nodelay: true, + linger: IggyDuration::new(linger_dur), + }; + let socket = build(false, &config); + assert!(socket.recv_buffer_size().unwrap() >= buffer_size as usize); + assert!(socket.send_buffer_size().unwrap() >= buffer_size as usize); + assert!(socket.keepalive().unwrap()); + assert!(socket.tcp_nodelay().unwrap()); + assert_eq!(socket.linger().unwrap(), Some(linger_dur)); + } +} diff --git a/core/server/src/tcp/tcp_tls_listener.rs b/core/server/src/tcp/tcp_tls_listener.rs new file mode 100644 index 0000000000..c4c200e059 --- /dev/null +++ b/core/server/src/tcp/tcp_tls_listener.rs @@ -0,0 +1,224 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::configs::tcp::TcpSocketConfig; +use crate::sender::SenderKind; +use crate::shard::IggyShard; +use crate::shard::task_registry::ShutdownToken; +use crate::shard::transmission::event::ShardEvent; +use crate::tcp::connection_handler::{handle_connection, handle_error}; +use compio::net::TcpListener; +use compio::tls::TlsAcceptor; +use err_trail::ErrContext; +use futures::FutureExt; +use iggy_common::{IggyError, TransportProtocol}; +use rustls::ServerConfig; +use rustls::pki_types::{CertificateDer, PrivateKeyDer}; +use rustls_pemfile::{certs, private_key}; +use std::io::BufReader; +use std::net::SocketAddr; +use std::rc::Rc; +use std::sync::Arc; +use std::time::Duration; +use tracing::{error, info, trace, warn}; + +pub(crate) async fn start( + server_name: &'static str, + mut addr: SocketAddr, + config: &TcpSocketConfig, + shard: Rc, + shutdown: ShutdownToken, +) -> Result<(), IggyError> { + if shard.id != 0 && addr.port() == 0 { + info!("Waiting for TCP address from shard 0..."); + loop { + if let Some(bound_addr) = shard.tcp_bound_address.get() { + addr = bound_addr; + info!("Received TCP address: {}", addr); + break; + } + compio::time::sleep(Duration::from_millis(50)).await; + } + } + + let listener = create_listener(addr, config) + .await + .map_err(|_| IggyError::CannotBindToSocket(addr.to_string())) + .error(|err: &IggyError| { + format!("Failed to bind {server_name} server to address: {addr}, {err}") + })?; + + let actual_addr = listener.local_addr().map_err(|_e| { + error!("Failed to get local address: {_e}"); + IggyError::CannotBindToSocket(addr.to_string()) + })?; + + if shard.id == 0 { + shard.tcp_bound_address.set(Some(actual_addr)); + if addr.port() == 0 { + // Notify config writer on shard 0 + let _ = shard.config_writer_notify.try_send(()); + + let event = ShardEvent::AddressBound { + protocol: TransportProtocol::Tcp, + address: actual_addr, + }; + shard.broadcast_event_to_all_shards(event).await?; + } + } + + // Ensure rustls crypto provider is installed + if rustls::crypto::CryptoProvider::get_default().is_none() + && let Err(e) = rustls::crypto::ring::default_provider().install_default() + { + warn!( + "Failed to install rustls crypto provider: {:?}. This may be normal if another thread installed it first.", + e + ); + } else { + trace!("Rustls crypto provider installed or already present"); + } + + // Load or generate TLS certificates + let tls_config = &shard.config.tcp.tls; + let (certs, key) = + if tls_config.self_signed && !std::path::Path::new(&tls_config.cert_file).exists() { + info!("Generating self-signed certificate for TCP TLS server"); + generate_self_signed_cert() + .unwrap_or_else(|e| panic!("Failed to generate self-signed certificate: {e}")) + } else { + info!( + "Loading certificates from cert_file: {}, key_file: {}", + tls_config.cert_file, tls_config.key_file + ); + load_certificates(&tls_config.cert_file, &tls_config.key_file) + .unwrap_or_else(|e| panic!("Failed to load certificates: {e}")) + }; + + let server_config = ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(certs, key) + .unwrap_or_else(|e| panic!("Unable to create TLS server config: {e}")); + + let acceptor = TlsAcceptor::from(Arc::new(server_config)); + + info!("{} server has started on: {:?}", server_name, actual_addr); + + accept_loop(server_name, listener, acceptor, shard, shutdown).await +} + +async fn create_listener( + addr: SocketAddr, + config: &TcpSocketConfig, +) -> Result { + crate::tcp::bind_reuseport_listener(addr, false, Some(config)).await +} + +async fn accept_loop( + server_name: &'static str, + listener: TcpListener, + acceptor: TlsAcceptor, + shard: Rc, + shutdown: ShutdownToken, +) -> Result<(), IggyError> { + loop { + let shard = shard.clone(); + let accept_future = listener.accept(); + futures::select! { + _ = shutdown.wait().fuse() => { + info!("{} received shutdown signal, no longer accepting connections", server_name); + break; + } + result = accept_future.fuse() => { + match result { + Ok((stream, address)) => { + if shard.is_shutting_down() { + info!("Rejecting new TLS connection from {} during shutdown", address); + continue; + } + info!("Accepted new TCP connection for TLS handshake: {}", address); + let shard_clone = shard.clone(); + let acceptor = acceptor.clone(); + + // Perform TLS handshake in a separate task to avoid blocking the accept loop + let registry = shard.task_registry.clone(); + let registry_clone = registry.clone(); + registry.spawn_connection(async move { + match acceptor.accept(stream).await { + Ok(tls_stream) => { + // TLS handshake successful, now create session + info!("TLS handshake successful, adding TCP client: {}", address); + let transport = TransportProtocol::Tcp; + let session = shard_clone.add_client(&address, transport); + info!("Added {} client with session: {} for IP address: {}", transport, session, address); + + let client_id = session.client_id; + info!("Created new session: {}", session); + + let conn_stop_receiver = registry_clone.add_connection(client_id); + let shard_for_conn = shard_clone.clone(); + let mut sender = SenderKind::get_tcp_tls_sender(tls_stream); + if let Err(error) = handle_connection(&session, &mut sender, &shard_for_conn, conn_stop_receiver).await { + handle_error(error); + } + shard_for_conn.delete_client(session.client_id).await; + registry_clone.remove_connection(&client_id); + + if let Err(error) = sender.shutdown().await { + error!("Failed to shutdown TCP TLS stream for client: {}, address: {}. {}", client_id, address, error); + } else { + info!("Successfully closed TCP TLS stream for client: {}, address: {}.", client_id, address); + } + } + Err(e) => { + error!("Failed to accept TLS connection from '{}': {}", address, e); + // No session was created, so no cleanup needed + } + } + }); + } + Err(error) => error!("Unable to accept TCP TLS socket. {}", error), + } + } + } + } + Ok(()) +} + +fn generate_self_signed_cert() +-> Result<(Vec>, PrivateKeyDer<'static>), Box> { + server_common::generate_self_signed_certificate("localhost") +} + +fn load_certificates( + cert_file: &str, + key_file: &str, +) -> Result<(Vec>, PrivateKeyDer<'static>), Box> { + let cert_file = std::fs::File::open(cert_file)?; + let mut cert_reader = BufReader::new(cert_file); + let certs: Vec<_> = certs(&mut cert_reader).collect::, _>>()?; + + if certs.is_empty() { + return Err("No certificates found in certificate file".into()); + } + + let key_file = std::fs::File::open(key_file)?; + let mut key_reader = BufReader::new(key_file); + let key = private_key(&mut key_reader)?.ok_or("No private key found in key file")?; + + Ok((certs, key)) +} diff --git a/core/server/src/websocket/connection_handler.rs b/core/server/src/websocket/connection_handler.rs new file mode 100644 index 0000000000..77ed24d7fc --- /dev/null +++ b/core/server/src/websocket/connection_handler.rs @@ -0,0 +1,175 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::binary::dispatch::{self, HandlerResult, MAX_CONTROL_FRAME_PAYLOAD}; +use crate::sender::SenderKind; +use crate::server_error::ConnectionError; +use crate::shard::IggyShard; +use crate::streaming::session::Session; +use async_channel::Receiver; +use bytes::BytesMut; +use futures::FutureExt; +use iggy_binary_protocol::RequestFrame; +use iggy_binary_protocol::codes::{SEND_MESSAGES_CODE, command_name}; +use iggy_common::IggyError; +use std::io::ErrorKind; +use std::rc::Rc; +use tracing::{debug, error, info, warn}; + +pub(crate) async fn handle_connection( + session: &Session, + sender: &mut SenderKind, + shard: &Rc, + stop_receiver: Receiver<()>, +) -> Result<(), ConnectionError> { + let mut header_buffer = BytesMut::with_capacity(RequestFrame::HEADER_SIZE); + + loop { + let read_future = sender.read(header_buffer); + let (_, mut header_buf) = futures::select! { + _ = stop_receiver.recv().fuse() => { + info!("Connection stop signal received for session: {}", session); + let _ = sender.send_error_response(IggyError::Disconnected).await; + return Ok(()); + } + result = read_future.fuse() => { + match result { + (Ok(_), buf) => (Ok::<(), IggyError>(()), buf), + (Err(error), buf) => { + header_buffer = buf; + if error.as_code() == IggyError::ConnectionClosed.as_code() { + return Err(ConnectionError::from(error)); + } else { + error!("got error: {:?}", error); + sender.send_error_response(error).await?; + continue; + } + } + } + } + }; + + let length = u32::from_le_bytes(header_buf[0..4].try_into().unwrap()); + let code = u32::from_le_bytes(header_buf[4..8].try_into().unwrap()); + header_buf.clear(); + header_buffer = header_buf; + + let cmd_name = command_name(code).unwrap_or("unknown"); + debug!("Received a WebSocket request, length: {length}, code: {code} ({cmd_name})"); + + let payload_length = match RequestFrame::payload_length(length) { + Ok(len) => len, + Err(_) => { + sender + .send_error_response(IggyError::InvalidCommand) + .await?; + continue; + } + }; + + let result = if code == SEND_MESSAGES_CODE { + dispatch::dispatch_send_messages(sender, payload_length, session, shard).await + } else { + if payload_length > MAX_CONTROL_FRAME_PAYLOAD { + sender + .send_error_response(IggyError::InvalidCommand) + .await?; + continue; + } + let payload = dispatch::read_payload(sender, payload_length).await?; + let frame = RequestFrame::from_parts(code, &payload); + dispatch::dispatch(frame, sender, session, shard).await + }; + + match result { + Ok(HandlerResult::Finished) => { + debug!( + "Command {code} ({cmd_name}) was handled successfully, session: {session}. WebSocket response was sent." + ); + } + Ok(HandlerResult::Migrated { to_shard }) => { + warn!("Unexpected migration on WebSocket: to_shard {to_shard}, session: {session}"); + } + Err(error) => match error { + IggyError::TcpError | IggyError::ConnectionClosed | IggyError::Disconnected => { + warn!( + "Client {} closed connection during request processing", + session.client_id + ); + return Err(ConnectionError::from(IggyError::ConnectionClosed)); + } + IggyError::ClientNotFound(_) | IggyError::StaleClient => { + error!("Command failed for session: {session}, error: {error}."); + sender.send_error_response(error.clone()).await?; + return Err(ConnectionError::from(error)); + } + _ => { + error!("Command failed for session: {session}, error: {error}."); + match sender.send_error_response(error).await { + Ok(_) => { + debug!("WebSocket error response was sent to: {session}."); + } + Err(IggyError::ConnectionClosed) => { + warn!( + "Could not send error response to {} - client already disconnected", + session.client_id + ); + return Err(ConnectionError::from(IggyError::ConnectionClosed)); + } + Err(send_err) => { + error!("Failed to send error response: {send_err}"); + return Err(ConnectionError::from(send_err)); + } + } + } + }, + } + } +} + +pub(crate) fn handle_error(error: ConnectionError) { + match error { + ConnectionError::IoError(e) => match e.kind() { + ErrorKind::UnexpectedEof => { + info!("WebSocket connection has been closed."); + } + ErrorKind::ConnectionAborted => { + info!("WebSocket connection has been aborted."); + } + ErrorKind::ConnectionRefused => { + info!("WebSocket connection has been refused."); + } + ErrorKind::ConnectionReset => { + info!("WebSocket connection has been reset."); + } + _ => { + error!("WebSocket connection has failed: {e}"); + } + }, + ConnectionError::SdkError(sdk_error) => match sdk_error { + IggyError::ConnectionClosed => { + debug!("Client closed WebSocket connection."); + } + _ => { + error!("Failure in internal SDK call: {sdk_error}"); + } + }, + _ => { + error!("WebSocket connection has failed: {error}"); + } + } +} diff --git a/core/server/src/websocket/mod.rs b/core/server/src/websocket/mod.rs new file mode 100644 index 0000000000..435648fe81 --- /dev/null +++ b/core/server/src/websocket/mod.rs @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +pub mod connection_handler; +pub mod websocket_listener; +pub mod websocket_server; +pub mod websocket_tls_listener; + +pub const COMPONENT: &str = "WEBSOCKET"; + +use crate::configs::websocket::WebSocketConfig; +use compio::ws::tungstenite::protocol::WebSocketConfig as CompioWsConfig; +use iggy_common::IggyByteSize; + +/// Build a `compio-ws`-compatible `WebSocketConfig` from the server config. +/// +/// The standalone `tungstenite` crate may be a different major version than +/// the one re-exported by `compio-ws`, so we construct the config through +/// `compio::ws::tungstenite` to guarantee type compatibility. +pub fn build_compio_ws_config(config: &WebSocketConfig) -> CompioWsConfig { + let mut ws = CompioWsConfig::default(); + + if let Some(ref s) = config.read_buffer_size + && let Ok(b) = s.parse::() + { + ws = ws.read_buffer_size(b.as_bytes_u64() as usize); + } + if let Some(ref s) = config.write_buffer_size + && let Ok(b) = s.parse::() + { + ws = ws.write_buffer_size(b.as_bytes_u64() as usize); + } + if let Some(ref s) = config.max_write_buffer_size + && let Ok(b) = s.parse::() + { + ws = ws.max_write_buffer_size(b.as_bytes_u64() as usize); + } + if let Some(ref s) = config.max_message_size + && let Ok(b) = s.parse::() + { + ws = ws.max_message_size(Some(b.as_bytes_u64() as usize)); + } + if let Some(ref s) = config.max_frame_size + && let Ok(b) = s.parse::() + { + ws = ws.max_frame_size(Some(b.as_bytes_u64() as usize)); + } + ws = ws.accept_unmasked_frames(config.accept_unmasked_frames); + ws +} diff --git a/core/server/src/websocket/websocket_listener.rs b/core/server/src/websocket/websocket_listener.rs new file mode 100644 index 0000000000..0f8512b154 --- /dev/null +++ b/core/server/src/websocket/websocket_listener.rs @@ -0,0 +1,182 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::configs::websocket::WebSocketConfig; +use crate::sender::{SenderKind, WebSocketSender}; +use crate::shard::IggyShard; +use crate::shard::task_registry::ShutdownToken; +use crate::shard::transmission::event::ShardEvent; +use crate::websocket::connection_handler::{handle_connection, handle_error}; +use compio::net::TcpListener; +use compio::ws::accept_async_with_config; +use err_trail::ErrContext; +use futures::FutureExt; +use iggy_common::IggyError; +use iggy_common::TransportProtocol; +use std::net::SocketAddr; +use std::rc::Rc; +use tracing::{debug, error, info}; + +async fn create_listener(addr: SocketAddr) -> Result { + // Thread-per-core: many sockets bind the same addr+port (SO_REUSEPORT). + crate::tcp::bind_reuseport_listener(addr, true, None).await +} + +pub async fn start( + config: WebSocketConfig, + shard: Rc, + shutdown: ShutdownToken, +) -> Result<(), IggyError> { + let mut addr: SocketAddr = config + .address + .parse() + .error(|e: &std::net::AddrParseError| { + format!( + "WebSocket (error: {e}) - failed to parse address: {}", + config.address + ) + }) + .map_err(|_| IggyError::InvalidConfiguration)?; + + if shard.id != 0 && addr.port() == 0 { + info!("Waiting for WebSocket address from shard 0..."); + loop { + if let Some(bound_addr) = shard.websocket_bound_address.get() { + addr = bound_addr; + info!("Received WebSocket address from shard 0: {}", addr); + break; + } + // Small delay to prevent busy waiting + compio::time::sleep(std::time::Duration::from_millis(10)).await; + } + } + + let listener = create_listener(addr) + .await + .error(|e: &std::io::Error| { + format!("WebSocket (error: {e}) - failed to bind to address: {addr}") + }) + .map_err(|_| IggyError::CannotBindToSocket(addr.to_string()))?; + + let local_addr = listener.local_addr().unwrap(); + info!("{} has started on: ws://{}", "WebSocket Server", local_addr); + + // Notify shard about the bound address + let event = ShardEvent::AddressBound { + protocol: TransportProtocol::WebSocket, + address: local_addr, + }; + + if shard.id == 0 { + // Store bound address locally first + shard.websocket_bound_address.set(Some(local_addr)); + + if addr.port() == 0 { + // Broadcast to other shards for SO_REUSEPORT binding + shard.broadcast_event_to_all_shards(event).await?; + } + } else { + // Non-shard0 just handles the event locally + crate::shard::handlers::handle_event(&shard, event) + .await + .ok(); + } + + let ws_config = super::build_compio_ws_config(&config); + info!( + "WebSocket config: max_message_size: {:?}, max_frame_size: {:?}, accept_unmasked_frames: {}", + config.max_message_size, config.max_frame_size, config.accept_unmasked_frames + ); + + accept_loop(listener, Some(ws_config), shard, shutdown).await +} + +async fn accept_loop( + listener: TcpListener, + ws_config: Option, + shard: Rc, + shutdown: ShutdownToken, +) -> Result<(), IggyError> { + loop { + let shard = shard.clone(); + let accept_future = listener.accept(); + + futures::select! { + _ = shutdown.wait().fuse() => { + debug!("WebSocket Server received shutdown signal, no longer accepting connections"); + break; + } + result = accept_future.fuse() => { + match result { + Ok((tcp_stream, remote_addr)) => { + if shard.is_shutting_down() { + info!("Rejecting new WebSocket connection from {} during shutdown", remote_addr); + continue; + } + info!("Accepted new WebSocket connection from: {}", remote_addr); + + let shard_clone = shard.clone(); + let ws_config_clone = ws_config; + let registry = shard.task_registry.clone(); + let registry_clone = registry.clone(); + + registry.spawn_connection(async move { + match accept_async_with_config(tcp_stream, ws_config_clone).await { + Ok(websocket) => { + info!("WebSocket handshake successful from: {}", remote_addr); + + let session = shard_clone.add_client(&remote_addr, TransportProtocol::WebSocket); + let client_id = session.client_id; + + let sender = WebSocketSender::new(websocket); + let mut sender_kind = SenderKind::get_websocket_sender(sender); + let client_stop_receiver = registry_clone.add_connection(client_id); + + if let Err(error) = handle_connection(&session, &mut sender_kind, &shard_clone, client_stop_receiver).await { + handle_error(error); + } + shard_clone.delete_client(session.client_id).await; + registry_clone.remove_connection(&client_id); + + match sender_kind.shutdown().await { + Ok(_) => { + info!("Successfully closed WebSocket stream for client: {}, address: {}.", client_id, remote_addr); + } + Err(_) => { + // shutdown failures during client disconnect are expected and normal + // real errors would have been caught earlier in handle_connection + debug!("WebSocket shutdown completed with error for client: {} (likely client already disconnected)", client_id); + } + } + } + Err(error) => { + error!("WebSocket handshake failed from {}: {:?}", remote_addr, error); + } + } + }); + } + Err(error) => { + error!("Failed to accept WebSocket connection: {}", error); + } + } + } + } + } + + info!("WebSocket Server listener has stopped"); + Ok(()) +} diff --git a/core/server/src/websocket/websocket_server.rs b/core/server/src/websocket/websocket_server.rs new file mode 100644 index 0000000000..9ce40cc3a4 --- /dev/null +++ b/core/server/src/websocket/websocket_server.rs @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::shard::IggyShard; +use crate::shard::task_registry::ShutdownToken; +use crate::websocket::websocket_listener; +use crate::websocket::websocket_tls_listener; +use iggy_common::IggyError; +use std::rc::Rc; +use tracing::{error, info}; + +pub async fn spawn_websocket_server( + shard: Rc, + shutdown: ShutdownToken, +) -> Result<(), IggyError> { + let config = shard.config.websocket.clone(); + + if !config.enabled { + info!("WebSocket server is disabled."); + return Ok(()); + } + + let server_name = if config.tls.enabled { + "WebSocket TLS" + } else { + "WebSocket" + }; + + info!( + "Starting {} server on: {} for shard: {}...", + server_name, config.address, shard.id + ); + + let result = match config.tls.enabled { + true => websocket_tls_listener::start(config, shard.clone(), shutdown).await, + false => websocket_listener::start(config, shard.clone(), shutdown).await, + }; + + if let Err(error) = result { + error!("{} server has failed to start, error: {error}", server_name); + return Err(error); + } + + Ok(()) +} diff --git a/core/server/src/websocket/websocket_tls_listener.rs b/core/server/src/websocket/websocket_tls_listener.rs new file mode 100644 index 0000000000..ea8787d9ed --- /dev/null +++ b/core/server/src/websocket/websocket_tls_listener.rs @@ -0,0 +1,272 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::configs::websocket::WebSocketConfig; +use crate::sender::{SenderKind, WebSocketTlsSender}; +use crate::shard::IggyShard; +use crate::shard::task_registry::ShutdownToken; +use crate::shard::transmission::event::ShardEvent; +use crate::websocket::connection_handler::{handle_connection, handle_error}; +use compio::net::TcpListener; +use compio::tls::{MaybeTlsStream, TlsAcceptor}; +use compio::ws::accept_async_with_config; +use err_trail::ErrContext; +use futures::FutureExt; +use iggy_common::{IggyError, TransportProtocol}; +use rustls::ServerConfig; +use rustls::pki_types::{CertificateDer, PrivateKeyDer}; +use rustls_pemfile::{certs, private_key}; +use std::io::BufReader; +use std::net::SocketAddr; +use std::rc::Rc; +use std::sync::Arc; +use tracing::{debug, error, info, trace, warn}; + +async fn create_listener(addr: SocketAddr) -> Result { + // Thread-per-core: many sockets bind the same addr+port (SO_REUSEPORT). + crate::tcp::bind_reuseport_listener(addr, true, None).await +} + +pub async fn start( + config: WebSocketConfig, + shard: Rc, + shutdown: ShutdownToken, +) -> Result<(), IggyError> { + let mut addr: SocketAddr = config + .address + .parse() + .error(|e: &std::net::AddrParseError| { + format!( + "WebSocket TLS (error: {e}) - failed to parse address: {}", + config.address + ) + }) + .map_err(|_| IggyError::InvalidConfiguration)?; + + if shard.id != 0 && addr.port() == 0 { + info!("Waiting for WebSocket TLS address from shard 0..."); + loop { + if let Some(bound_addr) = shard.websocket_bound_address.get() { + addr = bound_addr; + info!("Received WebSocket TLS address from shard 0: {}", addr); + break; + } + compio::time::sleep(std::time::Duration::from_millis(10)).await; + } + } + + let listener = create_listener(addr) + .await + .error(|e: &std::io::Error| { + format!("WebSocket TLS (error: {e}) - failed to bind to address: {addr}") + }) + .map_err(|_| IggyError::CannotBindToSocket(addr.to_string()))?; + + let local_addr = listener.local_addr().unwrap(); + + // Notify shard about the bound address + let event = ShardEvent::AddressBound { + protocol: TransportProtocol::WebSocket, + address: local_addr, + }; + + if shard.id == 0 { + // Store bound address locally first + shard.websocket_bound_address.set(Some(local_addr)); + + if addr.port() == 0 { + // Broadcast to other shards for SO_REUSEPORT binding + shard.broadcast_event_to_all_shards(event).await?; + } + } else { + // Non-shard0 just handles the event locally + crate::shard::handlers::handle_event(&shard, event) + .await + .ok(); + } + + // Ensure rustls crypto provider is installed + if rustls::crypto::CryptoProvider::get_default().is_none() + && let Err(e) = rustls::crypto::ring::default_provider().install_default() + { + warn!( + "Failed to install rustls crypto provider: {:?}. This may be normal if another thread installed it first.", + e + ); + } else { + trace!("Rustls crypto provider installed or already present"); + } + + // Load or generate TLS certificates + let tls_config = &shard.config.websocket.tls; + let (certs, key) = + if tls_config.self_signed && !std::path::Path::new(&tls_config.cert_file).exists() { + info!("Generating self-signed certificate for WebSocket TLS server"); + generate_self_signed_cert() + .unwrap_or_else(|e| panic!("Failed to generate self-signed certificate: {e}")) + } else { + info!( + "Loading certificates from cert_file: {}, key_file: {}", + tls_config.cert_file, tls_config.key_file + ); + load_certificates(&tls_config.cert_file, &tls_config.key_file) + .unwrap_or_else(|e| panic!("Failed to load certificates: {e}")) + }; + + let server_config = ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(certs, key) + .unwrap_or_else(|e| panic!("Unable to create TLS server config: {e}")); + + let acceptor = TlsAcceptor::from(Arc::new(server_config)); + + info!( + "{} has started on: wss://{}", + "WebSocket TLS Server", local_addr + ); + let ws_config = super::build_compio_ws_config(&config); + info!( + "WebSocket TLS config: max_message_size: {:?}, max_frame_size: {:?}, accept_unmasked_frames: {}", + config.max_message_size, config.max_frame_size, config.accept_unmasked_frames + ); + + let result = accept_loop(listener, acceptor, ws_config, shard.clone(), shutdown).await; + + info!( + "WebSocket TLS listener task exiting with result: {:?}", + result + ); + + result +} + +async fn accept_loop( + listener: TcpListener, + acceptor: TlsAcceptor, + ws_config: compio::ws::tungstenite::protocol::WebSocketConfig, + shard: Rc, + shutdown: ShutdownToken, +) -> Result<(), IggyError> { + info!("WebSocket TLS accept loop started, waiting for connections..."); + + loop { + let shard = shard.clone(); + let acceptor = acceptor.clone(); + let accept_future = listener.accept(); + + futures::select! { + _ = shutdown.wait().fuse() => { + debug!("WebSocket TLS Server received shutdown signal, no longer accepting connections"); + break; + } + result = accept_future.fuse() => { + match result { + Ok((tcp_stream, remote_addr)) => { + if shard.is_shutting_down() { + info!("Rejecting new WebSocket TLS connection from {} during shutdown", remote_addr); + continue; + } + info!("Accepted new TCP connection for WebSocket TLS handshake from: {}", remote_addr); + + let shard_clone = shard.clone(); + let ws_config_clone = ws_config; + let registry = shard.task_registry.clone(); + let registry_clone = registry.clone(); + + registry.spawn_connection(async move { + match acceptor.accept(tcp_stream).await { + Ok(tls_stream) => { + info!("TLS handshake successful for {}, performing WebSocket upgrade...", remote_addr); + + // compio-ws 0.4 drives TLS via `MaybeTlsStream` + // (`TlsStream` is not `Splittable`); wrap the handshaked + // stream so the WS layer owns it. + let tls_stream = MaybeTlsStream::new_tls(tls_stream); + match accept_async_with_config(tls_stream, Some(ws_config_clone)).await { + Ok(websocket) => { + info!("WebSocket TLS handshake successful from: {}", remote_addr); + + let session = shard_clone.add_client(&remote_addr, TransportProtocol::WebSocket); + let client_id = session.client_id; + + let sender = WebSocketTlsSender::new(websocket); + let mut sender_kind = SenderKind::WebSocketTls(sender); + let client_stop_receiver = registry_clone.add_connection(client_id); + + if let Err(error) = handle_connection(&session, &mut sender_kind, &shard_clone, client_stop_receiver).await { + handle_error(error); + } + shard_clone.delete_client(session.client_id).await; + registry_clone.remove_connection(&client_id); + + match sender_kind.shutdown().await { + Ok(_) => { + info!("Successfully closed WebSocket TLS stream for client: {}, address: {}.", client_id, remote_addr); + } + Err(_) => { + // shutdown failures during client disconnect are expected and normal + // real errors would have been caught earlier in handle_connection + debug!("WebSocket TLS shutdown completed with error for client: {} (likely client already disconnected)", client_id); + } + } + } + Err(error) => { + error!("WebSocket handshake failed on TLS connection from {}: {:?}", remote_addr, error); + } + } + } + Err(error) => { + error!("TLS handshake failed from {}: {:?}", remote_addr, error); + } + } + }); + } + Err(error) => { + error!("Failed to accept WebSocket TLS connection: {}", error); + } + } + } + } + } + + info!("WebSocket TLS Server listener has stopped"); + Ok(()) +} + +fn generate_self_signed_cert() +-> Result<(Vec>, PrivateKeyDer<'static>), Box> { + server_common::generate_self_signed_certificate("localhost") +} + +fn load_certificates( + cert_file: &str, + key_file: &str, +) -> Result<(Vec>, PrivateKeyDer<'static>), Box> { + let cert_file = std::fs::File::open(cert_file)?; + let mut cert_reader = BufReader::new(cert_file); + let certs: Vec<_> = certs(&mut cert_reader).collect::, _>>()?; + + if certs.is_empty() { + return Err("No certificates found in certificate file".into()); + } + + let key_file = std::fs::File::open(key_file)?; + let mut key_reader = BufReader::new(key_file); + let key = private_key(&mut key_reader)?.ok_or("No private key found in key file")?; + + Ok((certs, key)) +} diff --git a/core/server/src/wire.rs b/core/server/src/wire.rs deleted file mode 100644 index 02e21a5a3a..0000000000 --- a/core/server/src/wire.rs +++ /dev/null @@ -1,161 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Leaf wire helpers shared by the request-handling modules. -//! -//! Request-body slicing, the `usize -> u32` wire conversion, and the -//! transport-kind discriminant mapping. - -use bytes::Bytes; -use iggy_binary_protocol::RoutedRequestHeader; -use iggy_common::IggyError; -use message_bus::installer::conn_info::ClientTransportKind; -use server_common::Message; - -pub(crate) fn request_body(request: &Message) -> &[u8] { - &request.as_slice()[std::mem::size_of::()..request.header().size as usize] -} - -/// Check a client's `request_checksum` against the body it stamps. -/// -/// Must run BEFORE any body rewrite: PAT / password / consumer-group paths -/// substitute server-chosen bytes. Zero is "unstamped" and skips the check, so an -/// SDK predating the stamp still works. -/// -/// # Errors -/// [`IggyError::InvalidFormat`] when the stamp disagrees with the body. -pub(crate) fn verify_request_checksum( - request: &Message, -) -> Result<(), IggyError> { - let stamped = request.header().request_checksum; - if stamped == 0 || u128::from(iggy_common::calculate_checksum(request_body(request))) == stamped - { - return Ok(()); - } - Err(IggyError::InvalidFormat) -} - -/// Map the transport kind to the legacy wire discriminant -/// (`1=TCP, 2=QUIC, 4=WebSocket`); TLS variants report their base -/// transport. `ClientTransportKind` is `#[non_exhaustive]`, so any other -/// (TCP, TCP-TLS, or a future) variant falls back to TCP. -pub(crate) const fn transport_kind_to_wire(kind: ClientTransportKind) -> u8 { - match kind { - ClientTransportKind::Quic => 2, - ClientTransportKind::Ws | ClientTransportKind::Wss => 4, - _ => 1, - } -} - -pub(crate) fn usize_to_u32(value: usize) -> Result { - u32::try_from(value).map_err(|_| IggyError::InvalidIdentifier) -} - -/// Rebuild a request message with `body` replacing the original payload, -/// preserving the header (and fixing `size`). Used by the primary-side -/// request rewrites that swap a secret-bearing wire body for the -/// hash-carrying replicated body before consensus. -pub(crate) fn rewrite_request_body( - request: &Message, - body: &Bytes, -) -> Result, IggyError> { - let total_size = std::mem::size_of::() - .checked_add(body.len()) - .ok_or(IggyError::InvalidConfiguration)?; - let size = u32::try_from(total_size).map_err(|_| IggyError::InvalidConfiguration)?; - let mut rewritten = Message::::new(total_size); - let header = bytemuck::checked::try_from_bytes_mut::( - &mut rewritten.as_mut_slice()[..std::mem::size_of::()], - ) - .expect("zeroed bytes are a valid request header"); - *header = *request.header(); - header.size = size; - // Both describe the body just replaced, and nothing recomputes them for a - // `RoutedRequestHeader` -- the prepare projection derives its own `checksum_body` - // downstream. Clear rather than recompute; carrying them forward is a stale claim. - header.checksum = 0; - header.checksum_body = 0; - // `request_checksum` is deliberately NOT touched: it stamps what the CLIENT sent, - // already validated at admission. Re-stamping it over the substituted body would - // make the client-table reuse check compare a value no client ever produced. - rewritten.as_mut_slice()[std::mem::size_of::()..].copy_from_slice(body); - Ok(rewritten) -} - -#[cfg(test)] -mod tests { - use super::{request_body, rewrite_request_body}; - use bytes::Bytes; - use iggy_binary_protocol::{Command2, Operation, RoutedRequestHeader}; - use server_common::Message; - use std::mem::size_of; - - fn request(body: &[u8], request_checksum: u128) -> Message { - let total_size = size_of::() + body.len(); - let mut message = Message::::new(total_size).transmute_header( - |_, header: &mut RoutedRequestHeader| { - header.command = Command2::Request; - header.operation = Operation::CreateStream; - header.client = 1; - header.session = 1; - header.request = 9; - header.size = u32::try_from(total_size).expect("fits u32"); - header.request_checksum = request_checksum; - header.checksum = 0xdead; - header.checksum_body = 0xbeef; - }, - ); - message.as_mut_slice()[size_of::()..].copy_from_slice(body); - message - } - - #[test] - fn given_a_body_rewrite_should_keep_the_client_stamp_and_clear_the_stale_seals() { - // The secret-bearing wire body is swapped for the hash-carrying replicated - // one. `request_checksum` describes what the client sent and admission has - // already checked it, so it must survive; the other two describe the body - // that just went away. - let original = request(b"plaintext-secret", 0x1234); - let rewritten = rewrite_request_body(&original, &Bytes::from_static(b"argon2-hash")) - .expect("the rewritten body fits a request message"); - - assert_eq!( - rewritten.header().request_checksum, - 0x1234, - "the client's stamp must not be re-signed over server-substituted bytes" - ); - assert_eq!(rewritten.header().checksum, 0); - assert_eq!(rewritten.header().checksum_body, 0); - assert_eq!(request_body(&rewritten), b"argon2-hash"); - assert_eq!( - rewritten.header().size as usize, - size_of::() + b"argon2-hash".len(), - "`size` follows the new body, so `request_body` bounds it correctly" - ); - } - - #[test] - fn given_an_unstamped_request_when_rewriting_should_stay_unstamped() { - // Zero means "unstamped" all the way through the client table, so a rewrite - // must not manufacture a stamp for a client that sent none. - let original = request(b"plaintext-secret", 0); - let rewritten = rewrite_request_body(&original, &Bytes::from_static(b"argon2-hash")) - .expect("the rewritten body fits a request message"); - - assert_eq!(rewritten.header().request_checksum, 0); - } -} diff --git a/core/server_common/src/consensus_message.rs b/core/server_common/src/consensus_message.rs index 66f973a999..4fe65638be 100644 --- a/core/server_common/src/consensus_message.rs +++ b/core/server_common/src/consensus_message.rs @@ -20,14 +20,11 @@ use iggy_binary_protocol::{ Command2, CommitHeader, ConsensusError, ConsensusHeader, DoViewChangeHeader, GenericHeader, Operation, PrepareHeader, PrepareOkHeader, RepairPrepareHeader, RepairRangeReplyHeader, RequestHeader, RequestPreparesHeader, RequestStartViewHeader, RequestStateChunkHeader, - RequestStateTransferHeader, RoutedRequestHeader, StartViewChangeHeader, StartViewHeader, - StateChunkHeader, StateTransferTargetHeader, + RequestStateTransferHeader, StartViewChangeHeader, StartViewHeader, StateChunkHeader, + StateTransferTargetHeader, }; use smallvec::SmallVec; -use std::{ - marker::PhantomData, - mem::{offset_of, size_of}, -}; +use std::{marker::PhantomData, mem::size_of}; pub const MESSAGE_ALIGN: usize = 4096; @@ -240,9 +237,6 @@ where let bytes = >::header_storage(&self.backing); let typed = bytemuck::checked::try_from_bytes::(&bytes[..size_of::()]) .map_err(|_| ConsensusError::InvalidBitPattern)?; - // Before `validate`: a header that did not survive the link intact cannot - // have any of its fields believed, and `validate` reads them. - typed.verify_frame()?; typed.validate()?; Ok(Message { @@ -279,9 +273,6 @@ where let bytes = >::header_storage(&self.backing); let typed = bytemuck::checked::try_from_bytes::(&bytes[..size_of::()]) .map_err(|_| ConsensusError::InvalidBitPattern)?; - // Before `validate`: a header that did not survive the link intact cannot - // have any of its fields believed, and `validate` reads them. - typed.verify_frame()?; typed.validate()?; let typed_message = unsafe { &*std::ptr::from_ref(self).cast::>() }; @@ -381,31 +372,6 @@ where } } -impl Message { - /// Retype the client-wire request into the server-internal - /// [`RoutedRequestHeader`] shape in place, with `group` starting unset. - /// - /// The two layouts share every field offset (const-asserted where they - /// are declared) and `group` claims the client header's reserved tail, - /// so the promotion zeroes those eight bytes instead of rebuilding the - /// whole 256-byte header. This is the only sanctioned crossing between - /// the two layouts: transmute-based reads across them would alias - /// `group` with reserved bytes a client may have sent nonzero. - /// - /// # Panics - /// - /// Panics if the retyped header fails [`RoutedRequestHeader`] validation; - /// unreachable when `self` already passed [`RequestHeader`] validation, - /// which enforces the same field rules. - #[must_use] - pub fn into_routed(self) -> Message { - let group_offset = offset_of!(RoutedRequestHeader, group); - let mut owned = self.into_owned(); - owned.as_mut_slice()[group_offset..group_offset + size_of::()].fill(0); - Message::try_from(owned).expect("retyped request message must stay valid") - } -} - impl Message where H: ConsensusHeader, @@ -529,7 +495,7 @@ where #[derive(Debug)] pub enum MessageBag { - Request(Message), + Request(Message), Prepare(Message), PrepareOk(Message), StartViewChange(Message), @@ -634,9 +600,7 @@ where match command { Command2::Prepare => Ok(Self::Prepare(value.try_into_typed::()?)), - Command2::Request => Ok(Self::Request( - value.try_into_typed::()?, - )), + Command2::Request => Ok(Self::Request(value.try_into_typed::()?)), Command2::PrepareOk => Ok(Self::PrepareOk(value.try_into_typed::()?)), Command2::StartViewChange => Ok(Self::StartViewChange( value.try_into_typed::()?, @@ -690,19 +654,17 @@ where #[cfg(test)] mod tests { use super::*; - use iggy_binary_protocol::{ - HEADER_SIZE, Operation, ReplyHeader, RequestHeader, frame_checksum_bytes, - }; + use iggy_binary_protocol::{Operation, ReplyHeader}; use smallvec::smallvec; // Field offsets via `offset_of!`: a field reorder fails to compile here // rather than silently corrupting test bytes. - const SIZE_OFF: usize = std::mem::offset_of!(RoutedRequestHeader, size); - const COMMAND_OFF: usize = std::mem::offset_of!(RoutedRequestHeader, command); - const REQUEST_CLIENT_OFF: usize = std::mem::offset_of!(RoutedRequestHeader, client); - const REQUEST_OPERATION_OFF: usize = std::mem::offset_of!(RoutedRequestHeader, operation); - const REQUEST_SESSION_OFF: usize = std::mem::offset_of!(RoutedRequestHeader, session); - const REQUEST_REQUEST_OFF: usize = std::mem::offset_of!(RoutedRequestHeader, request); + const SIZE_OFF: usize = std::mem::offset_of!(RequestHeader, size); + const COMMAND_OFF: usize = std::mem::offset_of!(RequestHeader, command); + const REQUEST_CLIENT_OFF: usize = std::mem::offset_of!(RequestHeader, client); + const REQUEST_OPERATION_OFF: usize = std::mem::offset_of!(RequestHeader, operation); + const REQUEST_SESSION_OFF: usize = std::mem::offset_of!(RequestHeader, session); + const REQUEST_REQUEST_OFF: usize = std::mem::offset_of!(RequestHeader, request); fn header_bytes(command: Command2, size: u32) -> Owned { header_bytes_sized(command, size, 256) @@ -722,68 +684,10 @@ mod tests { // `Register` needs session 0 and request 0, which zeroed bytes // already satisfy. buf[REQUEST_OPERATION_OFF] = Operation::Register as u8; - seal_header_bytes(buf); } o } - /// Seal a hand-built frame the way a real sender does. - /// - /// Control headers are rejected on the typed parse unless `checksum` covers the - /// rest of the header, so a fixture that skips this tests the rejection path. - fn seal_header_bytes(buf: &mut [u8]) { - let header: &[u8; HEADER_SIZE] = buf[..HEADER_SIZE].try_into().expect("frame is a header"); - let checksum = frame_checksum_bytes(header); - buf[..size_of::()].copy_from_slice(&checksum.to_le_bytes()); - } - - /// A `DoViewChange` frame carrying a one-entry suffix, sealed. - /// - /// One entry rather than none because a bitset bit is only legal within the - /// suffix, so an empty frame cannot express the attack this seals against. - fn sealed_do_view_change() -> Owned { - const DVC_SIZE: usize = HEADER_SIZE * 2; - let mut owned = Owned::::zeroed(DVC_SIZE); - { - let buf = owned.as_mut_slice(); - buf[SIZE_OFF..SIZE_OFF + 4].copy_from_slice(&(DVC_SIZE as u32).to_le_bytes()); - buf[COMMAND_OFF] = Command2::DoViewChange as u8; - seal_header_bytes(buf); - } - owned - } - - #[test] - fn given_a_sealed_do_view_change_when_dispatching_should_accept() { - let generic = Message::::try_from(sealed_do_view_change()) - .expect("a sealed DoViewChange frames correctly"); - assert!(matches!( - MessageBag::try_from(generic), - Ok(MessageBag::DoViewChange(_)) - )); - } - - #[test] - fn given_a_flipped_nack_bit_when_dispatching_should_reject_the_frame() { - // Why the header seal exists. `validate` accepts this frame: the bit sits - // inside the one-entry suffix, where a legitimate nack lives. Downstream the - // bitset goes to the merge unchanged and authorises truncating a committed op. - const NACK_OFF: usize = std::mem::offset_of!(DoViewChangeHeader, nack_bitset); - - let mut owned = sealed_do_view_change(); - owned.as_mut_slice()[NACK_OFF] ^= 0x01; - - let generic = - Message::::try_from(owned).expect("framing does not inspect the bitset"); - assert!( - matches!( - MessageBag::try_from(generic), - Err(ConsensusError::FrameChecksumMismatch { .. }) - ), - "a manufactured nack must not reach the merge" - ); - } - // MessageBag round-trip for the probe + repair command family. Locks // RangeEvicted delivery in particular: RepairDone and RangeEvicted share // one header layout and BOTH must survive the typed parse -- a strict @@ -808,8 +712,6 @@ mod tests { let buf = owned.as_mut_slice(); buf[FROM_OP_OFF..FROM_OP_OFF + 8].copy_from_slice(&1u64.to_le_bytes()); buf[TO_OP_OFF..TO_OP_OFF + 8].copy_from_slice(&1u64.to_le_bytes()); - // Re-seal: the range was written after `header_bytes` sealed. - seal_header_bytes(buf); } let generic = Message::::try_from(owned) .unwrap_or_else(|e| panic!("{command:?} failed generic framing: {e}")); @@ -835,7 +737,7 @@ mod tests { #[test] #[should_panic(expected = "size must be at least header size")] fn message_new_smaller_than_header_panics() { - let _ = Message::::new(100); + let _ = Message::::new(100); } // try_from(Owned): validation gates the unsafe construction @@ -843,7 +745,7 @@ mod tests { #[test] fn try_from_owned_too_short_returns_err() { let owned = Owned::::zeroed(100); - let result = Message::::try_from(owned); + let result = Message::::try_from(owned); assert!(matches!(result, Err(ConsensusError::InvalidCommand { .. }))); } @@ -851,13 +753,13 @@ mod tests { fn try_from_owned_invalid_bit_pattern_returns_err() { let mut owned = Owned::::zeroed(256); owned.as_mut_slice()[COMMAND_OFF] = 99; // outside Command2's discriminant range - let result = Message::::try_from(owned); + let result = Message::::try_from(owned); assert!(matches!(result, Err(ConsensusError::InvalidBitPattern))); } #[test] fn try_from_owned_buffer_shorter_than_claimed_size_returns_err() { - // Header parses cleanly (RoutedRequestHeader::validate doesn't gate on + // Header parses cleanly (RequestHeader::validate doesn't gate on // size), but the encoded `size` field claims more bytes than the // backing buffer holds. The buffer-bounds check at the bottom of // `Message::try_from` must reject. (Both this case and the @@ -867,7 +769,7 @@ mod tests { let owned = header_bytes(Command2::Request, 999); // header_bytes already produces a 256-byte buffer; size=999 > 256, // so try_from rejects via `bytes.len() < header.size()`. - let result = Message::::try_from(owned); + let result = Message::::try_from(owned); assert!(matches!(result, Err(ConsensusError::InvalidCommand { .. }))); } @@ -877,11 +779,8 @@ mod tests { // so only the construction-time `size` floor rejects it (the // buffer-length check passes). Guards the `[size_of::()..size]` // underflow at every downstream call site. - let owned = header_bytes( - Command2::Request, - size_of::() as u32 - 1, - ); - let result = Message::::try_from(owned); + let owned = header_bytes(Command2::Request, size_of::() as u32 - 1); + let result = Message::::try_from(owned); assert!(matches!(result, Err(ConsensusError::InvalidCommand { .. }))); } @@ -890,7 +789,7 @@ mod tests { #[test] fn as_generic_view_reads_command_byte() { let owned = header_bytes(Command2::Request, 256); - let typed = Message::::try_from(owned).expect("valid"); + let typed = Message::::try_from(owned).expect("valid"); let generic = typed.as_generic(); assert_eq!(generic.header().command, Command2::Request); assert_eq!(generic.total_len(), 256); @@ -900,11 +799,11 @@ mod tests { #[test] fn try_as_typed_command_mismatch_returns_err_without_unsafe_cast() { - // bytes are a valid Prepare; asking for RoutedRequestHeader must fail + // bytes are a valid Prepare; asking for RequestHeader must fail // *before* the unsafe ptr-cast inside try_as_typed. let owned = header_bytes(Command2::Prepare, 256); let generic = Message::::try_from(owned).expect("valid"); - let result = generic.try_as_typed::(); + let result = generic.try_as_typed::(); assert!(matches!( result, Err(ConsensusError::InvalidCommand { @@ -916,8 +815,7 @@ mod tests { #[test] fn try_as_typed_invalid_validation_returns_err() { - // `RequestHeader::validate` rejects operation=Register with non-zero - // session; the routed shape shares the same field rules. + // RequestHeader::validate rejects operation=Register with non-zero session. let mut owned = header_bytes(Command2::Request, 256); { let buf = owned.as_mut_slice(); @@ -935,7 +833,7 @@ mod tests { fn try_into_typed_command_mismatch_returns_err() { let owned = header_bytes(Command2::Prepare, 256); let generic = Message::::try_from(owned).expect("valid"); - let result = generic.try_into_typed::(); + let result = generic.try_into_typed::(); assert!(matches!( result, Err(ConsensusError::InvalidCommand { @@ -999,7 +897,7 @@ mod tests { } #[test] - fn client_wire_decode_of_request_with_invalid_register_session_returns_err() { + fn messagebag_dispatch_request_with_invalid_register_session_returns_err() { // `RequestHeader::validate` rejects Register with non-zero session. let mut owned = header_bytes(Command2::Request, 256); { @@ -1008,16 +906,16 @@ mod tests { buf[REQUEST_SESSION_OFF..REQUEST_SESSION_OFF + 8].copy_from_slice(&5u64.to_le_bytes()); } let generic = Message::::try_from(owned).expect("valid generic"); - let result = generic.try_into_typed::(); + let result = MessageBag::try_from(generic); assert!(matches!(result, Err(ConsensusError::InvalidField(_)))); } // Ingress validation runs on every client frame at the network boundary, - // reached through `try_into_typed` -> `RequestHeader::validate` before - // dispatch promotes the frame to `RoutedRequestHeader`. Several dedup and - // authz conclusions rest on it running, so pin the field rules rather than - // the plumbing: whatever `request_preflight` and the operation gate see - // downstream has already passed these. + // reached through `MessageBag::try_from` -> `try_into_typed` -> + // `RequestHeader::validate`. Several dedup and authz conclusions rest on + // it running, so pin the field rules rather than the plumbing: whatever + // `request_preflight` and the operation gate see downstream has already + // passed these. #[test] fn ingress_validation_enforces_the_request_header_field_rules() { // (operation, session, request, must_pass) @@ -1048,7 +946,7 @@ mod tests { .copy_from_slice(&request.to_le_bytes()); } let generic = Message::::try_from(owned).expect("valid generic"); - let accepted = generic.try_into_typed::().is_ok(); + let accepted = MessageBag::try_from(generic).is_ok(); assert_eq!( accepted, must_pass, "{operation:?} with session={session} request={request}" @@ -1070,7 +968,7 @@ mod tests { } let generic = Message::::try_from(owned).expect("valid generic"); assert!(matches!( - generic.try_into_typed::(), + MessageBag::try_from(generic), Err(ConsensusError::InvalidField(_)) )); } @@ -1080,7 +978,7 @@ mod tests { #[test] fn request_message_deep_copy_independent() { let owned = header_bytes(Command2::Request, 256); - let mut msg = Message::::try_from(owned).expect("valid"); + let mut msg = Message::::try_from(owned).expect("valid"); let copy = msg.deep_copy(); // Mutate the original's bytes; the deep copy must be untouched. msg.as_mut_slice()[200] = 0xab; @@ -1093,7 +991,7 @@ mod tests { #[test] fn transmute_header_request_to_prepare() { let owned = header_bytes(Command2::Request, 256); - let msg = Message::::try_from(owned).expect("valid"); + let msg = Message::::try_from(owned).expect("valid"); let prepared: Message = msg.transmute_header::(|_old, new| { new.command = Command2::Prepare; @@ -1102,71 +1000,6 @@ mod tests { assert_eq!(prepared.header().command, Command2::Prepare); } - // into_routed: in-place client-wire -> routed retype - - // Promotion must carry the data-bearing reserved prefix verbatim (the - // non-replicated op code lives in `reserved[0..4]`) and unset only the - // `group` tail, whatever junk the client sent in those eight bytes. - #[test] - fn into_routed_keeps_reserved_prefix_and_unsets_group() { - const RESERVED_OFF: usize = std::mem::offset_of!(RequestHeader, reserved); - - let mut owned = header_bytes(Command2::Request, 256); - { - let buf = owned.as_mut_slice(); - for (index, byte) in buf[RESERVED_OFF..RESERVED_OFF + 60].iter_mut().enumerate() { - *byte = u8::try_from(index).expect("60 fits u8") + 1; - } - } - let request = Message::::try_from(owned).expect("valid client frame"); - let client_header = *request.header(); - - let routed = request.into_routed(); - let header = routed.header(); - assert_eq!( - header.reserved[..], - client_header.reserved[..52], - "the reserved prefix carries data and must survive promotion" - ); - assert_eq!( - header.group, 0, - "the client-sent reserved tail must not leak into `group`" - ); - assert_eq!(header.client, client_header.client); - assert_eq!(header.operation, client_header.operation); - assert_eq!(header.session, client_header.session); - assert_eq!(header.request, client_header.request); - assert_eq!(header.user_id, client_header.user_id); - } - - // A peer-wire `Command2::Request` decodes as `RoutedRequestHeader`, so its - // validate must enforce the client-boundary field rules: a forged - // `client = 0` frame would otherwise reach the client table's hard assert - // and abort the metadata primary, and a `Reserved` operation would replay - // that client's cached register reply. - #[test] - fn messagebag_dispatch_rejects_request_with_zero_client() { - let mut owned = header_bytes(Command2::Request, 256); - owned.as_mut_slice()[REQUEST_CLIENT_OFF..REQUEST_CLIENT_OFF + 16] - .copy_from_slice(&0u128.to_le_bytes()); - let generic = Message::::try_from(owned).expect("valid generic"); - assert!(matches!( - MessageBag::try_from(generic), - Err(ConsensusError::InvalidField(_)) - )); - } - - #[test] - fn messagebag_dispatch_rejects_request_with_reserved_operation() { - let mut owned = header_bytes(Command2::Request, 256); - owned.as_mut_slice()[REQUEST_OPERATION_OFF] = Operation::Reserved as u8; - let generic = Message::::try_from(owned).expect("valid generic"); - assert!(matches!( - MessageBag::try_from(generic), - Err(ConsensusError::InvalidField(_)) - )); - } - // ResponseBacking via SmallVec #[test] diff --git a/core/server_common/src/indexes_mut.rs b/core/server_common/src/indexes_mut.rs index 394b234cb5..b31c59089f 100644 --- a/core/server_common/src/indexes_mut.rs +++ b/core/server_common/src/indexes_mut.rs @@ -171,7 +171,7 @@ impl IggyIndexesMut { // Bound to exactly one entry rather than to the buffer end: a file // whose length is not a whole multiple of INDEX_SIZE (e.g. a 24-byte - // server sparse index read back through this 16-byte reader on a + // server-ng sparse index read back through this 16-byte reader on a // mixed-format recovery) would otherwise hand an oversized slice to the // view and trip its length assertion. let start = (self.count() - 1) as usize * INDEX_SIZE; diff --git a/core/server_common/src/send_messages2.rs b/core/server_common/src/send_messages2.rs index 4dacba2e2b..f0d9a7f88d 100644 --- a/core/server_common/src/send_messages2.rs +++ b/core/server_common/src/send_messages2.rs @@ -19,7 +19,7 @@ use crate::consensus_message::{MESSAGE_ALIGN, Message}; use crate::iobuf::Owned; use crate::sharding::IggyNamespace; use bytes::{Bytes, BytesMut}; -use iggy_binary_protocol::{PrepareHeader, RoutedRequestHeader}; +use iggy_binary_protocol::{PrepareHeader, RequestHeader}; use iggy_common::{EncryptorKind, INDEX_SIZE, IggyError, random_id}; use std::hash::Hasher; use twox_hash::XxHash3_64; @@ -196,20 +196,20 @@ impl SendMessages2Owned { pub fn encode_request( self, - mut request_header: RoutedRequestHeader, - ) -> Result, IggyError> { - let total_size = std::mem::size_of::() + self.header.total_size(); + mut request_header: RequestHeader, + ) -> Result, IggyError> { + let total_size = std::mem::size_of::() + self.header.total_size(); // The converted body differs in size from the legacy wire body the // header described; a stale `size` truncates the rebuilt blob for // every downstream slice (stamping, journal reads). request_header.size = u32::try_from(total_size).map_err(|_| IggyError::InvalidCommand)?; let mut buffer = Owned::::zeroed(total_size); let bytes = buffer.as_mut_slice(); - bytes[0..std::mem::size_of::()] + bytes[0..std::mem::size_of::()] .copy_from_slice(bytemuck::bytes_of(&request_header)); self.header.encode_into( - &mut bytes[std::mem::size_of::() - ..std::mem::size_of::() + COMMAND_HEADER_SIZE], + &mut bytes[std::mem::size_of::() + ..std::mem::size_of::() + COMMAND_HEADER_SIZE], ); bytes[PREPARE_SPLIT_POINT..PREPARE_SPLIT_POINT + self.blob.len()] .copy_from_slice(&self.blob); @@ -473,12 +473,12 @@ pub(crate) type FrozenBatchHeader = crate::iobuf::Frozen; /// [`IggyError::InvalidCommand`] on an undecodable batch; encryption errors /// propagate from the encryptor. pub fn encrypt_batch_request( - message: Message, + message: Message, encryptor: &EncryptorKind, -) -> Result, IggyError> { +) -> Result, IggyError> { let request_header = *message.header(); let total_size = request_header.size as usize; - let body = &message.as_slice()[std::mem::size_of::()..total_size]; + let body = &message.as_slice()[std::mem::size_of::()..total_size]; let batch = decode_batch_slice(body)?; let mut blob = BytesMut::with_capacity(batch.blob().len() * 2); @@ -537,27 +537,26 @@ pub enum ChecksumMode { pub fn convert_request_message( namespace: IggyNamespace, - message: Message, + message: Message, checksum: ChecksumMode, -) -> Result, IggyError> { +) -> Result, IggyError> { let request_header = *message.header(); let total_size = request_header.size as usize; - let body = &message.as_slice()[std::mem::size_of::()..total_size]; + let body = &message.as_slice()[std::mem::size_of::()..total_size]; // A canonical body enters the pipeline verbatim, so it must end exactly at // `batch_length`: `size` and `batch_length` are independent client-supplied // fields and `decode_batch_slice` only lower-bounds the frame. A suffix past // `batch_length` is covered by no checksum, still rides the buffer to disk, // and desyncs the segment walk that advances by `batch_length`. - match decode_batch_slice(body) { - Ok(batch) if batch.message_count() == 0 => Err(IggyError::InvalidCommand), - Ok(batch) if body.len() == batch.header.total_size() => Ok(message), + match decode_batch_slice(body).map(|batch| batch.header.total_size()) { + Ok(batch_length) if body.len() == batch_length => Ok(message), Ok(_) => Err(IggyError::InvalidCommand), Err(_) => transcode_legacy_request(namespace, body, request_header, checksum), } } /// Transcode a legacy `SendMessages` request body directly into the canonical -/// `[RoutedRequestHeader][256B SendMessages2Header][blob]` form, writing each message +/// `[RequestHeader][256B SendMessages2Header][blob]` form, writing each message /// record straight into the final aligned buffer. /// /// Fused replacement for the `from_legacy_request(..).encode_request(..)` @@ -572,13 +571,10 @@ pub fn convert_request_message( fn transcode_legacy_request( namespace: IggyNamespace, body: &[u8], - mut request_header: RoutedRequestHeader, + mut request_header: RequestHeader, checksum: ChecksumMode, -) -> Result, IggyError> { +) -> Result, IggyError> { let (message_count, messages) = legacy_messages_slice(body)?; - if message_count == 0 { - return Err(IggyError::InvalidCommand); - } let mut parsed = Vec::with_capacity(message_count as usize); let mut origin_timestamp = u64::MAX; let mut cursor = 0usize; @@ -602,7 +598,7 @@ fn transcode_legacy_request( origin_timestamp = 0; } - let header_size = std::mem::size_of::(); + let header_size = std::mem::size_of::(); let batch_length = COMMAND_HEADER_SIZE .checked_add(blob_len) .ok_or(IggyError::InvalidCommand)?; @@ -682,37 +678,6 @@ fn transcode_legacy_request( /// chunk and steps by `batch_length`. Callers whose buffer is meant to BE the /// batch must reject the surplus themselves - see [`convert_request_message`]. pub fn decode_batch_slice(body: &[u8]) -> Result, IggyError> { - decode_batch_slice_with(body, BatchIntegrity::Verify) -} - -/// How much of a batch record [`decode_batch_slice_with`] proves before returning it. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum BatchIntegrity { - /// Re-hash the batch and reject it unless it matches its own `batch_checksum`. - Verify, - /// Check the framing only, and hand back whatever it describes. The caller is - /// accepting bytes that may not be the ones written. - LayoutOnly, -} - -/// [`decode_batch_slice`] with the integrity level chosen by the caller. -/// -/// An enum, not a bool: the one caller that passes anything but -/// [`BatchIntegrity::Verify`] is the disk poll under its operator knob, and -/// `..., false)` there reads like a detail rather than opting a read out of -/// corruption detection. -/// -/// Layout checks are not optional either way: a short or self-inconsistent record is -/// rejected regardless, because the caller would otherwise index past it. -/// -/// # Errors -/// [`IggyError::InvalidCommand`] for a short or inconsistent record, and -/// [`IggyError::InvalidBatchChecksum`] under [`BatchIntegrity::Verify`] when the batch -/// does not match. Callers that must tell corruption from a partial tail need both. -pub fn decode_batch_slice_with( - body: &[u8], - integrity: BatchIntegrity, -) -> Result, IggyError> { if body.len() < COMMAND_HEADER_SIZE { return Err(IggyError::InvalidCommand); } @@ -725,18 +690,13 @@ pub fn decode_batch_slice_with( let blob = &body[COMMAND_HEADER_SIZE..COMMAND_HEADER_SIZE + blob_len]; let batch = SendMessages2Ref { header, blob }; - match integrity { - BatchIntegrity::Verify => { - let expected_checksum = verify_and_recompute_batch_checksum(&batch)?; - if header.batch_checksum != expected_checksum { - return Err(IggyError::InvalidBatchChecksum( - header.batch_checksum, - expected_checksum, - header.base_offset, - )); - } - } - BatchIntegrity::LayoutOnly => validate_batch_layout(&batch)?, + let expected_checksum = verify_and_recompute_batch_checksum(&batch)?; + if header.batch_checksum != expected_checksum { + return Err(IggyError::InvalidBatchChecksum( + header.batch_checksum, + expected_checksum, + header.base_offset, + )); } Ok(batch) @@ -766,13 +726,15 @@ pub fn decode_prepare_slice(bytes: &[u8]) -> Result, IggyEr /// /// INVARIANT: `bytes` MUST be node-local self-stamped - /// [`stamp_prepare_for_persistence`] recomputed the batch checksum over the -/// exact blob on the local node - or already integrity-checked at network +/// exact blob on THIS node - or already integrity-checked at their network /// ingress. There is no consensus-layer blob validation: the `PrepareHeader` -/// integrity fields are inert zeros. Replicated and repaired prepares are -/// validated via [`decode_prepare_slice`] before the bytes reach any trusted -/// decode. Calling this on unvalidated network bytes would let a corrupted blob -/// pass undetected. The full-body per-message checksum pass dominates -/// produce-path CPU, so trusted call sites that only read header meta skip it. +/// integrity fields are inert zeros. A replicated `SendMessages` prepare is +/// gated per-message on receipt by [`verify_received_send_messages`], and a +/// repaired prepare is validated via [`decode_prepare_slice`]; both run BEFORE +/// the bytes reach any trusted decode. NEVER call this on unvalidated network +/// bytes - it would let a corrupted blob pass undetected. The full-body +/// per-message checksum pass dominates produce-path CPU, so trusted call sites +/// that only read header meta skip it. /// /// # Errors /// @@ -862,6 +824,35 @@ pub fn stamp_prepare_for_persistence( Ok((message, command, command.message_count)) } +/// Verify every per-message checksum in a received `SendMessages` prepare. +/// +/// The FIRST blob-integrity check on the replicated path: the `PrepareHeader` +/// integrity fields are inert zeros and the batch checksum is recomputed +/// locally at stamp, so transit corruption of a message body would otherwise +/// reach apply undetected. Backups call this before journaling a replicated +/// prepare; on a mismatch the caller fails closed (drop, no `PrepareOk`) and the +/// primary retransmits on prepare-timeout. +/// +/// The stored `batch_checksum` is not consulted (a received prepare is +/// pre-stamp, `base_offset` / `base_timestamp` zero): integrity rests on the +/// per-message checksums, recomputed over the stamp-invariant cover +/// (`header[8..48] || payload || user_headers`), which excludes the 256B command +/// header, so it holds whether or not this node has stamped yet. Shares the +/// frame walk with the validating decoders via +/// [`verify_and_recompute_batch_checksum`], discarding its recomputed batch +/// value. +/// +/// # Errors +/// +/// [`IggyError::InvalidCommand`] if the records do not tile `message_count` +/// exactly (a length-field corruption desyncs the walk); +/// [`IggyError::InvalidMessageChecksum`] on the first per-message mismatch. +pub fn verify_received_send_messages(bytes: &[u8]) -> Result<(), IggyError> { + let batch = decode_prepare_slice_trusted(bytes)?; + verify_and_recompute_batch_checksum(&batch)?; + Ok(()) +} + fn legacy_messages_slice(body: &[u8]) -> Result<(u32, &[u8]), IggyError> { if body.len() < 4 { return Err(IggyError::InvalidCommand); @@ -1009,24 +1000,6 @@ fn verify_and_recompute_batch_checksum(batch: &SendMessages2Ref<'_>) -> Result) -> Result<(), IggyError> { - let mut framed = 0u32; - let mut covered = 0usize; - for message in batch.iter_with_offsets() { - framed += 1; - covered = message.end; - } - if framed != batch.message_count() || covered != batch.blob().len() { - return Err(IggyError::InvalidCommand); - } - Ok(()) -} - fn read_u32(bytes: &[u8], offset: usize) -> Result { bytes .get(offset..offset + 4) @@ -1206,8 +1179,8 @@ mod tests { } /// `[PrepareHeader][256B batch header][blob]` carrying real per-message - /// records and checksums from the production encoder, with the initial zero - /// base offset and timestamp. + /// records + checksums from the production encoder, left pre-stamp + /// (`base_offset` / `base_timestamp` zero) as a follower receives it. fn prepare_with_messages(messages: &IggyMessages2) -> Owned { let namespace = IggyNamespace::new(1, 1, 7); let owned = @@ -1215,6 +1188,42 @@ mod tests { prepare_from_owned(&owned) } + #[test] + fn verify_received_send_messages_accepts_clean_batch() { + let owned = prepare_with_messages(&sample_messages()); + verify_received_send_messages(owned.as_slice()) + .expect("a clean batch passes the receive gate"); + } + + #[test] + fn verify_received_send_messages_rejects_flipped_payload_byte() { + let mut owned = prepare_with_messages(&sample_messages()); + // First payload begins right after the first message's 48B header. + let payload_index = PREPARE_SPLIT_POINT + MESSAGE_HEADER_SIZE; + owned.as_mut_slice()[payload_index] ^= 0xFF; + assert!( + matches!( + verify_received_send_messages(owned.as_slice()), + Err(IggyError::InvalidMessageChecksum(..)) + ), + "a flipped payload byte must fail the per-message checksum", + ); + } + + #[test] + fn verify_received_send_messages_rejects_flipped_stored_checksum() { + let mut owned = prepare_with_messages(&sample_messages()); + // The first message's stored checksum is the first 8 bytes of the blob. + owned.as_mut_slice()[PREPARE_SPLIT_POINT] ^= 0xFF; + assert!( + matches!( + verify_received_send_messages(owned.as_slice()), + Err(IggyError::InvalidMessageChecksum(..)) + ), + "a flipped stored checksum must fail the per-message check", + ); + } + #[test] fn checksum_oneshot_matches_streaming_reference() { // Formula pin: the per-message checksum is XxHash3-64 (default seed) @@ -1372,14 +1381,14 @@ mod tests { body } - fn legacy_request_message(body: &[u8]) -> Message { - let header_size = std::mem::size_of::(); + fn legacy_request_message(body: &[u8]) -> Message { + let header_size = std::mem::size_of::(); let total = header_size + body.len(); let mut buffer = Owned::::zeroed(total); { - let header: &mut RoutedRequestHeader = + let header: &mut RequestHeader = bytemuck::checked::try_from_bytes_mut(&mut buffer.as_mut_slice()[..header_size]) - .expect("zeroed bytes form a valid RoutedRequestHeader"); + .expect("zeroed bytes form a valid RequestHeader"); header.command = Command2::Request; header.operation = Operation::SendMessages; header.client = 1; @@ -1391,29 +1400,6 @@ mod tests { Message::try_from(buffer).expect("legacy request message is valid") } - #[test] - fn convert_request_message_rejects_empty_canonical_and_legacy_batches() { - let namespace = IggyNamespace::new(1, 1, 3); - let messages = IggyMessages2::with_capacity(0); - let canonical = SendMessages2Owned::from_messages(namespace, &messages) - .expect("build empty canonical batch"); - let mut canonical_body = vec![0; canonical.header.total_size()]; - canonical - .header - .encode_into(&mut canonical_body[..COMMAND_HEADER_SIZE]); - let legacy_body = legacy_send_messages_body(&messages); - - for mode in [ChecksumMode::Compute, ChecksumMode::Skip] { - let canonical_result = - convert_request_message(namespace, legacy_request_message(&canonical_body), mode); - assert!(matches!(canonical_result, Err(IggyError::InvalidCommand))); - - let legacy_result = - convert_request_message(namespace, legacy_request_message(&legacy_body), mode); - assert!(matches!(legacy_result, Err(IggyError::InvalidCommand))); - } - } - #[test] fn convert_request_message_transcodes_legacy_to_canonical_bytes() { // Golden: the fused legacy transcode must emit the exact canonical batch @@ -1434,7 +1420,7 @@ mod tests { let legacy = legacy_request_message(&legacy_send_messages_body(&messages)); let converted = convert_request_message(namespace, legacy, ChecksumMode::Compute) .expect("legacy body transcodes"); - let header_size = std::mem::size_of::(); + let header_size = std::mem::size_of::(); let actual_body = &converted.as_slice()[header_size..converted.header().size as usize]; assert_eq!( @@ -1461,7 +1447,7 @@ mod tests { let namespace = IggyNamespace::new(1, 1, 3); let messages = sample_messages(); let body = legacy_send_messages_body(&messages); - let header_size = std::mem::size_of::(); + let header_size = std::mem::size_of::(); let computed = convert_request_message( namespace, @@ -1504,7 +1490,7 @@ mod tests { // batch and returns it unchanged. Every decode must succeed. let namespace = IggyNamespace::new(1, 1, 3); let messages = sample_messages(); - let header_size = std::mem::size_of::(); + let header_size = std::mem::size_of::(); let legacy = legacy_request_message(&legacy_send_messages_body(&messages)); let canonical = convert_request_message(namespace, legacy, ChecksumMode::Compute) @@ -1537,19 +1523,19 @@ mod tests { const TRAILING_JUNK_CASES: [&[u8]; 2] = [&[0xAA], &[0xFF; 64]]; /// Canonical `SendMessages` request carrying `junk` past `batch_length`, with - /// `RoutedRequestHeader.size` inflated to cover it. `size` and `batch_length` are + /// `RequestHeader.size` inflated to cover it. `size` and `batch_length` are /// independent wire fields, so a non-conforming client can emit this. - fn canonical_request_with_trailing_bytes(junk: &[u8]) -> Message { + fn canonical_request_with_trailing_bytes(junk: &[u8]) -> Message { let namespace = IggyNamespace::new(1, 1, 3); let owned = SendMessages2Owned::from_messages(namespace, &sample_messages()).expect("build batch"); - let header_size = std::mem::size_of::(); + let header_size = std::mem::size_of::(); let total = header_size + owned.header.total_size() + junk.len(); let mut buffer = Owned::::zeroed(total); { - let header: &mut RoutedRequestHeader = + let header: &mut RequestHeader = bytemuck::checked::try_from_bytes_mut(&mut buffer.as_mut_slice()[..header_size]) - .expect("zeroed bytes form a valid RoutedRequestHeader"); + .expect("zeroed bytes form a valid RequestHeader"); header.command = Command2::Request; header.operation = Operation::SendMessages; header.client = 1; @@ -1567,7 +1553,8 @@ mod tests { Message::try_from(buffer).expect("request message is valid") } - /// A `Prepare` whose `size` covers `junk` past `batch_length`. + /// The replicated counterpart: a pre-stamp `Prepare` whose `size` covers + /// `junk` past `batch_length`. fn prepare_with_trailing_bytes(junk: &[u8]) -> Owned { let namespace = IggyNamespace::new(1, 1, 7); let owned = @@ -1625,11 +1612,18 @@ mod tests { } #[test] - fn decode_prepare_slice_rejects_trailing_bytes_past_batch_length() { - // Replica ingest must reject bytes beyond `batch_length` because no - // per-message checksum covers them. + fn verify_received_send_messages_rejects_trailing_bytes_past_batch_length() { + // Replica ingest boundary. The gate clamps the blob to `batch_length` + // before verifying, so without an exact-frame check a primary could plant + // bytes that no per-message checksum covers on every backup. for junk in TRAILING_JUNK_CASES { let owned = prepare_with_trailing_bytes(junk); + let result = verify_received_send_messages(owned.as_slice()); + assert!( + matches!(result, Err(IggyError::InvalidCommand)), + "{} trailing bytes must fail the receive gate, got {result:?}", + junk.len(), + ); assert!( matches!( decode_prepare_slice(owned.as_slice()), diff --git a/core/server_common/src/sharding/mod.rs b/core/server_common/src/sharding/mod.rs index b73169f526..761e41d045 100644 --- a/core/server_common/src/sharding/mod.rs +++ b/core/server_common/src/sharding/mod.rs @@ -22,9 +22,10 @@ mod shard_id; pub use local_idx::LocalIdx; pub use namespace::{ - IggyNamespace, MAX_PARTITIONS, MAX_STREAMS, MAX_TOPICS, METADATA_GROUP, NamespaceCapacityError, - PACKED_NAMESPACE_BITS, PACKED_NAMESPACE_MAX, PARTITION_BITS, PARTITION_MASK, PARTITION_SHIFT, - STREAM_BITS, STREAM_MASK, STREAM_SHIFT, TOPIC_BITS, TOPIC_MASK, TOPIC_SHIFT, + IggyNamespace, MAX_PARTITIONS, MAX_STREAMS, MAX_TOPICS, METADATA_CONSENSUS_NAMESPACE, + NamespaceCapacityError, PACKED_NAMESPACE_BITS, PACKED_NAMESPACE_MAX, PARTITION_BITS, + PARTITION_MASK, PARTITION_SHIFT, STREAM_BITS, STREAM_MASK, STREAM_SHIFT, TOPIC_BITS, + TOPIC_MASK, TOPIC_SHIFT, }; pub use partition_location::PartitionLocation; pub use shard_id::ShardId; diff --git a/core/server_common/src/sharding/namespace.rs b/core/server_common/src/sharding/namespace.rs index 435b076c41..839edc0a24 100644 --- a/core/server_common/src/sharding/namespace.rs +++ b/core/server_common/src/sharding/namespace.rs @@ -27,7 +27,7 @@ // shard. Re-exported here for ergonomics of existing call sites. pub use iggy_binary_protocol::namespace::{ - MAX_PARTITIONS, MAX_STREAMS, MAX_TOPICS, METADATA_GROUP, PACKED_NAMESPACE_BITS, + MAX_PARTITIONS, MAX_STREAMS, MAX_TOPICS, METADATA_CONSENSUS_NAMESPACE, PACKED_NAMESPACE_BITS, PACKED_NAMESPACE_MAX, PARTITION_BITS, PARTITION_MASK, PARTITION_SHIFT, STREAM_BITS, STREAM_MASK, STREAM_SHIFT, TOPIC_BITS, TOPIC_MASK, TOPIC_SHIFT, bits_required, }; @@ -149,7 +149,7 @@ impl IggyNamespace { #[cfg(test)] mod tests { use super::{ - IggyNamespace, MAX_PARTITIONS, MAX_STREAMS, MAX_TOPICS, METADATA_GROUP, + IggyNamespace, MAX_PARTITIONS, MAX_STREAMS, MAX_TOPICS, METADATA_CONSENSUS_NAMESPACE, NamespaceCapacityError, PACKED_NAMESPACE_BITS, PACKED_NAMESPACE_MAX, }; @@ -158,26 +158,26 @@ mod tests { // STREAM_BITS / TOPIC_BITS / PARTITION_BITS that closes the gap between // the packed range and the sentinel will fail to build. const _: () = { - assert!(METADATA_GROUP > PACKED_NAMESPACE_MAX); + assert!(METADATA_CONSENSUS_NAMESPACE > PACKED_NAMESPACE_MAX); assert!(PACKED_NAMESPACE_BITS == 12 + 12 + 20); assert!(PACKED_NAMESPACE_MAX == (1u64 << PACKED_NAMESPACE_BITS) - 1); }; #[test] fn metadata_sentinel_cannot_collide_with_any_packable_namespace() { - assert!(!IggyNamespace::is_packable(METADATA_GROUP)); + assert!(!IggyNamespace::is_packable(METADATA_CONSENSUS_NAMESPACE)); // The (0, 0, 0) corner is intentionally a legal partition, which is // precisely why `0` is unsuitable as the metadata sentinel. let zero = IggyNamespace::new(0, 0, 0); assert_eq!(zero.inner(), 0); assert!(IggyNamespace::is_packable(zero.inner())); - assert_ne!(zero.inner(), METADATA_GROUP); + assert_ne!(zero.inner(), METADATA_CONSENSUS_NAMESPACE); // Maximum packable triple stays inside the packed range. let max = IggyNamespace::new(MAX_STREAMS - 1, MAX_TOPICS - 1, MAX_PARTITIONS - 1); assert!(IggyNamespace::is_packable(max.inner())); - assert_ne!(max.inner(), METADATA_GROUP); + assert_ne!(max.inner(), METADATA_CONSENSUS_NAMESPACE); } #[test] diff --git a/core/shard/Cargo.toml b/core/shard/Cargo.toml index 874fb53532..439221c447 100644 --- a/core/shard/Cargo.toml +++ b/core/shard/Cargo.toml @@ -25,7 +25,7 @@ publish = false [features] # Simulator-only test hook (`IggyShard::init_partition`): bypasses the # reconciler's `ReconcileOp::InsertOwned` funnel, mutating `IggyPartitions` -# off the pump task. A `-p iggy-server` build excludes it; `cargo build +# off the pump task. A `-p iggy-server-ng` build excludes it; `cargo build # --workspace` unifies features and compiles it into the shared `shard` # unit (simulator requests it). Benign: no production caller. simulator = [] diff --git a/core/shard/src/lib.rs b/core/shard/src/lib.rs index d898e6a909..28537d1c5f 100644 --- a/core/shard/src/lib.rs +++ b/core/shard/src/lib.rs @@ -28,22 +28,21 @@ pub use router::CONSENSUS_TICK_INTERVAL; #[cfg(any(test, feature = "simulator"))] use consensus::LocalPipeline; use consensus::{ - ChunkProgress, CommitOutcome, Consensus, ConsensusClock, DVC_HEADERS_MAX, DvcHeaderKind, - DvcSuffix, MergedLog, MetadataHandle, MuxPlane, PartitionsHandle, Pipeline, Plane, PlaneKind, - STATE_TRANSFER_MAX_DECODE_RETRIES, STATE_TRANSFER_MAX_STALL_RETRIES, Sequencer, Status, - VsrAction, VsrConsensus, build_deny_reply_from_request_header, dvc_blank, dvc_header_kind, - encode_prepare_headers, restamp_prepare_view, verify_prepare_integrity, + ChunkProgress, CommitOutcome, Consensus, ConsensusClock, MetadataHandle, MuxPlane, + PartitionsHandle, Pipeline, Plane, PlaneKind, STATE_TRANSFER_MAX_DECODE_RETRIES, + STATE_TRANSFER_MAX_STALL_RETRIES, Sequencer, VsrAction, VsrConsensus, + build_deny_reply_from_request_header, }; #[cfg(any(test, feature = "simulator"))] use crossfire::AsyncRxTrait; use crossfire::TrySendError; use futures::FutureExt; use iggy_binary_protocol::{ - CHECKSUM_UNSEALED, Command2, CommitHeader, ConsensusHeader, DoViewChangeHeader, GenericHeader, - Operation, PrepareHeader, PrepareOkHeader, RepairPrepareHeader, RepairRangeReplyHeader, + Command2, CommitHeader, DoViewChangeHeader, GenericHeader, Operation, PrepareHeader, + PrepareOkHeader, RepairPrepareHeader, RepairRangeReplyHeader, RequestHeader, RequestPreparesHeader, RequestStartViewHeader, RequestStateChunkHeader, - RequestStateTransferHeader, RoutedRequestHeader, StartViewChangeHeader, StartViewHeader, - StateChunkHeader, StateTransferTargetHeader, + RequestStateTransferHeader, StartViewChangeHeader, StartViewHeader, StateChunkHeader, + StateTransferTargetHeader, }; #[cfg(any(test, feature = "simulator"))] use iggy_common::PartitionStats; @@ -217,7 +216,7 @@ pub enum MetadataSubmit { /// Handler shard 0 runs for an inbound [`MetadataSubmit`]. /// -/// The server wires it to `submit_register_in_process` / +/// server-ng wires it to `submit_register_in_process` / /// `submit_logout_in_process` / `submit_request_in_process` and sends the /// result back over the frame's `reply` sender. A peer shard (no consensus) /// must never receive this frame. @@ -250,7 +249,7 @@ pub struct ConnectedClientInfo { } /// Handler each shard runs for an inbound [`LifecycleFrame::ListClients`]. -/// The server wires it to read the shard's `SessionManager` and push its +/// server-ng wires it to read the shard's `SessionManager` and push its /// connected clients back over the carried reply sender. pub type ListClientsHandler = Rc>)>; @@ -333,7 +332,7 @@ pub enum PartitionReadReply { } /// Handler the owning shard runs for an inbound -/// [`LifecycleFrame::PartitionRead`]. The server wires it to its partitions +/// [`LifecycleFrame::PartitionRead`]. server-ng wires it to its partitions /// plane; the handler pushes the result back over the carried reply sender. pub type PartitionReadHandler = Rc)>; @@ -729,7 +728,7 @@ impl ShardFrame { /// the receiver pulls the window chunk by chunk instead (each walked /// `RepairDone` immediately requests the next chunk while progress holds). /// -/// Runtime default; the server overrides the live ceiling per shard from +/// Runtime default; server-ng overrides the live ceiling per shard from /// `[cluster] repair_chunk_max` at bootstrap. pub const REPAIR_CHUNK_MAX: u64 = 128; @@ -866,7 +865,7 @@ const SEGMENT_SIZE_CEILING_BYTES: u64 = 1 << 30; /// The most one segment can overshoot its size cap: rotation checks the cap /// AFTER appending, so a segment closes at most one maximum-size batch past it. /// -/// Derived from the BUS frame cap, not `MAX_PAYLOAD_SIZE`: the server never +/// Derived from the BUS frame cap, not `MAX_PAYLOAD_SIZE`: server-ng never /// enforces the latter (its only enforcement sites are the legacy server and the /// SDK batch types), so the largest appendable batch is whatever the message bus /// will frame. This tracks the shipped `message_bus.max_message_size` default; an @@ -879,7 +878,7 @@ const SEGMENT_SIZE_OVERSHOOT_BYTES: u64 = 64 * 1024 * 1024; /// /// Mirrors `[partition] transfer_artifact_bytes_max`. Free const so the config /// crate's copy can be pinned to it by a `const _: () = assert!(..)` at the -/// server build edge, the way every other runtime default is. +/// server-ng build edge, the way every other runtime default is. pub const PARTITION_ARTIFACT_LEN_DEFAULT: u64 = SEGMENT_SIZE_CEILING_BYTES + SEGMENT_SIZE_OVERSHOOT_BYTES; @@ -1187,13 +1186,13 @@ where on_metadata_submit: MetadataSubmitHandler, /// Handler for inbound [`LifecycleFrame::ListClients`] broadcast - /// queries. Every shard receives these (not just shard 0); the server + /// queries. Every shard receives these (not just shard 0); server-ng /// wires it to its per-shard `SessionManager`. Defaults to a no-op for /// the simulator stub ctor. on_list_clients: ListClientsHandler, /// Handler for inbound [`LifecycleFrame::PartitionRead`] queries. - /// The server wires it to this shard's partitions plane. Defaults to a + /// server-ng wires it to this shard's partitions plane. Defaults to a /// no-op for the simulator stub ctor. on_partition_read: PartitionReadHandler, @@ -1289,30 +1288,30 @@ where shard_park_shedding: Cell, /// Live ceiling on prepares served per `RequestPrepares` round. Defaults - /// to [`REPAIR_CHUNK_MAX`]; the server overrides it from + /// to [`REPAIR_CHUNK_MAX`]; server-ng overrides it from /// `[cluster] repair_chunk_max` at bootstrap. repair_chunk_max: Cell, /// Live stalled-repair retry threshold in consensus ticks. Defaults to - /// [`partitions::REPAIR_RETRY_TICKS`]; the server overrides it from + /// [`partitions::REPAIR_RETRY_TICKS`]; server-ng overrides it from /// `[cluster] repair_retry_interval` at bootstrap. repair_retry_ticks: Cell, /// Live `[partition] transfer_served_cache_bytes_max`: the byte budget for /// segment payloads this shard keeps resident to serve chunk requests. - /// Defaults to [`SERVED_SEGMENT_CACHE_BYTES_DEFAULT`]; the server + /// Defaults to [`SERVED_SEGMENT_CACHE_BYTES_DEFAULT`]; server-ng /// overrides it at bootstrap. served_segment_cache_bytes_max: Cell, /// Live `[partition] transfer_artifact_bytes_max`: the alloc ceiling for one /// RECEIVED artifact. Defaults to [`PARTITION_ARTIFACT_LEN_DEFAULT`]; - /// the server overrides it at bootstrap. + /// server-ng overrides it at bootstrap. partition_artifact_len_max: Cell, /// Live `[message_bus] max_message_size`. Bounds a served state chunk: a /// frame above this is rejected by the RECEIVING transport, which tears /// down the whole replica connection. Defaults to a value that leaves - /// [`STATE_CHUNK_LEN`] usable; the server overrides it at bootstrap. + /// [`STATE_CHUNK_LEN`] usable; server-ng overrides it at bootstrap. bus_max_message_size: Cell, /// Consecutive metadata state-transfer rounds that made no progress. @@ -1776,34 +1775,6 @@ where let _ = sender.try_send(ShardFrame::lifecycle(LifecycleFrame::ReconcileApply)); } - /// `true` when an `InsertOwned` for `namespace` is built and queued but not - /// yet applied. - /// - /// The reconciler's own "already handled" test is `IggyPartitions::contains`, - /// which only turns true once the pump applies, so without this a pass run - /// during that lag rebuilds a namespace an earlier pass already built. The - /// queue IS the record of that in-flight work, so asking it cannot drift - /// from reality the way a parallel set would: every op leaves the queue - /// through `apply_reconcile_ops`, which either inserts or discards. - /// - /// Deliberately blind to `epoch`. Matching it would let a delete + recreate - /// landing inside the lag build a second incarnation over the queued one's - /// on-disk path, which is the case this exists to prevent; the recreate is - /// not lost, it costs one pass. The queued (dead-epoch) op applies, and the - /// next pass reads the epoch mismatch off the routing row and takes the - /// stale-incarnation teardown into a clean rebuild. - pub fn has_staged_insert_owned(&self, namespace: IggyNamespace) -> bool { - self.reconcile_queue.borrow().iter().any(|op| { - matches!( - op, - ReconcileOp::InsertOwned { - namespace: staged_namespace, - .. - } if *staged_namespace == namespace - ) - }) - } - /// Stage a segment-cleaner pass for `namespace` on this shard's pump. The /// timer task resolves retention config off-pump and stamps `now`; the pump /// is the single writer of partition state, so the deletion runs there, @@ -1911,30 +1882,18 @@ where epoch, } => { // Idempotent apply, mirroring `ConfirmRemove` (idempotent - // via `remove`'s `None` early-return). An unconditional - // `insert` over a live namespace would push a duplicate - // partition and overwrite the `ns -> idx` entry, orphaning - // the first: its VSR group + segment writers leak and `len` - // inflates. - // - // A backstop, not the mechanism. `reconcile_additions` - // skips a namespace whose `InsertOwned` is already staged - // ([`Self::has_staged_insert_owned`]), so a second op for a - // live namespace should not be built at all. Dropping one - // here is damage control rather than a free no-op: the - // build already planted its initial segment over the live - // incarnation's path and folded that into the namespace's - // shared stats. + // via `remove`'s `None` early-return). The reconciler + // stages this from a task separate from the pump, so under + // a commit burst two passes can each observe + // `!contains(ns)` and build the same namespace before + // either drains here. A second unconditional `insert` + // would push a duplicate partition and overwrite the + // `ns -> idx` entry, orphaning the first (its VSR group + + // segment writers leak and `len` inflates). The discarded + // build is a fresh empty incarnation over the same on-disk + // path the kept one owns, so dropping it just closes a few + // fds. if partitions.contains(&namespace) { - tracing::error!( - shard = self_shard_id, - ns_raw = namespace.inner(), - epoch, - "discarding duplicate InsertOwned for a live namespace: the \ - staged-op guard was bypassed and the build re-planted segment 0 \ - over the live incarnation's path" - ); - self.metrics.record_duplicate_partition_build_discarded(); drop(partition); continue; } @@ -2288,7 +2247,7 @@ where { match MessageBag::try_from(message) { Ok(MessageBag::Request(request)) => { - let routing = (request.header().operation, request.header().group); + let routing = (request.header().operation, request.header().namespace); match self.park_if_unmaterialised(request, routing.0, routing.1) { // The incarnation fence runs only here, on client traffic. // A backup denying what the primary admitted would diverge @@ -2313,7 +2272,7 @@ where } } Ok(MessageBag::Prepare(prepare)) => { - let routing = (prepare.header().operation, prepare.header().group); + let routing = (prepare.header().operation, prepare.header().namespace); // A tombstoned prepare still flows to the plane: replicated // traffic has no client awaiting a reply on this node, and // the plane's own tombstone guard drops it. @@ -2370,7 +2329,7 @@ where Ok(MessageBag::RequestStateChunk(ref msg)) => self.on_request_state_chunk(msg).await, Ok(MessageBag::StateChunk(ref msg)) => self.on_state_chunk(msg).await, Err(e) => { - tracing::warn!(shard = self.id, error = %e, "dropping unparsable consensus frame"); + tracing::warn!(shard = self.id, error = %e, "dropping message with invalid command"); } } } @@ -2749,7 +2708,7 @@ where /// `frame_drops_total{variant=partition,reason=park_dropped}`. fn deny_parked_client_request(&self, frame: ParkedFrame) -> bool { if frame.message.header().command == Command2::Request - && let Ok(request) = frame.message.try_into_typed::() + && let Ok(request) = frame.message.try_into_typed::() { return self.stage_transient_deny(request.header()); } @@ -2954,7 +2913,7 @@ where /// budget. Sent directly over the bus; delivery failure is terminal for /// this reply (the client recovers via its own read-timeout). #[allow(clippy::future_not_send)] - async fn deny_partition_request_transient(&self, request_header: &RoutedRequestHeader) { + async fn deny_partition_request_transient(&self, request_header: &RequestHeader) { let reply = build_deny_reply_from_request_header( request_header, IggyError::TransientNotAccepted.as_code(), @@ -2991,7 +2950,7 @@ where /// /// Returns whether the pump took it. A shard with no sender stages nothing, /// so assuming success logs an answer for a request destroyed unanswered. - fn stage_transient_deny(&self, request_header: &RoutedRequestHeader) -> bool { + fn stage_transient_deny(&self, request_header: &RequestHeader) -> bool { let reply = build_deny_reply_from_request_header( request_header, IggyError::TransientNotAccepted.as_code(), @@ -3024,7 +2983,7 @@ where } #[allow(clippy::future_not_send)] - pub async fn on_request(&self, request: Message) + pub async fn on_request(&self, request: Message) where B: MessageBus, MJ: JournalHandle, @@ -3175,7 +3134,7 @@ where /// production runtime path; bootstrap recovery uses `load_partition`), /// so it must never run in production. VSR replica id comes from /// `PartitionConsensusConfig`, not `self.id` (the local shard index). A - /// `-p iggy-server` build excludes the `simulator` feature and this + /// `-p iggy-server-ng` build excludes the `simulator` feature and this /// method; `cargo build --workspace` compiles it in but with no /// production caller. /// `superblock` is this group's durable `(view, log_view)` store. Passing @@ -3242,7 +3201,7 @@ where } /// Resolve the single partition a VSR control frame addresses, keyed by - /// `header.group`. Warns and returns `None` when the namespace matches + /// `header.namespace`. Warns and returns `None` when the namespace matches /// neither metadata nor a live partition consensus. Returns `&mut` because /// `on_do_view_change` / `on_commit` need it for `commit_journal`; the read- /// only callers reborrow `&`. Pump-only (sole mutator), so the `&mut` formed @@ -3271,7 +3230,7 @@ where return None; }; debug_assert_eq!( - partition.consensus().group(), + partition.consensus().namespace(), namespace, "keyed partition lookup must match the frame namespace" ); @@ -3296,9 +3255,8 @@ where let planes = self.plane.inner(); if let Some(ref consensus) = planes.0.consensus - && consensus.group() == header.group + && consensus.namespace() == header.namespace { - refresh_metadata_dvc_suffix(consensus, planes.0.journal.as_ref()); let actions = consensus.handle_start_view_change(PlaneKind::Metadata, &header); let (local_actions, wire_actions) = split_local_actions(actions); dispatch_vsr_actions(consensus, planes.0.journal.as_ref(), &local_actions).await; @@ -3310,14 +3268,13 @@ where let Some(partition) = self.resolve_partition_target( &planes.1.0, - header.group, + header.namespace, header.view, header.replica, "StartViewChange", ) else { return; }; - refresh_partition_dvc_suffix(partition); let consensus = partition.consensus(); let actions = consensus.handle_start_view_change(PlaneKind::Partitions, &header); let (local_actions, wire_actions) = split_local_actions(actions); @@ -3347,20 +3304,9 @@ where let planes = self.plane.inner(); if let Some(ref consensus) = planes.0.consensus - && consensus.group() == header.group + && consensus.namespace() == header.namespace { - refresh_metadata_dvc_suffix(consensus, planes.0.journal.as_ref()); - let Some(suffix_body) = control_suffix_body_verified(&msg, header.checksum_body) else { - tracing::warn!( - shard = self.id, - from_replica = header.replica, - view = header.view, - "dropping do_view_change whose body failed its checksum" - ); - return; - }; - let actions = - consensus.handle_do_view_change(PlaneKind::Metadata, &header, suffix_body); + let actions = consensus.handle_do_view_change(PlaneKind::Metadata, &header); let (local_actions, wire_actions) = split_local_actions(actions); dispatch_vsr_actions(consensus, planes.0.journal.as_ref(), &local_actions).await; if planes.0.persist_superblock_if_needed(consensus).await { @@ -3381,25 +3327,15 @@ where let config = planes.1.0.config(); let Some(partition) = self.resolve_partition_target( &planes.1.0, - header.group, + header.namespace, header.view, header.replica, "DoViewChange", ) else { return; }; - refresh_partition_dvc_suffix(partition); let consensus = partition.consensus(); - let Some(suffix_body) = control_suffix_body_verified(&msg, header.checksum_body) else { - tracing::warn!( - shard = self.id, - from_replica = header.replica, - view = header.view, - "dropping do_view_change whose body failed its checksum" - ); - return; - }; - let actions = consensus.handle_do_view_change(PlaneKind::Partitions, &header, suffix_body); + let actions = consensus.handle_do_view_change(PlaneKind::Partitions, &header); let (local_actions, wire_actions) = split_local_actions(actions); // Locals go to the partition dispatcher ONLY: `RebuildPipeline` // executes there (`dispatch_vsr_actions` bails on `journal: None`) @@ -3421,7 +3357,8 @@ where } } - #[allow(clippy::future_not_send, clippy::too_many_lines)] + #[allow(clippy::future_not_send)] + #[allow(clippy::too_many_lines)] async fn on_start_view(&self, msg: Message) where B: MessageBus, @@ -3437,31 +3374,15 @@ where let planes = self.plane.inner(); if let Some(ref consensus) = planes.0.consensus - && consensus.group() == header.group + && consensus.namespace() == header.namespace { - let Some(suffix_body) = control_suffix_body_verified(&msg, header.checksum_body) else { - tracing::warn!( - shard = self.id, - from_replica = header.replica, - view = header.view, - "dropping start_view whose body failed its checksum" - ); - return; - }; - let actions = consensus.handle_start_view(PlaneKind::Metadata, &header, suffix_body); + let actions = consensus.handle_start_view(PlaneKind::Metadata, &header); // Every rejection path (wrong primary, old view, stale incarnation, // below the commit floor, self-sent) returns no actions, and an // adopted StartView always emits at least `CommitJournal`. That // makes emptiness the adoption signal -- and the arms below must // not fire on a StartView this replica did not adopt. let adopted = !actions.is_empty(); - if adopted { - // First chance to spot a local entry disagreeing with the view's log. - // Ahead of the local dispatch below: it truncates the journal that - // `RebuildPipeline` reads back, so a rebuild before it would seed the - // pipeline from the entries this is about to drop. - self.reconcile_metadata_view_divergence().await; - } let (local_actions, wire_actions) = split_local_actions(actions); dispatch_vsr_actions(consensus, planes.0.journal.as_ref(), &local_actions).await; if planes.0.persist_superblock_if_needed(consensus).await { @@ -3539,40 +3460,24 @@ where // entirely, but `IggyPartition::transfer` is `pub` and cleared inside the // partitions crate, so an externally maintained count would drift; that // refactor is a prerequisite, not a detail.) - let transfers_inflight = if Self::may_arm_partition_transfer(&planes.1.0, header.group) { + let transfers_inflight = if Self::may_arm_partition_transfer(&planes.1.0, header.namespace) + { self.partition_transfers_inflight() } else { 0 }; let Some(partition) = self.resolve_partition_target( &planes.1.0, - header.group, + header.namespace, header.view, header.replica, "StartView", ) else { return; }; - let Some(suffix_body) = control_suffix_body_verified(&msg, header.checksum_body) else { - tracing::warn!( - shard = self.id, - from_replica = header.replica, - view = header.view, - "dropping start_view whose body failed its checksum" - ); - return; - }; - let actions = - partition - .consensus() - .handle_start_view(PlaneKind::Partitions, &header, suffix_body); - let adopted = !actions.is_empty(); - if adopted && let Some(pending) = partition.consensus().pending_view_log() { - // Ahead of the local dispatch, which rebuilds the pipeline out of the - // journal this rewrites. Same position as the metadata arm's twin. - reconcile_partition_view_divergence(self.id, partition, &pending).await; - } let consensus = partition.consensus(); + let actions = consensus.handle_start_view(PlaneKind::Partitions, &header); + let adopted = !actions.is_empty(); let (local_actions, wire_actions) = split_local_actions(actions); // Locals go to the partition dispatcher ONLY: `RebuildPipeline` // executes there (`dispatch_vsr_actions` bails on `journal: None`) @@ -3591,7 +3496,7 @@ where { tracing::info!( shard = self.id, - namespace_raw = header.group, + namespace_raw = header.namespace, peer = header.replica, "adopted a live view while awaiting transfer; requesting partition state transfer" ); @@ -3645,7 +3550,7 @@ where let planes = self.plane.inner(); if let Some(ref consensus) = planes.0.consensus - && consensus.group() == header.group + && consensus.namespace() == header.namespace { match consensus.handle_commit(&header) { CommitOutcome::Advanced => { @@ -3693,7 +3598,7 @@ where let config = planes.1.0.config(); let Some(partition) = self.resolve_partition_target( &planes.1.0, - header.group, + header.namespace, header.view, header.replica, "Commit", @@ -3747,7 +3652,7 @@ where let header = *msg.header(); let planes = self.plane.inner(); if let Some(ref consensus) = planes.0.consensus - && consensus.group() == header.group + && consensus.namespace() == header.namespace { let actions = consensus.handle_request_start_view(PlaneKind::Metadata, &header); let (local_actions, wire_actions) = split_local_actions(actions); @@ -3760,7 +3665,7 @@ where let Some(partition) = planes .1 .0 - .get_mut_by_ns(&IggyNamespace::from_raw(header.group)) + .get_mut_by_ns(&IggyNamespace::from_raw(header.namespace)) else { return; }; @@ -3804,16 +3709,9 @@ where let repair_chunk_max = self.repair_chunk_max.get(); let planes = self.plane.inner(); if let Some(ref consensus) = planes.0.consensus - && consensus.group() == header.group + && consensus.namespace() == header.namespace { - // Served in `ViewChange` too: the replicas holding a missing body are - // exactly those in `ViewChange`, so refusing would deadlock the repair - // the new primary waits on. Read-only; the requester decides. - if consensus.is_transferring() { - // A transfer rewrites local state wholesale; journal not stable yet. - return; - } - if !matches!(consensus.status(), Status::Normal | Status::ViewChange) { + if !consensus.is_normal() { return; } let Some(journal) = planes.0.journal.as_ref() else { @@ -3822,21 +3720,11 @@ where let journal = journal.handle(); let cluster = consensus.cluster(); let self_id = consensus.replica(); - let to_op = repair_serve_ceiling( - header.to_op, - consensus.commit_max(), - consensus.sequencer().current_sequence(), - ); + let to_op = header.to_op.min(consensus.commit_max()); // Skip the compacted prefix (below the snapshot floor) in one // RangeEvicted notice, then serve contiguously until the range // ends or the WAL runs out. - // - // Floored, not just walked up to: `validate` asks only for - // `1 <= from_op <= to_op`, and the walk steps op by op with no `.await`, - // so a peer sending `from_op = 1` against a large compacted frontier pins - // the pump against a 10 ms tick. Nothing at or below the watermark is - // servable anyway. The partition arm jumps to `retained_from` likewise. - let mut from_op = header.from_op.max(journal.snapshot_op() + 1); + let mut from_op = header.from_op; #[allow(clippy::cast_possible_truncation)] while from_op <= to_op && journal.header(from_op as usize).is_none() { from_op += 1; @@ -3853,7 +3741,7 @@ where Command2::RangeEvicted, header.nonce, from_op, - header.group, + header.namespace, ) .await; self.send_repair_range_reply( @@ -3863,7 +3751,7 @@ where Command2::RepairDone, header.nonce, header.from_op.saturating_sub(1), - header.group, + header.namespace, ) .await; return; @@ -3876,7 +3764,7 @@ where Command2::RangeEvicted, header.nonce, from_op, - header.group, + header.namespace, ) .await; } @@ -3905,13 +3793,16 @@ where Command2::RepairDone, header.nonce, served_through, - header.group, + header.namespace, ) .await; return; } - let namespace = IggyNamespace::from_raw(header.group); - let Some(partition) = planes.1.0.get_mut_by_ns(&namespace) else { + let Some(partition) = planes + .1 + .0 + .get_mut_by_ns(&IggyNamespace::from_raw(header.namespace)) + else { return; }; if !partition.consensus().is_normal() { @@ -3926,6 +3817,7 @@ where // Defer instead: no RepairDone is sent, the rejoiner's stall retry // re-asks, and the local purge (one reconciler wake away) installs // the floor the fence below serves behind. + let namespace = IggyNamespace::from_raw(header.namespace); let committed_purge = self .plane .metadata() @@ -3940,7 +3832,7 @@ where self.metrics.record_partition_repair_serve_deferred(); tracing::debug!( shard = self.id, - namespace_raw = header.group, + namespace_raw = header.namespace, committed_purge, applied_purge = partition.applied_purge_generation(), "deferring repair serve until the committed purge applies locally" @@ -3984,7 +3876,7 @@ where Command2::RangeEvicted, header.nonce, retained_from, - header.group, + header.namespace, ) .await; from_op = retained_from; @@ -4007,12 +3899,12 @@ where Command2::RepairDone, header.nonce, served_through, - header.group, + header.namespace, ) .await; tracing::info!( shard = self.id, - namespace_raw = header.group, + namespace_raw = header.namespace, target, from_op = header.from_op, to_op, @@ -4024,7 +3916,7 @@ where /// Ingest one repaired prepare. Metadata journals it into the WAL (the /// commit walk at `RepairDone` applies it); partitions journal + stage it /// through the same apply path as live replication, minus fence and ack. - #[allow(clippy::future_not_send, clippy::too_many_lines)] + #[allow(clippy::future_not_send)] async fn on_repair_prepare(&self, msg: Message) where B: MessageBus, @@ -4038,7 +3930,7 @@ where tracing::debug!( shard = self.id, op = msg.header().0.op, - namespace_raw = msg.header().0.group, + namespace_raw = msg.header().0.namespace, "repair prepare received" ); // Convert to a live-prepare frame exactly once, here at the apply @@ -4054,7 +3946,7 @@ where let header = *msg.header(); let planes = self.plane.inner(); // Legacy acceptance: pre-upgrade metadata WAL entries were journaled - // before prepares stamped `consensus.group()`, and repair ships + // before prepares stamped `consensus.namespace()`, and repair ships // stored bytes verbatim, so without it a mixed-version metadata repair // re-ships the same 0-stamped entries forever. // @@ -4068,58 +3960,15 @@ where // being metadata mutations, so `is_metadata` alone is too narrow), which // is why both sites share it rather than re-deriving the set. let metadata_plane_op = header.operation.is_metadata_plane(); - let legacy_metadata_claim = header.group == 0 && metadata_plane_op; + let legacy_metadata_claim = header.namespace == 0 && metadata_plane_op; if let Some(ref consensus) = planes.0.consensus - && (consensus.group() == header.group || legacy_metadata_claim) + && (consensus.namespace() == header.namespace || legacy_metadata_claim) { let session = *self.metadata_repair.borrow(); let Some(session) = session else { return; }; - if header.op > session.to_op { - return; - } - let is_primary = consensus.is_primary_for_view(consensus.view()); - let commit_min = consensus.commit_min(); - // In place: runs once per repaired prepare, and the clone is two Vecs. - let in_scope = consensus - .with_pending_view_log(|pending| { - repair_op_in_scope(Some(pending), is_primary, commit_min, header.op) - }) - .unwrap_or_else(|| repair_op_in_scope(None, is_primary, commit_min, header.op)); - if !in_scope { - return; - } - // Applies to both planes, and is why a backup parks a log at all. The - // view already decided which prepare belongs at this op; a different - // one forks the log. An op the parked log omits is unconstrained. - let disagrees = consensus - .with_pending_view_log(|pending| { - pending - .headers - .iter() - .chain(pending.committed_elsewhere.iter()) - .find(|expected| expected.op == header.op) - .is_some_and(|expected| expected.checksum != header.checksum) - }) - .unwrap_or(false); - if disagrees { - tracing::warn!( - shard = self.id, - op = header.op, - "discarding repaired prepare that disagrees with the merged log" - ); - return; - } - // Recompute both integrity fields before durable storage: everything - // above treats `header.checksum` as an opaque token, so a corrupted - // frame passes whenever its flipped value satisfies the comparisons. - if let Err(reason) = verify_prepare_integrity(&header, msg.as_slice()) { - tracing::warn!( - shard = self.id, - op = header.op, - "discarding repaired prepare: {reason}" - ); + if header.op > session.to_op || header.op <= consensus.commit_min() { return; } let Some(journal) = planes.0.journal.as_ref() else { @@ -4168,7 +4017,7 @@ where shard = self.id, op = header.op, operation = ?header.operation, - namespace_raw = header.group, + namespace_raw = header.namespace, "dropping a metadata-plane repair prepare this shard cannot journal" ); return; @@ -4176,22 +4025,10 @@ where let Some(partition) = planes .1 .0 - .get_mut_by_ns(&IggyNamespace::from_raw(header.group)) + .get_mut_by_ns(&IggyNamespace::from_raw(header.namespace)) else { return; }; - // The partition arm reaches the WAL via `apply_repaired_prepare` with no - // view fence and no ack, so this is its only integrity gate. Without it a - // repaired partition prepare is journaled on the serving peer's word alone. - if let Err(reason) = verify_prepare_integrity(&header, msg.as_slice()) { - tracing::warn!( - shard = self.id, - op = header.op, - namespace_raw = header.group, - "discarding repaired partition prepare: {reason}" - ); - return; - } partition.apply_repaired_prepare(msg).await; } @@ -4213,7 +4050,7 @@ where let header = *msg.header(); let planes = self.plane.inner(); if let Some(ref consensus) = planes.0.consensus - && consensus.group() == header.group + && consensus.namespace() == header.namespace { let session = *self.metadata_repair.borrow(); let Some(session) = session else { @@ -4253,7 +4090,7 @@ where session.nonce, commit_min + 1, session.to_op, - header.group, + header.namespace, ) .await; } @@ -4306,13 +4143,14 @@ where // Counted BEFORE the `&mut partition` below exists, and only when an arm // is possible at all: see the StartView site for why the scan is gated // rather than replaced with a counter. - let transfers_inflight = if Self::may_arm_partition_transfer(&planes.1.0, header.group) { + let transfers_inflight = if Self::may_arm_partition_transfer(&planes.1.0, header.namespace) + { self.partition_transfers_inflight() } else { 0 }; let config = planes.1.0.config().clone(); - let namespace = IggyNamespace::from_raw(header.group); + let namespace = IggyNamespace::from_raw(header.namespace); let Some(partition) = planes.1.0.get_mut_by_ns(&namespace) else { return; }; @@ -4350,7 +4188,7 @@ where self.metrics.record_partition_repair_serve_deferred(); tracing::debug!( shard = self.id, - namespace_raw = header.group, + namespace_raw = header.namespace, committed_purge, applied_purge = partition.applied_purge_generation(), command = ?header.command, @@ -4389,7 +4227,7 @@ where // arming here would defeat its backoff. tracing::info!( shard = self.id, - namespace_raw = header.group, + namespace_raw = header.namespace, floor, to_op, peer = header.replica, @@ -4415,7 +4253,7 @@ where // the scheduled re-arm) owns recovery from here. tracing::info!( shard = self.id, - namespace_raw = header.group, + namespace_raw = header.namespace, floor, to_op, "partition repair floor refused; transfer in flight or scheduled" @@ -4426,7 +4264,7 @@ where if partition.repair.is_none() { tracing::info!( shard = self.id, - namespace_raw = header.group, + namespace_raw = header.namespace, through_op = header.op, "partition journal repair complete" ); @@ -4445,7 +4283,7 @@ where nonce, commit_min + 1, to_op, - header.group, + header.namespace, ) .await; } @@ -4481,9 +4319,8 @@ where h.nonce = nonce; h.from_op = from_op; h.to_op = to_op; - h.group = namespace; + h.namespace = namespace; h.size = size_of::() as u32; - h.seal(); }); if self .bus @@ -4556,9 +4393,8 @@ where h.replica = self_id; h.nonce = nonce; h.op = op; - h.group = namespace; + h.namespace = namespace; h.size = size_of::() as u32; - h.seal(); }); let _ = self .bus @@ -4566,503 +4402,92 @@ where .await; } - /// Partition-plane twin of [`Self::advance_pending_metadata_view`]. - /// - /// No `RequestPrepares` stream to arm: the partition journal is not durable - /// yet, so coverage either holds or a peer must retransmit. Same invariant - /// either way: the view does not start until this replica can serve its log. + /// Start metadata tail journal-repair from `peer` when the commit walk + /// gap-stopped below the known frontier. Shared by `StartView` adoption + /// and the post-install step of a state transfer. #[allow(clippy::future_not_send)] - async fn advance_pending_partition_view(&self, namespace: IggyNamespace) + async fn maybe_request_metadata_repair

(&self, consensus: &VsrConsensus, peer: u8) where B: MessageBus, - MJ: JournalHandle, - ::Target: Journal< - ::Storage, - Entry = Message, - Header = PrepareHeader, - >, + P: Pipeline, { - let partitions = self.plane.partitions(); - let started = { - let Some(partition) = partitions.get_mut_by_ns(&namespace) else { - return; - }; - if !partition - .consensus() - .is_primary_for_view(partition.consensus().view()) - { - return; - } - let Some(pending) = partition.consensus().pending_view_log() else { - return; - }; - // Before the scan can mean anything: the repair ingest skips an op it - // already holds a header for, so the scan would report a gap nothing - // fills. Backups reach this on StartView adoption; a primary-elect has no - // adoption to hang it off. - reconcile_partition_view_divergence(self.id, partition, &pending).await; - let consensus = partition.consensus(); - // Identity, not presence: see the metadata twin. The floor is the local - // commit point, the partition twin of the metadata snapshot floor: - // `evict_prefix` clears the header vec for the flushed (committed) - // prefix, so a survivor that flushes on every commit holds NO resident - // header for the op the merged window opens on. Demanding one parks the - // primary-elect in `ViewChange` forever, and the rotation lands - // primaryship on whichever replica still has its window resident -- a - // fresh rejoiner with nothing but repair-ingested entries, which then - // cannot serve the state transfer it itself needs. A committed op - // cannot diverge from the merged log, and its bytes stay serveable - // from the evicted ring or the flushed segments. - let missing = { - let journal = partition.log.journal(); - first_op_not_covered(&pending, consensus.commit_min(), |op| { - journal.inner.header_by_op(op) - }) - }; - if let Some(missing_op) = missing { - tracing::debug!( - shard = self.id, - namespace_raw = namespace.inner(), - missing_op, - op_head = pending.op_head, - "partition view change waiting on op {missing_op} before starting the view" - ); - return; - } - - let actions = consensus.start_pending_view(PlaneKind::Partitions); - let (local_actions, wire_actions) = split_local_actions(actions); - // Locals go to the partition dispatcher ONLY: `RebuildPipeline` - // executes there (`dispatch_vsr_actions` bails on `journal: None`) - // and `CommitJournal` is a no-op in both. - dispatch_partition_journal_actions(consensus, partition, &local_actions).await; - // `start_pending_view` flips this replica into `Normal` for the new - // view, so the `StartView` it emits advertises a view the superblock - // must already record. Same gate as the `on_do_view_change` and - // `on_start_view` partition arms. - if partition.persist_superblock_if_needed().await { - dispatch_vsr_actions::(consensus, None, &wire_actions).await; - dispatch_partition_journal_actions(consensus, partition, &wire_actions).await; - } - local_actions - .iter() - .any(|action| matches!(action, VsrAction::CommitJournal)) - }; - if started { - let config = partitions.config(); - if let Some(partition) = partitions.get_mut_by_ns(&namespace) { - partition.commit_journal(config).await; - } + if consensus.is_normal() + && !consensus.is_transferring() + && consensus.commit_min() < consensus.commit_max() + && self.metadata_repair.borrow().is_none() + { + let nonce = iggy_common::random_id::get_uuid(); + let to_op = consensus.commit_max(); + let from_op = consensus.commit_min() + 1; + *self.metadata_repair.borrow_mut() = Some(MetadataRepairSession { + nonce, + to_op, + peer, + idle_ticks: 0, + }); + tracing::info!( + shard = self.id, + from_op, + to_op, + "metadata behind the group frontier; requesting repair" + ); + self.send_request_prepares( + consensus.cluster(), + consensus.replica(), + peer, + nonce, + from_op, + to_op, + consensus.namespace(), + ) + .await; } } - /// Re-request the remaining repair window when the stream has gone quiet. - /// - /// Repair frames are fire-and-forget, so a lost one leaves the session armed - /// forever with the commit walk pinned below the frontier. - #[allow(clippy::future_not_send)] - async fn retry_stalled_metadata_repair

(&self, consensus: &VsrConsensus) - where + #[allow(clippy::future_not_send, clippy::cast_possible_truncation)] + async fn send_request_state_transfer

( + &self, + consensus: &VsrConsensus, + target: u8, + nonce: u128, + ) where B: MessageBus, P: Pipeline, { - // Stall retry (mirrors `tick_partitions`): a lost frame must not wedge it. - let repair_retry_ticks = self.repair_retry_ticks.get(); - let stalled = { - // `ViewChange` too: a parked view change repairs toward its merged log - // and cannot start until the window fills. Gating on `Normal` alone - // defers a dropped frame to the 500-tick escalation. - let repairing_view = - consensus.view_log_is_pending() && consensus.is_primary_for_view(consensus.view()); - let mut session = self.metadata_repair.borrow_mut(); - session.as_mut().and_then(|session| { - if !consensus.is_normal() && !repairing_view { - return None; - } - session.idle_ticks += 1; - if session.idle_ticks < repair_retry_ticks { - return None; - } - session.idle_ticks = 0; - Some((session.peer, session.nonce, session.to_op)) - }) - }; - if let Some((peer, nonce, to_op)) = stalled { - // Primary-elect only. Its window starts at the merged log's commit - // point, which can sit below local `commit_min` (the headers inherited - // from senders behind the canonical log_view live there), so - // `commit_min + 1` would skip them. A backup's parked `StartView` - // suffix is only a verification reference; resuming from its commit - // point would restart at the view's opening head, not at the gap. - let from_op = consensus - .is_primary_for_view(consensus.view()) - .then(|| consensus.with_pending_view_log(|pending| pending.commit_max.max(1))) - .flatten() - .unwrap_or_else(|| consensus.commit_min() + 1); - if from_op <= to_op { - tracing::info!( - shard = self.id, - from_op, - to_op, - peer, - "metadata repair stalled; re-requesting remaining window" - ); - self.send_request_prepares( - consensus.cluster(), - consensus.replica(), - peer, - nonce, - from_op, - to_op, - consensus.group(), - ) - .await; - } - } + let msg = + Message::::new(size_of::()) + .transmute_header(|_, h: &mut RequestStateTransferHeader| { + h.command = Command2::RequestStateTransfer; + h.cluster = consensus.cluster(); + h.replica = consensus.replica(); + h.nonce = nonce; + h.namespace = consensus.namespace(); + h.size = size_of::() as u32; + }); + let _ = self + .bus + .send_to_replica(target, msg.into_generic().into_frozen()) + .await; } - /// Compare this replica's log against the headers the view decided, and drop or - /// report where they disagree. - /// - /// Without this, divergence is silent and permanent: the replica acks with its - /// own checksum, the primary rejects the ack, and journal repair skips an op - /// it already has a header for. - /// - /// Both roles. A backup runs it against the `StartView` suffix it adopted, the - /// primary-elect against the merged log before the coverage scan in - /// [`Self::advance_pending_metadata_view`]. The merge does NOT reconcile the - /// primary's log for it: nothing installs the merged headers into the journal, - /// `RebuildPipeline` reads the pipeline back out of it, and `CommitJournal` - /// applies whatever sits at each op up to the merged commit point. - /// - /// The split at the announced commit point is what matters. Above it a - /// disagreement is ordinary, so the entry is dropped and the primary's - /// retransmission refills the range. At or below it, this replica applied - /// something the view says was different, which only state transfer fixes, so - /// it is reported and left alone. - /// - /// Truncation uses `Journal::truncate_from`, not `drain`: `drain` advances - /// `snapshot_op` past what it removed, marking ops that must stay refillable - /// as evictable. - #[allow(clippy::future_not_send)] - async fn reconcile_metadata_view_divergence(&self) - where - B: MessageBus, - MJ: JournalHandle, - ::Target: Journal< - ::Storage, - Entry = Message, - Header = PrepareHeader, - >, - M: MetadataStm, - { - let metadata = self.plane.metadata(); - let Some(ref consensus) = metadata.consensus else { - return; - }; - let Some(pending) = consensus.pending_view_log() else { - return; - }; - let Some(journal) = metadata.journal.as_ref() else { - return; - }; - - // Truncation is safe only above what this replica has *applied*, which is - // not the view's commit point: `pending.commit_max` is the new primary's - // number and a backup can sit above it. Splitting on the view's number - // would drop already-executed ops with no rollback, and silently. - let applied_floor = pending.commit_max.max(consensus.commit_min()); - - let mut repairable_from: Option = None; - for canonical in &pending.headers { - let Some(local) = usize::try_from(canonical.op) - .ok() - .and_then(|slot| journal.handle().header(slot)) - else { - continue; - }; - if header_is_view_entry(&local, canonical) { - continue; - } - if canonical.op <= applied_floor { - tracing::error!( - shard = self.id, - op = canonical.op, - view = consensus.view(), - commit_max = pending.commit_max, - commit_min = consensus.commit_min(), - local_checksum = local.checksum, - canonical_checksum = canonical.checksum, - "committed op {} disagrees with the view that just started; this replica \ - applied a different op as committed and cannot be reconciled by log repair", - canonical.op - ); - continue; - } - repairable_from = Some(repairable_from.map_or(canonical.op, |op| op.min(canonical.op))); - } - - // A suffix ABOVE the announced head is named by nobody, so the loop cannot - // see it, and it is exactly the log that AGREES in-window (restart and - // re-adopt), where `repairable_from` never arms. Adoption drops the head - // under it, and the next prepare at `op_head + 1` then collides in `append`, - // which refuses the slot even when the ops match. Floored at the applied - // point too: an executed op is not rollback-able whatever the head says. - let above_head = pending.op_head.max(applied_floor) + 1; - if journal - .handle() - .last_op() - .is_some_and(|last_op| last_op >= above_head) - { - repairable_from = Some(repairable_from.map_or(above_head, |op| op.min(above_head))); - } - - let Some(from_op) = repairable_from else { - return; - }; - // SERIALIZATION: the drain guard excludes `truncate_from` against `drain`, - // NOT against an append; that is metadata's private `journal_gate`. It holds - // by call-site placement, not construction: the pump is single-threaded, and - // both callers run with a view change parked, so no submit is admitted and no - // repair prepare is in flight for these ops. Routing shard-side journal - // mutations through gate-taking metadata methods would make it structural. - match journal.handle().truncate_from(from_op).await { - Ok(removed) => { - // The snapshot's `(op, commit)` tag does not move when entries are - // removed under it, so the next `DoViewChange` would advertise the - // dropped headers and offer bodies this replica cannot serve. - consensus.invalidate_local_dvc_suffix(); - tracing::warn!( - shard = self.id, - from_op, - removed, - op_head = pending.op_head, - view = consensus.view(), - "dropped {removed} uncommitted entries from op {from_op} that disagreed with \ - the view's log; the primary's retransmission refills the range" - ); - } - Err(error) => { - tracing::error!( - shard = self.id, - from_op, - %error, - "could not drop the diverging uncommitted entries from op {from_op}; journal \ - repair skips ops it already holds a header for, so this replica will not \ - converge at those ops until it is restarted" - ); - } - } - } - - /// Drive a parked view change to completion. - /// - /// A DVC quorum decides the log before this replica necessarily holds it, so - /// the merged log parks in consensus and this replica stays in `ViewChange`, - /// announcing and preparing nothing: `StartView` promises it can serve every - /// op it names, and a backup adopting that head asks for the bodies at once. - /// - /// Check coverage, then start the view or pull missing bodies from a peer that - /// offered them in its DVC. Only those peers: a cleared present bit means the - /// body was never held or cannot be read back. - #[allow(clippy::future_not_send)] - async fn advance_pending_metadata_view(&self) - where - B: MessageBus, - MJ: JournalHandle, - ::Target: Journal< - ::Storage, - Entry = Message, - Header = PrepareHeader, - >, - M: MetadataStm, - { - let metadata = self.plane.metadata(); - let Some(ref consensus) = metadata.consensus else { - return; - }; - // Primary-elect only. A backup's parked `StartView` suffix is only what - // its ingest verifies bodies against; driving repair from it would put a - // rejoining node on the tail-repair path when its gap sits below every - // peer's retention floor, racing the view probe that picks state transfer. - if !consensus.is_primary_for_view(consensus.view()) { - return; - } - // Before the coverage scan, and a primary-elect's only shot at it: the repair - // ingest skips an op it already holds a header for, so a diverging entry would - // never be replaced. No-op when nothing diverges. - self.reconcile_metadata_view_divergence().await; - let Some(pending) = consensus.pending_view_log() else { - return; - }; - let Some(journal) = metadata.journal.as_ref() else { - return; - }; - - // Floor on what this replica can be asked to hold before starting the view. - // Entries at or below the snapshot watermark are compacted, so no repair puts - // one back: demanding one parks the view change forever on an op already - // applied and durable in the snapshot. - let repair_floor = journal.handle().snapshot_op(); - let missing = first_op_not_covered(&pending, repair_floor, |op| { - usize::try_from(op) - .ok() - .and_then(|slot| journal.handle().header(slot)) - .map(|header| *header) - }); - - let Some(missing_op) = missing else { - let actions = consensus.start_pending_view(PlaneKind::Metadata); - tracing::info!( - shard = self.id, - view = consensus.view(), - op_head = pending.op_head, - commit_max = pending.commit_max, - "merged log is locally serveable; starting the view" - ); - if metadata.persist_superblock_if_needed(consensus).await { - dispatch_vsr_actions(consensus, metadata.journal.as_ref(), &actions).await; - } - if actions - .iter() - .any(|action| matches!(action, VsrAction::CommitJournal)) - && !consensus.is_transferring() - { - metadata.commit_journal().await; - } - return; - }; - - if self.metadata_repair.borrow().is_some() { - // Stream already running; the stall retry covers it drying up. - return; - } - let sources = consensus.pending_view_body_sources(missing_op); - let Some(peer) = sources.first().copied() else { - // The merge only returns a startable log when some replica offered each - // body, so an empty source list means that offer was withdrawn (peer - // restarted, or moved on). Let the view-change timeout escalate. - tracing::warn!( - shard = self.id, - missing_op, - "no replica offers op {missing_op} for the merged log; view change is stalled" - ); - return; - }; - - let nonce = iggy_common::random_id::get_uuid(); - *self.metadata_repair.borrow_mut() = Some(MetadataRepairSession { - nonce, - to_op: pending.op_head, - peer, - idle_ticks: 0, - }); - tracing::info!( - shard = self.id, - missing_op, - peer, - to_op = pending.op_head, - "repairing toward the merged log before starting the view" - ); - self.send_request_prepares( - consensus.cluster(), - consensus.replica(), - peer, - nonce, - missing_op, - pending.op_head, - consensus.group(), - ) - .await; - } - - /// Start metadata tail journal-repair from `peer` when the commit walk - /// gap-stopped below the known frontier. Shared by `StartView` adoption - /// and the post-install step of a state transfer. - #[allow(clippy::future_not_send)] - async fn maybe_request_metadata_repair

(&self, consensus: &VsrConsensus, peer: u8) - where - B: MessageBus, - P: Pipeline, - { - if consensus.is_normal() - && !consensus.is_transferring() - && consensus.commit_min() < consensus.commit_max() - && self.metadata_repair.borrow().is_none() - { - let nonce = iggy_common::random_id::get_uuid(); - let to_op = consensus.commit_max(); - let from_op = consensus.commit_min() + 1; - *self.metadata_repair.borrow_mut() = Some(MetadataRepairSession { - nonce, - to_op, - peer, - idle_ticks: 0, - }); - tracing::info!( - shard = self.id, - from_op, - to_op, - "metadata behind the group frontier; requesting repair" - ); - self.send_request_prepares( - consensus.cluster(), - consensus.replica(), - peer, - nonce, - from_op, - to_op, - consensus.group(), - ) - .await; - } - } - - #[allow(clippy::future_not_send, clippy::cast_possible_truncation)] - async fn send_request_state_transfer

( - &self, - consensus: &VsrConsensus, - target: u8, - nonce: u128, - ) where - B: MessageBus, - P: Pipeline, - { - let msg = - Message::::new(size_of::()) - .transmute_header(|_, h: &mut RequestStateTransferHeader| { - h.command = Command2::RequestStateTransfer; - h.cluster = consensus.cluster(); - h.replica = consensus.replica(); - h.nonce = nonce; - h.group = consensus.group(); - h.size = size_of::() as u32; - h.seal(); - }); - let _ = self - .bus - .send_to_replica(target, msg.into_generic().into_frozen()) - .await; - } - - /// Answer a `RequestStateTransfer`: `offer = None` sends a header-only - /// `available = 0` (the requester falls back to journal repair or - /// retries elsewhere); an offer ships its encoded state manifest as the - /// frame body. - #[allow( - clippy::future_not_send, - clippy::cast_possible_truncation, - clippy::too_many_arguments - )] - async fn send_state_transfer_target( - &self, - cluster: u128, - self_id: u8, - target: u8, - nonce: u128, - namespace: u64, - descriptor: TransferDescriptor<'_>, - ) where + /// Answer a `RequestStateTransfer`: `offer = None` sends a header-only + /// `available = 0` (the requester falls back to journal repair or + /// retries elsewhere); an offer ships its encoded state manifest as the + /// frame body. + #[allow( + clippy::future_not_send, + clippy::cast_possible_truncation, + clippy::too_many_arguments + )] + async fn send_state_transfer_target( + &self, + cluster: u128, + self_id: u8, + target: u8, + nonce: u128, + namespace: u64, + descriptor: TransferDescriptor<'_>, + ) where B: MessageBus, { let manifest = descriptor @@ -5079,7 +4504,7 @@ where h.cluster = cluster; h.replica = self_id; h.nonce = nonce; - h.group = namespace; + h.namespace = namespace; h.size = total_size as u32; // The serving replica's own progress travels with every descriptor, // available or not: it is what lets a receiver refuse an offer from @@ -5091,7 +4516,6 @@ where h.available = 1; h.commit_op = commit_op; } - h.seal(); }); let _ = self .bus @@ -5123,12 +4547,11 @@ where h.cluster = cluster; h.replica = self_id; h.nonce = nonce; - h.group = namespace; + h.namespace = namespace; h.artifact = artifact; h.offset = offset; h.len = len; h.size = size_of::() as u32; - h.seal(); }); let _ = self .bus @@ -5156,7 +4579,7 @@ where .0 .consensus .as_ref() - .is_some_and(|consensus| consensus.group() == header.group); + .is_some_and(|consensus| consensus.namespace() == header.namespace); if !metadata_frame { return self.on_partition_request_state_transfer(msg).await; } @@ -5176,7 +4599,7 @@ where let cached = self .state_transfer_offers .borrow_mut() - .get_mut(&(header.group, header.replica)) + .get_mut(&(header.namespace, header.replica)) .filter(|served| served.nonce == header.nonce) .and_then(|served| { let ServedOffer::Metadata(offer) = &served.offer else { @@ -5200,7 +4623,7 @@ where self_id, header.replica, header.nonce, - header.group, + header.namespace, TransferDescriptor::available( &offer.manifest(), offer.commit_op, @@ -5228,7 +4651,7 @@ where self_id, header.replica, header.nonce, - header.group, + header.namespace, TransferDescriptor::available( &offer.manifest(), offer.commit_op, @@ -5238,7 +4661,7 @@ where ) .await; self.state_transfer_offers.borrow_mut().insert( - (header.group, header.replica), + (header.namespace, header.replica), ServedStateTransfer { nonce: header.nonce, offer: ServedOffer::Metadata(offer), @@ -5263,7 +4686,7 @@ where self_id, header.replica, header.nonce, - header.group, + header.namespace, TransferDescriptor::unavailable( false, consensus.view(), @@ -5308,7 +4731,7 @@ where .0 .consensus .as_ref() - .is_some_and(|consensus| consensus.group() == header.group); + .is_some_and(|consensus| consensus.namespace() == header.namespace); if !metadata_frame { return self.on_partition_state_transfer_target(msg).await; } @@ -5486,7 +4909,7 @@ where consensus.replica(), peer, nonce, - consensus.group(), + consensus.namespace(), artifact, offset, len, @@ -5599,7 +5022,7 @@ where .0 .consensus .as_ref() - .is_some_and(|consensus| consensus.group() == header.group); + .is_some_and(|consensus| consensus.namespace() == header.namespace); if !metadata_frame { return self.on_partition_request_state_chunk(msg).await; } @@ -5621,7 +5044,7 @@ where let reply = { let mut offers = self.state_transfer_offers.borrow_mut(); let served = offers - .get_mut(&(header.group, header.replica)) + .get_mut(&(header.namespace, header.replica)) .filter(|served| served.nonce == header.nonce); served.map_or( Some(ChunkReply::Unavailable { transient: true }), @@ -5671,11 +5094,10 @@ where h.cluster = cluster; h.replica = self_id; h.nonce = header.nonce; - h.group = header.group; + h.namespace = header.namespace; h.artifact = header.artifact; h.offset = header.offset; h.size = total_size as u32; - h.seal(); }, ))) }, @@ -5700,7 +5122,7 @@ where self_id, header.replica, header.nonce, - header.group, + header.namespace, TransferDescriptor::unavailable( transient, consensus.view(), @@ -5742,7 +5164,7 @@ where .0 .consensus .as_ref() - .is_some_and(|consensus| consensus.group() == header.group); + .is_some_and(|consensus| consensus.namespace() == header.namespace); if !metadata_frame { return self.on_partition_state_chunk(msg).await; } @@ -6082,16 +5504,6 @@ where }; let consensus = partition.consensus(); - // Only while a view change is live. A `Normal` tick has no consumer: - // `start_election` records no DoViewChange, and every path that does - // either refreshes at its own call site (the SVC and DVC handlers, - // still `Normal` at that point) or runs in `ViewChange`. - // - // Ungated, this rebuilt a 128-entry window every 10 ms per advancing - // partition: a linear `header_by_op` scan per entry plus 32 KiB. - if consensus.status() != Status::Normal { - refresh_partition_dvc_suffix(partition); - } let actions = consensus.tick(PlaneKind::Partitions); // The tick emits view-scoped sends (heartbeats, view-change // retransmits), so it persists first like every dispatch site; @@ -6105,9 +5517,6 @@ where dispatch_partition_journal_actions(consensus, partition, &wire_actions).await; } - // Finish a view change whose quorum decided ahead of the local log. - self.advance_pending_partition_view(namespace).await; - // Stall retry: repair frames are fire-and-forget, so a lost // frame (or a peer that went silent mid-stream) would leave the // session armed forever with commit_min pinned below commit_max. @@ -6378,7 +5787,7 @@ where let Some(partition) = planes .1 .0 - .get_mut_by_ns(&IggyNamespace::from_raw(header.group)) + .get_mut_by_ns(&IggyNamespace::from_raw(header.namespace)) else { return; }; @@ -6392,7 +5801,7 @@ where let cached = self .state_transfer_offers .borrow_mut() - .get_mut(&(header.group, header.replica)) + .get_mut(&(header.namespace, header.replica)) .filter(|served| served.nonce == header.nonce) .and_then(|served| { let ServedOffer::Partition(offer) = &served.offer else { @@ -6409,14 +5818,14 @@ where Some(offer) => { tracing::debug!( shard = self.id, - namespace_raw = header.group, + namespace_raw = header.namespace, requester = header.replica, "re-answering a partition state transfer request from the offer \ already served" ); Some(offer) } - None if !self.may_serve_another_partition_transfer(header.group) => { + None if !self.may_serve_another_partition_transfer(header.namespace) => { // Admission control, because the served-payload budget is a // BYTE budget and the pulls that overrun it do not degrade // gracefully. Each concurrent pull holds a different segment @@ -6429,7 +5838,7 @@ where // Refusing the surplus is what makes the admitted ones finish. tracing::info!( shard = self.id, - namespace_raw = header.group, + namespace_raw = header.namespace, requester = header.replica, "already serving as many partition transfers as the served-payload \ budget holds; refusing until one completes" @@ -6440,7 +5849,7 @@ where self_id, header.replica, header.nonce, - header.group, + header.namespace, TransferDescriptor::unavailable(true, view, commit_max), ) .await; @@ -6452,15 +5861,15 @@ where // and holds nothing in the offers map meanwhile. self.partition_offer_builds .borrow_mut() - .insert(header.group, 0); + .insert(header.namespace, 0); match partition.state_transfer_offer(&config).await { Ok(offer) => { self.partition_offer_builds .borrow_mut() - .remove(&header.group); + .remove(&header.namespace); tracing::info!( shard = self.id, - namespace_raw = header.group, + namespace_raw = header.namespace, requester = header.replica, commit_op = offer.commit_op, artifacts = offer.artifact_count(), @@ -6468,7 +5877,7 @@ where "serving partition state transfer" ); self.state_transfer_offers.borrow_mut().insert( - (header.group, header.replica), + (header.namespace, header.replica), ServedStateTransfer { nonce: header.nonce, offer: ServedOffer::Partition(Rc::clone(&offer)), @@ -6496,12 +5905,12 @@ where if !building { self.partition_offer_builds .borrow_mut() - .remove(&header.group); + .remove(&header.namespace); } let transient = reason.transient(); tracing::info!( shard = self.id, - namespace_raw = header.group, + namespace_raw = header.namespace, requester = header.replica, transient, %reason, @@ -6513,7 +5922,7 @@ where self_id, header.replica, header.nonce, - header.group, + header.namespace, TransferDescriptor::unavailable(transient, view, commit_max), ) .await; @@ -6537,7 +5946,7 @@ where self_id, header.replica, header.nonce, - header.group, + header.namespace, TransferDescriptor::available(&offer.manifest(), offer.commit_op, view, commit_max), ) .await; @@ -6576,7 +5985,11 @@ where return; } let planes = self.plane.inner(); - let Some(partition) = planes.1.0.get_by_ns(&IggyNamespace::from_raw(header.group)) else { + let Some(partition) = planes + .1 + .0 + .get_by_ns(&IggyNamespace::from_raw(header.namespace)) + else { return; }; let cluster = partition.consensus().cluster(); @@ -6588,7 +6001,7 @@ where let attempt = 'attempt: { let mut offers = self.state_transfer_offers.borrow_mut(); let served = offers - .get_mut(&(header.group, header.replica)) + .get_mut(&(header.namespace, header.replica)) .filter(|served| served.nonce == header.nonce); let Some(served) = served else { break 'attempt ChunkAttempt::Reply(Some(ChunkReply::Unavailable { @@ -6612,7 +6025,7 @@ where match self .served_segment_cache .borrow_mut() - .get(header.group, source.entry.checksum) + .get(header.namespace, source.entry.checksum) { Some(payload) => { segment_payload = payload; @@ -6649,7 +6062,7 @@ where // the serving side's proof that the pull ran to completion. tracing::info!( shard = self.id, - namespace_raw = header.group, + namespace_raw = header.namespace, requester = header.replica, "partition state transfer fully served" ); @@ -6664,14 +6077,10 @@ where h.cluster = cluster; h.replica = self_id; h.nonce = header.nonce; - h.group = header.group; + h.namespace = header.namespace; h.artifact = header.artifact; h.offset = header.offset; h.size = total_size as u32; - // `StateChunk` is `FRAME_SEALED`: the receiver's router - // drops an unsealed frame before any handler sees it, so - // a missing seal starves the pull silently. - h.seal(); }, )))) }; @@ -6691,7 +6100,7 @@ where let reason = match loaded { Ok(bytes) => { self.served_segment_cache.borrow_mut().insert( - header.group, + header.namespace, entry.checksum, Rc::new(bytes), self.served_segment_cache_bytes_max.get(), @@ -6707,7 +6116,7 @@ where let transient = reason.transient(); tracing::warn!( shard = self.id, - namespace_raw = header.group, + namespace_raw = header.namespace, artifact = header.artifact, path = %log_path, transient, @@ -6716,7 +6125,7 @@ where ); self.state_transfer_offers .borrow_mut() - .remove(&(header.group, header.replica)); + .remove(&(header.namespace, header.replica)); // The builder cache too: it is keyed by commit_op alone, // and GC unlinks files WITHOUT a commit, so the restarted // requester would otherwise be handed the same offer with @@ -6736,7 +6145,7 @@ where Some(ChunkReply::Unavailable { transient }) => { tracing::info!( shard = self.id, - namespace_raw = header.group, + namespace_raw = header.namespace, requester = header.replica, transient, "partition chunk request for an unknown offer; telling requester to restart" @@ -6746,7 +6155,7 @@ where self_id, header.replica, header.nonce, - header.group, + header.namespace, // Usually TRANSIENT -- retention GC'd a served segment, or // the offer aged out between two chunks, and the restarted // session converges -- but a load that failed on a local @@ -6759,7 +6168,7 @@ where None => { tracing::warn!( shard = self.id, - namespace_raw = header.group, + namespace_raw = header.namespace, requester = header.replica, artifact = header.artifact, offset = header.offset, @@ -7005,7 +6414,7 @@ where { tracing::info!( shard = self.id, - namespace_raw = partition.consensus().group(), + namespace_raw = partition.consensus().namespace(), cap = Self::PARTITION_TRANSFERS_INFLIGHT_MAX, "partition transfer slots exhausted; deferring this arm" ); @@ -7058,7 +6467,7 @@ where let to_op = consensus.commit_max(); let cluster = consensus.cluster(); let self_id = consensus.replica(); - let namespace = consensus.group(); + let namespace = consensus.namespace(); partition.repair = Some(partitions::RepairSession { nonce, to_op, @@ -7098,7 +6507,7 @@ where let Some(partition) = planes .1 .0 - .get_mut_by_ns(&IggyNamespace::from_raw(header.group)) + .get_mut_by_ns(&IggyNamespace::from_raw(header.namespace)) else { return; }; @@ -7121,7 +6530,7 @@ where let transient = header.unavailable_transient == 1; tracing::info!( shard = self.id, - namespace_raw = header.group, + namespace_raw = header.namespace, peer = header.replica, transient, "partition transfer peer cannot serve; backing off before re-arming" @@ -7167,7 +6576,7 @@ where if header.commit_op > header.commit_max { tracing::warn!( shard = self.id, - namespace_raw = header.group, + namespace_raw = header.namespace, peer = header.replica, serving_commit_op = header.commit_op, serving_commit_max = header.commit_max, @@ -7181,7 +6590,7 @@ where if header.view < local_view || header.commit_max < local_commit_max { tracing::warn!( shard = self.id, - namespace_raw = header.group, + namespace_raw = header.namespace, peer = header.replica, serving_view = header.view, serving_commit_max = header.commit_max, @@ -7202,7 +6611,7 @@ where Err(error) => { tracing::warn!( shard = self.id, - namespace_raw = header.group, + namespace_raw = header.namespace, %error, "partition transfer manifest rejected" ); @@ -7231,7 +6640,7 @@ where if !kind_capped || total_len > Self::PARTITION_TRANSFER_TOTAL_LEN_MAX { tracing::warn!( shard = self.id, - namespace_raw = header.group, + namespace_raw = header.namespace, total_len, "partition transfer manifest exceeds artifact caps; refusing descriptor" ); @@ -7251,7 +6660,7 @@ where if !reused.is_empty() { tracing::info!( shard = self.id, - namespace_raw = header.group, + namespace_raw = header.namespace, peer = header.replica, adopted = reused.len(), artifacts = entries.len(), @@ -7298,7 +6707,7 @@ where if consensus.state_transfer_stage() == consensus::StateTransferStage::AwaitingTarget { consensus.set_state_transfer_stage(consensus::StateTransferStage::Fetching); } - self.on_partition_transfer_progress(header.group).await; + self.on_partition_transfer_progress(header.namespace).await; } /// Receive one partition chunk; spill a completed segment artifact, and @@ -7315,7 +6724,7 @@ where let Some(partition) = planes .1 .0 - .get_mut_by_ns(&IggyNamespace::from_raw(header.group)) + .get_mut_by_ns(&IggyNamespace::from_raw(header.namespace)) else { return; }; @@ -7349,7 +6758,7 @@ where session.idle_ticks = 0; } partition.note_transfer_progress(); - self.on_partition_transfer_progress(header.group).await; + self.on_partition_transfer_progress(header.namespace).await; } /// Drive an in-flight partition transfer: spill newly completed segment @@ -7712,7 +7121,7 @@ where // cannot tell the two apart; the serving node's own logs can. tracing::warn!( shard = self.id, - namespace_raw = partition.consensus().group(), + namespace_raw = partition.consensus().namespace(), peer, refusals, "partition state transfer has been refused {refusals} times in a row; the peer \ @@ -7760,7 +7169,7 @@ where }; tracing::info!( shard = self.id, - namespace_raw = partition.consensus().group(), + namespace_raw = partition.consensus().namespace(), failures, next_peer, after_ticks, @@ -7906,7 +7315,7 @@ where let metadata_served = metadata.consensus.as_ref().is_some_and(|consensus| { offers .keys() - .any(|(namespace, _)| *namespace == consensus.group()) + .any(|(namespace, _)| *namespace == consensus.namespace()) }); if !metadata_served { metadata.clear_state_transfer_offer_cache(); @@ -7951,10 +7360,6 @@ where return; }; - // See the partition tick: no snapshot consumer on a `Normal` tick. - if consensus.status() != Status::Normal { - refresh_metadata_dvc_suffix(consensus, metadata.journal.as_ref()); - } let actions = consensus.tick(PlaneKind::Metadata); let (local_actions, wire_actions) = split_local_actions(actions); @@ -7978,9 +7383,6 @@ where // nothing is stranded. metadata.resume_stranded_commits().await; - self.advance_pending_metadata_view().await; - self.expire_idle_state_transfer_offers(); - // Stall retry for an in-flight state transfer: descriptor or chunk // frames are fire-and-forget, so a lost one must not wedge the // session (and the boot flow behind it) forever. @@ -8036,7 +7438,45 @@ where } } - self.retry_stalled_metadata_repair(consensus).await; + // Stall retry, mirroring `tick_partitions`: a lost repair frame must + // not wedge the session forever. + let repair_retry_ticks = self.repair_retry_ticks.get(); + let stalled = { + let mut session = self.metadata_repair.borrow_mut(); + session.as_mut().and_then(|session| { + if !consensus.is_normal() { + return None; + } + session.idle_ticks += 1; + if session.idle_ticks < repair_retry_ticks { + return None; + } + session.idle_ticks = 0; + Some((session.peer, session.nonce, session.to_op)) + }) + }; + if let Some((peer, nonce, to_op)) = stalled { + let from_op = consensus.commit_min() + 1; + if from_op <= to_op { + tracing::info!( + shard = self.id, + from_op, + to_op, + peer, + "metadata repair stalled; re-requesting remaining window" + ); + self.send_request_prepares( + consensus.cluster(), + consensus.replica(), + peer, + nonce, + from_op, + to_op, + consensus.namespace(), + ) + .await; + } + } } } @@ -8058,7 +7498,7 @@ where view = consensus.view(), op = consensus.sequencer().current_sequence(), commit = consensus.commit_max(), - namespace = consensus.group(), + namespace = consensus.namespace(), "answering stale-view heartbeat with StartView" ); // Unsolicited, answering a stale-view heartbeat rather than a probe, so there is @@ -8071,600 +7511,55 @@ where commit: consensus.commit_max(), incarnation: 0, target: None, - group: consensus.group(), - // Correcting a peer on a stale view, not concluding a view change: this - // publishes the settled frontier, which the peer reaches by repair. - suffix: Vec::new(), + namespace: consensus.namespace(), }; dispatch_vsr_actions::(consensus, None, &[action]).await; } -/// Rebuild the new primary's pipeline over `from_op..=to_op` from local journal -/// headers. -/// -/// A gap means the caller started the view before its journal could serve the -/// merged log: a bug in the transition, not a data condition. Nothing is -/// truncated, because truncating to the last findable op discards ops committed -/// on a quorum and already acknowledged. The pipeline is left short, the commit -/// walk stalls at the gap, and repair fills it in. -fn rebuild_pipeline_entries( - consensus: &VsrConsensus, - self_id: u8, - from_op: u64, - to_op: u64, - header_at: impl Fn(u64) -> Option, -) where - B: MessageBus, - P: Pipeline, -{ - let mut gap_at = None; - let entries: Vec<_> = (from_op..=to_op) - .map_while(|op| { - let header = header_at(op).or_else(|| { - gap_at = Some(op); - None - })?; - // Lift the monotonic timestamp floor to the rebuilt log so - // post-view-change prepares cannot stamp below committed ones. - consensus.observe_prepare_timestamp(header.timestamp); - let mut entry = consensus::PipelineEntry::new(header); - entry.add_ack(self_id); - Some(entry) - }) - .collect(); - - if let Some(missing_op) = gap_at { - tracing::error!( - replica = self_id, - missing_op, - range_start = from_op, - range_end = to_op, - rebuilt = entries.len(), - "RebuildPipeline: journal gap at op {missing_op} while starting a view; leaving the \ - sequencer at {to_op} and stalling the commit walk. Truncating here would discard ops \ - the view change proved recoverable." - ); - } - - let mut pipeline = consensus.pipeline().borrow_mut(); - for entry in entries { - pipeline.push(entry); - } +/// Re-stamp a stored prepare with the current view before retransmission. +/// After a view change the primary re-sends its uncommitted suffix as its +/// own prepares (VSR), but the journal keeps the original view stamp and +/// `replicate_preflight` fences `header.view < view` as deposed-primary +/// traffic -- a verbatim replay of the stored bytes would be ignored +/// forever, wedging the commit walk on every peer. The stored buffer is +/// shared with the journal, so the patch runs on an owned copy. +fn restamp_prepare_view(stored: &[u8], view: u32) -> Option> { + const VIEW_OFFSET: usize = std::mem::offset_of!(PrepareHeader, view); + let mut owned = server_common::iobuf::Owned::::copy_from_slice(stored); + owned.as_mut_slice()[VIEW_OFFSET..VIEW_OFFSET + std::mem::size_of::()] + .copy_from_slice(&view.to_ne_bytes()); + Message::::try_from(owned) + .ok() + .map(Message::into_frozen) } -/// Snapshot this replica's uncommitted suffix into consensus, if the journal has -/// moved since the last snapshot. -/// -/// Called before every handler that could start or join a view change: consensus -/// records its own `DoViewChange` there and has no journal to read. A stale -/// snapshot is never reused; consensus tags it with its `(op, commit)` and falls -/// back to an empty suffix, stalling the view change rather than nacking an op -/// since acquired. -fn refresh_metadata_dvc_suffix(consensus: &VsrConsensus, journal: Option<&MJ>) -where +/// Dispatch a list of `VsrAction`s by constructing the appropriate +/// protocol messages and sending them via the consensus message bus. +#[allow( + clippy::future_not_send, + clippy::too_many_lines, + clippy::cast_possible_truncation +)] +async fn dispatch_vsr_actions( + consensus: &VsrConsensus, + journal: Option<&J>, + actions: &[VsrAction], +) where B: MessageBus, P: Pipeline, - MJ: JournalHandle, - ::Target: Journal< - ::Storage, + J: JournalHandle, + ::Target: Journal< + ::Storage, Entry = Message, Header = PrepareHeader, >, { - if !consensus.local_dvc_suffix_stale() { - return; - } - let op = consensus.sequencer().current_sequence(); - let commit = consensus.commit_max().min(op); - let pending = adopted_view_headers(consensus); - consensus.set_local_dvc_suffix(build_metadata_dvc_suffix( - journal, - commit, - op, - pending.as_ref().map(|pending| pending.headers.as_slice()), - )); -} + use std::mem::size_of; -/// The adopted view's headers, when they describe a log this replica has NOT itself -/// decided. -/// -/// `None` for the primary-elect holding the log its own merge produced: that log is -/// a proposal it is still repairing toward and may contain ops a later view -/// truncated, so stitching it into its own `DoViewChange` would re-assert them. -/// -/// A backup's parked log is the opposite: headers the view already decided and -/// announced, which this replica acknowledged and is repairing to hold. -fn adopted_view_headers(consensus: &VsrConsensus) -> Option -where - B: MessageBus, - P: Pipeline, -{ - if consensus.is_primary_for_view(consensus.view()) { - return None; - } - consensus.pending_view_log() -} - -/// Snapshot a partition's uncommitted suffix into its consensus. -/// -/// Same contract as [`Self::refresh_metadata_dvc_suffix`]. The partition journal -/// is in-memory only, so after a restart it reads empty and this replica votes -/// all-nack: correct, since the ops really are lost and the merge needs a peer -/// that still holds them. -/// -/// Read through `repair_header`, not the resident headers: the committed prefix -/// leaves those as soon as its bytes reach a segment, which on a caught-up -/// replica includes the commit point itself. -fn refresh_partition_dvc_suffix(partition: &partitions::IggyPartition) -where - B: MessageBus, - SB: SuperblockStore, -{ - let consensus = partition.consensus(); - if !consensus.local_dvc_suffix_stale() { - return; - } - let op = consensus.sequencer().current_sequence(); - let commit = consensus.commit_max().min(op); - let journal = partition.log.journal(); - let pending = adopted_view_headers(consensus); - // The window materialized once: probing `repair_header` per op is two linear - // scans each, up to `DVC_HEADERS_MAX` of them, on every SVC/DVC arrival and - // non-Normal tick, on the pump. The internal clamp only narrows this range. - let head = op.max( - pending - .as_ref() - .and_then(|pending| pending.headers.first()) - .map_or(0, |header| header.op), - ); - let window = journal.inner.repair_headers_in(commit.max(1)..=head); - let suffix = build_dvc_suffix( - commit, - op, - |entry_op| window.get(&entry_op).copied(), - pending.as_ref().map(|pending| pending.headers.as_slice()), - ); - consensus.set_local_dvc_suffix(suffix); -} - -/// The suffix headers a `DoViewChange` or `StartView` carries, as raw bytes. -/// -/// `size` is attacker-controlled, so it is clamped to what arrived; a short read -/// decodes as a malformed suffix and the DVC is dropped. -fn control_suffix_body(msg: &Message) -> &[u8] -where - H: iggy_binary_protocol::ConsensusHeader, -{ - let slice = msg.as_slice(); - let start = size_of::(); - let end = (msg.header().size() as usize).min(slice.len()); - if end <= start { - return &[]; - } - &slice[start..end] -} - -/// Seal a control-message body. Zero for an empty body, which is the unsealed -/// sentinel every other integrity field in this protocol uses. -fn control_body_checksum(body: &[u8]) -> u128 { - if body.is_empty() { - return 0; - } - u128::from(iggy_common::calculate_checksum(body)) -} - -/// The body of a control frame, once it matches the checksum its header carries. -/// -/// `None` means corruption in transit and the frame must be dropped whole: the -/// header numbers describe a body that did not arrive intact, so neither half is -/// trustworthy. This is what covers a body-carrying control message end to end. -/// -/// Keyed on whether a body is present, NOT on whether `checksum_body` looks -/// sealed: skipping the check when that field reads zero makes the layer -/// bypassable by clearing the one field that decides whether anything is checked. -/// A frame legitimately carries no body (a sender with nothing uncommitted, a -/// probe-answer `StartView`), so emptiness is the only exemption. A non-empty body -/// always came from a sender that seals it, and a zero checksum there is corruption. -fn control_suffix_body_verified(msg: &Message, checksum_body: u128) -> Option<&[u8]> -where - H: iggy_binary_protocol::ConsensusHeader, -{ - let body = control_suffix_body(msg); - if body.is_empty() { - // Nothing to verify. `checksum_body` is irrelevant either way. - return Some(body); - } - if control_body_checksum(body) == checksum_body { - Some(body) - } else { - None - } -} - -/// Whether a repaired prepare at `op` falls inside the range this replica is -/// currently repairing. -/// -/// A parked log means two things depending on who parked it, and only one is a -/// repair window. The primary-elect parked the log its merge decided and repairs -/// toward exactly that range, so the range IS its scope, including ops at or -/// below `commit_min`: those are the headers inherited from senders behind the -/// canonical `log_view`, which the ordinary rule would reject and header repair -/// cannot walk back to. A backup's parked `StartView` suffix is only what its -/// ingest verifies bodies against, and its repair runs for the whole view, so -/// reading that range as a scope would discard every later op. -fn repair_op_in_scope( - pending: Option<&MergedLog>, - is_primary_elect: bool, - commit_min: u64, - op: u64, -) -> bool { - pending - .filter(|_| is_primary_elect) - .map_or(op > commit_min, |pending| { - (op >= pending.commit_max.max(1) && op <= pending.op_head) - || pending - .committed_elsewhere - .iter() - .any(|expected| expected.op == op) - }) -} - -/// Ceiling on the op range a repair request may ask this replica to walk. -/// -/// Not `commit_max` alone: a new primary repairing toward a merged log needs the -/// uncommitted suffix the view change kept, which sits above every commit point. -/// -/// Bounded by the local frontier all the same. `RequestPreparesHeader::validate` -/// accepts any `from_op <= to_op`, so `u64::MAX` is legal, and the metadata serve -/// path then walks op by op with no `.await` -- on a single-threaded shard pump -/// that ends the shard rather than merely serving slowly. Nothing above the -/// frontier is servable, so the clamp costs nothing. -fn repair_serve_ceiling(requested_to_op: u64, commit_max: u64, head: u64) -> u64 { - requested_to_op.min(commit_max.max(head)) -} - -/// Read this replica's uncommitted suffix out of the metadata journal, for the -/// window `commit..=op`. -/// -/// The nack bit is load-bearing, and is set only where absence *proves* this -/// replica never prepared the op: -/// * Above the commit point, a missing header is proof: the WAL refuses to boot -/// on interior corruption, so a hole in a journal that opened never arrived. -/// * At or below it, a checkpoint may have compacted the header away. Those slots -/// go out blank and un-nacked, read as "no information" rather than licence to -/// truncate an op this replica considers committed. -/// -/// Deriving the suffix on demand is also why it needs no durable record: the -/// merged log is in memory and bodies are fetched whole, so the WAL is the only -/// thing that ever backs a nack and recomputing after a restart gives the same -/// answer. A torn tail is the one exception, and it changes the answer correctly: -/// recovery truncates the incomplete append, which fsyncs before the ack, so no -/// replication quorum could have counted it. -fn build_metadata_dvc_suffix( - journal: Option<&J>, - commit: u64, - op: u64, - view_headers: Option<&[PrepareHeader]>, -) -> DvcSuffix -where - J: JournalHandle, - ::Target: Journal< - ::Storage, - Entry = Message, - Header = PrepareHeader, - >, -{ - let Some(journal) = journal else { - return DvcSuffix::empty(); - }; - let handle = journal.handle(); - build_dvc_suffix( - commit, - op, - |entry_op| { - usize::try_from(entry_op) - .ok() - .and_then(|slot| handle.header(slot)) - .map(|header| *header) - }, - view_headers, - ) -} - -/// Plane-independent core of the suffix read. `header_at` answers "do I hold -/// this op, and what is its header". -fn build_dvc_suffix( - commit: u64, - op: u64, - header_at: impl Fn(u64) -> Option, - view_headers: Option<&[PrepareHeader]>, -) -> DvcSuffix { - // Stitch the adopted view's headers over the journal, high-to-low. - // - // Reading the journal alone is only correct for a replica whose journal IS its - // log. A backup that adopted a `StartView` is header-poor by design: the suffix - // went to `pending_view_log` and the bodies are still being repaired, so the - // journal holds nothing at those ops and would report them blank AND nacked, - // since a hole above the commit point is normally proof the op never arrived. - // Here it proves only unfinished repair, and enough such senders reach a nack - // quorum against ops the view just decided to keep. - // - // The head rises to the view's head too, so a later view change cannot let the - // op backtrack below what this replica already acknowledged. - let view_head = view_headers - .and_then(<[PrepareHeader]>::first) - .map_or(0, |header| header.op); - let op = op.max(view_head); - if op == 0 { - return DvcSuffix::empty(); - } - // Window runs from the commit point up, floored at 1 because ops are 1-based. - // That floor is a scan bound only: the lines below can raise it above the - // commit point, so no reader may read it back as one. See `merge_commit_max`. - let mut low = commit.max(1); - if low > op { - return DvcSuffix::empty(); - } - if op - low + 1 > DVC_HEADERS_MAX as u64 { - // Defensive: every plane's `prepare_queue_depth` is capped below - // `DVC_HEADERS_MAX` so `op - commit` cannot reach this. If it does, the - // clamped-away ops go out described by nobody and the merge stalls rather - // than deciding wrongly. Keep the highest entries, whose fate the view - // change decides, and log it rather than shipping a different window. - let clamped = op - DVC_HEADERS_MAX as u64 + 1; - tracing::warn!( - commit, - op, - window_from = clamped, - "uncommitted suffix wider than {DVC_HEADERS_MAX} entries; truncating the DVC window \ - from below. Ops {}..={} are now undecidable and will stall the view change", - commit + 1, - clamped - 1 - ); - low = clamped; - } - - let len = usize::try_from(op - low + 1).unwrap_or(DVC_HEADERS_MAX); - let mut headers = Vec::with_capacity(len); - let mut nack_bitset = 0u128; - let mut present_bitset = 0u128; - for (index, entry_op) in (low..=op).rev().enumerate() { - if let Some(header) = header_at(entry_op) { - headers.push(header); - // A header in the index means the entry is in the WAL at a known - // offset, the same condition `on_request_prepares` serves from. - present_bitset |= 1u128 << index; - } else if let Some(header) = - view_headers.and_then(|headers| view_header_at(headers, entry_op)) - { - // Held from the adopted view rather than from the journal, so the - // header is reported and the op is NOT nacked: this replica knows - // the op exists and simply cannot serve its body yet. No present - // bit for the same reason. - headers.push(*header); - } else { - headers.push(dvc_blank(entry_op)); - if entry_op > commit { - nack_bitset |= 1u128 << index; - } else { - // The commit point, the one slot that goes out blank AND - // un-nacked. The merge scans it and may not discard it, so a - // sender is asking the new primary to take the header from - // someone else; if every sender in the quorum does that, the - // op is undecidable and the view never starts. - // - // Every compaction path is supposed to leave this header behind - // (the metadata checkpoint drain stops one op short, a - // partition serves it from the evicted ring), so reaching here - // means a replica whose log genuinely starts above its own - // commit point: a state-transfer receiver that jumped its - // commit floor to a snapshot whose prepares it never held. - tracing::warn!( - op = entry_op, - commit, - "no header at this replica's commit point; the DVC reports it blank and \ - cannot nack it, so the view change stalls unless a peer supplies it" - ); - } - } - } - DvcSuffix::new(headers, nack_bitset, present_bitset) -} - -/// Partition-plane twin of `Shard::reconcile_metadata_view_divergence`: same split -/// at the announced commit point, dropping above it and reporting at or below. -/// -/// Worse to skip here than on the metadata plane, which is why this exists. -/// Partition `append` has no slot-collision check, so a re-prepared op pushes a -/// duplicate header and rewrites `op_to_storage_offset`, and `committed_prefix` walks -/// positionally, so the stale entry is what `evict_prefix` flushes to the segment: -/// durable divergent bytes, no error anywhere. -#[allow(clippy::future_not_send)] -async fn reconcile_partition_view_divergence( - shard: u16, - partition: &mut IggyPartition, - pending: &MergedLog, -) where - B: MessageBus, - SB: journal::superblock::SuperblockStore, -{ - // Truncation is safe only above what this replica has *applied*, which is not - // the view's commit point: a backup can sit above it. - let applied_floor = pending.commit_max.max(partition.consensus().commit_min()); - - let mut repairable_from: Option = None; - for canonical in &pending.headers { - let Some(local) = partition.log.journal().inner.header_by_op(canonical.op) else { - continue; - }; - if header_is_view_entry(&local, canonical) { - continue; - } - if canonical.op <= applied_floor { - tracing::error!( - shard, - namespace_raw = partition.consensus().group(), - op = canonical.op, - view = partition.consensus().view(), - commit_max = pending.commit_max, - commit_min = partition.consensus().commit_min(), - local_checksum = local.checksum, - canonical_checksum = canonical.checksum, - "committed partition op {} disagrees with the view that just started; this \ - replica applied a different op and log repair cannot reconcile it", - canonical.op - ); - continue; - } - repairable_from = Some(repairable_from.map_or(canonical.op, |op| op.min(canonical.op))); - } - - // The suffix above the announced head, which no canonical header names. As on - // the metadata twin, except here `append` pushes a duplicate rather than erroring. - let above_head = pending.op_head.max(applied_floor) + 1; - if partition - .log - .journal() - .inner - .last_op() - .is_some_and(|last_op| last_op >= above_head) - { - repairable_from = Some(repairable_from.map_or(above_head, |op| op.min(above_head))); - } - - let Some(from_op) = repairable_from else { - return; - }; - match partition.truncate_uncommitted_from(from_op).await { - Ok(removed) => { - tracing::warn!( - shard, - namespace_raw = partition.consensus().group(), - from_op, - removed, - op_head = pending.op_head, - view = partition.consensus().view(), - "dropped {removed} uncommitted partition entries from op {from_op} that \ - disagreed with the view's log; the primary's retransmission refills the range" - ); - } - Err(error) => { - tracing::error!( - shard, - namespace_raw = partition.consensus().group(), - from_op, - %error, - "could not drop the diverging uncommitted partition entries from op \ - {from_op}; repair skips ops it already holds, so this replica will not \ - converge there until restarted" - ); - } - } -} - -/// Whether a locally journaled header IS the entry the view's log names at that op. -/// -/// Identity, not presence: otherwise a stale prepare at the right op reads as -/// coverage everywhere: the repair ingest skips it as already held, -/// `RebuildPipeline` seeds the pipeline from it and self-acks, `CommitJournal` -/// applies it. `identity_checksum` excludes `view`, so a restamp still compares equal. -/// -/// An unsealed checksum on either side is not evidence (pre-seal WAL, partition-plane -/// prepare), so it counts as agreement, as in `dvc_suffix_decode`. -const fn header_is_view_entry(local: &PrepareHeader, canonical: &PrepareHeader) -> bool { - local.checksum == CHECKSUM_UNSEALED - || canonical.checksum == CHECKSUM_UNSEALED - || local.checksum == canonical.checksum -} - -/// The lowest op in the merged log this replica cannot serve, or `None` when the -/// view can start. -/// -/// Coverage is identity, not presence (see [`header_is_view_entry`]): starting a view -/// over a differing entry commits this replica's own operation where the view says -/// another belongs. -/// -/// Covers every op the merged log names, including headers inherited from senders -/// behind the canonical `log_view`, which sit below the canonical window where header -/// repair cannot walk back to them. `repair_floor` drops the ops whose journal entry -/// is legitimately gone AND whose identity is already settled: on the metadata plane -/// ops compacted under a snapshot, on the partition plane ops at or below the local -/// commit point, whose flushed entries `evict_prefix` moves out of the header vec. -/// Neither can diverge from the merged log (a committed or compacted op is the -/// quorum's op), and no repair puts the journal entry back, so demanding one parks -/// the view change forever. -fn first_op_not_covered( - pending: &MergedLog, - repair_floor: u64, - header_at: impl Fn(u64) -> Option, -) -> Option { - let held = |op: u64| { - let Some(local) = header_at(op) else { - return false; - }; - pending - .headers - .iter() - .chain(pending.committed_elsewhere.iter()) - .find(|header| header.op == op) - .is_none_or(|canonical| header_is_view_entry(&local, canonical)) - }; - (pending.commit_max.max(1).max(repair_floor + 1)..=pending.op_head) - .find(|op| !held(*op)) - .or_else(|| { - pending - .committed_elsewhere - .iter() - .map(|header| header.op) - .filter(|op| *op > repair_floor) - .find(|op| !held(*op)) - }) -} - -/// The adopted view's header at `op`, or `None` when the view says nothing about -/// it. -/// -/// Headers run high-to-low from the view's head, so the slot is arithmetic. The -/// op is re-checked rather than assumed: a mismatch means the range is not the -/// contiguous run this indexing needs, and inventing a header for the wrong op -/// is worse than reporting none. -fn view_header_at(view_headers: &[PrepareHeader], op: u64) -> Option<&PrepareHeader> { - let head = view_headers.first()?.op; - let index = usize::try_from(head.checked_sub(op)?).ok()?; - let header = view_headers.get(index)?; - if header.op != op || matches!(dvc_header_kind(header), DvcHeaderKind::Blank) { - return None; - } - Some(header) -} - -/// Dispatch a list of `VsrAction`s by constructing the appropriate -/// protocol messages and sending them via the consensus message bus. -#[allow( - clippy::future_not_send, - clippy::too_many_lines, - clippy::cast_possible_truncation -)] -async fn dispatch_vsr_actions( - consensus: &VsrConsensus, - journal: Option<&J>, - actions: &[VsrAction], -) where - B: MessageBus, - P: Pipeline, - J: JournalHandle, - ::Target: Journal< - ::Storage, - Entry = Message, - Header = PrepareHeader, - >, -{ - use std::mem::size_of; - - let bus = consensus.message_bus(); - let self_id = consensus.replica(); - let cluster = consensus.cluster(); - let replica_count = consensus.replica_count(); + let bus = consensus.message_bus(); + let self_id = consensus.replica(); + let cluster = consensus.cluster(); + let replica_count = consensus.replica_count(); let send = |target: u8, msg: Frozen| async move { if let Err(e) = bus.send_to_replica(target, msg).await { @@ -8711,23 +7606,22 @@ async fn dispatch_vsr_actions( !advertises_view || !consensus.needs_superblock_persist(), "durable-before-send violated: dispatching a view-scoped action for \ namespace {} while the superblock is behind the in-memory view {}", - consensus.group(), + consensus.namespace(), consensus.view(), ); } for action in actions { match action { - VsrAction::SendStartViewChange { view, group } => { + VsrAction::SendStartViewChange { view, namespace } => { let msg = Message::::new(size_of::()) .transmute_header(|_, h: &mut StartViewChangeHeader| { h.command = Command2::StartViewChange; h.cluster = cluster; h.replica = self_id; h.view = *view; - h.group = *group; + h.namespace = *namespace; h.size = size_of::() as u32; - h.seal(); }); broadcast(msg.into_generic().into_frozen()).await; } @@ -8737,41 +7631,23 @@ async fn dispatch_vsr_actions( log_view, op, commit, - group, - suffix, + namespace, } => { - let header_size = size_of::(); - let total_size = header_size + suffix.encoded_len(); - let mut msg = Message::::new(total_size); - // Body first: `transmute_header` zeroes only the header region, so - // anything past it survives. Same order as the manifest build. - suffix.encode_into(&mut msg.as_mut_slice()[header_size..total_size]); - let body_checksum = control_body_checksum(&msg.as_slice()[header_size..total_size]); - let nack_bitset = suffix.nack_bitset(); - let present_bitset = suffix.present_bitset(); - let msg = msg.transmute_header(|_, h: &mut DoViewChangeHeader| { - h.command = Command2::DoViewChange; - h.cluster = cluster; - h.replica = self_id; - h.view = *view; - h.log_view = *log_view; - h.op = *op; - h.commit = *commit; - h.group = *group; - h.nack_bitset = nack_bitset; - h.present_bitset = present_bitset; - h.checksum_body = body_checksum; - h.size = total_size as u32; - // Last: covers the bitsets a new primary truncates on. - h.seal(); - }); - // Broadcast, not unicast to `target`: a backup seeing a DVC for a - // newer view adopts it instead of waiting out its heartbeat - // timeout, which converges the view change in one round. - let _ = target; - broadcast(msg.into_generic().into_frozen()).await; + let msg = Message::::new(size_of::()) + .transmute_header(|_, h: &mut DoViewChangeHeader| { + h.command = Command2::DoViewChange; + h.cluster = cluster; + h.replica = self_id; + h.view = *view; + h.log_view = *log_view; + h.op = *op; + h.commit = *commit; + h.namespace = *namespace; + h.size = size_of::() as u32; + }); + send(*target, msg.into_generic().into_frozen()).await; } - VsrAction::SendRequestStartView { view, group } => { + VsrAction::SendRequestStartView { view, namespace } => { // Stamp this replica's incarnation so the answering StartView can // echo it, proving to us the reply post-dates our restart. let incarnation = consensus.incarnation(); @@ -8783,9 +7659,8 @@ async fn dispatch_vsr_actions( h.replica = self_id; h.view = *view; h.incarnation = incarnation; - h.group = *group; + h.namespace = *namespace; h.size = size_of::() as u32; - h.seal(); }); broadcast(msg.into_generic().into_frozen()).await; } @@ -8795,28 +7670,20 @@ async fn dispatch_vsr_actions( commit, incarnation, target, - group, - suffix, + namespace, } => { - let header_size = size_of::(); - let total_size = header_size + suffix.len() * size_of::(); - let mut msg = Message::::new(total_size); - // Body first: `transmute_header` zeroes only the header region. - encode_prepare_headers(suffix, &mut msg.as_mut_slice()[header_size..total_size]); - let body_checksum = control_body_checksum(&msg.as_slice()[header_size..total_size]); - let msg = msg.transmute_header(|_, h: &mut StartViewHeader| { - h.checksum_body = body_checksum; - h.command = Command2::StartView; - h.cluster = cluster; - h.replica = self_id; - h.view = *view; - h.op = *op; - h.commit = *commit; - h.incarnation = *incarnation; - h.group = *group; - h.size = total_size as u32; - h.seal(); - }); + let msg = Message::::new(size_of::()) + .transmute_header(|_, h: &mut StartViewHeader| { + h.command = Command2::StartView; + h.cluster = cluster; + h.replica = self_id; + h.view = *view; + h.op = *op; + h.commit = *commit; + h.incarnation = *incarnation; + h.namespace = *namespace; + h.size = size_of::() as u32; + }); let frozen = msg.into_generic().into_frozen(); // A probe echo is addressed to its requester: the incarnation it // carries is that replica's freshness proof, and a peer recovering @@ -8832,7 +7699,7 @@ async fn dispatch_vsr_actions( from_op, to_op, target, - group, + namespace, } => { let Some(journal) = journal else { continue; @@ -8855,9 +7722,8 @@ async fn dispatch_vsr_actions( h.prepare_checksum = prepare_header.checksum; h.request = prepare_header.request; h.operation = prepare_header.operation; - h.group = *group; + h.namespace = *namespace; h.size = size_of::() as u32; - h.seal(); }); send(*target, msg.into_generic().into_frozen()).await; } @@ -8872,10 +7738,15 @@ async fn dispatch_vsr_actions( continue; }; // Freeze the retransmit payload once; clone per target. - let Some(frozen) = - restamp_prepare_view(prepare.into_generic().into_frozen(), current_view) - else { - continue; + let frozen = if prepare.header().view == current_view { + prepare.into_generic().into_frozen() + } else { + let Some(restamped) = + restamp_prepare_view(prepare.as_slice(), current_view) + else { + continue; + }; + restamped }; for replica in replicas { send(*replica, frozen.clone()).await; @@ -8886,12 +7757,49 @@ async fn dispatch_vsr_actions( let Some(journal) = journal else { continue; }; - rebuild_pipeline_entries(consensus, self_id, *from_op, *to_op, |op| { - usize::try_from(op) - .ok() - .and_then(|slot| journal.handle().header(slot)) - .map(|header| *header) - }); + // Collect headers before borrowing the pipeline to avoid + // holding borrow_mut() across journal reads. + let mut gap_at = None; + let entries: Vec<_> = (*from_op..=*to_op) + .map_while(|op| { + let Some(header) = journal.handle().header(op as usize) else { + gap_at = Some(op); + return None; + }; + // New-primary path: lift the monotonic timestamp + // floor to the rebuilt log so post-view-change + // prepares cannot stamp below committed ones. + consensus.observe_prepare_timestamp(header.timestamp); + let mut entry = consensus::PipelineEntry::new(*header); + entry.add_ack(self_id); + Some(entry) + }) + .collect(); + if let Some(missing_op) = gap_at { + // A primary's own uncommitted suffix has no repair + // source: peers ack'd nothing above the gap or the DVC + // merge would have carried it, so the range is decided + // lost. Truncate the sequencer to the last op we could + // rebuild so the next client prepare chains correctly. + let rebuilt_up_to = missing_op.saturating_sub(1); + tracing::warn!( + replica = self_id, + missing_op, + range_start = from_op, + range_end = to_op, + rebuilt = entries.len(), + "RebuildPipeline: journal gap at op {missing_op}, \ + truncating sequencer from {to_op} to {rebuilt_up_to} \ + ({}/{} ops rebuilt)", + entries.len(), + to_op - from_op + 1, + ); + consensus.sequencer().set_sequence(rebuilt_up_to); + } + let mut pipeline = consensus.pipeline().borrow_mut(); + for entry in entries { + pipeline.push(entry); + } } // Handled by the caller (shard view change handlers) since it // requires access to the plane's commit_journal method. @@ -8899,7 +7807,7 @@ async fn dispatch_vsr_actions( VsrAction::SendCommit { view, commit, - group, + namespace, timestamp_monotonic, } => { let msg = Message::::new(size_of::()).transmute_header( @@ -8909,10 +7817,9 @@ async fn dispatch_vsr_actions( h.replica = self_id; h.view = *view; h.commit = *commit; - h.group = *group; + h.namespace = *namespace; h.timestamp_monotonic = *timestamp_monotonic; h.size = size_of::() as u32; - h.seal(); }, ); broadcast(msg.into_generic().into_frozen()).await; @@ -8958,7 +7865,7 @@ async fn dispatch_partition_journal_actions( || !consensus.needs_superblock_persist(), "durable-before-send violated: dispatching a view-scoped action for \ namespace {} while the superblock is behind the in-memory view {}", - consensus.group(), + consensus.namespace(), consensus.view(), ); } @@ -8970,7 +7877,7 @@ async fn dispatch_partition_journal_actions( from_op, to_op, target, - group, + namespace, } => { for op in *from_op..=*to_op { let Some(prepare_header) = journal.header_by_op(op) else { @@ -8989,9 +7896,8 @@ async fn dispatch_partition_journal_actions( h.prepare_checksum = prepare_header.checksum; h.request = prepare_header.request; h.operation = prepare_header.operation; - h.group = *group; + h.namespace = *namespace; h.size = size_of::() as u32; - h.seal(); }); send(*target, msg.into_generic().into_frozen()).await; } @@ -9019,8 +7925,15 @@ async fn dispatch_partition_journal_actions( // above and avoids both the per-target 4 KiB memcpy // and the prior `.expect` that would panic the shard // on a corrupted journal entry. - let Some(prepare) = restamp_prepare_view(prepare, current_view) else { - continue; + let prepare = if header.view == current_view { + prepare + } else { + let Some(restamped) = + restamp_prepare_view(prepare.as_slice(), current_view) + else { + continue; + }; + restamped }; for replica in replicas { send(*replica, prepare.clone()).await; @@ -9028,9 +7941,42 @@ async fn dispatch_partition_journal_actions( } } VsrAction::RebuildPipeline { from_op, to_op } => { - rebuild_pipeline_entries(consensus, self_id, *from_op, *to_op, |op| { - journal.header_by_op(op) - }); + let mut gap_at = None; + let entries: Vec<_> = (*from_op..=*to_op) + .map_while(|op| { + let Some(header) = journal.header_by_op(op) else { + gap_at = Some(op); + return None; + }; + // New-primary path: lift the monotonic timestamp + // floor to the rebuilt log so post-view-change + // prepares cannot stamp below committed ones. + consensus.observe_prepare_timestamp(header.timestamp); + let mut entry = consensus::PipelineEntry::new(header); + entry.add_ack(self_id); + Some(entry) + }) + .collect(); + if let Some(missing_op) = gap_at { + let rebuilt_up_to = missing_op.saturating_sub(1); + tracing::warn!( + replica = self_id, + missing_op, + range_start = from_op, + range_end = to_op, + rebuilt = entries.len(), + "RebuildPipeline: journal gap at op {missing_op}, \ + truncating sequencer from {to_op} to {rebuilt_up_to} \ + ({}/{} ops rebuilt)", + entries.len(), + to_op - from_op + 1, + ); + consensus.sequencer().set_sequence(rebuilt_up_to); + } + let mut pipeline = consensus.pipeline().borrow_mut(); + for entry in entries { + pipeline.push(entry); + } } _ => {} } @@ -9062,8 +8008,7 @@ mod persist_gate_tests { commit: 3, incarnation: 0, target: None, - group: 7, - suffix: Vec::new(), + namespace: 7, }, VsrAction::CommitJournal, rebuild(), @@ -9083,422 +8028,12 @@ mod persist_gate_tests { #[test] fn given_send_only_actions_when_split_should_leave_locals_empty() { - let actions = vec![VsrAction::SendStartViewChange { view: 2, group: 7 }]; + let actions = vec![VsrAction::SendStartViewChange { + view: 2, + namespace: 7, + }]; let (local, wire) = split_local_actions(actions); assert!(local.is_empty()); assert_eq!(wire.len(), 1); } } - -#[cfg(test)] -mod repair_scope_tests { - //! Who parked the log decides what it means. - - use super::{MergedLog, repair_op_in_scope, repair_serve_ceiling}; - use iggy_binary_protocol::{Command2, PrepareHeader}; - - fn header(op: u64) -> PrepareHeader { - PrepareHeader { - command: Command2::Prepare, - op, - ..Default::default() - } - } - - /// A view that started at op 100 with commit 98. - fn parked() -> MergedLog { - MergedLog { - op_head: 100, - commit_max: 98, - headers: (98..=100).rev().map(header).collect(), - committed_elsewhere: Vec::new(), - } - } - - #[test] - fn given_a_backup_with_a_parked_log_when_repairing_above_the_view_head_should_accept() { - // A backup keeps its parked `StartView` suffix for the whole view, so at - // op 200 the parked head is 100 ops stale. Reading it as a repair scope - // silently discards the served op: the retry loops, the commit walk - // freezes, checkpointing stops, and the backup stops acking. - assert!( - repair_op_in_scope(Some(&parked()), false, 149, 150), - "a backup repairs for the whole view, not just the view-start range" - ); - } - - #[test] - fn given_a_backup_with_a_parked_log_when_repairing_below_commit_min_should_reject() { - // A backup's parked log grants no licence to re-ingest committed ops. - assert!(!repair_op_in_scope(Some(&parked()), false, 149, 149)); - } - - #[test] - fn given_a_primary_elect_when_repairing_toward_its_merged_log_should_use_it_as_the_scope() { - let pending = parked(); - // Inside the merged range, including inherited headers below `commit_min`. - assert!(repair_op_in_scope(Some(&pending), true, 99, 98)); - assert!(repair_op_in_scope(Some(&pending), true, 99, 100)); - // Outside it: the primary-elect is not repairing toward these. - assert!(!repair_op_in_scope(Some(&pending), true, 99, 101)); - assert!(!repair_op_in_scope(Some(&pending), true, 99, 97)); - // With nothing parked, the ordinary commit-point rule applies. - assert!(!repair_op_in_scope(None, false, 149, 149)); - assert!(repair_op_in_scope(None, false, 149, 150)); - } - - #[test] - fn given_a_primary_elect_when_an_op_is_committed_elsewhere_should_accept_it() { - let mut pending = parked(); - pending.committed_elsewhere.push(header(42)); - assert!(repair_op_in_scope(Some(&pending), true, 99, 42)); - } - - #[test] - fn given_a_repair_request_when_serving_should_clamp_to_the_frontier_but_not_below_it() { - // `validate` accepts any `to_op >= from_op` and the serve path walks op by - // op with no `.await`, so an unclamped ceiling hangs the whole shard. - assert_eq!(repair_serve_ceiling(u64::MAX, 40, 90), 90); - assert_eq!(repair_serve_ceiling(50, 40, 90), 50); - // The suffix a new primary repairs toward sits above every commit point, - // so clamping to `commit_max` alone deadlocks the view change. - assert_eq!(repair_serve_ceiling(90, 40, 90), 90); - // `commit_max` above the local head still counts: heartbeats outrun prepares. - assert_eq!(repair_serve_ceiling(u64::MAX, 120, 90), 120); - } -} - -#[cfg(test)] -mod view_coverage_tests { - //! Holding an op is not holding the view's op. - - use super::{MergedLog, first_op_not_covered}; - use iggy_binary_protocol::{Command2, Operation, PrepareHeader}; - - fn sealed(op: u64, request: u64) -> PrepareHeader { - let mut header = PrepareHeader { - command: Command2::Prepare, - operation: Operation::CreateStream, - op, - request, - ..Default::default() - }; - header.checksum = header.identity_checksum(); - header - } - - #[test] - fn given_a_diverging_entry_when_scanning_should_report_it_like_a_hole() { - // Op 99 is present and is not the view's op 99. Reading presence as coverage - // starts the view over an operation the view says is something else, which - // `CommitJournal` then applies at or below the commit point unchecked. - let pending = MergedLog { - op_head: 100, - commit_max: 98, - headers: (98..=100).rev().map(|op| sealed(op, 1)).collect(), - committed_elsewhere: Vec::new(), - }; - let held = [sealed(100, 1), sealed(99, 7), sealed(98, 1)]; - let missing = first_op_not_covered(&pending, 0, |op| { - held.iter().find(|header| header.op == op).copied() - }); - assert_eq!(missing, Some(99)); - } - - #[test] - fn given_an_evicted_committed_window_when_floored_should_start_the_view() { - // The wedge behind the partition_state_transfer regressions: a survivor - // that flushes on every commit holds NO resident journal header (the - // flush evicts them), so a merged window opening on its own committed op - // reads as a hole nothing can fill -- no repair re-journals a committed - // op. The floor (local commit point) must count it as covered, or the - // primary-elect parks in `ViewChange` forever and the rotation hands - // primaryship to an empty rejoiner that then cannot be served the state - // transfer it needs. - let pending = MergedLog { - op_head: 256, - commit_max: 256, - headers: vec![sealed(256, 1)], - committed_elsewhere: Vec::new(), - }; - let nothing_resident = |_: u64| None; - assert_eq!( - first_op_not_covered(&pending, 0, nothing_resident), - Some(256), - "unfloored, the evicted committed op reads as an unfillable hole" - ); - assert_eq!( - first_op_not_covered(&pending, 256, nothing_resident), - None, - "floored at the local commit point, the view starts" - ); - } -} - -#[cfg(test)] -mod dvc_suffix_window_tests { - //! The suffix window's floor is a scan bound, not a commit point. - //! - //! Reading the lowest suffix op back as a proven commit point assumes suffix - //! generation stops at the sender's commit. These pin the two paths that break - //! that premise, so it cannot be quietly reintroduced. - - use super::{DVC_HEADERS_MAX, build_dvc_suffix}; - use iggy_binary_protocol::{Command2, Operation, PrepareHeader}; - - /// A real prepare at `op`. The operation must not be `Reserved`: that is - /// exactly `dvc_blank`, and `dvc_header_kind` classifies by equality with it. - fn held(op: u64) -> PrepareHeader { - PrepareHeader { - command: Command2::Prepare, - operation: Operation::CreateStream, - op, - ..Default::default() - } - } - - /// The lowest op the built window describes. - fn floor(suffix: &consensus::DvcSuffix) -> Option { - suffix.headers().last().map(|header| header.op) - } - - /// A view's headers for `low..=high`, high-to-low as the suffix carries them. - fn view_headers(low: u64, high: u64) -> Vec { - (low..=high).rev().map(held).collect() - } - - #[test] - fn given_an_adopted_view_when_the_journal_is_empty_should_report_its_headers_unnacked() { - // A backup that adopted a `StartView` put the suffix in `pending_view_log` - // and is still repairing bodies, so its journal holds nothing at those ops. - // Reading the journal alone reports them blank AND nacked, which reaches a - // nack quorum against ops the view had just decided to keep. - let view = view_headers(3, 5); - let suffix = build_dvc_suffix(2, 0, |_| None, Some(&view)); - - assert_eq!( - suffix.len(), - 4, - "the window rises to the view's head even with an empty journal" - ); - assert_eq!( - floor(&suffix), - Some(2), - "the floor is still the commit point" - ); - assert_eq!( - suffix.nack_bitset(), - 0, - "a header held from the adopted view is not a nack" - ); - assert_eq!( - suffix.present_bitset(), - 0, - "and its body is not servable, so no present bit either" - ); - } - - #[test] - fn given_no_adopted_view_when_the_journal_is_empty_should_nack() { - // The contrast: without an adopted view the same holes really are proof. - let suffix = build_dvc_suffix(2, 5, |_| None, None); - assert_eq!( - suffix.nack_bitset(), - 0b0111, - "ops 5, 4 and 3 nack; op 2 is the commit point" - ); - } - - #[test] - fn given_an_adopted_view_when_the_journal_covers_part_should_prefer_the_journal() { - // Journal first, so an op whose body this replica can serve keeps its - // present bit; the view fills only what the journal is missing. - let view = view_headers(3, 5); - let suffix = build_dvc_suffix(2, 5, |op| (op == 5).then(|| held(op)), Some(&view)); - - assert_eq!(suffix.len(), 4); - assert_eq!(suffix.present_bitset(), 0b0001, "only op 5 is servable"); - assert_eq!(suffix.nack_bitset(), 0, "the view covers ops 4 and 3"); - - // The head is the max of the two, never the view's alone. - let short_view = view_headers(3, 4); - let deeper = build_dvc_suffix(2, 6, |op| Some(held(op)), Some(&short_view)); - assert_eq!(deeper.headers().first().map(|header| header.op), Some(6)); - assert_eq!(deeper.present_bitset(), 0b1_1111, "ops 6 down to 2"); - } - - #[test] - fn given_a_blank_view_entry_should_not_report_it_as_held() { - // A blank is the view saying "no header here", not one this replica holds. - let mut view = view_headers(3, 5); - view[1] = consensus::dvc_blank(4); - let suffix = build_dvc_suffix(2, 0, |_| None, Some(&view)); - - assert_eq!(suffix.nack_bitset(), 0b010, "only the blank op nacks"); - } - - #[test] - fn given_no_header_at_the_commit_point_should_report_it_blank_and_undecidable() { - // The window's floor is the commit point, and a blank there is the one - // entry that goes out with neither a header nor a nack. The merge scans - // that op and may not discard it, so a quorum of these deadlocks the view - // change. Pinned here because both compaction paths are meant to keep the - // header alive precisely so this shape never leaves a healthy replica. - let suffix = build_dvc_suffix(5, 5, |_| None, None); - - assert_eq!(suffix.len(), 1); - assert_eq!(floor(&suffix), Some(5)); - assert_eq!( - suffix.nack_bitset(), - 0, - "the commit point is never nacked, whatever the journal says" - ); - assert_eq!(suffix.present_bitset(), 0); - } - - #[test] - fn given_a_window_at_the_depth_ceiling_when_building_should_floor_at_the_commit() { - // At the deepest legal prepare-queue depth the window still starts exactly - // at the commit point, so nothing is clamped and no op goes undescribed. - // Config ceilings and `LocalPipeline::with_capacities` enforce the depth. - let depth = DVC_HEADERS_MAX as u64 - 1; - let commit = 500; - let op = commit + depth; - let suffix = build_dvc_suffix(commit, op, |op| Some(held(op)), None); - - assert_eq!(suffix.len(), DVC_HEADERS_MAX, "the widest window that fits"); - assert_eq!( - floor(&suffix), - Some(commit), - "at the ceiling the floor is still the commit point" - ); - } - - #[test] - fn given_a_window_past_the_depth_ceiling_when_building_should_clamp_above_the_commit() { - // One op deeper and the window clamps: the floor sits 501 ops above the - // sender's commit, with no marker on the frame saying so. - let commit = 500; - let op = commit + DVC_HEADERS_MAX as u64; - let suffix = build_dvc_suffix(commit, op, |op| Some(held(op)), None); - - assert_eq!(suffix.len(), DVC_HEADERS_MAX); - assert_eq!( - floor(&suffix), - Some(op - DVC_HEADERS_MAX as u64 + 1), - "the clamped floor sits above the commit point" - ); - assert!(floor(&suffix) > Some(commit)); - - // Second path, at any depth: ops are 1-based, so commit 0 floors at op 1. - let from_zero = build_dvc_suffix(0, 3, |op| Some(held(op)), None); - assert_eq!(floor(&from_zero), Some(1)); - } - - #[test] - fn given_a_compacted_log_when_building_should_still_describe_the_commit_point() { - // The commit point goes out blank AND un-nacked, so the merge can neither - // adopt nor discard it: a quorum that all compacted to the same op deadlocks - // and no further message fixes it. Both planes must keep that header - // reachable (metadata's drain stops one op short, a partition serves it from - // the evicted ring); nothing in `build_dvc_suffix` enforces it. - let commit = 500; - let compacted = |op: u64| (op >= commit).then(|| held(op)); - let suffix = build_dvc_suffix(commit, commit + 3, compacted, None); - - let commit_index = suffix.index_of(commit + 3, commit).expect("in window"); - assert!( - suffix.valid_header_at(commit_index).is_some(), - "a blank at the commit point is undecidable for the merge" - ); - assert!( - suffix.offers_body(commit_index), - "the commit point must be servable, or the merge stalls waiting for a peer" - ); - assert!( - !suffix.nacks(commit_index), - "the commit point can never be nacked" - ); - } -} - -#[cfg(test)] -mod control_frame_tests { - //! A control frame's body must be verified on a rule corruption cannot switch - //! off. Keying on `checksum_body` looking sealed is bypassable by zeroing it. - - use super::{control_body_checksum, control_suffix_body_verified}; - use iggy_binary_protocol::{Command2, DoViewChangeHeader, PrepareHeader}; - use server_common::Message; - use std::mem::size_of; - - /// A `DoViewChange` frame carrying `entries` blank suffix headers. - fn frame(entries: usize, checksum_body: u128) -> Message { - let header_size = size_of::(); - let total = header_size + entries * size_of::(); - let mut msg = Message::::new(total); - for (index, byte) in msg.as_mut_slice()[header_size..total] - .iter_mut() - .enumerate() - { - *byte = u8::try_from(index % 251).expect("modulus fits u8"); - } - msg.transmute_header(|_, header: &mut DoViewChangeHeader| { - header.command = Command2::DoViewChange; - header.checksum_body = checksum_body; - header.size = u32::try_from(total).expect("frame fits u32"); - }) - } - - #[test] - fn given_a_sealed_body_when_verifying_should_accept() { - let header_size = size_of::(); - let unsealed = frame(2, 0); - let sealed_value = control_body_checksum( - &unsealed.as_slice()[header_size..unsealed.header().size as usize], - ); - let msg = frame(2, sealed_value); - - assert!( - control_suffix_body_verified(&msg, msg.header().checksum_body).is_some(), - "a correctly sealed body must be accepted" - ); - } - - #[test] - fn given_a_body_with_a_zeroed_checksum_when_verifying_should_reject() { - // A non-empty body always came from a sender that seals it, so a zero here is - // corruption. Treating it as "unsealed, skip" disables the layer by clearing - // the one field that decides whether anything is checked. - let msg = frame(2, 0); - assert!( - control_suffix_body_verified(&msg, msg.header().checksum_body).is_none(), - "a non-empty body with a zeroed checksum must be rejected, not waved through" - ); - } - - #[test] - fn given_a_corrupted_body_when_verifying_should_reject() { - let header_size = size_of::(); - let unsealed = frame(2, 0); - let sealed_value = control_body_checksum( - &unsealed.as_slice()[header_size..unsealed.header().size as usize], - ); - let mut msg = frame(2, sealed_value); - msg.as_mut_slice()[header_size] ^= 0xFF; - - assert!( - control_suffix_body_verified(&msg, msg.header().checksum_body).is_none(), - "a body that does not match its checksum must be rejected" - ); - } - - #[test] - fn given_a_header_only_frame_when_verifying_should_accept() { - // A sender with nothing uncommitted contributes numbers only, no body. - let msg = frame(0, 0); - let body = control_suffix_body_verified(&msg, msg.header().checksum_body) - .expect("a header-only frame has nothing to verify"); - assert!(body.is_empty()); - } -} diff --git a/core/shard/src/metrics.rs b/core/shard/src/metrics.rs index 2df15c357e..659550217e 100644 --- a/core/shard/src/metrics.rs +++ b/core/shard/src/metrics.rs @@ -186,7 +186,6 @@ pub struct ShardMetrics { partitions_materialised_total: Counter, partitions_removed_total: Counter, partitions_reconcile_failures_total: Counter, - partitions_duplicate_builds_discarded_total: Counter, partition_transfer_refusals_total: Counter, partition_frames_rejected_stale_total: Counter, partition_frames_rejected_ahead_total: Counter, @@ -221,7 +220,6 @@ impl ShardMetrics { partitions_materialised_total: Counter::default(), partitions_removed_total: Counter::default(), partitions_reconcile_failures_total: Counter::default(), - partitions_duplicate_builds_discarded_total: Counter::default(), partition_transfer_refusals_total: Counter::default(), partition_frames_rejected_stale_total: Counter::default(), partition_frames_rejected_ahead_total: Counter::default(), @@ -261,16 +259,6 @@ impl ShardMetrics { self.partitions_removed_total.inc(); } - /// Bumped when the pump discards a duplicate `InsertOwned` for a namespace - /// that is already live. The reconciler's staged-op guard should make this - /// unreachable, so a non-zero value is a caught correctness anomaly, not - /// routine churn: the discarded build re-planted segment 0 over the live - /// incarnation's path and folded its initial segment into the shared stats - /// before the pump caught it. - pub fn record_duplicate_partition_build_discarded(&self) { - self.partitions_duplicate_builds_discarded_total.inc(); - } - /// Bumped each time `build_partition_fresh` or /// `delete_partitions_from_disk` returns `Err`. The reconciler retries /// next tick, but a sustained climb surfaces a stuck partition (disk diff --git a/core/shard/src/router.rs b/core/shard/src/router.rs index b683af2bc0..0dbf0c1f61 100644 --- a/core/shard/src/router.rs +++ b/core/shard/src/router.rs @@ -27,7 +27,7 @@ use iggy_binary_protocol::{ConsensusHeader, GenericHeader, Operation, PrepareHea use journal::superblock::SuperblockStore; use journal::{Journal, JournalHandle}; use message_bus::{ConnectionInstaller, MessageBus, ReplicaHandshakeDoneFn}; -use server_common::sharding::{IggyNamespace, METADATA_GROUP}; +use server_common::sharding::{IggyNamespace, METADATA_CONSENSUS_NAMESPACE}; use server_common::{Message, MessageBag}; /// How often the shard pump drives `VsrConsensus::tick`. @@ -46,63 +46,63 @@ fn extract_routing(bag: MessageBag) -> (Operation, u64, Message) match bag { MessageBag::Request(r) => { let h = *r.header(); - (h.operation, h.group, r.into_generic()) + (h.operation, h.namespace, r.into_generic()) } MessageBag::Prepare(p) => { let h = *p.header(); - (h.operation, h.group, p.into_generic()) + (h.operation, h.namespace, p.into_generic()) } MessageBag::PrepareOk(p) => { let h = *p.header(); - (h.operation, h.group, p.into_generic()) + (h.operation, h.namespace, p.into_generic()) } MessageBag::StartViewChange(m) => { let h = *m.header(); - (h.operation(), h.group, m.into_generic()) + (h.operation(), h.namespace, m.into_generic()) } MessageBag::DoViewChange(m) => { let h = *m.header(); - (h.operation(), h.group, m.into_generic()) + (h.operation(), h.namespace, m.into_generic()) } MessageBag::StartView(m) => { let h = *m.header(); - (h.operation(), h.group, m.into_generic()) + (h.operation(), h.namespace, m.into_generic()) } MessageBag::Commit(m) => { let h = *m.header(); - (h.operation(), h.group, m.into_generic()) + (h.operation(), h.namespace, m.into_generic()) } MessageBag::RequestStartView(m) => { let h = *m.header(); - (h.operation(), h.group, m.into_generic()) + (h.operation(), h.namespace, m.into_generic()) } MessageBag::RequestPrepares(m) => { let h = *m.header(); - (h.operation(), h.group, m.into_generic()) + (h.operation(), h.namespace, m.into_generic()) } MessageBag::RepairPrepare(m) => { let h = *m.header(); - (h.0.operation, h.0.group, m.into_generic()) + (h.0.operation, h.0.namespace, m.into_generic()) } MessageBag::RepairRangeReply(m) => { let h = *m.header(); - (h.operation(), h.group, m.into_generic()) + (h.operation(), h.namespace, m.into_generic()) } MessageBag::RequestStateTransfer(m) => { let h = *m.header(); - (h.operation(), h.group, m.into_generic()) + (h.operation(), h.namespace, m.into_generic()) } MessageBag::StateTransferTarget(m) => { let h = *m.header(); - (h.operation(), h.group, m.into_generic()) + (h.operation(), h.namespace, m.into_generic()) } MessageBag::RequestStateChunk(m) => { let h = *m.header(); - (h.operation(), h.group, m.into_generic()) + (h.operation(), h.namespace, m.into_generic()) } MessageBag::StateChunk(m) => { let h = *m.header(); - (h.operation(), h.group, m.into_generic()) + (h.operation(), h.namespace, m.into_generic()) } } } @@ -150,7 +150,7 @@ where // consensus-op additions need a version fence (release_min / // release_max bounds on the replica plane) before this arm is // safe to hit. - tracing::warn!(shard = self.id, error = %e, "dropping unparsable consensus frame"); + tracing::warn!(shard = self.id, error = %e, "dropping message with invalid command"); return; } }; @@ -163,7 +163,7 @@ where /// The simulator's dispatch shell uses this to drive `SimClient` requests /// through the real `on_client_request` path (auth, session binding, /// consensus submit, reply) instead of the raw `dispatch` routing the - /// shell-off fast path takes. No production caller: a `-p iggy-server` + /// shell-off fast path takes. No production caller: a `-p iggy-server-ng` /// build excludes the `simulator` feature and this method. #[cfg(any(test, feature = "simulator"))] pub fn deliver_client_request(&self, client_id: u128, message: Message) { @@ -195,7 +195,7 @@ where /// frame (`StartViewChange`, `DoViewChange`, `StartView`, `Commit`) /// or a client `Register` request. The owning consensus group is /// identified by `namespace_u64`: - /// - `METADATA_GROUP` -> shard 0. + /// - `METADATA_CONSENSUS_NAMESPACE` -> shard 0. /// - packable `IggyNamespace::inner()` -> the shard owning that /// partition's consensus group. fn route_typed( @@ -242,7 +242,7 @@ where "route_typed: operation {operation:?} fell through unclassified; \ expected is_metadata / is_partition / is_vsr_reserved" ); - if namespace_u64 == METADATA_GROUP { + if namespace_u64 == METADATA_CONSENSUS_NAMESPACE { self.try_send_to_target(0, generic, operation); return; } @@ -572,7 +572,7 @@ where // Only shard 0 owns the metadata consensus group, and // `forward_metadata_submit` always addresses shard 0, so a // non-zero shard here is a routing bug. The handler (wired - // by the server) replies `None` on the carried sender if it + // by server-ng) replies `None` on the carried sender if it // cannot submit, so the awaiting peer never blocks forever. debug_assert_eq!( self.id, 0, @@ -583,7 +583,7 @@ where LifecycleFrame::ListClients { reply } => { // Every shard handles this (not shard-0-only): each replies // with the clients whose connections it homes. The handler - // (wired by the server) reads this shard's `SessionManager` + // (wired by server-ng) reads this shard's `SessionManager` // and pushes the list over `reply`. (self.on_list_clients)(reply); } @@ -594,7 +594,7 @@ where } => { // Addressed to the shard owning `namespace` (the sender // resolved it via the shards table). The handler (wired by - // the server) runs the read against this shard's partitions + // server-ng) runs the read against this shard's partitions // plane and pushes the result over `reply`; a dropped // sender means the read is skipped and the gather side // times out. diff --git a/core/shard_allocator/src/lib.rs b/core/shard_allocator/src/lib.rs index f0aadae26b..c1cf2395ae 100644 --- a/core/shard_allocator/src/lib.rs +++ b/core/shard_allocator/src/lib.rs @@ -28,7 +28,6 @@ use cpu_allocation::{CpuAllocation, NumaConfig, allowed_cpus}; use hwlocality::Topology; use hwlocality::bitmap::SpecializedBitmapRef; use hwlocality::cpu::cpuset::CpuSet; -#[cfg(target_os = "linux")] use hwlocality::memory::binding::{MemoryBindingFlags, MemoryBindingPolicy}; use hwlocality::object::types::ObjectType::{self, NUMANode}; #[cfg(target_os = "linux")] @@ -237,45 +236,36 @@ impl ShardInfo { /// Pin the calling thread's memory to this shard's NUMA node so /// allocations stay local and fast. Does nothing if no node is set. - /// On non-Linux this does nothing (no-op), mirroring [`Self::bind_cpu`]. pub fn bind_memory(&self) -> Result<(), ShardingError> { - #[cfg(target_os = "linux")] - { - if let Some(node_id) = self.numa_node { - let topology = Topology::new().map_err(|err| ShardingError::TopologyDetection { - msg: err.to_string(), + if let Some(node_id) = self.numa_node { + let topology = Topology::new().map_err(|err| ShardingError::TopologyDetection { + msg: err.to_string(), + })?; + + let node = topology + .objects_with_type(ObjectType::NUMANode) + .nth(node_id) + .ok_or(ShardingError::InvalidNode { + requested: node_id, + available: topology.objects_with_type(ObjectType::NUMANode).count(), })?; - let node = topology - .objects_with_type(ObjectType::NUMANode) - .nth(node_id) - .ok_or(ShardingError::InvalidNode { - requested: node_id, - available: topology.objects_with_type(ObjectType::NUMANode).count(), + if let Some(nodeset) = node.nodeset() { + topology + .bind_memory( + nodeset, + MemoryBindingPolicy::Bind, + MemoryBindingFlags::THREAD | MemoryBindingFlags::STRICT, + ) + .map_err(|err| { + tracing::error!("Failed to bind memory {:?}", err); + ShardingError::BindingFailed })?; - if let Some(nodeset) = node.nodeset() { - topology - .bind_memory( - nodeset, - MemoryBindingPolicy::Bind, - MemoryBindingFlags::THREAD | MemoryBindingFlags::STRICT, - ) - .map_err(|err| { - tracing::error!("Failed to bind memory {:?}", err); - ShardingError::BindingFailed - })?; - - info!("Memory bound to NUMA node {node_id}"); - } + info!("Memory bound to NUMA node {node_id}"); } } - #[cfg(not(target_os = "linux"))] - { - tracing::debug!("NUMA memory binding skipped on non-Linux platform"); - } - Ok(()) } } diff --git a/core/simulator/Cargo.toml b/core/simulator/Cargo.toml index 37df9a52e6..32a3dae471 100644 --- a/core/simulator/Cargo.toml +++ b/core/simulator/Cargo.toml @@ -43,7 +43,7 @@ rand_xoshiro = { workspace = true } secrecy = { workspace = true } # `default-features = false` drops the mimalloc global allocator and the # web-embed feature; the sim only needs the dispatch/bootstrap library. -server = { path = "../server", default-features = false } +server-ng = { path = "../server-ng", default-features = false } server_common = { path = "../server_common", features = ["simulator"] } shard = { path = "../shard", features = ["simulator"] } strum = { workspace = true } diff --git a/core/simulator/src/client.rs b/core/simulator/src/client.rs index 02ea373231..028adb38cf 100644 --- a/core/simulator/src/client.rs +++ b/core/simulator/src/client.rs @@ -43,12 +43,12 @@ use iggy_binary_protocol::requests::users::{ UpdatePermissionsRequest, UpdateUserRequest, }; use iggy_binary_protocol::{ - AckLevel, ClientVersionInfo, IGGY_PROTOCOL_VERSION, Operation, RoutedRequestHeader, WireEncode, + AckLevel, ClientVersionInfo, IGGY_PROTOCOL_VERSION, Operation, RequestHeader, WireEncode, WireIdentifier, WireName, WirePartitioning, WirePollingStrategy, }; use metadata::stm::user::{CreatePersonalAccessTokenRequest, DeletePersonalAccessTokenRequest}; use secrecy::SecretString; -use server_common::sharding::{IggyNamespace, METADATA_GROUP}; +use server_common::sharding::{IggyNamespace, METADATA_CONSENSUS_NAMESPACE}; use server_common::{Message, iobuf::Owned}; use std::cell::Cell; @@ -130,7 +130,7 @@ impl SimClient { /// offset into a disjoint range ([`PARTITION_ID_BASE`]). A partition id can /// therefore never equal a metadata id, so a delayed or duplicated partition /// reply is never misattributed to a metadata entry in the auditor's - /// `(client, request)` map (which would trip the group guard and drop a + /// `(client, request)` map (which would trip the namespace guard and drop a /// live metadata op). This holds regardless of reply duplication, not only /// while clients are one-in-flight. fn request_id_for(&self, operation: Operation) -> u64 { @@ -162,9 +162,9 @@ impl SimClient { /// # Panics /// Panics if the register request buffer is invalid. #[allow(clippy::cast_possible_truncation)] - pub fn register(&self) -> Message { - let header_size = std::mem::size_of::(); - let header = RoutedRequestHeader { + pub fn register(&self) -> Message { + let header_size = std::mem::size_of::(); + let header = RequestHeader { command: iggy_binary_protocol::Command2::Request, operation: Operation::Register, size: header_size as u32, @@ -173,8 +173,8 @@ impl SimClient { request: 0, // Register is a vsr-reserved op: the shard router picks its // target by comparing this against the metadata consensus - // group, not by op class. - group: METADATA_GROUP, + // namespace, not by op class. + namespace: METADATA_CONSENSUS_NAMESPACE, ..Default::default() }; @@ -198,7 +198,7 @@ impl SimClient { /// Panics if a credential exceeds the wire name/secret bounds or the /// request buffer is invalid. #[allow(clippy::cast_possible_truncation)] - pub fn login(&self, username: &str, password: &str) -> Message { + pub fn login(&self, username: &str, password: &str) -> Message { let body = LoginRegisterRequest { version_info: ClientVersionInfo { protocol_version: IGGY_PROTOCOL_VERSION, @@ -211,16 +211,16 @@ impl SimClient { } .to_bytes(); - let header_size = std::mem::size_of::(); + let header_size = std::mem::size_of::(); let total_size = header_size + body.len(); - let header = RoutedRequestHeader { + let header = RequestHeader { command: iggy_binary_protocol::Command2::Request, operation: Operation::Register, size: total_size as u32, client: self.client_id, session: 0, request: 0, - group: METADATA_GROUP, + namespace: METADATA_CONSENSUS_NAMESPACE, ..Default::default() }; @@ -233,7 +233,7 @@ impl SimClient { /// # Panics /// Panics if the stream name is not a valid wire name. - pub fn create_stream(&self, name: &str) -> Message { + pub fn create_stream(&self, name: &str) -> Message { let wire = CreateStreamRequest { name: WireName::new(name).expect("stream name must be valid"), }; @@ -244,7 +244,7 @@ impl SimClient { /// # Panics /// Panics if the stream name cannot be converted to a `WireIdentifier`. - pub fn delete_stream(&self, name: &str) -> Message { + pub fn delete_stream(&self, name: &str) -> Message { let wire = DeleteStreamRequest { stream_id: WireIdentifier::named(name).expect("stream name must be valid"), }; @@ -256,7 +256,7 @@ impl SimClient { /// # Panics /// Panics if the new name or the existing stream name is not a valid /// `WireName`. - pub fn update_stream(&self, stream: &str, new_name: &str) -> Message { + pub fn update_stream(&self, stream: &str, new_name: &str) -> Message { let wire = UpdateStreamRequest { stream_id: WireIdentifier::named(stream).expect("stream name must be valid"), name: WireName::new(new_name).expect("stream name must be valid"), @@ -266,7 +266,7 @@ impl SimClient { /// # Panics /// Panics if `stream` is not a valid `WireName`. - pub fn purge_stream(&self, stream: &str) -> Message { + pub fn purge_stream(&self, stream: &str) -> Message { let wire = PurgeStreamRequest { stream_id: WireIdentifier::named(stream).expect("stream name must be valid"), }; @@ -280,7 +280,7 @@ impl SimClient { stream: &str, name: &str, partitions_count: u32, - ) -> Message { + ) -> Message { let wire = CreateTopicRequest { stream_id: WireIdentifier::named(stream).expect("stream name must be valid"), partitions_count, @@ -300,7 +300,7 @@ impl SimClient { stream: &str, topic: &str, new_name: &str, - ) -> Message { + ) -> Message { let wire = UpdateTopicRequest { stream_id: WireIdentifier::named(stream).expect("stream name must be valid"), topic_id: WireIdentifier::named(topic).expect("topic name must be valid"), @@ -315,7 +315,7 @@ impl SimClient { /// # Panics /// Panics if `stream` or `topic` is not a valid `WireName`. - pub fn delete_topic(&self, stream: &str, topic: &str) -> Message { + pub fn delete_topic(&self, stream: &str, topic: &str) -> Message { let wire = DeleteTopicRequest { stream_id: WireIdentifier::named(stream).expect("stream name must be valid"), topic_id: WireIdentifier::named(topic).expect("topic name must be valid"), @@ -325,7 +325,7 @@ impl SimClient { /// # Panics /// Panics if `stream` or `topic` is not a valid `WireName`. - pub fn purge_topic(&self, stream: &str, topic: &str) -> Message { + pub fn purge_topic(&self, stream: &str, topic: &str) -> Message { let wire = PurgeTopicRequest { stream_id: WireIdentifier::named(stream).expect("stream name must be valid"), topic_id: WireIdentifier::named(topic).expect("topic name must be valid"), @@ -340,7 +340,7 @@ impl SimClient { stream: &str, topic: &str, partitions_count: u32, - ) -> Message { + ) -> Message { let wire = CreatePartitionsRequest { stream_id: WireIdentifier::named(stream).expect("stream name must be valid"), topic_id: WireIdentifier::named(topic).expect("topic name must be valid"), @@ -356,7 +356,7 @@ impl SimClient { stream: &str, topic: &str, partitions_count: u32, - ) -> Message { + ) -> Message { let wire = DeletePartitionsRequest { stream_id: WireIdentifier::named(stream).expect("stream name must be valid"), topic_id: WireIdentifier::named(topic).expect("topic name must be valid"), @@ -373,7 +373,7 @@ impl SimClient { topic: &str, partition_id: u32, segments_count: u32, - ) -> Message { + ) -> Message { let wire = DeleteSegmentsRequest { stream_id: WireIdentifier::named(stream).expect("stream name must be valid"), topic_id: WireIdentifier::named(topic).expect("topic name must be valid"), @@ -390,7 +390,7 @@ impl SimClient { stream: &str, topic: &str, name: &str, - ) -> Message { + ) -> Message { let wire = CreateConsumerGroupRequest { stream_id: WireIdentifier::named(stream).expect("stream name must be valid"), topic_id: WireIdentifier::named(topic).expect("topic name must be valid"), @@ -406,7 +406,7 @@ impl SimClient { stream: &str, topic: &str, group: &str, - ) -> Message { + ) -> Message { let wire = DeleteConsumerGroupRequest { stream_id: WireIdentifier::named(stream).expect("stream name must be valid"), topic_id: WireIdentifier::named(topic).expect("topic name must be valid"), @@ -422,7 +422,7 @@ impl SimClient { username: &str, password: &str, status: u8, - ) -> Message { + ) -> Message { let wire = CreateUserRequest { username: WireName::new(username).expect("username must be valid"), password: password.to_string(), @@ -440,7 +440,7 @@ impl SimClient { user: &str, new_username: Option<&str>, status: Option, - ) -> Message { + ) -> Message { let wire = UpdateUserRequest { user_id: WireIdentifier::named(user).expect("username must be valid"), username: new_username.map(|n| WireName::new(n).expect("username must be valid")), @@ -451,7 +451,7 @@ impl SimClient { /// # Panics /// Panics if `user` is not a valid `WireName`. - pub fn delete_user(&self, user: &str) -> Message { + pub fn delete_user(&self, user: &str) -> Message { let wire = DeleteUserRequest { user_id: WireIdentifier::named(user).expect("username must be valid"), }; @@ -465,7 +465,7 @@ impl SimClient { user: &str, current_password: &str, new_password: &str, - ) -> Message { + ) -> Message { let wire = ChangePasswordRequest { user_id: WireIdentifier::named(user).expect("username must be valid"), current_password: current_password.to_string(), @@ -476,7 +476,7 @@ impl SimClient { /// # Panics /// Panics if `user` is not a valid `WireName`. - pub fn update_permissions(&self, user: &str) -> Message { + pub fn update_permissions(&self, user: &str) -> Message { let wire = UpdatePermissionsRequest { user_id: WireIdentifier::named(user).expect("username must be valid"), permissions: None, @@ -486,11 +486,7 @@ impl SimClient { /// # Panics /// Panics if `name` is not a valid `WireName`. - pub fn create_personal_access_token( - &self, - name: &str, - expiry: u64, - ) -> Message { + pub fn create_personal_access_token(&self, name: &str, expiry: u64) -> Message { let wire = CreatePersonalAccessTokenRequest { user_id: 0, name: WireName::new(name).expect("PAT name must be valid"), @@ -506,7 +502,7 @@ impl SimClient { /// # Panics /// Panics if `name` is not a valid `WireName`. - pub fn delete_personal_access_token(&self, name: &str) -> Message { + pub fn delete_personal_access_token(&self, name: &str) -> Message { let wire = DeletePersonalAccessTokenRequest { user_id: 0, name: WireName::new(name).expect("PAT name must be valid"), @@ -523,16 +519,16 @@ impl SimClient { /// path converts it to `SendMessages2` via `transcode_legacy_request`. /// /// # Panics - /// Panics if a group id exceeds `u32` or the request buffer is invalid. + /// Panics if a namespace id exceeds `u32` or the request buffer is invalid. pub fn send_messages( &self, - group: IggyNamespace, + namespace: IggyNamespace, messages: &[Bytes], - ) -> Message { - let to_u32 = |v: usize| u32::try_from(v).expect("group id fits u32"); - let stream_id = WireIdentifier::Numeric(to_u32(group.stream_id())); - let topic_id = WireIdentifier::Numeric(to_u32(group.topic_id())); - let partitioning = WirePartitioning::PartitionId(to_u32(group.partition_id())); + ) -> Message { + let to_u32 = |v: usize| u32::try_from(v).expect("namespace id fits u32"); + let stream_id = WireIdentifier::Numeric(to_u32(namespace.stream_id())); + let topic_id = WireIdentifier::Numeric(to_u32(namespace.topic_id())); + let partitioning = WirePartitioning::PartitionId(to_u32(namespace.partition_id())); // Stamp a deterministic, non-zero id per message. `id: 0` (the real // SDK's server-assigned path) would make the server mint an unseeded @@ -553,11 +549,11 @@ impl SimClient { let mut buf = BytesMut::with_capacity(size); SendMessagesEncoder::encode(&mut buf, &stream_id, &topic_id, &partitioning, &raw); - self.build_request_with_namespace(Operation::SendMessages, &buf, group) + self.build_request_with_namespace(Operation::SendMessages, &buf, namespace) } /// Build a `POLL_MESSAGES` request for an individual consumer, reading - /// `count` messages from offset 0 of `group`'s partition. + /// `count` messages from offset 0 of `namespace`'s partition. /// /// A `NonReplicated` read: the command code sits in the header's /// `reserved` prefix, and the request id ECHOES the current metadata @@ -568,8 +564,8 @@ impl SimClient { /// # Panics /// Panics if the session is unbound or the request buffer is invalid. #[allow(clippy::cast_possible_truncation)] - pub fn poll_messages(&self, group: IggyNamespace, count: u32) -> Message { - let (stream_id, topic_id, partition_id) = namespace_ids(group); + pub fn poll_messages(&self, namespace: IggyNamespace, count: u32) -> Message { + let (stream_id, topic_id, partition_id) = namespace_ids(namespace); let body = PollMessagesRequest { consumer: WireConsumer::consumer(WireIdentifier::Numeric(self.client_id as u32)), stream_id, @@ -581,11 +577,11 @@ impl SimClient { } .to_bytes(); - let header_size = std::mem::size_of::(); + let header_size = std::mem::size_of::(); let total_size = header_size + body.len(); let mut reserved = [0u8; 52]; reserved[..4].copy_from_slice(&POLL_MESSAGES_CODE.to_le_bytes()); - let header = RoutedRequestHeader { + let header = RequestHeader { command: iggy_binary_protocol::Command2::Request, operation: Operation::NonReplicated, size: total_size as u32, @@ -593,7 +589,7 @@ impl SimClient { session: self.session_id(), request: self.request_counter.get(), reserved, - group: group.inner(), + namespace: namespace.inner(), ..Default::default() }; @@ -606,12 +602,12 @@ impl SimClient { pub fn store_consumer_offset( &self, - group: IggyNamespace, + namespace: IggyNamespace, consumer_kind: u8, consumer_id: u32, offset: u64, - ) -> Message { - let (stream_id, topic_id, partition_id) = namespace_ids(group); + ) -> Message { + let (stream_id, topic_id, partition_id) = namespace_ids(namespace); let request = StoreConsumerOffsetRequest { consumer: namespace_consumer(consumer_kind, consumer_id), stream_id, @@ -622,17 +618,17 @@ impl SimClient { self.build_request_with_namespace( Operation::StoreConsumerOffset, &request.to_bytes(), - group, + namespace, ) } pub fn delete_consumer_offset( &self, - group: IggyNamespace, + namespace: IggyNamespace, consumer_kind: u8, consumer_id: u32, - ) -> Message { - let (stream_id, topic_id, partition_id) = namespace_ids(group); + ) -> Message { + let (stream_id, topic_id, partition_id) = namespace_ids(namespace); let request = DeleteConsumerOffsetRequest { consumer: namespace_consumer(consumer_kind, consumer_id), stream_id, @@ -642,7 +638,7 @@ impl SimClient { self.build_request_with_namespace( Operation::DeleteConsumerOffset, &request.to_bytes(), - group, + namespace, ) } @@ -651,16 +647,16 @@ impl SimClient { /// /// # Panics /// Panics on payload too large for `Owned::<4096>` or invalid - /// `Message` parse; both are simulator misconfig. + /// `Message` parse; both are simulator misconfig. pub fn store_consumer_offset_2( &self, - group: IggyNamespace, + namespace: IggyNamespace, consumer_kind: u8, consumer_id: u32, offset: u64, ack: AckLevel, - ) -> Message { - let (stream_id, topic_id, partition_id) = namespace_ids(group); + ) -> Message { + let (stream_id, topic_id, partition_id) = namespace_ids(namespace); let request = StoreConsumerOffset2Request { consumer: namespace_consumer(consumer_kind, consumer_id), stream_id, @@ -672,7 +668,7 @@ impl SimClient { self.build_request_with_namespace( Operation::StoreConsumerOffset2, &request.to_bytes(), - group, + namespace, ) } @@ -680,15 +676,15 @@ impl SimClient { /// /// # Panics /// Panics on payload too large for `Owned::<4096>` or invalid - /// `Message` parse; both are simulator misconfig. + /// `Message` parse; both are simulator misconfig. pub fn delete_consumer_offset_2( &self, - group: IggyNamespace, + namespace: IggyNamespace, consumer_kind: u8, consumer_id: u32, ack: AckLevel, - ) -> Message { - let (stream_id, topic_id, partition_id) = namespace_ids(group); + ) -> Message { + let (stream_id, topic_id, partition_id) = namespace_ids(namespace); let request = DeleteConsumerOffset2Request { consumer: namespace_consumer(consumer_kind, consumer_id), stream_id, @@ -699,7 +695,7 @@ impl SimClient { self.build_request_with_namespace( Operation::DeleteConsumerOffset2, &request.to_bytes(), - group, + namespace, ) } @@ -707,12 +703,12 @@ impl SimClient { &self, operation: Operation, payload: &[u8], - group: IggyNamespace, - ) -> Message { - let header_size = std::mem::size_of::(); + namespace: IggyNamespace, + ) -> Message { + let header_size = std::mem::size_of::(); let total_size = header_size + payload.len(); - let header = self.header(operation, group.inner(), total_size); + let header = self.header(operation, namespace.inner(), total_size); let header_bytes = bytemuck::bytes_of(&header); let mut buffer = Vec::with_capacity(total_size); @@ -723,14 +719,14 @@ impl SimClient { .expect("request buffer must contain a valid request message") } - fn build_request(&self, operation: Operation, payload: &[u8]) -> Message { - let header_size = std::mem::size_of::(); + fn build_request(&self, operation: Operation, payload: &[u8]) -> Message { + let header_size = std::mem::size_of::(); let total_size = header_size + payload.len(); // Every `build_request` caller is a metadata-plane op (partition // ops go through `build_request_with_namespace`), and metadata - // requests carry the metadata consensus group on the wire. - let header = self.header(operation, METADATA_GROUP, total_size); + // requests carry the metadata consensus namespace on the wire. + let header = self.header(operation, METADATA_CONSENSUS_NAMESPACE, total_size); let header_bytes = bytemuck::bytes_of(&header); let mut buffer = Vec::with_capacity(total_size); @@ -742,8 +738,8 @@ impl SimClient { } #[allow(clippy::cast_possible_truncation)] - fn header(&self, operation: Operation, group: u64, total_size: usize) -> RoutedRequestHeader { - RoutedRequestHeader { + fn header(&self, operation: Operation, namespace: u64, total_size: usize) -> RequestHeader { + RequestHeader { command: iggy_binary_protocol::Command2::Request, operation, size: total_size as u32, @@ -759,7 +755,7 @@ impl SimClient { timestamp: 0, // TODO: Use actual timestamp session: self.session_id(), request: self.request_id_for(operation), - group, + namespace, ..Default::default() } } @@ -776,11 +772,11 @@ const fn namespace_consumer(kind: u8, consumer_id: u32) -> WireConsumer { } } -/// Decompose a group into the `(stream_id, topic_id, partition_id)` wire +/// Decompose a namespace into the `(stream_id, topic_id, partition_id)` wire /// identifiers the consumer-offset requests carry. Namespace ids are small /// test values that always fit `u32`. fn namespace_ids(ns: IggyNamespace) -> (WireIdentifier, WireIdentifier, Option) { - let to_u32 = |v: usize| u32::try_from(v).expect("group id fits u32"); + let to_u32 = |v: usize| u32::try_from(v).expect("namespace id fits u32"); ( WireIdentifier::Numeric(to_u32(ns.stream_id())), WireIdentifier::Numeric(to_u32(ns.topic_id())), diff --git a/core/simulator/src/deps.rs b/core/simulator/src/deps.rs index 2502b8e9b5..de7c2240ed 100644 --- a/core/simulator/src/deps.rs +++ b/core/simulator/src/deps.rs @@ -176,37 +176,6 @@ impl>> Journal for SimJournal { where Self: 'a; - fn last_op(&self) -> Option { - self.last_op.get() - } - - /// Drop the suffix, so a simulated backup whose entries disagree with a started - /// view reconciles the way a real one does. Mirrors - /// `PrepareJournal::truncate_from`, whose watermark stays put; here it never moves. - async fn truncate_from(&self, from_op: u64) -> std::io::Result { - if from_op == 0 { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidInput, - "truncate_from: ops are 1-based, so 0 would discard the whole journal", - )); - } - #[cfg(debug_assertions)] - let _guard = JournalAccessGuard::new(&self.accessing); - let headers = unsafe { &mut *self.headers.get() }; - let offsets = unsafe { &mut *self.offsets.get() }; - let doomed: Vec = headers - .keys() - .copied() - .filter(|op| *op >= from_op) - .collect(); - for op in &doomed { - headers.remove(op); - offsets.remove(op); - } - self.last_op.set(headers.keys().copied().max()); - Ok(doomed.len()) - } - /// The simulated journal retains everything for the run, so nothing is /// ever superseded by a snapshot. Answered explicitly (the trait has no /// default) so a simulated state transfer has to opt into a watermark @@ -299,24 +268,6 @@ impl SimJournal { self.last_op.get() } - /// Forget one op, leaving a hole exactly where a lost prepare would. - /// - /// Tests only. The alternative is choreographing `Prepare`, `Commit` and - /// `RepairPrepare` drops on a directed link until a replica falls behind, which - /// is fragile to tune; the scenarios are about what a replica does with a hole, - /// not how it got one. - /// - /// `last_op` is deliberately left alone: a hole below the head must not look like - /// a shorter log, since that is the state a view change has to survive. - pub fn forget_op(&self, op: u64) -> bool { - #[cfg(debug_assertions)] - let _guard = JournalAccessGuard::new(&self.accessing); - let headers = unsafe { &mut *self.headers.get() }; - let offsets = unsafe { &mut *self.offsets.get() }; - offsets.remove(&op); - headers.remove(&op).is_some() - } - /// The committed watermark to restore after a restart, mirroring /// `metadata::recover`. On a solo cluster every appended op commits the instant /// it is durable, so the head IS the commit point; otherwise the highest diff --git a/core/simulator/src/lib.rs b/core/simulator/src/lib.rs index d4c2ba0c79..64aecc440a 100644 --- a/core/simulator/src/lib.rs +++ b/core/simulator/src/lib.rs @@ -164,7 +164,7 @@ impl Simulator { } /// [`Simulator::new`] with `shards_per_replica` shards on every replica, - /// meshed exactly like the server bootstrap: metadata plane on shard 0, + /// meshed exactly like server-ng bootstrap: metadata plane on shard 0, /// partitions hash-assigned, one pump task per shard. /// /// # Panics @@ -186,7 +186,7 @@ impl Simulator { } /// [`Simulator::with_shards`] with the deterministic dispatch shell on: - /// every shard wires the server's real dispatch handlers, so a client + /// every shard wires server-ng's real dispatch handlers, so a client /// request runs as a task the seeded executor interleaves with the /// pump. Off (the default) keeps the raw-`on_message` fast path. /// @@ -276,7 +276,7 @@ impl Simulator { let mut shards = Vec::with_capacity(usize::from(shards_per_replica)); let mut stop_txs = Vec::with_capacity(usize::from(shards_per_replica)); let mut pump_tasks = Vec::with_capacity(usize::from(shards_per_replica)); - // Single-writer metadata (mirrors the server bootstrap): shard 0 + // Single-writer metadata (mirrors server-ng bootstrap): shard 0 // builds the writable STM and mints a factory bundle; every peer // shard rebuilds a reader-mode mirror from it and sees committed // metadata through the shared read handle. Shards are built in index @@ -317,7 +317,7 @@ impl Simulator { ); } - // Same wiring as the server bootstrap: one pump task per + // Same wiring as server-ng bootstrap: one pump task per // shard, stopped only by the (held) stop channel or a // crash abort. let (stop_tx, stop_rx) = shard::channel::<()>(1); @@ -743,7 +743,7 @@ impl Simulator { let metadata_incarnation = self.replicas[idx].metadata_incarnation + 1; // Partition superblocks carry forward too: a group re-materialised after // the restart must recover its recorded view from the same store, exactly - // as a rebooted server partition reads the record in its directory. + // as a rebooted server-ng partition reads the record in its directory. let partition_superblocks = std::mem::take(&mut *self.replicas[idx].partition_superblocks.borrow_mut()); @@ -814,7 +814,7 @@ impl Simulator { }; // Re-materialise every group this replica had before the crash, as a - // rebooted the server re-opens every partition directory it owns. This + // rebooted server-ng re-opens every partition directory it owns. This // is what makes the carried-forward superblock load-bearing: the group // recovers the `(view, log_view)` it recorded instead of re-entering // view 0. @@ -945,7 +945,7 @@ impl Simulator { /// the routing row on every shard of that replica. /// /// Shared by [`SimCluster::init_partition`] and the restart path: a rebooted -/// server re-opens every partition directory it owns, so the sim has to +/// server-ng re-opens every partition directory it owns, so the sim has to /// re-materialise too, otherwise the superblock a restart carries forward is /// never read back and the recovered-view branch is dead code. fn materialise_partition(replica: &SimReplica, namespace: IggyNamespace) { @@ -1667,16 +1667,13 @@ mod tests { // shifts the trace. Re-lock on intentional changes; expect re-locks until // error discriminants and reply bodies stabilize the wire format. // - // Re-locked when the sim adopted METADATA_GROUP (1<<63) + // Re-locked when the sim adopted METADATA_CONSENSUS_NAMESPACE (1<<63) // for metadata requests and the metadata consensus group, replacing // the sim-only 0: reply headers and the per-group timeout-jitter seed // (replica_id ^ namespace) both changed. The old 0 only ever routed // correctly because `hash % 1 == 0` at one shard per replica. - // Re-locked again when replies stopped echoing a group id (the - // client wire lost its namespace field): the reply-hash tuple - // dropped that component. assert_eq!( - h1, 0xCF1F_BC79_B44A_65F7, + h1, 0x530D_499C_5DBE_A2BE, "workload reply hash drifted from locked baseline" ); } @@ -2322,7 +2319,15 @@ mod tests { } for reply in sim.step() { let h = reply.header(); - (h.client, h.request, h.op, h.commit, h.operation as u8).hash(&mut hasher); + ( + h.client, + h.request, + h.op, + h.commit, + h.namespace, + h.operation as u8, + ) + .hash(&mut hasher); let cmds = wl.on_reply(&reply); apply_sim_commands(&mut sim, &cmds); replies_seen += 1; @@ -2613,7 +2618,7 @@ mod tests { sim.schedule_hash() } - /// Turning the dispatch shell on wires the server's real deferred + /// Turning the dispatch shell on wires server-ng's real deferred /// handlers on every shard. With no client traffic none of them is /// reached, so the consensus plane both replays deterministically and /// matches the shell-off schedule: the toggle is genuinely off the @@ -2946,7 +2951,7 @@ mod tests { let header: &PrepareOkHeader = bytemuck::checked::from_bytes( &packet.message.as_slice()[..std::mem::size_of::()], ); - header.group == BLOCKED_NS.load(Ordering::Relaxed) + header.namespace == BLOCKED_NS.load(Ordering::Relaxed) } server_common::MemoryPool::init_pool(&server_common::MemoryPoolConfigOther { @@ -3002,16 +3007,13 @@ mod tests { // ns_b shares the client, the replicas, and the shard, but has its // own consensus group: it must commit while ns_a stays wedged. let msg = client.send_messages(ns_b, &[Bytes::from_static(b"independent")]); - // Replies no longer carry a group id; correlate by the request id - // this send was stamped with. - let ns_b_request = msg.header().request; sim.submit_request(client_id, 0, msg.into_generic()); let mut independent_replies = 0usize; for _ in 0..100 { for reply in sim.step() { assert_eq!( - reply.header().request, - ns_b_request, + reply.header().namespace, + ns_b.inner(), "only ns_b may commit while ns_a's acks are blocked" ); independent_replies += 1; @@ -3030,9 +3032,7 @@ mod tests { let mut drained_replies = 0usize; for _ in 0..800 { for reply in sim.step() { - // Only ns_a replies are outstanding once ns_b committed - // above, so every reply counts toward the drain. - if reply.header().request != ns_b_request { + if reply.header().namespace == ns_a.inner() { drained_replies += 1; } } @@ -3097,162 +3097,3 @@ mod tests { assert_no_frame_drops(&sim); } } - -#[cfg(test)] -mod view_change_data_loss_tests { - //! A committed, client-acknowledged op must survive a view change even when - //! the replica that becomes primary is the one missing it. - //! - //! Without the sender's log suffix on the `DoViewChange`, the new primary adopts - //! the winner's op NUMBER, rebuilds its pipeline from its OWN journal, hits the - //! hole, and truncates the range as "decided lost" -- discarding an op journaled - //! on a quorum and already replied to. The next client op then reuses the number - //! and collides with the stale entry on the up-to-date backup. - //! - //! The hole here is punched at the commit point, so the assertion that catches a - //! regression is "the op came back", not "the head did not regress": with nothing - //! uncommitted there is no pipeline rebuild to truncate. The `dvc_merge` unit - //! tests cover the sequencer-truncation path directly. - - use super::*; - use consensus::{Sequencer, Status}; - use journal::Journal; - - /// Whether a replica's shard-0 metadata consensus is a settled primary in a - /// view past the one that crashed. - fn is_new_metadata_primary(sim: &Simulator, replica: u8) -> bool { - sim.replicas[replica as usize].shards[0] - .plane - .metadata() - .consensus - .as_ref() - .is_some_and(|consensus| { - consensus.view() > 0 - && consensus.status() == Status::Normal - && consensus.is_primary() - }) - } - - /// `(head op, commit_max)` of a replica's shard-0 metadata consensus. - fn metadata_progress(sim: &Simulator, replica: u8) -> (u64, u64) { - let consensus = sim.replicas[replica as usize].shards[0] - .plane - .metadata() - .consensus - .as_ref() - .expect("shard 0 owns metadata consensus"); - ( - consensus.sequencer().current_sequence(), - consensus.commit_max(), - ) - } - - /// Whether a replica's metadata journal holds `op`. - fn metadata_holds(sim: &Simulator, replica: u8, op: u64) -> bool { - let journal = sim.replicas[replica as usize].shards[0] - .plane - .metadata() - .journal - .as_ref() - .expect("shard 0 owns the metadata journal"); - let slot = usize::try_from(op).expect("op fits usize"); - Journal::header(journal.as_ref(), slot).is_some() - } - - /// Drop `op` from a replica's metadata journal, leaving a hole. - fn metadata_forget(sim: &Simulator, replica: u8, op: u64) -> bool { - sim.replicas[replica as usize].shards[0] - .plane - .metadata() - .journal - .as_ref() - .expect("shard 0 owns the metadata journal") - .forget_op(op) - } - - #[test] - fn given_committed_op_missing_on_next_primary_when_primary_crashes_should_survive_view_change() - { - server_common::MemoryPool::init_pool(&server_common::MemoryPoolConfigOther { - enabled: false, - size: iggy_common::IggyByteSize::from(0u64), - bucket_capacity: 1, - }); - - let replica_count: u8 = 3; - let client_id: u128 = 1; - let network_opts = packet::PacketSimulatorOptions { - node_count: replica_count, - client_count: 1, - ..packet::PacketSimulatorOptions::default() - }; - let mut sim = Simulator::new( - replica_count as usize, - std::iter::once(client_id), - network_opts, - ); - let client = SimClient::new(client_id); - - // Commit some metadata ops so there is a log to lose. Registering binds - // a session, and seeding a stream/topic/partition commits several more. - sim.register_client_with_primary(&client); - sim.seed_stream_topic_partition(IggyNamespace::new(1, 1, 0)); - for _ in 0..200 { - sim.step(); - } - - // Replica 0 is primary for view 0, so replica 1 is primary-elect for view 1 - // (view % replica_count): the replica whose hole decides the outcome. - let next_primary: u8 = 1; - let (_, committed) = metadata_progress(&sim, next_primary); - assert!( - committed > 0, - "the test needs committed metadata ops to be able to lose one" - ); - - // Every replica must hold the op: the point is that it IS recoverable, and - // only the incoming primary lacks it. - for replica in 0..replica_count { - assert!( - metadata_holds(&sim, replica, committed), - "replica {replica} must hold op {committed} before the hole is punched" - ); - } - - // Punch the hole: the incoming primary forgets an op its peers still hold. - assert!( - metadata_forget(&sim, next_primary, committed), - "op {committed} must have been present to forget" - ); - - sim.replica_crash(0); - for _ in 0..1500 { - sim.step(); - } - - // A primary must emerge among the survivors. - let primary = (1..replica_count) - .find(|&replica| is_new_metadata_primary(&sim, replica)) - .expect("a metadata primary must be elected after the old one crashes"); - - let (head, commit_max) = metadata_progress(&sim, primary); - - // The committed op must not have been discarded. - assert!( - head >= committed, - "the new primary's head ({head}) regressed below the committed op ({committed}); \ - a committed, acknowledged op was discarded by the view change" - ); - assert!( - commit_max >= committed, - "commit_max ({commit_max}) regressed below the committed op ({committed})" - ); - - // And back in the new primary's journal: the view change repaired the hole - // from a peer that offered the body, rather than declaring the op lost. - assert!( - metadata_holds(&sim, primary, committed), - "op {committed} must be repaired back into the new primary's journal" - ); - } -} diff --git a/core/simulator/src/replica.rs b/core/simulator/src/replica.rs index 2764b96da6..9abf73c325 100644 --- a/core/simulator/src/replica.rs +++ b/core/simulator/src/replica.rs @@ -19,7 +19,7 @@ use crate::bus::{SharedSimOutbox, SimOutbox}; use crate::deps::SimSuperblock; use crate::deps::{MemStorage, SimJournal, SimMuxStateMachine, SimSnapshot}; use configs::server::PersonalAccessTokenConfig; -use configs::server::ServerSystemConfig; +use configs::server_ng::NgSystemConfig; use consensus::{ConsensusClock, LocalPipeline, Sequencer, VsrConsensus, VsrState}; use iggy_common::IggyByteSize; use iggy_common::variadic; @@ -28,9 +28,9 @@ use metadata::stm::stream::{Streams, StreamsInner}; use metadata::stm::user::{Users, UsersInner}; use metadata::{IggyMetadata, apply_committed_prepare}; use partitions::{IggyPartitions, PartitionsConfig}; -use server::bootstrap::{ShellHandlers, ShellShardHandle, wire_shell_handlers}; use server_common::crypto; -use server_common::sharding::{METADATA_GROUP, ShardId}; +use server_common::sharding::{METADATA_CONSENSUS_NAMESPACE, ShardId}; +use server_ng::bootstrap::{ShellHandlers, ShellShardHandle, wire_shell_handlers}; use shard::shards_table::PapayaShardsTable; use std::cell::RefCell; use std::rc::Rc; @@ -52,7 +52,7 @@ pub const SHELL_ROOT_PASSWORD: &str = "iggy"; // // `PapayaShardsTable` (the production namespace -> shard routing table) // instead of the always-`None` `()` impl: each shard owns its own table -// instance, exactly as the server wires it. Until rows are seeded the +// instance, exactly as server-ng wires it. Until rows are seeded the // router falls back to the deterministic hash assignment, which at one // shard per replica always resolves to shard 0. pub type Replica = shard::IggyShard< @@ -70,7 +70,7 @@ pub type Replica = shard::IggyShard< /// /// Shard 0 (the sole writer) mints one via `factory_bundle`; every peer shard /// rebuilds a reader-mode mirror from it with `from_factory_bundle`, exactly as -/// the server bootstrap does with its `ServerMetadataBundle`. +/// server-ng bootstrap does with its `ServerNgMetadataBundle`. /// `Clone + Send + Sync`. pub type SimMetadataBundle = ::Bundle; @@ -90,11 +90,11 @@ pub const SIM_INBOX_CAPACITY: usize = 8192; /// /// `shell` selects the dispatch handlers. Off is the fast path: inert /// no-ops, so the simulator drives raw client frames straight into -/// `IggyShard::on_message`. On wires the server's real deferred dispatch +/// `IggyShard::on_message`. On wires server-ng's real deferred dispatch /// handlers (via [`wire_shell_handlers`]), exactly as production does, so /// a client request runs as a task concurrent with the pump. /// -/// Mirrors the server bootstrap's single-writer metadata: the consensus +/// Mirrors server-ng bootstrap's single-writer metadata: the consensus /// group, journal, snapshot, and the only writable metadata STM live on /// shard 0. Shard 0 mints a [`SimMetadataBundle`] (returned as the second /// tuple element); every peer shard passes it back in as `reader_bundle` @@ -126,7 +126,7 @@ pub fn new_shard( recovered_state: Option, incarnation: u128, ) -> (Rc, Option) { - // Metadata is single-writer, mirroring the server bootstrap. Shard 0 owns + // Metadata is single-writer, mirroring server-ng bootstrap. Shard 0 owns // the only writable STM; every peer shard rebuilds a reader-mode mirror from // shard 0's factory bundle and sees committed metadata through the shared // left-right read handle (each apply `publish`es, bounding reader staleness @@ -182,7 +182,7 @@ pub fn new_shard( CLUSTER_ID, replica_id, replica_count, - METADATA_GROUP, + METADATA_CONSENSUS_NAMESPACE, SharedSimOutbox(Rc::clone(bus)), LocalPipeline::new(), clock.clone(), @@ -193,7 +193,7 @@ pub fn new_shard( consensus.set_incarnation(incarnation); // View/log_view come from the durable superblock; op, commit, and the // last-prepare markers come from the retained WAL. Independent inputs, - // mirroring the server's restore_metadata_consensus. + // mirroring server-ng's restore_metadata_consensus. let last_header = metadata_journal .as_ref() .and_then(|journal| journal.last_header()); @@ -283,9 +283,7 @@ pub fn new_shard( messages_required_to_save: 1000, size_of_messages_required_to_save: IggyByteSize::from(4 * 1024 * 1024), enforce_fsync: false, //Disable fsync for simulation - validate_checksum: true, segment_size: IggyByteSize::from(1024 * 1024 * 1024), - preallocate_segments: false, encryptor: None, }; @@ -316,7 +314,7 @@ pub fn new_shard( wire_shell_handlers( &SharedSimOutbox(Rc::clone(bus)), &shard_handle, - Arc::new(ServerSystemConfig::default()), + Arc::new(NgSystemConfig::default()), // Default-config PAT cap, like the system config above, so sim // ingress admits exactly what a default-configured server does. PersonalAccessTokenConfig::default().max_tokens_per_user, diff --git a/core/simulator/src/workload/auditor.rs b/core/simulator/src/workload/auditor.rs index d9616ec4ec..2cfe2c28db 100644 --- a/core/simulator/src/workload/auditor.rs +++ b/core/simulator/src/workload/auditor.rs @@ -122,7 +122,7 @@ impl ServerAuditor { /// classifies, applies effects, decrements the counter. /// - [`OnReply::NsMismatch`]: entry consumed but reply namespace /// diverged from the request namespace. Caller decrements but - /// skips effects + `note_committed`. Unreachable today (the server + /// skips effects + `note_committed`. Unreachable today (server-ng /// echoes the request namespace); guards future routing/dedup /// bugs from wedging a client at `CLIENT_REQUEST_QUEUE_MAX = 1`. /// - [`OnReply::Unknown`]: no matching entry (duplicate cached @@ -148,11 +148,16 @@ impl ServerAuditor { return OnReply::Unknown; }; - // Replies no longer echo a group id (the client wire has no - // namespace field at all), so correlation rests entirely on the - // (client, request) key that fetched `entry`; the group the request - // was submitted to comes from the sim's own bookkeeping. - let ns_key = (header.client, entry.request_namespace); + // Reply's namespace must match the namespace the request was + // submitted to. A mismatch means the reply landed in the wrong + // VSR group's bookkeeping; refuse to apply effects against the + // wrong shadow bucket. Entry already consumed. + if entry.request_namespace != header.namespace { + self.stats.replies_unknown += 1; + return OnReply::NsMismatch; + } + + let ns_key = (header.client, header.namespace); let last_commit = self .last_commit_watermark_per_client_ns .entry(ns_key) diff --git a/core/simulator/src/workload/effect.rs b/core/simulator/src/workload/effect.rs index a9d8c868c2..f9adb7edd1 100644 --- a/core/simulator/src/workload/effect.rs +++ b/core/simulator/src/workload/effect.rs @@ -42,16 +42,6 @@ pub enum Effect { stream: String, name: String, }, - AddPartitions { - stream: String, - topic: String, - count: u32, - }, - RemovePartitions { - stream: String, - topic: String, - count: u32, - }, AddUser { name: String, }, diff --git a/core/simulator/src/workload/mod.rs b/core/simulator/src/workload/mod.rs index 2e641066cb..045e946fa3 100644 --- a/core/simulator/src/workload/mod.rs +++ b/core/simulator/src/workload/mod.rs @@ -40,7 +40,7 @@ use crate::workload::ops::InFlight; use actions::Action; use auditor::{OnReply, ServerAuditor}; use effect::SimCommand; -use iggy_binary_protocol::{ReplyHeader, RoutedRequestHeader, result_code}; +use iggy_binary_protocol::{ReplyHeader, RequestHeader, result_code}; use invariants::Invariants; use metadata::stm::result::result_code_recognized; use options::WorkloadOptions; @@ -138,10 +138,7 @@ impl Workload { /// PRNG before `sample` runs, so they advance the trace even when `sample` /// returns `None` (a targeted outcome whose precondition is unmet, e.g. a /// duplicate-name target with an empty shadow). `samples_none` counts these. - pub fn build_request( - &mut self, - client: &SimClient, - ) -> Option<(u8, Message)> { + pub fn build_request(&mut self, client: &SimClient) -> Option<(u8, Message)> { if !self.client_idle(client.client_id()) { return None; } @@ -170,7 +167,7 @@ impl Workload { action, input, outcome, - request_namespace: header.group, + request_namespace: header.namespace, }, ); *self diff --git a/core/simulator/src/workload/ops/change_password.rs b/core/simulator/src/workload/ops/change_password.rs index 59668c92c9..edd2e0cf90 100644 --- a/core/simulator/src/workload/ops/change_password.rs +++ b/core/simulator/src/workload/ops/change_password.rs @@ -18,7 +18,7 @@ //! `ChangePassword` op. Targets `Ok` (rotate live user's password) or //! `UserNotFound` (fabricated user). -use iggy_binary_protocol::RoutedRequestHeader; +use iggy_binary_protocol::RequestHeader; use rand::RngExt; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -68,7 +68,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.change_password(&input.user, &input.current_password, &input.new_password) } diff --git a/core/simulator/src/workload/ops/create_consumer_group.rs b/core/simulator/src/workload/ops/create_consumer_group.rs index df1bc5c639..66113e65b9 100644 --- a/core/simulator/src/workload/ops/create_consumer_group.rs +++ b/core/simulator/src/workload/ops/create_consumer_group.rs @@ -21,7 +21,7 @@ //! (a fabricated parent stream), `TopicNotFound` (a live stream with a //! fabricated topic), or `NameAlreadyExists` (an existing group name). -use iggy_binary_protocol::RoutedRequestHeader; +use iggy_binary_protocol::RequestHeader; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -87,7 +87,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.create_consumer_group(&input.stream, &input.topic, &input.name) } diff --git a/core/simulator/src/workload/ops/create_partitions.rs b/core/simulator/src/workload/ops/create_partitions.rs index 2fdc6a2c4f..4ee8eb3004 100644 --- a/core/simulator/src/workload/ops/create_partitions.rs +++ b/core/simulator/src/workload/ops/create_partitions.rs @@ -19,11 +19,10 @@ //! //! Targets `Ok` (live topic), `StreamNotFound` (fabricated parent stream), or //! `TopicNotFound` (live stream, fabricated topic). `InvalidPartitionsCount` -//! not targeted (only reachable through partition-id overflow). A committed -//! `Ok` grows the shadow's per-topic partition count, which -//! `delete_partitions` samples against. +//! not targeted. Shadow tracks no partition counts, so every outcome predicts +//! `Effect::None`. -use iggy_binary_protocol::RoutedRequestHeader; +use iggy_binary_protocol::RequestHeader; use rand::RngExt; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -87,7 +86,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.create_partitions(&input.stream, &input.topic, input.partitions_count) } @@ -101,13 +100,7 @@ pub const fn classify_reply(code: u32) -> Outcome { } #[must_use] -pub fn predicted_effect(input: &Input, outcome: Outcome) -> Effect { - match outcome { - Outcome::Ok => Effect::AddPartitions { - stream: input.stream.clone(), - topic: input.topic.clone(), - count: input.partitions_count, - }, - _ => Effect::None, - } +pub const fn predicted_effect(_input: &Input, _outcome: Outcome) -> Effect { + // Shadow tracks no per-topic partition counts. + Effect::None } diff --git a/core/simulator/src/workload/ops/create_personal_access_token.rs b/core/simulator/src/workload/ops/create_personal_access_token.rs index 561b515343..da01cbc48c 100644 --- a/core/simulator/src/workload/ops/create_personal_access_token.rs +++ b/core/simulator/src/workload/ops/create_personal_access_token.rs @@ -19,7 +19,7 @@ //! (a live token name). `InvalidExpiry` is not targeted; tokens never expire //! (expiry = 0). -use iggy_binary_protocol::RoutedRequestHeader; +use iggy_binary_protocol::RequestHeader; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -56,7 +56,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.create_personal_access_token(&input.name, input.expiry) } diff --git a/core/simulator/src/workload/ops/create_stream.rs b/core/simulator/src/workload/ops/create_stream.rs index cf5e68ace8..c0b3047efb 100644 --- a/core/simulator/src/workload/ops/create_stream.rs +++ b/core/simulator/src/workload/ops/create_stream.rs @@ -18,7 +18,7 @@ //! `CreateStream` op. Targets `Ok` with a fresh name, or `NameAlreadyExists` //! by reusing a live stream name from the shadow. -use iggy_binary_protocol::RoutedRequestHeader; +use iggy_binary_protocol::RequestHeader; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -53,7 +53,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.create_stream(&input.name) } diff --git a/core/simulator/src/workload/ops/create_topic.rs b/core/simulator/src/workload/ops/create_topic.rs index 07fd654a1d..7d7b2f7612 100644 --- a/core/simulator/src/workload/ops/create_topic.rs +++ b/core/simulator/src/workload/ops/create_topic.rs @@ -19,7 +19,7 @@ //! `StreamNotFound` (a fabricated parent stream), or `NameAlreadyExists` (an //! existing topic name under its live stream). -use iggy_binary_protocol::RoutedRequestHeader; +use iggy_binary_protocol::RequestHeader; use rand::RngExt; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -84,7 +84,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.create_topic(&input.stream, &input.name, input.partitions_count) } diff --git a/core/simulator/src/workload/ops/create_user.rs b/core/simulator/src/workload/ops/create_user.rs index c5bce99db5..55fb8993ef 100644 --- a/core/simulator/src/workload/ops/create_user.rs +++ b/core/simulator/src/workload/ops/create_user.rs @@ -18,7 +18,7 @@ //! `CreateUser` op. Targets `Ok` (fresh username) or `UserAlreadyExists` (a //! live username from the shadow). Status fixed at 1 (Active). -use iggy_binary_protocol::RoutedRequestHeader; +use iggy_binary_protocol::RequestHeader; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -59,7 +59,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.create_user(&input.username, &input.password, input.status) } diff --git a/core/simulator/src/workload/ops/delete_consumer_group.rs b/core/simulator/src/workload/ops/delete_consumer_group.rs index 996ea1d3c1..cfbe89961e 100644 --- a/core/simulator/src/workload/ops/delete_consumer_group.rs +++ b/core/simulator/src/workload/ops/delete_consumer_group.rs @@ -22,7 +22,7 @@ //! `ConsumerGroupNotFound` (a live stream/topic with a fabricated group //! name), mirroring the legacy resolution ladder. -use iggy_binary_protocol::RoutedRequestHeader; +use iggy_binary_protocol::RequestHeader; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -86,7 +86,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.delete_consumer_group(&input.stream, &input.topic, &input.group) } diff --git a/core/simulator/src/workload/ops/delete_consumer_offset.rs b/core/simulator/src/workload/ops/delete_consumer_offset.rs index c05c7960ac..0c71b6031e 100644 --- a/core/simulator/src/workload/ops/delete_consumer_offset.rs +++ b/core/simulator/src/workload/ops/delete_consumer_offset.rs @@ -17,7 +17,7 @@ //! `DeleteConsumerOffset` op. Live namespace via shadow. -use iggy_binary_protocol::RoutedRequestHeader; +use iggy_binary_protocol::RequestHeader; use rand::RngExt; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -63,7 +63,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.delete_consumer_offset(input.ns, input.consumer_kind, input.consumer_id) } diff --git a/core/simulator/src/workload/ops/delete_consumer_offset_2.rs b/core/simulator/src/workload/ops/delete_consumer_offset_2.rs index ad2a6201a5..5913b018d1 100644 --- a/core/simulator/src/workload/ops/delete_consumer_offset_2.rs +++ b/core/simulator/src/workload/ops/delete_consumer_offset_2.rs @@ -17,7 +17,7 @@ //! `DeleteConsumerOffset2` op. Namespace-routed with `AckLevel`. -use iggy_binary_protocol::{AckLevel, RoutedRequestHeader}; +use iggy_binary_protocol::{AckLevel, RequestHeader}; use rand::RngExt; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -71,7 +71,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.delete_consumer_offset_2(input.ns, input.consumer_kind, input.consumer_id, input.ack) } diff --git a/core/simulator/src/workload/ops/delete_partitions.rs b/core/simulator/src/workload/ops/delete_partitions.rs index 3711f89ddf..46ee23e73f 100644 --- a/core/simulator/src/workload/ops/delete_partitions.rs +++ b/core/simulator/src/workload/ops/delete_partitions.rs @@ -17,14 +17,11 @@ //! `DeletePartitions` op. //! -//! Targets `Ok` (a live topic, count within its shadow-tracked partition -//! count), `StreamNotFound` (a fabricated parent stream), `TopicNotFound` (a -//! live stream with a fabricated topic), or `InvalidPartitionsCount` (a live -//! topic, count one past its partition count - the server commits the typed -//! rejection instead of acking a silent no-op). A committed `Ok` shrinks the -//! shadow's per-topic partition count. +//! Targets `Ok` (a live topic), `StreamNotFound` (a fabricated parent stream), +//! or `TopicNotFound` (a live stream with a fabricated topic). Partition counts +//! are not tracked in the shadow, so every outcome predicts `Effect::None`. -use iggy_binary_protocol::{MAX_PARTITIONS_PER_REQUEST, RoutedRequestHeader}; +use iggy_binary_protocol::RequestHeader; use rand::RngExt; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -43,12 +40,7 @@ pub struct Input { pub partitions_count: u32, } -pub const OUTCOMES: &[Outcome] = &[ - Outcome::Ok, - Outcome::StreamNotFound, - Outcome::TopicNotFound, - Outcome::InvalidPartitionsCount, -]; +pub const OUTCOMES: &[Outcome] = &[Outcome::Ok, Outcome::StreamNotFound, Outcome::TopicNotFound]; pub fn sample( shadow: &mut Shadow, @@ -59,15 +51,7 @@ pub fn sample( match outcome { Outcome::Ok => { let (stream, topic) = shadow.pick_topic_pair(prng)?; - let live = *shadow - .topic_partitions - .get(&(stream.clone(), topic.clone()))?; - if live == 0 { - // Any nonzero count would over-delete; that is the - // `InvalidPartitionsCount` target, not `Ok`. - return None; - } - let partitions_count = 1 + prng.random_range(0..live.min(4)); + let partitions_count = 1 + prng.random_range(0..4u32); Some(Input { stream, topic, @@ -94,29 +78,11 @@ pub fn sample( partitions_count, }) } - Outcome::InvalidPartitionsCount => { - let (stream, topic) = shadow.pick_topic_pair(prng)?; - let live = *shadow - .topic_partitions - .get(&(stream.clone(), topic.clone()))?; - // One past the live count is the smallest guaranteed over-delete. - // Past the per-request cap the pre-consensus gate would answer - // `TooManyPartitions` instead, so the target is unrealizable. - let partitions_count = live.checked_add(1)?; - if partitions_count > MAX_PARTITIONS_PER_REQUEST { - return None; - } - Some(Input { - stream, - topic, - partitions_count, - }) - } } } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.delete_partitions(&input.stream, &input.topic, input.partitions_count) } @@ -130,13 +96,7 @@ pub const fn classify_reply(code: u32) -> Outcome { } #[must_use] -pub fn predicted_effect(input: &Input, outcome: Outcome) -> Effect { - match outcome { - Outcome::Ok => Effect::RemovePartitions { - stream: input.stream.clone(), - topic: input.topic.clone(), - count: input.partitions_count, - }, - _ => Effect::None, - } +pub const fn predicted_effect(_input: &Input, _outcome: Outcome) -> Effect { + // Partition counts are not tracked per topic in the shadow. + Effect::None } diff --git a/core/simulator/src/workload/ops/delete_personal_access_token.rs b/core/simulator/src/workload/ops/delete_personal_access_token.rs index a829f5fa73..2bd54304eb 100644 --- a/core/simulator/src/workload/ops/delete_personal_access_token.rs +++ b/core/simulator/src/workload/ops/delete_personal_access_token.rs @@ -18,7 +18,7 @@ //! `DeletePersonalAccessToken` op. Targets `Ok` (a live token) or `NotFound` //! (a fabricated name). -use iggy_binary_protocol::RoutedRequestHeader; +use iggy_binary_protocol::RequestHeader; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -51,7 +51,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.delete_personal_access_token(&input.name) } diff --git a/core/simulator/src/workload/ops/delete_segments.rs b/core/simulator/src/workload/ops/delete_segments.rs index abbb173ff5..399016f77e 100644 --- a/core/simulator/src/workload/ops/delete_segments.rs +++ b/core/simulator/src/workload/ops/delete_segments.rs @@ -26,7 +26,7 @@ //! in `Action` and the dispatch table so the surface compiles and the v2.4 //! outcome expansion lands cleanly post-upgrade. -use iggy_binary_protocol::RoutedRequestHeader; +use iggy_binary_protocol::RequestHeader; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -61,7 +61,7 @@ pub const fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.delete_segments( &input.stream, &input.topic, diff --git a/core/simulator/src/workload/ops/delete_stream.rs b/core/simulator/src/workload/ops/delete_stream.rs index 97aa7f050c..008b9f04a0 100644 --- a/core/simulator/src/workload/ops/delete_stream.rs +++ b/core/simulator/src/workload/ops/delete_stream.rs @@ -18,7 +18,7 @@ //! `DeleteStream` op. Targets `Ok` with a live stream name, or `StreamNotFound` //! with a fabricated name that was never created. -use iggy_binary_protocol::RoutedRequestHeader; +use iggy_binary_protocol::RequestHeader; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -51,7 +51,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.delete_stream(&input.name) } diff --git a/core/simulator/src/workload/ops/delete_topic.rs b/core/simulator/src/workload/ops/delete_topic.rs index 4dfd8241fb..32e640ba7e 100644 --- a/core/simulator/src/workload/ops/delete_topic.rs +++ b/core/simulator/src/workload/ops/delete_topic.rs @@ -19,7 +19,7 @@ //! fabricated parent stream), or `TopicNotFound` (a live stream with a //! fabricated topic). -use iggy_binary_protocol::RoutedRequestHeader; +use iggy_binary_protocol::RequestHeader; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -63,7 +63,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.delete_topic(&input.stream, &input.topic) } diff --git a/core/simulator/src/workload/ops/delete_user.rs b/core/simulator/src/workload/ops/delete_user.rs index 55c039eb72..964b8eee87 100644 --- a/core/simulator/src/workload/ops/delete_user.rs +++ b/core/simulator/src/workload/ops/delete_user.rs @@ -18,7 +18,7 @@ //! `DeleteUser` op. Targets `Ok` (a live user) or `UserNotFound` (a fabricated //! username). -use iggy_binary_protocol::RoutedRequestHeader; +use iggy_binary_protocol::RequestHeader; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -53,7 +53,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.delete_user(&input.user) } diff --git a/core/simulator/src/workload/ops/mod.rs b/core/simulator/src/workload/ops/mod.rs index df6ffed485..be965e9108 100644 --- a/core/simulator/src/workload/ops/mod.rs +++ b/core/simulator/src/workload/ops/mod.rs @@ -55,7 +55,7 @@ pub mod update_stream; pub mod update_topic; pub mod update_user; -use iggy_binary_protocol::RoutedRequestHeader; +use iggy_binary_protocol::RequestHeader; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -84,7 +84,7 @@ macro_rules! op_dispatch { /// In-flight entry recorded on submit, removed on reply. /// - /// `request_namespace` is the `header.group` the request was + /// `request_namespace` is the `header.namespace` the request was /// submitted with; the auditor cross-checks it against the /// reply's namespace so a misrouted reply cannot update the /// wrong VSR group's bookkeeping. @@ -130,7 +130,7 @@ macro_rules! op_dispatch { } #[must_use] - pub fn build_message(client: &SimClient, input: &InFlightInput) -> Message { + pub fn build_message(client: &SimClient, input: &InFlightInput) -> Message { match input { $( InFlightInput::$variant(i) => $module::build_message(client, i), )* } diff --git a/core/simulator/src/workload/ops/purge_stream.rs b/core/simulator/src/workload/ops/purge_stream.rs index a7f89c66d8..50033df79d 100644 --- a/core/simulator/src/workload/ops/purge_stream.rs +++ b/core/simulator/src/workload/ops/purge_stream.rs @@ -18,7 +18,7 @@ //! `PurgeStream` op. Targets `Ok` (live stream) or `StreamNotFound` //! (fabricated, never-created name). -use iggy_binary_protocol::RoutedRequestHeader; +use iggy_binary_protocol::RequestHeader; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -51,7 +51,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.purge_stream(&input.stream) } diff --git a/core/simulator/src/workload/ops/purge_topic.rs b/core/simulator/src/workload/ops/purge_topic.rs index 875c625442..00ae96e30e 100644 --- a/core/simulator/src/workload/ops/purge_topic.rs +++ b/core/simulator/src/workload/ops/purge_topic.rs @@ -18,7 +18,7 @@ //! `PurgeTopic` op. Targets `Ok` (live topic), `StreamNotFound` (fabricated //! parent stream), or `TopicNotFound` (live stream, fabricated topic). -use iggy_binary_protocol::RoutedRequestHeader; +use iggy_binary_protocol::RequestHeader; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -62,7 +62,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.purge_topic(&input.stream, &input.topic) } diff --git a/core/simulator/src/workload/ops/send_messages.rs b/core/simulator/src/workload/ops/send_messages.rs index fda0c06565..7e58a27f99 100644 --- a/core/simulator/src/workload/ops/send_messages.rs +++ b/core/simulator/src/workload/ops/send_messages.rs @@ -22,7 +22,7 @@ //! 3. one `prng.random()` per payload to disambiguate body bytes use bytes::Bytes; -use iggy_binary_protocol::RoutedRequestHeader; +use iggy_binary_protocol::RequestHeader; use rand::RngExt; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -74,7 +74,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.send_messages(input.ns, &input.payloads) } diff --git a/core/simulator/src/workload/ops/store_consumer_offset.rs b/core/simulator/src/workload/ops/store_consumer_offset.rs index edefbfc722..feda1b47cf 100644 --- a/core/simulator/src/workload/ops/store_consumer_offset.rs +++ b/core/simulator/src/workload/ops/store_consumer_offset.rs @@ -18,7 +18,7 @@ //! `StoreConsumerOffset` op. Pre-`AckLevel` manual encoding. Live //! namespace via shadow, fabricated consumer kind/id. Samples Success. -use iggy_binary_protocol::RoutedRequestHeader; +use iggy_binary_protocol::RequestHeader; use rand::RngExt; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -58,7 +58,7 @@ pub fn sample( // Draw against the configured ceiling, then clamp to committed // reality so the offset is reachable. Clamping post-draw keeps // the PRNG draw order (and determinism hash baseline) intact - // while staying valid once the server validates offsets. + // while staying valid once server-ng validates offsets. let raw: u64 = prng.random_range(0..options.max_offset.max(1)); let high = shadow.sends_committed(ns).max(1); let offset = raw % high; @@ -73,7 +73,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.store_consumer_offset( input.ns, input.consumer_kind, diff --git a/core/simulator/src/workload/ops/store_consumer_offset_2.rs b/core/simulator/src/workload/ops/store_consumer_offset_2.rs index baac692d07..c4b2785a8b 100644 --- a/core/simulator/src/workload/ops/store_consumer_offset_2.rs +++ b/core/simulator/src/workload/ops/store_consumer_offset_2.rs @@ -23,7 +23,7 @@ //! 4. `offset` range draw //! 5. `ack` ratio draw -use iggy_binary_protocol::{AckLevel, RoutedRequestHeader}; +use iggy_binary_protocol::{AckLevel, RequestHeader}; use rand::RngExt; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -64,7 +64,7 @@ pub fn sample( // Draw against the configured ceiling, then clamp to committed // reality so the offset is reachable. Clamping post-draw keeps // the PRNG draw order (and determinism hash baseline) intact - // while staying valid once the server validates offsets. + // while staying valid once server-ng validates offsets. let raw: u64 = prng.random_range(0..options.max_offset.max(1)); let high = shadow.sends_committed(ns).max(1); let offset = raw % high; @@ -86,7 +86,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.store_consumer_offset_2( input.ns, input.consumer_kind, diff --git a/core/simulator/src/workload/ops/update_permissions.rs b/core/simulator/src/workload/ops/update_permissions.rs index b3aad70e8e..75169a3822 100644 --- a/core/simulator/src/workload/ops/update_permissions.rs +++ b/core/simulator/src/workload/ops/update_permissions.rs @@ -18,7 +18,7 @@ //! `UpdatePermissions` op. Targets `Ok` (live user) or `UserNotFound` //! (fabricated user). No permissions payload, so every outcome is `Effect::None`. -use iggy_binary_protocol::RoutedRequestHeader; +use iggy_binary_protocol::RequestHeader; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -53,7 +53,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.update_permissions(&input.user) } diff --git a/core/simulator/src/workload/ops/update_stream.rs b/core/simulator/src/workload/ops/update_stream.rs index f6f2285096..ef70236d8a 100644 --- a/core/simulator/src/workload/ops/update_stream.rs +++ b/core/simulator/src/workload/ops/update_stream.rs @@ -21,7 +21,7 @@ //! (fabricated stream). `NameAlreadyExists` (rename onto a live name) not //! targeted, but the server still classifies it on a race. -use iggy_binary_protocol::RoutedRequestHeader; +use iggy_binary_protocol::RequestHeader; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -63,7 +63,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.update_stream(&input.stream, &input.new_name) } diff --git a/core/simulator/src/workload/ops/update_topic.rs b/core/simulator/src/workload/ops/update_topic.rs index a4e36367e5..a06f2a61dc 100644 --- a/core/simulator/src/workload/ops/update_topic.rs +++ b/core/simulator/src/workload/ops/update_topic.rs @@ -21,7 +21,7 @@ //! parent stream), or `TopicNotFound` (live stream, fabricated topic). //! `NameAlreadyExists` not targeted. -use iggy_binary_protocol::RoutedRequestHeader; +use iggy_binary_protocol::RequestHeader; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -77,7 +77,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.update_topic(&input.stream, &input.topic, &input.new_name) } diff --git a/core/simulator/src/workload/ops/update_user.rs b/core/simulator/src/workload/ops/update_user.rs index c0c15aade4..045c377b11 100644 --- a/core/simulator/src/workload/ops/update_user.rs +++ b/core/simulator/src/workload/ops/update_user.rs @@ -18,7 +18,7 @@ //! `UpdateUser` op. Targets `Ok` (rename live user to fresh name) or //! `UserNotFound` (fabricated user). `UsernameAlreadyExists` not targeted. -use iggy_binary_protocol::RoutedRequestHeader; +use iggy_binary_protocol::RequestHeader; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -74,7 +74,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.update_user(&input.user, input.new_username.as_deref(), input.status) } diff --git a/core/simulator/src/workload/shadow.rs b/core/simulator/src/workload/shadow.rs index 69f64b2e47..3d02b693e1 100644 --- a/core/simulator/src/workload/shadow.rs +++ b/core/simulator/src/workload/shadow.rs @@ -46,11 +46,6 @@ pub struct Shadow { pub stream_names: IndexSet, /// Live topics by `(stream, topic)`. Only added if parent stream lives. pub topic_names: IndexSet<(String, String)>, - /// Partition count per live topic, keyed like `topic_names`. Lets - /// `create/delete_partitions` sample in-bounds vs over-count deliberately - /// (the server rejects an over-count delete with a committed - /// `InvalidPartitionsCount`, so the count must be known at sample time). - pub topic_partitions: HashMap<(String, String), u32>, pub user_names: IndexSet, pub pat_names: IndexSet, pub consumer_group_names: IndexSet<(String, String, String)>, @@ -85,7 +80,6 @@ impl Shadow { namespaces_live, stream_names: IndexSet::new(), topic_names: IndexSet::new(), - topic_partitions: HashMap::new(), user_names: IndexSet::new(), pat_names: IndexSet::new(), consumer_group_names: IndexSet::new(), @@ -197,23 +191,31 @@ impl Shadow { let applied = match e { Effect::None => true, Effect::AddStream { name } => self.stream_names.insert(name), - Effect::RemoveStream { name } => self.remove_stream(&name), + Effect::RemoveStream { name } => { + let removed = self.stream_names.shift_remove(&name); + self.topic_names.retain(|(s, _)| s != &name); + self.consumer_group_names.retain(|(s, _, _)| s != &name); + removed + } Effect::AddTopic { stream, name, - partitions, - } => self.add_topic(stream, name, partitions), - Effect::RemoveTopic { stream, name } => self.remove_topic(&stream, &name), - Effect::AddPartitions { - stream, - topic, - count, - } => self.add_partitions(stream, topic, count), - Effect::RemovePartitions { - stream, - topic, - count, - } => self.remove_partitions(stream, topic, count), + partitions: _, + } => { + if self.stream_names.contains(&stream) { + self.topic_names.insert((stream, name)) + } else { + false + } + } + Effect::RemoveTopic { stream, name } => { + let removed = self + .topic_names + .shift_remove(&(stream.clone(), name.clone())); + self.consumer_group_names + .retain(|(s, t, _)| !(s == &stream && t == &name)); + removed + } Effect::AddUser { name } => { // Matches `create_user::sample`'s pw-{name} baseline. let password = format!("pw-{name}"); @@ -284,60 +286,6 @@ impl Shadow { } } - fn remove_stream(&mut self, name: &str) -> bool { - let removed = self.stream_names.shift_remove(name); - self.topic_names.retain(|(s, _)| s != name); - self.topic_partitions.retain(|(s, _), _| s != name); - self.consumer_group_names.retain(|(s, _, _)| s != name); - removed - } - - fn add_topic(&mut self, stream: String, name: String, partitions: u32) -> bool { - if self.stream_names.contains(&stream) { - self.topic_partitions - .insert((stream.clone(), name.clone()), partitions); - self.topic_names.insert((stream, name)) - } else { - false - } - } - - fn remove_topic(&mut self, stream: &str, name: &str) -> bool { - let removed = self - .topic_names - .shift_remove(&(stream.to_string(), name.to_string())); - self.topic_partitions - .remove(&(stream.to_string(), name.to_string())); - self.consumer_group_names - .retain(|(s, t, _)| !(s == stream && t == name)); - removed - } - - fn add_partitions(&mut self, stream: String, topic: String, count: u32) -> bool { - self.topic_partitions - .get_mut(&(stream, topic)) - .is_some_and(|partitions| { - *partitions = partitions.saturating_add(count); - true - }) - } - - /// The committed delete was in bounds on the server; a shadow count below - /// it means a concurrent commit defeated the sample-time precondition, - /// same as the other `applied = false` paths. - fn remove_partitions(&mut self, stream: String, topic: String, count: u32) -> bool { - self.topic_partitions - .get_mut(&(stream, topic)) - .is_some_and(|partitions| { - if *partitions >= count { - *partitions -= count; - true - } else { - false - } - }) - } - fn rename_stream(&mut self, old: &str, new: &str) -> bool { if !self.stream_names.shift_remove(old) { return false; @@ -356,16 +304,6 @@ impl Shadow { } }) .collect(); - self.topic_partitions = std::mem::take(&mut self.topic_partitions) - .into_iter() - .map(|((s, t), partitions)| { - if s == old { - ((new_owned.clone(), t), partitions) - } else { - ((s, t), partitions) - } - }) - .collect(); self.consumer_group_names = std::mem::take(&mut self.consumer_group_names) .into_iter() .map(|(s, t, g)| { @@ -388,13 +326,6 @@ impl Shadow { } self.topic_names .insert((stream.to_string(), new.to_string())); - if let Some(partitions) = self - .topic_partitions - .remove(&(stream.to_string(), old.to_string())) - { - self.topic_partitions - .insert((stream.to_string(), new.to_string()), partitions); - } let new_owned = new.to_string(); self.consumer_group_names = std::mem::take(&mut self.consumer_group_names) .into_iter() diff --git a/examples/go/README.md b/examples/go/README.md index d463dee8bc..0a57967e7c 100644 --- a/examples/go/README.md +++ b/examples/go/README.md @@ -9,14 +9,16 @@ To run any example, first start a VSR server and then run the desired example. For server configuration options and help: ```bash -cargo run --bin iggy-server -- --help +# TODO: change to iggy-server once legacy server is removed (core/server has VSR support) +cargo run --bin iggy-server-ng --features vsr -- --help ``` You can also customize the server using environment variables: ```bash ## Example: Enable HTTP transport and set custom address -IGGY_HTTP_ENABLED=true IGGY_TCP_ADDRESS=0.0.0.0:8090 cargo run --bin iggy-server +# TODO: change to iggy-server once legacy server is removed (core/server has VSR support) +IGGY_HTTP_ENABLED=true IGGY_TCP_ADDRESS=0.0.0.0:8090 cargo run --bin iggy-server-ng --features vsr ``` You can run multiple producers and consumers simultaneously to observe how messages are distributed across clients. @@ -49,7 +51,8 @@ All examples can be executed directly from the repository. Follow these steps: 1. **Start the Iggy server**: the Go SDK speaks the VSR wire protocol, so the examples need a VSR server. - `cargo run --bin iggy-server` + + `cargo run --bin iggy-server-ng --features vsr` 2. **Run desired example**: `go run ./xxx/xxx/main.go` 3. **Check source code**: Examples include detailed comments explaining concepts and usage patterns diff --git a/examples/java/README.md b/examples/java/README.md index 4507d90095..aca1b8be65 100644 --- a/examples/java/README.md +++ b/examples/java/README.md @@ -6,24 +6,29 @@ Java 17 and Gradle 9.2.1 are recommended for running the examples. ## Running Examples -The Java SDK speaks the VSR (Viewstamped Replication) wire protocol, so the examples run against the VSR server. +Iggy requires valid credentials to authenticate client requests. The examples assume that the server is using the default root credentials, which can be enabled in one of two ways: -Iggy requires valid credentials to authenticate client requests. The examples assume that the server is using the default root credentials, set through environment variables before starting the server: +1. Start the server with default credentials: -macOS/Linux: + ```bash + cargo run --bin iggy-server -- --with-default-root-credentials + ``` -```bash -export IGGY_ROOT_USERNAME=iggy -export IGGY_ROOT_PASSWORD=iggy -cargo run --bin iggy-server -``` +2. Set the appropriate environment variables before starting the server with `cargo run --bin iggy-server`: -Windows(Powershell): + macOS/Linux: -```bash -$env:IGGY_ROOT_USERNAME = "iggy" -$env:IGGY_ROOT_PASSWORD = "iggy" -``` + ```bash + export IGGY_ROOT_USERNAME=iggy + export IGGY_ROOT_PASSWORD=iggy + ``` + + Windows(Powershell): + + ```bash + $env:IGGY_ROOT_USERNAME = "iggy" + $env:IGGY_ROOT_PASSWORD = "iggy" + ``` > **Note**
> This setup is intended only for development and testing, not production use. @@ -31,15 +36,32 @@ $env:IGGY_ROOT_PASSWORD = "iggy" By default, all server data is stored in the `local_data` directory (this can be changed via `system.path` in `config.toml`). Root credentials are applied **only on the very first startup**, when no data directory exists yet. -Once the server has created and populated the data directory, the existing stored credentials will always be used, and setting the environment variables will no longer override them. +Once the server has created and populated the data directory, the existing stored credentials will always be used, and supplying the `--with-default-root-credentials` flag or setting the environment variables will no longer override them. + +If the server has already been started once and your example returns `Error: InvalidCredentials`, then this means the stored credentials differ from the defaults. -If the server has already been started once and your example returns `Error: InvalidCredentials`, then this means the stored credentials differ from the defaults. Delete the existing data directory, then start the server again with the environment variables set. +You can reset the credentials in one of two ways: + +1. Delete the existing data directory, then start the server again with the default-credential flag or environment variables. +2. Use the `--fresh` flag to force a reset: + + ```bash + cargo run --bin iggy-server -- --with-default-root-credentials --fresh + ``` + + This will ignore any existing data directory and re-initialize it with the default credentials. + +For server configuration options and help: + +```bash +cargo run --bin iggy-server -- --help +``` You can also customize the server using environment variables: ```bash -## Example: set a custom TCP address -IGGY_TCP_ADDRESS=0.0.0.0:8090 cargo run --bin iggy-server +## Example: Enable HTTP transport and set custom address +IGGY_HTTP_ENABLED=true IGGY_TCP_ADDRESS=0.0.0.0:8090 cargo run --bin iggy-server ``` ## Basic Examples @@ -114,7 +136,7 @@ Shows how to use the stream builder API to create and configure streams with cus ### Async Producer -Non-blocking batch production with concurrent request submission: +High-throughput async production with pipelining: ```bash ./gradlew runAsyncProducer @@ -123,7 +145,7 @@ Non-blocking batch production with concurrent request submission: Shows: - CompletableFuture chaining patterns -- Submitting multiple sends without blocking +- Pipelining multiple sends without blocking - Performance comparison with blocking client ### Async Consumer @@ -188,7 +210,7 @@ The Iggy Java SDK provides two client types: **blocking (synchronous)** and **as - Need high throughput - Application is already async/reactive (Spring WebFlux, Vert.x) -- Want to compose non-blocking requests with `CompletableFuture` +- Want to pipeline multiple requests over a single connection - Building services that handle many concurrent streams ## Key Async Patterns @@ -206,20 +228,16 @@ client.connect() }); ``` -### Submitting Multiple Sends +### Pipelining for Throughput ```java -List> sends = new ArrayList<>(); +List> sends = new ArrayList<>(); for (int i = 0; i < 10; i++) { sends.add(client.messages().sendMessages(...)); } CompletableFuture.allOf(sends.toArray(new CompletableFuture[0])).join(); ``` -The client accepts these calls without blocking, but its single VSR-pinned TCP -connection processes them in order. Batch more messages into each send to improve -throughput. - ### Thread Pool Offloading ```java diff --git a/examples/node/src/tcp-tls/consumer.ts b/examples/node/src/tcp-tls/consumer.ts index 6817193123..9a7ec5229a 100644 --- a/examples/node/src/tcp-tls/consumer.ts +++ b/examples/node/src/tcp-tls/consumer.ts @@ -23,7 +23,6 @@ // // Prerequisites: // Start the Iggy server with TLS enabled: -// IGGY_ROOT_USERNAME=iggy IGGY_ROOT_PASSWORD=iggy \ // IGGY_TCP_TLS_ENABLED=true \ // IGGY_TCP_TLS_CERT_FILE=core/certs/iggy_cert.pem \ // IGGY_TCP_TLS_KEY_FILE=core/certs/iggy_key.pem \ diff --git a/examples/node/src/tcp-tls/producer.ts b/examples/node/src/tcp-tls/producer.ts index 98f7dc99c2..7b51cc541e 100644 --- a/examples/node/src/tcp-tls/producer.ts +++ b/examples/node/src/tcp-tls/producer.ts @@ -23,7 +23,6 @@ // // Prerequisites: // Start the Iggy server with TLS enabled: -// IGGY_ROOT_USERNAME=iggy IGGY_ROOT_PASSWORD=iggy \ // IGGY_TCP_TLS_ENABLED=true \ // IGGY_TCP_TLS_CERT_FILE=core/certs/iggy_cert.pem \ // IGGY_TCP_TLS_KEY_FILE=core/certs/iggy_key.pem \ @@ -34,15 +33,7 @@ import { readFileSync } from 'node:fs'; import { Client, Partitioning } from 'apache-iggy'; -import { - BATCHES_LIMIT, - cleanup, - initSystem, - log, - MESSAGES_PER_BATCH, - PARTITION_ID, - sleep -} from '../utils'; +import { BATCHES_LIMIT, cleanup, initSystem, log, MESSAGES_PER_BATCH, sleep } from '../utils'; async function produceMessages( client: Client, @@ -71,14 +62,11 @@ async function produceMessages( }); try { - // The VSR client routes each send to an explicit partition. - // TODO(hubcio): Balanced partitioning to be implemented; not decided - // yet whether it'll be on server side or client side. await client.message.send({ streamId, topicId, messages, - partition: Partitioning.PartitionId(PARTITION_ID), + partition: Partitioning.Balanced, }); } catch (error) { log('Error sending messages: %o', error); diff --git a/examples/python/getting-started/consumer.py b/examples/python/getting-started/consumer.py index db0a6edb1f..bb10e91382 100755 --- a/examples/python/getting-started/consumer.py +++ b/examples/python/getting-started/consumer.py @@ -19,16 +19,8 @@ import asyncio import typing import urllib.parse -from datetime import timedelta - -from apache_iggy import ( - AutoLogin, - IggyClient, - PollingStrategy, - ReceiveMessage, - TcpConfig, - TcpReconnectionConfig, -) + +from apache_iggy import IggyClient, PollingStrategy, ReceiveMessage from loguru import logger STREAM_NAME = "sample-stream" @@ -99,34 +91,34 @@ def parse_args() -> ArgNamespace: return ArgNamespace(**vars(args)) -def build_config(args: ArgNamespace) -> TcpConfig: - """Build a TCP client configuration with auto-login and reconnection.""" +def build_connection_string(args) -> str: + """Build a connection string with TLS support.""" - return TcpConfig( - server_address=args.tcp_server_address, - auto_login=AutoLogin.username_password(args.username, args.password), - reconnection=TcpReconnectionConfig( - enabled=True, - interval=timedelta(seconds=1), - ), - tls_enabled=args.tls, - tls_ca_file=args.tls_ca_file or None, - ) + conn_str = f"iggy://{args.username}:{args.password}@{args.tcp_server_address}" + + if args.tls: + # Extract domain from server address (host:port -> host) + host = args.tcp_server_address.split(":")[0] + query_params = ["tls=true", f"tls_domain={host}"] + + # Add CA file if provided + if args.tls_ca_file: + query_params.append(f"tls_ca_file={args.tls_ca_file}") + conn_str += "?" + "&".join(query_params) + + return conn_str async def main(): args: ArgNamespace = parse_args() - try: - config = build_config(args) - except ValueError as error: - logger.error(f"Invalid client configuration: {error}") - return - logger.info(f"Connecting to {args.tcp_server_address} (TLS: {args.tls})") - client = IggyClient(config) + # Build connection string with TLS support + connection_string = build_connection_string(args) + logger.info(f"Connection string: {connection_string}") + + client = IggyClient.from_connection_string(connection_string) try: logger.info("Connecting to IggyClient...") - # No login_user() call: auto_login replays the credentials on every connect. await client.connect() logger.info("Connected.") await consume_messages(client) diff --git a/examples/python/getting-started/producer.py b/examples/python/getting-started/producer.py index 23f964fa81..642399edb0 100755 --- a/examples/python/getting-started/producer.py +++ b/examples/python/getting-started/producer.py @@ -19,16 +19,8 @@ import asyncio import typing import urllib.parse -from datetime import timedelta - -from apache_iggy import ( - AutoLogin, - IggyClient, - StreamDetails, - TcpConfig, - TcpReconnectionConfig, - TopicDetails, -) + +from apache_iggy import IggyClient, StreamDetails, TopicDetails from apache_iggy import SendMessage as Message from loguru import logger @@ -100,33 +92,33 @@ def parse_args() -> ArgNamespace: return ArgNamespace(**vars(args)) -def build_config(args: ArgNamespace) -> TcpConfig: - """Build a TCP client configuration with auto-login and reconnection.""" +def build_connection_string(args) -> str: + """Build a connection string with TLS support.""" - return TcpConfig( - server_address=args.tcp_server_address, - auto_login=AutoLogin.username_password(args.username, args.password), - reconnection=TcpReconnectionConfig( - enabled=True, - interval=timedelta(seconds=1), - ), - tls_enabled=args.tls, - tls_ca_file=args.tls_ca_file or None, - ) + conn_str = f"iggy://{args.username}:{args.password}@{args.tcp_server_address}" + + if args.tls: + # Extract domain from server address (host:port -> host) + host = args.tcp_server_address.split(":")[0] + query_params = ["tls=true", f"tls_domain={host}"] + + # Add CA file if provided + if args.tls_ca_file: + query_params.append(f"tls_ca_file={args.tls_ca_file}") + conn_str += "?" + "&".join(query_params) + + return conn_str async def main(): args: ArgNamespace = parse_args() - try: - config = build_config(args) - except ValueError as error: - logger.error(f"Invalid client configuration: {error}") - return + # Build connection string with TLS support + connection_string = build_connection_string(args) + logger.info(f"Connection string: {connection_string}") logger.info(f"Connecting to {args.tcp_server_address} (TLS: {args.tls})") - client = IggyClient(config) + client = IggyClient.from_connection_string(connection_string) logger.info("Connecting to IggyClient") - # No login_user() call: auto_login replays the credentials on every connect. await client.connect() logger.info("Connected.") await init_system(client) diff --git a/foreign/cpp/tests/e2e/client.cpp b/foreign/cpp/tests/e2e/client.cpp index 53c1a96a51..64405fe92b 100644 --- a/foreign/cpp/tests/e2e/client.cpp +++ b/foreign/cpp/tests/e2e/client.cpp @@ -448,32 +448,40 @@ TEST_F(LowLevelE2E_Client, GetClientsReflectsSessionRemovalAfterDisconnect) { } TEST_F(LowLevelE2E_Client, GetClientsReflectsLoggedOutSessionAsUnauthenticated) { - RecordProperty("description", "Drops a logged out session from get_clients and reports it missing in get_client."); + RecordProperty("description", + "Keeps a logged out session visible in get_clients and get_client, but marks it unauthenticated."); iggy::ffi::Client *first_client = GetLoggedInClient(); iggy::ffi::Client *second_client = GetLoggedInClient(); iggy::ffi::ClientInfoDetails first_me{}; + iggy::ffi::ClientInfoDetails logged_out_client{}; + rust::Vec clients_after_logout; ASSERT_NO_THROW({ first_me = first_client->get_me(); }); - // The VSR server drops the client-table entry on logout (an unauthenticated - // session is not tracked), unlike the legacy server which kept it visible - // without a user id. ASSERT_NO_THROW(first_client->logout_user()); - constexpr auto removal_timeout = std::chrono::seconds(5); - constexpr auto removal_poll_interval = std::chrono::milliseconds(10); - const auto deadline = std::chrono::steady_clock::now() + removal_timeout; - bool removed = false; - do { - const auto clients = second_client->get_clients(); - removed = std::none_of(clients.begin(), clients.end(), - [&first_me](const auto &client) { return client.client_id == first_me.client_id; }); - if (removed) { - break; + ASSERT_NO_THROW({ + clients_after_logout = second_client->get_clients(); + logged_out_client = second_client->get_client(first_me.client_id); + }); + + bool found_first = false; + for (const auto &client : clients_after_logout) { + if (client.client_id != first_me.client_id) { + continue; } - std::this_thread::sleep_for(removal_poll_interval); - } while (std::chrono::steady_clock::now() < deadline); - ASSERT_TRUE(removed); - ASSERT_THROW(second_client->get_client(first_me.client_id), std::exception); + + found_first = true; + EXPECT_FALSE(client.has_user_id); + EXPECT_EQ(static_cast(client.address), static_cast(first_me.address)); + EXPECT_EQ(static_cast(client.transport), static_cast(first_me.transport)); + break; + } + + EXPECT_TRUE(found_first); + EXPECT_EQ(logged_out_client.client_id, first_me.client_id); + EXPECT_FALSE(logged_out_client.has_user_id); + EXPECT_EQ(static_cast(logged_out_client.address), static_cast(first_me.address)); + EXPECT_EQ(static_cast(logged_out_client.transport), static_cast(first_me.transport)); } TEST_F(LowLevelE2E_Client, LoginWithoutConnect) { @@ -559,12 +567,9 @@ TEST_F(LowLevelE2E_Client, GetStatsBeforeLoginThrows) { ASSERT_THROW(client->get_stats(), std::exception); } -// The VSR server has no unsaved-buffer primitive (writes are journaled at -// commit); FLUSH_UNSAVED_BUFFER denies typed with FeatureUnavailable even for -// resolvable targets. -TEST_F(LowLevelE2E_Client, FlushUnsavedBufferThrowsForExistingPartition) { +TEST_F(LowLevelE2E_Client, FlushUnsavedBufferSucceedsForExistingPartition) { RecordProperty("description", - "Rejects flush_unsaved_buffer with the feature-unavailable error for an existing partition."); + "Creates a stream and topic, sends one message, and flushes the partition buffer successfully."); const std::string stream_name = GetRandomName(); const std::string topic_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); @@ -580,14 +585,13 @@ TEST_F(LowLevelE2E_Client, FlushUnsavedBufferThrowsForExistingPartition) { ASSERT_NO_THROW(client->send_messages(make_numeric_identifier(stream.id), make_numeric_identifier(0), "partition_id", partition_id_bytes(0), std::move(messages))); - ASSERT_THROW(client->flush_unsaved_buffer(make_numeric_identifier(stream.id), make_numeric_identifier(0), 0, true), - std::exception); + ASSERT_NO_THROW( + client->flush_unsaved_buffer(make_numeric_identifier(stream.id), make_numeric_identifier(0), 0, true)); } -TEST_F(LowLevelE2E_Client, FlushUnsavedBufferThrowsForExistingEmptyPartition) { - RecordProperty( - "description", - "Rejects flush_unsaved_buffer with the feature-unavailable error for a partition with no unsaved messages."); +TEST_F(LowLevelE2E_Client, FlushUnsavedBufferSucceedsForExistingEmptyPartition) { + RecordProperty("description", + "Succeeds when flush_unsaved_buffer is called for an existing partition with no unsaved messages."); const std::string stream_name = GetRandomName(); const std::string topic_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); @@ -598,8 +602,8 @@ TEST_F(LowLevelE2E_Client, FlushUnsavedBufferThrowsForExistingEmptyPartition) { ASSERT_NO_THROW(client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, "server_default")); - ASSERT_THROW(client->flush_unsaved_buffer(make_numeric_identifier(stream.id), make_numeric_identifier(0), 0, true), - std::exception); + ASSERT_NO_THROW( + client->flush_unsaved_buffer(make_numeric_identifier(stream.id), make_numeric_identifier(0), 0, true)); } TEST_F(LowLevelE2E_Client, FlushUnsavedBufferBeforeLoginThrows) { @@ -690,9 +694,8 @@ TEST_F(LowLevelE2E_Client, FlushUnsavedBufferAfterTopicDeletedThrows) { std::exception); } -TEST_F(LowLevelE2E_Client, FlushUnsavedBufferTwiceThrows) { - RecordProperty("description", - "Rejects flush_unsaved_buffer with the feature-unavailable error consistently across repeat calls."); +TEST_F(LowLevelE2E_Client, FlushUnsavedBufferTwiceSucceeds) { + RecordProperty("description", "Allows flush_unsaved_buffer to be called twice in a row for the same partition."); const std::string stream_name = GetRandomName(); const std::string topic_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); @@ -708,10 +711,10 @@ TEST_F(LowLevelE2E_Client, FlushUnsavedBufferTwiceThrows) { ASSERT_NO_THROW(client->send_messages(make_numeric_identifier(stream.id), make_numeric_identifier(0), "partition_id", partition_id_bytes(0), std::move(messages))); - ASSERT_THROW(client->flush_unsaved_buffer(make_numeric_identifier(stream.id), make_numeric_identifier(0), 0, true), - std::exception); - ASSERT_THROW(client->flush_unsaved_buffer(make_numeric_identifier(stream.id), make_numeric_identifier(0), 0, true), - std::exception); + ASSERT_NO_THROW( + client->flush_unsaved_buffer(make_numeric_identifier(stream.id), make_numeric_identifier(0), 0, true)); + ASSERT_NO_THROW( + client->flush_unsaved_buffer(make_numeric_identifier(stream.id), make_numeric_identifier(0), 0, true)); } TEST_F(LowLevelE2E_Client, FlushUnsavedBufferWithInvalidPartitionIdsThrows) { @@ -1563,19 +1566,15 @@ TEST_F(LowLevelE2E_Client, GetClientsReflectsAdditionalSession) { EXPECT_TRUE(found_after); } -TEST_F(LowLevelE2E_Client, GetClusterMetadataBeforeLoginSucceeds) { +TEST_F(LowLevelE2E_Client, GetClusterMetadataBeforeLoginThrows) { RecordProperty( "description", - "Serves get_cluster_metadata to a connected but unauthenticated client, and rejects it without a connection."); + "Rejects get_cluster_metadata before connect, after connect but before login, and after disconnect."); iggy::ffi::Client *client = GetLoggedOutClient(); ASSERT_THROW(client->get_cluster_metadata(), std::exception); ASSERT_NO_THROW(client->connect()); - // By design pre-login on the VSR server: an SDK must read the roster to - // find the primary before it can authenticate (redirect bootstrap). - iggy::ffi::ClusterMetadata metadata{}; - ASSERT_NO_THROW({ metadata = client->get_cluster_metadata(); }); - ASSERT_EQ(metadata.nodes.size(), 1u); + ASSERT_THROW(client->get_cluster_metadata(), std::exception); ASSERT_NO_THROW(client->login_user("iggy", "iggy")); ASSERT_NO_THROW(client->disconnect()); ASSERT_THROW(client->get_cluster_metadata(), std::exception); @@ -1632,8 +1631,6 @@ TEST_F(LowLevelE2E_Client, PingSucceedsForNewConnection) { RecordProperty("description", "Successfully pings the server from a fresh unauthenticated client session."); iggy::ffi::Client *client = GetLoggedOutClient(); - // The VSR client has no lazy connect; ping still needs no authentication. - ASSERT_NO_THROW(client->connect()); ASSERT_NO_THROW(client->ping()); } diff --git a/foreign/cpp/tests/e2e/consumer_group.cpp b/foreign/cpp/tests/e2e/consumer_group.cpp index 7863b66300..3ca41d3e17 100644 --- a/foreign/cpp/tests/e2e/consumer_group.cpp +++ b/foreign/cpp/tests/e2e/consumer_group.cpp @@ -722,21 +722,19 @@ TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupsReflectsJoinedGroupMembersCou EXPECT_NE(groups[0].members_count, groups[1].members_count); } -// The VSR server rejects consumer-group reads whose parent stream or topic is -// absent with the legacy typed not-found; the legacy server answered them with -// an empty list. -TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupsOnNonExistentStreamThrows) { - RecordProperty("description", "Throws when the stream does not exist."); +TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupsOnNonExistentStreamReturnsEmpty) { + RecordProperty("description", "Returns an empty list when the stream does not exist."); const std::string stream_name = GetRandomName(); const std::string topic_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - ASSERT_THROW(client->get_consumer_groups(make_string_identifier(stream_name), make_string_identifier(topic_name)), - std::exception); + const auto groups = + client->get_consumer_groups(make_string_identifier(stream_name), make_string_identifier(topic_name)); + EXPECT_TRUE(groups.empty()); } -TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupsOnNonExistentTopicThrows) { - RecordProperty("description", "Throws when the topic does not exist."); +TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupsOnNonExistentTopicReturnsEmpty) { + RecordProperty("description", "Returns an empty list when the topic does not exist."); const std::string stream_name = GetRandomName(); const std::string topic_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); @@ -744,8 +742,9 @@ TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupsOnNonExistentTopicThrows) { ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - ASSERT_THROW(client->get_consumer_groups(make_string_identifier(stream_name), make_string_identifier(topic_name)), - std::exception); + const auto groups = + client->get_consumer_groups(make_string_identifier(stream_name), make_string_identifier(topic_name)); + EXPECT_TRUE(groups.empty()); } TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupsIsStableAcrossBackToBackCalls) { @@ -837,8 +836,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupsReturnsCorrectNumberOfGroups) EXPECT_TRUE(groups_after_delete.empty()); } -TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupsAfterStreamDeletionThrows) { - RecordProperty("description", "Throws after deleting the stream that owned the groups."); +TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupsAfterStreamDeletionReturnsEmpty) { + RecordProperty("description", "Returns an empty list after deleting the stream that owned the groups."); const std::string stream_name = GetRandomName(); const std::string topic_name = GetRandomName(); const std::string first_group_name = GetRandomName(); @@ -860,12 +859,13 @@ TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupsAfterStreamDeletionThrows) { ForgetTrackedConsumerGroup(stream_name, topic_name, second_group_name); ForgetTrackedStream(stream_name); - ASSERT_THROW(client->get_consumer_groups(make_string_identifier(stream_name), make_string_identifier(topic_name)), - std::exception); + const auto groups = + client->get_consumer_groups(make_string_identifier(stream_name), make_string_identifier(topic_name)); + EXPECT_TRUE(groups.empty()); } -TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupsAfterTopicDeletionThrows) { - RecordProperty("description", "Throws after deleting the topic that owned the groups."); +TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupsAfterTopicDeletionReturnsEmpty) { + RecordProperty("description", "Returns an empty list after deleting the topic that owned the groups."); const std::string stream_name = GetRandomName(); const std::string topic_name = GetRandomName(); const std::string first_group_name = GetRandomName(); @@ -887,8 +887,9 @@ TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupsAfterTopicDeletionThrows) { ForgetTrackedConsumerGroup(stream_name, topic_name, first_group_name); ForgetTrackedConsumerGroup(stream_name, topic_name, second_group_name); - ASSERT_THROW(client->get_consumer_groups(make_string_identifier(stream_name), make_string_identifier(topic_name)), - std::exception); + const auto groups = + client->get_consumer_groups(make_string_identifier(stream_name), make_string_identifier(topic_name)); + EXPECT_TRUE(groups.empty()); } TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupBeforeLoginThrows) { @@ -1133,10 +1134,7 @@ TEST_F(LowLevelE2E_ConsumerGroup, DeleteConsumerGroupAndRecreateWithSameNameSucc const auto recreated_group = client->create_consumer_group(make_string_identifier(stream_name), make_string_identifier(topic_name), group_name); TrackConsumerGroup(stream_name, topic_name, group_name); - // The VSR server mints group ids monotonically; a recreate gets a - // fresh id (the deleted group held 0), unlike the legacy server which - // reused the freed slot. - ASSERT_GT(recreated_group.id, 0u); + ASSERT_EQ(recreated_group.id, 0u); ASSERT_EQ(recreated_group.name, group_name); ASSERT_EQ(recreated_group.members_count, 0u); ASSERT_TRUE(recreated_group.members.empty()); diff --git a/foreign/cpp/tests/e2e/message.cpp b/foreign/cpp/tests/e2e/message.cpp index f805127a34..e20ef6008f 100644 --- a/foreign/cpp/tests/e2e/message.cpp +++ b/foreign/cpp/tests/e2e/message.cpp @@ -51,10 +51,9 @@ TEST_F(LowLevelE2E_Message, SendAndPollMessagesRoundTrip) { ASSERT_NO_THROW(sent = client->send_messages(make_numeric_identifier(stream.id), make_numeric_identifier(0), "partition_id", partition_id_bytes(0), std::move(messages))); - ASSERT_EQ(sent.confirmations.size(), 1u) - << "The VSR server reports the written partition's offsets, so a single-partition send " - << "must carry exactly one confirmation"; - EXPECT_EQ(sent.confirmations.front().partition_id, 0u); + ASSERT_TRUE(sent.confirmations.empty()) + << "The legacy server reports no offsets, so the confirmation list must stay empty, got " + << sent.confirmations.size(); auto polled = client->poll_messages(make_numeric_identifier(stream.id), make_numeric_identifier(0), 0, "consumer", make_numeric_identifier(1), "offset", 0, 100, false); diff --git a/foreign/csharp/Directory.Packages.props b/foreign/csharp/Directory.Packages.props index c0f382e4fa..14e554e308 100644 --- a/foreign/csharp/Directory.Packages.props +++ b/foreign/csharp/Directory.Packages.props @@ -30,14 +30,13 @@ - - + + - - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/ClusterRedirectionTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/ClusterRedirectionTests.cs index 006946f2aa..1eafa8ea10 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/ClusterRedirectionTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/ClusterRedirectionTests.cs @@ -26,15 +26,15 @@ namespace Apache.Iggy.Tests.Integrations; public class ClusterRedirectionTests { - [ClassDataSource(Shared = SharedType.PerAssembly)] - public required RedirectionClusterFixture Fixture { get; init; } + [ClassDataSource(Shared = SharedType.PerAssembly)] + public required IggyClusterFixture Fixture { get; init; } [Test] - public async Task ConnectToFollower_Should_ReturnClusterMetadataWithAllNodes() + public async Task ConnectToFollower_Should_ReturnClusterMetadataWithTwoNodes() { using var client = IggyClientFactory.CreateClient(new IggyClientConfigurator { - BaseAddress = await Fixture.GetFollowerTcpAddressAsync(), + BaseAddress = Fixture.GetFollowerAddress(), Protocol = Protocol.Tcp, ReconnectionSettings = new ReconnectionSettings { Enabled = false }, AutoLoginSettings = new AutoLoginSettings { Enabled = false } @@ -45,8 +45,8 @@ public async Task ConnectToFollower_Should_ReturnClusterMetadataWithAllNodes() var metadata = await client.GetClusterMetadataAsync(); metadata.ShouldNotBeNull(); - metadata.Name.ShouldBe("test-vsr-cluster"); - metadata.Nodes.Length.ShouldBe(3); + metadata.Name.ShouldBe("test-cluster"); + metadata.Nodes.Length.ShouldBe(2); metadata.Nodes.ShouldContain(n => n.Role == ClusterNodeRole.Leader); metadata.Nodes.ShouldContain(n => n.Role == ClusterNodeRole.Follower); } @@ -56,7 +56,7 @@ public async Task ConnectToFollowerWithAutoLogin_Should_RedirectToLeader() { using var client = IggyClientFactory.CreateClient(new IggyClientConfigurator { - BaseAddress = await Fixture.GetFollowerTcpAddressAsync(), + BaseAddress = Fixture.GetFollowerAddress(), Protocol = Protocol.Tcp, ReconnectionSettings = new ReconnectionSettings { Enabled = true }, AutoLoginSettings = new AutoLoginSettings @@ -70,7 +70,7 @@ public async Task ConnectToFollowerWithAutoLogin_Should_RedirectToLeader() var address = client.GetCurrentAddress(); address.ShouldNotBeNullOrEmpty(); - address.ShouldBe(await Fixture.GetIggyAddressAsync(Protocol.Tcp)); + address.ShouldBe(Fixture.GetLeaderAddress()); } [Test] @@ -78,7 +78,7 @@ public async Task ConnectToFollowerWithManualLogin_Should_RedirectToLeader() { using var client = IggyClientFactory.CreateClient(new IggyClientConfigurator { - BaseAddress = await Fixture.GetFollowerTcpAddressAsync(), + BaseAddress = Fixture.GetFollowerAddress(), Protocol = Protocol.Tcp, ReconnectionSettings = new ReconnectionSettings { Enabled = true }, AutoLoginSettings = new AutoLoginSettings { Enabled = false } @@ -88,15 +88,16 @@ public async Task ConnectToFollowerWithManualLogin_Should_RedirectToLeader() var address = client.GetCurrentAddress(); address.ShouldNotBeNullOrEmpty(); - address.ShouldBe(await Fixture.GetIggyAddressAsync(Protocol.Tcp)); + address.ShouldBe(Fixture.GetLeaderAddress()); } [Test] + [Skip("Currently personal access token exist only on leader. Unskip when it will be available on follower.")] public async Task ConnectToFollowerWithPersonalAccessToken_Should_RedirectToLeader() { using var leaderClient = IggyClientFactory.CreateClient(new IggyClientConfigurator { - BaseAddress = await Fixture.GetIggyAddressAsync(Protocol.Tcp), + BaseAddress = Fixture.GetLeaderAddress(), Protocol = Protocol.Tcp, ReconnectionSettings = new ReconnectionSettings { Enabled = false }, AutoLoginSettings = new AutoLoginSettings { Enabled = false } @@ -110,7 +111,7 @@ public async Task ConnectToFollowerWithPersonalAccessToken_Should_RedirectToLead using var client = IggyClientFactory.CreateClient(new IggyClientConfigurator { - BaseAddress = await Fixture.GetFollowerTcpAddressAsync(), + BaseAddress = Fixture.GetFollowerAddress(), Protocol = Protocol.Tcp, ReconnectionSettings = new ReconnectionSettings { Enabled = true }, AutoLoginSettings = new AutoLoginSettings { Enabled = false } @@ -123,6 +124,6 @@ public async Task ConnectToFollowerWithPersonalAccessToken_Should_RedirectToLead var address = client.GetCurrentAddress(); address.ShouldNotBeNullOrEmpty(); - address.ShouldBe(await Fixture.GetIggyAddressAsync(Protocol.Tcp)); + address.ShouldBe(Fixture.GetLeaderAddress()); } } diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/ConsumerGroupTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/ConsumerGroupTests.cs index c6ed9ed56a..ee13df7d74 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/ConsumerGroupTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/ConsumerGroupTests.cs @@ -81,8 +81,8 @@ public async Task GetConsumerGroupById_Should_Return_ValidResponse(Protocol prot var cg = await client.CreateConsumerGroupAsync(Identifier.String(streamName), Identifier.String(TopicName), GroupName); - var response = await client.GetConsumerGroupByIdAsync(Identifier.String(streamName), - Identifier.String(TopicName), + var response = await client.GetConsumerGroupByIdAsync( + Identifier.String(streamName), Identifier.String(TopicName), Identifier.Numeric(cg!.Id)); response.ShouldNotBeNull(); @@ -194,14 +194,15 @@ public async Task GetConsumerGroupById_WithMembers_Should_Return_ValidResponse(P var clients = new List(); for (var i = 0; i < 2; i++) { - var memberClient = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); + var memberClient = await Fixture.CreateClient(Protocol.Tcp); clients.Add(memberClient); + await memberClient.LoginUserAsync("iggy", "iggy"); await memberClient.JoinConsumerGroupAsync(Identifier.String(streamName), Identifier.String(TopicName), Identifier.Numeric(cg!.Id)); } - var response = await client.GetConsumerGroupByIdAsync(Identifier.String(streamName), - Identifier.String(TopicName), + var response = await client.GetConsumerGroupByIdAsync( + Identifier.String(streamName), Identifier.String(TopicName), Identifier.Numeric(cg!.Id)); response.ShouldNotBeNull(); diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/FetchMessagesTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/FetchMessagesTests.cs index f9ae474a23..d75d95cee1 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/FetchMessagesTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/FetchMessagesTests.cs @@ -23,7 +23,6 @@ using Apache.Iggy.IggyClient; using Apache.Iggy.Kinds; using Apache.Iggy.Messages; -using Apache.Iggy.Tests.Integrations.Attributes; using Apache.Iggy.Tests.Integrations.Fixtures; using Shouldly; using Partitioning = Apache.Iggy.Kinds.Partitioning; @@ -91,23 +90,23 @@ public async Task PollMessages_WithNoHeaders_Should_PollMessages_Successfully(Pr [Test] [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] - public async Task PollMessages_InvalidTopic_Should_Throw_NotFound(Protocol protocol) + public async Task PollMessages_InvalidTopic_Should_Throw_InvalidResponse(Protocol protocol) { var (client, streamName) = await CreateStreamWithMessages(protocol); - // A missing topic is an addressing error, not an empty partition: - // HTTP answers 404, TCP the typed topic-not-found rejection. + var invalidFetchRequest = new MessageFetchRequest + { + Count = 10, + AutoCommit = true, + Consumer = Consumer.New(1), + PartitionId = 0, + PollingStrategy = PollingStrategy.Next(), + StreamId = Identifier.String(streamName), + TopicId = Identifier.Numeric(2137) + }; + await Should.ThrowAsync(() => - client.PollMessagesAsync(new MessageFetchRequest - { - Count = 10, - AutoCommit = true, - Consumer = Consumer.New(1), - PartitionId = 0, - PollingStrategy = PollingStrategy.Next(), - StreamId = Identifier.String(streamName), - TopicId = Identifier.Numeric(2137) - })); + client.PollMessagesAsync(invalidFetchRequest)); } [Test] diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyClusterFixture.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyClusterFixture.cs new file mode 100644 index 0000000000..73c33edeac --- /dev/null +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyClusterFixture.cs @@ -0,0 +1,250 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Net; +using System.Net.Sockets; +using DotNet.Testcontainers.Builders; +using DotNet.Testcontainers.Containers; +using DotNet.Testcontainers.Networks; +using TUnit.Core.Interfaces; + +namespace Apache.Iggy.Tests.Integrations.Fixtures; + +public class IggyClusterFixture : IAsyncInitializer, IAsyncDisposable +{ + private const string LeaderAlias = "iggy-leader"; + private const string FollowerAlias = "iggy-follower"; + + // Split the host-port pool by .NET major version so parallel `dotnet test` + // processes (net8.0 + net10.0) can never pick the same port and race docker's + // allocator. The ranges sit below Linux's default ephemeral range (32768+), so + // the kernel's auto-allocation won't steal from us either. + // net8.0 - 30800..30899 + // net10.0 - 31000..31099 + // Future TFMs slot in without overlap (e.g. net12.0 - 31200..31299). + private const ushort PortRangeSize = 100; + private static readonly ushort BasePort = (ushort)(30000 + Environment.Version.Major * 100); + private static readonly ushort EndPort = (ushort)(BasePort + PortRangeSize); + + // Listeners only need to outlive the eight ReservePort() calls in the + // constructor so we don't pick the same port twice within one fixture. + // Partitioned ranges already guarantee sibling processes can't race us, so + // we can release them as soon as picking is done. + private readonly List _portReservations = []; + private readonly IContainer _followerContainer; + private readonly ushort _followerHttpPort; + private readonly ushort _followerQuicPort; + + private readonly ushort _followerTcpPort; + private readonly ushort _followerWsPort; + private readonly IContainer _leaderContainer; + private readonly ushort _leaderHttpPort; + private readonly ushort _leaderQuicPort; + + private readonly ushort _leaderTcpPort; + private readonly ushort _leaderWsPort; + + private readonly INetwork _network; + + private string DockerImage => + Environment.GetEnvironmentVariable("IGGY_SERVER_DOCKER_IMAGE") ?? "apache/iggy:edge"; + + private static string? LogDirectory => + Environment.GetEnvironmentVariable("IGGY_TEST_LOGS_DIR"); + + public IggyClusterFixture() + { + try + { + _leaderTcpPort = ReservePort(); + _leaderHttpPort = ReservePort(); + _leaderQuicPort = ReservePort(); + _leaderWsPort = ReservePort(); + _followerTcpPort = ReservePort(); + _followerHttpPort = ReservePort(); + _followerQuicPort = ReservePort(); + _followerWsPort = ReservePort(); + } + finally + { + ReleaseReservedPorts(); + } + + _network = new NetworkBuilder() + .WithName($"iggy-cluster-{Guid.NewGuid():N}") + .Build(); + + // Cluster.nodes roster env vars are byte-identical on both + // containers; only the bind addresses and the --replica-id CLI arg + // differ per node. + var clusterRosterEnv = new Dictionary + { + ["IGGY_CLUSTER_ENABLED"] = "true", + ["IGGY_CLUSTER_NAME"] = "test-cluster", + ["IGGY_CLUSTER_NODES_0_NAME"] = "leader-node", + ["IGGY_CLUSTER_NODES_0_IP"] = "127.0.0.1", + ["IGGY_CLUSTER_NODES_0_REPLICA_ID"] = "0", + ["IGGY_CLUSTER_NODES_0_PORTS_TCP"] = _leaderTcpPort.ToString(), + ["IGGY_CLUSTER_NODES_0_PORTS_QUIC"] = _leaderQuicPort.ToString(), + ["IGGY_CLUSTER_NODES_0_PORTS_HTTP"] = _leaderHttpPort.ToString(), + ["IGGY_CLUSTER_NODES_0_PORTS_WEBSOCKET"] = _leaderWsPort.ToString(), + ["IGGY_CLUSTER_NODES_1_NAME"] = "follower-node", + ["IGGY_CLUSTER_NODES_1_IP"] = "127.0.0.1", + ["IGGY_CLUSTER_NODES_1_REPLICA_ID"] = "1", + ["IGGY_CLUSTER_NODES_1_PORTS_TCP"] = _followerTcpPort.ToString(), + ["IGGY_CLUSTER_NODES_1_PORTS_QUIC"] = _followerQuicPort.ToString(), + ["IGGY_CLUSTER_NODES_1_PORTS_HTTP"] = _followerHttpPort.ToString(), + ["IGGY_CLUSTER_NODES_1_PORTS_WEBSOCKET"] = _followerWsPort.ToString(), + }; + + _leaderContainer = new ContainerBuilder(DockerImage) + .WithName($"iggy-leader-{Guid.NewGuid():N}") + .WithCommand("--replica-id", "0") + .WithNetwork(_network) + .WithNetworkAliases(LeaderAlias) + .WithPortBinding(_leaderTcpPort.ToString(), _leaderTcpPort.ToString()) + .WithPortBinding(_leaderHttpPort.ToString(), _leaderHttpPort.ToString()) + .WithEnvironment("RUST_LOG", "trace") + .WithEnvironment("IGGY_SYSTEM_LOGGING_LEVEL", "trace") + .WithEnvironment("IGGY_ROOT_USERNAME", "iggy") + .WithEnvironment("IGGY_ROOT_PASSWORD", "iggy") + .WithEnvironment("IGGY_SYSTEM_PATH", "local_data_leader") + .WithEnvironment("IGGY_TCP_ADDRESS", $"0.0.0.0:{_leaderTcpPort}") + .WithEnvironment("IGGY_HTTP_ADDRESS", $"0.0.0.0:{_leaderHttpPort}") + .WithEnvironment("IGGY_QUIC_ADDRESS", $"0.0.0.0:{_leaderQuicPort}") + .WithEnvironment("IGGY_WEBSOCKET_ADDRESS", $"0.0.0.0:{_leaderWsPort}") + .WithEnvironment(clusterRosterEnv) + .WithPrivileged(true) + .WithCleanUp(true) + .WithWaitStrategy(Wait.ForUnixContainer().UntilInternalTcpPortIsAvailable(_leaderTcpPort)) + .Build(); + + _followerContainer = new ContainerBuilder(DockerImage) + .WithName($"iggy-follower-{Guid.NewGuid():N}") + .WithCommand("--follower", "--replica-id", "1") + .WithNetwork(_network) + .WithNetworkAliases(FollowerAlias) + .WithPortBinding(_followerTcpPort.ToString(), _followerTcpPort.ToString()) + .WithPortBinding(_followerHttpPort.ToString(), _followerHttpPort.ToString()) + .WithEnvironment("RUST_LOG", "trace") + .WithEnvironment("IGGY_SYSTEM_LOGGING_LEVEL", "trace") + .WithEnvironment("IGGY_ROOT_USERNAME", "iggy") + .WithEnvironment("IGGY_ROOT_PASSWORD", "iggy") + .WithEnvironment("IGGY_SYSTEM_PATH", "local_data_follower") + .WithEnvironment("IGGY_TCP_ADDRESS", $"0.0.0.0:{_followerTcpPort}") + .WithEnvironment("IGGY_HTTP_ADDRESS", $"0.0.0.0:{_followerHttpPort}") + .WithEnvironment("IGGY_QUIC_ADDRESS", $"0.0.0.0:{_followerQuicPort}") + .WithEnvironment("IGGY_WEBSOCKET_ADDRESS", $"0.0.0.0:{_followerWsPort}") + .WithEnvironment(clusterRosterEnv) + .WithPrivileged(true) + .WithCleanUp(true) + .WithWaitStrategy(Wait.ForUnixContainer().UntilInternalTcpPortIsAvailable(_followerTcpPort)) + .Build(); + } + + public async ValueTask DisposeAsync() + { + await SaveContainerLogsAsync(_leaderContainer, "leader"); + await SaveContainerLogsAsync(_followerContainer, "follower"); + await _followerContainer.StopAsync(); + await _leaderContainer.StopAsync(); + await _network.DeleteAsync(); + } + + public async Task InitializeAsync() + { + await _network.CreateAsync(); + await Task.WhenAll(_leaderContainer.StartAsync(), _followerContainer.StartAsync()); + } + + public string GetLeaderAddress() + { + return $"127.0.0.1:{_leaderTcpPort}"; + } + + public string GetFollowerAddress() + { + return $"127.0.0.1:{_followerTcpPort}"; + } + + private ushort ReservePort() + { + for (ushort candidate = BasePort; candidate < EndPort; candidate++) + { + try + { + var listener = new TcpListener(IPAddress.Loopback, candidate); + listener.Start(); + _portReservations.Add(listener); + return candidate; + } + catch (SocketException) + { + // Port is held by a previous ReservePort() in this fixture + // (the common case) or by something else on the host; keep + // walking the range. + } + } + + throw new InvalidOperationException( + $"No free ports available in [{BasePort}, {EndPort}) for .NET {Environment.Version.Major}.x."); + } + + private void ReleaseReservedPorts() + { + foreach (var listener in _portReservations) + { + listener.Stop(); + } + + _portReservations.Clear(); + } + + private static async Task SaveContainerLogsAsync(IContainer container, string role) + { + if (string.IsNullOrEmpty(LogDirectory)) + { + return; + } + + try + { + Directory.CreateDirectory(LogDirectory); + var dotnetVersion = $"net{Environment.Version.Major}.{Environment.Version.Minor}"; + var logFilePath = Path.Combine(LogDirectory, $"iggy-{role}-{dotnetVersion}-{container.Name}.log"); + + var (stdout, stderr) = await container.GetLogsAsync(); + + await using var writer = new StreamWriter(logFilePath); + if (!string.IsNullOrEmpty(stdout)) + { + await writer.WriteLineAsync("=== STDOUT ==="); + await writer.WriteLineAsync(stdout); + } + + if (!string.IsNullOrEmpty(stderr)) + { + await writer.WriteLineAsync("=== STDERR ==="); + await writer.WriteLineAsync(stderr); + } + } + catch (Exception ex) + { + Console.WriteLine($"Failed to save {role} container logs: {ex.Message}"); + } + } +} diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyServerFixture.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyServerFixture.cs index 1670cbcdf4..8c6718b607 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyServerFixture.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyServerFixture.cs @@ -21,35 +21,35 @@ using Apache.Iggy.Factory; using Apache.Iggy.IggyClient; using Apache.Iggy.Tests.Integrations.Helpers; +using DotNet.Testcontainers.Builders; +using DotNet.Testcontainers.Containers; using TUnit.Core.Interfaces; namespace Apache.Iggy.Tests.Integrations.Fixtures; -///

-/// Runs the suite against an iggy-server : a standalone node by default, -/// or a replicated cluster when IGGY_TEST_CLUSTER_NODES asks for one, so every test commits through -/// consensus. The SDK frames TCP with the VSR wire protocol. -/// public class IggyServerFixture : IAsyncInitializer, IAsyncDisposable { private readonly string _containerId = Guid.NewGuid().ToString(); - private readonly SemaphoreSlim _startGate = new(1, 1); - - private VsrCluster? _cluster; + protected IContainer? IggyContainer; /// /// Docker image to use. Can be overridden via IGGY_SERVER_DOCKER_IMAGE environment variable - /// or by subclasses. Defaults to the locally built iggy-server:test; build it with - /// docker build -f core/server/Dockerfile -t iggy-server:test . from the repository root. + /// or by subclasses. Defaults to apache/iggy:edge if not specified. /// - protected virtual string DockerImage => - Environment.GetEnvironmentVariable("IGGY_SERVER_DOCKER_IMAGE") ?? "iggy-server:test"; + private string DockerImage => + Environment.GetEnvironmentVariable("IGGY_SERVER_DOCKER_IMAGE") ?? "apache/iggy:edge"; /// - /// Names the containers and network of this fixture's cluster, so a `docker ps` during a run - /// shows which fixture owns which node. + /// Environment variables for the container. Override in subclasses to customize. /// - protected virtual string ClusterName => "general"; + protected virtual Dictionary EnvironmentVariables => new() + { + { "IGGY_ROOT_USERNAME", "iggy" }, + { "IGGY_ROOT_PASSWORD", "iggy" }, + { "IGGY_TCP_ADDRESS", "0.0.0.0:8090" }, + { "IGGY_HTTP_ADDRESS", "0.0.0.0:3000" }, + { "IGGY_SYSTEM_TOPIC_MESSAGE_EXPIRY", "10m" } + }; /// /// Enables iggy server trace logs. @@ -57,85 +57,159 @@ public class IggyServerFixture : IAsyncInitializer, IAsyncDisposable protected bool EnabledServerTraceLogs => true; /// - /// Extra environment variables for every node, layered over the cluster's base configuration. - /// Override in subclasses to customize. - /// - protected virtual Dictionary EnvironmentVariables => []; - - /// - /// Resource mappings (certificates, etc.) mounted into every node. Override in subclasses. + /// Resource mappings (volumes, etc.) for the container. Override in subclasses to add custom mappings. /// protected virtual ResourceMapping[] ResourceMappings => []; /// - /// Cluster size, mirroring the Rust integration harness knob of the same name: the - /// IGGY_TEST_CLUSTER_NODES environment variable decides (default 1, a standalone node with - /// clustering disabled; 2 or more, a replicated cluster). A subclass can pin its own size - /// instead, the way the redirection cluster stays three nodes whatever the knob says. + /// Directory for container log files. Set via IGGY_TEST_LOGS_DIR environment variable. + /// If not set, container logs will not be saved to file. /// - protected virtual int NodeCount => - int.TryParse(Environment.GetEnvironmentVariable("IGGY_TEST_CLUSTER_NODES"), out var count) && count >= 1 - ? count - : 1; + private static string? LogDirectory => + Environment.GetEnvironmentVariable("IGGY_TEST_LOGS_DIR"); + + public IggyServerFixture() + { + var builder = new ContainerBuilder(DockerImage) + .WithPortBinding(3000, true) + .WithPortBinding(8090, true) + .WithWaitStrategy(Wait.ForUnixContainer() + .UntilInternalTcpPortIsAvailable(8090) + .UntilHttpRequestIsSucceeded(request => request + .ForPort(3000) + .ForPath("/ping"))) + .WithName(_containerId) + .WithPrivileged(true) + .WithCleanUp(true); + + foreach (var (key, value) in EnvironmentVariables) + { + builder = builder.WithEnvironment(key, value); + } + + if (EnabledServerTraceLogs) + { + builder = builder + .WithEnvironment("IGGY_SYSTEM_LOGGING_LEVEL", "trace") + .WithEnvironment("RUST_LOG", "trace"); + } + + foreach (var mapping in ResourceMappings) + { + builder = builder.WithResourceMapping(mapping.Source, mapping.Destination); + } + + IggyContainer = builder.Build(); + } public async ValueTask DisposeAsync() { - if (_cluster != null) + if (IggyContainer == null) { - await _cluster.DisposeAsync(); + return; } + + await SaveContainerLogsAsync(); + await IggyContainer.StopAsync(); } - /// - /// The cluster starts on first use, so a run that never dials the server does not pay for it. - /// - public Task InitializeAsync() + public virtual async Task InitializeAsync() { - return Task.CompletedTask; + await IggyContainer!.StartAsync(); + + await CreateTcpClient(); + await CreateHttpClient(); + } + + private async Task SaveContainerLogsAsync() + { + if (string.IsNullOrEmpty(LogDirectory)) + { + return; + } + + try + { + Directory.CreateDirectory(LogDirectory); + var dotnetVersion = $"net{Environment.Version.Major}.{Environment.Version.Minor}"; + var logFilePath = Path.Combine(LogDirectory, $"iggy-server-{dotnetVersion}-{_containerId}.log"); + + var (stdout, stderr) = await IggyContainer!.GetLogsAsync(); + + await using var writer = new StreamWriter(logFilePath); + if (!string.IsNullOrEmpty(stdout)) + { + await writer.WriteLineAsync("=== STDOUT ==="); + await writer.WriteLineAsync(stdout); + } + + if (!string.IsNullOrEmpty(stderr)) + { + await writer.WriteLineAsync("=== STDERR ==="); + await writer.WriteLineAsync(stderr); + } + } + catch (Exception ex) + { + Console.WriteLine($"Failed to save container logs: {ex.Message}"); + } + } + + public async Task> CreateClients() + { + var dictionary = new Dictionary(); + dictionary[Protocol.Tcp] = await CreateTcpClient(); + dictionary[Protocol.Http] = await CreateHttpClient(); + + return dictionary; } - /// - /// Under VSR the register handshake is the login, so auto-login on top of the explicit one would register - /// twice on the same connection: the server answers the second one by replaying the binding it already - /// holds, and the client then carries a client id the server never bound, which consumer-group - /// membership is keyed by. - /// public async Task CreateAuthenticatedClient(Protocol protocol, string userName = "iggy", - string password = "iggy", IMessageEncryptor? encryptor = null) + string password = "iggy") { - var client = await CreateClient(protocol, protocol == Protocol.Http, - encryptor: encryptor, userName: userName, password: password); - await client.LoginUserAsync(userName, password); + return protocol == Protocol.Tcp + ? await CreateTcpClient(userName, password) + : await CreateHttpClient(userName, password); + } + + public async Task CreateTcpClient(string userName = "iggy", string password = "iggy", + bool connect = true, IMessageEncryptor? encryptor = null) + { + var client = await CreateClient(Protocol.Tcp, connect: connect, encryptor: encryptor); + + if (connect) + { + await client.LoginUserAsync(userName, password); + } return client; } - /// - /// A connected client that has not logged in, so the caller owns the handshake. - /// - public async Task CreateUnauthenticatedClient(Protocol protocol) + public async Task CreateHttpClient(string userName = "iggy", string password = "iggy", + IMessageEncryptor? encryptor = null) { - return await CreateClient(protocol); + var client = await CreateClient(Protocol.Http, encryptor: encryptor); + + await client.LoginUserAsync(userName, password); + + return client; } - /// - /// overrides the cluster address, so a test can dial the server through a - /// proxy while keeping the rest of the configuration identical. - /// - public async Task CreateClient(Protocol protocol, bool autoLogin = false, bool connect = true, - IMessageEncryptor? encryptor = null, string? address = null, string userName = "iggy", - string password = "iggy") + public async Task CreateClient(Protocol protocol, Protocol? targetContainer = null, + bool connect = true, IMessageEncryptor? encryptor = null) { + var address = GetIggyAddress(protocol); + var client = IggyClientFactory.CreateClient(new IggyClientConfigurator { - BaseAddress = address ?? await GetIggyAddressAsync(protocol), + BaseAddress = address, Protocol = protocol, ReconnectionSettings = new ReconnectionSettings { Enabled = true }, AutoLoginSettings = new AutoLoginSettings { - Enabled = autoLogin, - Username = userName, - Password = password + Enabled = true, + Username = "iggy", + Password = "iggy" }, MessageEncryptor = encryptor }); @@ -148,58 +222,20 @@ public async Task CreateClient(Protocol protocol, bool autoLogin = return client; } - public async Task GetIggyAddressAsync(Protocol protocol) - { - var cluster = await EnsureClusterStartedAsync(); - - return protocol == Protocol.Tcp ? cluster.LeaderTcpAddress : cluster.LeaderHttpAddress; - } - - /// - /// A backup node of the initial view, for tests that dial a follower and expect the client to be - /// redirected to the primary. - /// - public async Task GetFollowerTcpAddressAsync() + public virtual string GetIggyAddress(Protocol protocol) { - var cluster = await EnsureClusterStartedAsync(); + var port = protocol == Protocol.Tcp + ? IggyContainer!.GetMappedPublicPort(8090) + : IggyContainer!.GetMappedPublicPort(3000); - return cluster.FollowerTcpAddress; + return protocol == Protocol.Tcp + ? $"127.0.0.1:{port}" + : $"http://127.0.0.1:{port}"; } public static IEnumerable> ProtocolData() { - return [() => Protocol.Http, () => Protocol.Tcp]; - } - - private async Task EnsureClusterStartedAsync() - { - await _startGate.WaitAsync(); - try - { - if (_cluster == null) - { - var cluster = new VsrCluster(DockerImage, ClusterName, _containerId, - EnabledServerTraceLogs, NodeCount, EnvironmentVariables, ResourceMappings); - try - { - await cluster.StartAsync(); - } - catch - { - // A later retry rebuilds the cluster under the same name, so a half-started one - // must not leave its network or containers behind. - await cluster.DisposeAsync(); - throw; - } - - _cluster = cluster; - } - - return _cluster; - } - finally - { - _startGate.Release(); - } + yield return () => Protocol.Http; + yield return () => Protocol.Tcp; } } diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyTlsServerFixture.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyTlsServerFixture.cs index 2b412503b2..0d7ff109df 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyTlsServerFixture.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyTlsServerFixture.cs @@ -25,8 +25,6 @@ namespace Apache.Iggy.Tests.Integrations.Fixtures; /// public class IggyTlsServerFixture : IggyServerFixture { - protected override string ClusterName => "tls"; - /// /// Environment variables with TLS configuration enabled. /// @@ -45,5 +43,8 @@ public class IggyTlsServerFixture : IggyServerFixture new("Certs", "/app/certs/") ]; - protected override int NodeCount => 1; + public override async Task InitializeAsync() + { + await IggyContainer!.StartAsync(); + } } diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/VsrCluster.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/VsrCluster.cs deleted file mode 100644 index 512b86a0e2..0000000000 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/VsrCluster.cs +++ /dev/null @@ -1,438 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Net; -using System.Net.Sockets; -using Apache.Iggy.Tests.Integrations.Helpers; -using Docker.DotNet; -using Docker.DotNet.Models; -using DotNet.Testcontainers.Builders; -using DotNet.Testcontainers.Containers; -using DotNet.Testcontainers.Networks; - -namespace Apache.Iggy.Tests.Integrations.Fixtures; - -/// -/// The iggy-server deployment a run scoped to that server uses. A single node runs with -/// clustering disabled, exercising the wire protocol without replication; two or more nodes form -/// a real roster that puts every test through consensus. -/// -internal sealed class VsrCluster : IAsyncDisposable -{ - private const string ClusterName = "test-vsr-cluster"; - - // TCP and HTTP get real host ports on every node: a view change can make any replica the - // primary clients are redirected to, so every advertised 127.0.0.1:port must be dialable from - // the host. QUIC/WebSocket/replica listeners are reachable only on the cluster network and get - // base + node. The offset is not for the network (each node has its own IP there) but because - // the server rejects a roster where two advertised endpoints collide. - private const ushort InternalQuicPortBase = 8200; - private const ushort InternalWebSocketPortBase = 8300; - private const ushort InternalReplicaPortBase = 8400; - - // Host-port pool, partitioned per .NET major so the parallel `dotnet test` processes - // (net8.0 -> 29800..29949, net10.0 -> 30000..30149) never race each other for a port. - private static readonly ushort BasePort = (ushort)(29000 + Environment.Version.Major * 100); - private static readonly ushort EndPort = (ushort)(BasePort + 150); - - // A reserved port is released before its container binds it, so another cluster in this process - // could scan onto it in that window. Ports stay claimed for the process lifetime to close it. - private static readonly HashSet ClaimedPorts = []; - - private readonly IContainer?[] _containers; - private readonly IReadOnlyDictionary? _extraEnvironment; - private readonly string _idSuffix; - private readonly string _image; - private readonly string _name; - private INetwork? _network; - private readonly int _nodeCount; - private readonly NodePorts[] _ports; - private readonly IReadOnlyList? _resourceMappings; - private readonly bool _traceLogs; - - /// Replica 0 - primary of the initial view, and the node the tests talk to. - public string LeaderTcpAddress => $"127.0.0.1:{_ports[0].Tcp}"; - - /// The same node's REST surface, which serves classic framing rather than VSR. - public string LeaderHttpAddress => $"http://127.0.0.1:{_ports[0].Http}"; - - /// A backup of the initial view, so a client dialing it gets redirected to the primary. - public string FollowerTcpAddress => ClusterEnabled - ? $"127.0.0.1:{_ports[1].Tcp}" - : throw new InvalidOperationException( - "A follower address requires a cluster; set IGGY_TEST_CLUSTER_NODES to 2 or more."); - - /// The same backup node's REST surface. - public string FollowerHttpAddress => ClusterEnabled - ? $"http://127.0.0.1:{_ports[1].Http}" - : throw new InvalidOperationException( - "A follower address requires a cluster; set IGGY_TEST_CLUSTER_NODES to 2 or more."); - - private bool ClusterEnabled => _nodeCount > 1; - - private static string? LogDirectory => - Environment.GetEnvironmentVariable("IGGY_TEST_LOGS_DIR"); - - /// - /// of 1 starts a standalone node with clustering disabled, higher - /// values a cluster with that many replicas. labels the containers and - /// network after the owning fixture. wins over the - /// base configuration, and are mounted into every node. - /// - public VsrCluster(string image, string name, string idSuffix, bool traceLogs, int nodeCount, - IReadOnlyDictionary? extraEnvironment = null, - IReadOnlyList? resourceMappings = null) - { - _image = image; - _name = name; - _idSuffix = idSuffix; - _traceLogs = traceLogs; - _extraEnvironment = extraEnvironment; - _resourceMappings = resourceMappings; - _nodeCount = nodeCount; - _ports = ReservePorts(_nodeCount); - _containers = new IContainer[_nodeCount]; - } - - public async ValueTask DisposeAsync() - { - for (var node = 0; node < _nodeCount; node++) - { - if (_containers[node] == null) - { - continue; - } - - try - { - await SaveContainerLogsAsync(_containers[node]!, $"iggy-server-{node}"); - } - catch (Exception e) - { - Console.WriteLine($"Failed to save the logs of iggy-server-{node}: {e}"); - } - } - - foreach (var container in _containers) - { - if (container == null) - { - continue; - } - - try - { - await container.DisposeAsync(); - } - catch (Exception e) - { - Console.WriteLine($"Failed to dispose an iggy-server container: {e}"); - } - } - - if (_network != null) - { - try - { - await _network.DeleteAsync(); - } - catch (Exception e) - { - Console.WriteLine($"Failed to delete the iggy-server network: {e}"); - } - } - } - - /// - /// The containers are built here rather than in the constructor because a roster needs every - /// node's IP before any node starts, and those IPs come from the subnet the network is - /// created with. - /// - public async Task StartAsync() - { - var nodeAddresses = Array.Empty(); - if (ClusterEnabled) - { - var subnetPrefix = await CreateNetworkAsync(); - // The gateway takes .1, so node addresses start at .10. - nodeAddresses = Enumerable.Range(0, _nodeCount) - .Select(node => $"{subnetPrefix}.{10 + node}") - .ToArray(); - } - - Dictionary clusterEnvironment = BuildClusterEnvironment(nodeAddresses); - for (var node = 0; node < _nodeCount; node++) - { - _containers[node] = BuildNodeContainer(node, nodeAddresses, clusterEnvironment); - } - - // A multi-node roster needs a quorum of replicas, so nothing commits until enough nodes are up. - await Task.WhenAll(_containers.Select(container => container!.StartAsync())); - } - - /// - /// Roster peers dial each other by literal IP - the ip field is parsed, not resolved - so every - /// node gets a static address, and Docker only honors static addresses on networks created with - /// an explicit subnet. The subnet is picked from a private /16 here; a candidate overlapping - /// another network on the host fails the create and the next one is tried. - /// - private async Task CreateNetworkAsync() - { - for (var candidate = 0; candidate < 256; candidate++) - { - var subnetPrefix = $"10.213.{candidate}"; - var network = new NetworkBuilder() - .WithName($"iggy-vsr-{_name}-{_idSuffix}") - .WithCreateParameterModifier(parameters => parameters.IPAM = new IPAM - { - Config = [new IPAMConfig { Subnet = $"{subnetPrefix}.0/24" }] - }) - .Build(); - - try - { - await network.CreateAsync(); - _network = network; - return subnetPrefix; - } - catch (DockerApiException) - { - // The subnet overlaps a network already on the host. - } - } - - throw new InvalidOperationException("No free /24 subnet in 10.213.0.0/16 for the cluster network."); - } - - private IContainer BuildNodeContainer(int node, IReadOnlyList nodeAddresses, - IReadOnlyDictionary clusterEnvironment) - { - var ports = _ports[node]; - var builder = new ContainerBuilder(_image) - .WithName($"iggy-vsr-{_name}-{node}-{_idSuffix}") - .WithEnvironment("IGGY_ROOT_USERNAME", "iggy") - .WithEnvironment("IGGY_ROOT_PASSWORD", "iggy") - .WithEnvironment("IGGY_SYSTEM_TOPIC_MESSAGE_EXPIRY", "10m") - .WithEnvironment("IGGY_SYSTEM_PATH", $"local_data_vsr_{node}") - .WithEnvironment("IGGY_TCP_ADDRESS", $"0.0.0.0:{ports.Tcp}") - .WithEnvironment("IGGY_HTTP_ADDRESS", $"0.0.0.0:{ports.Http}") - .WithEnvironment("IGGY_QUIC_ADDRESS", $"0.0.0.0:{ports.Quic}") - .WithEnvironment("IGGY_WEBSOCKET_ADDRESS", $"0.0.0.0:{ports.WebSocket}") - .WithEnvironment(clusterEnvironment) - .WithPrivileged(true) - .WithCleanUp(true) - .WithWaitStrategy(Wait.ForUnixContainer() - .UntilInternalTcpPortIsAvailable(ports.Tcp) - .UntilInternalTcpPortIsAvailable(ports.Http)); - - // Host bindings mirror the container port so the loopback address advertised in cluster - // metadata resolves to the node that advertised it. Every node needs one: view changes - // can make any replica the primary clients get redirected to. - builder = builder - .WithPortBinding(ports.Tcp.ToString(), ports.Tcp.ToString()) - .WithPortBinding(ports.Http.ToString(), ports.Http.ToString()); - - if (ClusterEnabled) - { - var address = nodeAddresses[node]; - builder = builder - .WithCommand("--replica-id", node.ToString()) - .WithNetwork(_network) - .WithNetworkAliases($"vsr-node-{node}") - .WithCreateParameterModifier(parameters => - AssignStaticAddress(parameters, _network!.Name, address)); - } - - if (_traceLogs) - { - builder = builder - .WithEnvironment("IGGY_SYSTEM_LOGGING_LEVEL", "trace") - .WithEnvironment("RUST_LOG", "trace"); - } - - if (_extraEnvironment != null) - { - foreach (var (key, value) in _extraEnvironment) - { - builder = builder.WithEnvironment(key, value); - } - } - - if (_resourceMappings != null) - { - foreach (var mapping in _resourceMappings) - { - builder = builder.WithResourceMapping(mapping.Source, mapping.Destination); - } - } - - return builder.Build(); - } - - private Dictionary BuildClusterEnvironment(IReadOnlyList nodeAddresses) - { - var environment = new Dictionary - { - ["IGGY_CLUSTER_ENABLED"] = ClusterEnabled ? "true" : "false" - }; - - if (!ClusterEnabled) - { - return environment; - } - - environment["IGGY_CLUSTER_NAME"] = ClusterName; - environment["IGGY_MESSAGE_BUS_RECONNECT_PERIOD"] = "100ms"; - - for (var node = 0; node < _nodeCount; node++) - { - var ports = _ports[node]; - environment[$"IGGY_CLUSTER_NODES_{node}_NAME"] = $"vsr-node-{node}"; - environment[$"IGGY_CLUSTER_NODES_{node}_IP"] = nodeAddresses[node]; - environment[$"IGGY_CLUSTER_NODES_{node}_ADVERTISED_ADDRESS"] = "127.0.0.1"; - environment[$"IGGY_CLUSTER_NODES_{node}_REPLICA_ID"] = node.ToString(); - environment[$"IGGY_CLUSTER_NODES_{node}_PORTS_TCP"] = ports.Tcp.ToString(); - environment[$"IGGY_CLUSTER_NODES_{node}_PORTS_HTTP"] = ports.Http.ToString(); - environment[$"IGGY_CLUSTER_NODES_{node}_PORTS_QUIC"] = ports.Quic.ToString(); - environment[$"IGGY_CLUSTER_NODES_{node}_PORTS_WEBSOCKET"] = ports.WebSocket.ToString(); - environment[$"IGGY_CLUSTER_NODES_{node}_PORTS_TCP_REPLICA"] = ports.Replica.ToString(); - } - - return environment; - } - - /// - /// Pins the container's address on the cluster network. Testcontainers has no first-class knob for it, - /// and the roster needs the address before any container starts. - /// - private static void AssignStaticAddress(CreateContainerParameters parameters, string networkName, - string address) - { - parameters.NetworkingConfig ??= new NetworkingConfig(); - parameters.NetworkingConfig.EndpointsConfig ??= new Dictionary(); - - if (!parameters.NetworkingConfig.EndpointsConfig.TryGetValue(networkName, out var endpoint)) - { - endpoint = new EndpointSettings(); - parameters.NetworkingConfig.EndpointsConfig[networkName] = endpoint; - } - - endpoint.IPAMConfig = new EndpointIPAMConfig { IPv4Address = address }; - } - - /// - /// Finds free host ports for the host-dialed endpoints by briefly binding them, so concurrent - /// clusters cannot pick the same port. The listeners are released before the containers start. - /// - private static NodePorts[] ReservePorts(int nodeCount) - { - var reservations = new List(); - try - { - var ports = new NodePorts[nodeCount]; - for (var node = 0; node < nodeCount; node++) - { - ports[node] = new NodePorts(ReservePort(reservations), - ReservePort(reservations), - (ushort)(InternalQuicPortBase + node), - (ushort)(InternalWebSocketPortBase + node), - (ushort)(InternalReplicaPortBase + node)); - } - - return ports; - } - finally - { - foreach (var listener in reservations) - { - listener.Stop(); - } - } - } - - private static ushort ReservePort(List reservations) - { - lock (ClaimedPorts) - { - for (var candidate = BasePort; candidate < EndPort; candidate++) - { - if (ClaimedPorts.Contains(candidate)) - { - continue; - } - - try - { - var listener = new TcpListener(IPAddress.Loopback, candidate); - listener.Start(); - reservations.Add(listener); - ClaimedPorts.Add(candidate); - return candidate; - } - catch (SocketException) - { - // Held by something else on the host. - } - } - } - - throw new InvalidOperationException( - $"No free ports available in [{BasePort}, {EndPort}) for .NET {Environment.Version.Major}.x."); - } - - private static async Task SaveContainerLogsAsync(IContainer container, string role) - { - if (string.IsNullOrEmpty(LogDirectory)) - { - return; - } - - try - { - Directory.CreateDirectory(LogDirectory); - var dotnetVersion = $"net{Environment.Version.Major}.{Environment.Version.Minor}"; - // Docker hands back names with a leading slash, which Path.Combine would read as a directory. - var containerName = container.Name.TrimStart('/'); - var logFilePath = Path.Combine(LogDirectory, $"{role}-{dotnetVersion}-{containerName}.log"); - - var (stdout, stderr) = await container.GetLogsAsync(); - - await using var writer = new StreamWriter(logFilePath); - if (!string.IsNullOrEmpty(stdout)) - { - await writer.WriteLineAsync("=== STDOUT ==="); - await writer.WriteLineAsync(stdout); - } - - if (!string.IsNullOrEmpty(stderr)) - { - await writer.WriteLineAsync("=== STDERR ==="); - await writer.WriteLineAsync(stderr); - } - } - catch (Exception ex) - { - Console.WriteLine($"Failed to save {role} container logs: {ex.Message}"); - } - } - - /// - /// The ports one node listens on. The leader's and first follower's TCP/HTTP are host ports - /// mirrored into the container; the rest are the fixed cluster-network ports. - /// - private readonly record struct NodePorts(ushort Tcp, ushort Http, ushort Quic, ushort WebSocket, ushort Replica); -} diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/FlushMessagesTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/FlushMessagesTests.cs index ea258700b8..778b707bac 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/FlushMessagesTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/FlushMessagesTests.cs @@ -17,8 +17,11 @@ using Apache.Iggy.Enums; using Apache.Iggy.Exceptions; +using Apache.Iggy.IggyClient; +using Apache.Iggy.Messages; using Apache.Iggy.Tests.Integrations.Fixtures; using Shouldly; +using Partitioning = Apache.Iggy.Kinds.Partitioning; namespace Apache.Iggy.Tests.Integrations; @@ -27,14 +30,62 @@ public class FlushMessagesTests [ClassDataSource(Shared = SharedType.PerAssembly)] public required IggyServerFixture Fixture { get; init; } + private async Task<(IIggyClient client, string streamName, string topicName)> CreateStreamWithMessages( + Protocol protocol) + { + var client = await Fixture.CreateAuthenticatedClient(protocol); + + var streamName = $"flush-{Guid.NewGuid():N}"; + var topicName = "test-topic"; + + await client.CreateStreamAsync(streamName); + await client.CreateTopicAsync(Identifier.String(streamName), topicName, 1); + + await client.SendMessagesAsync(Identifier.String(streamName), + Identifier.String(topicName), Partitioning.None(), + [ + new Message(Guid.NewGuid(), "Test message 1"u8.ToArray()), + new Message(Guid.NewGuid(), "Test message 2"u8.ToArray()), + new Message(Guid.NewGuid(), "Test message 3"u8.ToArray()), + new Message(Guid.NewGuid(), "Test message 4"u8.ToArray()) + ]); + + return (client, streamName, topicName); + } + [Test] [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] - public async Task FlushUnsavedBuffer_Should_Throw_FeatureUnavailable(Protocol protocol) + public async Task FlushUnsavedBuffer_WithFsync_Should_Flush_Successfully(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var (client, streamName, topicName) = await CreateStreamWithMessages(protocol); + + await Should.NotThrowAsync(() => + client.FlushUnsavedBufferAsync( + Identifier.String(streamName), + Identifier.String(topicName), 0, true)); + } + + [Test] + [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] + public async Task FlushUnsavedBuffer_WithOutFsync_Should_Flush_Successfully(Protocol protocol) + { + var (client, streamName, topicName) = await CreateStreamWithMessages(protocol); + + await Should.NotThrowAsync(() => + client.FlushUnsavedBufferAsync( + Identifier.String(streamName), + Identifier.String(topicName), 0, false)); + } + + [Test] + [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] + public async Task FlushUnsavedBuffer_Should_Throw_WhenPartition_DoesNotExist(Protocol protocol) + { + var (client, streamName, topicName) = await CreateStreamWithMessages(protocol); - await Should.ThrowAsync(() => - client.FlushUnsavedBufferAsync(Identifier.String("any-stream"), - Identifier.String("any-topic"), 1, false)); + await Should.ThrowAsync(() => + client.FlushUnsavedBufferAsync( + Identifier.String(streamName), + Identifier.String(topicName), 55, false)); } } diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/HeaderEncryptionIntegrationTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/HeaderEncryptionIntegrationTests.cs index 9de96652d1..4004a5929a 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/HeaderEncryptionIntegrationTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/HeaderEncryptionIntegrationTests.cs @@ -45,8 +45,12 @@ public async Task SendMessages_WithEncryptedHeaders_Should_NotBeReadableWithoutD // Publisher on an encrypting client; a plain client raw-polls the same topic to prove the wire bytes // stay encrypted. - var encryptingClient = await Fixture.CreateAuthenticatedClient(protocol, encryptor: encryptor); - var plainClient = await Fixture.CreateAuthenticatedClient(protocol); + var encryptingClient = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient(encryptor: encryptor) + : await Fixture.CreateHttpClient(encryptor: encryptor); + var plainClient = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStream(plainClient, protocol); var streamId = Identifier.String(testStream.StreamId); @@ -89,11 +93,7 @@ public async Task SendMessages_WithEncryptedHeaders_Should_NotBeReadableWithoutD Dictionary decryptedHeaders = BinaryMapper.MapHeaders(decryptedHeaderBytesResult); decryptedHeaders.Count.ShouldBe(3); - var typeHeader = decryptedHeaders[new HeaderKey - { - Kind = HeaderKind.String, - Value = "type"u8.ToArray() - }]; + var typeHeader = decryptedHeaders[new HeaderKey { Kind = HeaderKind.String, Value = "type"u8.ToArray() }]; Encoding.UTF8.GetString(typeHeader.Value).ShouldBe("test-message"); } @@ -104,7 +104,9 @@ public async Task ReceiveAsync_WithEncryptingClient_Should_DecryptHeadersCorrect var encryptor = CreateEncryptor(); // One encrypting client serves both publisher and consumer: it encrypts on send and decrypts on poll. - var client = await Fixture.CreateAuthenticatedClient(protocol, encryptor: encryptor); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient(encryptor: encryptor) + : await Fixture.CreateHttpClient(encryptor: encryptor); var testStream = await CreateTestStream(client, protocol); var streamId = Identifier.String(testStream.StreamId); @@ -160,25 +162,13 @@ public async Task ReceiveAsync_WithEncryptingClient_Should_DecryptHeadersCorrect received.Message.UserHeaders.ShouldNotBeNull(); received.Message.UserHeaders!.Count.ShouldBe(3); - var batchHeader = received.Message.UserHeaders[new HeaderKey - { - Kind = HeaderKind.String, - Value = "batch"u8.ToArray() - }]; + var batchHeader = received.Message.UserHeaders[new HeaderKey { Kind = HeaderKind.String, Value = "batch"u8.ToArray() }]; BitConverter.ToUInt64(batchHeader.Value).ShouldBe(1UL); - var typeHeader = received.Message.UserHeaders[new HeaderKey - { - Kind = HeaderKind.String, - Value = "type"u8.ToArray() - }]; + var typeHeader = received.Message.UserHeaders[new HeaderKey { Kind = HeaderKind.String, Value = "type"u8.ToArray() }]; Encoding.UTF8.GetString(typeHeader.Value).ShouldBe("test-message"); - var encHeader = received.Message.UserHeaders[new HeaderKey - { - Kind = HeaderKind.String, - Value = "encrypted"u8.ToArray() - }]; + var encHeader = received.Message.UserHeaders[new HeaderKey { Kind = HeaderKind.String, Value = "encrypted"u8.ToArray() }]; encHeader.Value[0].ShouldBe((byte)1); } @@ -192,41 +182,17 @@ private static Dictionary CreateTestHeaders() return new Dictionary { { - new HeaderKey - { - Kind = HeaderKind.String, - Value = "batch"u8.ToArray() - }, - new HeaderValue - { - Kind = HeaderKind.Uint64, - Value = BitConverter.GetBytes(1UL) - } + new HeaderKey { Kind = HeaderKind.String, Value = "batch"u8.ToArray() }, + new HeaderValue { Kind = HeaderKind.Uint64, Value = BitConverter.GetBytes(1UL) } }, { - new HeaderKey - { - Kind = HeaderKind.String, - Value = "type"u8.ToArray() - }, - new HeaderValue - { - Kind = HeaderKind.String, - Value = "test-message"u8.ToArray() - } + new HeaderKey { Kind = HeaderKind.String, Value = "type"u8.ToArray() }, + new HeaderValue { Kind = HeaderKind.String, Value = "test-message"u8.ToArray() } }, { - new HeaderKey - { - Kind = HeaderKind.String, - Value = "encrypted"u8.ToArray() - }, - new HeaderValue - { - Kind = HeaderKind.Bool, - Value = [1] - } - } + new HeaderKey { Kind = HeaderKind.String, Value = "encrypted"u8.ToArray() }, + new HeaderValue { Kind = HeaderKind.Bool, Value = [1] } + }, }; } diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/Helpers/Eventually.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/Helpers/Eventually.cs deleted file mode 100644 index 17cef93eee..0000000000 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/Helpers/Eventually.cs +++ /dev/null @@ -1,43 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -namespace Apache.Iggy.Tests.Integrations.Helpers; - -public static class Eventually -{ - private static readonly TimeSpan PollInterval = TimeSpan.FromMilliseconds(100); - - /// - /// Polls until the read satisfies the condition. Some server work commits ahead of the state it changes - - /// the server applies a purge by advancing a generation that its reconciler acts on a tick later - so the - /// first read after an acknowledged command can still show the old value. - /// - public static async Task ReadAsync(Func> read, Func condition, TimeSpan timeout) - { - var deadline = DateTime.UtcNow + timeout; - while (true) - { - var value = await read(); - if (condition(value) || DateTime.UtcNow >= deadline) - { - return value; - } - - await Task.Delay(PollInterval); - } - } -} diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/IggyConsumerTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/IggyConsumerTests.cs index ef7c062f05..72f7987b8d 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/IggyConsumerTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/IggyConsumerTests.cs @@ -38,7 +38,9 @@ public class IggyConsumerTests [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task InitAsync_WithSingleConsumer_Should_Initialize_Successfully(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -61,7 +63,9 @@ public async Task InitAsync_WithSingleConsumer_Should_Initialize_Successfully(Pr [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task InitAsync_WithConsumerGroup_Should_Initialize_Successfully(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -84,11 +88,13 @@ public async Task InitAsync_WithConsumerGroup_Should_Initialize_Successfully(Pro [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task InitAsync_NewClient_Should_Initialize_Successfully(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStreamWithMessages(client, protocol); - var clientAddress = await Fixture.GetIggyAddressAsync(protocol); + var clientAddress = Fixture.GetIggyAddress(protocol); ; var consumer = IggyConsumerBuilder .Create(Identifier.String(testStream.StreamId), @@ -108,7 +114,9 @@ public async Task InitAsync_NewClient_Should_Initialize_Successfully(Protocol pr [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task InitAsync_CalledTwice_Should_NotThrow(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -131,7 +139,9 @@ public async Task InitAsync_CalledTwice_Should_NotThrow(Protocol protocol) [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task ReceiveAsync_WithoutInit_Should_Throw_ConsumerNotInitializedException(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -159,7 +169,9 @@ await Should.ThrowAsync(async () => [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task InitAsync_WithConsumerGroup_Should_CreateGroup_WhenNotExists(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -192,7 +204,9 @@ public async Task InitAsync_WithConsumerGroup_Should_CreateGroup_WhenNotExists(P public async Task InitAsync_WithConsumerGroup_Should_Throw_WhenGroupNotExists_AndAutoCreateDisabled( Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -216,7 +230,9 @@ public async Task InitAsync_WithConsumerGroup_Should_Throw_WhenGroupNotExists_An [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task InitAsync_WithConsumerGroup_Should_JoinGroup_Successfully(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -245,7 +261,9 @@ await client.CreateConsumerGroupAsync(Identifier.String(testStream.StreamId), [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task DisposeAsync_Should_LeaveConsumerGroup(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -276,7 +294,9 @@ public async Task DisposeAsync_Should_LeaveConsumerGroup(Protocol protocol) [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task ReceiveAsync_WithSingleConsumer_Should_ReceiveMessages_Successfully(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -319,7 +339,9 @@ public async Task ReceiveAsync_WithSingleConsumer_Should_ReceiveMessages_Success [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task ReceiveAsync_WithBatchSize_Should_RespectBatchSize(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -358,7 +380,9 @@ public async Task ReceiveAsync_WithBatchSize_Should_RespectBatchSize(Protocol pr [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task ReceiveAsync_WithPollingInterval_Should_RespectInterval(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -398,7 +422,9 @@ public async Task ReceiveAsync_WithPollingInterval_Should_RespectInterval(Protoc [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task ReceiveAsync_WithAutoCommitAfterReceive_Should_StoreOffset(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -446,7 +472,9 @@ public async Task ReceiveAsync_WithAutoCommitAfterReceive_Should_StoreOffset(Pro [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task ReceiveAsync_WithAutoCommitAfterPoll_Should_StoreOffset(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -493,7 +521,9 @@ public async Task ReceiveAsync_WithAutoCommitAfterPoll_Should_StoreOffset(Protoc [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task StoreOffsetAsync_Should_StoreOffset_Successfully(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -530,7 +560,9 @@ public async Task StoreOffsetAsync_Should_StoreOffset_Successfully(Protocol prot [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task DeleteOffsetAsync_Should_DeleteOffset_Successfully(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -573,7 +605,9 @@ public async Task DeleteOffsetAsync_Should_DeleteOffset_Successfully(Protocol pr [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task DisposeAsync_Should_NotThrow(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -595,7 +629,9 @@ public async Task DisposeAsync_Should_NotThrow(Protocol protocol) [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task DisposeAsync_CalledTwice_Should_NotThrow(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -618,7 +654,9 @@ public async Task DisposeAsync_CalledTwice_Should_NotThrow(Protocol protocol) [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task DisposeAsync_WithoutInit_Should_NotThrow(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -639,7 +677,9 @@ public async Task DisposeAsync_WithoutInit_Should_NotThrow(Protocol protocol) [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task OnPollingError_Should_Fire_WhenPollingFails(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -688,7 +728,9 @@ public async Task OnPollingError_Should_Fire_WhenPollingFails(Protocol protocol) [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task ReceiveAsync_WithOffsetStrategy_Should_StartFromOffset(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -725,7 +767,9 @@ public async Task ReceiveAsync_WithOffsetStrategy_Should_StartFromOffset(Protoco [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task ReceiveAsync_WithFirstStrategy_Should_StartFromBeginning(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -757,7 +801,9 @@ public async Task ReceiveAsync_WithFirstStrategy_Should_StartFromBeginning(Proto [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task ReceiveAsync_WithLastStrategy_Should_StartFromEnd(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStreamWithMessages(client, protocol); diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/IggyPublisherTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/IggyPublisherTests.cs index c50c1ba603..c1028ec0da 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/IggyPublisherTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/IggyPublisherTests.cs @@ -35,8 +35,7 @@ public class IggyPublisherTests [ClassDataSource(Shared = SharedType.PerAssembly)] public required IggyServerFixture Fixture { get; init; } - private async Task CreateTestStream(IIggyClient client, Protocol protocol, - uint partitionsCount = 5) + private async Task CreateTestStream(IIggyClient client, Protocol protocol, uint partitionsCount = 5) { var streamId = $"stream_{Guid.NewGuid()}_{protocol.ToString().ToLowerInvariant()}"; var topicId = "test_topic"; @@ -51,7 +50,9 @@ private async Task CreateTestStream(IIggyClient client, Protocol [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task InitAsync_Should_Initialize_Successfully(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStream(client, protocol); @@ -69,7 +70,7 @@ public async Task InitAsync_Should_Initialize_Successfully(Protocol protocol) [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task InitAsync_NewClient_Should_Initialize_Successfully(Protocol protocol) { - var client = await Fixture.GetIggyAddressAsync(protocol); + var client = Fixture.GetIggyAddress(protocol); var stream = Guid.NewGuid().ToString(); var topic = Guid.NewGuid().ToString(); @@ -90,7 +91,9 @@ public async Task InitAsync_NewClient_Should_Initialize_Successfully(Protocol pr [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task InitAsync_CalledTwice_Should_NotThrow(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStream(client, protocol); @@ -108,7 +111,9 @@ public async Task InitAsync_CalledTwice_Should_NotThrow(Protocol protocol) [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task SendMessages_WithoutInit_Should_Throw_PublisherNotInitializedException(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStream(client, protocol); @@ -127,7 +132,9 @@ public async Task SendMessages_WithoutInit_Should_Throw_PublisherNotInitializedE [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task SendMessages_Should_SendMessages_Successfully(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStream(client, protocol); @@ -163,7 +170,9 @@ public async Task SendMessages_Should_SendMessages_Successfully(Protocol protoco [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task SendMessages_WithEmptyList_Should_NotThrow(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStream(client, protocol); @@ -183,7 +192,9 @@ public async Task SendMessages_WithEmptyList_Should_NotThrow(Protocol protocol) [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task InitAsync_WithStreamAutoCreate_Should_CreateStream(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var streamId = $"auto_stream_{Guid.NewGuid()}_{protocol.ToString().ToLowerInvariant()}"; var topicId = "auto_topic"; @@ -210,7 +221,9 @@ public async Task InitAsync_WithStreamAutoCreate_Should_CreateStream(Protocol pr [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task InitAsync_WithTopicAutoCreate_Should_CreateTopic(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var streamId = $"stream_{Guid.NewGuid()}_{protocol.ToString().ToLowerInvariant()}"; var topicId = "auto_topic"; @@ -239,7 +252,9 @@ public async Task InitAsync_WithTopicAutoCreate_Should_CreateTopic(Protocol prot [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task InitAsync_WithoutAutoCreate_Should_Throw_WhenStreamNotExists(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var streamId = $"nonexistent_stream_{Guid.NewGuid()}"; var topicId = "test_topic"; @@ -257,7 +272,9 @@ public async Task InitAsync_WithoutAutoCreate_Should_Throw_WhenStreamNotExists(P [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task InitAsync_WithoutAutoCreate_Should_Throw_WhenTopicNotExists(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var streamId = $"stream_{Guid.NewGuid()}_{protocol.ToString().ToLowerInvariant()}"; var topicId = "nonexistent_topic"; @@ -278,7 +295,9 @@ public async Task InitAsync_WithoutAutoCreate_Should_Throw_WhenTopicNotExists(Pr [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task SendMessages_WithBackgroundSending_Should_SendMessages_Successfully(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStream(client, protocol); @@ -316,7 +335,9 @@ public async Task SendMessages_WithBackgroundSending_Should_SendMessages_Success [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task WaitUntilAllSends_Should_WaitForPendingMessages(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStream(client, protocol); @@ -351,7 +372,9 @@ public async Task WaitUntilAllSends_Should_WaitForPendingMessages(Protocol proto [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task WaitUntilAllSends_WithoutBackgroundSending_Should_ReturnImmediately(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStream(client, protocol); @@ -372,7 +395,9 @@ public async Task WaitUntilAllSends_WithoutBackgroundSending_Should_ReturnImmedi [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task SendMessages_ToMultiplePartitions_Should_DistributeMessages(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStream(client, protocol); @@ -417,7 +442,9 @@ public async Task SendMessages_ToMultiplePartitions_Should_DistributeMessages(Pr [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task SendMessages_WithBalancedPartitioning_Should_DistributeAcrossPartitions(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStream(client, protocol, 3); @@ -459,7 +486,9 @@ public async Task SendMessages_WithBalancedPartitioning_Should_DistributeAcrossP [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task DisposeAsync_Should_NotThrow(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStream(client, protocol); @@ -476,7 +505,9 @@ public async Task DisposeAsync_Should_NotThrow(Protocol protocol) [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task DisposeAsync_CalledTwice_Should_NotThrow(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStream(client, protocol); @@ -494,7 +525,9 @@ public async Task DisposeAsync_CalledTwice_Should_NotThrow(Protocol protocol) [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task DisposeAsync_WithoutInit_Should_NotThrow(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStream(client, protocol); @@ -511,7 +544,9 @@ public async Task DisposeAsync_WithoutInit_Should_NotThrow(Protocol protocol) [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task StreamId_Should_ReturnConfiguredStreamId(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStream(client, protocol); @@ -528,7 +563,9 @@ public async Task StreamId_Should_ReturnConfiguredStreamId(Protocol protocol) [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task TopicId_Should_ReturnConfiguredTopicId(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStream(client, protocol); @@ -545,7 +582,9 @@ public async Task TopicId_Should_ReturnConfiguredTopicId(Protocol protocol) [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task SendMessages_LargeMessageCount_Should_HandleCorrectly(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStream(client, protocol); @@ -583,7 +622,9 @@ public async Task SendMessages_LargeMessageCount_Should_HandleCorrectly(Protocol [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task SendAsync_RentedBatch_WithBackgroundSending_Should_RoundTrip(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStream(client, protocol); @@ -634,7 +675,9 @@ public async Task SendAsync_RentedBatch_WithBackgroundSending_Should_RoundTrip(P [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task SendAsync_SingleMessageRentedBatch_WithBackgroundSending_Should_RoundTrip(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStream(client, protocol); diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTlsConnectionTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTlsConnectionTests.cs index 2255bbd4f5..4b4200a746 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTlsConnectionTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTlsConnectionTests.cs @@ -34,7 +34,7 @@ public async Task Connect_WithTls_Should_Connect_Successfully() { using var client = IggyClientFactory.CreateClient(new IggyClientConfigurator { - BaseAddress = await Fixture.GetIggyAddressAsync(Protocol.Tcp), + BaseAddress = Fixture.GetIggyAddress(Protocol.Tcp), Protocol = Protocol.Tcp, ReconnectionSettings = new ReconnectionSettings { Enabled = false }, AutoLoginSettings = new AutoLoginSettings @@ -62,15 +62,13 @@ public async Task Connect_WithoutTls_Should_Throw_WhenTlsIsRequired() { using var client = IggyClientFactory.CreateClient(new IggyClientConfigurator { - BaseAddress = await Fixture.GetIggyAddressAsync(Protocol.Tcp), + BaseAddress = Fixture.GetIggyAddress(Protocol.Tcp), Protocol = Protocol.Tcp, ReconnectionSettings = new ReconnectionSettings { Enabled = false } }); - // The VSR register handshake runs inside ConnectAsync and dies against the TLS listener, so the - // client never reaches the connected state. await client.ConnectAsync(); - await Should.ThrowAsync(client.LoginUserAsync("iggy", "iggy")); + await Should.ThrowAsync(client.LoginUserAsync("iggy", "iggy")); } [Test] @@ -78,7 +76,7 @@ public async Task Connect_WithTls_CA_Should_Connect_Successfully() { using var client = IggyClientFactory.CreateClient(new IggyClientConfigurator { - BaseAddress = await Fixture.GetIggyAddressAsync(Protocol.Tcp), + BaseAddress = Fixture.GetIggyAddress(Protocol.Tcp), Protocol = Protocol.Tcp, ReconnectionSettings = new ReconnectionSettings { Enabled = false }, AutoLoginSettings = new AutoLoginSettings diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedConsumerTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedConsumerTests.cs index 460b1c8cef..3861ae812f 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedConsumerTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedConsumerTests.cs @@ -38,7 +38,9 @@ public class IggyTypedConsumerTests [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task ReceiveDeserializedAsync_Should_YieldMessages_WithCorrectData(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -75,7 +77,9 @@ public async Task ReceiveDeserializedAsync_Should_YieldMessages_WithCorrectData( public async Task ReceiveDeserializedAsync_WithoutInit_Should_Throw_ConsumerNotInitializedException( Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -97,7 +101,9 @@ await Should.ThrowAsync(async () => [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task ReceiveDeserializedAsync_WithAutoCommitAfterReceive_Should_StoreOffset(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -136,7 +142,9 @@ public async Task ReceiveDeserializedAsync_WithAutoCommitAfterReceive_Should_Sto public async Task ReceiveDeserializedAsync_WithFailingDeserializer_Should_YieldDeserializationFailed( Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -164,7 +172,9 @@ public async Task ReceiveDeserializedAsync_WithFailingDeserializer_Should_YieldD [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task ReceiveDeserializedAsync_Should_StopCleanly_OnCancellation(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); var testStream = await CreateTestStreamWithMessages(client, protocol); diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedPublisherTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedPublisherTests.cs index 84c5d50816..64d32e40b4 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedPublisherTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedPublisherTests.cs @@ -249,7 +249,9 @@ public async Task SendAsync_WithEncryptor_Should_RoundTrip_Decrypted(Protocol pr // Encryption is configured on the client. The publisher uses an encrypting client; a plain client polls // to prove the wire bytes are ciphertext, then decrypts manually. - var encryptingClient = await Fixture.CreateAuthenticatedClient(protocol, encryptor: encryptor); + var encryptingClient = protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient(encryptor: encryptor) + : await Fixture.CreateHttpClient(encryptor: encryptor); var plainClient = await Client(protocol); var stream = await CreateTestStream(plainClient, protocol); @@ -290,7 +292,9 @@ IggyPublisher publisher private async Task Client(Protocol protocol) { - return await Fixture.CreateAuthenticatedClient(protocol); + return protocol == Protocol.Tcp + ? await Fixture.CreateTcpClient() + : await Fixture.CreateHttpClient(); } // Base fluent methods return the non-generic builder, so apply them as statements to keep the typed Build(). diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/MessageEncryptionIntegrationTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/MessageEncryptionIntegrationTests.cs index 96426ee55f..8e29ec1ba1 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/MessageEncryptionIntegrationTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/MessageEncryptionIntegrationTests.cs @@ -195,7 +195,9 @@ private async Task SendBatch(IIggyClient client, Identifier streamId, Identifier private Task CreateClient(Protocol protocol, IMessageEncryptor encryptor) { - return Fixture.CreateAuthenticatedClient(protocol, encryptor: encryptor); + return protocol == Protocol.Tcp + ? Fixture.CreateTcpClient(encryptor: encryptor) + : Fixture.CreateHttpClient(encryptor: encryptor); } private static AesMessageEncryptor CreateEncryptor() diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/OffsetTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/OffsetTests.cs index f96ced773f..82cc193d7b 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/OffsetTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/OffsetTests.cs @@ -98,13 +98,15 @@ await client.CreateConsumerGroupAsync(Identifier.String(streamName), // For HTTP, a separate TCP client joins (HTTP is stateless and doesn't track membership). if (protocol == Protocol.Tcp) { - await client.JoinConsumerGroupAsync(Identifier.String(streamName), + await client.JoinConsumerGroupAsync( + Identifier.String(streamName), Identifier.String(topicName), Identifier.String("test_consumer_group")); } else { - var tcpClient = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); - await tcpClient.JoinConsumerGroupAsync(Identifier.String(streamName), + var tcpClient = await Fixture.CreateTcpClient(); + await tcpClient.JoinConsumerGroupAsync( + Identifier.String(streamName), Identifier.String(topicName), Identifier.String("test_consumer_group")); } @@ -124,13 +126,15 @@ await client.CreateConsumerGroupAsync(Identifier.String(streamName), if (protocol == Protocol.Tcp) { - await client.JoinConsumerGroupAsync(Identifier.String(streamName), + await client.JoinConsumerGroupAsync( + Identifier.String(streamName), Identifier.String(topicName), Identifier.String("test_consumer_group")); } else { - var tcpClient = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); - await tcpClient.JoinConsumerGroupAsync(Identifier.String(streamName), + var tcpClient = await Fixture.CreateTcpClient(); + await tcpClient.JoinConsumerGroupAsync( + Identifier.String(streamName), Identifier.String(topicName), Identifier.String("test_consumer_group")); } @@ -157,13 +161,15 @@ await client.CreateConsumerGroupAsync(Identifier.String(streamName), if (protocol == Protocol.Tcp) { - await client.JoinConsumerGroupAsync(Identifier.String(streamName), + await client.JoinConsumerGroupAsync( + Identifier.String(streamName), Identifier.String(topicName), Identifier.String("test_consumer_group")); } else { - var tcpClient = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); - await tcpClient.JoinConsumerGroupAsync(Identifier.String(streamName), + var tcpClient = await Fixture.CreateTcpClient(); + await tcpClient.JoinConsumerGroupAsync( + Identifier.String(streamName), Identifier.String(topicName), Identifier.String("test_consumer_group")); } @@ -189,7 +195,8 @@ public async Task DeleteOffset_ConsumerGroup_Should_DeleteOffset_Successfully(Pr await client.CreateConsumerGroupAsync(Identifier.String(streamName), Identifier.String(topicName), "test_consumer_group"); - await client.JoinConsumerGroupAsync(Identifier.String(streamName), + await client.JoinConsumerGroupAsync( + Identifier.String(streamName), Identifier.String(topicName), Identifier.String("test_consumer_group")); await client.StoreOffsetAsync(Consumer.Group("test_consumer_group"), Identifier.String(streamName), diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/PartitionsTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/PartitionsTests.cs index 64e281ac48..406b1d3ade 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/PartitionsTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/PartitionsTests.cs @@ -43,7 +43,8 @@ await Should.NotThrowAsync(() => client.CreatePartitionsAsync(Identifier.String(streamName), Identifier.String(topicName), 3)); - var response = await client.GetTopicByIdAsync(Identifier.String(streamName), Identifier.String(topicName)); + var response = await client.GetTopicByIdAsync( + Identifier.String(streamName), Identifier.String(topicName)); response.ShouldNotBeNull(); response.PartitionsCount.ShouldBe(4u); } @@ -64,7 +65,8 @@ await Should.NotThrowAsync(() => client.DeletePartitionsAsync(Identifier.String(streamName), Identifier.String(topicName), 1)); - var response = await client.GetTopicByIdAsync(Identifier.String(streamName), Identifier.String(topicName)); + var response = await client.GetTopicByIdAsync( + Identifier.String(streamName), Identifier.String(topicName)); response.ShouldNotBeNull(); response.PartitionsCount.ShouldBe(3u); } diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/PersonalAccessTokenTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/PersonalAccessTokenTests.cs index b8ffc39b91..2e14ff637a 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/PersonalAccessTokenTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/PersonalAccessTokenTests.cs @@ -87,7 +87,7 @@ public async Task LoginWithPersonalAccessToken_Should_Be_Successfully(Protocol p var name = $"lgn-{Guid.NewGuid():N}"[..20]; var response = await client.CreatePersonalAccessTokenAsync(name, Expiry); - var loginClient = await Fixture.CreateClient(protocol, true); + var loginClient = await Fixture.CreateClient(protocol); var authResponse = await loginClient.LoginWithPersonalAccessTokenAsync(response!.Token); authResponse.ShouldNotBeNull(); diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/RawCommandTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/RawCommandTests.cs index 1cbfcd059c..49d156fe42 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/RawCommandTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/RawCommandTests.cs @@ -51,9 +51,8 @@ public async Task SendBinaryRequest_Tcp_ShouldRejectSessionControlCodes(Protocol foreach (var code in new uint[] { 38, 39, 40, 44, 45 }) { - var exception - = await Should.ThrowAsync(() => - client.SendBinaryRequestAsync(code, [])); + var exception = await Should.ThrowAsync( + () => client.SendBinaryRequestAsync(code, [])); exception.StatusCode.ShouldBe(3); } } @@ -65,8 +64,8 @@ public async Task SendBinaryRequest_Tcp_ShouldPropagateServerError(Protocol prot { var client = await Fixture.CreateAuthenticatedClient(protocol); - var exception - = await Should.ThrowAsync(() => client.SendBinaryRequestAsync(60_000, [])); + var exception = await Should.ThrowAsync( + () => client.SendBinaryRequestAsync(60_000, [])); exception.StatusCode.ShouldBe(3); } @@ -78,6 +77,7 @@ public async Task SendBinaryRequest_Http_ShouldThrowFeatureUnavailable(Protocol { var client = await Fixture.CreateAuthenticatedClient(protocol); - await Should.ThrowAsync(() => client.SendBinaryRequestAsync(1, [])); + await Should.ThrowAsync( + () => client.SendBinaryRequestAsync(1, [])); } } diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/SegmentsTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/SegmentsTests.cs index 64a80a3905..eed1d01b0a 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/SegmentsTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/SegmentsTests.cs @@ -42,7 +42,8 @@ public async Task DeleteSegments_WithZeroCount_Should_Succeed(Protocol protocol) // Deleting 0 segments should succeed without error (no-op) await Should.NotThrowAsync(() => - client.DeleteSegmentsAsync(Identifier.String(streamName), + client.DeleteSegmentsAsync( + Identifier.String(streamName), Identifier.String(topicName), 0, // partition_id (0-indexed) 0)); // segments_count = 0 @@ -61,7 +62,8 @@ public async Task DeleteSegments_Http_Should_Throw_FeatureUnavailable(Protocol p await client.CreateTopicAsync(Identifier.String(streamName), topicName, 1); await Should.ThrowAsync(() => - client.DeleteSegmentsAsync(Identifier.String(streamName), + client.DeleteSegmentsAsync( + Identifier.String(streamName), Identifier.String(topicName), 0, 0)); @@ -78,7 +80,8 @@ public async Task DeleteSegments_Should_Throw_WhenTopic_DoesNotExist(Protocol pr await client.CreateStreamAsync(streamName); await Should.ThrowAsync(() => - client.DeleteSegmentsAsync(Identifier.String(streamName), + client.DeleteSegmentsAsync( + Identifier.String(streamName), Identifier.String("non-existent-topic"), 0, // partition_id (0-indexed) 1)); // segments_count @@ -92,7 +95,8 @@ public async Task DeleteSegments_Should_Throw_WhenStream_DoesNotExist(Protocol p var client = await Fixture.CreateAuthenticatedClient(protocol); await Should.ThrowAsync(() => - client.DeleteSegmentsAsync(Identifier.String($"nonexistent-stream-{Guid.NewGuid():N}"), + client.DeleteSegmentsAsync( + Identifier.String($"nonexistent-stream-{Guid.NewGuid():N}"), Identifier.String("any-topic"), 0, // partition_id (0-indexed) 1)); // segments_count diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/SendMessagesTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/SendMessagesTests.cs index 085f4d9b04..db109bfbdb 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/SendMessagesTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/SendMessagesTests.cs @@ -16,12 +16,10 @@ // under the License. using System.Text; -using Apache.Iggy.Contracts; using Apache.Iggy.Enums; using Apache.Iggy.Exceptions; using Apache.Iggy.Headers; using Apache.Iggy.IggyClient; -using Apache.Iggy.Kinds; using Apache.Iggy.Messages; using Apache.Iggy.Tests.Integrations.Fixtures; using Shouldly; @@ -43,6 +41,20 @@ public class SendMessagesTests [ClassDataSource(Shared = SharedType.PerAssembly)] public required IggyServerFixture Fixture { get; init; } + private async Task<(IIggyClient client, string streamName, string topicName)> CreateStreamAndTopic( + Protocol protocol) + { + var client = await Fixture.CreateAuthenticatedClient(protocol); + + var streamName = $"send-msg-{Guid.NewGuid():N}"; + var topicName = "test-topic"; + + await client.CreateStreamAsync(streamName); + await client.CreateTopicAsync(Identifier.String(streamName), topicName, 1); + + return (client, streamName, topicName); + } + [Test] [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task SendMessages_NoHeaders_Should_SendMessages_Successfully(Protocol protocol) @@ -130,131 +142,4 @@ await Should.ThrowAsync(() => client.SendMessagesAsync(Identifier.String(streamName), Identifier.Numeric(69), Partitioning.None(), messages)); } - - [Test] - [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] - public async Task SendMessages_Should_ReturnConfirmation_WithStreamTopicPartitionAndBaseOffset(Protocol protocol) - { - var context = await CreateStreamAndTopicWithIds(protocol, 4); - - var response = await SendBatchAsync(context.Client, context.StreamName, context.TopicName, - Partitioning.PartitionId(2), 2); - - var confirmation = response.Confirmations.ShouldHaveSingleItem(); - confirmation.StreamId.ShouldBe(context.StreamId); - confirmation.TopicId.ShouldBe(context.TopicId); - confirmation.PartitionId.ShouldBe(2u); - confirmation.BaseOffset.ShouldBe(0u); - } - - [Test] - [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] - public async Task SendMessages_Consecutive_Should_AdvanceBaseOffset_ByBatchSize(Protocol protocol) - { - var (client, streamName, topicName) = await CreateStreamAndTopic(protocol); - - var first = await SendBatchAsync(client, streamName, topicName, Partitioning.PartitionId(0), 3); - var second = await SendBatchAsync(client, streamName, topicName, Partitioning.PartitionId(0), 2); - var third = await SendBatchAsync(client, streamName, topicName, Partitioning.PartitionId(0), 1); - - first.Confirmations.ShouldHaveSingleItem().BaseOffset.ShouldBe(0u); - second.Confirmations.ShouldHaveSingleItem().BaseOffset.ShouldBe(3u); - third.Confirmations.ShouldHaveSingleItem().BaseOffset.ShouldBe(5u); - } - - [Test] - [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] - public async Task SendMessages_Confirmation_Should_MatchPolledMessageOffsets(Protocol protocol) - { - var (client, streamName, topicName) = await CreateStreamAndTopic(protocol); - - await SendBatchAsync(client, streamName, topicName, Partitioning.PartitionId(0), 4); - var response = await SendBatchAsync(client, streamName, topicName, Partitioning.PartitionId(0), 3); - var confirmation = response.Confirmations.ShouldHaveSingleItem(); - - var polled = await PollAsync(client, streamName, topicName, 0, - PollingStrategy.Offset(confirmation.BaseOffset)); - - polled.Messages.Count.ShouldBe(3); - polled.Messages[0].Header.Offset.ShouldBe(confirmation.BaseOffset); - } - - /// - /// Balanced partitioning is resolved client-side, so the confirmation is the only place the caller - /// learns where the batch landed. It must name the partition the messages actually poll back from. - /// - [Test] - [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] - public async Task SendMessages_Balanced_Confirmation_Should_NameThePartition_MessagesLandedIn(Protocol protocol) - { - const uint partitionsCount = 4; - var context = await CreateStreamAndTopicWithIds(protocol, partitionsCount); - - var response = await SendBatchAsync(context.Client, context.StreamName, context.TopicName, - Partitioning.None(), 2); - - var confirmation = response.Confirmations.ShouldHaveSingleItem(); - confirmation.PartitionId.ShouldBeLessThan(partitionsCount); - - var polled = await PollAsync(context.Client, context.StreamName, context.TopicName, confirmation.PartitionId, - PollingStrategy.Offset(0)); - polled.Messages.Count.ShouldBe(2); - polled.Messages[0].Header.Offset.ShouldBe(confirmation.BaseOffset); - } - - private static Task SendBatchAsync(IIggyClient client, string streamName, string topicName, - Partitioning partitioning, int count) - { - Message[] messages = Enumerable.Range(0, count) - .Select(index => new Message(Guid.NewGuid(), Encoding.UTF8.GetBytes($"confirm-payload-{index}"))) - .ToArray(); - - return client.SendMessagesAsync(Identifier.String(streamName), Identifier.String(topicName), partitioning, - messages); - } - - private static Task PollAsync(IIggyClient client, string streamName, string topicName, - uint partitionId, PollingStrategy strategy) - { - return client.PollMessagesAsync(new MessageFetchRequest - { - Count = 100, - AutoCommit = false, - Consumer = Consumer.New(1), - PartitionId = partitionId, - PollingStrategy = strategy, - StreamId = Identifier.String(streamName), - TopicId = Identifier.String(topicName) - }); - } - - private async Task<(IIggyClient client, string streamName, string topicName)> CreateStreamAndTopic( - Protocol protocol) - { - var context = await CreateStreamAndTopicWithIds(protocol); - - return (context.Client, context.StreamName, context.TopicName); - } - - private async Task CreateStreamAndTopicWithIds(Protocol protocol, uint partitionsCount = 1) - { - var client = await Fixture.CreateAuthenticatedClient(protocol); - - var streamName = $"send-msg-{Guid.NewGuid():N}"; - var topicName = "test-topic"; - - var stream = await client.CreateStreamAsync(streamName); - stream.ShouldNotBeNull(); - var topic = await client.CreateTopicAsync(Identifier.String(streamName), topicName, partitionsCount); - topic.ShouldNotBeNull(); - - return new StreamTopicContext(client, streamName, topicName, stream.Id, topic.Id); - } - - private sealed record StreamTopicContext( - IIggyClient Client, - string StreamName, - string TopicName, - uint StreamId, - uint TopicId); } diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/StreamsTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/StreamsTests.cs index 41d762bbf6..aa47d00c8b 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/StreamsTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/StreamsTests.cs @@ -20,7 +20,6 @@ using Apache.Iggy.Exceptions; using Apache.Iggy.Messages; using Apache.Iggy.Tests.Integrations.Fixtures; -using Apache.Iggy.Tests.Integrations.Helpers; using Shouldly; using Partitioning = Apache.Iggy.Kinds.Partitioning; @@ -217,9 +216,7 @@ await client.SendMessagesAsync(Identifier.String(streamName), await Should.NotThrowAsync(() => client.PurgeStreamAsync(Identifier.String(streamName))); - // The server commits the purge by advancing a generation its reconciler acts on a tick later. - stream = await Eventually.ReadAsync(() => client.GetStreamByIdAsync(Identifier.String(streamName)), - purged => purged?.MessagesCount == 0, TimeSpan.FromSeconds(10)); + stream = await client.GetStreamByIdAsync(Identifier.String(streamName)); stream.ShouldNotBeNull(); stream.MessagesCount.ShouldBe(0u); stream.TopicsCount.ShouldBe(1); diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/SystemTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/SystemTests.cs index dd2b26ab94..3d803b4cc8 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/SystemTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/SystemTests.cs @@ -55,7 +55,8 @@ public async Task GetClient_Should_Return_CorrectClient(Protocol protocol) { var client = await Fixture.CreateAuthenticatedClient(protocol); - var tcpClient = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); + var tcpClient = await Fixture.CreateClient(Protocol.Tcp); + await tcpClient.LoginUserAsync("iggy", "iggy"); var clientInfo = await tcpClient.GetMeAsync(); clientInfo.ShouldNotBeNull(); @@ -75,7 +76,7 @@ public async Task GetClient_Should_Return_CorrectClient(Protocol protocol) [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task GetMe_Tcp_Should_Return_MyClient(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = await Fixture.CreateTcpClient(); var me = await client.GetMeAsync(); me.ShouldNotBeNull(); @@ -90,7 +91,7 @@ public async Task GetMe_Tcp_Should_Return_MyClient(Protocol protocol) [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task GetMe_HTTP_Should_Throw_FeatureUnavailableException(Protocol protocol) { - var client = await Fixture.CreateAuthenticatedClient(protocol); + var client = await Fixture.CreateHttpClient(); await Should.ThrowAsync(() => client.GetMeAsync()); } @@ -102,7 +103,8 @@ public async Task GetClient_WithConsumerGroup_Should_Return_CorrectClient(Protoc var client = await Fixture.CreateAuthenticatedClient(protocol); var streamName = $"sys-cg-{Guid.NewGuid():N}"; - var tcpClient = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); + var tcpClient = await Fixture.CreateClient(Protocol.Tcp); + await tcpClient.LoginUserAsync("iggy", "iggy"); var stream = await tcpClient.CreateStreamAsync(streamName); await tcpClient.CreateTopicAsync(Identifier.String(streamName), "first_topic", 2); @@ -157,7 +159,7 @@ await client.SendMessagesAsync(Identifier.String(streamName), response.PartitionsCount.ShouldBeGreaterThanOrEqualTo(1); response.SegmentsCount.ShouldBeGreaterThanOrEqualTo(1); response.MessagesCount.ShouldBeGreaterThanOrEqualTo(1u); - // iggy-server leaves the connected-client tally out of its stats reply, so ClientsCount goes unchecked. + response.ClientsCount.ShouldBeGreaterThanOrEqualTo(1); response.Hostname.ShouldNotBeNullOrEmpty(); response.OsName.ShouldNotBeNullOrEmpty(); response.OsVersion.ShouldNotBeNullOrEmpty(); diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/TopicsTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/TopicsTests.cs index 88d2c23f36..fa6a57796b 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/TopicsTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/TopicsTests.cs @@ -21,7 +21,6 @@ using Apache.Iggy.Exceptions; using Apache.Iggy.Messages; using Apache.Iggy.Tests.Integrations.Fixtures; -using Apache.Iggy.Tests.Integrations.Helpers; using Shouldly; using Partitioning = Apache.Iggy.Kinds.Partitioning; @@ -41,8 +40,8 @@ public async Task Create_NewTopic_Should_Return_Successfully(Protocol protocol) var streamName = $"topic-create-{Guid.NewGuid():N}"; await client.CreateStreamAsync(streamName); - var response = await client.CreateTopicAsync(Identifier.String(streamName), "Test Topic", 2, - CompressionAlgorithm.Gzip, + var response = await client.CreateTopicAsync( + Identifier.String(streamName), "Test Topic", 2, CompressionAlgorithm.Gzip, 1, TimeSpan.FromMinutes(10), 2_000_000_000); response.ShouldNotBeNull(); @@ -69,8 +68,8 @@ public async Task Create_DuplicateTopic_Should_Throw_InvalidResponse(Protocol pr await client.CreateStreamAsync(streamName); await client.CreateTopicAsync(Identifier.String(streamName), "Dup Topic", 1); - await Should.ThrowAsync(client.CreateTopicAsync(Identifier.String(streamName), - "Dup Topic", 1)); + await Should.ThrowAsync( + client.CreateTopicAsync(Identifier.String(streamName), "Dup Topic", 1)); } [Test] @@ -198,11 +197,13 @@ public async Task Update_ExistingTopic_Should_UpdateTopic_Successfully(Protocol var topicToUpdate = await client.CreateTopicAsync(Identifier.String(streamName), "topic-to-update", 1); topicToUpdate.ShouldNotBeNull(); - await Should.NotThrowAsync(client.UpdateTopicAsync(Identifier.String(streamName), + await Should.NotThrowAsync(client.UpdateTopicAsync( + Identifier.String(streamName), Identifier.Numeric(topicToUpdate.Id), "Updated Topic", CompressionAlgorithm.Gzip, 3_000_000_000, TimeSpan.FromMinutes(10), 3)); - var result = await client.GetTopicByIdAsync(Identifier.String(streamName), + var result = await client.GetTopicByIdAsync( + Identifier.String(streamName), Identifier.Numeric(topicToUpdate.Id)); result.ShouldNotBeNull(); result!.Name.ShouldBe("Updated Topic"); @@ -231,13 +232,11 @@ await client.SendMessagesAsync(Identifier.String(streamName), beforePurge.MessagesCount.ShouldBe(5u); beforePurge.Size.ShouldBeGreaterThan(0u); - await Should.NotThrowAsync(client.PurgeTopicAsync(Identifier.String(streamName), - Identifier.String("Purge Topic"))); + await Should.NotThrowAsync(client.PurgeTopicAsync( + Identifier.String(streamName), Identifier.String("Purge Topic"))); - // The server commits the purge by advancing a generation its reconciler acts on a tick later. - var afterPurge = await Eventually.ReadAsync( - () => client.GetTopicByIdAsync(Identifier.String(streamName), Identifier.String("Purge Topic")), - topic => topic?.MessagesCount == 0, TimeSpan.FromSeconds(10)); + var afterPurge = await client.GetTopicByIdAsync(Identifier.String(streamName), + Identifier.String("Purge Topic")); afterPurge.ShouldNotBeNull(); afterPurge!.MessagesCount.ShouldBe(0u); afterPurge.Size.ShouldBe(0u); @@ -254,8 +253,8 @@ public async Task Delete_ExistingTopic_Should_DeleteTopic_Successfully(Protocol var topicToDelete = await client.CreateTopicAsync(Identifier.String(streamName), "topic-to-delete", 1); topicToDelete.ShouldNotBeNull(); - await Should.NotThrowAsync(client.DeleteTopicAsync(Identifier.String(streamName), - Identifier.Numeric(topicToDelete.Id))); + await Should.NotThrowAsync(client.DeleteTopicAsync( + Identifier.String(streamName), Identifier.Numeric(topicToDelete.Id))); } [Test] @@ -267,8 +266,8 @@ public async Task Delete_NonExistingTopic_Should_Throw_InvalidResponse(Protocol var streamName = $"topic-delnone-{Guid.NewGuid():N}"; await client.CreateStreamAsync(streamName); - await Should.ThrowAsync(client.DeleteTopicAsync(Identifier.String(streamName), - Identifier.String("nonexistent-topic"))); + await Should.ThrowAsync(client.DeleteTopicAsync( + Identifier.String(streamName), Identifier.String("nonexistent-topic"))); } [Test] @@ -280,8 +279,8 @@ public async Task Get_NonExistingTopic_Should_Throw_InvalidResponse(Protocol pro var streamName = $"topic-getnone-{Guid.NewGuid():N}"; await client.CreateStreamAsync(streamName); - var topic = await client.GetTopicByIdAsync(Identifier.String(streamName), - Identifier.String("nonexistent-topic")); + var topic = await client.GetTopicByIdAsync( + Identifier.String(streamName), Identifier.String("nonexistent-topic")); topic.ShouldBeNull(); } diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/UsersTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/UsersTests.cs index 3c86131b29..c3820a3079 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/UsersTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/UsersTests.cs @@ -17,6 +17,7 @@ using Apache.Iggy.Contracts; using Apache.Iggy.Contracts.Auth; +using Apache.Iggy.Contracts.Http.Auth; using Apache.Iggy.Enums; using Apache.Iggy.Exceptions; using Apache.Iggy.Tests.Integrations.Fixtures; @@ -53,8 +54,8 @@ public async Task CreateUser_Duplicate_Should_Throw_InvalidResponse(Protocol pro var username = $"dup-{Guid.NewGuid():N}"[..20]; await client.CreateUserAsync(username, "test1", UserStatus.Active); - await Should.ThrowAsync(client.CreateUserAsync(username, "test1", - UserStatus.Active)); + await Should.ThrowAsync( + client.CreateUserAsync(username, "test1", UserStatus.Active)); } [Test] @@ -168,11 +169,10 @@ public async Task ChangePassword_Should_ChangePassword_Successfully(Protocol pro var username = $"chpw-{Guid.NewGuid():N}"[..20]; await client.CreateUserAsync(username, "old_password", UserStatus.Active); - await Should.NotThrowAsync(client.ChangePasswordAsync(Identifier.String(username), "old_password", - "new_password")); + await Should.NotThrowAsync(client.ChangePasswordAsync(Identifier.String(username), "old_password", "new_password")); // Verify password was actually changed by logging in with the new credentials - var loginClient = await Fixture.CreateClient(protocol, true); + var loginClient = await Fixture.CreateClient(protocol); var loginResponse = await loginClient.LoginUserAsync(username, "new_password"); loginResponse.ShouldNotBeNull(); loginResponse.UserId.ShouldBeGreaterThan(0); @@ -187,8 +187,8 @@ public async Task ChangePassword_WrongCurrentPassword_Should_Throw_InvalidRespon var username = $"chpwf-{Guid.NewGuid():N}"[..20]; await client.CreateUserAsync(username, "correct_password", UserStatus.Active); - await Should.ThrowAsync(client.ChangePasswordAsync(Identifier.String(username), - "wrong_password", "new_password")); + await Should.ThrowAsync( + client.ChangePasswordAsync(Identifier.String(username), "wrong_password", "new_password")); } [Test] @@ -200,7 +200,7 @@ public async Task LoginUser_Should_LoginUser_Successfully(Protocol protocol) var username = $"login-{Guid.NewGuid():N}"[..20]; await client.CreateUserAsync(username, "login_password", UserStatus.Active); - var loginClient = await Fixture.CreateClient(protocol, true); + var loginClient = await Fixture.CreateClient(protocol); var response = await loginClient.LoginUserAsync(username, "login_password"); response.ShouldNotBeNull(); diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/Vsr/VsrConsumerGroupTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/Vsr/VsrConsumerGroupTests.cs deleted file mode 100644 index 5731c84abc..0000000000 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/Vsr/VsrConsumerGroupTests.cs +++ /dev/null @@ -1,228 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Text; -using Apache.Iggy.Contracts; -using Apache.Iggy.Enums; -using Apache.Iggy.Exceptions; -using Apache.Iggy.IggyClient; -using Apache.Iggy.Kinds; -using Apache.Iggy.Messages; -using Apache.Iggy.Tests.Integrations.Fixtures; -using Shouldly; -using Partitioning = Apache.Iggy.Kinds.Partitioning; - -namespace Apache.Iggy.Tests.Integrations.Vsr; - -/// -/// Group polls under VSR: the server hands out an assignment, the client caches it and round-robins the -/// assigned partitions itself, so a poll without an explicit partition id never reaches the broker as one. -/// -public class VsrConsumerGroupTests -{ - private const uint PartitionsCount = 3; - private const string TopicName = "vsr-group-topic"; - - [ClassDataSource(Shared = SharedType.PerAssembly)] - public required IggyServerFixture Fixture { get; init; } - - [Test] - public async Task GroupPoll_Should_Drain_EveryAssignedPartition() - { - var (client, streamName, groupName) = await CreateGroup(); - await client.JoinConsumerGroupAsync(Identifier.String(streamName), Identifier.String(TopicName), - Identifier.String(groupName)); - - for (uint partitionId = 0; partitionId < PartitionsCount; partitionId++) - { - await SendAsync(client, streamName, partitionId); - } - - // One poll per assigned partition drains it; the sole member owns them all, so the round-robin has to - // hand out every partition exactly once before it wraps. - var polled = await DrainGroupAsync(client, streamName, groupName, PartitionsCount); - - polled.ShouldBe((int)PartitionsCount); - } - - [Test] - public async Task GroupPoll_Without_Joining_Should_Throw_MemberNotFound() - { - var (client, streamName, groupName) = await CreateGroup(); - - var exception = await Should.ThrowAsync( - PollGroupAsync(client, streamName, groupName)); - - exception.StatusCode.ShouldBe(5006); - } - - [Test] - public async Task GroupPoll_After_Leaving_Should_Throw_MemberNotFound() - { - var (client, streamName, groupName) = await CreateGroup(); - await client.JoinConsumerGroupAsync(Identifier.String(streamName), Identifier.String(TopicName), - Identifier.String(groupName)); - await PollGroupAsync(client, streamName, groupName); - - await client.LeaveConsumerGroupAsync(Identifier.String(streamName), Identifier.String(TopicName), - Identifier.String(groupName)); - - var exception = await Should.ThrowAsync( - PollGroupAsync(client, streamName, groupName)); - - exception.StatusCode.ShouldBe(5006); - } - - /// - /// A partition count change widens the assignment, and the ping is where the client re-syncs it. The - /// cached generation is asserted first: without it the test would pass on a client that re-synced on - /// every poll and never needed the heartbeat. - /// - [Test] - public async Task Ping_Should_Refresh_TheGroupAssignment_After_PartitionsAreAdded() - { - var (client, streamName, groupName) = await CreateGroup(); - await client.JoinConsumerGroupAsync(Identifier.String(streamName), Identifier.String(TopicName), - Identifier.String(groupName)); - await PollGroupAsync(client, streamName, groupName); - - await client.CreatePartitionsAsync(Identifier.String(streamName), Identifier.String(TopicName), 1); - await SendAsync(client, streamName, PartitionsCount); - - (await DrainGroupAsync(client, streamName, groupName, PartitionsCount + 1)).ShouldBe(0); - - await client.PingAsync(); - - (await DrainGroupAsync(client, streamName, groupName, PartitionsCount + 1)).ShouldBe(1); - } - - /// - /// A member holding no partitions is still a member, so its poll has to come back empty instead of - /// surfacing the not-a-member error the unassigned cursor otherwise looks like. One partition and two - /// members guarantees exactly one of them is in that state. - /// - [Test] - public async Task GroupPoll_By_AMemberWithoutPartitions_Should_ReturnEmpty() - { - var (first, streamName, groupName) = await CreateGroup(); - var topicName = $"vsr-single-partition-{Guid.NewGuid():N}"; - await first.CreateTopicAsync(Identifier.String(streamName), topicName, 1); - await first.CreateConsumerGroupAsync(Identifier.String(streamName), Identifier.String(topicName), groupName); - - var second = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); - await JoinAsync(first, streamName, topicName, groupName); - await JoinAsync(second, streamName, topicName, groupName); - - await first.SendMessagesAsync(Identifier.String(streamName), Identifier.String(topicName), - Partitioning.PartitionId(0), - [new Message(Guid.NewGuid(), Encoding.UTF8.GetBytes("vsr-single-partition-payload"))]); - - var firstPoll = await PollGroupAsync(first, streamName, topicName, groupName); - var secondPoll = await PollGroupAsync(second, streamName, topicName, groupName); - - // Whichever member drew the partition drains it, and the other one has nothing to poll. - (firstPoll.Messages.Count + secondPoll.Messages.Count).ShouldBe(1); - Math.Min(firstPoll.Messages.Count, secondPoll.Messages.Count).ShouldBe(0); - } - - /// - /// A second member rebalances the group, which leaves the first one round-robining partitions it no - /// longer owns. The fence has to re-sync the assignment underneath the poll: a client that surfaced the - /// ownership error instead would break every group app that did not special-case it. - /// - [Test] - public async Task GroupPoll_After_ASecondMemberJoins_Should_ResyncTheStaleAssignment() - { - var (first, streamName, groupName) = await CreateGroup(); - await JoinAsync(first, streamName, TopicName, groupName); - await DrainGroupAsync(first, streamName, groupName, PartitionsCount); - - var second = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); - await JoinAsync(second, streamName, TopicName, groupName); - - for (uint partitionId = 0; partitionId < PartitionsCount; partitionId++) - { - await SendAsync(first, streamName, partitionId); - } - - // The first member still holds the pre-rebalance assignment, so its polls fence until they re-sync. - // Between them the two members have to see every partition; a lost one means a fence was swallowed. - var drained = await DrainGroupAsync(first, streamName, groupName, PartitionsCount); - drained += await DrainGroupAsync(second, streamName, groupName, PartitionsCount); - - drained.ShouldBe((int)PartitionsCount); - } - - private static Task JoinAsync(IIggyClient client, string streamName, string topicName, string groupName) - { - return client.JoinConsumerGroupAsync(Identifier.String(streamName), Identifier.String(topicName), - Identifier.String(groupName)); - } - - private async Task<(IIggyClient Client, string StreamName, string GroupName)> CreateGroup() - { - var client = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); - var streamName = $"vsr-group-{Guid.NewGuid():N}"; - var groupName = $"vsr-group-name-{Guid.NewGuid():N}"; - - await client.CreateStreamAsync(streamName); - await client.CreateTopicAsync(Identifier.String(streamName), TopicName, PartitionsCount); - await client.CreateConsumerGroupAsync(Identifier.String(streamName), Identifier.String(TopicName), groupName); - - return (client, streamName, groupName); - } - - private static Task SendAsync(IIggyClient client, string streamName, uint partitionId) - { - return client.SendMessagesAsync(Identifier.String(streamName), Identifier.String(TopicName), - Partitioning.PartitionId((int)partitionId), - [new Message(Guid.NewGuid(), Encoding.UTF8.GetBytes($"vsr-group-payload-{partitionId}"))]); - } - - /// Polls once per assigned partition and returns how many messages came back in total. - private static async Task DrainGroupAsync(IIggyClient client, string streamName, string groupName, - uint polls) - { - var drained = 0; - for (var poll = 0; poll < polls; poll++) - { - drained += (await PollGroupAsync(client, streamName, groupName)).Messages.Count; - } - - return drained; - } - - private static Task PollGroupAsync(IIggyClient client, string streamName, string groupName) - { - return PollGroupAsync(client, streamName, TopicName, groupName); - } - - private static Task PollGroupAsync(IIggyClient client, string streamName, string topicName, - string groupName) - { - return client.PollMessagesAsync(new MessageFetchRequest - { - Count = 10, - AutoCommit = true, - Consumer = Consumer.Group(groupName), - PartitionId = null, - PollingStrategy = PollingStrategy.Next(), - StreamId = Identifier.String(streamName), - TopicId = Identifier.String(topicName) - }); - } -} diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/Vsr/VsrHandshakeTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/Vsr/VsrHandshakeTests.cs deleted file mode 100644 index 6f77ae26c5..0000000000 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/Vsr/VsrHandshakeTests.cs +++ /dev/null @@ -1,164 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using Apache.Iggy.Contracts; -using Apache.Iggy.Enums; -using Apache.Iggy.Exceptions; -using Apache.Iggy.Tests.Integrations.Fixtures; -using Shouldly; - -namespace Apache.Iggy.Tests.Integrations.Vsr; - -/// -/// The register handshake and the session it binds. Every other VSR suite depends on this one passing: -/// without a bound session the server fences every replicated request. -/// -public class VsrHandshakeTests -{ - [ClassDataSource(Shared = SharedType.PerAssembly)] - public required IggyServerFixture Fixture { get; init; } - - [Test] - public async Task Login_Should_BindSession_And_ServeReplicatedRequests() - { - var client = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); - - var name = $"vsr-handshake-{Guid.NewGuid():N}"; - var stream = await client.CreateStreamAsync(name); - - stream.ShouldNotBeNull(); - stream.Name.ShouldBe(name); - } - - /// - /// A re-login first logs out the bound session, then registers a fresh client identity. The metadata write - /// after re-login proves the request counter belongs to that new binding instead of replaying a cached - /// response from the old client table entry. - /// - [Test] - public async Task ReLogin_Should_AllowReplicatedRequests() - { - var client = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); - var name = $"vsr-relogin-{Guid.NewGuid():N}"; - await client.CreateStreamAsync(name); - - var response = await client.LoginUserAsync("iggy", "iggy"); - response.ShouldNotBeNull(); - - var afterRelogin = $"vsr-relogin-after-{Guid.NewGuid():N}"; - var stream = await client.CreateStreamAsync(afterRelogin); - stream.ShouldNotBeNull(); - stream.Name.ShouldBe(afterRelogin); - } - - [Test] - public async Task Logout_Should_UnbindTheSession_Until_TheClientRegistersAgain() - { - var client = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); - await client.LogoutUserAsync(); - - await Should.ThrowAsync(client.CreateStreamAsync($"vsr-logout-{Guid.NewGuid():N}")); - - await client.LoginUserAsync("iggy", "iggy"); - - var name = $"vsr-logout-relogin-{Guid.NewGuid():N}"; - (await client.CreateStreamAsync(name)).ShouldNotBeNull(); - } - - /// - /// A rejected register is a consumed one on the server, so the failure has to reset the session and - /// unwind the connection state. The retry with valid credentials is the assertion that matters: a client - /// left in Authenticating with the failed register's session still bound would fence it. - /// Wrong credentials come back as an InvalidCredentials eviction, not as the empty register reply, - /// and an eviction is terminal for the connection: the retry has to reconnect first. - /// - [Test] - public async Task Login_WithInvalidCredentials_Should_ResetTheSession_And_AllowARetry() - { - var client = await Fixture.CreateUnauthenticatedClient(Protocol.Tcp); - - var exception = await Should.ThrowAsync( - client.LoginUserAsync("iggy", "not-the-password")); - - exception.StatusCode.ShouldBe(42); - exception.Message.ShouldContain("Invalid credentials"); - - await client.ConnectAsync(); - (await client.LoginUserAsync("iggy", "iggy")).ShouldNotBeNull(); - - var name = $"vsr-failed-login-{Guid.NewGuid():N}"; - (await client.CreateStreamAsync(name)).ShouldNotBeNull(); - } - - [Test] - public async Task LoginWithPersonalAccessToken_Should_BindTheSession() - { - var client = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); - var token = await client.CreatePersonalAccessTokenAsync($"vsr-pat-{Guid.NewGuid():N}"); - token.ShouldNotBeNull(); - - var patClient = await Fixture.CreateUnauthenticatedClient(Protocol.Tcp); - var response = await patClient.LoginWithPersonalAccessTokenAsync(token.Token); - - response.ShouldNotBeNull(); - - var name = $"vsr-pat-stream-{Guid.NewGuid():N}"; - (await patClient.CreateStreamAsync(name)).ShouldNotBeNull(); - } - - [Test] - public async Task Ping_Should_Succeed_Before_TheSessionIsBound() - { - var client = await Fixture.CreateUnauthenticatedClient(Protocol.Tcp); - - // Non-replicated ops are sessionless, so an unbound client still pings. - await client.PingAsync(); - - await Should.ThrowAsync(client.CreateStreamAsync($"vsr-unbound-{Guid.NewGuid():N}")); - } - - [Test] - public async Task Ping_Should_Ride_NonReplicated_Without_GappingTheRequestCounter() - { - var client = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); - - // A ping that consumed a request id would gap the next metadata request, and the primary silently - // drops a gapped one - the create below would hang instead of failing loudly. - await client.PingAsync(); - var first = await client.CreateStreamAsync($"vsr-ping-first-{Guid.NewGuid():N}"); - await client.PingAsync(); - var second = await client.CreateStreamAsync($"vsr-ping-second-{Guid.NewGuid():N}"); - - first.ShouldNotBeNull(); - second.ShouldNotBeNull(); - } - - [Test] - public async Task Reads_Should_Ride_NonReplicated_Between_MetadataWrites() - { - var client = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); - - var name = $"vsr-reads-{Guid.NewGuid():N}"; - await client.CreateStreamAsync(name); - - IReadOnlyList streams = await client.GetStreamsAsync(); - streams.ShouldContain(stream => stream.Name == name); - - var second = $"vsr-reads-second-{Guid.NewGuid():N}"; - (await client.CreateStreamAsync(second)).ShouldNotBeNull(); - } -} diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/Vsr/VsrMessagingTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/Vsr/VsrMessagingTests.cs deleted file mode 100644 index d1fdbd3e47..0000000000 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/Vsr/VsrMessagingTests.cs +++ /dev/null @@ -1,165 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Text; -using Apache.Iggy.Contracts; -using Apache.Iggy.Enums; -using Apache.Iggy.IggyClient; -using Apache.Iggy.Kinds; -using Apache.Iggy.Messages; -using Apache.Iggy.Tests.Integrations.Fixtures; -using Shouldly; -using Partitioning = Apache.Iggy.Kinds.Partitioning; - -namespace Apache.Iggy.Tests.Integrations.Vsr; - -/// -/// The partition plane. Under VSR the broker never picks a partition, so the client resolves every -/// partitioning kind to an explicit id before the request leaves - these tests assert the resolution -/// lands where the Rust SDK's does. -/// -public class VsrMessagingTests -{ - private const uint PartitionsCount = 4; - private const string TopicName = "vsr-messages"; - - [ClassDataSource(Shared = SharedType.PerAssembly)] - public required IggyServerFixture Fixture { get; init; } - - [Test] - public async Task SendMessages_ToAnExplicitPartition_Should_PollBack_FromThatPartition() - { - var (client, streamName) = await CreateStreamAndTopic(); - - await SendAsync(client, streamName, Partitioning.PartitionId(2), 5); - - var polled = await PollAsync(client, streamName, 2); - polled.Messages.Count.ShouldBe(5); - polled.PartitionId.ShouldBe(2); - - (await PollAsync(client, streamName, 1)).Messages.ShouldBeEmpty(); - } - - /// - /// Balanced partitioning is resolved client-side by round-robin over the topic's partition count, so - /// a batch per partition ends up one message on each. - /// - [Test] - public async Task SendMessages_Balanced_Should_RoundRobin_AcrossEveryPartition() - { - var (client, streamName) = await CreateStreamAndTopic(); - - for (var i = 0; i < PartitionsCount; i++) - { - await SendAsync(client, streamName, Partitioning.None(), 1); - } - - List counts = await PollEveryPartitionAsync(client, streamName); - - counts.Sum().ShouldBe((int)PartitionsCount); - counts.ShouldAllBe(count => count == 1); - } - - /// - /// The message key hashes to one partition, so every message under the same key lands together and - /// two different keys are free to differ. Only the first is asserted: the hash is pinned by the unit - /// tests against the Rust vectors, and asserting two keys differ would be a coin flip. - /// - [Test] - public async Task SendMessages_ByMessageKey_Should_LandOn_ASinglePartition() - { - var (client, streamName) = await CreateStreamAndTopic(); - var key = Partitioning.EntityIdString($"key-{Guid.NewGuid():N}"); - - for (var i = 0; i < 6; i++) - { - await SendAsync(client, streamName, key, 1); - } - - List counts = await PollEveryPartitionAsync(client, streamName); - - counts.Sum().ShouldBe(6); - counts.Count(count => count > 0).ShouldBe(1); - } - - [Test] - public async Task ConsumerOffsets_Should_RoundTrip_ThroughTheResultSection() - { - var (client, streamName) = await CreateStreamAndTopic(); - await SendAsync(client, streamName, Partitioning.PartitionId(0), 3); - - var consumer = Consumer.New($"vsr-offset-{Guid.NewGuid():N}"); - await client.StoreOffsetAsync(consumer, Identifier.String(streamName), Identifier.String(TopicName), 1, 0); - - var stored = await client.GetOffsetAsync(consumer, Identifier.String(streamName), - Identifier.String(TopicName), 0); - stored.ShouldNotBeNull(); - stored.StoredOffset.ShouldBe(1u); - - await client.DeleteOffsetAsync(consumer, Identifier.String(streamName), Identifier.String(TopicName), 0); - - var cleared = await client.GetOffsetAsync(consumer, Identifier.String(streamName), - Identifier.String(TopicName), 0); - cleared.ShouldSatisfyAllConditions(() => (cleared is null || cleared.StoredOffset == 0).ShouldBeTrue()); - } - - private async Task<(IIggyClient Client, string StreamName)> CreateStreamAndTopic() - { - var client = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); - var streamName = $"vsr-msg-{Guid.NewGuid():N}"; - - await client.CreateStreamAsync(streamName); - await client.CreateTopicAsync(Identifier.String(streamName), TopicName, PartitionsCount); - - return (client, streamName); - } - - private static Task SendAsync(IIggyClient client, string streamName, Partitioning partitioning, int count) - { - Message[] messages = Enumerable.Range(0, count) - .Select(index => new Message(Guid.NewGuid(), Encoding.UTF8.GetBytes($"vsr-payload-{index}"))) - .ToArray(); - - return client.SendMessagesAsync(Identifier.String(streamName), Identifier.String(TopicName), partitioning, - messages); - } - - private static Task PollAsync(IIggyClient client, string streamName, uint partitionId) - { - return client.PollMessagesAsync(new MessageFetchRequest - { - Count = 100, - AutoCommit = false, - Consumer = Consumer.New(1), - PartitionId = partitionId, - PollingStrategy = PollingStrategy.Offset(0), - StreamId = Identifier.String(streamName), - TopicId = Identifier.String(TopicName) - }); - } - - private static async Task> PollEveryPartitionAsync(IIggyClient client, string streamName) - { - var counts = new List(); - for (uint partitionId = 0; partitionId < PartitionsCount; partitionId++) - { - counts.Add((await PollAsync(client, streamName, partitionId)).Messages.Count); - } - - return counts; - } -} diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/Vsr/VsrMetadataTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/Vsr/VsrMetadataTests.cs deleted file mode 100644 index 8f898d8e24..0000000000 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/Vsr/VsrMetadataTests.cs +++ /dev/null @@ -1,132 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using Apache.Iggy.Enums; -using Apache.Iggy.Exceptions; -using Apache.Iggy.Kinds; -using Apache.Iggy.Messages; -using Apache.Iggy.Tests.Integrations.Fixtures; -using Shouldly; -using Partitioning = Apache.Iggy.Kinds.Partitioning; - -namespace Apache.Iggy.Tests.Integrations.Vsr; - -/// -/// Control-plane operations through the consensus path: every one of these consumes a request id and -/// comes back with a committed result section the decoder has to strip before the typed mapper runs. -/// -public class VsrMetadataTests -{ - [ClassDataSource(Shared = SharedType.PerAssembly)] - public required IggyServerFixture Fixture { get; init; } - - [Test] - public async Task StreamLifecycle_Should_CommitThroughConsensus() - { - var client = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); - - var name = $"vsr-meta-stream-{Guid.NewGuid():N}"; - var created = await client.CreateStreamAsync(name); - created.ShouldNotBeNull(); - - var fetched = await client.GetStreamByIdAsync(Identifier.Numeric(created.Id)); - fetched.ShouldNotBeNull(); - fetched.Name.ShouldBe(name); - - await client.DeleteStreamAsync(Identifier.Numeric(created.Id)); - (await client.GetStreamsAsync()).ShouldNotContain(stream => stream.Name == name); - } - - [Test] - public async Task TopicLifecycle_Should_CommitThroughConsensus() - { - var client = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); - - var streamName = $"vsr-meta-topic-{Guid.NewGuid():N}"; - await client.CreateStreamAsync(streamName); - - var topic = await client.CreateTopicAsync(Identifier.String(streamName), "vsr-topic", 3); - topic.ShouldNotBeNull(); - topic.PartitionsCount.ShouldBe(3u); - - var fetched = await client.GetTopicByIdAsync(Identifier.String(streamName), Identifier.Numeric(topic.Id)); - fetched.ShouldNotBeNull(); - fetched.PartitionsCount.ShouldBe(3u); - - await client.DeleteTopicAsync(Identifier.String(streamName), Identifier.Numeric(topic.Id)); - (await client.GetTopicsAsync(Identifier.String(streamName))).ShouldBeEmpty(); - } - - /// - /// A committed rejection rides the result section with status 0 in the header, so the decoder has to - /// read the first result entry to see it. A silent success here would mean the section was skipped. - /// - [Test] - public async Task DuplicateStream_Should_Surface_TheCommittedRejection() - { - var client = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); - - var name = $"vsr-meta-dup-{Guid.NewGuid():N}"; - await client.CreateStreamAsync(name); - - await Should.ThrowAsync(client.CreateStreamAsync(name)); - } - - [Test] - public async Task UserLifecycle_Should_CommitThroughConsensus() - { - var client = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); - - var name = $"vsr-user-{Guid.NewGuid():N}"; - var user = await client.CreateUserAsync(name, "secret-password", UserStatus.Active); - user.ShouldNotBeNull(); - - var fetched = await client.GetUserAsync(Identifier.Numeric(user.Id)); - fetched.ShouldNotBeNull(); - fetched.Username.ShouldBe(name); - - await client.DeleteUserAsync(Identifier.Numeric(user.Id)); - (await client.GetUserAsync(Identifier.Numeric(user.Id))).ShouldBeNull(); - } - - /// - /// Partition ops read the request counter without advancing it. Interleaving them with metadata - /// writes catches the asymmetry: a partition op that consumed an id would gap the next metadata one - /// and the primary would silently drop it. - /// - [Test] - public async Task MetadataWrites_Should_KeepCommitting_Around_PartitionOps() - { - var client = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); - - var streamName = $"vsr-meta-mixed-{Guid.NewGuid():N}"; - await client.CreateStreamAsync(streamName); - await client.CreateTopicAsync(Identifier.String(streamName), "vsr-mixed-topic", 1); - - await client.SendMessagesAsync(Identifier.String(streamName), Identifier.String("vsr-mixed-topic"), - Partitioning.PartitionId(0), - [new Message(Guid.NewGuid(), "vsr-mixed"u8.ToArray())]); - await client.StoreOffsetAsync(Consumer.New(1), Identifier.String(streamName), - Identifier.String("vsr-mixed-topic"), 0, 0); - - var second = $"vsr-meta-mixed-second-{Guid.NewGuid():N}"; - var created = await client.CreateStreamAsync(second); - - created.ShouldNotBeNull(); - created.Name.ShouldBe(second); - } -} diff --git a/foreign/csharp/Iggy_SDK/Configuration/AutoLoginSettings.cs b/foreign/csharp/Iggy_SDK/Configuration/AutoLoginSettings.cs index 3c88446eca..b1b434f563 100644 --- a/foreign/csharp/Iggy_SDK/Configuration/AutoLoginSettings.cs +++ b/foreign/csharp/Iggy_SDK/Configuration/AutoLoginSettings.cs @@ -36,19 +36,4 @@ public class AutoLoginSettings /// Specifies the password for auto-login authentication /// public string Password { get; set; } = string.Empty; - - /// - /// Settings for a builder-owned client that signs in with the given credentials. The credentials must - /// reach the client and not only the explicit login the wrapper performs at startup: a reconnect or a - /// leader redirect drops the session, and without them the client comes back unauthenticated. - /// - internal static AutoLoginSettings For(string username, string password) - { - return new AutoLoginSettings - { - Enabled = !string.IsNullOrEmpty(username), - Username = username, - Password = password - }; - } } diff --git a/foreign/csharp/Iggy_SDK/Configuration/IggyClientConfigurator.cs b/foreign/csharp/Iggy_SDK/Configuration/IggyClientConfigurator.cs index fad8604a6f..b8c555a812 100644 --- a/foreign/csharp/Iggy_SDK/Configuration/IggyClientConfigurator.cs +++ b/foreign/csharp/Iggy_SDK/Configuration/IggyClientConfigurator.cs @@ -37,12 +37,6 @@ public sealed class IggyClientConfigurator /// public required Protocol Protocol { get; set; } - /// - /// The largest response frame accepted over , in bytes. - /// Default is 64 MiB, minimum is the 256-byte header. - /// - public int MaxResponseFrameSize { get; set; } = 64 * 1024 * 1024; - /// /// The size of the receive buffer in bytes. Default is 4096. /// diff --git a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Rented.cs b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Rented.cs index 9b4bbfaeb8..e75b60ae31 100644 --- a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Rented.cs +++ b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Rented.cs @@ -198,7 +198,13 @@ protected async Task PollRentedMessagesAsync(CancellationToken ct) LogFailedToDecryptMessage(ex, ex.Offset, ex.PartitionId); throw; } - catch (Exception ex) when (ex is not (MalformedResponseException or VsrRequestOutcomeUnknownException)) + catch (MalformedResponseException) + { + // Non-transient poison: rethrow so the generic catch below does not swallow it and re-poll forever. + // Base InvalidResponseException (server error status, possibly transient) falls through to retry. + throw; + } + catch (Exception ex) { LogFailedToPollMessages(ex); _consumerErrorEvents.Publish(new ConsumerErrorEventArgs(ex, "Failed to poll messages")); diff --git a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs index b5317cfdef..c1201d0274 100644 --- a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs +++ b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs @@ -154,7 +154,7 @@ public async Task InitAsync(CancellationToken ct = default) await _client.ConnectAsync(ct); - if (!string.IsNullOrEmpty(_config.Login) && !_config.CreateIggyClient) + if (_config.CreateIggyClient) { await _client.LoginUserAsync(_config.Login, _config.Password, ct); } @@ -441,7 +441,13 @@ private async Task PollMessagesAsync(CancellationToken ct) LogFailedToDecryptMessage(ex, ex.Offset, ex.PartitionId); throw; } - catch (Exception ex) when (ex is not (MalformedResponseException or VsrRequestOutcomeUnknownException)) + catch (MalformedResponseException) + { + // Non-transient poison: rethrow so the generic catch below does not swallow it and re-poll forever. + // Base InvalidResponseException (server error status, possibly transient) falls through to retry. + throw; + } + catch (Exception ex) { LogFailedToPollMessages(ex); _consumerErrorEvents.Publish(new ConsumerErrorEventArgs(ex, "Failed to poll messages")); diff --git a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerBuilder.cs b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerBuilder.cs index fc768662bf..937ca10c5a 100644 --- a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerBuilder.cs +++ b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerBuilder.cs @@ -31,7 +31,7 @@ namespace Apache.Iggy.Consumers; /// public class IggyConsumerBuilder { - private protected IMessageEncryptor? _encryptor; + private IMessageEncryptor? _encryptor; internal Func? OnPollingError { get; set; } internal IggyConsumerConfig Config { get; set; } = new(); @@ -39,7 +39,7 @@ public class IggyConsumerBuilder /// /// Creates a new consumer builder that will create its own Iggy client. - /// You must configure connection settings using WithConnection. + /// You must configure connection settings using . /// /// The stream identifier to consume from /// The topic identifier to consume from @@ -245,7 +245,6 @@ public IggyConsumer Build() ReceiveBufferSize = Config.ReceiveBufferSize, SendBufferSize = Config.SendBufferSize, ReconnectionSettings = Config.ReconnectionSettings ?? new ReconnectionSettings(), - AutoLoginSettings = AutoLoginSettings.For(Config.Login, Config.Password), LoggerFactory = Config.LoggerFactory ?? NullLoggerFactory.Instance, MessageEncryptor = _encryptor }); diff --git a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerBuilderOfT.cs b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerBuilderOfT.cs index ad640ebeff..0ccf68cc18 100644 --- a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerBuilderOfT.cs +++ b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerBuilderOfT.cs @@ -19,6 +19,7 @@ using Apache.Iggy.Factory; using Apache.Iggy.IggyClient; using Apache.Iggy.Kinds; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; namespace Apache.Iggy.Consumers; @@ -94,11 +95,7 @@ public static IggyConsumerBuilder Create(IIggyClient iggyClient, Identifier s Protocol = Config.Protocol, BaseAddress = Config.Address, ReceiveBufferSize = Config.ReceiveBufferSize, - SendBufferSize = Config.SendBufferSize, - ReconnectionSettings = Config.ReconnectionSettings ?? new ReconnectionSettings(), - AutoLoginSettings = AutoLoginSettings.For(Config.Login, Config.Password), - LoggerFactory = Config.LoggerFactory ?? NullLoggerFactory.Instance, - MessageEncryptor = _encryptor + SendBufferSize = Config.SendBufferSize }); } @@ -136,7 +133,8 @@ protected override void Validate() } else { - throw new InvalidOperationException($"Config must be of type IggyConsumerConfig<{typeof(T).Name}>."); + throw new InvalidOperationException( + $"Config must be of type IggyConsumerConfig<{typeof(T).Name}>."); } } } diff --git a/foreign/csharp/Iggy_SDK/Contracts/SendMessagesResponse.cs b/foreign/csharp/Iggy_SDK/Contracts/SendMessagesResponse.cs deleted file mode 100644 index 199bdc4c52..0000000000 --- a/foreign/csharp/Iggy_SDK/Contracts/SendMessagesResponse.cs +++ /dev/null @@ -1,77 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -namespace Apache.Iggy.Contracts; - -/// -/// Commit confirmation for one partition written by a send messages request. -/// -/// -/// is the offset assigned to the first message of the batch in that -/// partition, bounded by two properties of the send path: -/// -/// -/// Delivery is at-least-once. An earlier retry of the same batch may already have committed -/// at a lower offset, so the value never implies uniqueness. -/// -/// -/// A batch is confirmed once it is committed in memory, not once it is fsynced. A -/// crash-restart can stamp a later batch with an offset a client has already recorded. -/// -/// -/// -public sealed class SendMessagesConfirmation -{ - /// - /// Stream identifier the batch was written to. - /// - public required uint StreamId { get; init; } - - /// - /// Topic identifier the batch was written to. - /// - public required uint TopicId { get; init; } - - /// - /// Partition the batch landed in. - /// - public required uint PartitionId { get; init; } - - /// - /// Offset assigned to the first message of the batch in that partition. - /// - public required ulong BaseOffset { get; init; } -} - -/// -/// Result of a send messages request. -/// -/// -/// An empty list means the batch committed with no offsets to -/// report (e.g. a background send merged into a larger batch). The server currently reports a -/// single partition per request; the list shape lets a later multi-partition send grow the -/// count without an API change. -/// -public sealed class SendMessagesResponse -{ - /// - /// One confirmation per partition the batch was committed to. - /// - public required IReadOnlyList Confirmations { get; init; } - - internal static SendMessagesResponse Empty { get; } = new() { Confirmations = [] }; -} diff --git a/foreign/csharp/Iggy_SDK/Exceptions/IggyInvalidStatusCodeException.cs b/foreign/csharp/Iggy_SDK/Exceptions/IggyInvalidStatusCodeException.cs index 9d73640d5c..6ba6e14d3e 100644 --- a/foreign/csharp/Iggy_SDK/Exceptions/IggyInvalidStatusCodeException.cs +++ b/foreign/csharp/Iggy_SDK/Exceptions/IggyInvalidStatusCodeException.cs @@ -25,18 +25,10 @@ public sealed class IggyInvalidStatusCodeException : Exception /// /// Status code returned by the server. /// - public int StatusCode { get; } + public int StatusCode { get; init; } - /// - /// Whether the status code was reported by the server rather than raised by the client. The two share one - /// code space, and only a server verdict may drive retry or failover: a locally raised code says nothing - /// about what the cluster did with the request. - /// - public bool FromServer { get; } - - internal IggyInvalidStatusCodeException(int statusCode, string message, bool fromServer = false) : base(message) + internal IggyInvalidStatusCodeException(int statusCode, string message) : base(message) { StatusCode = statusCode; - FromServer = fromServer; } } diff --git a/foreign/csharp/Iggy_SDK/Exceptions/VsrRequestOutcomeUnknownException.cs b/foreign/csharp/Iggy_SDK/Exceptions/VsrRequestOutcomeUnknownException.cs deleted file mode 100644 index f79334b282..0000000000 --- a/foreign/csharp/Iggy_SDK/Exceptions/VsrRequestOutcomeUnknownException.cs +++ /dev/null @@ -1,33 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -namespace Apache.Iggy.Exceptions; - -/// -/// The request produced no server verdict after transmission began, so the server may already have committed -/// it. Replaying it on a fresh consensus session would bypass server-side deduplication, so the SDK refuses to -/// retry and surfaces this instead: the caller decides whether re-issuing the operation is safe. -/// -/// -/// Deliberately derived from rather than or -/// , whichever ended the request. Both of those are routinely caught -/// and either retried or swallowed, which are the two responses this type exists to prevent. The triggering -/// exception is preserved as . -/// -public sealed class VsrRequestOutcomeUnknownException(Exception innerException) - : Exception("The VSR request outcome is unknown because no server verdict arrived after transmission began.", - innerException); diff --git a/foreign/csharp/Iggy_SDK/Exceptions/VsrSessionEvictedException.cs b/foreign/csharp/Iggy_SDK/Exceptions/VsrSessionEvictedException.cs deleted file mode 100644 index 824356618a..0000000000 --- a/foreign/csharp/Iggy_SDK/Exceptions/VsrSessionEvictedException.cs +++ /dev/null @@ -1,34 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -namespace Apache.Iggy.Exceptions; - -/// -/// An eviction frame arrived where a reply was expected. Internal: the transport decides whether the caller -/// sees or an unknown outcome, depending on whether the outstanding request could -/// still have committed. -/// -/// -/// The server emits evictions off its own heartbeat timer rather than as an answer, so the frame carries no -/// correlation with the request it interrupts and that request's real reply is never read. -/// -internal sealed class VsrSessionEvictedException(Exception verdict) - : Exception("The consensus session was evicted by the server.", verdict) -{ - /// The error the eviction reason maps to, for the requests it can safely be reported as. - internal Exception Verdict { get; } = verdict; -} diff --git a/foreign/csharp/Iggy_SDK/Factory/IggyClientFactory.cs b/foreign/csharp/Iggy_SDK/Factory/IggyClientFactory.cs index ef965f34bf..4300464900 100644 --- a/foreign/csharp/Iggy_SDK/Factory/IggyClientFactory.cs +++ b/foreign/csharp/Iggy_SDK/Factory/IggyClientFactory.cs @@ -20,7 +20,6 @@ using Apache.Iggy.Enums; using Apache.Iggy.IggyClient; using Apache.Iggy.IggyClient.Implementations; -using Apache.Iggy.Vsr; namespace Apache.Iggy.Factory; @@ -45,13 +44,8 @@ public static class IggyClientFactory /// Thrown when the specified protocol in is not /// supported. /// - /// - /// Thrown when is below the 256-byte header. - /// public static IIggyClient CreateClient(IggyClientConfigurator options) { - Validate(options); - return options.Protocol switch { Protocol.Http => CreateIggyHttpClient(options), @@ -60,15 +54,6 @@ public static IIggyClient CreateClient(IggyClientConfigurator options) }; } - private static void Validate(IggyClientConfigurator options) - { - if (options.Protocol == Protocol.Tcp && options.MaxResponseFrameSize < VsrHeader.HEADER_SIZE) - { - throw new ArgumentOutOfRangeException(nameof(options), options.MaxResponseFrameSize, - $"MaxResponseFrameSize must be at least {VsrHeader.HEADER_SIZE} bytes."); - } - } - private static IIggyClient CreateIggyTcpClient(IggyClientConfigurator options) { return new TcpMessageStream(options, options.LoggerFactory); @@ -82,7 +67,7 @@ private static IIggyClient CreateIggyHttpClient(IggyClientConfigurator options) private static HttpClient CreateHttpClient(IggyClientConfigurator options) { - var client = new HttpClient(new TransientHttpRetryHandler(new HttpClientHandler())); + var client = new HttpClient(); client.BaseAddress = new Uri(options.BaseAddress); return client; } diff --git a/foreign/csharp/Iggy_SDK/IggyClient/IIggyPublisher.cs b/foreign/csharp/Iggy_SDK/IggyClient/IIggyPublisher.cs index 7263c220cd..483814a630 100644 --- a/foreign/csharp/Iggy_SDK/IggyClient/IIggyPublisher.cs +++ b/foreign/csharp/Iggy_SDK/IggyClient/IIggyPublisher.cs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -using Apache.Iggy.Contracts; using Apache.Iggy.Kinds; using Apache.Iggy.Messages; @@ -47,11 +46,9 @@ public interface IIggyPublisher /// The partitioning strategy that determines which partition receives the messages. /// The collection of messages to be sent. /// The cancellation token to cancel the operation. - /// - /// Commit confirmations carrying the partition each batch landed in and its base offset. - /// - Task SendMessagesAsync(Identifier streamId, Identifier topicId, Partitioning partitioning, - IList messages, CancellationToken token = default); + /// A task representing the asynchronous operation. + Task SendMessagesAsync(Identifier streamId, Identifier topicId, Partitioning partitioning, IList messages, + CancellationToken token = default); /// /// Sends a single message to the specified stream and topic. See @@ -63,18 +60,19 @@ Task SendMessagesAsync(Identifier streamId, Identifier top /// payload memory (e.g. a pooled buffer) may be released /// once it completes. /// - Task SendMessagesAsync(Identifier streamId, Identifier topicId, Partitioning partitioning, - Message message, CancellationToken token = default) - { - return SendMessagesAsync(streamId, topicId, partitioning, new[] { message }, token); - } + Task SendMessagesAsync(Identifier streamId, Identifier topicId, Partitioning partitioning, Message message, + CancellationToken token = default) + => SendMessagesAsync(streamId, topicId, partitioning, new[] { message }, token); /// /// Forces a flush of the unsaved buffer to disk for a specific partition. /// /// - /// This feature is not supported by the server. Durability is handled by replication and the journal, - /// so there is no client-flushable in-memory buffer. + /// This method ensures that all pending messages in the in-memory buffer for the specified partition are written to + /// disk. + /// If is true, the data is both flushed to disk and synchronized (fsync), ensuring + /// durability. + /// If false, the data is only flushed to disk without synchronization. /// /// The stream identifier (numeric ID or name). /// The topic identifier (numeric ID or name). @@ -82,7 +80,6 @@ Task SendMessagesAsync(Identifier streamId, Identifier top /// If true, the data is flushed and synchronized to disk (fsync). If false, only flushed. /// The cancellation token to cancel the operation. /// A task representing the asynchronous operation. - /// Always thrown; the server does not support this command. Task FlushUnsavedBufferAsync(Identifier streamId, Identifier topicId, uint partitionId, bool fsync, CancellationToken token = default); } diff --git a/foreign/csharp/Iggy_SDK/IggyClient/IIggySystem.cs b/foreign/csharp/Iggy_SDK/IggyClient/IIggySystem.cs index aab60f1c6c..32acfcec89 100644 --- a/foreign/csharp/Iggy_SDK/IggyClient/IIggySystem.cs +++ b/foreign/csharp/Iggy_SDK/IggyClient/IIggySystem.cs @@ -86,10 +86,7 @@ public interface IIggySystem /// Sends a ping request to the server to verify connectivity. /// /// - /// This is a simple health check operation that can be used to verify the connection is active. On the - /// VSR wire protocol it also re-syncs the assignment of every consumer group this client has joined, so - /// it costs one extra round trip per joined group. The SDK never calls it on its own: an application - /// that wants assignments refreshed has to ping on its own cadence. + /// This is a simple health check operation that can be used to verify the connection is active. /// /// The cancellation token to cancel the operation. /// A task representing the asynchronous operation. diff --git a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs index b8f51c7a5e..4aba83255e 100644 --- a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs +++ b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs @@ -16,7 +16,6 @@ // under the License. using System.Buffers; -using System.IO.Hashing; using System.Net; using System.Net.Http.Headers; using System.Net.Http.Json; @@ -36,7 +35,6 @@ using Apache.Iggy.Messages; using Apache.Iggy.StringHandlers; using Apache.Iggy.Utils; -using Apache.Iggy.Vsr; using Partitioning = Apache.Iggy.Kinds.Partitioning; namespace Apache.Iggy.IggyClient.Implementations; @@ -49,7 +47,6 @@ public class HttpMessageStream : IIggyClient private const string Context = "csharp-sdk"; private readonly bool _allowAutoCommitWithEncryptor; - private readonly ConsumerGroupClientState _groupState = new(); private readonly HttpClient _httpClient; //TODO - create mechanism for refreshing jwt token @@ -207,8 +204,6 @@ public async Task DeleteTopicAsync(Identifier streamId, Identifier topicId, Canc { await HandleResponseAsync(response); } - - _groupState.InvalidatePartitionCount(new TopicKey(streamId, topicId)); } /// @@ -256,8 +251,8 @@ public async Task> GetTopicsAsync(Identifier stream } /// - public async Task SendMessagesAsync(Identifier streamId, Identifier topicId, - Partitioning partitioning, IList messages, + public async Task SendMessagesAsync(Identifier streamId, Identifier topicId, Partitioning partitioning, + IList messages, CancellationToken token = default) { if (MessageEncryptor is not null) @@ -271,11 +266,6 @@ public async Task SendMessagesAsync(Identifier streamId, I messages = encrypted; } - if (partitioning.Kind != Enums.Partitioning.PartitionId) - { - partitioning = await ResolvePartitioningAsync(streamId, topicId, partitioning, token); - } - var request = new MessageSendRequest { StreamId = streamId, @@ -294,19 +284,20 @@ public async Task SendMessagesAsync(Identifier streamId, I { await HandleResponseAsync(response); } - - return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token) - ?? throw new InvalidResponseException("Send messages reply carried no confirmation body."); } - /// - /// This feature is not supported by the server. - /// - /// - public Task FlushUnsavedBufferAsync(Identifier streamId, Identifier topicId, uint partitionId, bool fsync, + /// + public async Task FlushUnsavedBufferAsync(Identifier streamId, Identifier topicId, uint partitionId, bool fsync, CancellationToken token = default) { - throw new FeatureUnavailableException(); + var url = CreateUrl($"/streams/{streamId}/topics/{topicId}/messages/flush/{partitionId}/{fsync}"); + + var response = await _httpClient.GetAsync(url, token); + + if (!response.IsSuccessStatusCode) + { + await HandleResponseAsync(response, true); + } } /// @@ -624,8 +615,6 @@ var response { await HandleResponseAsync(response); } - - _groupState.InvalidatePartitionCount(new TopicKey(streamId, topicId)); } /// @@ -657,8 +646,6 @@ public async Task CreatePartitionsAsync(Identifier streamId, Identifier topicId, { await HandleResponseAsync(response); } - - _groupState.InvalidatePartitionCount(new TopicKey(streamId, topicId)); } /// @@ -898,41 +885,6 @@ public string GetCurrentAddress() return _httpClient.BaseAddress?.ToString() ?? string.Empty; } - /// - /// Resolves balanced and message-key partitioning to an explicit partition id, mirroring the TCP client. - /// Server-side balanced resolution races partition-count changes (a send right after CreatePartitions can - /// land on a stale round-robin cycle), so the client picks the partition and sends it explicitly. - /// - private async ValueTask ResolvePartitioningAsync(Identifier streamId, Identifier topicId, - Partitioning partitioning, CancellationToken token) - { - var key = new TopicKey(streamId, topicId); - var partitionCount = _groupState.PartitionCount(key); - if (partitionCount is null) - { - var topic = await GetTopicByIdAsync(streamId, topicId, token) - ?? throw new IggyInvalidStatusCodeException((int)HttpStatusCode.NotFound, - $"Topic {topicId} was not found in stream {streamId}.", true); - _groupState.SetPartitionCount(key, topic.PartitionsCount); - partitionCount = topic.PartitionsCount; - } - - if (partitionCount == 0) - { - throw new IggyInvalidStatusCodeException((int)HttpStatusCode.NotFound, - $"Topic {topicId} in stream {streamId} has no partitions to resolve the message to.", true); - } - - var partition = partitioning.Kind switch - { - Enums.Partitioning.Balanced => _groupState.NextBalancedPartition(key, partitionCount.Value), - Enums.Partitioning.MessageKey => XxHash32.HashToUInt32(partitioning.Value) % partitionCount.Value, - _ => throw new FeatureUnavailableException() - }; - - return Partitioning.PartitionId((int)partition); - } - private void DecryptMessages(IReadOnlyList messages, uint partitionId) { foreach (var message in messages) @@ -1029,30 +981,20 @@ private static byte[] Decrypt(IMessageEncryptor encryptor, ReadOnlySpan da private static async Task HandleResponseAsync(HttpResponseMessage response, bool shouldThrowOnGetNotFound = false) { - if (response.IsSuccessStatusCode) + if ((int)response.StatusCode > 300 + && (int)response.StatusCode < 500 + && !(response.RequestMessage!.Method == HttpMethod.Get && response.StatusCode == HttpStatusCode.NotFound && + !shouldThrowOnGetNotFound)) { - return; + var err = await response.Content.ReadAsStringAsync(); + var errorModel = JsonSerializer.Deserialize(err); + throw new IggyInvalidStatusCodeException(errorModel?.Id ?? -1, err); } - if (response.RequestMessage!.Method == HttpMethod.Get && response.StatusCode == HttpStatusCode.NotFound && - !shouldThrowOnGetNotFound) + if (response.StatusCode == HttpStatusCode.InternalServerError) { - return; + throw new Exception("Internal server error"); } - - var err = await response.Content.ReadAsStringAsync(); - ErrorResponse? errorModel = null; - try - { - errorModel = JsonSerializer.Deserialize(err); - } - catch (JsonException) - { - // A gateway or proxy error body is not the server's JSON schema; the raw text still travels in - // the exception message. - } - - throw new IggyInvalidStatusCodeException(errorModel?.Id ?? -1, err, true); } private static string CreateUrl(ref MessageRequestInterpolationHandler message) diff --git a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.Vsr.cs b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.Vsr.cs deleted file mode 100644 index 81898c0994..0000000000 --- a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.Vsr.cs +++ /dev/null @@ -1,978 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Buffers; -using System.Buffers.Binary; -using System.IO.Hashing; -using System.Runtime.ExceptionServices; -using Apache.Iggy.ConnectionStream; -using Apache.Iggy.Contracts; -using Apache.Iggy.Contracts.Auth; -using Apache.Iggy.Contracts.Tcp; -using Apache.Iggy.Enums; -using Apache.Iggy.Exceptions; -using Apache.Iggy.Kinds; -using Apache.Iggy.Messages; -using Apache.Iggy.Utils; -using Apache.Iggy.Vsr; -using Microsoft.Extensions.Logging; -using Partitioning = Apache.Iggy.Kinds.Partitioning; - -namespace Apache.Iggy.IggyClient.Implementations; - -/// -/// The consensus (VSR) half of the TCP client: the framed request path, the leader redirection, the -/// register handshake, and the client-side partitioning and consumer-group assignment the broker does not -/// resolve server-side. The command surface lives in . -/// -public sealed partial class TcpMessageStream -{ - /// - /// Upper bound for a whole VSR request: the transient replays and the leader failovers share it, and so - /// do the reply header and body reads. The connection is lockstep, so an unanswered read would hold the - /// sending semaphore forever and wedge every later request. - /// - private const int VsrRequestTimeoutMs = 30_000; - - /// Backoff between replays of a transiently refused request. - private const int VsrTransientRetryIntervalMs = 50; - - /// - /// Largest body still sent as one contiguous frame with its header. Beyond this the copy outweighs the - /// syscall and the extra segment it saves, so header and body go out as two writes. - /// - private const int VsrContiguousFrameLimit = 4 * 1024; - - /// - /// How long a request replays on the same connection - /// before the leader roster is re-checked. A node that stopped being primary refuses forever, so - /// replaying alone never recovers. - /// - private const int VsrTransientFailoverCheckMs = 2_000; - - /// - /// How long a transiently leaderless roster is polled before the connection proceeds on the current - /// node anyway. A restarted node cedes the primaryship its stale view assigns it, and the peers need - /// about one heartbeat timeout to elect. - /// - private const int VsrLeaderlessWaitMs = 5_000; - - private const int VsrLeaderlessPollMs = 250; - - /// - /// Cap on consecutive leader redirects, so a flapping roster cannot spin the connect loop or the - /// transient failover path. The budget is client-wide and resets on a roster check that finds the - /// current node is the leader, and on every request that completes, so a client that outlives more - /// leader changes than the cap does not latch onto a follower for good. - /// - private const int VsrMaxLeaderRedirects = 3; - - /// - /// Attempts a consumer-group poll gets before it gives up and reports an empty poll: one re-sync after - /// the coordinator fences a stale assignment, then one retry. - /// - private const int VsrGroupPollMaxAttempts = 2; - - /// - /// Partition id a fenced group poll echoes instead of a typed error, matching - /// RESYNC_REQUIRED_PARTITION_SENTINEL (u32::MAX). The reply header carries no status for an - /// empty poll, so the sentinel is the only channel the coordinator has to ask for a re-sync. - /// - private const int VsrResyncRequiredPartitionSentinel = -1; - - /// - /// Shared empty poll result. An idle consumer loop returns one on every iteration, and the instance owns - /// no rented buffer - disposes to nothing - so it is safe to hand out - /// repeatedly even after a caller disposes it. - /// - private static readonly PolledMessagesRental EmptyPolledMessages = new(EmptyMemoryOwner.Instance) - { - PartitionId = 0, - CurrentOffset = 0, - Messages = [] - }; - - private readonly ConsensusSession _consensusSession = new(); - private readonly ConsumerGroupClientState _groupState = new(); - private readonly byte[] _vsrReplyHeaderBuffer = new byte[VsrHeader.HEADER_SIZE]; - - // The redirect budget is refunded by a completed request, and the roster check a redirect runs is itself a - // request. Without this the refund lands between the check and the increment that reads the budget, and the - // counter never leaves zero. Nonzero for the duration of a roster read, so that refund is skipped. - private int _leaderProbeDepth; - - /// - /// Runs the consensus register handshake and binds the session it commits. Everything before the bind - /// is a consumed register on the server, so any failure resets the session: the next attempt must - /// re-register under a fresh client id rather than send requests the primary would fence. - /// - /// - /// A bound connection must commit logout before it can register again. The server treats a register on - /// an already-bound transport as an idempotent replay of the existing binding, so re-arming only the - /// local session would pair a fresh client id and request counter with the old server session. - /// - private async Task LoginRegisterAsync(int code, byte[] message, CancellationToken token) - { - for (var redirects = 0; ; redirects++) - { - if (_consensusSession.IsBound) - { - await LogoutUserAsync(token); - } - else if (_state == ConnectionState.Authenticated) - { - SetConnectionStateAsync(ConnectionState.Connected); - } - - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, code); - - SetConnectionStateAsync(ConnectionState.Authenticating); - - LoginRegisterResponse response; - try - { - Interlocked.Exchange(ref _skipAutoLoginOnce, 1); - using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); - - response = LoginRegister.Deserialize(responseBuffer.Memory.Span); - _consensusSession.Bind(response.Session); - } - catch - { - await ResetConsensusSessionAsync(); - if (_state == ConnectionState.Authenticating) - { - SetConnectionStateAsync(ConnectionState.Connected); - } - - throw; - } - finally - { - Interlocked.Exchange(ref _skipAutoLoginOnce, 0); - } - - _logger.LogInformation( - "Authenticated against the server, version {ServerVersion}, protocol version {ServerProtocolVersion}", - response.ServerVersion, response.ServerProtocolVersion); - SetConnectionStateAsync(ConnectionState.Authenticated); - - var authResponse = new AuthResponse((int)response.UserId, null); - if (IsConnecting) - { - return authResponse; - } - - if (redirects >= VsrMaxLeaderRedirects) - { - _logger.LogWarning("Maximum leader redirections reached while registering, staying on {Address}", - _currentAddress); - - return authResponse; - } - - if (!await RedirectAsync(token)) - { - return authResponse; - } - - await ConnectAsync(false, token); - } - } - - /// - /// Whether the partitioning has to be resolved to an explicit partition id before the request is framed. - /// The broker never picks a partition, so balanced and message-key kinds resolve client-side. - /// - private static bool NeedsClientSidePartitioning(Partitioning partitioning) - { - return partitioning.Kind != Enums.Partitioning.PartitionId; - } - - private async Task SendMessagesResolvedAsync(Identifier streamId, Identifier topicId, - Partitioning partitioning, IList messages, CancellationToken token) - { - var resolved = await ResolvePartitioningAsync(streamId, topicId, partitioning, token); - - return await SendMessagesCoreAsync(streamId, topicId, resolved, AsSpan(messages), token); - } - - /// - /// Resolves balanced and message-key partitioning to an explicit partition id, mirroring - /// core/common/src/traits/binary_impls/messages.rs. The VSR broker never picks a partition, so - /// sending either kind on the wire would fail to route. - /// - private async ValueTask ResolvePartitioningAsync(Identifier streamId, Identifier topicId, - Partitioning partitioning, CancellationToken token) - { - var partitionCount = await TopicPartitionCountAsync(streamId, topicId, token); - if (partitionCount == 0) - { - throw VsrError.Exception(VsrError.TOPIC_ID_NOT_FOUND, - $"Topic {topicId} in stream {streamId} has no partitions to resolve the message to."); - } - - var partition = partitioning.Kind switch - { - Enums.Partitioning.Balanced => _groupState.NextBalancedPartition(new TopicKey(streamId, topicId), - partitionCount), - Enums.Partitioning.MessageKey => XxHash32.HashToUInt32(partitioning.Value) % partitionCount, - _ => throw VsrError.Exception(VsrError.FEATURE_UNAVAILABLE, - $"Partitioning kind {partitioning.Kind} cannot be resolved to a partition id.") - }; - - return Partitioning.PartitionId((int)partition); - } - - private async ValueTask TopicPartitionCountAsync(Identifier streamId, Identifier topicId, - CancellationToken token) - { - var key = new TopicKey(streamId, topicId); - if (_groupState.PartitionCount(key) is { } cached) - { - return cached; - } - - var topic = await GetTopicByIdAsync(streamId, topicId, token); - if (topic is null) - { - throw VsrError.Exception(VsrError.TOPIC_ID_NOT_FOUND, - $"Topic {topicId} was not found in stream {streamId}."); - } - - _groupState.SetPartitionCount(key, topic.PartitionsCount); - - return topic.PartitionsCount; - } - - /// - /// Polls one of the group member's assigned partitions, round-robin. A fence rejection - either the typed - /// error or the sentinel partition id an empty poll carries - re-syncs the assignment and retries once. - /// - private async Task PollGroupMessagesRentedAsync(Identifier streamId, Identifier topicId, - Consumer consumer, PollingStrategy pollingStrategy, uint count, bool autoCommit, CancellationToken token) - { - var key = new GroupKey(streamId, topicId, consumer.ConsumerId); - if (!_groupState.HasAssignment(key)) - { - await SyncGroupAssignmentAsync(streamId, topicId, consumer.ConsumerId, token); - } - - for (var attempt = 0; attempt < VsrGroupPollMaxAttempts; attempt++) - { - if (_groupState.NextGroupPartition(key) is not { } partitionId) - { - if (!_groupState.IsRegistered(key)) - { - throw VsrError.Exception(VsrError.CONSUMER_GROUP_MEMBER_NOT_FOUND, - $"Client is not a member of consumer group {consumer.ConsumerId} on topic {topicId}."); - } - - return EmptyPolledMessages; - } - - PolledMessagesRental? rental = null; - try - { - rental = await PollPartitionMessagesRentedAsync(streamId, topicId, partitionId, consumer, - pollingStrategy, count, autoCommit, token); - } - catch (IggyInvalidStatusCodeException e) when (e is - { - StatusCode: VsrError.CONSUMER_GROUP_PARTITION_NOT_OWNED, - FromServer: true - }) - { - // Both fence shapes - the typed error and the sentinel an empty poll carries - land on the same - // re-sync below. - } - - if (rental is not null) - { - if (rental.Messages.Count != 0 || rental.PartitionId != VsrResyncRequiredPartitionSentinel) - { - return rental; - } - - rental.Dispose(); - } - - _groupState.InvalidateAssignment(key); - await SyncGroupAssignmentAsync(streamId, topicId, consumer.ConsumerId, token); - } - - return EmptyPolledMessages; - } - - /// - /// Pulls the requesting member's assignment from the coordinator into the cache. An empty reply means the - /// client is not a member: the coordinator answers with an assignment header for any member, including - /// one holding zero partitions. - /// - private async Task SyncGroupAssignmentAsync(Identifier streamId, Identifier topicId, Identifier groupId, - CancellationToken token) - { - var message = TcpContracts.GetGroup(streamId, topicId, groupId); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.SYNC_CONSUMER_GROUP_CODE); - - using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); - - var key = new GroupKey(streamId, topicId, groupId); - if (responseBuffer.Memory.Length == 0) - { - // Deregistering is the only thing that observes a server-side removal (group deleted, member - // evicted); without it membership latches true and every later poll returns empty. - _groupState.DeregisterGroup(key); - - return; - } - - var assignment = SyncConsumerGroupAssignment.Decode(responseBuffer.Memory.Span); - _groupState.RegisterGroup(key, streamId, topicId, groupId); - _groupState.SetAssignment(key, assignment.Generation, assignment.Partitions); - } - - /// - /// Re-syncs every joined group so a widened assignment (a partition-count change, say) is picked up - /// without first hitting an ownership fence. One failing group is logged and skipped so it cannot stall - /// the rest. - /// - private async Task RefreshGroupAssignmentsAsync(CancellationToken token) - { - foreach (var group in _groupState.RegisteredGroups()) - { - try - { - await SyncGroupAssignmentAsync(group.StreamId, group.TopicId, group.GroupId, token); - } - catch (Exception e) when (e is not OperationCanceledException) - { - _logger.LogWarning(e, - "Failed to refresh the consumer group assignment for {StreamId}|{TopicId}|{GroupId}", - group.StreamId, group.TopicId, group.GroupId); - } - } - } - - - /// - /// Points the client at the current leader when it is not the node this connection is on, leaving the - /// stream closed for the caller to reconnect. The redirect budget is client-wide: the connect loop and - /// the transient failover path spend the same counter, and it is refunded as soon as a roster check - /// lands on the leader. - /// - private async Task RedirectAsync(CancellationToken token) - { - // The probe issues a request of its own, which takes the sending semaphore, so it cannot run under that - // lock. Only the commit below does. - var currentLeaderNode = await GetCurrentLeaderNodeAsync(token); - if (currentLeaderNode == null) - { - Interlocked.Exchange(ref _leaderRedirectCount, 0); - return false; - } - - var leaderAddress = ServerAddress.HostPort(currentLeaderNode.Ip, currentLeaderNode.Endpoints.Tcp); - if (ServerAddress.IsSame(leaderAddress, _currentAddress)) - { - Interlocked.Exchange(ref _leaderRedirectCount, 0); - return false; - } - - if (Interlocked.Increment(ref _leaderRedirectCount) > VsrMaxLeaderRedirects) - { - _logger.LogWarning("Maximum leader redirections reached, continuing on {Address}", _currentAddress); - return false; - } - - _logger.LogInformation("Leader address changed. Trying to reconnect to {Address}", - leaderAddress); - - // The address move and the drop are one step: a reader that saw the new address but the old stream would - // find every later redirect short-circuited by the address check above, with no path back to the leader. - await _sendingSemaphore.WaitAsync(token); - try - { - _currentAddress = leaderAddress; - DropVsrConnectionLocked(_stream); - } - finally - { - _sendingSemaphore.Release(); - } - - return true; - } - - private async Task GetCurrentLeaderNodeAsync(CancellationToken token) - { - var leaderlessDeadline = Environment.TickCount64 + VsrLeaderlessWaitMs; - Interlocked.Increment(ref _leaderProbeDepth); - try - { - while (true) - { - var clusterMetadata = await GetClusterMetadataAsync(token); - if (clusterMetadata == null) - { - return null; - } - - if (clusterMetadata.Nodes.Count() == 1) - { - return null; - } - - var leaderNode = clusterMetadata.Nodes.FirstOrDefault(x => - x.Role == ClusterNodeRole.Leader && x.Status == ClusterNodeStatus.Healthy); - if (leaderNode != null) - { - return leaderNode; - } - - if (Environment.TickCount64 >= leaderlessDeadline) - { - _logger.LogWarning("No leader in the cluster metadata after {WaitMs} ms, continuing on {Address}", - VsrLeaderlessWaitMs, _currentAddress); - - return null; - } - - await Task.Delay(VsrLeaderlessPollMs, token); - } - } - // todo: change after error refactoring, error code 5 is for feature not supported - catch (IggyInvalidStatusCodeException e) when (e is - { - StatusCode: VsrError.FEATURE_UNAVAILABLE, FromServer: true - }) - { - return null; - } - catch (Exception e) when (e is not OperationCanceledException) - { - _logger.LogWarning(e, "Failed to read the cluster metadata, continuing on {Address}", _currentAddress); - - return null; - } - finally - { - Interlocked.Decrement(ref _leaderProbeDepth); - } - } - - /// - /// Sends a consensus-framed request. The call sites still build the classic - /// [size u32][code u32][body] buffer, so the code is read back from it here and the body is written - /// right after the 256-byte consensus header - two writes, no concatenation. - /// - /// - /// One deadline bounds the whole request across transient replays AND leader failovers. Login and - /// register replay on this connection for the whole budget instead: the connect flow owns leader - /// redirection for the handshake, and reconnecting from underneath it would recurse. - /// - private async Task> SendRawVsrAsync(ReadOnlyMemory payload, CancellationToken token) - { - var code = (int)BinaryPrimitives.ReadUInt32LittleEndian(payload.Span.Slice(4, 4)); - ReadOnlyMemory body = payload[8..]; - var isLoginRegister = code is CommandCodes.LOGIN_REGISTER_CODE or CommandCodes.LOGIN_REGISTER_WITH_PAT_CODE; - var overallDeadline = Environment.TickCount64 + VsrRequestTimeoutMs; - var headerBuffer = ArrayPool.Shared.Rent(VsrHeader.HEADER_SIZE); - Memory header = headerBuffer.AsMemory(0, VsrHeader.HEADER_SIZE); - var requestEncoded = false; - TcpConnectionStream? lastStream = null; - - try - { - while (true) - { - var transientDeadline = isLoginRegister - ? overallDeadline - : Math.Min(overallDeadline, Environment.TickCount64 + VsrTransientFailoverCheckMs); - - var attempt = await SendVsrAttemptAsync(code, body, header, transientDeadline, overallDeadline, - token); - requestEncoded |= attempt.Encoded; - lastStream = attempt.Stream; - - if (attempt.Error is null) - { - // A roster read taken by RedirectAsync must not refund the budget it is about to be charged - // against, or the cap can never be reached. - if (Volatile.Read(ref _leaderRedirectCount) != 0 && Volatile.Read(ref _leaderProbeDepth) == 0) - { - Interlocked.Exchange(ref _leaderRedirectCount, 0); - } - - return attempt.Response!; - } - - if (attempt.Error is IggyInvalidStatusCodeException - { - StatusCode: VsrError.TRANSIENT_NOT_ACCEPTED, FromServer: true - } - && !isLoginRegister - && Environment.TickCount64 < overallDeadline) - { - if (await RedirectAsync(token)) - { - await ConnectAsync(token); - } - - continue; - } - - if (attempt.Error is VsrSessionEvictedException evicted) - { - if (attempt.RequestStarted && !VsrOperations.IsReplaySafeRead(code, isLoginRegister, body.Span)) - { - throw new VsrRequestOutcomeUnknownException(evicted); - } - - ExceptionDispatchInfo.Throw(evicted.Verdict); - } - - if (attempt.RequestStarted - && !VsrOperations.IsReplaySafeRead(code, isLoginRegister, body.Span) - && !IsDefinitiveVerdict(attempt.Error)) - { - throw new VsrRequestOutcomeUnknownException(attempt.Error); - } - - ExceptionDispatchInfo.Throw(attempt.Error); - } - } - catch (OperationCanceledException) - { - if (requestEncoded) - { - await DropVsrConnectionAsync(lastStream); - } - - throw; - } - finally - { - ArrayPool.Shared.Return(headerBuffer); - } - } - - /// - /// Whether the failure carries the server's verdict on this request. A lost connection, a reply frame - /// the client refused or discarded, and a NOT_COMMITTED that outlived its replay deadline all leave the - /// outcome of a request the server may still commit unknowable. - /// - private static bool IsDefinitiveVerdict(Exception error) - { - return error is IggyInvalidStatusCodeException - { - FromServer: true, - StatusCode: not VsrError.TRANSIENT_NOT_COMMITTED - }; - } - - /// - /// One attempt on the current connection: encode the header into , write the - /// frame, and replay it - same session, same request id - while the server answers transiently. - /// - private async ValueTask SendVsrAttemptAsync(int code, ReadOnlyMemory body, Memory header, - long transientDeadline, long readDeadline, CancellationToken token) - { - await _sendingSemaphore.WaitAsync(token); - - var encoded = false; - var requestStarted = false; - byte[]? frameBuffer = null; - - // Read the stream once for the whole attempt. Nothing may swap the field without the sending lock this - // call holds, so the frame and its reply cannot be split across two sockets, and a teardown after the - // lock is gone can tell this connection from a replacement a reconnect installed since. - var stream = _stream; - try - { - // A small request goes out as one write. Two writes cost two syscalls and, with Nagle disabled, two - // TCP segments (two TLS records when encrypted) for what the reference encoder sends as a single - // contiguous frame. Above the threshold the copy costs more than the extra write saves. Renting - // inside the try keeps the semaphore paired with its release even if the pool throws. - if (body.Length <= VsrContiguousFrameLimit) - { - frameBuffer = ArrayPool.Shared.Rent(VsrHeader.HEADER_SIZE + body.Length); - } - - VsrHeader.EncodeRequestHeader(header.Span, _consensusSession, code, body.Span); - - encoded = true; - - var frame = Memory.Empty; - if (frameBuffer is not null) - { - frame = frameBuffer.AsMemory(0, VsrHeader.HEADER_SIZE + body.Length); - header.CopyTo(frame); - body.CopyTo(frame[VsrHeader.HEADER_SIZE..]); - } - - while (true) - { - try - { - // Everything that fails without reaching the socket has to fail before this point: past it a - // failure is reported as an outcome the server alone knows, which for a replicated write - // tells the caller its request may have committed twice. - token.ThrowIfCancellationRequested(); - requestStarted = true; - - if (frameBuffer is not null) - { - await stream.SendAsync(frame, token); - } - else - { - await stream.SendAsync(header, token); - await stream.SendAsync(body, token); - } - - await stream.FlushAsync(token); - - IMemoryOwner response = await ReadVsrReplyAsync(stream, readDeadline, token); - - return VsrAttempt.Ok(response, stream); - } - catch (IggyInvalidStatusCodeException e) when (IsReplayableTransient(e, transientDeadline, - readDeadline)) - { - var governingDeadline = e.StatusCode == VsrError.TRANSIENT_NOT_COMMITTED - ? readDeadline - : transientDeadline; - var remaining = governingDeadline - Environment.TickCount64; - await Task.Delay((int)Math.Clamp(remaining, 0, VsrTransientRetryIntervalMs), token); - } - catch (Exception e) when (IsConnectionException(e)) - { - DropVsrConnectionLocked(stream); - - return VsrAttempt.Failed(encoded, e, requestStarted, stream); - } - catch (OperationCanceledException e) - { - DropVsrConnectionLocked(stream); - - return VsrAttempt.Failed(encoded, e, requestStarted, stream); - } - catch (Exception e) - { - return VsrAttempt.Failed(encoded, e, requestStarted, stream); - } - } - } - catch (OperationCanceledException e) - { - if (encoded) - { - DropVsrConnectionLocked(stream); - } - - return VsrAttempt.Failed(encoded, e, requestStarted, stream); - } - catch (Exception e) - { - return VsrAttempt.Failed(encoded, e, requestStarted, stream); - } - finally - { - if (frameBuffer is not null) - { - ArrayPool.Shared.Return(frameBuffer); - } - - _sendingSemaphore.Release(); - } - } - - private async Task> ReadVsrReplyAsync(TcpConnectionStream stream, long readDeadline, - CancellationToken token) - { - var remaining = readDeadline - Environment.TickCount64; - if (remaining <= 0) - { - throw new IOException($"Timed out after {VsrRequestTimeoutMs} ms waiting for a consensus reply."); - } - - // One timer for the whole reply: the deadline covers the frame, not each partial read, so a per-read - // source would both re-arm the budget and allocate a timer per socket read. - using var readCancellation = CancellationTokenSource.CreateLinkedTokenSource(token); - readCancellation.CancelAfter((int)Math.Min(remaining, VsrRequestTimeoutMs)); - - await ReadExactVsrAsync(stream, _vsrReplyHeaderBuffer, readCancellation.Token, token); - - var command = VsrHeader.PeekCommand(_vsrReplyHeaderBuffer); - if (command == Command2.Eviction) - { - var eviction = VsrHeader.ReadEviction(_vsrReplyHeaderBuffer); - _logger.LogWarning("Consensus session evicted by the server: {Reason}", eviction.Reason); - DropVsrConnectionLocked(stream); - - throw new VsrSessionEvictedException(VsrReplyDecoder.ToException(eviction)); - } - - if (command != Command2.Reply) - { - // Neither a reply nor an eviction: this frame was never an answer to the outstanding request, so - // whatever the peer does send for it would be read as the next request's reply and handed to the - // wrong caller. The size field of a frame the client cannot model is no basis for resynchronising. - DropVsrConnectionLocked(stream); - - throw VsrError.Exception(VsrError.INVALID_COMMAND, - $"Unexpected consensus frame {command} on a client connection."); - } - - int bodySize; - try - { - bodySize = VsrReplyDecoder.ReadBodySize(_vsrReplyHeaderBuffer); - if (VsrHeader.HEADER_SIZE + (long)bodySize > _configuration.MaxResponseFrameSize) - { - throw VsrError.Exception(VsrError.INVALID_COMMAND, - $"Reply frame of {VsrHeader.HEADER_SIZE + bodySize} bytes exceeds the configured maximum of " + - $"{_configuration.MaxResponseFrameSize} bytes."); - } - } - catch - { - // An announced size the client refuses to read - undersized, oversized - leaves the body on the - // wire, so the stream no longer sits on a frame boundary and the next reply would decode body bytes - // as a header. - DropVsrConnectionLocked(stream); - - throw; - } - - if (bodySize == 0) - { - VsrReplyDecoder.Decode(_vsrReplyHeaderBuffer, ReadOnlyMemory.Empty); - - return EmptyMemoryOwner.Instance; - } - - var buffer = ArrayPool.Shared.Rent(bodySize); - try - { - await ReadExactVsrAsync(stream, buffer.AsMemory(0, bodySize), readCancellation.Token, token); - ReadOnlyMemory decoded = VsrReplyDecoder.Decode(_vsrReplyHeaderBuffer, buffer.AsMemory(0, bodySize)); - if (decoded.IsEmpty) - { - ArrayPool.Shared.Return(buffer); - - return EmptyMemoryOwner.Instance; - } - - // The decoded payload is always a suffix of the body - the funnel only strips the leading - // committed result section. - return new PooledMemoryOwner(buffer, bodySize - decoded.Length, decoded.Length); - } - catch - { - ArrayPool.Shared.Return(buffer); - throw; - } - } - - private async ValueTask ReadExactVsrAsync(TcpConnectionStream stream, Memory buffer, - CancellationToken readToken, - CancellationToken token) - { - var totalRead = 0; - while (totalRead < buffer.Length) - { - int readBytes; - try - { - readBytes = await stream.ReadAsync(buffer[totalRead..], readToken); - } - catch (OperationCanceledException) when (!token.IsCancellationRequested) - { - throw new IOException($"Timed out after {VsrRequestTimeoutMs} ms waiting for a consensus reply."); - } - - if (readBytes == 0) - { - throw new IggyZeroBytesException(); - } - - totalRead += readBytes; - } - } - - /// - /// Drops the consensus session and the group state scoped to it. Consumer-group assignments are fenced by - /// a generation the coordinator tracks per session, so carrying them into a new session would fence every - /// poll until the first re-sync. The balanced cursors and the partition counts survive: neither is bound - /// to a session, and dropping them costs a metadata round trip per topic on the next produce. - /// - private void ResetConsensusSession() - { - _consensusSession.Reset(); - _groupState.ClearSessionScoped(); - } - - /// - /// Resets the session on behalf of a caller that does not hold the sending lock, so no request can be - /// encoding against the identity while it is re-armed. - /// - private async ValueTask ResetConsensusSessionAsync() - { - // Taking a disposed semaphore would replace the failure the caller is about to rethrow with an - // ObjectDisposedException, and a disposed client has nothing left to fence. Dispose can still land - // between the check and the wait, so the wait itself has to tolerate it. - if (_disposed || !await TryEnterSendingSemaphoreAsync()) - { - return; - } - - try - { - ResetConsensusSession(); - } - finally - { - _sendingSemaphore.Release(); - } - } - - /// - /// Takes the sending lock for a teardown that must not fail. Returns false once the client is disposed: - /// the caller is unwinding an earlier failure and has nothing left to fence. - /// - private async ValueTask TryEnterSendingSemaphoreAsync() - { - try - { - await _sendingSemaphore.WaitAsync(CancellationToken.None); - return true; - } - catch (ObjectDisposedException) - { - return false; - } - } - - /// - /// Drops the connection along with the session. A late or half-read reply would desync the framing of the - /// next request, so the stream cannot be reused. The caller must hold , - /// which owns every write to . - /// - /// - /// The connection the caller was using. A reconnect that completed in the meantime already closed it and - /// re-armed the session, so dropping anything but the live one would tear down a healthy replacement. - /// - private void DropVsrConnectionLocked(TcpConnectionStream? stream) - { - if (!ReferenceEquals(_stream, stream)) - { - return; - } - - ResetConsensusSession(); - _stream?.Close(); - SetConnectionStateAsync(ConnectionState.Disconnected); - } - - /// Drops the connection on behalf of a caller that no longer holds the sending lock. - private async ValueTask DropVsrConnectionAsync(TcpConnectionStream? stream) - { - // Dispose already closed the stream, and taking a disposed semaphore here would replace the - // cancellation the caller is about to rethrow with an ObjectDisposedException. Dispose can still land - // between the check and the wait, so the wait itself has to tolerate it. - // The request this drop belongs to was cancelled; the drop itself still has to run to completion. - if (_disposed || !await TryEnterSendingSemaphoreAsync()) - { - return; - } - - try - { - DropVsrConnectionLocked(stream); - } - finally - { - _sendingSemaphore.Release(); - } - } - - private static bool IsReplayableTransient(IggyInvalidStatusCodeException error, long transientDeadline, - long readDeadline) - { - if (!error.FromServer) - { - return false; - } - - return error.StatusCode switch - { - VsrError.TRANSIENT_NOT_COMMITTED => Environment.TickCount64 < readDeadline, - VsrError.TRANSIENT_NOT_ACCEPTED => Environment.TickCount64 < transientDeadline, - _ => false - }; - } - - /// Outcome of one call on the current connection. - /// Whether the header was encoded, i.e. whether a request id may have been consumed. - /// The decoded reply payload, non-null exactly when is null. - /// The failure that ended the attempt, or null on success. - /// - /// Whether any byte of the frame was written, which makes the server-side outcome unknowable on failure. - /// - /// - /// The connection the attempt ran on, so a caller that drops it after releasing the sending lock can tell - /// its own connection from a replacement a reconnect installed since. - /// - private readonly record struct VsrAttempt( - bool Encoded, - IMemoryOwner? Response, - Exception? Error, - bool RequestStarted, - TcpConnectionStream? Stream) - { - public static VsrAttempt Ok(IMemoryOwner response, TcpConnectionStream stream) - { - return new VsrAttempt(true, response, null, true, stream); - } - - public static VsrAttempt Failed(bool encoded, Exception error, bool requestStarted, - TcpConnectionStream? stream) - { - return new VsrAttempt(encoded, null, error, requestStarted, stream); - } - } - - /// Owns a pooled buffer while exposing only the decoded payload slice inside it. - internal sealed class PooledMemoryOwner(byte[] buffer, int start, int length) : IMemoryOwner - { - private int _disposed; - - public Memory Memory => buffer.AsMemory(start, length); - - public void Dispose() - { - if (Interlocked.Exchange(ref _disposed, 1) == 0) - { - ArrayPool.Shared.Return(buffer); - } - } - } -} diff --git a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs index 146c93ee13..a2071c3513 100644 --- a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs +++ b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs @@ -22,6 +22,7 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Security.Cryptography.X509Certificates; +using System.Text; using Apache.Iggy.Configuration; using Apache.Iggy.ConnectionStream; using Apache.Iggy.Contracts; @@ -34,17 +35,15 @@ using Apache.Iggy.Mappers; using Apache.Iggy.Messages; using Apache.Iggy.Utils; -using Apache.Iggy.Vsr; using Microsoft.Extensions.Logging; using Partitioning = Apache.Iggy.Kinds.Partitioning; namespace Apache.Iggy.IggyClient.Implementations; /// -/// A TCP client for interacting with the Iggy server over the consensus (VSR) framing. The framed request -/// path, leader redirection and register handshake live in TcpMessageStream.Vsr.cs. +/// A TCP client for interacting with the Iggy server. /// -public sealed partial class TcpMessageStream : IIggyClient +public sealed class TcpMessageStream : IIggyClient { private const int InvalidCommandStatus = 3; @@ -58,28 +57,18 @@ public sealed partial class TcpMessageStream : IIggyClient ]; private readonly IggyClientConfigurator _configuration; - private readonly SemaphoreSlim _connectGate = new(1, 1); private readonly EventAggregator _connectionEvents; private readonly SemaphoreSlim _connectionSemaphore; private readonly ILogger _logger; + private readonly byte[] _responseHeaderBuffer = new byte[BufferSizes.EXPECTED_RESPONSE_SIZE]; private readonly SemaphoreSlim _sendingSemaphore; private string _currentAddress = string.Empty; private X509Certificate2Collection _customCaStore = []; - private volatile bool _disposed; - private int _isConnecting; + private bool _isConnecting; private DateTimeOffset _lastConnectionTime; - private int _leaderRedirectCount; - - // Both are written by the connect and redirect paths, which do not hold the sending semaphore the request - // paths read them under, so they are accessed through Interlocked rather than as plain fields. Losing an - // update to the skip flag leaves a connection reporting Connected that never authenticated; losing one to - // the redirect counter over- or under-spends the redirect budget. - private int _skipAutoLoginOnce; - private volatile ConnectionState _state = ConnectionState.Disconnected; + private ConnectionState _state = ConnectionState.Disconnected; private TcpConnectionStream _stream = null!; - private bool IsConnecting => Volatile.Read(ref _isConnecting) != 0; - internal TcpMessageStream(IggyClientConfigurator configuration, ILoggerFactory loggerFactory) { _configuration = configuration; @@ -95,14 +84,10 @@ internal TcpMessageStream(IggyClientConfigurator configuration, ILoggerFactory l /// public void Dispose() { - _disposed = true; _stream?.Close(); _stream?.Dispose(); - - SetConnectionStateAsync(ConnectionState.Disconnected); _sendingSemaphore.Dispose(); _connectionSemaphore.Dispose(); - _connectGate.Dispose(); _connectionEvents.Clear(); } @@ -288,7 +273,6 @@ public async Task DeleteTopicAsync(Identifier streamId, Identifier topicId, Canc TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_TOPIC_CODE); await SendAckAsync(payload, token); - _groupState.InvalidatePartitionCount(new TopicKey(streamId, topicId)); } /// @@ -303,38 +287,30 @@ public async Task PurgeTopicAsync(Identifier streamId, Identifier topicId, Cance /// - public Task SendMessagesAsync(Identifier streamId, Identifier topicId, - Partitioning partitioning, IList messages, CancellationToken token = default) + public Task SendMessagesAsync(Identifier streamId, Identifier topicId, Partitioning partitioning, + IList messages, CancellationToken token = default) { - if (NeedsClientSidePartitioning(partitioning)) - { - return SendMessagesResolvedAsync(streamId, topicId, partitioning, messages, token); - } - return SendMessagesCoreAsync(streamId, topicId, partitioning, AsSpan(messages), token); } /// - public Task SendMessagesAsync(Identifier streamId, Identifier topicId, - Partitioning partitioning, Message message, CancellationToken token = default) + public Task SendMessagesAsync(Identifier streamId, Identifier topicId, Partitioning partitioning, + Message message, CancellationToken token = default) { - if (NeedsClientSidePartitioning(partitioning)) - { - return SendMessagesResolvedAsync(streamId, topicId, partitioning, [message], token); - } - ReadOnlySpan span = [message]; return SendMessagesCoreAsync(streamId, topicId, partitioning, span, token); } - /// - /// This feature is not supported by the server. - /// - /// - public Task FlushUnsavedBufferAsync(Identifier streamId, Identifier topicId, uint partitionId, bool fsync, + /// + public async Task FlushUnsavedBufferAsync(Identifier streamId, Identifier topicId, uint partitionId, bool fsync, CancellationToken token = default) { - throw new FeatureUnavailableException(); + var message = TcpContracts.FlushUnsavedBuffer(streamId, topicId, partitionId, fsync); + + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.FLUSH_UNSAVED_BUFFER_CODE); + + await SendAckAsync(payload, token); } /// @@ -348,23 +324,38 @@ public async Task PollMessagesAsync(Identifier streamId, Identif } /// - public Task PollMessagesRentedAsync(Identifier streamId, Identifier topicId, + public async Task PollMessagesRentedAsync(Identifier streamId, Identifier topicId, uint? partitionId, Consumer consumer, PollingStrategy pollingStrategy, uint count, bool autoCommit, CancellationToken token = default) { ThrowIfAutoCommitWithEncryptor(autoCommit); - // The broker routes explicit partitions only, so a group poll picks one of the member's assigned - // partitions client-side. - if (consumer.Type == ConsumerType.ConsumerGroup && partitionId is null) + var messageBufferSize = CalculateMessageBufferSize(streamId, topicId, consumer); + var payloadBufferSize = CalculatePayloadBufferSize(messageBufferSize); + var payload = ArrayPool.Shared.Rent(payloadBufferSize); + IMemoryOwner? responseBuffer = null; + + try { - return PollGroupMessagesRentedAsync(streamId, topicId, consumer, pollingStrategy, count, autoCommit, - token); - } + TcpContracts.GetMessages(payload.AsSpan().Slice(8, messageBufferSize), consumer, streamId, + topicId, pollingStrategy, count, autoCommit, partitionId); + BinaryPrimitives.WriteInt32LittleEndian(payload.AsSpan()[..4], messageBufferSize + 4); + BinaryPrimitives.WriteInt32LittleEndian(payload.AsSpan()[4..8], CommandCodes.POLL_MESSAGES_CODE); - return PollPartitionMessagesRentedAsync(streamId, topicId, partitionId, consumer, pollingStrategy, count, - autoCommit, token); + responseBuffer = await SendWithResponseAsync(payload.AsMemory(0, payloadBufferSize), token); + return BinaryMapper.MapRentedMessages(responseBuffer.Memory, responseBuffer, + _configuration.MessageEncryptor); + } + catch + { + responseBuffer?.Dispose(); + throw; + } + finally + { + ArrayPool.Shared.Return(payload); + } } /// @@ -471,7 +462,6 @@ public async Task DeleteConsumerGroupAsync(Identifier streamId, Identifier topic TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_CONSUMER_GROUP_CODE); await SendAckAsync(payload, token); - _groupState.DeregisterGroup(new GroupKey(streamId, topicId, groupId)); } /// @@ -483,10 +473,6 @@ public async Task JoinConsumerGroupAsync(Identifier streamId, Identifier topicId TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.JOIN_CONSUMER_GROUP_CODE); await SendAckAsync(payload, token); - - // A join rebalances the group, so whatever this client holds for it is a generation behind and every - // poll under it would be fenced until the first re-sync. - _groupState.InvalidateAssignment(new GroupKey(streamId, topicId, groupId)); } /// @@ -498,7 +484,6 @@ public async Task LeaveConsumerGroupAsync(Identifier streamId, Identifier topicI TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LEAVE_CONSUMER_GROUP_CODE); await SendAckAsync(payload, token); - _groupState.DeregisterGroup(new GroupKey(streamId, topicId, groupId)); } /// @@ -510,7 +495,6 @@ public async Task DeletePartitionsAsync(Identifier streamId, Identifier topicId, TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_PARTITIONS_CODE); await SendAckAsync(payload, token); - _groupState.InvalidatePartitionCount(new TopicKey(streamId, topicId)); } /// @@ -522,7 +506,6 @@ public async Task CreatePartitionsAsync(Identifier streamId, Identifier topicId, TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_PARTITIONS_CODE); await SendAckAsync(payload, token); - _groupState.InvalidatePartitionCount(new TopicKey(streamId, topicId)); } /// @@ -595,8 +578,6 @@ public async Task PingAsync(CancellationToken token = default) TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.PING_CODE); await SendAckAsync(payload, token); - - await RefreshGroupAssignmentsAsync(token); } /// @@ -630,9 +611,31 @@ public async Task SendBinaryRequestAsync(uint code, byte[] payload, Canc } /// - public Task ConnectAsync(CancellationToken token = default) + public async Task ConnectAsync(CancellationToken token = default) { - return ConnectAsync(true, token); + if (_state is ConnectionState.Connected + or ConnectionState.Authenticating + or ConnectionState.Authenticated) + { + _logger.LogWarning("Connection is already connected"); + return; + } + + if (_lastConnectionTime != DateTimeOffset.MinValue) + { + await Task.Delay(_configuration.ReconnectionSettings.InitialDelay, token); + } + + SetConnectionStateAsync(ConnectionState.Connecting); + _isConnecting = true; + try + { + await TryEstablishConnectionAsync(token); + } + finally + { + _isConnecting = false; + } } /// @@ -772,8 +775,30 @@ public async Task ChangePasswordAsync(Identifier userId, string currentPassword, throw new NotConnectedException(); } - return await LoginRegisterAsync(CommandCodes.LOGIN_REGISTER_CODE, - LoginRegister.Serialize(userName, password), token); + // TODO: Add binary protocol version + var message = TcpContracts.LoginUser(userName, password, SdkVersion.Value, "csharp-sdk"); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LOGIN_USER_CODE); + + SetConnectionStateAsync(ConnectionState.Authenticating); + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + if (responseBuffer.Memory.Length == 0) + { + return null; + } + + var userId = BinaryPrimitives.ReadInt32LittleEndian(responseBuffer.Memory.Span[..responseBuffer.Memory.Length]); + SetConnectionStateAsync(ConnectionState.Authenticated); + + if (await RedirectAsync(token)) + { + await ConnectAsync(token); + return await LoginUserAsync(userName, password, token); + } + + var authResponse = new AuthResponse(userId, null); + return authResponse; } /// @@ -783,19 +808,7 @@ public async Task LogoutUserAsync(CancellationToken token = default) var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LOGOUT_USER_CODE); - try - { - await SendAckAsync(payload, token); - } - finally - { - await ResetConsensusSessionAsync(); - - if (_state == ConnectionState.Authenticated) - { - SetConnectionStateAsync(ConnectionState.Connected); - } - } + await SendAckAsync(payload, token); } /// @@ -847,86 +860,29 @@ public async Task DeletePersonalAccessTokenAsync(string name, CancellationToken /// public async Task LoginWithPersonalAccessTokenAsync(string token, CancellationToken ct = default) { - return await LoginRegisterAsync(CommandCodes.LOGIN_REGISTER_WITH_PAT_CODE, - LoginRegister.SerializeWithPersonalAccessToken(token), ct); - } - - /// - /// Connects, optionally without the configured auto login. A caller that authenticates itself right - /// after the connect passes false, so the connect does not spend a round trip on credentials the - /// caller is about to replace. - /// - private async Task ConnectAsync(bool autoLogin, CancellationToken token) - { - if (_state is ConnectionState.Connected - or ConnectionState.Authenticating - or ConnectionState.Authenticated) - { - _logger.LogWarning("Connection is already connected"); - return; - } - - await _connectGate.WaitAsync(token); - Interlocked.Exchange(ref _isConnecting, 1); - try - { - if (_state is ConnectionState.Connected - or ConnectionState.Authenticating - or ConnectionState.Authenticated) - { - return; - } + var message = TcpContracts.LoginWithPersonalAccessToken(token); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE); - if (_lastConnectionTime != DateTimeOffset.MinValue) - { - await Task.Delay(_configuration.ReconnectionSettings.InitialDelay, token); - } + SetConnectionStateAsync(ConnectionState.Authenticating); + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, ct); - SetConnectionStateAsync(ConnectionState.Connecting); - await TryEstablishConnectionAsync(autoLogin, token); - } - finally + if (responseBuffer.Memory.Length == 0) { - Interlocked.Exchange(ref _isConnecting, 0); - _connectGate.Release(); + return null; } - } - private async Task PollPartitionMessagesRentedAsync(Identifier streamId, Identifier topicId, - uint? partitionId, Consumer consumer, PollingStrategy pollingStrategy, uint count, bool autoCommit, - CancellationToken token) - { - var messageBufferSize = CalculateMessageBufferSize(streamId, topicId, consumer); - var payloadBufferSize = CalculatePayloadBufferSize(messageBufferSize); - var payload = ArrayPool.Shared.Rent(payloadBufferSize); - IMemoryOwner? responseBuffer = null; + var userId = BinaryPrimitives.ReadInt32LittleEndian(responseBuffer.Memory.Span[..4]); - try - { - TcpContracts.GetMessages(payload.AsSpan().Slice(8, messageBufferSize), consumer, streamId, - topicId, pollingStrategy, count, autoCommit, partitionId); - BinaryPrimitives.WriteInt32LittleEndian(payload.AsSpan()[..4], messageBufferSize + 4); - BinaryPrimitives.WriteInt32LittleEndian(payload.AsSpan()[4..8], CommandCodes.POLL_MESSAGES_CODE); + SetConnectionStateAsync(ConnectionState.Authenticated); - responseBuffer = await SendWithResponseAsync(payload.AsMemory(0, payloadBufferSize), token); - if (responseBuffer.Memory.Length == 0) - { - responseBuffer.Dispose(); - return EmptyPolledMessages; - } - - return BinaryMapper.MapRentedMessages(responseBuffer.Memory, responseBuffer, - _configuration.MessageEncryptor); - } - catch - { - responseBuffer?.Dispose(); - throw; - } - finally + if (await RedirectAsync(ct)) { - ArrayPool.Shared.Return(payload); + await ConnectAsync(ct); + return await LoginWithPersonalAccessTokenAsync(token, ct); } + + return new AuthResponse(userId, null); } // Server-side autoCommit commits the batch offset before the client decrypts, so a decryption failure @@ -941,8 +897,8 @@ private void ThrowIfAutoCommitWithEncryptor(bool autoCommit) } } - private Task SendMessagesCoreAsync(Identifier streamId, Identifier topicId, - Partitioning partitioning, ReadOnlySpan messages, CancellationToken token) + private Task SendMessagesCoreAsync(Identifier streamId, Identifier topicId, Partitioning partitioning, + ReadOnlySpan messages, CancellationToken token) { var encryptor = _configuration.MessageEncryptor; @@ -968,17 +924,15 @@ private Task SendMessagesCoreAsync(Identifier streamId, Id throw; } - return SendConfirmedAndDisposeAsync(payloadBuffer, payloadBufferSize, token); + return SendAckAndDisposeAsync(payloadBuffer, payloadBufferSize, token); } - private async Task SendConfirmedAndDisposeAsync(IMemoryOwner payloadBuffer, - int payloadBufferSize, CancellationToken token) + private async Task SendAckAndDisposeAsync(IMemoryOwner payloadBuffer, int payloadBufferSize, + CancellationToken token) { try { - using IMemoryOwner responseBuffer = - await SendWithResponseAsync(payloadBuffer.Memory[..payloadBufferSize], token); - return BinaryMapper.MapSendMessages(responseBuffer.Memory.Span); + await SendAckAsync(payloadBuffer.Memory[..payloadBufferSize], token); } finally { @@ -1008,101 +962,59 @@ private static int FillSendMessagesPayload(Span buffer, int maxMessageBuff return messageBufferSize; } - private async Task TryEstablishConnectionAsync(bool autoLogin, CancellationToken token) + private async Task TryEstablishConnectionAsync(CancellationToken token) { var retryCount = 0; - var redirects = 0; var delay = _configuration.ReconnectionSettings.InitialDelay; do { - // The sending semaphore owns every write to _stream, so an in-flight request never observes the - // field changing between its write and its reply. - await _sendingSemaphore.WaitAsync(token); - try - { - _stream?.Dispose(); - - ResetConsensusSession(); - } - finally - { - _sendingSemaphore.Release(); - } + // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract + _stream?.Close(); + _stream?.Dispose(); if (string.IsNullOrEmpty(_currentAddress)) { _currentAddress = _configuration.BaseAddress; } - if (!ServerAddress.TryParse(_currentAddress, out var host, out var port)) + var urlPortSplitter = _currentAddress.Split(":"); + if (urlPortSplitter.Length > 2) { throw new InvalidBaseAddressException(); } - Socket? socket = null; try { - socket = new Socket(ServerAddress.AddressFamilyOf(host), SocketType.Stream, ProtocolType.Tcp); + var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); socket.SendBufferSize = _configuration.SendBufferSize; socket.ReceiveBufferSize = _configuration.ReceiveBufferSize; - // The protocol is request/reply, so a write is always the last one before - // the client blocks on the answer and Nagle has nothing to coalesce it with - it only delays the - // trailing segment of a large request until the previous one is acked. - socket.NoDelay = true; - - await socket.ConnectAsync(host, port, token); + await socket.ConnectAsync(urlPortSplitter[0], int.Parse(urlPortSplitter[1]), token); socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.KeepAlive, true); socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.TcpKeepAliveTime, 5); - var connectionStream = _configuration.TlsSettings.Enabled switch - { - true => await CreateSslStreamAndAuthenticate(socket, _configuration.TlsSettings), - false => new TcpConnectionStream(new NetworkStream(socket, true)) - }; - - await _sendingSemaphore.WaitAsync(token); - try - { - _stream = connectionStream; - } - finally - { - _sendingSemaphore.Release(); - } - SetConnectionStateAsync(ConnectionState.Connected); _lastConnectionTime = DateTimeOffset.UtcNow; - socket = null; - - if (await RedirectAsync(token)) + _stream = _configuration.TlsSettings.Enabled switch { - await BackoffOrThrowAsync(); - continue; - } + true => await CreateSslStreamAndAuthenticate(socket, _configuration.TlsSettings), + false => new TcpConnectionStream(new NetworkStream(socket, true)) + }; - if (autoLogin && _configuration.AutoLoginSettings.Enabled && !ConsumeSkipAutoLogin()) + if (_configuration.AutoLoginSettings.Enabled) { _logger.LogInformation("Auto login enabled. Trying to login with credentials: {Username}", _configuration.AutoLoginSettings.Username); await LoginUserAsync(_configuration.AutoLoginSettings.Username, _configuration.AutoLoginSettings.Password, token); - - if (await RedirectAsync(token)) - { - await BackoffOrThrowAsync(); - continue; - } } break; } catch (Exception e) { - socket?.Dispose(); - _logger.LogError(e, "Failed to connect"); if (!_configuration.ReconnectionSettings.Enabled || @@ -1133,38 +1045,37 @@ await LoginUserAsync(_configuration.AutoLoginSettings.Username, await Task.Delay(delay, token); } } while (true); + } - // A redirect restarts the loop without passing through the catch, so it spends no retry and waits for - // nothing. Its own budget rather than the reconnection one: following the roster to the leader is how a - // VSR connect succeeds, and it has to work with reconnection turned off. - async Task BackoffOrThrowAsync() + private async Task GetCurrentLeaderNodeAsync(CancellationToken token) + { + try { - if (++redirects > VsrMaxLeaderRedirects) + var clusterMetadata = await GetClusterMetadataAsync(token); + if (clusterMetadata == null) { - SetConnectionStateAsync(ConnectionState.Disconnected); - throw new MissingLeaderException(); + return null; } - _logger.LogInformation("Following leader redirect {Redirect} to {Address}", redirects, _currentAddress); + // Single-node cluster (clustering disabled) - no redirection needed + if (clusterMetadata.Nodes.Count() == 1) + { + return null; + } - await Task.Delay(delay, token); - } - } + var leaderNode = clusterMetadata.Nodes.FirstOrDefault(x => x.Role == ClusterNodeRole.Leader); + if (leaderNode == null) + { + throw new MissingLeaderException(); + } - /// - /// Whether this connect was triggered by a login or register request that will re-authenticate itself, - /// so the auto-login must sit this one out. Consumes the flag. - /// - private bool ConsumeSkipAutoLogin() - { - if (Interlocked.Exchange(ref _skipAutoLoginOnce, 0) == 0) + return leaderNode; + } + // todo: change after error refactoring, error code 5 is for feature not supported + catch (IggyInvalidStatusCodeException e) when (e.StatusCode == 5) { - return false; + return null; } - - _logger.LogInformation("Skipping auto login for a replayed register request"); - - return true; } private async Task CreateSslStreamAndAuthenticate(Socket socket, TlsSettings tlsSettings) @@ -1193,7 +1104,7 @@ private async Task> SendWithResponseAsync(ReadOnlyMemory> HandleReconnectionAsync(ReadOnlyMemory> SendRawAsync(ReadOnlyMemory payload, CancellationToken token) + private async Task> SendRawAsync(ReadOnlyMemory payload, CancellationToken token) { - ObjectDisposedException.ThrowIf(_disposed, this); - if (_state is ConnectionState.Disconnected or ConnectionState.Connecting) { throw new NotConnectedException(); } - return SendRawVsrAsync(payload, token); + await _sendingSemaphore.WaitAsync(token); + + try + { + await _stream.SendAsync(payload, token); + await _stream.FlushAsync(token); + + // Read the 8-byte header (4 bytes status + 4 bytes length) + var totalRead = 0; + while (totalRead < BufferSizes.EXPECTED_RESPONSE_SIZE) + { + var readBytes + = await _stream.ReadAsync( + _responseHeaderBuffer.AsMemory(totalRead, BufferSizes.EXPECTED_RESPONSE_SIZE - totalRead), + token); + if (readBytes == 0) + { + throw new IggyZeroBytesException(); + } + + totalRead += readBytes; + } + + var response = TcpMessageStreamHelpers.GetResponseLengthAndStatus(_responseHeaderBuffer); + + if (response.Status != 0) + { + if (response.Length == 0) + { + throw new IggyInvalidStatusCodeException(response.Status, + $"Invalid response status code: {response.Status}"); + } + + + using var errorBuffer = ArrayPoolHelper.Rent(response.Length); + totalRead = 0; + while (totalRead < response.Length) + { + var readBytes + = await _stream.ReadAsync(errorBuffer.Memory.Slice(totalRead, response.Length - totalRead), + token); + if (readBytes == 0) + { + throw new IggyZeroBytesException(); + } + + totalRead += readBytes; + } + + throw new InvalidResponseException(Encoding.UTF8.GetString(errorBuffer.Memory.Span)); + } + + if (response.Length == 0) + { + return EmptyMemoryOwner.Instance; + } + + var responseBuffer = ArrayPoolHelper.Rent(response.Length); + try + { + totalRead = 0; + while (totalRead < response.Length) + { + var readBytes + = await _stream.ReadAsync(responseBuffer.Memory.Slice(totalRead, response.Length - totalRead), + token); + + if (readBytes == 0) + { + throw new IggyZeroBytesException(); + } + + totalRead += readBytes; + } + } + catch + { + responseBuffer.Dispose(); + throw; + } + + return responseBuffer; + } + finally + { + _sendingSemaphore.Release(); + } } private static bool IsConnectionException(Exception ex) @@ -1362,6 +1357,30 @@ private bool RemoteCertificateValidationCallback(object sender, X509Certificate? return false; } + private async Task RedirectAsync(CancellationToken token) + { + var currentLeaderNode = await GetCurrentLeaderNodeAsync(token); + if (currentLeaderNode == null) + { + return false; + } + + var leaderAddress = $"{currentLeaderNode.Ip}:{currentLeaderNode.Endpoints.Tcp}"; + if (leaderAddress == _currentAddress) + { + return false; + } + + _currentAddress = leaderAddress; + + _logger.LogInformation("Leader address changed. Trying to reconnect to {Address}", + leaderAddress); + + _stream.Close(); + SetConnectionStateAsync(ConnectionState.Disconnected); + return true; + } + internal sealed class EmptyMemoryOwner : IMemoryOwner { public static readonly EmptyMemoryOwner Instance = new(); diff --git a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TransientHttpRetryHandler.cs b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TransientHttpRetryHandler.cs deleted file mode 100644 index 014ee53f3b..0000000000 --- a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TransientHttpRetryHandler.cs +++ /dev/null @@ -1,109 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Net; - -namespace Apache.Iggy.IggyClient.Implementations; - -/// -/// Replays requests the server answered with a retryable status: 503 (no caught-up primary, full -/// pipeline, view-change cancel, shard write budget) and 429 (session write cap). The server signals -/// both as transient with a Retry-After; the binary transports absorb the equivalent frames in their -/// in-client replay loop, so HTTP replays here for transport parity. A write 504 stays terminal: the -/// request may still commit, so its outcome is unknown and must surface to the caller. -/// -internal sealed class TransientHttpRetryHandler : DelegatingHandler -{ - private static readonly TimeSpan RetryDeadline = TimeSpan.FromSeconds(30); - private static readonly TimeSpan RetryInterval = TimeSpan.FromMilliseconds(50); - - public TransientHttpRetryHandler(HttpMessageHandler innerHandler) : base(innerHandler) - { - } - - protected override async Task SendAsync(HttpRequestMessage request, - CancellationToken cancellationToken) - { - var deadline = Environment.TickCount64 + (long)RetryDeadline.TotalMilliseconds; - byte[]? bufferedContent = null; - if (request.Content is not null) - { - bufferedContent = await request.Content.ReadAsByteArrayAsync(cancellationToken); - } - - while (true) - { - HttpResponseMessage response = await base.SendAsync(CloneRequest(request, bufferedContent), - cancellationToken); - if (!IsRetryable(request.Method, response.StatusCode) || Environment.TickCount64 >= deadline) - { - return response; - } - - var delay = response.Headers.RetryAfter?.Delta ?? RetryInterval; - response.Dispose(); - await Task.Delay(delay, cancellationToken); - } - } - - private static bool IsRetryable(HttpMethod method, HttpStatusCode statusCode) - { - if (statusCode is HttpStatusCode.ServiceUnavailable or HttpStatusCode.TooManyRequests) - { - return true; - } - - // A read that timed out in the partition plane (e.g. a poll racing the reconciler that has not yet - // materialised a fresh topic's partition group) has no side effects, so it replays. A write 504 is an - // unknown outcome and stays terminal. - return statusCode == HttpStatusCode.GatewayTimeout && method == HttpMethod.Get; - } - - /// - /// An is single-use, so every attempt sends a copy built from the - /// buffered body. - /// - private static HttpRequestMessage CloneRequest(HttpRequestMessage request, byte[]? bufferedContent) - { - var clone = new HttpRequestMessage(request.Method, request.RequestUri) - { - Version = request.Version, - VersionPolicy = request.VersionPolicy - }; - - foreach (var header in request.Headers) - { - clone.Headers.TryAddWithoutValidation(header.Key, header.Value); - } - - if (bufferedContent is not null) - { - clone.Content = new ByteArrayContent(bufferedContent); - foreach (var header in request.Content!.Headers) - { - clone.Content.Headers.TryAddWithoutValidation(header.Key, header.Value); - } - } - - foreach (var option in request.Options) - { - clone.Options.Set(new HttpRequestOptionsKey(option.Key), option.Value); - } - - return clone; - } -} diff --git a/foreign/csharp/Iggy_SDK/Iggy_SDK.csproj b/foreign/csharp/Iggy_SDK/Iggy_SDK.csproj index 79a6390366..776883aacd 100644 --- a/foreign/csharp/Iggy_SDK/Iggy_SDK.csproj +++ b/foreign/csharp/Iggy_SDK/Iggy_SDK.csproj @@ -27,7 +27,7 @@ net8.0;net10.0 Apache.Iggy Apache.Iggy - 0.9.0-edge.1 + 0.8.1-edge.4 true @@ -69,7 +69,6 @@ - diff --git a/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs b/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs index f5be657f20..e65a4ef909 100644 --- a/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs +++ b/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs @@ -604,16 +604,8 @@ internal static Dictionary MapHeaders(ReadOnlySpan ReadOnlySpan value = payload[position..(position + valueLength)]; position += valueLength; - headers[new HeaderKey - { - Kind = keyKind, - Value = keyValue - }] = - new HeaderValue - { - Kind = valueKind, - Value = value.ToArray() - }; + headers[new HeaderKey { Kind = keyKind, Value = keyValue }] = + new HeaderValue { Kind = valueKind, Value = value.ToArray() }; } return headers; @@ -690,16 +682,8 @@ headers[new HeaderKey ReadOnlySpan value = payload[position..(position + valueLength)]; position += valueLength; - headers[new HeaderKey - { - Kind = keyKind, - Value = keyValue - }] = - new HeaderValue - { - Kind = valueKind, - Value = value.ToArray() - }; + headers[new HeaderKey { Kind = keyKind, Value = keyValue }] = + new HeaderValue { Kind = valueKind, Value = value.ToArray() }; } return headers; @@ -756,47 +740,6 @@ internal static IReadOnlyList MapStreams(ReadOnlySpan payl return streams.AsReadOnly(); } - /// - /// Maps a send messages reply: [count:4][stream_id:4][topic_id:4][partition_id:4][base_offset:8]*. - /// - /// - /// Strict: the payload is the whole reply body, so any length that does not match the count - /// is a shape this build cannot read, not a prefix of a larger value. Entries are read at a - /// fixed 20-byte stride, so tolerating a tail would return garbage as a successful decode. - /// - internal static SendMessagesResponse MapSendMessages(ReadOnlySpan payload) - { - const int confirmationSize = 4 + 4 + 4 + 8; - - if (payload.Length < 4) - { - throw new InvalidResponseException("Send messages reply is shorter than the confirmation count prefix."); - } - - var count = BinaryPrimitives.ReadUInt32LittleEndian(payload[..4]); - if (payload.Length - 4 != count * (long)confirmationSize) - { - throw new InvalidResponseException( - $"Send messages reply length {payload.Length} does not match {count} confirmations."); - } - - var confirmations = new SendMessagesConfirmation[count]; - var position = 4; - for (var i = 0; i < confirmations.Length; i++) - { - confirmations[i] = new SendMessagesConfirmation - { - StreamId = BinaryPrimitives.ReadUInt32LittleEndian(payload[position..(position + 4)]), - TopicId = BinaryPrimitives.ReadUInt32LittleEndian(payload[(position + 4)..(position + 8)]), - PartitionId = BinaryPrimitives.ReadUInt32LittleEndian(payload[(position + 8)..(position + 12)]), - BaseOffset = BinaryPrimitives.ReadUInt64LittleEndian(payload[(position + 12)..(position + 20)]) - }; - position += confirmationSize; - } - - return new SendMessagesResponse { Confirmations = confirmations }; - } - internal static StreamResponse MapStream(ReadOnlySpan payload) { var (stream, position) = MapToStream(payload, 0); diff --git a/foreign/csharp/Iggy_SDK/Publishers/BackgroundMessageProcessor.cs b/foreign/csharp/Iggy_SDK/Publishers/BackgroundMessageProcessor.cs index cae62bfb5d..969118e408 100644 --- a/foreign/csharp/Iggy_SDK/Publishers/BackgroundMessageProcessor.cs +++ b/foreign/csharp/Iggy_SDK/Publishers/BackgroundMessageProcessor.cs @@ -17,7 +17,6 @@ using System.Threading.Channels; using Apache.Iggy.Enums; -using Apache.Iggy.Exceptions; using Apache.Iggy.IggyClient; using Apache.Iggy.Messages; using Apache.Iggy.Utils; @@ -56,11 +55,6 @@ internal sealed partial class BackgroundMessageProcessor : IAsyncDisposable private int _inFlight; private PooledBufferWriter _payloadBuffer; - // Set by DisposeAsync so the loop finishes what it has instead of being cancelled mid-send. A send cancelled - // after its first byte is reported as an outcome only the server knows, which would make every ordinary - // shutdown publish a batch that may have committed twice. - private int _stopping; - public BackgroundMessageProcessor(IIggyClient client, IggyPublisherConfig config, ILoggerFactory loggerFactory) { _client = client; @@ -103,7 +97,7 @@ public async ValueTask DisposeAsync() _client.UnsubscribeConnectionEvents(ClientOnOnConnectionStateChanged); - Volatile.Write(ref _stopping, 1); + await _cancellationTokenSource.CancelAsync(); _writer.TryComplete(); var backgroundTaskTimedOut = false; @@ -124,14 +118,9 @@ public async ValueTask DisposeAsync() { LogBackgroundProcessorError(e); } - finally - { - await _cancellationTokenSource.CancelAsync(); - } } else { - await _cancellationTokenSource.CancelAsync(); DrainAndDispose(); } @@ -255,11 +244,9 @@ private async Task RunBackgroundProcessor(CancellationToken ct) { while (!ct.IsCancellationRequested) { - var stopping = Volatile.Read(ref _stopping) != 0; - if (!_canSend) { - if (stopping || !await timer.WaitForNextTickAsync(ct)) + if (!await timer.WaitForNextTickAsync(ct)) { break; } @@ -269,7 +256,7 @@ private async Task RunBackgroundProcessor(CancellationToken ct) if (!AccumulateBatch()) { - if (stopping || !await timer.WaitForNextTickAsync(ct)) + if (!await timer.WaitForNextTickAsync(ct)) { break; } @@ -490,16 +477,6 @@ private async Task SendWithRetry(List wire, CancellationToken ct) // Disposal cancellation, not a send failure; let the loop's cancellation handling take over. throw; } - catch (VsrRequestOutcomeUnknownException ex) - { - // May already have committed - report it as its own type rather than folding it into the - // generic failure path, so a subscriber can tell "not sent" from "possibly sent twice". - LogFailedToSendBatch(ex, wire.Count); - if (_messageBatchErrorAggregator.HasSubscribers) - { - _messageBatchErrorAggregator.Publish(new MessageBatchFailedEventArgs(ex, SnapshotForFailure(wire))); - } - } catch (Exception ex) { LogFailedToSendBatch(ex, wire.Count); @@ -530,20 +507,6 @@ private async Task SendWithRetry(List wire, CancellationToken ct) // remaining attempts instantly and publish a misleading "failed after N attempts" event. throw; } - catch (VsrRequestOutcomeUnknownException ex) - { - // The send may already have committed. The partition plane is sessionless - it keeps no - // client-table entry to deduplicate an append against - so a retry cannot be matched to the - // original under any client id and would simply append the batch twice. Report it instead. - LogFailedToSendBatch(ex, wire.Count); - if (_messageBatchErrorAggregator.HasSubscribers) - { - _messageBatchErrorAggregator.Publish(new MessageBatchFailedEventArgs(ex, SnapshotForFailure(wire), - attempt + 1)); - } - - return; - } catch (Exception ex) { lastException = ex; diff --git a/foreign/csharp/Iggy_SDK/Publishers/IggyPublisher.cs b/foreign/csharp/Iggy_SDK/Publishers/IggyPublisher.cs index 7ec63bebf7..c90bedfa60 100644 --- a/foreign/csharp/Iggy_SDK/Publishers/IggyPublisher.cs +++ b/foreign/csharp/Iggy_SDK/Publishers/IggyPublisher.cs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -using Apache.Iggy.Contracts; using Apache.Iggy.Enums; using Apache.Iggy.Exceptions; using Apache.Iggy.IggyClient; @@ -174,7 +173,7 @@ public async Task InitAsync(CancellationToken ct = default) await Client.ConnectAsync(ct); LogInitializingPublisher(Config.StreamId, Config.TopicId); - if (!string.IsNullOrEmpty(Config.Login) && !Config.CreateIggyClient) + if (Config.CreateIggyClient) { await Client.LoginUserAsync(Config.Login, Config.Password, ct); LogUserLoggedIn(Config.Login); @@ -282,12 +281,8 @@ await Client.CreateTopicAsync(Config.StreamId, Config.TopicName, Config.TopicPar /// /// The messages to send. /// Cancellation token to cancel the send operation. - /// - /// Commit confirmations for a direct send. Empty when background sending is enabled: the - /// processor merges queued batches, so no 1:1 mapping to this call exists. - /// /// Thrown when attempting to send before initialization. - public async Task SendMessagesAsync(IList messages, CancellationToken ct = default) + public async Task SendMessagesAsync(IList messages, CancellationToken ct = default) { if (!IsInitialized) { @@ -297,19 +292,20 @@ public async Task SendMessagesAsync(IList message if (messages.Count == 0) { - return SendMessagesResponse.Empty; + return; } if (Config.EnableBackgroundSending && BackgroundProcessor != null) { LogQueuingMessages(messages.Count); // Snapshot so a caller mutating the list after enqueue cannot change the batch read at flush time. - return await SendReadyAsync(messages.ToArray(), null, ct); + await SendReadyAsync(messages.ToArray(), null, ct); + } + else + { + await SendReadyAsync(messages, null, ct); + LogSuccessfullySentMessages(messages.Count); } - - var response = await SendReadyAsync(messages, null, ct); - LogSuccessfullySentMessages(messages.Count); - return response; } /// @@ -319,11 +315,8 @@ public async Task SendMessagesAsync(IList message /// /// The rented batch to send. /// Cancellation token to cancel the send operation. - /// - /// Commit confirmations for a direct send; empty when background sending is enabled. - /// /// Thrown when attempting to send before initialization. - public async Task SendAsync(RentedMessageBatch batch, CancellationToken ct = default) + public async Task SendAsync(RentedMessageBatch batch, CancellationToken ct = default) { ArgumentNullException.ThrowIfNull(batch); @@ -339,29 +332,29 @@ public async Task SendAsync(RentedMessageBatch batch, Canc if (messages.Count == 0) { batch.Dispose(); - return SendMessagesResponse.Empty; + return; } - return await SendReadyAsync(messages, batch, ct); + await SendReadyAsync(messages, batch, ct); } // Background: queued as one unit, owner disposed after its flush. Direct: sent now, owner disposed after. - private async Task SendReadyAsync(IList messages, IDisposable? owner, - CancellationToken ct) + private async Task SendReadyAsync(IList messages, IDisposable? owner, CancellationToken ct) { if (Config.EnableBackgroundSending && BackgroundProcessor != null) { await BackgroundProcessor.EnqueueAsync(new ReadyUnit(messages, owner), ct); - return SendMessagesResponse.Empty; } - - try - { - return await Client.SendMessagesAsync(Config.StreamId, Config.TopicId, Config.Partitioning, messages, ct); - } - finally + else { - owner?.Dispose(); + try + { + await Client.SendMessagesAsync(Config.StreamId, Config.TopicId, Config.Partitioning, messages, ct); + } + finally + { + owner?.Dispose(); + } } } diff --git a/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherBuilder.cs b/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherBuilder.cs index b69b7e99db..47fa6f09df 100644 --- a/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherBuilder.cs +++ b/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherBuilder.cs @@ -33,7 +33,7 @@ namespace Apache.Iggy.Publishers; /// public class IggyPublisherBuilder { - private protected IMessageEncryptor? _encryptor; + private IMessageEncryptor? _encryptor; internal Func? OnBackgroundError { get; set; } internal Func? OnMessageBatchFailed { get; set; } @@ -305,7 +305,6 @@ public IggyPublisher Build() ReceiveBufferSize = Config.ReceiveBufferSize, SendBufferSize = Config.SendBufferSize, ReconnectionSettings = Config.ReconnectionSettings ?? new ReconnectionSettings(), - AutoLoginSettings = AutoLoginSettings.For(Config.Login, Config.Password), LoggerFactory = Config.LoggerFactory ?? NullLoggerFactory.Instance, MessageEncryptor = _encryptor }); diff --git a/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherBuilderOfT.cs b/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherBuilderOfT.cs index 6198c76433..962b401af6 100644 --- a/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherBuilderOfT.cs +++ b/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherBuilderOfT.cs @@ -90,11 +90,7 @@ public static IggyPublisherBuilder Create(IIggyClient iggyClient, Identifier Protocol = Config.Protocol, BaseAddress = Config.Address, ReceiveBufferSize = Config.ReceiveBufferSize, - SendBufferSize = Config.SendBufferSize, - ReconnectionSettings = Config.ReconnectionSettings ?? new ReconnectionSettings(), - AutoLoginSettings = AutoLoginSettings.For(Config.Login, Config.Password), - LoggerFactory = Config.LoggerFactory ?? NullLoggerFactory.Instance, - MessageEncryptor = _encryptor + SendBufferSize = Config.SendBufferSize }); } @@ -138,7 +134,8 @@ protected override void Validate() } else { - throw new InvalidOperationException($"Config must be of type IggyPublisherConfig<{typeof(T).Name}>."); + throw new InvalidOperationException( + $"Config must be of type IggyPublisherConfig<{typeof(T).Name}>."); } } } diff --git a/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherOfT.cs b/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherOfT.cs index 1c9fdf91fb..ac8c790c6a 100644 --- a/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherOfT.cs +++ b/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherOfT.cs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -using Apache.Iggy.Contracts; using Apache.Iggy.Exceptions; using Apache.Iggy.Extensions; using Apache.Iggy.Headers; @@ -61,10 +60,7 @@ public IggyPublisher(IIggyClient client, IggyPublisherConfig config, ILogger< /// Optional message ID. If null, the message is sent with ID 0 and the server assigns one /// Optional user headers to attach to the message /// Cancellation token - /// - /// Commit confirmations for a direct send; empty when background sending is enabled. - /// - public async Task SendAsync(T data, Guid? messageId = null, + public async Task SendAsync(T data, Guid? messageId = null, Dictionary? userHeaders = null, CancellationToken ct = default) { EnsureInitialized(); @@ -74,7 +70,7 @@ public async Task SendAsync(T data, Guid? messageId = null if (BackgroundProcessor != null) { await BackgroundProcessor.EnqueueAsync(TypedUnit.Single(data, id, userHeaders, _serializer), ct); - return SendMessagesResponse.Empty; + return; } var writer = new PooledBufferWriter(); @@ -82,7 +78,7 @@ public async Task SendAsync(T data, Guid? messageId = null { _serializer.Serialize(data, writer); var message = new Message(id, writer.Written, userHeaders); - return await Client.SendMessagesAsync(Config.StreamId, Config.TopicId, Config.Partitioning, message, ct); + await Client.SendMessagesAsync(Config.StreamId, Config.TopicId, Config.Partitioning, message, ct); } finally { @@ -95,10 +91,7 @@ public async Task SendAsync(T data, Guid? messageId = null /// /// The collection of objects to serialize and send /// Cancellation token - /// - /// Commit confirmations for a direct send; empty when background sending is enabled. - /// - public async Task SendAsync(IEnumerable data, CancellationToken ct = default) + public async Task SendAsync(IEnumerable data, CancellationToken ct = default) { EnsureInitialized(); @@ -110,7 +103,7 @@ public async Task SendAsync(IEnumerable data, Cancellat await BackgroundProcessor.EnqueueAsync(unit, ct); } - return SendMessagesResponse.Empty; + return; } using var builder = new RentedMessageBatchBuilder(); @@ -120,7 +113,7 @@ public async Task SendAsync(IEnumerable data, Cancellat static (state, writer) => state.Serializer.Serialize(state.Data, writer)); } - return await SendDirectBatchAsync(builder, ct); + await SendDirectBatchAsync(builder, ct); } /// @@ -128,10 +121,7 @@ public async Task SendAsync(IEnumerable data, Cancellat /// /// The collection of items to send, each with optional message ID and headers /// Cancellation token - /// - /// Commit confirmations for a direct send; empty when background sending is enabled. - /// - public async Task SendAsync( + public async Task SendAsync( IEnumerable<(T data, Guid? messageId, Dictionary? userHeaders)> items, CancellationToken ct = default) { @@ -145,7 +135,7 @@ public async Task SendAsync( await BackgroundProcessor.EnqueueAsync(unit, ct); } - return SendMessagesResponse.Empty; + return; } using var builder = new RentedMessageBatchBuilder(); @@ -156,11 +146,10 @@ public async Task SendAsync( item.userHeaders); } - return await SendDirectBatchAsync(builder, ct); + await SendDirectBatchAsync(builder, ct); } - private async Task SendDirectBatchAsync(RentedMessageBatchBuilder builder, - CancellationToken ct) + private async Task SendDirectBatchAsync(RentedMessageBatchBuilder builder, CancellationToken ct) { var batch = builder.Build(); try @@ -168,10 +157,10 @@ private async Task SendDirectBatchAsync(RentedMessageBatch IList messages = batch.Messages; if (messages.Count == 0) { - return SendMessagesResponse.Empty; + return; } - return await Client.SendMessagesAsync(Config.StreamId, Config.TopicId, Config.Partitioning, messages, ct); + await Client.SendMessagesAsync(Config.StreamId, Config.TopicId, Config.Partitioning, messages, ct); } finally { diff --git a/foreign/csharp/Iggy_SDK/Utils/BufferSizes.cs b/foreign/csharp/Iggy_SDK/Utils/BufferSizes.cs index cd5a9e871f..90bc6a1955 100644 --- a/foreign/csharp/Iggy_SDK/Utils/BufferSizes.cs +++ b/foreign/csharp/Iggy_SDK/Utils/BufferSizes.cs @@ -20,4 +20,5 @@ namespace Apache.Iggy.Utils; internal static class BufferSizes { internal const int INITIAL_BYTES_LENGTH = 4; + internal const int EXPECTED_RESPONSE_SIZE = 8; } diff --git a/foreign/csharp/Iggy_SDK/Utils/CommandCodes.cs b/foreign/csharp/Iggy_SDK/Utils/CommandCodes.cs index c7a3051e59..8922d37d61 100644 --- a/foreign/csharp/Iggy_SDK/Utils/CommandCodes.cs +++ b/foreign/csharp/Iggy_SDK/Utils/CommandCodes.cs @@ -47,8 +47,6 @@ internal static class CommandCodes internal const int GET_CONSUMER_OFFSET_CODE = 120; internal const int STORE_CONSUMER_OFFSET_CODE = 121; internal const int DELETE_CONSUMER_OFFSET_CODE = 122; - internal const int STORE_CONSUMER_OFFSET_2_CODE = 123; - internal const int DELETE_CONSUMER_OFFSET_2_CODE = 124; internal const int GET_STREAM_CODE = 200; internal const int GET_STREAMS_CODE = 201; internal const int CREATE_STREAM_CODE = 202; @@ -70,5 +68,4 @@ internal static class CommandCodes internal const int DELETE_CONSUMER_GROUP_CODE = 603; internal const int JOIN_CONSUMER_GROUP_CODE = 604; internal const int LEAVE_CONSUMER_GROUP_CODE = 605; - internal const int SYNC_CONSUMER_GROUP_CODE = 606; } diff --git a/foreign/csharp/Iggy_SDK/Utils/ServerAddress.cs b/foreign/csharp/Iggy_SDK/Utils/ServerAddress.cs deleted file mode 100644 index 2f85784610..0000000000 --- a/foreign/csharp/Iggy_SDK/Utils/ServerAddress.cs +++ /dev/null @@ -1,139 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Net; -using System.Net.Sockets; - -namespace Apache.Iggy.Utils; - -/// -/// Endpoint comparison for leader redirection. The configured address is whatever the caller wrote -/// (localhost:8090), while the cluster roster reports IPs, so the two are compared as parsed -/// endpoints rather than as strings. -/// -internal static class ServerAddress -{ - internal static bool IsSame(string first, string second) - { - return Normalize(first) == Normalize(second); - } - - /// - /// Renders host:port, bracketing a host that carries colons of its own (a bare IPv6 address). - /// Without the brackets the rendering can never round-trip through , so - /// it would compare unequal to every normalized address. - /// - internal static string HostPort(string host, ushort port) - { - return host.Contains(':') && !host.StartsWith('[') ? $"[{host}]:{port}" : $"{host}:{port}"; - } - - internal static bool TryParse(string address, out string host, out int port) - { - var parsed = TrySplitHostPort(address, out host, out var hostPort); - port = hostPort; - - return parsed; - } - - /// - /// The socket family a host has to be dialled on. A name resolves to whatever the resolver returns, and - /// the connect call handles that itself, so only a literal decides the family here. - /// - internal static AddressFamily AddressFamilyOf(string host) - { - return IPAddress.TryParse(host, out var ip) ? ip.AddressFamily : AddressFamily.InterNetwork; - } - - /// - /// Canonical host:port rendering: the host is lowercased, the loopback and unspecified aliases - /// collapse onto the loopback address, and an IP is re-rendered from its parsed form. The host is - /// replaced as a whole, never as a substring, so a node named my-localhost-1 keeps its name. - /// An address that is not host:port is only lowercased, which leaves it comparable but distinct. - /// - internal static string Normalize(string address) - { - if (!TrySplitHostPort(address, out var host, out var port)) - { - return address.ToLowerInvariant(); - } - - if (!IPAddress.TryParse(NormalizeHostAlias(host), out var ip)) - { - return $"{host.ToLowerInvariant()}:{port}"; - } - - return ip.AddressFamily == AddressFamily.InterNetworkV6 ? $"[{ip}]:{port}" : $"{ip}:{port}"; - } - - /// - /// A server bound to the unspecified address is reachable on the loopback one, and the roster may - /// report either, so both render the same way. - /// - private static string NormalizeHostAlias(string host) - { - if (host.Equals("localhost", StringComparison.OrdinalIgnoreCase)) - { - return "127.0.0.1"; - } - - if (!IPAddress.TryParse(host, out var ip)) - { - return host; - } - - if (ip.Equals(IPAddress.Any)) - { - return IPAddress.Loopback.ToString(); - } - - return ip.Equals(IPAddress.IPv6Any) ? IPAddress.IPv6Loopback.ToString() : host; - } - - private static bool TrySplitHostPort(string address, out string host, out ushort port) - { - host = string.Empty; - port = 0; - string portText; - - // A bracketed host is the only form that may carry colons of its own. - if (address.StartsWith('[')) - { - var closing = address.IndexOf(']'); - if (closing < 0 || closing + 1 >= address.Length || address[closing + 1] != ':') - { - return false; - } - - host = address[1..closing]; - portText = address[(closing + 2)..]; - } - else - { - var separator = address.IndexOf(':'); - if (separator <= 0 || address.IndexOf(':', separator + 1) >= 0) - { - return false; - } - - host = address[..separator]; - portText = address[(separator + 1)..]; - } - - return host.Length > 0 && ushort.TryParse(portText, out port); - } -} diff --git a/foreign/csharp/Iggy_SDK/Vsr/ConsensusSession.cs b/foreign/csharp/Iggy_SDK/Vsr/ConsensusSession.cs deleted file mode 100644 index 8cdbdcc16c..0000000000 --- a/foreign/csharp/Iggy_SDK/Vsr/ConsensusSession.cs +++ /dev/null @@ -1,223 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Buffers.Binary; -using System.Security.Cryptography; -using Apache.Iggy.Exceptions; - -namespace Apache.Iggy.Vsr; - -/// -/// Consensus-level session state: the ephemeral client id, the session number the server hands back -/// when a register commits, and the monotonic request counter. -/// -/// -/// Every mutation and every read of the identity runs under one lock. The transport serialises the -/// re-arms against the requests with its sending lock, so a request always encodes from the identity -/// that is live for the whole time it is on the wire. -/// -internal sealed class ConsensusSession -{ -#if NET10_0_OR_GREATER - private readonly Lock _gate = new(); -#else - private readonly object _gate = new(); -#endif - private UInt128 _clientId; - private bool _registerPending; - private ulong _requestCounter; - private ulong? _session; - - /// Ephemeral client identifier, never persisted, non-zero. - internal UInt128 ClientId - { - get - { - lock (_gate) - { - return _clientId; - } - } - } - - /// Session fence epoch assigned by the server, null until a register commits. - internal ulong? Session - { - get - { - lock (_gate) - { - return _session; - } - } - } - - /// The id the next request id resolution will return. - internal ulong RequestCounter - { - get - { - lock (_gate) - { - return _requestCounter; - } - } - } - - internal bool IsBound - { - get - { - lock (_gate) - { - return _session.HasValue; - } - } - } - - internal ConsensusSession() : this(GenerateClientId()) - { - } - - internal ConsensusSession(UInt128 clientId) - { - _clientId = clientId; - _requestCounter = 1; - } - - /// - /// Resolves the identity one request header is encoded from, in a single atomic step so no reset can - /// interleave between the client id, the request id and the session. - /// - internal SessionFrame Resolve(VsrOperation operation) - { - lock (_gate) - { - return operation switch - { - VsrOperation.Register => RegisterFrameLocked(), - VsrOperation.NonReplicated => new SessionFrame(_clientId, _requestCounter, _session ?? 0), - _ => ReplicatedFrameLocked(operation) - }; - } - } - - /// Bind the session from a committed register reply. - /// The session number the register reply carried. - /// - /// No register is awaiting a binding, so the identity was re-armed while this one was in flight. Binding - /// regardless would pair the session with a client id the server never registered, and it would fence - /// every later request. - /// - /// - /// The register reply carried no session. The value comes off the wire, so a malformed reply has to - /// surface as a protocol error rather than as argument validation. - /// - internal void Bind(ulong session) - { - if (session == 0) - { - throw VsrError.Exception(VsrError.INVALID_FORMAT, "Register reply carried no session."); - } - - lock (_gate) - { - if (!_registerPending) - { - throw new NotConnectedException(); - } - - _registerPending = false; - _session = session; - } - } - - /// Forget the binding and the client id, e.g. after an eviction or a torn connection. - internal void Reset() - { - lock (_gate) - { - ReArmLocked(); - } - } - - /// - /// Begin a registration, re-arming the session if it was already used. A re-login mints a fresh - /// client id and clears the binding, so the register encodes cleanly and the server never has to - /// disambiguate a repeat register for the same client. Always returns 0. - /// - private SessionFrame RegisterFrameLocked() - { - // A second register while one is still unbound would re-arm the identity out from under the first, and - // the winner's Bind would then attach its session to the re-armed client id - the server answers every - // later request with NoSession and evicts. Refuse instead: the caller retries against a clean session. - if (_registerPending) - { - throw VsrError.Exception(VsrError.UNAUTHENTICATED, "A consensus register is already in flight."); - } - - if (_session.HasValue) - { - ReArmLocked(); - } - - _registerPending = true; - - return new SessionFrame(_clientId, 0, 0); - } - - private SessionFrame ReplicatedFrameLocked(VsrOperation operation) - { - var sessionId = _session ?? throw VsrError.Exception(VsrError.UNAUTHENTICATED, - "A replicated request requires a bound consensus session."); - - // Partition ops replicate in their own per-partition group with no client-table dedup, so they too - // must leave the metadata counter untouched. Only metadata operations and logout consume an id: the - // server tracks request ids for those alone, and it accepts any id above the client's watermark. - if (operation.IsPartition()) - { - return new SessionFrame(_clientId, _requestCounter, sessionId); - } - - var requestId = _requestCounter; - _requestCounter = checked(_requestCounter + 1); - - return new SessionFrame(_clientId, requestId, sessionId); - } - - private void ReArmLocked() - { - _clientId = GenerateClientId(); - _session = null; - _requestCounter = 1; - _registerPending = false; - } - - private static UInt128 GenerateClientId() - { - Span bytes = stackalloc byte[16]; - RandomNumberGenerator.Fill(bytes); - var lower = BinaryPrimitives.ReadUInt64LittleEndian(bytes[..8]); - var upper = BinaryPrimitives.ReadUInt64LittleEndian(bytes[8..]); - var clientId = new UInt128(upper, lower); - - return clientId == UInt128.Zero ? UInt128.One : clientId; - } -} - -/// The session identity one request header is encoded from, resolved as a single atomic snapshot. -internal readonly record struct SessionFrame(UInt128 ClientId, ulong RequestId, ulong SessionId); diff --git a/foreign/csharp/Iggy_SDK/Vsr/ConsumerGroupClientState.cs b/foreign/csharp/Iggy_SDK/Vsr/ConsumerGroupClientState.cs deleted file mode 100644 index e594443ae9..0000000000 --- a/foreign/csharp/Iggy_SDK/Vsr/ConsumerGroupClientState.cs +++ /dev/null @@ -1,301 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -namespace Apache.Iggy.Vsr; - -/// -/// Per-connection cache of consumer-group assignments and topic partition counts, mirroring -/// core/common/src/consumer_group_client_state.rs. Under VSR the broker never picks a partition, so -/// the client resolves group polls and balanced / message-key produce locally. The cursors have to survive -/// across calls, which is why this lives on the long-lived transport rather than on a request. -/// -internal sealed class ConsumerGroupClientState -{ - private readonly Dictionary _assignments = []; - private readonly Dictionary _balancedCursors = []; -#if NET10_0_OR_GREATER - private readonly Lock _gate = new(); -#else - private readonly object _gate = new(); -#endif - private readonly Dictionary _joinedGroups = []; - private readonly Dictionary _partitionCounts = []; - - /// Nothing tells this client when another one resizes a topic, so the count expires on its own. - private const long PartitionCountTtlMs = 30_000; - - /// True when a non-empty assignment is cached for the group. - internal bool HasAssignment(GroupKey key) - { - lock (_gate) - { - return _assignments.TryGetValue(key, out var assignment) && assignment.Partitions.Count > 0; - } - } - - /// - /// Replaces a group's cached assignment. A generation change is a rebalance, so the round-robin cursor - /// restarts rather than carrying an index that meant something else. - /// - internal void SetAssignment(GroupKey key, ulong generation, IReadOnlyList partitions) - { - lock (_gate) - { - if (!_assignments.TryGetValue(key, out var assignment)) - { - assignment = new GroupAssignment(); - _assignments[key] = assignment; - } - - if (assignment.Generation != generation) - { - assignment.Cursor = 0; - } - - assignment.Generation = generation; - assignment.Partitions = partitions; - } - } - - internal void InvalidateAssignment(GroupKey key) - { - lock (_gate) - { - _assignments.Remove(key); - } - } - - /// - /// The next assigned partition for a group poll, advancing the cursor. null when nothing is cached - /// or the member holds no partitions. - /// - internal uint? NextGroupPartition(GroupKey key) - { - lock (_gate) - { - if (!_assignments.TryGetValue(key, out var assignment) || assignment.Partitions.Count == 0) - { - return null; - } - - var index = assignment.Cursor % assignment.Partitions.Count; - assignment.Cursor = assignment.Cursor == int.MaxValue ? 0 : assignment.Cursor + 1; - - return assignment.Partitions[index]; - } - } - - /// The next balanced produce partition for a topic, advancing the cursor. - internal uint NextBalancedPartition(TopicKey key, uint partitionCount) - { - if (partitionCount == 0) - { - return 0; - } - - lock (_gate) - { - _balancedCursors.TryGetValue(key, out var cursor); - var partition = (uint)(cursor % partitionCount); - _balancedCursors[key] = cursor == int.MaxValue ? 0 : cursor + 1; - - return partition; - } - } - - internal uint? PartitionCount(TopicKey key) - { - lock (_gate) - { - if (!_partitionCounts.TryGetValue(key, out var cached)) - { - return null; - } - - if (Environment.TickCount64 >= cached.ExpiresAt) - { - _partitionCounts.Remove(key); - - return null; - } - - return cached.Count; - } - } - - internal void SetPartitionCount(TopicKey key, uint partitionCount) - { - lock (_gate) - { - _partitionCounts[key] = new CachedPartitionCount(partitionCount, - Environment.TickCount64 + PartitionCountTtlMs); - } - } - - /// - /// Forgets a topic's cached partition count immediately, for the changes this client makes itself. - /// Changes made by anyone else are covered by . - /// - internal void InvalidatePartitionCount(TopicKey key) - { - lock (_gate) - { - _partitionCounts.Remove(key); - } - } - - /// Records a joined group's identifiers so a later refresh can rebuild its sync request. - internal void RegisterGroup(GroupKey key, Identifier streamId, Identifier topicId, Identifier groupId) - { - lock (_gate) - { - _joinedGroups[key] = new GroupIdentifiers(streamId, topicId, groupId); - } - } - - internal void DeregisterGroup(GroupKey key) - { - lock (_gate) - { - _joinedGroups.Remove(key); - _assignments.Remove(key); - } - } - - /// - /// True when the last assignment sync saw this client as a member. A member mid-rebalance, or one holding - /// zero partitions, is still registered, so this asks a different question than - /// . - /// - internal bool IsRegistered(GroupKey key) - { - lock (_gate) - { - return _joinedGroups.ContainsKey(key); - } - } - - internal IReadOnlyList RegisteredGroups() - { - lock (_gate) - { - return _joinedGroups.Count == 0 ? [] : [.. _joinedGroups.Values]; - } - } - - /// - /// Drops what a consensus session owns. The assignments are fenced by a generation the coordinator tracks - /// per session, so carrying them across a reset would fence every poll, and membership has to be re-synced - /// before it can be trusted again. The balanced cursors and the cached partition counts stay: they belong - /// to a topic, not to a session, and clearing them restarts the produce round-robin at partition 0 and - /// costs a metadata round trip per topic on every reconnect. - /// - internal void ClearSessionScoped() - { - lock (_gate) - { - _assignments.Clear(); - _joinedGroups.Clear(); - } - } - - internal readonly record struct GroupIdentifiers(Identifier StreamId, Identifier TopicId, Identifier GroupId); - - private readonly record struct CachedPartitionCount(uint Count, long ExpiresAt); - - private sealed class GroupAssignment - { - internal IReadOnlyList Partitions { get; set; } = []; - internal ulong Generation { get; set; } - internal int Cursor { get; set; } - } -} - -/// -/// Cache key for a topic. The identifier kind is part of the key because a stream named "1" and the stream -/// with id 1 are different streams that share a rendering. Identifiers are compared by their wire bytes: -/// compares its value array by reference, and every call site builds a fresh one. -/// -internal readonly struct TopicKey(Identifier streamId, Identifier topicId) : IEquatable -{ - private Identifier StreamId { get; } = streamId; - private Identifier TopicId { get; } = topicId; - - public bool Equals(TopicKey other) - { - return IdentifierKey.Equal(StreamId, other.StreamId) && IdentifierKey.Equal(TopicId, other.TopicId); - } - - public override bool Equals(object? obj) - { - return obj is TopicKey other && Equals(other); - } - - public override int GetHashCode() - { - return HashCode.Combine(IdentifierKey.Hash(StreamId), IdentifierKey.Hash(TopicId)); - } - - public override string ToString() - { - return $"{StreamId}|{TopicId}"; - } -} - -/// Cache key for a consumer group on a topic. -internal readonly struct GroupKey(Identifier streamId, Identifier topicId, Identifier groupId) : IEquatable -{ - private TopicKey Topic { get; } = new(streamId, topicId); - private Identifier GroupId { get; } = groupId; - - public bool Equals(GroupKey other) - { - return Topic.Equals(other.Topic) && IdentifierKey.Equal(GroupId, other.GroupId); - } - - public override bool Equals(object? obj) - { - return obj is GroupKey other && Equals(other); - } - - public override int GetHashCode() - { - return HashCode.Combine(Topic.GetHashCode(), IdentifierKey.Hash(GroupId)); - } - - public override string ToString() - { - return $"{Topic}|{GroupId}"; - } -} - -internal static class IdentifierKey -{ - internal static bool Equal(Identifier first, Identifier second) - { - return first.Kind == second.Kind && first.Value.AsSpan().SequenceEqual(second.Value); - } - - internal static int Hash(Identifier identifier) - { - var hash = new HashCode(); - hash.Add((byte)identifier.Kind); - hash.AddBytes(identifier.Value); - - return hash.ToHashCode(); - } -} diff --git a/foreign/csharp/Iggy_SDK/Vsr/CredentialBounds.cs b/foreign/csharp/Iggy_SDK/Vsr/CredentialBounds.cs deleted file mode 100644 index 11fde11871..0000000000 --- a/foreign/csharp/Iggy_SDK/Vsr/CredentialBounds.cs +++ /dev/null @@ -1,66 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Text; - -namespace Apache.Iggy.Vsr; - -/// -/// The credential bounds every server enforces, checked before encoding so an oversized credential is -/// reported as the typed status code instead of desyncing the u8 length prefix on the wire. Mirrors -/// core/common/src/http/users/defaults.rs. -/// -internal static class CredentialBounds -{ - internal const int MIN_USERNAME_LENGTH = 3; - internal const int MAX_USERNAME_LENGTH = 50; - internal const int MIN_PASSWORD_LENGTH = 3; - internal const int MAX_PASSWORD_LENGTH = 100; - - internal const int MIN_TOKEN_LENGTH = 1; - internal const int MAX_TOKEN_LENGTH = 255; - - internal static void ValidateUsername(string username) - { - var length = Encoding.UTF8.GetByteCount(username); - if (length is < MIN_USERNAME_LENGTH or > MAX_USERNAME_LENGTH) - { - throw VsrError.Exception(VsrError.INVALID_USERNAME, - $"Username must be {MIN_USERNAME_LENGTH}-{MAX_USERNAME_LENGTH} bytes, got {length}."); - } - } - - internal static void ValidatePassword(string password) - { - var length = Encoding.UTF8.GetByteCount(password); - if (length is < MIN_PASSWORD_LENGTH or > MAX_PASSWORD_LENGTH) - { - throw VsrError.Exception(VsrError.INVALID_PASSWORD, - $"Password must be {MIN_PASSWORD_LENGTH}-{MAX_PASSWORD_LENGTH} bytes, got {length}."); - } - } - - internal static void ValidateToken(string token) - { - var length = Encoding.UTF8.GetByteCount(token); - if (length is < MIN_TOKEN_LENGTH or > MAX_TOKEN_LENGTH) - { - throw VsrError.Exception(VsrError.INVALID_PERSONAL_ACCESS_TOKEN, - $"Personal access token must be {MIN_TOKEN_LENGTH}-{MAX_TOKEN_LENGTH} bytes, got {length}."); - } - } -} diff --git a/foreign/csharp/Iggy_SDK/Vsr/LoginRegister.cs b/foreign/csharp/Iggy_SDK/Vsr/LoginRegister.cs deleted file mode 100644 index 6d2958e139..0000000000 --- a/foreign/csharp/Iggy_SDK/Vsr/LoginRegister.cs +++ /dev/null @@ -1,168 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Buffers.Binary; -using System.Text; -using Apache.Iggy.Utils; - -namespace Apache.Iggy.Vsr; - -/// -/// The register handshake bodies. VSR replaces the legacy login codes with -/// LOGIN_REGISTER / LOGIN_REGISTER_WITH_PAT, whose bodies lead with the client version -/// info the server gates on before it touches credentials. -/// -internal static class LoginRegister -{ - internal const string SDK_NAME = "csharp-sdk"; - - /// Semver of the iggy_binary_protocol crate this SDK is built against. - internal const int PROTOCOL_VERSION_MAJOR = 0; - - internal const int PROTOCOL_VERSION_MINOR = 11; - internal const int PROTOCOL_VERSION_PATCH = 0; - - /// Packed protocol version: major << 20 | minor << 10 | patch, 10 bits each. - internal const uint PROTOCOL_VERSION = - ((uint)PROTOCOL_VERSION_MAJOR << 20) | ((uint)PROTOCOL_VERSION_MINOR << 10) | PROTOCOL_VERSION_PATCH; - - private const int MaxWireNameLength = 255; - - private static int VersionInfoLength => 4 + NameLength(SDK_NAME) + NameLength(SdkVersion.Value); - - internal static byte[] Serialize(string username, string password, string? clientContext = null) - { - CredentialBounds.ValidateUsername(username); - CredentialBounds.ValidatePassword(password); - - var writer = new BodyWriter(VersionInfoLength + NameLength(username) + NameLength(password) + 4 + - ContextLength(clientContext)); - writer.WriteVersionInfo(); - writer.WriteName(username, nameof(username)); - writer.WriteName(password, nameof(password)); - writer.WriteContext(clientContext); - - return writer.Buffer; - } - - internal static byte[] SerializeWithPersonalAccessToken(string token, string? clientContext = null) - { - CredentialBounds.ValidateToken(token); - - var writer = new BodyWriter(VersionInfoLength + NameLength(token) + 4 + ContextLength(clientContext)); - writer.WriteVersionInfo(); - writer.WriteName(token, nameof(token)); - writer.WriteContext(clientContext); - - return writer.Buffer; - } - - /// - /// [user_id u32][session u64][server_protocol_version u32][server_version len u8 + bytes]. - /// - internal static LoginRegisterResponse Deserialize(ReadOnlySpan body) - { - if (body.IsEmpty) - { - // The server fast-fails a terminal register failure (invalid credentials, invalid token, - // inactive user) with an empty reply instead of a typed error frame, and the reason is not - // recoverable from the wire. Same INVALID_FORMAT surface as the Rust SDK. - throw VsrError.Exception(VsrError.INVALID_FORMAT, - "Server rejected the login. The register reply is empty, which the server sends for invalid " + - "credentials, an invalid personal access token, or an inactive user."); - } - - if (body.Length < 17) - { - throw VsrError.Exception(VsrError.INVALID_FORMAT, "Register reply is truncated."); - } - - var serverVersionLength = body[16]; - if (serverVersionLength == 0 || body.Length < 17 + serverVersionLength) - { - throw VsrError.Exception(VsrError.INVALID_FORMAT, "Register reply carries a malformed server version."); - } - - return new LoginRegisterResponse(BinaryPrimitives.ReadUInt32LittleEndian(body[..4]), - BinaryPrimitives.ReadUInt64LittleEndian(body[4..12]), - BinaryPrimitives.ReadUInt32LittleEndian(body[12..16]), - Encoding.UTF8.GetString(body.Slice(17, serverVersionLength))); - } - - private static int NameLength(string value) - { - return 1 + Encoding.UTF8.GetByteCount(value); - } - - private static int ContextLength(string? clientContext) - { - return clientContext is null ? 0 : Encoding.UTF8.GetByteCount(clientContext); - } - - private struct BodyWriter - { - private int _position; - - internal BodyWriter(int length) - { - Buffer = new byte[length]; - _position = 0; - } - - internal byte[] Buffer { get; } - - internal void WriteVersionInfo() - { - BinaryPrimitives.WriteUInt32LittleEndian(Buffer.AsSpan(_position, 4), PROTOCOL_VERSION); - _position += 4; - WriteName(SDK_NAME, nameof(SDK_NAME)); - WriteName(SdkVersion.Value, nameof(SdkVersion)); - } - - internal void WriteName(string value, string name) - { - var length = Encoding.UTF8.GetByteCount(value); - if (length is 0 or > MaxWireNameLength) - { - throw new ArgumentException($"{name} must be 1-{MaxWireNameLength} UTF-8 bytes, got {length}.", name); - } - - Buffer[_position] = (byte)length; - _position += 1; - Encoding.UTF8.GetBytes(value, Buffer.AsSpan(_position, length)); - _position += length; - } - - internal void WriteContext(string? clientContext) - { - var length = ContextLength(clientContext); - BinaryPrimitives.WriteUInt32LittleEndian(Buffer.AsSpan(_position, 4), (uint)length); - _position += 4; - if (clientContext is not null && length > 0) - { - Encoding.UTF8.GetBytes(clientContext, Buffer.AsSpan(_position, length)); - _position += length; - } - } - } -} - -internal readonly record struct LoginRegisterResponse( - uint UserId, - ulong Session, - uint ServerProtocolVersion, - string ServerVersion); diff --git a/foreign/csharp/Iggy_SDK/Vsr/SyncConsumerGroup.cs b/foreign/csharp/Iggy_SDK/Vsr/SyncConsumerGroup.cs deleted file mode 100644 index 5bc8d55fb8..0000000000 --- a/foreign/csharp/Iggy_SDK/Vsr/SyncConsumerGroup.cs +++ /dev/null @@ -1,61 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Buffers.Binary; - -namespace Apache.Iggy.Vsr; - -/// -/// Reply body of a SyncConsumerGroup request: the requesting member's current partition assignment -/// and the generation it belongs to. Mirrors -/// core/binary_protocol/src/responses/consumer_groups/sync_consumer_group.rs. -/// -/// -/// Wire format: [generation u64][partitions_count u32][partition_id u32]*. The request body is the -/// plain [stream][topic][group] identifier triple every other group request already builds. -/// -internal readonly record struct SyncConsumerGroupAssignment(ulong Generation, IReadOnlyList Partitions) -{ - internal static SyncConsumerGroupAssignment Decode(ReadOnlySpan body) - { - if (body.Length < 12) - { - throw Malformed(); - } - - var generation = BinaryPrimitives.ReadUInt64LittleEndian(body[..8]); - var count = BinaryPrimitives.ReadUInt32LittleEndian(body.Slice(8, 4)); - if (body.Length < 12 + (long)count * 4) - { - throw Malformed(); - } - - var partitions = new uint[count]; - for (var i = 0; i < partitions.Length; i++) - { - partitions[i] = BinaryPrimitives.ReadUInt32LittleEndian(body.Slice(12 + i * 4, 4)); - } - - return new SyncConsumerGroupAssignment(generation, partitions); - } - - private static Exception Malformed() - { - return VsrError.Exception(VsrError.INVALID_COMMAND, - "Consumer group assignment reply is too short for the partition count it declares."); - } -} diff --git a/foreign/csharp/Iggy_SDK/Vsr/VsrError.cs b/foreign/csharp/Iggy_SDK/Vsr/VsrError.cs deleted file mode 100644 index 07e5306a83..0000000000 --- a/foreign/csharp/Iggy_SDK/Vsr/VsrError.cs +++ /dev/null @@ -1,61 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using Apache.Iggy.Exceptions; - -namespace Apache.Iggy.Vsr; - -/// -/// Server error codes the VSR paths raise locally or surface from the wire. Values match -/// core/common/src/error/iggy_error.rs; the server shares this code space across the reply -/// status word, the committed result section and the eviction mapping. -/// -internal static class VsrError -{ - internal const int INVALID_COMMAND = 3; - internal const int INVALID_FORMAT = 4; - internal const int FEATURE_UNAVAILABLE = 5; - internal const int INVALID_IDENTIFIER = 6; - internal const int STALE_CLIENT = 30; - internal const int UNAUTHENTICATED = 40; - internal const int INVALID_CREDENTIALS = 42; - internal const int INVALID_USERNAME = 43; - internal const int INVALID_PASSWORD = 44; - internal const int INVALID_PERSONAL_ACCESS_TOKEN = 53; - internal const int TRANSIENT_NOT_COMMITTED = 57; - internal const int TRANSIENT_NOT_ACCEPTED = 58; - internal const int EMPTY_RESPONSE = 304; - internal const int TOPIC_ID_NOT_FOUND = 2010; - internal const int CONSUMER_GROUP_MEMBER_NOT_FOUND = 5006; - internal const int CONSUMER_GROUP_PARTITION_NOT_OWNED = 5009; - internal const int INCOMPATIBLE_PROTOCOL_VERSION = 14003; - - /// A failure the client raised itself, before or instead of a server verdict. - internal static IggyInvalidStatusCodeException Exception(int code, string message) - { - return new IggyInvalidStatusCodeException(code, message); - } - - /// - /// A verdict the server reported, on the reply status word, in the committed result section or in an - /// eviction frame. Only these may drive retry and failover. - /// - internal static IggyInvalidStatusCodeException FromServer(int code, string message) - { - return new IggyInvalidStatusCodeException(code, message, true); - } -} diff --git a/foreign/csharp/Iggy_SDK/Vsr/VsrHeader.cs b/foreign/csharp/Iggy_SDK/Vsr/VsrHeader.cs deleted file mode 100644 index ad58e0c56e..0000000000 --- a/foreign/csharp/Iggy_SDK/Vsr/VsrHeader.cs +++ /dev/null @@ -1,156 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Buffers.Binary; - -namespace Apache.Iggy.Vsr; - -/// -/// The 256-byte consensus header, read and written by wire offset. Offsets mirror -/// core/binary_protocol/src/consensus/header.rs. Checksums stay zero: the server does not verify -/// them for client frames. -/// -internal static class VsrHeader -{ - internal const int HEADER_SIZE = 256; - - internal const int SIZE_OFFSET = 48; - internal const int COMMAND_OFFSET = 60; - - // The client wire carries no routing group: the server derives it (plane from - // the operation, partition target from the payload) and stamps it into its own - // internal header. Every field past the operation therefore sits eight bytes - // earlier than it did while the client header still carried one. - internal const int REQUEST_CLIENT_OFFSET = 128; - internal const int REQUEST_TIMESTAMP_OFFSET = 160; - internal const int REQUEST_ID_OFFSET = 168; - internal const int REQUEST_OPERATION_OFFSET = 176; - internal const int REQUEST_SESSION_OFFSET = 184; - internal const int REQUEST_RESERVED_OFFSET = 196; - - internal const int REPLY_OPERATION_OFFSET = 208; - internal const int REPLY_STATUS_OFFSET = 216; - - internal const int EVICTION_CLIENT_OFFSET = 128; - internal const int EVICTION_PROTOCOL_VERSION_OFFSET = 144; - internal const int EVICTION_PROTOCOL_VERSION_MIN_OFFSET = 148; - internal const int EVICTION_REASON_OFFSET = 255; - - /// - /// Encodes the request header for a classic command code and its body. Returns the total frame size - /// (header plus body). - /// - /// - /// Everything that can fail runs before a request id is consumed. The primary accepts any id above the - /// watermark, so a gap costs nothing, but a consumed id can never be handed back: re-encoding it for a - /// different request would let the client table answer that request from the first one's cached reply. - /// - internal static int EncodeRequestHeader(Span header, ConsensusSession session, int code, - ReadOnlySpan payload) - { - if (header.Length < HEADER_SIZE) - { - throw new ArgumentException($"Header buffer must be at least {HEADER_SIZE} bytes.", nameof(header)); - } - - header = header[..HEADER_SIZE]; - header.Clear(); - - var operation = VsrOperations.ForCode(code); - - if (payload.Length > int.MaxValue - HEADER_SIZE) - { - throw VsrError.Exception(VsrError.INVALID_COMMAND, "Request body exceeds the maximum frame size."); - } - - var frame = session.Resolve(operation); - var totalSize = HEADER_SIZE + payload.Length; - - BinaryPrimitives.WriteUInt32LittleEndian(header[SIZE_OFFSET..], (uint)totalSize); - header[COMMAND_OFFSET] = (byte)Command2.Request; - WriteUInt128(header[REQUEST_CLIENT_OFFSET..], frame.ClientId); - BinaryPrimitives.WriteUInt64LittleEndian(header[REQUEST_TIMESTAMP_OFFSET..], 0); - BinaryPrimitives.WriteUInt64LittleEndian(header[REQUEST_ID_OFFSET..], frame.RequestId); - header[REQUEST_OPERATION_OFFSET] = (byte)operation; - BinaryPrimitives.WriteUInt64LittleEndian(header[REQUEST_SESSION_OFFSET..], frame.SessionId); - - if (operation == VsrOperation.NonReplicated) - { - BinaryPrimitives.WriteUInt32LittleEndian(header[REQUEST_RESERVED_OFFSET..], (uint)code); - } - - return totalSize; - } - - internal static Command2 PeekCommand(ReadOnlySpan header) - { - return header[COMMAND_OFFSET] switch - { - (byte)Command2.Reply => Command2.Reply, - (byte)Command2.Eviction => Command2.Eviction, - _ => Command2.Reserved - }; - } - - /// Total frame size (header plus body) the peer announced. - internal static uint ReadSize(ReadOnlySpan header) - { - return BinaryPrimitives.ReadUInt32LittleEndian(header[SIZE_OFFSET..]); - } - - /// - /// Pre-commit deny channel. Nonzero means refused before commit with an empty body; a committed - /// rejection stamps 0 here and rides the result section instead. - /// - internal static uint ReadStatus(ReadOnlySpan header) - { - return BinaryPrimitives.ReadUInt32LittleEndian(header[REPLY_STATUS_OFFSET..]); - } - - internal static VsrOperation ReadReplyOperation(ReadOnlySpan header) - { - var operation = header[REPLY_OPERATION_OFFSET]; - if (!VsrOperations.IsKnown(operation)) - { - throw VsrError.Exception(VsrError.INVALID_COMMAND, $"Reply carries an unknown operation ({operation})."); - } - - return (VsrOperation)operation; - } - - internal static EvictionFrame ReadEviction(ReadOnlySpan header) - { - var reason = header[EVICTION_REASON_OFFSET]; - - return new EvictionFrame( - reason is > 0 and <= (byte)EvictionReason.MalformedLogin ? (EvictionReason)reason : null, - BinaryPrimitives.ReadUInt32LittleEndian(header[EVICTION_PROTOCOL_VERSION_OFFSET..]), - BinaryPrimitives.ReadUInt32LittleEndian(header[EVICTION_PROTOCOL_VERSION_MIN_OFFSET..])); - } - - private static void WriteUInt128(Span destination, UInt128 value) - { - BinaryPrimitives.WriteUInt64LittleEndian(destination, (ulong)value); - BinaryPrimitives.WriteUInt64LittleEndian(destination[8..], (ulong)(value >> 64)); - } -} - -/// Session-terminal eviction frame. Version fields are zero unless the reason is a protocol mismatch. -internal readonly record struct EvictionFrame( - EvictionReason? Reason, - uint ServerProtocolVersion, - uint ServerProtocolVersionMin); diff --git a/foreign/csharp/Iggy_SDK/Vsr/VsrOperation.cs b/foreign/csharp/Iggy_SDK/Vsr/VsrOperation.cs deleted file mode 100644 index 5653a23c06..0000000000 --- a/foreign/csharp/Iggy_SDK/Vsr/VsrOperation.cs +++ /dev/null @@ -1,275 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using Apache.Iggy.Utils; - -namespace Apache.Iggy.Vsr; - -/// -/// Replicated operation discriminant, byte 176 of a request header and byte 208 of a reply header. -/// Mirrors core/binary_protocol/src/consensus/operation.rs; discriminants are wire-pinned. -/// -internal enum VsrOperation : byte -{ - Reserved = 0, - Register = 1, - NonReplicated = 2, - Logout = 3, - - CreateTopicWithAssignments = 64, - CreatePartitionsWithAssignments = 65, - RemoveConsumerGroupMember = 66, - CompleteConsumerGroupRevocation = 67, - TruncatePartition = 68, - - CreateStream = 128, - UpdateStream = 129, - DeleteStream = 130, - PurgeStream = 131, - CreateTopic = 132, - UpdateTopic = 133, - DeleteTopic = 134, - PurgeTopic = 135, - CreatePartitions = 136, - DeletePartitions = 137, - DeleteSegments = 138, - CreateConsumerGroup = 139, - DeleteConsumerGroup = 140, - CreateUser = 141, - UpdateUser = 142, - DeleteUser = 143, - ChangePassword = 144, - UpdatePermissions = 145, - CreatePersonalAccessToken = 146, - DeletePersonalAccessToken = 147, - JoinConsumerGroup = 148, - LeaveConsumerGroup = 149, - - SendMessages = 160, - StoreConsumerOffset = 161, - DeleteConsumerOffset = 162, - StoreConsumerOffset2 = 164, - DeleteConsumerOffset2 = 165 -} - -internal static class VsrOperations -{ - private const byte InternalStart = (byte)VsrOperation.CreateTopicWithAssignments; - private const byte MetadataStart = (byte)VsrOperation.CreateStream; - private const byte PartitionStart = (byte)VsrOperation.SendMessages; - - /// - /// Non-replicated codes this build knows to leave no server-side state behind, so re-sending one after a - /// lost connection is indistinguishable from sending it once. Flushing an unsaved buffer is included: it - /// is idempotent by construction, a second flush writes nothing new. - /// - private static readonly HashSet NonReplicatedReadCodes = - [ - CommandCodes.PING_CODE, - CommandCodes.GET_STATS_CODE, - CommandCodes.GET_SNAPSHOT_CODE, - CommandCodes.GET_CLUSTER_METADATA_CODE, - CommandCodes.GET_ME_CODE, - CommandCodes.GET_CLIENT_CODE, - CommandCodes.GET_CLIENTS_CODE, - CommandCodes.GET_USER_CODE, - CommandCodes.GET_USERS_CODE, - CommandCodes.GET_PERSONAL_ACCESS_TOKENS_CODE, - CommandCodes.FLUSH_UNSAVED_BUFFER_CODE, - CommandCodes.GET_CONSUMER_OFFSET_CODE, - CommandCodes.GET_STREAM_CODE, - CommandCodes.GET_STREAMS_CODE, - CommandCodes.GET_TOPIC_CODE, - CommandCodes.GET_TOPICS_CODE, - CommandCodes.GET_CONSUMER_GROUP_CODE, - CommandCodes.GET_CONSUMER_GROUPS_CODE, - CommandCodes.SYNC_CONSUMER_GROUP_CODE - ]; - - /// - /// Maps a legacy command code to the operation its request header carries. An unmapped code rides - /// : the command table is a protocol registry, not a - /// per-server capability list, so the server is the authority on codes this SDK build does not know. - /// - internal static VsrOperation ForCode(int code) - { - return code switch - { - CommandCodes.LOGIN_REGISTER_CODE or CommandCodes.LOGIN_REGISTER_WITH_PAT_CODE => VsrOperation.Register, - - // VSR replaces the legacy login codes with the register handshake. A legacy login code arriving here - // means a caller bypassed the typed path, and sending it non-replicated would look like a working - // login while no session is ever bound. - CommandCodes.LOGIN_USER_CODE or CommandCodes.LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE => - throw VsrError.Exception(VsrError.INVALID_COMMAND, - $"Command {code} cannot be sent as a consensus request."), - CommandCodes.LOGOUT_USER_CODE => VsrOperation.Logout, - CommandCodes.CREATE_USER_CODE => VsrOperation.CreateUser, - CommandCodes.DELETE_USER_CODE => VsrOperation.DeleteUser, - CommandCodes.UPDATE_USER_CODE => VsrOperation.UpdateUser, - CommandCodes.UPDATE_PERMISSIONS_CODE => VsrOperation.UpdatePermissions, - CommandCodes.CHANGE_PASSWORD_CODE => VsrOperation.ChangePassword, - CommandCodes.CREATE_PERSONAL_ACCESS_TOKEN_CODE => VsrOperation.CreatePersonalAccessToken, - CommandCodes.DELETE_PERSONAL_ACCESS_TOKEN_CODE => VsrOperation.DeletePersonalAccessToken, - CommandCodes.SEND_MESSAGES_CODE => VsrOperation.SendMessages, - CommandCodes.STORE_CONSUMER_OFFSET_CODE => VsrOperation.StoreConsumerOffset, - CommandCodes.DELETE_CONSUMER_OFFSET_CODE => VsrOperation.DeleteConsumerOffset, - CommandCodes.STORE_CONSUMER_OFFSET_2_CODE => VsrOperation.StoreConsumerOffset2, - CommandCodes.DELETE_CONSUMER_OFFSET_2_CODE => VsrOperation.DeleteConsumerOffset2, - CommandCodes.CREATE_STREAM_CODE => VsrOperation.CreateStream, - CommandCodes.DELETE_STREAM_CODE => VsrOperation.DeleteStream, - CommandCodes.UPDATE_STREAM_CODE => VsrOperation.UpdateStream, - CommandCodes.PURGE_STREAM_CODE => VsrOperation.PurgeStream, - CommandCodes.CREATE_TOPIC_CODE => VsrOperation.CreateTopic, - CommandCodes.DELETE_TOPIC_CODE => VsrOperation.DeleteTopic, - CommandCodes.UPDATE_TOPIC_CODE => VsrOperation.UpdateTopic, - CommandCodes.PURGE_TOPIC_CODE => VsrOperation.PurgeTopic, - CommandCodes.CREATE_PARTITIONS_CODE => VsrOperation.CreatePartitions, - CommandCodes.DELETE_PARTITIONS_CODE => VsrOperation.DeletePartitions, - CommandCodes.DELETE_SEGMENTS_CODE => VsrOperation.DeleteSegments, - CommandCodes.CREATE_CONSUMER_GROUP_CODE => VsrOperation.CreateConsumerGroup, - CommandCodes.DELETE_CONSUMER_GROUP_CODE => VsrOperation.DeleteConsumerGroup, - CommandCodes.JOIN_CONSUMER_GROUP_CODE => VsrOperation.JoinConsumerGroup, - CommandCodes.LEAVE_CONSUMER_GROUP_CODE => VsrOperation.LeaveConsumerGroup, - _ => VsrOperation.NonReplicated - }; - } - - /// - /// Whether losing the connection mid-request leaves nothing for the server to deduplicate, so the request - /// can be re-issued on a fresh session by the reconnect path instead of failing the caller. - /// - internal static bool IsReplaySafeRead(int code, bool isLoginRegister, ReadOnlySpan body) - { - // A register lost mid-flight is replayable: the retry re-arms the session under a fresh client id, so - // it cannot be mistaken for the first one. At worst the server keeps an entry nobody binds, which it - // ages out. Reporting an unknown outcome here instead would deny the caller the retry that is in fact - // the only correct response. - if (isLoginRegister) - { - return true; - } - - var operation = ForCode(code); - - // A consumer offset write carries an absolute offset and the server applies it as an unconditional - // overwrite, on a plane that keeps no client table to dedup against, so a replay lands on the same - // value. Denying the retry here reports an unknown outcome for a blip on an offset commit, which - // takes down the consume loop over a write that was safe to repeat. - if (operation is VsrOperation.StoreConsumerOffset or VsrOperation.StoreConsumerOffset2 - or VsrOperation.DeleteConsumerOffset or VsrOperation.DeleteConsumerOffset2) - { - return true; - } - - if (operation != VsrOperation.NonReplicated) - { - return false; - } - - // A poll that auto-commits moves the consumer offset server-side, so a reply lost after the commit - // would make the replay start past a batch the caller never saw. auto_commit is the last body byte. - if (code == CommandCodes.POLL_MESSAGES_CODE) - { - return body.Length > 0 && body[^1] == 0; - } - - // Everything else non-replicated is replay-safe only if this build knows it to be a read. An unmapped - // code also lands on NonReplicated, and re-sending one the server implements as a mutation would apply - // it twice with nothing to deduplicate against. - return NonReplicatedReadCodes.Contains(code); - } - - /// - /// Whether the byte is a declared discriminant. Replies carry a server-controlled operation byte, so - /// an undeclared value is rejected rather than classified by range. - /// - internal static bool IsKnown(byte value) - { - return (VsrOperation)value switch - { - VsrOperation.Reserved or VsrOperation.Register or VsrOperation.NonReplicated or VsrOperation.Logout => - true, - >= VsrOperation.CreateTopicWithAssignments and <= VsrOperation.TruncatePartition => true, - >= VsrOperation.CreateStream and <= VsrOperation.LeaveConsumerGroup => true, - VsrOperation.SendMessages or VsrOperation.StoreConsumerOffset or VsrOperation.DeleteConsumerOffset - or VsrOperation.StoreConsumerOffset2 or VsrOperation.DeleteConsumerOffset2 => true, - _ => false - }; - } - - /// Replica / journal only; never emitted by a client. - internal static bool IsInternal(this VsrOperation operation) - { - return (byte)operation >= InternalStart && (byte)operation < MetadataStart; - } - - /// - /// Control-plane operations handled by shard 0. Enumerated member by member, mirroring - /// Operation::is_metadata, rather than tested as a numeric range: the metadata block runs to 159 - /// but the last member declared today is 149, so a range would silently exclude the next operation added - /// upstream. That operation would then also read as not result-framed, and a committed rejection in its - /// reply would decode as a successful payload. - /// - internal static bool IsMetadata(this VsrOperation operation) - { - return operation.IsInternal() || operation is VsrOperation.CreateStream - or VsrOperation.UpdateStream - or VsrOperation.DeleteStream - or VsrOperation.PurgeStream - or VsrOperation.CreateTopic - or VsrOperation.UpdateTopic - or VsrOperation.DeleteTopic - or VsrOperation.PurgeTopic - or VsrOperation.CreatePartitions - or VsrOperation.DeletePartitions - or VsrOperation.CreateConsumerGroup - or VsrOperation.DeleteConsumerGroup - or VsrOperation.CreateUser - or VsrOperation.UpdateUser - or VsrOperation.DeleteUser - or VsrOperation.ChangePassword - or VsrOperation.UpdatePermissions - or VsrOperation.CreatePersonalAccessToken - or VsrOperation.DeletePersonalAccessToken - or VsrOperation.JoinConsumerGroup - or VsrOperation.LeaveConsumerGroup; - } - - /// - /// Data-plane operations routed by namespace to the shard owning the partition. - /// is deliberately neither metadata nor partition: the - /// server resolves it to an internal TruncatePartition, yet it still carries a packed namespace. - /// - internal static bool IsPartition(this VsrOperation operation) - { - return (byte)operation >= PartitionStart; - } - - /// - /// Whether a reply for this operation leads its body with the committed result section. Metadata ops - /// always do; on the partition plane only the consumer-offset ops do. Register is result-framed only - /// when its body is non-empty, which is why that case stays in . - /// - internal static bool IsResultFramed(this VsrOperation operation) - { - return operation.IsMetadata() || operation is VsrOperation.StoreConsumerOffset - or VsrOperation.StoreConsumerOffset2 - or VsrOperation.DeleteConsumerOffset - or VsrOperation.DeleteConsumerOffset2; - } -} diff --git a/foreign/csharp/Iggy_SDK/Vsr/VsrReplyDecoder.cs b/foreign/csharp/Iggy_SDK/Vsr/VsrReplyDecoder.cs deleted file mode 100644 index efb2130157..0000000000 --- a/foreign/csharp/Iggy_SDK/Vsr/VsrReplyDecoder.cs +++ /dev/null @@ -1,206 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Buffers.Binary; - -namespace Apache.Iggy.Vsr; - -/// -/// Decodes a reply frame into the typed payload the classic response readers expect. -/// -internal static class VsrReplyDecoder -{ - internal const int RESULT_COUNT_LENGTH = 4; - internal const int RESULT_ENTRY_LENGTH = 8; - - /// - /// Runs the decode funnel in the order the wire contract requires: eviction frames first, then the - /// announced size, then the pre-commit status word, and only then the body and its committed result - /// section. Reading the status before the body is what lets an empty deny reply fail with the right - /// error instead of a truncation. - /// - internal static ReadOnlyMemory Decode(ReadOnlySpan header, ReadOnlyMemory body) - { - if (header.Length < VsrHeader.HEADER_SIZE) - { - throw VsrError.Exception(VsrError.EMPTY_RESPONSE, "Reply header is shorter than the consensus header."); - } - - switch (VsrHeader.PeekCommand(header)) - { - case Command2.Eviction: - throw ToException(VsrHeader.ReadEviction(header)); - case Command2.Reply: - break; - default: - throw VsrError.Exception(VsrError.INVALID_COMMAND, - $"Unexpected consensus frame ({header[VsrHeader.COMMAND_OFFSET]})."); - } - - var expectedBody = ReadBodySize(header); - if (body.Length < expectedBody) - { - throw VsrError.Exception(VsrError.INVALID_COMMAND, "Reply body is shorter than the announced frame size."); - } - - var status = VsrHeader.ReadStatus(header); - if (status != 0) - { - throw VsrError.FromServer((int)status, $"Server rejected the request with status {status}."); - } - - return SplitResultSection(VsrHeader.ReadReplyOperation(header), body[..expectedBody]); - } - - /// Body length announced by the frame, i.e. the bytes to read after the header. - internal static int ReadBodySize(ReadOnlySpan header) - { - var size = VsrHeader.ReadSize(header); - if (size < VsrHeader.HEADER_SIZE || size > int.MaxValue) - { - throw VsrError.Exception(VsrError.INVALID_COMMAND, $"Reply announced an invalid frame size ({size})."); - } - - return (int)size - VsrHeader.HEADER_SIZE; - } - - internal static Exception ToException(EvictionFrame eviction) - { - var (code, message) = eviction.Reason switch - { - // The five reasons below fall into the catch-all of the shared grader, so they carry INVALID_COMMAND - // even where a narrower status would read better. The message keeps the detail; the code is the part - // six SDKs agree on. - EvictionReason.ClientReleaseTooLow => (VsrError.INVALID_COMMAND, - "Client release is below the cluster minimum."), - EvictionReason.ClientReleaseTooHigh => (VsrError.INVALID_COMMAND, - "Client release is above the cluster maximum."), - EvictionReason.InvalidRequestOperation => (VsrError.INVALID_COMMAND, - "Server rejected the request operation."), - EvictionReason.InvalidRequestBody => (VsrError.INVALID_COMMAND, "Server rejected the request body."), - EvictionReason.InvalidRequestBodySize => (VsrError.INVALID_COMMAND, - "Server rejected the request body size."), - EvictionReason.InvalidCredentials => (VsrError.INVALID_CREDENTIALS, "Invalid credentials."), - EvictionReason.InvalidToken => (VsrError.INVALID_PERSONAL_ACCESS_TOKEN, "Invalid personal access token."), - EvictionReason.UserInactive => (VsrError.UNAUTHENTICATED, "User is inactive."), - EvictionReason.SessionError => (VsrError.UNAUTHENTICATED, "Session error."), - EvictionReason.NoSession => (VsrError.UNAUTHENTICATED, "No session for this client."), - EvictionReason.SessionTooLow => (VsrError.UNAUTHENTICATED, "Session is below the cluster minimum."), - EvictionReason.SessionReleaseMismatch => (VsrError.UNAUTHENTICATED, "Session release mismatch."), - EvictionReason.StaleClient => (VsrError.STALE_CLIENT, "Client missed too many heartbeats."), - EvictionReason.IncompatibleProtocol => IncompatibleProtocol(eviction), - EvictionReason.MalformedLogin => (VsrError.INVALID_FORMAT, "Malformed login body."), - - // Reserved and any reason this build cannot decode share the grader's catch-all. - null => (VsrError.INVALID_COMMAND, "Session evicted for an unrecognized reason."), - _ => (VsrError.INVALID_COMMAND, $"Session evicted ({eviction.Reason}).") - }; - - return VsrError.FromServer(code, message); - } - - /// - /// Leading result code of a committed reply body: 0 for success, otherwise the first entry's result. - /// null when the body cannot hold what the count claims - corruption, never a silent success. - /// - internal static uint? ReadResultCode(ReadOnlySpan body) - { - if (!TryReadUInt32(body, 0, out var count)) - { - return null; - } - - if (count == 0) - { - return 0; - } - - return TryReadUInt32(body, RESULT_COUNT_LENGTH + 4, out var result) ? result : null; - } - - /// Byte length of the leading result section, i.e. where the typed payload starts. - internal static int? ReadResultSectionLength(ReadOnlySpan body) - { - if (!TryReadUInt32(body, 0, out var count)) - { - return null; - } - - var length = RESULT_COUNT_LENGTH + (long)count * RESULT_ENTRY_LENGTH; - - return body.Length >= length ? (int)length : null; - } - - private static (int Code, string Message) IncompatibleProtocol(EvictionFrame eviction) - { - if (eviction.ServerProtocolVersionMin == 0 || - eviction.ServerProtocolVersion < eviction.ServerProtocolVersionMin) - { - return (VsrError.UNAUTHENTICATED, "Server rejected the client protocol version."); - } - - return (VsrError.INCOMPATIBLE_PROTOCOL_VERSION, - $"Client protocol version {LoginRegister.PROTOCOL_VERSION} is outside the range accepted by the server " + - $"({eviction.ServerProtocolVersionMin}..{eviction.ServerProtocolVersion})."); - } - - /// - /// Strips the committed result section from a result-framed reply and maps a committed rejection to - /// its error. Register replies are result-framed only when non-empty: a terminal register failure - /// ships an empty body and is passed through to fail the typed response decode. - /// - private static ReadOnlyMemory SplitResultSection(VsrOperation operation, ReadOnlyMemory body) - { - var resultFramed = operation.IsResultFramed() || - (operation == VsrOperation.Register && !body.IsEmpty); - if (!resultFramed) - { - return body; - } - - var code = ReadResultCode(body.Span); - if (code is null) - { - throw VsrError.Exception(VsrError.INVALID_COMMAND, "Reply carries a malformed committed result section."); - } - - if (code != 0) - { - throw VsrError.FromServer((int)code.Value, $"Server rejected the request with status {code.Value}."); - } - - var payloadStart = ReadResultSectionLength(body.Span) - ?? throw VsrError.Exception(VsrError.INVALID_COMMAND, - "Reply carries a truncated committed result section."); - - return body[payloadStart..]; - } - - private static bool TryReadUInt32(ReadOnlySpan body, int offset, out uint value) - { - if (body.Length < offset + 4) - { - value = 0; - - return false; - } - - value = BinaryPrimitives.ReadUInt32LittleEndian(body.Slice(offset, 4)); - - return true; - } -} diff --git a/foreign/csharp/Iggy_SDK_Tests/ClientTests/IggyClientFactoryTests.cs b/foreign/csharp/Iggy_SDK_Tests/ClientTests/IggyClientFactoryTests.cs deleted file mode 100644 index de665603d1..0000000000 --- a/foreign/csharp/Iggy_SDK_Tests/ClientTests/IggyClientFactoryTests.cs +++ /dev/null @@ -1,67 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using Apache.Iggy.Configuration; -using Apache.Iggy.Enums; -using Apache.Iggy.Factory; - -namespace Apache.Iggy.Tests.ClientTests; - -public sealed class IggyClientFactoryTests -{ - [Fact] - public void CreateClient_CreatesTcpClient() - { - var options = new IggyClientConfigurator - { - BaseAddress = "127.0.0.1:8090", - Protocol = Protocol.Tcp - }; - - Assert.Equal(64 * 1024 * 1024, options.MaxResponseFrameSize); - - using var client = IggyClientFactory.CreateClient(options) as IDisposable; - Assert.NotNull(client); - } - - [Fact] - public void CreateClient_RejectsMaxResponseFrameSizeBelowHeader() - { - var options = new IggyClientConfigurator - { - BaseAddress = "127.0.0.1:8090", - Protocol = Protocol.Tcp, - MaxResponseFrameSize = 255 - }; - - Assert.Throws(() => IggyClientFactory.CreateClient(options)); - } - - [Fact] - public void CreateClient_AcceptsMaxResponseFrameSizeUnderHttp() - { - var options = new IggyClientConfigurator - { - BaseAddress = "http://127.0.0.1:3000", - Protocol = Protocol.Http, - MaxResponseFrameSize = 1 - }; - - using var client = IggyClientFactory.CreateClient(options) as IDisposable; - Assert.NotNull(client); - } -} diff --git a/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/IggyConsumerBuilderTests.cs b/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/IggyConsumerBuilderTests.cs index 8178484708..db0af7c3fc 100644 --- a/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/IggyConsumerBuilderTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/IggyConsumerBuilderTests.cs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -using System.Text; using Apache.Iggy.Consumers; using Apache.Iggy.Encryption; using Apache.Iggy.Enums; @@ -70,22 +69,4 @@ public void Build_WithEncryptorAndAfterReceiveCommit_DoesNotThrow() Assert.NotNull(consumer); } - - [Fact] - public void TypedBuild_OverTcp_CreatesTheClient() - { - IggyConsumerBuilder builder = IggyConsumerBuilder - .Create(StreamId, TopicId, Consumer.New(1), new StringDeserializer()); - builder.WithConnection(Protocol.Tcp, "127.0.0.1:8090", "user", "pass"); - - Assert.NotNull(builder.Build()); - } - - private sealed class StringDeserializer : IDeserializer - { - public string Deserialize(ReadOnlyMemory data) - { - return Encoding.UTF8.GetString(data.Span); - } - } } diff --git a/foreign/csharp/Iggy_SDK_Tests/MapperTests/BinaryMapper.cs b/foreign/csharp/Iggy_SDK_Tests/MapperTests/BinaryMapper.cs index e37aa35750..c86fb463db 100644 --- a/foreign/csharp/Iggy_SDK_Tests/MapperTests/BinaryMapper.cs +++ b/foreign/csharp/Iggy_SDK_Tests/MapperTests/BinaryMapper.cs @@ -432,82 +432,6 @@ public void MapRentedMessages_WithEncryptor_TamperedCiphertext_ThrowsMessageDecr Assert.IsAssignableFrom(ex.InnerException); } - [Fact] - public void MapSendMessages_ReturnsConfirmations() - { - // Wire layout mirrors core/binary_protocol responses/messages/send_messages.rs: - // [count:4][stream_id:4][topic_id:4][partition_id:4][base_offset:8]* - var payload = new byte[4 + 20]; - BinaryPrimitives.WriteUInt32LittleEndian(payload.AsSpan(0, 4), 1); - BinaryPrimitives.WriteUInt32LittleEndian(payload.AsSpan(4, 4), 1); - BinaryPrimitives.WriteUInt32LittleEndian(payload.AsSpan(8, 4), 2); - BinaryPrimitives.WriteUInt32LittleEndian(payload.AsSpan(12, 4), 3); - BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(16, 8), 42); - - var response = Mappers.BinaryMapper.MapSendMessages(payload); - - var confirmation = Assert.Single(response.Confirmations); - Assert.Equal(1u, confirmation.StreamId); - Assert.Equal(2u, confirmation.TopicId); - Assert.Equal(3u, confirmation.PartitionId); - Assert.Equal(42ul, confirmation.BaseOffset); - } - - [Fact] - public void MapSendMessages_MultipleConfirmations_ReturnsAll() - { - var payload = new byte[4 + 3 * 20]; - BinaryPrimitives.WriteUInt32LittleEndian(payload.AsSpan(0, 4), 3); - for (var i = 0; i < 3; i++) - { - var position = 4 + i * 20; - BinaryPrimitives.WriteUInt32LittleEndian(payload.AsSpan(position, 4), 1); - BinaryPrimitives.WriteUInt32LittleEndian(payload.AsSpan(position + 4, 4), 2); - BinaryPrimitives.WriteUInt32LittleEndian(payload.AsSpan(position + 8, 4), (uint)i); - BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(position + 12, 8), (ulong)(100 + i)); - } - - var response = Mappers.BinaryMapper.MapSendMessages(payload); - - Assert.Equal(3, response.Confirmations.Count); - Assert.Equal(2u, response.Confirmations[2].PartitionId); - Assert.Equal(102ul, response.Confirmations[2].BaseOffset); - } - - [Fact] - public void MapSendMessages_EmptyBody_Throws() - { - Assert.Throws(() => Mappers.BinaryMapper.MapSendMessages([])); - } - - [Fact] - public void MapSendMessages_ZeroCount_ReturnsNoConfirmations() - { - var response = Mappers.BinaryMapper.MapSendMessages(new byte[4]); - - Assert.Empty(response.Confirmations); - } - - [Theory] - [InlineData(4 + 19)] // truncated entry - [InlineData(4 + 21)] // trailing byte - public void MapSendMessages_ShapeMismatch_Throws(int payloadLength) - { - var payload = new byte[payloadLength]; - BinaryPrimitives.WriteUInt32LittleEndian(payload.AsSpan(0, 4), 1); - - Assert.Throws(() => Mappers.BinaryMapper.MapSendMessages(payload)); - } - - [Fact] - public void MapSendMessages_BogusCount_DoesNotOverflow() - { - var payload = new byte[4]; - BinaryPrimitives.WriteUInt32LittleEndian(payload, uint.MaxValue); - - Assert.Throws(() => Mappers.BinaryMapper.MapSendMessages(payload)); - } - private static byte[] BuildEncryptedFrame(AesMessageEncryptor encryptor, ulong offset, ReadOnlySpan plainPayload, ReadOnlySpan plainHeaders) { diff --git a/foreign/csharp/Iggy_SDK_Tests/PublisherTests/IggyPublisherBuilderTests.cs b/foreign/csharp/Iggy_SDK_Tests/PublisherTests/IggyPublisherBuilderTests.cs deleted file mode 100644 index 1172a752dd..0000000000 --- a/foreign/csharp/Iggy_SDK_Tests/PublisherTests/IggyPublisherBuilderTests.cs +++ /dev/null @@ -1,47 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Buffers; -using System.Text; -using Apache.Iggy.Enums; -using Apache.Iggy.Publishers; - -namespace Apache.Iggy.Tests.PublisherTests; - -public class IggyPublisherBuilderTests -{ - private static readonly Identifier StreamId = Identifier.Numeric(1); - private static readonly Identifier TopicId = Identifier.Numeric(1); - - [Fact] - public void TypedBuild_OverTcp_CreatesTheClient() - { - IggyPublisherBuilder builder - = IggyPublisherBuilder.Create(StreamId, TopicId, new StringSerializer()); - builder.WithConnection(Protocol.Tcp, "127.0.0.1:8090", "user", "pass"); - - Assert.NotNull(builder.Build()); - } - - private sealed class StringSerializer : ISerializer - { - public void Serialize(string data, IBufferWriter writer) - { - writer.Write(Encoding.UTF8.GetBytes(data)); - } - } -} diff --git a/foreign/csharp/Iggy_SDK_Tests/PublisherTests/IggyTypedPublisherTests.cs b/foreign/csharp/Iggy_SDK_Tests/PublisherTests/IggyTypedPublisherTests.cs index 2537b2f3bb..61776aec4c 100644 --- a/foreign/csharp/Iggy_SDK_Tests/PublisherTests/IggyTypedPublisherTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/PublisherTests/IggyTypedPublisherTests.cs @@ -308,7 +308,7 @@ private static IIggyClient BuildClient(SendRecorder recorder) CancellationToken _) => { recorder.Record(stream, topic, partitioning, messages); - return Task.FromResult(new SendMessagesResponse { Confirmations = [] }); + return Task.CompletedTask; }); mock.Setup(c => c.SendMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) @@ -316,7 +316,7 @@ private static IIggyClient BuildClient(SendRecorder recorder) CancellationToken _) => { recorder.Record(stream, topic, partitioning, new[] { message }); - return Task.FromResult(new SendMessagesResponse { Confirmations = [] }); + return Task.CompletedTask; }); return mock.Object; diff --git a/foreign/csharp/Iggy_SDK_Tests/UtilityTests/SendUnitTests.cs b/foreign/csharp/Iggy_SDK_Tests/UtilityTests/SendUnitTests.cs index ec5736bcd6..c7095f312a 100644 --- a/foreign/csharp/Iggy_SDK_Tests/UtilityTests/SendUnitTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/UtilityTests/SendUnitTests.cs @@ -17,7 +17,6 @@ using System.Buffers; using System.Diagnostics; -using Apache.Iggy.Contracts; using Apache.Iggy.Extensions; using Apache.Iggy.IggyClient; using Apache.Iggy.Messages; @@ -212,7 +211,7 @@ public async Task Processor_Dispose_DisposesQueuedOwners() public async Task Processor_DrainStaysPending_UntilSendCompletes() { // Gate the send so the unit is in flight; the drain must not complete mid-flight. - var gate = new TaskCompletionSource(); + var gate = new TaskCompletionSource(); var client = new Mock(); client.Setup(c => c.SendMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>(), It.IsAny())) @@ -230,7 +229,7 @@ public async Task Processor_DrainStaysPending_UntilSendCompletes() await Task.Delay(100, TestContext.Current.CancellationToken); Assert.False(drain.IsCompleted); - gate.SetResult(new SendMessagesResponse { Confirmations = [] }); + gate.SetResult(); await drain.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); } @@ -239,7 +238,7 @@ private static Mock MockClient(List sentCounts) var client = new Mock(); client.Setup(c => c.SendMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>(), It.IsAny())) - .ReturnsAsync(new SendMessagesResponse { Confirmations = [] }) + .Returns(Task.CompletedTask) .Callback((Identifier _, Identifier _, Partitioning _, IList messages, CancellationToken _) => sentCounts.Add(messages.Count)); return client; diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/ConsensusSessionTests.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/ConsensusSessionTests.cs deleted file mode 100644 index 51e4d7f0a1..0000000000 --- a/foreign/csharp/Iggy_SDK_Tests/VsrTests/ConsensusSessionTests.cs +++ /dev/null @@ -1,218 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using Apache.Iggy.Exceptions; -using Apache.Iggy.Vsr; - -namespace Apache.Iggy.Tests.VsrTests; - -public sealed class ConsensusSessionTests -{ - [Fact] - public void NewSession_IsUnboundWithNonZeroClientId() - { - var session = new ConsensusSession(); - - Assert.False(session.IsBound); - Assert.Null(session.Session); - Assert.NotEqual(UInt128.Zero, session.ClientId); - } - - [Fact] - public void NewSession_MintsUniqueClientIds() - { - Assert.NotEqual(new ConsensusSession().ClientId, new ConsensusSession().ClientId); - } - - [Fact] - public void BeginRegister_OnFreshSessionKeepsClientId() - { - var session = new ConsensusSession(7); - - Assert.Equal(0UL, session.Resolve(VsrOperation.Register).RequestId); - Assert.Equal((UInt128)7, session.ClientId); - Assert.False(session.IsBound); - } - - /// - /// The re-arm mints the client id the register is encoded with, so the frame must carry the new one, not - /// the one the previous session used. - /// - [Fact] - public void Resolve_RegisterAfterBindReportsTheReArmedClientId() - { - var session = new ConsensusSession(7); - session.Resolve(VsrOperation.Register); - session.Bind(42); - - var frame = session.Resolve(VsrOperation.Register); - - Assert.Equal(session.ClientId, frame.ClientId); - Assert.NotEqual((UInt128)7, frame.ClientId); - } - - [Fact] - public void BeginRegister_AfterBindReArmsWithFreshClientId() - { - var session = new ConsensusSession(7); - session.Resolve(VsrOperation.Register); - session.Bind(42); - session.Resolve(VsrOperation.CreateStream); - - Assert.Equal(0UL, session.Resolve(VsrOperation.Register).RequestId); - Assert.NotEqual((UInt128)7, session.ClientId); - Assert.False(session.IsBound); - Assert.Equal(1UL, session.RequestCounter); - } - - /// - /// A register that never bound is cleared by the reset its failure path runs, and the retry re-arms onto a - /// fresh client id rather than reusing the one the server may have already seen. - /// - [Fact] - public void BeginRegister_AfterConsumedRegisterAndResetReArms() - { - var session = new ConsensusSession(7); - session.Resolve(VsrOperation.Register); - - session.Reset(); - - Assert.Equal(0UL, session.Resolve(VsrOperation.Register).RequestId); - Assert.False(session.IsBound); - Assert.NotEqual((UInt128)7, session.ClientId); - } - - [Fact] - public void NextRequestId_IsMonotonicAfterBind() - { - var session = new ConsensusSession(1); - session.Resolve(VsrOperation.Register); - session.Bind(10); - - Assert.Equal(1UL, session.Resolve(VsrOperation.CreateStream).RequestId); - Assert.Equal(2UL, session.Resolve(VsrOperation.CreateStream).RequestId); - Assert.Equal(3UL, session.RequestCounter); - } - - [Fact] - public void NextRequestId_BeforeBindThrows() - { - var session = new ConsensusSession(1); - - var error = Assert.Throws(() => session.Resolve(VsrOperation.CreateStream)); - - Assert.Equal(VsrError.UNAUTHENTICATED, error.StatusCode); - } - - [Fact] - public void Resolve_DoesNotConsumeAnIdForNonReplicatedOrPartitionOps() - { - var session = new ConsensusSession(1); - session.Resolve(VsrOperation.Register); - session.Bind(10); - - Assert.Equal(1UL, session.Resolve(VsrOperation.NonReplicated).RequestId); - Assert.Equal(1UL, session.Resolve(VsrOperation.SendMessages).RequestId); - Assert.Equal(1UL, session.RequestCounter); - } - - [Fact] - public void Bind_TwiceThrows() - { - var session = new ConsensusSession(1); - session.Resolve(VsrOperation.Register); - session.Bind(10); - - Assert.Throws(() => session.Bind(20)); - Assert.Equal(10UL, session.Session); - } - - [Fact] - public void Bind_ZeroThrows() - { - var session = new ConsensusSession(1); - - var exception = Assert.Throws(() => session.Bind(0)); - Assert.Equal(VsrError.INVALID_FORMAT, exception.StatusCode); - } - - [Fact] - public void Bind_WithoutAnInFlightRegisterThrows() - { - var session = new ConsensusSession(1); - - Assert.Throws(() => session.Bind(10)); - } - - /// - /// Binding runs after the sending lock is released, so a drop can have re-armed the identity since the - /// register committed. The reset clears the pending register, and binding regardless would pair the - /// session the server issued to the old client id with the one the re-arm minted. - /// - [Fact] - public void Bind_AfterTheIdentityReArmedThrows() - { - var session = new ConsensusSession(1); - session.Resolve(VsrOperation.Register); - - session.Reset(); - - Assert.Throws(() => session.Bind(10)); - Assert.False(session.IsBound); - } - - /// - /// Two concurrent registers would otherwise re-arm the identity under the first one, so its bind would - /// pair a committed session with a client id the server never saw. - /// - [Fact] - public void Resolve_SecondRegisterWhileOneIsInFlightThrows() - { - var session = new ConsensusSession(1); - session.Resolve(VsrOperation.Register); - - var error = Assert.Throws(() => session.Resolve(VsrOperation.Register)); - - Assert.Equal(VsrError.UNAUTHENTICATED, error.StatusCode); - } - - [Fact] - public void Resolve_RegisterIsAllowedAgainAfterReset() - { - var session = new ConsensusSession(1); - session.Resolve(VsrOperation.Register); - - session.Reset(); - - Assert.Equal(0UL, session.Resolve(VsrOperation.Register).RequestId); - } - - [Fact] - public void Reset_ClearsBindingAndCounter() - { - var session = new ConsensusSession(1); - session.Resolve(VsrOperation.Register); - session.Bind(10); - session.Resolve(VsrOperation.CreateStream); - - session.Reset(); - - Assert.False(session.IsBound); - Assert.Equal(1UL, session.RequestCounter); - Assert.NotEqual((UInt128)1, session.ClientId); - } -} diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/ConsumerGroupClientStateTests.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/ConsumerGroupClientStateTests.cs deleted file mode 100644 index c1c4f00ada..0000000000 --- a/foreign/csharp/Iggy_SDK_Tests/VsrTests/ConsumerGroupClientStateTests.cs +++ /dev/null @@ -1,176 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.IO.Hashing; -using System.Text; -using Apache.Iggy.Vsr; - -namespace Apache.Iggy.Tests.VsrTests; - -public sealed class ConsumerGroupClientStateTests -{ - private static readonly GroupKey Key = - new(Identifier.Numeric(1), Identifier.Numeric(2), Identifier.Numeric(3)); - - private static readonly TopicKey Topic = new(Identifier.Numeric(1), Identifier.Numeric(2)); - - [Fact] - public void NextGroupPartition_RoundRobinsThenWraps() - { - var state = new ConsumerGroupClientState(); - state.SetAssignment(Key, 1, [0, 1, 2]); - - Assert.Equal([0, 1, 2, 0], [ - state.NextGroupPartition(Key), state.NextGroupPartition(Key), - state.NextGroupPartition(Key), state.NextGroupPartition(Key) - ]); - } - - [Fact] - public void SetAssignment_OnNewGenerationResetsCursor() - { - var state = new ConsumerGroupClientState(); - state.SetAssignment(Key, 1, [0, 1, 2]); - state.NextGroupPartition(Key); - state.NextGroupPartition(Key); - - state.SetAssignment(Key, 2, [5]); - - Assert.Equal(5u, state.NextGroupPartition(Key)); - } - - [Fact] - public void SetAssignment_OnSameGenerationKeepsCursor() - { - var state = new ConsumerGroupClientState(); - state.SetAssignment(Key, 1, [0, 1, 2]); - state.NextGroupPartition(Key); - - state.SetAssignment(Key, 1, [0, 1, 2]); - - Assert.Equal(1u, state.NextGroupPartition(Key)); - } - - [Fact] - public void NextGroupPartition_WithoutAssignmentIsNull() - { - var state = new ConsumerGroupClientState(); - - Assert.False(state.HasAssignment(Key)); - Assert.Null(state.NextGroupPartition(Key)); - } - - [Fact] - public void InvalidateAssignment_KeepsMembership() - { - var state = new ConsumerGroupClientState(); - state.RegisterGroup(Key, Identifier.Numeric(1), Identifier.Numeric(2), Identifier.Numeric(3)); - state.SetAssignment(Key, 1, [0]); - - state.InvalidateAssignment(Key); - - Assert.False(state.HasAssignment(Key)); - Assert.True(state.IsRegistered(Key)); - } - - [Fact] - public void MemberHoldingNoPartitions_StaysRegistered() - { - var state = new ConsumerGroupClientState(); - state.RegisterGroup(Key, Identifier.Numeric(1), Identifier.Numeric(2), Identifier.Numeric(3)); - state.SetAssignment(Key, 1, []); - - Assert.False(state.HasAssignment(Key)); - Assert.True(state.IsRegistered(Key)); - - state.DeregisterGroup(Key); - - Assert.False(state.IsRegistered(Key)); - } - - [Fact] - public void RegisteredGroups_ReturnsJoinedIdentifiers() - { - var state = new ConsumerGroupClientState(); - state.RegisterGroup(Key, Identifier.Numeric(1), Identifier.Numeric(2), Identifier.String("group")); - - IReadOnlyList groups = state.RegisteredGroups(); - - var group = Assert.Single(groups); - - Assert.Equal("1", group.StreamId.ToString()); - Assert.Equal("2", group.TopicId.ToString()); - Assert.Equal("group", group.GroupId.ToString()); - } - - [Fact] - public void NextBalancedPartition_RoundRobinsThenWraps() - { - var state = new ConsumerGroupClientState(); - - Assert.Equal([0, 1, 2, 0], [ - state.NextBalancedPartition(Topic, 3), state.NextBalancedPartition(Topic, 3), - state.NextBalancedPartition(Topic, 3), state.NextBalancedPartition(Topic, 3) - ]); - } - - [Fact] - public void NextBalancedPartition_WithNoPartitionsIsZero() - { - Assert.Equal(0u, new ConsumerGroupClientState().NextBalancedPartition(Topic, 0)); - } - - [Fact] - public void ClearSessionScoped_DropsMembershipAndAssignmentsButKeepsTopicState() - { - var state = new ConsumerGroupClientState(); - state.RegisterGroup(Key, Identifier.Numeric(1), Identifier.Numeric(2), Identifier.Numeric(3)); - state.SetAssignment(Key, 1, [0]); - state.SetPartitionCount(Topic, 4); - state.NextBalancedPartition(Topic, 4); - - state.ClearSessionScoped(); - - Assert.False(state.IsRegistered(Key)); - Assert.False(state.HasAssignment(Key)); - Assert.Equal(4u, state.PartitionCount(Topic)); - Assert.Equal(1u, state.NextBalancedPartition(Topic, 4)); - } - - [Fact] - public void TopicKey_SeparatesNumericFromNamedIdentifiers() - { - Assert.NotEqual(new TopicKey(Identifier.Numeric(1), Identifier.Numeric(1)), - new TopicKey(Identifier.String("1"), Identifier.String("1"))); - } - - /// - /// Message-key partitioning has to agree with the Rust client byte for byte, or the two SDKs put the same - /// key on different partitions. Vectors come from calculate_32 (XxHash32::oneshot(0, data)). - /// - [Theory] - [InlineData("", 0x02cc5d05u)] - [InlineData("a", 0x550d7456u)] - [InlineData("abc", 0x32d153ffu)] - [InlineData("hello world", 0xcebb6622u)] - [InlineData("iggy-message-key", 0xf54b51c9u)] - [InlineData("1234567890123456789012345", 0xb10c970eu)] - public void XxHash32_MatchesRustVectors(string value, uint expected) - { - Assert.Equal(expected, XxHash32.HashToUInt32(Encoding.UTF8.GetBytes(value))); - } -} diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/CredentialBoundsTests.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/CredentialBoundsTests.cs deleted file mode 100644 index 4fc2e6b306..0000000000 --- a/foreign/csharp/Iggy_SDK_Tests/VsrTests/CredentialBoundsTests.cs +++ /dev/null @@ -1,72 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using Apache.Iggy.Exceptions; -using Apache.Iggy.Vsr; - -namespace Apache.Iggy.Tests.VsrTests; - -/// -/// Mirrors validate_username / validate_password in -/// core/common/src/traits/binary_impls/mod.rs. -/// -public sealed class CredentialBoundsTests -{ - [Fact] - public void ValidateUsername_AcceptsTheServerBounds() - { - CredentialBounds.ValidateUsername(new string('a', CredentialBounds.MIN_USERNAME_LENGTH)); - CredentialBounds.ValidateUsername(new string('a', CredentialBounds.MAX_USERNAME_LENGTH)); - } - - [Fact] - public void ValidateUsername_RejectsOutOfBounds() - { - AssertRejects(VsrError.INVALID_USERNAME, - () => CredentialBounds.ValidateUsername(new string('a', CredentialBounds.MIN_USERNAME_LENGTH - 1))); - AssertRejects(VsrError.INVALID_USERNAME, - () => CredentialBounds.ValidateUsername(new string('a', CredentialBounds.MAX_USERNAME_LENGTH + 1))); - } - - [Fact] - public void ValidatePassword_AcceptsTheServerBounds() - { - CredentialBounds.ValidatePassword(new string('a', CredentialBounds.MIN_PASSWORD_LENGTH)); - CredentialBounds.ValidatePassword(new string('a', CredentialBounds.MAX_PASSWORD_LENGTH)); - } - - [Fact] - public void ValidatePassword_RejectsOutOfBounds() - { - AssertRejects(VsrError.INVALID_PASSWORD, - () => CredentialBounds.ValidatePassword(new string('a', CredentialBounds.MIN_PASSWORD_LENGTH - 1))); - AssertRejects(VsrError.INVALID_PASSWORD, - () => CredentialBounds.ValidatePassword(new string('a', CredentialBounds.MAX_PASSWORD_LENGTH + 1))); - } - - [Fact] - public void ValidatePassword_CountsUtf8BytesNotChars() - { - // 51 two-byte code points encode to 102 bytes, over the server's limit despite fitting in chars. - AssertRejects(VsrError.INVALID_PASSWORD, () => CredentialBounds.ValidatePassword(new string('ż', 51))); - } - - private static void AssertRejects(int statusCode, Action action) - { - Assert.Equal(statusCode, Assert.Throws(action).StatusCode); - } -} diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/LoginRegisterTests.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/LoginRegisterTests.cs deleted file mode 100644 index ea752241b2..0000000000 --- a/foreign/csharp/Iggy_SDK_Tests/VsrTests/LoginRegisterTests.cs +++ /dev/null @@ -1,165 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Buffers.Binary; -using System.Text; -using Apache.Iggy.Exceptions; -using Apache.Iggy.Utils; -using Apache.Iggy.Vsr; - -namespace Apache.Iggy.Tests.VsrTests; - -public sealed class LoginRegisterTests -{ - private static int AssertVersionInfo(byte[] body) - { - Assert.Equal(LoginRegister.PROTOCOL_VERSION, BinaryPrimitives.ReadUInt32LittleEndian(body)); - var position = 4; - position += AssertName(body, position, LoginRegister.SDK_NAME); - position += AssertName(body, position, SdkVersion.Value); - - return position; - } - - private static int AssertName(byte[] body, int position, string expected) - { - var length = body[position]; - Assert.Equal(Encoding.UTF8.GetByteCount(expected), length); - Assert.Equal(expected, Encoding.UTF8.GetString(body, position + 1, length)); - - return 1 + length; - } - - [Fact] - public void Serialize_WritesVersionInfoThenCredentialsThenContext() - { - var body = LoginRegister.Serialize("admin", "secret"); - - var position = AssertVersionInfo(body); - position += AssertName(body, position, "admin"); - position += AssertName(body, position, "secret"); - - Assert.Equal(0u, BinaryPrimitives.ReadUInt32LittleEndian(body.AsSpan(position))); - Assert.Equal(position + 4, body.Length); - } - - [Fact] - public void Serialize_AppendsTheClientContextWithAUInt32Length() - { - var body = LoginRegister.Serialize("admin", "secret", "ctx"); - - var position = AssertVersionInfo(body); - position += AssertName(body, position, "admin"); - position += AssertName(body, position, "secret"); - - Assert.Equal(3u, BinaryPrimitives.ReadUInt32LittleEndian(body.AsSpan(position))); - Assert.Equal("ctx", Encoding.UTF8.GetString(body, position + 4, 3)); - Assert.Equal(position + 7, body.Length); - } - - [Fact] - public void SerializeWithPersonalAccessToken_PutsTheTokenInTheCredentialSlot() - { - var body = LoginRegister.SerializeWithPersonalAccessToken("token"); - - var position = AssertVersionInfo(body); - position += AssertName(body, position, "token"); - - Assert.Equal(0u, BinaryPrimitives.ReadUInt32LittleEndian(body.AsSpan(position))); - Assert.Equal(position + 4, body.Length); - } - - [Fact] - public void Serialize_RejectsAnEmptyCredential() - { - var error = Assert.Throws(() => - LoginRegister.Serialize("admin", string.Empty)); - - Assert.Equal(VsrError.INVALID_PASSWORD, error.StatusCode); - } - - [Fact] - public void Serialize_RejectsACredentialAboveTheLengthPrefix() - { - var error = Assert.Throws(() => - LoginRegister.Serialize("admin", new string('x', 256))); - - Assert.Equal(VsrError.INVALID_PASSWORD, error.StatusCode); - } - - /// - /// The u8 length prefix is also guarded inside the writer, for the fields no credential check covers. - /// - [Fact] - public void SerializeWithPersonalAccessToken_RejectsATokenAboveTheLengthPrefix() - { - var error = Assert.Throws(() => - LoginRegister.SerializeWithPersonalAccessToken(new string('x', 256))); - - Assert.Equal(VsrError.INVALID_PERSONAL_ACCESS_TOKEN, error.StatusCode); - } - - [Fact] - public void SerializeWithPersonalAccessToken_RejectsAnEmptyToken() - { - var error = Assert.Throws(() => - LoginRegister.SerializeWithPersonalAccessToken(string.Empty)); - - Assert.Equal(VsrError.INVALID_PERSONAL_ACCESS_TOKEN, error.StatusCode); - } - - [Fact] - public void Deserialize_ReadsTheRegisterReply() - { - var body = VsrTestPayloads.Concat(VsrTestPayloads.UInt32(42), new byte[8], VsrTestPayloads.UInt32(10243), - [5], "0.8.0"u8.ToArray()); - BinaryPrimitives.WriteUInt64LittleEndian(body.AsSpan(4), 100); - - var response = LoginRegister.Deserialize(body); - - Assert.Equal(42u, response.UserId); - Assert.Equal(100UL, response.Session); - Assert.Equal(10243u, response.ServerProtocolVersion); - Assert.Equal("0.8.0", response.ServerVersion); - } - - [Fact] - public void Deserialize_TruncatedReplyIsInvalidFormat() - { - var body = VsrTestPayloads.Concat(VsrTestPayloads.UInt32(42), new byte[8], VsrTestPayloads.UInt32(10243), - [5], "0.8.0"u8.ToArray()); - - for (var length = 0; length < body.Length; length++) - { - var truncated = body[..length]; - var exception = Assert.Throws(() => - LoginRegister.Deserialize(truncated)); - - Assert.Equal(VsrError.INVALID_FORMAT, exception.StatusCode); - } - } - - [Fact] - public void Deserialize_EmptyReplyIsTheTerminalRegisterRejection() - { - var exception = Assert.Throws(() => - LoginRegister.Deserialize(ReadOnlySpan.Empty)); - - Assert.Equal(VsrError.INVALID_FORMAT, exception.StatusCode); - Assert.Contains("rejected the login", exception.Message); - } -} diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/ServerAddressTests.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/ServerAddressTests.cs deleted file mode 100644 index 7288b45170..0000000000 --- a/foreign/csharp/Iggy_SDK_Tests/VsrTests/ServerAddressTests.cs +++ /dev/null @@ -1,85 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using Apache.Iggy.Utils; - -namespace Apache.Iggy.Tests.VsrTests; - -/// -/// Mirrors core/sdk/src/leader_aware.rs test_is_same_address and -/// test_normalize_address: the leader redirect check has to agree with the Rust SDK, or the two -/// disagree on whether a roster entry names the node this connection is already on. -/// -public sealed class ServerAddressTests -{ - [Theory] - [InlineData("127.0.0.1:8090", "127.0.0.1:8090")] - [InlineData("localhost:8090", "127.0.0.1:8090")] - [InlineData("LOCALHOST:8090", "127.0.0.1:8090")] - [InlineData("[::1]:8090", "[::1]:8090")] - public void IsSame_MatchesEquivalentEndpoints(string first, string second) - { - Assert.True(ServerAddress.IsSame(first, second)); - Assert.True(ServerAddress.IsSame(second, first)); - } - - [Theory] - [InlineData("127.0.0.1:8090", "127.0.0.1:8091")] - [InlineData("192.168.1.1:8090", "127.0.0.1:8090")] - [InlineData("localhost:8090", "127.0.0.1:8091")] - [InlineData("iggy-1:8090", "iggy-2:8090")] - [InlineData("127.0.0.1:8090", "")] - public void IsSame_SeparatesDistinctEndpoints(string first, string second) - { - Assert.False(ServerAddress.IsSame(first, second)); - Assert.False(ServerAddress.IsSame(second, first)); - } - - [Theory] - [InlineData("localhost:8090", "127.0.0.1:8090")] - [InlineData("LOCALHOST:8090", "127.0.0.1:8090")] - [InlineData("[::]:8090", "[::1]:8090")] - [InlineData("0.0.0.0:8090", "127.0.0.1:8090")] - [InlineData("my-localhost-1:8090", "my-localhost-1:8090")] - public void Normalize_ResolvesHostAliases(string address, string expected) - { - Assert.Equal(expected, ServerAddress.Normalize(address)); - } - - /// - /// A host name that merely contains an alias is a different node, and a server bound to the unspecified - /// address answers on the loopback one. - /// - [Theory] - [InlineData("my-localhost-1:8090", "127.0.0.1:8090", false)] - [InlineData("localhost.example.com:8090", "127.0.0.1:8090", false)] - [InlineData("0.0.0.0:8090", "127.0.0.1:8090", true)] - [InlineData("[::]:8090", "[::1]:8090", true)] - [InlineData("0.0.0.0:8090", "[::1]:8090", false)] - public void IsSame_ResolvesHostAliasesWithoutSubstringMatching(string first, string second, bool same) - { - Assert.Equal(same, ServerAddress.IsSame(first, second)); - Assert.Equal(same, ServerAddress.IsSame(second, first)); - } - - [Fact] - public void IsSame_FallsBackToNormalizedStringsForUnparsableAddresses() - { - Assert.True(ServerAddress.IsSame("Iggy-Node:8090", "iggy-node:8090")); - Assert.False(ServerAddress.IsSame("iggy-node:8090", "iggy-node:8091")); - } -} diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/SyncConsumerGroupTests.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/SyncConsumerGroupTests.cs deleted file mode 100644 index 1d185668b5..0000000000 --- a/foreign/csharp/Iggy_SDK_Tests/VsrTests/SyncConsumerGroupTests.cs +++ /dev/null @@ -1,93 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Buffers.Binary; -using Apache.Iggy.Exceptions; -using Apache.Iggy.Vsr; - -namespace Apache.Iggy.Tests.VsrTests; - -public sealed class SyncConsumerGroupTests -{ - [Fact] - public void Decode_ReadsGenerationAndPartitions() - { - var assignment = SyncConsumerGroupAssignment.Decode(Encode(7, [0, 2, 4])); - - Assert.Equal(7ul, assignment.Generation); - Assert.Equal([0, 2, 4], [.. assignment.Partitions]); - } - - [Fact] - public void Decode_ReadsEmptyAssignment() - { - var assignment = SyncConsumerGroupAssignment.Decode(Encode(1, [])); - - Assert.Equal(1ul, assignment.Generation); - Assert.Empty(assignment.Partitions); - } - - [Fact] - public void Decode_IgnoresTrailingBytes() - { - var body = Encode(1, [3]).Concat(new byte[8]).ToArray(); - - Assert.Equal([3], [.. SyncConsumerGroupAssignment.Decode(body).Partitions]); - } - - [Fact] - public void Decode_TruncatedBodyThrows() - { - var body = Encode(7, [1, 2]); - for (var length = 0; length < body.Length; length++) - { - var truncated = body[..length]; - var error = Assert.Throws(() => - SyncConsumerGroupAssignment.Decode(truncated)); - - Assert.Equal(VsrError.INVALID_COMMAND, error.StatusCode); - } - } - - /// - /// A count the body cannot back must fail rather than allocate for it: the value is attacker-reachable - /// through a corrupted frame. - /// - [Fact] - public void Decode_ImplausiblePartitionCountThrows() - { - var body = new byte[12]; - BinaryPrimitives.WriteUInt32LittleEndian(body.AsSpan(8, 4), uint.MaxValue); - - var error = Assert.Throws(() => SyncConsumerGroupAssignment.Decode(body)); - - Assert.Equal(VsrError.INVALID_COMMAND, error.StatusCode); - } - - private static byte[] Encode(ulong generation, uint[] partitions) - { - var body = new byte[12 + partitions.Length * 4]; - BinaryPrimitives.WriteUInt64LittleEndian(body.AsSpan(0, 8), generation); - BinaryPrimitives.WriteUInt32LittleEndian(body.AsSpan(8, 4), (uint)partitions.Length); - for (var i = 0; i < partitions.Length; i++) - { - BinaryPrimitives.WriteUInt32LittleEndian(body.AsSpan(12 + i * 4, 4), partitions[i]); - } - - return body; - } -} diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrHeaderTests.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrHeaderTests.cs deleted file mode 100644 index 73eb1fbb4d..0000000000 --- a/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrHeaderTests.cs +++ /dev/null @@ -1,283 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Buffers.Binary; -using Apache.Iggy.Exceptions; -using Apache.Iggy.Utils; -using Apache.Iggy.Vsr; - -namespace Apache.Iggy.Tests.VsrTests; - -public sealed class VsrHeaderTests -{ - private static ConsensusSession BoundSession(ulong session = 5) - { - var consensusSession = new ConsensusSession(0x0102_0304_0506_0708); - consensusSession.Resolve(VsrOperation.Register); - consensusSession.Bind(session); - - return consensusSession; - } - - private static byte[] Encode(ConsensusSession session, int code, byte[] payload, out int totalSize) - { - var header = new byte[VsrHeader.HEADER_SIZE]; - totalSize = VsrHeader.EncodeRequestHeader(header, session, code, payload); - - return header; - } - - private static ulong ReadUInt64(byte[] header, int offset) - { - return BinaryPrimitives.ReadUInt64LittleEndian(header.AsSpan(offset)); - } - - private static uint ReadUInt32(byte[] header, int offset) - { - return BinaryPrimitives.ReadUInt32LittleEndian(header.AsSpan(offset)); - } - - /// - /// The client no longer inspects a payload to route, so a partition-plane body it cannot interpret is - /// passed through for the server to resolve instead of failing at encode time. - /// - [Fact] - public void Encode_UnroutablePartitionPayloadIsPassedThroughToTheServer() - { - var session = BoundSession(); - var payload = VsrTestPayloads.ConsumerOffset(VsrTestPayloads.NumericIdentifier(4), - VsrTestPayloads.NumericIdentifier(5), null); - var header = new byte[VsrHeader.HEADER_SIZE]; - - var totalSize = VsrHeader.EncodeRequestHeader(header, session, - CommandCodes.STORE_CONSUMER_OFFSET_CODE, payload); - - Assert.Equal(VsrHeader.HEADER_SIZE + payload.Length, totalSize); - Assert.Equal((byte)VsrOperation.StoreConsumerOffset, header[VsrHeader.REQUEST_OPERATION_OFFSET]); - Assert.True(session.IsBound); - } - - [Fact] - public void Encode_RegisterUsesZeroRequestAndSession() - { - var session = new ConsensusSession(7); - var payload = LoginRegister.Serialize("admin", "secret"); - - var header = Encode(session, CommandCodes.LOGIN_REGISTER_CODE, payload, out var totalSize); - - Assert.Equal(VsrHeader.HEADER_SIZE + payload.Length, totalSize); - Assert.Equal((uint)totalSize, ReadUInt32(header, VsrHeader.SIZE_OFFSET)); - Assert.Equal((byte)Command2.Request, header[VsrHeader.COMMAND_OFFSET]); - Assert.Equal((byte)VsrOperation.Register, header[VsrHeader.REQUEST_OPERATION_OFFSET]); - Assert.Equal(0UL, ReadUInt64(header, VsrHeader.REQUEST_ID_OFFSET)); - Assert.Equal(0UL, ReadUInt64(header, VsrHeader.REQUEST_SESSION_OFFSET)); - Assert.Equal(0UL, ReadUInt64(header, VsrHeader.REQUEST_TIMESTAMP_OFFSET)); - Assert.Equal(7UL, ReadUInt64(header, VsrHeader.REQUEST_CLIENT_OFFSET)); - Assert.Equal(0UL, ReadUInt64(header, VsrHeader.REQUEST_CLIENT_OFFSET + 8)); - } - - [Fact] - public void Encode_WritesClientIdAsTwoLittleEndianHalvesLowFirst() - { - var session = new ConsensusSession(new UInt128(0xAABB_CCDD_EEFF_0011, 0x1122_3344_5566_7788)); - - var header = Encode(session, CommandCodes.PING_CODE, [], out _); - - Assert.Equal(0x1122_3344_5566_7788UL, ReadUInt64(header, VsrHeader.REQUEST_CLIENT_OFFSET)); - Assert.Equal(0xAABB_CCDD_EEFF_0011UL, ReadUInt64(header, VsrHeader.REQUEST_CLIENT_OFFSET + 8)); - } - - [Fact] - public void Encode_NonReplicatedDoesNotAdvanceCounterAndCarriesCodeInReserved() - { - var session = BoundSession(); - - var header = Encode(session, CommandCodes.PING_CODE, [], out _); - - Assert.Equal((byte)VsrOperation.NonReplicated, header[VsrHeader.REQUEST_OPERATION_OFFSET]); - Assert.Equal(1UL, ReadUInt64(header, VsrHeader.REQUEST_ID_OFFSET)); - Assert.Equal(1UL, session.RequestCounter); - Assert.Equal(5UL, ReadUInt64(header, VsrHeader.REQUEST_SESSION_OFFSET)); - Assert.Equal((uint)CommandCodes.PING_CODE, ReadUInt32(header, VsrHeader.REQUEST_RESERVED_OFFSET)); - } - - [Fact] - public void Encode_UnknownCodeRidesNonReplicated() - { - var session = BoundSession(); - - var header = Encode(session, 9999, [], out _); - - Assert.Equal((byte)VsrOperation.NonReplicated, header[VsrHeader.REQUEST_OPERATION_OFFSET]); - Assert.Equal(9999u, ReadUInt32(header, VsrHeader.REQUEST_RESERVED_OFFSET)); - } - - [Fact] - public void Encode_NonReplicatedWithoutSessionSendsSessionZero() - { - var session = new ConsensusSession(1); - - var header = Encode(session, CommandCodes.PING_CODE, [], out _); - - Assert.Equal(0UL, ReadUInt64(header, VsrHeader.REQUEST_SESSION_OFFSET)); - } - - [Fact] - public void Encode_MetadataAdvancesTheCounter() - { - var session = BoundSession(); - - var header = Encode(session, CommandCodes.CREATE_STREAM_CODE, [1, 2, 3], out _); - - Assert.Equal((byte)VsrOperation.CreateStream, header[VsrHeader.REQUEST_OPERATION_OFFSET]); - Assert.Equal(1UL, ReadUInt64(header, VsrHeader.REQUEST_ID_OFFSET)); - Assert.Equal(2UL, session.RequestCounter); - Assert.Equal(0u, ReadUInt32(header, VsrHeader.REQUEST_RESERVED_OFFSET)); - } - - [Fact] - public void Encode_LogoutAdvancesTheCounter() - { - var session = BoundSession(); - - var header = Encode(session, CommandCodes.LOGOUT_USER_CODE, [], out _); - - Assert.Equal((byte)VsrOperation.Logout, header[VsrHeader.REQUEST_OPERATION_OFFSET]); - Assert.Equal(1UL, ReadUInt64(header, VsrHeader.REQUEST_ID_OFFSET)); - Assert.Equal(2UL, session.RequestCounter); - } - - [Fact] - public void Encode_PartitionOpDoesNotAdvanceTheCounter() - { - var session = BoundSession(); - var payload = VsrTestPayloads.SendMessagesToPartition(2, 3, 4); - - var header = Encode(session, CommandCodes.SEND_MESSAGES_CODE, payload, out _); - - Assert.Equal((byte)VsrOperation.SendMessages, header[VsrHeader.REQUEST_OPERATION_OFFSET]); - Assert.Equal(1UL, ReadUInt64(header, VsrHeader.REQUEST_ID_OFFSET)); - Assert.Equal(1UL, session.RequestCounter); - } - - [Fact] - public void Encode_ReplicatedOpWithoutSessionIsUnauthenticated() - { - var session = new ConsensusSession(1); - - var exception = Assert.Throws(() => - Encode(session, CommandCodes.CREATE_STREAM_CODE, [1], out _)); - - Assert.Equal(VsrError.UNAUTHENTICATED, exception.StatusCode); - Assert.Equal(1UL, session.RequestCounter); - } - - [Fact] - public void Encode_ClearsStaleBytesFromAReusedBuffer() - { - var header = new byte[VsrHeader.HEADER_SIZE]; - Array.Fill(header, (byte)0xFF); - var session = BoundSession(); - - VsrHeader.EncodeRequestHeader(header, session, CommandCodes.CREATE_STREAM_CODE, [1]); - - Assert.Equal(0u, ReadUInt32(header, VsrHeader.REQUEST_RESERVED_OFFSET)); - Assert.All(header[..VsrHeader.SIZE_OFFSET], stale => Assert.Equal(0, stale)); - } - - [Fact] - public void Encode_RejectsAShortBuffer() - { - var session = BoundSession(); - - Assert.Throws(() => - VsrHeader.EncodeRequestHeader(new byte[VsrHeader.HEADER_SIZE - 1], session, CommandCodes.PING_CODE, [])); - } - - [Fact] - public void PeekCommand_MapsOnlyTheClientVisibleFrames() - { - var header = new byte[VsrHeader.HEADER_SIZE]; - - header[VsrHeader.COMMAND_OFFSET] = (byte)Command2.Reply; - Assert.Equal(Command2.Reply, VsrHeader.PeekCommand(header)); - - header[VsrHeader.COMMAND_OFFSET] = (byte)Command2.Eviction; - Assert.Equal(Command2.Eviction, VsrHeader.PeekCommand(header)); - - header[VsrHeader.COMMAND_OFFSET] = 6; - Assert.Equal(Command2.Reserved, VsrHeader.PeekCommand(header)); - } - - [Fact] - public void ReadReplyOperation_RejectsAnUnknownDiscriminant() - { - var header = new byte[VsrHeader.HEADER_SIZE]; - header[VsrHeader.REPLY_OPERATION_OFFSET] = 200; - - var exception = Assert.Throws(() => VsrHeader.ReadReplyOperation(header)); - - Assert.Equal(VsrError.INVALID_COMMAND, exception.StatusCode); - } - - [Fact] - public void ReadEviction_ReadsReasonAndProtocolWindow() - { - var header = new byte[VsrHeader.HEADER_SIZE]; - header[VsrHeader.EVICTION_REASON_OFFSET] = (byte)EvictionReason.IncompatibleProtocol; - BinaryPrimitives.WriteUInt32LittleEndian(header.AsSpan(VsrHeader.EVICTION_PROTOCOL_VERSION_OFFSET), 10243); - BinaryPrimitives.WriteUInt32LittleEndian(header.AsSpan(VsrHeader.EVICTION_PROTOCOL_VERSION_MIN_OFFSET), 10240); - - var eviction = VsrHeader.ReadEviction(header); - - Assert.Equal(EvictionReason.IncompatibleProtocol, eviction.Reason); - Assert.Equal(10243u, eviction.ServerProtocolVersion); - Assert.Equal(10240u, eviction.ServerProtocolVersionMin); - } - - /// - /// An undecodable reason stays null rather than collapsing onto a named one, and grades through the - /// shared grader's catch-all like every reason the Rust mapping does not name. - /// - [Fact] - public void ReadEviction_UnknownReasonDecodesAsUndecodable() - { - var header = new byte[VsrHeader.HEADER_SIZE]; - header[VsrHeader.EVICTION_REASON_OFFSET] = 200; - - Assert.Null(VsrHeader.ReadEviction(header).Reason); - Assert.Equal(VsrError.INVALID_COMMAND, - Assert.IsType(VsrReplyDecoder.ToException(VsrHeader.ReadEviction(header))) - .StatusCode); - } - - /// - /// Reason 0 is the Reserved sentinel the server rejects on the wire, so it decodes as an unrecognized - /// reason rather than as one. Rust grades it through the same catch-all. - /// - [Fact] - public void ReadEviction_ReservedReasonDecodesAsUnknown() - { - var header = new byte[VsrHeader.HEADER_SIZE]; - header[VsrHeader.EVICTION_REASON_OFFSET] = (byte)EvictionReason.Reserved; - - Assert.Null(VsrHeader.ReadEviction(header).Reason); - Assert.Equal(VsrError.INVALID_COMMAND, - Assert.IsType(VsrReplyDecoder.ToException(VsrHeader.ReadEviction(header))) - .StatusCode); - } -} diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrOperationTests.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrOperationTests.cs deleted file mode 100644 index bd5686e031..0000000000 --- a/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrOperationTests.cs +++ /dev/null @@ -1,167 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using Apache.Iggy.Utils; -using Apache.Iggy.Exceptions; -using Apache.Iggy.Vsr; - -namespace Apache.Iggy.Tests.VsrTests; - -public sealed class VsrOperationTests -{ - [Theory] - [InlineData(CommandCodes.LOGOUT_USER_CODE, (byte)VsrOperation.Logout)] - [InlineData(CommandCodes.CREATE_USER_CODE, (byte)VsrOperation.CreateUser)] - [InlineData(CommandCodes.DELETE_USER_CODE, (byte)VsrOperation.DeleteUser)] - [InlineData(CommandCodes.UPDATE_USER_CODE, (byte)VsrOperation.UpdateUser)] - [InlineData(CommandCodes.UPDATE_PERMISSIONS_CODE, (byte)VsrOperation.UpdatePermissions)] - [InlineData(CommandCodes.CHANGE_PASSWORD_CODE, (byte)VsrOperation.ChangePassword)] - [InlineData(CommandCodes.CREATE_PERSONAL_ACCESS_TOKEN_CODE, (byte)VsrOperation.CreatePersonalAccessToken)] - [InlineData(CommandCodes.DELETE_PERSONAL_ACCESS_TOKEN_CODE, (byte)VsrOperation.DeletePersonalAccessToken)] - [InlineData(CommandCodes.SEND_MESSAGES_CODE, (byte)VsrOperation.SendMessages)] - [InlineData(CommandCodes.STORE_CONSUMER_OFFSET_CODE, (byte)VsrOperation.StoreConsumerOffset)] - [InlineData(CommandCodes.DELETE_CONSUMER_OFFSET_CODE, (byte)VsrOperation.DeleteConsumerOffset)] - [InlineData(CommandCodes.STORE_CONSUMER_OFFSET_2_CODE, (byte)VsrOperation.StoreConsumerOffset2)] - [InlineData(CommandCodes.DELETE_CONSUMER_OFFSET_2_CODE, (byte)VsrOperation.DeleteConsumerOffset2)] - [InlineData(CommandCodes.CREATE_STREAM_CODE, (byte)VsrOperation.CreateStream)] - [InlineData(CommandCodes.DELETE_STREAM_CODE, (byte)VsrOperation.DeleteStream)] - [InlineData(CommandCodes.UPDATE_STREAM_CODE, (byte)VsrOperation.UpdateStream)] - [InlineData(CommandCodes.PURGE_STREAM_CODE, (byte)VsrOperation.PurgeStream)] - [InlineData(CommandCodes.CREATE_TOPIC_CODE, (byte)VsrOperation.CreateTopic)] - [InlineData(CommandCodes.DELETE_TOPIC_CODE, (byte)VsrOperation.DeleteTopic)] - [InlineData(CommandCodes.UPDATE_TOPIC_CODE, (byte)VsrOperation.UpdateTopic)] - [InlineData(CommandCodes.PURGE_TOPIC_CODE, (byte)VsrOperation.PurgeTopic)] - [InlineData(CommandCodes.CREATE_PARTITIONS_CODE, (byte)VsrOperation.CreatePartitions)] - [InlineData(CommandCodes.DELETE_PARTITIONS_CODE, (byte)VsrOperation.DeletePartitions)] - [InlineData(CommandCodes.DELETE_SEGMENTS_CODE, (byte)VsrOperation.DeleteSegments)] - [InlineData(CommandCodes.CREATE_CONSUMER_GROUP_CODE, (byte)VsrOperation.CreateConsumerGroup)] - [InlineData(CommandCodes.DELETE_CONSUMER_GROUP_CODE, (byte)VsrOperation.DeleteConsumerGroup)] - [InlineData(CommandCodes.JOIN_CONSUMER_GROUP_CODE, (byte)VsrOperation.JoinConsumerGroup)] - [InlineData(CommandCodes.LEAVE_CONSUMER_GROUP_CODE, (byte)VsrOperation.LeaveConsumerGroup)] - public void ForCode_MapsReplicatedCommands(int code, byte expected) - { - Assert.Equal((VsrOperation)expected, VsrOperations.ForCode(code)); - } - - [Theory] - [InlineData(CommandCodes.PING_CODE)] - [InlineData(CommandCodes.POLL_MESSAGES_CODE)] - [InlineData(CommandCodes.GET_STREAM_CODE)] - [InlineData(CommandCodes.GET_CONSUMER_OFFSET_CODE)] - [InlineData(9999)] - public void ForCode_ReadsAndUnknownCodesRideNonReplicated(int code) - { - Assert.Equal(VsrOperation.NonReplicated, VsrOperations.ForCode(code)); - } - - [Theory] - [InlineData(CommandCodes.LOGIN_REGISTER_CODE)] - [InlineData(CommandCodes.LOGIN_REGISTER_WITH_PAT_CODE)] - public void ForCode_MapsTheRegisterHandshake(int code) - { - Assert.Equal(VsrOperation.Register, VsrOperations.ForCode(code)); - } - - /// - /// A classic login sent as a consensus request would ride NonReplicated and look like it worked while no - /// session is ever bound, so the classification rejects it instead. - /// - [Theory] - [InlineData(CommandCodes.LOGIN_USER_CODE)] - [InlineData(CommandCodes.LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE)] - public void ForCode_RejectsLoginCodes(int code) - { - var error = Assert.Throws(() => VsrOperations.ForCode(code)); - - Assert.Equal(VsrError.INVALID_COMMAND, error.StatusCode); - } - - [Fact] - public void Classification_MatchesTheServerSideRanges() - { - Assert.True(VsrOperation.CreateTopicWithAssignments.IsInternal()); - Assert.True(VsrOperation.CreateTopicWithAssignments.IsMetadata()); - Assert.True(VsrOperation.CreateStream.IsMetadata()); - Assert.False(VsrOperation.CreateStream.IsPartition()); - Assert.True(VsrOperation.SendMessages.IsPartition()); - Assert.False(VsrOperation.SendMessages.IsMetadata()); - - // Resolved server-side to an internal truncate, so it is neither plane despite carrying a namespace. - Assert.False(VsrOperation.DeleteSegments.IsPartition()); - Assert.False(VsrOperation.DeleteSegments.IsMetadata()); - } - - [Fact] - public void IsResultFramed_CoversMetadataAndConsumerOffsetsOnly() - { - Assert.True(VsrOperation.CreateStream.IsResultFramed()); - Assert.True(VsrOperation.StoreConsumerOffset.IsResultFramed()); - Assert.True(VsrOperation.DeleteConsumerOffset2.IsResultFramed()); - Assert.False(VsrOperation.SendMessages.IsResultFramed()); - Assert.False(VsrOperation.NonReplicated.IsResultFramed()); - Assert.False(VsrOperation.Register.IsResultFramed()); - Assert.False(VsrOperation.Logout.IsResultFramed()); - } - - [Fact] - public void IsKnown_RejectsUndefinedDiscriminants() - { - Assert.True(VsrOperations.IsKnown((byte)VsrOperation.SendMessages)); - Assert.False(VsrOperations.IsKnown(163)); - Assert.False(VsrOperations.IsKnown(200)); - } - - /// Every arm of the control-plane table: shard 0 owns these, so a miss would route to a data shard. - [Theory] - [InlineData((byte)VsrOperation.CreateStream)] - [InlineData((byte)VsrOperation.UpdateStream)] - [InlineData((byte)VsrOperation.DeleteStream)] - [InlineData((byte)VsrOperation.PurgeStream)] - [InlineData((byte)VsrOperation.CreateTopic)] - [InlineData((byte)VsrOperation.UpdateTopic)] - [InlineData((byte)VsrOperation.DeleteTopic)] - [InlineData((byte)VsrOperation.PurgeTopic)] - [InlineData((byte)VsrOperation.CreatePartitions)] - [InlineData((byte)VsrOperation.DeletePartitions)] - [InlineData((byte)VsrOperation.CreateConsumerGroup)] - [InlineData((byte)VsrOperation.DeleteConsumerGroup)] - [InlineData((byte)VsrOperation.CreateUser)] - [InlineData((byte)VsrOperation.UpdateUser)] - [InlineData((byte)VsrOperation.DeleteUser)] - [InlineData((byte)VsrOperation.ChangePassword)] - [InlineData((byte)VsrOperation.UpdatePermissions)] - [InlineData((byte)VsrOperation.CreatePersonalAccessToken)] - [InlineData((byte)VsrOperation.DeletePersonalAccessToken)] - [InlineData((byte)VsrOperation.JoinConsumerGroup)] - [InlineData((byte)VsrOperation.LeaveConsumerGroup)] - public void IsMetadata_CoversTheWholeControlPlane(byte operation) - { - Assert.True(((VsrOperation)operation).IsMetadata()); - } - - [Theory] - [InlineData((byte)VsrOperation.SendMessages)] - [InlineData((byte)VsrOperation.StoreConsumerOffset)] - [InlineData((byte)VsrOperation.DeleteConsumerOffset2)] - [InlineData((byte)VsrOperation.DeleteSegments)] - [InlineData((byte)VsrOperation.NonReplicated)] - [InlineData((byte)VsrOperation.Logout)] - public void IsMetadata_LeavesTheDataPlaneOut(byte operation) - { - Assert.False(((VsrOperation)operation).IsMetadata()); - } -} diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrProtocolDriftTests.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrProtocolDriftTests.cs deleted file mode 100644 index afc9a6b7f5..0000000000 --- a/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrProtocolDriftTests.cs +++ /dev/null @@ -1,810 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Buffers.Binary; -using System.Globalization; -using System.Reflection; -using System.Text.RegularExpressions; -using Apache.Iggy.Exceptions; -using Apache.Iggy.Utils; -using Apache.Iggy.Vsr; - -namespace Apache.Iggy.Tests.VsrTests; - -/// -/// Asserts the .NET VSR constants still match the Rust wire definitions they were ported from. The unit -/// tests above pin the .NET side against itself; this one pins it against -/// core/binary_protocol/, so a Rust-side change fails here instead of in production framing. -/// Skipped when the Rust sources are not on disk (packaged SDK builds). -/// -public sealed class VsrProtocolDriftTests -{ - private const string HeaderPath = "core/binary_protocol/src/consensus/header.rs"; - private const string CommandPath = "core/binary_protocol/src/consensus/command.rs"; - private const string OperationPath = "core/binary_protocol/src/consensus/operation.rs"; - private const string CodesPath = "core/binary_protocol/src/codes.rs"; - private const string DispatchPath = "core/binary_protocol/src/dispatch.rs"; - private const string ErrorPath = "core/common/src/error/iggy_error.rs"; - private const string ManifestPath = "core/binary_protocol/Cargo.toml"; - private const string EvictionGraderPath = "core/common/src/error/eviction.rs"; - private const string ReplyResultPath = "core/binary_protocol/src/consensus/reply_result.rs"; - private const string CredentialDefaultsPath = "core/common/src/http/users/defaults.rs"; - - private const string LoginRegisterResponsePath = - "core/binary_protocol/src/responses/users/login_register.rs"; - - private const string SyncConsumerGroupResponsePath = - "core/binary_protocol/src/responses/consumer_groups/sync_consumer_group.rs"; - - /// - /// Codes the SDK deliberately does not resolve the way the Rust dispatch table declares them, with the - /// reason each one is exempt. Anything not listed here must agree with the table. - /// - private static readonly Dictionary CodeMappingExceptions = new() - { - // Non-replicated in the table because the legacy transport routes them; under VSR they are the - // consensus handshake itself and carry their own operations. - ["LOGIN_REGISTER_CODE"] = VsrOperation.Register, - ["LOGIN_REGISTER_WITH_PAT_CODE"] = VsrOperation.Register, - ["LOGOUT_USER_CODE"] = VsrOperation.Logout, - - // VSR has no legacy login. ForCode throws rather than encoding a request that would look like a - // working login while binding no session; null means "asserted to throw". - ["LOGIN_USER_CODE"] = null, - ["LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE"] = null - }; - - [Fact] - public void RequestHeaderOffsets_MatchTheRustLayout() - { - IReadOnlyDictionary offsets = RustStruct.Offsets(ReadRustSource(HeaderPath), "RequestHeader"); - - Assert.Equal(VsrHeader.SIZE_OFFSET, offsets["size"]); - Assert.Equal(VsrHeader.COMMAND_OFFSET, offsets["command"]); - Assert.Equal(VsrHeader.REQUEST_CLIENT_OFFSET, offsets["client"]); - Assert.Equal(VsrHeader.REQUEST_TIMESTAMP_OFFSET, offsets["timestamp"]); - Assert.Equal(VsrHeader.REQUEST_ID_OFFSET, offsets["request"]); - Assert.Equal(VsrHeader.REQUEST_OPERATION_OFFSET, offsets["operation"]); - Assert.Equal(VsrHeader.REQUEST_SESSION_OFFSET, offsets["session"]); - Assert.Equal(VsrHeader.REQUEST_RESERVED_OFFSET, offsets["reserved"]); - - // A routing group reintroduced on the client header would shift every field - // after it and silently reinstate a derivation this SDK no longer performs. - Assert.DoesNotContain("namespace", offsets.Keys); - Assert.DoesNotContain("group", offsets.Keys); - } - - [Fact] - public void ReplyHeaderOffsets_MatchTheRustLayout() - { - IReadOnlyDictionary offsets = RustStruct.Offsets(ReadRustSource(HeaderPath), "ReplyHeader"); - - Assert.Equal(VsrHeader.SIZE_OFFSET, offsets["size"]); - Assert.Equal(VsrHeader.COMMAND_OFFSET, offsets["command"]); - Assert.Equal(VsrHeader.REPLY_OPERATION_OFFSET, offsets["operation"]); - Assert.Equal(VsrHeader.REPLY_STATUS_OFFSET, offsets["status"]); - - Assert.DoesNotContain("namespace", offsets.Keys); - Assert.DoesNotContain("group", offsets.Keys); - } - - [Fact] - public void EvictionHeaderOffsets_MatchTheRustLayout() - { - IReadOnlyDictionary offsets = RustStruct.Offsets(ReadRustSource(HeaderPath), "EvictionHeader"); - - Assert.Equal(VsrHeader.COMMAND_OFFSET, offsets["command"]); - Assert.Equal(VsrHeader.EVICTION_CLIENT_OFFSET, offsets["client"]); - Assert.Equal(VsrHeader.EVICTION_PROTOCOL_VERSION_OFFSET, offsets["server_protocol_version"]); - Assert.Equal(VsrHeader.EVICTION_PROTOCOL_VERSION_MIN_OFFSET, offsets["server_protocol_version_min"]); - Assert.Equal(VsrHeader.EVICTION_REASON_OFFSET, offsets["reason"]); - } - - [Fact] - public void HeaderSize_MatchesTheRustConstant() - { - var source = ReadRustSource(HeaderPath); - var declared = Regex.Match(source, @"pub const HEADER_SIZE: usize = (\d+);"); - - Assert.True(declared.Success, "HEADER_SIZE is no longer declared in header.rs."); - Assert.Equal(VsrHeader.HEADER_SIZE, int.Parse(declared.Groups[1].Value, CultureInfo.InvariantCulture)); - Assert.Equal(VsrHeader.HEADER_SIZE, RustStruct.Size(source, "RequestHeader")); - Assert.Equal(VsrHeader.HEADER_SIZE, RustStruct.Size(source, "ReplyHeader")); - Assert.Equal(VsrHeader.HEADER_SIZE, RustStruct.Size(source, "EvictionHeader")); - } - - [Fact] - public void Command2Discriminants_MatchTheRustEnum() - { - IReadOnlyDictionary rust = RustEnum.Discriminants(ReadRustSource(CommandPath), "Command2"); - - AssertSubsetMatches(rust); - } - - [Fact] - public void EvictionReasons_MatchTheRustEnumExactly() - { - IReadOnlyDictionary rust = RustEnum.Discriminants(ReadRustSource(HeaderPath), "EvictionReason"); - - AssertExactMatch(rust); - } - - [Fact] - public void Operations_MatchTheRustEnumExactly() - { - IReadOnlyDictionary rust = RustEnum.Discriminants(ReadRustSource(OperationPath), "Operation"); - - AssertExactMatch(rust); - } - - [Fact] - public void ProtocolVersion_MatchesTheBinaryProtocolCrateVersion() - { - var manifest = ReadRustSource(ManifestPath); - var declared = Regex.Match(manifest, @"^version\s*=\s*""(\d+)\.(\d+)\.(\d+)", RegexOptions.Multiline); - - Assert.True(declared.Success, "iggy_binary_protocol no longer declares a semver version."); - Assert.Equal(LoginRegister.PROTOCOL_VERSION_MAJOR, Group(declared, 1)); - Assert.Equal(LoginRegister.PROTOCOL_VERSION_MINOR, Group(declared, 2)); - Assert.Equal(LoginRegister.PROTOCOL_VERSION_PATCH, Group(declared, 3)); - } - - /// - /// Pins against codes.rs as a set of values, the way the Node mirror - /// does. A name-keyed lookup alone lets a command added upstream pass unnoticed, because the constant it - /// would have to match does not exist here yet. Names are still compared where both sides declare them, - /// which catches a renumber that keeps the set size intact. - /// - [Fact] - public void CommandCodes_MatchTheRustCodes() - { - var rust = Regex.Matches(ReadRustSource(CodesPath), @"pub const (\w+_CODE): u32 = (\d+);") - .ToDictionary(code => code.Groups[1].Value, code => Group(code, 2)); - Assert.NotEmpty(rust); - - var dotnet = typeof(CommandCodes) - .GetFields(BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Static) - .Where(field => field is { IsLiteral: true, FieldType.Name: nameof(Int32) }) - .ToDictionary(field => field.Name, field => (int)field.GetRawConstantValue()!); - - var mismatches = rust - .Where(code => !dotnet.ContainsValue(code.Value)) - .Select(code => $"{code.Key} = {code.Value} is declared in Rust and missing from CommandCodes") - .Concat(dotnet - .Where(code => !rust.ContainsValue(code.Value)) - .Select(code => $"{code.Key} = {code.Value} is declared in CommandCodes and missing from Rust")) - .Concat(rust - .Where(code => dotnet.TryGetValue(code.Key, out var actual) && actual != code.Value) - .Select(code => $"{code.Key}: rust {code.Value}, .NET {dotnet[code.Key]}")) - .ToList(); - - Assert.Empty(mismatches); - } - - /// - /// Pins against the Rust dispatch table. The .NET mapping is a hand - /// written switch whose default arm is , so a command that gains - /// replication upstream would otherwise keep encoding as non-replicated here: it would apply on the node - /// that received it and never reach the others, diverging the replicas. This is the sweep - /// no_replicated_command_ever_resolves_to_non_replicated performs on the Rust side. - /// - [Fact] - public void CommandOperationTable_MatchesTheRustDispatchTable() - { - var source = ReadRustSource(DispatchPath); - - // Table entries wrap across lines once the arguments are long enough, so the separators have to match - // arbitrary whitespace rather than a single line. - var replicated = Regex.Matches(source, - @"CommandMeta::replicated\(\s*(\w+_CODE)\s*,\s*""[^""]*""\s*,\s*Operation::(\w+)\s*,?\s*\)"); - var nonReplicated = Regex.Matches(source, @"CommandMeta::non_replicated\(\s*(\w+_CODE)\s*,"); - - Assert.NotEmpty(replicated); - Assert.NotEmpty(nonReplicated); - - var mismatches = new List(); - var matched = 0; - - foreach (Match entry in replicated) - { - var codeName = entry.Groups[1].Value; - if (CommandCode(codeName) is not { } code) - { - // Skipping here would exempt the exact command this test exists to catch: one that gains - // replication upstream while the .NET side has no code for it, so every call encodes as - // non-replicated and applies on one node only. - mismatches.Add($"{codeName}: rust replicates it, absent from .NET CommandCodes"); - - continue; - } - - matched++; - if (!Enum.TryParse(entry.Groups[2].Value, out var expected)) - { - mismatches.Add($"{codeName}: rust maps to Operation::{entry.Groups[2].Value}, absent from .NET"); - - continue; - } - - var actual = ResolveOperation(codeName, code); - if (actual != expected) - { - mismatches.Add($"{codeName}: rust {expected}, .NET {ActualText(actual)}"); - } - } - - foreach (Match entry in nonReplicated) - { - var codeName = entry.Groups[1].Value; - if (CommandCode(codeName) is not { } code) - { - continue; - } - - matched++; - var expected = CodeMappingExceptions.TryGetValue(codeName, out var exempt) - ? exempt - : VsrOperation.NonReplicated; - var actual = ResolveOperation(codeName, code); - if (actual != expected) - { - mismatches.Add($"{codeName}: expected {ActualText(expected)}, .NET {ActualText(actual)}"); - } - } - - Assert.Empty(mismatches); - Assert.True(matched > 40, $"Only {matched} dispatch entries were matched by name; the Rust naming drifted."); - } - - /// - /// Pins against Operation::is_metadata. Metadata replies - /// lead their body with a committed result section, so an operation misclassified here has its rejection - /// entry decoded as payload and a refused command reads as a success. - /// - [Fact] - public void MetadataOperations_MatchTheRustClassifier() - { - var body = MatchesArmBody(ReadRustSource(OperationPath), "is_metadata"); - var rust = Regex.Matches(body, @"Self::(\w+)").Select(match => match.Groups[1].Value).ToHashSet(); - Assert.NotEmpty(rust); - - var mismatches = new List(); - foreach (VsrOperation operation in Enum.GetValues()) - { - // is_internal() short-circuits ahead of the match arm on both sides, so those members are metadata - // without appearing in the list. - var expected = rust.Contains(operation.ToString()) || operation.IsInternal(); - if (operation.IsMetadata() != expected) - { - mismatches.Add($"{operation}: rust {expected}, .NET {operation.IsMetadata()}"); - } - } - - Assert.Empty(mismatches); - } - - /// - /// Pins against Operation::is_result_framed. The Rust - /// side composes it from is_metadata plus its own partition-plane list, so pinning - /// is_metadata alone leaves the second half free to drift. An operation that gains result framing - /// upstream and not here has skip the strip, and a - /// committed rejection decodes as payload: a refused command reads as a success. - /// - [Fact] - public void ResultFramedOperations_MatchTheRustClassifier() - { - var body = MatchesArmBody(ReadRustSource(OperationPath), "is_result_framed"); - var rust = Regex.Matches(body, @"Self::(\w+)").Select(match => match.Groups[1].Value).ToHashSet(); - Assert.NotEmpty(rust); - - var mismatches = new List(); - foreach (VsrOperation operation in Enum.GetValues()) - { - // The Rust body is `is_metadata() || matches!(...)`, so the metadata members carry over without - // appearing in the list. - var expected = rust.Contains(operation.ToString()) || operation.IsMetadata(); - if (operation.IsResultFramed() != expected) - { - mismatches.Add($"{operation}: rust {expected}, .NET {operation.IsResultFramed()}"); - } - } - - Assert.Empty(mismatches); - } - - /// - /// Pins against the Rust error discriminants. These are copied by hand and drive - /// more than error text: TRANSIENT_NOT_COMMITTED and TRANSIENT_NOT_ACCEPTED are what tells a - /// same-session replay from a fresh-session reissue, so a renumber upstream would reissue a write whose - /// outcome is unknown under a client id the server cannot deduplicate against, applying it twice. - /// - [Fact] - public void ErrorCodes_MatchTheRustDiscriminants() - { - var rust = Regex.Matches(ReadRustSource(ErrorPath), @"^\s{4}(\w+)(?:\([^)]*\))?\s*=\s*(\d+),", - RegexOptions.Multiline) - .ToDictionary(match => Normalize(match.Groups[1].Value), - match => int.Parse(match.Groups[2].Value, CultureInfo.InvariantCulture)); - Assert.NotEmpty(rust); - - var mismatches = new List(); - var matched = 0; - - foreach (FieldInfo field in typeof(VsrError).GetFields(BindingFlags.NonPublic | BindingFlags.Static)) - { - if (!field.IsLiteral || field.FieldType != typeof(int)) - { - continue; - } - - if (!rust.TryGetValue(Normalize(field.Name), out var expected)) - { - mismatches.Add($"{field.Name}: no Rust variant of that name remains"); - - continue; - } - - matched++; - var actual = (int)field.GetValue(null)!; - if (actual != expected) - { - mismatches.Add($"{field.Name}: rust {expected}, .NET {actual}"); - } - } - - Assert.Empty(mismatches); - Assert.True(matched > 10, $"Only {matched} error codes were matched by name; the Rust naming drifted."); - } - - private static VsrOperation? ResolveOperation(string codeName, int code) - { - try - { - return VsrOperations.ForCode(code); - } - catch (IggyInvalidStatusCodeException) - { - // The legacy login codes reject rather than resolve; CodeMappingExceptions records that as null. - _ = codeName; - - return null; - } - } - - private static string ActualText(VsrOperation? operation) - { - return operation?.ToString() ?? "rejected"; - } - - private static int? CommandCode(string name) - { - var field = typeof(CommandCodes).GetField(name, - BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Static); - - return field is null ? null : (int)field.GetValue(null)!; - } - - /// Extracts the body of the matches! invocation inside the named method. - private static string MatchesArmBody(string source, string method) - { - var start = source.IndexOf($"fn {method}(", StringComparison.Ordinal); - Assert.True(start >= 0, $"Operation::{method} is gone from the Rust source."); - - var matches = source.IndexOf("matches!(", start, StringComparison.Ordinal); - Assert.True(matches >= 0, $"Operation::{method} no longer classifies with matches!."); - - var depth = 0; - for (var i = matches + "matches!".Length; i < source.Length; i++) - { - depth += source[i] switch - { - '(' => 1, - ')' => -1, - _ => 0 - }; - - if (depth == 0) - { - return source[matches..(i + 1)]; - } - } - - Assert.Fail($"Operation::{method} has an unbalanced matches! invocation."); - - return string.Empty; - } - - /// - /// Pins the eviction reason to error mapping against eviction_reason_to_error, which the Rust - /// module documents as the single grader both its callers share "so the mappings cannot drift apart". - /// The .NET decoder is a third copy outside that guarantee, and the status it produces is what a caller - /// branches on, so an arm that silently regrades here changes public behaviour in one SDK only. - /// - [Fact] - public void EvictionReasonErrors_MatchTheRustGrader() - { - IReadOnlyDictionary errorCodes = RustErrorCodes(); - var body = RustItem.Body(ReadRustSource(EvictionGraderPath), "pub fn eviction_reason_to_error("); - - // Arms list one or more reasons separated by `|`; IncompatibleProtocol is a block arm and is asserted - // separately below, on both of its branches. - var arms = Regex.Matches(body, - @"((?:\s*\|?\s*EvictionReason::\w+)+)\s*=>\s*IggyError::(\w+)"); - Assert.NotEmpty(arms); - - var expected = new Dictionary(StringComparer.Ordinal); - int? catchAll = null; - - foreach (Match arm in arms) - { - var code = errorCodes[Normalize(arm.Groups[2].Value)]; - foreach (Match reason in Regex.Matches(arm.Groups[1].Value, @"EvictionReason::(\w+)")) - { - expected[reason.Groups[1].Value] = code; - } - } - - var fallback = Regex.Match(body, @"_\s*=>\s*IggyError::(\w+)"); - Assert.True(fallback.Success, "eviction_reason_to_error no longer has a catch-all arm."); - catchAll = errorCodes[Normalize(fallback.Groups[1].Value)]; - - var mismatches = new List(); - - foreach (EvictionReason reason in Enum.GetValues()) - { - if (reason == EvictionReason.IncompatibleProtocol) - { - continue; - } - - var want = expected.TryGetValue(reason.ToString(), out var mapped) ? mapped : catchAll.Value; - var got = StatusOf(reason); - if (got != want) - { - mismatches.Add($"{reason}: rust {want}, .NET {got}"); - } - } - - // A reason this build cannot decode arrives as null and must grade like the Rust catch-all. - var unknown = StatusOf(null); - if (unknown != catchAll.Value) - { - mismatches.Add($": rust {catchAll.Value}, .NET {unknown}"); - } - - Assert.Empty(mismatches); - - // IncompatibleProtocol reports the window, except when it is degenerate - a zero minimum or an - // inverted range - which falls back to re-authentication. - Assert.Equal(errorCodes["INCOMPATIBLEPROTOCOLVERSION"], - StatusOf(EvictionReason.IncompatibleProtocol, serverVersion: 12, serverVersionMin: 8)); - Assert.Equal(errorCodes["UNAUTHENTICATED"], - StatusOf(EvictionReason.IncompatibleProtocol, serverVersion: 12, serverVersionMin: 0)); - Assert.Equal(errorCodes["UNAUTHENTICATED"], - StatusOf(EvictionReason.IncompatibleProtocol, serverVersion: 8, serverVersionMin: 12)); - } - - /// - /// Pins the result-section widths. They are hardcoded on the .NET side, and the Rust module notes that - /// the decoder and its encode mirror "share the widths below so they cannot drift". - /// - [Fact] - public void ResultSectionWidths_MatchTheRustConstants() - { - var source = ReadRustSource(ReplyResultPath); - - Assert.Equal(VsrReplyDecoder.RESULT_COUNT_LENGTH, RustConstant(source, "RESULT_COUNT_LEN")); - Assert.Equal(VsrReplyDecoder.RESULT_ENTRY_LENGTH, RustConstant(source, "RESULT_ENTRY_LEN")); - } - - /// - /// Pins the credential bounds the client rejects on before spending a consensus round trip. Bounds that - /// drift wide let the server evict the session instead; bounds that drift narrow reject logins the - /// server would accept. - /// - [Fact] - public void CredentialBounds_MatchTheRustDefaults() - { - var source = ReadRustSource(CredentialDefaultsPath); - - Assert.Equal(CredentialBounds.MIN_USERNAME_LENGTH, RustConstant(source, "MIN_USERNAME_LENGTH")); - Assert.Equal(CredentialBounds.MAX_USERNAME_LENGTH, RustConstant(source, "MAX_USERNAME_LENGTH")); - Assert.Equal(CredentialBounds.MIN_PASSWORD_LENGTH, RustConstant(source, "MIN_PASSWORD_LENGTH")); - Assert.Equal(CredentialBounds.MAX_PASSWORD_LENGTH, RustConstant(source, "MAX_PASSWORD_LENGTH")); - } - - /// - /// Pins the register reply body layout by reading the field offsets out of the Rust decoder and feeding - /// .NET a buffer laid out to them. The header offsets are pinned structurally above, but the bodies were - /// only ever checked against hand-written .NET expectations, which move together with the code. - /// - [Fact] - public void LoginRegisterResponseLayout_MatchesTheRustDecoder() - { - var body = RustItem.Body(ReadRustSource(LoginRegisterResponsePath), "fn decode(buf: &[u8])"); - - Assert.Equal(0, RustReadOffset(body, "read_u32_le", "user_id")); - Assert.Equal(4, RustReadOffset(body, "read_u64_le", "session")); - Assert.Equal(12, RustReadOffset(body, "read_u32_le", "server_protocol_version")); - - var versionOffset = Regex.Match(body, @"WireName::decode\(&buf\[(\d+)\.\.\]\)"); - Assert.True(versionOffset.Success, "The register reply no longer decodes its server version by offset."); - Assert.Equal(16, Group(versionOffset, 1)); - - Span reply = stackalloc byte[16 + 1 + 3]; - BinaryPrimitives.WriteUInt32LittleEndian(reply, 7); - BinaryPrimitives.WriteUInt64LittleEndian(reply[4..], 42); - BinaryPrimitives.WriteUInt32LittleEndian(reply[12..], 99); - reply[16] = 3; - "1.2"u8.CopyTo(reply[17..]); - - LoginRegisterResponse decoded = LoginRegister.Deserialize(reply); - - Assert.Equal(7u, decoded.UserId); - Assert.Equal(42ul, decoded.Session); - } - - /// - /// Pins the consumer-group assignment reply layout the same way. A silent offset shift here reassigns - /// partitions rather than failing, so every member of a group polls the wrong partitions. - /// - [Fact] - public void SyncConsumerGroupResponseLayout_MatchesTheRustDecoder() - { - var body = RustItem.Body(ReadRustSource(SyncConsumerGroupResponsePath), "fn decode(buf: &[u8])"); - - Assert.Equal(0, RustReadOffset(body, "read_u64_le", "generation")); - Assert.Equal(8, RustReadOffset(body, "read_u32_le", "partitions_count")); - - var payloadOffset = Regex.Match(body, @"let mut offset = (\d+);"); - Assert.True(payloadOffset.Success, "The assignment reply no longer decodes its partitions by offset."); - Assert.Equal(12, Group(payloadOffset, 1)); - - Span reply = stackalloc byte[12 + 8]; - BinaryPrimitives.WriteUInt64LittleEndian(reply, 5); - BinaryPrimitives.WriteUInt32LittleEndian(reply[8..], 2); - BinaryPrimitives.WriteUInt32LittleEndian(reply[12..], 11); - BinaryPrimitives.WriteUInt32LittleEndian(reply[16..], 13); - - SyncConsumerGroupAssignment decoded = SyncConsumerGroupAssignment.Decode(reply); - - Assert.Equal(5ul, decoded.Generation); - Assert.Equal([11u, 13u], decoded.Partitions); - } - - private static IReadOnlyDictionary RustErrorCodes() - { - return Regex.Matches(ReadRustSource(ErrorPath), @"^\s{4}(\w+)(?:\([^)]*\))?\s*=\s*(\d+),", - RegexOptions.Multiline) - .ToDictionary(match => Normalize(match.Groups[1].Value), - match => int.Parse(match.Groups[2].Value, CultureInfo.InvariantCulture)); - } - - private static int StatusOf(EvictionReason? reason, uint serverVersion = 0, uint serverVersionMin = 0) - { - var thrown = Assert.IsType( - VsrReplyDecoder.ToException(new EvictionFrame(reason, serverVersion, serverVersionMin))); - - return thrown.StatusCode; - } - - private static int RustConstant(string source, string name) - { - var declared = Regex.Match(source, $@"pub const {Regex.Escape(name)}\s*:\s*\w+\s*=\s*(\d+)\s*;"); - Assert.True(declared.Success, $"Rust no longer declares {name}."); - - return Group(declared, 1); - } - - private static int RustReadOffset(string body, string reader, string field) - { - var read = Regex.Match(body, $@"let {Regex.Escape(field)} = {Regex.Escape(reader)}\(buf,\s*(\d+)\)"); - Assert.True(read.Success, $"Rust no longer decodes {field} with {reader} at a literal offset."); - - return Group(read, 1); - } - - private static string Normalize(string name) - { - return name.Replace("_", string.Empty, StringComparison.Ordinal).ToUpperInvariant(); - } - - private static int Group(Match match, int index) - { - return int.Parse(match.Groups[index].Value, CultureInfo.InvariantCulture); - } - - /// Every .NET member matches Rust, and Rust carries no member the .NET enum is missing. - private static void AssertExactMatch(IReadOnlyDictionary rust) where TEnum : struct, Enum - { - AssertSubsetMatches(rust); - - HashSet ported = Enum.GetNames().ToHashSet(); - Assert.DoesNotContain(rust.Keys, name => !ported.Contains(name)); - } - - /// Every .NET member matches Rust; Rust members with no .NET counterpart are allowed. - private static void AssertSubsetMatches(IReadOnlyDictionary rust) where TEnum : struct, Enum - { - Assert.NotEmpty(rust); - - var mismatches = new List(); - foreach (var name in Enum.GetNames()) - { - var value = Convert.ToInt32(Enum.Parse(name), CultureInfo.InvariantCulture); - if (!rust.TryGetValue(name, out var rustValue)) - { - mismatches.Add($"{name}: missing on the Rust side"); - } - else if (rustValue != value) - { - mismatches.Add($"{name}: rust {rustValue}, .NET {value}"); - } - } - - Assert.Empty(mismatches); - } - - private static string ReadRustSource(string relativePath) - { - var root = RepositoryRoot(); - if (root is null) - { - // Skipping suits a consumer running the suite outside a checkout, but in CI it would turn every - // assertion in this file green on a Rust-side path change, which is the one drift the suite - // cannot afford to miss. - Assert.False(Environment.GetEnvironmentVariable("GITHUB_ACTIONS") == "true", - $"Rust sources are unavailable in CI: no ancestor of {AppContext.BaseDirectory} holds {HeaderPath}."); - Assert.Skip("Rust sources are not available; run the drift check from a repository checkout."); - } - - return File.ReadAllText(Path.Combine(root, relativePath)); - } - - private static string? RepositoryRoot() - { - var directory = new DirectoryInfo(AppContext.BaseDirectory); - while (directory is not null) - { - if (File.Exists(Path.Combine(directory.FullName, HeaderPath))) - { - return directory.FullName; - } - - directory = directory.Parent; - } - - return null; - } -} - -/// Discriminants of a #[repr(u8)] Rust enum, by member name. -internal static class RustEnum -{ - internal static IReadOnlyDictionary Discriminants(string source, string name) - { - var body = RustItem.Body(source, $"pub enum {name} {{"); - var members = new Dictionary(); - - foreach (Match member in Regex.Matches(body, @"^\s*(\w+) = (\d+),", RegexOptions.Multiline)) - { - members[member.Groups[1].Value] = int.Parse(member.Groups[2].Value, CultureInfo.InvariantCulture); - } - - return members; - } -} - -/// -/// Field offsets of a #[repr(C)] Rust struct, computed from the declared field order the same way -/// rustc lays them out: each field starts at the next multiple of its alignment. -/// -internal static class RustStruct -{ - private static readonly Dictionary ScalarLayouts = new() - { - ["u8"] = (1, 1), - ["u16"] = (2, 2), - ["u32"] = (4, 4), - ["u64"] = (8, 8), - ["u128"] = (16, 16), - // Every enum the headers embed is `#[repr(u8)]`. - ["Command2"] = (1, 1), - ["Operation"] = (1, 1), - ["EvictionReason"] = (1, 1) - }; - - internal static IReadOnlyDictionary Offsets(string source, string name) - { - return Layout(source, name).Offsets; - } - - internal static int Size(string source, string name) - { - return Layout(source, name).Size; - } - - private static (Dictionary Offsets, int Size) Layout(string source, string name) - { - var body = RustItem.Body(source, $"pub struct {name} {{"); - var offsets = new Dictionary(); - var offset = 0; - var structAlign = 1; - - foreach (Match field in Regex.Matches(body, @"^\s*pub (\w+): ([^,]+),", RegexOptions.Multiline)) - { - var (size, align) = FieldLayout(field.Groups[2].Value.Trim(), name); - offset = Align(offset, align); - offsets[field.Groups[1].Value] = offset; - offset += size; - structAlign = Math.Max(structAlign, align); - } - - return (offsets, Align(offset, structAlign)); - } - - private static (int Size, int Align) FieldLayout(string type, string structName) - { - if (ScalarLayouts.TryGetValue(type, out var scalar)) - { - return scalar; - } - - var array = Regex.Match(type, @"^\[u8; (\d+)\]$"); - if (array.Success) - { - return (int.Parse(array.Groups[1].Value, CultureInfo.InvariantCulture), 1); - } - - throw new InvalidOperationException($"{structName} gained a field of unmapped type '{type}'."); - } - - private static int Align(int offset, int alignment) - { - return (offset + alignment - 1) / alignment * alignment; - } -} - -internal static class RustItem -{ - /// The brace-delimited body following , comments stripped. - internal static string Body(string source, string declaration) - { - var start = source.IndexOf(declaration, StringComparison.Ordinal); - if (start < 0) - { - throw new InvalidOperationException($"'{declaration}' is no longer declared in the Rust sources."); - } - - var cursor = start + declaration.Length; - var depth = 1; - while (cursor < source.Length && depth > 0) - { - depth += source[cursor] switch - { - '{' => 1, - '}' => -1, - _ => 0 - }; - cursor++; - } - - var body = source[(start + declaration.Length)..(cursor - 1)]; - - return Regex.Replace(body, @"^\s*//.*$", string.Empty, RegexOptions.Multiline); - } -} diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrReplyDecoderTests.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrReplyDecoderTests.cs deleted file mode 100644 index a6707b8905..0000000000 --- a/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrReplyDecoderTests.cs +++ /dev/null @@ -1,293 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Buffers.Binary; -using Apache.Iggy.Exceptions; -using Apache.Iggy.Vsr; - -namespace Apache.Iggy.Tests.VsrTests; - -public sealed class VsrReplyDecoderTests -{ - private static byte[] ReplyHeader(VsrOperation operation, int bodyLength, uint status = 0) - { - var header = new byte[VsrHeader.HEADER_SIZE]; - header[VsrHeader.COMMAND_OFFSET] = (byte)Command2.Reply; - header[VsrHeader.REPLY_OPERATION_OFFSET] = (byte)operation; - BinaryPrimitives.WriteUInt32LittleEndian(header.AsSpan(VsrHeader.SIZE_OFFSET), - (uint)(VsrHeader.HEADER_SIZE + bodyLength)); - BinaryPrimitives.WriteUInt32LittleEndian(header.AsSpan(VsrHeader.REPLY_STATUS_OFFSET), status); - - return header; - } - - private static byte[] EvictionHeader(EvictionReason reason, uint version = 0, uint versionMin = 0) - { - var header = new byte[VsrHeader.HEADER_SIZE]; - header[VsrHeader.COMMAND_OFFSET] = (byte)Command2.Eviction; - header[VsrHeader.EVICTION_REASON_OFFSET] = (byte)reason; - BinaryPrimitives.WriteUInt32LittleEndian(header.AsSpan(VsrHeader.EVICTION_PROTOCOL_VERSION_OFFSET), version); - BinaryPrimitives.WriteUInt32LittleEndian(header.AsSpan(VsrHeader.EVICTION_PROTOCOL_VERSION_MIN_OFFSET), - versionMin); - - return header; - } - - private static byte[] SuccessBody(params byte[] payload) - { - return VsrTestPayloads.Concat(VsrTestPayloads.UInt32(0), payload); - } - - private static byte[] RejectionBody(uint code) - { - return VsrTestPayloads.Concat(VsrTestPayloads.UInt32(1), VsrTestPayloads.UInt32(0), - VsrTestPayloads.UInt32((int)code)); - } - - [Fact] - public void Decode_StripsTheResultSectionFromACommittedMetadataReply() - { - var body = SuccessBody(1, 2, 3); - - ReadOnlyMemory payload - = VsrReplyDecoder.Decode(ReplyHeader(VsrOperation.CreateStream, body.Length), body); - - Assert.Equal([1, 2, 3], payload.ToArray()); - } - - [Fact] - public void Decode_CommittedRejectionThrowsTheTypedError() - { - var body = RejectionBody(1009); - - var exception = Assert.Throws(() => - VsrReplyDecoder.Decode(ReplyHeader(VsrOperation.CreateStream, body.Length), body)); - - Assert.Equal(1009, exception.StatusCode); - Assert.True(exception.FromServer); - } - - /// - /// Retry and failover key off the transient status codes, and a locally raised code says nothing about - /// what the cluster did, so the two origins have to stay distinguishable. - /// - [Fact] - public void ServerVerdictsAreDistinguishableFromClientSideFailures() - { - var header = ReplyHeader(VsrOperation.CreateStream, 0, VsrError.TRANSIENT_NOT_ACCEPTED); - - var fromWire = Assert.Throws(() => - VsrReplyDecoder.Decode(header, ReadOnlyMemory.Empty)); - - Assert.True(fromWire.FromServer); - Assert.False(VsrError.Exception(VsrError.TRANSIENT_NOT_ACCEPTED, "raised locally").FromServer); - } - - [Fact] - public void Decode_TruncatedResultSectionIsInvalidCommandNeverSuccess() - { - var body = VsrTestPayloads.UInt32(1); - - var exception = Assert.Throws(() => - VsrReplyDecoder.Decode(ReplyHeader(VsrOperation.CreateStream, body.Length), body)); - - Assert.Equal(VsrError.INVALID_COMMAND, exception.StatusCode); - } - - [Fact] - public void Decode_NonZeroStatusIsReadBeforeTheBody() - { - var exception = Assert.Throws(() => - VsrReplyDecoder.Decode(ReplyHeader(VsrOperation.CreateStream, 0, 1009), ReadOnlyMemory.Empty)); - - Assert.Equal(1009, exception.StatusCode); - } - - [Fact] - public void Decode_NonResultFramedReplyPassesTheBodyThrough() - { - byte[] body = [1, 2, 3, 4, 5]; - - ReadOnlyMemory payload - = VsrReplyDecoder.Decode(ReplyHeader(VsrOperation.NonReplicated, body.Length), body); - - Assert.Equal(body, payload.ToArray()); - } - - [Fact] - public void Decode_PartitionSendReplyPassesTheBodyThrough() - { - byte[] body = [7, 7]; - - ReadOnlyMemory payload - = VsrReplyDecoder.Decode(ReplyHeader(VsrOperation.SendMessages, body.Length), body); - - Assert.Equal(body, payload.ToArray()); - } - - [Fact] - public void Decode_ConsumerOffsetReplyIsResultFramed() - { - var body = SuccessBody(); - - ReadOnlyMemory payload - = VsrReplyDecoder.Decode(ReplyHeader(VsrOperation.StoreConsumerOffset, body.Length), body); - - Assert.True(payload.IsEmpty); - } - - [Fact] - public void Decode_NonEmptyRegisterReplyIsResultFramed() - { - var body = SuccessBody(9, 9); - - ReadOnlyMemory payload = VsrReplyDecoder.Decode(ReplyHeader(VsrOperation.Register, body.Length), body); - - Assert.Equal([9, 9], payload.ToArray()); - } - - [Fact] - public void Decode_EmptyRegisterReplyPassesThroughToFailTheTypedDecode() - { - ReadOnlyMemory payload - = VsrReplyDecoder.Decode(ReplyHeader(VsrOperation.Register, 0), ReadOnlyMemory.Empty); - - Assert.True(payload.IsEmpty); - } - - [Fact] - public void Decode_ShortBodyIsInvalidCommand() - { - var exception = Assert.Throws(() => - VsrReplyDecoder.Decode(ReplyHeader(VsrOperation.CreateStream, 8), new byte[4])); - - Assert.Equal(VsrError.INVALID_COMMAND, exception.StatusCode); - } - - [Fact] - public void Decode_FrameSmallerThanTheHeaderIsInvalidCommand() - { - var header = ReplyHeader(VsrOperation.CreateStream, 0); - BinaryPrimitives.WriteUInt32LittleEndian(header.AsSpan(VsrHeader.SIZE_OFFSET), 128); - - var exception = Assert.Throws(() => - VsrReplyDecoder.Decode(header, ReadOnlyMemory.Empty)); - - Assert.Equal(VsrError.INVALID_COMMAND, exception.StatusCode); - } - - [Fact] - public void Decode_UnexpectedFrameIsInvalidCommand() - { - var header = new byte[VsrHeader.HEADER_SIZE]; - header[VsrHeader.COMMAND_OFFSET] = 6; - - var exception = Assert.Throws(() => - VsrReplyDecoder.Decode(header, ReadOnlyMemory.Empty)); - - Assert.Equal(VsrError.INVALID_COMMAND, exception.StatusCode); - } - - [Fact] - public void Decode_ShortHeaderIsEmptyResponse() - { - var exception = Assert.Throws(() => - VsrReplyDecoder.Decode(new byte[8], ReadOnlyMemory.Empty)); - - Assert.Equal(VsrError.EMPTY_RESPONSE, exception.StatusCode); - } - - [Fact] - public void Decode_ReadsAnEvictionFromAMisalignedBuffer() - { - var frame = VsrTestPayloads.Concat([0, 0, 0], EvictionHeader(EvictionReason.InvalidCredentials)); - - var exception = Assert.Throws(() => - VsrReplyDecoder.Decode(frame.AsSpan(3), ReadOnlyMemory.Empty)); - - Assert.Equal(VsrError.INVALID_CREDENTIALS, exception.StatusCode); - } - - [Theory] - [InlineData((byte)EvictionReason.InvalidCredentials, VsrError.INVALID_CREDENTIALS)] - [InlineData((byte)EvictionReason.InvalidToken, VsrError.INVALID_PERSONAL_ACCESS_TOKEN)] - [InlineData((byte)EvictionReason.UserInactive, VsrError.UNAUTHENTICATED)] - [InlineData((byte)EvictionReason.SessionError, VsrError.UNAUTHENTICATED)] - [InlineData((byte)EvictionReason.NoSession, VsrError.UNAUTHENTICATED)] - [InlineData((byte)EvictionReason.SessionTooLow, VsrError.UNAUTHENTICATED)] - [InlineData((byte)EvictionReason.SessionReleaseMismatch, VsrError.UNAUTHENTICATED)] - [InlineData((byte)EvictionReason.StaleClient, VsrError.STALE_CLIENT)] - [InlineData((byte)EvictionReason.MalformedLogin, VsrError.INVALID_FORMAT)] - [InlineData((byte)EvictionReason.ClientReleaseTooLow, VsrError.INVALID_COMMAND)] - [InlineData((byte)EvictionReason.ClientReleaseTooHigh, VsrError.INVALID_COMMAND)] - [InlineData((byte)EvictionReason.InvalidRequestOperation, VsrError.INVALID_COMMAND)] - [InlineData((byte)EvictionReason.InvalidRequestBody, VsrError.INVALID_COMMAND)] - [InlineData((byte)EvictionReason.InvalidRequestBodySize, VsrError.INVALID_COMMAND)] - - // Reserved and every reason this build cannot decode land in the shared grader's catch-all. - [InlineData((byte)EvictionReason.Reserved, VsrError.INVALID_COMMAND)] - public void Decode_MapsEachEvictionReasonToItsError(byte reason, int expected) - { - var exception = Assert.Throws(() => - VsrReplyDecoder.Decode(EvictionHeader((EvictionReason)reason), ReadOnlyMemory.Empty)); - - Assert.Equal(expected, exception.StatusCode); - } - - [Fact] - public void Decode_IncompatibleProtocolReportsTheAcceptedWindow() - { - var exception = Assert.Throws(() => - VsrReplyDecoder.Decode(EvictionHeader(EvictionReason.IncompatibleProtocol, 10243, 10240), - ReadOnlyMemory.Empty)); - - Assert.Equal(VsrError.INCOMPATIBLE_PROTOCOL_VERSION, exception.StatusCode); - Assert.Contains("10240..10243", exception.Message); - } - - [Theory] - [InlineData(10243u, 0u)] - [InlineData(10240u, 10243u)] - public void Decode_IncompatibleProtocolWithAnUnusableWindowDegradesToUnauthenticated(uint version, uint min) - { - var exception = Assert.Throws(() => - VsrReplyDecoder.Decode(EvictionHeader(EvictionReason.IncompatibleProtocol, version, min), - ReadOnlyMemory.Empty)); - - Assert.Equal(VsrError.UNAUTHENTICATED, exception.StatusCode); - } - - [Fact] - public void ReadResultCode_MalformedSectionIsNullNeverZero() - { - Assert.Null(VsrReplyDecoder.ReadResultCode([])); - Assert.Null(VsrReplyDecoder.ReadResultCode(VsrTestPayloads.UInt32(1))); - Assert.Null(VsrReplyDecoder.ReadResultCode([1, 0, 0, 0, 0, 0, 0, 0])); - Assert.Equal(0u, VsrReplyDecoder.ReadResultCode(SuccessBody())); - } - - [Fact] - public void ReadResultSectionLength_CoversTheCountAndItsEntries() - { - Assert.Equal(4, VsrReplyDecoder.ReadResultSectionLength(SuccessBody(1, 2))); - Assert.Equal(12, VsrReplyDecoder.ReadResultSectionLength(RejectionBody(1009))); - Assert.Null(VsrReplyDecoder.ReadResultSectionLength(VsrTestPayloads.UInt32(1))); - - // Shorter than the count itself, so there is no section to measure. - Assert.Null(VsrReplyDecoder.ReadResultSectionLength([1, 2, 3])); - } -} diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrTestPayloads.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrTestPayloads.cs deleted file mode 100644 index 1e4914766c..0000000000 --- a/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrTestPayloads.cs +++ /dev/null @@ -1,102 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Buffers.Binary; -using System.Text; - -namespace Apache.Iggy.Tests.VsrTests; - -/// -/// Builders for the classic request bodies the VSR encoder peeks into, matching what -/// TcpContracts writes. -/// -internal static class VsrTestPayloads -{ - internal static byte[] NumericIdentifier(uint value) - { - var bytes = new byte[6]; - bytes[0] = 1; - bytes[1] = 4; - BinaryPrimitives.WriteUInt32LittleEndian(bytes.AsSpan(2), value); - - return bytes; - } - - internal static byte[] NamedIdentifier(string value) - { - var name = Encoding.UTF8.GetBytes(value); - var bytes = new byte[2 + name.Length]; - bytes[0] = 2; - bytes[1] = (byte)name.Length; - name.CopyTo(bytes, 2); - - return bytes; - } - - internal static byte[] SendMessages(byte[] streamId, byte[] topicId, byte partitioningKind, - byte[] partitioningValue, int messagesCount = 1) - { - var metadata = Concat(streamId, topicId, - Concat([partitioningKind, (byte)partitioningValue.Length], partitioningValue), - UInt32(messagesCount)); - - return Concat(UInt32(metadata.Length), metadata); - } - - internal static byte[] SendMessagesToPartition(uint streamId, uint topicId, uint partitionId) - { - return SendMessages(NumericIdentifier(streamId), NumericIdentifier(topicId), 2, UInt32((int)partitionId)); - } - - internal static byte[] ConsumerOffset(byte[] streamId, byte[] topicId, uint? partitionId) - { - var partition = new byte[5]; - if (partitionId.HasValue) - { - partition[0] = 1; - BinaryPrimitives.WriteUInt32LittleEndian(partition.AsSpan(1), partitionId.Value); - } - - return Concat([1], NumericIdentifier(1), streamId, topicId, partition); - } - - internal static byte[] DeleteSegments(byte[] streamId, byte[] topicId, uint partitionId, uint segmentsCount = 1) - { - return Concat(streamId, topicId, UInt32((int)partitionId), UInt32((int)segmentsCount)); - } - - internal static byte[] UInt32(int value) - { - var bytes = new byte[4]; - BinaryPrimitives.WriteUInt32LittleEndian(bytes, (uint)value); - - return bytes; - } - - internal static byte[] Concat(params byte[][] parts) - { - var result = new byte[parts.Sum(part => part.Length)]; - var position = 0; - foreach (var part in parts) - { - part.CopyTo(result, position); - position += part.Length; - } - - return result; - } -} diff --git a/foreign/csharp/README.md b/foreign/csharp/README.md index 508990e391..633d4145bf 100644 --- a/foreign/csharp/README.md +++ b/foreign/csharp/README.md @@ -37,10 +37,6 @@ The SDK supports two transport protocols: - **TCP** - Binary protocol for optimal performance and lower latency (recommended) - **HTTP** - RESTful JSON API for stateless operations -Over TCP the SDK speaks the VSR consensus framing, which is the only wire protocol the server accepts. - -See [Viewstamped Replication (VSR)](#viewstamped-replication-vsr) for what that means for the client API. - ### Creating a Client The SDK is built around the `IIggyClient` interface. To create a client instance: @@ -124,89 +120,6 @@ var client = IggyClientFactory.CreateClient(new IggyClientConfigurator await client.ConnectAsync(); ``` -## Viewstamped Replication (VSR) - -Over TCP every request is wrapped in a 256-byte consensus header, the client registers a consensus session at -login, and writes are replicated before they are acknowledged. The `IIggyClient` surface is unchanged, with the -few exceptions listed under [Limitations](#limitations). - -```c# -var client = IggyClientFactory.CreateClient(new IggyClientConfigurator -{ - BaseAddress = "127.0.0.1:8090", - Protocol = Protocol.Tcp, - - // Upper bound on a reply frame the server announces, 64 MiB by default. - MaxResponseFrameSize = 64 * 1024 * 1024, - - AutoLoginSettings = new AutoLoginSettings - { - Enabled = true, - Username = "iggy", - Password = "iggy" - } -}); - -await client.ConnectAsync(); -``` - -### What changes under VSR - -- **Login binds a session.** `LoginUserAsync` / `LoginWithPersonalAccessTokenAsync` run the register handshake - at connect time, and the session lives for as long as the connection. Logging out, being evicted - or losing the connection ends it, and the next login registers a fresh one. -- **Leader redirection is automatic.** The client reads the cluster roster, follows the current leader and - re-checks it when a request is refused because the node stopped being primary. -- **The client picks partitions.** The broker never routes: balanced and message-key partitioning are resolved - client-side (the message-key hash matches the Rust SDK byte for byte), and consumer-group polls round-robin - over the partitions the coordinator assigned to this client. -- **Consumer groups are assignment-based.** `JoinConsumerGroupAsync` makes this client a member; the assignment - is synced on demand and refreshed on every `PingAsync`. Partition counts are cached for 30 seconds, so a topic - another client widens is picked up without waiting for a ping. -- **Credentials are bounds-checked locally.** A username outside 3-50 bytes, a password outside 3-100 bytes or a - personal access token outside 1-255 bytes is rejected before the register body is framed. -- **`PingAsync` costs more than a ping.** Besides the ping it re-syncs the assignment of every consumer group - this client has joined, so it makes one extra round trip per joined group. The SDK runs no background - heartbeat: an application that wants assignments refreshed calls `PingAsync` on its own cadence. - -### Retries and failed requests - -The SDK replays a request whenever the server says it never admitted it. Two cases surface to the caller: - -- `IggyInvalidStatusCodeException` carries the server status code, with `FromServer` telling apart a verdict the - cluster reported from a failure the client raised itself. -- `VsrRequestOutcomeUnknownException` means no server verdict arrived after the request was written - the - connection was lost, the call was cancelled, or the server evicted the session while the request was in - flight - so the cluster may or may not have committed it. The SDK will not replay it on a new session, - because that would bypass server-side deduplication - re-issuing it is the caller's decision. - `IggyPublisher` will not retry it either: it reports the batch through the message-batch-failed event, and - `IggyConsumer` rethrows it rather than swallowing it, because an auto-committing poll may have advanced the - offset already. Rethrowing ends the consumer's polling loop: catch it around the enumeration, decide whether - the operation is safe to re-issue, and start consuming again. - -### Limitations - -- VSR requires `Protocol.Tcp`; configuring it with `Protocol.Http` throws at client creation. -- `StoreOffsetAsync` / `DeleteOffsetAsync` need an explicit partition id under VSR: the broker does not - resolve a `null` partition for a consumer-offset request, so passing one throws client-side. -- `FlushUnsavedBufferAsync` is not available under VSR; the server refuses it. -- Polling a topic that does not exist returns an empty poll rather than throwing. The server - answers an unresolved topic with the empty-poll reply shape, so the client cannot tell it apart from a topic - with no messages. Check the topic exists first if the distinction matters. - -### Behaviour changes for existing clients - -- `MaxResponseFrameSize` bounds the reply frames the **VSR** reader accepts. A reply larger than the 64 MiB - default is refused and the connection is dropped, so raise it if a single response legitimately exceeds that - - a large `GetSnapshotAsync` is the usual case. -- Clients built with `IggyConsumerBuilder` / `IggyPublisherBuilder` now auto-login with the credentials passed - to `WithConnection`. Before, a builder-created client came back from a - reconnect unauthenticated; now the credentials are held for the lifetime of the connection and replayed. -- The SDK now ships a dependency on `System.IO.Hashing`, used for the client-side message-key partitioner. -- TCP sockets are opened with `NoDelay`. The protocol is request/reply, so a write is - always the last one before the client waits for the answer and Nagle has nothing to coalesce it with - it - only held back the trailing segment of a large request until the previous one was acked. - ## Authentication ### User Login @@ -781,14 +694,10 @@ Integration tests are located in `Iggy_SDK.Tests.Integration/`. Tests can run ag #### 1. Dockerization -The suite runs against `iggy-server`. TCP only: the SDK frames TCP with the -VSR wire protocol, the cluster serves reads from the primary, and the HTTP surface has no equivalent path to -route them through. - ```bash -cargo build --bin iggy-server --bin iggy +cargo build -docker build --no-cache -f core/server/Dockerfile --platform linux/amd64 --target runtime-prebuilt --build-arg PREBUILT_IGGY_SERVER=target/debug/iggy-server --build-arg PREBUILT_IGGY_CLI=target/debug/iggy -t iggy-server:test . +docker build --no-cache -f core/server/Dockerfile --platform linux/amd64 --target runtime-prebuilt --build-arg PREBUILT_IGGY_SERVER=target/debug/iggy-server --build-arg PREBUILT_IGGY_CLI=target/debug/iggy -t local-iggy-server . ``` #### 2. Build the Test Project @@ -801,13 +710,10 @@ dotnet build foreign/csharp/Iggy_SDK.Tests.Integration ```bash cd foreign/csharp -export IGGY_SERVER_DOCKER_IMAGE=iggy-server:test +export IGGY_SERVER_DOCKER_IMAGE=local-iggy-server dotnet test -f net10.0 --project Iggy_SDK.Tests.Integration --no-build --verbosity diagnostic ``` -`IGGY_SERVER_DOCKER_IMAGE` defaults to `iggy-server:test`, so the export above is only needed to point -at a different image. Rider and Visual Studio need nothing configured. - ## Useful Resources - [Iggy Documentation](https://iggy.apache.org/docs/) diff --git a/foreign/csharp/scripts/pack.sh b/foreign/csharp/scripts/pack.sh index cb66da0b4f..09b23e34e1 100755 --- a/foreign/csharp/scripts/pack.sh +++ b/foreign/csharp/scripts/pack.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/foreign/go/README.md b/foreign/go/README.md index dc41052d70..b9fe45e97e 100644 --- a/foreign/go/README.md +++ b/foreign/go/README.md @@ -11,7 +11,9 @@ Official Go client SDK for [Apache Iggy](https://iggy.apache.org) message streaming. The client speaks the VSR wire protocol over TCP, with or without TLS, in a -blocking implementation. VSR is the only protocol it supports. +blocking implementation. VSR is the only protocol it supports: there is no +option to fall back to the classic framing, so the SDK requires a server that +speaks VSR and no longer works with the legacy `iggy-server`. > Apache Iggy (Incubating) is an effort undergoing incubation at the Apache Software Foundation (ASF), sponsored by the Apache Incubator PMC. > @@ -29,14 +31,16 @@ go get github.com/apache/iggy/foreign/go Build and start a VSR server from a checkout of this repository: + + ```bash -cargo build --bin iggy-server +cargo build --bin iggy-server-ng --features vsr IGGY_SYSTEM_PATH=/tmp/iggy-go \ IGGY_TCP_ADDRESS=127.0.0.1:8090 \ IGGY_HTTP_ENABLED=false IGGY_QUIC_ENABLED=false IGGY_WEBSOCKET_ENABLED=false \ IGGY_ROOT_USERNAME=iggy IGGY_ROOT_PASSWORD=iggy \ -target/debug/iggy-server +target/debug/iggy-server-ng ``` QUIC, WebSocket and HTTP are enabled by default on ports 8080, 8092 and 3000. diff --git a/foreign/go/client/tcp/tcp_connect_test.go b/foreign/go/client/tcp/tcp_connect_test.go index 9e7cf56132..5a812b7257 100644 --- a/foreign/go/client/tcp/tcp_connect_test.go +++ b/foreign/go/client/tcp/tcp_connect_test.go @@ -198,6 +198,7 @@ func TestConnect_DiscoversTheLeaderAndSignsIn(t *testing.T) { assert.Zero(t, recorded[1].code(), "only a non-replicated frame carries the code") assert.Zero(t, recorded[1].requestID(), "a register is always request zero") assert.Zero(t, recorded[1].sessionID()) + assert.Equal(t, uint64(1)<<63, recorded[1].namespace()) assert.True(t, client.session.Bound()) assert.Equal(t, uint64(128), client.session.SessionID()) diff --git a/foreign/go/client/tcp/tcp_core_test.go b/foreign/go/client/tcp/tcp_core_test.go index 1ae9a438fa..f4cc00e150 100644 --- a/foreign/go/client/tcp/tcp_core_test.go +++ b/foreign/go/client/tcp/tcp_core_test.go @@ -389,7 +389,7 @@ func TestSendMessages_DecodesTheConfirmations(t *testing.T) { recorded := server.recorded() require.Len(t, recorded, 1) assert.Equal(t, vsr.OperationSendMessages, recorded[0].operation()) - assert.Equal(t, uint32(1), recorded[0].partitionID(t)) + assert.Equal(t, uint64(3)<<32|uint64(2)<<20|1, recorded[0].namespace()) assert.Equal(t, uint64(1), recorded[0].requestID(), "a partition request reads the watermark without consuming it") } @@ -471,7 +471,7 @@ func TestSendMessages_ResolvesKeyPartitioningToAnExplicitPartition(t *testing.T) require.Len(t, recorded, 2) assert.Equal(t, uint32(command.GetTopicCode), recorded[0].code()) // 0x0D3FE0E1 modulo four partitions. - assert.Equal(t, uint32(1), recorded[1].partitionID(t)) + assert.Equal(t, uint64(1)<<32|uint64(1)<<20|1, recorded[1].namespace()) } func TestSendMessages_RoundRobinsBalancedPartitioning(t *testing.T) { @@ -492,13 +492,13 @@ func TestSendMessages_RoundRobinsBalancedPartitioning(t *testing.T) { require.NoError(t, err) } - var partitions []uint32 + var partitions []uint64 for _, recorded := range server.recorded() { if recorded.operation() == vsr.OperationSendMessages { - partitions = append(partitions, recorded.partitionID(t)) + partitions = append(partitions, recorded.namespace()&0xFFFFF) } } - assert.Equal(t, []uint32{0, 1, 2, 0}, partitions) + assert.Equal(t, []uint64{0, 1, 2, 0}, partitions) metadataRequests := 0 for _, recorded := range server.recorded() { diff --git a/foreign/go/client/tcp/tcp_testing_test.go b/foreign/go/client/tcp/tcp_testing_test.go index f6f5db98b8..b63671b0e1 100644 --- a/foreign/go/client/tcp/tcp_testing_test.go +++ b/foreign/go/client/tcp/tcp_testing_test.go @@ -38,12 +38,13 @@ const ( frameOffsetClient = 128 frameOffsetRequest = 168 frameOffsetOperation = 176 - frameOffsetSession = 184 - frameOffsetReserved = 196 + frameOffsetNamespace = 184 + frameOffsetSession = 192 + frameOffsetReserved = 204 replyFrameOffsetRequest = 200 replyFrameOffsetOperation = 208 - replyFrameOffsetStatus = 216 + replyFrameOffsetStatus = 224 evictionFrameOffsetVersion = 144 evictionFrameOffsetVersionMin = 148 @@ -66,6 +67,9 @@ func (r request) requestID() uint64 { func (r request) sessionID() uint64 { return binary.LittleEndian.Uint64(r.header[frameOffsetSession:]) } +func (r request) namespace() uint64 { + return binary.LittleEndian.Uint64(r.header[frameOffsetNamespace:]) +} func (r request) clientID() vsr.ClientID { return vsr.ClientID{ Lo: binary.LittleEndian.Uint64(r.header[frameOffsetClient:]), @@ -73,27 +77,6 @@ func (r request) clientID() vsr.ClientID { } } -// partitionID reads the resolved partition out of a recorded SendMessages -// payload: [metadata length u32][stream id][topic id][partitioning], where the -// identifiers and the partitioning are each [kind u8][length u8][value]. The -// frame carries no routing namespace, so the payload is the only place the -// client's partitioning decision is observable. -func (r request) partitionID(t *testing.T) uint32 { - t.Helper() - - cursor := 4 - for range 2 { - require.Greater(t, len(r.payload), cursor+1) - cursor += 2 + int(r.payload[cursor+1]) - } - require.Greater(t, len(r.payload), cursor+1) - require.Equal(t, byte(iggcon.PartitionIdKind), r.payload[cursor], - "the client resolves every strategy to an explicit partition") - require.Equal(t, byte(4), r.payload[cursor+1]) - require.GreaterOrEqual(t, len(r.payload), cursor+6) - return binary.LittleEndian.Uint32(r.payload[cursor+2:]) -} - // replyFrame builds a committed reply carrying body. func replyFrame(operation vsr.Operation, body []byte) []byte { return statusReplyFrame(operation, 0, body) diff --git a/foreign/go/internal/vsr/envelope.go b/foreign/go/internal/vsr/envelope.go index 912881a2c1..690b5d9102 100644 --- a/foreign/go/internal/vsr/envelope.go +++ b/foreign/go/internal/vsr/envelope.go @@ -31,7 +31,7 @@ func EncodeRequest(session *Session, code uint32, payload []byte) ([]byte, error } // StampRequestHeader writes the request header into the first HeaderSize bytes -// of frame, deriving sequencing from the command code and the session state. +// of frame, deriving routing and sequencing from the payload that follows it. // Callers that encode a payload into a pooled buffer reserve the prologue up // front and stamp it here, which keeps the frame a single allocation. // @@ -40,17 +40,25 @@ func StampRequestHeader(session *Session, code uint32, frame []byte) error { if len(frame) > MaxFrameSize { return ierror.ErrInvalidConfiguration } + payload := frame[HeaderSize:] operation := OperationForCode(code) replicated := operation != OperationRegister && operation != OperationNonReplicated - // A replicated operation needs a bound session; refusing it here leaves - // the request-id counter untouched. + // A replicated operation needs a bound session. Checking that ahead of the + // payload means an unauthenticated caller hears about the missing session + // rather than about the request it could not have sent anyway. if replicated && !session.Bound() { return ierror.ErrUnauthenticated } + // Namespace derivation can fail on a malformed payload. Run it before + // taking a request id so a local failure leaves the counter untouched. + namespace, err := NamespaceForRequest(code, payload, operation) + if err != nil { + return err + } + var request, sessionID uint64 - var err error switch operation { case OperationRegister: request = session.BeginRegister() @@ -77,6 +85,7 @@ func StampRequestHeader(session *Session, code uint32, frame []byte) error { Client: session.ClientID(), Request: request, Operation: operation, + Namespace: namespace, Session: sessionID, } if operation == OperationNonReplicated { diff --git a/foreign/go/internal/vsr/envelope_test.go b/foreign/go/internal/vsr/envelope_test.go index c497b07fc5..f3c4f9ac40 100644 --- a/foreign/go/internal/vsr/envelope_test.go +++ b/foreign/go/internal/vsr/envelope_test.go @@ -55,6 +55,8 @@ func TestEncodeRequest_EncodesARegisterFrame(t *testing.T) { assert.Equal(t, byte(OperationRegister), header[requestOffsetOperation]) assert.Zero(t, binary.LittleEndian.Uint64(header[requestOffsetRequest:])) assert.Zero(t, binary.LittleEndian.Uint64(header[requestOffsetSession:])) + assert.Equal(t, MetadataConsensusNamespace, + binary.LittleEndian.Uint64(header[requestOffsetNamespace:])) assert.Zero(t, binary.LittleEndian.Uint32(header[requestOffsetReserved:])) } @@ -75,6 +77,8 @@ func TestEncodeRequest_EncodesALogoutFrame(t *testing.T) { header := frameHeader(t, frame) assert.Equal(t, byte(OperationLogout), header[requestOffsetOperation]) + assert.Equal(t, MetadataConsensusNamespace, + binary.LittleEndian.Uint64(header[requestOffsetNamespace:])) assert.Equal(t, uint64(1), binary.LittleEndian.Uint64(header[requestOffsetRequest:]), "logout advances the metadata watermark") assert.Equal(t, uint64(2), session.CurrentRequestID()) @@ -124,8 +128,10 @@ func TestEncodeRequest_DoesNotAdvanceTheWatermarkOffTheMetadataPlane(t *testing. } assert.Equal(t, uint64(1), session.CurrentRequestID()) + payload := sendMessagesPayload( + numericIdentifier(1), numericIdentifier(1), partitionIDPartitioning(0)) for range 3 { - _, err := EncodeRequest(session, uint32(command.SendMessagesCode), []byte{1}) + _, err := EncodeRequest(session, uint32(command.SendMessagesCode), payload) require.NoError(t, err) } assert.Equal(t, uint64(1), session.CurrentRequestID()) @@ -140,6 +146,7 @@ func TestEncodeRequest_AdvancesTheWatermarkPerMetadataCommand(t *testing.T) { header := frameHeader(t, frame) assert.Equal(t, expected, binary.LittleEndian.Uint64(header[requestOffsetRequest:])) assert.Equal(t, byte(OperationCreateStream), header[requestOffsetOperation]) + assert.Zero(t, binary.LittleEndian.Uint64(header[requestOffsetNamespace:])) } assert.Equal(t, uint64(4), session.CurrentRequestID()) } @@ -154,19 +161,51 @@ func TestEncodeRequest_RejectsAReplicatedCommandOnAnUnboundSession(t *testing.T) func TestEncodeRequest_RejectsAPartitionCommandOnAnUnboundSession(t *testing.T) { session := NewSessionWithClientID(ClientID{Lo: 1}) + payload := sendMessagesPayload( + numericIdentifier(1), numericIdentifier(1), partitionIDPartitioning(0)) - _, err := EncodeRequest(session, uint32(command.SendMessagesCode), []byte{1}) + _, err := EncodeRequest(session, uint32(command.SendMessagesCode), payload) assert.ErrorIs(t, err, ierror.ErrUnauthenticated) } -func TestEncodeRequest_EncodesASendMessagesFrame(t *testing.T) { +func TestEncodeRequest_ReportsTheMissingSessionAheadOfAnUnroutablePayload(t *testing.T) { + session := NewSessionWithClientID(ClientID{Lo: 1}) + unroutable := sendMessagesPayload( + numericIdentifier(MaxStreams), numericIdentifier(1), partitionIDPartitioning(0)) + + _, err := EncodeRequest(session, uint32(command.SendMessagesCode), unroutable) + assert.ErrorIs(t, err, ierror.ErrUnauthenticated, + "an unauthenticated caller could not have sent the request either way") +} + +func TestEncodeRequest_BurnsNoIDWhenTheNamespaceFails(t *testing.T) { + session := boundSession(t) + malformed := sendMessagesPayload( + numericIdentifier(1), numericIdentifier(1), []byte{1, 4, 0, 0, 0, 0}) + + _, err := EncodeRequest(session, uint32(command.SendMessagesCode), malformed) + assert.ErrorIs(t, err, ierror.ErrFeatureUnavailable) + assert.Equal(t, uint64(1), session.CurrentRequestID()) + + // The next metadata command still takes id 1, so the sequence has no gap. + frame, err := EncodeRequest(session, uint32(command.CreateStreamCode), []byte{1}) + require.NoError(t, err) + assert.Equal(t, uint64(1), + binary.LittleEndian.Uint64(frameHeader(t, frame)[requestOffsetRequest:])) +} + +func TestEncodeRequest_RoutesSendMessagesToItsPartition(t *testing.T) { session := boundSession(t) + payload := sendMessagesPayload( + numericIdentifier(3), numericIdentifier(2), partitionIDPartitioning(7)) - frame, err := EncodeRequest(session, uint32(command.SendMessagesCode), []byte{1}) + frame, err := EncodeRequest(session, uint32(command.SendMessagesCode), payload) require.NoError(t, err) header := frameHeader(t, frame) assert.Equal(t, byte(OperationSendMessages), header[requestOffsetOperation]) + assert.Equal(t, uint64(3<<32|2<<20|7), + binary.LittleEndian.Uint64(header[requestOffsetNamespace:])) assert.Zero(t, binary.LittleEndian.Uint32(header[requestOffsetReserved:]), "only non-replicated frames carry the code") } diff --git a/foreign/go/internal/vsr/header.go b/foreign/go/internal/vsr/header.go index 789237efd1..99cc742f9c 100644 --- a/foreign/go/internal/vsr/header.go +++ b/foreign/go/internal/vsr/header.go @@ -33,12 +33,6 @@ const ( // Request header field offsets the client writes. Fields are addressed by // byte offset rather than a struct cast because the header leads with // unaligned u128 values. -// -// The client wire carries no routing namespace: the server derives the -// consensus group (plane from the operation, partition target from the -// payload) and stamps it into its own internal header. Everything that -// followed the removed field therefore sits eight bytes earlier than in the -// pre-derivation layout. const ( requestOffsetSize = 48 requestOffsetCommand = 60 @@ -46,8 +40,9 @@ const ( requestOffsetTimestamp = 160 requestOffsetRequest = 168 requestOffsetOperation = 176 - requestOffsetSession = 184 - requestOffsetReserved = 196 + requestOffsetNamespace = 184 + requestOffsetSession = 192 + requestOffsetReserved = 204 ) // Reply header field offsets the client reads. @@ -56,7 +51,8 @@ const ( replyOffsetCommand = 60 replyOffsetRequest = 200 replyOffsetOperation = 208 - replyOffsetStatus = 216 + replyOffsetNamespace = 216 + replyOffsetStatus = 224 ) // Eviction header field offsets the client reads. @@ -126,6 +122,8 @@ type RequestFields struct { Request uint64 // Operation is the consensus operation discriminant. Operation Operation + // Namespace routes the request to the owning shard. + Namespace uint64 // Session is the bound session fence, or 0 while unbound. Session uint64 // NonReplicatedCode is the raw command code the server reads out of @@ -144,6 +142,7 @@ func EncodeRequestHeader(dst *[HeaderSize]byte, fields RequestFields) { binary.LittleEndian.PutUint64(dst[requestOffsetClient+8:], fields.Client.Hi) binary.LittleEndian.PutUint64(dst[requestOffsetRequest:], fields.Request) dst[requestOffsetOperation] = byte(fields.Operation) + binary.LittleEndian.PutUint64(dst[requestOffsetNamespace:], fields.Namespace) binary.LittleEndian.PutUint64(dst[requestOffsetSession:], fields.Session) if fields.NonReplicatedCode != 0 { binary.LittleEndian.PutUint32(dst[requestOffsetReserved:], fields.NonReplicatedCode) diff --git a/foreign/go/internal/vsr/header_test.go b/foreign/go/internal/vsr/header_test.go index 87cdadc7cb..6db3b7b271 100644 --- a/foreign/go/internal/vsr/header_test.go +++ b/foreign/go/internal/vsr/header_test.go @@ -33,6 +33,7 @@ func TestEncodeRequestHeader_WritesEveryFieldAtItsCanonicalOffset(t *testing.T) Client: ClientID{Lo: 0x99AABBCCDDEEFF00, Hi: 0x1122334455667788}, Request: 0x0102030405060708, Operation: OperationNonReplicated, + Namespace: 0x8877665544332211, Session: 0x1020304050607080, NonReplicatedCode: 60001, }) @@ -43,6 +44,7 @@ func TestEncodeRequestHeader_WritesEveryFieldAtItsCanonicalOffset(t *testing.T) assert.Equal(t, uint64(0x1122334455667788), binary.LittleEndian.Uint64(header[requestOffsetClient+8:])) assert.Equal(t, uint64(0x0102030405060708), binary.LittleEndian.Uint64(header[requestOffsetRequest:])) assert.Equal(t, byte(OperationNonReplicated), header[requestOffsetOperation]) + assert.Equal(t, uint64(0x8877665544332211), binary.LittleEndian.Uint64(header[requestOffsetNamespace:])) assert.Equal(t, uint64(0x1020304050607080), binary.LittleEndian.Uint64(header[requestOffsetSession:])) assert.Equal(t, uint32(60001), binary.LittleEndian.Uint32(header[requestOffsetReserved:])) assert.Zero(t, binary.LittleEndian.Uint64(header[requestOffsetTimestamp:]), "timestamp stays zero") @@ -54,6 +56,7 @@ func TestEncodeRequestHeader_LeavesEveryUnwrittenByteZero(t *testing.T) { Size: HeaderSize, Client: ClientID{Lo: 1}, Operation: OperationRegister, + Namespace: MetadataConsensusNamespace, }) var expected [HeaderSize]byte @@ -61,6 +64,7 @@ func TestEncodeRequestHeader_LeavesEveryUnwrittenByteZero(t *testing.T) { expected[requestOffsetCommand] = byte(FrameRequest) binary.LittleEndian.PutUint64(expected[requestOffsetClient:], 1) expected[requestOffsetOperation] = byte(OperationRegister) + binary.LittleEndian.PutUint64(expected[requestOffsetNamespace:], MetadataConsensusNamespace) assert.Equal(t, expected, header) } @@ -88,12 +92,14 @@ func TestEncodeRequestHeader_SupportsMaximumWidthFields(t *testing.T) { Client: ClientID{Lo: math.MaxUint64, Hi: math.MaxUint64}, Request: math.MaxUint64, Operation: OperationSendMessages, + Namespace: math.MaxUint64, Session: math.MaxUint64, }) assert.Equal(t, uint32(math.MaxUint32), binary.LittleEndian.Uint32(header[requestOffsetSize:])) assert.Equal(t, uint64(math.MaxUint64), binary.LittleEndian.Uint64(header[requestOffsetRequest:])) assert.Equal(t, uint64(math.MaxUint64), binary.LittleEndian.Uint64(header[requestOffsetSession:])) + assert.Equal(t, uint64(math.MaxUint64), binary.LittleEndian.Uint64(header[requestOffsetNamespace:])) } func TestEncodeRequestHeader_OmitsTheReservedCodeForReplicatedOperations(t *testing.T) { @@ -116,6 +122,7 @@ func TestEncodeRequestHeader_ProducesTheGoldenFrame(t *testing.T) { Client: ClientID{Lo: 0x0807060504030201, Hi: 0x100F0E0D0C0B0A09}, Request: 2, Operation: OperationCreateStream, + Namespace: 0, Session: 11, }) @@ -126,7 +133,7 @@ func TestEncodeRequestHeader_ProducesTheGoldenFrame(t *testing.T) { 136: {0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10}, 168: {0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, 176: {0x80}, - 184: {0x0B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + 192: {0x0B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, } written := 0 for offset, want := range golden { @@ -202,14 +209,16 @@ func TestHeaderOffsets_MatchTheWireContract(t *testing.T) { assert.Equal(t, 160, requestOffsetTimestamp) assert.Equal(t, 168, requestOffsetRequest) assert.Equal(t, 176, requestOffsetOperation) - assert.Equal(t, 184, requestOffsetSession) - assert.Equal(t, 196, requestOffsetReserved) + assert.Equal(t, 184, requestOffsetNamespace) + assert.Equal(t, 192, requestOffsetSession) + assert.Equal(t, 204, requestOffsetReserved) assert.Equal(t, 48, replyOffsetSize) assert.Equal(t, 60, replyOffsetCommand) assert.Equal(t, 208, replyOffsetOperation) - assert.Equal(t, 216, replyOffsetStatus) - assert.Equal(t, replyOffsetOperation+8, replyOffsetStatus, "status follows the operation") + assert.Equal(t, 216, replyOffsetNamespace) + assert.Equal(t, 224, replyOffsetStatus) + assert.Equal(t, replyOffsetNamespace+8, replyOffsetStatus, "status follows namespace") assert.Equal(t, 60, evictionOffsetCommand) assert.Equal(t, 128, evictionOffsetClient) diff --git a/foreign/go/internal/vsr/namespace.go b/foreign/go/internal/vsr/namespace.go new file mode 100644 index 0000000000..22490f8749 --- /dev/null +++ b/foreign/go/internal/vsr/namespace.go @@ -0,0 +1,246 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package vsr + +import ( + "encoding/binary" + + ierror "github.com/apache/iggy/foreign/go/errors" + "github.com/apache/iggy/foreign/go/internal/command" +) + +// Namespace packing limits, a port of +// core/binary_protocol/src/namespace.rs. +const ( + MaxStreams = 4096 + MaxTopics = 4096 + MaxPartitions = 1_000_000 + + streamBits = 12 + topicBits = 12 + partitionBits = 20 + + partitionShift = 0 + topicShift = partitionShift + partitionBits + streamShift = topicShift + topicBits +) + +// MetadataConsensusNamespace routes a control-plane request to the metadata +// replica on shard 0. Plain 0 falls into namespace hashing and can land a +// Register on a peer shard that has no consensus instance. +const MetadataConsensusNamespace uint64 = 1 << 63 + +// Identifier kinds in the wire prefix [kind u8][len u8][value]. +const ( + identifierKindNumeric = 1 + identifierKindString = 2 +) + +// partitioningPartitionID is the only Partitioning kind that carries an +// explicit partition, which is what routing needs. +const partitioningPartitionID = 2 + +// PackNamespace packs stream, topic and partition ids into a routing +// namespace. +func PackNamespace(streamID, topicID, partitionID uint32) (uint64, error) { + if err := validateNamespaceField(streamID, MaxStreams); err != nil { + return 0, err + } + if err := validateNamespaceField(topicID, MaxTopics); err != nil { + return 0, err + } + if err := validateNamespaceField(partitionID, MaxPartitions); err != nil { + return 0, err + } + return uint64(streamID)< 0: + return peekedIdentifier{named: true, length: 2 + length}, nil + default: + return peekedIdentifier{}, ierror.ErrInvalidCommand + } +} + +func validateNamespaceField(value uint32, exclusiveMax uint32) error { + if value >= exclusiveMax { + return ierror.ErrInvalidIdentifier + } + return nil +} + +func namespaceFromIdentifiers(stream, topic peekedIdentifier, partitionID uint32) (uint64, error) { + if stream.named || topic.named { + return 0, nil + } + return PackNamespace(stream.numeric, topic.numeric, partitionID) +} + +// namespaceFromSendMessages peeks +// [metadata_len u32][stream ident][topic ident][partitioning kind u8, len u8, value]. +// Only explicit PartitionId partitioning is routable: the broker never picks a +// partition under consensus. +func namespaceFromSendMessages(payload []byte) (uint64, error) { + if len(payload) < 4 { + return 0, ierror.ErrInvalidCommand + } + metadataLength := uint64(binary.LittleEndian.Uint32(payload)) + if uint64(len(payload)) < 4+metadataLength { + return 0, ierror.ErrInvalidCommand + } + // A read past the declared metadata region must fail rather than spill + // into message bytes and derive a namespace the server would not compute. + metadata := payload[4 : 4+metadataLength] + + offset := 0 + stream, err := peekIdentifier(metadata, offset) + if err != nil { + return 0, err + } + offset += stream.length + topic, err := peekIdentifier(metadata, offset) + if err != nil { + return 0, err + } + offset += topic.length + + if len(metadata) < offset+2 { + return 0, ierror.ErrInvalidCommand + } + partitioningKind := metadata[offset] + partitioningLength := int(metadata[offset+1]) + if partitioningKind != partitioningPartitionID { + return 0, ierror.ErrFeatureUnavailable + } + if partitioningLength != 4 || len(metadata) < offset+2+4 { + return 0, ierror.ErrInvalidCommand + } + partitionID := binary.LittleEndian.Uint32(metadata[offset+2:]) + + return namespaceFromIdentifiers(stream, topic, partitionID) +} + +// namespaceFromConsumerOffset peeks +// [consumer kind u8][consumer ident][stream ident][topic ident] +// [partition flag u8][partition u32]. The v1 and v2 request layouts share this +// prefix and differ only in the trailing fields, which routing ignores. +func namespaceFromConsumerOffset(payload []byte) (uint64, error) { + if len(payload) < 1 || (payload[0] != 1 && payload[0] != 2) { + return 0, ierror.ErrInvalidCommand + } + offset := 1 + consumer, err := peekIdentifier(payload, offset) + if err != nil { + return 0, err + } + offset += consumer.length + stream, err := peekIdentifier(payload, offset) + if err != nil { + return 0, err + } + offset += stream.length + topic, err := peekIdentifier(payload, offset) + if err != nil { + return 0, err + } + offset += topic.length + + if len(payload) < offset+5 { + return 0, ierror.ErrInvalidCommand + } + if payload[offset] != 1 { + return 0, ierror.ErrInvalidIdentifier + } + partitionID := binary.LittleEndian.Uint32(payload[offset+1:]) + + return namespaceFromIdentifiers(stream, topic, partitionID) +} + +// namespaceFromDeleteSegments peeks +// [stream ident][topic ident][partition u32]. +func namespaceFromDeleteSegments(payload []byte) (uint64, error) { + offset := 0 + stream, err := peekIdentifier(payload, offset) + if err != nil { + return 0, err + } + offset += stream.length + topic, err := peekIdentifier(payload, offset) + if err != nil { + return 0, err + } + offset += topic.length + + if len(payload) < offset+4 { + return 0, ierror.ErrInvalidCommand + } + partitionID := binary.LittleEndian.Uint32(payload[offset:]) + + return namespaceFromIdentifiers(stream, topic, partitionID) +} diff --git a/foreign/go/internal/vsr/namespace_test.go b/foreign/go/internal/vsr/namespace_test.go new file mode 100644 index 0000000000..2a45b63768 --- /dev/null +++ b/foreign/go/internal/vsr/namespace_test.go @@ -0,0 +1,376 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package vsr + +import ( + "encoding/binary" + "fmt" + "testing" + + ierror "github.com/apache/iggy/foreign/go/errors" + "github.com/apache/iggy/foreign/go/internal/command" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// numericIdentifier builds the [kind u8][len u8][value u32] prefix. +func numericIdentifier(id uint32) []byte { + out := []byte{identifierKindNumeric, 4} + return binary.LittleEndian.AppendUint32(out, id) +} + +// namedIdentifier builds the [kind u8][len u8][utf-8] prefix. +func namedIdentifier(name string) []byte { + out := []byte{identifierKindString, byte(len(name))} + return append(out, name...) +} + +// sendMessagesPayload builds [metadata len u32][stream][topic][partitioning]. +func sendMessagesPayload(stream, topic, partitioning []byte) []byte { + metadata := make([]byte, 0, len(stream)+len(topic)+len(partitioning)) + metadata = append(metadata, stream...) + metadata = append(metadata, topic...) + metadata = append(metadata, partitioning...) + payload := binary.LittleEndian.AppendUint32(nil, uint32(len(metadata))) + return append(payload, metadata...) +} + +func partitionIDPartitioning(id uint32) []byte { + out := []byte{partitioningPartitionID, 4} + return binary.LittleEndian.AppendUint32(out, id) +} + +// consumerOffsetPayload builds the peeked prefix shared by codes 121 to 124. +func consumerOffsetPayload(stream, topic []byte, partitionID uint32, hasPartition bool) []byte { + payload := []byte{1} + payload = append(payload, numericIdentifier(1)...) + payload = append(payload, stream...) + payload = append(payload, topic...) + if hasPartition { + payload = append(payload, 1) + } else { + payload = append(payload, 0) + } + return binary.LittleEndian.AppendUint32(payload, partitionID) +} + +func TestPackNamespace_PacksTheTripleAtItsShifts(t *testing.T) { + tests := []struct { + name string + stream, topic, partitionKey uint32 + want uint64 + }{ + {name: "zero", want: 0}, + {name: "partition only", partitionKey: 1, want: 1}, + {name: "topic only", topic: 1, want: 1 << 20}, + {name: "stream only", stream: 1, want: 1 << 32}, + {name: "triple", stream: 3, topic: 2, partitionKey: 7, want: 3<<32 | 2<<20 | 7}, + { + name: "maximum", + stream: MaxStreams - 1, + topic: MaxTopics - 1, + partitionKey: MaxPartitions - 1, + want: 4095<<32 | 4095<<20 | 999999, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := PackNamespace(test.stream, test.topic, test.partitionKey) + require.NoError(t, err) + assert.Equal(t, test.want, got) + }) + } +} + +func TestPackNamespace_RejectsOutOfRangeFields(t *testing.T) { + tests := []struct { + name string + stream, topic, partitionKey uint32 + }{ + {name: "stream", stream: MaxStreams}, + {name: "topic", topic: MaxTopics}, + {name: "partition", partitionKey: MaxPartitions}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := PackNamespace(test.stream, test.topic, test.partitionKey) + assert.ErrorIs(t, err, ierror.ErrInvalidIdentifier) + }) + } +} + +func TestNamespaceShifts_MatchTheWireContract(t *testing.T) { + assert.Equal(t, 12, streamBits) + assert.Equal(t, 12, topicBits) + assert.Equal(t, 20, partitionBits) + assert.Equal(t, 0, partitionShift) + assert.Equal(t, 20, topicShift) + assert.Equal(t, 32, streamShift) + assert.Equal(t, uint64(1)<<63, MetadataConsensusNamespace) +} + +func TestNamespaceForRequest_RoutesControlPlaneToTheMetadataSentinel(t *testing.T) { + for _, operation := range []Operation{OperationRegister, OperationLogout} { + got, err := NamespaceForRequest(uint32(command.LoginRegisterCode), nil, operation) + require.NoError(t, err) + assert.Equal(t, MetadataConsensusNamespace, got) + } +} + +func TestNamespaceForRequest_RoutesReadsAndMetadataToZero(t *testing.T) { + tests := []struct { + name string + code command.Code + operation Operation + }{ + {name: "ping", code: command.PingCode, operation: OperationNonReplicated}, + {name: "poll", code: command.PollMessagesCode, operation: OperationNonReplicated}, + {name: "create stream", code: command.CreateStreamCode, operation: OperationCreateStream}, + {name: "join group", code: command.JoinGroupCode, operation: OperationJoinConsumerGroup}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := NamespaceForRequest(uint32(test.code), []byte{0xFF}, test.operation) + require.NoError(t, err) + assert.Zero(t, got) + }) + } +} + +func TestNamespaceForRequest_RejectsAnUnpeekablePartitionOperation(t *testing.T) { + _, err := NamespaceForRequest(uint32(command.PollMessagesCode), nil, OperationSendMessages+64) + assert.ErrorIs(t, err, ierror.ErrFeatureUnavailable) +} + +func TestNamespaceForRequest_SendMessages(t *testing.T) { + t.Run("packs numeric identifiers", func(t *testing.T) { + payload := sendMessagesPayload( + numericIdentifier(3), numericIdentifier(2), partitionIDPartitioning(7)) + got, err := NamespaceForRequest( + uint32(command.SendMessagesCode), payload, OperationSendMessages) + require.NoError(t, err) + assert.Equal(t, uint64(3<<32|2<<20|7), got) + }) + + t.Run("ignores message bytes past the metadata region", func(t *testing.T) { + payload := sendMessagesPayload( + numericIdentifier(1), numericIdentifier(1), partitionIDPartitioning(0)) + payload = append(payload, 0xDE, 0xAD, 0xBE, 0xEF) + got, err := NamespaceForRequest( + uint32(command.SendMessagesCode), payload, OperationSendMessages) + require.NoError(t, err) + assert.Equal(t, uint64(1<<32|1<<20), got) + }) + + t.Run("defers a named stream to the server", func(t *testing.T) { + payload := sendMessagesPayload( + namedIdentifier("orders"), numericIdentifier(2), partitionIDPartitioning(7)) + got, err := NamespaceForRequest( + uint32(command.SendMessagesCode), payload, OperationSendMessages) + require.NoError(t, err) + assert.Zero(t, got) + }) + + t.Run("defers a named topic to the server", func(t *testing.T) { + payload := sendMessagesPayload( + numericIdentifier(3), namedIdentifier("events"), partitionIDPartitioning(7)) + got, err := NamespaceForRequest( + uint32(command.SendMessagesCode), payload, OperationSendMessages) + require.NoError(t, err) + assert.Zero(t, got) + }) + + t.Run("rejects partitioning the client cannot route", func(t *testing.T) { + for _, kind := range []byte{0, 1, 3} { + payload := sendMessagesPayload( + numericIdentifier(1), numericIdentifier(1), []byte{kind, 4, 0, 0, 0, 0}) + _, err := NamespaceForRequest( + uint32(command.SendMessagesCode), payload, OperationSendMessages) + assert.ErrorIs(t, err, ierror.ErrFeatureUnavailable, "kind %d", kind) + } + }) + + t.Run("rejects a partition value of the wrong width", func(t *testing.T) { + payload := sendMessagesPayload( + numericIdentifier(1), numericIdentifier(1), + []byte{partitioningPartitionID, 2, 0, 0}) + _, err := NamespaceForRequest( + uint32(command.SendMessagesCode), payload, OperationSendMessages) + assert.ErrorIs(t, err, ierror.ErrInvalidCommand) + }) + + t.Run("rejects an out-of-range identifier", func(t *testing.T) { + payload := sendMessagesPayload( + numericIdentifier(MaxStreams), numericIdentifier(1), partitionIDPartitioning(0)) + _, err := NamespaceForRequest( + uint32(command.SendMessagesCode), payload, OperationSendMessages) + assert.ErrorIs(t, err, ierror.ErrInvalidIdentifier) + }) + + t.Run("rejects an out-of-range partition", func(t *testing.T) { + payload := sendMessagesPayload( + numericIdentifier(1), numericIdentifier(1), partitionIDPartitioning(MaxPartitions)) + _, err := NamespaceForRequest( + uint32(command.SendMessagesCode), payload, OperationSendMessages) + assert.ErrorIs(t, err, ierror.ErrInvalidIdentifier) + }) + + t.Run("rejects every truncation", func(t *testing.T) { + payload := sendMessagesPayload( + numericIdentifier(1), numericIdentifier(2), partitionIDPartitioning(3)) + for length := range len(payload) { + _, err := NamespaceForRequest( + uint32(command.SendMessagesCode), payload[:length], OperationSendMessages) + assert.Error(t, err, "truncated to %d bytes", length) + } + }) + + t.Run("rejects a metadata length past the payload", func(t *testing.T) { + payload := binary.LittleEndian.AppendUint32(nil, 1024) + payload = append(payload, numericIdentifier(1)...) + _, err := NamespaceForRequest( + uint32(command.SendMessagesCode), payload, OperationSendMessages) + assert.ErrorIs(t, err, ierror.ErrInvalidCommand) + }) +} + +func TestNamespaceForRequest_ConsumerOffsets(t *testing.T) { + variants := []struct { + code command.Code + operation Operation + }{ + {code: command.StoreOffsetCode, operation: OperationStoreConsumerOffset}, + {code: command.DeleteConsumerOffsetCode, operation: OperationDeleteConsumerOffset}, + {code: command.StoreOffset2Code, operation: OperationStoreConsumerOffset2}, + {code: command.DeleteConsumerOffset2Code, operation: OperationDeleteConsumerOffset2}, + } + + for _, variant := range variants { + t.Run(fmt.Sprintf("code_%d", variant.code), func(t *testing.T) { + runConsumerOffsetNamespaceCases(t, uint32(variant.code), variant.operation) + }) + } +} + +func runConsumerOffsetNamespaceCases(t *testing.T, code uint32, operation Operation) { + t.Helper() + + t.Run("packs the triple", func(t *testing.T) { + payload := consumerOffsetPayload( + numericIdentifier(5), numericIdentifier(4), 9, true) + got, err := NamespaceForRequest(code, payload, operation) + require.NoError(t, err) + assert.Equal(t, uint64(5<<32|4<<20|9), got) + }) + + t.Run("ignores the trailing offset and ack bytes", func(t *testing.T) { + payload := consumerOffsetPayload( + numericIdentifier(5), numericIdentifier(4), 9, true) + payload = binary.LittleEndian.AppendUint64(payload, 42) + payload = append(payload, 1) + got, err := NamespaceForRequest(code, payload, operation) + require.NoError(t, err) + assert.Equal(t, uint64(5<<32|4<<20|9), got) + }) + + t.Run("rejects a missing explicit partition", func(t *testing.T) { + payload := consumerOffsetPayload( + numericIdentifier(5), numericIdentifier(4), 0, false) + _, err := NamespaceForRequest(code, payload, operation) + assert.ErrorIs(t, err, ierror.ErrInvalidIdentifier) + }) + + t.Run("defers a named stream to the server", func(t *testing.T) { + payload := consumerOffsetPayload( + namedIdentifier("orders"), numericIdentifier(4), 9, true) + got, err := NamespaceForRequest(code, payload, operation) + require.NoError(t, err) + assert.Zero(t, got) + }) + + t.Run("rejects an unknown consumer kind", func(t *testing.T) { + payload := consumerOffsetPayload( + numericIdentifier(5), numericIdentifier(4), 9, true) + payload[0] = 3 + _, err := NamespaceForRequest(code, payload, operation) + assert.ErrorIs(t, err, ierror.ErrInvalidCommand) + }) + + t.Run("rejects every truncation", func(t *testing.T) { + payload := consumerOffsetPayload( + numericIdentifier(5), numericIdentifier(4), 9, true) + for length := range len(payload) { + _, err := NamespaceForRequest(code, payload[:length], operation) + assert.Error(t, err, "truncated to %d bytes", length) + } + }) +} + +func TestNamespaceForRequest_DeleteSegments(t *testing.T) { + t.Run("packs the triple", func(t *testing.T) { + payload := append(numericIdentifier(6), numericIdentifier(5)...) + payload = binary.LittleEndian.AppendUint32(payload, 4) + got, err := NamespaceForRequest( + uint32(command.DeleteSegmentsCode), payload, OperationDeleteSegments) + require.NoError(t, err) + assert.Equal(t, uint64(6<<32|5<<20|4), got) + }) + + t.Run("rejects every truncation", func(t *testing.T) { + payload := append(numericIdentifier(6), numericIdentifier(5)...) + payload = binary.LittleEndian.AppendUint32(payload, 4) + for length := range len(payload) { + _, err := NamespaceForRequest( + uint32(command.DeleteSegmentsCode), payload[:length], OperationDeleteSegments) + assert.Error(t, err, "truncated to %d bytes", length) + } + }) +} + +func TestPeekIdentifier_RejectsMalformedPrefixes(t *testing.T) { + tests := []struct { + name string + payload []byte + }{ + {name: "empty", payload: nil}, + {name: "kind only", payload: []byte{identifierKindNumeric}}, + {name: "numeric of the wrong width", payload: []byte{identifierKindNumeric, 2, 0, 0}}, + {name: "empty name", payload: []byte{identifierKindString, 0}}, + {name: "unknown kind", payload: []byte{9, 4, 0, 0, 0, 0}}, + {name: "value past the end", payload: []byte{identifierKindNumeric, 4, 0, 0}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := peekIdentifier(test.payload, 0) + assert.ErrorIs(t, err, ierror.ErrInvalidCommand) + }) + } +} + +func TestPeekIdentifier_ReportsTheConsumedLength(t *testing.T) { + numeric, err := peekIdentifier(numericIdentifier(9), 0) + require.NoError(t, err) + assert.Equal(t, uint32(9), numeric.numeric) + assert.False(t, numeric.named) + assert.Equal(t, 6, numeric.length) + + named, err := peekIdentifier(namedIdentifier("abc"), 0) + require.NoError(t, err) + assert.True(t, named.named) + assert.Equal(t, 5, named.length) +} diff --git a/foreign/go/internal/vsr/protocol_parity_test.go b/foreign/go/internal/vsr/protocol_parity_test.go index b510c8eaba..f995ce25bc 100644 --- a/foreign/go/internal/vsr/protocol_parity_test.go +++ b/foreign/go/internal/vsr/protocol_parity_test.go @@ -46,8 +46,10 @@ var rustSources = map[string]string{ "header": "core/binary_protocol/src/consensus/header.rs", "command": "core/binary_protocol/src/consensus/command.rs", "operation": "core/binary_protocol/src/consensus/operation.rs", + "namespace": "core/binary_protocol/src/namespace.rs", "cargo": "core/binary_protocol/Cargo.toml", "eviction": "core/common/src/error/eviction.rs", + "sdk": "core/sdk/src/vsr.rs", } // goOperations names every discriminant the codec declares. It exists so the @@ -129,6 +131,7 @@ var goHeaderOffsets = map[string]map[string]int{ "timestamp": requestOffsetTimestamp, "request": requestOffsetRequest, "operation": requestOffsetOperation, + "namespace": requestOffsetNamespace, "session": requestOffsetSession, "reserved": requestOffsetReserved, }, @@ -136,6 +139,7 @@ var goHeaderOffsets = map[string]map[string]int{ "size": replyOffsetSize, "command": replyOffsetCommand, "operation": replyOffsetOperation, + "namespace": replyOffsetNamespace, "status": replyOffsetStatus, }, "EvictionHeader": { @@ -329,6 +333,47 @@ func TestProtocolParity_ReplicatedOperationMap(t *testing.T) { assert.Equal(t, want, replicatedOperation) } +func TestProtocolParity_NamespaceLayout(t *testing.T) { + sources := loadRustSources(t) + limits := namedNumbers(sources["namespace"], + regexp.MustCompile(`(?m)^pub const (MAX_[A-Z]+): usize = ([0-9_]+);$`)) + + require.Equal(t, uint64(MaxStreams), limits["MAX_STREAMS"]) + require.Equal(t, uint64(MaxTopics), limits["MAX_TOPICS"]) + require.Equal(t, uint64(MaxPartitions), limits["MAX_PARTITIONS"]) + + // Rust derives the shifts from the limits rather than declaring literals, + // so the parity assertion derives them the same way. + assert.Equal(t, bitsRequired(limits["MAX_STREAMS"]-1), streamBits) + assert.Equal(t, bitsRequired(limits["MAX_TOPICS"]-1), topicBits) + assert.Equal(t, bitsRequired(limits["MAX_PARTITIONS"]-1), partitionBits) + + literalShift := namedNumbers(sources["namespace"], + regexp.MustCompile(`(?m)^pub const (PARTITION_SHIFT): u32 = ([0-9]+);$`)) + require.Contains(t, literalShift, "PARTITION_SHIFT") + assert.Equal(t, uint64(partitionShift), literalShift["PARTITION_SHIFT"]) + assert.Equal(t, partitionShift+partitionBits, topicShift) + assert.Equal(t, topicShift+topicBits, streamShift) + + packed, err := PackNamespace(1, 1, 1) + require.NoError(t, err) + assert.Equal(t, uint64(1)< 0 { + bits++ + value >>= 1 + } + return bits +} + func TestProtocolParity_EvictionReasons(t *testing.T) { sources := loadRustSources(t) rustValues := rustEnumValues(sources["header"], "EvictionReason") @@ -516,19 +561,48 @@ func TestProtocolParity_EvictionReasonMapping(t *testing.T) { // dedicated eviction tests in reply_test.go. } -// The client wire carries no routing namespace: the server derives the -// consensus group (plane from the operation, partition target from the -// payload) and stamps it into its own internal header, so this SDK has no -// packing rules to mirror. Growing the field back would move every field -// behind it, which is what makes this worth asserting on its own rather than -// leaving to the offset recomputation. -func TestProtocolParity_ClientHeadersCarryNoNamespace(t *testing.T) { +func TestProtocolParity_NamespaceRouting(t *testing.T) { sources := loadRustSources(t) + codeValues := rustCommandCodes(sources["codes"]) + require.NotEmpty(t, codeValues) + + body := captureBlock(sources["sdk"], `fn namespace_for_request\(`) + require.NotEmpty(t, body, "the Rust namespace_for_request routing was not found") + + armed := make(map[uint32]string) + for _, arm := range regexp.MustCompile(`(?m)^\s*([A-Z0-9_]+_CODE) => \{`). + FindAllStringSubmatch(body, -1) { + value, ok := codeValues[arm[1]] + require.True(t, ok, "unknown command constant %s", arm[1]) + armed[uint32(value)] = arm[1] + } + require.NotEmpty(t, armed, "no payload-peek arms were parsed") + + for name, value := range codeValues { + code := uint32(value) + operation := OperationForCode(code) + if operation == OperationRegister || operation == OperationLogout || + operation == OperationNonReplicated || IsMetadata(operation) { + continue + } + // A code that reaches the payload peek fails on an empty payload; a + // code Rust does not route is refused outright. The two errors keep + // the routing decisions distinguishable without crafting payloads. + _, err := NamespaceForRequest(code, nil, operation) + if _, peeked := armed[code]; peeked { + assert.ErrorIs(t, err, ierror.ErrInvalidCommand, + "%s must derive its namespace from the payload", name) + } else { + assert.ErrorIs(t, err, ierror.ErrFeatureUnavailable, + "%s must be refused rather than routed blindly", name) + } + } - for _, structName := range []string{"RequestHeader", "ReplyHeader"} { - offsets := rustStructOffsets(t, sources["header"], structName) - assert.NotContains(t, offsets, "namespace", - "%s must not carry a routing namespace", structName) + for code, name := range armed { + operation := OperationForCode(code) + shortCircuited := operation == OperationRegister || operation == OperationLogout || + operation == OperationNonReplicated || IsMetadata(operation) + assert.False(t, shortCircuited, "%s never reaches the namespace peek in Go", name) } } diff --git a/foreign/go/tests/e2e_helpers_test.go b/foreign/go/tests/e2e_helpers_test.go index 75e680c24d..505c63b852 100644 --- a/foreign/go/tests/e2e_helpers_test.go +++ b/foreign/go/tests/e2e_helpers_test.go @@ -19,12 +19,13 @@ // // The suite skips unless IGGY_TCP_ADDRESS points at a server. Start one with: // -// cargo build --bin iggy-server +// # TODO: change to iggy-server once legacy server is removed (core/server has VSR support) +// cargo build --bin iggy-server-ng --features vsr // IGGY_SYSTEM_PATH=/tmp/iggy-go-e2e \ // IGGY_TCP_ADDRESS=127.0.0.1:8090 \ // IGGY_HTTP_ENABLED=false IGGY_QUIC_ENABLED=false IGGY_WEBSOCKET_ENABLED=false \ // IGGY_ROOT_USERNAME=iggy IGGY_ROOT_PASSWORD=iggy \ -// target/debug/iggy-server +// target/debug/iggy-server-ng // // IGGY_TCP_ADDRESS=127.0.0.1:8090 go test ./tests // diff --git a/foreign/java/README.md b/foreign/java/README.md index fa3fdbd345..83ee696eb0 100644 --- a/foreign/java/README.md +++ b/foreign/java/README.md @@ -183,6 +183,7 @@ var client = Iggy.tcpClientBuilder() .port(8090) .connectionTimeout(Duration.ofSeconds(10)) .requestTimeout(Duration.ofSeconds(30)) + .connectionPoolSize(10) .retryPolicy(RetryPolicy.exponentialBackoff()) .credentials("iggy", "iggy") .buildAndLogin(); @@ -212,7 +213,7 @@ See the **[Java Examples](../../examples/java/)** directory for runnable applica - **BlockingProducer**: synchronous message production with batch sending - **BlockingConsumer**: synchronous consumption with polling loops -- **AsyncProducer**: non-blocking batch production with concurrent request submission +- **AsyncProducer**: high-throughput async production with pipelining - **AsyncConsumer**: async consumption with backpressure and error recovery Each example includes comprehensive documentation on when to use blocking vs. async clients, CompletableFuture patterns, thread pool management, and performance characteristics. diff --git a/foreign/java/bench/src/main/java/org/apache/iggy/bench/benchmarks/actors/tcp/async/TcpAsyncPinnedProducerActor.java b/foreign/java/bench/src/main/java/org/apache/iggy/bench/benchmarks/actors/tcp/async/TcpAsyncPinnedProducerActor.java index e764697131..479b7a1a2c 100644 --- a/foreign/java/bench/src/main/java/org/apache/iggy/bench/benchmarks/actors/tcp/async/TcpAsyncPinnedProducerActor.java +++ b/foreign/java/bench/src/main/java/org/apache/iggy/bench/benchmarks/actors/tcp/async/TcpAsyncPinnedProducerActor.java @@ -100,6 +100,7 @@ public CompletableFuture run() { rateLimit); return AsyncIggyTcpClient.builder() .credentials(globalCliArgs.username(), globalCliArgs.password()) + .connectionPoolSize(1) .buildAndLogin() .thenCompose(client -> { this.client = client; diff --git a/foreign/java/bench/src/main/java/org/apache/iggy/bench/report/ServerStatsCollector.java b/foreign/java/bench/src/main/java/org/apache/iggy/bench/report/ServerStatsCollector.java index 7ffb009c7e..fd99ac1483 100644 --- a/foreign/java/bench/src/main/java/org/apache/iggy/bench/report/ServerStatsCollector.java +++ b/foreign/java/bench/src/main/java/org/apache/iggy/bench/report/ServerStatsCollector.java @@ -41,6 +41,7 @@ public ServerStatsCollector(GlobalCliArgs globalCliArgs) { public BenchmarkServerStats collect() { try (var client = IggyTcpClient.builder() .credentials(globalCliArgs.username(), globalCliArgs.password()) + .connectionPoolSize(1) .buildAndLogin()) { Stats stats = client.system().getStats(); Map cacheMetrics = new HashMap<>(); diff --git a/foreign/java/external-processors/iggy-connector-flink/iggy-connector-library/src/main/java/org/apache/iggy/connector/flink/source/IggySource.java b/foreign/java/external-processors/iggy-connector-flink/iggy-connector-library/src/main/java/org/apache/iggy/connector/flink/source/IggySource.java index f4402e9184..82fead0006 100644 --- a/foreign/java/external-processors/iggy-connector-flink/iggy-connector-library/src/main/java/org/apache/iggy/connector/flink/source/IggySource.java +++ b/foreign/java/external-processors/iggy-connector-flink/iggy-connector-library/src/main/java/org/apache/iggy/connector/flink/source/IggySource.java @@ -168,6 +168,7 @@ private AsyncIggyTcpClient createAsyncIggyClient() { .retryPolicy(RetryPolicy.fixedDelay( connectionConfig.getMaxRetries(), connectionConfig.getRetryBackoff())) .tls(connectionConfig.isEnableTls()) + .connectionPoolSize(4) .buildAndLogin() .join(); diff --git a/foreign/java/external-processors/iggy-connector-flink/iggy-flink-examples/src/test/java/org/apache/iggy/flink/example/AsyncTcpMessageSendTest.java b/foreign/java/external-processors/iggy-connector-flink/iggy-flink-examples/src/test/java/org/apache/iggy/flink/example/AsyncTcpMessageSendTest.java index ffa1f85528..1cd0ab3b0c 100644 --- a/foreign/java/external-processors/iggy-connector-flink/iggy-flink-examples/src/test/java/org/apache/iggy/flink/example/AsyncTcpMessageSendTest.java +++ b/foreign/java/external-processors/iggy-connector-flink/iggy-flink-examples/src/test/java/org/apache/iggy/flink/example/AsyncTcpMessageSendTest.java @@ -24,7 +24,6 @@ import org.apache.iggy.identifier.TopicId; import org.apache.iggy.message.Message; import org.apache.iggy.message.Partitioning; -import org.apache.iggy.message.SendMessagesResponse; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.DisplayName; @@ -117,7 +116,7 @@ void testSendSingleMessage() throws ExecutionException, InterruptedException { List messages = new ArrayList<>(); messages.add(message); - CompletableFuture sendFuture = + CompletableFuture sendFuture = client.messages().sendMessages(streamId, topicId, Partitioning.balanced(), messages); sendFuture.get(); @@ -143,7 +142,7 @@ void testSendBatchMessages() throws ExecutionException, InterruptedException { messages.add(Message.of(content)); } - CompletableFuture sendFuture = + CompletableFuture sendFuture = client.messages().sendMessages(streamId, topicId, Partitioning.balanced(), messages); sendFuture.get(); @@ -167,7 +166,7 @@ void testSendToSpecificPartition() throws ExecutionException, InterruptedExcepti List messages = new ArrayList<>(); messages.add(message); - CompletableFuture sendFuture = + CompletableFuture sendFuture = client.messages().sendMessages(streamId, topicId, Partitioning.partitionId(targetPartition), messages); sendFuture.get(); @@ -191,7 +190,7 @@ void testSendWithMessageKey() throws ExecutionException, InterruptedException { messages.add(message); String messageKey = "test-key-123"; - CompletableFuture sendFuture = + CompletableFuture sendFuture = client.messages().sendMessages(streamId, topicId, Partitioning.messagesKey(messageKey), messages); sendFuture.get(); @@ -215,7 +214,7 @@ void testSendJsonMessages() throws ExecutionException, InterruptedException { List messages = new ArrayList<>(); messages.add(message); - CompletableFuture sendFuture = + CompletableFuture sendFuture = client.messages().sendMessages(streamId, topicId, Partitioning.balanced(), messages); sendFuture.get(); @@ -233,7 +232,7 @@ void testSendMultipleMessagesInParallel() throws ExecutionException, Interrupted TopicId topicId = TopicId.of("lines"); int parallelRequests = 5; - List> futures = new ArrayList<>(); + List> futures = new ArrayList<>(); for (int i = 0; i < parallelRequests; i++) { String content = "Parallel message #" + i; @@ -241,7 +240,7 @@ void testSendMultipleMessagesInParallel() throws ExecutionException, Interrupted List messages = new ArrayList<>(); messages.add(message); - CompletableFuture future = + CompletableFuture future = client.messages().sendMessages(streamId, topicId, Partitioning.balanced(), messages); futures.add(future); } @@ -270,7 +269,7 @@ void testSendLargeBatch() throws ExecutionException, InterruptedException { } long startTime = System.currentTimeMillis(); - CompletableFuture sendFuture = + CompletableFuture sendFuture = client.messages().sendMessages(streamId, topicId, Partitioning.balanced(), messages); sendFuture.get(); long duration = System.currentTimeMillis() - startTime; diff --git a/foreign/java/external-processors/iggy-connector-pinot/integration-test.sh b/foreign/java/external-processors/iggy-connector-pinot/integration-test.sh index f83c5f8ec1..e438afe4bc 100755 --- a/foreign/java/external-processors/iggy-connector-pinot/integration-test.sh +++ b/foreign/java/external-processors/iggy-connector-pinot/integration-test.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/foreign/java/gradle.properties b/foreign/java/gradle.properties index 5747b90f8a..f9eb048aea 100644 --- a/foreign/java/gradle.properties +++ b/foreign/java/gradle.properties @@ -15,5 +15,5 @@ # specific language governing permissions and limitations # under the License. -version=0.9.0-SNAPSHOT +version=0.8.3-SNAPSHOT group=org.apache.iggy diff --git a/foreign/java/gradle/libs.versions.toml b/foreign/java/gradle/libs.versions.toml index 6b5b54d941..ed00082551 100644 --- a/foreign/java/gradle/libs.versions.toml +++ b/foreign/java/gradle/libs.versions.toml @@ -30,7 +30,7 @@ jackson2 = "2.22.1" commons-lang3 = "3.20.0" # HTTP Client -httpclient5 = "5.6.3" +httpclient5 = "5.6.2" # Logging slf4j = "2.0.18" diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/ConsumerGroupsClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/ConsumerGroupsClient.java index a8f2d0dba3..9a8af01083 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/ConsumerGroupsClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/ConsumerGroupsClient.java @@ -20,7 +20,6 @@ package org.apache.iggy.client.async; import org.apache.iggy.consumergroup.ConsumerGroup; -import org.apache.iggy.consumergroup.ConsumerGroupAssignment; import org.apache.iggy.consumergroup.ConsumerGroupDetails; import org.apache.iggy.identifier.ConsumerId; import org.apache.iggy.identifier.StreamId; @@ -182,26 +181,4 @@ default CompletableFuture deleteConsumerGroup(Long streamId, Long topicId, * group */ CompletableFuture leaveConsumerGroup(StreamId streamId, TopicId topicId, ConsumerId groupId); - - /** - * Fetches this client's current partition assignment for a consumer group. - * - *

The server identifies the member by the connection's session, so no - * member identifier is sent. The returned assignment carries the group - * generation; it advances on every rebalance, at which point cached - * assignments become stale and polls against revoked partitions are - * fenced by the server. - * - *

Group polling calls this internally; explicit calls are only needed - * for custom partition-selection logic. - * - * @param streamId the stream identifier containing the topic - * @param topicId the topic identifier - * @param groupId the consumer group identifier - * @return a {@link CompletableFuture} completing with the member's - * {@link ConsumerGroupAssignment}, or empty when this client is - * not a member of the group - */ - CompletableFuture> syncConsumerGroup( - StreamId streamId, TopicId topicId, ConsumerId groupId); } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/MessagesClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/MessagesClient.java index f3b9ad115b..8215665b5c 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/MessagesClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/MessagesClient.java @@ -26,7 +26,6 @@ import org.apache.iggy.message.Partitioning; import org.apache.iggy.message.PolledMessages; import org.apache.iggy.message.PollingStrategy; -import org.apache.iggy.message.SendMessagesResponse; import java.util.List; import java.util.Optional; @@ -84,10 +83,8 @@ public interface MessagesClient { * * @param streamId the stream identifier (numeric or string-based) * @param topicId the topic identifier (numeric or string-based) - * @param partitionId optional partition ID to poll from; when empty and polling with a - * group consumer, the client selects the partition round-robin from - * the member's synced group assignment (the group must be joined - * first) + * @param partitionId optional partition ID to poll from; if empty, the server selects + * the partition (required when using consumer groups) * @param consumer the consumer identity, either individual ({@link Consumer#of(Long)}) * or group ({@link Consumer#group(Long)}) * @param strategy the polling strategy controlling where to start reading @@ -151,11 +148,6 @@ default CompletableFuture pollMessages( * messages with the same key always go to the same partition, preserving order * * - *

Over TCP the VSR broker routes explicit partitions only, so balanced and - * key-based partitioning are resolved to a concrete partition by the client - * (round-robin cursor and {@code xxh32(key) % partitionCount} respectively), - * consistently with the other Iggy SDKs. - * *

Messages are batched into a single network request for efficiency. For high * throughput, accumulate messages and send them in larger batches rather than one * at a time. @@ -164,11 +156,11 @@ default CompletableFuture pollMessages( * @param topicId the topic identifier (numeric or string-based) * @param partitioning the partitioning strategy for routing messages * @param messages the list of messages to send - * @return a {@link CompletableFuture} that completes with the {@link SendMessagesResponse} - * confirming the partition and base offset of the committed batch + * @return a {@link CompletableFuture} that completes when all messages have been + * acknowledged by the server * @throws org.apache.iggy.exception.IggyException if the stream or topic does not exist */ - CompletableFuture sendMessages( + CompletableFuture sendMessages( StreamId streamId, TopicId topicId, Partitioning partitioning, List messages); /** @@ -181,9 +173,9 @@ CompletableFuture sendMessages( * @param topicId the numeric topic ID * @param partitioning the partitioning strategy * @param messages the list of messages to send - * @return a {@link CompletableFuture} that completes with the {@link SendMessagesResponse} + * @return a {@link CompletableFuture} that completes when messages are acknowledged */ - default CompletableFuture sendMessages( + default CompletableFuture sendMessages( Long streamId, Long topicId, Partitioning partitioning, List messages) { return sendMessages(StreamId.of(streamId), TopicId.of(topicId), partitioning, messages); } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java index 04f46ec492..5a78acdbaf 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java @@ -19,9 +19,7 @@ package org.apache.iggy.client.async.tcp; -import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; -import io.netty.channel.ConnectTimeoutException; import org.apache.iggy.IggyVersion; import org.apache.iggy.client.ConnectionInfo; import org.apache.iggy.client.async.ConsumerGroupsClient; @@ -35,26 +33,20 @@ import org.apache.iggy.client.async.UsersClient; import org.apache.iggy.client.async.tcp.AsyncTcpConnection.TcpConnectionPoolConfig; import org.apache.iggy.client.async.tcp.LeaderAwareness.LeaderRedirectionState; -import org.apache.iggy.client.async.tcp.vsr.VsrFrameDecoder; import org.apache.iggy.config.RetryPolicy; import org.apache.iggy.exception.IggyMissingCredentialsException; import org.apache.iggy.exception.IggyNotConnectedException; import org.apache.iggy.exception.IggyServerException; -import org.apache.iggy.exception.IggyTimeoutException; import org.apache.iggy.serde.CommandCode; import org.apache.iggy.user.IdentityInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; -import java.io.IOException; import java.time.Duration; import java.util.Objects; import java.util.Optional; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Executor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.function.Supplier; @@ -113,24 +105,19 @@ public class AsyncIggyTcpClient { private static final int INVALID_COMMAND_ERROR_CODE = 3; private static final Logger log = LoggerFactory.getLogger(AsyncIggyTcpClient.class); - private static final RetryPolicy DEFAULT_RECONNECT_POLICY = RetryPolicy.fixedDelay(12, Duration.ofSeconds(5)); - private final ConnectionInfo seedConnectionInfo; - private final AtomicBoolean reconnecting = new AtomicBoolean(); private final Optional username; private final Optional password; private final Optional connectionTimeout; private final Optional acquireTimeout; private final Optional requestTimeout; - private final Duration heartbeatInterval; - private final int maxVsrFrameSize; + private final Optional connectionPoolSize; private final Optional retryPolicy; private final boolean enableTls; private final Optional tlsCertificate; private final TcpConnectionPoolConfig poolConfig; - private final ClientRoutingState routingState = new ClientRoutingState(); private final AtomicReference connection = new AtomicReference<>(); - private final AtomicReference> loginChain = + private final AtomicReference> redirectChain = new AtomicReference<>(CompletableFuture.completedFuture(null)); private volatile ConnectionInfo connectionInfo; private volatile boolean closed; @@ -153,19 +140,7 @@ public class AsyncIggyTcpClient { * @param port the server port */ public AsyncIggyTcpClient(String host, int port) { - this( - host, - port, - null, - null, - null, - null, - null, - Duration.ofSeconds(5), - VsrFrameDecoder.DEFAULT_MAX_FRAME_SIZE, - null, - false, - Optional.empty()); + this(host, port, null, null, null, null, null, null, null, false, Optional.empty()); } @SuppressWarnings("checkstyle:ParameterNumber") @@ -177,25 +152,23 @@ public AsyncIggyTcpClient(String host, int port) { Duration connectionTimeout, Duration acquireTimeout, Duration requestTimeout, - Duration heartbeatInterval, - int maxVsrFrameSize, + Integer connectionPoolSize, RetryPolicy retryPolicy, boolean enableTls, Optional tlsCertificate) { this.connectionInfo = new ConnectionInfo(host, port); - this.seedConnectionInfo = this.connectionInfo; this.username = Optional.ofNullable(username); this.password = Optional.ofNullable(password); this.connectionTimeout = Optional.ofNullable(connectionTimeout); this.acquireTimeout = Optional.ofNullable(acquireTimeout); this.requestTimeout = Optional.ofNullable(requestTimeout); - this.heartbeatInterval = heartbeatInterval; - this.maxVsrFrameSize = maxVsrFrameSize; + this.connectionPoolSize = Optional.ofNullable(connectionPoolSize); this.retryPolicy = Optional.ofNullable(retryPolicy); this.enableTls = enableTls; this.tlsCertificate = tlsCertificate; var poolConfigBuilder = TcpConnectionPoolConfig.builder(); + this.connectionPoolSize.ifPresent(poolConfigBuilder::setMaxConnections); this.acquireTimeout.ifPresent(timeout -> poolConfigBuilder.setAcquireTimeoutMillis(timeout.toMillis())); this.poolConfig = poolConfigBuilder.build(); } @@ -226,18 +199,18 @@ public CompletableFuture connect() { if (previousConnection != null) { previousConnection.close(); } - routingState.clearAssignments(); Supplier currentConnection = connection::get; return newConnection.connect().thenRun(() -> { log.debug("Connected to {} | {}", target.serverAddress(), IggyVersion.getInstance()); - messagesClient = new MessagesTcpClient(currentConnection, routingState); + messagesClient = new MessagesTcpClient(currentConnection); consumerGroupsClient = new ConsumerGroupsTcpClient(currentConnection); consumerOffsetsClient = new ConsumerOffsetsTcpClient(currentConnection); streamsClient = new StreamsTcpClient(currentConnection); topicsClient = new TopicsTcpClient(currentConnection); - usersClient = new UsersTcpClient(currentConnection, this::loginOnLeader); + usersClient = new UsersTcpClient(currentConnection, this::checkLeaderAndRedirect); systemClient = new SystemTcpClient(currentConnection); - personalAccessTokensClient = new PersonalAccessTokensTcpClient(currentConnection, this::loginOnLeader); + personalAccessTokensClient = + new PersonalAccessTokensTcpClient(currentConnection, this::checkLeaderAndRedirect); partitionsClient = new PartitionsTcpClient(currentConnection); }); } @@ -435,9 +408,9 @@ public CompletableFuture close() { } /** - * Returns the server address this client currently targets. Pre-login - * leader discovery can change it, so it may differ from the address the - * client was built with. + * Returns the server address this client currently targets. Leader + * redirection can change it after login, so it may differ from the + * address the client was built with. * * @return the current {@link ConnectionInfo} */ @@ -447,227 +420,48 @@ public ConnectionInfo getConnectionInfo() { private AsyncTcpConnection openConnection(ConnectionInfo target) { return new AsyncTcpConnection( - target.host(), - target.port(), - enableTls, - tlsCertificate, - poolConfig, - connectionTimeout, - requestTimeout, - heartbeatInterval, - maxVsrFrameSize, - this::retryTransientOnLeader, - routingState::clearAssignments, - this::onConnectionFailure); + target.host(), target.port(), enableTls, tlsCertificate, poolConfig, connectionTimeout); } /** - * A not-accepted request was never admitted, so it is safe to recheck the - * leader, restore authentication on a new connection, and retry it within - * the original request deadline. + * Post-login leader check, serialized across concurrent logins: fetches + * the cluster roster and, while a healthy leader lives elsewhere, + * reconnects to it and replays the login. A queued login waits for the + * in-flight redirection and then re-checks the roster, so it either + * confirms the new node or retries a redirection that failed. Every + * failure except the replayed login's own is non-fatal; the client then + * stays on the current node. Each check runs with a fresh redirection + * budget, so hitting the cap parks only that login, not the client. */ - private CompletableFuture retryTransientOnLeader( - AsyncTcpConnection source, - int commandCode, - ByteBuf payload, - long requestDeadlineNanos, - IggyServerException rejection) { - AtomicReference requestPayload = new AtomicReference<>(payload); - Optional authentication = source.authenticationSnapshot(); - AtomicReference authenticationPayload = new AtomicReference<>(authentication - .map(AsyncTcpConnection.AuthenticationSnapshot::payload) - .orElse(null)); - - CompletableFuture ready = prepareTransientFailover( - source, authentication, authenticationPayload, requestDeadlineNanos, rejection); - CompletableFuture retried = ready.thenCompose(ignored -> { - if (requestDeadlineNanos - System.nanoTime() <= 0) { - return CompletableFuture.failedFuture(rejection); - } - AsyncTcpConnection currentConnection = connection.get(); - if (currentConnection == null) { - return CompletableFuture.failedFuture(new IggyNotConnectedException()); - } - return currentConnection.send(commandCode, takePayload(requestPayload), requestDeadlineNanos); - }); - return retried.whenComplete((response, error) -> { - releasePayload(requestPayload); - releasePayload(authenticationPayload); - }); - } - - private CompletableFuture prepareTransientFailover( - AsyncTcpConnection source, - Optional authentication, - AtomicReference authenticationPayload, - long requestDeadlineNanos, - IggyServerException rejection) { + private CompletableFuture checkLeaderAndRedirect(Supplier> reLogin) { CompletableFuture gate = new CompletableFuture<>(); - CompletableFuture previous = loginChain.getAndSet(gate); + CompletableFuture previous = redirectChain.getAndSet(gate); LeaderRedirectionState redirectionState = new LeaderRedirectionState(); - CompletableFuture transaction = previous.thenCompose(ignored -> { - if (closed) { - return CompletableFuture.failedFuture(new IggyNotConnectedException()); - } - if (requestDeadlineNanos - System.nanoTime() <= 0) { - return CompletableFuture.failedFuture(rejection); - } - if (connection.get() != source) { - return CompletableFuture.completedFuture(null); - } - return redirectToLeader(redirectionState).thenCompose(redirected -> { - AsyncTcpConnection currentConnection = connection.get(); - if (currentConnection == null || currentConnection == source || authentication.isEmpty()) { - return CompletableFuture.completedFuture(null); - } - if (requestDeadlineNanos - System.nanoTime() <= 0) { - return CompletableFuture.failedFuture(rejection); - } - return currentConnection - .send( - authentication.orElseThrow().commandCode(), - takePayload(authenticationPayload), - requestDeadlineNanos) - .thenAccept(ByteBuf::release); - }); - }); - transaction.whenComplete((ignored, error) -> gate.complete(null)); - return transaction; - } - - private static ByteBuf takePayload(AtomicReference payload) { - ByteBuf owned = payload.getAndSet(null); - if (owned == null) { - throw new IllegalStateException("Request payload ownership was already transferred"); - } - return owned; - } - - private static void releasePayload(AtomicReference payload) { - ByteBuf owned = payload.getAndSet(null); - if (owned != null) { - owned.release(); - } - } - - /** - * Entry point of the background redial after a pool acquire failure or an - * expired reply. Requests that were in flight stay failed (their outcome - * is unknown); the redial only restores the client for subsequent calls. - * Alternates the current endpoint with the seed, paced by the configured - * retry policy, and replays the builder credentials on the restored - * connection. Personal-access-token logins cannot be replayed here; those - * clients must log in again themselves. - */ - private void onConnectionFailure(Throwable cause) { - if (closed || !isConnectionLoss(cause)) { - return; - } - if (!reconnecting.compareAndSet(false, true)) { - return; - } - log.warn("Connection to {} lost ({}), starting redial", connectionInfo.serverAddress(), cause.getMessage()); - RetryPolicy policy = retryPolicy.orElse(DEFAULT_RECONNECT_POLICY); - redialAttempt(1, policy).whenComplete((ignored, error) -> reconnecting.set(false)); - } - - private static boolean isConnectionLoss(Throwable cause) { - return cause instanceof ConnectTimeoutException - || cause instanceof IOException - || cause instanceof IggyTimeoutException; - } - - private CompletableFuture redialAttempt(int attempt, RetryPolicy policy) { - if (closed) { - return CompletableFuture.completedFuture(null); - } - if (attempt > policy.getMaxRetries()) { - log.error("Redial gave up after {} attempts, next request will fail fast", policy.getMaxRetries()); - return CompletableFuture.completedFuture(null); - } - ConnectionInfo target = ReconnectPlan.target(connectionInfo, seedConnectionInfo, attempt); - Duration delay = ReconnectPlan.delay(policy, attempt); - Executor delayedExecutor = CompletableFuture.delayedExecutor(delay.toMillis(), TimeUnit.MILLISECONDS); - return CompletableFuture.supplyAsync(() -> null, delayedExecutor).thenCompose(ignored -> { - if (closed) { - return CompletableFuture.completedFuture(null); - } - log.info("Redial attempt {}/{} to {}", attempt, policy.getMaxRetries(), target.serverAddress()); - return retarget(target) - .thenCompose(retargeted -> replayLogin()) - .handle((ok, error) -> { - if (error == null) { - log.info("Reconnected to {}", target.serverAddress()); - return CompletableFuture.completedFuture(null); - } - log.warn( - "Redial attempt {} to {} failed: {}", - attempt, - target.serverAddress(), - error.getMessage()); - return redialAttempt(attempt + 1, policy); - }) - .thenCompose(Function.identity()); - }); - } - - /** - * Replays the builder credentials on the freshly published connection. - * The login runs through the users client, so leader discovery retargets - * again before Register when the redialed node is not the leader. - */ - private CompletableFuture replayLogin() { - if (username.isEmpty() || password.isEmpty() || usersClient == null) { - return CompletableFuture.completedFuture(null); - } - return usersClient.login(username.get(), password.get()).thenApply(identity -> null); - } - - /** - * Serializes pre-login leader discovery and Register across concurrent - * logins. A queued login waits for the entire in-flight transaction, then - * checks the roster from the connection that transaction published. Each - * transaction has a fresh redirection budget, so hitting the cap affects - * only that login. Metadata and retargeting failures retain best-effort - * behavior and let Register run against the current target. - */ - CompletableFuture loginOnLeader(Supplier> loginAttempt) { - CompletableFuture gate = new CompletableFuture<>(); - CompletableFuture previous = loginChain.getAndSet(gate); - LeaderRedirectionState redirectionState = new LeaderRedirectionState(); - CompletableFuture transaction = previous.thenCompose( - ignored -> redirectToLeader(redirectionState)) - .thenCompose(ignored -> loginAttempt.get()); - CompletableFuture callerFuture = new CompletableFuture<>(); - transaction.whenComplete((identity, error) -> { - gate.complete(null); - if (error != null) { - callerFuture.completeExceptionally(error); - } else { - callerFuture.complete(identity); - } - }); - return callerFuture; + return previous.thenCompose(ignored -> redirectToLeader(reLogin, null, redirectionState)) + .whenComplete((identity, error) -> gate.complete(null)); } /** - * One authentication-independent discovery hop. When the roster names a - * healthy leader elsewhere, reconnect to it and re-check from the new - * node, since mid-election metadata can point at a node that is itself not - * the leader. Register is sent only after this bounded process settles. + * One redirection hop: when the roster names a healthy leader elsewhere, + * reconnect to it, replay the login and re-check from the new node, since + * mid-election metadata can point at a node that is itself not the + * leader. Bounded by the per-check redirection budget. */ - private CompletableFuture redirectToLeader(LeaderRedirectionState redirectionState) { + private CompletableFuture redirectToLeader( + Supplier> reLogin, + IdentityInfo redirectedIdentity, + LeaderRedirectionState redirectionState) { ConnectionInfo currentTarget = connectionInfo; return findLeaderElsewhere(currentTarget).thenCompose(leaderTarget -> { if (leaderTarget.isEmpty()) { - return CompletableFuture.completedFuture(null); + return CompletableFuture.completedFuture(redirectedIdentity); } if (!redirectionState.canRedirect()) { log.warn( "Maximum leader redirections ({}) reached, connection will continue on server node {}", LeaderAwareness.MAX_LEADER_REDIRECTS, currentTarget.serverAddress()); - return CompletableFuture.completedFuture(null); + return CompletableFuture.completedFuture(redirectedIdentity); } return retarget(leaderTarget.get()) .handle((ignored, error) -> { @@ -678,10 +472,11 @@ private CompletableFuture redirectToLeader(LeaderRedirectionState redirect leaderTarget.get().serverAddress(), error.getMessage(), currentTarget.serverAddress()); - return CompletableFuture.completedFuture(null); + return CompletableFuture.completedFuture(redirectedIdentity); } redirectionState.recordRedirect(); - return redirectToLeader(redirectionState); + return reLogin.get() + .thenCompose(identity -> redirectToLeader(reLogin, identity, redirectionState)); }) .thenCompose(Function.identity()); }); @@ -693,7 +488,7 @@ private CompletableFuture redirectToLeader(LeaderRedirectionState redirect * (metadata fetch, malformed roster) so the redirection path never fails * the login that triggered it. */ - CompletableFuture> findLeaderElsewhere(ConnectionInfo currentTarget) { + private CompletableFuture> findLeaderElsewhere(ConnectionInfo currentTarget) { SystemClient currentSystemClient = systemClient; if (currentSystemClient == null) { return CompletableFuture.completedFuture(Optional.empty()); @@ -701,7 +496,7 @@ CompletableFuture> findLeaderElsewhere(ConnectionInfo c return LeaderAwareness.findLeaderElsewhere(currentSystemClient::getClusterMetadata, currentTarget); } - CompletableFuture retarget(ConnectionInfo newTarget) { + private CompletableFuture retarget(ConnectionInfo newTarget) { AsyncTcpConnection oldConnection = connection.get(); AsyncTcpConnection newConnection; try { @@ -732,7 +527,6 @@ private CompletableFuture publishConnection( new IggyNotConnectedException("Client closed during leader redirection")); } connectionInfo = newTarget; - routingState.clearAssignments(); oldConnection.close().whenComplete((ignored, closeError) -> { if (closeError != null) { log.warn("Failed to close previous connection: {}", closeError.getMessage()); diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientBuilder.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientBuilder.java index e6a6ed510b..85664e5cf9 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientBuilder.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientBuilder.java @@ -20,8 +20,6 @@ package org.apache.iggy.client.async.tcp; import org.apache.commons.lang3.StringUtils; -import org.apache.iggy.client.async.tcp.vsr.VsrFrameDecoder; -import org.apache.iggy.client.async.tcp.vsr.VsrHeaders; import org.apache.iggy.config.RetryPolicy; import org.apache.iggy.exception.IggyInvalidArgumentException; import org.apache.iggy.exception.IggyMissingCredentialsException; @@ -74,10 +72,9 @@ public final class AsyncIggyTcpClientBuilder { private File tlsCertificate; private Duration connectionTimeout; private Duration requestTimeout; + private Integer connectionPoolSize; private RetryPolicy retryPolicy; private Duration acquireTimeout; - private Duration heartbeatInterval = Duration.ofSeconds(5); - private long maxVsrFrameSize = VsrFrameDecoder.DEFAULT_MAX_FRAME_SIZE; public AsyncIggyTcpClientBuilder() {} @@ -171,29 +168,6 @@ public AsyncIggyTcpClientBuilder acquireTimeout(Duration acquireTimeout) { return this; } - /** - * Sets how often the client sends a heartbeat while connected. The value - * should not exceed the server's configured heartbeat interval. - * - * @param heartbeatInterval the heartbeat interval - * @return this builder - */ - public AsyncIggyTcpClientBuilder heartbeatInterval(Duration heartbeatInterval) { - this.heartbeatInterval = heartbeatInterval; - return this; - } - - /** - * Sets the largest inbound VSR frame the client will buffer. - * - * @param maxVsrFrameSize maximum frame size in bytes, including the VSR header - * @return this builder - */ - public AsyncIggyTcpClientBuilder maxVsrFrameSize(long maxVsrFrameSize) { - this.maxVsrFrameSize = maxVsrFrameSize; - return this; - } - /** * Sets the connection timeout. * @@ -216,6 +190,17 @@ public AsyncIggyTcpClientBuilder requestTimeout(Duration requestTimeout) { return this; } + /** + * Sets the connection pool size. + * + * @param connectionPoolSize the connection pool size + * @return this builder + */ + public AsyncIggyTcpClientBuilder connectionPoolSize(Integer connectionPoolSize) { + this.connectionPoolSize = connectionPoolSize; + return this; + } + /** * Sets the retry policy. * @@ -237,11 +222,9 @@ public AsyncIggyTcpClientBuilder retryPolicy(RetryPolicy retryPolicy) { public AsyncIggyTcpClient build() { validateHost(); validatePort(); + validateConnectionPoolSize(); validateConnectionTimeout(); validateAcquireTimeout(); - validateRequestTimeout(); - validateHeartbeatInterval(); - validateMaxVsrFrameSize(); return new AsyncIggyTcpClient( host, @@ -251,8 +234,7 @@ public AsyncIggyTcpClient build() { connectionTimeout, acquireTimeout, requestTimeout, - heartbeatInterval, - (int) maxVsrFrameSize, + connectionPoolSize, retryPolicy, enableTls, Optional.ofNullable(tlsCertificate)); @@ -270,6 +252,12 @@ private void validatePort() { } } + private void validateConnectionPoolSize() { + if (connectionPoolSize != null && connectionPoolSize <= 0) { + throw new IggyInvalidArgumentException("Connection pool size cannot by 0 or negative"); + } + } + private void validateConnectionTimeout() { if (connectionTimeout == null) { return; @@ -289,25 +277,6 @@ private void validateAcquireTimeout() { } } - private void validateRequestTimeout() { - if (requestTimeout != null && (requestTimeout.equals(Duration.ZERO) || requestTimeout.isNegative())) { - throw new IggyInvalidArgumentException("RequestTimeout Cannot be 0 or Negative"); - } - } - - private void validateHeartbeatInterval() { - if (heartbeatInterval == null || heartbeatInterval.isZero() || heartbeatInterval.isNegative()) { - throw new IggyInvalidArgumentException("HeartbeatInterval Cannot be null, 0 or Negative"); - } - } - - private void validateMaxVsrFrameSize() { - if (maxVsrFrameSize < VsrHeaders.HEADER_SIZE || maxVsrFrameSize > Integer.MAX_VALUE) { - throw new IggyInvalidArgumentException("MaxVsrFrameSize must be between " + VsrHeaders.HEADER_SIZE + " and " - + Integer.MAX_VALUE + " bytes"); - } - } - /** * Builds, connects, and logs in using the provided credentials. * This is a convenience method equivalent to calling {@code build()}, {@code connect()}, diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncTcpConnection.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncTcpConnection.java index c421947c86..38eb30f05e 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncTcpConnection.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncTcpConnection.java @@ -21,14 +21,15 @@ import io.netty.bootstrap.Bootstrap; import io.netty.buffer.ByteBuf; -import io.netty.buffer.Unpooled; import io.netty.channel.Channel; import io.netty.channel.ChannelFutureListener; +import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelOption; import io.netty.channel.ChannelPipeline; import io.netty.channel.ConnectTimeoutException; import io.netty.channel.IoEventLoopGroup; import io.netty.channel.MultiThreadIoEventLoopGroup; +import io.netty.channel.SimpleChannelInboundHandler; import io.netty.channel.nio.NioIoHandler; import io.netty.channel.pool.AbstractChannelPoolHandler; import io.netty.channel.pool.ChannelHealthChecker; @@ -37,18 +38,12 @@ import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslContextBuilder; import io.netty.util.concurrent.FutureListener; -import io.netty.util.concurrent.ScheduledFuture; -import org.apache.iggy.client.async.tcp.vsr.ConsensusSession; -import org.apache.iggy.client.async.tcp.vsr.VsrFrameDecoder; -import org.apache.iggy.client.async.tcp.vsr.VsrRequestEncoder; -import org.apache.iggy.client.async.tcp.vsr.VsrResponseHandler; import org.apache.iggy.exception.IggyClientException; import org.apache.iggy.exception.IggyConnectionException; import org.apache.iggy.exception.IggyEmptyResponseException; import org.apache.iggy.exception.IggyInvalidArgumentException; import org.apache.iggy.exception.IggyNotConnectedException; import org.apache.iggy.exception.IggyServerException; -import org.apache.iggy.exception.IggyTimeoutException; import org.apache.iggy.exception.IggyTlsException; import org.apache.iggy.serde.CommandCode; import org.slf4j.Logger; @@ -60,13 +55,11 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; +import java.util.Queue; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; -import java.util.function.Consumer; import java.util.function.Function; /** @@ -76,35 +69,12 @@ public class AsyncTcpConnection { private static final Logger log = LoggerFactory.getLogger(AsyncTcpConnection.class); private static final Duration DEFAULT_CONNECTION_TIMEOUT = Duration.ofMillis(3000); - // A missing reply must not hold the single VSR-pinned channel forever. - private static final Duration DEFAULT_REQUEST_TIMEOUT = Duration.ofSeconds(30); - // Transient VSR denials (not-committed / not-accepted) are replayed with - // the same encoded frame so the server's dedup sees the same request id. - // A not-committed outcome is unknown, so it replays for the whole budget. - // A not-accepted deny was refused outright (typically a demoted primary), - // so after a short same-node retry it is handed to the owning client for - // a leader recheck and safe replay; mirrors TRANSIENT_FAILOVER_CHECK_INTERVAL - // in core/sdk/src/tcp/tcp_client.rs. - private static final int TRANSIENT_NOT_COMMITTED = 57; - private static final int TRANSIENT_NOT_ACCEPTED = 58; - private static final long TRANSIENT_RETRY_INTERVAL_MS = 50; - private static final Duration TRANSIENT_RETRY_BUDGET = Duration.ofSeconds(30); - private static final Duration NOT_ACCEPTED_RETRY_BUDGET = Duration.ofSeconds(2); private final IoEventLoopGroup eventLoopGroup; private final FixedChannelPool channelPool; private final AtomicBoolean isClosed = new AtomicBoolean(false); private final AtomicLong authGeneration = new AtomicLong(0); - private final VsrRequestEncoder vsrEncoder; - private final TransientFailoverHandler transientFailoverHandler; - private final Runnable sessionResetListener; - private final Consumer connectionFailureListener; - private final long requestTimeoutNanos; - private final long heartbeatIntervalNanos; - private final Object heartbeatLock = new Object(); private ByteBuf loginPayload; - private ScheduledFuture heartbeatTask; - private boolean heartbeatRunning; private volatile int loginCommandCode; private volatile boolean authenticated = false; @@ -116,40 +86,6 @@ public AsyncTcpConnection( Optional tlsCertificate, TcpConnectionPoolConfig poolConfig, Optional connectionTimeout) { - this( - host, - port, - enableTls, - tlsCertificate, - poolConfig, - connectionTimeout, - Optional.empty(), - Duration.ofSeconds(5), - VsrFrameDecoder.DEFAULT_MAX_FRAME_SIZE, - null, - () -> {}, - ignored -> {}); - } - - @SuppressWarnings("checkstyle:ParameterNumber") - AsyncTcpConnection( - String host, - int port, - boolean enableTls, - Optional tlsCertificate, - TcpConnectionPoolConfig poolConfig, - Optional connectionTimeout, - Optional requestTimeout, - Duration heartbeatInterval, - int maxVsrFrameSize, - TransientFailoverHandler transientFailoverHandler, - Runnable sessionResetListener, - Consumer connectionFailureListener) { - this.transientFailoverHandler = transientFailoverHandler; - this.sessionResetListener = sessionResetListener; - this.connectionFailureListener = connectionFailureListener; - this.requestTimeoutNanos = toTimeoutNanos(requestTimeout.orElse(DEFAULT_REQUEST_TIMEOUT)); - this.heartbeatIntervalNanos = toTimeoutNanos(heartbeatInterval); SslContext sslContext = null; if (enableTls) { try { @@ -161,8 +97,6 @@ public AsyncTcpConnection( } } - ConsensusSession consensusSession = new ConsensusSession(); - this.vsrEncoder = new VsrRequestEncoder(consensusSession); this.eventLoopGroup = new MultiThreadIoEventLoopGroup(NioIoHandler.newFactory()); var bootstrap = new Bootstrap() @@ -174,20 +108,16 @@ public AsyncTcpConnection( .option(ChannelOption.SO_KEEPALIVE, true) .remoteAddress(host, port); - // The VSR session (client id, fence epoch, request counter) is bound - // to one transport connection server-side; sharing it across channels - // would interleave request ids, so the pool holds a single channel. this.channelPool = new FixedChannelPool( bootstrap, - new PoolChannelHandler( - host, port, enableTls, sslContext, consensusSession, maxVsrFrameSize, this::onSessionEvicted), + new PoolChannelHandler(host, port, enableTls, sslContext), ChannelHealthChecker.ACTIVE, FixedChannelPool.AcquireTimeoutAction.FAIL, poolConfig.getAcquireTimeoutMillis(), - 1, + poolConfig.getMaxConnections(), poolConfig.getMaxPendingAcquires()); - log.info("Connection pool initialized with a single VSR-pinned connection"); + log.info("Connection pool initialized with max connections: {}", poolConfig.getMaxConnections()); } /** @@ -197,14 +127,8 @@ public CompletableFuture connect() { CompletableFuture future = new CompletableFuture<>(); channelPool.acquire().addListener((FutureListener) f -> { if (f.isSuccess()) { - channelPool.release(f.getNow()).addListener(release -> { - if (release.isSuccess()) { - startHeartbeat(); - future.complete(null); - } else { - future.completeExceptionally(release.cause()); - } - }); + channelPool.release(f.getNow()); + future.complete(null); } else { Throwable cause = f.cause(); if (cause instanceof ConnectTimeoutException) { @@ -217,62 +141,6 @@ public CompletableFuture connect() { return future; } - private void startHeartbeat() { - synchronized (heartbeatLock) { - if (heartbeatRunning || isClosed.get()) { - return; - } - heartbeatRunning = true; - scheduleNextHeartbeat(); - } - } - - private void scheduleNextHeartbeat() { - synchronized (heartbeatLock) { - if (!heartbeatRunning || isClosed.get()) { - return; - } - heartbeatTask = - eventLoopGroup.next().schedule(this::sendHeartbeat, heartbeatIntervalNanos, TimeUnit.NANOSECONDS); - } - } - - private void sendHeartbeat() { - synchronized (heartbeatLock) { - heartbeatTask = null; - if (!heartbeatRunning || isClosed.get()) { - return; - } - } - CompletableFuture heartbeat; - try { - heartbeat = send(CommandCode.System.PING.getValue(), Unpooled.EMPTY_BUFFER); - } catch (RuntimeException error) { - log.warn("Failed to send heartbeat: {}", error.getMessage()); - scheduleNextHeartbeat(); - return; - } - heartbeat.whenComplete((response, error) -> { - if (response != null) { - response.release(); - } - if (error != null && !isClosed.get()) { - log.warn("Heartbeat failed: {}", error.getMessage()); - } - scheduleNextHeartbeat(); - }); - } - - private void stopHeartbeat() { - synchronized (heartbeatLock) { - heartbeatRunning = false; - if (heartbeatTask != null) { - heartbeatTask.cancel(false); - heartbeatTask = null; - } - } - } - public CompletableFuture exchangeForEntity( CommandCode commandCode, ByteBuf payload, Function func) { return send(commandCode, payload).thenApply(response -> { @@ -325,341 +193,93 @@ public CompletableFuture send(CommandCode commandCode, ByteBuf payload) } public CompletableFuture send(int commandCode, ByteBuf payload) { - return send(commandCode, payload, 0); - } - - CompletableFuture send(int commandCode, ByteBuf payload, long requestDeadlineNanos) { - if (isLoginCode(commandCode) && authenticated) { - return logoutThenLogin(commandCode, payload); - } captureLoginPayloadIfNeeded(commandCode, payload); CompletableFuture responseFuture = new CompletableFuture<>(); CompletableFuture callerFuture = new CompletableFuture<>(); - ByteBuf failoverPayload = - transientFailoverHandler != null && !isLoginCode(commandCode) ? payload.retainedDuplicate() : null; channelPool.acquire().addListener((FutureListener) f -> { if (!f.isSuccess()) { payload.release(); - releaseIfPresent(failoverPayload); - notifyConnectionFailure(f.cause()); callerFuture.completeExceptionally(mapAcquireException(f.cause())); return; } - dispatchAcquiredChannel( - f.getNow(), - commandCode, - payload, - failoverPayload, - responseFuture, - callerFuture, - requestDeadlineNanos); - }); - - return callerFuture; - } - - @SuppressWarnings("checkstyle:ParameterNumber") - private void dispatchAcquiredChannel( - Channel channel, - int commandCode, - ByteBuf payload, - ByteBuf failoverPayload, - CompletableFuture responseFuture, - CompletableFuture callerFuture, - long requestDeadlineNanos) { - Runnable dispatch = () -> dispatchOnChannel( - channel, commandCode, payload, failoverPayload, responseFuture, callerFuture, requestDeadlineNanos); - if (channel.eventLoop().inEventLoop()) { - dispatch.run(); - return; - } - try { - channel.eventLoop().execute(dispatch); - } catch (RejectedExecutionException error) { - payload.release(); - releaseIfPresent(failoverPayload); - releaseChannel(channel); - callerFuture.completeExceptionally(error); - } - } - private void dispatchOnChannel( - Channel channel, - int commandCode, - ByteBuf payload, - ByteBuf failoverPayload, - CompletableFuture responseFuture, - CompletableFuture callerFuture, - long inheritedRequestDeadlineNanos) { - boolean isLoginCommand = isLoginCode(commandCode); - boolean holdLeaseUntilResponse = mutatesSessionState(commandCode); - long requestDeadlineNanos = inheritedRequestDeadlineNanos == 0 - ? System.nanoTime() + requestTimeoutNanos - : inheritedRequestDeadlineNanos; - - responseFuture.whenComplete((response, error) -> completeResponse( - channel, - commandCode, - isLoginCommand, - failoverPayload, - requestDeadlineNanos, - holdLeaseUntilResponse, - callerFuture, - response, - error)); - authenticationStep(channel, commandCode, requestDeadlineNanos) - .whenComplete((ignored, authError) -> completeAuthenticationStep( - channel, - commandCode, - payload, - responseFuture, - requestDeadlineNanos, - holdLeaseUntilResponse, - authError)); - } + Channel channel = f.getNow(); + boolean isLoginCommand = (commandCode == CommandCode.User.LOGIN.getValue() + || commandCode == CommandCode.PersonalAccessToken.LOGIN.getValue()); + boolean requiresAuth = !isLoginCommand + && commandCode != CommandCode.System.PING.getValue() + && commandCode != CommandCode.System.GET_STATS.getValue(); - private CompletableFuture authenticationStep(Channel channel, int commandCode, long requestDeadlineNanos) { - if (isLoginCode(commandCode) || !requiresAuthentication(commandCode)) { - return CompletableFuture.completedFuture(null); - } - if (!authenticated) { - return CompletableFuture.failedFuture(new IggyNotConnectedException("Not authenticated, call login first")); - } - ByteBuf loginPayloadCopy = getLoginPayloadCopy(); - if (loginPayloadCopy == null) { - return CompletableFuture.failedFuture(new IggyNotConnectedException("Not authenticated, call login first")); - } - return IggyAuthenticator.ensureAuthenticated( - channel, - loginPayloadCopy, - authGeneration, - payloadToLogin -> - sendAuthenticationFrame(channel, payloadToLogin, loginCommandCode, requestDeadlineNanos)); - } + responseFuture.whenComplete((response, error) -> { + try { + handlePostResponse(channel, commandCode, isLoginCommand, error); + } catch (RuntimeException bookkeepingError) { + log.error("Post-response bookkeeping failed: {}", bookkeepingError.getMessage()); + } + if (error != null) { + callerFuture.completeExceptionally(error); + } else { + callerFuture.complete(response); + } + }); - @SuppressWarnings("checkstyle:ParameterNumber") - private void completeAuthenticationStep( - Channel channel, - int commandCode, - ByteBuf payload, - CompletableFuture responseFuture, - long requestDeadlineNanos, - boolean holdLeaseUntilResponse, - Throwable authError) { - try { - if (authError != null) { + CompletableFuture authStep; + if (!requiresAuth) { + authStep = CompletableFuture.completedFuture(null); + } else if (!authenticated) { payload.release(); - responseFuture.completeExceptionally(authError); + responseFuture.completeExceptionally( + new IggyNotConnectedException("Not authenticated, call login first")); return; - } - sendFrame(channel, payload, commandCode, responseFuture, requestDeadlineNanos); - } finally { - if (!holdLeaseUntilResponse) { - releaseChannel(channel); - } - } - } - - @SuppressWarnings("checkstyle:ParameterNumber") - private void completeResponse( - Channel channel, - int commandCode, - boolean isLoginCommand, - ByteBuf failoverPayload, - long requestDeadlineNanos, - boolean holdLeaseUntilResponse, - CompletableFuture callerFuture, - ByteBuf response, - Throwable error) { - try { - completeRequest( - channel, - commandCode, - isLoginCommand, - failoverPayload, - requestDeadlineNanos, - callerFuture, - response, - error); - } finally { - if (holdLeaseUntilResponse) { - releaseChannel(channel); - } - } - } - - @SuppressWarnings("checkstyle:ParameterNumber") - private void completeRequest( - Channel channel, - int commandCode, - boolean isLoginCommand, - ByteBuf failoverPayload, - long requestDeadlineNanos, - CompletableFuture callerFuture, - ByteBuf response, - Throwable error) { - try { - handlePostResponse(channel, commandCode, isLoginCommand, error); - } catch (RuntimeException bookkeepingError) { - log.error("Post-response bookkeeping failed: {}", bookkeepingError.getMessage()); - } - if (error == null) { - releaseIfPresent(failoverPayload); - completeWithResponse(callerFuture, response); - return; - } - completeFailedRequest(commandCode, failoverPayload, requestDeadlineNanos, callerFuture, error); - } - - private void completeFailedRequest( - int commandCode, - ByteBuf failoverPayload, - long requestDeadlineNanos, - CompletableFuture callerFuture, - Throwable error) { - IggyTimeoutException timeout = findResponseTimeout(error); - if (timeout != null) { - notifyConnectionFailure(timeout); - } - IggyServerException serverError = findServerError(error); - if (shouldRecheckLeader(serverError, failoverPayload, requestDeadlineNanos)) { - retryAfterLeaderRecheck(commandCode, failoverPayload, requestDeadlineNanos, serverError, callerFuture); - return; - } - releaseIfPresent(failoverPayload); - callerFuture.completeExceptionally(error); - } - - private static boolean shouldRecheckLeader( - IggyServerException serverError, ByteBuf failoverPayload, long requestDeadlineNanos) { - return serverError != null - && serverError.getRawErrorCode() == TRANSIENT_NOT_ACCEPTED - && failoverPayload != null - && requestDeadlineNanos - System.nanoTime() > 0; - } - - private void retryAfterLeaderRecheck( - int commandCode, - ByteBuf payload, - long requestDeadlineNanos, - IggyServerException rejection, - CompletableFuture callerFuture) { - CompletableFuture retry; - try { - retry = transientFailoverHandler.retry(this, commandCode, payload, requestDeadlineNanos, rejection); - } catch (RuntimeException retryError) { - payload.release(); - callerFuture.completeExceptionally(retryError); - return; - } - retry.whenComplete((response, error) -> { - if (error != null) { - callerFuture.completeExceptionally(error); } else { - completeWithResponse(callerFuture, response); + ByteBuf loginPayloadCopy = getLoginPayloadCopy(); + if (loginPayloadCopy == null) { + payload.release(); + responseFuture.completeExceptionally( + new IggyNotConnectedException("Not authenticated, call login first")); + return; + } + authStep = IggyAuthenticator.ensureAuthenticated( + channel, loginPayloadCopy, loginCommandCode, authGeneration); } - }); - } - private CompletableFuture sendAuthenticationFrame( - Channel channel, ByteBuf payload, int commandCode, long requestDeadlineNanos) { - CompletableFuture loginFuture = new CompletableFuture<>(); - sendFrame(channel, payload, commandCode, loginFuture, requestDeadlineNanos); - return loginFuture; - } + authStep.thenRun(() -> sendFrame(channel, payload, commandCode, responseFuture)) + .exceptionally(ex -> { + responseFuture.completeExceptionally(ex); + return null; + }); + }); - /** - * Pool acquire failures and expired replies both make the current target - * unusable. The listener lets the owning client run its redial strategy - * while the failed request surfaces to its caller. - */ - private void notifyConnectionFailure(Throwable cause) { - try { - connectionFailureListener.accept(cause); - } catch (RuntimeException listenerError) { - log.warn("Connection failure listener threw: {}", listenerError.getMessage()); - } + return callerFuture; } private static Throwable mapAcquireException(Throwable cause) { if (cause instanceof IllegalStateException) { return new IggyNotConnectedException("Connection pool is closed"); } - if (cause instanceof TimeoutException) { - return new IggyTimeoutException("Timed out acquiring a connection from the pool", cause); - } return cause; } - /** - * A Register on an already-bound VSR connection is answered with a replay - * of the original register reply, while the client has re-armed a fresh - * identity; its reset request counter would then collide with the - * server's dedup table and mutations would be silently swallowed. Unbind - * first, then login fresh. - */ - private CompletableFuture logoutThenLogin(int commandCode, ByteBuf payload) { - return send(CommandCode.User.LOGOUT.getValue(), Unpooled.EMPTY_BUFFER) - .handle((logoutResponse, logoutError) -> { - if (logoutResponse != null) { - logoutResponse.release(); - } - return null; - }) - .thenCompose(ignored -> send(commandCode, payload)); - } - - private static boolean isLoginCode(int commandCode) { - return commandCode == CommandCode.User.LOGIN.getValue() - || commandCode == CommandCode.PersonalAccessToken.LOGIN.getValue(); - } - - private static boolean mutatesSessionState(int commandCode) { - return isLoginCode(commandCode) || commandCode == CommandCode.User.LOGOUT.getValue(); - } - - /** - * Ping and cluster metadata are the only sessionless bootstrap commands. - * Cluster metadata must be available before Register so a VSR client can - * select the leader; every other non-login command requires a bound - * session. - */ - private static boolean requiresAuthentication(int commandCode) { - return !isAllowedBeforeAuthentication(commandCode); - } - - private static boolean isAllowedBeforeAuthentication(int commandCode) { - return commandCode == CommandCode.System.PING.getValue() - || commandCode == CommandCode.System.GET_CLUSTER_METADATA.getValue(); - } - private void sendFrame( - Channel channel, - ByteBuf payload, - int commandCode, - CompletableFuture responseFuture, - long requestDeadlineNanos) { + Channel channel, ByteBuf payload, int commandCode, CompletableFuture responseFuture) { try { - VsrResponseHandler handler = channel.pipeline().get(VsrResponseHandler.class); + IggyResponseHandler handler = channel.pipeline().get(IggyResponseHandler.class); if (handler == null) { - throw new IggyClientException("Channel missing VsrResponseHandler"); + throw new IggyClientException("Channel missing IggyResponseHandler"); } - ByteBuf frame = vsrEncoder.encode(channel.alloc(), commandCode, payload); - long nowNanos = System.nanoTime(); - long deadlineNanos = nowNanos + TRANSIENT_RETRY_BUDGET.toNanos(); - long notAcceptedDeadlineNanos = - isLoginCode(commandCode) ? deadlineNanos : nowNanos + NOT_ACCEPTED_RETRY_BUDGET.toNanos(); - writeVsrFrame( - channel, - handler, - frame, - responseFuture, - requestDeadlineNanos, - deadlineNanos, - notAcceptedDeadlineNanos, - commandCode); + handler.enqueueRequest(responseFuture); + ByteBuf frame = IggyFrameEncoder.encode(channel.alloc(), commandCode, payload); + + channel.writeAndFlush(frame).addListener((ChannelFutureListener) future -> { + if (!future.isSuccess()) { + log.error("Failed to send frame: {}", future.cause().getMessage()); + responseFuture.completeExceptionally(future.cause()); + } else { + log.trace("Frame sent successfully to {}", channel.remoteAddress()); + } + }); } catch (RuntimeException e) { responseFuture.completeExceptionally(e); } finally { @@ -667,145 +287,6 @@ private void sendFrame( } } - /** - * One VSR write attempt. A transient denial (the cluster could not commit - * or accept yet) replays the SAME encoded frame so the server's dedup - * sees the same request id; everything else resolves the caller. - */ - @SuppressWarnings("checkstyle:ParameterNumber") - private void writeVsrFrame( - Channel channel, - VsrResponseHandler handler, - ByteBuf frame, - CompletableFuture responseFuture, - long requestDeadlineNanos, - long deadlineNanos, - long notAcceptedDeadlineNanos, - int commandCode) { - if (requestDeadlineNanos - System.nanoTime() <= 0) { - IggyTimeoutException timeout = responseTimeout(commandCode); - handler.closeChannel(channel, timeout); - frame.release(); - responseFuture.completeExceptionally(timeout); - return; - } - CompletableFuture attempt = new CompletableFuture<>(); - try { - handler.registerRequest(channel, frame, attempt, requestDeadlineNanos, commandCode); - } catch (RuntimeException error) { - handler.closeChannel(channel, error); - frame.release(); - responseFuture.completeExceptionally(error); - return; - } - channel.writeAndFlush(frame.retainedDuplicate()).addListener((ChannelFutureListener) future -> { - if (!future.isSuccess()) { - log.error("Failed to send frame: {}", future.cause().getMessage()); - // A failed write leaves framing undefined. Closing removes and - // fails every pending entry before the channel can be reused. - handler.closeChannel(channel, future.cause()); - } - }); - attempt.whenComplete((response, error) -> { - if (shouldRetryTransient(error, deadlineNanos, notAcceptedDeadlineNanos) && channel.isActive()) { - try { - channel.eventLoop() - .schedule( - () -> writeVsrFrame( - channel, - handler, - frame, - responseFuture, - requestDeadlineNanos, - deadlineNanos, - notAcceptedDeadlineNanos, - commandCode), - TRANSIENT_RETRY_INTERVAL_MS, - TimeUnit.MILLISECONDS); - return; - } catch (RejectedExecutionException retryRejected) { - log.warn("Event loop rejected a VSR retry, failing the request: {}", retryRejected.getMessage()); - } - } - frame.release(); - if (error != null) { - responseFuture.completeExceptionally(error); - } else { - responseFuture.complete(response); - } - }); - } - - private static IggyTimeoutException responseTimeout(int commandCode) { - return new IggyTimeoutException("Timed out waiting for a response to command code " + commandCode); - } - - private static IggyTimeoutException findResponseTimeout(Throwable error) { - Throwable cause = error; - while (cause != null) { - if (cause instanceof IggyTimeoutException timeout) { - return timeout; - } - cause = cause.getCause(); - } - return null; - } - - private static IggyServerException findServerError(Throwable error) { - Throwable cause = error; - while (cause != null) { - if (cause instanceof IggyServerException serverError) { - return serverError; - } - cause = cause.getCause(); - } - return null; - } - - private static void releaseIfPresent(ByteBuf payload) { - if (payload != null) { - payload.release(); - } - } - - private static void completeWithResponse(CompletableFuture future, ByteBuf response) { - if (!future.complete(response)) { - response.release(); - } - } - - private void releaseChannel(Channel channel) { - channelPool.release(channel).addListener(future -> { - if (!future.isSuccess()) { - log.warn( - "Failed to release VSR channel lease: {}", - future.cause().getMessage()); - channel.close(); - } - }); - } - - private static long toTimeoutNanos(Duration timeout) { - try { - return timeout.toNanos(); - } catch (ArithmeticException ignored) { - return Long.MAX_VALUE; - } - } - - private static boolean shouldRetryTransient(Throwable error, long deadlineNanos, long notAcceptedDeadlineNanos) { - if (!(error instanceof IggyServerException serverError)) { - return false; - } - if (serverError.getRawErrorCode() == TRANSIENT_NOT_COMMITTED) { - return System.nanoTime() < deadlineNanos; - } - if (serverError.getRawErrorCode() == TRANSIENT_NOT_ACCEPTED) { - return System.nanoTime() < notAcceptedDeadlineNanos; - } - return false; - } - private void handlePostResponse(Channel channel, int commandCode, boolean isLoginOp, Throwable ex) { if (isLoginOp) { if (ex == null) { @@ -821,21 +302,12 @@ private void handlePostResponse(Channel channel, int commandCode, boolean isLogi authGeneration.incrementAndGet(); IggyAuthenticator.clearAuthGeneration(channel); } - } - - /** - * A server-side eviction unbinds the transport session and closes its - * channel. Bumping the generation makes the replacement channel re-run - * login and Register. The fresh session invalidates cached routing state - * such as consumer-group assignments. - */ - private void onSessionEvicted() { - authGeneration.incrementAndGet(); - sessionResetListener.run(); + channelPool.release(channel); } private void captureLoginPayloadIfNeeded(int commandCode, ByteBuf payload) { - if (isLoginCode(commandCode)) { + if (commandCode == CommandCode.User.LOGIN.getValue() + || commandCode == CommandCode.PersonalAccessToken.LOGIN.getValue()) { updateLoginPayload(commandCode, payload); } } @@ -855,13 +327,6 @@ private synchronized ByteBuf getLoginPayloadCopy() { return null; } - synchronized Optional authenticationSnapshot() { - if (!authenticated || loginPayload == null) { - return Optional.empty(); - } - return Optional.of(new AuthenticationSnapshot(loginCommandCode, loginPayload.retainedDuplicate())); - } - private synchronized void releaseLoginPayload() { if (this.loginPayload != null) { loginPayload.release(); @@ -873,7 +338,6 @@ public CompletableFuture close() { if (!isClosed.compareAndSet(false, true)) { return CompletableFuture.completedFuture(null); } - stopHeartbeat(); releaseLoginPayload(); CompletableFuture shutdownFuture = new CompletableFuture<>(); channelPool @@ -893,25 +357,12 @@ private static final class PoolChannelHandler extends AbstractChannelPoolHandler private final int port; private final boolean enableTls; private final SslContext sslContext; - private final ConsensusSession consensusSession; - private final int maxVsrFrameSize; - private final Runnable onEviction; - - PoolChannelHandler( - String host, - int port, - boolean enableTls, - SslContext sslContext, - ConsensusSession consensusSession, - int maxVsrFrameSize, - Runnable onEviction) { + + PoolChannelHandler(String host, int port, boolean enableTls, SslContext sslContext) { this.host = host; this.port = port; this.enableTls = enableTls; this.sslContext = sslContext; - this.consensusSession = consensusSession; - this.maxVsrFrameSize = maxVsrFrameSize; - this.onEviction = onEviction; } @Override @@ -920,34 +371,74 @@ public void channelCreated(Channel ch) { if (enableTls) { pipeline.addLast("ssl", sslContext.newHandler(ch.alloc(), host, port)); } - pipeline.addLast("frameDecoder", new VsrFrameDecoder(maxVsrFrameSize)); - pipeline.addLast("responseHandler", new VsrResponseHandler(consensusSession, onEviction)); + pipeline.addLast("frameDecoder", new IggyFrameDecoder()); + pipeline.addLast("responseHandler", new IggyResponseHandler()); } } - record AuthenticationSnapshot(int commandCode, ByteBuf payload) {} + public static class IggyResponseHandler extends SimpleChannelInboundHandler { + private final Queue> responseQueue = new ConcurrentLinkedQueue<>(); + + public void enqueueRequest(CompletableFuture future) { + responseQueue.add(future); + } - @FunctionalInterface - interface TransientFailoverHandler { - CompletableFuture retry( - AsyncTcpConnection source, - int commandCode, - ByteBuf payload, - long requestDeadlineNanos, - IggyServerException rejection); + @Override + protected void channelRead0(ChannelHandlerContext ctx, ByteBuf msg) { + int status = msg.readIntLE(); + int length = msg.readIntLE(); + + CompletableFuture future = responseQueue.poll(); + + if (future != null) { + if (status == 0) { + future.complete(msg.retainedSlice()); + } else { + byte[] errorBytes = length > 0 ? new byte[length] : new byte[0]; + msg.readBytes(errorBytes); + future.completeExceptionally(IggyServerException.fromTcpResponse(status, errorBytes)); + } + } else { + log.error( + "Received response on channel {} but no request was waiting!", + ctx.channel().id()); + } + } + + @Override + public void channelInactive(ChannelHandlerContext ctx) { + failPendingRequests(new IggyConnectionException("Connection closed before a response arrived")); + ctx.fireChannelInactive(); + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { + failPendingRequests(cause); + ctx.close(); + } + + private void failPendingRequests(Throwable cause) { + CompletableFuture pending; + while ((pending = responseQueue.poll()) != null) { + pending.completeExceptionally(cause); + } + } } public static class TcpConnectionPoolConfig { + private final int maxConnections; private final int maxPendingAcquires; private final long acquireTimeoutMillis; public TcpConnectionPoolConfig() { this( + TcpConnectionPoolConfigBuilder.DEFAULT_MAX_CONNECTION, TcpConnectionPoolConfigBuilder.DEFAULT_MAX_PENDING_ACQUIRES, TcpConnectionPoolConfigBuilder.DEFAULT_ACQUIRE_TIMEOUT_MILLIS); } - public TcpConnectionPoolConfig(int maxPendingAcquires, long acquireTimeoutMillis) { + public TcpConnectionPoolConfig(int maxConnections, int maxPendingAcquires, long acquireTimeoutMillis) { + this.maxConnections = maxConnections; this.maxPendingAcquires = maxPendingAcquires; this.acquireTimeoutMillis = acquireTimeoutMillis; } @@ -956,6 +447,10 @@ public static TcpConnectionPoolConfigBuilder builder() { return new TcpConnectionPoolConfigBuilder(); } + public int getMaxConnections() { + return this.maxConnections; + } + public int getMaxPendingAcquires() { return this.maxPendingAcquires; } @@ -965,14 +460,24 @@ public long getAcquireTimeoutMillis() { } public static final class TcpConnectionPoolConfigBuilder { + public static final int DEFAULT_MAX_CONNECTION = 5; public static final int DEFAULT_MAX_PENDING_ACQUIRES = 1000; public static final int DEFAULT_ACQUIRE_TIMEOUT_MILLIS = 3000; + private int maxConnections; private int maxPendingAcquires; private long acquireTimeoutMillis; public TcpConnectionPoolConfigBuilder() {} + public TcpConnectionPoolConfigBuilder setMaxConnections(int maxConnections) { + if (maxConnections <= 0) { + throw new IggyInvalidArgumentException("Connection pool size cannot be 0 or negative"); + } + this.maxConnections = maxConnections; + return this; + } + public TcpConnectionPoolConfigBuilder setMaxPendingAcquires(int maxPendingAcquires) { if (maxPendingAcquires <= 0) { throw new IggyInvalidArgumentException("Max Pending Acquires cannot be 0 or negative"); @@ -990,13 +495,16 @@ public TcpConnectionPoolConfigBuilder setAcquireTimeoutMillis(long acquireTimeou } public TcpConnectionPoolConfig build() { + if (this.maxConnections == 0) { + this.maxConnections = DEFAULT_MAX_CONNECTION; + } if (this.acquireTimeoutMillis == 0) { this.acquireTimeoutMillis = DEFAULT_ACQUIRE_TIMEOUT_MILLIS; } if (this.maxPendingAcquires == 0) { this.maxPendingAcquires = DEFAULT_MAX_PENDING_ACQUIRES; } - return new TcpConnectionPoolConfig(maxPendingAcquires, acquireTimeoutMillis); + return new TcpConnectionPoolConfig(maxConnections, maxPendingAcquires, acquireTimeoutMillis); } } } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ClientRoutingState.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ClientRoutingState.java deleted file mode 100644 index ef28136d91..0000000000 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ClientRoutingState.java +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iggy.client.async.tcp; - -import org.apache.iggy.identifier.ConsumerId; -import org.apache.iggy.identifier.Identifier; -import org.apache.iggy.identifier.StreamId; -import org.apache.iggy.identifier.TopicId; - -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalLong; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Per-client cache of the routing facts needed to resolve partitioning and - * consumer-group polling client-side: topic partition counts, balanced - * round-robin cursors, and consumer-group assignments. Typed keys retain both - * the kind and value of every identifier, so numeric identifiers cannot - * collide with same-text named identifiers. - * - *

Partition counts and balanced cursors survive reconnects; group - * assignments are bound to the server-side VSR session (the member is keyed - * by the connection's client id) and must be cleared whenever that session is - * reset or the client moves to another node. Partition counts carry their - * fetch timestamp so callers can refresh them past a staleness budget — a - * count cached forever would keep hashing keys with the wrong modulus after - * a partition-count change (the Rust SDK still has that gap). - */ -final class ClientRoutingState { - - private final Map partitionCounts = new ConcurrentHashMap<>(); - private final Map balancedCursors = new ConcurrentHashMap<>(); - private final Map assignments = new ConcurrentHashMap<>(); - - static TopicKey topicKey(StreamId streamId, TopicId topicId) { - return new TopicKey(IdentifierKey.from(streamId), IdentifierKey.from(topicId)); - } - - static GroupKey groupKey(StreamId streamId, TopicId topicId, ConsumerId groupId) { - return new GroupKey(topicKey(streamId, topicId), IdentifierKey.from(groupId)); - } - - Optional partitionCount(TopicKey topicKey) { - return Optional.ofNullable(partitionCounts.get(topicKey)); - } - - void setPartitionCount(TopicKey topicKey, long count, long fetchedAtNanos) { - partitionCounts.put(topicKey, new CachedPartitionCount(count, fetchedAtNanos)); - } - - /** - * Drops the cached partition count for a topic. Called when a send that - * was routed with the cached count is refused with a not-found error, - * meaning the topic shrank or was recreated and the count is stale. - */ - void invalidatePartitionCount(TopicKey topicKey) { - partitionCounts.remove(topicKey); - } - - /** - * The next balanced produce partition for a topic, advancing the cursor. - * A per-topic monotonic counter modulo the partition count, so three - * partitions yield 0, 1, 2, 0, ... - */ - long nextBalancedPartition(TopicKey topicKey, long partitionCount) { - if (partitionCount <= 0) { - return 0; - } - var cursor = balancedCursors.computeIfAbsent(topicKey, ignored -> new AtomicInteger()); - return Integer.toUnsignedLong(cursor.getAndIncrement()) % partitionCount; - } - - /** - * Replaces the cached assignment for a group. A generation change (a - * rebalance) resets the round-robin cursor so selection restarts cleanly; - * an unchanged generation keeps the cursor so an assignment refresh does - * not disturb the polling rotation. - */ - void setAssignment(GroupKey groupKey, long generation, List partitions, long syncedAtNanos) { - assignments.compute(groupKey, (ignored, existing) -> { - var cursor = existing != null && existing.generation() == generation ? existing.cursor() : 0; - return new GroupAssignment(generation, List.copyOf(partitions), cursor, syncedAtNanos); - }); - } - - /** - * The next assigned partition to poll for a group, advancing the - * round-robin cursor. Empty when the group has no cached assignment or - * the member currently owns no partitions. - */ - OptionalLong nextGroupPartition(GroupKey groupKey) { - var next = new long[] {-1}; - assignments.computeIfPresent(groupKey, (ignored, assignment) -> { - if (assignment.partitions().isEmpty()) { - return assignment; - } - var index = - Math.floorMod(assignment.cursor(), assignment.partitions().size()); - next[0] = assignment.partitions().get(index); - return new GroupAssignment( - assignment.generation(), - assignment.partitions(), - assignment.cursor() + 1, - assignment.syncedAtNanos()); - }); - return next[0] < 0 ? OptionalLong.empty() : OptionalLong.of(next[0]); - } - - Optional assignment(GroupKey groupKey) { - return Optional.ofNullable(assignments.get(groupKey)); - } - - void invalidateAssignment(GroupKey groupKey) { - assignments.remove(groupKey); - } - - /** - * Drops every cached group assignment. Called when the VSR session is - * reset or the client retargets to another node, since the server keys - * group membership by the session's client id. - */ - void clearAssignments() { - assignments.clear(); - } - - record IdentifierKey(int kind, Long id, String name) { - static IdentifierKey from(Identifier identifier) { - return new IdentifierKey(identifier.getKind(), identifier.getId(), identifier.getName()); - } - } - - record TopicKey(IdentifierKey stream, IdentifierKey topic) {} - - record GroupKey(TopicKey topic, IdentifierKey consumer) {} - - record GroupAssignment(long generation, List partitions, int cursor, long syncedAtNanos) {} - - record CachedPartitionCount(long count, long fetchedAtNanos) {} -} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ConsumerGroupsTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ConsumerGroupsTcpClient.java index 786bcc766c..9b4561fc64 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ConsumerGroupsTcpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ConsumerGroupsTcpClient.java @@ -22,7 +22,6 @@ import io.netty.buffer.Unpooled; import org.apache.iggy.client.async.ConsumerGroupsClient; import org.apache.iggy.consumergroup.ConsumerGroup; -import org.apache.iggy.consumergroup.ConsumerGroupAssignment; import org.apache.iggy.consumergroup.ConsumerGroupDetails; import org.apache.iggy.identifier.ConsumerId; import org.apache.iggy.identifier.StreamId; @@ -185,22 +184,4 @@ public CompletableFuture leaveConsumerGroup(StreamId streamId, TopicId top response.release(); }); } - - @Override - public CompletableFuture> syncConsumerGroup( - StreamId streamId, TopicId topicId, ConsumerId groupId) { - var payload = Unpooled.buffer(); - payload.writeBytes(BytesSerializer.toBytes(streamId)); - payload.writeBytes(BytesSerializer.toBytes(topicId)); - payload.writeBytes(BytesSerializer.toBytes(groupId)); - - log.debug("Syncing consumer group assignment - Stream: {}, Topic: {}, Group: {}", streamId, topicId, groupId); - - // An empty body means "not a member", which exchangeForOptional maps - // to an empty Optional; a member owning zero partitions still gets a - // non-empty body with a zero partition count. - return connection() - .exchangeForOptional( - CommandCode.ConsumerGroup.SYNC, payload, BytesDeserializer::readConsumerGroupAssignment); - } } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/IggyAuthenticator.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/IggyAuthenticator.java index c54d7c43e7..71bfcdb60c 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/IggyAuthenticator.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/IggyAuthenticator.java @@ -21,13 +21,15 @@ import io.netty.buffer.ByteBuf; import io.netty.channel.Channel; +import io.netty.channel.ChannelFutureListener; import io.netty.util.AttributeKey; +import org.apache.iggy.client.async.tcp.AsyncTcpConnection.IggyResponseHandler; +import org.apache.iggy.exception.IggyNotConnectedException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicLong; -import java.util.function.Function; final class IggyAuthenticator { private static final Logger log = LoggerFactory.getLogger(IggyAuthenticator.class); @@ -42,15 +44,12 @@ private IggyAuthenticator() {} * * @param channel the channel to authenticate * @param loginPayload the login payload to send (will be released by this method) + * @param commandCode the login command code * @param currentGeneration the current authentication generation counter - * @param login sends the login payload through the connection's retry path * @return a future that completes when authentication is done */ static CompletableFuture ensureAuthenticated( - Channel channel, - ByteBuf loginPayload, - AtomicLong currentGeneration, - Function> login) { + Channel channel, ByteBuf loginPayload, int commandCode, AtomicLong currentGeneration) { Long channelGeneration = channel.attr(AUTH_GENERATION_KEY).get(); long requiredGeneration = currentGeneration.get(); @@ -59,14 +58,21 @@ static CompletableFuture ensureAuthenticated( return CompletableFuture.completedFuture(null); } - CompletableFuture loginFuture; - try { - loginFuture = login.apply(loginPayload); - } catch (RuntimeException loginError) { - loginPayload.release(); - return CompletableFuture.failedFuture(loginError); + if (loginPayload == null) { + return CompletableFuture.failedFuture(new IggyNotConnectedException("Not authenticated, call login first")); } + CompletableFuture loginFuture = new CompletableFuture<>(); + IggyResponseHandler handler = channel.pipeline().get(IggyResponseHandler.class); + handler.enqueueRequest(loginFuture); + ByteBuf frame = IggyFrameEncoder.encode(channel.alloc(), commandCode, loginPayload); + loginPayload.release(); + channel.writeAndFlush(frame).addListener((ChannelFutureListener) f -> { + if (!f.isSuccess()) { + loginFuture.completeExceptionally(f.cause()); + } + }); + return loginFuture.thenAccept(result -> { try { channel.attr(AUTH_GENERATION_KEY).set(currentGeneration.get()); diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/IggyFrameDecoder.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/IggyFrameDecoder.java new file mode 100644 index 0000000000..3ea104b42f --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/IggyFrameDecoder.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.client.async.tcp; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.ByteToMessageDecoder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + +/** + * Decoder for Iggy protocol responses. + * Response format: [4-byte status LE] [4-byte length LE] [payload] + */ +public class IggyFrameDecoder extends ByteToMessageDecoder { + private static final Logger log = LoggerFactory.getLogger(IggyFrameDecoder.class); + private static final int HEADER_SIZE = 8; // status (4) + length (4) + + @Override + protected void decode(ChannelHandlerContext ctx, ByteBuf in, List out) { + // Wait until we have at least the header + if (in.readableBytes() < HEADER_SIZE) { + return; + } + + // Mark the current reader index + in.markReaderIndex(); + + // Read status and length + int status = in.readIntLE(); + int length = in.readIntLE(); + + log.trace("Received response with status={}, length={}", status, length); + + // Check if we have the complete payload + if (in.readableBytes() < length) { + // Not enough data, reset and wait for more + in.resetReaderIndex(); + return; + } + + // Create a new buffer with the complete response + ByteBuf response = ctx.alloc().buffer(HEADER_SIZE + length); + response.writeIntLE(status); + response.writeIntLE(length); + + if (length > 0) { + response.writeBytes(in, length); + } + + log.trace("Decoded complete response, forwarding to handler"); + out.add(response); + } +} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/IggyFrameEncoder.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/IggyFrameEncoder.java new file mode 100644 index 0000000000..87f0eb479c --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/IggyFrameEncoder.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.client.async.tcp; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.ByteBufAllocator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +final class IggyFrameEncoder { + private static final Logger log = LoggerFactory.getLogger(IggyFrameEncoder.class); + + private IggyFrameEncoder() {} + + /** + * Encodes a command into the Iggy TCP frame format: [payload_size:4][command:4][payload:N] + */ + static ByteBuf encode(ByteBufAllocator alloc, int commandCode, ByteBuf payload) { + int payloadSize = payload.readableBytes(); + int framePayloadSize = 4 + payloadSize; + ByteBuf frame = alloc.buffer(4 + framePayloadSize); + frame.writeIntLE(framePayloadSize); + frame.writeIntLE(commandCode); + frame.writeBytes(payload); + + if (log.isTraceEnabled()) { + byte[] frameBytes = new byte[Math.min(frame.readableBytes(), 30)]; + frame.getBytes(0, frameBytes); + StringBuilder hex = new StringBuilder(); + for (byte b : frameBytes) { + hex.append(String.format("%02x ", b)); + } + log.trace( + "Sending frame with command: {}, payload size: {}, frame payload size (with command): {}, total frame size: {}", + commandCode, + payloadSize, + framePayloadSize, + frame.readableBytes()); + log.trace("Frame bytes (hex): {}", hex); + } + + return frame; + } +} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/LoginRoutingHook.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/LoginRedirectionHook.java similarity index 56% rename from foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/LoginRoutingHook.java rename to foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/LoginRedirectionHook.java index a6acb7dc09..d1c269f9ef 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/LoginRoutingHook.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/LoginRedirectionHook.java @@ -25,20 +25,23 @@ import java.util.function.Supplier; /** - * Routes a login to the cluster leader before executing its Register - * operation. + * Runs after a successful login to redirect the client to the cluster + * leader when it is connected to a follower. */ @FunctionalInterface -interface LoginRoutingHook { +interface LoginRedirectionHook { - LoginRoutingHook NONE = loginAttempt -> loginAttempt.get(); + LoginRedirectionHook NONE = reLogin -> CompletableFuture.completedFuture(null); /** - * Discovers and selects the current leader, then executes the supplied - * login exactly once against the selected connection. + * Checks the cluster roster and, while the current node is not the + * leader, retargets the connection and re-runs the login. * - * @param loginAttempt sends Register against the active connection - * @return the identity returned by the successful Register response + * @param reLogin replays the just-completed login against the current + * target; it must not trigger another redirection check itself, since + * the hook drives any further hops and serializes concurrent checks + * @return the redirected login's identity, or {@code null} when the client + * stays on the current node */ - CompletableFuture loginOnLeader(Supplier> loginAttempt); + CompletableFuture afterLogin(Supplier> reLogin); } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/MessagesTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/MessagesTcpClient.java index 6873c670fa..5cb2d12fc5 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/MessagesTcpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/MessagesTcpClient.java @@ -20,34 +20,20 @@ package org.apache.iggy.client.async.tcp; import io.netty.buffer.Unpooled; -import org.apache.iggy.client.async.ConsumerGroupsClient; import org.apache.iggy.client.async.MessagesClient; -import org.apache.iggy.client.async.TopicsClient; import org.apache.iggy.consumergroup.Consumer; -import org.apache.iggy.exception.IggyErrorCode; -import org.apache.iggy.exception.IggyResourceNotFoundException; -import org.apache.iggy.exception.IggyServerException; -import org.apache.iggy.hash.XxHash32; import org.apache.iggy.identifier.StreamId; import org.apache.iggy.identifier.TopicId; import org.apache.iggy.message.Message; import org.apache.iggy.message.Partitioning; -import org.apache.iggy.message.PartitioningKind; import org.apache.iggy.message.PolledMessages; import org.apache.iggy.message.PollingStrategy; -import org.apache.iggy.message.SendMessagesResponse; import org.apache.iggy.serde.BytesDeserializer; import org.apache.iggy.serde.CommandCode; -import org.apache.iggy.topic.TopicDetails; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.math.BigInteger; -import java.time.Duration; import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; import java.util.function.Supplier; import static org.apache.iggy.serde.BytesSerializer.toBytes; @@ -57,41 +43,10 @@ */ public class MessagesTcpClient implements MessagesClient { - private static final Logger log = LoggerFactory.getLogger(MessagesTcpClient.class); - - /** - * A generation-fenced group poll is answered with an empty poll body - * carrying this sentinel partition id, telling the client to re-sync its - * assignment and retry; mirrors RESYNC_REQUIRED_PARTITION_SENTINEL in - * core/common/src/lib.rs. - */ - private static final long RESYNC_REQUIRED_PARTITION_SENTINEL = 0xFFFF_FFFFL; - - private static final int GROUP_POLL_MAX_ATTEMPTS = 2; - private static final int PARTITION_NOT_OWNED_ERROR_CODE = 5009; - - /** - * Staleness budget for the client-side routing caches: group assignments - * and topic partition counts. Both re-fetch lazily on the next use once - * this old, so a partition-count change or rebalance is picked up without - * a background refresher thread. - */ - private static final Duration ROUTING_CACHE_REFRESH = Duration.ofSeconds(5); - private final Supplier connectionSupplier; - private final ClientRoutingState routingState; - private final TopicsClient topicsClient; - private final ConsumerGroupsClient consumerGroupsClient; public MessagesTcpClient(Supplier connectionSupplier) { - this(connectionSupplier, new ClientRoutingState()); - } - - MessagesTcpClient(Supplier connectionSupplier, ClientRoutingState routingState) { this.connectionSupplier = connectionSupplier; - this.routingState = routingState; - this.topicsClient = new TopicsTcpClient(connectionSupplier); - this.consumerGroupsClient = new ConsumerGroupsTcpClient(connectionSupplier); } private AsyncTcpConnection connection() { @@ -107,23 +62,6 @@ public CompletableFuture pollMessages( PollingStrategy strategy, Long count, boolean autoCommit) { - if (consumer.kind() == Consumer.Kind.ConsumerGroup && partitionId.isEmpty()) { - // The VSR broker fences group polls against unowned partitions - // instead of picking one, so the partition is selected here from - // the member's synced assignment, matching the Rust SDK. - return pollGroupMessages(streamId, topicId, consumer, strategy, count, autoCommit, GROUP_POLL_MAX_ATTEMPTS); - } - return pollPartition(streamId, topicId, partitionId, consumer, strategy, count, autoCommit); - } - - private CompletableFuture pollPartition( - StreamId streamId, - TopicId topicId, - Optional partitionId, - Consumer consumer, - PollingStrategy strategy, - Long count, - boolean autoCommit) { var payload = Unpooled.buffer(); @@ -156,29 +94,7 @@ private CompletableFuture pollPartition( } @Override - public CompletableFuture sendMessages( - StreamId streamId, TopicId topicId, Partitioning partitioning, List messages) { - if (partitioning.kind() == PartitioningKind.PartitionId) { - return sendToPartition(streamId, topicId, partitioning, messages); - } - // The VSR broker routes explicit partitions only, so balanced and - // message-key partitioning resolve to a partition id client-side, - // matching the Rust SDK (round-robin cursor, xxh32(key) % count). - return resolvePartitioning(streamId, topicId, partitioning) - .thenCompose(resolved -> sendToPartition(streamId, topicId, resolved, messages)) - .exceptionallyCompose(error -> { - // A resolved send refused with not-found means the cached - // partition count is stale (the topic shrank or was - // recreated); drop it so the next send re-fetches now - // instead of waiting out the staleness budget. - if (unwrapCompletion(error) instanceof IggyResourceNotFoundException) { - routingState.invalidatePartitionCount(ClientRoutingState.topicKey(streamId, topicId)); - } - return CompletableFuture.failedFuture(error); - }); - } - - private CompletableFuture sendToPartition( + public CompletableFuture sendMessages( StreamId streamId, TopicId topicId, Partitioning partitioning, List messages) { // Build metadata section following the blocking client pattern @@ -211,161 +127,10 @@ private CompletableFuture sendToPartition( payload.writeBytes(toBytes(message)); } - return connection().send(CommandCode.Messages.SEND.getValue(), payload).thenApply(response -> { - try { - return BytesDeserializer.readSendMessagesResponse(response); - } catch (RuntimeException e) { - // The batch is already committed server-side; failing here would - // trigger a spurious resend, so a malformed confirmation degrades - // to an empty one. - log.warn("Discarding malformed send confirmation: {}", e.getMessage()); - return SendMessagesResponse.empty(); - } finally { - response.release(); - } + // Send async request (no response data expected for send) + return connection().send(CommandCode.Messages.SEND.getValue(), payload).thenAccept(response -> { + // Response received, messages sent successfully + response.release(); // Release the buffer }); } - - /** - * One group-poll attempt: sync the assignment when missing or stale, pick - * the next assigned partition round-robin, poll it explicitly, and on a - * generation fence (the re-sync sentinel or a partition-not-owned error) - * drop the cached assignment and retry. The attempt budget allows one - * re-sync after the coordinator rejects a stale assignment, then one - * retry; an exhausted budget is an empty poll, not an error. - */ - private CompletableFuture pollGroupMessages( - StreamId streamId, - TopicId topicId, - Consumer consumer, - PollingStrategy strategy, - Long count, - boolean autoCommit, - int attemptsLeft) { - if (attemptsLeft == 0) { - return CompletableFuture.completedFuture(emptyPolledMessages()); - } - var groupKey = ClientRoutingState.groupKey(streamId, topicId, consumer.id()); - return ensureFreshAssignment(streamId, topicId, consumer, groupKey).thenCompose(ignored -> { - var partitionId = routingState.nextGroupPartition(groupKey); - if (partitionId.isEmpty()) { - if (routingState.assignment(groupKey).isPresent()) { - // a member owning no partitions polls nothing - return CompletableFuture.completedFuture(emptyPolledMessages()); - } - return CompletableFuture.failedFuture(new IggyResourceNotFoundException( - IggyErrorCode.CONSUMER_GROUP_NOT_JOINED, - IggyErrorCode.CONSUMER_GROUP_NOT_JOINED.getCode(), - "Cannot poll consumer group " + consumer.id() + " for topic " + topicId + " in stream " - + streamId + ": this client is not a member, join the group first", - Optional.empty(), - Optional.empty())); - } - return pollPartition( - streamId, - topicId, - Optional.of(partitionId.getAsLong()), - consumer, - strategy, - count, - autoCommit) - .thenCompose(polled -> { - if (polled.messages().isEmpty() && polled.partitionId() == RESYNC_REQUIRED_PARTITION_SENTINEL) { - routingState.invalidateAssignment(groupKey); - return pollGroupMessages( - streamId, topicId, consumer, strategy, count, autoCommit, attemptsLeft - 1); - } - return CompletableFuture.completedFuture(polled); - }) - .exceptionallyCompose(error -> { - if (!isPartitionNotOwned(error)) { - return CompletableFuture.failedFuture(error); - } - routingState.invalidateAssignment(groupKey); - return pollGroupMessages( - streamId, topicId, consumer, strategy, count, autoCommit, attemptsLeft - 1); - }); - }); - } - - private CompletableFuture ensureFreshAssignment( - StreamId streamId, TopicId topicId, Consumer consumer, ClientRoutingState.GroupKey groupKey) { - var cached = routingState.assignment(groupKey); - if (cached.isPresent() && System.nanoTime() - cached.get().syncedAtNanos() < ROUTING_CACHE_REFRESH.toNanos()) { - return CompletableFuture.completedFuture(null); - } - return consumerGroupsClient - .syncConsumerGroup(streamId, topicId, consumer.id()) - .thenAccept(assignment -> { - if (assignment.isEmpty()) { - // an empty sync reply means "not a member" - routingState.invalidateAssignment(groupKey); - return; - } - routingState.setAssignment( - groupKey, - assignment.get().generation(), - assignment.get().partitions(), - System.nanoTime()); - }); - } - - private static boolean isPartitionNotOwned(Throwable error) { - return unwrapCompletion(error) instanceof IggyServerException serverError - && serverError.getRawErrorCode() == PARTITION_NOT_OWNED_ERROR_CODE; - } - - private static Throwable unwrapCompletion(Throwable error) { - return error instanceof CompletionException && error.getCause() != null ? error.getCause() : error; - } - - private static PolledMessages emptyPolledMessages() { - return new PolledMessages(0L, BigInteger.ZERO, 0L, List.of()); - } - - private CompletableFuture resolvePartitioning( - StreamId streamId, TopicId topicId, Partitioning partitioning) { - return partitionCount(streamId, topicId).thenApply(partitionsCount -> switch (partitioning.kind()) { - case Balanced -> - Partitioning.partitionId(routingState.nextBalancedPartition( - ClientRoutingState.topicKey(streamId, topicId), partitionsCount)); - case MessagesKey -> Partitioning.partitionId(XxHash32.hashUnsigned(partitioning.value()) % partitionsCount); - case PartitionId -> partitioning; - }); - } - - private CompletableFuture partitionCount(StreamId streamId, TopicId topicId) { - var topicKey = ClientRoutingState.topicKey(streamId, topicId); - var cached = routingState.partitionCount(topicKey); - if (cached.isPresent() && System.nanoTime() - cached.get().fetchedAtNanos() < ROUTING_CACHE_REFRESH.toNanos()) { - return CompletableFuture.completedFuture(cached.get().count()); - } - return topicsClient - .getTopic(streamId, topicId) - .thenApply(topicDetails -> { - var partitionsCount = - topicDetails.map(TopicDetails::partitionsCount).orElse(0L); - if (partitionsCount == 0) { - throw new IggyResourceNotFoundException( - IggyErrorCode.TOPIC_ID_NOT_FOUND, - IggyErrorCode.TOPIC_ID_NOT_FOUND.getCode(), - "Cannot resolve partitioning: topic " + topicId + " in stream " + streamId - + " was not found or has no partitions", - Optional.empty(), - Optional.empty()); - } - routingState.setPartitionCount(topicKey, partitionsCount, System.nanoTime()); - return partitionsCount; - }) - .exceptionallyCompose(error -> { - // A failed refresh should not stop routing while a stale - // count is still on hand, except when the topic itself is - // gone; serving the stale value keeps sends flowing - // through transient metadata-fetch failures. - if (cached.isPresent() && !(unwrapCompletion(error) instanceof IggyResourceNotFoundException)) { - return CompletableFuture.completedFuture(cached.get().count()); - } - return CompletableFuture.failedFuture(error); - }); - } } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/PersonalAccessTokensTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/PersonalAccessTokensTcpClient.java index 6803843796..b5b97842fd 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/PersonalAccessTokensTcpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/PersonalAccessTokensTcpClient.java @@ -38,22 +38,22 @@ import java.util.function.Supplier; /** - * Async TCP implementation of personal access tokens client. Login discovers - * the active cluster leader before sending the VSR Register operation. + * Async TCP implementation of personal access tokens client. */ public class PersonalAccessTokensTcpClient implements PersonalAccessTokensClient { private static final Logger log = LoggerFactory.getLogger(PersonalAccessTokensTcpClient.class); private final Supplier connectionSupplier; - private final LoginRoutingHook routingHook; + private final LoginRedirectionHook redirectionHook; public PersonalAccessTokensTcpClient(Supplier connectionSupplier) { - this(connectionSupplier, LoginRoutingHook.NONE); + this(connectionSupplier, LoginRedirectionHook.NONE); } - PersonalAccessTokensTcpClient(Supplier connectionSupplier, LoginRoutingHook routingHook) { + PersonalAccessTokensTcpClient( + Supplier connectionSupplier, LoginRedirectionHook redirectionHook) { this.connectionSupplier = connectionSupplier; - this.routingHook = routingHook; + this.redirectionHook = redirectionHook; } private AsyncTcpConnection connection() { @@ -115,7 +115,9 @@ public CompletableFuture deletePersonalAccessToken(String name) { @Override public CompletableFuture loginWithPersonalAccessToken(String token) { - return routingHook.loginOnLeader(() -> loginWithoutRedirect(token)); + return loginWithoutRedirect(token).thenCompose(identity -> redirectionHook + .afterLogin(() -> loginWithoutRedirect(token)) + .thenApply(redirectedIdentity -> redirectedIdentity != null ? redirectedIdentity : identity)); } private CompletableFuture loginWithoutRedirect(String token) { diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ReconnectPlan.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ReconnectPlan.java deleted file mode 100644 index ae596731f4..0000000000 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ReconnectPlan.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iggy.client.async.tcp; - -import org.apache.iggy.client.ConnectionInfo; -import org.apache.iggy.config.RetryPolicy; - -import java.time.Duration; - -/** - * Pure redial planning: which address to dial on a given reconnect attempt - * and how long to wait before it. - */ -final class ReconnectPlan { - - private ReconnectPlan() {} - - /** - * Alternates reconnect dials between the current endpoint and the - * configured seed. After a leader redirect the current endpoint may die - * with the leader, and the seed is the way back to the rest of the - * cluster. Attempts are 1-based; odd attempts dial the current endpoint. - */ - static ConnectionInfo target(ConnectionInfo current, ConnectionInfo seed, int attempt) { - if (current.equals(seed)) { - return current; - } - return attempt % 2 == 1 ? current : seed; - } - - /** - * The delay before the given 1-based attempt: the policy's initial delay - * scaled by its multiplier per prior attempt, capped at its max delay. - */ - static Duration delay(RetryPolicy policy, int attempt) { - double scaled = policy.getInitialDelay().toMillis() * Math.pow(policy.getMultiplier(), attempt - 1L); - long millis = (long) Math.min(scaled, policy.getMaxDelay().toMillis()); - return Duration.ofMillis(millis); - } -} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/UsersTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/UsersTcpClient.java index f6e3be1f2d..6ed7c3bef4 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/UsersTcpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/UsersTcpClient.java @@ -41,22 +41,21 @@ import static org.apache.iggy.serde.BytesSerializer.toBytes; /** - * Async TCP implementation of users client. Login discovers the active - * cluster leader before sending the VSR Register operation. + * Async TCP implementation of users client. */ public class UsersTcpClient implements UsersClient { private static final Logger log = LoggerFactory.getLogger(UsersTcpClient.class); private final Supplier connectionSupplier; - private final LoginRoutingHook routingHook; + private final LoginRedirectionHook redirectionHook; public UsersTcpClient(Supplier connectionSupplier) { - this(connectionSupplier, LoginRoutingHook.NONE); + this(connectionSupplier, LoginRedirectionHook.NONE); } - UsersTcpClient(Supplier connectionSupplier, LoginRoutingHook routingHook) { + UsersTcpClient(Supplier connectionSupplier, LoginRedirectionHook redirectionHook) { this.connectionSupplier = connectionSupplier; - this.routingHook = routingHook; + this.redirectionHook = redirectionHook; } private AsyncTcpConnection connection() { @@ -146,7 +145,9 @@ public CompletableFuture changePassword(UserId userId, String currentPassw @Override public CompletableFuture login(String username, String password) { - return routingHook.loginOnLeader(() -> loginWithoutRedirect(username, password)); + return loginWithoutRedirect(username, password).thenCompose(identity -> redirectionHook + .afterLogin(() -> loginWithoutRedirect(username, password)) + .thenApply(redirectedIdentity -> redirectedIdentity != null ? redirectedIdentity : identity)); } private CompletableFuture loginWithoutRedirect(String username, String password) { diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/package-info.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/package-info.java index cab5768ef3..261a1cbd47 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/package-info.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/package-info.java @@ -34,12 +34,13 @@ * * *

Protocol Details

- *

The transport speaks the VSR (Viewstamped Replication) wire protocol: - * every frame starts with a 256-byte consensus header ({@code RequestHeader} - * on the way out, {@code ReplyHeader} or {@code EvictionHeader} on the way - * back) followed by the command payload; see the {@code vsr} subpackage. - *

Responses are matched to requests in FIFO order. The - * {@link org.apache.iggy.client.async.tcp.AsyncTcpConnection} + *

The Iggy binary protocol uses a simple framing scheme: + *

    + *
  • Request: {@code [payload_size:4 LE][command:4 LE][payload:N]}
    • + *
  • Response: {@code [status:4 LE][length:4 LE][payload:N]}
    • + *
+ *

Responses are matched to requests in FIFO order (the protocol does not include + * request IDs). The {@link org.apache.iggy.client.async.tcp.AsyncTcpConnection} * serializes all writes through Netty's event loop to maintain ordering. * * @see org.apache.iggy.client.async.tcp.AsyncIggyTcpClient diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/ConsensusSession.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/ConsensusSession.java deleted file mode 100644 index ab79ea19ee..0000000000 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/ConsensusSession.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iggy.client.async.tcp.vsr; - -import org.apache.iggy.exception.IggyNotConnectedException; - -import java.security.SecureRandom; - -/** - * VSR client identity and dedup state, mirroring - * {@code core/sdk/src/session.rs}. - * - *

The (client id, request id) pair is the server's dedup key for - * replicated operations, and the session value is the fence epoch of the - * latest committed {@code Register}. None of these are bearer tokens; auth - * is bound to the transport connection server-side. - */ -public final class ConsensusSession { - - private static final SecureRandom RANDOM = new SecureRandom(); - - private long clientIdLow; - private long clientIdHigh; - private Long session; - private long requestCounter = 1; - private long correlationCounter = 1; - private boolean registerConsumed; - - public ConsensusSession() { - regenerateClientId(); - } - - /** - * Arms a {@code Register}: on re-login (or a consumed one-shot register) - * the whole identity re-arms with a fresh client id so the server sees a - * brand-new registration. Returns the request id a Register carries, - * which is always zero. - */ - synchronized long beginRegister() { - if (registerConsumed || session != null) { - regenerateClientId(); - session = null; - requestCounter = 1; - } - registerConsumed = true; - return 0; - } - - /** Binds the fence epoch returned by a committed Register reply. */ - synchronized void bind(long sessionEpoch) { - if (sessionEpoch <= 0) { - throw new IllegalStateException("Register reply carried a non-positive session epoch: " + sessionEpoch); - } - this.session = sessionEpoch; - } - - /** Replicated metadata ops consume the monotonic VSR dedup counter. */ - synchronized long nextRequestId() { - if (session == null) { - throw new IggyNotConnectedException("Not authenticated, call login first"); - } - return requestCounter++; - } - - /** - * Partition and non-replicated ops use an independent sequence for reply - * correlation, so they do not create gaps in the metadata dedup sequence. - */ - synchronized long nextCorrelationId() { - return correlationCounter++; - } - - synchronized long currentRequestId() { - return requestCounter; - } - - synchronized long sessionOrZero() { - return session == null ? 0 : session; - } - - synchronized long boundSession() { - if (session == null) { - throw new IggyNotConnectedException("Not authenticated, call login first"); - } - return session; - } - - synchronized boolean isBound() { - return session != null; - } - - /** Clears the bound epoch (logout / eviction); next login re-registers. */ - synchronized void reset() { - session = null; - } - - synchronized long clientIdLow() { - return clientIdLow; - } - - synchronized long clientIdHigh() { - return clientIdHigh; - } - - private void regenerateClientId() { - do { - clientIdLow = RANDOM.nextLong(); - clientIdHigh = RANDOM.nextLong(); - } while (clientIdLow == 0 && clientIdHigh == 0); - } -} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrFrameDecoder.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrFrameDecoder.java deleted file mode 100644 index 7398f527f7..0000000000 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrFrameDecoder.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iggy.client.async.tcp.vsr; - -import io.netty.buffer.ByteBuf; -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.ByteToMessageDecoder; -import io.netty.handler.codec.DecoderException; - -import java.util.List; - -/** - * Decoder for VSR response frames: a 256-byte consensus header whose total - * frame size (header included) sits at byte offset 48, followed by an - * optional body. There is no length delimiter outside the header. - */ -public class VsrFrameDecoder extends ByteToMessageDecoder { - - /** Matches the server's default {@code max_message_size} (64 MB). */ - public static final int DEFAULT_MAX_FRAME_SIZE = 64 * 1024 * 1024; - - private final int maxFrameSize; - - public VsrFrameDecoder() { - this(DEFAULT_MAX_FRAME_SIZE); - } - - public VsrFrameDecoder(int maxFrameSize) { - if (maxFrameSize < VsrHeaders.HEADER_SIZE) { - throw new IllegalArgumentException( - "Maximum VSR frame size must be at least " + VsrHeaders.HEADER_SIZE + " bytes"); - } - this.maxFrameSize = maxFrameSize; - } - - @Override - protected void decode(ChannelHandlerContext ctx, ByteBuf in, List out) { - if (in.readableBytes() < VsrHeaders.HEADER_SIZE) { - return; - } - long totalSize = VsrHeaders.readSize(in); - if (totalSize < VsrHeaders.HEADER_SIZE || totalSize > maxFrameSize) { - throw new DecoderException("Invalid VSR frame size " + totalSize + ", connection is desynchronized"); - } - if (in.readableBytes() < totalSize) { - return; - } - out.add(in.readRetainedSlice((int) totalSize)); - } -} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrHeaders.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrHeaders.java deleted file mode 100644 index 356c722a63..0000000000 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrHeaders.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iggy.client.async.tcp.vsr; - -import io.netty.buffer.ByteBuf; -import org.apache.iggy.exception.IggyServerException; - -/** - * Byte offsets and readers for the 256-byte consensus headers, mirroring the - * {@code #[repr(C)]} layouts in - * {@code core/binary_protocol/src/consensus/header.rs}. All fields are - * little-endian; the checksum fields stay zero by protocol contract. - */ -public final class VsrHeaders { - - public static final int HEADER_SIZE = 256; - - // GenericHeader (shared prefix) - static final int SIZE_OFFSET = 48; - static final int COMMAND_OFFSET = 60; - - // RequestHeader. The client wire carries no routing group: the server - // derives it (plane from the operation, partition target from the payload), - // so every field past the operation sits eight bytes earlier than it did - // while the header still carried one. - static final int REQUEST_CLIENT_OFFSET = 128; - static final int REQUEST_ID_OFFSET = 168; - static final int REQUEST_OPERATION_OFFSET = 176; - static final int REQUEST_SESSION_OFFSET = 184; - static final int REQUEST_RESERVED_CODE_OFFSET = 196; - - // ReplyHeader - static final int REPLY_REQUEST_OFFSET = 200; - static final int REPLY_OPERATION_OFFSET = 208; - static final int REPLY_STATUS_OFFSET = 216; - - // EvictionHeader - static final int EVICTION_PROTOCOL_VERSION_OFFSET = 144; - static final int EVICTION_PROTOCOL_VERSION_MIN_OFFSET = 148; - static final int EVICTION_REASON_OFFSET = 255; - - // Command2 discriminants - static final int COMMAND_REQUEST = 5; - static final int COMMAND_REPLY = 8; - static final int COMMAND_EVICTION = 13; - - // EvictionReason discriminants - static final int REASON_NO_SESSION = 1; - static final int REASON_SESSION_TOO_LOW = 7; - static final int REASON_SESSION_RELEASE_MISMATCH = 8; - static final int REASON_INVALID_CREDENTIALS = 9; - static final int REASON_INVALID_TOKEN = 10; - static final int REASON_USER_INACTIVE = 11; - static final int REASON_SESSION_ERROR = 12; - static final int REASON_STALE_CLIENT = 13; - static final int REASON_INCOMPATIBLE_PROTOCOL = 14; - static final int REASON_MALFORMED_LOGIN = 15; - - // IggyError codes the eviction reasons grade to, mirroring - // core/common/src/error/eviction.rs. - static final int ERROR_INVALID_COMMAND = 3; - static final int ERROR_INVALID_FORMAT = 4; - static final int ERROR_STALE_CLIENT = 30; - static final int ERROR_UNAUTHENTICATED = 40; - static final int ERROR_INVALID_CREDENTIALS = 42; - static final int ERROR_INVALID_PERSONAL_ACCESS_TOKEN = 53; - static final int ERROR_TRANSIENT_NOT_COMMITTED = 57; - static final int ERROR_TRANSIENT_NOT_ACCEPTED = 58; - static final int ERROR_INCOMPATIBLE_PROTOCOL_VERSION = 14003; - - private VsrHeaders() {} - - static int peekCommand(ByteBuf frame) { - return frame.getUnsignedByte(frame.readerIndex() + COMMAND_OFFSET); - } - - static long readSize(ByteBuf frame) { - return frame.getUnsignedIntLE(frame.readerIndex() + SIZE_OFFSET); - } - - static long readStatus(ByteBuf frame) { - return frame.getUnsignedIntLE(frame.readerIndex() + REPLY_STATUS_OFFSET); - } - - static int readReplyOperation(ByteBuf frame) { - return frame.getUnsignedByte(frame.readerIndex() + REPLY_OPERATION_OFFSET); - } - - static long readReplyRequestId(ByteBuf frame) { - return frame.getLongLE(frame.readerIndex() + REPLY_REQUEST_OFFSET); - } - - static int readRequestOperation(ByteBuf frame) { - return frame.getUnsignedByte(frame.readerIndex() + REQUEST_OPERATION_OFFSET); - } - - static long readRequestId(ByteBuf frame) { - return frame.getLongLE(frame.readerIndex() + REQUEST_ID_OFFSET); - } - - /** - * Grades a session-terminal eviction frame to the error the caller sees, - * mirroring {@code eviction_reason_to_error}. A degenerate protocol - * window (zero minimum or inverted range) degrades to unauthenticated. - */ - static IggyServerException evictionToException(ByteBuf frame) { - int base = frame.readerIndex(); - int reason = frame.getUnsignedByte(base + EVICTION_REASON_OFFSET); - int errorCode; - switch (reason) { - case REASON_INVALID_CREDENTIALS -> errorCode = ERROR_INVALID_CREDENTIALS; - case REASON_INVALID_TOKEN -> errorCode = ERROR_INVALID_PERSONAL_ACCESS_TOKEN; - case REASON_STALE_CLIENT -> errorCode = ERROR_STALE_CLIENT; - case REASON_MALFORMED_LOGIN -> errorCode = ERROR_INVALID_FORMAT; - case REASON_INCOMPATIBLE_PROTOCOL -> { - long max = frame.getUnsignedIntLE(base + EVICTION_PROTOCOL_VERSION_OFFSET); - long min = frame.getUnsignedIntLE(base + EVICTION_PROTOCOL_VERSION_MIN_OFFSET); - errorCode = (min == 0 || max < min) ? ERROR_UNAUTHENTICATED : ERROR_INCOMPATIBLE_PROTOCOL_VERSION; - } - case REASON_NO_SESSION, - REASON_SESSION_TOO_LOW, - REASON_SESSION_RELEASE_MISMATCH, - REASON_USER_INACTIVE, - REASON_SESSION_ERROR -> errorCode = ERROR_UNAUTHENTICATED; - default -> errorCode = ERROR_INVALID_COMMAND; - } - return IggyServerException.fromTcpResponse(errorCode, new byte[0]); - } -} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrLoginCodec.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrLoginCodec.java deleted file mode 100644 index 36c856c3a2..0000000000 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrLoginCodec.java +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iggy.client.async.tcp.vsr; - -import io.netty.buffer.ByteBuf; -import io.netty.buffer.ByteBufAllocator; -import org.apache.iggy.IggyVersion; -import org.apache.iggy.exception.IggyInvalidArgumentException; - -import java.nio.charset.StandardCharsets; - -/** - * Rewrites serialized login payloads into the - * {@code LoginRegister} / {@code LoginRegisterWithPat} bodies, mirroring - * {@code core/binary_protocol/src/requests/users/login_register.rs} and - * {@code login_register_with_pat.rs}. Both bodies start with a - * {@code ClientVersionInfo} prefix carrying the packed protocol version and - * the SDK identity ({@code core/binary_protocol/src/version.rs}). - */ -final class VsrLoginCodec { - - /** - * Packed semver of the {@code iggy_binary_protocol} crate this codec - * targets: {@code major << 20 | minor << 10 | patch}, 10 bits per field. - * Keep in sync with {@code core/binary_protocol/Cargo.toml}; the server - * accepts any client whose major.minor is not newer than its own. - */ - static final int PROTOCOL_VERSION = (11 << 10); // 0.11.0 - - static final String SDK_NAME = "java-sdk"; - - private VsrLoginCodec() {} - - /** - * {@code LoginUser} (code 38) payload in: - * {@code [username:u8-len][password:u8-len][version:u32-len][context:u32-len]}. - * The trailing version/context strings are superseded by the - * {@code ClientVersionInfo} prefix and dropped. - */ - static ByteBuf rewriteUserLogin(ByteBufAllocator alloc, ByteBuf loginPayload) { - ByteBuf in = loginPayload.slice(); - byte[] username = readShortField(in, "username"); - byte[] password = readShortField(in, "password"); - - ByteBuf body = alloc.buffer(); - writeVersionInfo(body); - writeShortField(body, username); - body.writeByte(password.length); - body.writeBytes(password); - body.writeIntLE(0); - return body; - } - - /** - * {@code LoginWithPersonalAccessToken} (code 44) payload in: - * {@code [token:u8-len]}. - */ - static ByteBuf rewritePatLogin(ByteBufAllocator alloc, ByteBuf loginPayload) { - ByteBuf in = loginPayload.slice(); - byte[] token = readShortField(in, "token"); - - ByteBuf body = alloc.buffer(); - writeVersionInfo(body); - writeShortField(body, token); - body.writeIntLE(0); - return body; - } - - /** - * Register reply body after result-section stripping: - * {@code [user_id:u32][session:u64][server_protocol_version:u32][server_version:u8-len]}. - */ - static long readSessionEpoch(ByteBuf registerBody) { - return registerBody.getLongLE(registerBody.readerIndex() + 4); - } - - private static void writeVersionInfo(ByteBuf body) { - body.writeIntLE(PROTOCOL_VERSION); - writeShortField(body, SDK_NAME.getBytes(StandardCharsets.UTF_8)); - writeShortField(body, sdkVersion().getBytes(StandardCharsets.UTF_8)); - } - - private static String sdkVersion() { - String version = IggyVersion.getInstance().getVersion(); - if (version == null || version.isEmpty()) { - return "unknown"; - } - return version.length() > 255 ? version.substring(0, 255) : version; - } - - private static byte[] readShortField(ByteBuf in, String field) { - if (!in.isReadable()) { - throw new IggyInvalidArgumentException("Login payload is missing the " + field + " field"); - } - int length = in.readUnsignedByte(); - if (in.readableBytes() < length) { - throw new IggyInvalidArgumentException("Login payload " + field + " field is truncated"); - } - byte[] value = new byte[length]; - in.readBytes(value); - return value; - } - - private static void writeShortField(ByteBuf out, byte[] value) { - if (value.length == 0 || value.length > 255) { - throw new IggyInvalidArgumentException("Wire name fields must be 1..255 bytes, got " + value.length); - } - out.writeByte(value.length); - out.writeBytes(value); - } -} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrOperation.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrOperation.java deleted file mode 100644 index 6bd6349ac1..0000000000 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrOperation.java +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iggy.client.async.tcp.vsr; - -import java.util.BitSet; -import java.util.Map; - -/** - * VSR {@code Operation} discriminants and the command-code mapping, mirroring - * {@code core/binary_protocol/src/consensus/operation.rs} and the SDK-side - * mapping in {@code core/sdk/src/vsr.rs}. - */ -public final class VsrOperation { - - public static final int RESERVED = 0; - public static final int REGISTER = 1; - public static final int NON_REPLICATED = 2; - public static final int LOGOUT = 3; - - public static final int CREATE_TOPIC_WITH_ASSIGNMENTS = 64; - public static final int CREATE_PARTITIONS_WITH_ASSIGNMENTS = 65; - public static final int REMOVE_CONSUMER_GROUP_MEMBER = 66; - public static final int COMPLETE_CONSUMER_GROUP_REVOCATION = 67; - public static final int TRUNCATE_PARTITION = 68; - - public static final int CREATE_STREAM = 128; - public static final int UPDATE_STREAM = 129; - public static final int DELETE_STREAM = 130; - public static final int PURGE_STREAM = 131; - public static final int CREATE_TOPIC = 132; - public static final int UPDATE_TOPIC = 133; - public static final int DELETE_TOPIC = 134; - public static final int PURGE_TOPIC = 135; - public static final int CREATE_PARTITIONS = 136; - public static final int DELETE_PARTITIONS = 137; - public static final int DELETE_SEGMENTS = 138; - public static final int CREATE_CONSUMER_GROUP = 139; - public static final int DELETE_CONSUMER_GROUP = 140; - public static final int CREATE_USER = 141; - public static final int UPDATE_USER = 142; - public static final int DELETE_USER = 143; - public static final int CHANGE_PASSWORD = 144; - public static final int UPDATE_PERMISSIONS = 145; - public static final int CREATE_PERSONAL_ACCESS_TOKEN = 146; - public static final int DELETE_PERSONAL_ACCESS_TOKEN = 147; - public static final int JOIN_CONSUMER_GROUP = 148; - public static final int LEAVE_CONSUMER_GROUP = 149; - - public static final int SEND_MESSAGES = 160; - public static final int STORE_CONSUMER_OFFSET = 161; - public static final int DELETE_CONSUMER_OFFSET = 162; - public static final int STORE_CONSUMER_OFFSET_2 = 164; - public static final int DELETE_CONSUMER_OFFSET_2 = 165; - - private static final int INTERNAL_START = 64; - private static final int METADATA_START = 128; - private static final int PARTITION_START = 160; - - /** - * Replicated command code to operation, from the server's - * {@code COMMAND_TABLE}. Codes absent here travel as - * {@link #NON_REPLICATED} with the code stamped into the header's - * reserved bytes; the server is the authority on unknown codes. - */ - private static final Map REPLICATED_OPERATIONS = Map.ofEntries( - Map.entry(33, CREATE_USER), - Map.entry(34, DELETE_USER), - Map.entry(35, UPDATE_USER), - Map.entry(36, UPDATE_PERMISSIONS), - Map.entry(37, CHANGE_PASSWORD), - Map.entry(42, CREATE_PERSONAL_ACCESS_TOKEN), - Map.entry(43, DELETE_PERSONAL_ACCESS_TOKEN), - Map.entry(101, SEND_MESSAGES), - Map.entry(121, STORE_CONSUMER_OFFSET), - Map.entry(122, DELETE_CONSUMER_OFFSET), - Map.entry(123, STORE_CONSUMER_OFFSET_2), - Map.entry(124, DELETE_CONSUMER_OFFSET_2), - Map.entry(202, CREATE_STREAM), - Map.entry(203, DELETE_STREAM), - Map.entry(204, UPDATE_STREAM), - Map.entry(205, PURGE_STREAM), - Map.entry(302, CREATE_TOPIC), - Map.entry(303, DELETE_TOPIC), - Map.entry(304, UPDATE_TOPIC), - Map.entry(305, PURGE_TOPIC), - Map.entry(402, CREATE_PARTITIONS), - Map.entry(403, DELETE_PARTITIONS), - Map.entry(503, DELETE_SEGMENTS), - Map.entry(602, CREATE_CONSUMER_GROUP), - Map.entry(603, DELETE_CONSUMER_GROUP), - Map.entry(604, JOIN_CONSUMER_GROUP), - Map.entry(605, LEAVE_CONSUMER_GROUP)); - - private static final int LOGOUT_USER_CODE = 39; - - private static final BitSet KNOWN_OPERATIONS = new BitSet(); - - static { - KNOWN_OPERATIONS.set(RESERVED, LOGOUT + 1); - KNOWN_OPERATIONS.set(CREATE_TOPIC_WITH_ASSIGNMENTS, TRUNCATE_PARTITION + 1); - KNOWN_OPERATIONS.set(CREATE_STREAM, LEAVE_CONSUMER_GROUP + 1); - KNOWN_OPERATIONS.set(SEND_MESSAGES); - KNOWN_OPERATIONS.set(STORE_CONSUMER_OFFSET); - KNOWN_OPERATIONS.set(DELETE_CONSUMER_OFFSET); - KNOWN_OPERATIONS.set(STORE_CONSUMER_OFFSET_2); - KNOWN_OPERATIONS.set(DELETE_CONSUMER_OFFSET_2); - } - - private VsrOperation() {} - - /** - * Maps a command code to its operation. Login codes are handled upstream - * (they select {@link #REGISTER}); everything unmapped is forwarded as - * {@link #NON_REPLICATED}. - */ - static int operationForCode(int commandCode) { - if (commandCode == LOGOUT_USER_CODE) { - return LOGOUT; - } - return REPLICATED_OPERATIONS.getOrDefault(commandCode, NON_REPLICATED); - } - - static boolean isMetadata(int operation) { - if (operation >= INTERNAL_START && operation < METADATA_START) { - return true; - } - // DeleteSegments replicates in its per-partition group, not the - // metadata group, so it is excluded from the metadata band. - if (operation == DELETE_SEGMENTS) { - return false; - } - return operation >= METADATA_START && operation <= LEAVE_CONSUMER_GROUP; - } - - static boolean isPartition(int operation) { - return operation >= PARTITION_START; - } - - /** - * Whether a reply body for this operation starts with a committed result - * section ({@code [count:u32][{index,result} x count]}). - */ - static boolean isResultFramed(int operation) { - return isMetadata(operation) - || operation == STORE_CONSUMER_OFFSET - || operation == DELETE_CONSUMER_OFFSET - || operation == STORE_CONSUMER_OFFSET_2 - || operation == DELETE_CONSUMER_OFFSET_2; - } - - /** - * The server controls the reply's operation byte; an undeclared value must - * be rejected rather than routed through a predicate that happens to match. - */ - static boolean isKnown(int operation) { - return operation >= 0 && KNOWN_OPERATIONS.get(operation); - } - - /** - * Maps an internal operation returned by the server to the client - * operation that initiated it. The server preserves the request id when - * it enriches these commands before replication. - */ - static int correlationOperation(int operation) { - return switch (operation) { - case CREATE_TOPIC_WITH_ASSIGNMENTS -> CREATE_TOPIC; - case CREATE_PARTITIONS_WITH_ASSIGNMENTS -> CREATE_PARTITIONS; - case TRUNCATE_PARTITION -> DELETE_SEGMENTS; - default -> operation; - }; - } -} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrRequestEncoder.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrRequestEncoder.java deleted file mode 100644 index 45859f06d7..0000000000 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrRequestEncoder.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iggy.client.async.tcp.vsr; - -import io.netty.buffer.ByteBuf; -import io.netty.buffer.ByteBufAllocator; - -/** - * Encodes a (code, payload) command into a VSR request frame: - * a 256-byte {@code RequestHeader} followed by the unchanged command payload. - * Login codes are rewritten to the {@code LoginRegister} exchange; every - * other payload is passed through byte-identical. - * - *

Mirrors {@code encode_request_header} in {@code core/sdk/src/vsr.rs}. - */ -public final class VsrRequestEncoder { - - private static final int LOGIN_USER_CODE = 38; - private static final int LOGIN_REGISTER_CODE = 40; - private static final int LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE = 44; - private static final int LOGIN_REGISTER_WITH_PAT_CODE = 45; - - private final ConsensusSession session; - - public VsrRequestEncoder(ConsensusSession session) { - this.session = session; - } - - /** - * Builds the full request frame. The caller keeps ownership of - * {@code payload}; its reader index is not advanced. - */ - public ByteBuf encode(ByteBufAllocator alloc, int commandCode, ByteBuf payload) { - int operation; - long requestId; - long sessionId; - ByteBuf body; - boolean releaseBody = false; - - if (commandCode == LOGIN_USER_CODE || commandCode == LOGIN_REGISTER_CODE) { - body = VsrLoginCodec.rewriteUserLogin(alloc, payload); - releaseBody = true; - operation = VsrOperation.REGISTER; - requestId = session.beginRegister(); - sessionId = 0; - } else if (commandCode == LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE - || commandCode == LOGIN_REGISTER_WITH_PAT_CODE) { - body = VsrLoginCodec.rewritePatLogin(alloc, payload); - releaseBody = true; - operation = VsrOperation.REGISTER; - requestId = session.beginRegister(); - sessionId = 0; - } else { - body = payload; - operation = VsrOperation.operationForCode(commandCode); - if (operation == VsrOperation.NON_REPLICATED) { - // Non-replicated ops bypass dedup but still need a unique - // correlation id. Bootstrap ping and cluster metadata remain - // sessionless before login. - requestId = session.nextCorrelationId(); - sessionId = session.sessionOrZero(); - } else if (VsrOperation.isPartition(operation)) { - // Partition ops replicate in their own group without client - // table dedup, so use the independent correlation sequence. - sessionId = session.boundSession(); - requestId = session.nextCorrelationId(); - } else { - sessionId = session.boundSession(); - requestId = session.nextRequestId(); - } - } - - try { - int totalSize = VsrHeaders.HEADER_SIZE + body.readableBytes(); - ByteBuf frame = alloc.buffer(totalSize); - frame.writeZero(VsrHeaders.HEADER_SIZE); - frame.setIntLE(VsrHeaders.SIZE_OFFSET, totalSize); - frame.setByte(VsrHeaders.COMMAND_OFFSET, VsrHeaders.COMMAND_REQUEST); - frame.setLongLE(VsrHeaders.REQUEST_CLIENT_OFFSET, session.clientIdLow()); - frame.setLongLE(VsrHeaders.REQUEST_CLIENT_OFFSET + 8, session.clientIdHigh()); - frame.setLongLE(VsrHeaders.REQUEST_ID_OFFSET, requestId); - frame.setByte(VsrHeaders.REQUEST_OPERATION_OFFSET, operation); - frame.setLongLE(VsrHeaders.REQUEST_SESSION_OFFSET, sessionId); - if (operation == VsrOperation.NON_REPLICATED) { - frame.setIntLE(VsrHeaders.REQUEST_RESERVED_CODE_OFFSET, commandCode); - } - frame.writeBytes(body, body.readerIndex(), body.readableBytes()); - return frame; - } finally { - if (releaseBody) { - body.release(); - } - } - } -} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandler.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandler.java deleted file mode 100644 index 113c288e27..0000000000 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandler.java +++ /dev/null @@ -1,261 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iggy.client.async.tcp.vsr; - -import io.netty.buffer.ByteBuf; -import io.netty.channel.Channel; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.SimpleChannelInboundHandler; -import io.netty.util.concurrent.ScheduledFuture; -import org.apache.iggy.exception.IggyConnectionException; -import org.apache.iggy.exception.IggyServerException; -import org.apache.iggy.exception.IggyTimeoutException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicReference; - -/** - * Correlates multiplexed requests by operation and request id, decodes VSR - * reply frames, and completes each pending future with the command payload - * the typed deserializers expect. Mirrors {@code decode_response} in {@code - * core/sdk/src/vsr.rs}: eviction frames become typed errors, a nonzero header - * status is a pre-commit deny, and result-framed bodies have their committed - * result section stripped (or raised as the typed error). - */ -public class VsrResponseHandler extends SimpleChannelInboundHandler { - - private static final Logger log = LoggerFactory.getLogger(VsrResponseHandler.class); - - private static final int RESULT_COUNT_LEN = 4; - private static final int RESULT_ENTRY_LEN = 8; - private static final int REGISTER_BODY_MIN_LEN = 17; - - private final ConcurrentMap> pendingRequests = new ConcurrentHashMap<>(); - private final ConsensusSession session; - private final Runnable onEviction; - private final AtomicReference closeCause = new AtomicReference<>(); - - public VsrResponseHandler(ConsensusSession session, Runnable onEviction) { - this.session = session; - this.onEviction = onEviction; - } - - void registerRequest(CompletableFuture future, int operation, long requestId) { - registerRequest(new RequestKey(operation, requestId), future); - } - - public void registerRequest( - Channel channel, - ByteBuf requestFrame, - CompletableFuture future, - long deadlineNanos, - int commandCode) { - RequestKey key = - new RequestKey(VsrHeaders.readRequestOperation(requestFrame), VsrHeaders.readRequestId(requestFrame)); - registerRequest(key, future); - long timeoutNanos = Math.max(0, deadlineNanos - System.nanoTime()); - ScheduledFuture timeoutFuture; - try { - timeoutFuture = channel.eventLoop() - .schedule( - () -> { - if (!future.isDone()) { - closeChannel( - channel, - new IggyTimeoutException( - "Timed out waiting for a response to command code " + commandCode)); - } - }, - timeoutNanos, - TimeUnit.NANOSECONDS); - } catch (RuntimeException error) { - pendingRequests.remove(key, future); - throw error; - } - future.whenComplete((response, error) -> { - pendingRequests.remove(key, future); - timeoutFuture.cancel(false); - }); - } - - private void registerRequest(RequestKey key, CompletableFuture future) { - CompletableFuture existing = pendingRequests.putIfAbsent(key, future); - if (existing != null) { - throw new IllegalStateException( - "A request is already pending for operation " + key.operation() + " and request id " + key.id()); - } - } - - public void closeChannel(Channel channel, Throwable cause) { - closeCause.compareAndSet(null, cause); - channel.close(); - failPendingRequests(closeCause.get()); - } - - @Override - public void channelInactive(ChannelHandlerContext ctx) { - Throwable cause = closeCause.get(); - if (cause == null) { - cause = new IggyConnectionException("Connection closed before a response arrived"); - } - failPendingRequests(cause); - ctx.fireChannelInactive(); - } - - @Override - public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { - closeChannel(ctx.channel(), cause); - } - - @Override - protected void channelRead0(ChannelHandlerContext ctx, ByteBuf msg) { - if (VsrHeaders.peekCommand(msg) == VsrHeaders.COMMAND_EVICTION) { - handleEviction(ctx, msg); - return; - } - int replyOperation = VsrHeaders.readReplyOperation(msg); - RequestKey key = - new RequestKey(VsrOperation.correlationOperation(replyOperation), VsrHeaders.readReplyRequestId(msg)); - CompletableFuture future = pendingRequests.remove(key); - if (future == null) { - closeChannel( - ctx.channel(), - invalidReply( - "no request was pending for operation " + replyOperation + " and request id " + key.id())); - return; - } - ByteBuf body; - try { - body = decodeReply(msg); - } catch (RuntimeException error) { - future.completeExceptionally(error); - return; - } - if (!future.complete(body)) { - body.release(); - } - } - - private void handleEviction(ChannelHandlerContext ctx, ByteBuf frame) { - IggyServerException error = VsrHeaders.evictionToException(frame); - session.reset(); - try { - onEviction.run(); - } catch (RuntimeException listenerError) { - log.warn("Eviction listener failed: {}", listenerError.getMessage()); - } - // The server has already unbound this transport. Closing it before - // completing requests prevents the pool from recycling the channel - // and assigning a late reply to a request from the next session. - closeChannel(ctx.channel(), error); - } - - private ByteBuf decodeReply(ByteBuf frame) { - int operation = validatedOperation(frame); - - int totalSize = (int) VsrHeaders.readSize(frame); - int bodyStart = frame.readerIndex() + VsrHeaders.HEADER_SIZE; - int bodyLength = totalSize - VsrHeaders.HEADER_SIZE; - - boolean resultFramed = - VsrOperation.isResultFramed(operation) || (operation == VsrOperation.REGISTER && bodyLength > 0); - if (resultFramed) { - int sectionLength = resultSectionLength(frame, bodyStart, bodyLength); - bodyStart += sectionLength; - bodyLength -= sectionLength; - } - - applySessionEffects(frame, operation, bodyStart, bodyLength); - return frame.retainedSlice(bodyStart, bodyLength); - } - - private int validatedOperation(ByteBuf frame) { - int command = VsrHeaders.peekCommand(frame); - if (command != VsrHeaders.COMMAND_REPLY) { - throw invalidReply("unexpected consensus command " + command); - } - long status = VsrHeaders.readStatus(frame); - if (status != 0) { - throw IggyServerException.fromTcpResponse(status, new byte[0]); - } - int operation = VsrHeaders.readReplyOperation(frame); - if (!VsrOperation.isKnown(operation)) { - throw invalidReply("unknown reply operation " + operation); - } - return operation; - } - - /** - * Validates the committed result section leading the body and returns its - * length; a nonzero committed result surfaces as the typed error. - */ - private static int resultSectionLength(ByteBuf frame, int bodyStart, int bodyLength) { - if (bodyLength < RESULT_COUNT_LEN) { - throw invalidReply("result-framed body shorter than its count field"); - } - long count = frame.getUnsignedIntLE(bodyStart); - long sectionLength = RESULT_COUNT_LEN + count * RESULT_ENTRY_LEN; - if (bodyLength < sectionLength) { - throw invalidReply("result section truncated"); - } - if (count > 0) { - long resultCode = frame.getUnsignedIntLE(bodyStart + RESULT_COUNT_LEN + 4); - if (resultCode != 0) { - throw IggyServerException.fromTcpResponse(resultCode, new byte[0]); - } - } - return (int) sectionLength; - } - - private void applySessionEffects(ByteBuf frame, int operation, int bodyStart, int bodyLength) { - if (operation == VsrOperation.REGISTER) { - // A terminal register failure ships an empty body (no result - // section); anything shorter than the typed response is not a - // successful registration. - if (bodyLength < REGISTER_BODY_MIN_LEN) { - throw IggyServerException.fromTcpResponse(VsrHeaders.ERROR_UNAUTHENTICATED, new byte[0]); - } - session.bind(frame.getLongLE(bodyStart + 4)); - } - if (operation == VsrOperation.LOGOUT) { - session.reset(); - } - } - - private static IggyServerException invalidReply(String detail) { - log.error("Malformed VSR reply: {}", detail); - return IggyServerException.fromTcpResponse(VsrHeaders.ERROR_INVALID_COMMAND, new byte[0]); - } - - private void failPendingRequests(Throwable cause) { - pendingRequests.forEach((key, pending) -> { - if (pendingRequests.remove(key, pending)) { - pending.completeExceptionally(cause); - } - }); - } - - private record RequestKey(int operation, long id) {} -} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/ConsumerGroupsClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/ConsumerGroupsClient.java index 99a0742564..bf91344315 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/ConsumerGroupsClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/ConsumerGroupsClient.java @@ -20,7 +20,6 @@ package org.apache.iggy.client.blocking; import org.apache.iggy.consumergroup.ConsumerGroup; -import org.apache.iggy.consumergroup.ConsumerGroupAssignment; import org.apache.iggy.consumergroup.ConsumerGroupDetails; import org.apache.iggy.identifier.ConsumerId; import org.apache.iggy.identifier.StreamId; @@ -66,14 +65,4 @@ default void leaveConsumerGroup(Long streamId, Long topicId, Long groupId) { } void leaveConsumerGroup(StreamId streamId, TopicId topicId, ConsumerId groupId); - - default Optional syncConsumerGroup(Long streamId, Long topicId, Long groupId) { - return syncConsumerGroup(StreamId.of(streamId), TopicId.of(topicId), ConsumerId.of(groupId)); - } - - /** - * Fetches this client's current partition assignment for a consumer group, - * or empty when this client is not a member. - */ - Optional syncConsumerGroup(StreamId streamId, TopicId topicId, ConsumerId groupId); } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/MessagesClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/MessagesClient.java index 8099126765..fb6705686c 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/MessagesClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/MessagesClient.java @@ -26,7 +26,6 @@ import org.apache.iggy.message.Partitioning; import org.apache.iggy.message.PolledMessages; import org.apache.iggy.message.PollingStrategy; -import org.apache.iggy.message.SendMessagesResponse; import java.util.List; import java.util.Optional; @@ -60,11 +59,9 @@ PolledMessages pollMessages( Long count, boolean autoCommit); - default SendMessagesResponse sendMessages( - Long streamId, Long topicId, Partitioning partitioning, List messages) { - return sendMessages(StreamId.of(streamId), TopicId.of(topicId), partitioning, messages); + default void sendMessages(Long streamId, Long topicId, Partitioning partitioning, List messages) { + sendMessages(StreamId.of(streamId), TopicId.of(topicId), partitioning, messages); } - SendMessagesResponse sendMessages( - StreamId streamId, TopicId topicId, Partitioning partitioning, List messages); + void sendMessages(StreamId streamId, TopicId topicId, Partitioning partitioning, List messages); } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/ConsumerGroupsHttpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/ConsumerGroupsHttpClient.java index bf155ceb60..1d73f56f6e 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/ConsumerGroupsHttpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/ConsumerGroupsHttpClient.java @@ -21,7 +21,6 @@ import org.apache.iggy.client.blocking.ConsumerGroupsClient; import org.apache.iggy.consumergroup.ConsumerGroup; -import org.apache.iggy.consumergroup.ConsumerGroupAssignment; import org.apache.iggy.consumergroup.ConsumerGroupDetails; import org.apache.iggy.exception.IggyOperationNotSupportedException; import org.apache.iggy.identifier.ConsumerId; @@ -74,11 +73,6 @@ public void leaveConsumerGroup(StreamId streamId, TopicId topicId, ConsumerId gr throw new IggyOperationNotSupportedException("leaveConsumerGroup", "HTTP"); } - @Override - public Optional syncConsumerGroup(StreamId streamId, TopicId topicId, ConsumerId groupId) { - throw new IggyOperationNotSupportedException("syncConsumerGroup", "HTTP"); - } - private static String path(StreamId streamId, TopicId topicId) { return "/streams/" + streamId + "/topics/" + topicId + "/consumer-groups"; } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/MessagesHttpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/MessagesHttpClient.java index 998637f4cb..3d2ce27978 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/MessagesHttpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/MessagesHttpClient.java @@ -28,7 +28,6 @@ import org.apache.iggy.message.Partitioning; import org.apache.iggy.message.PolledMessages; import org.apache.iggy.message.PollingStrategy; -import org.apache.iggy.message.SendMessagesResponse; import java.util.List; import java.util.Optional; @@ -64,14 +63,9 @@ public PolledMessages pollMessages( } @Override - public SendMessagesResponse sendMessages( - StreamId streamId, TopicId topicId, Partitioning partitioning, List messages) { + public void sendMessages(StreamId streamId, TopicId topicId, Partitioning partitioning, List messages) { var request = httpClient.preparePostRequest(path(streamId, topicId), new SendMessages(partitioning, messages)); - var body = httpClient.executeWithStringResponse(request); - if (body.isBlank()) { - return SendMessagesResponse.empty(); - } - return ObjectMapperFactory.getInstance().readValue(body, SendMessagesResponse.class); + httpClient.execute(request); } private static String path(StreamId streamId, TopicId topicId) { diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/ConsumerGroupsTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/ConsumerGroupsTcpClient.java index d8b452e596..b69416d1c2 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/ConsumerGroupsTcpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/ConsumerGroupsTcpClient.java @@ -21,7 +21,6 @@ import org.apache.iggy.client.blocking.ConsumerGroupsClient; import org.apache.iggy.consumergroup.ConsumerGroup; -import org.apache.iggy.consumergroup.ConsumerGroupAssignment; import org.apache.iggy.consumergroup.ConsumerGroupDetails; import org.apache.iggy.identifier.ConsumerId; import org.apache.iggy.identifier.StreamId; @@ -67,9 +66,4 @@ public void joinConsumerGroup(StreamId streamId, TopicId topicId, ConsumerId gro public void leaveConsumerGroup(StreamId streamId, TopicId topicId, ConsumerId groupId) { FutureUtil.resolve(delegate.leaveConsumerGroup(streamId, topicId, groupId)); } - - @Override - public Optional syncConsumerGroup(StreamId streamId, TopicId topicId, ConsumerId groupId) { - return FutureUtil.resolve(delegate.syncConsumerGroup(streamId, topicId, groupId)); - } } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/IggyTcpClientBuilder.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/IggyTcpClientBuilder.java index 65a09d215a..90ba272f05 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/IggyTcpClientBuilder.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/IggyTcpClientBuilder.java @@ -125,6 +125,17 @@ public IggyTcpClientBuilder requestTimeout(Duration requestTimeout) { return this; } + /** + * Sets the connection pool size. + * + * @param connectionPoolSize the size of the connection pool + * @return this builder + */ + public IggyTcpClientBuilder connectionPoolSize(Integer connectionPoolSize) { + asyncBuilder.connectionPoolSize(connectionPoolSize); + return this; + } + /** * Sets the retry policy. * diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/MessagesTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/MessagesTcpClient.java index 47e561b0dd..2ec3ed2fed 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/MessagesTcpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/MessagesTcpClient.java @@ -27,7 +27,6 @@ import org.apache.iggy.message.Partitioning; import org.apache.iggy.message.PolledMessages; import org.apache.iggy.message.PollingStrategy; -import org.apache.iggy.message.SendMessagesResponse; import java.util.List; import java.util.Optional; @@ -54,8 +53,7 @@ public PolledMessages pollMessages( } @Override - public SendMessagesResponse sendMessages( - StreamId streamId, TopicId topicId, Partitioning partitioning, List messages) { - return FutureUtil.resolve(delegate.sendMessages(streamId, topicId, partitioning, messages)); + public void sendMessages(StreamId streamId, TopicId topicId, Partitioning partitioning, List messages) { + FutureUtil.resolve(delegate.sendMessages(streamId, topicId, partitioning, messages)); } } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/consumergroup/ConsumerGroupAssignment.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/consumergroup/ConsumerGroupAssignment.java deleted file mode 100644 index a12a04d39d..0000000000 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/consumergroup/ConsumerGroupAssignment.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iggy.consumergroup; - -import java.util.List; - -/** - * A consumer-group member's current partition assignment and the group - * generation it belongs to, as returned by the sync-consumer-group command. - * - *

The generation advances on every rebalance; a member holding an - * assignment from an older generation is fenced by the server when it polls. - * An assignment with no partitions still means the client is a group member, - * just one that currently owns nothing. - * - * @param generation the group generation this assignment belongs to - * @param partitions the partition IDs this member may poll, possibly empty - */ -public record ConsumerGroupAssignment(long generation, List partitions) {} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/exception/IggyErrorCode.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/exception/IggyErrorCode.java index 3780efc318..426c4f5f71 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/exception/IggyErrorCode.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/exception/IggyErrorCode.java @@ -34,12 +34,10 @@ public enum IggyErrorCode { ERROR(1), INVALID_COMMAND(3), INVALID_FORMAT(4), - FEATURE_UNAVAILABLE(5), - INVALID_IDENTIFIER(6), + FEATURE_UNAVAILABLE(6), CANNOT_PARSE_INT(7), CANNOT_PARSE_SLICE(8), CANNOT_PARSE_UTF8(9), - STALE_CLIENT(30), // Resource errors RESOURCE_NOT_FOUND(20), @@ -61,8 +59,6 @@ public enum IggyErrorCode { CLIENT_NOT_FOUND(52), INVALID_PAT_TOKEN(53), PAT_NAME_ALREADY_EXISTS(54), - TRANSIENT_NOT_COMMITTED(57), - TRANSIENT_NOT_ACCEPTED(58), PASSWORD_DOES_NOT_MATCH(77), PASSWORD_HASH_INTERNAL_ERROR(78), @@ -106,9 +102,6 @@ public enum IggyErrorCode { INVALID_MESSAGE_CHECKSUM(7003), MESSAGE_NOT_FOUND(7004), - // VSR protocol errors - INCOMPATIBLE_PROTOCOL_VERSION(14003), - // Unknown error code UNKNOWN(-1); diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/exception/IggyValidationException.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/exception/IggyValidationException.java index 02ae079752..84e0b8eb41 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/exception/IggyValidationException.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/exception/IggyValidationException.java @@ -35,7 +35,6 @@ public class IggyValidationException extends IggyServerException { IggyErrorCode.INVALID_COMMAND, IggyErrorCode.INVALID_FORMAT, IggyErrorCode.FEATURE_UNAVAILABLE, - IggyErrorCode.INVALID_IDENTIFIER, IggyErrorCode.CANNOT_PARSE_INT, IggyErrorCode.CANNOT_PARSE_SLICE, IggyErrorCode.CANNOT_PARSE_UTF8, diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/hash/XxHash32.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/hash/XxHash32.java deleted file mode 100644 index b28ea6ec40..0000000000 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/hash/XxHash32.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iggy.hash; - -/** - * XXH32 (32-bit xxHash), one-shot, seed 0. - * - *

Vendored so message-key partitioning hashes identically across every - * Iggy SDK and the server (the Rust side uses {@code twox_hash::XxHash32} - * with seed 0). The routing contract is - * {@code xxh32(key, 0) % partitionCount}, so any deviation here silently - * breaks per-key ordering against other clients. - */ -public final class XxHash32 { - - private static final int PRIME_1 = 0x9E3779B1; - private static final int PRIME_2 = 0x85EBCA77; - private static final int PRIME_3 = 0xC2B2AE3D; - private static final int PRIME_4 = 0x27D4EB2F; - private static final int PRIME_5 = 0x165667B1; - - private XxHash32() {} - - /** - * Hashes the given bytes with XXH32, seed 0. - * - * @param data the bytes to hash - * @return the 32-bit hash, as an unsigned value in a long - */ - public static long hashUnsigned(byte[] data) { - return Integer.toUnsignedLong(hash(data)); - } - - static int hash(byte[] data) { - final int length = data.length; - int offset = 0; - int hash; - - if (length >= 16) { - int v1 = PRIME_1 + PRIME_2; - int v2 = PRIME_2; - int v3 = 0; - int v4 = -PRIME_1; - for (; offset <= length - 16; offset += 16) { - v1 = round(v1, readIntLE(data, offset)); - v2 = round(v2, readIntLE(data, offset + 4)); - v3 = round(v3, readIntLE(data, offset + 8)); - v4 = round(v4, readIntLE(data, offset + 12)); - } - hash = Integer.rotateLeft(v1, 1) - + Integer.rotateLeft(v2, 7) - + Integer.rotateLeft(v3, 12) - + Integer.rotateLeft(v4, 18); - } else { - hash = PRIME_5; - } - - hash += length; - - for (; offset <= length - 4; offset += 4) { - hash = Integer.rotateLeft(hash + readIntLE(data, offset) * PRIME_3, 17) * PRIME_4; - } - for (; offset < length; offset++) { - hash = Integer.rotateLeft(hash + (data[offset] & 0xFF) * PRIME_5, 11) * PRIME_1; - } - - hash ^= hash >>> 15; - hash *= PRIME_2; - hash ^= hash >>> 13; - hash *= PRIME_3; - hash ^= hash >>> 16; - return hash; - } - - private static int round(int accumulator, int lane) { - return Integer.rotateLeft(accumulator + lane * PRIME_2, 13) * PRIME_1; - } - - private static int readIntLE(byte[] data, int offset) { - return (data[offset] & 0xFF) - | (data[offset + 1] & 0xFF) << 8 - | (data[offset + 2] & 0xFF) << 16 - | (data[offset + 3] & 0xFF) << 24; - } -} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/SendConfirmation.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/SendConfirmation.java deleted file mode 100644 index 836c348526..0000000000 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/SendConfirmation.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iggy.message; - -import java.math.BigInteger; - -/** - * Confirmation of a committed send: the partition the batch landed on and the - * offset of its first message. - * - * @param streamId the numeric stream ID the batch was written to - * @param topicId the numeric topic ID the batch was written to - * @param partitionId the partition the batch landed on - * @param baseOffset the offset assigned to the first message of the batch - */ -public record SendConfirmation(Long streamId, Long topicId, Long partitionId, BigInteger baseOffset) {} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/SendMessagesResponse.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/SendMessagesResponse.java deleted file mode 100644 index 79519d374f..0000000000 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/SendMessagesResponse.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iggy.message; - -import java.util.List; - -/** - * Server reply to a send: one {@link SendConfirmation} per committed batch. - * - *

An empty confirmation list means the server acknowledged the send without - * reporting where it landed; the messages were still committed. - * - * @param confirmations the committed-batch confirmations, possibly empty - */ -public record SendMessagesResponse(List confirmations) { - - public static SendMessagesResponse empty() { - return new SendMessagesResponse(List.of()); - } -} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/BytesDeserializer.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/BytesDeserializer.java index 623d62b36e..67afa08523 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/BytesDeserializer.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/BytesDeserializer.java @@ -27,7 +27,6 @@ import org.apache.iggy.cluster.ClusterNodeStatus; import org.apache.iggy.cluster.TransportEndpoints; import org.apache.iggy.consumergroup.ConsumerGroup; -import org.apache.iggy.consumergroup.ConsumerGroupAssignment; import org.apache.iggy.consumergroup.ConsumerGroupDetails; import org.apache.iggy.consumergroup.ConsumerGroupMember; import org.apache.iggy.consumeroffset.ConsumerOffsetInfo; @@ -39,8 +38,6 @@ import org.apache.iggy.message.Message; import org.apache.iggy.message.MessageHeader; import org.apache.iggy.message.PolledMessages; -import org.apache.iggy.message.SendConfirmation; -import org.apache.iggy.message.SendMessagesResponse; import org.apache.iggy.partition.Partition; import org.apache.iggy.personalaccesstoken.PersonalAccessTokenInfo; import org.apache.iggy.personalaccesstoken.RawPersonalAccessToken; @@ -77,8 +74,6 @@ */ public final class BytesDeserializer { - private static final int CONSUMER_GROUP_ASSIGNMENT_ENTRY_BYTES = Integer.BYTES; - private static final int SEND_CONFIRMATION_BYTES = 3 * Integer.BYTES + Long.BYTES; private static final int MIN_CLUSTER_NODE_BYTES = 18; private BytesDeserializer() {} @@ -182,23 +177,6 @@ public static ConsumerGroup readConsumerGroup(ByteBuf response) { return new ConsumerGroup(groupId, name, partitionsCount, membersCount); } - public static ConsumerGroupAssignment readConsumerGroupAssignment(ByteBuf response) { - // The generation is a monotonic rebalance counter compared only for - // equality, so reading the u64 as a signed long is safe. - var generation = response.readLongLE(); - var partitionsCount = response.readUnsignedIntLE(); - int capacity = validatedCollectionSize( - partitionsCount, - response.readableBytes(), - CONSUMER_GROUP_ASSIGNMENT_ENTRY_BYTES, - "Consumer group partitions count"); - List partitions = new ArrayList<>(capacity); - for (long i = 0; i < partitionsCount; i++) { - partitions.add(response.readUnsignedIntLE()); - } - return new ConsumerGroupAssignment(generation, partitions); - } - public static ConsumerOffsetInfo readConsumerOffsetInfo(ByteBuf response) { var partitionId = response.readUnsignedIntLE(); var currentOffset = readU64AsBigInteger(response); @@ -206,28 +184,6 @@ public static ConsumerOffsetInfo readConsumerOffsetInfo(ByteBuf response) { return new ConsumerOffsetInfo(partitionId, currentOffset, storedOffset); } - public static SendMessagesResponse readSendMessagesResponse(ByteBuf response) { - if (!response.isReadable()) { - return SendMessagesResponse.empty(); - } - var confirmationsCount = response.readUnsignedIntLE(); - int capacity = validatedCollectionSize( - confirmationsCount, response.readableBytes(), SEND_CONFIRMATION_BYTES, "Send confirmations count"); - var confirmations = new ArrayList(capacity); - for (long i = 0; i < confirmationsCount; i++) { - var streamId = response.readUnsignedIntLE(); - var topicId = response.readUnsignedIntLE(); - var partitionId = response.readUnsignedIntLE(); - var baseOffset = readU64AsBigInteger(response); - confirmations.add(new SendConfirmation(streamId, topicId, partitionId, baseOffset)); - } - if (response.isReadable()) { - throw new IggyMalformedResponseException( - "send messages response has " + response.readableBytes() + " trailing bytes"); - } - return new SendMessagesResponse(confirmations); - } - public static PolledMessages readPolledMessages(ByteBuf response) { var partitionId = response.readUnsignedIntLE(); var currentOffset = readU64AsBigInteger(response); @@ -539,14 +495,6 @@ private static String readU32PrefixedString(ByteBuf buffer, String field) { return buffer.readCharSequence(toInt(length), StandardCharsets.UTF_8).toString(); } - private static int validatedCollectionSize(long count, int readableBytes, int entryBytes, String field) { - if (count > readableBytes / entryBytes) { - throw new IggyMalformedResponseException( - field + " " + count + " exceeds remaining payload of " + readableBytes + " bytes"); - } - return Math.toIntExact(count); - } - static BigInteger readU64AsBigInteger(ByteBuf buffer) { var bytesArray = new byte[8]; buffer.readBytes(bytesArray, 0, 8); diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/CommandCode.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/CommandCode.java index 4c2e56ceac..337494d185 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/CommandCode.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/CommandCode.java @@ -183,8 +183,7 @@ enum ConsumerGroup implements CommandCode { CREATE(602), DELETE(603), JOIN(604), - LEAVE(605), - SYNC(606); + LEAVE(605); private final int value; diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/BaseIntegrationTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/BaseIntegrationTest.java index fd24f171c2..ad1d3f1646 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/BaseIntegrationTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/BaseIntegrationTest.java @@ -32,21 +32,6 @@ import java.util.List; -/** - * Base for integration tests. The SDK speaks the VSR wire protocol, so the - * server under test must support it. - * - *

With {@code USE_EXTERNAL_SERVER} set, tests target an externally - * started VSR server on localhost, running standalone (single-node) mode. - * Start it from the repo root: - * 

{@code
- * IGGY_ROOT_USERNAME=iggy IGGY_ROOT_PASSWORD=iggy cargo run --bin iggy-server
- * }
- * - *

Otherwise a server container is started via testcontainers. This works - * once the published image ships a VSR-capable server; until then the - * external-server mode is the only one that can pass. - */ @Testcontainers public abstract class BaseIntegrationTest { @@ -73,10 +58,6 @@ public static String serverHost() { static void setupContainer() { ResourceLeakDetector.setLevel(ResourceLeakDetector.Level.PARANOID); if (!USE_EXTERNAL_SERVER) { - // The published image still ships the legacy server, which does - // not speak the VSR wire protocol, so tests against this - // container fail until a VSR-capable server is released. Use - // USE_EXTERNAL_SERVER until then. log.info("Starting Iggy Server Container..."); iggyServer = new GenericContainer<>(DockerImageName.parse("apache/iggy:edge")) .withExposedPorts(HTTP_PORT, TCP_PORT) diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncClientIntegrationTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncClientIntegrationTest.java index 4f1496f4da..fbe1045d1d 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncClientIntegrationTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncClientIntegrationTest.java @@ -27,7 +27,6 @@ import org.apache.iggy.message.Message; import org.apache.iggy.message.Partitioning; import org.apache.iggy.message.PollingStrategy; -import org.apache.iggy.message.SendMessagesResponse; import org.apache.iggy.topic.CompressionAlgorithm; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -219,7 +218,7 @@ void shouldSendAndPollLargeVolume() throws Exception { .get(TIMEOUT_SECONDS, TimeUnit.SECONDS); // when — send messages in concurrent batches - List> sendFutures = new ArrayList<>(); + List> sendFutures = new ArrayList<>(); for (int batch = 0; batch < 10; batch++) { List batchMessages = new ArrayList<>(); for (int i = 0; i < 10; i++) { diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncConnectionPoolAuthTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncConnectionPoolAuthTest.java index 5a5c4e5001..ce7fd59f6b 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncConnectionPoolAuthTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncConnectionPoolAuthTest.java @@ -25,7 +25,6 @@ import org.apache.iggy.identifier.StreamId; import org.apache.iggy.message.Message; import org.apache.iggy.message.Partitioning; -import org.apache.iggy.message.SendMessagesResponse; import org.apache.iggy.topic.CompressionAlgorithm; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -47,9 +46,9 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; /** - * Integration tests for connection authentication lifecycle. + * Integration tests for connection pool authentication lifecycle. * Verifies that lazy per-channel authentication works correctly across - * login, logout, and re-login cycles on the pooled connection. + * login, logout, and re-login cycles with a pooled connection. */ @DisplayName("Connection Pool Authentication") class AsyncConnectionPoolAuthTest extends BaseIntegrationTest { @@ -57,7 +56,7 @@ class AsyncConnectionPoolAuthTest extends BaseIntegrationTest { private static final String USERNAME = "iggy"; private static final String PASSWORD = "iggy"; - private static final int CONCURRENCY = 3; + private static final int POOL_SIZE = 3; private AsyncIggyTcpClient client; @@ -66,6 +65,7 @@ void setUp() throws Exception { client = AsyncIggyTcpClient.builder() .host(serverHost()) .port(serverTcpPort()) + .connectionPoolSize(POOL_SIZE) .build(); client.connect().get(5, TimeUnit.SECONDS); } @@ -78,15 +78,9 @@ void tearDown() throws Exception { } @Test - @DisplayName("should allow only bootstrap commands before login") - void shouldAllowOnlyBootstrapCommandsBeforeLogin() throws Exception { - // when - var ping = client.system().ping().get(5, TimeUnit.SECONDS); - var metadata = client.system().getClusterMetadata().get(5, TimeUnit.SECONDS); - - // then - assertThat(ping).isEqualTo("pong"); - assertThat(metadata).isNotNull(); + @DisplayName("should reject commands before login") + void shouldRejectCommandsBeforeLogin() { + // when/then assertThatThrownBy(() -> client.streams().getStreams().get(5, TimeUnit.SECONDS)) .isInstanceOf(ExecutionException.class) .hasCauseInstanceOf(IggyNotConnectedException.class); @@ -165,10 +159,10 @@ void shouldAuthenticatePoolChannelsLazily() throws Exception { "test-topic") .get(5, TimeUnit.SECONDS); - // when - fire a burst of concurrent requests to exercise lazy - // authentication on the shared channel - int concurrentRequests = CONCURRENCY * 3; - List> futures = new ArrayList<>(); + // when - fire more concurrent requests than the pool size to force + // multiple channels to be created and lazily authenticated + int concurrentRequests = POOL_SIZE * 3; + List> futures = new ArrayList<>(); for (int i = 0; i < concurrentRequests; i++) { var future = client.messages() .sendMessages( @@ -206,8 +200,8 @@ void shouldReAuthenticateStaleChannelsAfterReLogin() throws Exception { "test-topic") .get(5, TimeUnit.SECONDS); - List> warmupFutures = new ArrayList<>(); - for (int i = 0; i < CONCURRENCY * 2; i++) { + List> warmupFutures = new ArrayList<>(); + for (int i = 0; i < POOL_SIZE * 2; i++) { warmupFutures.add(client.messages() .sendMessages( StreamId.of(streamName), @@ -224,8 +218,8 @@ void shouldReAuthenticateStaleChannelsAfterReLogin() throws Exception { log.info("Logout + re-login complete"); // then - all channels should re-authenticate transparently - List> postReLoginFutures = new ArrayList<>(); - for (int i = 0; i < CONCURRENCY * 2; i++) { + List> postReLoginFutures = new ArrayList<>(); + for (int i = 0; i < POOL_SIZE * 2; i++) { postReLoginFutures.add(client.messages() .sendMessages( StreamId.of(streamName), diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncConsumerGroupsTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncConsumerGroupsTest.java index 5a60691cd9..f1cc13111c 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncConsumerGroupsTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncConsumerGroupsTest.java @@ -71,10 +71,7 @@ public class AsyncConsumerGroupsTest extends BaseIntegrationTest { @BeforeAll public static void setup() throws Exception { log.info("Setting up async consumer groups test"); - client = AsyncIggyTcpClient.builder() - .host(serverHost()) - .port(serverTcpPort()) - .build(); + client = new AsyncIggyTcpClient(serverHost(), serverTcpPort()); client.connect() .thenCompose(v -> client.users().login(USERNAME, PASSWORD)) @@ -169,10 +166,7 @@ void shouldHandleMultipleClientsJoiningGroup() throws Exception { .get(TIMEOUT_SECONDS, TimeUnit.SECONDS); ConsumerId groupId = ConsumerId.of(created.id()); - AsyncIggyTcpClient secondClient = AsyncIggyTcpClient.builder() - .host(serverHost()) - .port(serverTcpPort()) - .build(); + AsyncIggyTcpClient secondClient = new AsyncIggyTcpClient(serverHost(), serverTcpPort()); try { secondClient .connect() @@ -317,14 +311,8 @@ void shouldHandleConcurrentJoinAndLeaveOperations() throws Exception { .get(TIMEOUT_SECONDS, TimeUnit.SECONDS); ConsumerId groupId = ConsumerId.of(created.id()); - AsyncIggyTcpClient secondClient = AsyncIggyTcpClient.builder() - .host(serverHost()) - .port(serverTcpPort()) - .build(); - AsyncIggyTcpClient thirdClient = AsyncIggyTcpClient.builder() - .host(serverHost()) - .port(serverTcpPort()) - .build(); + AsyncIggyTcpClient secondClient = new AsyncIggyTcpClient(serverHost(), serverTcpPort()); + AsyncIggyTcpClient thirdClient = new AsyncIggyTcpClient(serverHost(), serverTcpPort()); try { secondClient .connect() diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientBuilderTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientBuilderTest.java index ce7234a35f..6f33b106a1 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientBuilderTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientBuilderTest.java @@ -147,6 +147,24 @@ void shouldThrowExceptionForNullPort() { assertThatThrownBy(builder::build).isInstanceOf(IggyInvalidArgumentException.class); } + @Test + void shouldThrowExceptionForZeroConnectionPoolSize() { + // Given: Builder with 0 connection pool size + AsyncIggyTcpClientBuilder builder = AsyncIggyTcpClient.builder().connectionPoolSize(0); + + // When/Then: Building should throw IggyInvalidArgumentException + assertThatThrownBy(builder::build).isInstanceOf(IggyInvalidArgumentException.class); + } + + @Test + void shouldThrowExceptionForNegativeConnectionPoolSize() { + // Given: Builder with negative connection pool size + AsyncIggyTcpClientBuilder builder = AsyncIggyTcpClient.builder().connectionPoolSize(-1); + + // When/Then: Building should throw IggyInvalidArgumentException + assertThatThrownBy(builder::build).isInstanceOf(IggyInvalidArgumentException.class); + } + @Test void shouldThrowExceptionForZeroConnectionTimeout() { // Given: Builder with 0 connection timeout @@ -193,63 +211,6 @@ void shouldThrowExceptionForNegativeAcquireTimeout() { assertThatThrownBy(builder::build).isInstanceOf(IggyInvalidArgumentException.class); } - @Test - void shouldThrowExceptionForZeroRequestTimeout() { - AsyncIggyTcpClientBuilder builder = AsyncIggyTcpClient.builder().requestTimeout(Duration.ZERO); - - assertThatThrownBy(builder::build).isInstanceOf(IggyInvalidArgumentException.class); - } - - @Test - void shouldThrowExceptionForNegativeRequestTimeout() { - AsyncIggyTcpClientBuilder builder = AsyncIggyTcpClient.builder().requestTimeout(Duration.ofMillis(-1)); - - assertThatThrownBy(builder::build).isInstanceOf(IggyInvalidArgumentException.class); - } - - @Test - void shouldAcceptCustomMaximumVsrFrameSize() { - client = - AsyncIggyTcpClient.builder().maxVsrFrameSize(128L * 1024 * 1024).build(); - - assertThat(client).isNotNull(); - } - - @Test - void shouldRejectMaximumVsrFrameSizeBelowHeader() { - AsyncIggyTcpClientBuilder builder = AsyncIggyTcpClient.builder().maxVsrFrameSize(255); - - assertThatThrownBy(builder::build).isInstanceOf(IggyInvalidArgumentException.class); - } - - @Test - void shouldRejectNonPositiveMaximumVsrFrameSize() { - assertThatThrownBy(() -> AsyncIggyTcpClient.builder().maxVsrFrameSize(0).build()) - .isInstanceOf(IggyInvalidArgumentException.class); - assertThatThrownBy( - () -> AsyncIggyTcpClient.builder().maxVsrFrameSize(-1).build()) - .isInstanceOf(IggyInvalidArgumentException.class); - } - - @Test - void shouldRejectMaximumVsrFrameSizeAboveJavaBufferLimit() { - AsyncIggyTcpClientBuilder builder = AsyncIggyTcpClient.builder().maxVsrFrameSize((long) Integer.MAX_VALUE + 1); - - assertThatThrownBy(builder::build).isInstanceOf(IggyInvalidArgumentException.class); - } - - @Test - void shouldRejectNonPositiveHeartbeatInterval() { - assertThatThrownBy(() -> AsyncIggyTcpClient.builder() - .heartbeatInterval(Duration.ZERO) - .build()) - .isInstanceOf(IggyInvalidArgumentException.class); - assertThatThrownBy(() -> AsyncIggyTcpClient.builder() - .heartbeatInterval(Duration.ofMillis(-1)) - .build()) - .isInstanceOf(IggyInvalidArgumentException.class); - } - @Test void shouldMaintainBackwardCompatibilityWithOldConstructor() throws Exception { // Given: Old constructor approach @@ -487,6 +448,18 @@ void testBuildClientWithRequestTimeout() throws Exception { assertThat(client.users()).isNotNull(); } + @Test + void testBuildClientWithConnectionPoolSize() throws Exception { + client = AsyncIggyTcpClient.builder() + .host(serverHost()) + .port(serverTcpPort()) + .connectionPoolSize(5) + .build(); + client.connect().get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS); + + assertThat(client.users()).isNotNull(); + } + @Test void testBuildClientWithExponentialBackoffRetryPolicy() throws Exception { client = AsyncIggyTcpClient.builder() @@ -554,6 +527,7 @@ void testBuildClientWithAllConfigurationOptions() throws Exception { .credentials(TEST_USERNAME, TEST_PASSWORD) .connectionTimeout(Duration.ofSeconds(10)) .requestTimeout(Duration.ofSeconds(30)) + .connectionPoolSize(5) .retryPolicy(RetryPolicy.exponentialBackoff()) .tls(false) .build(); diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientLoginRoutingTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientLoginRoutingTest.java deleted file mode 100644 index b03974154f..0000000000 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientLoginRoutingTest.java +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iggy.client.async.tcp; - -import org.apache.iggy.client.ConnectionInfo; -import org.apache.iggy.user.IdentityInfo; -import org.junit.jupiter.api.Test; - -import java.util.ArrayDeque; -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.Queue; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -class AsyncIggyTcpClientLoginRoutingTest { - - private static final IdentityInfo FIRST_IDENTITY = new IdentityInfo(1L, Optional.empty()); - private static final IdentityInfo SECOND_IDENTITY = new IdentityInfo(2L, Optional.empty()); - - @Test - void shouldLoginOnceAfterAllLeaderRedirects() { - var client = new TestClient(); - client.enqueueLeader(new ConnectionInfo("leader-a", 8091)); - client.enqueueLeader(new ConnectionInfo("leader-b", 8092)); - client.enqueueStay(); - var attempts = new AtomicInteger(); - - var identity = client.loginOnLeader(() -> { - client.events.add("login"); - attempts.incrementAndGet(); - return CompletableFuture.completedFuture(FIRST_IDENTITY); - }) - .join(); - - assertThat(identity).isEqualTo(FIRST_IDENTITY); - assertThat(attempts).hasValue(1); - assertThat(client.retargets) - .containsExactly(new ConnectionInfo("leader-a", 8091), new ConnectionInfo("leader-b", 8092)); - assertThat(client.events).containsExactly("discover", "retarget", "discover", "retarget", "discover", "login"); - } - - @Test - void shouldSerializeDiscoveryAndLoginAcrossConcurrentCalls() { - var client = new TestClient(); - client.enqueueStay(); - client.enqueueStay(); - var firstAttempt = new CompletableFuture(); - - var firstLogin = client.loginOnLeader(() -> { - client.events.add("first-login"); - return firstAttempt; - }); - var secondLogin = client.loginOnLeader(() -> { - client.events.add("second-login"); - return CompletableFuture.completedFuture(SECOND_IDENTITY); - }); - - assertThat(client.events).containsExactly("discover", "first-login"); - assertThat(secondLogin).isNotDone(); - - firstAttempt.complete(FIRST_IDENTITY); - - assertThat(firstLogin.join()).isEqualTo(FIRST_IDENTITY); - assertThat(secondLogin.join()).isEqualTo(SECOND_IDENTITY); - assertThat(client.events).containsExactly("discover", "first-login", "discover", "second-login"); - } - - @Test - void shouldReleaseSerializationGateAfterRoutingFailure() { - var routingFailure = new IllegalStateException("metadata failed"); - var client = new TestClient(); - client.enqueueDiscovery(CompletableFuture.failedFuture(routingFailure)); - client.enqueueStay(); - var failedAttemptCount = new AtomicInteger(); - - var failedLogin = client.loginOnLeader(() -> { - failedAttemptCount.incrementAndGet(); - return CompletableFuture.completedFuture(FIRST_IDENTITY); - }); - var nextLogin = client.loginOnLeader(() -> CompletableFuture.completedFuture(SECOND_IDENTITY)); - - assertThatThrownBy(failedLogin::join) - .isInstanceOf(CompletionException.class) - .hasCause(routingFailure); - assertThat(failedAttemptCount).hasValue(0); - assertThat(nextLogin.join()).isEqualTo(SECOND_IDENTITY); - } - - @Test - void shouldReleaseSerializationGateAfterLoginFailure() { - var loginFailure = new IllegalStateException("register failed"); - var client = new TestClient(); - client.enqueueStay(); - client.enqueueStay(); - - var failedLogin = client.loginOnLeader(() -> CompletableFuture.failedFuture(loginFailure)); - var nextLogin = client.loginOnLeader(() -> CompletableFuture.completedFuture(SECOND_IDENTITY)); - - assertThatThrownBy(failedLogin::join) - .isInstanceOf(CompletionException.class) - .hasCause(loginFailure); - assertThat(nextLogin.join()).isEqualTo(SECOND_IDENTITY); - } - - @Test - void shouldKeepGateUntilCancelledCallFinishesItsRegister() { - var client = new TestClient(); - client.enqueueStay(); - client.enqueueStay(); - var committedRegister = new CompletableFuture(); - - var cancelledLogin = client.loginOnLeader(() -> committedRegister); - cancelledLogin.cancel(false); - var nextLogin = client.loginOnLeader(() -> CompletableFuture.completedFuture(SECOND_IDENTITY)); - - assertThat(nextLogin).isNotDone(); - - committedRegister.complete(FIRST_IDENTITY); - - assertThat(nextLogin.join()).isEqualTo(SECOND_IDENTITY); - } - - private static final class TestClient extends AsyncIggyTcpClient { - private final Queue>> discoveries = new ArrayDeque<>(); - private final List retargets = new ArrayList<>(); - private final List events = new ArrayList<>(); - - private TestClient() { - super("seed", 8090); - } - - private void enqueueLeader(ConnectionInfo target) { - enqueueDiscovery(CompletableFuture.completedFuture(Optional.of(target))); - } - - private void enqueueStay() { - enqueueDiscovery(CompletableFuture.completedFuture(Optional.empty())); - } - - private void enqueueDiscovery(CompletableFuture> discovery) { - discoveries.add(discovery); - } - - @Override - CompletableFuture> findLeaderElsewhere(ConnectionInfo currentTarget) { - events.add("discover"); - return discoveries.remove(); - } - - @Override - CompletableFuture retarget(ConnectionInfo newTarget) { - events.add("retarget"); - retargets.add(newTarget); - return CompletableFuture.completedFuture(null); - } - } -} diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java deleted file mode 100644 index ba481bec98..0000000000 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java +++ /dev/null @@ -1,337 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iggy.client.async.tcp; - -import io.netty.buffer.ByteBuf; -import io.netty.buffer.Unpooled; -import org.apache.iggy.exception.IggyServerException; -import org.junit.jupiter.api.Test; - -import java.io.EOFException; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.InetAddress; -import java.net.ServerSocket; -import java.net.Socket; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.charset.StandardCharsets; -import java.time.Duration; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -class AsyncIggyTcpClientTransientFailoverTest { - private static final int HEADER_SIZE = 256; - private static final int SIZE_OFFSET = 48; - private static final int COMMAND_OFFSET = 60; - private static final int REQUEST_ID_OFFSET = 168; - private static final int REQUEST_OPERATION_OFFSET = 176; - private static final int REQUEST_CODE_OFFSET = 196; - private static final int REPLY_REQUEST_ID_OFFSET = 200; - private static final int REPLY_OPERATION_OFFSET = 208; - private static final int REPLY_STATUS_OFFSET = 216; - private static final int EVICTION_REASON_OFFSET = 255; - - private static final int COMMAND_REPLY = 8; - private static final int COMMAND_EVICTION = 13; - private static final int OPERATION_REGISTER = 1; - private static final int OPERATION_NON_REPLICATED = 2; - private static final int OPERATION_CREATE_STREAM = 128; - private static final int GET_CLUSTER_METADATA_CODE = 12; - private static final int CREATE_STREAM_CODE = 202; - private static final int TRANSIENT_NOT_ACCEPTED = 58; - private static final int EVICTION_STALE_CLIENT = 13; - - @Test - void shouldRecheckLeaderAndReplayNotAcceptedMutation() throws Exception { - InetAddress loopback = InetAddress.getLoopbackAddress(); - try (ServerSocket oldLeaderSocket = new ServerSocket(0, 1, loopback); - ServerSocket newLeaderSocket = new ServerSocket(0, 1, loopback)) { - AtomicInteger denials = new AtomicInteger(); - AtomicInteger retriedMutations = new AtomicInteger(); - CompletableFuture oldLeader = serve( - oldLeaderSocket, - request -> handleOldLeader( - request, oldLeaderSocket.getLocalPort(), newLeaderSocket.getLocalPort(), denials)); - CompletableFuture newLeader = serve( - newLeaderSocket, - request -> handleNewLeader( - request, oldLeaderSocket.getLocalPort(), newLeaderSocket.getLocalPort(), retriedMutations)); - - AsyncIggyTcpClient client = AsyncIggyTcpClient.builder() - .host(loopback.getHostAddress()) - .port(oldLeaderSocket.getLocalPort()) - .credentials("iggy", "iggy") - .requestTimeout(Duration.ofSeconds(10)) - .build(); - try { - client.connect().get(5, TimeUnit.SECONDS); - client.login().get(5, TimeUnit.SECONDS); - - byte[] response = client.sendBinaryRequest(CREATE_STREAM_CODE, new byte[0]) - .get(10, TimeUnit.SECONDS); - - assertThat(response).isEmpty(); - assertThat(client.getConnectionInfo().port()).isEqualTo(newLeaderSocket.getLocalPort()); - assertThat(denials).hasValueGreaterThan(1); - assertThat(retriedMutations).hasValue(1); - } finally { - client.close().get(5, TimeUnit.SECONDS); - } - oldLeader.get(5, TimeUnit.SECONDS); - newLeader.get(5, TimeUnit.SECONDS); - } - } - - @Test - void shouldReplayTransientImplicitLoginAfterEviction() throws Exception { - InetAddress loopback = InetAddress.getLoopbackAddress(); - try (ServerSocket serverSocket = new ServerSocket(0, 1, loopback)) { - AtomicInteger registrations = new AtomicInteger(); - AtomicInteger mutations = new AtomicInteger(); - CompletableFuture server = serve(serverSocket, 2, request -> { - if (request.is(GET_CLUSTER_METADATA_CODE, OPERATION_NON_REPLICATED)) { - return Response.success(OPERATION_NON_REPLICATED, singleNodeMetadata(serverSocket.getLocalPort())); - } - if (request.operation() == OPERATION_REGISTER) { - int attempt = registrations.incrementAndGet(); - if (attempt == 2) { - return Response.success(OPERATION_REGISTER, transientResult(TRANSIENT_NOT_ACCEPTED)); - } - return Response.success(OPERATION_REGISTER, registerBody(attempt)); - } - if (request.operation() == OPERATION_CREATE_STREAM) { - if (mutations.incrementAndGet() == 1) { - return Response.eviction(EVICTION_STALE_CLIENT); - } - ByteBuf body = Unpooled.buffer(Integer.BYTES); - body.writeIntLE(0); - return Response.success(OPERATION_CREATE_STREAM, body); - } - throw new IllegalStateException("Unexpected request: " + request); - }); - - AsyncIggyTcpClient client = AsyncIggyTcpClient.builder() - .host(loopback.getHostAddress()) - .port(serverSocket.getLocalPort()) - .credentials("iggy", "iggy") - .requestTimeout(Duration.ofSeconds(5)) - .build(); - try { - client.connect().get(5, TimeUnit.SECONDS); - client.login().get(5, TimeUnit.SECONDS); - - assertThatThrownBy(() -> client.sendBinaryRequest(CREATE_STREAM_CODE, new byte[0]) - .get(5, TimeUnit.SECONDS)) - .hasCauseInstanceOf(IggyServerException.class); - - assertThat(client.sendBinaryRequest(CREATE_STREAM_CODE, new byte[0]) - .get(5, TimeUnit.SECONDS)) - .isEmpty(); - assertThat(registrations).hasValue(3); - assertThat(mutations).hasValue(2); - } finally { - client.close().get(5, TimeUnit.SECONDS); - } - server.get(5, TimeUnit.SECONDS); - } - } - - private static Response handleOldLeader( - Request request, int oldLeaderPort, int newLeaderPort, AtomicInteger denials) { - if (request.is(GET_CLUSTER_METADATA_CODE, OPERATION_NON_REPLICATED)) { - boolean demoted = denials.get() > 0; - return Response.success( - OPERATION_NON_REPLICATED, - clusterMetadata(oldLeaderPort, newLeaderPort, demoted ? newLeaderPort : oldLeaderPort)); - } - if (request.operation() == OPERATION_REGISTER) { - return Response.success(OPERATION_REGISTER, registerBody(1)); - } - if (request.operation() == OPERATION_CREATE_STREAM) { - denials.incrementAndGet(); - return Response.error(OPERATION_CREATE_STREAM, TRANSIENT_NOT_ACCEPTED); - } - throw new IllegalStateException("Unexpected request to old leader: " + request); - } - - private static Response handleNewLeader( - Request request, int oldLeaderPort, int newLeaderPort, AtomicInteger retriedMutations) { - if (request.is(GET_CLUSTER_METADATA_CODE, OPERATION_NON_REPLICATED)) { - return Response.success( - OPERATION_NON_REPLICATED, clusterMetadata(oldLeaderPort, newLeaderPort, newLeaderPort)); - } - if (request.operation() == OPERATION_REGISTER) { - return Response.success(OPERATION_REGISTER, registerBody(2)); - } - if (request.operation() == OPERATION_CREATE_STREAM) { - retriedMutations.incrementAndGet(); - ByteBuf body = Unpooled.buffer(Integer.BYTES); - body.writeIntLE(0); - return Response.success(OPERATION_CREATE_STREAM, body); - } - throw new IllegalStateException("Unexpected request to new leader: " + request); - } - - private static CompletableFuture serve(ServerSocket server, RequestHandler handler) { - return serve(server, 1, handler); - } - - private static CompletableFuture serve(ServerSocket server, int connectionCount, RequestHandler handler) { - return CompletableFuture.runAsync(() -> { - try { - for (int connection = 0; connection < connectionCount; connection++) { - try (Socket socket = server.accept()) { - InputStream input = socket.getInputStream(); - OutputStream output = socket.getOutputStream(); - Request request; - while ((request = readRequest(input)) != null) { - writeResponse(output, request, handler.handle(request)); - } - } - } - } catch (IOException error) { - throw new IllegalStateException("Mock VSR server failed", error); - } - }); - } - - private static Request readRequest(InputStream input) throws IOException { - byte[] header = input.readNBytes(HEADER_SIZE); - if (header.length == 0) { - return null; - } - if (header.length != HEADER_SIZE) { - throw new EOFException("Truncated VSR request header"); - } - ByteBuffer fields = ByteBuffer.wrap(header).order(ByteOrder.LITTLE_ENDIAN); - int size = fields.getInt(SIZE_OFFSET); - byte[] body = input.readNBytes(size - HEADER_SIZE); - if (body.length != size - HEADER_SIZE) { - throw new EOFException("Truncated VSR request body"); - } - return new Request( - Byte.toUnsignedInt(header[REQUEST_OPERATION_OFFSET]), - fields.getInt(REQUEST_CODE_OFFSET), - fields.getLong(REQUEST_ID_OFFSET)); - } - - private static void writeResponse(OutputStream output, Request request, Response response) throws IOException { - byte[] body = new byte[response.body().readableBytes()]; - response.body().readBytes(body); - response.body().release(); - byte[] header = new byte[HEADER_SIZE]; - ByteBuffer fields = ByteBuffer.wrap(header).order(ByteOrder.LITTLE_ENDIAN); - fields.putInt(SIZE_OFFSET, HEADER_SIZE + body.length); - header[COMMAND_OFFSET] = (byte) response.command(); - if (response.command() == COMMAND_EVICTION) { - header[EVICTION_REASON_OFFSET] = (byte) response.evictionReason(); - } else { - fields.putLong(REPLY_REQUEST_ID_OFFSET, request.requestId()); - header[REPLY_OPERATION_OFFSET] = (byte) response.operation(); - fields.putInt(REPLY_STATUS_OFFSET, response.status()); - } - output.write(header); - output.write(body); - output.flush(); - } - - private static ByteBuf registerBody(long session) { - ByteBuf body = Unpooled.buffer(); - body.writeIntLE(0); - body.writeIntLE(1); - body.writeLongLE(session); - body.writeIntLE(11 << 10); - body.writeByte(0); - return body; - } - - private static ByteBuf transientResult(int errorCode) { - ByteBuf body = Unpooled.buffer(3 * Integer.BYTES); - body.writeIntLE(1); - body.writeIntLE(0); - body.writeIntLE(errorCode); - return body; - } - - private static ByteBuf singleNodeMetadata(int port) { - ByteBuf body = Unpooled.buffer(); - writeString(body, "test-cluster"); - body.writeIntLE(1); - writeNode(body, "node", port, true); - return body; - } - - private static ByteBuf clusterMetadata(int oldLeaderPort, int newLeaderPort, int leaderPort) { - ByteBuf body = Unpooled.buffer(); - writeString(body, "test-cluster"); - body.writeIntLE(2); - writeNode(body, "old-node", oldLeaderPort, oldLeaderPort == leaderPort); - writeNode(body, "new-node", newLeaderPort, newLeaderPort == leaderPort); - return body; - } - - private static void writeNode(ByteBuf body, String name, int port, boolean leader) { - writeString(body, name); - writeString(body, InetAddress.getLoopbackAddress().getHostAddress()); - body.writeShortLE(port); - body.writeShortLE(0); - body.writeShortLE(0); - body.writeShortLE(0); - body.writeByte(leader ? 0 : 1); - body.writeByte(0); - } - - private static void writeString(ByteBuf body, String value) { - byte[] bytes = value.getBytes(StandardCharsets.UTF_8); - body.writeIntLE(bytes.length); - body.writeBytes(bytes); - } - - private record Request(int operation, int commandCode, long requestId) { - boolean is(int expectedCode, int expectedOperation) { - return commandCode == expectedCode && operation == expectedOperation; - } - } - - private record Response(int command, int operation, int status, int evictionReason, ByteBuf body) { - static Response success(int operation, ByteBuf body) { - return new Response(COMMAND_REPLY, operation, 0, 0, body); - } - - static Response error(int operation, int status) { - return new Response(COMMAND_REPLY, operation, status, 0, Unpooled.EMPTY_BUFFER); - } - - static Response eviction(int reason) { - return new Response(COMMAND_EVICTION, 0, 0, reason, Unpooled.EMPTY_BUFFER); - } - } - - @FunctionalInterface - private interface RequestHandler { - Response handle(Request request); - } -} diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionConcurrencyTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionConcurrencyTest.java deleted file mode 100644 index 442dfd5387..0000000000 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionConcurrencyTest.java +++ /dev/null @@ -1,409 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iggy.client.async.tcp; - -import io.netty.buffer.ByteBuf; -import io.netty.buffer.Unpooled; -import org.junit.jupiter.api.Test; - -import java.io.EOFException; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.InetAddress; -import java.net.ServerSocket; -import java.net.Socket; -import java.net.SocketTimeoutException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.charset.StandardCharsets; -import java.time.Duration; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -class AsyncTcpConnectionConcurrencyTest { - private static final int HEADER_SIZE = 256; - private static final int SIZE_OFFSET = 48; - private static final int COMMAND_OFFSET = 60; - private static final int REQUEST_ID_OFFSET = 168; - private static final int REQUEST_OPERATION_OFFSET = 176; - private static final int REQUEST_CODE_OFFSET = 196; - private static final int REPLY_REQUEST_ID_OFFSET = 200; - private static final int REPLY_OPERATION_OFFSET = 208; - private static final int REPLY_STATUS_OFFSET = 216; - - private static final int COMMAND_REPLY = 8; - private static final int OPERATION_REGISTER = 1; - private static final int OPERATION_NON_REPLICATED = 2; - private static final int OPERATION_LOGOUT = 3; - private static final int OPERATION_SEND_MESSAGES = 160; - private static final int PING_CODE = 1; - private static final int GET_CLUSTER_METADATA_CODE = 12; - private static final int LOGIN_CODE = 38; - private static final int LOGOUT_CODE = 39; - private static final int SEND_MESSAGES_CODE = 101; - private static final int TRANSIENT_NOT_COMMITTED = 57; - - @Test - void shouldCorrelateConcurrentPartitionResponsesInReverseOrder() throws Exception { - InetAddress loopback = InetAddress.getLoopbackAddress(); - try (ServerSocket serverSocket = new ServerSocket(0, 1, loopback)) { - CompletableFuture firstRequestRead = new CompletableFuture<>(); - CompletableFuture server = CompletableFuture.runAsync(() -> { - try (Socket socket = serverSocket.accept()) { - socket.setSoTimeout((int) TimeUnit.SECONDS.toMillis(2)); - InputStream input = socket.getInputStream(); - OutputStream output = socket.getOutputStream(); - - Request register = readRequest(input); - assertThat(register.operation()).isEqualTo(OPERATION_REGISTER); - writeResponse(output, register, registerBody()); - - Request firstRequest = readRequest(input); - assertThat(firstRequest.operation()).isEqualTo(OPERATION_SEND_MESSAGES); - firstRequestRead.complete(null); - Request secondRequest = readRequest(input); - assertThat(secondRequest.operation()).isEqualTo(OPERATION_SEND_MESSAGES); - assertThat(secondRequest.requestId()).isNotEqualTo(firstRequest.requestId()); - - Thread.sleep(200); - writeResponse(output, secondRequest, "second".getBytes(StandardCharsets.UTF_8)); - writeResponse(output, firstRequest, "first".getBytes(StandardCharsets.UTF_8)); - } catch (IOException error) { - firstRequestRead.completeExceptionally(error); - throw new IllegalStateException("Mock VSR server failed", error); - } catch (InterruptedException error) { - Thread.currentThread().interrupt(); - throw new IllegalStateException("Mock VSR server interrupted", error); - } - }); - - AsyncTcpConnection connection = new AsyncTcpConnection( - loopback.getHostAddress(), - serverSocket.getLocalPort(), - false, - Optional.empty(), - new AsyncTcpConnection.TcpConnectionPoolConfig(1000, 50), - Optional.of(Duration.ofSeconds(1)), - Optional.of(Duration.ofSeconds(2)), - Duration.ofHours(1), - 1024 * 1024, - null, - () -> {}, - ignored -> {}); - try { - connection.connect().get(5, TimeUnit.SECONDS); - connection - .send(LOGIN_CODE, loginPayload()) - .get(5, TimeUnit.SECONDS) - .release(); - - CompletableFuture first = connection.send(SEND_MESSAGES_CODE, sendMessagesPayload()); - firstRequestRead.get(5, TimeUnit.SECONDS); - CompletableFuture second = connection.send(SEND_MESSAGES_CODE, sendMessagesPayload()); - - assertResponse(first, "first"); - assertResponse(second, "second"); - } finally { - connection.close().get(5, TimeUnit.SECONDS); - } - server.get(5, TimeUnit.SECONDS); - } - } - - @Test - void shouldNotStarveHeartbeatWhileApplicationResponseIsPending() throws Exception { - InetAddress loopback = InetAddress.getLoopbackAddress(); - try (ServerSocket serverSocket = new ServerSocket(0, 1, loopback)) { - CompletableFuture applicationRequestRead = new CompletableFuture<>(); - CompletableFuture heartbeatRead = new CompletableFuture<>(); - CompletableFuture server = CompletableFuture.runAsync(() -> { - try (Socket socket = serverSocket.accept()) { - socket.setSoTimeout((int) TimeUnit.SECONDS.toMillis(2)); - InputStream input = socket.getInputStream(); - OutputStream output = socket.getOutputStream(); - - Request register = readRequest(input); - writeResponse(output, register, registerBody()); - - Request application = readRequest(input); - assertThat(application.operation()).isEqualTo(OPERATION_SEND_MESSAGES); - applicationRequestRead.complete(null); - - Request heartbeat = readRequest(input); - assertThat(heartbeat.operation()).isEqualTo(OPERATION_NON_REPLICATED); - assertThat(heartbeat.commandCode()).isEqualTo(PING_CODE); - heartbeatRead.complete(null); - writeResponse(output, heartbeat, new byte[0]); - writeResponse(output, application, "application".getBytes(StandardCharsets.UTF_8)); - } catch (IOException error) { - applicationRequestRead.completeExceptionally(error); - heartbeatRead.completeExceptionally(error); - throw new IllegalStateException("Mock VSR server failed", error); - } - }); - - AsyncTcpConnection connection = newConnection(serverSocket, Duration.ofMillis(300), 50); - try { - connection.connect().get(5, TimeUnit.SECONDS); - connection - .send(LOGIN_CODE, loginPayload()) - .get(5, TimeUnit.SECONDS) - .release(); - - CompletableFuture application = connection.send(SEND_MESSAGES_CODE, sendMessagesPayload()); - applicationRequestRead.get(5, TimeUnit.SECONDS); - heartbeatRead.get(5, TimeUnit.SECONDS); - assertResponse(application, "application"); - } finally { - connection.close().get(5, TimeUnit.SECONDS); - } - server.get(5, TimeUnit.SECONDS); - } - } - - @Test - void shouldReuseCorrelationIdForTransientReplay() throws Exception { - InetAddress loopback = InetAddress.getLoopbackAddress(); - try (ServerSocket serverSocket = new ServerSocket(0, 1, loopback)) { - CompletableFuture server = CompletableFuture.runAsync(() -> { - try (Socket socket = serverSocket.accept()) { - socket.setSoTimeout((int) TimeUnit.SECONDS.toMillis(2)); - InputStream input = socket.getInputStream(); - OutputStream output = socket.getOutputStream(); - - Request register = readRequest(input); - writeResponse(output, register, registerBody()); - - Request firstAttempt = readRequest(input); - writeResponse(output, firstAttempt, TRANSIENT_NOT_COMMITTED, new byte[0]); - Request secondAttempt = readRequest(input); - assertThat(secondAttempt.operation()).isEqualTo(firstAttempt.operation()); - assertThat(secondAttempt.requestId()).isEqualTo(firstAttempt.requestId()); - writeResponse(output, secondAttempt, "retried".getBytes(StandardCharsets.UTF_8)); - } catch (IOException error) { - throw new IllegalStateException("Mock VSR server failed", error); - } - }); - - AsyncTcpConnection connection = newConnection(serverSocket, Duration.ofHours(1), 50); - try { - connection.connect().get(5, TimeUnit.SECONDS); - connection - .send(LOGIN_CODE, loginPayload()) - .get(5, TimeUnit.SECONDS) - .release(); - - assertResponse(connection.send(SEND_MESSAGES_CODE, sendMessagesPayload()), "retried"); - } finally { - connection.close().get(5, TimeUnit.SECONDS); - } - server.get(5, TimeUnit.SECONDS); - } - } - - @Test - void shouldSerializeRegisterAndLogoutThroughTheirResponses() throws Exception { - InetAddress loopback = InetAddress.getLoopbackAddress(); - try (ServerSocket serverSocket = new ServerSocket(0, 1, loopback)) { - CompletableFuture registerRead = new CompletableFuture<>(); - CompletableFuture registerConcurrentSendStarted = new CompletableFuture<>(); - CompletableFuture logoutRead = new CompletableFuture<>(); - CompletableFuture logoutConcurrentSendStarted = new CompletableFuture<>(); - CompletableFuture server = CompletableFuture.runAsync(() -> { - try (Socket socket = serverSocket.accept()) { - socket.setSoTimeout((int) TimeUnit.SECONDS.toMillis(2)); - InputStream input = socket.getInputStream(); - OutputStream output = socket.getOutputStream(); - - Request register = readRequest(input); - registerRead.complete(null); - registerConcurrentSendStarted.get(2, TimeUnit.SECONDS); - assertNoRequest(input, socket); - writeResponse(output, register, registerBody()); - - Request metadataDuringRegister = readRequest(input); - assertThat(metadataDuringRegister.commandCode()).isEqualTo(GET_CLUSTER_METADATA_CODE); - writeResponse(output, metadataDuringRegister, new byte[0]); - - Request logout = readRequest(input); - assertThat(logout.operation()).isEqualTo(OPERATION_LOGOUT); - logoutRead.complete(null); - logoutConcurrentSendStarted.get(2, TimeUnit.SECONDS); - assertNoRequest(input, socket); - writeResponse(output, logout, new byte[0]); - - Request metadataDuringLogout = readRequest(input); - assertThat(metadataDuringLogout.commandCode()).isEqualTo(GET_CLUSTER_METADATA_CODE); - writeResponse(output, metadataDuringLogout, new byte[0]); - } catch (InterruptedException error) { - Thread.currentThread().interrupt(); - registerRead.completeExceptionally(error); - logoutRead.completeExceptionally(error); - throw new IllegalStateException("Mock VSR server interrupted", error); - } catch (IOException | ExecutionException | TimeoutException error) { - registerRead.completeExceptionally(error); - logoutRead.completeExceptionally(error); - throw new IllegalStateException("Mock VSR server failed", error); - } - }); - - AsyncTcpConnection connection = newConnection(serverSocket, Duration.ofHours(1), 500); - try { - connection.connect().get(5, TimeUnit.SECONDS); - CompletableFuture login = connection.send(LOGIN_CODE, loginPayload()); - registerRead.get(5, TimeUnit.SECONDS); - CompletableFuture metadataDuringRegister = - connection.send(GET_CLUSTER_METADATA_CODE, Unpooled.EMPTY_BUFFER); - registerConcurrentSendStarted.complete(null); - login.get(5, TimeUnit.SECONDS).release(); - metadataDuringRegister.get(5, TimeUnit.SECONDS).release(); - - CompletableFuture logout = connection.send(LOGOUT_CODE, Unpooled.EMPTY_BUFFER); - logoutRead.get(5, TimeUnit.SECONDS); - CompletableFuture metadataDuringLogout = - connection.send(GET_CLUSTER_METADATA_CODE, Unpooled.EMPTY_BUFFER); - logoutConcurrentSendStarted.complete(null); - logout.get(5, TimeUnit.SECONDS).release(); - metadataDuringLogout.get(5, TimeUnit.SECONDS).release(); - } finally { - connection.close().get(5, TimeUnit.SECONDS); - } - server.get(5, TimeUnit.SECONDS); - } - } - - private static AsyncTcpConnection newConnection( - ServerSocket serverSocket, Duration heartbeatInterval, long acquireTimeoutMillis) { - return new AsyncTcpConnection( - serverSocket.getInetAddress().getHostAddress(), - serverSocket.getLocalPort(), - false, - Optional.empty(), - new AsyncTcpConnection.TcpConnectionPoolConfig(1000, acquireTimeoutMillis), - Optional.of(Duration.ofSeconds(1)), - Optional.of(Duration.ofSeconds(2)), - heartbeatInterval, - 1024 * 1024, - null, - () -> {}, - ignored -> {}); - } - - private static Request readRequest(InputStream input) throws IOException { - byte[] header = input.readNBytes(HEADER_SIZE); - if (header.length != HEADER_SIZE) { - throw new EOFException("Truncated VSR request header"); - } - ByteBuffer fields = ByteBuffer.wrap(header).order(ByteOrder.LITTLE_ENDIAN); - int size = fields.getInt(SIZE_OFFSET); - byte[] body = input.readNBytes(size - HEADER_SIZE); - if (body.length != size - HEADER_SIZE) { - throw new EOFException("Truncated VSR request body"); - } - return new Request( - Byte.toUnsignedInt(header[REQUEST_OPERATION_OFFSET]), - fields.getLong(REQUEST_ID_OFFSET), - fields.getInt(REQUEST_CODE_OFFSET)); - } - - private static void writeResponse(OutputStream output, Request request, byte[] payload) throws IOException { - writeResponse(output, request, 0, payload); - } - - private static void writeResponse(OutputStream output, Request request, int status, byte[] payload) - throws IOException { - byte[] header = new byte[HEADER_SIZE]; - ByteBuffer fields = ByteBuffer.wrap(header).order(ByteOrder.LITTLE_ENDIAN); - fields.putInt(SIZE_OFFSET, HEADER_SIZE + payload.length); - fields.putLong(REPLY_REQUEST_ID_OFFSET, request.requestId()); - fields.putInt(REPLY_STATUS_OFFSET, status); - header[COMMAND_OFFSET] = COMMAND_REPLY; - header[REPLY_OPERATION_OFFSET] = (byte) request.operation(); - output.write(header); - output.write(payload); - output.flush(); - } - - private static void assertNoRequest(InputStream input, Socket socket) throws IOException { - socket.setSoTimeout(200); - assertThatThrownBy(input::read).isInstanceOf(SocketTimeoutException.class); - socket.setSoTimeout((int) TimeUnit.SECONDS.toMillis(2)); - } - - private static byte[] registerBody() { - return ByteBuffer.allocate(21) - .order(ByteOrder.LITTLE_ENDIAN) - .putInt(0) - .putInt(1) - .putLong(42) - .putInt(11 << 10) - .put((byte) 0) - .array(); - } - - private static ByteBuf loginPayload() { - ByteBuf payload = Unpooled.buffer(); - payload.writeByte(4); - payload.writeBytes("iggy".getBytes(StandardCharsets.UTF_8)); - payload.writeByte(4); - payload.writeBytes("iggy".getBytes(StandardCharsets.UTF_8)); - payload.writeIntLE(0); - payload.writeIntLE(0); - return payload; - } - - private static ByteBuf sendMessagesPayload() { - ByteBuf payload = Unpooled.buffer(); - payload.writeIntLE(22); - writeNumericIdentifier(payload, 1); - writeNumericIdentifier(payload, 2); - payload.writeByte(2); - payload.writeByte(4); - payload.writeIntLE(3); - payload.writeIntLE(0); - return payload; - } - - private static void writeNumericIdentifier(ByteBuf payload, int id) { - payload.writeByte(1); - payload.writeByte(4); - payload.writeIntLE(id); - } - - private static void assertResponse(CompletableFuture responseFuture, String expected) throws Exception { - ByteBuf response = responseFuture.get(5, TimeUnit.SECONDS); - try { - byte[] body = new byte[response.readableBytes()]; - response.readBytes(body); - assertThat(body).isEqualTo(expected.getBytes(StandardCharsets.UTF_8)); - } finally { - response.release(); - } - } - - private record Request(int operation, long requestId, int commandCode) {} -} diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionHeartbeatTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionHeartbeatTest.java deleted file mode 100644 index 8d0bd24c1e..0000000000 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionHeartbeatTest.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iggy.client.async.tcp; - -import org.junit.jupiter.api.Test; - -import java.io.EOFException; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.InetAddress; -import java.net.ServerSocket; -import java.net.Socket; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.time.Duration; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.assertj.core.api.Assertions.assertThat; - -class AsyncTcpConnectionHeartbeatTest { - private static final int HEADER_SIZE = 256; - private static final int SIZE_OFFSET = 48; - private static final int COMMAND_OFFSET = 60; - private static final int REQUEST_OPERATION_OFFSET = 176; - private static final int REQUEST_CODE_OFFSET = 196; - private static final int REQUEST_ID_OFFSET = 168; - private static final int REPLY_REQUEST_ID_OFFSET = 200; - private static final int REPLY_OPERATION_OFFSET = 208; - - private static final int COMMAND_REPLY = 8; - private static final int OPERATION_NON_REPLICATED = 2; - private static final int PING_CODE = 1; - - @Test - void shouldSendHeartbeatsUntilConnectionCloses() throws Exception { - InetAddress loopback = InetAddress.getLoopbackAddress(); - try (ServerSocket serverSocket = new ServerSocket(0, 1, loopback)) { - AtomicInteger requests = new AtomicInteger(); - CompletableFuture receivedTwoHeartbeats = new CompletableFuture<>(); - CompletableFuture server = CompletableFuture.runAsync(() -> { - try (Socket socket = serverSocket.accept()) { - socket.setSoTimeout((int) TimeUnit.SECONDS.toMillis(2)); - InputStream input = socket.getInputStream(); - OutputStream output = socket.getOutputStream(); - while (true) { - Request request = readRequest(input); - if (request == null) { - return; - } - assertThat(request.operation()).isEqualTo(OPERATION_NON_REPLICATED); - assertThat(request.commandCode()).isEqualTo(PING_CODE); - writeResponse(output, request); - if (requests.incrementAndGet() == 2) { - receivedTwoHeartbeats.complete(null); - } - } - } catch (IOException error) { - receivedTwoHeartbeats.completeExceptionally(error); - throw new IllegalStateException("Mock VSR server failed", error); - } - }); - - AsyncIggyTcpClient client = AsyncIggyTcpClient.builder() - .host(loopback.getHostAddress()) - .port(serverSocket.getLocalPort()) - .heartbeatInterval(Duration.ofMillis(25)) - .requestTimeout(Duration.ofSeconds(1)) - .build(); - client.connect().get(5, TimeUnit.SECONDS); - receivedTwoHeartbeats.get(5, TimeUnit.SECONDS); - client.close().get(5, TimeUnit.SECONDS); - int requestsAtClose = requests.get(); - - server.get(5, TimeUnit.SECONDS); - Thread.sleep(100); - assertThat(requests).hasValue(requestsAtClose); - } - } - - private static Request readRequest(InputStream input) throws IOException { - byte[] header = input.readNBytes(HEADER_SIZE); - if (header.length == 0) { - return null; - } - if (header.length != HEADER_SIZE) { - throw new EOFException("Truncated VSR request header"); - } - ByteBuffer fields = ByteBuffer.wrap(header).order(ByteOrder.LITTLE_ENDIAN); - int size = fields.getInt(SIZE_OFFSET); - byte[] body = input.readNBytes(size - HEADER_SIZE); - if (body.length != size - HEADER_SIZE) { - throw new EOFException("Truncated VSR request body"); - } - return new Request( - Byte.toUnsignedInt(header[REQUEST_OPERATION_OFFSET]), - fields.getInt(REQUEST_CODE_OFFSET), - fields.getLong(REQUEST_ID_OFFSET)); - } - - private static void writeResponse(OutputStream output, Request request) throws IOException { - byte[] header = new byte[HEADER_SIZE]; - ByteBuffer fields = ByteBuffer.wrap(header).order(ByteOrder.LITTLE_ENDIAN); - fields.putInt(SIZE_OFFSET, HEADER_SIZE); - fields.putLong(REPLY_REQUEST_ID_OFFSET, request.requestId()); - header[COMMAND_OFFSET] = COMMAND_REPLY; - header[REPLY_OPERATION_OFFSET] = OPERATION_NON_REPLICATED; - output.write(header); - output.flush(); - } - - private record Request(int operation, int commandCode, long requestId) {} -} diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionRequestTimeoutTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionRequestTimeoutTest.java deleted file mode 100644 index bf4270ddbb..0000000000 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionRequestTimeoutTest.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iggy.client.async.tcp; - -import org.apache.iggy.exception.IggyTimeoutException; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.net.InetAddress; -import java.net.ServerSocket; -import java.net.Socket; -import java.time.Duration; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -class AsyncTcpConnectionRequestTimeoutTest { - private static final int TEST_TIMEOUT_SECONDS = 5; - private static final Duration REQUEST_TIMEOUT = Duration.ofMillis(100); - - private AsyncIggyTcpClient client; - - @AfterEach - void tearDown() throws Exception { - if (client != null) { - client.close().get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS); - } - } - - @Test - void shouldReplaceChannelAfterResponseTimeout() throws Exception { - try (ServerSocket server = new ServerSocket(0, 2, InetAddress.getLoopbackAddress())) { - client = AsyncIggyTcpClient.builder() - .host(server.getInetAddress().getHostAddress()) - .port(server.getLocalPort()) - .requestTimeout(REQUEST_TIMEOUT) - .build(); - - CompletableFuture firstAccepted = accept(server); - client.connect().get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS); - try (Socket firstSocket = firstAccepted.get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { - verifyRequestTimesOutAndClosesSocket(client.system().ping(), firstSocket); - } - - CompletableFuture secondAccepted = accept(server); - CompletableFuture secondResponse = client.system().ping(); - try (Socket secondSocket = secondAccepted.get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { - verifyRequestTimesOutAndClosesSocket(secondResponse, secondSocket); - } - } - } - - @Test - void shouldFailMultiplexedRequestsAndUseReplacementChannelAfterTimeout() throws Exception { - try (ServerSocket server = new ServerSocket(0, 2, InetAddress.getLoopbackAddress())) { - client = AsyncIggyTcpClient.builder() - .host(server.getInetAddress().getHostAddress()) - .port(server.getLocalPort()) - .requestTimeout(REQUEST_TIMEOUT) - .build(); - - CompletableFuture firstAccepted = accept(server); - client.connect().get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS); - try (Socket firstSocket = firstAccepted.get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { - CompletableFuture first = client.system().ping(); - CompletableFuture second = client.system().ping(); - - assertThat(verifyRequestTimesOutAndClosesSocket(first, firstSocket)) - .hasSize(2 * 256); - assertThatThrownBy(() -> second.get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS)) - .isInstanceOf(ExecutionException.class) - .hasCauseInstanceOf(IggyTimeoutException.class); - } - - CompletableFuture secondAccepted = accept(server); - CompletableFuture third = client.system().ping(); - try (Socket secondSocket = secondAccepted.get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { - assertThat(verifyRequestTimesOutAndClosesSocket(third, secondSocket)) - .hasSize(256); - } - } - } - - private static byte[] verifyRequestTimesOutAndClosesSocket(CompletableFuture response, Socket socket) - throws Exception { - assertThatThrownBy(() -> response.get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS)) - .isInstanceOf(ExecutionException.class) - .hasCauseInstanceOf(IggyTimeoutException.class); - socket.setSoTimeout((int) TimeUnit.SECONDS.toMillis(TEST_TIMEOUT_SECONDS)); - byte[] request = socket.getInputStream().readAllBytes(); - assertThat(request).isNotEmpty(); - return request; - } - - private static CompletableFuture accept(ServerSocket server) { - return CompletableFuture.supplyAsync(() -> { - try { - return server.accept(); - } catch (IOException e) { - throw new IllegalStateException("Failed to accept test connection", e); - } - }); - } -} diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/ClientRoutingStateTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/ClientRoutingStateTest.java deleted file mode 100644 index 74a508bdc6..0000000000 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/ClientRoutingStateTest.java +++ /dev/null @@ -1,251 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iggy.client.async.tcp; - -import org.apache.iggy.identifier.ConsumerId; -import org.apache.iggy.identifier.StreamId; -import org.apache.iggy.identifier.TopicId; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; - -import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; - -class ClientRoutingStateTest { - - private final ClientRoutingState state = new ClientRoutingState(); - - @Nested - class Keys { - - @Test - void shouldDistinguishNumericAndNamedIdentifiersWithTheSameText() { - assertThat(ClientRoutingState.topicKey(StreamId.of(1L), TopicId.of(2L))) - .isNotEqualTo(ClientRoutingState.topicKey(StreamId.of("1"), TopicId.of("2"))); - } - - @Test - void shouldDistinguishIdentifiersContainingTheFormerDelimiter() { - assertThat(ClientRoutingState.topicKey(StreamId.of("orders|created"), TopicId.of("events"))) - .isNotEqualTo(ClientRoutingState.topicKey(StreamId.of("orders"), TopicId.of("created|events"))); - } - - @Test - void shouldDistinguishConsumerIdentifiersWithTheSameText() { - assertThat(ClientRoutingState.groupKey(StreamId.of(1L), TopicId.of(2L), ConsumerId.of(3L))) - .isNotEqualTo(ClientRoutingState.groupKey(StreamId.of(1L), TopicId.of(2L), ConsumerId.of("3"))); - } - } - - @Nested - class BalancedCursor { - - @Test - void shouldRoundRobinAcrossPartitions() { - // pinned to the Rust SDK: 3 partitions give 0, 1, 2, 0 - var topic = topic("s", "t"); - - assertThat(state.nextBalancedPartition(topic, 3)).isEqualTo(0); - assertThat(state.nextBalancedPartition(topic, 3)).isEqualTo(1); - assertThat(state.nextBalancedPartition(topic, 3)).isEqualTo(2); - assertThat(state.nextBalancedPartition(topic, 3)).isEqualTo(0); - } - - @Test - void shouldKeepIndependentCursorsPerTopic() { - var first = topic("s", "a"); - var second = topic("s", "b"); - - assertThat(state.nextBalancedPartition(first, 2)).isEqualTo(0); - assertThat(state.nextBalancedPartition(second, 2)).isEqualTo(0); - assertThat(state.nextBalancedPartition(first, 2)).isEqualTo(1); - } - - @Test - void shouldKeepIndependentCursorsForFormerlyCollidingTopics() { - var numeric = ClientRoutingState.topicKey(StreamId.of(1L), TopicId.of(2L)); - var named = ClientRoutingState.topicKey(StreamId.of("1"), TopicId.of("2")); - - assertThat(state.nextBalancedPartition(numeric, 2)).isEqualTo(0); - assertThat(state.nextBalancedPartition(named, 2)).isEqualTo(0); - assertThat(state.nextBalancedPartition(numeric, 2)).isEqualTo(1); - } - - @Test - void shouldFallBackToZeroWithoutPartitions() { - assertThat(state.nextBalancedPartition(topic("s", "t"), 0)).isEqualTo(0); - } - } - - @Nested - class GroupAssignments { - - @Test - void shouldRoundRobinAcrossAssignedPartitions() { - // pinned to the Rust SDK: assignment [0, 1, 2] gives 0, 1, 2, 0 - var group = group("s", "t", "g"); - state.setAssignment(group, 1, List.of(0L, 1L, 2L), 0); - - assertThat(state.nextGroupPartition(group)).hasValue(0); - assertThat(state.nextGroupPartition(group)).hasValue(1); - assertThat(state.nextGroupPartition(group)).hasValue(2); - assertThat(state.nextGroupPartition(group)).hasValue(0); - } - - @Test - void shouldReturnEmptyWithoutAssignment() { - assertThat(state.nextGroupPartition(group("s", "t", "g"))).isEmpty(); - } - - @Test - void shouldReturnEmptyForMemberOwningNoPartitions() { - var group = group("s", "t", "g"); - state.setAssignment(group, 1, List.of(), 0); - - assertThat(state.nextGroupPartition(group)).isEmpty(); - } - - @Test - void shouldResetCursorWhenGenerationAdvances() { - var group = group("s", "t", "g"); - state.setAssignment(group, 1, List.of(0L, 1L, 2L), 0); - state.nextGroupPartition(group); - state.nextGroupPartition(group); - - state.setAssignment(group, 2, List.of(0L, 1L, 2L), 0); - - assertThat(state.nextGroupPartition(group)).hasValue(0); - } - - @Test - void shouldKeepCursorWhenGenerationIsUnchanged() { - var group = group("s", "t", "g"); - state.setAssignment(group, 1, List.of(0L, 1L, 2L), 0); - state.nextGroupPartition(group); - - state.setAssignment(group, 1, List.of(0L, 1L, 2L), 100); - - assertThat(state.nextGroupPartition(group)).hasValue(1); - } - - @Test - void shouldWrapCursorPositionWhenAssignmentShrinks() { - var group = group("s", "t", "g"); - state.setAssignment(group, 1, List.of(0L, 1L, 2L), 0); - state.nextGroupPartition(group); - state.nextGroupPartition(group); - - state.setAssignment(group, 1, List.of(5L), 0); - - assertThat(state.nextGroupPartition(group)).hasValue(5); - } - - @Test - void shouldInvalidateSingleAssignment() { - var group = group("s", "t", "g"); - state.setAssignment(group, 1, List.of(0L), 0); - state.invalidateAssignment(group); - - assertThat(state.assignment(group)).isEmpty(); - } - - @Test - void shouldClearAllAssignments() { - var first = group("s", "t", "g1"); - var second = group("s", "t", "g2"); - state.setAssignment(first, 1, List.of(0L), 0); - state.setAssignment(second, 1, List.of(1L), 0); - - state.clearAssignments(); - - assertThat(state.assignment(first)).isEmpty(); - assertThat(state.assignment(second)).isEmpty(); - } - - @Test - void shouldKeepAssignmentsIndependentForFormerlyCollidingGroups() { - var numeric = ClientRoutingState.groupKey(StreamId.of(1L), TopicId.of(2L), ConsumerId.of(3L)); - var named = ClientRoutingState.groupKey(StreamId.of("1"), TopicId.of("2"), ConsumerId.of("3")); - state.setAssignment(numeric, 1, List.of(1L), 0); - state.setAssignment(named, 1, List.of(2L), 0); - - assertThat(state.nextGroupPartition(numeric)).hasValue(1); - assertThat(state.nextGroupPartition(named)).hasValue(2); - } - - @Test - void shouldExposeSyncTimestampForStalenessChecks() { - var group = group("s", "t", "g"); - state.setAssignment(group, 1, List.of(0L), 42L); - - assertThat(state.assignment(group)).hasValueSatisfying(assignment -> assertThat(assignment.syncedAtNanos()) - .isEqualTo(42L)); - } - } - - @Nested - class PartitionCounts { - - @Test - void shouldCachePartitionCountWithFetchTimestamp() { - var topic = topic("s", "t"); - assertThat(state.partitionCount(topic)).isEmpty(); - - state.setPartitionCount(topic, 4L, 42L); - - assertThat(state.partitionCount(topic)).hasValueSatisfying(cached -> { - assertThat(cached.count()).isEqualTo(4L); - assertThat(cached.fetchedAtNanos()).isEqualTo(42L); - }); - } - - @Test - void shouldInvalidatePartitionCount() { - var topic = topic("s", "t"); - state.setPartitionCount(topic, 4L, 0L); - - state.invalidatePartitionCount(topic); - - assertThat(state.partitionCount(topic)).isEmpty(); - } - - @Test - void shouldKeepCountsIndependentForFormerlyCollidingTopics() { - var numeric = ClientRoutingState.topicKey(StreamId.of(1L), TopicId.of(2L)); - var named = ClientRoutingState.topicKey(StreamId.of("1"), TopicId.of("2")); - state.setPartitionCount(numeric, 4L, 0L); - state.setPartitionCount(named, 8L, 0L); - - assertThat(state.partitionCount(numeric)) - .hasValueSatisfying(cached -> assertThat(cached.count()).isEqualTo(4L)); - assertThat(state.partitionCount(named)) - .hasValueSatisfying(cached -> assertThat(cached.count()).isEqualTo(8L)); - } - } - - private static ClientRoutingState.TopicKey topic(String stream, String topic) { - return ClientRoutingState.topicKey(StreamId.of(stream), TopicId.of(topic)); - } - - private static ClientRoutingState.GroupKey group(String stream, String topic, String consumer) { - return ClientRoutingState.groupKey(StreamId.of(stream), TopicId.of(topic), ConsumerId.of(consumer)); - } -} diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/IggyFrameDecoderTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/IggyFrameDecoderTest.java new file mode 100644 index 0000000000..448f07de7b --- /dev/null +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/IggyFrameDecoderTest.java @@ -0,0 +1,455 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.client.async.tcp; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.channel.embedded.EmbeddedChannel; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class IggyFrameDecoderTest { + + private EmbeddedChannel channel; + + @AfterEach + void tearDown() { + if (channel != null) { + // Release any remaining messages + Object msg; + while ((msg = channel.readInbound()) != null) { + if (msg instanceof ByteBuf) { + ((ByteBuf) msg).release(); + } + } + channel.finishAndReleaseAll(); + } + } + + @Nested + class CompleteFrames { + + @Test + void shouldDecodeCompleteFrameWithSmallPayload() { + // given + channel = new EmbeddedChannel(new IggyFrameDecoder()); + ByteBuf input = Unpooled.buffer(); + input.writeIntLE(0); // status = success + input.writeIntLE(5); // length = 5 bytes + input.writeBytes("hello".getBytes()); // payload + + // when + channel.writeInbound(input); + + // then + ByteBuf decoded = channel.readInbound(); + assertThat(decoded).isNotNull(); + assertThat(decoded.readableBytes()).isEqualTo(8 + 5); // header + payload + assertThat(decoded.readIntLE()).isEqualTo(0); // status + assertThat(decoded.readIntLE()).isEqualTo(5); // length + byte[] payload = new byte[5]; + decoded.readBytes(payload); + assertThat(new String(payload)).isEqualTo("hello"); + decoded.release(); + } + + @Test + void shouldDecodeCompleteFrameWithZeroLengthPayload() { + // given + channel = new EmbeddedChannel(new IggyFrameDecoder()); + ByteBuf input = Unpooled.buffer(); + input.writeIntLE(0); // status + input.writeIntLE(0); // length = 0 + + // when + channel.writeInbound(input); + + // then + ByteBuf decoded = channel.readInbound(); + assertThat(decoded).isNotNull(); + assertThat(decoded.readableBytes()).isEqualTo(8); // header only + assertThat(decoded.readIntLE()).isEqualTo(0); + assertThat(decoded.readIntLE()).isEqualTo(0); + decoded.release(); + } + + @Test + void shouldDecodeCompleteFrameWithLargePayload() { + // given + channel = new EmbeddedChannel(new IggyFrameDecoder()); + byte[] largePayload = new byte[10000]; + for (int i = 0; i < largePayload.length; i++) { + largePayload[i] = (byte) (i % 256); + } + + ByteBuf input = Unpooled.buffer(); + input.writeIntLE(200); // error status + input.writeIntLE(10000); // large length + input.writeBytes(largePayload); + + // when + channel.writeInbound(input); + + // then + ByteBuf decoded = channel.readInbound(); + assertThat(decoded).isNotNull(); + assertThat(decoded.readableBytes()).isEqualTo(8 + 10000); + assertThat(decoded.readIntLE()).isEqualTo(200); + assertThat(decoded.readIntLE()).isEqualTo(10000); + decoded.release(); + } + + @Test + void shouldDecodeFrameWithVariousStatusCodes() { + // given + channel = new EmbeddedChannel(new IggyFrameDecoder()); + + for (int status = 0; status <= 5; status++) { + ByteBuf input = Unpooled.buffer(); + input.writeIntLE(status); + input.writeIntLE(1); + input.writeByte(42); + + // when + channel.writeInbound(input); + + // then + ByteBuf decoded = channel.readInbound(); + assertThat(decoded).isNotNull(); + assertThat(decoded.readIntLE()).isEqualTo(status); + decoded.skipBytes(4); // length + decoded.skipBytes(1); // payload + decoded.release(); + } + } + } + + @Nested + class IncompleteFrames { + + @Test + void shouldWaitForCompleteHeaderWhenOnlyPartialHeaderAvailable() { + // given + channel = new EmbeddedChannel(new IggyFrameDecoder()); + ByteBuf input = Unpooled.buffer(); + input.writeIntLE(0); // only status, missing length (4 bytes total, need 8) + + // when + boolean hasMessage = channel.writeInbound(input); + + // then + assertThat(hasMessage).isFalse(); // No complete frame yet + ByteBuf decoded = channel.readInbound(); + assertThat(decoded).isNull(); // Nothing to read + } + + @Test + void shouldWaitForCompletePayloadWhenOnlyHeaderAvailable() { + // given + channel = new EmbeddedChannel(new IggyFrameDecoder()); + ByteBuf input = Unpooled.buffer(); + input.writeIntLE(0); // status + input.writeIntLE(100); // expects 100 bytes payload + // But no payload written + + // when + boolean hasMessage = channel.writeInbound(input); + + // then + assertThat(hasMessage).isFalse(); + ByteBuf decoded = channel.readInbound(); + assertThat(decoded).isNull(); + } + + @Test + void shouldWaitForCompletePayloadWhenPartialPayloadAvailable() { + // given + channel = new EmbeddedChannel(new IggyFrameDecoder()); + ByteBuf input = Unpooled.buffer(); + input.writeIntLE(0); + input.writeIntLE(100); // expects 100 bytes + input.writeBytes(new byte[50]); // only 50 bytes available + + // when + boolean hasMessage = channel.writeInbound(input); + + // then + assertThat(hasMessage).isFalse(); + assertThat((ByteBuf) channel.readInbound()).isNull(); + } + + @Test + void shouldEventuallyDecodeWhenMoreDataArrives() { + // given + channel = new EmbeddedChannel(new IggyFrameDecoder()); + ByteBuf firstChunk = Unpooled.buffer(); + firstChunk.writeIntLE(0); + firstChunk.writeIntLE(10); + firstChunk.writeBytes(new byte[5]); // only 5 of 10 bytes + + // when - first chunk + boolean hasMessage1 = channel.writeInbound(firstChunk); + assertThat(hasMessage1).isFalse(); + + // when - second chunk completes the frame + ByteBuf secondChunk = Unpooled.buffer(); + secondChunk.writeBytes(new byte[5]); // remaining 5 bytes + boolean hasMessage2 = channel.writeInbound(secondChunk); + + // then + assertThat(hasMessage2).isTrue(); + ByteBuf decoded = channel.readInbound(); + assertThat(decoded).isNotNull(); + assertThat(decoded.readableBytes()).isEqualTo(8 + 10); + decoded.release(); + } + } + + @Nested + class MultipleFrames { + + @Test + void shouldDecodeMultipleFramesInSequence() { + // given + channel = new EmbeddedChannel(new IggyFrameDecoder()); + ByteBuf input = Unpooled.buffer(); + + // Frame 1 + input.writeIntLE(0); + input.writeIntLE(3); + input.writeBytes("abc".getBytes()); + + // Frame 2 + input.writeIntLE(1); + input.writeIntLE(2); + input.writeBytes("de".getBytes()); + + // when + channel.writeInbound(input); + + // then - frame 1 + ByteBuf decoded1 = channel.readInbound(); + assertThat(decoded1).isNotNull(); + assertThat(decoded1.readableBytes()).isEqualTo(8 + 3); + decoded1.release(); + + // then - frame 2 + ByteBuf decoded2 = channel.readInbound(); + assertThat(decoded2).isNotNull(); + assertThat(decoded2.readableBytes()).isEqualTo(8 + 2); + decoded2.release(); + + // No more frames + assertThat((ByteBuf) channel.readInbound()).isNull(); + } + + @Test + void shouldDecodeThreeFramesCorrectly() { + // given + channel = new EmbeddedChannel(new IggyFrameDecoder()); + ByteBuf input = Unpooled.buffer(); + + for (int i = 0; i < 3; i++) { + input.writeIntLE(i); + input.writeIntLE(1); + input.writeByte(i); + } + + // when + channel.writeInbound(input); + + // then + for (int i = 0; i < 3; i++) { + ByteBuf decoded = channel.readInbound(); + assertThat(decoded).isNotNull(); + assertThat(decoded.readIntLE()).isEqualTo(i); + decoded.skipBytes(4 + 1); // skip length and payload + decoded.release(); + } + + assertThat((ByteBuf) channel.readInbound()).isNull(); + } + } + + @Nested + class EdgeCases { + + @Test + void shouldNotAdvanceReaderIndexWhenPayloadIncomplete() { + // given + channel = new EmbeddedChannel(new IggyFrameDecoder()); + ByteBuf input = Unpooled.buffer(); + input.writeIntLE(0); + input.writeIntLE(100); // expects 100 bytes + input.writeBytes(new byte[50]); // only 50 bytes + int readerIndexBefore = input.readerIndex(); + + // when + boolean hasMessage = channel.writeInbound(input); + + // then - decoder should wait for complete payload + assertThat(hasMessage).isFalse(); + ByteBuf decoded = channel.readInbound(); + assertThat(decoded).isNull(); + assertThat(input.readerIndex()).isEqualTo(readerIndexBefore); + } + + @Test + void shouldHandleEmptyInput() { + // given + channel = new EmbeddedChannel(new IggyFrameDecoder()); + ByteBuf input = Unpooled.buffer(); // empty buffer + + // when + boolean hasMessage = channel.writeInbound(input); + + // then + assertThat(hasMessage).isFalse(); + assertThat((ByteBuf) channel.readInbound()).isNull(); + } + + @Test + void shouldHandleSingleByteInput() { + // given + channel = new EmbeddedChannel(new IggyFrameDecoder()); + ByteBuf input = Unpooled.buffer(); + input.writeByte(0); // only 1 byte + + // when + boolean hasMessage = channel.writeInbound(input); + + // then + assertThat(hasMessage).isFalse(); + assertThat((ByteBuf) channel.readInbound()).isNull(); + } + + @Test + void shouldHandleExactlyHeaderSizeWithoutPayload() { + // given + channel = new EmbeddedChannel(new IggyFrameDecoder()); + ByteBuf input = Unpooled.buffer(); + input.writeIntLE(0); + input.writeIntLE(10); // expects payload, but none provided + // Exactly 8 bytes (header size) + + // when + boolean hasMessage = channel.writeInbound(input); + + // then + assertThat(hasMessage).isFalse(); // Waiting for payload + assertThat((ByteBuf) channel.readInbound()).isNull(); + } + + @Test + void shouldDecodeFrameFollowedByPartialNextFrame() { + // given + channel = new EmbeddedChannel(new IggyFrameDecoder()); + ByteBuf input = Unpooled.buffer(); + + // Complete frame 1 + input.writeIntLE(0); + input.writeIntLE(5); + input.writeBytes("hello".getBytes()); + + // Partial frame 2 (only header) + input.writeIntLE(1); + input.writeIntLE(10); + // No payload for frame 2 + + // when + channel.writeInbound(input); + + // then - should get frame 1 + ByteBuf decoded1 = channel.readInbound(); + assertThat(decoded1).isNotNull(); + assertThat(decoded1.readableBytes()).isEqualTo(8 + 5); + decoded1.release(); + + // Frame 2 should not be available yet + assertThat((ByteBuf) channel.readInbound()).isNull(); + } + + @Test + void shouldHandleMaxIntPayloadLength() { + // given - test with a reasonably large payload (not actual MAX_INT to avoid OOM) + channel = new EmbeddedChannel(new IggyFrameDecoder()); + int largeSize = 1000000; // 1MB + ByteBuf input = Unpooled.buffer(); + input.writeIntLE(0); + input.writeIntLE(largeSize); + input.writeBytes(new byte[largeSize]); + + // when + channel.writeInbound(input); + + // then + ByteBuf decoded = channel.readInbound(); + assertThat(decoded).isNotNull(); + assertThat(decoded.readableBytes()).isEqualTo(8 + largeSize); + decoded.release(); + } + } + + @Nested + class BufferManagement { + + @Test + void shouldCreateNewBufferForEachDecodedFrame() { + // given + channel = new EmbeddedChannel(new IggyFrameDecoder()); + ByteBuf input = Unpooled.buffer(); + input.writeIntLE(0); + input.writeIntLE(5); + input.writeBytes("hello".getBytes()); + + // when + channel.writeInbound(input); + ByteBuf decoded = channel.readInbound(); + + // then - decoded buffer should be independent + assertThat(decoded).isNotNull(); + assertThat(decoded).isNotSameAs(input); + assertThat(decoded.readableBytes()).isEqualTo(13); + decoded.release(); + } + + @Test + void shouldAllowManualReleaseOfDecodedBuffers() { + // given + channel = new EmbeddedChannel(new IggyFrameDecoder()); + ByteBuf input = Unpooled.buffer(); + input.writeIntLE(0); + input.writeIntLE(3); + input.writeBytes("foo".getBytes()); + + // when + channel.writeInbound(input); + ByteBuf decoded = channel.readInbound(); + + // then - should be releasable + assertThat(decoded.refCnt()).isEqualTo(1); + decoded.release(); + assertThat(decoded.refCnt()).isEqualTo(0); + } + } +} diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/IggyResponseHandlerTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/IggyResponseHandlerTest.java new file mode 100644 index 0000000000..fe542542fd --- /dev/null +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/IggyResponseHandlerTest.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.client.async.tcp; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.embedded.EmbeddedChannel; +import org.apache.iggy.exception.IggyConnectionException; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.CompletableFuture; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class IggyResponseHandlerTest { + + @Test + void shouldFailPendingRequestsWhenChannelBecomesInactive() { + var handler = new AsyncTcpConnection.IggyResponseHandler(); + var channel = new EmbeddedChannel(handler); + CompletableFuture pending = new CompletableFuture<>(); + handler.enqueueRequest(pending); + + channel.close(); + + assertThat(pending).isCompletedExceptionally(); + assertThatThrownBy(pending::join).hasCauseInstanceOf(IggyConnectionException.class); + } + + @Test + void shouldFailPendingRequestsOnPipelineException() { + var handler = new AsyncTcpConnection.IggyResponseHandler(); + var channel = new EmbeddedChannel(handler); + CompletableFuture pending = new CompletableFuture<>(); + handler.enqueueRequest(pending); + + var failure = new IllegalStateException("broken pipe"); + channel.pipeline().fireExceptionCaught(failure); + + assertThat(pending).isCompletedExceptionally(); + assertThatThrownBy(pending::join).hasCause(failure); + } +} diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/LoginRoutingHookTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/LoginRoutingHookTest.java deleted file mode 100644 index 500eef75fe..0000000000 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/LoginRoutingHookTest.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iggy.client.async.tcp; - -import org.apache.iggy.user.IdentityInfo; -import org.junit.jupiter.api.Test; - -import java.util.ArrayList; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Supplier; - -import static org.assertj.core.api.Assertions.assertThat; - -class LoginRoutingHookTest { - - private static final IdentityInfo IDENTITY = new IdentityInfo(1L, Optional.empty()); - - @Test - void shouldRouteUsernameLoginBeforeCreatingRegisterAttempt() { - List events = new ArrayList<>(); - var client = new UsersTcpClient(failingConnectionSupplier(events), routingHook(events)); - - var identity = client.login("iggy", "iggy").join(); - - assertThat(identity).isEqualTo(IDENTITY); - assertThat(events).containsExactly("route", "login-attempt"); - } - - @Test - void shouldRoutePersonalAccessTokenLoginBeforeCreatingRegisterAttempt() { - List events = new ArrayList<>(); - var client = new PersonalAccessTokensTcpClient(failingConnectionSupplier(events), routingHook(events)); - - var identity = client.loginWithPersonalAccessToken("token").join(); - - assertThat(identity).isEqualTo(IDENTITY); - assertThat(events).containsExactly("route", "login-attempt"); - } - - @Test - void shouldExecuteOneAttemptWithNoOpHook() { - var attempts = new AtomicInteger(); - - var identity = LoginRoutingHook.NONE - .loginOnLeader(() -> { - attempts.incrementAndGet(); - return CompletableFuture.completedFuture(IDENTITY); - }) - .join(); - - assertThat(identity).isEqualTo(IDENTITY); - assertThat(attempts).hasValue(1); - } - - private static Supplier failingConnectionSupplier(List events) { - return () -> { - events.add("login-attempt"); - throw new LoginAttemptReached(); - }; - } - - private static LoginRoutingHook routingHook(List events) { - return loginAttempt -> { - events.add("route"); - try { - loginAttempt.get(); - } catch (LoginAttemptReached expected) { - return CompletableFuture.completedFuture(IDENTITY); - } - throw new AssertionError("Expected the login attempt to acquire its connection"); - }; - } - - private static final class LoginAttemptReached extends RuntimeException {} -} diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/ReconnectPlanTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/ReconnectPlanTest.java deleted file mode 100644 index ce833a58cb..0000000000 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/ReconnectPlanTest.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iggy.client.async.tcp; - -import org.apache.iggy.client.ConnectionInfo; -import org.apache.iggy.config.RetryPolicy; -import org.junit.jupiter.api.Test; - -import java.time.Duration; - -import static org.assertj.core.api.Assertions.assertThat; - -class ReconnectPlanTest { - - private final ConnectionInfo seed = new ConnectionInfo("seed-node", 8090); - private final ConnectionInfo current = new ConnectionInfo("leader-node", 8090); - - @Test - void shouldAlternateBetweenCurrentAndSeed() { - assertThat(ReconnectPlan.target(current, seed, 1)).isEqualTo(current); - assertThat(ReconnectPlan.target(current, seed, 2)).isEqualTo(seed); - assertThat(ReconnectPlan.target(current, seed, 3)).isEqualTo(current); - assertThat(ReconnectPlan.target(current, seed, 4)).isEqualTo(seed); - } - - @Test - void shouldDialOnlyOneAddressWhenNeverRedirected() { - assertThat(ReconnectPlan.target(seed, seed, 1)).isEqualTo(seed); - assertThat(ReconnectPlan.target(seed, seed, 2)).isEqualTo(seed); - } - - @Test - void shouldKeepFixedDelayConstant() { - var policy = RetryPolicy.fixedDelay(12, Duration.ofSeconds(5)); - - assertThat(ReconnectPlan.delay(policy, 1)).isEqualTo(Duration.ofSeconds(5)); - assertThat(ReconnectPlan.delay(policy, 12)).isEqualTo(Duration.ofSeconds(5)); - } - - @Test - void shouldScaleExponentialDelayUpToTheCap() { - var policy = RetryPolicy.exponentialBackoff(5, Duration.ofMillis(100), Duration.ofSeconds(1), 2.0); - - assertThat(ReconnectPlan.delay(policy, 1)).isEqualTo(Duration.ofMillis(100)); - assertThat(ReconnectPlan.delay(policy, 2)).isEqualTo(Duration.ofMillis(200)); - assertThat(ReconnectPlan.delay(policy, 3)).isEqualTo(Duration.ofMillis(400)); - assertThat(ReconnectPlan.delay(policy, 5)).isEqualTo(Duration.ofSeconds(1)); - } -} diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrFrameDecoderTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrFrameDecoderTest.java deleted file mode 100644 index fe916cd9fe..0000000000 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrFrameDecoderTest.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iggy.client.async.tcp.vsr; - -import io.netty.buffer.ByteBuf; -import io.netty.buffer.Unpooled; -import io.netty.channel.embedded.EmbeddedChannel; -import io.netty.handler.codec.DecoderException; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Test; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -class VsrFrameDecoderTest { - - private final EmbeddedChannel channel = new EmbeddedChannel(new VsrFrameDecoder()); - - @AfterEach - void tearDown() { - try { - channel.finishAndReleaseAll(); - } catch (DecoderException ignored) { - // The desync test leaves the failed decoder in the pipeline and - // closing the channel replays its exception. - } - } - - @Test - void shouldWaitForMoreBytesOnPartialHeader() { - ByteBuf partial = Unpooled.buffer(); - partial.writeZero(100); - channel.writeInbound(partial); - - assertThat((Object) channel.readInbound()).isNull(); - } - - @Test - void shouldEmitOneFrameFromSplitReads() { - ByteBuf frame = Unpooled.buffer(); - frame.writeZero(VsrHeaders.HEADER_SIZE); - frame.setIntLE(VsrHeaders.SIZE_OFFSET, VsrHeaders.HEADER_SIZE + 8); - frame.writeLongLE(123456789L); - - channel.writeInbound(frame.retainedSlice(0, 200)); - assertThat((Object) channel.readInbound()).isNull(); - channel.writeInbound(frame.retainedSlice(200, frame.readableBytes() - 200)); - frame.release(); - - ByteBuf decoded = channel.readInbound(); - try { - assertThat(decoded.readableBytes()).isEqualTo(VsrHeaders.HEADER_SIZE + 8); - assertThat(decoded.getLongLE(VsrHeaders.HEADER_SIZE)).isEqualTo(123456789L); - } finally { - decoded.release(); - } - } - - @Test - void shouldFailConnectionOnInvalidSizeField() { - ByteBuf frame = Unpooled.buffer(); - frame.writeZero(VsrHeaders.HEADER_SIZE); - frame.setIntLE(VsrHeaders.SIZE_OFFSET, 8); - - assertThatThrownBy(() -> channel.writeInbound(frame)).isInstanceOf(DecoderException.class); - } - - @Test - void shouldWaitForLargeFrameWhenConfiguredLimitAllowsIt() { - int declaredSize = VsrFrameDecoder.DEFAULT_MAX_FRAME_SIZE + 1; - EmbeddedChannel largeFrameChannel = new EmbeddedChannel(new VsrFrameDecoder(declaredSize)); - ByteBuf header = Unpooled.buffer(VsrHeaders.HEADER_SIZE); - header.writeZero(VsrHeaders.HEADER_SIZE); - header.setIntLE(VsrHeaders.SIZE_OFFSET, declaredSize); - try { - largeFrameChannel.writeInbound(header); - - assertThat((Object) largeFrameChannel.readInbound()).isNull(); - } finally { - largeFrameChannel.finishAndReleaseAll(); - } - } - - @Test - void shouldRejectFrameAboveConfiguredLimit() { - int configuredLimit = VsrHeaders.HEADER_SIZE + 1; - EmbeddedChannel smallFrameChannel = new EmbeddedChannel(new VsrFrameDecoder(configuredLimit)); - ByteBuf header = Unpooled.buffer(VsrHeaders.HEADER_SIZE); - header.writeZero(VsrHeaders.HEADER_SIZE); - header.setIntLE(VsrHeaders.SIZE_OFFSET, configuredLimit + 1); - try { - assertThatThrownBy(() -> smallFrameChannel.writeInbound(header)).isInstanceOf(DecoderException.class); - } finally { - try { - smallFrameChannel.finishAndReleaseAll(); - } catch (DecoderException ignored) { - // Closing a failed decoder can replay its exception. - } - } - } -} diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrRequestEncoderTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrRequestEncoderTest.java deleted file mode 100644 index 426aa0b501..0000000000 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrRequestEncoderTest.java +++ /dev/null @@ -1,225 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iggy.client.async.tcp.vsr; - -import io.netty.buffer.ByteBuf; -import io.netty.buffer.ByteBufAllocator; -import io.netty.buffer.Unpooled; -import org.apache.iggy.exception.IggyNotConnectedException; -import org.junit.jupiter.api.Test; - -import java.nio.charset.StandardCharsets; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -class VsrRequestEncoderTest { - - private static final int LOGIN_USER_CODE = 38; - private static final int PING_CODE = 1; - private static final int GET_CLUSTER_METADATA_CODE = 12; - private static final int CREATE_STREAM_CODE = 202; - private static final int SEND_MESSAGES_CODE = 101; - - private final ConsensusSession session = new ConsensusSession(); - private final VsrRequestEncoder encoder = new VsrRequestEncoder(session); - private final ByteBufAllocator alloc = ByteBufAllocator.DEFAULT; - - @Test - void shouldBuildRegisterFrameForLogin() { - ByteBuf payload = loginUserPayload(); - ByteBuf frame = encoder.encode(alloc, LOGIN_USER_CODE, payload); - payload.release(); - - try { - assertThat(frame.getUnsignedByte(VsrHeaders.COMMAND_OFFSET)).isEqualTo((short) VsrHeaders.COMMAND_REQUEST); - assertThat(frame.getUnsignedByte(VsrHeaders.REQUEST_OPERATION_OFFSET)) - .isEqualTo((short) VsrOperation.REGISTER); - assertThat(frame.getLongLE(VsrHeaders.REQUEST_ID_OFFSET)).isZero(); - assertThat(frame.getLongLE(VsrHeaders.REQUEST_SESSION_OFFSET)).isZero(); - assertThat(frame.getUnsignedIntLE(VsrHeaders.SIZE_OFFSET)).isEqualTo(frame.readableBytes()); - - // Body starts with the ClientVersionInfo prefix. - int bodyStart = VsrHeaders.HEADER_SIZE; - assertThat(frame.getIntLE(bodyStart)).isEqualTo(VsrLoginCodec.PROTOCOL_VERSION); - int sdkNameLength = frame.getUnsignedByte(bodyStart + 4); - byte[] sdkName = new byte[sdkNameLength]; - frame.getBytes(bodyStart + 5, sdkName); - assertThat(new String(sdkName, StandardCharsets.UTF_8)).isEqualTo(VsrLoginCodec.SDK_NAME); - } finally { - frame.release(); - } - } - - @Test - void shouldStampCodeAndZeroSessionForPing() { - ByteBuf frame = encoder.encode(alloc, PING_CODE, Unpooled.EMPTY_BUFFER); - try { - assertThat(frame.getUnsignedByte(VsrHeaders.REQUEST_OPERATION_OFFSET)) - .isEqualTo((short) VsrOperation.NON_REPLICATED); - assertThat(frame.getIntLE(VsrHeaders.REQUEST_RESERVED_CODE_OFFSET)).isEqualTo(PING_CODE); - assertThat(frame.getLongLE(VsrHeaders.REQUEST_SESSION_OFFSET)).isZero(); - } finally { - frame.release(); - } - } - - @Test - void shouldKeepPreAuthClusterMetadataSessionlessWithoutAdvancingRequestId() { - ByteBuf frame = encoder.encode(alloc, GET_CLUSTER_METADATA_CODE, Unpooled.EMPTY_BUFFER); - try { - assertThat(frame.getUnsignedByte(VsrHeaders.REQUEST_OPERATION_OFFSET)) - .isEqualTo((short) VsrOperation.NON_REPLICATED); - assertThat(frame.getIntLE(VsrHeaders.REQUEST_RESERVED_CODE_OFFSET)).isEqualTo(GET_CLUSTER_METADATA_CODE); - assertThat(frame.getLongLE(VsrHeaders.REQUEST_ID_OFFSET)).isEqualTo(1); - assertThat(frame.getLongLE(VsrHeaders.REQUEST_SESSION_OFFSET)).isZero(); - assertThat(session.currentRequestId()).isEqualTo(1); - } finally { - frame.release(); - } - } - - @Test - void shouldRejectReplicatedCommandBeforeLogin() { - assertThatThrownBy(() -> encoder.encode(alloc, CREATE_STREAM_CODE, Unpooled.EMPTY_BUFFER)) - .isInstanceOf(IggyNotConnectedException.class); - } - - @Test - void shouldAdvanceRequestIdsForReplicatedCommands() { - session.beginRegister(); - session.bind(42); - - ByteBuf first = encoder.encode(alloc, CREATE_STREAM_CODE, Unpooled.EMPTY_BUFFER); - ByteBuf second = encoder.encode(alloc, CREATE_STREAM_CODE, Unpooled.EMPTY_BUFFER); - try { - assertThat(first.getLongLE(VsrHeaders.REQUEST_ID_OFFSET)).isEqualTo(1); - assertThat(second.getLongLE(VsrHeaders.REQUEST_ID_OFFSET)).isEqualTo(2); - assertThat(second.getLongLE(VsrHeaders.REQUEST_SESSION_OFFSET)).isEqualTo(42); - } finally { - first.release(); - second.release(); - } - } - - @Test - void shouldCorrelatePartitionOpsWithoutAdvancingTheDedupRequestId() { - // Partition ops replicate in their own group with no client-table dedup, so - // they take a correlation id and leave the dedup counter where it was. - session.beginRegister(); - session.bind(42); - - ByteBuf firstPayload = sendMessagesPayload(2, 3, 4); - ByteBuf secondPayload = sendMessagesPayload(2, 3, 4); - ByteBuf first = encoder.encode(alloc, SEND_MESSAGES_CODE, firstPayload); - ByteBuf second = encoder.encode(alloc, SEND_MESSAGES_CODE, secondPayload); - firstPayload.release(); - secondPayload.release(); - try { - assertThat(first.getLongLE(VsrHeaders.REQUEST_ID_OFFSET)).isEqualTo(1); - assertThat(second.getLongLE(VsrHeaders.REQUEST_ID_OFFSET)).isEqualTo(2); - assertThat(session.currentRequestId()).isEqualTo(1); - } finally { - first.release(); - second.release(); - } - } - - @Test - void shouldPassPartitioningThroughForTheServerToResolve() { - // The client no longer inspects the payload to route: balanced partitioning - // reaches the server byte-identical instead of failing at encode time. - session.beginRegister(); - session.bind(42); - - ByteBuf payload = balancedSendMessagesPayload(2, 3); - ByteBuf frame = encoder.encode(alloc, SEND_MESSAGES_CODE, payload); - try { - assertThat(frame.getUnsignedIntLE(VsrHeaders.SIZE_OFFSET)).isEqualTo(frame.readableBytes()); - byte[] encodedBody = new byte[payload.readableBytes()]; - frame.getBytes(VsrHeaders.HEADER_SIZE, encodedBody); - byte[] originalBody = new byte[payload.readableBytes()]; - payload.getBytes(payload.readerIndex(), originalBody); - assertThat(encodedBody).isEqualTo(originalBody); - } finally { - frame.release(); - payload.release(); - } - } - - @Test - void shouldReArmWithFreshClientIdOnSecondLogin() { - ByteBuf firstLogin = loginUserPayload(); - encoder.encode(alloc, LOGIN_USER_CODE, firstLogin).release(); - firstLogin.release(); - long firstLow = session.clientIdLow(); - long firstHigh = session.clientIdHigh(); - session.bind(7); - - ByteBuf secondLogin = loginUserPayload(); - encoder.encode(alloc, LOGIN_USER_CODE, secondLogin).release(); - secondLogin.release(); - - assertThat(session.isBound()).isFalse(); - assertThat(session.clientIdLow() != firstLow || session.clientIdHigh() != firstHigh) - .isTrue(); - } - - private static ByteBuf loginUserPayload() { - ByteBuf payload = Unpooled.buffer(); - payload.writeByte(4); - payload.writeBytes("iggy".getBytes(StandardCharsets.UTF_8)); - payload.writeByte(4); - payload.writeBytes("iggy".getBytes(StandardCharsets.UTF_8)); - payload.writeIntLE(0); - payload.writeIntLE(0); - return payload; - } - - private static ByteBuf sendMessagesPayload(long streamId, long topicId, long partitionId) { - ByteBuf payload = Unpooled.buffer(); - // metadata: stream ident (6) + topic ident (6) + partitioning (6) + count (4) - payload.writeIntLE(22); - writeNumericIdentifier(payload, streamId); - writeNumericIdentifier(payload, topicId); - payload.writeByte(2); - payload.writeByte(4); - payload.writeIntLE((int) partitionId); - payload.writeIntLE(0); - return payload; - } - - private static ByteBuf balancedSendMessagesPayload(long streamId, long topicId) { - ByteBuf payload = Unpooled.buffer(); - payload.writeIntLE(18); - writeNumericIdentifier(payload, streamId); - writeNumericIdentifier(payload, topicId); - payload.writeByte(1); - payload.writeByte(0); - payload.writeIntLE(0); - return payload; - } - - private static void writeNumericIdentifier(ByteBuf payload, long id) { - payload.writeByte(1); - payload.writeByte(4); - payload.writeIntLE((int) id); - } -} diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandlerTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandlerTest.java deleted file mode 100644 index e8069b9159..0000000000 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandlerTest.java +++ /dev/null @@ -1,334 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iggy.client.async.tcp.vsr; - -import io.netty.buffer.ByteBuf; -import io.netty.buffer.Unpooled; -import io.netty.channel.embedded.EmbeddedChannel; -import org.apache.iggy.exception.IggyConnectionException; -import org.apache.iggy.exception.IggyServerException; -import org.apache.iggy.exception.IggyTimeoutException; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Test; - -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -class VsrResponseHandlerTest { - - private final ConsensusSession session = new ConsensusSession(); - private final AtomicInteger evictions = new AtomicInteger(); - private final VsrResponseHandler handler = new VsrResponseHandler(session, evictions::incrementAndGet); - private final EmbeddedChannel channel = new EmbeddedChannel(handler); - - @AfterEach - void tearDown() { - channel.finishAndReleaseAll(); - } - - @Test - void shouldFailWithStatusCodeOnDeniedReply() { - CompletableFuture future = enqueue(VsrOperation.NON_REPLICATED, 7); - ByteBuf frame = replyFrame(VsrOperation.NON_REPLICATED, 7, Unpooled.EMPTY_BUFFER); - frame.setIntLE(VsrHeaders.REPLY_STATUS_OFFSET, 40); - - channel.writeInbound(frame); - - assertThat(rawErrorCode(future)).isEqualTo(40); - } - - @Test - void shouldPassNonReplicatedBodyThrough() throws Exception { - CompletableFuture future = enqueue(VsrOperation.NON_REPLICATED, 7); - ByteBuf body = Unpooled.buffer(); - body.writeIntLE(1234); - channel.writeInbound(replyFrame(VsrOperation.NON_REPLICATED, 7, body)); - - ByteBuf response = future.get(); - try { - assertThat(response.readIntLE()).isEqualTo(1234); - } finally { - response.release(); - } - } - - @Test - void shouldReleaseResponseBodyWhenRequestWasCancelled() { - CompletableFuture future = enqueue(VsrOperation.NON_REPLICATED, 7); - future.cancel(false); - ByteBuf frame = - replyFrame(VsrOperation.NON_REPLICATED, 7, Unpooled.buffer().writeIntLE(1234)); - - channel.writeInbound(frame); - - assertThat(frame.refCnt()).isZero(); - } - - @Test - void shouldStripResultSectionFromMetadataReply() throws Exception { - CompletableFuture future = enqueue(VsrOperation.CREATE_STREAM, 7); - ByteBuf body = Unpooled.buffer(); - body.writeIntLE(0); - body.writeIntLE(777); - channel.writeInbound(replyFrame(VsrOperation.CREATE_STREAM, 7, body)); - - ByteBuf response = future.get(); - try { - assertThat(response.readIntLE()).isEqualTo(777); - } finally { - response.release(); - } - } - - @Test - void shouldSurfaceCommittedErrorFromMetadataRejection() { - CompletableFuture future = enqueue(VsrOperation.DELETE_STREAM, 7); - ByteBuf body = Unpooled.buffer(); - body.writeIntLE(1); - body.writeIntLE(0); - body.writeIntLE(1010); - channel.writeInbound(replyFrame(VsrOperation.DELETE_STREAM, 7, body)); - - assertThat(rawErrorCode(future)).isEqualTo(1010); - } - - @Test - void shouldBindSessionOnRegisterReply() throws Exception { - CompletableFuture future = enqueue(VsrOperation.REGISTER, 0); - ByteBuf body = Unpooled.buffer(); - body.writeIntLE(0); // result section: success - body.writeIntLE(1); // user id - body.writeLongLE(99); // session epoch - body.writeIntLE(VsrLoginCodec.PROTOCOL_VERSION); - body.writeByte(3); - body.writeBytes("0.1".getBytes()); - channel.writeInbound(replyFrame(VsrOperation.REGISTER, 0, body)); - - ByteBuf response = future.get(); - try { - assertThat(response.readUnsignedIntLE()).isEqualTo(1); - assertThat(session.isBound()).isTrue(); - assertThat(session.boundSession()).isEqualTo(99); - } finally { - response.release(); - } - } - - @Test - void shouldResetSessionOnLogoutReply() throws Exception { - session.beginRegister(); - session.bind(42); - CompletableFuture future = enqueue(VsrOperation.LOGOUT, 7); - channel.writeInbound(replyFrame(VsrOperation.LOGOUT, 7, Unpooled.EMPTY_BUFFER)); - - future.get().release(); - assertThat(session.isBound()).isFalse(); - } - - @Test - void shouldMapEvictionReasonAndResetSession() { - session.beginRegister(); - session.bind(42); - CompletableFuture future = enqueue(VsrOperation.NON_REPLICATED, 7); - ByteBuf frame = emptyFrame(); - frame.setByte(VsrHeaders.COMMAND_OFFSET, VsrHeaders.COMMAND_EVICTION); - frame.setByte(VsrHeaders.EVICTION_REASON_OFFSET, VsrHeaders.REASON_INVALID_CREDENTIALS); - channel.writeInbound(frame); - - assertThat(rawErrorCode(future)).isEqualTo(42); - assertThat(session.isBound()).isFalse(); - assertThat(evictions).hasValue(1); - } - - @Test - void shouldCloseChannelOnEvictionWithoutPendingRequest() { - session.beginRegister(); - session.bind(42); - ByteBuf frame = evictionFrame(VsrHeaders.REASON_STALE_CLIENT); - - channel.writeInbound(frame); - - assertThat(session.isBound()).isFalse(); - assertThat(evictions).hasValue(1); - assertThat(frame.refCnt()).isZero(); - assertThat(channel.isActive()).isFalse(); - } - - @Test - void shouldFailEveryPendingRequestOnEviction() { - session.beginRegister(); - session.bind(42); - CompletableFuture first = enqueue(VsrOperation.NON_REPLICATED, 7); - CompletableFuture second = enqueue(VsrOperation.CREATE_STREAM, 8); - - channel.writeInbound(evictionFrame(VsrHeaders.REASON_STALE_CLIENT)); - - assertThat(rawErrorCode(first)).isEqualTo(VsrHeaders.ERROR_STALE_CLIENT); - assertThat(rawErrorCode(second)).isEqualTo(VsrHeaders.ERROR_STALE_CLIENT); - assertThat(evictions).hasValue(1); - assertThat(channel.isActive()).isFalse(); - } - - @Test - void shouldFailPendingRequestsWhenChannelBecomesInactive() { - CompletableFuture pending = enqueue(VsrOperation.NON_REPLICATED, 7); - - channel.close(); - - assertThat(pending).isCompletedExceptionally(); - assertThatThrownBy(pending::join).hasCauseInstanceOf(IggyConnectionException.class); - } - - @Test - void shouldFailPendingRequestsOnPipelineException() { - CompletableFuture pending = enqueue(VsrOperation.NON_REPLICATED, 7); - - var failure = new IllegalStateException("broken pipe"); - channel.pipeline().fireExceptionCaught(failure); - - assertThat(pending).isCompletedExceptionally(); - assertThatThrownBy(pending::join).hasCause(failure); - } - - @Test - void shouldCloseChannelWhenResponseDeadlineExpires() { - CompletableFuture pending = new CompletableFuture<>(); - ByteBuf request = requestFrame(VsrOperation.NON_REPLICATED, 7); - handler.registerRequest(channel, request, pending, System.nanoTime(), 1); - request.release(); - - channel.runScheduledPendingTasks(); - - assertThat(channel.isActive()).isFalse(); - assertThatThrownBy(pending::join).hasCauseInstanceOf(IggyTimeoutException.class); - } - - @Test - void shouldCorrelateRepliesArrivingInReverseOrder() throws Exception { - CompletableFuture first = enqueue(VsrOperation.SEND_MESSAGES, 7); - CompletableFuture second = enqueue(VsrOperation.SEND_MESSAGES, 8); - - channel.writeInbound(replyFrame(VsrOperation.SEND_MESSAGES, 8, Unpooled.wrappedBuffer(new byte[] {2}))); - channel.writeInbound(replyFrame(VsrOperation.SEND_MESSAGES, 7, Unpooled.wrappedBuffer(new byte[] {1}))); - - ByteBuf firstResponse = first.get(); - ByteBuf secondResponse = second.get(); - try { - assertThat(firstResponse.readByte()).isEqualTo((byte) 1); - assertThat(secondResponse.readByte()).isEqualTo((byte) 2); - } finally { - firstResponse.release(); - secondResponse.release(); - } - } - - @Test - void shouldCorrelateRepliesForServerRewrittenOperations() throws Exception { - int[][] rewrittenOperations = { - {VsrOperation.CREATE_TOPIC, VsrOperation.CREATE_TOPIC_WITH_ASSIGNMENTS}, - {VsrOperation.CREATE_PARTITIONS, VsrOperation.CREATE_PARTITIONS_WITH_ASSIGNMENTS}, - {VsrOperation.DELETE_SEGMENTS, VsrOperation.TRUNCATE_PARTITION} - }; - - for (int index = 0; index < rewrittenOperations.length; index++) { - int requestOperation = rewrittenOperations[index][0]; - int replyOperation = rewrittenOperations[index][1]; - long requestId = index + 1; - CompletableFuture future = enqueue(requestOperation, requestId); - ByteBuf body = Unpooled.buffer(); - body.writeIntLE(0); - body.writeIntLE(100 + index); - - channel.writeInbound(replyFrame(replyOperation, requestId, body)); - - ByteBuf response = future.get(); - try { - assertThat(response.readIntLE()).isEqualTo(100 + index); - } finally { - response.release(); - } - } - } - - @Test - void shouldRejectUnrelatedReplyOperation() { - CompletableFuture pending = enqueue(VsrOperation.CREATE_TOPIC, 7); - - channel.writeInbound( - replyFrame(VsrOperation.DELETE_TOPIC, 7, Unpooled.buffer().writeIntLE(0))); - - assertThat(channel.isActive()).isFalse(); - assertThat(rawErrorCode(pending)).isEqualTo(VsrHeaders.ERROR_INVALID_COMMAND); - } - - private CompletableFuture enqueue(int operation, long requestId) { - CompletableFuture future = new CompletableFuture<>(); - handler.registerRequest(future, operation, requestId); - return future; - } - - private static ByteBuf emptyFrame() { - ByteBuf frame = Unpooled.buffer(VsrHeaders.HEADER_SIZE); - frame.writeZero(VsrHeaders.HEADER_SIZE); - frame.setIntLE(VsrHeaders.SIZE_OFFSET, VsrHeaders.HEADER_SIZE); - return frame; - } - - private static ByteBuf requestFrame(int operation, long requestId) { - ByteBuf frame = emptyFrame(); - frame.setByte(VsrHeaders.REQUEST_OPERATION_OFFSET, operation); - frame.setLongLE(VsrHeaders.REQUEST_ID_OFFSET, requestId); - return frame; - } - - private static ByteBuf replyFrame(int operation, long requestId, ByteBuf body) { - ByteBuf frame = emptyFrame(); - frame.setByte(VsrHeaders.COMMAND_OFFSET, VsrHeaders.COMMAND_REPLY); - frame.setByte(VsrHeaders.REPLY_OPERATION_OFFSET, operation); - frame.setLongLE(VsrHeaders.REPLY_REQUEST_OFFSET, requestId); - frame.setIntLE(VsrHeaders.SIZE_OFFSET, VsrHeaders.HEADER_SIZE + body.readableBytes()); - frame.writeBytes(body); - body.release(); - return frame; - } - - private static ByteBuf evictionFrame(int reason) { - ByteBuf frame = emptyFrame(); - frame.setByte(VsrHeaders.COMMAND_OFFSET, VsrHeaders.COMMAND_EVICTION); - frame.setByte(VsrHeaders.EVICTION_REASON_OFFSET, reason); - return frame; - } - - private static int rawErrorCode(CompletableFuture future) { - try { - future.get().release(); - throw new AssertionError("Expected the response future to fail"); - } catch (ExecutionException e) { - assertThat(e.getCause()).isInstanceOf(IggyServerException.class); - return ((IggyServerException) e.getCause()).getRawErrorCode(); - } catch (InterruptedException e) { - throw new AssertionError(e); - } - } -} diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/ConsumerOffsetsClientBaseTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/ConsumerOffsetsClientBaseTest.java index da9fd267f7..633c84cd88 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/ConsumerOffsetsClientBaseTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/ConsumerOffsetsClientBaseTest.java @@ -54,9 +54,7 @@ void shouldGetConsumerOffset() { // when var consumer = new Consumer(Consumer.Kind.Consumer, ConsumerId.of(1223L)); - // The VSR client routes the store to its partition consensus group, so - // the partition id must be explicit. - consumerOffsetsClient.storeConsumerOffset(STREAM_NAME, TOPIC_NAME, Optional.of(0L), consumer, BigInteger.ZERO); + consumerOffsetsClient.storeConsumerOffset(STREAM_NAME, TOPIC_NAME, Optional.empty(), consumer, BigInteger.ZERO); var consumerOffset = consumerOffsetsClient.getConsumerOffset(STREAM_NAME, TOPIC_NAME, Optional.of(0L), consumer); diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/MessagesClientBaseTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/MessagesClientBaseTest.java index fd4034f2bb..040e1790b0 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/MessagesClientBaseTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/MessagesClientBaseTest.java @@ -69,70 +69,6 @@ void shouldSendAndGetMessages() { assertThat(polledMessages.messages()).hasSize(1); } - @Test - void shouldSendMessageWithBalancedPartitioning() { - // given - setUpStreamAndTopic(); - - // when - String text = "message from java sdk"; - messagesClient.sendMessages(STREAM_NAME, TOPIC_NAME, Partitioning.balanced(), List.of(Message.of(text))); - - var polledMessages = messagesClient.pollMessages( - STREAM_NAME, - TOPIC_NAME, - empty(), - Consumer.of(0L), - new PollingStrategy(PollingKind.Last, BigInteger.TEN), - 10L, - false); - - // then - assertThat(polledMessages.messages()).hasSize(1); - } - - @Test - void shouldSendMessageWithMessageKeyPartitioning() { - // given - setUpStreamAndTopic(); - - // when - String text = "message from java sdk"; - messagesClient.sendMessages( - STREAM_NAME, TOPIC_NAME, Partitioning.messagesKey("test-key"), List.of(Message.of(text))); - var polledMessages = messagesClient.pollMessages( - STREAM_NAME, - TOPIC_NAME, - empty(), - Consumer.of(0L), - new PollingStrategy(PollingKind.Last, BigInteger.TEN), - 10L, - false); - - // then - assertThat(polledMessages.messages()).hasSize(1); - } - - @Test - void shouldReturnSendConfirmations() { - // given - setUpStreamAndTopic(); - - // when - var firstResponse = messagesClient.sendMessages( - STREAM_NAME, TOPIC_NAME, Partitioning.partitionId(0L), List.of(Message.of("first"))); - var secondResponse = messagesClient.sendMessages( - STREAM_NAME, TOPIC_NAME, Partitioning.partitionId(0L), List.of(Message.of("second"))); - - // then - assertThat(firstResponse.confirmations()).hasSize(1); - var firstConfirmation = firstResponse.confirmations().get(0); - assertThat(firstConfirmation.partitionId()).isEqualTo(0L); - assertThat(firstConfirmation.baseOffset()).isEqualTo(BigInteger.ZERO); - assertThat(secondResponse.confirmations()).hasSize(1); - assertThat(secondResponse.confirmations().get(0).baseOffset()).isEqualTo(BigInteger.ONE); - } - @Test void shouldPollMessagesWithFirstStrategy() { // given @@ -213,4 +149,48 @@ void shouldVerifyMessageContentRoundTrip() { assertThat(polledMessages.messages()).hasSize(1); assertThat(new String(polledMessages.messages().get(0).payload())).isEqualTo(content); } + + @Test + void shouldSendMessageWithBalancedPartitioning() { + // given + setUpStreamAndTopic(); + + // when + String text = "message from java sdk"; + messagesClient.sendMessages(STREAM_NAME, TOPIC_NAME, Partitioning.balanced(), List.of(Message.of(text))); + + var polledMessages = messagesClient.pollMessages( + STREAM_NAME, + TOPIC_NAME, + empty(), + Consumer.of(0L), + new PollingStrategy(PollingKind.Last, BigInteger.TEN), + 10L, + false); + + // then + assertThat(polledMessages.messages()).hasSize(1); + } + + @Test + void shouldSendMessageWithMessageKeyPartitioning() { + // given + setUpStreamAndTopic(); + + // when + String text = "message from java sdk"; + messagesClient.sendMessages( + STREAM_NAME, TOPIC_NAME, Partitioning.messagesKey("test-key"), List.of(Message.of(text))); + var polledMessages = messagesClient.pollMessages( + STREAM_NAME, + TOPIC_NAME, + empty(), + Consumer.of(0L), + new PollingStrategy(PollingKind.Last, BigInteger.TEN), + 10L, + false); + + // then + assertThat(polledMessages.messages()).hasSize(1); + } } diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/SystemClientBaseTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/SystemClientBaseTest.java index 1e4f096f33..550b69b40b 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/SystemClientBaseTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/SystemClientBaseTest.java @@ -58,8 +58,6 @@ void shouldGetClusterMetadataForSingleNode() { // then assertThat(metadata).isNotNull(); - // The server runs standalone (cluster disabled), so it synthesizes - // itself as the sole leader of a single-node roster. assertThat(metadata.name()).isEqualTo("single-node"); assertThat(metadata.nodes()).hasSize(1); var node = metadata.nodes().get(0); diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/UsersClientBaseTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/UsersClientBaseTest.java index a5724ba051..78bcea06c1 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/UsersClientBaseTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/UsersClientBaseTest.java @@ -25,6 +25,7 @@ import org.apache.iggy.user.UserInfo; import org.apache.iggy.user.UserInfoDetails; import org.apache.iggy.user.UserStatus; +import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -220,7 +221,7 @@ void shouldReturnEmptyForNonExistingUser() { assertThat(user).isEmpty(); } - private static GlobalPermissions createGlobalPermissions(boolean manageServers) { + private static @NotNull GlobalPermissions createGlobalPermissions(boolean manageServers) { return new GlobalPermissions(manageServers, false, false, false, false, false, false, false, false, false); } } diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/ConsumerGroupsTcpClientTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/ConsumerGroupsTcpClientTest.java index 1ecb61b9dd..9ed7565554 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/ConsumerGroupsTcpClientTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/ConsumerGroupsTcpClientTest.java @@ -21,21 +21,12 @@ import org.apache.iggy.client.blocking.ConsumerGroupsClientBaseTest; import org.apache.iggy.client.blocking.IggyBaseClient; -import org.apache.iggy.consumergroup.Consumer; -import org.apache.iggy.exception.IggyResourceNotFoundException; import org.apache.iggy.identifier.ConsumerId; -import org.apache.iggy.message.Message; -import org.apache.iggy.message.Partitioning; -import org.apache.iggy.message.PollingStrategy; import org.junit.jupiter.api.Test; -import java.util.List; -import java.util.Optional; - import static org.apache.iggy.TestConstants.STREAM_NAME; import static org.apache.iggy.TestConstants.TOPIC_NAME; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; class ConsumerGroupsTcpClientTest extends ConsumerGroupsClientBaseTest { @@ -69,81 +60,4 @@ void shouldJoinAndLeaveConsumerGroup() { .get(); assertThat(group.membersCount()).isEqualTo(0); } - - @Test - void shouldSyncConsumerGroupAssignmentAfterJoin() { - // given - setUpStreamAndTopic(); - var group = consumerGroupsClient.createConsumerGroup(STREAM_NAME, TOPIC_NAME, "consumer-group-42"); - ConsumerId groupId = ConsumerId.of(group.id()); - - // when - consumerGroupsClient.joinConsumerGroup(STREAM_NAME, TOPIC_NAME, groupId); - var assignment = consumerGroupsClient.syncConsumerGroup(STREAM_NAME, TOPIC_NAME, groupId); - - // then — the only member owns the topic's single partition - assertThat(assignment).isPresent(); - assertThat(assignment.get().partitions()).hasSize(1); - } - - @Test - void shouldReturnEmptyAssignmentWhenNotMember() { - // given - setUpStreamAndTopic(); - var group = consumerGroupsClient.createConsumerGroup(STREAM_NAME, TOPIC_NAME, "consumer-group-42"); - ConsumerId groupId = ConsumerId.of(group.id()); - - // when - var assignment = consumerGroupsClient.syncConsumerGroup(STREAM_NAME, TOPIC_NAME, groupId); - - // then - assertThat(assignment).isEmpty(); - } - - @Test - void shouldPollAsGroupMemberWithoutExplicitPartition() { - // given - setUpStreamAndTopic(); - var group = consumerGroupsClient.createConsumerGroup(STREAM_NAME, TOPIC_NAME, "consumer-group-42"); - ConsumerId groupId = ConsumerId.of(group.id()); - consumerGroupsClient.joinConsumerGroup(STREAM_NAME, TOPIC_NAME, groupId); - client.messages() - .sendMessages( - STREAM_NAME, TOPIC_NAME, Partitioning.partitionId(0L), List.of(Message.of("group message"))); - - // when — the partition is selected client-side from the synced assignment - var polledMessages = client.messages() - .pollMessages( - STREAM_NAME, - TOPIC_NAME, - Optional.empty(), - Consumer.group(groupId), - PollingStrategy.first(), - 10L, - false); - - // then - assertThat(polledMessages.messages()).hasSize(1); - assertThat(new String(polledMessages.messages().get(0).payload())).isEqualTo("group message"); - } - - @Test - void shouldFailGroupPollWhenNotJoined() { - // given - setUpStreamAndTopic(); - var group = consumerGroupsClient.createConsumerGroup(STREAM_NAME, TOPIC_NAME, "consumer-group-42"); - ConsumerId groupId = ConsumerId.of(group.id()); - - // when / then - assertThatThrownBy(() -> client.messages() - .pollMessages( - STREAM_NAME, - TOPIC_NAME, - Optional.empty(), - Consumer.group(groupId), - PollingStrategy.first(), - 10L, - false)) - .isInstanceOf(IggyResourceNotFoundException.class); - } } diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/IggyTcpClientBuilderTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/IggyTcpClientBuilderTest.java index 96607457e3..7fa82e30d4 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/IggyTcpClientBuilderTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/IggyTcpClientBuilderTest.java @@ -91,6 +91,21 @@ void shouldCreateClientWithTimeoutConfiguration() { assertThat(clients).isNotNull(); } + @Test + void shouldCreateClientWithConnectionPoolSize() { + // Given: Builder with connection pool size + IggyTcpClient client = IggyTcpClient.builder() + .host(serverHost()) + .port(serverTcpPort()) + .connectionPoolSize(10) + .credentials("iggy", "iggy") + .buildAndLogin(); + + // Then: Should succeed + List clients = client.system().getClients(); + assertThat(clients).isNotNull(); + } + @Test void shouldCreateClientWithRetryPolicy() { // Given: Builder with exponential backoff retry policy @@ -144,6 +159,7 @@ void shouldCreateClientWithAllOptions() { .port(serverTcpPort()) .connectionTimeout(Duration.ofSeconds(30)) .requestTimeout(Duration.ofSeconds(10)) + .connectionPoolSize(10) .retryPolicy(RetryPolicy.exponentialBackoff(3, Duration.ofMillis(100), Duration.ofSeconds(5), 2.0)) .credentials("iggy", "iggy") .buildAndLogin(); diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/MessagesTcpClientTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/MessagesTcpClientTest.java index c644b50ac7..aabdf19e0c 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/MessagesTcpClientTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/MessagesTcpClientTest.java @@ -21,15 +21,6 @@ import org.apache.iggy.client.blocking.IggyBaseClient; import org.apache.iggy.client.blocking.MessagesClientBaseTest; -import org.apache.iggy.message.Message; -import org.apache.iggy.message.Partitioning; -import org.junit.jupiter.api.Test; - -import java.util.List; - -import static org.apache.iggy.TestConstants.STREAM_NAME; -import static org.apache.iggy.TestConstants.TOPIC_NAME; -import static org.assertj.core.api.Assertions.assertThat; class MessagesTcpClientTest extends MessagesClientBaseTest { @@ -37,26 +28,4 @@ class MessagesTcpClientTest extends MessagesClientBaseTest { protected IggyBaseClient getClient() { return TcpClientFactory.create(serverHost(), serverTcpPort()); } - - /* - * The TCP client resolves balanced and key-based partitioning to an - * explicit partition id before encoding the frame (the VSR broker routes - * explicit partitions only), so messages with the same key must land on - * the same partition. - */ - - @Test - void shouldRouteSameMessageKeyToSamePartition() { - setUpStreamAndTopic(); - - var firstResponse = messagesClient.sendMessages( - STREAM_NAME, TOPIC_NAME, Partitioning.messagesKey("test-key"), List.of(Message.of("first"))); - var secondResponse = messagesClient.sendMessages( - STREAM_NAME, TOPIC_NAME, Partitioning.messagesKey("test-key"), List.of(Message.of("second"))); - - assertThat(firstResponse.confirmations()).hasSize(1); - assertThat(secondResponse.confirmations()).hasSize(1); - assertThat(secondResponse.confirmations().get(0).partitionId()) - .isEqualTo(firstResponse.confirmations().get(0).partitionId()); - } } diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/exception/IggyErrorCodeTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/exception/IggyErrorCodeTest.java index 97e60579e5..3cb4c5118f 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/exception/IggyErrorCodeTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/exception/IggyErrorCodeTest.java @@ -156,12 +156,10 @@ void shouldParseCodeOne() { "1, ERROR", "3, INVALID_COMMAND", "4, INVALID_FORMAT", - "5, FEATURE_UNAVAILABLE", - "6, INVALID_IDENTIFIER", + "6, FEATURE_UNAVAILABLE", "7, CANNOT_PARSE_INT", "8, CANNOT_PARSE_SLICE", "9, CANNOT_PARSE_UTF8", - "30, STALE_CLIENT", // Resource errors "20, RESOURCE_NOT_FOUND", @@ -183,8 +181,6 @@ void shouldParseCodeOne() { "52, CLIENT_NOT_FOUND", "53, INVALID_PAT_TOKEN", "54, PAT_NAME_ALREADY_EXISTS", - "57, TRANSIENT_NOT_COMMITTED", - "58, TRANSIENT_NOT_ACCEPTED", "77, PASSWORD_DOES_NOT_MATCH", "78, PASSWORD_HASH_INTERNAL_ERROR", @@ -227,9 +223,6 @@ void shouldParseCodeOne() { "7002, TOO_BIG_MESSAGE", "7003, INVALID_MESSAGE_CHECKSUM", "7004, MESSAGE_NOT_FOUND", - - // VSR protocol errors - "14003, INCOMPATIBLE_PROTOCOL_VERSION", }) void fromCodeReturnsExpectedIggyErrorCodeWhenCodeIsValid(int code, IggyErrorCode expected) { var iggyErrorCode = IggyErrorCode.fromCode(code); diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/exception/IggyServerExceptionTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/exception/IggyServerExceptionTest.java index 9835c5b62b..cbfb55efea 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/exception/IggyServerExceptionTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/exception/IggyServerExceptionTest.java @@ -116,15 +116,6 @@ void shouldBuildMessageWithUnknownRawCode() { assertThat(exception.getMessage()).isEqualTo("Server error [code=88888]"); } - @Test - void shouldBuildMessageWithTransientErrorCode() { - IggyServerException exception = IggyServerException.fromTcpResponse(57, new byte[0]); - - assertThat(exception.getErrorCode()).isEqualTo(IggyErrorCode.TRANSIENT_NOT_COMMITTED); - assertThat(exception.getMessage()) - .isEqualTo("Server error [code=57 (TRANSIENT_NOT_COMMITTED)]: Server error"); - } - @Test void shouldBuildMessageWithEmptyReason() { // given diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/exception/IggyValidationExceptionTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/exception/IggyValidationExceptionTest.java index c0e3e12b25..c7f9877ebc 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/exception/IggyValidationExceptionTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/exception/IggyValidationExceptionTest.java @@ -49,7 +49,6 @@ void constructorCreatesExpectedIggyValidationException() { "INVALID_COMMAND", "INVALID_FORMAT", "FEATURE_UNAVAILABLE", - "INVALID_IDENTIFIER", "CANNOT_PARSE_INT", "CANNOT_PARSE_SLICE", "CANNOT_PARSE_UTF8", @@ -76,7 +75,6 @@ void matchesReturnsTrueForValidationRelatedCodes(IggyErrorCode code) { "INVALID_COMMAND", "INVALID_FORMAT", "FEATURE_UNAVAILABLE", - "INVALID_IDENTIFIER", "CANNOT_PARSE_INT", "CANNOT_PARSE_SLICE", "CANNOT_PARSE_UTF8", diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/hash/XxHash32Test.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/hash/XxHash32Test.java deleted file mode 100644 index e454d58122..0000000000 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/hash/XxHash32Test.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iggy.hash; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; - -import java.nio.charset.StandardCharsets; - -import static org.assertj.core.api.Assertions.assertThat; - -class XxHash32Test { - - /** - * Golden vectors for XXH32 with seed 0, matching the canonical xxHash - * reference implementation and {@code twox_hash::XxHash32::oneshot(0, ..)} - * used by the Rust SDK for message-key partitioning. - */ - @ParameterizedTest - @CsvSource({ - "'', 02CC5D05", - "a, 550D7456", - "abc, 32D153FF", - "message digest, 7C948494", - "abcdefghijklmnopqrstuvwxyz, 63A14D5F", - "user-123, 0CC9EC8C", - "order-key, 43A01006", - }) - void shouldMatchReferenceVectors(String input, String expectedHex) { - long hash = XxHash32.hashUnsigned(input.getBytes(StandardCharsets.UTF_8)); - - assertThat(hash).isEqualTo(Long.parseLong(expectedHex, 16)); - } - - @Test - void shouldHashAllByteValuesAcrossEveryLoopShape() { - // 256 bytes exercises the 16-byte stripes, the 4-byte tail chunks and - // the single-byte tail in one input - byte[] data = new byte[256]; - for (int i = 0; i < data.length; i++) { - data[i] = (byte) i; - } - - assertThat(XxHash32.hashUnsigned(data)).isEqualTo(0x59441253L); - } - - @Test - void shouldReturnUnsignedValueForHashesAboveIntegerMax() { - // "Nobody inspects the spammish repetition" hashes to 0xE2293B2F, - // which is negative as a signed int - long hash = XxHash32.hashUnsigned("Nobody inspects the spammish repetition".getBytes(StandardCharsets.UTF_8)); - - assertThat(hash).isEqualTo(0xE2293B2FL); - } -} diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/serde/BytesDeserializerTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/serde/BytesDeserializerTest.java index ee835f5b39..637456ab1d 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/serde/BytesDeserializerTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/serde/BytesDeserializerTest.java @@ -42,7 +42,6 @@ import static org.apache.iggy.serde.BytesDeserializer.readClientInfoDetails; import static org.apache.iggy.serde.BytesDeserializer.readClusterMetadata; import static org.apache.iggy.serde.BytesDeserializer.readConsumerGroup; -import static org.apache.iggy.serde.BytesDeserializer.readConsumerGroupAssignment; import static org.apache.iggy.serde.BytesDeserializer.readConsumerGroupDetails; import static org.apache.iggy.serde.BytesDeserializer.readConsumerGroupInfo; import static org.apache.iggy.serde.BytesDeserializer.readConsumerGroupMember; @@ -54,7 +53,6 @@ import static org.apache.iggy.serde.BytesDeserializer.readPolledMessage; import static org.apache.iggy.serde.BytesDeserializer.readPolledMessages; import static org.apache.iggy.serde.BytesDeserializer.readRawPersonalAccessToken; -import static org.apache.iggy.serde.BytesDeserializer.readSendMessagesResponse; import static org.apache.iggy.serde.BytesDeserializer.readStats; import static org.apache.iggy.serde.BytesDeserializer.readStreamBase; import static org.apache.iggy.serde.BytesDeserializer.readStreamDetails; @@ -316,55 +314,6 @@ void shouldDeserializeConsumerGroupDetails() { } } - @Nested - class ConsumerGroupAssignmentDeserialization { - - @Test - void shouldDeserializeAssignment() { - // given — [generation:8][partitions_count:4][partition_id:4]* - ByteBuf buffer = Unpooled.buffer(); - writeU64(buffer, BigInteger.valueOf(7)); // generation - buffer.writeIntLE(3); // partitions count - buffer.writeIntLE(0); - buffer.writeIntLE(1); - buffer.writeIntLE(2); - - // when - var assignment = readConsumerGroupAssignment(buffer); - - // then - assertThat(assignment.generation()).isEqualTo(7L); - assertThat(assignment.partitions()).containsExactly(0L, 1L, 2L); - assertThat(buffer.isReadable()).isFalse(); - } - - @Test - void shouldDeserializeMemberWithoutPartitions() { - // given — distinct from an empty body, which means "not a member" - ByteBuf buffer = Unpooled.buffer(); - writeU64(buffer, BigInteger.valueOf(2)); // generation - buffer.writeIntLE(0); // partitions count - - // when - var assignment = readConsumerGroupAssignment(buffer); - - // then - assertThat(assignment.generation()).isEqualTo(2L); - assertThat(assignment.partitions()).isEmpty(); - } - - @Test - void shouldRejectPartitionCountLargerThanPayload() { - ByteBuf buffer = Unpooled.buffer(); - buffer.writeLongLE(2); - buffer.writeIntLE(Integer.MAX_VALUE); - - assertThatThrownBy(() -> readConsumerGroupAssignment(buffer)) - .isInstanceOf(IggyMalformedResponseException.class) - .hasMessageContaining("partitions count"); - } - } - @Nested class ConsumerOffsetDeserialization { @@ -476,101 +425,6 @@ void shouldDeserializePolledMessages() { } } - @Nested - class SendMessagesResponseDeserialization { - - private ByteBuf singleConfirmation() { - ByteBuf buffer = Unpooled.buffer(); - buffer.writeIntLE(1); // confirmations count - buffer.writeIntLE(3); // stream ID - buffer.writeIntLE(5); // topic ID - buffer.writeIntLE(7); // partition ID - writeU64(buffer, BigInteger.valueOf(41)); // base offset - return buffer; - } - - @Test - void shouldDeserializeSingleConfirmation() { - // given - ByteBuf buffer = singleConfirmation(); - - // when - var response = readSendMessagesResponse(buffer); - - // then - assertThat(response.confirmations()).hasSize(1); - var confirmation = response.confirmations().get(0); - assertThat(confirmation.streamId()).isEqualTo(3L); - assertThat(confirmation.topicId()).isEqualTo(5L); - assertThat(confirmation.partitionId()).isEqualTo(7L); - assertThat(confirmation.baseOffset()).isEqualTo(BigInteger.valueOf(41)); - } - - @Test - void shouldDeserializeEmptyBodyAsNoConfirmations() { - // given — legacy servers ack a send with an empty body - ByteBuf buffer = Unpooled.buffer(); - - // when - var response = readSendMessagesResponse(buffer); - - // then - assertThat(response.confirmations()).isEmpty(); - } - - @Test - void shouldDeserializeZeroCountAsNoConfirmations() { - // given — the server sends count = 0 when it could not decode the batch - ByteBuf buffer = Unpooled.buffer(); - buffer.writeIntLE(0); - - // when - var response = readSendMessagesResponse(buffer); - - // then - assertThat(response.confirmations()).isEmpty(); - } - - @Test - void shouldRejectConfirmationCountLargerThanPayload() { - ByteBuf buffer = Unpooled.buffer(); - buffer.writeIntLE(Integer.MAX_VALUE); - - assertThatThrownBy(() -> readSendMessagesResponse(buffer)) - .isInstanceOf(IggyMalformedResponseException.class) - .hasMessageContaining("confirmations count"); - } - - @Test - void shouldFailOnTrailingBytes() { - // given - ByteBuf buffer = singleConfirmation(); - buffer.writeByte(0xAB); - - // when / then - assertThatThrownBy(() -> readSendMessagesResponse(buffer)) - .isInstanceOf(IggyMalformedResponseException.class) - .hasMessageContaining("trailing"); - } - - @Test - void shouldFailOnTruncationAtEveryByte() { - // given - ByteBuf complete = singleConfirmation(); - byte[] bytes = new byte[complete.readableBytes()]; - complete.getBytes(0, bytes); - - for (int length = 1; length < bytes.length; length++) { - ByteBuf truncated = Unpooled.wrappedBuffer(bytes, 0, length); - - // when / then - assertThatThrownBy(() -> readSendMessagesResponse(truncated)) - .as("truncated at byte %d", length) - .isInstanceOf(RuntimeException.class); - } - } - } - @Nested class StatsDeserialization { diff --git a/foreign/node/CHANGELOG.md b/foreign/node/CHANGELOG.md new file mode 100644 index 0000000000..732a4f587f --- /dev/null +++ b/foreign/node/CHANGELOG.md @@ -0,0 +1,95 @@ +# Changelog + +## [1.0.6](https://github.com/iggy-rs/iggy-node-client/compare/v1.0.5...v1.0.6) (2025-01-14) + +### Bug Fixes + +* add commitlint as husky pre-commit hook and ci check ([#25](https://github.com/iggy-rs/iggy-node-client/issues/25)) ([5dbac10](https://github.com/iggy-rs/iggy-node-client/commit/5dbac1071bea5c26f852a60361ac51217df09d25)) + +## [1.0.5](https://github.com/iggy-rs/iggy-node-client/compare/v1.0.4...v1.0.5) (2025-01-14) + +### Bug Fixes + +* set package.json license to Apache-2.0 as well ([#24](https://github.com/iggy-rs/iggy-node-client/issues/24)) ([2523c0f](https://github.com/iggy-rs/iggy-node-client/commit/2523c0fde82958e8a13bb95b2c2d0babe0d1d290)) + +## [1.0.4](https://github.com/iggy-rs/iggy-node-client/compare/v1.0.3...v1.0.4) (2024-12-16) + +### Bug Fixes + +* add tests, fix headers.double type, ehance typing ([431063f](https://github.com/iggy-rs/iggy-node-client/commit/431063f253e0fb1739188bbd6614cfc06ccb3fd4)) + +## [1.0.3](https://github.com/iggy-rs/iggy-node-client/compare/v1.0.2...v1.0.3) (2024-11-24) + +### Bug Fixes + +* upgrade dependencies ([#19](https://github.com/iggy-rs/iggy-node-client/issues/19)) ([a05d5c2](https://github.com/iggy-rs/iggy-node-client/commit/a05d5c2484f8711f72a23c4ee1222d29e323a5ba)) + +## [1.0.2](https://github.com/iggy-rs/iggy-node-client/compare/v1.0.1...v1.0.2) (2024-11-24) + +### Bug Fixes + +* **npm:** configure package as public ([#18](https://github.com/iggy-rs/iggy-node-client/issues/18)) ([7a56455](https://github.com/iggy-rs/iggy-node-client/commit/7a5645512a564322af343bc7ba748c8c58dfab1e)) + +## [1.0.1](https://github.com/iggy-rs/iggy-node-client/compare/v1.0.0...v1.0.1) (2024-11-24) + +### Bug Fixes + +* **npm:** publish package ([#17](https://github.com/iggy-rs/iggy-node-client/issues/17)) ([6e2f60e](https://github.com/iggy-rs/iggy-node-client/commit/6e2f60e2f4484b57596285ff79d76ea647962595)) + +## 1.0.0 (2024-11-24) + +### Bug Fixes + +* add e2e tests, fix create-group return ([ad52b43](https://github.com/iggy-rs/iggy-node-client/commit/ad52b43a6ee5e8f868eb1178a9b9f02f11cb1204)) +* add package's keywords, cleans up logs ([5287b74](https://github.com/iggy-rs/iggy-node-client/commit/5287b74006733aef4429d889c0cc38a80f375dc2)) +* correctly destroy socket when destroy function is called on TCPClient ([76f7d07](https://github.com/iggy-rs/iggy-node-client/commit/76f7d07a96e44f6701998bb9fb2dab8fa4d75489)) +* enforce check on message id to be either uuid string or 0 ([d05b898](https://github.com/iggy-rs/iggy-node-client/commit/d05b898d62b097ca9770cdab289707546c2fe6aa)) +* fix client auth quirks, add handleResponse & deserializePollMessage as transform stream ([0458d57](https://github.com/iggy-rs/iggy-node-client/commit/0458d579de45605588fff7b7d01119c9625364da)) +* fix consumer group commands ([a33094f](https://github.com/iggy-rs/iggy-node-client/commit/a33094f66eb2e433b2fc8cc6adf7f6a122b0f365)) +* fix createtopic, add new compressionAlgorithm param ([f93a444](https://github.com/iggy-rs/iggy-node-client/commit/f93a4441f4bcd790763d982ead981c4ac86b33c5)) +* fix getStats command (add new totalCpuUsage field) ([#3](https://github.com/iggy-rs/iggy-node-client/issues/3)) ([de4cfda](https://github.com/iggy-rs/iggy-node-client/commit/de4cfdad4046f556a51878fbadb8dde0df9302c5)) +* fix github ci ([78870e3](https://github.com/iggy-rs/iggy-node-client/commit/78870e389333c8b7ca2425748c8806fa5e36a7ae)) +* fix message header typing ([1276374](https://github.com/iggy-rs/iggy-node-client/commit/12763749f95d29a78028f95b5bc33281a62246c9)) +* fix message headers serialization bug ([22ffe16](https://github.com/iggy-rs/iggy-node-client/commit/22ffe1603db9b7a94deadb1fcf6a25f81cfea868)) +* fix module type export ([b0bcdc7](https://github.com/iggy-rs/iggy-node-client/commit/b0bcdc7945ba68d519a129ef33b4353538e9d64e)) +* fix npm test command for ci ([842fe69](https://github.com/iggy-rs/iggy-node-client/commit/842fe697548224a08012574d2e44f14100105b63)) +* fix Partitioning.MessageKey type, fix indent ([05d05b6](https://github.com/iggy-rs/iggy-node-client/commit/05d05b6db68b33bc9045fee16c9fca8c8eb0ae6d)) +* fix tcp client options ([f9bd442](https://github.com/iggy-rs/iggy-node-client/commit/f9bd44204f86f2becfe674c9b38f09657c00ac6c)) +* fix topic deserialisation bug ([3e787be](https://github.com/iggy-rs/iggy-node-client/commit/3e787be2546f29f1f8d31e9e1388e19a62729e50)) +* fix updateUser and changePassword command ([d01e086](https://github.com/iggy-rs/iggy-node-client/commit/d01e08621bbf976c7dc5578273a253a0dcc43e72)) +* fix var naming, add some test ([40d91dd](https://github.com/iggy-rs/iggy-node-client/commit/40d91ddfe41f6114ca3c7da2a30225e81e3226bc)) +* get rid of enums, add type helpers ([6e2613b](https://github.com/iggy-rs/iggy-node-client/commit/6e2613b2f1ab0112d401f87a2f0cfb6e77b8d99d)) +* more e2e tests ([d537aa7](https://github.com/iggy-rs/iggy-node-client/commit/d537aa7454b983edb471fa354e55b5e814fb1524)) +* no ssh pull ([#12](https://github.com/iggy-rs/iggy-node-client/issues/12)) ([f13f0ed](https://github.com/iggy-rs/iggy-node-client/commit/f13f0edd5adf4584ab7f02620a159317654251e5)) +* remove bad symbol in the CI definition ([1878a0c](https://github.com/iggy-rs/iggy-node-client/commit/1878a0c501224d17eefc986ccd5d89e481cc15b8)) +* remove console.error on normal close signal ([0f8c993](https://github.com/iggy-rs/iggy-node-client/commit/0f8c99330b2352220ee23df0b7923185d710cd89)) +* remove initial .gitignore and README.md to avoid rebase conflict ([931b9b8](https://github.com/iggy-rs/iggy-node-client/commit/931b9b8f5d0b8a249de8a30c3be9435c93a8461f)) +* update createUser, createStream & createTopic command return value ([8712b0f](https://github.com/iggy-rs/iggy-node-client/commit/8712b0f5021b7852361117f6fc764eac3851beb7)) +* update readme ([263f271](https://github.com/iggy-rs/iggy-node-client/commit/263f271fed2c529b89329e88d83fc4100b04c639)) +* use debug lib, make poolsize configurable ([2f66e89](https://github.com/iggy-rs/iggy-node-client/commit/2f66e89220b30b3e33aa773b6471b1669658f37f)) +* use pat as token for semantic release ([#10](https://github.com/iggy-rs/iggy-node-client/issues/10)) ([5eccfd2](https://github.com/iggy-rs/iggy-node-client/commit/5eccfd281763773140ce9be39a2c022b2de803b6)) + +### Features + +* add base ci workflow ([#2](https://github.com/iggy-rs/iggy-node-client/issues/2)) ([4cbde14](https://github.com/iggy-rs/iggy-node-client/commit/4cbde140409841bf3d440f01ccaca1a855f37b13)) +* add command client with socket pool management ([e747b29](https://github.com/iggy-rs/iggy-node-client/commit/e747b292374a7a8e1ee8f27d69a9203db3a6b09b)) +* add CommandResponseStream to wrap tcp socket, add parallel call safetiness ([0e6f38f](https://github.com/iggy-rs/iggy-node-client/commit/0e6f38fde621dc1cfe3f12376723f1b43ffee9bf)) +* add consumer stream facility ([8c19d3f](https://github.com/iggy-rs/iggy-node-client/commit/8c19d3fae30a5ab47d0f10b4747158e604aea37f)) +* add create, delete, join & leave consumer-group command ([e7ca376](https://github.com/iggy-rs/iggy-node-client/commit/e7ca3762a9fcb92a3bb5018cb66570079bec60f1)) +* add createPartition & deletePartition command ([2286c10](https://github.com/iggy-rs/iggy-node-client/commit/2286c106534704b02544613944e74f2549ca1a67)) +* add createUser and deleteUser command ([d25d837](https://github.com/iggy-rs/iggy-node-client/commit/d25d837d38abe89a038880651c19bd8925f1affe)) +* add getGroup and getGroups command ([cdfa766](https://github.com/iggy-rs/iggy-node-client/commit/cdfa766c9b800e59c2858599ccf87afa44c142dc)) +* add getOffset and storeOffset command, fix typos ([6f9a425](https://github.com/iggy-rs/iggy-node-client/commit/6f9a4256b713abcfdd6dc146b78268169c9bb198)) +* add pollMessage command ([fe59f82](https://github.com/iggy-rs/iggy-node-client/commit/fe59f825ebb0ba3a4c4c53e90cd8a28f9b21cca2)) +* add purgeTopic & purgeStream command ([86482d1](https://github.com/iggy-rs/iggy-node-client/commit/86482d12bfdf48b23973374e09cb9cc5e852ef18)) +* add SendMessages command ([e1d39d8](https://github.com/iggy-rs/iggy-node-client/commit/e1d39d887e36783e2ca23cd560060df3fc62099a)) +* add updateStream command ([89970c0](https://github.com/iggy-rs/iggy-node-client/commit/89970c0574f0bcade634b338522bcbe6990b4b43)) +* add updateTopic command ([f1e278e](https://github.com/iggy-rs/iggy-node-client/commit/f1e278e993a05c10f10a24761bd2e63f95d9891b)) +* add updateUser and changePassword command, fix permissions deserialization bug ([f3bcda3](https://github.com/iggy-rs/iggy-node-client/commit/f3bcda30b71f5d70145257534848ee4053e714f7)) +* better error, add some test ([f386854](https://github.com/iggy-rs/iggy-node-client/commit/f386854bd388e95a11251803dcb81292b33d7981)) +* publish to npm ([82268f5](https://github.com/iggy-rs/iggy-node-client/commit/82268f5963e72865355e2cfe919307d5bd1500ab)) +* reorganize client declaration ([7cb1bd8](https://github.com/iggy-rs/iggy-node-client/commit/7cb1bd857c5d1050ab6b8d2b0e6b70908b3bc105)) +* start low level command api and base tcp client ([7bb8b32](https://github.com/iggy-rs/iggy-node-client/commit/7bb8b32ec809e53d665295f9e0c069d535b88cae)) +* start unit test on serialization ([2082b71](https://github.com/iggy-rs/iggy-node-client/commit/2082b713a37d59a3d0482669a3a544a6a922ae41)) +* update modified commands for v0.3.0 server release (createTopic, updateTopic, login, createToken) ([cb4f0d1](https://github.com/iggy-rs/iggy-node-client/commit/cb4f0d17c61967e95a6b5502c70d6e8b8569e9c7)) +* wraps command to higher level api, starts client ([60f9466](https://github.com/iggy-rs/iggy-node-client/commit/60f9466cf92965f3bf4ff5918e50876e5bb9aa84)) diff --git a/foreign/node/README.md b/foreign/node/README.md index 994ffee608..ecb0453793 100644 --- a/foreign/node/README.md +++ b/foreign/node/README.md @@ -32,17 +32,18 @@ npm i --save apache-iggy ### Response frame limit -**Compatibility note:** response frames larger than `maxResponseFrameSize` (default 64 MiB) are rejected and close the connection. Raise the limit in the client configuration when polling very large batches. +**Compatibility note:** response frames larger than `maxResponseFrameSize` (default 64 MiB) are now rejected and close the connection under both framing modes. This is a behavior change for existing classic-framing clients. Raise the limit in the client configuration when polling very large batches. ### VSR framing -The SDK speaks the VSR wire protocol exclusively and requires an Iggy VSR -server: +Classic framing remains the default. Select VSR explicitly when connecting to +an Iggy VSR server: ```typescript import { SimpleClient, getRawClient } from "apache-iggy"; const config = { + protocol: "vsr" as const, transport: "TCP" as const, options: { host: "127.0.0.1", port: 8090 }, credentials: { username: "iggy", password: "iggy" }, @@ -51,18 +52,12 @@ const client = new SimpleClient(getRawClient(config)); const stats = await client.system.getStats(); ``` -Codes absent from the SDK command table use `Operation::NonReplicated` and -carry the command code in the request header's reserved field. The server -remains authoritative for classifying or rejecting extension commands. +VSR is a runtime protocol choice in Node.js, not a build feature. Codes absent +from the SDK command table use `Operation::NonReplicated` and carry the command +code in the request header's reserved field. The server remains authoritative +for classifying or rejecting extension commands. -Sends must use explicit `Partitioning.PartitionId` partitioning: the client -routes each request to a partition-scoped namespace, so broker-side balancing -(`Partitioning.Balanced`) and key hashing (`Partitioning.MessageKey`) are -rejected before the request is sent. - - -VSR works over TCP and TLS. It restricts `Client` to one pooled connection because authentication, request sequencing, and consumer-group assignments belong to one consensus session. Configurations requesting more than one pooled connection fail before a socket is opened. +The same npm package supports both framing modes over TCP and TLS. VSR restricts `Client` to one pooled connection because authentication, request sequencing, and consumer-group assignments belong to one consensus session. Configurations requesting more than one pooled connection fail before a socket is opened. VSR authentication translates the existing password and personal-access-token login APIs into the register handshake required by the consensus protocol. A @@ -73,7 +68,7 @@ new session. When the server's `[heartbeat]` eviction is enabled, configure the client's `heartbeatInterval` below the server heartbeat interval. Client heartbeats are disabled when `heartbeatInterval` is unset. -`sendBinaryRequest(code, payload)` sends an arbitrary command code. Known replicated commands use their registered operation, while unknown codes reach the server as non-replicated requests and are rejected by servers that do not register them. +`sendBinaryRequest(code, payload)` has the same signature under classic and VSR framing. Known replicated commands use their registered operation, while unknown codes reach the server as non-replicated requests and are rejected by servers that do not register them. Classic request bytes remain unchanged. ```typescript import { ResponseError } from "apache-iggy"; diff --git a/foreign/node/docker-compose.yml b/foreign/node/docker-compose.yml index 58f6049ba2..b30064648f 100644 --- a/foreign/node/docker-compose.yml +++ b/foreign/node/docker-compose.yml @@ -17,17 +17,14 @@ services: iggy-server: - # The SDK is vsr-only, so build the vsr server from the repo instead - # of pulling the legacy apache/iggy image. - build: - context: ../.. - dockerfile: core/server/Dockerfile + image: apache/iggy:latest container_name: iggy-server restart: unless-stopped networks: - iggy ports: - 3000:3000 + - 8080:8080 - 8090:8090 volumes: - iggy-server:/local_data diff --git a/foreign/node/package-lock.json b/foreign/node/package-lock.json index b1a49f7244..11295a84ab 100644 --- a/foreign/node/package-lock.json +++ b/foreign/node/package-lock.json @@ -1,12 +1,12 @@ { "name": "apache-iggy", - "version": "0.10.0-edge.1", + "version": "0.9.0-edge.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "apache-iggy", - "version": "0.10.0-edge.1", + "version": "0.9.0-edge.1", "license": "Apache-2.0", "dependencies": { "debug": "4.4.3", diff --git a/foreign/node/package.json b/foreign/node/package.json index b423ee3da3..7f4bcb6fc2 100644 --- a/foreign/node/package.json +++ b/foreign/node/package.json @@ -1,7 +1,7 @@ { "name": "apache-iggy", "type": "module", - "version": "0.10.0-edge.1", + "version": "0.9.0-edge.1", "description": "Official Apache Iggy NodeJS SDK", "keywords": [ "iggy", @@ -37,6 +37,7 @@ "scripts": { "test:unit": "node --import @swc-node/register/esm-register --test --experimental-test-coverage './src/**/*.test.ts'", "test:e2e": "node --import @swc-node/register/esm-register --test --experimental-test-coverage --test-force-exit './src/e2e/*.e2e.ts'", + "test:e2e:vsr": "IGGY_TEST_PROTOCOL=vsr node --import @swc-node/register/esm-register --test --experimental-test-coverage --test-force-exit './src/e2e/tcp.*.e2e.ts'", "test:bdd": "cucumber-js --exit", "test": "npm run test:unit && npm run test:bdd && npm run test:e2e", "clean": "rm -Rf dist/", diff --git a/foreign/node/scripts/check-vsr-protocol.mjs b/foreign/node/scripts/check-vsr-protocol.mjs index ebfcf66d2c..cdcd432e2e 100644 --- a/foreign/node/scripts/check-vsr-protocol.mjs +++ b/foreign/node/scripts/check-vsr-protocol.mjs @@ -31,9 +31,11 @@ const [ rustHeader, rustCommand, rustOperation, + rustNamespace, rustProtocolCargo, nodeCodes, nodeHeader, + nodeNamespace, nodeOperation, nodeRegister, ] = await Promise.all([ @@ -42,9 +44,11 @@ const [ read('core/binary_protocol/src/consensus/header.rs'), read('core/binary_protocol/src/consensus/command.rs'), read('core/binary_protocol/src/consensus/operation.rs'), + read('core/binary_protocol/src/namespace.rs'), read('core/binary_protocol/Cargo.toml'), readNode('src/wire/command.code.ts'), readNode('src/wire/vsr/header.ts'), + readNode('src/wire/vsr/namespace.ts'), readNode('src/wire/vsr/operation.ts'), readNode('src/wire/vsr/register.ts'), ]); @@ -121,14 +125,46 @@ assert.deepEqual( 'Node replicated code-to-operation map differs from Rust dispatch' ); -// No namespace-packing parity to check: the client wire carries no routing -// namespace, so the packing rules stay entirely server-side and this SDK has -// nothing to mirror. What still matters is that the client never grows a -// namespace field back -- the offset recomputation below catches that, since -// reintroducing one would move every field after it. -assert.ok( - !/namespace/i.test(nodeHeader.replace(/\/\*[\s\S]*?\*\/|\/\/.*/g, '')), - 'Node request header must not carry a namespace field' +const rustNamespaceValue = (name) => Number( + rustNamespace.match( + new RegExp(`pub const ${name}: usize = ([0-9_]+);`) + )?.[1].replaceAll('_', '') +); +const nodeNamespaceValue = (name) => Number( + nodeNamespace.match( + new RegExp(`const ${name} = ([0-9_]+);`) + )?.[1].replaceAll('_', '') +); +const namespaceLimits = new Map( + ['MAX_STREAMS', 'MAX_TOPICS', 'MAX_PARTITIONS'].map((name) => [ + name, + rustNamespaceValue(name) + ]) +); +for (const [name, value] of namespaceLimits) { + assert.ok(Number.isSafeInteger(value), `Rust ${name} was not found`); + assert.equal( + nodeNamespaceValue(name), + value, + `Node ${name} differs from Rust namespace layout` + ); +} + +const bitsRequired = (value) => BigInt(value).toString(2).length; +const expectedTopicShift = bitsRequired( + namespaceLimits.get('MAX_PARTITIONS') - 1 +); +const expectedStreamShift = expectedTopicShift + + bitsRequired(namespaceLimits.get('MAX_TOPICS') - 1); +const namespaceModule = await import( + pathToFileURL(resolve(nodeRoot, 'dist/wire/vsr/namespace.js')).href +); +assert.equal( + namespaceModule.packNamespace(1, 1, 1), + (1n << BigInt(expectedStreamShift)) | + (1n << BigInt(expectedTopicShift)) | + 1n, + 'Node namespace shifts differ from Rust namespace layout' ); const rustEvictionBlock = diff --git a/foreign/node/src/bdd/auth.ts b/foreign/node/src/bdd/auth.ts index 147327ee1d..a67e496923 100644 --- a/foreign/node/src/bdd/auth.ts +++ b/foreign/node/src/bdd/auth.ts @@ -18,14 +18,17 @@ import assert from 'node:assert/strict'; import { Client } from '../client/index.js'; +import type { Protocol } from '../client/index.js'; import { Given } from "@cucumber/cucumber"; import type { TestWorld } from './world.js'; import { getIggyAddress } from '../tcp.sm.utils.js'; const credentials = { username: 'iggy', password: 'iggy' }; const [host, port] = getIggyAddress(); +const protocol: Protocol = process.env.IGGY_TEST_PROTOCOL === 'vsr' ? 'vsr' : 'classic'; const opt = { + protocol, transport: 'TCP' as const, options: { host, port }, credentials diff --git a/foreign/node/src/client/client.config.test.ts b/foreign/node/src/client/client.config.test.ts index e4402e1b92..bc612da935 100644 --- a/foreign/node/src/client/client.config.test.ts +++ b/foreign/node/src/client/client.config.test.ts @@ -30,34 +30,55 @@ const config = (): ClientConfig => ({ }); describe('normalizeClientConfig', () => { - it('applies the default response frame limit', () => { - const normalized = normalizeClientConfig(config()); + it('defaults to classic without changing classic pool sizing', () => { + const normalized = normalizeClientConfig({ + ...config(), + poolSize: { min: 2, max: 4 } + }); + assert.equal(normalized.protocol, 'classic'); + assert.deepEqual(normalized.poolSize, { min: 2, max: 4 }); assert.equal( normalized.maxResponseFrameSize, DEFAULT_MAX_RESPONSE_FRAME_SIZE ); }); - it('restricts the client to one pooled connection', () => { - const normalized = normalizeClientConfig(config()); + it('restricts VSR to one pooled connection', () => { + const normalized = normalizeClientConfig({ + ...config(), + protocol: 'vsr' + }); assert.deepEqual(normalized.poolSize, { min: 1, max: 1 }); assert.throws( () => normalizeClientConfig({ ...config(), + protocol: 'vsr', poolSize: { max: 2 } }), /exactly one pooled connection/ ); }); - it('supports TLS transport', () => { + it('rejects invalid protocols before opening a socket', () => { + assert.throws( + () => normalizeClientConfig({ + ...config(), + protocol: 'auto' as 'vsr' + }), + /unsupported wire protocol/ + ); + }); + + it('supports VSR over TLS', () => { const normalized = normalizeClientConfig({ ...config(), + protocol: 'vsr', transport: 'TLS' }); + assert.equal(normalized.protocol, 'vsr'); assert.equal(normalized.transport, 'TLS'); assert.deepEqual(normalized.poolSize, { min: 1, max: 1 }); }); diff --git a/foreign/node/src/client/client.config.ts b/foreign/node/src/client/client.config.ts index ed4030819b..7fbb13e255 100644 --- a/foreign/node/src/client/client.config.ts +++ b/foreign/node/src/client/client.config.ts @@ -15,13 +15,20 @@ // specific language governing permissions and limitations // under the License. -import type { ClientConfig } from './client.type.js'; +import type { ClientConfig, Protocol } from './client.type.js'; export const DEFAULT_MAX_RESPONSE_FRAME_SIZE = 64 * 1024 * 1024; +const isProtocol = (value: unknown): value is Protocol => + value === 'classic' || value === 'vsr'; + export const normalizeClientConfig = ( config: ClientConfig -): ClientConfig => { +): ClientConfig & { protocol: Protocol } => { + const protocol = config.protocol ?? 'classic'; + if (!isProtocol(protocol)) + throw new TypeError(`unsupported wire protocol: ${String(protocol)}`); + const maxResponseFrameSize = config.maxResponseFrameSize ?? DEFAULT_MAX_RESPONSE_FRAME_SIZE; if (!Number.isSafeInteger(maxResponseFrameSize) || @@ -30,15 +37,17 @@ export const normalizeClientConfig = ( 'maxResponseFrameSize must be a safe integer of at least 256 bytes' ); - if ((config.poolSize?.min ?? 1) > 1 || (config.poolSize?.max ?? 1) > 1) + if (protocol === 'vsr' && + ((config.poolSize?.min ?? 1) > 1 || (config.poolSize?.max ?? 1) > 1)) throw new TypeError( 'VSR clients currently support exactly one pooled connection' ); return { ...config, + protocol, options: { ...config.options }, maxResponseFrameSize, - poolSize: { min: 1, max: 1 } + ...(protocol === 'vsr' ? { poolSize: { min: 1, max: 1 } } : {}) }; }; diff --git a/foreign/node/src/client/client.connection.test.ts b/foreign/node/src/client/client.connection.test.ts index 5c11c54ba6..d9639f432f 100644 --- a/foreign/node/src/client/client.connection.test.ts +++ b/foreign/node/src/client/client.connection.test.ts @@ -28,9 +28,6 @@ import { describe, it } from 'node:test'; import { ProtocolFrameError } from './client.frame.js'; import { IggyConnection } from './client.connection.js'; import type { ClientConfig } from './client.type.js'; -import { Command2, HEADER_SIZE, REPLY_OFFSET } from '../wire/vsr/header.js'; - -const FRAME_LIMIT = 2 * HEADER_SIZE; const startServer = async (): Promise => { const server = createServer(); @@ -40,6 +37,7 @@ const startServer = async (): Promise => { }; const connectionConfig = (server: Server): ClientConfig => ({ + protocol: 'classic', transport: 'TCP', options: { host: '127.0.0.1', @@ -47,17 +45,9 @@ const connectionConfig = (server: Server): ClientConfig => ({ }, credentials: { username: 'iggy', password: 'iggy' }, reconnect: { enabled: false, interval: 0, maxRetries: 0 }, - maxResponseFrameSize: FRAME_LIMIT + maxResponseFrameSize: 256 }); -const replyFrame = (body: Buffer): Buffer => { - const frame = Buffer.alloc(HEADER_SIZE + body.length); - frame.writeUInt32LE(frame.length, REPLY_OFFSET.size); - frame.writeUInt8(Command2.Reply, REPLY_OFFSET.command); - body.copy(frame, HEADER_SIZE); - return frame; -}; - const closeConnection = async ( connection: IggyConnection, server: Server @@ -85,7 +75,7 @@ describe('IggyConnection', () => { } ); - it('shares connection attempts, recognizes endpoints, and writes frames', + it('shares connection attempts, recognizes endpoints, and writes commands', async () => { const server = await startServer(); const received = new Promise((resolve) => { @@ -119,9 +109,10 @@ describe('IggyConnection', () => { true ); - const frame = replyFrame(Buffer.from('payload')); - connection.writeFrame(frame); - assert.deepEqual(await received, frame); + connection.writeCommand(1, Buffer.from('payload')); + const command = await received; + assert.equal(command.readUInt32LE(4), 1); + assert.deepEqual(command.subarray(8), Buffer.from('payload')); } finally { await closeConnection(connection, server); } @@ -134,14 +125,17 @@ describe('IggyConnection', () => { const connection = new IggyConnection(connectionConfig(server)); try { await connection.connect(); - const frame = replyFrame(Buffer.from('response')); + const body = Buffer.from('response'); + const frame = Buffer.alloc(8 + body.length); + frame.writeUInt32LE(body.length, 4); + body.copy(frame, 8); const response = once(connection, 'response'); connection._onData(frame.subarray(0, 6)); connection._onData(frame.subarray(6)); assert.deepEqual((await response)[0], frame); - const malformed = replyFrame(Buffer.alloc(0)); - malformed.writeUInt32LE(FRAME_LIMIT + 1, REPLY_OFFSET.size); + const malformed = Buffer.alloc(8); + malformed.writeUInt32LE(256, 4); const error = once(connection, 'error'); connection._onData(malformed); assert.ok((await error)[0] instanceof ProtocolFrameError); @@ -231,7 +225,10 @@ describe('IggyConnection', () => { assert.equal(connection.connected, true); assert.equal(connections, 2); - const frame = replyFrame(Buffer.from('fresh')); + const body = Buffer.from('fresh'); + const frame = Buffer.alloc(8 + body.length); + frame.writeUInt32LE(body.length, 4); + body.copy(frame, 8); const response = once(connection, 'response'); oldSocket.emit('data', Buffer.alloc(8)); connection._onData(frame); diff --git a/foreign/node/src/client/client.connection.ts b/foreign/node/src/client/client.connection.ts index 99bcda97c3..25e4274b20 100644 --- a/foreign/node/src/client/client.connection.ts +++ b/foreign/node/src/client/client.connection.ts @@ -21,6 +21,7 @@ import type { Socket } from 'node:net'; import { createConnection } from 'node:net'; import { connect as TLSConnect } from 'node:tls'; import type { ClientConfig, TlsOption, TcpOption, ReconnectOption } from "./client.type.js" +import { serializeCommand } from './client.utils.js'; import { debug } from './client.debug.js'; import { DEFAULT_MAX_RESPONSE_FRAME_SIZE } from './client.config.js'; import { @@ -140,6 +141,7 @@ export class IggyConnection extends EventEmitter { this.connectPromise = undefined; this.reconnectPromise = undefined; this.responseDecoder = new ResponseFrameDecoder( + config.protocol ?? 'classic', config.maxResponseFrameSize ?? DEFAULT_MAX_RESPONSE_FRAME_SIZE ); this.socket = this._installSocket(getTransport(config)); @@ -425,7 +427,8 @@ export class IggyConnection extends EventEmitter { try { for (const response of this.responseDecoder.push(data)) { - if (peekCommand(response) === Command2.Eviction) + if (this.config.protocol === 'vsr' && + peekCommand(response) === Command2.Eviction) this.emit('eviction', evictionError(response)); else this.emit('response', response); @@ -440,6 +443,18 @@ export class IggyConnection extends EventEmitter { } } + /** + * Writes a command to the socket. + * + * @param command - Command code + * @param payload - Command payload + * @returns True if the write was successful + */ + writeCommand(command: number, payload: Buffer): void { + const cmd = serializeCommand(command, payload); + this.socket.write(cmd); + } + writeFrame(frame: Buffer): void { this.socket.write(frame); } diff --git a/foreign/node/src/client/client.frame.test.ts b/foreign/node/src/client/client.frame.test.ts index bdeba1eda1..5b05481d8c 100644 --- a/foreign/node/src/client/client.frame.test.ts +++ b/foreign/node/src/client/client.frame.test.ts @@ -30,6 +30,13 @@ import { const LIMIT = 1024; +const classicFrame = (body: Buffer): Buffer => { + const frame = Buffer.alloc(8 + body.length); + frame.writeUInt32LE(body.length, 4); + body.copy(frame, 8); + return frame; +}; + const vsrFrame = (body: Buffer): Buffer => { const frame = Buffer.alloc(HEADER_SIZE + body.length); frame.writeUInt32LE(frame.length, REPLY_OFFSET.size); @@ -39,59 +46,83 @@ const vsrFrame = (body: Buffer): Buffer => { }; describe('extractResponseFrames', () => { - it('buffers headers split at every boundary', () => { - const frame = vsrFrame(Buffer.from('payload')); + for (const protocol of ['classic', 'vsr'] as const) { + const makeFrame = protocol === 'classic' ? classicFrame : vsrFrame; + + it(`buffers ${protocol} headers split at every boundary`, () => { + const frame = makeFrame(Buffer.from('payload')); + const headerSize = protocol === 'classic' ? 8 : HEADER_SIZE; + + for (let split = 0; split < headerSize; split += 1) { + const first = extractResponseFrames( + protocol, + frame.subarray(0, split), + LIMIT + ); + assert.equal(first.frames.length, 0); + const second = extractResponseFrames( + protocol, + Buffer.concat([first.remainder, frame.subarray(split)]), + LIMIT + ); + assert.deepEqual(second.frames, [frame]); + assert.equal(second.remainder.length, 0); + } + }); - for (let split = 0; split < HEADER_SIZE; split += 1) { - const first = extractResponseFrames(frame.subarray(0, split), LIMIT); + it(`buffers a fragmented ${protocol} body`, () => { + const frame = makeFrame(Buffer.from('payload')); + const split = frame.length - 2; + const first = extractResponseFrames( + protocol, + frame.subarray(0, split), + LIMIT + ); assert.equal(first.frames.length, 0); const second = extractResponseFrames( + protocol, Buffer.concat([first.remainder, frame.subarray(split)]), LIMIT ); assert.deepEqual(second.frames, [frame]); - assert.equal(second.remainder.length, 0); - } - }); + }); - it('buffers a fragmented body', () => { - const frame = vsrFrame(Buffer.from('payload')); - const split = frame.length - 2; - const first = extractResponseFrames(frame.subarray(0, split), LIMIT); - assert.equal(first.frames.length, 0); - const second = extractResponseFrames( - Buffer.concat([first.remainder, frame.subarray(split)]), - LIMIT - ); - assert.deepEqual(second.frames, [frame]); - }); + it(`extracts coalesced ${protocol} frames and a partial tail`, () => { + const first = makeFrame(Buffer.from('one')); + const second = makeFrame(Buffer.from('two')); + const third = makeFrame(Buffer.from('three')); + const input = Buffer.concat([first, second, third.subarray(0, 3)]); + const extracted = extractResponseFrames(protocol, input, LIMIT); - it('extracts coalesced frames and a partial tail', () => { - const first = vsrFrame(Buffer.from('one')); - const second = vsrFrame(Buffer.from('two')); - const third = vsrFrame(Buffer.from('three')); - const input = Buffer.concat([first, second, third.subarray(0, 3)]); - const extracted = extractResponseFrames(input, LIMIT); + assert.deepEqual(extracted.frames, [first, second]); + assert.deepEqual(extracted.remainder, third.subarray(0, 3)); + assert.equal(extracted.remainder.buffer, input.buffer); + }); + } - assert.deepEqual(extracted.frames, [first, second]); - assert.deepEqual(extracted.remainder, third.subarray(0, 3)); - assert.equal(extracted.remainder.buffer, input.buffer); - }); - - it('rejects a size below the header', () => { + it('rejects a VSR size below the header', () => { const frame = vsrFrame(Buffer.alloc(0)); frame.writeUInt32LE(0, REPLY_OFFSET.size); assert.throws( - () => extractResponseFrames(frame, LIMIT), + () => extractResponseFrames('vsr', frame, LIMIT), ProtocolFrameError ); }); - it('rejects an oversized frame before buffering its body', () => { + it('rejects an oversized VSR frame before buffering its body', () => { const header = vsrFrame(Buffer.alloc(0)); header.writeUInt32LE(LIMIT + 1, REPLY_OFFSET.size); assert.throws( - () => extractResponseFrames(header, LIMIT), + () => extractResponseFrames('vsr', header, LIMIT), + ProtocolFrameError + ); + }); + + it('rejects an oversized classic frame before buffering its body', () => { + const header = classicFrame(Buffer.alloc(0)); + header.writeUInt32LE(LIMIT, 4); + assert.throws( + () => extractResponseFrames('classic', header, LIMIT), ProtocolFrameError ); }); @@ -99,9 +130,9 @@ describe('extractResponseFrames', () => { describe('ResponseFrameDecoder', () => { it('decodes bytewise input without losing coalesced frames', () => { - const decoder = new ResponseFrameDecoder(LIMIT); - const first = vsrFrame(Buffer.from('first')); - const second = vsrFrame(Buffer.from('second')); + const decoder = new ResponseFrameDecoder('classic', LIMIT); + const first = classicFrame(Buffer.from('first')); + const second = classicFrame(Buffer.from('second')); const input = Buffer.concat([first, second]); const frames: Buffer[] = []; @@ -113,17 +144,17 @@ describe('ResponseFrameDecoder', () => { }); it('clears a partial frame', () => { - const decoder = new ResponseFrameDecoder(LIMIT); - decoder.push(vsrFrame(Buffer.from('body')).subarray(0, HEADER_SIZE - 2)); + const decoder = new ResponseFrameDecoder('vsr', LIMIT); + decoder.push(vsrFrame(Buffer.from('body')).subarray(0, 100)); assert.equal(decoder.hasBufferedData, true); decoder.clear(); assert.equal(decoder.hasBufferedData, false); }); it('rejects an oversized frame as soon as its header is complete', () => { - const decoder = new ResponseFrameDecoder(LIMIT); - const header = vsrFrame(Buffer.alloc(0)); - header.writeUInt32LE(LIMIT + 1, REPLY_OFFSET.size); + const decoder = new ResponseFrameDecoder('classic', LIMIT); + const header = Buffer.alloc(8); + header.writeUInt32LE(LIMIT, 4); assert.throws(() => decoder.push(header), ProtocolFrameError); }); }); diff --git a/foreign/node/src/client/client.frame.ts b/foreign/node/src/client/client.frame.ts index 38cc0566a1..4bd90ff5b6 100644 --- a/foreign/node/src/client/client.frame.ts +++ b/foreign/node/src/client/client.frame.ts @@ -15,11 +15,14 @@ // specific language governing permissions and limitations // under the License. +import type { Protocol } from './client.type.js'; import { - HEADER_SIZE, - readSize + HEADER_SIZE as VSR_HEADER_SIZE, + readSize as readVsrSize } from '../wire/vsr/header.js'; +const CLASSIC_HEADER_SIZE = 8; + export class ProtocolFrameError extends Error { constructor(message: string) { super(message); @@ -32,35 +35,45 @@ export type ExtractedFrames = { remainder: Buffer }; +const headerSizeFor = (protocol: Protocol): number => + protocol === 'vsr' ? VSR_HEADER_SIZE : CLASSIC_HEADER_SIZE; + const declaredFrameSize = ( + protocol: Protocol, header: Buffer, maximumFrameSize: number ): number => { - const declaredSize = readSize(header); + const headerSize = headerSizeFor(protocol); + const declaredSize = protocol === 'vsr' + ? readVsrSize(header) + : CLASSIC_HEADER_SIZE + header.readUInt32LE(4); - if (declaredSize < HEADER_SIZE) + if (declaredSize < headerSize) throw new ProtocolFrameError( - `declared frame size ${declaredSize} is below header size` + `declared ${protocol} frame size ${declaredSize} is below header size` ); if (declaredSize > maximumFrameSize) throw new ProtocolFrameError( - `declared frame size ${declaredSize} exceeds ` + + `declared ${protocol} frame size ${declaredSize} exceeds ` + `the ${maximumFrameSize} byte limit` ); return declaredSize; }; export const extractResponseFrames = ( + protocol: Protocol, buffer: Buffer, maximumFrameSize: number ): ExtractedFrames => { + const headerSize = headerSizeFor(protocol); const frames: Buffer[] = []; let offset = 0; - while (buffer.length - offset >= HEADER_SIZE) { + while (buffer.length - offset >= headerSize) { const available = buffer.length - offset; const declaredSize = declaredFrameSize( - buffer.subarray(offset, offset + HEADER_SIZE), + protocol, + buffer.subarray(offset, offset + headerSize), maximumFrameSize ); if (available < declaredSize) @@ -83,15 +96,19 @@ export const extractResponseFrames = ( * incomplete frame as new socket chunks arrive. */ export class ResponseFrameDecoder { + private readonly protocol: Protocol; private readonly maximumFrameSize: number; + private readonly headerSize: number; private chunks: Buffer[]; private chunkIndex: number; private chunkOffset: number; private bufferedLength: number; private expectedFrameSize?: number; - constructor(maximumFrameSize: number) { + constructor(protocol: Protocol, maximumFrameSize: number) { + this.protocol = protocol; this.maximumFrameSize = maximumFrameSize; + this.headerSize = headerSizeFor(protocol); this.chunks = []; this.chunkIndex = 0; this.chunkOffset = 0; @@ -120,10 +137,11 @@ export class ResponseFrameDecoder { const frames: Buffer[] = []; while (true) { if (this.expectedFrameSize === undefined) { - if (this.bufferedLength < HEADER_SIZE) + if (this.bufferedLength < this.headerSize) break; this.expectedFrameSize = declaredFrameSize( - this.peek(HEADER_SIZE), + this.protocol, + this.peek(this.headerSize), this.maximumFrameSize ); } diff --git a/foreign/node/src/client/client.socket.test.ts b/foreign/node/src/client/client.socket.test.ts index 33e841f9dc..76c8a783c8 100644 --- a/foreign/node/src/client/client.socket.test.ts +++ b/foreign/node/src/client/client.socket.test.ts @@ -203,6 +203,7 @@ const singleNodeHandler = (port: number): FrameHandler => }; const vsrConfig = (port: number): ClientConfig => ({ + protocol: 'vsr', transport: 'TCP', options: { host: '127.0.0.1', port }, credentials: { username: 'iggy', password: 'iggy' }, diff --git a/foreign/node/src/client/client.socket.ts b/foreign/node/src/client/client.socket.ts index 9bbafa401c..a29e328ddc 100644 --- a/foreign/node/src/client/client.socket.ts +++ b/foreign/node/src/client/client.socket.ts @@ -20,9 +20,10 @@ import { EventEmitter } from 'node:events'; import type { ClientConfig, ClientCredentials, CommandResponse, - PasswordCredentials, RawClient, SendCommandOptions, + PasswordCredentials, Protocol, RawClient, SendCommandOptions, TokenCredentials } from '../client/client.type.js'; +import { handleResponse } from './client.utils.js'; import { ResponseError, responseError } from '../wire/error.utils.js'; import { debug } from './client.debug.js'; import { IggyConnection } from './client.connection.js'; @@ -85,6 +86,8 @@ export class VsrResponseTimeoutError extends Error { * Implements command queuing, authentication, and heartbeat functionality. */ export class CommandResponseStream extends EventEmitter { + /** Server wire protocol used by this connection */ + readonly protocol: Protocol; /** Client configuration */ private options: ClientConfig; /** Underlying connection to the server */ @@ -117,6 +120,7 @@ export class CommandResponseStream extends EventEmitter { super(); const normalizedConfig = normalizeClientConfig(options); this.options = normalizedConfig; + this.protocol = normalizedConfig.protocol; this.connection = new IggyConnection(normalizedConfig); this.busy = false; this.isAuthenticated = false; @@ -172,7 +176,7 @@ export class CommandResponseStream extends EventEmitter { if (!this.connection.connected) await this.connection.connect() - if (isLoginCommand(command)) + if (this.options.protocol === 'vsr' && isLoginCommand(command)) await this._ensureVsrLeader(); if (!this.isAuthenticated && !this.isUnloggedCommand(command)) @@ -243,6 +247,8 @@ export class CommandResponseStream extends EventEmitter { payload: Buffer, handleResp = true ): Promise { + if (this.options.protocol !== 'vsr') + return this._processClassic(command, payload, handleResp); if (isLoginCommand(command) && this.isAuthenticated) return this._processVsrLogin(command, payload, handleResp); return this._processVsr(command, payload, handleResp); @@ -257,6 +263,22 @@ export class CommandResponseStream extends EventEmitter { return this._processVsr(command, payload, handleResp); } + private async _processClassic( + command: number, + payload: Buffer, + handleResp: boolean + ): Promise { + const response = await this._exchange( + () => this.connection.writeCommand(command, payload) + ); + if (!handleResp) + return response as unknown as CommandResponse; + const parsed = handleResponse(response); + if (parsed.status !== 0) + throw responseError(command, parsed.status); + return parsed; + } + private async _processVsr( command: number, payload: Buffer, @@ -387,7 +409,8 @@ export class CommandResponseStream extends EventEmitter { private isUnloggedCommand(command: number): boolean { return UNLOGGED_COMMAND_CODE.includes(command) || - command === COMMAND_CODE.GetClusterMetadata; + (this.options.protocol === 'vsr' && + command === COMMAND_CODE.GetClusterMetadata); } private async _ensureVsrLeader(): Promise { diff --git a/foreign/node/src/client/client.type.ts b/foreign/node/src/client/client.type.ts index a7e63723a3..080529891f 100644 --- a/foreign/node/src/client/client.type.ts +++ b/foreign/node/src/client/client.type.ts @@ -56,6 +56,8 @@ export type SendCommandOptions = { * Provides direct access to command sending and event handling. */ export type RawClient = { + /** Server wire protocol used by this connection */ + readonly protocol: Protocol, /** Sends a command to the server and returns the response */ sendCommand: ( code: number, @@ -112,6 +114,9 @@ export type ReconnectOption = { */ export type TransportOption = TcpOption | TlsOption; +/** Server wire protocol. */ +export type Protocol = 'classic' | 'vsr'; + /** * Token-based authentication credentials. */ @@ -150,6 +155,8 @@ export type PoolSizeOption = { * Complete client configuration for connecting to the Iggy server. */ export type ClientConfig = { + /** Server wire protocol (default: classic) */ + protocol?: Protocol, /** Transport protocol to use (TCP or TLS) */ transport: TransportType, /** Transport-specific connection options */ diff --git a/foreign/node/src/client/client.utils.test.ts b/foreign/node/src/client/client.utils.test.ts index 815058dd31..174f09cf5b 100644 --- a/foreign/node/src/client/client.utils.test.ts +++ b/foreign/node/src/client/client.utils.test.ts @@ -18,19 +18,32 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; -import { deserializeVoidResponse } from './client.utils.js'; +import { handleResponse, deserializeVoidResponse } from './client.utils.js'; -describe('deserializeVoidResponse', () => { +const SUCCESS = 0; - it('returns true only for a success status with an empty payload', () => { - const success = { status: 0, length: 0, data: Buffer.alloc(0) }; - assert.equal(deserializeVoidResponse(success), true); +describe('handleResponse', () => { - const failed = { status: 1, length: 0, data: Buffer.alloc(0) }; - assert.equal(deserializeVoidResponse(failed), false); + it('bounds data to the length field, not the full buffer', () => { + // Server says: status=0, length=0, no payload. + // But the raw buffer has 4 trailing bytes (e.g. start of next response). + const buf = Buffer.alloc(12); + buf.writeUInt32LE(SUCCESS, 0); // status + buf.writeUInt32LE(0, 4); // length = 0 (void response) + buf.writeUInt32LE(42, 8); // trailing bytes — NOT part of this response - const withData = { status: 0, length: 4, data: Buffer.alloc(4) }; - assert.equal(deserializeVoidResponse(withData), false); + const r = handleResponse(buf); + assert.equal(r.data.length, 0); + }); + + it('deserializeVoidResponse returns true for a valid void response with trailing buffer bytes', () => { + const buf = Buffer.alloc(12); + buf.writeUInt32LE(SUCCESS, 0); + buf.writeUInt32LE(0, 4); // length = 0 + buf.writeUInt32LE(42, 8); // trailing bytes + + const r = handleResponse(buf); + assert.equal(deserializeVoidResponse(r), true); }); }); diff --git a/foreign/node/src/client/client.utils.ts b/foreign/node/src/client/client.utils.ts index dfde49b820..bb9a922589 100644 --- a/foreign/node/src/client/client.utils.ts +++ b/foreign/node/src/client/client.utils.ts @@ -16,7 +16,45 @@ // under the License. // +import { Transform, type TransformCallback } from 'node:stream'; import type { CommandResponse } from './client.type.js'; +import { translateCommandCode } from '../wire/command.code.js'; +import { debug } from './client.debug.js'; + + +/** + * Parses a raw response buffer into a structured CommandResponse. + * Extracts status code, length, and payload data from the buffer. + * + * @param r - Raw response buffer from the server + * @returns Parsed command response with status, length, and data + */ +export const handleResponse = (r: Buffer) => { + const status = r.readUint32LE(0); + const length = r.readUint32LE(4); + debug('<== handleResponse', { status, length }); + return { + status, length, data: r.subarray(8, 8 + length) + } +}; + +/** + * Creates a Transform stream that parses response buffers. + * Transforms raw server responses into just the data payload. + * + * @returns Transform stream for processing server responses + */ +export const handleResponseTransform = () => new Transform({ + transform(chunk: Buffer, encoding: BufferEncoding, cb: TransformCallback) { + try { + const r = handleResponse(chunk); + debug('response::', r) + return cb(null, r.data); + } catch (err: unknown) { + return cb(new Error('handleResponseTransform error', { cause: err }), null); + } + } +}); /** * Deserializes a void response from the server. @@ -27,3 +65,33 @@ import type { CommandResponse } from './client.type.js'; */ export const deserializeVoidResponse = (r: CommandResponse) => r.status === 0 && r.data.length === 0; + +/** Length of the command code in bytes */ +const COMMAND_LENGTH = 4; + +/** + * Serializes a command and its payload into a buffer for sending to the server. + * Creates the wire format: [payload_size (4 bytes)][command (4 bytes)][payload] + * + * @param command - Command code to send + * @param payload - Command payload buffer + * @returns Buffer ready to be sent to the server + */ +export const serializeCommand = (command: number, payload: Buffer) => { + const payloadSize = payload.length + COMMAND_LENGTH; + const data = Buffer.allocUnsafe(8 + payload.length); + + data.writeUint32LE(payloadSize, 0); + data.writeUint32LE(command, 4); + data.fill(payload, 8); + + debug( + '==> CMD', command, + translateCommandCode(command), + 'LENGTH', payloadSize + ); + + debug('FullMessage#Base64', data.toString('base64')); + + return data; +} diff --git a/foreign/node/src/e2e/tcp.cluster.e2e.ts b/foreign/node/src/e2e/tcp.cluster.e2e.ts index d3e5c85e41..8fae9e93f4 100644 --- a/foreign/node/src/e2e/tcp.cluster.e2e.ts +++ b/foreign/node/src/e2e/tcp.cluster.e2e.ts @@ -20,14 +20,13 @@ import { after, describe, it } from "node:test"; import assert from "node:assert/strict"; import { getTestClient } from "./test-client.utils.js"; -// The suite runs against a single-node server (cluster mode off), which -// reports itself as the sole healthy leader with endpoints derived from -// its default config. +// cluster mode still in dev atm +// response is mocked from /core/server/config.toml const expectedMeta = { - name: "single-node", + name: "iggy-cluster", nodes: [ { - name: "iggy-node", + name: "iggy-node-1", ip: "127.0.0.1", endpoints: { tcp: 8090, @@ -38,6 +37,18 @@ const expectedMeta = { role: "Leader", status: "Healthy", }, + { + name: "iggy-node-2", + ip: "127.0.0.1", + endpoints: { + tcp: 8091, + quic: 8081, + http: 3001, + websocket: 8093, + }, + role: "Follower", + status: "Healthy", + }, ], }; diff --git a/foreign/node/src/e2e/tcp.consumer-group.e2e.ts b/foreign/node/src/e2e/tcp.consumer-group.e2e.ts index 58234fbb10..d0dd85497a 100644 --- a/foreign/node/src/e2e/tcp.consumer-group.e2e.ts +++ b/foreign/node/src/e2e/tcp.consumer-group.e2e.ts @@ -26,8 +26,10 @@ import { getIggyAddress } from '../tcp.sm.utils.js'; describe('e2e -> consumer-group', async () => { const [host, port] = getIggyAddress(); + const vsr = process.env.IGGY_TEST_PROTOCOL === 'vsr'; const c = new SingleClient({ + protocol: vsr ? 'vsr' : 'classic', transport: 'TCP', options: { host, port }, credentials: { username: 'iggy', password: 'iggy' } @@ -93,7 +95,9 @@ describe('e2e -> consumer-group', async () => { streamId: streamName, topicId: topicName, messages: generateMessages(mn), - partition: Partitioning.PartitionId((i / mn) % 3) + partition: vsr + ? Partitioning.PartitionId((i / mn) % 3) + : Partitioning.MessageKey(`key-${ i % 300 }`) })); } payloadLength = ct; diff --git a/foreign/node/src/e2e/tcp.consumer-stream.e2e.ts b/foreign/node/src/e2e/tcp.consumer-stream.e2e.ts index 40d9afe93a..e954e7e6a2 100644 --- a/foreign/node/src/e2e/tcp.consumer-stream.e2e.ts +++ b/foreign/node/src/e2e/tcp.consumer-stream.e2e.ts @@ -32,8 +32,10 @@ describe('e2e -> consumer-stream', async () => { const [host, port] = getIggyAddress(); const credentials = { username: 'iggy', password: 'iggy' }; + const vsr = process.env.IGGY_TEST_PROTOCOL === 'vsr'; const opt = { + protocol: (vsr ? 'vsr' : 'classic') as 'vsr' | 'classic', transport: 'TCP' as const, options: { host, port }, credentials @@ -60,7 +62,9 @@ describe('e2e -> consumer-stream', async () => { await sendSomeMessages(c.clientProvider)( streamName, topicName, - Partitioning.PartitionId((i / 100) % 3) + vsr + ? Partitioning.PartitionId((i / 100) % 3) + : Partitioning.MessageKey(`k-${i % 300}`) ); } }); diff --git a/foreign/node/src/e2e/tcp.send-message.e2e.ts b/foreign/node/src/e2e/tcp.send-message.e2e.ts index 8b170c7bad..e085e16128 100644 --- a/foreign/node/src/e2e/tcp.send-message.e2e.ts +++ b/foreign/node/src/e2e/tcp.send-message.e2e.ts @@ -27,6 +27,10 @@ describe('e2e -> message', async () => { const c = getTestClient(); + // Only the VSR lane reaches a server that reports offsets. The classic lane + // runs against the legacy server, which commits without confirming. + const vsr = process.env.IGGY_TEST_PROTOCOL === 'vsr'; + const streamName = 'e2e-stream-934'; const topicName = 'e2e-topic-832'; const partitionId = 0; @@ -49,6 +53,10 @@ describe('e2e -> message', async () => { it('e2e -> message::send', async () => { const { confirmations } = await c.message.send(msg); + if (!vsr) { + assert.equal(confirmations.length, 0); + return; + } assert.equal(confirmations.length, 1); assert.equal(confirmations[0].partitionId, partitionId); }); @@ -169,6 +177,10 @@ describe('e2e -> message', async () => { ...msg, messages: generateMessages(3) }); + if (!vsr) { + assert.equal(confirmations.length, 0); + return; + } // Landing behind the already committed batch is the part no placeholder // confirmation could reproduce. assert.equal(confirmations.length, 1); diff --git a/foreign/node/src/e2e/test-client.utils.ts b/foreign/node/src/e2e/test-client.utils.ts index 22090066ec..b5d2d77139 100644 --- a/foreign/node/src/e2e/test-client.utils.ts +++ b/foreign/node/src/e2e/test-client.utils.ts @@ -21,8 +21,10 @@ import { getIggyAddress } from '../tcp.sm.utils.js'; const credentials = { username: 'iggy', password: 'iggy' }; const [host, port] = getIggyAddress(); +const protocol = process.env.IGGY_TEST_PROTOCOL === 'vsr' ? 'vsr' : 'classic'; export const getTestClient = () => new Client({ + protocol, transport: 'TCP', options: { host, port }, credentials diff --git a/foreign/node/src/e2e/tls.system.e2e.ts b/foreign/node/src/e2e/tls.system.e2e.ts index e1dde409e4..c2f7b2d9bb 100644 --- a/foreign/node/src/e2e/tls.system.e2e.ts +++ b/foreign/node/src/e2e/tls.system.e2e.ts @@ -51,10 +51,14 @@ const caCertPath = process.env.E2E_ROOT_CA_CERT const getTlsClient = () => { const [, port] = getIggyAddress(); const caCert = readFileSync(caCertPath); + const protocol = process.env.IGGY_TEST_PROTOCOL === 'vsr' + ? 'vsr' + : 'classic'; // The server certificate SAN is DNS:localhost, so we connect via 'localhost' // for proper hostname verification (consistent with Python and C# TLS tests). return new Client({ + protocol, transport: 'TLS', options: { port, diff --git a/foreign/node/src/wire/command-set.test.ts b/foreign/node/src/wire/command-set.test.ts index dbc3a78c95..3c1cf6f755 100644 --- a/foreign/node/src/wire/command-set.test.ts +++ b/foreign/node/src/wire/command-set.test.ts @@ -23,6 +23,7 @@ import type { RawClient } from '../client/client.type.js'; import { COMMAND_CODE } from './command.code.js'; const mockRawClient = (): RawClient => ({ + protocol: 'classic', sendCommand: async () => { throw new Error('sendCommand should not be called by the session-control guard'); }, diff --git a/foreign/node/src/wire/message/poll-messages.command.ts b/foreign/node/src/wire/message/poll-messages.command.ts index 4023f387e6..335e3718e7 100644 --- a/foreign/node/src/wire/message/poll-messages.command.ts +++ b/foreign/node/src/wire/message/poll-messages.command.ts @@ -247,7 +247,8 @@ export const pollMessages = (getClient: ClientProvider) => const client = await getClient(); const release = client.hold?.(); try { - if (request.consumer.kind === ConsumerKind.Group && + if (client.protocol === 'vsr' && + request.consumer.kind === ConsumerKind.Group && request.partitionId === null) { const state = getGroupState(client); while (true) { diff --git a/foreign/node/src/wire/vsr/header.test.ts b/foreign/node/src/wire/vsr/header.test.ts index a8095c7446..91b92def0f 100644 --- a/foreign/node/src/wire/vsr/header.test.ts +++ b/foreign/node/src/wire/vsr/header.test.ts @@ -32,6 +32,7 @@ describe('VSR request header', () => { client, request: 0x0102030405060708n, operation: 2, + namespace: 0x8877665544332211n, session: 0x1020304050607080n, nonReplicatedCode: 60_001 }); @@ -52,6 +53,10 @@ describe('VSR request header', () => { 0x0102030405060708n ); assert.equal(header.readUInt8(REQUEST_OFFSET.operation), 2); + assert.equal( + header.readBigUInt64LE(REQUEST_OFFSET.namespace), + 0x8877665544332211n + ); assert.equal( header.readBigUInt64LE(REQUEST_OFFSET.session), 0x1020304050607080n @@ -66,6 +71,7 @@ describe('VSR request header', () => { client: 1n, request: 0n, operation: 1, + namespace: 1n << 63n, session: 0n }); const expected = Buffer.alloc(HEADER_SIZE); @@ -73,6 +79,7 @@ describe('VSR request header', () => { expected.writeUInt8(Command2.Request, REQUEST_OFFSET.command); expected.writeBigUInt64LE(1n, REQUEST_OFFSET.client); expected.writeUInt8(1, REQUEST_OFFSET.operation); + expected.writeBigUInt64LE(1n << 63n, REQUEST_OFFSET.namespace); assert.deepEqual(header, expected); }); @@ -83,6 +90,7 @@ describe('VSR request header', () => { client: maximum << 64n | maximum, request: maximum, operation: 160, + namespace: maximum, session: maximum }); assert.equal(header.readBigUInt64LE(REQUEST_OFFSET.request), maximum); diff --git a/foreign/node/src/wire/vsr/header.ts b/foreign/node/src/wire/vsr/header.ts index 9c51217dd1..bdee4f2706 100644 --- a/foreign/node/src/wire/vsr/header.ts +++ b/foreign/node/src/wire/vsr/header.ts @@ -27,15 +27,7 @@ /** Size of every consensus header, both directions. */ export const HEADER_SIZE = 256; -/** - * `RequestHeader` field offsets the client writes. - * - * The client wire carries no routing namespace: the server derives the - * consensus group (plane from `operation`, partition target from the payload) - * and stamps it into its own internal header. Everything that followed the - * removed field therefore sits eight bytes earlier than in the pre-derivation - * layout. - */ +/** `RequestHeader` field offsets the client writes. */ export const REQUEST_OFFSET = { size: 48, command: 60, @@ -43,8 +35,9 @@ export const REQUEST_OFFSET = { timestamp: 160, request: 168, operation: 176, - session: 184, - reserved: 196 + namespace: 184, + session: 192, + reserved: 204 } as const; /** `ReplyHeader` field offsets the client reads. */ @@ -52,7 +45,8 @@ export const REPLY_OFFSET = { size: 48, command: 60, operation: 208, - status: 216 + namespace: 216, + status: 224 } as const; /** `EvictionHeader` field offsets the client reads. */ @@ -104,6 +98,8 @@ export type RequestHeaderFields = { request: bigint, /** `Operation` discriminant. */ operation: number, + /** Routing namespace (u64). */ + namespace: bigint, /** Bound session (u64), or 0n. */ session: bigint, /** Command code for `NonReplicated`, placed in `reserved[0..4]`. */ @@ -113,7 +109,7 @@ export type RequestHeaderFields = { const U64_MASK = 0xFFFFFFFFFFFFFFFFn; /** - * Encodes a 256-byte request header. Only the six fields the server reads + * Encodes a 256-byte request header. Only the seven fields the server reads * are written; the checksums stay zero, matching the Rust SDK's contract * with the VSR server. */ @@ -126,6 +122,7 @@ export const encodeRequestHeader = (fields: RequestHeaderFields): Buffer => { header.writeBigUInt64LE(fields.client >> 64n, REQUEST_OFFSET.client + 8); header.writeBigUInt64LE(fields.request, REQUEST_OFFSET.request); header.writeUInt8(fields.operation, REQUEST_OFFSET.operation); + header.writeBigUInt64LE(fields.namespace, REQUEST_OFFSET.namespace); header.writeBigUInt64LE(fields.session, REQUEST_OFFSET.session); if (fields.nonReplicatedCode !== undefined) header.writeUInt32LE(fields.nonReplicatedCode, REQUEST_OFFSET.reserved); diff --git a/foreign/node/src/wire/vsr/index.ts b/foreign/node/src/wire/vsr/index.ts index ca88e7945b..5c079ba4f6 100644 --- a/foreign/node/src/wire/vsr/index.ts +++ b/foreign/node/src/wire/vsr/index.ts @@ -21,6 +21,7 @@ import type { CommandResponse } from '../../client/client.type.js'; import { COMMAND_CODE } from '../command.code.js'; import { responseError } from '../error.utils.js'; import { HEADER_SIZE, encodeRequestHeader } from './header.js'; +import { namespaceForRequest } from './namespace.js'; import { Operation, isPartition, @@ -65,6 +66,7 @@ export class VsrSession { const operation = registerCommand(command) ? Operation.Register : operationForCode(command); + const namespace = namespaceForRequest(command, payload, operation); const size = HEADER_SIZE + payload.length; if (size > MAX_U32) throw new RangeError('VSR request exceeds the u32 frame-size limit'); @@ -92,6 +94,7 @@ export class VsrSession { client: this.state.clientId, request, operation, + namespace, session, nonReplicatedCode: operation === Operation.NonReplicated ? command : undefined, diff --git a/foreign/node/src/wire/vsr/namespace.test.ts b/foreign/node/src/wire/vsr/namespace.test.ts new file mode 100644 index 0000000000..244ca39325 --- /dev/null +++ b/foreign/node/src/wire/vsr/namespace.test.ts @@ -0,0 +1,342 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; +import { serializeIdentifier } from '../identifier.utils.js'; +import { serializeSendMessages } from '../message/message.utils.js'; +import { Partitioning } from '../message/partitioning.utils.js'; +import { Consumer, serializeStoreOffset } from '../offset/offset.utils.js'; +import { COMMAND_CODE } from '../command.code.js'; +import { ResponseError } from '../error.utils.js'; +import { + METADATA_CONSENSUS_NAMESPACE, + namespaceForRequest, + packNamespace +} from './namespace.js'; +import { Operation } from './operation.js'; + +describe('VSR namespace routing', () => { + it('routes register and logout to metadata consensus', () => { + for (const operation of [Operation.Register, Operation.Logout]) + assert.equal( + namespaceForRequest(0, Buffer.alloc(0), operation), + METADATA_CONSENSUS_NAMESPACE + ); + }); + + it('routes metadata and non-replicated requests to zero', () => { + assert.equal( + namespaceForRequest( + COMMAND_CODE.CreateStream, + Buffer.alloc(0), + Operation.CreateStream + ), + 0n + ); + assert.equal( + namespaceForRequest( + COMMAND_CODE.GetStats, + Buffer.alloc(0), + Operation.NonReplicated + ), + 0n + ); + }); + + it('packs explicit send-message partition identifiers', () => { + const payload = serializeSendMessages( + 7, + 11, + [], + Partitioning.PartitionId(13) + ); + assert.equal( + namespaceForRequest( + COMMAND_CODE.SendMessages, + payload, + Operation.SendMessages + ), + packNamespace(7, 11, 13) + ); + }); + + it('defers named stream and topic routing to the server', () => { + const payload = serializeSendMessages( + 'stream', + 'topic', + [], + Partitioning.PartitionId(1) + ); + assert.equal( + namespaceForRequest( + COMMAND_CODE.SendMessages, + payload, + Operation.SendMessages + ), + 0n + ); + }); + + it('rejects server-selected message partitioning', () => { + const payload = serializeSendMessages( + 1, + 2, + [], + Partitioning.Balanced + ); + assert.throws( + () => namespaceForRequest( + COMMAND_CODE.SendMessages, + payload, + Operation.SendMessages + ), + (error: unknown) => + error instanceof ResponseError && error.errorCode === 5 + ); + }); + + it('routes consumer offsets from their explicit partition', () => { + const payload = serializeStoreOffset( + 1, + 2, + Consumer.Single, + 3, + 99n + ); + assert.equal( + namespaceForRequest( + COMMAND_CODE.StoreOffset, + payload, + Operation.StoreConsumerOffset + ), + packNamespace(1, 2, 3) + ); + }); + + it('routes delete-segments payloads', () => { + const payload = Buffer.concat([ + serializeIdentifier(1), + serializeIdentifier(2), + Buffer.from([3, 0, 0, 0]) + ]); + assert.equal( + namespaceForRequest( + COMMAND_CODE.DeleteSegments, + payload, + Operation.DeleteSegments + ), + packNamespace(1, 2, 3) + ); + }); + + it('accepts the maximum packable identifiers', () => { + assert.equal( + packNamespace(4095, 4095, 999_999), + (4095n << 32n) | (4095n << 20n) | 999_999n + ); + }); + + it('rejects peeks past the declared send-messages metadata region', () => { + const payload = serializeSendMessages( + 1, + 2, + [], + Partitioning.PartitionId(3) + ); + const underDeclared = Buffer.from(payload); + // Shrink the declared metadata region so the partitioning bytes sit + // outside it; the peek must fail instead of reading them. + underDeclared.writeUInt32LE(payload.readUInt32LE(0) - 6, 0); + assert.throws( + () => namespaceForRequest( + COMMAND_CODE.SendMessages, + underDeclared, + Operation.SendMessages + ), + (error: unknown) => + error instanceof ResponseError && error.errorCode === 3 + ); + }); + + it('rejects unknown codes in partition routing', () => { + assert.throws( + () => namespaceForRequest( + 60_001, + Buffer.alloc(0), + Operation.SendMessages + ), + (error: unknown) => + error instanceof ResponseError && error.errorCode === 5 + ); + }); + + it('requires an explicit consumer-offset partition', () => { + const payload = serializeStoreOffset( + 1, + 2, + Consumer.Single, + 3, + 99n + ); + // [kind u8][consumer 6][stream 6][topic 6] puts the partition flag at 19. + const withoutPartition = Buffer.from(payload); + withoutPartition.writeUInt8(0, 19); + assert.throws( + () => namespaceForRequest( + COMMAND_CODE.StoreOffset, + withoutPartition, + Operation.StoreConsumerOffset + ), + (error: unknown) => + error instanceof ResponseError && error.errorCode === 6 + ); + }); + + it('rejects namespace fields before masking', () => { + for (const [streamId, topicId, partitionId] of [ + [4096, 0, 0], + [0, 4096, 0], + [0, 0, 1_000_000] + ] as const) + assert.throws( + () => packNamespace(streamId, topicId, partitionId), + (error: unknown) => + error instanceof ResponseError && error.errorCode === 6 + ); + }); + + it('rejects negative and non-integer namespace fields', () => { + for (const [streamId, topicId, partitionId] of [ + [-1, 0, 0], + [0, -1, 0], + [0, 0, -1], + [0.5, 0, 0], + [0, Number.NaN, 0] + ] as const) + assert.throws( + () => packNamespace(streamId, topicId, partitionId), + (error: unknown) => + error instanceof ResponseError && error.errorCode === 6 + ); + }); + + it('rejects malformed identifiers at every prefix boundary', () => { + const payload = serializeSendMessages( + 1, + 2, + [], + Partitioning.PartitionId(3) + ); + for (let length = 0; length < payload.length; length += 1) + assert.throws( + () => namespaceForRequest( + COMMAND_CODE.SendMessages, + payload.subarray(0, length), + Operation.SendMessages + ), + ResponseError + ); + + const invalidKind = Buffer.from(payload); + invalidKind.writeUInt8(99, 4); + assert.throws( + () => namespaceForRequest( + COMMAND_CODE.SendMessages, + invalidKind, + Operation.SendMessages + ), + ResponseError + ); + }); + + it('rejects malformed consumer-offset and delete-segment payloads', () => { + const offsetPayload = serializeStoreOffset( + 1, + 2, + Consumer.Single, + 3, + 99n + ); + const invalidConsumerKind = Buffer.from(offsetPayload); + invalidConsumerKind.writeUInt8(0, 0); + assert.throws( + () => namespaceForRequest( + COMMAND_CODE.StoreOffset, + invalidConsumerKind, + Operation.StoreConsumerOffset + ), + ResponseError + ); + for (let length = 1; length < 20; length += 1) + assert.throws( + () => namespaceForRequest( + COMMAND_CODE.StoreOffset, + offsetPayload.subarray(0, length), + Operation.StoreConsumerOffset + ), + ResponseError + ); + + const deletePayload = Buffer.concat([ + serializeIdentifier(1), + serializeIdentifier(2), + Buffer.from([3, 0, 0, 0]) + ]); + for (let length = 0; length < deletePayload.length; length += 1) + assert.throws( + () => namespaceForRequest( + COMMAND_CODE.DeleteSegments, + deletePayload.subarray(0, length), + Operation.DeleteSegments + ), + ResponseError + ); + }); + + it('defers named offset and delete-segment routing to the server', () => { + const offsetPayload = serializeStoreOffset( + 'stream', + 'topic', + Consumer.Single, + 3, + 99n + ); + assert.equal( + namespaceForRequest( + COMMAND_CODE.StoreOffset, + offsetPayload, + Operation.StoreConsumerOffset + ), + 0n + ); + + const deletePayload = Buffer.concat([ + serializeIdentifier('stream'), + serializeIdentifier('topic'), + Buffer.from([3, 0, 0, 0]) + ]); + assert.equal( + namespaceForRequest( + COMMAND_CODE.DeleteSegments, + deletePayload, + Operation.DeleteSegments + ), + 0n + ); + }); +}); diff --git a/foreign/node/src/wire/vsr/namespace.ts b/foreign/node/src/wire/vsr/namespace.ts new file mode 100644 index 0000000000..1a6ec20e9d --- /dev/null +++ b/foreign/node/src/wire/vsr/namespace.ts @@ -0,0 +1,216 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// + +/** + * Namespace packing and the partition-plane payload peeks needed to derive + * it, ported from `core/binary_protocol/src/namespace.rs` and + * `namespace_for_request` in `core/sdk/src/vsr.rs`. + */ + +import { COMMAND_CODE } from '../command.code.js'; +import { responseError } from '../error.utils.js'; +import { Operation, isMetadata } from './operation.js'; + +/** `IggyError::InvalidCommand`. */ +const INVALID_COMMAND = 3; +/** `IggyError::FeatureUnavailable`. */ +const FEATURE_UNAVAILABLE = 5; +/** `IggyError::InvalidIdentifier`. */ +const INVALID_IDENTIFIER = 6; + +const MAX_STREAMS = 4096; +const MAX_TOPICS = 4096; +const MAX_PARTITIONS = 1_000_000; +const bitsRequired = (value: number): bigint => + BigInt(BigInt(value).toString(2).length); +const TOPIC_SHIFT = bitsRequired(MAX_PARTITIONS - 1); +const STREAM_SHIFT = TOPIC_SHIFT + bitsRequired(MAX_TOPICS - 1); + +/** + * Control-plane requests target the metadata replica (shard 0), selected by + * this exact sentinel. Plain 0 would fall into namespace hashing and land a + * Register on a peer shard. + */ +export const METADATA_CONSENSUS_NAMESPACE = 1n << 63n; + +/** Packs stream / topic / partition ids into a routing namespace. */ +export const packNamespace = ( + streamId: number, + topicId: number, + partitionId: number +): bigint => { + validateField(streamId, MAX_STREAMS); + validateField(topicId, MAX_TOPICS); + validateField(partitionId, MAX_PARTITIONS); + return (BigInt(streamId) << STREAM_SHIFT) | + (BigInt(topicId) << TOPIC_SHIFT) | + BigInt(partitionId); +}; + +/** + * Selects the routing namespace for a request. Partition-plane commands + * derive it from their own payload; a named stream or topic identifier + * yields 0 so the server resolves the name. + * + * @throws Error mirroring the Rust SDK: invalid-identifier for an + * out-of-range field, invalid-command for an undecodable payload, and + * feature-unavailable for a partition operation this SDK cannot derive. + */ +export const namespaceForRequest = ( + code: number, + payload: Buffer, + operation: number +): bigint => { + if (operation === Operation.Register || operation === Operation.Logout) + return METADATA_CONSENSUS_NAMESPACE; + if (operation === Operation.NonReplicated || isMetadata(operation)) + return 0n; + + switch (code) { + case COMMAND_CODE.SendMessages: + return namespaceFromSendMessages(payload); + case COMMAND_CODE.StoreOffset: + case COMMAND_CODE.DeleteConsumerOffset: + case COMMAND_CODE.StoreOffset2: + case COMMAND_CODE.DeleteConsumerOffset2: + return namespaceFromConsumerOffset(payload); + case COMMAND_CODE.DeleteSegments: + return namespaceFromDeleteSegments(payload); + default: + // The guard that keeps custom partition operations unreachable. + throw responseError(code, FEATURE_UNAVAILABLE); + } +}; + +/** A decoded identifier: numeric value, or null for a name (server resolves). */ +type PeekedIdentifier = { + numeric: number | null, + length: number +}; + +const IDENTIFIER_KIND_NUMERIC = 1; +const IDENTIFIER_KIND_STRING = 2; + +const peekIdentifier = (payload: Buffer, offset: number): PeekedIdentifier => { + if (payload.length < offset + 2) + throw responseError(0, INVALID_COMMAND); + const kind = payload.readUInt8(offset); + const length = payload.readUInt8(offset + 1); + if (payload.length < offset + 2 + length) + throw responseError(0, INVALID_COMMAND); + if (kind === IDENTIFIER_KIND_NUMERIC) { + if (length !== 4) + throw responseError(0, INVALID_COMMAND); + return { numeric: payload.readUInt32LE(offset + 2), length: 2 + length }; + } + if (kind === IDENTIFIER_KIND_STRING && length > 0) + return { numeric: null, length: 2 + length }; + throw responseError(0, INVALID_COMMAND); +}; + +const validateField = (value: number, exclusiveMax: number): void => { + if (!Number.isInteger(value) || value < 0 || value >= exclusiveMax) + throw responseError(0, INVALID_IDENTIFIER); +}; + +const namespaceFromIds = ( + stream: PeekedIdentifier, + topic: PeekedIdentifier, + partitionId: number +): bigint => { + // Named identifiers defer resolution to the server. + if (stream.numeric === null || topic.numeric === null) return 0n; + return packNamespace(stream.numeric, topic.numeric, partitionId); +}; + +/** + * `SendMessages`: `[metadata_len u32][stream ident][topic ident] + * [partitioning kind u8, len u8, value]...`. Only explicit `PartitionId` + * partitioning is routable under VSR; the broker never picks a partition. + */ +const namespaceFromSendMessages = (payload: Buffer): bigint => { + if (payload.length < 4) + throw responseError(COMMAND_CODE.SendMessages, INVALID_COMMAND); + const metadataLength = payload.readUInt32LE(0); + if (payload.length < 4 + metadataLength) + throw responseError(COMMAND_CODE.SendMessages, INVALID_COMMAND); + // Rust peeks inside payload[4..4 + metadata_length]; a read past the + // declared metadata region must fail rather than spill into message bytes + // and derive a namespace the server would never compute. + const metadata = payload.subarray(4, 4 + metadataLength); + + let offset = 0; + const stream = peekIdentifier(metadata, offset); + offset += stream.length; + const topic = peekIdentifier(metadata, offset); + offset += topic.length; + + if (metadata.length < offset + 2) + throw responseError(COMMAND_CODE.SendMessages, INVALID_COMMAND); + const partitioningKind = metadata.readUInt8(offset); + const partitioningLength = metadata.readUInt8(offset + 1); + const PARTITIONING_PARTITION_ID = 2; + if (partitioningKind !== PARTITIONING_PARTITION_ID) + throw responseError(COMMAND_CODE.SendMessages, FEATURE_UNAVAILABLE); + if (partitioningLength !== 4 || metadata.length < offset + 2 + 4) + throw responseError(COMMAND_CODE.SendMessages, INVALID_COMMAND); + const partitionId = metadata.readUInt32LE(offset + 2); + + return namespaceFromIds(stream, topic, partitionId); +}; + +/** + * Consumer-offset requests: `[consumer kind u8][consumer ident] + * [stream ident][topic ident][partition flag u8][partition u32]...`. + */ +const namespaceFromConsumerOffset = (payload: Buffer): bigint => { + if (payload.length < 1 || (payload.readUInt8(0) !== 1 && + payload.readUInt8(0) !== 2)) + throw responseError(COMMAND_CODE.StoreOffset, INVALID_COMMAND); + let offset = 1; + const consumer = peekIdentifier(payload, offset); + offset += consumer.length; + const stream = peekIdentifier(payload, offset); + offset += stream.length; + const topic = peekIdentifier(payload, offset); + offset += topic.length; + + if (payload.length < offset + 5) + throw responseError(COMMAND_CODE.StoreOffset, INVALID_COMMAND); + const hasPartition = payload.readUInt8(offset) === 1; + if (!hasPartition) + throw responseError(COMMAND_CODE.StoreOffset, INVALID_IDENTIFIER); + const partitionId = payload.readUInt32LE(offset + 1); + + return namespaceFromIds(stream, topic, partitionId); +}; + +/** `DeleteSegments`: `[stream ident][topic ident][partition u32]...`. */ +const namespaceFromDeleteSegments = (payload: Buffer): bigint => { + let offset = 0; + const stream = peekIdentifier(payload, offset); + offset += stream.length; + const topic = peekIdentifier(payload, offset); + offset += topic.length; + + if (payload.length < offset + 4) + throw responseError(COMMAND_CODE.DeleteSegments, INVALID_COMMAND); + const partitionId = payload.readUInt32LE(offset); + + return namespaceFromIds(stream, topic, partitionId); +}; diff --git a/foreign/node/src/wire/vsr/vsr.test.ts b/foreign/node/src/wire/vsr/vsr.test.ts index 9149938231..4e6cec6a2c 100644 --- a/foreign/node/src/wire/vsr/vsr.test.ts +++ b/foreign/node/src/wire/vsr/vsr.test.ts @@ -37,6 +37,23 @@ describe('VSR custom request framing', () => { assert.deepEqual(frame.subarray(256), payload); }); + it('does not consume a request ID when local routing fails', () => { + const session = new VsrSession(7n); + session.bind(42n); + assert.throws( + () => session.encode( + COMMAND_CODE.SendMessages, + Buffer.alloc(0) + ) + ); + + const frame = session.encode( + COMMAND_CODE.CreateStream, + Buffer.alloc(0) + ); + assert.equal(frame.readBigUInt64LE(REQUEST_OFFSET.request), 1n); + }); + it('rejects an unbound replicated request with a typed error', () => { const session = new VsrSession(); assert.throws( diff --git a/foreign/php/README.md b/foreign/php/README.md index 73a31cfa42..cdb5aab0d2 100644 --- a/foreign/php/README.md +++ b/foreign/php/README.md @@ -70,7 +70,7 @@ docker run --rm --name iggy-php-test \ You can also run a local server from the repository root: ```sh -cargo run --bin iggy-server -- --fresh --with-default-root-credentials +cargo run --bin iggy-server --fresh --with-default-root-credentials ``` The tests assume: diff --git a/foreign/php/docker-compose.test.yml b/foreign/php/docker-compose.test.yml index 3831e39bde..492779feac 100644 --- a/foreign/php/docker-compose.test.yml +++ b/foreign/php/docker-compose.test.yml @@ -16,15 +16,13 @@ # under the License. services: - # The PHP extension frames TCP with the VSR wire protocol only. iggy-server: build: context: ../.. dockerfile: core/server/Dockerfile args: PROFILE: debug - # The server takes only --replica-id; root credentials come from the env. - command: [] + command: ["--fresh", "--with-default-root-credentials"] container_name: iggy-server-php-test security_opt: - seccomp:unconfined @@ -33,8 +31,6 @@ services: - IGGY_TCP_ADDRESS=0.0.0.0:8090 - IGGY_QUIC_ADDRESS=0.0.0.0:8080 - IGGY_WEBSOCKET_ADDRESS=0.0.0.0:8092 - - IGGY_ROOT_USERNAME=iggy - - IGGY_ROOT_PASSWORD=iggy networks: - php-test-network ports: diff --git a/foreign/php/iggy-php.stubs.php b/foreign/php/iggy-php.stubs.php index caf4a7841c..b8cf52eaa7 100644 --- a/foreign/php/iggy-php.stubs.php +++ b/foreign/php/iggy-php.stubs.php @@ -218,8 +218,9 @@ public function pollMessages(mixed $stream, mixed $topic, int $partition_id, \Ig public function sendBinaryRequest(int $code, string $payload): string {} /** - * Sends messages to a topic and returns the commit confirmations, one per - * partition the batch landed in. + * Sends messages to a topic and returns the commit confirmations. + * + * The list is empty against the legacy server, which reports no offsets. * * @param mixed $stream * @param mixed $topic @@ -512,6 +513,9 @@ class SendMessagesConfirmation { * crash-restart can stamp a later batch with an offset a client has already * recorded. * + * The legacy server returns an empty confirmation list, so it reports no offset + * at all. + * * @var int */ public readonly int $base_offset; @@ -532,9 +536,9 @@ class SendMessagesResponse { /** * One confirmation per partition the batch landed in. * - * The list is empty when the server reports no offsets. A server can commit a - * batch it has no offsets to describe, so check for an empty array instead of - * indexing. + * The list is empty when the server reports no offsets. The legacy server never + * reports any, and a server that does can still commit a batch it has no offsets + * to describe, so check for an empty array instead of indexing. * * The confirmations are rebuilt on each getter call; cache the result in PHP if * they will be read repeatedly. diff --git a/foreign/php/scripts/test.sh b/foreign/php/scripts/test.sh index effe9e7a06..6c964ace24 100755 --- a/foreign/php/scripts/test.sh +++ b/foreign/php/scripts/test.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/foreign/php/src/client.rs b/foreign/php/src/client.rs index 118775a238..57e54d1512 100644 --- a/foreign/php/src/client.rs +++ b/foreign/php/src/client.rs @@ -184,8 +184,9 @@ impl IggyClient { }) } - /// Sends messages to a topic and returns the commit confirmations, one per - /// partition the batch landed in. + /// Sends messages to a topic and returns the commit confirmations. + /// + /// The list is empty against the legacy server, which reports no offsets. pub fn send_messages( &self, stream: PhpIdentifier, diff --git a/foreign/php/src/send_message.rs b/foreign/php/src/send_message.rs index 19b09a291e..cabbf2b8f0 100644 --- a/foreign/php/src/send_message.rs +++ b/foreign/php/src/send_message.rs @@ -99,9 +99,9 @@ impl From for SendMessagesResponse { impl SendMessagesResponse { /// One confirmation per partition the batch landed in. /// - /// The list is empty when the server reports no offsets. A server can commit a - /// batch it has no offsets to describe, so check for an empty array instead of - /// indexing. + /// The list is empty when the server reports no offsets. The legacy server never + /// reports any, and a server that does can still commit a batch it has no offsets + /// to describe, so check for an empty array instead of indexing. /// /// The confirmations are rebuilt on each getter call; cache the result in PHP if /// they will be read repeatedly. @@ -154,6 +154,9 @@ impl SendMessagesConfirmation { /// A batch is confirmed once it is committed in memory, not once it is fsynced. A /// crash-restart can stamp a later batch with an offset a client has already /// recorded. + /// + /// The legacy server returns an empty confirmation list, so it reports no offset + /// at all. #[php(getter)] pub fn base_offset(&self) -> u64 { self.inner.base_offset diff --git a/foreign/php/tests/IggySdkTest.php b/foreign/php/tests/IggySdkTest.php index 149106f1e5..978a6f7fba 100644 --- a/foreign/php/tests/IggySdkTest.php +++ b/foreign/php/tests/IggySdkTest.php @@ -222,8 +222,8 @@ public function testSendAndPollBinaryMessages(): void } } - #[TestDox('sendMessages reports one commit confirmation per written partition')] - public function testSendMessagesReportsCommitConfirmation(): void + #[TestDox('sendMessages against the legacy server reports no commit confirmations')] + public function testSendMessagesReportsNoConfirmationFromLegacyServer(): void { $client = new_client(); $streamName = unique_name('confirm-stream'); @@ -235,17 +235,7 @@ public function testSendMessagesReportsCommitConfirmation(): void $response = $client->sendMessages($streamName, $topicName, $partitionId, [new SendMessage('confirm-first')]); assert_instance_of(SendMessagesResponse::class, $response); - assert_count(1, $response->confirmations, 'a single-partition send commits in exactly one partition'); - assert_same($partitionId, $response->confirmations[0]->partition_id); - - // Offsets are per partition and start at 0, so the second send of - // the same size must be confirmed one message later. - $second = $client->sendMessages($streamName, $topicName, $partitionId, [new SendMessage('confirm-second')]); - assert_same( - $response->confirmations[0]->base_offset + 1, - $second->confirmations[0]->base_offset, - 'the confirmed offset must advance by the committed message count' - ); + assert_count(0, $response->confirmations, 'the legacy server answers a send with no offsets'); } finally { cleanup_stream_with_topics($client, $streamName, [$topicName]); } diff --git a/foreign/python/Cargo.toml b/foreign/python/Cargo.toml index 94ae86a223..f709947b69 100644 --- a/foreign/python/Cargo.toml +++ b/foreign/python/Cargo.toml @@ -37,7 +37,9 @@ doc = false [dependencies] bytes = "1.12.1" futures = "0.3.33" -iggy = { path = "../../core/sdk", version = "0.11.0-edge.1" } +iggy = { path = "../../core/sdk", version = "0.11.0-edge.1", features = [ + "vsr", +] } paste = "1" pyo3 = "0.29.0" pyo3-async-runtimes = { version = "0.29.0", features = [ @@ -45,5 +47,4 @@ pyo3-async-runtimes = { version = "0.29.0", features = [ "tokio-runtime", ] } pyo3-stub-gen = "0.23.0" -secrecy = "0.10" tokio = "1.53.1" diff --git a/foreign/python/README.md b/foreign/python/README.md index 8a5aff4250..99a9b0ed68 100644 --- a/foreign/python/README.md +++ b/foreign/python/README.md @@ -139,42 +139,6 @@ running prek / committing / pushing. This list is not exhaustive and other hook ./scripts/ci/markdownlint.sh --fix foreign/python/README.md # read the diff after applying this, sometimes it gives unwanted results, e.g. messing up enumerations ``` -## Client Configuration - -`IggyClient` takes either a server address or a `TcpConfig`: - -```python -import asyncio -from datetime import timedelta - -from apache_iggy import AutoLogin, IggyClient, TcpConfig, TcpReconnectionConfig - - -async def main(): - client = IggyClient( - TcpConfig( - server_address="127.0.0.1:8090", - auto_login=AutoLogin.username_password("iggy", "iggy"), - reconnection=TcpReconnectionConfig( - enabled=True, - max_retries=10, - interval=timedelta(seconds=2), - reestablish_after=timedelta(seconds=30), - ), - heartbeat_interval=timedelta(seconds=5), - # tls_enabled=True, - # tls_domain="localhost", - # tls_ca_file="../../core/certs/iggy_ca_cert.pem", - # tls_validate_certificate=True, - # nodelay=True, - ) - ) - await client.connect() - - -asyncio.run(main()) -``` - ## Examples Refer to the [examples/python/](https://github.com/apache/iggy/tree/master/examples/python) directory for usage examples. diff --git a/foreign/python/apache_iggy.pyi b/foreign/python/apache_iggy.pyi index ab63e66c5a..341b3885c6 100644 --- a/foreign/python/apache_iggy.pyi +++ b/foreign/python/apache_iggy.pyi @@ -29,7 +29,6 @@ __all__ = [ "AutoCommit", "AutoCommitAfter", "AutoCommitWhen", - "AutoLogin", "ConsumerGroup", "ConsumerGroupDetails", "ConsumerGroupMember", @@ -49,8 +48,6 @@ __all__ = [ "SendMessagesResponse", "StreamDetails", "StreamPermissions", - "TcpConfig", - "TcpReconnectionConfig", "Topic", "TopicDetails", "TopicPermissions", @@ -253,41 +250,6 @@ class AutoCommitWhen: ... -@typing.final -class AutoLogin: - r""" - The credentials replayed by the client every time it (re)connects. - - `IggyClient` only recovers a lost session when it has credentials to replay, - so a long-running consumer should pass one of the enabled variants. - """ - @property - def enabled(self) -> builtins.bool: - r""" - Whether automatic login is enabled. - """ - @property - def username(self) -> builtins.str | None: - r""" - The username to log in with, or `None` for the disabled and token variants. - """ - @staticmethod - def disabled() -> AutoLogin: - r""" - No automatic login. `login_user()` must be called by hand after every connect. - """ - @staticmethod - def username_password(username: builtins.str, password: builtins.str) -> AutoLogin: - r""" - Log in with the given username and password on every connect. - """ - @staticmethod - def personal_access_token(token: builtins.str) -> AutoLogin: - r""" - Log in with the given personal access token on every connect. - """ - def __repr__(self) -> builtins.str: ... - @typing.final class ConsumerGroup: @property @@ -810,25 +772,14 @@ class GlobalPermissions: class IggyClient: r""" A Python class representing the Iggy client. - It provides asynchronous functionality through the contained runtime. + It wraps the RustIggyClient and provides asynchronous functionality + through the contained runtime. """ - def __new__(cls, conn: TcpConfig | builtins.str | None = None) -> IggyClient: + def __new__(cls, conn: builtins.str | None = None) -> IggyClient: r""" - Constructs a new IggyClient from a TCP server address or a `TcpConfig`. + Constructs a new IggyClient from a TCP server address. This initializes a new runtime for asynchronous operations. Future versions might utilize asyncio for more Pythonic async. - - Args: - conn: Either a `host:port` address, or a `TcpConfig` carrying the full - transport configuration. Defaults to `127.0.0.1:8090` with auto-login - disabled. A malformed address is reported differently by the two - forms: the string form raises `RuntimeError` here, while `TcpConfig` - raises `ValueError` when it is constructed, before it ever reaches - this call. Neither exception is a subclass of the other. - - Raises: - RuntimeError: If the address passed as a string is not a valid - `host:port` pair. """ @classmethod def from_connection_string(cls, connection_string: builtins.str) -> IggyClient: @@ -839,14 +790,15 @@ class IggyClient: def ping(self) -> collections.abc.Awaitable[None]: r""" Sends a ping request to the server to check connectivity. - Raises `RuntimeError` if the connection fails. + Returns `Ok(())` if the server responds successfully, or a `PyRuntimeError` + if the connection fails. """ def login_user( self, username: builtins.str, password: builtins.str ) -> collections.abc.Awaitable[None]: r""" Logs in the user with the given credentials. - Raises `RuntimeError` on failure. + Returns `Ok(())` on success, or a PyRuntimeError on failure. """ def get_user( self, user_id: builtins.str | builtins.int @@ -862,8 +814,8 @@ class IggyClient: or `None` otherwise. Raises: - ValueError: If a string identifier is invalid. - RuntimeError: If the request fails. + PyValueError: If a string identifier is invalid. + PyRuntimeError: If the request fails. """ def get_users(self) -> collections.abc.Awaitable[list[UserInfo]]: r""" @@ -873,7 +825,7 @@ class IggyClient: An awaitable that resolves to `list[UserInfo]`. Raises: - RuntimeError: If the request fails. + PyRuntimeError: If the request fails. """ def create_user( self, @@ -895,7 +847,7 @@ class IggyClient: An awaitable that resolves to the created `UserInfoDetails`. Raises: - RuntimeError: If an argument is invalid or the request fails. + PyRuntimeError: If an argument is invalid or the request fails. """ def update_user( self, @@ -915,8 +867,8 @@ class IggyClient: An awaitable that resolves to `None` when the user is updated. Raises: - ValueError: If a string identifier is invalid. - RuntimeError: If the request fails. + PyValueError: If a string identifier is invalid. + PyRuntimeError: If the request fails. """ def delete_user( self, user_id: builtins.str | builtins.int @@ -931,8 +883,8 @@ class IggyClient: An awaitable that resolves to `None` when the user is deleted. Raises: - ValueError: If a string identifier is invalid. - RuntimeError: If the request fails. + PyValueError: If a string identifier is invalid. + PyRuntimeError: If the request fails. """ def update_permissions( self, user_id: builtins.str | builtins.int, permissions: Permissions | None @@ -951,8 +903,8 @@ class IggyClient: An awaitable that resolves to `None` when the permissions are updated. Raises: - ValueError: If a string identifier is invalid. - RuntimeError: If the request fails. + PyValueError: If a string identifier is invalid. + PyRuntimeError: If the request fails. """ def change_password( self, @@ -972,8 +924,8 @@ class IggyClient: An awaitable that resolves to `None` when the password is changed. Raises: - ValueError: If a string identifier is invalid. - RuntimeError: If the current password is wrong or the request fails. + PyValueError: If a string identifier is invalid. + PyRuntimeError: If the current password is wrong or the request fails. """ def logout_user(self) -> collections.abc.Awaitable[None]: r""" @@ -983,25 +935,24 @@ class IggyClient: An awaitable that resolves to `None` when the user is logged out. Raises: - RuntimeError: If the request fails. + PyRuntimeError: If the request fails. """ def connect(self) -> collections.abc.Awaitable[None]: r""" Connects the IggyClient to its service. - Raises `RuntimeError` if the connection fails. + Returns Ok(()) on successful connection or a PyRuntimeError on failure. """ def create_stream(self, name: builtins.str) -> collections.abc.Awaitable[None]: r""" Creates a new stream with the provided ID and name. - Raises `RuntimeError` if the stream cannot be created. + Returns Ok(()) on successful stream creation or a PyRuntimeError on failure. """ def get_stream( self, stream_id: builtins.str | builtins.int ) -> collections.abc.Awaitable[StreamDetails | None]: r""" Gets stream by id. - Returns the stream details, or `None` if the stream does not exist. - Raises `RuntimeError` on failure. + Returns Option of stream details or a PyRuntimeError on failure. """ def create_topic( self, @@ -1039,8 +990,7 @@ class IggyClient: ) -> collections.abc.Awaitable[TopicDetails | None]: r""" Gets topic by stream and id. - Returns the topic details, or `None` if the topic does not exist. - Raises `RuntimeError` on failure. + Returns Option of topic details or a PyRuntimeError on failure. """ def get_topics( self, stream_id: builtins.str | builtins.int @@ -1055,7 +1005,7 @@ class IggyClient: An awaitable that resolves to `list[Topic]`. Raises: - RuntimeError: If the identifier is invalid or the request fails. + PyRuntimeError: If the identifier is invalid or the request fails. """ def update_topic( self, @@ -1105,7 +1055,7 @@ class IggyClient: An awaitable that resolves to `None` when the topic is deleted. Raises: - RuntimeError: If an identifier is invalid or the request fails. + PyRuntimeError: If an identifier is invalid or the request fails. """ def purge_topic( self, @@ -1123,7 +1073,7 @@ class IggyClient: An awaitable that resolves to `None` when the topic is purged. Raises: - RuntimeError: If an identifier is invalid or the request fails. + PyRuntimeError: If an identifier is invalid or the request fails. """ def create_consumer_group( self, @@ -1143,8 +1093,8 @@ class IggyClient: An awaitable that resolves to `None` when the consumer group is created. Raises: - ValueError: If an identifier is invalid. - RuntimeError: If the request fails. + PyValueError: If an identifier is invalid. + PyRuntimeError: If the request fails. """ def get_consumer_group( self, @@ -1165,8 +1115,8 @@ class IggyClient: or `None` otherwise. Raises: - ValueError: If an identifier is invalid. - RuntimeError: If the request fails. + PyValueError: If an identifier is invalid. + PyRuntimeError: If the request fails. """ def get_consumer_groups( self, @@ -1184,8 +1134,8 @@ class IggyClient: An awaitable that resolves to `list[ConsumerGroup]`. Raises: - ValueError: If an identifier is invalid. - RuntimeError: If the request fails. + PyValueError: If an identifier is invalid. + PyRuntimeError: If the request fails. """ def delete_consumer_group( self, @@ -1205,8 +1155,8 @@ class IggyClient: An awaitable that resolves to `None` when the consumer group is deleted. Raises: - ValueError: If a string identifier is invalid. - RuntimeError: If the request fails. + PyValueError: If a string identifier is invalid. + PyRuntimeError: If the request fails. """ def join_consumer_group( self, @@ -1229,8 +1179,8 @@ class IggyClient: An awaitable that resolves to `None` when the client joins the consumer group. Raises: - ValueError: If a string identifier is invalid. - RuntimeError: If the request fails, including `Feature is unavailable` on HTTP transport. + PyValueError: If a string identifier is invalid. + PyRuntimeError: If the request fails, including `Feature is unavailable` on HTTP transport. """ def leave_consumer_group( self, @@ -1255,8 +1205,8 @@ class IggyClient: rejoin on their next poll. Raises: - ValueError: If a string identifier is invalid. - RuntimeError: If the request fails, including `Feature is unavailable` on HTTP transport. + PyValueError: If a string identifier is invalid. + PyRuntimeError: If the request fails, including `Feature is unavailable` on HTTP transport. """ def send_messages( self, @@ -1283,7 +1233,7 @@ class IggyClient: ) -> collections.abc.Awaitable[list[ReceiveMessage]]: r""" Polls for messages from the specified topic and partition. - Returns a list of received messages or a RuntimeError on failure. + Returns a list of received messages or a PyRuntimeError on failure. """ def consumer_group( self, @@ -1304,10 +1254,7 @@ class IggyClient: ) -> collections.abc.Awaitable[IggyConsumer]: r""" Creates a new consumer group consumer. - Returns the consumer or a RuntimeError on failure. Raises `ValueError` if - `poll_interval`, `polling_retry_interval`, `init_retry_interval` or an - `AutoCommit` interval is negative, or if any of those except `poll_interval` - is zero. + Returns the consumer or a PyRuntimeError on failure. """ def send_binary_request( self, code: builtins.int, payload: builtins.bytes @@ -1326,14 +1273,15 @@ class IggyClient: An awaitable that resolves to the raw response `bytes`. Raises: - RuntimeError: If the command cannot be sent or the server returns an error. + PyRuntimeError: If the command cannot be sent or the server returns an error. """ @typing.final class IggyConsumer: r""" A Python class representing the Iggy consumer. - It provides asynchronous functionality through the contained runtime. + It wraps the RustIggyConsumer and provides asynchronous functionality + through the contained runtime. """ def get_last_consumed_offset( self, partition_id: builtins.int @@ -1367,7 +1315,8 @@ class IggyConsumer: r""" Stores the provided offset for the provided partition id or if none is specified uses the current partition id for the consumer group. - Raises `RuntimeError` if the operation fails. + Returns `Ok(())` if the server responds successfully, or a `PyRuntimeError` + if the operation fails. """ def delete_offset( self, partition_id: builtins.int | None @@ -1375,13 +1324,14 @@ class IggyConsumer: r""" Deletes the offset for the provided partition id or if none is specified uses the current partition id for the consumer group. - Raises `RuntimeError` if the operation fails. + Returns `Ok(())` if the server responds successfully, or a `PyRuntimeError` + if the operation fails. """ def iter_messages(self) -> collections.abc.AsyncIterator[ReceiveMessage]: r""" Asynchronously iterate over `ReceiveMessage`s. Returns an async iterator that raises `StopAsyncIteration` when no more messages are available - or a `RuntimeError` on failure. + or a `PyRuntimeError` on failure. Note: This method does not currently support `AutoCommit.After`. For `AutoCommit.IntervalOrAfter(datetime.timedelta, AutoCommitAfter)`, only the interval part is applied; the `after` mode is ignored. @@ -1396,7 +1346,7 @@ class IggyConsumer: ) -> collections.abc.Awaitable[None]: r""" Consumes messages continuously using a callback function and an optional `asyncio.Event` for signaling shutdown. - Returns an awaitable that completes when shutdown is signaled or a RuntimeError on failure. + Returns an awaitable that completes when shutdown is signaled or a PyRuntimeError on failure. """ class IggyExpiry: @@ -1596,7 +1546,7 @@ class PollingStrategy: class ReceiveMessage: r""" A Python class representing a received message. - It provides access to the message payload and offset. + This class wraps a Rust message, allowing for access to its payload and offset from Python. """ def payload(self) -> bytes: r""" @@ -1646,6 +1596,8 @@ class ReceiveMessage: class SendMessage: r""" A Python class representing a message to be sent. + This class wraps a Rust message meant for sending, facilitating + the creation of such messages from Python and their subsequent use in Rust. """ def __new__( cls, @@ -1811,115 +1763,6 @@ class StreamPermissions: treated as `None`. """ -@typing.final -class TcpConfig: - r""" - Configuration for the TCP transport, accepted by `IggyClient(...)`. - - Every field is keyword-only and optional. - """ - @property - def server_address(self) -> builtins.str: ... - @property - def auto_login(self) -> AutoLogin: ... - @property - def reconnection(self) -> TcpReconnectionConfig: ... - @property - def heartbeat_interval(self) -> datetime.timedelta: ... - @property - def tls_enabled(self) -> builtins.bool: ... - @property - def tls_domain(self) -> builtins.str: ... - @property - def tls_ca_file(self) -> builtins.str | None: ... - @property - def tls_validate_certificate(self) -> builtins.bool: ... - @property - def nodelay(self) -> builtins.bool: ... - def __new__( - cls, - *, - server_address: builtins.str | None = None, - auto_login: AutoLogin | None = None, - reconnection: TcpReconnectionConfig | None = None, - heartbeat_interval: datetime.timedelta | None = None, - tls_enabled: builtins.bool | None = None, - tls_domain: builtins.str | None = None, - tls_ca_file: builtins.str | None = None, - tls_validate_certificate: builtins.bool | None = None, - nodelay: builtins.bool | None = None, - ) -> TcpConfig: - r""" - Constructs a TCP configuration. - - Args: - server_address: `host:port` of the Iggy server. Defaults to `127.0.0.1:8090`. - auto_login: Credentials replayed on every connect. Defaults to `AutoLogin.disabled()`. - reconnection: Reconnection policy. Defaults to `TcpReconnectionConfig()`. - heartbeat_interval: Interval of heartbeats sent by the client. Defaults to 5 seconds. - tls_enabled: Whether to connect over TLS. Defaults to disabled. - tls_domain: Domain to validate the certificate against. Empty means it is - taken from `server_address`. - tls_ca_file: Path to the CA file for TLS. Read only when `tls_enabled` - and `tls_validate_certificate` are both on; with either one off it - is kept but never consulted, so pairing it with - `tls_validate_certificate=False` pins nothing. - tls_validate_certificate: Whether to validate the server certificate. - Defaults to validating. Disabling this accepts any certificate the - server presents, including self-signed and mismatched ones, and - takes precedence over `tls_ca_file`; intended for local development - only. - nodelay: Disable the Nagle algorithm for the TCP socket. Defaults to - leaving it on. - - Raises: - ValueError: If `server_address` is not a valid `host:port` pair, if a - duration is negative, or if `heartbeat_interval` is zero. - """ - def __repr__(self) -> builtins.str: ... - -@typing.final -class TcpReconnectionConfig: - r""" - How the TCP client reconnects after the connection to the server is lost. - """ - @property - def enabled(self) -> builtins.bool: ... - @property - def max_retries(self) -> builtins.int | None: ... - @property - def interval(self) -> datetime.timedelta: ... - @property - def reestablish_after(self) -> datetime.timedelta: ... - def __new__( - cls, - *, - enabled: builtins.bool | None = None, - max_retries: builtins.int | None = None, - interval: datetime.timedelta | None = None, - reestablish_after: datetime.timedelta | None = None, - ) -> TcpReconnectionConfig: - r""" - Constructs a reconnection policy. - - Args: - enabled: Whether to reconnect at all. Defaults to enabled. - max_retries: Attempts before giving up, or `None` for unlimited. - Defaults to unlimited, which means a call awaited while the server - is down never returns: `connect()`, `send_messages()` and - `poll_messages()` all wait inside the retry loop. Set a finite - number for request/reply style usage, so a call fails instead. - interval: Delay between attempts. Defaults to 1 second. - reestablish_after: Cooldown before reconnecting after a previously - successful connection. Defaults to 5 seconds. - - Raises: - ValueError: If a duration is negative, if `max_retries` is outside the - range of an unsigned 32-bit integer, or if `interval` is zero while - reconnection is enabled and `max_retries` is unlimited. - """ - def __repr__(self) -> builtins.str: ... - @typing.final class Topic: @property diff --git a/foreign/python/docker-compose.test.yml b/foreign/python/docker-compose.test.yml index 78caed9929..129b941bf5 100644 --- a/foreign/python/docker-compose.test.yml +++ b/foreign/python/docker-compose.test.yml @@ -22,8 +22,7 @@ services: dockerfile: core/server/Dockerfile args: PROFILE: debug - # The server takes only --replica-id; root credentials come from the env. - command: [] + command: ["--fresh", "--with-default-root-credentials"] container_name: iggy-server-python-test security_opt: - seccomp:unconfined @@ -37,11 +36,8 @@ services: - IGGY_HTTP_ADDRESS=0.0.0.0:3000 - IGGY_TCP_ADDRESS=0.0.0.0:8090 - IGGY_QUIC_ADDRESS=0.0.0.0:8080 - - IGGY_WEBSOCKET_ADDRESS=0.0.0.0:8092 - - IGGY_ROOT_USERNAME=iggy - - IGGY_ROOT_PASSWORD=iggy healthcheck: - test: [ "CMD", "/usr/local/bin/iggy", "--tcp-server-address", "127.0.0.1:8090", "ping" ] + test: [ "CMD", "iggy", "--tcp-server-address", "127.0.0.1:8090", "ping" ] interval: 5s timeout: 5s retries: 12 diff --git a/foreign/python/scripts/test.sh b/foreign/python/scripts/test.sh index 295fe3d3df..5d7aff90e8 100755 --- a/foreign/python/scripts/test.sh +++ b/foreign/python/scripts/test.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/foreign/python/src/client.rs b/foreign/python/src/client.rs index cde56040ff..c7cbe39520 100644 --- a/foreign/python/src/client.rs +++ b/foreign/python/src/client.rs @@ -30,12 +30,10 @@ use pyo3_stub_gen::derive::{gen_stub_pyclass, gen_stub_pymethods}; use std::str::FromStr; use std::sync::Arc; -use crate::config::PyClientConfig; use crate::consumer::{ AutoCommit, ConsumerGroup as PyConsumerGroup, ConsumerGroupDetails as PyConsumerGroupDetails, - IggyConsumer, + IggyConsumer, py_delta_to_iggy_duration, }; -use crate::duration::{py_delta_to_iggy_duration, reject_zero}; use crate::identifier::PyIdentifier; use crate::permissions::Permissions as PyPermissions; use crate::receive_message::{PollingStrategy, ReceiveMessage}; @@ -48,7 +46,8 @@ use crate::user::{ use tokio::sync::Mutex; /// A Python class representing the Iggy client. -/// It provides asynchronous functionality through the contained runtime. +/// It wraps the RustIggyClient and provides asynchronous functionality +/// through the contained runtime. #[gen_stub_pyclass] #[pyclass] pub struct IggyClient { @@ -84,44 +83,21 @@ fn resolve_topic_params( #[gen_stub_pymethods] #[pymethods] impl IggyClient { - /// Constructs a new IggyClient from a TCP server address or a `TcpConfig`. + /// Constructs a new IggyClient from a TCP server address. /// This initializes a new runtime for asynchronous operations. /// Future versions might utilize asyncio for more Pythonic async. - /// - /// Args: - /// conn: Either a `host:port` address, or a `TcpConfig` carrying the full - /// transport configuration. Defaults to `127.0.0.1:8090` with auto-login - /// disabled. A malformed address is reported differently by the two - /// forms: the string form raises `RuntimeError` here, while `TcpConfig` - /// raises `ValueError` when it is constructed, before it ever reaches - /// this call. Neither exception is a subclass of the other. - /// - /// Raises: - /// RuntimeError: If the address passed as a string is not a valid - /// `host:port` pair. #[new] #[pyo3(signature = (conn=None))] fn new( - #[gen_stub(override_type(type_repr = "TcpConfig | builtins.str | None"))] conn: Option< - PyClientConfig, - >, + #[gen_stub(override_type(type_repr = "builtins.str | None"))] conn: Option, ) -> PyResult { - let config = match conn { - Some(PyClientConfig::Config(config)) => config.client_config(), - Some(PyClientConfig::ServerAddress(server_address)) => Arc::new( - TcpClientConfigBuilder::new() - .with_server_address(server_address) - .build() - .map_err(|e| { - PyErr::new::(e.to_string()) - })?, - ), - None => Arc::new(TcpClientConfig::default()), - }; - let tcp_client = TcpClient::create(config) + let client = IggyClientBuilder::new() + .with_tcp() + .with_server_address(conn.unwrap_or("127.0.0.1:8090".to_string())) + .build() .map_err(|e| PyErr::new::(e.to_string()))?; Ok(IggyClient { - inner: Arc::new(RustIggyClient::new(ClientWrapper::Tcp(tcp_client))), + inner: Arc::new(client), }) } @@ -143,7 +119,8 @@ impl IggyClient { } /// Sends a ping request to the server to check connectivity. - /// Raises `RuntimeError` if the connection fails. + /// Returns `Ok(())` if the server responds successfully, or a `PyRuntimeError` + /// if the connection fails. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn ping<'a>(&self, py: Python<'a>) -> PyResult> { let inner = self.inner.clone(); @@ -156,7 +133,7 @@ impl IggyClient { } /// Logs in the user with the given credentials. - /// Raises `RuntimeError` on failure. + /// Returns `Ok(())` on success, or a PyRuntimeError on failure. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn login_user<'a>( &self, @@ -184,8 +161,8 @@ impl IggyClient { /// or `None` otherwise. /// /// Raises: - /// ValueError: If a string identifier is invalid. - /// RuntimeError: If the request fails. + /// PyValueError: If a string identifier is invalid. + /// PyRuntimeError: If the request fails. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[UserInfoDetails | None]", imports=("collections.abc")))] fn get_user<'a>(&self, py: Python<'a>, user_id: PyIdentifier) -> PyResult> { let user_id = Identifier::try_from(user_id)?; @@ -206,7 +183,7 @@ impl IggyClient { /// An awaitable that resolves to `list[UserInfo]`. /// /// Raises: - /// RuntimeError: If the request fails. + /// PyRuntimeError: If the request fails. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[list[UserInfo]]", imports=("collections.abc")))] fn get_users<'a>(&self, py: Python<'a>) -> PyResult> { let inner = self.inner.clone(); @@ -232,7 +209,7 @@ impl IggyClient { /// An awaitable that resolves to the created `UserInfoDetails`. /// /// Raises: - /// RuntimeError: If an argument is invalid or the request fails. + /// PyRuntimeError: If an argument is invalid or the request fails. #[pyo3(signature = (username, password, status=None, permissions=None))] #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[UserInfoDetails]", imports=("collections.abc")))] fn create_user<'a>( @@ -269,8 +246,8 @@ impl IggyClient { /// An awaitable that resolves to `None` when the user is updated. /// /// Raises: - /// ValueError: If a string identifier is invalid. - /// RuntimeError: If the request fails. + /// PyValueError: If a string identifier is invalid. + /// PyRuntimeError: If the request fails. #[pyo3(signature = (user_id, username=None, status=None))] #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn update_user<'a>( @@ -302,8 +279,8 @@ impl IggyClient { /// An awaitable that resolves to `None` when the user is deleted. /// /// Raises: - /// ValueError: If a string identifier is invalid. - /// RuntimeError: If the request fails. + /// PyValueError: If a string identifier is invalid. + /// PyRuntimeError: If the request fails. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn delete_user<'a>(&self, py: Python<'a>, user_id: PyIdentifier) -> PyResult> { let user_id = Identifier::try_from(user_id)?; @@ -331,8 +308,8 @@ impl IggyClient { /// An awaitable that resolves to `None` when the permissions are updated. /// /// Raises: - /// ValueError: If a string identifier is invalid. - /// RuntimeError: If the request fails. + /// PyValueError: If a string identifier is invalid. + /// PyRuntimeError: If the request fails. #[pyo3(signature = (user_id, permissions))] #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn update_permissions<'a>( @@ -367,8 +344,8 @@ impl IggyClient { /// An awaitable that resolves to `None` when the password is changed. /// /// Raises: - /// ValueError: If a string identifier is invalid. - /// RuntimeError: If the current password is wrong or the request fails. + /// PyValueError: If a string identifier is invalid. + /// PyRuntimeError: If the current password is wrong or the request fails. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn change_password<'a>( &self, @@ -395,7 +372,7 @@ impl IggyClient { /// An awaitable that resolves to `None` when the user is logged out. /// /// Raises: - /// RuntimeError: If the request fails. + /// PyRuntimeError: If the request fails. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn logout_user<'a>(&self, py: Python<'a>) -> PyResult> { let inner = self.inner.clone(); @@ -410,7 +387,7 @@ impl IggyClient { } /// Connects the IggyClient to its service. - /// Raises `RuntimeError` if the connection fails. + /// Returns Ok(()) on successful connection or a PyRuntimeError on failure. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn connect<'a>(&self, py: Python<'a>) -> PyResult> { let inner = self.inner.clone(); @@ -424,7 +401,7 @@ impl IggyClient { } /// Creates a new stream with the provided ID and name. - /// Raises `RuntimeError` if the stream cannot be created. + /// Returns Ok(()) on successful stream creation or a PyRuntimeError on failure. #[pyo3(signature = (name))] #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn create_stream<'a>(&self, py: Python<'a>, name: String) -> PyResult> { @@ -439,8 +416,7 @@ impl IggyClient { } /// Gets stream by id. - /// Returns the stream details, or `None` if the stream does not exist. - /// Raises `RuntimeError` on failure. + /// Returns Option of stream details or a PyRuntimeError on failure. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[StreamDetails | None]", imports=("collections.abc")))] fn get_stream<'a>( &self, @@ -524,8 +500,7 @@ impl IggyClient { } /// Gets topic by stream and id. - /// Returns the topic details, or `None` if the topic does not exist. - /// Raises `RuntimeError` on failure. + /// Returns Option of topic details or a PyRuntimeError on failure. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[TopicDetails | None]", imports=("collections.abc")))] fn get_topic<'a>( &self, @@ -555,7 +530,7 @@ impl IggyClient { /// An awaitable that resolves to `list[Topic]`. /// /// Raises: - /// RuntimeError: If the identifier is invalid or the request fails. + /// PyRuntimeError: If the identifier is invalid or the request fails. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[list[Topic]]", imports=("collections.abc")))] fn get_topics<'a>( &self, @@ -652,7 +627,7 @@ impl IggyClient { /// An awaitable that resolves to `None` when the topic is deleted. /// /// Raises: - /// RuntimeError: If an identifier is invalid or the request fails. + /// PyRuntimeError: If an identifier is invalid or the request fails. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn delete_topic<'a>( &self, @@ -683,7 +658,7 @@ impl IggyClient { /// An awaitable that resolves to `None` when the topic is purged. /// /// Raises: - /// RuntimeError: If an identifier is invalid or the request fails. + /// PyRuntimeError: If an identifier is invalid or the request fails. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn purge_topic<'a>( &self, @@ -715,8 +690,8 @@ impl IggyClient { /// An awaitable that resolves to `None` when the consumer group is created. /// /// Raises: - /// ValueError: If an identifier is invalid. - /// RuntimeError: If the request fails. + /// PyValueError: If an identifier is invalid. + /// PyRuntimeError: If the request fails. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn create_consumer_group<'a>( &self, @@ -750,8 +725,8 @@ impl IggyClient { /// or `None` otherwise. /// /// Raises: - /// ValueError: If an identifier is invalid. - /// RuntimeError: If the request fails. + /// PyValueError: If an identifier is invalid. + /// PyRuntimeError: If the request fails. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[ConsumerGroupDetails | None]", imports=("collections.abc")))] fn get_consumer_group<'a>( &self, @@ -784,8 +759,8 @@ impl IggyClient { /// An awaitable that resolves to `list[ConsumerGroup]`. /// /// Raises: - /// ValueError: If an identifier is invalid. - /// RuntimeError: If the request fails. + /// PyValueError: If an identifier is invalid. + /// PyRuntimeError: If the request fails. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[list[ConsumerGroup]]", imports=("collections.abc")))] fn get_consumer_groups<'a>( &self, @@ -820,8 +795,8 @@ impl IggyClient { /// An awaitable that resolves to `None` when the consumer group is deleted. /// /// Raises: - /// ValueError: If a string identifier is invalid. - /// RuntimeError: If the request fails. + /// PyValueError: If a string identifier is invalid. + /// PyRuntimeError: If the request fails. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn delete_consumer_group<'a>( &self, @@ -858,8 +833,8 @@ impl IggyClient { /// An awaitable that resolves to `None` when the client joins the consumer group. /// /// Raises: - /// ValueError: If a string identifier is invalid. - /// RuntimeError: If the request fails, including `Feature is unavailable` on HTTP transport. + /// PyValueError: If a string identifier is invalid. + /// PyRuntimeError: If the request fails, including `Feature is unavailable` on HTTP transport. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn join_consumer_group<'a>( &self, @@ -898,8 +873,8 @@ impl IggyClient { /// rejoin on their next poll. /// /// Raises: - /// ValueError: If a string identifier is invalid. - /// RuntimeError: If the request fails, including `Feature is unavailable` on HTTP transport. + /// PyValueError: If a string identifier is invalid. + /// PyRuntimeError: If the request fails, including `Feature is unavailable` on HTTP transport. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn leave_consumer_group<'a>( &self, @@ -963,7 +938,7 @@ impl IggyClient { } /// Polls for messages from the specified topic and partition. - /// Returns a list of received messages or a RuntimeError on failure. + /// Returns a list of received messages or a PyRuntimeError on failure. #[allow(clippy::too_many_arguments)] #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[list[ReceiveMessage]]", imports=("collections.abc")))] fn poll_messages<'a>( @@ -1009,10 +984,7 @@ impl IggyClient { } /// Creates a new consumer group consumer. - /// Returns the consumer or a RuntimeError on failure. Raises `ValueError` if - /// `poll_interval`, `polling_retry_interval`, `init_retry_interval` or an - /// `AutoCommit` interval is negative, or if any of those except `poll_interval` - /// is zero. + /// Returns the consumer or a PyRuntimeError on failure. #[allow(clippy::too_many_arguments)] #[pyo3(signature = ( name, @@ -1088,10 +1060,8 @@ impl IggyClient { builder = builder.without_poll_interval() }; if let Some(polling_retry_interval) = polling_retry_interval { - builder = builder.polling_retry_interval(reject_zero( - py_delta_to_iggy_duration(&polling_retry_interval)?, - "polling_retry_interval", - )?) + builder = + builder.polling_retry_interval(py_delta_to_iggy_duration(&polling_retry_interval)?) } if init_retries.is_some() && init_retry_interval.is_none() { return Err(PyErr::new::( @@ -1107,10 +1077,7 @@ impl IggyClient { { builder = builder.init_retries( init_retries, - reject_zero( - py_delta_to_iggy_duration(&init_retry_interval)?, - "init_retry_interval", - )?, + py_delta_to_iggy_duration(&init_retry_interval)?, ); } if allow_replay { @@ -1142,7 +1109,7 @@ impl IggyClient { /// An awaitable that resolves to the raw response `bytes`. /// /// Raises: - /// RuntimeError: If the command cannot be sent or the server returns an error. + /// PyRuntimeError: If the command cannot be sent or the server returns an error. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[bytes]", imports=("collections.abc")))] fn send_binary_request<'a>( &self, diff --git a/foreign/python/src/config.rs b/foreign/python/src/config.rs deleted file mode 100644 index 4519c939fb..0000000000 --- a/foreign/python/src/config.rs +++ /dev/null @@ -1,423 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use iggy::prelude::{ - AutoLogin as RustAutoLogin, Credentials as RustCredentials, - TcpClientConfig as RustTcpClientConfig, TcpClientConfigBuilder, - TcpClientReconnectionConfig as RustTcpClientReconnectionConfig, -}; -use pyo3::exceptions::PyValueError; -use pyo3::prelude::*; -use pyo3::types::PyDelta; -use pyo3_stub_gen::derive::{gen_stub_pyclass, gen_stub_pymethods}; -use pyo3_stub_gen::impl_stub_type; -use secrecy::SecretString; -use std::sync::Arc; - -use crate::duration::{ - duration_repr, iggy_duration_to_py_delta, py_delta_to_iggy_duration, reject_zero, -}; - -/// The credentials replayed by the client every time it (re)connects. -/// -/// `IggyClient` only recovers a lost session when it has credentials to replay, -/// so a long-running consumer should pass one of the enabled variants. -#[gen_stub_pyclass] -#[pyclass(from_py_object)] -#[derive(Clone)] -pub struct AutoLogin { - pub(crate) inner: RustAutoLogin, -} - -#[gen_stub_pymethods] -#[pymethods] -impl AutoLogin { - /// No automatic login. `login_user()` must be called by hand after every connect. - #[staticmethod] - fn disabled() -> Self { - Self { - inner: RustAutoLogin::Disabled, - } - } - - /// Log in with the given username and password on every connect. - #[staticmethod] - fn username_password(username: String, password: String) -> Self { - Self { - inner: RustAutoLogin::Enabled(RustCredentials::UsernamePassword( - username, - SecretString::from(password), - )), - } - } - - /// Log in with the given personal access token on every connect. - #[staticmethod] - fn personal_access_token(token: String) -> Self { - Self { - inner: RustAutoLogin::Enabled(RustCredentials::PersonalAccessToken( - SecretString::from(token), - )), - } - } - - /// Whether automatic login is enabled. - #[getter] - fn enabled(&self) -> bool { - matches!(self.inner, RustAutoLogin::Enabled(_)) - } - - /// The username to log in with, or `None` for the disabled and token variants. - #[gen_stub(override_return_type(type_repr = "builtins.str | None"))] - #[getter] - fn username(&self) -> Option { - match &self.inner { - RustAutoLogin::Enabled(RustCredentials::UsernamePassword(username, _)) => { - Some(username.clone()) - } - _ => None, - } - } - - fn __repr__(&self) -> String { - match &self.inner { - RustAutoLogin::Disabled => "AutoLogin.disabled()".to_owned(), - RustAutoLogin::Enabled(RustCredentials::UsernamePassword(username, _)) => { - format!("AutoLogin.username_password({username:?}, ...)") - } - RustAutoLogin::Enabled(RustCredentials::PersonalAccessToken(_)) => { - "AutoLogin.personal_access_token(...)".to_owned() - } - } - } -} - -/// How the TCP client reconnects after the connection to the server is lost. -#[gen_stub_pyclass] -#[pyclass(from_py_object)] -#[derive(Clone)] -pub struct TcpReconnectionConfig { - pub(crate) inner: RustTcpClientReconnectionConfig, -} - -#[gen_stub_pymethods] -#[pymethods] -impl TcpReconnectionConfig { - /// Constructs a reconnection policy. - /// - /// Args: - /// enabled: Whether to reconnect at all. Defaults to enabled. - /// max_retries: Attempts before giving up, or `None` for unlimited. - /// Defaults to unlimited, which means a call awaited while the server - /// is down never returns: `connect()`, `send_messages()` and - /// `poll_messages()` all wait inside the retry loop. Set a finite - /// number for request/reply style usage, so a call fails instead. - /// interval: Delay between attempts. Defaults to 1 second. - /// reestablish_after: Cooldown before reconnecting after a previously - /// successful connection. Defaults to 5 seconds. - /// - /// Raises: - /// ValueError: If a duration is negative, if `max_retries` is outside the - /// range of an unsigned 32-bit integer, or if `interval` is zero while - /// reconnection is enabled and `max_retries` is unlimited. - #[new] - #[pyo3(signature = (*, enabled=None, max_retries=None, interval=None, reestablish_after=None))] - fn new( - #[gen_stub(override_type(type_repr = "builtins.bool | None"))] enabled: Option, - #[gen_stub(override_type(type_repr = "builtins.int | None"))] max_retries: Option, - #[gen_stub(override_type(type_repr = "datetime.timedelta | None", imports=("datetime")))] - interval: Option>, - #[gen_stub(override_type(type_repr = "datetime.timedelta | None", imports=("datetime")))] - reestablish_after: Option>, - ) -> PyResult { - let defaults = RustTcpClientReconnectionConfig::default(); - let enabled = enabled.unwrap_or(defaults.enabled); - let max_retries = max_retries - .map(|max_retries| { - u32::try_from(max_retries).map_err(|_| { - PyValueError::new_err(format!( - "'max_retries' must be between 0 and {}", - u32::MAX - )) - }) - }) - .transpose()?; - let interval = interval - .as_ref() - .map(py_delta_to_iggy_duration) - .transpose()? - .unwrap_or(defaults.interval); - // Unlimited retries at a zero interval reconnect in a continuous loop; - // a zero interval with a retry cap is a legitimate fast-retry policy, and - // with reconnection off the interval is never read at all. - if enabled && interval.is_zero() && max_retries.is_none() { - return Err(PyValueError::new_err( - "'interval' must not be zero unless 'max_retries' is set", - )); - } - Ok(Self { - inner: RustTcpClientReconnectionConfig { - enabled, - max_retries, - interval, - reestablish_after: reestablish_after - .as_ref() - .map(py_delta_to_iggy_duration) - .transpose()? - .unwrap_or(defaults.reestablish_after), - }, - }) - } - - #[getter] - fn enabled(&self) -> bool { - self.inner.enabled - } - - #[gen_stub(override_return_type(type_repr = "builtins.int | None"))] - #[getter] - fn max_retries(&self) -> Option { - self.inner.max_retries - } - - #[gen_stub(override_return_type(type_repr = "datetime.timedelta", imports=("datetime")))] - #[getter] - fn interval<'a>(&self, py: Python<'a>) -> PyResult> { - iggy_duration_to_py_delta(py, self.inner.interval) - } - - #[gen_stub(override_return_type(type_repr = "datetime.timedelta", imports=("datetime")))] - #[getter] - fn reestablish_after<'a>(&self, py: Python<'a>) -> PyResult> { - iggy_duration_to_py_delta(py, self.inner.reestablish_after) - } - - fn __repr__(&self) -> String { - let max_retries = match self.inner.max_retries { - Some(max_retries) => max_retries.to_string(), - None => "None".to_owned(), - }; - format!( - "TcpReconnectionConfig(enabled={}, max_retries={max_retries}, interval={}, reestablish_after={})", - python_bool(self.inner.enabled), - duration_repr(self.inner.interval), - duration_repr(self.inner.reestablish_after), - ) - } -} - -/// Configuration for the TCP transport, accepted by `IggyClient(...)`. -/// -/// Every field is keyword-only and optional. -#[gen_stub_pyclass] -#[pyclass(from_py_object)] -#[derive(Clone)] -pub struct TcpConfig { - inner: Arc, -} - -impl TcpConfig { - /// The configuration in the shape `TcpClient::create` expects. - pub(crate) fn client_config(&self) -> Arc { - self.inner.clone() - } -} - -#[gen_stub_pymethods] -#[pymethods] -impl TcpConfig { - /// Constructs a TCP configuration. - /// - /// Args: - /// server_address: `host:port` of the Iggy server. Defaults to `127.0.0.1:8090`. - /// auto_login: Credentials replayed on every connect. Defaults to `AutoLogin.disabled()`. - /// reconnection: Reconnection policy. Defaults to `TcpReconnectionConfig()`. - /// heartbeat_interval: Interval of heartbeats sent by the client. Defaults to 5 seconds. - /// tls_enabled: Whether to connect over TLS. Defaults to disabled. - /// tls_domain: Domain to validate the certificate against. Empty means it is - /// taken from `server_address`. - /// tls_ca_file: Path to the CA file for TLS. Read only when `tls_enabled` - /// and `tls_validate_certificate` are both on; with either one off it - /// is kept but never consulted, so pairing it with - /// `tls_validate_certificate=False` pins nothing. - /// tls_validate_certificate: Whether to validate the server certificate. - /// Defaults to validating. Disabling this accepts any certificate the - /// server presents, including self-signed and mismatched ones, and - /// takes precedence over `tls_ca_file`; intended for local development - /// only. - /// nodelay: Disable the Nagle algorithm for the TCP socket. Defaults to - /// leaving it on. - /// - /// Raises: - /// ValueError: If `server_address` is not a valid `host:port` pair, if a - /// duration is negative, or if `heartbeat_interval` is zero. - #[new] - #[pyo3(signature = ( - *, - server_address=None, - auto_login=None, - reconnection=None, - heartbeat_interval=None, - tls_enabled=None, - tls_domain=None, - tls_ca_file=None, - tls_validate_certificate=None, - nodelay=None, - ))] - #[allow(clippy::too_many_arguments)] - fn new( - #[gen_stub(override_type(type_repr = "builtins.str | None"))] server_address: Option< - String, - >, - #[gen_stub(override_type(type_repr = "AutoLogin | None"))] auto_login: Option, - #[gen_stub(override_type(type_repr = "TcpReconnectionConfig | None"))] reconnection: Option< - TcpReconnectionConfig, - >, - #[gen_stub(override_type(type_repr = "datetime.timedelta | None", imports=("datetime")))] - heartbeat_interval: Option>, - #[gen_stub(override_type(type_repr = "builtins.bool | None"))] tls_enabled: Option, - #[gen_stub(override_type(type_repr = "builtins.str | None"))] tls_domain: Option, - #[gen_stub(override_type(type_repr = "builtins.str | None"))] tls_ca_file: Option, - #[gen_stub(override_type(type_repr = "builtins.bool | None"))] - tls_validate_certificate: Option, - #[gen_stub(override_type(type_repr = "builtins.bool | None"))] nodelay: Option, - ) -> PyResult { - // The builder starts from `TcpClientConfig::default()`, and its `build()` - // trims and validates the address whether or not one was set here. - let mut builder = TcpClientConfigBuilder::new(); - if let Some(server_address) = server_address { - builder = builder.with_server_address(server_address); - } - let mut inner = builder - .build() - .map_err(|e| PyValueError::new_err(e.to_string()))?; - if let Some(auto_login) = auto_login { - inner.auto_login = auto_login.inner; - } - if let Some(reconnection) = reconnection { - inner.reconnection = reconnection.inner; - } - if let Some(heartbeat_interval) = heartbeat_interval { - inner.heartbeat_interval = reject_zero( - py_delta_to_iggy_duration(&heartbeat_interval)?, - "heartbeat_interval", - )?; - } - if let Some(tls_enabled) = tls_enabled { - inner.tls_enabled = tls_enabled; - } - if let Some(tls_domain) = tls_domain { - inner.tls_domain = tls_domain; - } - if tls_ca_file.is_some() { - inner.tls_ca_file = tls_ca_file; - } - if let Some(tls_validate_certificate) = tls_validate_certificate { - inner.tls_validate_certificate = tls_validate_certificate; - } - if let Some(nodelay) = nodelay { - inner.nodelay = nodelay; - } - - Ok(Self { - inner: Arc::new(inner), - }) - } - - #[getter] - fn server_address(&self) -> String { - self.inner.server_address.clone() - } - - #[getter] - fn auto_login(&self) -> AutoLogin { - AutoLogin { - inner: self.inner.auto_login.clone(), - } - } - - #[getter] - fn reconnection(&self) -> TcpReconnectionConfig { - TcpReconnectionConfig { - inner: self.inner.reconnection.clone(), - } - } - - #[gen_stub(override_return_type(type_repr = "datetime.timedelta", imports=("datetime")))] - #[getter] - fn heartbeat_interval<'a>(&self, py: Python<'a>) -> PyResult> { - iggy_duration_to_py_delta(py, self.inner.heartbeat_interval) - } - - #[getter] - fn tls_enabled(&self) -> bool { - self.inner.tls_enabled - } - - #[getter] - fn tls_domain(&self) -> String { - self.inner.tls_domain.clone() - } - - #[gen_stub(override_return_type(type_repr = "builtins.str | None"))] - #[getter] - fn tls_ca_file(&self) -> Option { - self.inner.tls_ca_file.clone() - } - - #[getter] - fn tls_validate_certificate(&self) -> bool { - self.inner.tls_validate_certificate - } - - #[getter] - fn nodelay(&self) -> bool { - self.inner.nodelay - } - - fn __repr__(&self) -> String { - let tls_ca_file = match &self.inner.tls_ca_file { - Some(tls_ca_file) => format!("{tls_ca_file:?}"), - None => "None".to_owned(), - }; - format!( - "TcpConfig(server_address={:?}, auto_login={}, reconnection={}, heartbeat_interval={}, tls_enabled={}, tls_domain={:?}, tls_ca_file={tls_ca_file}, tls_validate_certificate={}, nodelay={})", - self.inner.server_address, - self.auto_login().__repr__(), - self.reconnection().__repr__(), - duration_repr(self.inner.heartbeat_interval), - python_bool(self.inner.tls_enabled), - self.inner.tls_domain, - python_bool(self.inner.tls_validate_certificate), - python_bool(self.inner.nodelay), - ) - } -} - -fn python_bool(value: bool) -> &'static str { - if value { "True" } else { "False" } -} - -/// What `IggyClient(...)` accepts: a bare `host:port` or a full `TcpConfig`. -#[derive(FromPyObject)] -pub enum PyClientConfig { - #[pyo3(transparent)] - Config(TcpConfig), - #[pyo3(transparent, annotation = "str")] - ServerAddress(String), -} -impl_stub_type!(PyClientConfig = TcpConfig | String); diff --git a/foreign/python/src/consumer.rs b/foreign/python/src/consumer.rs index 6a6e69a877..4d64fc626c 100644 --- a/foreign/python/src/consumer.rs +++ b/foreign/python/src/consumer.rs @@ -16,6 +16,7 @@ // under the License. use std::sync::Arc; +use std::time::Duration; use futures::StreamExt; use iggy::consumer_ext::{IggyConsumerMessageExt, MessageConsumer}; @@ -26,8 +27,8 @@ use iggy::prelude::{ ConsumerGroupMember as RustConsumerGroupMember, IggyConsumer as RustIggyConsumer, IggyDuration, IggyError, ReceivedMessage, }; -use pyo3::exceptions::PyStopAsyncIteration; -use pyo3::types::PyDelta; +use pyo3::exceptions::{PyStopAsyncIteration, PyValueError}; +use pyo3::types::{PyDelta, PyDeltaAccess}; use pyo3::prelude::*; use pyo3_async_runtimes::TaskLocals; @@ -38,12 +39,12 @@ use tokio::sync::Mutex; use tokio::sync::oneshot::Sender; use tokio::task::JoinHandle; -use crate::duration::{py_delta_to_iggy_duration, reject_zero}; use crate::identifier::PyIdentifier; use crate::receive_message::ReceiveMessage; /// A Python class representing the Iggy consumer. -/// It provides asynchronous functionality through the contained runtime. +/// It wraps the RustIggyConsumer and provides asynchronous functionality +/// through the contained runtime. #[gen_stub_pyclass] #[pyclass] pub struct IggyConsumer { @@ -93,7 +94,8 @@ impl IggyConsumer { /// Stores the provided offset for the provided partition id or if none is specified /// uses the current partition id for the consumer group. - /// Raises `RuntimeError` if the operation fails. + /// Returns `Ok(())` if the server responds successfully, or a `PyRuntimeError` + /// if the operation fails. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn store_offset<'a>( &self, @@ -114,7 +116,8 @@ impl IggyConsumer { /// Deletes the offset for the provided partition id or if none is specified /// uses the current partition id for the consumer group. - /// Raises `RuntimeError` if the operation fails. + /// Returns `Ok(())` if the server responds successfully, or a `PyRuntimeError` + /// if the operation fails. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn delete_offset<'a>( &self, @@ -134,7 +137,7 @@ impl IggyConsumer { /// Asynchronously iterate over `ReceiveMessage`s. /// Returns an async iterator that raises `StopAsyncIteration` when no more messages are available - /// or a `RuntimeError` on failure. + /// or a `PyRuntimeError` on failure. /// Note: This method does not currently support `AutoCommit.After`. /// For `AutoCommit.IntervalOrAfter(datetime.timedelta, AutoCommitAfter)`, /// only the interval part is applied; the `after` mode is ignored. @@ -146,7 +149,7 @@ impl IggyConsumer { } /// Consumes messages continuously using a callback function and an optional `asyncio.Event` for signaling shutdown. - /// Returns an awaitable that completes when shutdown is signaled or a RuntimeError on failure. + /// Returns an awaitable that completes when shutdown is signaled or a PyRuntimeError on failure. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn consume_messages<'a>( &self, @@ -432,12 +435,17 @@ impl TryFrom<&AutoCommit> for RustAutoCommit { fn try_from(val: &AutoCommit) -> PyResult { Ok(match val { AutoCommit::Disabled() => RustAutoCommit::Disabled, - AutoCommit::Interval(delta) => RustAutoCommit::Interval(auto_commit_interval(delta)?), + AutoCommit::Interval(delta) => { + let duration = py_delta_to_iggy_duration(delta)?; + RustAutoCommit::Interval(duration) + } AutoCommit::IntervalOrWhen(delta, when) => { - RustAutoCommit::IntervalOrWhen(auto_commit_interval(delta)?, when.into()) + let duration = py_delta_to_iggy_duration(delta)?; + RustAutoCommit::IntervalOrWhen(duration, when.into()) } AutoCommit::IntervalOrAfter(delta, after) => { - RustAutoCommit::IntervalOrAfter(auto_commit_interval(delta)?, after.into()) + let duration = py_delta_to_iggy_duration(delta)?; + RustAutoCommit::IntervalOrAfter(duration, after.into()) } AutoCommit::When(when) => RustAutoCommit::When(when.into()), AutoCommit::After(after) => RustAutoCommit::After(after.into()), @@ -445,10 +453,6 @@ impl TryFrom<&AutoCommit> for RustAutoCommit { } } -fn auto_commit_interval(delta: &Py) -> PyResult { - reject_zero(py_delta_to_iggy_duration(delta)?, "AutoCommit interval") -} - /// The auto-commit mode for storing the offset on the server. #[derive(Debug, PartialEq, Copy, Clone)] #[gen_stub_pyclass_complex_enum(skip_stub_type)] @@ -514,3 +518,20 @@ impl PyStubType for AutoCommitAfter { TypeInfo::unqualified("AutoCommitAfter") } } + +pub fn py_delta_to_iggy_duration(delta1: &Py) -> PyResult { + Python::attach(|py| { + let delta = delta1.bind(py); + let total_seconds = i64::from(delta.get_days()) * 86_400 + i64::from(delta.get_seconds()); + if total_seconds < 0 { + return Err(PyValueError::new_err( + "duration must not be negative".to_string(), + )); + } + let nanos = (delta.get_microseconds() * 1_000) as u32; + Ok(IggyDuration::new(Duration::new( + total_seconds as u64, + nanos, + ))) + }) +} diff --git a/foreign/python/src/duration.rs b/foreign/python/src/duration.rs deleted file mode 100644 index 9b2b419fe5..0000000000 --- a/foreign/python/src/duration.rs +++ /dev/null @@ -1,65 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use iggy::prelude::IggyDuration; -use pyo3::exceptions::PyValueError; -use pyo3::prelude::*; -use pyo3::types::PyDelta; -use std::time::Duration; - -pub fn py_delta_to_iggy_duration(delta: &Py) -> PyResult { - Python::attach(|py| { - // The value is already a timedelta, so a negative one is the only failure - // left to map, and the Python surface must not name Rust types. - delta - .bind(py) - .extract::() - .map(IggyDuration::from) - .map_err(|_| PyValueError::new_err("duration must not be negative")) - }) -} - -pub fn iggy_duration_to_py_delta( - py: Python<'_>, - duration: IggyDuration, -) -> PyResult> { - duration.get_duration().into_pyobject(py) -} - -/// Renders a duration the way it would be written in Python, so that a `__repr__` -/// built from it can be pasted back into a constructor. -pub fn duration_repr(duration: IggyDuration) -> String { - // Read the std duration, whose micros are u128: `IggyDuration::as_micros()` - // truncates to u64, which a timedelta near the Python maximum overflows. - let micros = duration.get_duration().as_micros(); - if micros.is_multiple_of(1_000_000) { - format!("datetime.timedelta(seconds={})", micros / 1_000_000) - } else { - format!("datetime.timedelta(microseconds={micros})") - } -} - -/// Rejects a zero duration for parameters where zero means an unthrottled loop -/// rather than "disabled". -pub fn reject_zero(duration: IggyDuration, parameter: &str) -> PyResult { - if duration.is_zero() { - return Err(PyValueError::new_err(format!( - "'{parameter}' must not be zero" - ))); - } - Ok(duration) -} diff --git a/foreign/python/src/lib.rs b/foreign/python/src/lib.rs index 9c7f1efaa7..985476ff74 100644 --- a/foreign/python/src/lib.rs +++ b/foreign/python/src/lib.rs @@ -16,9 +16,7 @@ // under the License. pub mod client; -mod config; mod consumer; -mod duration; mod identifier; mod permissions; mod receive_message; @@ -29,7 +27,6 @@ mod user; mod user_headers; use client::IggyClient; -use config::{AutoLogin, TcpConfig, TcpReconnectionConfig}; use consumer::{ AutoCommit, AutoCommitAfter, AutoCommitWhen, ConsumerGroup, ConsumerGroupDetails, ConsumerGroupMember, IggyConsumer, ReceiveMessageIterator, @@ -43,7 +40,7 @@ use topic::{IggyExpiry, MaxTopicSize, Partition, Topic, TopicDetails}; use user::{UserInfo, UserInfoDetails, UserStatus}; use user_headers::{HeaderKey, HeaderValue, UserHeaders}; -/// Python client for Apache Iggy, the persistent message streaming platform. +/// A Python module implemented in Rust. #[pymodule] fn apache_iggy(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; @@ -51,9 +48,6 @@ fn apache_iggy(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/foreign/python/src/receive_message.rs b/foreign/python/src/receive_message.rs index ddb14e6233..ecabb6cf22 100644 --- a/foreign/python/src/receive_message.rs +++ b/foreign/python/src/receive_message.rs @@ -24,7 +24,7 @@ use pyo3_stub_gen::derive::{gen_stub_pyclass, gen_stub_pyclass_complex_enum, gen use crate::user_headers::{UserHeaders, rust_user_headers_to_py}; /// A Python class representing a received message. -/// It provides access to the message payload and offset. +/// This class wraps a Rust message, allowing for access to its payload and offset from Python. #[pyclass] #[gen_stub_pyclass] pub struct ReceiveMessage { diff --git a/foreign/python/src/send_message.rs b/foreign/python/src/send_message.rs index bfa3cdd671..e63b1a2f78 100644 --- a/foreign/python/src/send_message.rs +++ b/foreign/python/src/send_message.rs @@ -30,6 +30,8 @@ use pyo3_stub_gen::{ use crate::user_headers::py_user_headers_to_rust; /// A Python class representing a message to be sent. +/// This class wraps a Rust message meant for sending, facilitating +/// the creation of such messages from Python and their subsequent use in Rust. #[pyclass(from_py_object)] #[gen_stub_pyclass] pub struct SendMessage { diff --git a/foreign/python/src/topic.rs b/foreign/python/src/topic.rs index 40a7db228e..178f90dd5e 100644 --- a/foreign/python/src/topic.rs +++ b/foreign/python/src/topic.rs @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +use std::time::Duration; + use iggy::prelude::{ IggyByteSize, IggyExpiry as RustIggyExpiry, MaxTopicSize as RustMaxTopicSize, Partition as RustPartition, Topic as RustTopic, TopicDetails as RustTopicDetails, @@ -24,7 +26,7 @@ use pyo3::prelude::*; use pyo3::types::PyDelta; use pyo3_stub_gen::derive::{gen_stub_pyclass, gen_stub_pyclass_complex_enum, gen_stub_pymethods}; -use crate::duration::{iggy_duration_to_py_delta, py_delta_to_iggy_duration}; +use crate::consumer::py_delta_to_iggy_duration; /// The expiry of the messages in a topic. #[gen_stub_pyclass_complex_enum] @@ -53,14 +55,7 @@ impl TryFrom for IggyExpiry { Ok(match expiry { RustIggyExpiry::ServerDefault => IggyExpiry::ServerDefault(), RustIggyExpiry::ExpireDuration(duration) => IggyExpiry::ExpireDuration { - duration: Python::attach(|py| { - iggy_duration_to_py_delta(py, duration).map(|delta| delta.unbind()) - }) - .map_err(|err| { - PyValueError::new_err(format!( - "topic message expiry duration does not fit within timedelta bounds: {err}" - )) - })?, + duration: iggy_duration_to_py_delta(duration.get_duration())?, }, RustIggyExpiry::NeverExpire => IggyExpiry::NeverExpire(), }) @@ -98,6 +93,26 @@ impl TryFrom<&IggyExpiry> for RustIggyExpiry { } } +fn iggy_duration_to_py_delta(duration: Duration) -> PyResult> { + let days = duration.as_secs() / 86_400; + let secs_of_day = duration.as_secs() % 86_400; + Python::attach(|py| { + PyDelta::new( + py, + days as i32, + secs_of_day as i32, + duration.subsec_micros() as i32, + true, + ) + .map(|delta| delta.unbind()) + .map_err(|err| { + PyValueError::new_err(format!( + "topic message expiry duration does not fit within timedelta bounds: {err}" + )) + }) + }) +} + /// The maximum size of a topic. #[gen_stub_pyclass_complex_enum] #[pyclass] diff --git a/foreign/python/tests/conftest.py b/foreign/python/tests/conftest.py index 3ab97065f5..aa54ff50d8 100644 --- a/foreign/python/tests/conftest.py +++ b/foreign/python/tests/conftest.py @@ -131,10 +131,5 @@ def pytest_collection_modifyitems(items): path.name for path in Path(__file__).parent.glob("test_*.py") } for item in items: - # Tests explicitly marked as unit need no server; auto-marking them - # integration too would make `-m "not integration"` unable to select - # them. - if item.get_closest_marker("unit"): - continue if any(module in item.nodeid for module in integration_modules): item.add_marker(pytest.mark.integration) diff --git a/foreign/python/tests/test_client_config.py b/foreign/python/tests/test_client_config.py deleted file mode 100644 index 78df23459c..0000000000 --- a/foreign/python/tests/test_client_config.py +++ /dev/null @@ -1,459 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -""" -Tests for the TCP client configuration surface. - -`TcpConfig`, `TcpReconnectionConfig` and `AutoLogin` mirror the Rust SDK -types, so most of these assert that a value set from Python survives to the -getters and that unset fields fall back to the Rust defaults. The last class -proves the point of the configuration: with `auto_login` set, credentials are -replayed on connect and no manual `login_user()` is needed. -""" - -import ast -from collections.abc import Callable -from datetime import timedelta - -import pytest - -from apache_iggy import ( - AutoCommit, - AutoCommitAfter, - AutoCommitWhen, - AutoLogin, - IggyClient, - IggyExpiry, - TcpConfig, - TcpReconnectionConfig, -) - -from .utils import get_server_config, wait_for_ping, wait_for_server - - -@pytest.mark.unit -class TestAutoLogin: - """Test the credentials carried into the client.""" - - def test_disabled_has_no_username(self): - """Test that the disabled variant carries no credentials.""" - auto_login = AutoLogin.disabled() - - assert auto_login.enabled is False - assert auto_login.username is None - - def test_username_password_exposes_username_only(self): - """Test that the username is readable back but the password is not.""" - auto_login = AutoLogin.username_password("iggy", "secret") - - assert auto_login.enabled is True - assert auto_login.username == "iggy" - assert "secret" not in repr(auto_login) - - def test_personal_access_token_hides_the_token(self): - """Test that a token login exposes neither a username nor the token.""" - auto_login = AutoLogin.personal_access_token("secret-token") - - assert auto_login.enabled is True - assert auto_login.username is None - assert "secret-token" not in repr(auto_login) - - -@pytest.mark.unit -class TestTcpReconnectionConfig: - """Test the reconnection policy.""" - - def test_defaults_match_the_rust_sdk(self): - """Test that an unconfigured policy reconnects forever, one second apart.""" - reconnection = TcpReconnectionConfig() - - assert reconnection.enabled is True - assert reconnection.max_retries is None - assert reconnection.interval == timedelta(seconds=1) - assert reconnection.reestablish_after == timedelta(seconds=5) - - def test_every_field_round_trips(self): - """Test that each configured field is readable back unchanged.""" - reconnection = TcpReconnectionConfig( - enabled=False, - max_retries=10, - interval=timedelta(milliseconds=250), - reestablish_after=timedelta(seconds=30), - ) - - assert reconnection.enabled is False - assert reconnection.max_retries == 10 - assert reconnection.interval == timedelta(milliseconds=250) - assert reconnection.reestablish_after == timedelta(seconds=30) - - def test_arguments_are_keyword_only(self): - """Test that the adjacent flags cannot be passed positionally.""" - with pytest.raises(TypeError): - # pyrefly: ignore # bad-argument-count - TcpReconnectionConfig(True) - - @pytest.mark.parametrize( - "construct", - [ - lambda duration: TcpReconnectionConfig(interval=duration), - lambda duration: TcpReconnectionConfig(reestablish_after=duration), - ], - ids=["interval", "reestablish_after"], - ) - @pytest.mark.parametrize( - "negative", - [timedelta(microseconds=-1), timedelta(seconds=-1), timedelta(days=-1)], - ) - def test_negative_duration_is_rejected( - self, - construct: Callable[[timedelta], TcpReconnectionConfig], - negative: timedelta, - ): - """Test that a negative duration fails at construction, not at connect.""" - with pytest.raises(ValueError, match="negative"): - construct(negative) - - @pytest.mark.parametrize("out_of_range", [-1, 2**32]) - def test_out_of_range_max_retries_is_rejected(self, out_of_range: int): - """Test that a retry count outside the wire range names the argument. - - The conversion pyo3 does on its own raises OverflowError, which is not a - ValueError and so escapes the handler a caller wraps construction in. - """ - with pytest.raises(ValueError, match="max_retries"): - TcpReconnectionConfig(max_retries=out_of_range) - - def test_zero_reestablish_after_is_allowed(self): - """Test that a zero cooldown is legal and readable back.""" - reconnection = TcpReconnectionConfig(reestablish_after=timedelta(0)) - - assert reconnection.reestablish_after == timedelta(0) - - def test_zero_interval_is_allowed_with_bounded_retries(self): - """Test that a zero interval is legal as a bounded fast-retry policy.""" - reconnection = TcpReconnectionConfig(interval=timedelta(0), max_retries=5) - - assert reconnection.interval == timedelta(0) - - def test_zero_interval_is_allowed_when_reconnection_is_disabled(self): - """Test that a zero interval is legal when nothing ever reads it.""" - reconnection = TcpReconnectionConfig(enabled=False, interval=timedelta(0)) - - assert reconnection.interval == timedelta(0) - - def test_zero_interval_with_unlimited_retries_is_rejected(self): - """Test that the combination that reconnects in a continuous loop fails.""" - with pytest.raises(ValueError, match="zero"): - TcpReconnectionConfig(interval=timedelta(0)) - - def test_very_long_interval_round_trips(self): - """Test that an interval beyond 68 years survives the i32 boundary.""" - reconnection = TcpReconnectionConfig(interval=timedelta(days=30_000)) - - assert reconnection.interval == timedelta(days=30_000) - - def test_maximum_interval_round_trips(self): - """Test that the largest timedelta survives the day conversion.""" - reconnection = TcpReconnectionConfig(interval=timedelta(days=999_999_999)) - - assert reconnection.interval == timedelta(days=999_999_999) - - -@pytest.mark.unit -class TestTcpConfig: - """Test the transport configuration.""" - - def test_defaults_match_the_rust_sdk(self): - """Test that an unconfigured transport matches the Rust SDK defaults.""" - config = TcpConfig() - - assert config.server_address == "127.0.0.1:8090" - assert config.auto_login.enabled is False - assert config.reconnection.enabled is True - assert config.heartbeat_interval == timedelta(seconds=5) - assert config.tls_enabled is False - assert config.tls_domain == "" - assert config.tls_ca_file is None - assert config.tls_validate_certificate is True - assert config.nodelay is False - - def test_every_field_round_trips(self): - """Test that each configured field is readable back unchanged.""" - config = TcpConfig( - server_address="localhost:8090", - auto_login=AutoLogin.username_password("iggy", "iggy"), - reconnection=TcpReconnectionConfig(max_retries=3), - heartbeat_interval=timedelta(seconds=15), - tls_enabled=True, - tls_domain="localhost", - tls_ca_file="ca.pem", - tls_validate_certificate=False, - nodelay=True, - ) - - assert config.server_address == "localhost:8090" - assert config.auto_login.username == "iggy" - assert config.reconnection.max_retries == 3 - assert config.heartbeat_interval == timedelta(seconds=15) - assert config.tls_enabled is True - assert config.tls_domain == "localhost" - assert config.tls_ca_file == "ca.pem" - assert config.tls_validate_certificate is False - assert config.nodelay is True - - def test_arguments_are_keyword_only(self): - """Test that the address cannot be passed positionally.""" - with pytest.raises(TypeError): - # pyrefly: ignore # bad-argument-count - TcpConfig("127.0.0.1:8090") - - def test_repr_hides_the_password(self): - """Test that the password does not leak through repr.""" - config = TcpConfig(auto_login=AutoLogin.username_password("iggy", "secret")) - - assert "secret" not in repr(config) - - def test_repr_shows_every_field_as_python(self): - """Test that repr covers the TLS fields and parses as Python. - - The TLS fields are the ones a handshake is debugged with, and a repr is - only worth printing if it can be pasted back into a constructor. - """ - config = TcpConfig( - heartbeat_interval=timedelta(seconds=15), - tls_enabled=True, - tls_domain="localhost", - tls_ca_file="ca.pem", - tls_validate_certificate=False, - nodelay=True, - ) - - printed = repr(config) - - assert 'tls_domain="localhost"' in printed - assert 'tls_ca_file="ca.pem"' in printed - assert "tls_validate_certificate=False" in printed - assert "nodelay=True" in printed - assert "heartbeat_interval=datetime.timedelta(seconds=15)" in printed - ast.parse(printed) - - @pytest.mark.parametrize( - "invalid_address", - ["", "127.0.0.1", "127.0.0.1:not-a-port", "127.0.0.1:70000", "::1:8090"], - ) - def test_invalid_server_address_is_rejected(self, invalid_address: str): - """Test that a malformed address fails at construction, not at connect.""" - with pytest.raises(ValueError): - TcpConfig(server_address=invalid_address) - - def test_negative_heartbeat_interval_is_rejected(self): - """Test that a negative heartbeat interval fails at construction.""" - with pytest.raises(ValueError, match="negative"): - TcpConfig(heartbeat_interval=timedelta(seconds=-3)) - - def test_zero_heartbeat_interval_is_rejected(self): - """Test that a zero heartbeat interval fails at construction. - - Nothing downstream reads zero as "disabled"; it heartbeats in a - continuous loop for as long as the client lives. - """ - with pytest.raises(ValueError, match="zero"): - TcpConfig(heartbeat_interval=timedelta(0)) - - -@pytest.mark.unit -class TestClientConstruction: - """Test what the client constructor accepts.""" - - def test_accepts_a_config(self): - """Test that a client can be built from a config object.""" - assert IggyClient(TcpConfig(server_address="127.0.0.1:8090")) is not None - - def test_accepts_an_address(self): - """Test that the address form still works.""" - assert IggyClient("127.0.0.1:8090") is not None - - def test_accepts_nothing(self): - """Test that the default address is used when no argument is given.""" - assert IggyClient() is not None - - def test_rejects_an_invalid_address(self): - """Test that a malformed address is rejected.""" - with pytest.raises(RuntimeError): - IggyClient("nonsense") - - def test_negative_message_expiry_is_rejected(self): - """Test that the negative-duration rule reaches create_topic. - - The check runs at the call, before any I/O. - """ - client = IggyClient() - - with pytest.raises(ValueError, match="negative"): - client.create_topic( - stream="stream", - name="topic", - partitions_count=1, - message_expiry=IggyExpiry.ExpireDuration(timedelta(seconds=-1)), - ) - - @pytest.mark.parametrize( - "interval_kwargs", - [ - {"polling_retry_interval": timedelta(0)}, - {"init_retries": 3, "init_retry_interval": timedelta(0)}, - {"auto_commit": AutoCommit.Interval(timedelta(0))}, - { - "auto_commit": AutoCommit.IntervalOrWhen( - timedelta(0), AutoCommitWhen.PollingMessages() - ) - }, - { - "auto_commit": AutoCommit.IntervalOrAfter( - timedelta(0), AutoCommitAfter.ConsumingEachMessage() - ) - }, - ], - ids=[ - "polling_retry_interval", - "init_retry_interval", - "auto_commit_interval", - "auto_commit_interval_or_when", - "auto_commit_interval_or_after", - ], - ) - def test_zero_consumer_interval_is_rejected(self, interval_kwargs: dict): - """Test that a zero consumer interval fails at the call. - - Zero spins the retry loop, floods the server with offset stores, or - panics inside the runtime timer, and none of those name the argument - that caused it. - """ - client = IggyClient() - - with pytest.raises(ValueError, match="zero"): - client.consumer_group( - name="group", - stream="stream", - topic="topic", - **interval_kwargs, - ) - - def test_zero_poll_interval_is_allowed(self): - """Test that a zero poll interval passes validation. - - Zero there means "do not wait before polling" and is short-circuited - before the sleep, unlike the retry intervals. Reaching the awaitable is - what proves it: building one without a running loop is the next failure, - and a rejected value would have raised ValueError first. - """ - client = IggyClient() - - with pytest.raises(RuntimeError): - client.consumer_group( - name="group", - stream="stream", - topic="topic", - poll_interval=timedelta(0), - ) - - -@pytest.mark.integration -class TestAutoLoginAgainstServer: - """Test that configured credentials are actually replayed on connect.""" - - @pytest.mark.asyncio - async def test_auto_login_authenticates_without_login_user(self, unique_name): - """Test that a privileged call succeeds without a manual login_user().""" - host, port = get_server_config() - wait_for_server(host, port) - - client = IggyClient( - TcpConfig( - server_address=f"{host}:{port}", - auto_login=AutoLogin.username_password("iggy", "iggy"), - ) - ) - await client.connect() - await wait_for_ping(client) - - stream_name = unique_name() - await client.create_stream(stream_name) - assert await client.get_stream(stream_name) is not None - - @pytest.mark.asyncio - async def test_without_auto_login_a_privileged_call_is_unauthenticated( - self, unique_name - ): - """Test that the same call fails when no credentials are configured.""" - host, port = get_server_config() - wait_for_server(host, port) - - client = IggyClient(TcpConfig(server_address=f"{host}:{port}")) - await client.connect() - await wait_for_ping(client) - - with pytest.raises(RuntimeError): - await client.create_stream(unique_name()) - - @pytest.mark.asyncio - async def test_config_and_connection_string_both_authenticate(self, unique_name): - """Test that either form of configuring credentials logs the client in. - - The reconnection policy is set on both sides to mirror the connection - string, but the client exposes no getter for it, so this asserts only - what is observable: both clients reach an authenticated session. - """ - host, port = get_server_config() - wait_for_server(host, port) - - from_config = IggyClient( - TcpConfig( - server_address=f"{host}:{port}", - auto_login=AutoLogin.username_password("iggy", "iggy"), - reconnection=TcpReconnectionConfig( - max_retries=3, interval=timedelta(seconds=1) - ), - ) - ) - from_string = IggyClient.from_connection_string( - f"iggy+tcp://iggy:iggy@{host}:{port}" - "?reconnection_retries=3&reconnection_interval=1s" - ) - - stream_name = unique_name() - for client in (from_config, from_string): - await client.connect() - await wait_for_ping(client) - assert await client.get_stream(stream_name) is None - - @pytest.mark.asyncio - async def test_wrong_auto_login_credentials_fail(self): - """Test that bad configured credentials surface as a connect failure.""" - host, port = get_server_config() - wait_for_server(host, port) - - client = IggyClient( - TcpConfig( - server_address=f"{host}:{port}", - auto_login=AutoLogin.username_password("iggy", "invalid-password"), - reconnection=TcpReconnectionConfig(enabled=False), - ) - ) - - with pytest.raises(RuntimeError): - await client.connect() diff --git a/foreign/python/tests/test_tls.py b/foreign/python/tests/test_tls.py index f914e41466..fd8ecef7fb 100644 --- a/foreign/python/tests/test_tls.py +++ b/foreign/python/tests/test_tls.py @@ -27,8 +27,10 @@ - testcontainers[docker] installed (in [testing-docker] extras) - CA certificate available at core/certs/iggy_ca_cert.pem - server image built locally (or IGGY_SERVER_DOCKER_IMAGE set): - docker build -f core/server/Dockerfile --target runtime-prebuilt \ - --build-arg PREBUILT_IGGY_SERVER=target/debug/iggy-server \ + TODO(hubcio): change to iggy-server once legacy server is removed + (core/server has VSR support) + docker build -f core/server-ng/Dockerfile --target runtime-prebuilt \ + --build-arg PREBUILT_IGGY_SERVER_NG=target/debug/iggy-server-ng \ --build-arg PREBUILT_IGGY_CLI=target/debug/iggy \ -t iggy-server:local . """ @@ -71,7 +73,7 @@ def tls_container(): ) container.start() # Wait for the server to be ready inside the container - wait_for_logs(container, "server running", timeout=60) + wait_for_logs(container, "server-ng running", timeout=60) yield container container.stop() diff --git a/foreign/python/tests/test_topic.py b/foreign/python/tests/test_topic.py index 46434525ef..417f24a385 100644 --- a/foreign/python/tests/test_topic.py +++ b/foreign/python/tests/test_topic.py @@ -24,6 +24,7 @@ from .utils import ( get_server_config, wait_for_ping, + wait_for_purged_topic, wait_for_server, ) @@ -1383,10 +1384,9 @@ async def test_purge_topic_clears_messages_but_keeps_topic( await iggy_client.purge_topic(stream_name, topic_name) - after = await iggy_client.get_topic(stream_name, topic_name) - assert after is not None - assert after.messages_count == 0 - assert after.size == 0 + # The purge ack precedes the asynchronous partition prune, so poll + # until the stats catch up instead of asserting the counts directly. + after = await wait_for_purged_topic(iggy_client, stream_name, topic_name) # Purging clears messages and size only; topic config is unchanged. assert after.id == before.id assert after.name == before.name @@ -1437,10 +1437,7 @@ async def test_purge_topic_is_idempotent_when_called_repeatedly( await iggy_client.purge_topic(stream_name, topic_name) await iggy_client.purge_topic(stream_name, topic_name) - topic = await iggy_client.get_topic(stream_name, topic_name) - assert topic is not None - assert topic.messages_count == 0 - assert topic.size == 0 + await wait_for_purged_topic(iggy_client, stream_name, topic_name) @pytest.mark.asyncio async def test_purge_nonexistent_topic_fails( diff --git a/foreign/python/tests/utils.py b/foreign/python/tests/utils.py index b37a53831e..b3ea85c413 100644 --- a/foreign/python/tests/utils.py +++ b/foreign/python/tests/utils.py @@ -24,7 +24,7 @@ import socket import time -from apache_iggy import IggyClient +from apache_iggy import IggyClient, TopicDetails # Server-side limits: usernames are 3-50 bytes, passwords 3-100 bytes. MIN_USERNAME_BYTES = 3 @@ -115,6 +115,37 @@ async def wait_for_ping( await asyncio.sleep(interval) +async def wait_for_purged_topic( + client: IggyClient, stream: str, topic: str, timeout: float = 10.0 +) -> TopicDetails: + """ + Poll get_topic until a committed purge is reflected in the stats. + + The VSR server acknowledges a purge once it commits; partition data + is pruned asynchronously, so stats can transiently report pre-purge + counts. + + Returns: + TopicDetails once messages_count and size reach 0 + + Raises: + TimeoutError: If the purge is not reflected within timeout + """ + deadline = time.time() + timeout + + while True: + details = await client.get_topic(stream, topic) + assert details is not None, "purged topic must still exist" + if details.messages_count == 0 and details.size == 0: + return details + if time.time() >= deadline: + raise TimeoutError( + f"purge of {stream}/{topic} not reflected after {timeout}s: " + f"messages_count={details.messages_count} size={details.size}" + ) + await asyncio.sleep(0.05) + + def unique_credentials(unique_name) -> tuple[str, str]: """Return a unique (username, password) pair within the server limits.""" username = unique_name(max_bytes=MAX_USERNAME_BYTES) diff --git a/justfile b/justfile index 7e18398c80..c8bf1ba4f2 100644 --- a/justfile +++ b/justfile @@ -43,6 +43,9 @@ reap_test_containers := "docker ps -aqf 'name=^iggy-test-' | xargs -r docker rm build: cargo build +build-vsr: + cargo build --features vsr + test: build #!/usr/bin/env bash set -euo pipefail @@ -61,6 +64,17 @@ nextest: build trap "{{reap_test_containers}}" EXIT cargo nextest run --retries 2 +# Like `nextest` but with the `vsr` feature; builds vsr first so the +# harness-spawned iggy-server-ng carries the vsr wire format. `--no-fail-fast` +# because nextest otherwise cancels the run on the first red, which on a loaded +# box means a stray flake hides most of the suite and the reported counts +# understate what actually ran. +nextest-vsr: build-vsr + #!/usr/bin/env bash + set -euo pipefail + trap "{{reap_test_containers}}" EXIT + cargo nextest run --features vsr --no-fail-fast --retries 2 + nextests TEST: build #!/usr/bin/env bash set -euo pipefail @@ -87,6 +101,9 @@ miri: server *ARGS: cargo run --bin iggy-server {{ARGS}} +server-ng *ARGS: + cargo run --bin iggy-server-ng {{ARGS}} + run-benches: ./scripts/run-benches.sh diff --git a/scripts/bump-version.sh b/scripts/bump-version.sh index 88413af9d2..d1e44e334a 100755 --- a/scripts/bump-version.sh +++ b/scripts/bump-version.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/check-backwards-compat.sh b/scripts/check-backwards-compat.sh new file mode 100755 index 0000000000..f7990195f6 --- /dev/null +++ b/scripts/check-backwards-compat.sh @@ -0,0 +1,260 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +# ----------------------------- +# Config (overridable via args) +# ----------------------------- +MASTER_REF="${MASTER_REF:-master}" # branch or commit for "baseline" +PR_REF="${PR_REF:-HEAD}" # commit to test (assumes current checkout) +HOST="${HOST:-127.0.0.1}" +PORT="${PORT:-8090}" +WAIT_SECS="${WAIT_SECS:-60}" +BATCHES="${BATCHES:-50}" +MSGS_PER_BATCH="${MSGS_PER_BATCH:-100}" +KEEP_TMP="${KEEP_TMP:-false}" + +# ----------------------------- +# Helpers +# ----------------------------- +info(){ printf "\n\033[1;36m➤ %s\033[0m\n" "$*"; } +ok(){ printf "\033[0;32m✓ %s\033[0m\n" "$*"; } +err(){ printf "\033[0;31m✗ %s\033[0m\n" "$*" >&2; } +die(){ err "$*"; exit 1; } + +need() { + command -v "$1" >/dev/null 2>&1 || die "missing dependency: $1" +} + +wait_for_port() { + local host="$1" port="$2" deadline=$((SECONDS + WAIT_SECS)) + while (( SECONDS < deadline )); do + if command -v nc >/dev/null 2>&1; then + if nc -z "$host" "$port" 2>/dev/null; then return 0; fi + else + if (echo >"/dev/tcp/$host/$port") >/dev/null 2>&1; then return 0; fi + fi + sleep 1 + done + return 1 +} + +stop_pid() { + local pid="$1" name="${2:-process}" + if kill -0 "$pid" 2>/dev/null; then + kill -TERM "$pid" || true + for _ in $(seq 1 15); do + kill -0 "$pid" 2>/dev/null || { ok "stopped $name (pid $pid)"; return 0; } + sleep 1 + done + err "$name (pid $pid) still running; sending SIGKILL" + kill -KILL "$pid" || true + fi +} + +print_logs_if_any() { + local dir="$1" + if compgen -G "$dir/local_data/logs/iggy*" > /dev/null; then + echo "---- $dir/local_data/logs ----" + cat "$dir"/local_data/logs/iggy* || true + echo "------------------------------" + else + echo "(no iggy logs found in $dir/local_data/logs)" + fi +} + +# ----------------------------- +# Args +# ----------------------------- +usage() { + cat </dev/null 2>&1 || true # optional, we'll use it if present + +REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)" +cd "$REPO_ROOT" + +# Free the port proactively (best-effort) +pkill -f iggy-server >/dev/null 2>&1 || true + +TMP_ROOT="$(mktemp -d -t iggy-backcompat-XXXXXX)" +MASTER_DIR="$TMP_ROOT/master" +PR_DIR="$REPO_ROOT" # assume script is run from PR checkout +MASTER_LOG="$TMP_ROOT/server-master.stdout.log" +PR_LOG="$TMP_ROOT/server-pr.stdout.log" + +cleanup() { + # Stop any leftover iggy-server + pkill -f iggy-server >/dev/null 2>&1 || true + git worktree remove --force "$MASTER_DIR" >/dev/null 2>&1 || true + if [[ "$KEEP_TMP" != "true" ]]; then + rm -rf "$TMP_ROOT" || true + else + info "keeping temp dir: $TMP_ROOT" + fi +} +trap cleanup EXIT + +# ----------------------------- +# 1) Prepare master worktree +# ----------------------------- +info "Preparing baseline worktree at '$MASTER_REF'" +git fetch --all --tags --prune >/dev/null 2>&1 || true +git worktree add --force "$MASTER_DIR" "$MASTER_REF" +ok "worktree at $MASTER_DIR" + +# ----------------------------- +# 2) Build & run master server +# ----------------------------- +pushd "$MASTER_DIR" >/dev/null + +info "Building iggy-server & benches (baseline: $MASTER_REF)" +cargo build --locked --bin iggy-server --bin iggy-bench +ok "built baseline" + +info "Starting iggy-server (baseline)" +set +e +( IGGY_ROOT_USERNAME=iggy IGGY_ROOT_PASSWORD=iggy nohup target/debug/iggy-server >"$MASTER_LOG" 2>&1 & echo $! > "$TMP_ROOT/master.pid" ) +set -e +MASTER_PID="$(cat "$TMP_ROOT/master.pid")" +ok "iggy-server started (pid $MASTER_PID), logs: $MASTER_LOG" + +info "Waiting for $HOST:$PORT to be ready (up to ${WAIT_SECS}s)" +if ! wait_for_port "$HOST" "$PORT"; then + err "server did not become ready in ${WAIT_SECS}s" + print_logs_if_any "$MASTER_DIR" + [[ -f "$MASTER_LOG" ]] && { tail -n 200 "$MASTER_LOG" || true; } + exit 1 +fi +ok "server is ready" + +# Producer bench (baseline) +info "Running producer bench on baseline" +BENCH_CMD=( target/debug/iggy-bench --message-batches "$BATCHES" --messages-per-batch "$MSGS_PER_BATCH" pinned-producer tcp ) +if command -v timeout >/dev/null 2>&1; then timeout 60s "${BENCH_CMD[@]}"; else "${BENCH_CMD[@]}"; fi +ok "producer bench done" + +# Consumer bench (baseline) +info "Running consumer bench on baseline" +BENCH_CMD=( target/debug/iggy-bench --message-batches "$BATCHES" --messages-per-batch "$MSGS_PER_BATCH" pinned-consumer tcp ) +if command -v timeout >/dev/null 2>&1; then timeout 60s "${BENCH_CMD[@]}"; else "${BENCH_CMD[@]}"; fi +ok "consumer bench done (baseline)" + +# Stop baseline server +info "Stopping baseline server" +stop_pid "$MASTER_PID" "iggy-server(baseline)" +print_logs_if_any "$MASTER_DIR" + +# Clean baseline logs (like CI step) +if compgen -G "local_data/logs/iggy*" > /dev/null; then + rm -f local_data/logs/iggy* || true +fi + +# Snapshot local_data/ +info "Snapshotting baseline local_data/" +cp -a local_data "$TMP_ROOT/local_data" +ok "snapshot stored at $TMP_ROOT/local_data" + +popd >/dev/null + +# ----------------------------- +# 3) Build PR & restore data +# ----------------------------- +pushd "$PR_DIR" >/dev/null +info "Ensuring PR ref is present: $PR_REF" +git rev-parse --verify "$PR_REF^{commit}" >/dev/null 2>&1 || die "PR_REF '$PR_REF' not found" +git checkout -q "$PR_REF" + +info "Building iggy-server & benches (PR: $PR_REF)" +cargo build --locked --bin iggy-server --bin iggy-bench +ok "built PR" + +info "Restoring baseline local_data/ into PR workspace" +rm -rf local_data +cp -a "$TMP_ROOT/local_data" ./local_data +ok "restored local_data/" + +# ----------------------------- +# 4) Run PR server & consumer bench +# ----------------------------- +info "Starting iggy-server (PR)" +set +e +( IGGY_ROOT_USERNAME=iggy IGGY_ROOT_PASSWORD=iggy nohup target/debug/iggy-server >"$PR_LOG" 2>&1 & echo $! > "$TMP_ROOT/pr.pid" ) +set -e +PR_PID="$(cat "$TMP_ROOT/pr.pid")" +ok "iggy-server (PR) started (pid $PR_PID), logs: $PR_LOG" + +info "Waiting for $HOST:$PORT to be ready (up to ${WAIT_SECS}s)" +if ! wait_for_port "$HOST" "$PORT"; then + err "PR server did not become ready in ${WAIT_SECS}s" + print_logs_if_any "$PR_DIR" + [[ -f "$PR_LOG" ]] && { tail -n 200 "$PR_LOG" || true; } + exit 1 +fi +ok "PR server is ready" + +# Only consumer bench against PR +info "Running consumer bench on PR (compat check)" +BENCH_CMD=( target/debug/iggy-bench --message-batches "$BATCHES" --messages-per-batch "$MSGS_PER_BATCH" pinned-consumer tcp ) +if command -v timeout >/dev/null 2>&1; then timeout 60s "${BENCH_CMD[@]}"; else "${BENCH_CMD[@]}"; fi +ok "consumer bench done (PR)" + +# Stop PR server +info "Stopping PR server" +stop_pid "$PR_PID" "iggy-server(PR)" +print_logs_if_any "$PR_DIR" + +ok "backwards-compatibility check PASSED" +popd >/dev/null diff --git a/scripts/ci/binary-artifacts.sh b/scripts/ci/binary-artifacts.sh index bbc02760fd..4d63a71e2a 100755 --- a/scripts/ci/binary-artifacts.sh +++ b/scripts/ci/binary-artifacts.sh @@ -18,9 +18,6 @@ set -euo pipefail -# shellcheck source-path=SCRIPTDIR -source "$(dirname "${BASH_SOURCE[0]}")/lib/init.sh" - # binary-artifacts.sh -- Prevent compiled binaries from entering the repo. # # .gitignore catches common extensions (*.o, *.so, *.exe, *.out, etc.) but diff --git a/scripts/ci/lib/init.sh b/scripts/ci/lib/init.sh deleted file mode 100644 index 0cde9f27a1..0000000000 --- a/scripts/ci/lib/init.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Single entry point for the script-backed pre-commit hooks. Sourcing this runs -# every preflight check, so a new check reaches all hooks by being added here -# rather than to each of them. A check that fails exits the sourcing script. -# -# Keep this parseable and runnable by bash 3.2, or the version check cannot -# report on the shell it is meant to reject. - -ensure_bash_version() { - local min_major=4 - local min_minor=2 - - if [ "${BASH_VERSINFO[0]}" -gt "$min_major" ]; then - return 0 - fi - if [ "${BASH_VERSINFO[0]}" -eq "$min_major" ] && [ "${BASH_VERSINFO[1]}" -ge "$min_minor" ]; then - return 0 - fi - - echo "ERROR: this script requires bash >= ${min_major}.${min_minor}, but is running under ${BASH_VERSION}" >&2 - echo " interpreter: ${BASH}" >&2 - echo " bash on PATH: $(command -v bash || echo '')" >&2 - if [ "$(uname)" = "Darwin" ]; then - echo >&2 - echo "macOS ships bash 3.2. Install a current one and make sure it comes first on PATH:" >&2 - echo " brew install bash" >&2 - echo >&2 - echo "Git GUIs often launch hooks with a minimal PATH where /bin wins. If the command" >&2 - echo "above is already installed, commit from a terminal or fix the GUI's PATH." >&2 - fi - exit 1 -} - -ensure_bash_version diff --git a/scripts/ci/license-headers.sh b/scripts/ci/license-headers.sh index 3d0f4d9371..910820b555 100755 --- a/scripts/ci/license-headers.sh +++ b/scripts/ci/license-headers.sh @@ -18,9 +18,6 @@ set -euo pipefail -# shellcheck source-path=SCRIPTDIR -source "$(dirname "${BASH_SOURCE[0]}")/lib/init.sh" - # Parse arguments MODE="check" if [ $# -gt 0 ]; then diff --git a/scripts/ci/markdownlint.sh b/scripts/ci/markdownlint.sh index aae00f2b02..a937956e4e 100755 --- a/scripts/ci/markdownlint.sh +++ b/scripts/ci/markdownlint.sh @@ -18,9 +18,6 @@ set -euo pipefail -# shellcheck source-path=SCRIPTDIR -source "$(dirname "${BASH_SOURCE[0]}")/lib/init.sh" - MODE="check" FILES=() diff --git a/scripts/ci/python-sdk-version-sync.sh b/scripts/ci/python-sdk-version-sync.sh index 59cf88897a..fa9e3d33d1 100755 --- a/scripts/ci/python-sdk-version-sync.sh +++ b/scripts/ci/python-sdk-version-sync.sh @@ -18,9 +18,6 @@ set -euo pipefail -# shellcheck source-path=SCRIPTDIR -source "$(dirname "${BASH_SOURCE[0]}")/lib/init.sh" - # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' diff --git a/scripts/ci/shellcheck.sh b/scripts/ci/shellcheck.sh index 1f8477ed9c..131cd30201 100755 --- a/scripts/ci/shellcheck.sh +++ b/scripts/ci/shellcheck.sh @@ -18,9 +18,6 @@ set -euo pipefail -# shellcheck source-path=SCRIPTDIR -source "$(dirname "${BASH_SOURCE[0]}")/lib/init.sh" - MODE="check" FILES=() diff --git a/scripts/ci/skills.sh b/scripts/ci/skills.sh index 6399d15ec5..49787fd463 100755 --- a/scripts/ci/skills.sh +++ b/scripts/ci/skills.sh @@ -23,9 +23,6 @@ set -euo pipefail -# shellcheck source-path=SCRIPTDIR -source "$(dirname "${BASH_SOURCE[0]}")/lib/init.sh" - ROOT="$(git rev-parse --show-toplevel)" cd "$ROOT" diff --git a/scripts/ci/sync-python-interpreter-version.sh b/scripts/ci/sync-python-interpreter-version.sh index cc3ced1a36..2c33e475fc 100755 --- a/scripts/ci/sync-python-interpreter-version.sh +++ b/scripts/ci/sync-python-interpreter-version.sh @@ -18,9 +18,6 @@ set -euo pipefail -# shellcheck source-path=SCRIPTDIR -source "$(dirname "${BASH_SOURCE[0]}")/lib/init.sh" - # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' diff --git a/scripts/ci/sync-rustc-version.sh b/scripts/ci/sync-rustc-version.sh index 8cf0709a46..6e317df6be 100755 --- a/scripts/ci/sync-rustc-version.sh +++ b/scripts/ci/sync-rustc-version.sh @@ -18,9 +18,6 @@ set -euo pipefail -# shellcheck source-path=SCRIPTDIR -source "$(dirname "${BASH_SOURCE[0]}")/lib/init.sh" - # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' @@ -82,7 +79,7 @@ fi # Strip trailing ".0" -> e.g., 1.89.0 -> 1.89 (no change if it doesn't end in .0) RUST_VERSION_SHORT=$(echo "$RUST_VERSION" | sed -E 's/^([0-9]+)\.([0-9]+)\.0$/\1.\2/') RUST_IMAGE_VARIANT="slim-trixie" -RUST_IMAGE_PATTERN="FROM[[:space:]]+(.*[^[:alnum:]_])?rust:" +RUST_IMAGE_PATTERN="FROM[[:space:]].*\\brust:" RUST_IMAGE_TAG_PATTERN="(rust:[^-[:space:]]+)" echo "Rust version from rust-toolchain.toml: ${GREEN}$RUST_VERSION${NC} (using ${GREEN}$RUST_VERSION_SHORT${NC} for Dockerfiles)" @@ -111,9 +108,9 @@ for dockerfile in $DOCKERFILES; do SOURCE="arg" CURRENT_VERSION=$(grep "^ARG RUST_VERSION=" "$dockerfile" | head -1 | sed 's/^ARG RUST_VERSION=//') EXPECTED_VERSION="$RUST_VERSION_SHORT" - elif grep -qE "${RUST_IMAGE_PATTERN}[0-9]" "$dockerfile" 2>/dev/null; then + elif grep -qE "FROM[[:space:]].*\brust:[0-9]" "$dockerfile" 2>/dev/null; then SOURCE="from" - CURRENT_VERSION=$(grep -E "${RUST_IMAGE_PATTERN}[0-9]" "$dockerfile" | head -1 | sed -nE 's/.*rust:([0-9]+\.[0-9]+(\.[0-9]+)?).*/\1/p') + CURRENT_VERSION=$(grep -E "FROM[[:space:]].*\brust:[0-9]" "$dockerfile" | head -1 | sed -nE 's/.*\brust:([0-9]+\.[0-9]+(\.[0-9]+)?).*/\1/p') # Preserve the file's precision: full patch (1.96.0) or short (1.96). if [[ "$CURRENT_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then EXPECTED_VERSION="$RUST_VERSION" @@ -162,7 +159,7 @@ for dockerfile in $DOCKERFILES; do sed -i.bak "s/^ARG RUST_VERSION=.*/ARG RUST_VERSION=$EXPECTED_VERSION/" "$dockerfile" rm -f "$dockerfile.bak" elif [ -n "$SOURCE" ] && [ "$CURRENT_VERSION" != "$EXPECTED_VERSION" ]; then - sed -i.bak -E "/${RUST_IMAGE_PATTERN}[0-9]/ s#(rust:)[0-9]+\\.[0-9]+(\\.[0-9]+)?#\\1$EXPECTED_VERSION#g" "$dockerfile" + sed -i.bak -E "/FROM[[:space:]].*\\brust:[0-9]/ s#(\\brust:)[0-9]+\\.[0-9]+(\\.[0-9]+)?#\\1$EXPECTED_VERSION#g" "$dockerfile" rm -f "$dockerfile.bak" fi if [ "$RUST_IMAGE_MISMATCH" = "true" ]; then diff --git a/scripts/ci/taplo.sh b/scripts/ci/taplo.sh index 792be4feed..1568d17d93 100755 --- a/scripts/ci/taplo.sh +++ b/scripts/ci/taplo.sh @@ -18,9 +18,6 @@ set -euo pipefail -# shellcheck source-path=SCRIPTDIR -source "$(dirname "${BASH_SOURCE[0]}")/lib/init.sh" - # Default values MODE="check" FILE_MODE="all" diff --git a/scripts/ci/trailing-newline.sh b/scripts/ci/trailing-newline.sh index a2e928b1af..55916b1bb7 100755 --- a/scripts/ci/trailing-newline.sh +++ b/scripts/ci/trailing-newline.sh @@ -18,9 +18,6 @@ set -euo pipefail -# shellcheck source-path=SCRIPTDIR -source "$(dirname "${BASH_SOURCE[0]}")/lib/init.sh" - # Default values MODE="check" FILE_MODE="all" diff --git a/scripts/ci/trailing-whitespace.sh b/scripts/ci/trailing-whitespace.sh index e4d8299e54..b3c481dc5c 100755 --- a/scripts/ci/trailing-whitespace.sh +++ b/scripts/ci/trailing-whitespace.sh @@ -18,9 +18,6 @@ set -euo pipefail -# shellcheck source-path=SCRIPTDIR -source "$(dirname "${BASH_SOURCE[0]}")/lib/init.sh" - # Default values MODE="check" FILE_MODE="all" diff --git a/scripts/ci/uv-lock-check.sh b/scripts/ci/uv-lock-check.sh index 6e633249a1..de76ad7bfe 100755 --- a/scripts/ci/uv-lock-check.sh +++ b/scripts/ci/uv-lock-check.sh @@ -18,9 +18,6 @@ set -euo pipefail -# shellcheck source-path=SCRIPTDIR -source "$(dirname "${BASH_SOURCE[0]}")/lib/init.sh" - RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' diff --git a/scripts/copy-latest-from-master.sh b/scripts/copy-latest-from-master.sh index 37d817a9be..35665cf58f 100755 --- a/scripts/copy-latest-from-master.sh +++ b/scripts/copy-latest-from-master.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/dashboard/build_release.sh b/scripts/dashboard/build_release.sh index e06b9b46a7..7f46d6fa50 100755 --- a/scripts/dashboard/build_release.sh +++ b/scripts/dashboard/build_release.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/dashboard/run_dev.sh b/scripts/dashboard/run_dev.sh index 6cbd365283..5fb8c83acb 100755 --- a/scripts/dashboard/run_dev.sh +++ b/scripts/dashboard/run_dev.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/extract-version.sh b/scripts/extract-version.sh index 020cb56f46..6253db9342 100755 --- a/scripts/extract-version.sh +++ b/scripts/extract-version.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -61,9 +61,6 @@ set -euo pipefail -# shellcheck source-path=SCRIPTDIR -source "$(dirname "${BASH_SOURCE[0]}")/ci/lib/init.sh" - # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' diff --git a/scripts/performance/run-standard-performance-suite.sh b/scripts/performance/run-standard-performance-suite.sh index f99635ff81..31d2e2aea1 100755 --- a/scripts/performance/run-standard-performance-suite.sh +++ b/scripts/performance/run-standard-performance-suite.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/performance/utils.sh b/scripts/performance/utils.sh index b8513da2a4..ec790fb19e 100755 --- a/scripts/performance/utils.sh +++ b/scripts/performance/utils.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/profile.sh b/scripts/profile.sh index b6b7b56683..dad410ba9f 100755 --- a/scripts/profile.sh +++ b/scripts/profile.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/run-bdd-tests.sh b/scripts/run-bdd-tests.sh index 272f763061..37d47315ab 100755 --- a/scripts/run-bdd-tests.sh +++ b/scripts/run-bdd-tests.sh @@ -19,10 +19,12 @@ set -Eeuo pipefail COVERAGE=0 +VSR=0 ARGS=() for arg in "$@"; do case "$arg" in --coverage) COVERAGE=1 ;; + --vsr) VSR=1 ;; *) ARGS+=("$arg") ;; esac done @@ -33,21 +35,34 @@ FEATURE="${ARGS[1]:-all}" log(){ printf "%b\n" "$*"; } usage(){ - log "Usage: $0 [--coverage] [feature]" + log "Usage: $0 [--coverage] [--vsr] [feature]" log "" log " sdk: rust | python | php | go | go-race | node | csharp | java | cpp | all | clean (default: all)" log " feature: basic_messaging | leader_redirection | raw_command | all (default: all)" - log "" - log " Every suite runs against iggy-server, taken from IGGY_SERVER_PATH" - log " (default: target/debug/iggy-server) with an iggy CLI at IGGY_CLI_PATH." + # TODO: change to iggy-server once legacy server is removed (core/server has VSR support) + log " --vsr: run against iggy-server-ng built with --features vsr (rust, python, go, node);" + log " expects IGGY_SERVER_NG_PATH (default: target/debug/iggy-server-ng)" + log " and a vsr-built iggy CLI at IGGY_CLI_PATH." + log " The go suites imply it: the Go SDK speaks only the VSR protocol." log "" log "Examples:" log " $0 rust # run all features for Rust" log " $0 rust basic_messaging # run only basic_messaging for Rust" log " $0 all leader_redirection # run leader_redirection for all supporting SDKs" + log " $0 --vsr rust # run all features for Rust against server-ng" log " $0 --coverage go basic_messaging" } +if [ "$VSR" = "1" ]; then + case "$SDK" in + rust|python|go|go-race|node|clean) ;; + *) + log "❌ --vsr supports only the Rust, Python, Go, and Node SDKs so far" + usage + exit 2 ;; + esac +fi + case "$FEATURE" in basic_messaging|leader_redirection|raw_command|all) ;; *) @@ -65,6 +80,7 @@ ALL_COMPOSE_FILES=( -f docker-compose.server.yml -f docker-compose.cluster.yml -f docker-compose.coverage.yml + -f docker-compose.vsr.yml ) COMPOSE_FILES=(-f docker-compose.yml) @@ -80,6 +96,11 @@ if [ "$COVERAGE" = "1" ]; then COMPOSE_FILES+=(-f docker-compose.coverage.yml) mkdir -p ../reports fi +# vsr overrides must come last to win over the server/cluster/coverage files. +if [ "$VSR" = "1" ]; then + COMPOSE_FILES+=(-f docker-compose.vsr.yml) + export BDD_RUST_FEATURES="bdd,vsr" +fi cleanup(){ log "🧹 cleaning up containers & volumes…" @@ -89,7 +110,10 @@ trap cleanup EXIT INT TERM log "🧪 Running BDD tests for SDK: ${SDK}" log "📁 Feature file: ${FEATURE}" -log "🗳️ Server: iggy-server" +if [ "$VSR" = "1" ] || [ "$SDK" = "go" ] || [ "$SDK" = "go-race" ]; then + # TODO: change to iggy-server once legacy server is removed (core/server has VSR support) + log "🗳️ Server: iggy-server-ng (--features vsr)" +fi if [ "$COVERAGE" = "1" ]; then log "📊 Coverage collection enabled → reports will be in ./reports/" fi @@ -111,12 +135,20 @@ run_suite(){ esac fi + # The Go SDK speaks only the VSR wire protocol, so its suites always run + # against the VSR server even when the caller did not ask for it. Each suite + # tears its own stack down, so an `all` run can mix the two servers. + local files=("${COMPOSE_FILES[@]}") + if [ "$svc" = "go-bdd" ] && [ "$VSR" != "1" ]; then + files+=(-f docker-compose.vsr.yml) + fi + log "${emoji} ${label}..." local code=0 - docker compose "${COMPOSE_FILES[@]}" \ + docker compose "${files[@]}" \ up --build --exit-code-from "$svc" "$svc" \ || code=$? - docker compose "${COMPOSE_FILES[@]}" \ + docker compose "${files[@]}" \ down -v --remove-orphans >/dev/null 2>&1 || true return "$code" } diff --git a/scripts/run-benches.sh b/scripts/run-benches.sh index c8d8e30410..4a35491395 100755 --- a/scripts/run-benches.sh +++ b/scripts/run-benches.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/run-examples-from-readme.sh b/scripts/run-examples-from-readme.sh index 3a13fb0059..95674b99bf 100755 --- a/scripts/run-examples-from-readme.sh +++ b/scripts/run-examples-from-readme.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -179,12 +179,11 @@ run_language_examples() { # shellcheck disable=SC2329 run_rust_examples() { - resolve_server_binary "${TARGET}" "iggy-server" + resolve_server_binary "${TARGET}" resolve_cli_binary "${TARGET}" # The README documents credentials as / - # placeholders; the test server starts with iggy/iggy. The README keeps - # plain cargo run commands, so the cross-compile target is injected here. + # placeholders; the test server starts with iggy/iggy. if [ -n "${TARGET}" ]; then TRANSFORM_COMMAND() { echo "$1" | sed "s||iggy|g; s||iggy|g; s|cargo run |cargo run --target ${TARGET} |g" @@ -218,8 +217,7 @@ run_rust_examples() { # shellcheck disable=SC2329 run_node_examples() { - # The Node SDK is vsr-only, so examples run against the vsr server. - resolve_server_binary "${TARGET}" iggy-server + resolve_server_binary "${TARGET}" export DEBUG=iggy:examples unset -f TRANSFORM_COMMAND 2>/dev/null || true @@ -238,7 +236,8 @@ run_node_examples() { # shellcheck disable=SC2329 run_go_examples() { # The Go SDK speaks only the VSR wire protocol. - resolve_server_binary "${TARGET}" "iggy-server" + # TODO: change to iggy-server once legacy server is removed (core/server has VSR support) + resolve_server_binary "${TARGET}" "iggy-server-ng" "vsr" # The VSR server logs no startup line, so readiness is a connect poll. SERVER_READY_PROBE="tcp" unset -f TRANSFORM_COMMAND 2>/dev/null || true @@ -258,10 +257,12 @@ run_go_examples() { # shellcheck disable=SC2329 run_python_examples() { - # The Python SDK speaks only the VSR wire protocol, so examples run - # against the VSR server, started fresh by cleanup_server_state wiping - # local_data rather than by passing --fresh. - resolve_server_binary "${TARGET}" iggy-server + # Python wheels are vsr-built, so examples run against the vsr + # server. It takes no --fresh flag; cleanup_server_state wiping + # local_data is the fresh start. + # TODO(hubcio): change to iggy-server once legacy server is removed + # (core/server has VSR support) + resolve_server_binary "${TARGET}" iggy-server-ng unset -f TRANSFORM_COMMAND 2>/dev/null || true echo "" @@ -310,10 +311,7 @@ run_python_examples() { # shellcheck disable=SC2329 run_php_examples() { - # The PHP extension speaks only the VSR wire protocol, so examples run - # against the VSR server, started fresh by cleanup_server_state wiping - # local_data rather than by passing --fresh. - resolve_server_binary "${TARGET}" iggy-server + resolve_server_binary "${TARGET}" local php_bin="${PHP:-php}" if [ -z "${PHP_IGGY_EXTENSION:-}" ]; then @@ -349,14 +347,12 @@ run_php_examples() { "" \ "" \ 0 \ - "" + "--fresh" } # shellcheck disable=SC2329 run_java_examples() { - # Java examples run against the VSR server. - resolve_server_binary "${TARGET}" "iggy-server" - SERVER_READY_PATTERN="client listeners started" + resolve_server_binary "${TARGET}" unset -f TRANSFORM_COMMAND 2>/dev/null || true run_language_examples \ @@ -372,10 +368,7 @@ run_java_examples() { # shellcheck disable=SC2329 run_csharp_examples() { - # The .NET SDK speaks only the VSR wire protocol, so examples run against - # the VSR server, started fresh by cleanup_server_state wiping local_data - # rather than by passing --fresh. - resolve_server_binary "${TARGET}" iggy-server + resolve_server_binary "${TARGET}" unset -f TRANSFORM_COMMAND 2>/dev/null || true run_language_examples \ @@ -401,7 +394,6 @@ run_one() { EXAMPLES_EXIT_CODE=0 unset -f TRANSFORM_COMMAND 2>/dev/null || true - unset SERVER_READY_PATTERN 2>/dev/null || true set +e ${lang_fn} @@ -409,7 +401,6 @@ run_one() { set -e unset -f TRANSFORM_COMMAND 2>/dev/null || true - unset SERVER_READY_PATTERN 2>/dev/null || true if [ ${rc} -ne 0 ]; then echo "" diff --git a/scripts/utils.sh b/scripts/utils.sh index 0c77704877..fb84c8b6a2 100755 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -159,11 +159,14 @@ readonly EXAMPLES_SERVER_TIMEOUT=300 readonly EXAMPLES_STOP_TIMEOUT=5 # Resolve and validate the server binary path. -# Usage: resolve_server_binary [target] [binary_name] -# Sets global SERVER_BIN. binary_name defaults to iggy-server. +# Usage: resolve_server_binary [target] [binary_name] [cargo_features] +# Sets global SERVER_BIN. binary_name defaults to iggy-server; vsr lanes +# pass their server binary and the vsr feature (so the wire protocol +# matches vsr-built clients). function resolve_server_binary() { local target="${1:-}" local binary="${2:-iggy-server}" + local features="${3:-}" if [ -n "${target}" ]; then SERVER_BIN="target/${target}/debug/${binary}" @@ -176,6 +179,9 @@ function resolve_server_binary() { if [ -n "${target}" ]; then build_command="cargo build --target ${target} --bin ${binary}" fi + if [ -n "${features}" ]; then + build_command="${build_command} --features ${features}" + fi echo "Error: Server binary not found at ${SERVER_BIN}" echo "Please build the server binary before running this script:" echo " ${build_command}" @@ -237,14 +243,12 @@ function start_tls_server() { # How wait_for_server_ready decides the server is up. "log" greps the startup # lines: the legacy "has started" and the VSR server "client listeners -# started" (logged once the TCP socket is bound); override via -# SERVER_READY_PATTERN or the pattern arg. "tcp" polls the listener +# started" (logged once the TCP socket is bound). "tcp" polls the listener # instead for lanes that cannot rely on a startup line. : "${SERVER_READY_PROBE:=log}" : "${SERVER_READY_ADDRESS:=127.0.0.1:8090}" # Report whether the server is accepting work. -# Usage: server_is_ready [pattern] function server_is_ready() { if [ "${SERVER_READY_PROBE}" = "tcp" ]; then local host="${SERVER_READY_ADDRESS%:*}" @@ -253,7 +257,7 @@ function server_is_ready() { exec 3<&- return 0 fi - grep -qE "${1:-has started|client listeners started}" "${EXAMPLES_LOG_FILE}" + grep -qE "has started|client listeners started" "${EXAMPLES_LOG_FILE}" } # Report whether the server this script started is still running. @@ -265,13 +269,9 @@ function server_is_alive() { } # Block until the server is ready or the timeout elapses. -# Usage: wait_for_server_ready [label] [pattern] -# The default log pattern matches both the legacy line ("has started") and -# the vsr server line ("client listeners started"). Override via the -# pattern arg or SERVER_READY_PATTERN. +# Usage: wait_for_server_ready [label] function wait_for_server_ready() { local label="${1:-Iggy}" - local pattern="${2:-${SERVER_READY_PATTERN:-has started|client listeners started}}" local elapsed=0 while true; do # Liveness is checked first so a server that died leaves its log here @@ -282,7 +282,7 @@ function wait_for_server_ready() { cat "${EXAMPLES_LOG_FILE}" exit 1 fi - if server_is_ready "${pattern}"; then + if server_is_ready; then return 0 fi if [ ${elapsed} -gt ${EXAMPLES_SERVER_TIMEOUT} ]; then From 7e4a2e764824c36a0f872bae2d0f5ca1641982f8 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Wed, 12 Aug 2026 08:02:17 -0400 Subject: [PATCH 54/57] Reapply "Merge branch 'master' into feat(gateways)/kafka_to_iggy_listener" This reverts commit 3caa4c6f0b94b0f99615c6ba16ec63ecb69be35d. --- .claude/skills/connectors-overview/SKILL.md | 26 +- .dockerignore | 1 - .../actions/cpp-bazel/pre-merge/action.yml | 2 + .../csharp-dotnet/pre-merge/action.yml | 19 +- .github/actions/go/pre-merge/action.yml | 46 +- .../actions/java-gradle/pre-merge/action.yml | 8 + .github/actions/node-npm/pre-merge/action.yml | 94 +- .github/actions/php/pre-merge/action.yml | 3 + .../python-maturin/pre-merge/action.yml | 15 +- .github/actions/rust/pre-merge/action.yml | 21 +- .../utils/docker-build-test-server/action.yml | 105 - .github/actions/utils/server-start/action.yml | 9 +- .github/config/components.yml | 16 +- .github/dependabot.yml | 1 - .github/workflows/_test.yml | 3 +- .github/workflows/_test_bdd.yml | 52 +- .github/workflows/_test_examples.yml | 20 +- .github/workflows/coverage-baseline.yml | 42 +- .github/workflows/pr-title.yml | 1 - .github/workflows/pre-merge.yml | 2 +- AGENTS.md | 4 +- CONTRIBUTING.md | 22 + Cargo.lock | 74 +- Cargo.toml | 2 - Dockerfile | 3 +- README.md | 10 +- bdd/README.md | 11 +- bdd/docker-compose.cluster.yml | 50 +- bdd/docker-compose.server.yml | 8 +- bdd/docker-compose.vsr.yml | 101 - bdd/go/tests/tcp_test/test_helpers.go | 9 +- .../iggy/bdd/LeaderRedirectionSteps.java | 35 +- bdd/python/tests/test_basic_messaging.py | 3 +- bdd/rust/Cargo.toml | 1 - codecov.yml | 3 - core/ai/mcp/Cargo.toml | 1 - core/bench/Cargo.toml | 12 - .../frontend/scripts/select_index.sh | 2 +- core/bench/src/analytics/report_builder.rs | 4 +- core/bench/src/main.rs | 16 +- core/binary_protocol/Cargo.toml | 1 + core/binary_protocol/src/consensus/command.rs | 2 +- core/binary_protocol/src/consensus/error.rs | 17 + core/binary_protocol/src/consensus/header.rs | 967 +++- core/binary_protocol/src/consensus/mod.rs | 11 +- core/binary_protocol/src/framing.rs | 4 +- core/binary_protocol/src/lib.rs | 14 +- core/binary_protocol/src/namespace.rs | 36 +- .../src/requests/users/login_register.rs | 6 +- .../requests/users/login_register_with_pat.rs | 2 +- .../src/responses/topics/create_topic.rs | 2 +- .../src/responses/users/login_register.rs | 2 +- core/binary_protocol/src/version.rs | 4 +- core/cli/Cargo.toml | 1 - core/common/Cargo.toml | 3 - core/common/src/error/eviction.rs | 4 +- core/common/src/lib.rs | 3 - core/common/src/traits/binary_client.rs | 15 +- .../traits/binary_impls/consumer_groups.rs | 2 - .../src/traits/binary_impls/messages.rs | 16 - core/common/src/traits/binary_impls/mod.rs | 7 +- .../binary_impls/personal_access_tokens.rs | 110 +- core/common/src/traits/binary_impls/system.rs | 1 - core/common/src/traits/binary_impls/users.rs | 108 +- core/common/src/traits/binary_transport.rs | 5 - core/common/src/traits/message_client.rs | 2 +- core/common/src/utils/serde_secret.rs | 95 +- core/configs/Cargo.toml | 1 - .../cache_indexes.rs | 0 core/configs/src/common/defaults.rs | 431 ++ core/configs/src/common/displays.rs | 280 + .../src/{server_config => common}/http.rs | 0 .../src/state => configs/src/common}/mod.rs | 19 +- core/configs/src/common/server.rs | 144 + .../src/{server_config => common}/system.rs | 13 +- core/configs/src/common/validators.rs | 357 ++ core/configs/src/lib.rs | 11 +- core/configs/src/server_config/cluster.rs | 2450 +++++++- core/configs/src/server_config/defaults.rs | 620 +- core/configs/src/server_config/displays.rs | 311 +- .../message_bus.rs | 38 +- .../metadata.rs | 76 +- core/configs/src/server_config/mod.rs | 14 +- .../partition.rs | 56 +- core/configs/src/server_config/quic.rs | 157 +- core/configs/src/server_config/server.rs | 263 +- core/configs/src/server_config/sharding.rs | 383 +- core/configs/src/server_config/tcp.rs | 2 + core/configs/src/server_config/validators.rs | 1268 ++--- core/configs/src/server_config/websocket.rs | 100 +- core/configs/src/server_ng_config/cluster.rs | 2612 --------- core/configs/src/server_ng_config/defaults.rs | 335 -- core/configs/src/server_ng_config/displays.rs | 189 - core/configs/src/server_ng_config/mod.rs | 45 - core/configs/src/server_ng_config/quic.rs | 222 - .../configs/src/server_ng_config/server_ng.rs | 229 - core/configs/src/server_ng_config/sharding.rs | 431 -- core/configs/src/server_ng_config/tcp.rs | 62 - .../src/server_ng_config/validators.rs | 861 --- .../configs/src/server_ng_config/websocket.rs | 116 - core/connectors/runtime/Cargo.toml | 3 - core/consensus/Cargo.toml | 1 + core/consensus/src/client_table.rs | 9 +- core/consensus/src/dvc_merge.rs | 981 ++++ core/consensus/src/impls.rs | 909 ++- core/consensus/src/lib.rs | 9 +- core/consensus/src/observability.rs | 2 +- core/consensus/src/plane_helpers.rs | 1061 +++- core/consensus/src/view_change_quorum.rs | 429 +- core/harness_derive/src/attrs.rs | 7 +- core/harness_derive/src/codegen.rs | 4 +- core/integration/Cargo.toml | 14 +- core/integration/src/bench_utils.rs | 13 +- .../integration/src/harness/config/resolve.rs | 35 +- core/integration/src/harness/config/server.rs | 4 +- core/integration/src/harness/handle/server.rs | 23 +- .../src/harness/orchestrator/builder.rs | 13 +- .../src/harness/orchestrator/harness.rs | 14 +- .../cli/message/test_message_flush_command.rs | 14 +- .../cli/stream/test_stream_purge_command.rs | 2 +- .../cli/system/test_cli_session_scenario.rs | 4 +- .../tests/cli/system/test_me_command.rs | 23 +- .../cli/topic/test_topic_purge_command.rs | 2 +- .../tests/cluster/client_table_restart.rs | 7 - .../cluster/metadata_checkpoint_restart.rs | 10 +- .../tests/cluster/metadata_state_transfer.rs | 2 - .../multi_shard_partition_convergence.rs | 2 - .../tests/cluster/partition_state_transfer.rs | 14 +- core/integration/tests/config_provider/mod.rs | 2 +- core/integration/tests/data_integrity/mod.rs | 13 +- .../verify_after_server_restart.rs | 45 +- .../verify_auto_commit_offset_replicates.rs | 10 +- ...ify_consumer_group_partition_assignment.rs | 81 +- core/integration/tests/mod.rs | 17 +- .../tests/sdk/consumer_group_membership.rs | 2 +- core/integration/tests/sdk/hello_world.rs | 1 - core/integration/tests/sdk/http_refresh.rs | 2 +- core/integration/tests/sdk/mod.rs | 3 - .../integration/tests/sdk/protocol_version.rs | 4 - core/integration/tests/sdk/raw.rs | 11 - .../tests/sdk/send_confirmation.rs | 36 +- .../tests/server/a2a_jwt/jwt_tests.rs | 23 +- .../server/cluster_view_durability_vsr.rs | 4 +- core/integration/tests/server/flush_vsr.rs | 4 +- core/integration/tests/server/general.rs | 19 - core/integration/tests/server/http_client.rs | 4 +- core/integration/tests/server/http_rbac.rs | 4 +- core/integration/tests/server/http_tls.rs | 8 +- core/integration/tests/server/http_vsr.rs | 6 +- .../tests/server/legacy_login_vsr.rs | 6 +- .../tests/server/login_credentials_vsr.rs | 91 + core/integration/tests/server/mod.rs | 36 +- .../server/partition_view_durability_vsr.rs | 4 +- .../tests/server/poll_semantics_vsr.rs | 72 +- core/integration/tests/server/purge_delete.rs | 62 +- core/integration/tests/server/purge_vsr.rs | 101 +- .../scenarios/authentication_scenario.rs | 7 +- .../tests/server/scenarios/bench_scenario.rs | 45 - .../server/scenarios/encryption_scenario.rs | 40 +- .../integration/tests/server/scenarios/mod.rs | 6 +- .../server/scenarios/permissions_scenario.rs | 4 +- .../server/scenarios/purge_delete_scenario.rs | 207 +- .../reconnect_after_restart_scenario.rs | 2 +- .../scenarios/restart_offset_skip_scenario.rs | 14 +- .../stream_size_validation_scenario.rs | 13 +- .../tests/server/scenarios/system_scenario.rs | 7 +- core/integration/tests/server/specific.rs | 39 +- .../tests/server/stats_vsr.rs} | 42 +- .../tests/server/topic_admission_vsr.rs | 71 +- core/integration/tests/state/file.rs | 164 - core/integration/tests/state/mod.rs | 92 - core/integration/tests/state/system.rs | 212 - .../tests/storage/consumer_offsets.rs | 179 - core/integration/tests/storage/mod.rs | 18 - core/journal/src/lib.rs | 19 + core/journal/src/prepare_journal.rs | 442 +- core/message_bus/src/client_listener/quic.rs | 2 +- core/message_bus/src/config.rs | 50 +- core/message_bus/src/installer/conn_info.rs | 2 +- core/message_bus/src/installer/mod.rs | 2 +- core/message_bus/src/installer/wss.rs | 2 +- core/message_bus/src/lib.rs | 24 +- core/message_bus/src/replica/io.rs | 4 +- core/message_bus/src/transports/quic.rs | 2 +- core/message_bus/src/transports/tls/mod.rs | 4 +- core/message_bus/tests/ws_client_roundtrip.rs | 4 +- core/metadata/Cargo.toml | 2 +- core/metadata/src/impls/metadata.rs | 303 +- core/metadata/src/impls/recovery.rs | 182 +- core/metadata/src/stm/result.rs | 1 + core/metadata/src/stm/snapshot.rs | 59 +- core/metadata/src/stm/stream.rs | 415 +- core/metadata/src/stm/user.rs | 4 +- core/partitions/Cargo.toml | 6 +- core/partitions/src/iggy_partition.rs | 764 ++- core/partitions/src/iggy_partitions.rs | 125 +- core/partitions/src/journal.rs | 184 +- core/partitions/src/lib.rs | 2 +- core/partitions/src/log.rs | 8 + core/partitions/src/messages_writer.rs | 89 +- core/partitions/src/offset_storage.rs | 364 +- core/partitions/src/poll_plan.rs | 85 +- core/partitions/src/state_transfer.rs | 86 +- core/partitions/src/types.rs | 9 + core/sdk/Cargo.toml | 3 - core/sdk/src/clients/client.rs | 35 +- core/sdk/src/http/http_client.rs | 2 +- core/sdk/src/lib.rs | 2 - core/sdk/src/prelude.rs | 24 +- core/sdk/src/quic/quic_client.rs | 149 +- core/sdk/src/session.rs | 2 +- core/sdk/src/tcp/tcp_client.rs | 341 +- core/sdk/src/tcp/tcp_connection_stream.rs | 2 - .../sdk/src/tcp/tcp_connection_stream_kind.rs | 2 - core/sdk/src/tcp/tcp_tls_connection_stream.rs | 2 - core/sdk/src/vsr.rs | 256 +- core/sdk/src/websocket/websocket_client.rs | 126 +- core/server-ng/.dockerignore | 30 - core/server-ng/Cargo.toml | 195 - core/server-ng/Dockerfile | 179 - core/server-ng/build.rs | 86 - core/server-ng/config.toml | 1027 ---- core/server-ng/server.http | 369 -- core/server-ng/src/bootstrap.rs | 4385 -------------- core/server-ng/src/http/error.rs | 818 --- core/server-ng/src/http/metrics.rs | 297 - core/server-ng/src/lib.rs | 47 - core/server-ng/src/main.rs | 89 - core/server-ng/src/server_error.rs | 391 -- core/server-ng/src/wire.rs | 75 - core/server/Cargo.toml | 104 +- core/server/Dockerfile | 11 +- core/server/README.md | 37 +- core/server/config.toml | 495 +- core/server/server.http | 35 +- core/server/src/args.rs | 122 +- core/{server-ng => server}/src/auth.rs | 14 +- core/server/src/binary/dispatch.rs | 456 -- .../cluster/get_cluster_metadata_handler.rs | 60 - .../server/src/binary/handlers/cluster/mod.rs | 18 - .../create_consumer_group_handler.rs | 67 - .../delete_consumer_group_handler.rs | 56 - .../get_consumer_group_handler.rs | 78 - .../get_consumer_groups_handler.rs | 69 - .../join_consumer_group_handler.rs | 57 - .../leave_consumer_group_handler.rs | 57 - .../binary/handlers/consumer_groups/mod.rs | 25 - .../delete_consumer_offset_handler.rs | 56 - .../get_consumer_offset_handler.rs | 79 - .../binary/handlers/consumer_offsets/mod.rs | 22 - .../store_consumer_offset_handler.rs | 63 - .../messages/flush_unsaved_buffer_handler.rs | 65 - .../src/binary/handlers/messages/mod.rs | 22 - .../messages/poll_messages_handler.rs | 87 - .../messages/send_messages_handler.rs | 203 - core/server/src/binary/handlers/mod.rs | 28 - .../partitions/create_partitions_handler.rs | 61 - .../partitions/delete_partitions_handler.rs | 60 - .../src/binary/handlers/partitions/mod.rs | 21 - .../create_personal_access_token_handler.rs | 73 - .../delete_personal_access_token_handler.rs | 57 - .../get_personal_access_tokens_handler.rs | 60 - ...ogin_with_personal_access_token_handler.rs | 51 - .../handlers/personal_access_tokens/mod.rs | 23 - .../segments/delete_segments_handler.rs | 76 - .../src/binary/handlers/segments/mod.rs | 18 - .../handlers/streams/create_stream_handler.rs | 67 - .../handlers/streams/delete_stream_handler.rs | 56 - .../handlers/streams/get_stream_handler.rs | 121 - .../handlers/streams/get_streams_handler.rs | 62 - .../server/src/binary/handlers/streams/mod.rs | 25 - .../handlers/streams/purge_stream_handler.rs | 56 - .../handlers/streams/update_stream_handler.rs | 56 - .../handlers/system/get_client_handler.rs | 54 - .../handlers/system/get_clients_handler.rs | 44 - .../binary/handlers/system/get_me_handler.rs | 77 - .../handlers/system/get_snapshot_handler.rs | 53 - .../handlers/system/get_stats_handler.rs | 93 - core/server/src/binary/handlers/system/mod.rs | 25 - .../binary/handlers/system/ping_handler.rs | 41 - .../handlers/topics/create_topic_handler.rs | 103 - .../handlers/topics/delete_topic_handler.rs | 56 - .../handlers/topics/get_topic_handler.rs | 76 - .../handlers/topics/get_topics_handler.rs | 61 - core/server/src/binary/handlers/topics/mod.rs | 25 - .../handlers/topics/purge_topic_handler.rs | 56 - .../handlers/topics/update_topic_handler.rs | 56 - .../handlers/users/change_password_handler.rs | 66 - .../handlers/users/create_user_handler.rs | 83 - .../handlers/users/delete_user_handler.rs | 57 - .../binary/handlers/users/get_user_handler.rs | 62 - .../handlers/users/get_users_handler.rs | 53 - .../handlers/users/login_user_handler.rs | 71 - .../handlers/users/logout_user_handler.rs | 44 - core/server/src/binary/handlers/users/mod.rs | 28 - .../users/update_permissions_handler.rs | 59 - .../handlers/users/update_user_handler.rs | 65 - core/server/src/binary/mod.rs | 21 - core/server/src/bootstrap.rs | 5053 +++++++++++++++-- .../{server-ng => server}/src/cluster_meta.rs | 4 +- .../index_rebuilding/index_rebuilder.rs | 118 - .../server/src/compat/index_rebuilding/mod.rs | 18 - core/server/src/compat/mod.rs | 18 - .../src/config_writer.rs | 19 +- core/server/src/configs.rs | 21 - .../src/consumer_group.rs | 10 +- core/server/src/diagnostics.rs | 22 - core/{server-ng => server}/src/dispatch.rs | 231 +- .../src/dispatch/authz.rs | 8 +- core/{server-ng => server}/src/http.rs | 22 +- .../src/http/admission.rs | 0 core/server/src/http/consumer_groups.rs | 176 - core/server/src/http/consumer_offsets.rs | 150 - core/server/src/http/diagnostics.rs | 68 - core/server/src/http/error.rs | 780 ++- .../src/http/extractor.rs | 0 .../{server-ng => server}/src/http/forward.rs | 14 +- .../src/http/handlers.rs | 4 +- core/server/src/http/http_server.rs | 399 -- core/server/src/http/http_shard_wrapper.rs | 232 - core/{server-ng => server}/src/http/jwks.rs | 0 core/{server-ng => server}/src/http/jwt.rs | 8 +- core/server/src/http/jwt/json_web_token.rs | 155 - core/server/src/http/jwt/jwks.rs | 357 -- core/server/src/http/jwt/jwt_manager.rs | 457 -- core/server/src/http/jwt/middleware.rs | 105 - core/server/src/http/jwt/mod.rs | 24 - core/server/src/http/jwt/storage.rs | 149 - core/server/src/http/mapper.rs | 329 -- core/server/src/http/messages.rs | 164 - core/server/src/http/metrics.rs | 296 +- core/server/src/http/mod.rs | 40 - core/server/src/http/partitions.rs | 104 - .../server/src/http/personal_access_tokens.rs | 148 - core/{server-ng => server}/src/http/reads.rs | 18 +- core/{server-ng => server}/src/http/reply.rs | 4 +- core/server/src/http/segments.rs | 120 - .../{server-ng => server}/src/http/session.rs | 6 +- core/{server-ng => server}/src/http/state.rs | 14 +- core/server/src/http/streams.rs | 200 - core/{server-ng => server}/src/http/submit.rs | 18 +- core/server/src/http/system.rs | 168 - core/{server-ng => server}/src/http/tls.rs | 24 +- core/server/src/http/topics.rs | 263 - core/server/src/http/users.rs | 331 -- core/server/src/http/web.rs | 83 - core/{server-ng => server}/src/http/wire.rs | 14 +- core/server/src/io/mod.rs | 20 - core/server/src/io/storage.rs | 171 - core/server/src/lib.rs | 56 +- .../src/login_register.rs | 0 core/server/src/main.rs | 591 +- core/server/src/metadata/absorb.rs | 500 -- core/server/src/metadata/consumer_group.rs | 315 - .../src/metadata/consumer_group_member.rs | 58 - core/server/src/metadata/inner.rs | 54 - core/server/src/metadata/mod.rs | 71 - core/server/src/metadata/ops.rs | 134 - core/server/src/metadata/partition.rs | 39 - core/server/src/metadata/reader.rs | 1920 ------- core/server/src/metadata/stream.rs | 64 - core/server/src/metadata/topic.rs | 72 - core/server/src/metadata/writer.rs | 617 -- .../src/offset_recovery.rs | 61 +- .../src/partition_helpers.rs | 66 +- .../src/partition_reconciler.rs | 289 +- core/{server-ng => server}/src/pat.rs | 24 +- .../src/personal_access_token_cleaner.rs | 6 +- core/server/src/quic/listener.rs | 239 - core/server/src/quic/mod.rs | 22 - core/server/src/quic/quic_server.rs | 222 - core/server/src/quic/quic_socket.rs | 66 - core/{server-ng => server}/src/responses.rs | 70 +- .../src/segment_cleaner.rs | 6 +- .../src/segment_recovery.rs | 30 +- core/server/src/sender/mod.rs | 257 - core/server/src/sender/quic_sender.rs | 140 - core/server/src/sender/tcp_sender.rs | 90 - core/server/src/sender/tcp_tls_sender.rs | 96 - core/server/src/sender/websocket_sender.rs | 206 - .../server/src/sender/websocket_tls_sender.rs | 185 - core/server/src/server_error.rs | 453 +- .../src/session_manager.rs | 8 +- core/server/src/shard/builder.rs | 197 - core/server/src/shard/communication.rs | 198 - core/server/src/shard/execution.rs | 732 --- core/server/src/shard/handlers.rs | 614 -- core/server/src/shard/mod.rs | 482 -- core/server/src/shard/system/clients.rs | 92 - core/server/src/shard/system/cluster.rs | 196 - .../src/shard/system/consumer_groups.rs | 205 - .../src/shard/system/consumer_offsets.rs | 489 -- core/server/src/shard/system/info.rs | 78 - core/server/src/shard/system/messages.rs | 695 --- core/server/src/shard/system/mod.rs | 35 - core/server/src/shard/system/partitions.rs | 443 -- .../shard/system/personal_access_tokens.rs | 149 - core/server/src/shard/system/segments.rs | 586 -- core/server/src/shard/system/snapshot/mod.rs | 257 - .../src/shard/system/snapshot/procdump.rs | 213 - core/server/src/shard/system/stats.rs | 120 - core/server/src/shard/system/storage.rs | 99 - core/server/src/shard/system/streams.rs | 179 - core/server/src/shard/system/topics.rs | 250 - core/server/src/shard/system/users.rs | 301 - core/server/src/shard/system/utils.rs | 255 - core/server/src/shard/systemd.rs | 45 - .../src/shard/task_registry/builders.rs | 42 - .../task_registry/builders/continuous.rs | 109 - .../shard/task_registry/builders/oneshot.rs | 120 - .../shard/task_registry/builders/periodic.rs | 135 - core/server/src/shard/task_registry/mod.rs | 23 - .../src/shard/task_registry/registry.rs | 732 --- .../src/shard/task_registry/shutdown.rs | 232 - .../src/shard/tasks/continuous/http_server.rs | 40 - .../shard/tasks/continuous/message_pump.rs | 135 - core/server/src/shard/tasks/continuous/mod.rs | 28 - .../src/shard/tasks/continuous/quic_server.rs | 36 - .../src/shard/tasks/continuous/tcp_server.rs | 36 - .../tasks/continuous/websocket_server.rs | 38 - core/server/src/shard/tasks/mod.rs | 20 - .../src/shard/tasks/oneshot/config_writer.rs | 142 - core/server/src/shard/tasks/oneshot/mod.rs | 20 - .../tasks/periodic/heartbeat_verifier.rs | 86 - .../shard/tasks/periodic/jwt_token_cleaner.rs | 60 - .../shard/tasks/periodic/message_cleaner.rs | 120 - .../src/shard/tasks/periodic/message_saver.rs | 71 - core/server/src/shard/tasks/periodic/mod.rs | 36 - .../periodic/personal_access_token_cleaner.rs | 85 - .../tasks/periodic/revocation_timeout.rs | 105 - .../shard/tasks/periodic/sysinfo_printer.rs | 103 - .../src/shard/transmission/connector.rs | 106 - core/server/src/shard/transmission/event.rs | 70 - core/server/src/shard/transmission/frame.rs | 116 - core/server/src/shard/transmission/message.rs | 254 - core/server/src/shard/transmission/mod.rs | 21 - core/{server-ng => server}/src/snapshot.rs | 14 +- .../src/snapshot/procdump.rs | 0 core/server/src/state/command.rs | 268 - core/server/src/state/entry.rs | 150 - core/server/src/state/file.rs | 378 -- core/server/src/state/models.rs | 334 -- core/server/src/state/system.rs | 633 --- .../src/streaming/clients/client_manager.rs | 233 - core/server/src/streaming/clients/mod.rs | 18 - .../server/src/streaming/deduplication/mod.rs | 18 - .../src/streaming/diagnostics/metrics.rs | 149 - core/server/src/streaming/diagnostics/mod.rs | 18 - core/server/src/streaming/mod.rs | 31 - .../partitions/consumer_group_offsets.rs | 18 - .../streaming/partitions/consumer_offset.rs | 18 - .../streaming/partitions/consumer_offsets.rs | 18 - .../src/streaming/partitions/helpers.rs | 36 - .../src/streaming/partitions/in_flight.rs | 18 - .../src/streaming/partitions/journal.rs | 212 - .../streaming/partitions/local_partition.rs | 98 - .../streaming/partitions/local_partitions.rs | 213 - core/server/src/streaming/partitions/log.rs | 207 - core/server/src/streaming/partitions/mod.rs | 33 - core/server/src/streaming/partitions/ops.rs | 730 --- .../src/streaming/partitions/ops_tests.rs | 358 -- .../src/streaming/partitions/segments.rs | 123 - .../src/streaming/partitions/storage.rs | 346 -- core/server/src/streaming/persistence/mod.rs | 20 - .../src/streaming/persistence/persister.rs | 166 - core/server/src/streaming/polling_consumer.rs | 129 - .../segments/indexes/index_reader.rs | 19 - .../segments/indexes/index_writer.rs | 18 - .../src/streaming/segments/indexes/mod.rs | 22 - .../src/streaming/segments/memory_journal.rs | 17 - .../segments/messages/messages_reader.rs | 19 - .../segments/messages/messages_writer.rs | 18 - .../src/streaming/segments/messages/mod.rs | 21 - core/server/src/streaming/segments/mod.rs | 34 - core/server/src/streaming/segments/segment.rs | 18 - core/server/src/streaming/segments/storage.rs | 50 - .../src/streaming/segments/types/mod.rs | 19 - core/server/src/streaming/session.rs | 105 - core/server/src/streaming/stats/mod.rs | 18 - core/server/src/streaming/storage.rs | 39 - core/server/src/streaming/streams/mod.rs | 20 - core/server/src/streaming/streams/storage.rs | 65 - core/server/src/streaming/topics/helpers.rs | 33 - core/server/src/streaming/topics/mod.rs | 21 - core/server/src/streaming/topics/storage.rs | 82 - core/server/src/streaming/users/mod.rs | 18 - core/server/src/streaming/users/user.rs | 137 - core/server/src/streaming/utils/address.rs | 75 - core/server/src/streaming/utils/file.rs | 54 - core/server/src/streaming/utils/mod.rs | 22 - core/server/src/streaming/utils/ptr.rs | 70 - core/server/src/systemd.rs | 80 + core/server/src/tcp/connection_handler.rs | 184 - core/server/src/tcp/mod.rs | 73 - core/server/src/tcp/tcp_listener.rs | 171 - core/server/src/tcp/tcp_server.rs | 56 - core/server/src/tcp/tcp_socket.rs | 97 - core/server/src/tcp/tcp_tls_listener.rs | 224 - core/{server-ng => server}/src/users.rs | 6 +- core/{server-ng => server}/src/web.rs | 0 .../src/websocket/connection_handler.rs | 175 - core/server/src/websocket/mod.rs | 64 - .../src/websocket/websocket_listener.rs | 182 - core/server/src/websocket/websocket_server.rs | 59 - .../src/websocket/websocket_tls_listener.rs | 272 - core/server/src/wire.rs | 161 + core/{server-ng => server}/tests/sdk_e2e.rs | 18 +- core/server_common/src/consensus_message.rs | 237 +- core/server_common/src/indexes_mut.rs | 2 +- core/server_common/src/send_messages2.rs | 256 +- core/server_common/src/sharding/mod.rs | 7 +- core/server_common/src/sharding/namespace.rs | 12 +- core/shard/Cargo.toml | 2 +- core/shard/src/lib.rs | 2375 ++++++-- core/shard/src/metrics.rs | 12 + core/shard/src/router.rs | 46 +- core/shard_allocator/src/lib.rs | 54 +- core/simulator/Cargo.toml | 2 +- core/simulator/src/client.rs | 158 +- core/simulator/src/deps.rs | 49 + core/simulator/src/lib.rs | 205 +- core/simulator/src/replica.rs | 24 +- core/simulator/src/workload/auditor.rs | 17 +- core/simulator/src/workload/effect.rs | 10 + core/simulator/src/workload/mod.rs | 9 +- .../src/workload/ops/change_password.rs | 4 +- .../src/workload/ops/create_consumer_group.rs | 4 +- .../src/workload/ops/create_partitions.rs | 21 +- .../ops/create_personal_access_token.rs | 4 +- .../src/workload/ops/create_stream.rs | 4 +- .../src/workload/ops/create_topic.rs | 4 +- .../simulator/src/workload/ops/create_user.rs | 4 +- .../src/workload/ops/delete_consumer_group.rs | 4 +- .../workload/ops/delete_consumer_offset.rs | 4 +- .../workload/ops/delete_consumer_offset_2.rs | 4 +- .../src/workload/ops/delete_partitions.rs | 60 +- .../ops/delete_personal_access_token.rs | 4 +- .../src/workload/ops/delete_segments.rs | 4 +- .../src/workload/ops/delete_stream.rs | 4 +- .../src/workload/ops/delete_topic.rs | 4 +- .../simulator/src/workload/ops/delete_user.rs | 4 +- core/simulator/src/workload/ops/mod.rs | 6 +- .../src/workload/ops/purge_stream.rs | 4 +- .../simulator/src/workload/ops/purge_topic.rs | 4 +- .../src/workload/ops/send_messages.rs | 4 +- .../src/workload/ops/store_consumer_offset.rs | 6 +- .../workload/ops/store_consumer_offset_2.rs | 6 +- .../src/workload/ops/update_permissions.rs | 4 +- .../src/workload/ops/update_stream.rs | 4 +- .../src/workload/ops/update_topic.rs | 4 +- .../simulator/src/workload/ops/update_user.rs | 4 +- core/simulator/src/workload/shadow.rs | 113 +- examples/go/README.md | 9 +- examples/java/README.md | 70 +- examples/node/src/tcp-tls/consumer.ts | 1 + examples/node/src/tcp-tls/producer.ts | 16 +- examples/python/getting-started/consumer.py | 52 +- examples/python/getting-started/producer.py | 50 +- foreign/cpp/tests/e2e/client.cpp | 91 +- foreign/cpp/tests/e2e/consumer_group.cpp | 44 +- foreign/cpp/tests/e2e/message.cpp | 7 +- foreign/csharp/Directory.Packages.props | 7 +- .../ClusterRedirectionTests.cs | 27 +- .../ConsumerGroupTests.cs | 11 +- .../FetchMessagesTests.cs | 27 +- .../Fixtures/IggyClusterFixture.cs | 250 - .../Fixtures/IggyServerFixture.cs | 256 +- .../Fixtures/IggyTlsServerFixture.cs | 7 +- .../Fixtures/RedirectionClusterFixture.cs | 22 +- .../Fixtures/VsrCluster.cs | 438 ++ .../FlushMessagesTests.cs | 61 +- .../HeaderEncryptionIntegrationTests.cs | 74 +- .../Helpers/Eventually.cs | 43 + .../IggyConsumerTests.cs | 94 +- .../IggyPublisherTests.cs | 93 +- .../IggyTlsConnectionTests.cs | 10 +- .../IggyTypedConsumerTests.cs | 20 +- .../IggyTypedPublisherTests.cs | 8 +- .../MessageEncryptionIntegrationTests.cs | 4 +- .../Iggy_SDK.Tests.Integration/OffsetTests.cs | 27 +- .../PartitionsTests.cs | 6 +- .../PersonalAccessTokenTests.cs | 2 +- .../RawCommandTests.cs | 12 +- .../SegmentsTests.cs | 12 +- .../SendMessagesTests.cs | 143 +- .../StreamsTests.cs | 5 +- .../Iggy_SDK.Tests.Integration/SystemTests.cs | 12 +- .../Iggy_SDK.Tests.Integration/TopicsTests.cs | 37 +- .../Iggy_SDK.Tests.Integration/UsersTests.cs | 16 +- .../Vsr/VsrConsumerGroupTests.cs | 228 + .../Vsr/VsrHandshakeTests.cs | 164 + .../Vsr/VsrMessagingTests.cs | 165 + .../Vsr/VsrMetadataTests.cs | 132 + .../Configuration/AutoLoginSettings.cs | 15 + .../Configuration/IggyClientConfigurator.cs | 6 + .../Iggy_SDK/Consumers/IggyConsumer.Rented.cs | 8 +- .../csharp/Iggy_SDK/Consumers/IggyConsumer.cs | 10 +- .../Iggy_SDK/Consumers/IggyConsumerBuilder.cs | 5 +- .../Consumers/IggyConsumerBuilderOfT.cs | 10 +- .../Contracts/SendMessagesResponse.cs | 77 + .../IggyInvalidStatusCodeException.cs | 12 +- .../VsrRequestOutcomeUnknownException.cs | 33 + .../Exceptions/VsrSessionEvictedException.cs | 34 + .../Iggy_SDK/Factory/IggyClientFactory.cs | 17 +- .../Iggy_SDK/IggyClient/IIggyPublisher.cs | 25 +- .../csharp/Iggy_SDK/IggyClient/IIggySystem.cs | 5 +- .../Implementations/HttpMessageStream.cs | 100 +- .../Implementations/TcpMessageStream.Vsr.cs | 978 ++++ .../Implementations/TcpMessageStream.cs | 499 +- .../TransientHttpRetryHandler.cs | 109 + foreign/csharp/Iggy_SDK/Iggy_SDK.csproj | 3 +- .../csharp/Iggy_SDK/Mappers/BinaryMapper.cs | 65 +- .../Publishers/BackgroundMessageProcessor.cs | 43 +- .../Iggy_SDK/Publishers/IggyPublisher.cs | 51 +- .../Publishers/IggyPublisherBuilder.cs | 3 +- .../Publishers/IggyPublisherBuilderOfT.cs | 9 +- .../Iggy_SDK/Publishers/IggyPublisherOfT.cs | 35 +- foreign/csharp/Iggy_SDK/Utils/BufferSizes.cs | 1 - foreign/csharp/Iggy_SDK/Utils/CommandCodes.cs | 3 + .../csharp/Iggy_SDK/Utils/ServerAddress.cs | 139 + .../csharp/Iggy_SDK/Vsr/Command2.cs | 22 +- .../csharp/Iggy_SDK/Vsr/ConsensusSession.cs | 223 + .../Iggy_SDK/Vsr/ConsumerGroupClientState.cs | 301 + .../csharp/Iggy_SDK/Vsr/CredentialBounds.cs | 66 + .../csharp/Iggy_SDK/Vsr/EvictionReason.cs | 38 +- foreign/csharp/Iggy_SDK/Vsr/LoginRegister.cs | 168 + .../csharp/Iggy_SDK/Vsr/SyncConsumerGroup.cs | 61 + foreign/csharp/Iggy_SDK/Vsr/VsrError.cs | 61 + foreign/csharp/Iggy_SDK/Vsr/VsrHeader.cs | 156 + foreign/csharp/Iggy_SDK/Vsr/VsrOperation.cs | 275 + .../csharp/Iggy_SDK/Vsr/VsrReplyDecoder.cs | 206 + .../ClientTests/IggyClientFactoryTests.cs | 67 + .../ConsumerTests/IggyConsumerBuilderTests.cs | 19 + .../MapperTests/BinaryMapper.cs | 76 + .../IggyPublisherBuilderTests.cs | 47 + .../PublisherTests/IggyTypedPublisherTests.cs | 4 +- .../UtilityTests/SendUnitTests.cs | 7 +- .../VsrTests/ConsensusSessionTests.cs | 218 + .../VsrTests/ConsumerGroupClientStateTests.cs | 176 + .../VsrTests/CredentialBoundsTests.cs | 72 + .../VsrTests/LoginRegisterTests.cs | 165 + .../VsrTests/ServerAddressTests.cs | 85 + .../VsrTests/SyncConsumerGroupTests.cs | 93 + .../Iggy_SDK_Tests/VsrTests/VsrHeaderTests.cs | 283 + .../VsrTests/VsrOperationTests.cs | 167 + .../VsrTests/VsrProtocolDriftTests.cs | 810 +++ .../VsrTests/VsrReplyDecoderTests.cs | 293 + .../VsrTests/VsrTestPayloads.cs | 102 + foreign/csharp/README.md | 100 +- foreign/csharp/scripts/pack.sh | 2 +- foreign/go/README.md | 10 +- foreign/go/client/tcp/tcp_connect_test.go | 1 - foreign/go/client/tcp/tcp_core_test.go | 10 +- foreign/go/client/tcp/tcp_testing_test.go | 31 +- foreign/go/internal/vsr/envelope.go | 17 +- foreign/go/internal/vsr/envelope_test.go | 47 +- foreign/go/internal/vsr/header.go | 17 +- foreign/go/internal/vsr/header_test.go | 19 +- foreign/go/internal/vsr/namespace.go | 246 - foreign/go/internal/vsr/namespace_test.go | 376 -- .../go/internal/vsr/protocol_parity_test.go | 96 +- foreign/go/tests/e2e_helpers_test.go | 5 +- foreign/java/README.md | 3 +- .../async/TcpAsyncPinnedProducerActor.java | 1 - .../bench/report/ServerStatsCollector.java | 1 - .../connector/flink/source/IggySource.java | 1 - .../example/AsyncTcpMessageSendTest.java | 17 +- .../iggy-connector-pinot/integration-test.sh | 2 +- foreign/java/gradle.properties | 2 +- foreign/java/gradle/libs.versions.toml | 2 +- .../client/async/ConsumerGroupsClient.java | 23 + .../iggy/client/async/MessagesClient.java | 22 +- .../client/async/tcp/AsyncIggyTcpClient.java | 288 +- .../async/tcp/AsyncIggyTcpClientBuilder.java | 71 +- .../client/async/tcp/AsyncTcpConnection.java | 768 ++- .../client/async/tcp/ClientRoutingState.java | 159 + .../async/tcp/ConsumerGroupsTcpClient.java | 19 + .../client/async/tcp/IggyAuthenticator.java | 30 +- .../client/async/tcp/IggyFrameDecoder.java | 73 - .../client/async/tcp/IggyFrameEncoder.java | 61 - ...rectionHook.java => LoginRoutingHook.java} | 21 +- .../client/async/tcp/MessagesTcpClient.java | 245 +- .../tcp/PersonalAccessTokensTcpClient.java | 16 +- .../iggy/client/async/tcp/ReconnectPlan.java | 57 + .../iggy/client/async/tcp/UsersTcpClient.java | 15 +- .../iggy/client/async/tcp/package-info.java | 13 +- .../async/tcp/vsr/ConsensusSession.java | 128 + .../client/async/tcp/vsr/VsrFrameDecoder.java | 67 + .../iggy/client/async/tcp/vsr/VsrHeaders.java | 146 + .../client/async/tcp/vsr/VsrLoginCodec.java | 128 + .../client/async/tcp/vsr/VsrOperation.java | 189 + .../async/tcp/vsr/VsrRequestEncoder.java | 112 + .../async/tcp/vsr/VsrResponseHandler.java | 261 + .../client/blocking/ConsumerGroupsClient.java | 11 + .../iggy/client/blocking/MessagesClient.java | 9 +- .../http/ConsumerGroupsHttpClient.java | 6 + .../blocking/http/MessagesHttpClient.java | 10 +- .../blocking/tcp/ConsumerGroupsTcpClient.java | 6 + .../blocking/tcp/IggyTcpClientBuilder.java | 11 - .../blocking/tcp/MessagesTcpClient.java | 6 +- .../ConsumerGroupAssignment.java | 36 + .../apache/iggy/exception/IggyErrorCode.java | 9 +- .../exception/IggyValidationException.java | 1 + .../java/org/apache/iggy/hash/XxHash32.java | 102 + .../apache/iggy/message/SendConfirmation.java | 33 + .../iggy/message/SendMessagesResponse.java | 37 + .../apache/iggy/serde/BytesDeserializer.java | 52 + .../org/apache/iggy/serde/CommandCode.java | 3 +- .../iggy/client/BaseIntegrationTest.java | 19 + .../async/AsyncClientIntegrationTest.java | 3 +- .../async/AsyncConnectionPoolAuthTest.java | 36 +- .../client/async/AsyncConsumerGroupsTest.java | 20 +- .../tcp/AsyncIggyTcpClientBuilderTest.java | 88 +- .../AsyncIggyTcpClientLoginRoutingTest.java | 180 + ...yncIggyTcpClientTransientFailoverTest.java | 337 ++ .../AsyncTcpConnectionConcurrencyTest.java | 409 ++ .../tcp/AsyncTcpConnectionHeartbeatTest.java | 132 + .../AsyncTcpConnectionRequestTimeoutTest.java | 125 + .../async/tcp/ClientRoutingStateTest.java | 251 + .../async/tcp/IggyFrameDecoderTest.java | 455 -- .../async/tcp/IggyResponseHandlerTest.java | 60 - .../async/tcp/LoginRoutingHookTest.java | 95 + .../client/async/tcp/ReconnectPlanTest.java | 66 + .../async/tcp/vsr/VsrFrameDecoderTest.java | 118 + .../async/tcp/vsr/VsrRequestEncoderTest.java | 225 + .../async/tcp/vsr/VsrResponseHandlerTest.java | 334 ++ .../ConsumerOffsetsClientBaseTest.java | 4 +- .../blocking/MessagesClientBaseTest.java | 108 +- .../client/blocking/SystemClientBaseTest.java | 2 + .../client/blocking/UsersClientBaseTest.java | 3 +- .../tcp/ConsumerGroupsTcpClientTest.java | 86 + .../tcp/IggyTcpClientBuilderTest.java | 16 - .../blocking/tcp/MessagesTcpClientTest.java | 31 + .../iggy/exception/IggyErrorCodeTest.java | 9 +- .../exception/IggyServerExceptionTest.java | 9 + .../IggyValidationExceptionTest.java | 2 + .../org/apache/iggy/hash/XxHash32Test.java | 73 + .../iggy/serde/BytesDeserializerTest.java | 146 + foreign/node/CHANGELOG.md | 95 - foreign/node/README.md | 25 +- foreign/node/docker-compose.yml | 7 +- foreign/node/package-lock.json | 4 +- foreign/node/package.json | 3 +- foreign/node/scripts/check-vsr-protocol.mjs | 52 +- foreign/node/src/bdd/auth.ts | 3 - foreign/node/src/client/client.config.test.ts | 31 +- foreign/node/src/client/client.config.ts | 17 +- .../node/src/client/client.connection.test.ts | 37 +- foreign/node/src/client/client.connection.ts | 17 +- foreign/node/src/client/client.frame.test.ts | 113 +- foreign/node/src/client/client.frame.ts | 40 +- foreign/node/src/client/client.socket.test.ts | 1 - foreign/node/src/client/client.socket.ts | 29 +- foreign/node/src/client/client.type.ts | 7 - foreign/node/src/client/client.utils.test.ts | 31 +- foreign/node/src/client/client.utils.ts | 68 - foreign/node/src/e2e/tcp.cluster.e2e.ts | 21 +- .../node/src/e2e/tcp.consumer-group.e2e.ts | 6 +- .../node/src/e2e/tcp.consumer-stream.e2e.ts | 6 +- foreign/node/src/e2e/tcp.send-message.e2e.ts | 12 - foreign/node/src/e2e/test-client.utils.ts | 2 - foreign/node/src/e2e/tls.system.e2e.ts | 4 - foreign/node/src/wire/command-set.test.ts | 1 - .../src/wire/message/poll-messages.command.ts | 3 +- foreign/node/src/wire/vsr/header.test.ts | 8 - foreign/node/src/wire/vsr/header.ts | 23 +- foreign/node/src/wire/vsr/index.ts | 3 - foreign/node/src/wire/vsr/namespace.test.ts | 342 -- foreign/node/src/wire/vsr/namespace.ts | 216 - foreign/node/src/wire/vsr/vsr.test.ts | 17 - foreign/php/README.md | 2 +- foreign/php/docker-compose.test.yml | 6 +- foreign/php/iggy-php.stubs.php | 14 +- foreign/php/scripts/test.sh | 2 +- foreign/php/src/client.rs | 5 +- foreign/php/src/send_message.rs | 9 +- foreign/php/tests/IggySdkTest.php | 16 +- foreign/python/Cargo.toml | 5 +- foreign/python/README.md | 36 + foreign/python/apache_iggy.pyi | 263 +- foreign/python/docker-compose.test.yml | 8 +- foreign/python/scripts/test.sh | 2 +- foreign/python/src/client.rs | 135 +- foreign/python/src/config.rs | 423 ++ foreign/python/src/consumer.rs | 51 +- foreign/python/src/duration.rs | 65 + foreign/python/src/lib.rs | 8 +- foreign/python/src/receive_message.rs | 2 +- foreign/python/src/send_message.rs | 2 - foreign/python/src/topic.rs | 33 +- foreign/python/tests/conftest.py | 5 + foreign/python/tests/test_client_config.py | 459 ++ foreign/python/tests/test_tls.py | 8 +- foreign/python/tests/test_topic.py | 13 +- foreign/python/tests/utils.py | 33 +- justfile | 17 - scripts/bump-version.sh | 2 +- scripts/check-backwards-compat.sh | 260 - scripts/ci/binary-artifacts.sh | 3 + scripts/ci/lib/init.sh | 51 + scripts/ci/license-headers.sh | 3 + scripts/ci/markdownlint.sh | 3 + scripts/ci/python-sdk-version-sync.sh | 3 + scripts/ci/shellcheck.sh | 3 + scripts/ci/skills.sh | 3 + scripts/ci/sync-python-interpreter-version.sh | 3 + scripts/ci/sync-rustc-version.sh | 11 +- scripts/ci/taplo.sh | 3 + scripts/ci/trailing-newline.sh | 3 + scripts/ci/trailing-whitespace.sh | 3 + scripts/ci/uv-lock-check.sh | 3 + scripts/copy-latest-from-master.sh | 2 +- scripts/dashboard/build_release.sh | 2 +- scripts/dashboard/run_dev.sh | 2 +- scripts/extract-version.sh | 5 +- .../run-standard-performance-suite.sh | 2 +- scripts/performance/utils.sh | 2 +- scripts/profile.sh | 2 +- scripts/run-bdd-tests.sh | 46 +- scripts/run-benches.sh | 2 +- scripts/run-examples-from-readme.sh | 41 +- scripts/utils.sh | 26 +- 822 files changed, 39202 insertions(+), 57739 deletions(-) delete mode 100644 .github/actions/utils/docker-build-test-server/action.yml delete mode 100644 bdd/docker-compose.vsr.yml rename core/configs/src/{server_config => common}/cache_indexes.rs (100%) create mode 100644 core/configs/src/common/defaults.rs create mode 100644 core/configs/src/common/displays.rs rename core/configs/src/{server_config => common}/http.rs (100%) rename core/{server/src/state => configs/src/common}/mod.rs (70%) create mode 100644 core/configs/src/common/server.rs rename core/configs/src/{server_config => common}/system.rs (96%) create mode 100644 core/configs/src/common/validators.rs rename core/configs/src/{server_ng_config => server_config}/message_bus.rs (87%) rename core/configs/src/{server_ng_config => server_config}/metadata.rs (74%) rename core/configs/src/{server_ng_config => server_config}/partition.rs (81%) delete mode 100644 core/configs/src/server_ng_config/cluster.rs delete mode 100644 core/configs/src/server_ng_config/defaults.rs delete mode 100644 core/configs/src/server_ng_config/displays.rs delete mode 100644 core/configs/src/server_ng_config/mod.rs delete mode 100644 core/configs/src/server_ng_config/quic.rs delete mode 100644 core/configs/src/server_ng_config/server_ng.rs delete mode 100644 core/configs/src/server_ng_config/sharding.rs delete mode 100644 core/configs/src/server_ng_config/tcp.rs delete mode 100644 core/configs/src/server_ng_config/validators.rs delete mode 100644 core/configs/src/server_ng_config/websocket.rs create mode 100644 core/consensus/src/dvc_merge.rs create mode 100644 core/integration/tests/server/login_credentials_vsr.rs delete mode 100644 core/integration/tests/server/scenarios/bench_scenario.rs rename core/{server/src/shard/tasks/periodic/systemd_watchdog.rs => integration/tests/server/stats_vsr.rs} (51%) delete mode 100644 core/integration/tests/state/file.rs delete mode 100644 core/integration/tests/state/mod.rs delete mode 100644 core/integration/tests/state/system.rs delete mode 100644 core/integration/tests/storage/consumer_offsets.rs delete mode 100644 core/integration/tests/storage/mod.rs delete mode 100644 core/server-ng/.dockerignore delete mode 100644 core/server-ng/Cargo.toml delete mode 100644 core/server-ng/Dockerfile delete mode 100644 core/server-ng/build.rs delete mode 100644 core/server-ng/config.toml delete mode 100644 core/server-ng/server.http delete mode 100644 core/server-ng/src/bootstrap.rs delete mode 100644 core/server-ng/src/http/error.rs delete mode 100644 core/server-ng/src/http/metrics.rs delete mode 100644 core/server-ng/src/lib.rs delete mode 100644 core/server-ng/src/main.rs delete mode 100644 core/server-ng/src/server_error.rs delete mode 100644 core/server-ng/src/wire.rs rename core/{server-ng => server}/src/auth.rs (97%) delete mode 100644 core/server/src/binary/dispatch.rs delete mode 100644 core/server/src/binary/handlers/cluster/get_cluster_metadata_handler.rs delete mode 100644 core/server/src/binary/handlers/cluster/mod.rs delete mode 100644 core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs delete mode 100644 core/server/src/binary/handlers/consumer_groups/delete_consumer_group_handler.rs delete mode 100644 core/server/src/binary/handlers/consumer_groups/get_consumer_group_handler.rs delete mode 100644 core/server/src/binary/handlers/consumer_groups/get_consumer_groups_handler.rs delete mode 100644 core/server/src/binary/handlers/consumer_groups/join_consumer_group_handler.rs delete mode 100644 core/server/src/binary/handlers/consumer_groups/leave_consumer_group_handler.rs delete mode 100644 core/server/src/binary/handlers/consumer_groups/mod.rs delete mode 100644 core/server/src/binary/handlers/consumer_offsets/delete_consumer_offset_handler.rs delete mode 100644 core/server/src/binary/handlers/consumer_offsets/get_consumer_offset_handler.rs delete mode 100644 core/server/src/binary/handlers/consumer_offsets/mod.rs delete mode 100644 core/server/src/binary/handlers/consumer_offsets/store_consumer_offset_handler.rs delete mode 100644 core/server/src/binary/handlers/messages/flush_unsaved_buffer_handler.rs delete mode 100644 core/server/src/binary/handlers/messages/mod.rs delete mode 100644 core/server/src/binary/handlers/messages/poll_messages_handler.rs delete mode 100644 core/server/src/binary/handlers/messages/send_messages_handler.rs delete mode 100644 core/server/src/binary/handlers/mod.rs delete mode 100644 core/server/src/binary/handlers/partitions/create_partitions_handler.rs delete mode 100644 core/server/src/binary/handlers/partitions/delete_partitions_handler.rs delete mode 100644 core/server/src/binary/handlers/partitions/mod.rs delete mode 100644 core/server/src/binary/handlers/personal_access_tokens/create_personal_access_token_handler.rs delete mode 100644 core/server/src/binary/handlers/personal_access_tokens/delete_personal_access_token_handler.rs delete mode 100644 core/server/src/binary/handlers/personal_access_tokens/get_personal_access_tokens_handler.rs delete mode 100644 core/server/src/binary/handlers/personal_access_tokens/login_with_personal_access_token_handler.rs delete mode 100644 core/server/src/binary/handlers/personal_access_tokens/mod.rs delete mode 100644 core/server/src/binary/handlers/segments/delete_segments_handler.rs delete mode 100644 core/server/src/binary/handlers/segments/mod.rs delete mode 100644 core/server/src/binary/handlers/streams/create_stream_handler.rs delete mode 100644 core/server/src/binary/handlers/streams/delete_stream_handler.rs delete mode 100644 core/server/src/binary/handlers/streams/get_stream_handler.rs delete mode 100644 core/server/src/binary/handlers/streams/get_streams_handler.rs delete mode 100644 core/server/src/binary/handlers/streams/mod.rs delete mode 100644 core/server/src/binary/handlers/streams/purge_stream_handler.rs delete mode 100644 core/server/src/binary/handlers/streams/update_stream_handler.rs delete mode 100644 core/server/src/binary/handlers/system/get_client_handler.rs delete mode 100644 core/server/src/binary/handlers/system/get_clients_handler.rs delete mode 100644 core/server/src/binary/handlers/system/get_me_handler.rs delete mode 100644 core/server/src/binary/handlers/system/get_snapshot_handler.rs delete mode 100644 core/server/src/binary/handlers/system/get_stats_handler.rs delete mode 100644 core/server/src/binary/handlers/system/mod.rs delete mode 100644 core/server/src/binary/handlers/system/ping_handler.rs delete mode 100644 core/server/src/binary/handlers/topics/create_topic_handler.rs delete mode 100644 core/server/src/binary/handlers/topics/delete_topic_handler.rs delete mode 100644 core/server/src/binary/handlers/topics/get_topic_handler.rs delete mode 100644 core/server/src/binary/handlers/topics/get_topics_handler.rs delete mode 100644 core/server/src/binary/handlers/topics/mod.rs delete mode 100644 core/server/src/binary/handlers/topics/purge_topic_handler.rs delete mode 100644 core/server/src/binary/handlers/topics/update_topic_handler.rs delete mode 100644 core/server/src/binary/handlers/users/change_password_handler.rs delete mode 100644 core/server/src/binary/handlers/users/create_user_handler.rs delete mode 100644 core/server/src/binary/handlers/users/delete_user_handler.rs delete mode 100644 core/server/src/binary/handlers/users/get_user_handler.rs delete mode 100644 core/server/src/binary/handlers/users/get_users_handler.rs delete mode 100644 core/server/src/binary/handlers/users/login_user_handler.rs delete mode 100644 core/server/src/binary/handlers/users/logout_user_handler.rs delete mode 100644 core/server/src/binary/handlers/users/mod.rs delete mode 100644 core/server/src/binary/handlers/users/update_permissions_handler.rs delete mode 100644 core/server/src/binary/handlers/users/update_user_handler.rs delete mode 100644 core/server/src/binary/mod.rs rename core/{server-ng => server}/src/cluster_meta.rs (98%) delete mode 100644 core/server/src/compat/index_rebuilding/index_rebuilder.rs delete mode 100644 core/server/src/compat/index_rebuilding/mod.rs delete mode 100644 core/server/src/compat/mod.rs rename core/{server-ng => server}/src/config_writer.rs (87%) delete mode 100644 core/server/src/configs.rs rename core/{server-ng => server}/src/consumer_group.rs (97%) delete mode 100644 core/server/src/diagnostics.rs rename core/{server-ng => server}/src/dispatch.rs (95%) rename core/{server-ng => server}/src/dispatch/authz.rs (98%) rename core/{server-ng => server}/src/http.rs (98%) rename core/{server-ng => server}/src/http/admission.rs (100%) delete mode 100644 core/server/src/http/consumer_groups.rs delete mode 100644 core/server/src/http/consumer_offsets.rs delete mode 100644 core/server/src/http/diagnostics.rs rename core/{server-ng => server}/src/http/extractor.rs (100%) rename core/{server-ng => server}/src/http/forward.rs (98%) rename core/{server-ng => server}/src/http/handlers.rs (99%) delete mode 100644 core/server/src/http/http_server.rs delete mode 100644 core/server/src/http/http_shard_wrapper.rs rename core/{server-ng => server}/src/http/jwks.rs (100%) rename core/{server-ng => server}/src/http/jwt.rs (98%) delete mode 100644 core/server/src/http/jwt/json_web_token.rs delete mode 100644 core/server/src/http/jwt/jwks.rs delete mode 100644 core/server/src/http/jwt/jwt_manager.rs delete mode 100644 core/server/src/http/jwt/middleware.rs delete mode 100644 core/server/src/http/jwt/mod.rs delete mode 100644 core/server/src/http/jwt/storage.rs delete mode 100644 core/server/src/http/mapper.rs delete mode 100644 core/server/src/http/messages.rs delete mode 100644 core/server/src/http/mod.rs delete mode 100644 core/server/src/http/partitions.rs delete mode 100644 core/server/src/http/personal_access_tokens.rs rename core/{server-ng => server}/src/http/reads.rs (95%) rename core/{server-ng => server}/src/http/reply.rs (99%) delete mode 100644 core/server/src/http/segments.rs rename core/{server-ng => server}/src/http/session.rs (98%) rename core/{server-ng => server}/src/http/state.rs (97%) delete mode 100644 core/server/src/http/streams.rs rename core/{server-ng => server}/src/http/submit.rs (97%) delete mode 100644 core/server/src/http/system.rs rename core/{server-ng => server}/src/http/tls.rs (92%) delete mode 100644 core/server/src/http/topics.rs delete mode 100644 core/server/src/http/users.rs delete mode 100644 core/server/src/http/web.rs rename core/{server-ng => server}/src/http/wire.rs (97%) delete mode 100644 core/server/src/io/mod.rs delete mode 100644 core/server/src/io/storage.rs rename core/{server-ng => server}/src/login_register.rs (100%) delete mode 100644 core/server/src/metadata/absorb.rs delete mode 100644 core/server/src/metadata/consumer_group.rs delete mode 100644 core/server/src/metadata/consumer_group_member.rs delete mode 100644 core/server/src/metadata/inner.rs delete mode 100644 core/server/src/metadata/mod.rs delete mode 100644 core/server/src/metadata/ops.rs delete mode 100644 core/server/src/metadata/partition.rs delete mode 100644 core/server/src/metadata/reader.rs delete mode 100644 core/server/src/metadata/stream.rs delete mode 100644 core/server/src/metadata/topic.rs delete mode 100644 core/server/src/metadata/writer.rs rename core/{server-ng => server}/src/offset_recovery.rs (73%) rename core/{server-ng => server}/src/partition_helpers.rs (94%) rename core/{server-ng => server}/src/partition_reconciler.rs (94%) rename core/{server-ng => server}/src/pat.rs (93%) rename core/{server-ng => server}/src/personal_access_token_cleaner.rs (96%) delete mode 100644 core/server/src/quic/listener.rs delete mode 100644 core/server/src/quic/mod.rs delete mode 100644 core/server/src/quic/quic_server.rs delete mode 100644 core/server/src/quic/quic_socket.rs rename core/{server-ng => server}/src/responses.rs (97%) rename core/{server-ng => server}/src/segment_cleaner.rs (97%) rename core/{server-ng => server}/src/segment_recovery.rs (96%) delete mode 100644 core/server/src/sender/mod.rs delete mode 100644 core/server/src/sender/quic_sender.rs delete mode 100644 core/server/src/sender/tcp_sender.rs delete mode 100644 core/server/src/sender/tcp_tls_sender.rs delete mode 100644 core/server/src/sender/websocket_sender.rs delete mode 100644 core/server/src/sender/websocket_tls_sender.rs rename core/{server-ng => server}/src/session_manager.rs (99%) delete mode 100644 core/server/src/shard/builder.rs delete mode 100644 core/server/src/shard/communication.rs delete mode 100644 core/server/src/shard/execution.rs delete mode 100644 core/server/src/shard/handlers.rs delete mode 100644 core/server/src/shard/mod.rs delete mode 100644 core/server/src/shard/system/clients.rs delete mode 100644 core/server/src/shard/system/cluster.rs delete mode 100644 core/server/src/shard/system/consumer_groups.rs delete mode 100644 core/server/src/shard/system/consumer_offsets.rs delete mode 100644 core/server/src/shard/system/info.rs delete mode 100644 core/server/src/shard/system/messages.rs delete mode 100644 core/server/src/shard/system/mod.rs delete mode 100644 core/server/src/shard/system/partitions.rs delete mode 100644 core/server/src/shard/system/personal_access_tokens.rs delete mode 100644 core/server/src/shard/system/segments.rs delete mode 100644 core/server/src/shard/system/snapshot/mod.rs delete mode 100644 core/server/src/shard/system/snapshot/procdump.rs delete mode 100644 core/server/src/shard/system/stats.rs delete mode 100644 core/server/src/shard/system/storage.rs delete mode 100644 core/server/src/shard/system/streams.rs delete mode 100644 core/server/src/shard/system/topics.rs delete mode 100644 core/server/src/shard/system/users.rs delete mode 100644 core/server/src/shard/system/utils.rs delete mode 100644 core/server/src/shard/systemd.rs delete mode 100644 core/server/src/shard/task_registry/builders.rs delete mode 100644 core/server/src/shard/task_registry/builders/continuous.rs delete mode 100644 core/server/src/shard/task_registry/builders/oneshot.rs delete mode 100644 core/server/src/shard/task_registry/builders/periodic.rs delete mode 100644 core/server/src/shard/task_registry/mod.rs delete mode 100644 core/server/src/shard/task_registry/registry.rs delete mode 100644 core/server/src/shard/task_registry/shutdown.rs delete mode 100644 core/server/src/shard/tasks/continuous/http_server.rs delete mode 100644 core/server/src/shard/tasks/continuous/message_pump.rs delete mode 100644 core/server/src/shard/tasks/continuous/mod.rs delete mode 100644 core/server/src/shard/tasks/continuous/quic_server.rs delete mode 100644 core/server/src/shard/tasks/continuous/tcp_server.rs delete mode 100644 core/server/src/shard/tasks/continuous/websocket_server.rs delete mode 100644 core/server/src/shard/tasks/mod.rs delete mode 100644 core/server/src/shard/tasks/oneshot/config_writer.rs delete mode 100644 core/server/src/shard/tasks/oneshot/mod.rs delete mode 100644 core/server/src/shard/tasks/periodic/heartbeat_verifier.rs delete mode 100644 core/server/src/shard/tasks/periodic/jwt_token_cleaner.rs delete mode 100644 core/server/src/shard/tasks/periodic/message_cleaner.rs delete mode 100644 core/server/src/shard/tasks/periodic/message_saver.rs delete mode 100644 core/server/src/shard/tasks/periodic/mod.rs delete mode 100644 core/server/src/shard/tasks/periodic/personal_access_token_cleaner.rs delete mode 100644 core/server/src/shard/tasks/periodic/revocation_timeout.rs delete mode 100644 core/server/src/shard/tasks/periodic/sysinfo_printer.rs delete mode 100644 core/server/src/shard/transmission/connector.rs delete mode 100644 core/server/src/shard/transmission/event.rs delete mode 100644 core/server/src/shard/transmission/frame.rs delete mode 100644 core/server/src/shard/transmission/message.rs delete mode 100644 core/server/src/shard/transmission/mod.rs rename core/{server-ng => server}/src/snapshot.rs (96%) rename core/{server-ng => server}/src/snapshot/procdump.rs (100%) delete mode 100644 core/server/src/state/command.rs delete mode 100644 core/server/src/state/entry.rs delete mode 100644 core/server/src/state/file.rs delete mode 100644 core/server/src/state/models.rs delete mode 100644 core/server/src/state/system.rs delete mode 100644 core/server/src/streaming/clients/client_manager.rs delete mode 100644 core/server/src/streaming/clients/mod.rs delete mode 100644 core/server/src/streaming/deduplication/mod.rs delete mode 100644 core/server/src/streaming/diagnostics/metrics.rs delete mode 100644 core/server/src/streaming/diagnostics/mod.rs delete mode 100644 core/server/src/streaming/mod.rs delete mode 100644 core/server/src/streaming/partitions/consumer_group_offsets.rs delete mode 100644 core/server/src/streaming/partitions/consumer_offset.rs delete mode 100644 core/server/src/streaming/partitions/consumer_offsets.rs delete mode 100644 core/server/src/streaming/partitions/helpers.rs delete mode 100644 core/server/src/streaming/partitions/in_flight.rs delete mode 100644 core/server/src/streaming/partitions/journal.rs delete mode 100644 core/server/src/streaming/partitions/local_partition.rs delete mode 100644 core/server/src/streaming/partitions/local_partitions.rs delete mode 100644 core/server/src/streaming/partitions/log.rs delete mode 100644 core/server/src/streaming/partitions/mod.rs delete mode 100644 core/server/src/streaming/partitions/ops.rs delete mode 100644 core/server/src/streaming/partitions/ops_tests.rs delete mode 100644 core/server/src/streaming/partitions/segments.rs delete mode 100644 core/server/src/streaming/partitions/storage.rs delete mode 100644 core/server/src/streaming/persistence/mod.rs delete mode 100644 core/server/src/streaming/persistence/persister.rs delete mode 100644 core/server/src/streaming/polling_consumer.rs delete mode 100644 core/server/src/streaming/segments/indexes/index_reader.rs delete mode 100644 core/server/src/streaming/segments/indexes/index_writer.rs delete mode 100644 core/server/src/streaming/segments/indexes/mod.rs delete mode 100644 core/server/src/streaming/segments/memory_journal.rs delete mode 100644 core/server/src/streaming/segments/messages/messages_reader.rs delete mode 100644 core/server/src/streaming/segments/messages/messages_writer.rs delete mode 100644 core/server/src/streaming/segments/messages/mod.rs delete mode 100644 core/server/src/streaming/segments/mod.rs delete mode 100644 core/server/src/streaming/segments/segment.rs delete mode 100644 core/server/src/streaming/segments/storage.rs delete mode 100644 core/server/src/streaming/segments/types/mod.rs delete mode 100644 core/server/src/streaming/session.rs delete mode 100644 core/server/src/streaming/stats/mod.rs delete mode 100644 core/server/src/streaming/storage.rs delete mode 100644 core/server/src/streaming/streams/mod.rs delete mode 100644 core/server/src/streaming/streams/storage.rs delete mode 100644 core/server/src/streaming/topics/helpers.rs delete mode 100644 core/server/src/streaming/topics/mod.rs delete mode 100644 core/server/src/streaming/topics/storage.rs delete mode 100644 core/server/src/streaming/users/mod.rs delete mode 100644 core/server/src/streaming/users/user.rs delete mode 100644 core/server/src/streaming/utils/address.rs delete mode 100644 core/server/src/streaming/utils/file.rs delete mode 100644 core/server/src/streaming/utils/mod.rs delete mode 100644 core/server/src/streaming/utils/ptr.rs create mode 100644 core/server/src/systemd.rs delete mode 100644 core/server/src/tcp/connection_handler.rs delete mode 100644 core/server/src/tcp/mod.rs delete mode 100644 core/server/src/tcp/tcp_listener.rs delete mode 100644 core/server/src/tcp/tcp_server.rs delete mode 100644 core/server/src/tcp/tcp_socket.rs delete mode 100644 core/server/src/tcp/tcp_tls_listener.rs rename core/{server-ng => server}/src/users.rs (98%) rename core/{server-ng => server}/src/web.rs (100%) delete mode 100644 core/server/src/websocket/connection_handler.rs delete mode 100644 core/server/src/websocket/mod.rs delete mode 100644 core/server/src/websocket/websocket_listener.rs delete mode 100644 core/server/src/websocket/websocket_server.rs delete mode 100644 core/server/src/websocket/websocket_tls_listener.rs create mode 100644 core/server/src/wire.rs rename core/{server-ng => server}/tests/sdk_e2e.rs (91%) delete mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyClusterFixture.cs rename core/server/src/http/shared.rs => foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/RedirectionClusterFixture.cs (65%) create mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/VsrCluster.cs create mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/Helpers/Eventually.cs create mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/Vsr/VsrConsumerGroupTests.cs create mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/Vsr/VsrHandshakeTests.cs create mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/Vsr/VsrMessagingTests.cs create mode 100644 foreign/csharp/Iggy_SDK.Tests.Integration/Vsr/VsrMetadataTests.cs create mode 100644 foreign/csharp/Iggy_SDK/Contracts/SendMessagesResponse.cs create mode 100644 foreign/csharp/Iggy_SDK/Exceptions/VsrRequestOutcomeUnknownException.cs create mode 100644 foreign/csharp/Iggy_SDK/Exceptions/VsrSessionEvictedException.cs create mode 100644 foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.Vsr.cs create mode 100644 foreign/csharp/Iggy_SDK/IggyClient/Implementations/TransientHttpRetryHandler.cs create mode 100644 foreign/csharp/Iggy_SDK/Utils/ServerAddress.cs rename core/server/src/metadata/user.rs => foreign/csharp/Iggy_SDK/Vsr/Command2.cs (69%) create mode 100644 foreign/csharp/Iggy_SDK/Vsr/ConsensusSession.cs create mode 100644 foreign/csharp/Iggy_SDK/Vsr/ConsumerGroupClientState.cs create mode 100644 foreign/csharp/Iggy_SDK/Vsr/CredentialBounds.cs rename core/server-ng/src/args.rs => foreign/csharp/Iggy_SDK/Vsr/EvictionReason.cs (53%) create mode 100644 foreign/csharp/Iggy_SDK/Vsr/LoginRegister.cs create mode 100644 foreign/csharp/Iggy_SDK/Vsr/SyncConsumerGroup.cs create mode 100644 foreign/csharp/Iggy_SDK/Vsr/VsrError.cs create mode 100644 foreign/csharp/Iggy_SDK/Vsr/VsrHeader.cs create mode 100644 foreign/csharp/Iggy_SDK/Vsr/VsrOperation.cs create mode 100644 foreign/csharp/Iggy_SDK/Vsr/VsrReplyDecoder.cs create mode 100644 foreign/csharp/Iggy_SDK_Tests/ClientTests/IggyClientFactoryTests.cs create mode 100644 foreign/csharp/Iggy_SDK_Tests/PublisherTests/IggyPublisherBuilderTests.cs create mode 100644 foreign/csharp/Iggy_SDK_Tests/VsrTests/ConsensusSessionTests.cs create mode 100644 foreign/csharp/Iggy_SDK_Tests/VsrTests/ConsumerGroupClientStateTests.cs create mode 100644 foreign/csharp/Iggy_SDK_Tests/VsrTests/CredentialBoundsTests.cs create mode 100644 foreign/csharp/Iggy_SDK_Tests/VsrTests/LoginRegisterTests.cs create mode 100644 foreign/csharp/Iggy_SDK_Tests/VsrTests/ServerAddressTests.cs create mode 100644 foreign/csharp/Iggy_SDK_Tests/VsrTests/SyncConsumerGroupTests.cs create mode 100644 foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrHeaderTests.cs create mode 100644 foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrOperationTests.cs create mode 100644 foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrProtocolDriftTests.cs create mode 100644 foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrReplyDecoderTests.cs create mode 100644 foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrTestPayloads.cs delete mode 100644 foreign/go/internal/vsr/namespace.go delete mode 100644 foreign/go/internal/vsr/namespace_test.go create mode 100644 foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ClientRoutingState.java delete mode 100644 foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/IggyFrameDecoder.java delete mode 100644 foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/IggyFrameEncoder.java rename foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/{LoginRedirectionHook.java => LoginRoutingHook.java} (56%) create mode 100644 foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ReconnectPlan.java create mode 100644 foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/ConsensusSession.java create mode 100644 foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrFrameDecoder.java create mode 100644 foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrHeaders.java create mode 100644 foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrLoginCodec.java create mode 100644 foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrOperation.java create mode 100644 foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrRequestEncoder.java create mode 100644 foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandler.java create mode 100644 foreign/java/java-sdk/src/main/java/org/apache/iggy/consumergroup/ConsumerGroupAssignment.java create mode 100644 foreign/java/java-sdk/src/main/java/org/apache/iggy/hash/XxHash32.java create mode 100644 foreign/java/java-sdk/src/main/java/org/apache/iggy/message/SendConfirmation.java create mode 100644 foreign/java/java-sdk/src/main/java/org/apache/iggy/message/SendMessagesResponse.java create mode 100644 foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientLoginRoutingTest.java create mode 100644 foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java create mode 100644 foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionConcurrencyTest.java create mode 100644 foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionHeartbeatTest.java create mode 100644 foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionRequestTimeoutTest.java create mode 100644 foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/ClientRoutingStateTest.java delete mode 100644 foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/IggyFrameDecoderTest.java delete mode 100644 foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/IggyResponseHandlerTest.java create mode 100644 foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/LoginRoutingHookTest.java create mode 100644 foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/ReconnectPlanTest.java create mode 100644 foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrFrameDecoderTest.java create mode 100644 foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrRequestEncoderTest.java create mode 100644 foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandlerTest.java create mode 100644 foreign/java/java-sdk/src/test/java/org/apache/iggy/hash/XxHash32Test.java delete mode 100644 foreign/node/CHANGELOG.md delete mode 100644 foreign/node/src/wire/vsr/namespace.test.ts delete mode 100644 foreign/node/src/wire/vsr/namespace.ts create mode 100644 foreign/python/src/config.rs create mode 100644 foreign/python/src/duration.rs create mode 100644 foreign/python/tests/test_client_config.py delete mode 100755 scripts/check-backwards-compat.sh create mode 100644 scripts/ci/lib/init.sh diff --git a/.claude/skills/connectors-overview/SKILL.md b/.claude/skills/connectors-overview/SKILL.md index 91ff86ec7c..230ad5354b 100644 --- a/.claude/skills/connectors-overview/SKILL.md +++ b/.claude/skills/connectors-overview/SKILL.md @@ -96,14 +96,24 @@ The connectors codebase is intentionally repetitive across plugins. Cross-plugin ### Secrets -Any credential-bearing field (connection strings, API keys, bearer tokens, AWS keys) must be `SecretString` from the `secrecy` crate, with the workspace serde wrapper applied so `Debug` and serialization both redact. Runtime exposes plugin configs over the `/stats` HTTP surface via serialization - plain `String` leaks the secret to anyone who can hit the endpoint. Plain `String` for a credential is a review-blocker. Pattern (from `sinks/postgres_sink/src/lib.rs::PostgresSinkConfig`): +Any credential-bearing field (connection strings, API keys, bearer tokens, AWS keys) must be `SecretString` from the `secrecy` crate. Plain `String` for a credential is a review-blocker: `SecretString` redacts on `Debug`, so it is what keeps a credential out of a log line that formats the whole config. + +**`serde_secret::serialize_secret` EXPOSES the secret. It does not redact.** It calls `expose_secret()` and writes the plaintext. `SecretString` deliberately has no `Serialize` impl, and that absence is the protection - so adding `serialize_with` is what *unblocks* the derive and turns a compile-time guarantee into plaintext output. Use it only where the plaintext is the point: a wire payload, a persisted config, an API response that exposes credentials by design. + +So the default for a plugin config struct is **derive `Deserialize`, but not `Serialize`**. `Deserialize` is required: the SDK glue deserializes the config into the plugin's own struct (`sdk/src/{sink,source}.rs` call `serde_json::from_str::` under a `DeserializeOwned` bound). + +What never happens is the return trip. The runtime holds plugin configuration as a `serde_json::Value` - parsed from TOML, posted as JSON to the control API, or injected by env var - and hands that across the FFI, so nothing re-serializes the plugin's struct. Leaving `Serialize` off makes that compiler-enforced instead of convention-enforced (`sources/http_source/src/lib.rs::HttpSourceConfig` does this, and comments the omission so nobody adds it back). + +Pattern: ```rust use secrecy::{ExposeSecret, SecretString}; -#[derive(Debug, Clone, Serialize, Deserialize)] +// `Deserialize` only. Nothing re-serializes a plugin config, and leaving +// `Serialize` off is what makes the credential unserializable rather than +// merely un-serialized. +#[derive(Debug, Clone, Deserialize)] pub struct MyConfig { - #[serde(serialize_with = "iggy_common::serde_secret::serialize_secret")] pub connection_string: SecretString, } @@ -113,7 +123,13 @@ let pool = PgPoolOptions::new() .await?; ``` -In-tree uses: `sinks/{postgres,mongodb,elasticsearch,influxdb,delta}_sink`, `sources/{postgres,elasticsearch,influxdb}_source`. +If a config struct genuinely needs `Serialize`, `serde_secret::serialize_redacted` (and `serialize_optional_redacted`) write `[REDACTED]` in place of the value. Reach for `serialize_secret` only when the caller must get the real thing back. The sinks and sources listed below predate that helper and use the exposing one; the annotation is inert today, but it is not the protection it looks like. + +Note that none of this protects the credential from the runtime's own control API, which returns plugin configuration verbatim - see #3802. Plugin-side annotations are inert there because the runtime never routes through them. + +Plugin-side uses of the exposing helpers: `sinks/{postgres,mongodb,elasticsearch,influxdb,s3,surrealdb}_sink`, `sources/{postgres,elasticsearch,influxdb}_source`. + +That list is plugin-side only, not an inventory of every caller in the tree, and the others are not all mistakes: `runtime/src/api/config.rs` puts `serialize_secret` on `HttpConfig::api_key` (inert for the same reason), and several `core/common` wire-payload types (login, create-user, change-password, PAT) use these helpers by design, because there the credential *is* the payload. ### Errors @@ -193,7 +209,7 @@ Each implemented in at least one in-tree plugin or runtime path. | `flume::unbounded()` channel | `runtime/src/source.rs::spawn_source_handler` / `source_forwarding_loop` | MPSC handoff from SDK async task to runtime loop | | `tokio::sync::watch::channel(())` | `sdk/src/{sink,source}.rs`, `runtime/src/sink.rs`, `runtime/src/manager/*` | One-shot shutdown broadcast | | `dashmap::DashMap` | `runtime/src/manager/sink.rs`, `source.rs::SOURCE_SENDERS`, SDK `INSTANCES` | Lock-free concurrent keyed access | -| `secrecy::SecretString` + `iggy_common::serde_secret::serialize_secret` | `sinks/postgres_sink::PostgresSinkConfig::connection_string` | Auto-redact on Debug/Display + serialization | +| `secrecy::SecretString` + `iggy_common::serde_secret::serialize_secret` | `sinks/postgres_sink::PostgresSinkConfig::connection_string` | `Debug` redacts; `serialize_secret` EXPOSES | ## Drop accounting diff --git a/.dockerignore b/.dockerignore index b527cb132d..02100b3874 100644 --- a/.dockerignore +++ b/.dockerignore @@ -25,7 +25,6 @@ /target !/target/debug/iggy !/target/debug/iggy-server -!/target/debug/iggy-server-ng !/target/debug/iggy-mcp !/target/debug/iggy-connectors /web/node_modules diff --git a/.github/actions/cpp-bazel/pre-merge/action.yml b/.github/actions/cpp-bazel/pre-merge/action.yml index adce777398..20e7e465b1 100644 --- a/.github/actions/cpp-bazel/pre-merge/action.yml +++ b/.github/actions/cpp-bazel/pre-merge/action.yml @@ -83,6 +83,8 @@ runs: - name: Setup server for e2e tests if: inputs.task == 'e2e' uses: ./.github/actions/utils/server-start + with: + cargo-bin: iggy-server - name: Run e2e tests if: inputs.task == 'e2e' diff --git a/.github/actions/csharp-dotnet/pre-merge/action.yml b/.github/actions/csharp-dotnet/pre-merge/action.yml index 143a157006..c4dc35bde7 100644 --- a/.github/actions/csharp-dotnet/pre-merge/action.yml +++ b/.github/actions/csharp-dotnet/pre-merge/action.yml @@ -78,11 +78,17 @@ runs: - name: Build Iggy server Docker image id: docker_build if: inputs.task == 'e2e' - uses: ./.github/actions/utils/docker-build-test-server - with: - image-tag: "iggy-server:test" - libc: "glibc" - profile: "debug" + shell: bash + run: | + cargo build --locked --bin iggy-server --bin iggy + docker build \ + -f core/server/Dockerfile \ + --target runtime-prebuilt \ + -t iggy-server:test \ + --build-arg PREBUILT_IGGY_SERVER=target/debug/iggy-server \ + --build-arg PREBUILT_IGGY_CLI=target/debug/iggy \ + . + echo "docker_image=iggy-server:test" >> "$GITHUB_OUTPUT" - name: Run integration tests if: inputs.task == 'e2e' @@ -90,6 +96,7 @@ runs: env: IGGY_SERVER_DOCKER_IMAGE: ${{ steps.docker_build.outputs.docker_image }} IGGY_TEST_LOGS_DIR: ./reports/container-logs + IGGY_TEST_CLUSTER_NODES: "3" run: | dotnet test --project Iggy_SDK.Tests.Integration \ --no-build \ @@ -112,7 +119,7 @@ runs: uses: actions/upload-artifact@v7 if: inputs.task == 'e2e' && always() with: - name: dotnet-test-results + name: dotnet-test-results-${{ inputs.task }} path: foreign/csharp/reports retention-days: 7 diff --git a/.github/actions/go/pre-merge/action.yml b/.github/actions/go/pre-merge/action.yml index cfb5ba7e62..56e0162b70 100644 --- a/.github/actions/go/pre-merge/action.yml +++ b/.github/actions/go/pre-merge/action.yml @@ -128,9 +128,7 @@ runs: if: inputs.task == 'e2e' uses: ./.github/actions/utils/server-start with: - # TODO: change to iggy-server once legacy server is removed (core/server has VSR support) - cargo-bin: iggy-server-ng - cargo-features: vsr + cargo-bin: iggy-server replica-id: "0" pid-file: ${{ runner.temp }}/iggy-go-e2e.pid log-file: ${{ runner.temp }}/iggy-go-e2e.log @@ -184,9 +182,7 @@ runs: if: inputs.task == 'e2e' uses: ./.github/actions/utils/server-start with: - # TODO: change to iggy-server once legacy server is removed (core/server has VSR support) - cargo-bin: iggy-server-ng - cargo-features: vsr + cargo-bin: iggy-server replica-id: "0" pid-file: ${{ runner.temp }}/iggy-go-e2e-tls.pid log-file: ${{ runner.temp }}/iggy-go-e2e-tls.log @@ -215,14 +211,16 @@ runs: pid-file: ${{ steps.iggy-tls.outputs.pid_file }} log-file: ${{ steps.iggy-tls.outputs.log_file }} - - name: Start Iggy VSR cluster node + # Both replicas of the roster in `core/server/config.toml`, not just this + # one: a two-replica cluster commits on 2 acks (`quorum_replication`), so a + # lone node journals every op and commits none, and each client request + # blocks until it times out. + - name: Start Iggy VSR cluster node 0 id: iggy-cluster-0 if: inputs.task == 'e2e' uses: ./.github/actions/utils/server-start with: - # TODO: change to iggy-server once legacy server is removed (core/server has VSR support) - cargo-bin: iggy-server-ng - cargo-features: vsr + cargo-bin: iggy-server replica-id: "0" pid-file: ${{ runner.temp }}/iggy-go-cluster-0.pid log-file: ${{ runner.temp }}/iggy-go-cluster-0.log @@ -231,17 +229,40 @@ runs: IGGY_CLUSTER_ENABLED: "true" IGGY_SYSTEM_PATH: ${{ runner.temp }}/iggy-go-cluster-0-data + - name: Start Iggy VSR cluster node 1 + id: iggy-cluster-1 + if: inputs.task == 'e2e' + uses: ./.github/actions/utils/server-start + with: + cargo-bin: iggy-server + replica-id: "1" + tcp_address: 127.0.0.1:8091 + http_address: 127.0.0.1:3001 + pid-file: ${{ runner.temp }}/iggy-go-cluster-1.pid + log-file: ${{ runner.temp }}/iggy-go-cluster-1.log + wait-timeout-seconds: "90" + env: + IGGY_CLUSTER_ENABLED: "true" + IGGY_SYSTEM_PATH: ${{ runner.temp }}/iggy-go-cluster-1-data + - name: Run cluster e2e tests shell: bash if: inputs.task == 'e2e' env: IGGY_TCP_ADDRESS: 127.0.0.1:8090 run: | - echo "🗳️ Running Go e2e tests against a single-node cluster..." + echo "🗳️ Running Go e2e tests against a two-node cluster..." cd foreign/go go test -v -race ./tests/... - - name: Stop Iggy VSR cluster node + - name: Stop Iggy VSR cluster node 1 + if: always() && inputs.task == 'e2e' + uses: ./.github/actions/utils/server-stop + with: + pid-file: ${{ steps.iggy-cluster-1.outputs.pid_file }} + log-file: ${{ steps.iggy-cluster-1.outputs.log_file }} + + - name: Stop Iggy VSR cluster node 0 if: always() && inputs.task == 'e2e' uses: ./.github/actions/utils/server-stop with: @@ -257,4 +278,5 @@ runs: ${{ steps.iggy.outputs.log_file }} ${{ steps.iggy-tls.outputs.log_file }} ${{ steps.iggy-cluster-0.outputs.log_file }} + ${{ steps.iggy-cluster-1.outputs.log_file }} if-no-files-found: ignore diff --git a/.github/actions/java-gradle/pre-merge/action.yml b/.github/actions/java-gradle/pre-merge/action.yml index 47ccc4eb7b..565e33c30f 100644 --- a/.github/actions/java-gradle/pre-merge/action.yml +++ b/.github/actions/java-gradle/pre-merge/action.yml @@ -84,6 +84,11 @@ runs: if: inputs.task == 'test' id: iggy uses: ./.github/actions/utils/server-start + with: + cargo-bin: iggy-server + wait-timeout-seconds: "90" + env: + IGGY_SYSTEM_PATH: ${{ runner.temp }}/iggy-java-data - name: Test if: inputs.task == 'test' @@ -151,9 +156,12 @@ runs: if: inputs.task == 'test' uses: ./.github/actions/utils/server-start with: + cargo-bin: iggy-server + wait-timeout-seconds: "90" pid-file: ${{ runner.temp }}/iggy-server-tls.pid log-file: ${{ runner.temp }}/iggy-server-tls.log env: + IGGY_SYSTEM_PATH: ${{ runner.temp }}/iggy-java-tls-data IGGY_TCP_TLS_ENABLED: "true" IGGY_TCP_TLS_CERT_FILE: core/certs/iggy_cert.pem IGGY_TCP_TLS_KEY_FILE: core/certs/iggy_key.pem diff --git a/.github/actions/node-npm/pre-merge/action.yml b/.github/actions/node-npm/pre-merge/action.yml index 5dba122baa..34b83e0b2f 100644 --- a/.github/actions/node-npm/pre-merge/action.yml +++ b/.github/actions/node-npm/pre-merge/action.yml @@ -20,7 +20,7 @@ description: Node.js pre-merge testing github iggy actions inputs: task: - description: "Task to run (lint, test, build, e2e, e2e-vsr)" + description: "Task to run (lint, test, build, e2e)" required: true runs: @@ -39,11 +39,11 @@ runs: shell: bash - name: Setup Rust with cache - if: inputs.task == 'e2e' || inputs.task == 'e2e-vsr' + if: inputs.task == 'e2e' uses: ./.github/actions/utils/setup-rust-with-cache - name: Install netcat - if: inputs.task == 'e2e' || inputs.task == 'e2e-vsr' + if: inputs.task == 'e2e' run: sudo apt-get update && sudo apt-get install -y netcat-openbsd shell: bash @@ -95,12 +95,8 @@ runs: if: inputs.task == 'e2e' uses: ./.github/actions/utils/server-start with: - # Node e2e asserts full cluster metadata (2 nodes from - # config.toml's [[cluster.nodes]] list), so cluster mode must be - # on. --replica-id picks the current node out of that list. - replica-id: "0" - env: - IGGY_CLUSTER_ENABLED: "true" + cargo-bin: iggy-server + wait-timeout-seconds: "90" - name: E2E tests if: inputs.task == 'e2e' @@ -108,91 +104,29 @@ runs: cd foreign/node mkdir -p ../../reports/node-coverage/e2e npx c8 --reporter=lcov --reports-dir=../../reports/node-coverage/e2e npm run test:e2e - env: - IGGY_SERVER_HOST: 127.0.0.1 - IGGY_SERVER_TCP_PORT: 8090 shell: bash - - name: Start Iggy VSR server 0 - id: iggy-vsr-0 - if: inputs.task == 'e2e-vsr' - uses: ./.github/actions/utils/server-start - with: - cargo-bin: iggy-server-ng - cargo-features: vsr - replica-id: "0" - pid-file: ${{ runner.temp }}/iggy-node-vsr-0.pid - log-file: ${{ runner.temp }}/iggy-node-vsr-0.log - wait-timeout-seconds: "90" - env: - IGGY_CLUSTER_ENABLED: "true" - IGGY_SYSTEM_PATH: ${{ runner.temp }}/iggy-node-vsr-0-data - - - name: Start Iggy VSR server 1 - id: iggy-vsr-1 - if: inputs.task == 'e2e-vsr' - uses: ./.github/actions/utils/server-start - with: - cargo-bin: iggy-server-ng - cargo-features: vsr - replica-id: "1" - tcp_address: 127.0.0.1:8091 - http_address: 127.0.0.1:3001 - pid-file: ${{ runner.temp }}/iggy-node-vsr-1.pid - log-file: ${{ runner.temp }}/iggy-node-vsr-1.log - wait-timeout-seconds: "90" - env: - IGGY_CLUSTER_ENABLED: "true" - IGGY_SYSTEM_PATH: ${{ runner.temp }}/iggy-node-vsr-1-data - - - name: VSR E2E tests - if: inputs.task == 'e2e-vsr' - run: | - cd foreign/node - mkdir -p ../../reports/node-coverage/e2e-vsr - npx c8 --reporter=lcov \ - --reports-dir=../../reports/node-coverage/e2e-vsr \ - npm run test:e2e:vsr - env: - IGGY_TEST_PROTOCOL: vsr - shell: bash - - - name: Stop Iggy VSR server 1 - if: always() && inputs.task == 'e2e-vsr' - uses: ./.github/actions/utils/server-stop - with: - pid-file: ${{ steps.iggy-vsr-1.outputs.pid_file }} - log-file: ${{ steps.iggy-vsr-1.outputs.log_file }} - - - name: Stop Iggy VSR server 0 - if: always() && inputs.task == 'e2e-vsr' + - name: Stop Iggy server + if: always() && inputs.task == 'e2e' uses: ./.github/actions/utils/server-stop with: - pid-file: ${{ steps.iggy-vsr-0.outputs.pid_file }} - log-file: ${{ steps.iggy-vsr-0.outputs.log_file }} + pid-file: ${{ steps.iggy.outputs.pid_file }} + log-file: ${{ steps.iggy.outputs.log_file }} - - name: Upload VSR server logs - if: always() && inputs.task == 'e2e-vsr' + - name: Upload server logs + if: always() && inputs.task == 'e2e' uses: actions/upload-artifact@v7 with: - name: iggy-node-vsr-server-logs - path: | - ${{ steps.iggy-vsr-0.outputs.log_file }} - ${{ steps.iggy-vsr-1.outputs.log_file }} + name: iggy-node-server-logs + path: ${{ steps.iggy.outputs.log_file }} if-no-files-found: ignore - - name: Stop Iggy server (plain) - if: always() && inputs.task == 'e2e' - uses: ./.github/actions/utils/server-stop - with: - pid-file: ${{ steps.iggy.outputs.pid_file }} - log-file: ${{ steps.iggy.outputs.log_file }} - - name: Start Iggy server (TLS) id: iggy-tls if: inputs.task == 'e2e' uses: ./.github/actions/utils/server-start with: + cargo-bin: iggy-server pid-file: ${{ runner.temp }}/iggy-server-tls.pid log-file: ${{ runner.temp }}/iggy-server-tls.log env: diff --git a/.github/actions/php/pre-merge/action.yml b/.github/actions/php/pre-merge/action.yml index a0c817cc74..02b1c70e4b 100644 --- a/.github/actions/php/pre-merge/action.yml +++ b/.github/actions/php/pre-merge/action.yml @@ -154,6 +154,8 @@ runs: if: inputs.task == 'test' id: iggy uses: ./.github/actions/utils/server-start + with: + cargo-bin: iggy-server - name: Run PHP SDK tests if: inputs.task == 'test' @@ -180,6 +182,7 @@ runs: id: iggy-tls uses: ./.github/actions/utils/server-start with: + cargo-bin: iggy-server pid-file: ${{ runner.temp }}/iggy-server-tls.pid log-file: ${{ runner.temp }}/iggy-server-tls.log env: diff --git a/.github/actions/python-maturin/pre-merge/action.yml b/.github/actions/python-maturin/pre-merge/action.yml index a44062fc9a..f97462f34d 100644 --- a/.github/actions/python-maturin/pre-merge/action.yml +++ b/.github/actions/python-maturin/pre-merge/action.yml @@ -124,14 +124,12 @@ runs: if: inputs.task == 'test' run: | # test_tls.py spawns the server in a container; build it from the - # same vsr binaries the plain-TCP tests run against. - # TODO(hubcio): change to iggy-server once legacy server is removed - # (core/server has VSR support) - cargo build --locked --bin iggy-server-ng --bin iggy --features vsr + # same binaries the plain-TCP tests run against. + cargo build --locked --bin iggy-server --bin iggy docker build \ - -f core/server-ng/Dockerfile \ + -f core/server/Dockerfile \ --target runtime-prebuilt \ - --build-arg PREBUILT_IGGY_SERVER_NG=target/debug/iggy-server-ng \ + --build-arg PREBUILT_IGGY_SERVER=target/debug/iggy-server \ --build-arg PREBUILT_IGGY_CLI=target/debug/iggy \ -t iggy-server:local . shell: bash @@ -141,10 +139,7 @@ runs: id: iggy uses: ./.github/actions/utils/server-start with: - # TODO(hubcio): change to iggy-server once legacy server is removed - # (core/server has VSR support) - cargo-bin: iggy-server-ng - cargo-features: vsr + cargo-bin: iggy-server - name: Run Python integration tests if: inputs.task == 'test' diff --git a/.github/actions/rust/pre-merge/action.yml b/.github/actions/rust/pre-merge/action.yml index 25a8a8389f..71248a848b 100644 --- a/.github/actions/rust/pre-merge/action.yml +++ b/.github/actions/rust/pre-merge/action.yml @@ -20,7 +20,7 @@ description: Rust pre-merge testing and linting github iggy actions inputs: task: - description: "Task to run (check, check-msrv, fmt, clippy, sort, machete, doctest, verify-publish, test-1, test-2, compat, miri)" + description: "Task to run (check, check-msrv, fmt, clippy, sort, machete, doctest, verify-publish, test-1, test-2, test-3, miri)" required: true component: description: "Component name (for context)" @@ -208,13 +208,17 @@ runs: - name: Build and test with coverage if: startsWith(inputs.task, 'test-') run: | - # Parse partition index from task name (test-1 -> hash:1/2, test-2 -> hash:2/2) + # Parse partition index from task name (test-1 -> hash:1/3, test-2 -> hash:2/3, ...). + # TEST_PARTITIONS must match the number of test-N tasks in + # .github/config/components.yml. Cluster bootstrap makes each test + # CPU-heavy, so partitions stay small. + TEST_PARTITIONS=3 TASK="${{ inputs.task }}" PARTITION_FLAG="" if [[ "$TASK" =~ ^test-([0-9]+)$ ]]; then PARTITION_INDEX="${BASH_REMATCH[1]}" - PARTITION_FLAG="--partition hash:${PARTITION_INDEX}/2" - echo "::notice::Running test partition ${PARTITION_INDEX}/2" + PARTITION_FLAG="--partition hash:${PARTITION_INDEX}/${TEST_PARTITIONS}" + echo "::notice::Running test partition ${PARTITION_INDEX}/${TEST_PARTITIONS}" fi # Read DAG-based affected crate filter (computed in earlier step) @@ -360,15 +364,6 @@ runs: ls -la codecov.json shell: bash - - name: Backwards compatibility check - if: inputs.task == 'compat' && (github.event_name != 'pull_request' || !contains(join(github.event.pull_request.labels.*.name, ','), 'breaking:storage')) - run: | - scripts/check-backwards-compat.sh \ - --master-ref master \ - --pr-ref ${{ github.sha }} \ - --port 8090 --wait-secs 180 - shell: bash - # Miri (UB detector) on the unsafe-heavy crates that don't pull tokio / # compio. Pinned nightly so MIRIFLAGS behavior is stable across CI runs; # bump the date quarterly. Tree-borrows is the future-default aliasing diff --git a/.github/actions/utils/docker-build-test-server/action.yml b/.github/actions/utils/docker-build-test-server/action.yml deleted file mode 100644 index a82ae762a2..0000000000 --- a/.github/actions/utils/docker-build-test-server/action.yml +++ /dev/null @@ -1,105 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -name: docker-build-test-server -description: Build Iggy server binaries and Docker image with prebuilt binaries -inputs: - dockerfile: - description: "Path to Dockerfile" - required: false - default: "core/server/Dockerfile" - dockerfile-target: - description: "Dockerfile target stage (e.g., runtime-prebuilt)" - required: false - default: "runtime-prebuilt" - image-tag: - description: "Docker image tag (without registry/name)" - required: false - default: "iggy-server:test" - docker-build-context: - description: "Docker build context" - required: false - default: "." - prebuilt-server-binary: - description: "Path to prebuilt iggy-server binary" - required: false - default: "target/debug/iggy-server" - prebuilt-cli-binary: - description: "Path to prebuilt iggy CLI binary" - required: false - default: "target/debug/iggy" - libc: - description: "Libc type (musl or glibc)" - required: false - default: "glibc" - profile: - description: "Build profile (debug or release)" - required: false - default: "debug" -outputs: - docker_image: - description: "Full Docker image name built" - value: ${{ steps.build.outputs.image_tag }} -runs: - using: "composite" - steps: - - name: Build Iggy server and CLI binaries - shell: bash - run: | - echo "Building Iggy server and CLI binaries with cargo..." - cargo build --locked --bin iggy-server --bin iggy - - echo "✅ Binaries built successfully:" - ls -lh ${{ inputs.prebuilt-server-binary }} ${{ inputs.prebuilt-cli-binary }} - - - name: Build Docker image with prebuilt binaries - id: build - shell: bash - run: | - set -euo pipefail - DOCKERFILE="${{ inputs.dockerfile }}" - DOCKERFILE_TARGET="${{ inputs.dockerfile-target }}" - IMAGE_TAG="${{ inputs.image-tag }}" - BUILD_CONTEXT="${{ inputs.docker-build-context }}" - PREBUILT_IGGY_SERVER="${{ inputs.prebuilt-server-binary }}" - PREBUILT_IGGY_CLI="${{ inputs.prebuilt-cli-binary }}" - LIBC="${{ inputs.libc }}" - PROFILE="${{ inputs.profile }}" - - echo "Building Docker image with prebuilt binaries..." - echo " Dockerfile: $DOCKERFILE" - echo " Target: $DOCKERFILE_TARGET" - echo " Image Tag: $IMAGE_TAG" - echo " Context: $BUILD_CONTEXT" - echo " Server Binary: $PREBUILT_IGGY_SERVER" - echo " CLI Binary: $PREBUILT_IGGY_CLI" - echo " Libc: $LIBC" - echo " Profile: $PROFILE" - - docker build \ - -f "$DOCKERFILE" \ - --target "$DOCKERFILE_TARGET" \ - -t "$IMAGE_TAG" \ - --build-arg PREBUILT_IGGY_SERVER="$PREBUILT_IGGY_SERVER" \ - --build-arg PREBUILT_IGGY_CLI="$PREBUILT_IGGY_CLI" \ - --build-arg LIBC="$LIBC" \ - --build-arg PROFILE="$PROFILE" \ - "$BUILD_CONTEXT" - - echo "image_tag=$IMAGE_TAG" >> "$GITHUB_OUTPUT" - echo "✅ Docker image built successfully: $IMAGE_TAG" diff --git a/.github/actions/utils/server-start/action.yml b/.github/actions/utils/server-start/action.yml index 3eb06a1b97..664582f7e0 100644 --- a/.github/actions/utils/server-start/action.yml +++ b/.github/actions/utils/server-start/action.yml @@ -30,10 +30,6 @@ inputs: description: "Cargo profile: release|debug" required: false default: "debug" - cargo-features: - description: "Comma-separated Cargo features" - required: false - default: "" bin: description: "Path to server binary (when mode=bin)" required: false @@ -116,15 +112,12 @@ runs: else OUT="target/debug/$NAME" fi - if [[ ! -x "$OUT" || -n "${{ inputs.cargo-features }}" ]]; then + if [[ ! -x "$OUT" ]]; then echo "Building $NAME with cargo ($PROFILE)…" CARGO_ARGS=(build --locked --bin "$NAME") if [[ "$PROFILE" == "release" ]]; then CARGO_ARGS+=(--release) fi - if [[ -n "${{ inputs.cargo-features }}" ]]; then - CARGO_ARGS+=(--features "${{ inputs.cargo-features }}") - fi cargo "${CARGO_ARGS[@]}" fi BIN_PATH="$OUT" diff --git a/.github/config/components.yml b/.github/config/components.yml index 545b5cb5b7..c3c28c6b71 100644 --- a/.github/config/components.yml +++ b/.github/config/components.yml @@ -121,7 +121,6 @@ components: - "rust-cluster" paths: - "core/server/**" - - "core/server-ng/**" rust-cluster: depends_on: @@ -177,7 +176,7 @@ components: - "machete" - "test-1" - "test-2" - - "compat" + - "test-3" - "build-aarch64-gnu" - "build-aarch64-musl" - "build-macos-aarch64" @@ -251,7 +250,7 @@ components: - "ci-infrastructure" # CI changes trigger full regression paths: - "foreign/node/**" - tasks: ["lint", "test", "build", "e2e", "e2e-vsr"] + tasks: ["lint", "test", "build", "e2e"] sdk-go: depends_on: @@ -265,8 +264,7 @@ components: # The SDK compiles against the VSR wire contract, so a change to the # protocol crate or the VSR server must rerun it. - "core/binary_protocol/**" - # TODO: change to core/server once legacy server is removed (core/server has VSR support) - - "core/server-ng/**" + - "core/server/**" # VSR is the only protocol the Go SDK speaks, so there is no separate lane. tasks: ["lint", "test", "build", "e2e"] @@ -307,7 +305,6 @@ components: - "bdd/docker-compose.server.yml" - "bdd/docker-compose.cluster.yml" - "bdd/docker-compose.coverage.yml" - - "bdd/docker-compose.vsr.yml" # Individual BDD tests per SDK - only run when specific SDK changes bdd-rust: @@ -319,7 +316,7 @@ components: paths: - "bdd/rust/**" - "bdd/scenarios/**" - tasks: ["bdd-rust", "bdd-rust-vsr"] + tasks: ["bdd-rust"] bdd-python: depends_on: @@ -357,8 +354,7 @@ components: - "bdd/scenarios/**" # The Go suites run against the VSR server, so its sources gate them. - "core/binary_protocol/**" - # TODO: change to core/server once legacy server is removed (core/server has VSR support) - - "core/server-ng/**" + - "core/server/**" tasks: ["bdd-go", "bdd-go-race"] bdd-node: @@ -371,7 +367,7 @@ components: paths: - "bdd/node/**" - "bdd/scenarios/**" - tasks: ["bdd-node", "bdd-node-vsr"] + tasks: ["bdd-node"] bdd-csharp: depends_on: diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 8d2932f980..eb28a9fb3b 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -230,7 +230,6 @@ updates: directories: - "/" - "/core/server" - - "/core/server-ng" - "/core/connectors/runtime" - "/core/ai/mcp" - "/core/bench/dashboard/server" diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index 80edc184bc..a2b6a61dbb 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -127,8 +127,7 @@ jobs: if: >- inputs.component == 'sdk-node' && (inputs.task == 'test' || - inputs.task == 'e2e' || - inputs.task == 'e2e-vsr') + inputs.task == 'e2e') uses: codecov/codecov-action@v7.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/_test_bdd.yml b/.github/workflows/_test_bdd.yml index 5640e1bbb4..87ab40756f 100644 --- a/.github/workflows/_test_bdd.yml +++ b/.github/workflows/_test_bdd.yml @@ -51,24 +51,9 @@ jobs: - name: Build server for BDD tests if: startsWith(inputs.component, 'bdd-') && startsWith(inputs.task, 'bdd-') run: | - # The VSR lanes need the vsr feature on both the server and the CLI, - # otherwise the CLI cannot frame requests for the VSR wire protocol - # and the healthcheck ping fails. The Go SDK speaks only VSR and the - # Python wheels are vsr-built, so those suites are always on this - # branch. - case "${{ inputs.task }}" in - bdd-rust-vsr|bdd-go|bdd-go-race|bdd-python|bdd-node-vsr) - # TODO: change to iggy-server once legacy server is removed (core/server has VSR support) - SERVER_BIN="iggy-server-ng" - echo "Building the VSR server binary and CLI (--features vsr) for BDD tests..." - cargo build --locked --bin iggy-server-ng --bin iggy --features vsr - ;; - *) - SERVER_BIN="iggy-server" - echo "Building server binary and CLI for BDD tests..." - cargo build --locked --bin iggy-server --bin iggy - ;; - esac + SERVER_BIN="iggy-server" + echo "Building server binary and CLI for BDD tests..." + cargo build --locked --bin iggy-server --bin iggy echo "Server binary built at: target/debug/${SERVER_BIN}" ls -lh "target/debug/${SERVER_BIN}" @@ -90,39 +75,20 @@ jobs: - name: Run BDD tests if: startsWith(inputs.component, 'bdd-') && startsWith(inputs.task, 'bdd-') run: | - # Extract SDK name from task (format: bdd-, or bdd--vsr - # for an explicit vsr lane). Python has no legacy lane, so its - # plain task name runs vsr. - SDK_NAME=$(echo "${{ inputs.task }}" | sed 's/^bdd-//; s/-vsr$//') - EXTRA_FLAGS=() - case "${{ inputs.task }}" in - bdd-rust-vsr|bdd-python|bdd-node-vsr) - EXTRA_FLAGS+=(--vsr) - # TODO: change to iggy-server once legacy server is removed (core/server has VSR support) - export IGGY_SERVER_NG_PATH="target/debug/iggy-server-ng" - echo "Server binary location: $(ls -lh target/debug/iggy-server-ng)" - ;; - bdd-go|bdd-go-race) - # The Go SDK speaks only VSR, so the runner forces the overlay. - # TODO: change to iggy-server once legacy server is removed (core/server has VSR support) - export IGGY_SERVER_NG_PATH="target/debug/iggy-server-ng" - echo "Server binary location: $(ls -lh target/debug/iggy-server-ng)" - ;; - *) - echo "Server binary location: $(ls -lh target/debug/iggy-server)" - ;; - esac + # Extract SDK name from task (format: bdd-). + SDK_NAME=$(echo "${{ inputs.task }}" | sed 's/^bdd-//') + export IGGY_SERVER_PATH="target/debug/iggy-server" + echo "Server binary location: $(ls -lh target/debug/iggy-server)" echo "Running BDD tests for SDK: $SDK_NAME" echo "Current directory: $(pwd)" echo "CLI binary location: $(ls -lh target/debug/iggy)" - # Export path to the pre-built server and cli binaries (relative to repo root) - export IGGY_SERVER_PATH="target/debug/iggy-server" + # Export path to the pre-built cli binary (relative to repo root) export IGGY_CLI_PATH="target/debug/iggy" export IGGY_ROOT_USERNAME="iggy" export IGGY_ROOT_PASSWORD="iggy" - ./scripts/run-bdd-tests.sh "${EXTRA_FLAGS[@]}" "$SDK_NAME" + ./scripts/run-bdd-tests.sh "$SDK_NAME" - name: Clean up Docker resources (BDD) if: always() && startsWith(inputs.component, 'bdd-') && startsWith(inputs.task, 'bdd-') diff --git a/.github/workflows/_test_examples.yml b/.github/workflows/_test_examples.yml index 3fdb104745..1382f9ee55 100644 --- a/.github/workflows/_test_examples.yml +++ b/.github/workflows/_test_examples.yml @@ -113,22 +113,12 @@ jobs: echo "Building common binaries for all examples tests..." echo "Current directory: $(pwd)" - # The Go SDK speaks only the VSR wire protocol and the Python wheels - # are vsr-built, so those lanes need the VSR server. Every other - # language still runs against the legacy one until its own migration - # lands. - if [[ "${{ inputs.task }}" == "examples-go" || "${{ inputs.task }}" == "examples-python" ]]; then - # TODO: change to iggy-server once legacy server is removed (core/server has VSR support) - SERVER_BIN="iggy-server-ng" - echo "Building ${SERVER_BIN} (--features vsr)..." - cargo build --locked --bin "${SERVER_BIN}" --features vsr - else - SERVER_BIN="iggy-server" - echo "Building ${SERVER_BIN}..." - cargo build --locked --bin "${SERVER_BIN}" - fi + # Every SDK speaks only the VSR wire protocol, so every lane runs + # against the VSR server. + SERVER_BIN="iggy-server" + echo "Building ${SERVER_BIN}..." + cargo build --locked --bin "${SERVER_BIN}" - # For Rust examples, also build CLI and example binaries if [[ "${{ inputs.task }}" == "examples-rust" ]]; then echo "Building additional binaries for Rust examples..." cargo build --locked --bin iggy --examples diff --git a/.github/workflows/coverage-baseline.yml b/.github/workflows/coverage-baseline.yml index eaebf82eb2..9fed1dc1b0 100644 --- a/.github/workflows/coverage-baseline.yml +++ b/.github/workflows/coverage-baseline.yml @@ -216,11 +216,16 @@ jobs: - name: Build Iggy server Docker image id: docker_build - uses: ./.github/actions/utils/docker-build-test-server - with: - image-tag: "iggy-server:test" - libc: "glibc" - profile: "debug" + run: | + cargo build --locked --bin iggy-server --bin iggy + docker build \ + -f core/server/Dockerfile \ + --target runtime-prebuilt \ + -t iggy-server:test \ + --build-arg PREBUILT_IGGY_SERVER=target/debug/iggy-server \ + --build-arg PREBUILT_IGGY_CLI=target/debug/iggy \ + . + echo "docker_image=iggy-server:test" >> "$GITHUB_OUTPUT" - name: Restore and build working-directory: foreign/csharp @@ -318,14 +323,12 @@ jobs: - name: Build server Docker image for TLS tests run: | # test_tls.py spawns the server in a container; build it from the - # same vsr binaries the plain-TCP tests run against. - # TODO(hubcio): change to iggy-server once legacy server is removed - # (core/server has VSR support) - cargo build --locked --bin iggy-server-ng --bin iggy --features vsr + # same binaries the plain-TCP tests run against. + cargo build --locked --bin iggy-server --bin iggy docker build \ - -f core/server-ng/Dockerfile \ + -f core/server/Dockerfile \ --target runtime-prebuilt \ - --build-arg PREBUILT_IGGY_SERVER_NG=target/debug/iggy-server-ng \ + --build-arg PREBUILT_IGGY_SERVER=target/debug/iggy-server \ --build-arg PREBUILT_IGGY_CLI=target/debug/iggy \ -t iggy-server:local . shell: bash @@ -334,10 +337,7 @@ jobs: id: iggy uses: ./.github/actions/utils/server-start with: - # TODO(hubcio): change to iggy-server once legacy server is removed - # (core/server has VSR support) - cargo-bin: iggy-server-ng - cargo-features: vsr + cargo-bin: iggy-server - name: Run tests run: | @@ -443,8 +443,9 @@ jobs: - name: Start Iggy server id: iggy uses: ./.github/actions/utils/server-start - env: - IGGY_CLUSTER_ENABLED: true + with: + cargo-bin: iggy-server + wait-timeout-seconds: "90" - name: Run unit tests with coverage run: | @@ -457,9 +458,6 @@ jobs: cd foreign/node mkdir -p ../../reports/node-coverage/e2e npx c8 --reporter=lcov --reports-dir=../../reports/node-coverage/e2e npm run test:e2e - env: - IGGY_SERVER_HOST: 127.0.0.1 - IGGY_SERVER_TCP_PORT: 8090 - name: Stop Iggy server if: always() @@ -500,9 +498,7 @@ jobs: id: iggy uses: ./.github/actions/utils/server-start with: - # TODO: change to iggy-server once legacy server is removed (core/server has VSR support) - cargo-bin: iggy-server-ng - cargo-features: vsr + cargo-bin: iggy-server replica-id: "0" wait-timeout-seconds: "90" diff --git a/.github/workflows/pr-title.yml b/.github/workflows/pr-title.yml index f485734caa..69b4fdcdc3 100644 --- a/.github/workflows/pr-title.yml +++ b/.github/workflows/pr-title.yml @@ -84,7 +84,6 @@ jobs: sdk security server - server-ng shard test web diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 03b5254675..edc9452f58 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -16,7 +16,7 @@ # under the License. # PR gate: detects changed components, builds test matrices, and runs -# lint/test/build/compat/BDD/examples jobs only for affected languages. +# lint/test/build/BDD/examples jobs only for affected languages. # All jobs must pass before merge. name: Pre-merge diff --git a/AGENTS.md b/AGENTS.md index 8631c013f8..676ea42422 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -82,8 +82,7 @@ install` (Python) or `prek install` (Rust drop-in) - both read ```text iggy/ ├── core/ -│ ├── server/ Iggy server binary -│ ├── server-ng/ Next-gen server (Viewstamped Replication, WIP) +│ ├── server/ Iggy server binary (Viewstamped Replication) │ ├── sdk/ Rust client SDK │ ├── cli/ iggy CLI │ ├── connectors/ Connectors runtime + SDK + sinks/sources @@ -114,7 +113,6 @@ iggy/ | --------------------- | ---------------------------------------- | | Wire protocol | `core/binary_protocol/` | | Server | `core/server/src/` | -| Next-gen server (WIP) | `core/server-ng/` | | Rust client SDK | `core/sdk/src/` | | Connectors | `core/connectors/` -> connector-* skills | | Integration tests | `core/integration/tests/` | diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f7850662e6..8297d919dd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -84,6 +84,28 @@ cargo install prek prek install ``` +The hooks require **bash >= 4.2** and refuse to run on anything older. Every current +Linux distribution already satisfies this. + +#### macOS + +macOS ships bash 3.2 and never updates it, so this is the one platform that needs a +step: + +```bash +brew install bash +``` + +Homebrew's bash has to precede `/bin` on `PATH`, which is the default for a Homebrew +install but not guaranteed. Check with: + +```bash +bash --version +``` + +Git GUIs launched from the Dock get a minimal `PATH` where `/bin` wins, so the hook can +still find bash 3.2 after the install. Committing from a terminal avoids this. + ## Code Style ### Comments: WHY, Not WHAT diff --git a/Cargo.lock b/Cargo.lock index 1cf7f53614..051b51babc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3198,7 +3198,6 @@ dependencies = [ "static-toml", "strum 0.28.0", "tracing", - "tungstenite 0.30.0", ] [[package]] @@ -3215,6 +3214,7 @@ dependencies = [ name = "consensus" version = "0.1.0" dependencies = [ + "aligned-vec", "bit-set 0.11.1", "bytemuck", "bytes", @@ -6856,6 +6856,7 @@ dependencies = [ "enumset", "secrecy", "thiserror 2.0.19", + "twox-hash", ] [[package]] @@ -7501,7 +7502,6 @@ dependencies = [ "serde", "serde_json", "serial_test", - "server", "sqlx", "sysinfo 0.39.6", "tempfile", @@ -9621,10 +9621,12 @@ dependencies = [ "iggy_common", "journal", "message_bus", + "nix", "papaya", "ringbuffer", "server_common", "smallvec", + "tempfile", "tokio", "tracing", ] @@ -11905,71 +11907,6 @@ dependencies = [ [[package]] name = "server" -version = "0.8.2-edge.1" -dependencies = [ - "ahash 0.8.12", - "anyhow", - "async-channel", - "async_zip", - "axum", - "axum-server", - "bytes", - "chrono", - "clap", - "compio", - "configs", - "ctrlc", - "cyper", - "cyper-axum", - "dashmap", - "dotenvy", - "err_trail", - "error_set", - "figlet-rs", - "flume", - "fs2", - "futures", - "hash32 1.0.0", - "human-repr", - "iggy_binary_protocol", - "iggy_common", - "jsonwebtoken", - "left-right", - "mimalloc", - "mime_guess", - "nix", - "papaya", - "prometheus-client", - "ringbuffer", - "rmp-serde", - "rust-embed", - "rustls", - "rustls-pemfile", - "sd-notify", - "secrecy", - "send_wrapper", - "serde", - "serde_json", - "server_common", - "shard_allocator", - "slab", - "socket2 0.6.5", - "strum 0.28.0", - "sysinfo 0.39.6", - "system_stats", - "tempfile", - "thiserror 2.0.19", - "tokio", - "toml 1.1.3+spec-1.1.0", - "tower-http 0.7.0", - "tracing", - "ulid", - "uuid", - "vergen-git2", -] - -[[package]] -name = "server-ng" version = "0.9.0-edge.2" dependencies = [ "ahash 0.8.12", @@ -12029,6 +11966,7 @@ dependencies = [ "rust-embed", "rustls", "rustls-pemfile", + "sd-notify", "secrecy", "send_wrapper", "serde", @@ -12315,7 +12253,7 @@ dependencies = [ "rand 0.10.2", "rand_xoshiro", "secrecy", - "server-ng", + "server", "server_common", "shard", "strum 0.28.0", diff --git a/Cargo.toml b/Cargo.toml index 3e5db8fb62..d6c7d0c544 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,7 +61,6 @@ members = [ "core/partitions", "core/sdk", "core/server", - "core/server-ng", "core/server_common", "core/shard", "core/shard_allocator", @@ -297,7 +296,6 @@ serde_with = { version = "3.21.0", features = ["base64", "macros"] } serde_yaml_ng = "0.10.0" serial_test = "3.5.0" server = { path = "core/server" } -server-ng = { path = "core/server-ng" } server_common = { path = "core/server_common" } shard = { path = "core/shard" } shard_allocator = { path = "core/shard_allocator" } diff --git a/Dockerfile b/Dockerfile index 6f4f99b64a..f340b38f40 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,7 +43,7 @@ RUN apt-get update && apt-get install -y \ COPY . . RUN npm --prefix web ci && npm --prefix web run build:static RUN cargo build --bin iggy --release -RUN cargo build --bin iggy-server --release +RUN cargo build --bin iggy-server -p server --release FROM debian:trixie-slim RUN apt-get update && apt-get install -y \ @@ -51,7 +51,6 @@ RUN apt-get update && apt-get install -y \ liblzma5 \ libhwloc15 \ && rm -rf /var/lib/apt/lists/* -COPY ./core/configs ./configs COPY --from=builder /build/target/release/iggy . COPY --from=builder /build/target/release/iggy-server . diff --git a/README.md b/README.md index 3db9535827..0e1925eeb8 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ The name is an abbreviation for the Italian Greyhound - small yet extremely fast - Built-in **CLI** to manage the streaming server installable via `cargo install iggy-cli` - Built-in **benchmarking app** to test the performance - **Single binary deployment** (no external dependencies) -- Running as a single node (clustering based on Viewstamped Replication will be implemented in the near future) +- Running as a single node or as a **cluster**, with data replication based on **[Viewstamped Replication (VSR)](https://github.com/apache/iggy/blob/master/assets/vsr.pdf)** ![server](assets/server.png) @@ -117,12 +117,6 @@ We do also publish edge/dev/nightly releases (e.g. `0.7.0-edge.1` or `apache/igg --- -## Roadmap - -- **Clustering** & data replication based on **[VSR](https://github.com/apache/iggy/blob/master/assets/vsr.pdf)** (coming soon) - ---- - ## Supported languages SDK - [Rust](https://crates.io/crates/iggy) @@ -285,7 +279,7 @@ it should only be used for development and testing. `cargo run --bin iggy-server -- --with-default-root-credentials` -Root credentials are only set on the first server startup when the data directory doesn't exist yet. Once the server has been started and persisted data exists, the existing root credentials will be reused, and the `--with-default-root-credentials` flag or environment variables will have no effect. To reset credentials, delete the data directory. +Root credentials are only set on the first server startup when the data directory doesn't exist yet. Once the server has been started and persisted data exists, the existing root credentials will be reused, and the `--with-default-root-credentials` flag or environment variables are ignored. They are still validated, though: a half-set pair or an out-of-range value aborts the boot instead of being silently dropped. To reset credentials, delete the data directory. For configuration options and detailed help: diff --git a/bdd/README.md b/bdd/README.md index 2509006f4f..b50246b03f 100644 --- a/bdd/README.md +++ b/bdd/README.md @@ -39,7 +39,6 @@ bdd/ ├── docker-compose.server.yml # Single iggy-server test setup ├── docker-compose.cluster.yml # Leader + follower test setup ├── docker-compose.coverage.yml # Coverage collection overlay -├── docker-compose.vsr.yml # server-ng (VSR) overlay, Rust SDK only ├── Dockerfile # Debug build of Iggy server └── README.md ``` @@ -71,14 +70,8 @@ bdd/ # Run only leader_redirection ../scripts/run-bdd-tests.sh all leader_redirection -# Run against iggy-server-ng with VSR (Rust SDK only; other SDKs do not -# speak the VSR wire protocol yet). Build the vsr binaries first: -# cargo build --bin iggy-server-ng --bin iggy --features vsr -# NOTE: `target/debug/iggy` is shared between lanes and the vsr flavour -# speaks a different wire protocol. When switching back to the legacy -# lane, rebuild without `--features vsr` first, or the in-container -# healthcheck ping cannot talk to the legacy server. -../scripts/run-bdd-tests.sh --vsr rust +# Every suite runs against iggy-server. Build the binaries first: +# cargo build --bin iggy-server --bin iggy # Clean up Docker resources ../scripts/run-bdd-tests.sh clean diff --git a/bdd/docker-compose.cluster.yml b/bdd/docker-compose.cluster.yml index 063daf0db4..9976d342b1 100644 --- a/bdd/docker-compose.cluster.yml +++ b/bdd/docker-compose.cluster.yml @@ -15,9 +15,20 @@ # specific language governing permissions and limitations # under the License. -# Iggy leader + follower cluster test setup. +# Iggy leader + follower cluster test setup: a real 2-node VSR cluster with +# replica 0 as the initial leader. # Activated by: ./scripts/run-bdd-tests.sh leader_redirection # ./scripts/run-bdd-tests.sh all +# +# Server constraints that shape this file: +# - `--replica-id ` is the only CLI arg each node needs; there is no +# `--follower`, and a new container per run makes `--fresh` redundant. +# - root credentials are mandatory once the cluster is enabled and come +# from IGGY_ROOT_USERNAME / IGGY_ROOT_PASSWORD. +# - `cluster.nodes[*].ip` is parsed as a strict IpAddr (no hostnames) and +# cluster listeners bind to that roster IP, so nodes need static IPs and +# healthchecks must ping the roster address rather than loopback. +# - replica-to-replica consensus requires `ports.tcp_replica`. x-cluster-node: &cluster-node image: iggy-bdd-server @@ -39,26 +50,34 @@ x-cluster-node: &cluster-node memlock: soft: -1 hard: -1 - networks: - - iggy-bdd-network x-cluster-topology: &cluster-topology + IGGY_ROOT_USERNAME: iggy + IGGY_ROOT_PASSWORD: iggy IGGY_CLUSTER_ENABLED: "true" IGGY_CLUSTER_NAME: test-cluster IGGY_CLUSTER_NODES_0_NAME: leader-node - IGGY_CLUSTER_NODES_0_IP: iggy-leader + IGGY_CLUSTER_NODES_0_IP: 172.28.0.101 IGGY_CLUSTER_NODES_0_REPLICA_ID: "0" IGGY_CLUSTER_NODES_0_PORTS_TCP: "8091" IGGY_CLUSTER_NODES_0_PORTS_QUIC: "8081" IGGY_CLUSTER_NODES_0_PORTS_HTTP: "3001" IGGY_CLUSTER_NODES_0_PORTS_WEBSOCKET: "8071" + IGGY_CLUSTER_NODES_0_PORTS_TCP_REPLICA: "8191" IGGY_CLUSTER_NODES_1_NAME: follower-node - IGGY_CLUSTER_NODES_1_IP: iggy-follower + IGGY_CLUSTER_NODES_1_IP: 172.28.0.102 IGGY_CLUSTER_NODES_1_REPLICA_ID: "1" IGGY_CLUSTER_NODES_1_PORTS_TCP: "8092" IGGY_CLUSTER_NODES_1_PORTS_QUIC: "8082" IGGY_CLUSTER_NODES_1_PORTS_HTTP: "3002" IGGY_CLUSTER_NODES_1_PORTS_WEBSOCKET: "8072" + IGGY_CLUSTER_NODES_1_PORTS_TCP_REPLICA: "8192" + # http.enabled with cluster.enabled requires a JWT key every node can + # verify; cluster auth provides it (derived from the shared secret). + # Enabling auth activates follower-to-primary forwarding, so the config + # validator requires ports.http on every roster node. + IGGY_CLUSTER_AUTH_ENABLED: "true" + IGGY_CLUSTER_AUTH_SHARED_SECRET: "bdd-vsr-cluster-shared-secret-0123456789" x-cluster-bdd-deps: &cluster-bdd-deps depends_on: @@ -73,9 +92,9 @@ x-cluster-bdd-deps: &cluster-bdd-deps services: iggy-leader: <<: *cluster-node - command: [ "--fresh", "--with-default-root-credentials", "--replica-id", "0" ] + command: [ "--replica-id", "0" ] healthcheck: - test: [ "CMD", "/usr/local/bin/iggy", "--tcp-server-address", "127.0.0.1:8091", "ping" ] + test: [ "CMD", "/usr/local/bin/iggy", "--tcp-server-address", "172.28.0.101:8091", "ping" ] interval: 5s timeout: 3s retries: 30 @@ -88,14 +107,17 @@ services: IGGY_HTTP_ADDRESS: 0.0.0.0:3001 IGGY_QUIC_ADDRESS: 0.0.0.0:8081 IGGY_WEBSOCKET_ADDRESS: 0.0.0.0:8071 + networks: + iggy-bdd-network: + ipv4_address: 172.28.0.101 volumes: - iggy_leader_data:/app/local_data_leader iggy-follower: <<: *cluster-node - command: [ "--fresh", "--with-default-root-credentials", "--follower", "--replica-id", "1" ] + command: [ "--replica-id", "1" ] healthcheck: - test: [ "CMD", "/usr/local/bin/iggy", "--tcp-server-address", "127.0.0.1:8092", "ping" ] + test: [ "CMD", "/usr/local/bin/iggy", "--tcp-server-address", "172.28.0.102:8092", "ping" ] interval: 5s timeout: 3s retries: 30 @@ -108,6 +130,9 @@ services: IGGY_HTTP_ADDRESS: 0.0.0.0:3002 IGGY_QUIC_ADDRESS: 0.0.0.0:8082 IGGY_WEBSOCKET_ADDRESS: 0.0.0.0:8072 + networks: + iggy-bdd-network: + ipv4_address: 172.28.0.102 volumes: - iggy_follower_data:/app/local_data_follower @@ -123,6 +148,13 @@ services: java-bdd: <<: *cluster-bdd-deps +networks: + iggy-bdd-network: + driver: bridge + ipam: + config: + - subnet: 172.28.0.0/24 + volumes: iggy_leader_data: iggy_follower_data: diff --git a/bdd/docker-compose.server.yml b/bdd/docker-compose.server.yml index 54158aa1b1..e22484533c 100644 --- a/bdd/docker-compose.server.yml +++ b/bdd/docker-compose.server.yml @@ -19,6 +19,10 @@ # Activated by: ./scripts/run-bdd-tests.sh basic_messaging # ./scripts/run-bdd-tests.sh leader_redirection # ./scripts/run-bdd-tests.sh all +# +# Each run starts from a new container, so `--fresh` would be redundant; root +# credentials arrive through IGGY_ROOT_USERNAME / IGGY_ROOT_PASSWORD rather +# than through `--with-default-root-credentials`. x-server-bdd-deps: &server-bdd-deps depends_on: @@ -40,7 +44,7 @@ services: PREBUILT_IGGY_CLI: ${IGGY_CLI_PATH:-target/debug/iggy} LIBC: glibc PROFILE: debug - command: [ "--fresh", "--with-default-root-credentials" ] + command: [] cap_add: - SYS_NICE security_opt: @@ -57,6 +61,8 @@ services: start_period: 2s environment: - RUST_LOG=info + - IGGY_ROOT_USERNAME=iggy + - IGGY_ROOT_PASSWORD=iggy - IGGY_SYSTEM_PATH=local_data - IGGY_TCP_ADDRESS=0.0.0.0:8090 - IGGY_HTTP_ADDRESS=0.0.0.0:3000 diff --git a/bdd/docker-compose.vsr.yml b/bdd/docker-compose.vsr.yml deleted file mode 100644 index 9aaf51d37f..0000000000 --- a/bdd/docker-compose.vsr.yml +++ /dev/null @@ -1,101 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# server-ng (VSR) override. Swaps the legacy iggy-server binary for -# iggy-server-ng built with `--features vsr` and adapts args, credentials, -# and cluster topology to server-ng semantics: -# - server-ng takes no `--fresh` / `--with-default-root-credentials` / -# `--follower` flags; the only CLI arg is `--replica-id `. -# - Root credentials come from IGGY_ROOT_USERNAME / IGGY_ROOT_PASSWORD -# (mandatory when the cluster is enabled). -# - `cluster.nodes[*].ip` is parsed as a strict IpAddr (no hostnames) -# and cluster listeners bind to that roster IP, so nodes get static -# IPs and node healthchecks ping the roster address, not loopback. -# - Replica-to-replica consensus requires `ports.tcp_replica`. -# The leader_redirection feature runs a REAL 2-node VSR cluster here -# (initial leader = replica 0), not the legacy mock `--follower` server. -# -# Activated by: ./scripts/run-bdd-tests.sh --vsr [feature] -# Must be passed LAST so its overrides win over the server/cluster files. - -x-server-ng-image: &server-ng-image - image: iggy-bdd-server-ng - build: - context: .. - dockerfile: core/server-ng/Dockerfile - target: runtime-prebuilt - args: - PREBUILT_IGGY_SERVER_NG: ${IGGY_SERVER_NG_PATH:-target/debug/iggy-server-ng} - PREBUILT_IGGY_CLI: ${IGGY_CLI_PATH:-target/debug/iggy} - -x-vsr-cluster-env: &vsr-cluster-env - IGGY_ROOT_USERNAME: iggy - IGGY_ROOT_PASSWORD: iggy - IGGY_CLUSTER_NODES_0_IP: 172.28.0.101 - IGGY_CLUSTER_NODES_1_IP: 172.28.0.102 - IGGY_CLUSTER_NODES_0_PORTS_TCP_REPLICA: "8191" - IGGY_CLUSTER_NODES_1_PORTS_TCP_REPLICA: "8192" - # http.enabled with cluster.enabled requires a JWT key every node can - # verify; cluster auth provides it (derived from the shared secret). - # Enabling auth activates follower-to-primary forwarding, so the config - # validator requires ports.http on every roster node - those ports are set - # in docker-compose.cluster.yml (merged for the leader_redirection flow). - IGGY_CLUSTER_AUTH_ENABLED: "true" - IGGY_CLUSTER_AUTH_SHARED_SECRET: "bdd-vsr-cluster-shared-secret-0123456789" - -services: - iggy-server: - <<: *server-ng-image - command: [] - environment: - - IGGY_ROOT_USERNAME=iggy - - IGGY_ROOT_PASSWORD=iggy - - # The Node SDK picks the wire protocol at runtime; classic is its default, - # so the VSR lane has to opt in explicitly. - node-bdd: - environment: - - IGGY_TEST_PROTOCOL=vsr - - iggy-leader: - <<: *server-ng-image - command: [ "--replica-id", "0" ] - environment: - <<: *vsr-cluster-env - networks: - iggy-bdd-network: - ipv4_address: 172.28.0.101 - healthcheck: - test: [ "CMD", "/usr/local/bin/iggy", "--tcp-server-address", "172.28.0.101:8091", "ping" ] - - iggy-follower: - <<: *server-ng-image - command: [ "--replica-id", "1" ] - environment: - <<: *vsr-cluster-env - networks: - iggy-bdd-network: - ipv4_address: 172.28.0.102 - healthcheck: - test: [ "CMD", "/usr/local/bin/iggy", "--tcp-server-address", "172.28.0.102:8092", "ping" ] - -networks: - iggy-bdd-network: - driver: bridge - ipam: - config: - - subnet: 172.28.0.0/24 diff --git a/bdd/go/tests/tcp_test/test_helpers.go b/bdd/go/tests/tcp_test/test_helpers.go index 646b741da8..efcdd18310 100644 --- a/bdd/go/tests/tcp_test/test_helpers.go +++ b/bdd/go/tests/tcp_test/test_helpers.go @@ -59,11 +59,10 @@ func createClient() iggcon.Client { return cli } -// maxRoutableId is the highest stream or topic id the wire namespace can -// address. A larger id cannot be routed at all, so the SDK rejects it before -// it reaches the server. Specs that want an id the server has never seen must -// stay inside this range to get the server's answer rather than a local -// rejection. +// maxRoutableId is the highest stream or topic id the server's routing +// namespace can address. A larger id cannot be routed at all, so specs that +// want an id the server has never seen must stay inside this range to get the +// server's answer rather than a routing failure. const maxRoutableId = 4095 func createRandomUInt32() uint32 { diff --git a/bdd/java/src/test/java/org/apache/iggy/bdd/LeaderRedirectionSteps.java b/bdd/java/src/test/java/org/apache/iggy/bdd/LeaderRedirectionSteps.java index 5dc9778b7d..4f9645377a 100644 --- a/bdd/java/src/test/java/org/apache/iggy/bdd/LeaderRedirectionSteps.java +++ b/bdd/java/src/test/java/org/apache/iggy/bdd/LeaderRedirectionSteps.java @@ -23,6 +23,7 @@ import io.cucumber.java.en.Given; import io.cucumber.java.en.Then; import io.cucumber.java.en.When; +import org.apache.iggy.client.ConnectionInfo; import org.apache.iggy.client.blocking.tcp.IggyTcpClient; import org.apache.iggy.cluster.ClusterNode; import org.apache.iggy.cluster.ClusterNodeRole; @@ -30,6 +31,9 @@ import org.apache.iggy.exception.IggyException; import org.apache.iggy.stream.StreamDetails; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Arrays; import java.util.LinkedHashMap; import java.util.Map; import java.util.Optional; @@ -168,11 +172,15 @@ public void clientConnectsWithoutRedirection() { public void bothClientsUseTheSameServer() { IggyTcpClient clientA = client("A"); IggyTcpClient clientB = client("B"); + ConnectionInfo connectionA = clientA.getConnectionInfo(); + ConnectionInfo connectionB = clientB.getConnectionInfo(); - assertEquals( - clientA.getConnectionInfo().serverAddress(), - clientB.getConnectionInfo().serverAddress(), - "Both clients should be connected to the same server"); + assertTrue( + isSameEndpoint(connectionA, connectionB), + () -> "Both clients should be connected to the same server, got " + + connectionA.serverAddress() + + " and " + + connectionB.serverAddress()); clientA.system().ping(); clientB.system().ping(); @@ -242,6 +250,25 @@ private static Optional leaderFromMetadata(IggyTcpClient client) { } } + private static boolean isSameEndpoint(ConnectionInfo left, ConnectionInfo right) { + if (left.port() != right.port()) { + return false; + } + + InetAddress[] leftAddresses = resolveHost(left.host()); + InetAddress[] rightAddresses = resolveHost(right.host()); + return Arrays.stream(leftAddresses) + .anyMatch(leftAddress -> Arrays.stream(rightAddresses).anyMatch(leftAddress::equals)); + } + + private static InetAddress[] resolveHost(String host) { + try { + return InetAddress.getAllByName(host); + } catch (UnknownHostException error) { + throw new AssertionError("Failed to resolve server host " + host, error); + } + } + private static void assertAddressMatchesPort(String address, int port, String description) { assertTrue(address.endsWith(":" + port), description + " " + address + " should use port " + port); } diff --git a/bdd/python/tests/test_basic_messaging.py b/bdd/python/tests/test_basic_messaging.py index 363c95c4e1..d489ef4e90 100644 --- a/bdd/python/tests/test_basic_messaging.py +++ b/bdd/python/tests/test_basic_messaging.py @@ -64,8 +64,7 @@ async def _login(): @given("I have no streams in the system") def no_streams_in_system(context): """Ensure no streams exist in the system""" - # With --fresh flag on server, this should already be clean - # Just verify by attempting to get a stream that shouldn't exist + # Every run gets a new server container, so the system starts empty pass diff --git a/bdd/rust/Cargo.toml b/bdd/rust/Cargo.toml index f63ac45647..0b761fb421 100644 --- a/bdd/rust/Cargo.toml +++ b/bdd/rust/Cargo.toml @@ -25,7 +25,6 @@ publish = false [features] bdd = [] -vsr = ["iggy/vsr"] [dev-dependencies] bytes = { workspace = true } diff --git a/codecov.yml b/codecov.yml index f16b838653..7c71b59c96 100644 --- a/codecov.yml +++ b/codecov.yml @@ -68,9 +68,6 @@ flag_management: - name: node-e2e paths: - foreign/node/ - - name: node-e2e-vsr - paths: - - foreign/node/ - name: go paths: - foreign/go/ diff --git a/core/ai/mcp/Cargo.toml b/core/ai/mcp/Cargo.toml index 8f4e55fbea..9cc785d274 100644 --- a/core/ai/mcp/Cargo.toml +++ b/core/ai/mcp/Cargo.toml @@ -29,7 +29,6 @@ publish = false [features] systemd = ["dep:sd-notify", "dep:tokio-util"] -vsr = ["iggy/vsr"] [dependencies] axum = { workspace = true } diff --git a/core/bench/Cargo.toml b/core/bench/Cargo.toml index faa3269a6e..9842a4413b 100644 --- a/core/bench/Cargo.toml +++ b/core/bench/Cargo.toml @@ -31,18 +31,6 @@ publish = false name = "iggy-bench" path = "src/main.rs" -[features] -# Switches the SDK to the vsr Register-handshake framing spoken by server-ng -# clusters. The framing is chosen at compile time, so a default-features bench -# binary cannot talk to a vsr cluster at all (the first request never frames). -# TRAP: `cargo test -p integration --features vsr` does NOT rebuild this -# binary -- the harness spawns whatever the last build produced, and a -# default-featured leftover trips the bench timeout in -# `run_bench_and_wait_for_finish` (one per restart-matrix case). Build the -# workspace (or this crate with --features vsr) first; `just nextest-vsr` -# does. -vsr = ["iggy/vsr"] - [dependencies] async-trait = { workspace = true } bench-report = { workspace = true } diff --git a/core/bench/dashboard/frontend/scripts/select_index.sh b/core/bench/dashboard/frontend/scripts/select_index.sh index 93f0b699ff..087c383ca4 100755 --- a/core/bench/dashboard/frontend/scripts/select_index.sh +++ b/core/bench/dashboard/frontend/scripts/select_index.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/core/bench/src/analytics/report_builder.rs b/core/bench/src/analytics/report_builder.rs index 35119a32e4..8e5bad92b6 100644 --- a/core/bench/src/analytics/report_builder.rs +++ b/core/bench/src/analytics/report_builder.rs @@ -35,8 +35,8 @@ use iggy::prelude::{ }; use tracing::warn; -/// Both the legacy server and server-ng synthesize exactly this cluster name -/// for a non-clustered instance, so it is the single-node sentinel. +/// The server synthesizes exactly this cluster name for a non-clustered +/// instance, so it is the single-node sentinel. const SINGLE_NODE_CLUSTER_NAME: &str = "single-node"; pub struct BenchmarkReportBuilder; diff --git a/core/bench/src/main.rs b/core/bench/src/main.rs index 70e06be02c..292356b1cf 100644 --- a/core/bench/src/main.rs +++ b/core/bench/src/main.rs @@ -33,18 +33,10 @@ use tracing::{error, info}; use tracing_subscriber::{EnvFilter, fmt, layer::SubscriberExt, util::SubscriberInitExt}; use utils::cpu_name::append_cpu_name_lowercase; -/// Which SDK framing this binary was compiled with. -/// -/// One binary name, two wire dialects, and the mismatch is asymmetric: a -/// default-features bench against a vsr server HANGS rather than fails, because -/// the server reads a full 256-byte header before validating anything and the -/// client's own response timeout is itself vsr-gated. Printing the flavor in the -/// always-on banner turns that from a silent hang into a one-second diagnosis. -const SDK_FRAMING: &str = if cfg!(feature = "vsr") { - "vsr (server-ng Register handshake)" -} else { - "legacy (classic server framing)" -}; +/// Which SDK framing this binary speaks, printed in the always-on banner so a +/// server that answers a Register handshake with silence is diagnosed in a +/// second rather than mistaken for a hang. +const SDK_FRAMING: &str = "vsr (Register handshake)"; #[tokio::main] async fn main() -> Result<(), IggyError> { diff --git a/core/binary_protocol/Cargo.toml b/core/binary_protocol/Cargo.toml index ec9b25e751..4fcdab1c5d 100644 --- a/core/binary_protocol/Cargo.toml +++ b/core/binary_protocol/Cargo.toml @@ -35,6 +35,7 @@ bytes = { workspace = true } enumset = { workspace = true } secrecy = { workspace = true } thiserror = { workspace = true } +twox-hash = { workspace = true } [dev-dependencies] aligned-vec = { workspace = true } diff --git a/core/binary_protocol/src/consensus/command.rs b/core/binary_protocol/src/consensus/command.rs index 6dcf1298ae..8e60655fc1 100644 --- a/core/binary_protocol/src/consensus/command.rs +++ b/core/binary_protocol/src/consensus/command.rs @@ -41,7 +41,7 @@ pub enum Command2 { StartView = 12, Eviction = 13, - // Replica-to-replica auth handshake (server-ng consensus plane). + // Replica-to-replica auth handshake (the server consensus plane). ReplicaHello = 14, ReplicaChallenge = 15, ReplicaFinish = 16, diff --git a/core/binary_protocol/src/consensus/error.rs b/core/binary_protocol/src/consensus/error.rs index 13af4a92e6..b424660534 100644 --- a/core/binary_protocol/src/consensus/error.rs +++ b/core/binary_protocol/src/consensus/error.rs @@ -29,6 +29,23 @@ pub enum ConsensusError { #[error("invalid checksum")] InvalidChecksum, + #[error( + "{command:?}: header checksum {found:#034x} does not cover the frame (expected \ + {expected:#034x}){}", + if *found == 0 { + ". A zeroed checksum is the signature of a peer predating the frame seal, \ + which is a hard version break: replicas must be upgraded together, with the \ + cluster down" + } else { + "" + } + )] + FrameChecksumMismatch { + command: Command2, + expected: u128, + found: u128, + }, + #[error("invalid cluster ID")] InvalidCluster, diff --git a/core/binary_protocol/src/consensus/header.rs b/core/binary_protocol/src/consensus/header.rs index a4661cd27a..4f8fffe950 100644 --- a/core/binary_protocol/src/consensus/header.rs +++ b/core/binary_protocol/src/consensus/header.rs @@ -18,6 +18,23 @@ //! All consensus headers are exactly 256 bytes with `#[repr(C)]` layout. //! Size and field offsets are enforced at compile time. Deserialization //! is a pointer cast (zero-copy) via `bytemuck::try_from_bytes`. +//! +//! # Wire compatibility +//! +//! The replica-to-replica control headers are a BREAKING, non-negotiable change +//! against any build predating [`ConsensusHeader::FRAME_SEALED`]: `checksum` went +//! from a field nobody wrote to one every receiver verifies, so each side reads the +//! other's frames as corrupt. `release` must be zero on every header, so there is no +//! version channel to gate on and no way for the two to detect each other. +//! +//! Replicas must therefore be upgraded together, with the cluster down. A rolling +//! upgrade does not degrade, it stops the cluster: every control frame between a +//! mixed pair is dropped, so no view change reaches a quorum. Nothing enforces this, +//! because there is nothing left to enforce it with; this note is the declaration. +//! +//! `Prepare`, `Request`, `Reply`, and `Eviction` are unaffected. Prepares keep +//! `checksum` as their view-independent identity, and the three client-facing +//! headers are sealed on neither side, so SDKs are untouched. use super::{Command2, ConsensusError, Operation}; use bytemuck::{CheckedBitPattern, NoUninit}; @@ -54,6 +71,16 @@ pub fn read_size_field(header: &[u8]) -> Option { .map(u32::from_le_bytes) } +/// Frame checksum over a raw header: every byte past `checksum` itself. +/// +/// Byte-level twin of [`ConsensusHeader::frame_checksum`], which delegates here so +/// the typed and raw seals cannot disagree. For callers that do not know the +/// concrete header type statically, such as a wire-level test fixture. +#[must_use] +pub fn frame_checksum_bytes(header: &[u8; HEADER_SIZE]) -> u128 { + u128::from(twox_hash::XxHash3_64::oneshot(&header[size_of::()..])) +} + /// Trait implemented by all consensus header types. /// /// Every header is exactly [`HEADER_SIZE`] bytes, `#[repr(C)]`, and supports @@ -78,12 +105,86 @@ pub trait ConsensusHeader: Sized + CheckedBitPattern + NoUninit { command == Self::COMMAND } + /// Whether this header's `checksum` field seals the frame. + /// + /// True for replica-to-replica control frames, whose header carries every + /// decision field: view number, commit point, and the nack bitset that + /// authorises truncation. TCP's 16-bit checksum does not reliably catch a + /// flipped bit on a plaintext replica link. + /// + /// False for three groups: [`PrepareHeader`] / [`RepairPrepareHeader`] spend + /// `checksum` on [`PrepareHeader::identity_checksum`], which excludes `view` so + /// a re-stamped prepare keeps one identity, and a seal cannot share the field; + /// [`RequestHeader`] / [`ReplyHeader`] / [`EvictionHeader`] cross the client + /// boundary, so sealing them is an SDK change on both ends; [`GenericHeader`] is + /// the type-erased pre-dispatch view and defers to the typed parse, where + /// [`Self::verify_frame`] runs. + /// + /// Required, not defaulted: [`Self::seal`] on an unsealed type overwrites the + /// identity checksum with a frame checksum, and in release only a `debug_assert` + /// stands in the way. + const FRAME_SEALED: bool; + /// # Errors /// Returns `ConsensusError` if the header fields are inconsistent. fn validate(&self) -> Result<(), ConsensusError>; fn operation(&self) -> Operation; fn command(&self) -> Command2; fn size(&self) -> u32; + + /// The `checksum` field, whatever this header spends it on. + fn checksum(&self) -> u128; + + /// Overwrite the `checksum` field. + fn set_checksum(&mut self, checksum: u128); + + /// Checksum over every byte of the header past `checksum` itself. + /// + /// `checksum_body` sits inside that range, so sealing the header also + /// pins the body seal, and the two together cover the whole frame. + #[must_use] + fn frame_checksum(&self) -> u128 { + let bytes: &[u8; HEADER_SIZE] = bytemuck::bytes_of(self) + .try_into() + .expect("every consensus header is HEADER_SIZE bytes"); + frame_checksum_bytes(bytes) + } + + /// Stamp [`Self::frame_checksum`]. Call last when building a frame: it covers + /// every other field, `checksum_body` included, so later writes are uncovered. + fn seal(&mut self) { + debug_assert!( + Self::FRAME_SEALED, + "sealing a header whose checksum field means something else", + ); + let checksum = self.frame_checksum(); + self.set_checksum(checksum); + } + + /// Reject a frame whose header does not match its own checksum. + /// + /// Runs before [`Self::validate`] on every typed parse: a header that did not + /// arrive intact cannot have any field believed, `validate`'s included. + /// + /// # Errors + /// [`ConsensusError::FrameChecksumMismatch`] on a bad seal. Unsealed header + /// types return `Ok` unconditionally. + fn verify_frame(&self) -> Result<(), ConsensusError> { + if !Self::FRAME_SEALED { + return Ok(()); + } + let expected = self.frame_checksum(); + let found = self.checksum(); + if found == expected { + Ok(()) + } else { + Err(ConsensusError::FrameChecksumMismatch { + command: self.command(), + expected, + found, + }) + } + } } // GenericHeader - type-erased dispatch @@ -121,6 +222,15 @@ const _: () = { impl ConsensusHeader for GenericHeader { const COMMAND: Command2 = Command2::Reserved; + const FRAME_SEALED: bool = false; + + fn checksum(&self) -> u128 { + self.checksum + } + + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } fn operation(&self) -> Operation { Operation::Reserved } @@ -162,7 +272,6 @@ pub struct RequestHeader { pub request: u64, pub operation: Operation, pub operation_padding: [u8; 7], - pub namespace: u64, /// Session fence epoch: the commit op of the latest committed `Register` /// for this `client`. Handed to the client by that register's reply and /// echoed on every subsequent request. @@ -183,7 +292,7 @@ pub struct RequestHeader { /// The submitter's wire value is never trusted. Zero for `Logout`, /// partition-plane, and server-internal ops. pub user_id: u32, - pub reserved: [u8; 52], + pub reserved: [u8; 60], } const _: () = { assert!(size_of::() == HEADER_SIZE); @@ -194,10 +303,63 @@ const _: () = { assert!( offset_of!(RequestHeader, user_id) == offset_of!(RequestHeader, session) + size_of::() ); - assert!(offset_of!(RequestHeader, reserved) + size_of::<[u8; 52]>() == HEADER_SIZE); + assert!(offset_of!(RequestHeader, reserved) + size_of::<[u8; 60]>() == HEADER_SIZE); }; -impl Default for RequestHeader { +/// A [`RequestHeader`] AFTER the receiving node resolved its target. +/// +/// The server-internal shape a client request travels in between shards +/// (and follower to primary), never on the client wire. `group` carries the +/// resolved consensus group so the owner shard can route, park, and fence +/// WITHOUT re-decoding the payload -- clients no longer send any namespace +/// (it is derived: plane from `operation`, partition group from the body), +/// so this is where the derivation result lives for the internal hop. +/// +/// Layout: identical to [`RequestHeader`] with `group` claiming the LAST +/// eight reserved bytes (the tail, bytes 248..256); the leading 52 reserved +/// bytes keep their client-wire meaning, so promotion is a same-size copy. +#[repr(C)] +#[derive(Debug, Clone, Copy, CheckedBitPattern, NoUninit)] +pub struct RoutedRequestHeader { + pub checksum: u128, + pub checksum_body: u128, + pub cluster: u128, + pub size: u32, + pub view: u32, + pub release: u32, + pub command: Command2, + pub replica: u8, + pub reserved_frame: [u8; 66], + + pub client: u128, + pub request_checksum: u128, + pub timestamp: u64, + pub request: u64, + pub operation: Operation, + pub operation_padding: [u8; 7], + pub session: u64, + pub user_id: u32, + /// Same offset and meaning as the leading 52 bytes of + /// `RequestHeader::reserved` -- this region CARRIES DATA (the + /// non-replicated op code range), so `group` must not displace it. + pub reserved: [u8; 52], + /// The resolved consensus group id (see `binary_protocol::namespace`), + /// claiming the TAIL of the client header's reserved area. + pub group: u64, +} +const _: () = { + assert!(size_of::() == HEADER_SIZE); + // Every field shared with `RequestHeader` sits at the same offset -- + // including the data-bearing prefix of `reserved` (the non-replicated + // code range) -- so promotion preserves everything a client sent. + assert!(offset_of!(RoutedRequestHeader, client) == offset_of!(RequestHeader, client)); + assert!(offset_of!(RoutedRequestHeader, session) == offset_of!(RequestHeader, session)); + assert!(offset_of!(RoutedRequestHeader, user_id) == offset_of!(RequestHeader, user_id)); + assert!(offset_of!(RoutedRequestHeader, reserved) == offset_of!(RequestHeader, reserved)); + assert!(offset_of!(RoutedRequestHeader, group) + size_of::() == HEADER_SIZE); +}; + +impl Default for RoutedRequestHeader { fn default() -> Self { Self { checksum: 0, @@ -215,16 +377,112 @@ impl Default for RequestHeader { request: 0, operation: Operation::Reserved, operation_padding: [0; 7], - namespace: 0, session: 0, user_id: 0, reserved: [0; 52], + group: 0, } } } -impl ConsensusHeader for RequestHeader { +impl Default for RequestHeader { + fn default() -> Self { + Self { + checksum: 0, + checksum_body: 0, + cluster: 0, + size: 0, + view: 0, + release: 0, + command: Command2::Reserved, + replica: 0, + reserved_frame: [0; 66], + client: 0, + request_checksum: 0, + timestamp: 0, + request: 0, + operation: Operation::Reserved, + operation_padding: [0; 7], + session: 0, + user_id: 0, + reserved: [0; 60], + } + } +} + +/// Field rules shared by the client-wire [`RequestHeader`] and the +/// server-internal [`RoutedRequestHeader`]. The routed shape is decoded +/// straight off the peer wire (`MessageBag`), so it must reject everything +/// the client boundary rejects; validating only the command byte there +/// would let a peer frame carry `client = 0` into the client table's hard +/// assert, or `operation = Reserved` into a cached-register replay. +fn validate_request_fields( + client: u128, + operation: Operation, + session: u64, + request: u64, +) -> Result<(), ConsensusError> { + if client == 0 { + return Err(ConsensusError::InvalidField( + "request: client must be != 0".to_string(), + )); + } + // Reserved is the zero value, never a real client op + // (`is_client_allowed` rejects it). Refusing it here rather than after + // the dedup preflight matters: a bound client sending + // `Reserved, request = 0` used to pass validation, reach + // `request_preflight`, hit its own watermark and get its register + // reply replayed before the operation gate ever ran. + if operation == Operation::Reserved { + return Err(ConsensusError::InvalidField( + "operation must not be Reserved".to_string(), + )); + } + // Register: session must be 0, request must be 0. + // NonReplicated: sessionless by design (the `ClientTable` ignores + // these ops and the server routes/auth-gates them by transport id), + // so a pre-register client may legitimately send session 0 -- + // ping must work before authentication. + // Other non-register ops: session must be > 0, request must be > 0. + if operation == Operation::Register { + if session != 0 { + return Err(ConsensusError::InvalidField( + "register: session must be 0".to_string(), + )); + } + if request != 0 { + return Err(ConsensusError::InvalidField( + "register: request must be 0".to_string(), + )); + } + } else if operation != Operation::NonReplicated { + if session == 0 { + return Err(ConsensusError::InvalidField( + "non-register: session must be > 0".to_string(), + )); + } + if request == 0 { + return Err(ConsensusError::InvalidField( + "non-register: request must be > 0".to_string(), + )); + } + } + Ok(()) +} + +impl ConsensusHeader for RoutedRequestHeader { const COMMAND: Command2 = Command2::Request; + /// The client-wire [`RequestHeader`] this is promoted from is unsealed, and the + /// promotion copies `checksum` verbatim, so there is nothing here to verify. + const FRAME_SEALED: bool = false; + + fn checksum(&self) -> u128 { + self.checksum + } + + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } fn operation(&self) -> Operation { self.operation } @@ -242,52 +500,39 @@ impl ConsensusHeader for RequestHeader { found: self.command, }); } - if self.client == 0 { - return Err(ConsensusError::InvalidField( - "request: client must be != 0".to_string(), - )); - } - // Reserved is the zero value, never a real client op - // (`is_client_allowed` rejects it). Refusing it here rather than after - // the dedup preflight matters: a bound client sending - // `Reserved, request = 0` used to pass validation, reach - // `request_preflight`, hit its own watermark and get its register - // reply replayed before the operation gate ever ran. - if self.operation == Operation::Reserved { - return Err(ConsensusError::InvalidField( - "operation must not be Reserved".to_string(), - )); - } - // Register: session must be 0, request must be 0. - // NonReplicated: sessionless by design (the `ClientTable` ignores - // these ops and the server routes/auth-gates them by transport id), - // so a pre-register client may legitimately send session 0 -- - // ping must work before authentication. - // Other non-register ops: session must be > 0, request must be > 0. - if self.operation == Operation::Register { - if self.session != 0 { - return Err(ConsensusError::InvalidField( - "register: session must be 0".to_string(), - )); - } - if self.request != 0 { - return Err(ConsensusError::InvalidField( - "register: request must be 0".to_string(), - )); - } - } else if self.operation != Operation::NonReplicated { - if self.session == 0 { - return Err(ConsensusError::InvalidField( - "non-register: session must be > 0".to_string(), - )); - } - if self.request == 0 { - return Err(ConsensusError::InvalidField( - "non-register: request must be > 0".to_string(), - )); - } + validate_request_fields(self.client, self.operation, self.session, self.request) + } +} + +impl ConsensusHeader for RequestHeader { + const COMMAND: Command2 = Command2::Request; + const FRAME_SEALED: bool = false; + + fn checksum(&self) -> u128 { + self.checksum + } + + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } + fn operation(&self) -> Operation { + self.operation + } + fn command(&self) -> Command2 { + self.command + } + fn size(&self) -> u32 { + self.size + } + + fn validate(&self) -> Result<(), ConsensusError> { + if self.command != Command2::Request { + return Err(ConsensusError::InvalidCommand { + expected: Command2::Request, + found: self.command, + }); } - Ok(()) + validate_request_fields(self.client, self.operation, self.session, self.request) } } @@ -318,7 +563,6 @@ pub struct ReplyHeader { pub request: u64, pub operation: Operation, pub operation_padding: [u8; 7], - pub namespace: u64, /// Request-level status: 0 = ok; nonzero = the `IggyError` code for a /// failure decided before commit (e.g. a dispatch-time authorization /// denial, or the partition primary rejecting a consumer-offset op). @@ -332,7 +576,7 @@ pub struct ReplyHeader { /// `user_id` in `RequestHeader` / `PrepareHeader`; no existing field offset /// moves and `validate` does not inspect it. pub status: u32, - pub reserved: [u8; 28], + pub reserved: [u8; 36], } const _: () = { assert!(size_of::() == HEADER_SIZE); @@ -340,10 +584,7 @@ const _: () = { offset_of!(ReplyHeader, request_checksum) == offset_of!(ReplyHeader, reserved_frame) + size_of::<[u8; 66]>() ); - assert!( - offset_of!(ReplyHeader, status) == offset_of!(ReplyHeader, namespace) + size_of::() - ); - assert!(offset_of!(ReplyHeader, reserved) + size_of::<[u8; 28]>() == HEADER_SIZE); + assert!(offset_of!(ReplyHeader, reserved) + size_of::<[u8; 36]>() == HEADER_SIZE); }; impl Default for ReplyHeader { @@ -367,15 +608,23 @@ impl Default for ReplyHeader { request: 0, operation: Operation::Reserved, operation_padding: [0; 7], - namespace: 0, status: 0, - reserved: [0; 28], + reserved: [0; 36], } } } impl ConsensusHeader for ReplyHeader { const COMMAND: Command2 = Command2::Reply; + const FRAME_SEALED: bool = false; + + fn checksum(&self) -> u128 { + self.checksum + } + + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } fn operation(&self) -> Operation { self.operation } @@ -558,6 +807,15 @@ impl EvictionHeader { impl ConsensusHeader for EvictionHeader { const COMMAND: Command2 = Command2::Eviction; + const FRAME_SEALED: bool = false; + + fn checksum(&self) -> u128 { + self.checksum + } + + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } /// Session-level (not per-op): always `Reserved`. fn operation(&self) -> Operation { Operation::Reserved @@ -633,7 +891,7 @@ impl ConsensusHeader for EvictionHeader { /// Primary -> replicas: replicate this operation. #[repr(C)] -#[derive(Debug, Clone, Copy, CheckedBitPattern, NoUninit)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, CheckedBitPattern, NoUninit)] pub struct PrepareHeader { pub checksum: u128, pub checksum_body: u128, @@ -655,7 +913,12 @@ pub struct PrepareHeader { pub request: u64, pub operation: Operation, pub operation_padding: [u8; 7], - pub namespace: u64, + /// Consensus group id: which of the node's multiplexed VSR groups this + /// frame belongs to. `METADATA_GROUP` (top bit) for the metadata plane, + /// otherwise the partition's packed stream-topic-partition key. The + /// demux and repair-replay routing key; see `binary_protocol::namespace` + /// for the value-space contract. + pub group: u64, /// Acting user id, copied verbatim from the admitted `RequestHeader`; see /// that field for the stamping contract. pub user_id: u32, @@ -668,8 +931,7 @@ const _: () = { == offset_of!(PrepareHeader, reserved_frame) + size_of::<[u8; 66]>() ); assert!( - offset_of!(PrepareHeader, user_id) - == offset_of!(PrepareHeader, namespace) + size_of::() + offset_of!(PrepareHeader, user_id) == offset_of!(PrepareHeader, group) + size_of::() ); assert!(offset_of!(PrepareHeader, reserved) + size_of::<[u8; 28]>() == HEADER_SIZE); }; @@ -695,7 +957,7 @@ impl Default for PrepareHeader { request: 0, operation: Operation::Reserved, operation_padding: [0; 7], - namespace: 0, + group: 0, user_id: 0, reserved: [0; 28], } @@ -704,6 +966,15 @@ impl Default for PrepareHeader { impl ConsensusHeader for PrepareHeader { const COMMAND: Command2 = Command2::Prepare; + const FRAME_SEALED: bool = false; + + fn checksum(&self) -> u128 { + self.checksum + } + + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } fn operation(&self) -> Operation { self.operation } @@ -721,10 +992,63 @@ impl ConsensusHeader for PrepareHeader { found: self.command, }); } + // Both reserved regions must be zero. They sit inside + // [`Self::identity_checksum`], so a peer that fills them changes the op's + // identity while changing nothing the merge can see; and `dvc_blank` + // classifies a slot by exact struct equality, so a non-zero reserved byte + // turns a blank into a `Valid` header the merge then indexes. + if self.reserved_frame.iter().any(|&byte| byte != 0) { + return Err(ConsensusError::InvalidField( + "prepare: reserved_frame bytes must be zero".to_string(), + )); + } + if self.reserved.iter().any(|&byte| byte != 0) { + return Err(ConsensusError::InvalidField( + "prepare: reserved bytes must be zero".to_string(), + )); + } Ok(()) } } +/// `checksum` of a prepare no producer sealed. +/// +/// Written by a build predating the identity seal, or by the partition plane. +/// Verification skips such entries so an older build's WAL still replays. +pub const CHECKSUM_UNSEALED: u128 = 0; + +/// The frame's body, bounded by `size`. What `checksum_body` covers. +/// +/// Not `&frame[HEADER_SIZE..]`: `Message::try_from` accepts a buffer longer than +/// `size` without trimming, while the WAL scan reads exactly `size`, so slicing to +/// the end makes the two disagree. Empty when `size` overruns the buffer. +#[must_use] +pub fn frame_body(frame: &[u8], size: u32) -> &[u8] { + let end = size as usize; + if end <= HEADER_SIZE || end > frame.len() { + return &[]; + } + &frame[HEADER_SIZE..end] +} + +impl PrepareHeader { + /// Which prepare this is, independent of which view re-sent it. + /// + /// Covers the whole 256-byte header except `checksum` (a field cannot hash + /// itself) and `view`, so a retransmission that re-stamps `view` stays valid. + /// The body reaches the value through the covered `checksum_body`. + /// + /// Lives here, not in the consensus crate, because the WAL scan verifies it too + /// and the two must agree byte for byte: it hashes this struct's layout. + #[must_use] + pub fn identity_checksum(&self) -> u128 { + let mut covered = *self; + covered.checksum = 0; + covered.view = 0; + u128::from(twox_hash::XxHash3_64::oneshot(bytemuck::bytes_of(&covered))) + } +} + // RepairPrepareHeader - repair peer -> recovering replica (journal repair) /// A stored prepare served for journal repair. @@ -740,6 +1064,15 @@ pub struct RepairPrepareHeader(pub PrepareHeader); impl ConsensusHeader for RepairPrepareHeader { const COMMAND: Command2 = Command2::RepairPrepare; + const FRAME_SEALED: bool = false; + + fn checksum(&self) -> u128 { + self.0.checksum + } + + fn set_checksum(&mut self, checksum: u128) { + self.0.checksum = checksum; + } fn operation(&self) -> Operation { self.0.operation } @@ -757,6 +1090,21 @@ impl ConsensusHeader for RepairPrepareHeader { found: self.0.command, }); } + // Same rule as `PrepareHeader::validate`, same reason: the regions sit inside + // `identity_checksum`, and a repaired prepare is journaled and later re-read + // as a DVC suffix entry, where `dvc_blank`'s exact-equality classification is + // what a dirty byte defeats. Not delegated, so the command check above stays + // `RepairPrepare`. + if self.0.reserved_frame.iter().any(|&byte| byte != 0) { + return Err(ConsensusError::InvalidField( + "repair_prepare: reserved_frame bytes must be zero".to_string(), + )); + } + if self.0.reserved.iter().any(|&byte| byte != 0) { + return Err(ConsensusError::InvalidField( + "repair_prepare: reserved bytes must be zero".to_string(), + )); + } Ok(()) } } @@ -785,7 +1133,7 @@ pub struct PrepareOkHeader { pub request: u64, pub operation: Operation, pub operation_padding: [u8; 7], - pub namespace: u64, + pub group: u64, pub reserved: [u8; 48], } const _: () = { @@ -817,14 +1165,24 @@ impl Default for PrepareOkHeader { request: 0, operation: Operation::Reserved, operation_padding: [0; 7], - namespace: 0, + group: 0, reserved: [0; 48], } } } impl ConsensusHeader for PrepareOkHeader { + const FRAME_SEALED: bool = true; + const COMMAND: Command2 = Command2::PrepareOk; + + fn checksum(&self) -> u128 { + self.checksum + } + + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } fn operation(&self) -> Operation { self.operation } @@ -866,7 +1224,7 @@ pub struct CommitHeader { pub timestamp_monotonic: u64, pub commit: u64, pub checkpoint_op: u64, - pub namespace: u64, + pub group: u64, pub reserved: [u8; 80], } const _: () = { @@ -879,7 +1237,17 @@ const _: () = { }; impl ConsensusHeader for CommitHeader { + const FRAME_SEALED: bool = true; + const COMMAND: Command2 = Command2::Commit; + + fn checksum(&self) -> u128 { + self.checksum + } + + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } fn operation(&self) -> Operation { Operation::Reserved } @@ -917,20 +1285,30 @@ pub struct StartViewChangeHeader { pub replica: u8, pub reserved_frame: [u8; 66], - pub namespace: u64, + pub group: u64, pub reserved: [u8; 120], } const _: () = { assert!(size_of::() == HEADER_SIZE); assert!( - offset_of!(StartViewChangeHeader, namespace) + offset_of!(StartViewChangeHeader, group) == offset_of!(StartViewChangeHeader, reserved_frame) + size_of::<[u8; 66]>() ); assert!(offset_of!(StartViewChangeHeader, reserved) + size_of::<[u8; 120]>() == HEADER_SIZE); }; impl ConsensusHeader for StartViewChangeHeader { + const FRAME_SEALED: bool = true; + const COMMAND: Command2 = Command2::StartViewChange; + + fn checksum(&self) -> u128 { + self.checksum + } + + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } fn operation(&self) -> Operation { Operation::Reserved } @@ -975,10 +1353,31 @@ pub struct DoViewChangeHeader { pub op: u64, /// Highest committed op. pub commit: u64, - pub namespace: u64, + pub group: u64, /// View when status was last normal (key for log selection). pub log_view: u32, - pub reserved: [u8; 100], + pub reserved: [u8; 68], + /// Bit `i` set means the sender proves it never prepared suffix entry `i`, so + /// that entry never reached a replication quorum through this replica. A new + /// primary may truncate only once `quorum_nack_prepare` senders nack an entry; + /// short of that it might be committed and must be preserved. + /// + /// A corrupt local entry is deliberately NOT nacked: the sender cannot tell a + /// prepare it never saw from one it saw and lost, and only the former is proof. + /// Silence costs availability; a false nack costs data. + /// + /// Carved from the tail of the former `reserved` region, with `present_bitset` + /// LAST so both land 16-aligned with no padding and `op`/`commit`/`group`/ + /// `log_view` keep their offsets. A sender with nothing to nack sends zeros, + /// decoding as "nacks nothing": safe, since that can only slow a view change. + pub nack_bitset: u128, + /// Bit `i` set means the sender can serve the BODY of suffix entry `i`, not just + /// its header. The new primary needs one such sender per surviving entry, since + /// a header whose body it cannot fetch is an entry it can never commit. + /// + /// Zero from a sender offering nothing, reading as "offers no bodies": safe, + /// since the new primary waits rather than adopting an entry it cannot complete. + pub present_bitset: u128, } const _: () = { assert!(size_of::() == HEADER_SIZE); @@ -986,11 +1385,38 @@ const _: () = { offset_of!(DoViewChangeHeader, op) == offset_of!(DoViewChangeHeader, reserved_frame) + size_of::<[u8; 66]>() ); - assert!(offset_of!(DoViewChangeHeader, reserved) + size_of::<[u8; 100]>() == HEADER_SIZE); + // op/commit/group/log_view keep their pre-bitset offsets. + assert!(offset_of!(DoViewChangeHeader, reserved) == 156); + // Both bitsets are last and 16-aligned, so the struct has no padding + // (`NoUninit` would reject any). + assert!(offset_of!(DoViewChangeHeader, nack_bitset) % 16 == 0); + assert!(offset_of!(DoViewChangeHeader, present_bitset) % 16 == 0); + assert!( + offset_of!(DoViewChangeHeader, nack_bitset) + == offset_of!(DoViewChangeHeader, reserved) + size_of::<[u8; 68]>() + ); + assert!(offset_of!(DoViewChangeHeader, present_bitset) + size_of::() == HEADER_SIZE); }; +/// Suffix headers a `DoViewChange` may carry: one bit per entry in each of the two +/// `u128` bitsets. +/// +/// Mirrors `consensus::DVC_HEADERS_MAX` as a literal so this crate need not depend +/// on the consensus crate, as with `REPLICAS_MAX` in [`EvictionHeader::new`]. +pub const DVC_HEADERS_MAX: usize = 128; + impl ConsensusHeader for DoViewChangeHeader { + const FRAME_SEALED: bool = true; + const COMMAND: Command2 = Command2::DoViewChange; + + fn checksum(&self) -> u128 { + self.checksum + } + + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } fn operation(&self) -> Operation { Operation::Reserved } @@ -1023,10 +1449,64 @@ impl ConsensusHeader for DoViewChangeHeader { "commit cannot exceed op".to_string(), )); } + let suffix_len = self.suffix_len()?; + // Bits past the suffix describe entries never sent: unchecked, a peer could + // smuggle a nack for an op the new primary would then truncate. + if suffix_len < DVC_HEADERS_MAX { + let beyond = !((1u128 << suffix_len) - 1); + if self.nack_bitset & beyond != 0 || self.present_bitset & beyond != 0 { + return Err(ConsensusError::InvalidField(format!( + "do_view_change: bitset bits set past the {suffix_len}-entry suffix" + ))); + } + } Ok(()) } } +impl DoViewChangeHeader { + /// Number of `PrepareHeader`s in the body. + /// + /// Zero is valid and means "no suffix": a replica with nothing uncommitted + /// contributes numbers only. + /// + /// # Errors + /// [`ConsensusError::InvalidField`] when `size` is short of the header, is not a + /// whole number of headers, or exceeds what the bitsets can address. + pub fn suffix_len(&self) -> Result { + suffix_len_of("do_view_change", self.size) + } +} + +/// Body length of a suffix-carrying control frame, in whole [`PrepareHeader`]s. +/// +/// Shared by `DoViewChange` and `StartView`: same layout, same `DVC_HEADERS_MAX` +/// bound. `frame` only names the sender in the error text. +/// +/// # Errors +/// [`ConsensusError::InvalidField`] when `size` is short of the header, is not a +/// whole number of headers, or exceeds what a view change can address. +fn suffix_len_of(frame: &str, size: u32) -> Result { + let size = size as usize; + let Some(body_len) = size.checked_sub(HEADER_SIZE) else { + return Err(ConsensusError::InvalidField(format!( + "{frame}: size {size} is shorter than the {HEADER_SIZE}-byte header" + ))); + }; + if body_len % HEADER_SIZE != 0 { + return Err(ConsensusError::InvalidField(format!( + "{frame}: body of {body_len} bytes is not a whole number of headers" + ))); + } + let suffix_len = body_len / HEADER_SIZE; + if suffix_len > DVC_HEADERS_MAX { + return Err(ConsensusError::InvalidField(format!( + "{frame}: {suffix_len} suffix entries exceeds the maximum {DVC_HEADERS_MAX}" + ))); + } + Ok(suffix_len) +} + // StartViewHeader - new view announcement (header-only) /// New primary -> all replicas: start new view. Header-only. @@ -1047,7 +1527,7 @@ pub struct StartViewHeader { pub op: u64, /// max(commit) from all DVCs. pub commit: u64, - pub namespace: u64, + pub group: u64, pub reserved: [u8; 88], /// Sender's incarnation, echoed from the `RequestStartView` this answers so a /// recovering replica can prove the reply post-dates its restart (see @@ -1055,16 +1535,15 @@ pub struct StartViewHeader { /// (a normal view-change completion), which carries no freshness claim. /// /// Carved from the tail of the former `reserved` region and placed LAST so it - /// lands 16-aligned with no padding WITHOUT moving `op`/`commit`/`namespace`. - /// A peer that predates it sends zeros, decoding as `incarnation == 0`, which - /// the `handle_start_view` guard treats as no claim rather than as a foreign - /// one, so a mixed-version rolling upgrade is wire-compatible: the pre-upgrade - /// peer's `StartView` is judged by the view checks alone, as before the field. + /// lands 16-aligned with no padding WITHOUT moving `op`/`commit`/`group`. + /// Zero is "no claim", which is what `handle_start_view` keys on and what the + /// unsolicited completion path sends. NOT mixed-version tolerance: the frame seal + /// drops a pre-seal peer before any field is read (see this module's header). pub incarnation: u128, } const _: () = { assert!(size_of::() == HEADER_SIZE); - // op/commit/namespace keep their pre-incarnation offsets. + // op/commit/group keep their pre-incarnation offsets. assert!( offset_of!(StartViewHeader, op) == offset_of!(StartViewHeader, reserved_frame) + size_of::<[u8; 66]>() @@ -1075,7 +1554,17 @@ const _: () = { }; impl ConsensusHeader for StartViewHeader { + const FRAME_SEALED: bool = true; + const COMMAND: Command2 = Command2::StartView; + + fn checksum(&self) -> u128 { + self.checksum + } + + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } fn operation(&self) -> Operation { Operation::Reserved } @@ -1103,10 +1592,26 @@ impl ConsensusHeader for StartViewHeader { "commit cannot exceed op".to_string(), )); } + self.suffix_len()?; Ok(()) } } +impl StartViewHeader { + /// Number of `PrepareHeader`s in the body: the view's suffix, high-to-low op + /// from `op` down toward `commit`. + /// + /// Zero means numbers only, which is what the probe-answer path sends. A backup + /// then falls back to trusting `op`. + /// + /// # Errors + /// [`ConsensusError::InvalidField`] when `size` is short of the header, is not a + /// whole number of headers, or exceeds what a view change can address. + pub fn suffix_len(&self) -> Result { + suffix_len_of("start_view", self.size) + } +} + // RequestStartViewHeader - restarted replica asking for the current view /// Recovering replica -> all replicas: resend me the current `StartView`. @@ -1130,22 +1635,22 @@ pub struct RequestStartViewHeader { pub replica: u8, pub reserved_frame: [u8; 66], - pub namespace: u64, + pub group: u64, pub reserved: [u8; 104], /// The requester's per-boot incarnation, echoed back in the answering /// `StartView` so a reply from a previous incarnation is detectable. /// /// Carved from the tail of the former `reserved` region and placed LAST so it - /// lands 16-aligned with no padding WITHOUT moving `namespace`. A peer that - /// predates it sends zeros, decoding as `incarnation == 0`, so a mixed-version - /// rolling upgrade is wire-compatible. + /// lands 16-aligned with no padding WITHOUT moving `group`. Zero is "no claim + /// to echo"; see [`StartViewHeader::incarnation`] on why that is not + /// mixed-version tolerance. pub incarnation: u128, } const _: () = { assert!(size_of::() == HEADER_SIZE); - // namespace keeps its pre-incarnation offset. + // group keeps its pre-incarnation offset. assert!( - offset_of!(RequestStartViewHeader, namespace) + offset_of!(RequestStartViewHeader, group) == offset_of!(RequestStartViewHeader, reserved_frame) + size_of::<[u8; 66]>() ); // `incarnation` is last and 16-aligned, so the struct has no padding. @@ -1154,7 +1659,17 @@ const _: () = { }; impl ConsensusHeader for RequestStartViewHeader { + const FRAME_SEALED: bool = true; + const COMMAND: Command2 = Command2::RequestStartView; + + fn checksum(&self) -> u128 { + self.checksum + } + + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } fn operation(&self) -> Operation { Operation::Reserved } @@ -1186,7 +1701,7 @@ impl ConsensusHeader for RequestStartViewHeader { /// Recovering/holed replica -> a Normal peer: request a repair stream. /// /// Sent to the primary first. Asks for the journaled prepares in -/// `[from_op, to_op]` for `namespace`. Header-only. The peer answers with +/// `[from_op, to_op]` for `group`. Header-only. The peer answers with /// `RepairPrepare` frames in op order, terminated by `RepairDone` or /// `RangeEvicted`. #[derive(Debug, Clone, Copy, PartialEq, Eq, CheckedBitPattern, NoUninit)] @@ -1205,7 +1720,7 @@ pub struct RequestPreparesHeader { pub nonce: u128, pub from_op: u64, pub to_op: u64, - pub namespace: u64, + pub group: u64, pub reserved: [u8; 88], } const _: () = { @@ -1218,7 +1733,17 @@ const _: () = { }; impl ConsensusHeader for RequestPreparesHeader { + const FRAME_SEALED: bool = true; + const COMMAND: Command2 = Command2::RequestPrepares; + + fn checksum(&self) -> u128 { + self.checksum + } + + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } fn operation(&self) -> Operation { Operation::Reserved } @@ -1268,7 +1793,7 @@ pub struct RepairRangeReplyHeader { pub nonce: u128, /// `RepairDone`: last op served. `RangeEvicted`: oldest retained op. pub op: u64, - pub namespace: u64, + pub group: u64, pub reserved: [u8; 96], } const _: () = { @@ -1281,7 +1806,17 @@ const _: () = { }; impl ConsensusHeader for RepairRangeReplyHeader { + const FRAME_SEALED: bool = true; + const COMMAND: Command2 = Command2::RepairDone; + + fn checksum(&self) -> u128 { + self.checksum + } + + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } // One layout, two commands: `RepairDone` terminates a stream, // `RangeEvicted` prefixes it. Without this widening, `try_into_typed` // rejects `RangeEvicted` frames before `validate` ever sees them. @@ -1338,7 +1873,7 @@ pub struct RequestStateTransferHeader { pub reserved_frame: [u8; 66], pub nonce: u128, - pub namespace: u64, + pub group: u64, pub reserved: [u8; 104], } const _: () = { @@ -1353,7 +1888,17 @@ const _: () = { }; impl ConsensusHeader for RequestStateTransferHeader { + const FRAME_SEALED: bool = true; + const COMMAND: Command2 = Command2::RequestStateTransfer; + + fn checksum(&self) -> u128 { + self.checksum + } + + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } fn operation(&self) -> Operation { Operation::Reserved } @@ -1413,7 +1958,7 @@ pub struct StateTransferTargetHeader { /// Serving primary's applied frontier (`commit_min`) when the descriptor /// was built. The receiver's tail repair targets past this. pub commit_op: u64, - pub namespace: u64, + pub group: u64, pub available: u8, /// Set on an `available == 0` refusal that means "not right now" rather than /// "this node is broken". @@ -1459,7 +2004,7 @@ const _: () = { // The pre-existing published offsets. New fields grow into the reserved // tail only; a change that moves one of these is a wire break. assert!(offset_of!(StateTransferTargetHeader, commit_op) == 144); - assert!(offset_of!(StateTransferTargetHeader, namespace) == 152); + assert!(offset_of!(StateTransferTargetHeader, group) == 152); assert!(offset_of!(StateTransferTargetHeader, available) == 160); assert!(offset_of!(StateTransferTargetHeader, unavailable_transient) == 161); assert!(offset_of!(StateTransferTargetHeader, commit_max) == 168); @@ -1467,7 +2012,17 @@ const _: () = { }; impl ConsensusHeader for StateTransferTargetHeader { + const FRAME_SEALED: bool = true; + const COMMAND: Command2 = Command2::StateTransferTarget; + + fn checksum(&self) -> u128 { + self.checksum + } + + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } fn operation(&self) -> Operation { Operation::Reserved } @@ -1541,7 +2096,7 @@ pub struct RequestStateChunkHeader { pub nonce: u128, pub offset: u64, - pub namespace: u64, + pub group: u64, pub len: u32, /// Index into the offered state manifest. Range-checked by the serving /// handler against the cached offer (the header cannot know the count). @@ -1558,7 +2113,17 @@ const _: () = { }; impl ConsensusHeader for RequestStateChunkHeader { + const FRAME_SEALED: bool = true; + const COMMAND: Command2 = Command2::RequestStateChunk; + + fn checksum(&self) -> u128 { + self.checksum + } + + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } fn operation(&self) -> Operation { Operation::Reserved } @@ -1616,7 +2181,7 @@ pub struct StateChunkHeader { pub nonce: u128, pub offset: u64, - pub namespace: u64, + pub group: u64, /// Index into the offered state manifest. Range-checked by the receiving /// handler against its accepted manifest. pub artifact: u32, @@ -1632,7 +2197,17 @@ const _: () = { }; impl ConsensusHeader for StateChunkHeader { + const FRAME_SEALED: bool = true; + const COMMAND: Command2 = Command2::StateChunk; + + fn checksum(&self) -> u128 { + self.checksum + } + + fn set_checksum(&mut self, checksum: u128) { + self.checksum = checksum; + } fn operation(&self) -> Operation { Operation::Reserved } @@ -1664,9 +2239,12 @@ impl ConsensusHeader for StateChunkHeader { #[cfg(test)] mod tests { use super::{ - Command2, CommitHeader, ConsensusHeader, DoViewChangeHeader, EvictionHeader, - EvictionReason, GenericHeader, HEADER_SIZE, Operation, PrepareHeader, PrepareOkHeader, - ReplyHeader, RequestHeader, StartViewChangeHeader, StartViewHeader, + Command2, CommitHeader, ConsensusError, ConsensusHeader, DoViewChangeHeader, + EvictionHeader, EvictionReason, GenericHeader, HEADER_SIZE, Operation, PrepareHeader, + PrepareOkHeader, RepairPrepareHeader, RepairRangeReplyHeader, ReplyHeader, RequestHeader, + RequestPreparesHeader, RequestStartViewHeader, RequestStateChunkHeader, + RequestStateTransferHeader, RoutedRequestHeader, StartViewChangeHeader, StartViewHeader, + StateChunkHeader, StateTransferTargetHeader, }; use aligned_vec::{AVec, ConstAlign}; @@ -1678,6 +2256,128 @@ mod tests { v } + /// A header-sized frame that satisfies `bytemuck`'s 16-byte alignment. + #[repr(C, align(16))] + struct AlignedFrame([u8; HEADER_SIZE]); + + /// A minimal well-formed header of type `H`: own command and size, everything + /// else zero. Enough for the seal, which reads bytes rather than fields. + fn control_header() -> H { + const COMMAND_OFF: usize = std::mem::offset_of!(GenericHeader, command); + const SIZE_OFF: usize = std::mem::offset_of!(GenericHeader, size); + + let frame_len = u32::try_from(HEADER_SIZE).expect("HEADER_SIZE fits u32"); + let mut frame = AlignedFrame([0u8; HEADER_SIZE]); + frame.0[COMMAND_OFF] = H::COMMAND as u8; + frame.0[SIZE_OFF..SIZE_OFF + 4].copy_from_slice(&frame_len.to_le_bytes()); + *bytemuck::checked::try_from_bytes::(&frame.0).expect("a zeroed frame is a valid header") + } + + /// Seal a header, flip one bit at `offset`, and report `verify_frame`'s verdict. + fn tamper(mut header: H, offset: usize) -> Result<(), ConsensusError> { + header.seal(); + let mut frame = AlignedFrame([0u8; HEADER_SIZE]); + frame.0.copy_from_slice(bytemuck::bytes_of(&header)); + frame.0[offset] ^= 0x01; + let tampered = bytemuck::checked::try_from_bytes::(&frame.0) + .expect("a single flipped bit stays a valid bit pattern here"); + tampered.verify_frame() + } + + #[test] + fn given_a_sealed_control_header_when_verifying_should_accept() { + macro_rules! seals { + ($($header:ty),+ $(,)?) => {$({ + assert!( + <$header>::FRAME_SEALED, + "{} is a replica-to-replica control header and must seal", + stringify!($header), + ); + let mut header = control_header::<$header>(); + header.seal(); + assert_eq!( + header.verify_frame(), + Ok(()), + "{} must accept its own seal", + stringify!($header), + ); + })+}; + } + seals!( + PrepareOkHeader, + CommitHeader, + StartViewChangeHeader, + DoViewChangeHeader, + StartViewHeader, + RequestStartViewHeader, + RequestPreparesHeader, + RepairRangeReplyHeader, + RequestStateTransferHeader, + StateTransferTargetHeader, + RequestStateChunkHeader, + StateChunkHeader, + ); + } + + #[test] + fn given_any_covered_byte_when_flipped_should_reject() { + // Why this seal exists. A `DoViewChange` nack bitset is a new primary's + // authority to truncate: two nacks on three replicas reach + // `quorum_nack_prepare` and discard a committed, client-acked op. The bitsets + // ride the header, and TCP's checksum will not reliably catch one bit. + // Every byte past `checksum` is covered, `checksum_body` included. + for offset in size_of::()..HEADER_SIZE { + let header = control_header::(); + // Skip offsets where the flipped bit is an invalid bit pattern, which + // `try_from_bytes` rejects one layer earlier. + if offset == std::mem::offset_of!(DoViewChangeHeader, command) { + continue; + } + assert!( + matches!( + tamper(header, offset), + Err(ConsensusError::FrameChecksumMismatch { .. }) + ), + "byte {offset} is inside the seal and must be covered" + ); + } + } + + #[test] + fn given_an_unsealed_control_header_when_verifying_should_reject() { + // No presence-keying: a zero checksum is a corrupt frame, not an old one. + // Keying on "does this look sealed" leaves the layer bypassable by zeroing + // the one field that decides whether anything is checked. + let header = control_header::(); + assert_eq!(header.checksum, 0); + assert!(matches!( + header.verify_frame(), + Err(ConsensusError::FrameChecksumMismatch { found: 0, .. }) + )); + } + + #[test] + fn given_an_identity_or_client_header_when_verifying_should_abstain() { + // `PrepareHeader` spends `checksum` on `identity_checksum`, which excludes + // `view` so a re-stamped prepare keeps one identity; the client-facing three + // are sealed on neither side yet. All must parse unchanged. + const { + assert!(!PrepareHeader::FRAME_SEALED); + assert!(!RepairPrepareHeader::FRAME_SEALED); + assert!(!RequestHeader::FRAME_SEALED); + assert!(!ReplyHeader::FRAME_SEALED); + assert!(!EvictionHeader::FRAME_SEALED); + assert!(!GenericHeader::FRAME_SEALED); + } + + let prepare = PrepareHeader { + command: Command2::Prepare, + checksum: 0xdead_beef, + ..Default::default() + }; + assert_eq!(prepare.verify_frame(), Ok(())); + } + #[test] fn all_headers_are_256_bytes() { assert_eq!(size_of::(), 256); @@ -1808,21 +2508,88 @@ mod tests { } // `status` is carved from the reserved tail; the SDK reply funnel peeks it - // at this offset before any body decode, so a layout drift must trip here. + // at this offset before any body decode, and four foreign SDKs hardcode + // it, so a layout drift must trip here. #[test] fn reply_header_status_offset_and_size_pinned() { use std::mem::offset_of; assert_eq!(size_of::(), HEADER_SIZE); + assert_eq!(offset_of!(ReplyHeader, status), 216); + assert_eq!( + offset_of!(ReplyHeader, reserved) + size_of::<[u8; 36]>(), + HEADER_SIZE + ); + } + + // `group` claims the TAIL of the client header's reserved area; the + // leading 52 reserved bytes carry data (the non-replicated op code lives + // in `reserved[0..4]`), so a reshuffle of the carve must trip here rather + // than silently move the code range. + #[test] + fn routed_request_group_claims_reserved_tail() { + use std::mem::offset_of; + assert_eq!( + offset_of!(RoutedRequestHeader, reserved), + offset_of!(RequestHeader, reserved) + ); assert_eq!( - offset_of!(ReplyHeader, status), - offset_of!(ReplyHeader, namespace) + size_of::() + offset_of!(RoutedRequestHeader, group), + offset_of!(RequestHeader, reserved) + 52 ); + assert_eq!(offset_of!(RoutedRequestHeader, group), 248); assert_eq!( - offset_of!(ReplyHeader, reserved) + size_of::<[u8; 28]>(), + offset_of!(RoutedRequestHeader, group) + size_of::(), HEADER_SIZE ); } + // The routed shape is decoded straight off the peer wire, so it must + // enforce the same field rules as the client boundary; a command-only + // validate lets a forged `client = 0` frame reach the client table's + // hard assert. + #[test] + fn routed_request_zero_client_rejected() { + let header = RoutedRequestHeader { + command: Command2::Request, + operation: Operation::SendMessages, + session: 10, + request: 1, + ..RoutedRequestHeader::default() + }; + assert!(header.validate().is_err()); + } + + #[test] + fn routed_request_reserved_operation_rejected() { + let header = RoutedRequestHeader { + command: Command2::Request, + client: 0xCAFE, + session: 10, + request: 1, + ..RoutedRequestHeader::default() + }; + assert!(header.validate().is_err()); + } + + #[test] + fn routed_request_default_fails_validate() { + assert!(RoutedRequestHeader::default().validate().is_err()); + } + + #[test] + fn routed_request_valid_fields_accepted() { + let header = RoutedRequestHeader { + command: Command2::Request, + operation: Operation::SendMessages, + client: 0xCAFE, + session: 10, + request: 1, + group: 7, + ..RoutedRequestHeader::default() + }; + assert!(header.validate().is_ok()); + } + // A nonzero status rides the reserved region, which reply `validate` does // not inspect: a status-bearing reply stays valid so the SDK can peek it. #[test] diff --git a/core/binary_protocol/src/consensus/mod.rs b/core/binary_protocol/src/consensus/mod.rs index 83bf5ae616..7738256ac3 100644 --- a/core/binary_protocol/src/consensus/mod.rs +++ b/core/binary_protocol/src/consensus/mod.rs @@ -45,11 +45,12 @@ mod reply_result; pub use command::Command2; pub use error::ConsensusError; pub use header::{ - CommitHeader, ConsensusHeader, DoViewChangeHeader, EvictionHeader, EvictionReason, - GenericHeader, HEADER_SIZE, PrepareHeader, PrepareOkHeader, RESERVED_COMMAND_LEN, - RepairPrepareHeader, RepairRangeReplyHeader, ReplyHeader, RequestHeader, RequestPreparesHeader, - RequestStartViewHeader, RequestStateChunkHeader, RequestStateTransferHeader, SIZE_FIELD_OFFSET, - StartViewChangeHeader, StartViewHeader, StateChunkHeader, StateTransferTargetHeader, + CHECKSUM_UNSEALED, CommitHeader, ConsensusHeader, DVC_HEADERS_MAX, DoViewChangeHeader, + EvictionHeader, EvictionReason, GenericHeader, HEADER_SIZE, PrepareHeader, PrepareOkHeader, + RESERVED_COMMAND_LEN, RepairPrepareHeader, RepairRangeReplyHeader, ReplyHeader, RequestHeader, + RequestPreparesHeader, RequestStartViewHeader, RequestStateChunkHeader, + RequestStateTransferHeader, RoutedRequestHeader, SIZE_FIELD_OFFSET, StartViewChangeHeader, + StartViewHeader, StateChunkHeader, StateTransferTargetHeader, frame_body, frame_checksum_bytes, read_size_field, }; pub use operation::Operation; diff --git a/core/binary_protocol/src/framing.rs b/core/binary_protocol/src/framing.rs index 6ead7d6e09..9603834d4b 100644 --- a/core/binary_protocol/src/framing.rs +++ b/core/binary_protocol/src/framing.rs @@ -184,7 +184,7 @@ impl<'a> ResponseFrame<'a> { } /// Decoded request frame with request ID for request-response correlation -/// and consensus-level duplicate detection (server-ng framing). +/// and consensus-level duplicate detection (the server framing). /// /// Wire format: `[length:4 LE][code:4 LE][request_id:8 LE][payload:N]` /// where `length` = 4 (code) + 8 (`request_id`) + N (payload). @@ -282,7 +282,7 @@ impl<'a> RequestFrame2<'a> { } /// Decoded response frame with request ID for request-response correlation -/// (server-ng framing). +/// (the server framing). /// /// Wire format: `[status:4 LE][length:4 LE][request_id:8 LE][payload:N]` /// where `status` = 0 for success, non-zero for error code. diff --git a/core/binary_protocol/src/lib.rs b/core/binary_protocol/src/lib.rs index 1f4baf3fb6..35109765fe 100644 --- a/core/binary_protocol/src/lib.rs +++ b/core/binary_protocol/src/lib.rs @@ -71,12 +71,14 @@ pub mod version; pub use codec::{WireDecode, WireEncode}; pub use consensus::{ - Command2, CommitHeader, ConsensusError, ConsensusHeader, DoViewChangeHeader, EvictionHeader, - EvictionReason, GenericHeader, HEADER_SIZE, Operation, PrepareHeader, PrepareOkHeader, - RESERVED_COMMAND_LEN, RepairPrepareHeader, RepairRangeReplyHeader, ReplyHeader, RequestHeader, - RequestPreparesHeader, RequestStartViewHeader, RequestStateChunkHeader, - RequestStateTransferHeader, SIZE_FIELD_OFFSET, StartViewChangeHeader, StartViewHeader, - StateChunkHeader, StateTransferTargetHeader, read_size_field, result_code, result_section_len, + CHECKSUM_UNSEALED, Command2, CommitHeader, ConsensusError, ConsensusHeader, DVC_HEADERS_MAX, + DoViewChangeHeader, EvictionHeader, EvictionReason, GenericHeader, HEADER_SIZE, Operation, + PrepareHeader, PrepareOkHeader, RESERVED_COMMAND_LEN, RepairPrepareHeader, + RepairRangeReplyHeader, ReplyHeader, RequestHeader, RequestPreparesHeader, + RequestStartViewHeader, RequestStateChunkHeader, RequestStateTransferHeader, + RoutedRequestHeader, SIZE_FIELD_OFFSET, StartViewChangeHeader, StartViewHeader, + StateChunkHeader, StateTransferTargetHeader, frame_body, frame_checksum_bytes, read_size_field, + result_code, result_section_len, }; pub use dispatch::{COMMAND_TABLE, CommandMeta, lookup_by_operation, lookup_command}; pub use error::WireError; diff --git a/core/binary_protocol/src/namespace.rs b/core/binary_protocol/src/namespace.rs index 454e81461e..88dbdfa93d 100644 --- a/core/binary_protocol/src/namespace.rs +++ b/core/binary_protocol/src/namespace.rs @@ -15,14 +15,16 @@ // specific language governing permissions and limitations // under the License. -//! Wire-format namespace routing constants. +//! Consensus group-id packing constants. //! -//! Both the SDK encoder (which writes `RequestHeader.namespace`) and the -//! server-side sharding layer (which hashes it to a shard) must agree on -//! how stream/topic/partition triples pack into the namespace `u64`. Any -//! drift between the two silently routes writes to the wrong shard, so the -//! single source of truth lives here in the wire-format crate that both -//! sides already depend on. +//! Clients send no group or namespace at all -- the server derives the +//! target from the operation and the request payload, stamps it into +//! `RoutedRequestHeader.group` at the dispatch boundary, and every internal +//! layer (sharding hash, consensus demux, repair replay) routes on that +//! stamped value. How stream/topic/partition triples pack into the `u64` +//! is therefore a server-side agreement between the resolver and the +//! sharding layer; the single source of truth lives here in the wire-format +//! crate both already depend on. pub const MAX_STREAMS: usize = 4096; pub const MAX_TOPICS: usize = 4096; @@ -61,20 +63,26 @@ pub const PACKED_NAMESPACE_BITS: u32 = STREAM_BITS + TOPIC_BITS + PARTITION_BITS /// Equivalent to `(1 << PACKED_NAMESPACE_BITS) - 1`. pub const PACKED_NAMESPACE_MAX: u64 = (1u64 << PACKED_NAMESPACE_BITS) - 1; -/// Reserved consensus-namespace identifier for the cluster's metadata replica. +/// Reserved consensus GROUP id for the cluster's metadata plane. /// -/// The packed layout uses only bits `0..PACKED_NAMESPACE_BITS`, so the top -/// bit is unreachable from any packed namespace value. Routers distinguish -/// metadata's single global consensus group from per-partition consensus -/// groups by value alone. -pub const METADATA_CONSENSUS_NAMESPACE: u64 = 1u64 << 63; +/// The group-id space is not a free namespace: values inside the packed +/// range are partition groups (the packed stream-topic-partition key), the +/// top bit is the control plane, and 0 is "unset", legal only on client +/// request headers. The packed layout uses only bits +/// `0..PACKED_NAMESPACE_BITS` (compile-asserted below), so the top bit is +/// unreachable from any packed value and routers distinguish metadata's +/// single global consensus group from per-partition groups by value alone. +/// Reserving the BOTTOM of the range instead (Redpanda's raft group 0) +/// only works for allocated ids; ours are derived, and packed 0 is the +/// legal partition `(0, 0, 0)`. +pub const METADATA_GROUP: u64 = 1u64 << 63; // Compile-time invariants. Bumping `MAX_STREAMS`/`MAX_TOPICS`/`MAX_PARTITIONS` // past the values here would silently collapse the sentinel-above-packed-range // guarantee and route writes to the wrong shard; the assertions guard against // that in every build (release included), not only under `cargo test`. const _: () = { - assert!(METADATA_CONSENSUS_NAMESPACE > PACKED_NAMESPACE_MAX); + assert!(METADATA_GROUP > PACKED_NAMESPACE_MAX); assert!(PACKED_NAMESPACE_BITS == STREAM_BITS + TOPIC_BITS + PARTITION_BITS); assert!(PACKED_NAMESPACE_MAX == (1u64 << PACKED_NAMESPACE_BITS) - 1); }; diff --git a/core/binary_protocol/src/requests/users/login_register.rs b/core/binary_protocol/src/requests/users/login_register.rs index b3c3fc3234..f52a2d76cb 100644 --- a/core/binary_protocol/src/requests/users/login_register.rs +++ b/core/binary_protocol/src/requests/users/login_register.rs @@ -22,7 +22,7 @@ use crate::version::ClientVersionInfo; use bytes::{BufMut, BytesMut}; use secrecy::{ExposeSecret, SecretString}; -/// Combined login + register request for server-ng. +/// Combined login + register request for the server. /// /// The server gates on `version_info.protocol_version` (see /// [`crate::version::is_protocol_compatible`]), verifies credentials @@ -49,8 +49,8 @@ use secrecy::{ExposeSecret, SecretString}; /// This wire shape is gated by the `vsr` cargo feature and lives under /// `LOGIN_REGISTER_CODE`. The legacy `LOGIN_USER_CODE` shape (still in use /// by non-`vsr` builds against the legacy `iggy-server`) is untouched. -/// server-ng speaks VSR framing only; a non-`vsr` SDK cannot log in to -/// server-ng. Foreign-language SDKs (C++, C#, Python, Go, Java) adopt this +/// The server speaks VSR framing only; a non-`vsr` SDK cannot log in to +/// the server. Foreign-language SDKs (C++, C#, Python, Go, Java) adopt this /// shape, with their own `sdk_name`, when they wire VSR framing. Bump /// [`crate::version::IGGY_PROTOCOL_VERSION`] on any wire-incompatible /// change. diff --git a/core/binary_protocol/src/requests/users/login_register_with_pat.rs b/core/binary_protocol/src/requests/users/login_register_with_pat.rs index 5e2ce8f860..544cd0de30 100644 --- a/core/binary_protocol/src/requests/users/login_register_with_pat.rs +++ b/core/binary_protocol/src/requests/users/login_register_with_pat.rs @@ -21,7 +21,7 @@ use crate::version::ClientVersionInfo; use bytes::{BufMut, BytesMut}; use secrecy::{ExposeSecret, SecretString}; -/// Combined login-with-PAT + register request for server-ng. +/// Combined login-with-PAT + register request for the server. /// /// Shares the `ClientVersionInfo` prefix with `LoginRegisterRequest` so the /// server gates on the protocol version once before attempting either body diff --git a/core/binary_protocol/src/responses/topics/create_topic.rs b/core/binary_protocol/src/responses/topics/create_topic.rs index 33ea71a750..397cc15013 100644 --- a/core/binary_protocol/src/responses/topics/create_topic.rs +++ b/core/binary_protocol/src/responses/topics/create_topic.rs @@ -19,6 +19,6 @@ /// /// Same `[TopicHeader][PartitionResponse]*` layout as `GetTopicResponse`, /// so the SDK reuses one decoder for both calls. Legacy server's -/// `create_topic_handler` builds this shape directly; server-ng's metadata +/// `create_topic_handler` builds this shape directly; the server's metadata /// STM emits the same bytes from `apply`. pub type CreateTopicResponse = super::GetTopicResponse; diff --git a/core/binary_protocol/src/responses/users/login_register.rs b/core/binary_protocol/src/responses/users/login_register.rs index dd9714a938..cd22d7a6b1 100644 --- a/core/binary_protocol/src/responses/users/login_register.rs +++ b/core/binary_protocol/src/responses/users/login_register.rs @@ -20,7 +20,7 @@ use crate::codec::{WireDecode, WireEncode, read_u32_le, read_u64_le}; use crate::primitives::identifier::WireName; use bytes::{BufMut, BytesMut}; -/// Combined login + register response for server-ng. +/// Combined login + register response for the server. /// /// Returns the authenticated user's ID, the consensus session number /// (commit op number from the Register operation), and the server's diff --git a/core/binary_protocol/src/version.rs b/core/binary_protocol/src/version.rs index 1eaba07c52..670c460b02 100644 --- a/core/binary_protocol/src/version.rs +++ b/core/binary_protocol/src/version.rs @@ -63,8 +63,8 @@ //! `ClientVersionInfo` is the leading bytes of the login-register request //! *body*, which itself rides inside a 256-byte VSR `RequestHeader` (see //! `consensus::header`): `command` = `Command2::Request`, `operation` = -//! `Operation::Register`, `namespace` = `METADATA_CONSENSUS_NAMESPACE`, -//! client id in `RequestHeader.client`. A foreign SDK emits that header, +//! `Operation::Register`, client id in `RequestHeader.client`. The client +//! sends no group; the server derives it. A foreign SDK emits that header, //! then the body starting with this prefix, to reach the gate. //! //! ## Login gate diff --git a/core/cli/Cargo.toml b/core/cli/Cargo.toml index b838b062c5..9bb24742ce 100644 --- a/core/cli/Cargo.toml +++ b/core/cli/Cargo.toml @@ -51,7 +51,6 @@ login-session = [ "dep:apple-native-keyring-store", "dep:windows-native-keyring-store", ] -vsr = ["iggy/vsr"] [dependencies] anyhow = { workspace = true } diff --git a/core/common/Cargo.toml b/core/common/Cargo.toml index e3a4b30d34..2e41e25251 100644 --- a/core/common/Cargo.toml +++ b/core/common/Cargo.toml @@ -29,9 +29,6 @@ documentation = "https://iggy.apache.org/docs" repository = "https://github.com/apache/iggy" readme = "README.md" -[features] -vsr = [] - [dependencies] aes-gcm = { workspace = true } async-broadcast = { workspace = true } diff --git a/core/common/src/error/eviction.rs b/core/common/src/error/eviction.rs index 8510274252..ef1fe88db9 100644 --- a/core/common/src/error/eviction.rs +++ b/core/common/src/error/eviction.rs @@ -17,7 +17,7 @@ //! Shared grading of a wire [`EvictionReason`] to the typed [`IggyError`] an //! evicted session surfaces. The SDK's binary-transport Eviction-frame decoder -//! (TCP / QUIC / WebSocket) and the server-ng HTTP write path both call this, +//! (TCP / QUIC / WebSocket) and the server HTTP write path both call this, //! so every transport sees one status per reason. use iggy_binary_protocol::consensus::EvictionReason; @@ -32,7 +32,7 @@ use super::iggy_error::IggyError; /// zero minimum or an inverted range), which also falls back to /// re-authentication. /// -/// The two callers extract the fields differently - server-ng from an aligned +/// The two callers extract the fields differently - the server from an aligned /// `EvictionHeader`, the SDK by wire offset off an unaligned buffer - then grade /// through here so the mappings cannot drift apart. #[must_use] diff --git a/core/common/src/lib.rs b/core/common/src/lib.rs index dde606a386..b3bfc1c8fc 100644 --- a/core/common/src/lib.rs +++ b/core/common/src/lib.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -#[cfg(feature = "vsr")] pub mod consumer_group_client_state; mod error; pub mod http; @@ -31,7 +30,6 @@ pub use error::iggy_error::{IggyError, IggyErrorDiscriminants}; // Locking is feature gated, thus only mod level re-export. pub mod locking; pub use chrono::{DateTime, Duration as ChronoDuration, Utc}; -#[cfg(feature = "vsr")] pub use consumer_group_client_state::ConsumerGroupClientState; /// Sentinel `partition_id` in an otherwise-empty poll reply that tells the @@ -57,7 +55,6 @@ pub use iggy_binary_protocol::responses::messages::{ }; pub use traits::binary_client::BinaryClient; pub use traits::binary_transport::BinaryTransport; -#[cfg(feature = "vsr")] pub use traits::binary_transport::{VsrSessionControl, VsrSessionSealed}; pub use traits::client::Client; pub use traits::cluster_client::ClusterClient; diff --git a/core/common/src/traits/binary_client.rs b/core/common/src/traits/binary_client.rs index 2d3b618fc2..a38004575d 100644 --- a/core/common/src/traits/binary_client.rs +++ b/core/common/src/traits/binary_client.rs @@ -18,15 +18,10 @@ use crate::{BinaryTransport, Client}; use async_trait::async_trait; -/// A client that can send and receive binary messages. In `vsr` builds it -/// also exposes the sealed [`VsrSessionControl`](crate::VsrSessionControl) -/// for the SDK's login/logout flows; that surface stays out of -/// [`BinaryTransport`] so external `&dyn BinaryTransport` consumers can't -/// touch consensus session state. -#[cfg(feature = "vsr")] +/// A client that can send and receive binary messages. It also exposes the +/// sealed [`VsrSessionControl`](crate::VsrSessionControl) for the SDK's +/// login/logout flows; that surface stays out of [`BinaryTransport`] so +/// external `&dyn BinaryTransport` consumers can't touch consensus session +/// state. #[async_trait] pub trait BinaryClient: BinaryTransport + Client + crate::VsrSessionControl {} - -#[cfg(not(feature = "vsr"))] -#[async_trait] -pub trait BinaryClient: BinaryTransport + Client {} diff --git a/core/common/src/traits/binary_impls/consumer_groups.rs b/core/common/src/traits/binary_impls/consumer_groups.rs index a47dbb97ad..ab4e95ebd5 100644 --- a/core/common/src/traits/binary_impls/consumer_groups.rs +++ b/core/common/src/traits/binary_impls/consumer_groups.rs @@ -159,7 +159,6 @@ impl ConsumerGroupClient for B { // Joining changes this member's assignment (and the group generation); // drop any cached assignment so the next group poll re-syncs. Key // format matches `binary_impls::messages::group_cache_key`. - #[cfg(feature = "vsr")] self.consumer_group_state() .invalidate_assignment(&format!("{stream_id}|{topic_id}|{group_id}")); Ok(()) @@ -185,7 +184,6 @@ impl ConsumerGroupClient for B { .to_bytes(), ) .await?; - #[cfg(feature = "vsr")] { let key = format!("{stream_id}|{topic_id}|{group_id}"); self.consumer_group_state().invalidate_assignment(&key); diff --git a/core/common/src/traits/binary_impls/messages.rs b/core/common/src/traits/binary_impls/messages.rs index 8ffe2fd485..ab1f260040 100644 --- a/core/common/src/traits/binary_impls/messages.rs +++ b/core/common/src/traits/binary_impls/messages.rs @@ -24,43 +24,34 @@ use crate::{ Consumer, Identifier, IggyError, IggyMessage, MessageClient, Partitioning, PolledMessages, PollingStrategy, SendMessagesResponse, }; -#[cfg(feature = "vsr")] use crate::{ConsumerKind, PartitioningKind, TopicClient, calculate_32}; use bytes::BytesMut; -#[cfg(feature = "vsr")] use iggy_binary_protocol::codec::WireDecode; use iggy_binary_protocol::codec::WireEncode; -#[cfg(feature = "vsr")] use iggy_binary_protocol::codes::SYNC_CONSUMER_GROUP_CODE; use iggy_binary_protocol::codes::{ FLUSH_UNSAVED_BUFFER_CODE, POLL_MESSAGES_CODE, SEND_MESSAGES_CODE, }; -#[cfg(feature = "vsr")] use iggy_binary_protocol::requests::consumer_groups::SyncConsumerGroupRequest; use iggy_binary_protocol::requests::messages::{ FlushUnsavedBufferRequest, PollMessagesRequest, RawMessage, SendMessagesEncoder, }; -#[cfg(feature = "vsr")] use iggy_binary_protocol::responses::consumer_groups::SyncConsumerGroupResponse; /// Max attempts to resolve a fenced consumer-group poll: one re-sync after the /// coordinator rejects a stale assignment, then retry once. -#[cfg(feature = "vsr")] const GROUP_POLL_MAX_ATTEMPTS: usize = 2; -#[cfg(feature = "vsr")] fn group_cache_key(stream_id: &Identifier, topic_id: &Identifier, group_id: &Identifier) -> String { format!("{stream_id}|{topic_id}|{group_id}") } -#[cfg(feature = "vsr")] fn topic_cache_key(stream_id: &Identifier, topic_id: &Identifier) -> String { format!("{stream_id}|{topic_id}") } /// Sync the requesting member's assignment from the coordinator into the /// transport cache. An empty reply means the client is not a member. -#[cfg(feature = "vsr")] async fn sync_group_assignment( client: &B, stream_id: &Identifier, @@ -106,7 +97,6 @@ async fn sync_group_assignment( /// driven so a member picks up a widened assignment (e.g. after a /// partition-count change) without first hitting an ownership fence. A failed /// per-group sync is logged and skipped so one bad group can't stall the rest. -#[cfg(feature = "vsr")] pub(crate) async fn refresh_group_assignments(client: &B) { for (stream_id, topic_id, group_id) in client.consumer_group_state().registered_groups() { if let Err(error) = sync_group_assignment(client, &stream_id, &topic_id, &group_id).await { @@ -119,7 +109,6 @@ pub(crate) async fn refresh_group_assignments(client: &B) { /// Resolve (and cache) the topic's partition count for client-side produce /// partitioning. -#[cfg(feature = "vsr")] async fn topic_partition_count( client: &B, stream_id: &Identifier, @@ -140,7 +129,6 @@ async fn topic_partition_count( /// Resolve `Balanced` / `MessagesKey` to an explicit `PartitionId` client-side /// (the VSR broker only routes explicit partitions, matching Kafka). -#[cfg(feature = "vsr")] async fn resolve_partitioning( client: &B, stream_id: &Identifier, @@ -180,7 +168,6 @@ async fn resolve_partitioning( /// Poll a consumer group: select one of the member's assigned partitions /// (round-robin) and send an explicit-partition poll. A coordinator fence /// rejection (stale assignment after a rebalance) triggers one re-sync + retry. -#[cfg(feature = "vsr")] async fn poll_group_messages( client: &B, stream_id: &Identifier, @@ -301,7 +288,6 @@ impl MessageClient for B { // VSR: a consumer-group poll without an explicit partition is resolved // client-side from the member's cached assignment (the broker routes // explicit partitions only). - #[cfg(feature = "vsr")] if consumer.kind == ConsumerKind::ConsumerGroup && partition_id.is_none() { return poll_group_messages( self, @@ -340,9 +326,7 @@ impl MessageClient for B { // VSR: resolve Balanced/MessagesKey to an explicit partition client-side. // An explicit `PartitionId` needs no resolution, so borrow the input // directly on that fast path instead of cloning its `value: Vec`. - #[cfg(feature = "vsr")] let resolved_partitioning; - #[cfg(feature = "vsr")] let partitioning = if partitioning.kind == PartitioningKind::PartitionId { partitioning } else { diff --git a/core/common/src/traits/binary_impls/mod.rs b/core/common/src/traits/binary_impls/mod.rs index 23e3c590ec..58239f5056 100644 --- a/core/common/src/traits/binary_impls/mod.rs +++ b/core/common/src/traits/binary_impls/mod.rs @@ -33,21 +33,17 @@ use crate::IggyError; use crate::http::users::defaults::{ MAX_PASSWORD_LENGTH, MAX_USERNAME_LENGTH, MIN_PASSWORD_LENGTH, MIN_USERNAME_LENGTH, }; -#[cfg(feature = "vsr")] use crate::{BinaryClient, ClientState}; use iggy_binary_protocol::WireDecode; -#[cfg(feature = "vsr")] use iggy_binary_protocol::{ClientVersionInfo, IGGY_PROTOCOL_VERSION, WireName}; /// SDK identifier sent in the login-register version prefix. Foreign SDKs /// send their own (e.g. `go-sdk`) once they adopt VSR framing. -#[cfg(feature = "vsr")] pub(crate) const RUST_SDK_NAME: &str = "rust-sdk"; /// Version prefix for both login-register request shapes. `sdk_version` /// comes from [`crate::VsrSessionControl::sdk_version`] so it is the SDK /// crate's version, not this crate's. -#[cfg(feature = "vsr")] pub(crate) fn rust_sdk_version_info(sdk_version: &str) -> Result { Ok(ClientVersionInfo { protocol_version: IGGY_PROTOCOL_VERSION, @@ -58,14 +54,13 @@ pub(crate) fn rust_sdk_version_info(sdk_version: &str) -> Result(client: &B) -> Result<(), IggyError> { if client.get_state().await == ClientState::Authenticated { client.logout_user().await?; diff --git a/core/common/src/traits/binary_impls/personal_access_tokens.rs b/core/common/src/traits/binary_impls/personal_access_tokens.rs index 202b4f41f9..7e1299f284 100644 --- a/core/common/src/traits/binary_impls/personal_access_tokens.rs +++ b/core/common/src/traits/binary_impls/personal_access_tokens.rs @@ -21,33 +21,22 @@ use crate::{ BinaryClient, ClientState, DiagnosticEvent, IdentityInfo, IggyError, PersonalAccessTokenClient, PersonalAccessTokenExpiry, PersonalAccessTokenInfo, RawPersonalAccessToken, }; -#[cfg(feature = "vsr")] use iggy_binary_protocol::MAX_WIRE_NAME_LENGTH; use iggy_binary_protocol::WireName; use iggy_binary_protocol::codec::WireEncode; -#[cfg(feature = "vsr")] use iggy_binary_protocol::codes::LOGIN_REGISTER_WITH_PAT_CODE; -#[cfg(not(feature = "vsr"))] -use iggy_binary_protocol::codes::LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE; use iggy_binary_protocol::codes::{ CREATE_PERSONAL_ACCESS_TOKEN_CODE, DELETE_PERSONAL_ACCESS_TOKEN_CODE, GET_PERSONAL_ACCESS_TOKENS_CODE, }; -#[cfg(not(feature = "vsr"))] -use iggy_binary_protocol::requests::personal_access_tokens::LoginWithPersonalAccessTokenRequest; use iggy_binary_protocol::requests::personal_access_tokens::{ CreatePersonalAccessTokenRequest, DeletePersonalAccessTokenRequest, GetPersonalAccessTokensRequest, }; -#[cfg(feature = "vsr")] use iggy_binary_protocol::requests::users::LoginRegisterWithPatRequest; use iggy_binary_protocol::responses::personal_access_tokens::create_personal_access_token::RawPersonalAccessTokenResponse; use iggy_binary_protocol::responses::personal_access_tokens::get_personal_access_tokens::GetPersonalAccessTokensResponse; -#[cfg(feature = "vsr")] use iggy_binary_protocol::responses::users::LoginRegisterResponse; -#[cfg(not(feature = "vsr"))] -use iggy_binary_protocol::responses::users::login_user::IdentityResponse; -#[cfg(feature = "vsr")] use secrecy::SecretString; #[async_trait::async_trait] @@ -103,73 +92,52 @@ impl PersonalAccessTokenClient for B { &self, token: &str, ) -> Result { - #[cfg(feature = "vsr")] + super::logout_before_relogin(self).await?; + // The request stores a `SecretString` rather than a `WireName`, so the + // `WireName` bounds are enforced here to keep the u8 length prefix + // consistent with the realized bytes. + if token.is_empty() || token.len() > MAX_WIRE_NAME_LENGTH { + return Err(IggyError::InvalidFormat); + } + let response = match self + .send_raw_with_response( + LOGIN_REGISTER_WITH_PAT_CODE, + LoginRegisterWithPatRequest { + version_info: super::rust_sdk_version_info(self.sdk_version())?, + token: SecretString::from(token.to_string()), + client_context: None, + } + .to_bytes(), + ) + .await { - super::logout_before_relogin(self).await?; - // Same bounds the non-vsr branch gets from `WireName::new(token)`; - // the request stores a `SecretString`, so enforce them here to keep - // the u8 length prefix consistent with the realized bytes. - if token.is_empty() || token.len() > MAX_WIRE_NAME_LENGTH { - return Err(IggyError::InvalidFormat); + Ok(response) => response, + Err(error) => { + self.reset_vsr_session().await?; + return Err(error); } - let response = match self - .send_raw_with_response( - LOGIN_REGISTER_WITH_PAT_CODE, - LoginRegisterWithPatRequest { - version_info: super::rust_sdk_version_info(self.sdk_version())?, - token: SecretString::from(token.to_string()), - client_context: None, - } - .to_bytes(), - ) - .await - { - Ok(response) => response, - Err(error) => { - self.reset_vsr_session().await?; - return Err(error); - } - }; - let wire_resp = match super::decode_response::(&response) { - Ok(wire_resp) => wire_resp, - Err(error) => { - self.reset_vsr_session().await?; - return Err(error); - } - }; - if let Err(error) = self.bind_vsr_session(wire_resp.session).await { + }; + let wire_resp = match super::decode_response::(&response) { + Ok(wire_resp) => wire_resp, + Err(error) => { self.reset_vsr_session().await?; return Err(error); } - tracing::debug!( - server_version = %wire_resp.server_version, - server_protocol_version = wire_resp.server_protocol_version, - "authenticated against iggy server" - ); - self.set_state(ClientState::Authenticated).await; - self.publish_event(DiagnosticEvent::SignedIn).await; - return Ok(IdentityInfo { - user_id: wire_resp.user_id, - access_token: None, - }); + }; + if let Err(error) = self.bind_vsr_session(wire_resp.session).await { + self.reset_vsr_session().await?; + return Err(error); } - - #[cfg(not(feature = "vsr"))] - let wire_token = WireName::new(token).map_err(|_| IggyError::InvalidFormat)?; - #[cfg(not(feature = "vsr"))] - let response = self - .send_raw_with_response( - LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE, - LoginWithPersonalAccessTokenRequest { token: wire_token }.to_bytes(), - ) - .await?; - #[cfg(not(feature = "vsr"))] + tracing::debug!( + server_version = %wire_resp.server_version, + server_protocol_version = wire_resp.server_protocol_version, + "authenticated against iggy server" + ); self.set_state(ClientState::Authenticated).await; - #[cfg(not(feature = "vsr"))] self.publish_event(DiagnosticEvent::SignedIn).await; - #[cfg(not(feature = "vsr"))] - let wire_resp = super::decode_response::(&response)?; - #[cfg(not(feature = "vsr"))] - Ok(IdentityInfo::from(wire_resp)) + Ok(IdentityInfo { + user_id: wire_resp.user_id, + access_token: None, + }) } } diff --git a/core/common/src/traits/binary_impls/system.rs b/core/common/src/traits/binary_impls/system.rs index f3868fc905..09f82a997d 100644 --- a/core/common/src/traits/binary_impls/system.rs +++ b/core/common/src/traits/binary_impls/system.rs @@ -87,7 +87,6 @@ impl SystemClient for B { self.get_heartbeat_interval() } - #[cfg(feature = "vsr")] async fn refresh_consumer_group_assignments(&self) { super::messages::refresh_group_assignments(self).await; } diff --git a/core/common/src/traits/binary_impls/users.rs b/core/common/src/traits/binary_impls/users.rs index 1a7f15b707..a375803668 100644 --- a/core/common/src/traits/binary_impls/users.rs +++ b/core/common/src/traits/binary_impls/users.rs @@ -23,28 +23,18 @@ use crate::{ }; use iggy_binary_protocol::WireName; use iggy_binary_protocol::codec::WireEncode; -#[cfg(feature = "vsr")] use iggy_binary_protocol::codes::LOGIN_REGISTER_CODE; -#[cfg(not(feature = "vsr"))] -use iggy_binary_protocol::codes::LOGIN_USER_CODE; use iggy_binary_protocol::codes::{ CHANGE_PASSWORD_CODE, CREATE_USER_CODE, DELETE_USER_CODE, GET_USER_CODE, GET_USERS_CODE, LOGOUT_USER_CODE, UPDATE_PERMISSIONS_CODE, UPDATE_USER_CODE, }; -#[cfg(feature = "vsr")] use iggy_binary_protocol::requests::users::LoginRegisterRequest; -#[cfg(not(feature = "vsr"))] -use iggy_binary_protocol::requests::users::LoginUserRequest; use iggy_binary_protocol::requests::users::{ ChangePasswordRequest, CreateUserRequest, DeleteUserRequest, GetUserRequest, GetUsersRequest, LogoutUserRequest, UpdatePermissionsRequest, UpdateUserRequest, }; -#[cfg(feature = "vsr")] use iggy_binary_protocol::responses::users::LoginRegisterResponse; -#[cfg(not(feature = "vsr"))] -use iggy_binary_protocol::responses::users::login_user::IdentityResponse; use iggy_binary_protocol::responses::users::{GetUsersResponse, UserDetailsResponse}; -#[cfg(feature = "vsr")] use secrecy::SecretString; #[async_trait::async_trait] @@ -187,83 +177,55 @@ impl UserClient for B { async fn login_user(&self, username: &str, password: &str) -> Result { super::validate_username(username)?; super::validate_password(password)?; - #[cfg(feature = "vsr")] - { - super::logout_before_relogin(self).await?; - let wire_name = WireName::new(username).map_err(|_| IggyError::InvalidFormat)?; - let response = match self - .send_raw_with_response( - LOGIN_REGISTER_CODE, - LoginRegisterRequest { - version_info: super::rust_sdk_version_info(self.sdk_version())?, - username: wire_name, - password: SecretString::from(password.to_string()), - client_context: None, - } - .to_bytes(), - ) - .await - { - Ok(response) => response, - Err(error) => { - self.reset_vsr_session().await?; - return Err(error); - } - }; - let wire_resp = match super::decode_response::(&response) { - Ok(wire_resp) => wire_resp, - Err(error) => { - self.reset_vsr_session().await?; - return Err(error); - } - }; - if let Err(error) = self.bind_vsr_session(wire_resp.session).await { - self.reset_vsr_session().await?; - return Err(error); - } - tracing::debug!( - server_version = %wire_resp.server_version, - server_protocol_version = wire_resp.server_protocol_version, - "authenticated against iggy server" - ); - self.set_state(ClientState::Authenticated).await; - self.publish_event(DiagnosticEvent::SignedIn).await; - return Ok(IdentityInfo { - user_id: wire_resp.user_id, - access_token: None, - }); - } - - #[cfg(not(feature = "vsr"))] + super::logout_before_relogin(self).await?; let wire_name = WireName::new(username).map_err(|_| IggyError::InvalidFormat)?; - #[cfg(not(feature = "vsr"))] - let response = self + let response = match self .send_raw_with_response( - LOGIN_USER_CODE, - LoginUserRequest { + LOGIN_REGISTER_CODE, + LoginRegisterRequest { + version_info: super::rust_sdk_version_info(self.sdk_version())?, username: wire_name, - password: password.to_string(), - version: Some(env!("CARGO_PKG_VERSION").to_string()), - context: Some(String::new()), + password: SecretString::from(password.to_string()), + client_context: None, } .to_bytes(), ) - .await?; - #[cfg(not(feature = "vsr"))] + .await + { + Ok(response) => response, + Err(error) => { + self.reset_vsr_session().await?; + return Err(error); + } + }; + let wire_resp = match super::decode_response::(&response) { + Ok(wire_resp) => wire_resp, + Err(error) => { + self.reset_vsr_session().await?; + return Err(error); + } + }; + if let Err(error) = self.bind_vsr_session(wire_resp.session).await { + self.reset_vsr_session().await?; + return Err(error); + } + tracing::debug!( + server_version = %wire_resp.server_version, + server_protocol_version = wire_resp.server_protocol_version, + "authenticated against iggy server" + ); self.set_state(ClientState::Authenticated).await; - #[cfg(not(feature = "vsr"))] self.publish_event(DiagnosticEvent::SignedIn).await; - #[cfg(not(feature = "vsr"))] - let wire_resp = super::decode_response::(&response)?; - #[cfg(not(feature = "vsr"))] - Ok(IdentityInfo::from(wire_resp)) + Ok(IdentityInfo { + user_id: wire_resp.user_id, + access_token: None, + }) } async fn logout_user(&self) -> Result<(), IggyError> { fail_if_not_authenticated(self).await?; self.send_raw_with_response(LOGOUT_USER_CODE, LogoutUserRequest.to_bytes()) .await?; - #[cfg(feature = "vsr")] self.reset_vsr_session().await?; self.set_state(ClientState::Connected).await; self.publish_event(DiagnosticEvent::SignedOut).await; diff --git a/core/common/src/traits/binary_transport.rs b/core/common/src/traits/binary_transport.rs index 3dacff9ad0..d3c2bd2e3a 100644 --- a/core/common/src/traits/binary_transport.rs +++ b/core/common/src/traits/binary_transport.rs @@ -18,7 +18,6 @@ use crate::{ClientState, DiagnosticEvent, IggyDuration, IggyError}; use async_trait::async_trait; use bytes::Bytes; -#[cfg(feature = "vsr")] use std::sync::Arc; #[async_trait] @@ -34,7 +33,6 @@ pub trait BinaryTransport { /// Per-transport consumer-group + partitioning cache used to resolve /// partitioning client-side under VSR (the broker never picks a /// partition). Shared via `Arc` so a refresh task can hold it. - #[cfg(feature = "vsr")] fn consumer_group_state(&self) -> Arc; } @@ -42,7 +40,6 @@ pub trait BinaryTransport { /// [`VsrSessionControl`] because they cannot name /// `vsr_session_sealed::Sealed`. The session-mutation methods stay /// in-crate so only the SDK's login/logout flows can call them. -#[cfg(feature = "vsr")] mod vsr_session_sealed { pub trait Sealed {} } @@ -50,7 +47,6 @@ mod vsr_session_sealed { /// VSR-internal session control. Distinct from [`BinaryTransport`] so /// `&dyn BinaryTransport` cannot reach `bind`/`reset` -- mid-session /// mutation corrupts the dedup counter or silently breaks at-most-once. -#[cfg(feature = "vsr")] #[async_trait] pub trait VsrSessionControl: vsr_session_sealed::Sealed + BinaryTransport { async fn bind_vsr_session(&self, session: u64) -> Result<(), IggyError>; @@ -61,5 +57,4 @@ pub trait VsrSessionControl: vsr_session_sealed::Sealed + BinaryTransport { fn sdk_version(&self) -> &'static str; } -#[cfg(feature = "vsr")] pub use vsr_session_sealed::Sealed as VsrSessionSealed; diff --git a/core/common/src/traits/message_client.rs b/core/common/src/traits/message_client.rs index 23e79c5eaf..0c5fa11e4b 100644 --- a/core/common/src/traits/message_client.rs +++ b/core/common/src/traits/message_client.rs @@ -28,7 +28,7 @@ pub trait MessageClient { /// /// Authentication is required, and the permission to poll the messages. /// - /// Under the `vsr` feature, polling a consumer group the client is not (or no longer) a member of fails with `ConsumerGroupMemberNotFound` rather than returning an empty batch, so the caller can rejoin. + /// Polling a consumer group the client is not (or no longer) a member of fails with `ConsumerGroupMemberNotFound` rather than returning an empty batch, so the caller can rejoin. #[allow(clippy::too_many_arguments)] async fn poll_messages( &self, diff --git a/core/common/src/utils/serde_secret.rs b/core/common/src/utils/serde_secret.rs index 7f8bd2feb3..b10492e438 100644 --- a/core/common/src/utils/serde_secret.rs +++ b/core/common/src/utils/serde_secret.rs @@ -17,22 +17,42 @@ //! Serde serialization helpers for `SecretString` fields. //! -//! `SecretString` intentionally does not implement `Serialize` to prevent -//! accidental secret exposure. These helpers are for fields that **must** be -//! serialized (e.g., wire protocol payloads, persisted TOML configs, API -//! responses that already expose credentials by design). +//! `SecretString` intentionally does not implement `Serialize`, and that +//! absence is the protection: a struct holding one cannot derive `Serialize` +//! at all. Adding `serialize_with` is therefore what *unblocks* the derive, so +//! reaching for a helper here is a decision to serialize a credential, never a +//! way to avoid it. +//! +//! [`serialize_secret`] and [`serialize_optional_secret`] write the plaintext. +//! Use them only where the plaintext is the point: wire protocol payloads, +//! persisted configs, API responses that expose credentials by design. //! -//! Usage: //! ```ignore //! #[serde(serialize_with = "crate::utils::serde_secret::serialize_secret")] //! pub password: SecretString, //! ``` //! -//! Do **not** add `serialize_with` to fields that should remain redacted in -//! serialized output — rely on `SecretString`'s default behavior instead. +//! [`serialize_redacted`] and [`serialize_optional_redacted`] write +//! [`REDACTED`] in place of the value, for a struct that must be serializable +//! for unrelated reasons but whose credential no reader is entitled to. +//! +//! **Redacted output is not a config.** Deserializing it hands back the literal +//! [`REDACTED`] as the secret, silently, so a redact-then-reload round trip +//! replaces the credential with the placeholder instead of failing. Nothing +//! in-tree can reach that today: these helpers have no consumers, and the one +//! persist/reload path round-trips a raw `serde_json::Value` rather than a +//! typed struct. If a consumer ever needs the round trip closed mechanically, +//! the shape that cannot be half-applied is a newtype owning both directions, +//! not a paired `deserialize_with` that a caller can forget to add. +//! +//! If neither applies, leave `serialize_with` off and let the missing impl keep +//! the field unserializable. use secrecy::{ExposeSecret, SecretString}; +/// Placeholder written in place of a redacted secret. +pub const REDACTED: &str = "[REDACTED]"; + pub fn serialize_secret( secret: &SecretString, serializer: S, @@ -50,6 +70,28 @@ pub fn serialize_optional_secret( } } +/// Writes [`REDACTED`] instead of the secret. +pub fn serialize_redacted( + _secret: &SecretString, + serializer: S, +) -> Result { + serializer.serialize_str(REDACTED) +} + +/// Writes [`REDACTED`] instead of the secret, keeping `None` distinguishable. +/// +/// Whether a credential is configured at all is not itself a secret, and +/// collapsing `Some` to `null` would tell a reader the field is unset. +pub fn serialize_optional_redacted( + secret: &Option, + serializer: S, +) -> Result { + match secret { + Some(_) => serializer.serialize_some(REDACTED), + None => serializer.serialize_none(), + } +} + #[cfg(test)] mod tests { use super::*; @@ -101,4 +143,43 @@ mod tests { let json = serde_json::to_string(&s).unwrap(); assert_eq!(json, r#"{"token":null}"#); } + + #[derive(Serialize)] + struct WithRedactedSecret { + #[serde(serialize_with = "serialize_redacted")] + password: SecretString, + } + + #[derive(Serialize)] + struct WithOptionalRedactedSecret { + #[serde(serialize_with = "serialize_optional_redacted")] + token: Option, + } + + #[test] + fn serialize_redacted_replaces_value_in_json() { + let s = WithRedactedSecret { + password: SecretString::from("my_password"), + }; + let json = serde_json::to_string(&s).unwrap(); + assert_eq!(json, r#"{"password":"[REDACTED]"}"#); + assert!(!json.contains("my_password")); + } + + #[test] + fn serialize_optional_redacted_keeps_some_distinguishable_from_none() { + let present = WithOptionalRedactedSecret { + token: Some(SecretString::from("tok_123")), + }; + let absent = WithOptionalRedactedSecret { token: None }; + + let present_json = serde_json::to_string(&present).unwrap(); + assert_eq!(present_json, r#"{"token":"[REDACTED]"}"#); + assert!(!present_json.contains("tok_123")); + assert_eq!( + serde_json::to_string(&absent).unwrap(), + r#"{"token":null}"#, + "a configured credential must not read as an unset one" + ); + } } diff --git a/core/configs/Cargo.toml b/core/configs/Cargo.toml index 858ea2b176..0b2f2079a1 100644 --- a/core/configs/Cargo.toml +++ b/core/configs/Cargo.toml @@ -37,4 +37,3 @@ server_common = { workspace = true } static-toml = { workspace = true } strum = { workspace = true } tracing = { workspace = true } -tungstenite = { workspace = true } diff --git a/core/configs/src/server_config/cache_indexes.rs b/core/configs/src/common/cache_indexes.rs similarity index 100% rename from core/configs/src/server_config/cache_indexes.rs rename to core/configs/src/common/cache_indexes.rs diff --git a/core/configs/src/common/defaults.rs b/core/configs/src/common/defaults.rs new file mode 100644 index 0000000000..2cc6d22007 --- /dev/null +++ b/core/configs/src/common/defaults.rs @@ -0,0 +1,431 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::http::{HttpConfig, HttpCorsConfig, HttpJwtConfig, HttpMetricsConfig, HttpTlsConfig}; +use super::server::{ + ConsumerGroupConfig, HeartbeatConfig, MemoryPoolConfig, MessageSaverConfig, + MessagesMaintenanceConfig, PersonalAccessTokenCleanerConfig, PersonalAccessTokenConfig, + TelemetryConfig, TelemetryLogsConfig, TelemetryTracesConfig, +}; +use super::system::{ + BackupConfig, CompatibilityConfig, CompressionConfig, EncryptionConfig, LoggingConfig, + MessageDeduplicationConfig, PartitionConfig, RecoveryConfig, RuntimeConfig, SegmentConfig, + StateConfig, StreamConfig, SystemConfig, TopicConfig, +}; +use configs::ConfigEnvMappings; + +static_toml::static_toml! { + // static_toml resolves relative to CARGO_MANIFEST_DIR (core/configs/). + pub static SERVER_CONFIG = include_toml!("../server/config.toml"); +} + +impl Default for MessagesMaintenanceConfig { + fn default() -> MessagesMaintenanceConfig { + MessagesMaintenanceConfig { + cleaner_enabled: SERVER_CONFIG.data_maintenance.messages.cleaner_enabled, + interval: SERVER_CONFIG + .data_maintenance + .messages + .interval + .parse() + .unwrap(), + } + } +} + +impl Default for HttpConfig { + fn default() -> HttpConfig { + HttpConfig { + enabled: SERVER_CONFIG.http.enabled, + address: SERVER_CONFIG.http.address.parse().unwrap(), + max_request_size: SERVER_CONFIG.http.max_request_size.parse().unwrap(), + web_ui: SERVER_CONFIG.http.web_ui, + cors: HttpCorsConfig::default(), + jwt: HttpJwtConfig::default(), + metrics: HttpMetricsConfig::default(), + tls: HttpTlsConfig::default(), + } + } +} + +impl Default for HttpCorsConfig { + fn default() -> HttpCorsConfig { + HttpCorsConfig { + enabled: SERVER_CONFIG.http.cors.enabled, + allowed_methods: SERVER_CONFIG + .http + .cors + .allowed_methods + .iter() + .map(|s| s.parse().unwrap()) + .collect(), + allowed_origins: SERVER_CONFIG + .http + .cors + .allowed_origins + .iter() + .map(|s| s.parse().unwrap()) + .collect(), + allowed_headers: SERVER_CONFIG + .http + .cors + .allowed_headers + .iter() + .map(|s| s.parse().unwrap()) + .collect(), + exposed_headers: SERVER_CONFIG + .http + .cors + .exposed_headers + .iter() + .map(|s| s.parse().unwrap()) + .collect(), + allow_credentials: SERVER_CONFIG.http.cors.allow_credentials, + allow_private_network: SERVER_CONFIG.http.cors.allow_private_network, + } + } +} + +impl Default for HttpJwtConfig { + fn default() -> HttpJwtConfig { + HttpJwtConfig { + algorithm: SERVER_CONFIG.http.jwt.algorithm.parse().unwrap(), + issuer: SERVER_CONFIG.http.jwt.issuer.parse().unwrap(), + audience: SERVER_CONFIG.http.jwt.audience.parse().unwrap(), + valid_issuers: SERVER_CONFIG + .http + .jwt + .valid_issuers + .iter() + .map(|s| s.parse().unwrap()) + .collect(), + valid_audiences: SERVER_CONFIG + .http + .jwt + .valid_audiences + .iter() + .map(|s| s.parse().unwrap()) + .collect(), + access_token_expiry: SERVER_CONFIG.http.jwt.access_token_expiry.parse().unwrap(), + clock_skew: SERVER_CONFIG.http.jwt.clock_skew.parse().unwrap(), + not_before: SERVER_CONFIG.http.jwt.not_before.parse().unwrap(), + encoding_secret: SERVER_CONFIG.http.jwt.encoding_secret.parse().unwrap(), + decoding_secret: SERVER_CONFIG.http.jwt.decoding_secret.parse().unwrap(), + use_base64_secret: SERVER_CONFIG.http.jwt.use_base_64_secret, + trusted_issuers: None, + } + } +} + +impl Default for HttpMetricsConfig { + fn default() -> HttpMetricsConfig { + HttpMetricsConfig { + enabled: SERVER_CONFIG.http.metrics.enabled, + endpoint: SERVER_CONFIG.http.metrics.endpoint.parse().unwrap(), + } + } +} + +impl Default for HttpTlsConfig { + fn default() -> HttpTlsConfig { + HttpTlsConfig { + enabled: SERVER_CONFIG.http.tls.enabled, + cert_file: SERVER_CONFIG.http.tls.cert_file.parse().unwrap(), + key_file: SERVER_CONFIG.http.tls.key_file.parse().unwrap(), + } + } +} + +impl Default for MessageSaverConfig { + fn default() -> MessageSaverConfig { + MessageSaverConfig { + enabled: SERVER_CONFIG.message_saver.enabled, + enforce_fsync: SERVER_CONFIG.message_saver.enforce_fsync, + interval: SERVER_CONFIG.message_saver.interval.parse().unwrap(), + } + } +} + +impl Default for PersonalAccessTokenConfig { + fn default() -> PersonalAccessTokenConfig { + PersonalAccessTokenConfig { + max_tokens_per_user: SERVER_CONFIG.personal_access_token.max_tokens_per_user as u32, + cleaner: PersonalAccessTokenCleanerConfig::default(), + } + } +} + +impl Default for PersonalAccessTokenCleanerConfig { + fn default() -> PersonalAccessTokenCleanerConfig { + PersonalAccessTokenCleanerConfig { + enabled: SERVER_CONFIG.personal_access_token.cleaner.enabled, + interval: SERVER_CONFIG + .personal_access_token + .cleaner + .interval + .parse() + .unwrap(), + } + } +} + +impl Default for SystemConfig { + fn default() -> Self { + Self { + path: SERVER_CONFIG.system.path.parse().unwrap(), + backup: BackupConfig::default(), + runtime: RuntimeConfig::default(), + logging: LoggingConfig::default(), + stream: StreamConfig::default(), + encryption: EncryptionConfig::default(), + topic: TopicConfig::default(), + partition: PartitionConfig::default(), + segment: SegmentConfig::default(), + state: StateConfig::default(), + compression: CompressionConfig::default(), + message_deduplication: MessageDeduplicationConfig::default(), + recovery: RecoveryConfig::default(), + memory_pool: MemoryPoolConfig::default(), + sharding: S::default(), + } + } +} + +impl Default for BackupConfig { + fn default() -> BackupConfig { + BackupConfig { + path: SERVER_CONFIG.system.backup.path.parse().unwrap(), + compatibility: CompatibilityConfig::default(), + } + } +} + +impl Default for CompatibilityConfig { + fn default() -> Self { + CompatibilityConfig { + path: SERVER_CONFIG + .system + .backup + .compatibility + .path + .parse() + .unwrap(), + } + } +} + +impl Default for HeartbeatConfig { + fn default() -> HeartbeatConfig { + HeartbeatConfig { + enabled: SERVER_CONFIG.heartbeat.enabled, + interval: SERVER_CONFIG.heartbeat.interval.parse().unwrap(), + } + } +} + +impl Default for ConsumerGroupConfig { + fn default() -> ConsumerGroupConfig { + ConsumerGroupConfig { + rebalancing_timeout: SERVER_CONFIG + .consumer_group + .rebalancing_timeout + .parse() + .unwrap(), + rebalancing_check_interval: SERVER_CONFIG + .consumer_group + .rebalancing_check_interval + .parse() + .unwrap(), + } + } +} + +impl Default for RuntimeConfig { + fn default() -> RuntimeConfig { + RuntimeConfig { + path: SERVER_CONFIG.system.runtime.path.parse().unwrap(), + } + } +} + +impl Default for CompressionConfig { + fn default() -> Self { + CompressionConfig { + allow_override: SERVER_CONFIG.system.compression.allow_override, + default_algorithm: SERVER_CONFIG + .system + .compression + .default_algorithm + .parse() + .unwrap(), + } + } +} + +impl Default for LoggingConfig { + fn default() -> LoggingConfig { + LoggingConfig { + path: SERVER_CONFIG.system.logging.path.parse().unwrap(), + level: SERVER_CONFIG.system.logging.level.parse().unwrap(), + file_enabled: SERVER_CONFIG.system.logging.file_enabled, + max_file_size: SERVER_CONFIG.system.logging.max_file_size.parse().unwrap(), + max_total_size: SERVER_CONFIG.system.logging.max_total_size.parse().unwrap(), + rotation_check_interval: SERVER_CONFIG + .system + .logging + .rotation_check_interval + .parse() + .unwrap(), + retention: SERVER_CONFIG.system.logging.retention.parse().unwrap(), + sysinfo_print_interval: SERVER_CONFIG + .system + .logging + .sysinfo_print_interval + .parse() + .unwrap(), + } + } +} + +impl Default for EncryptionConfig { + fn default() -> EncryptionConfig { + EncryptionConfig { + enabled: SERVER_CONFIG.system.encryption.enabled, + key: SERVER_CONFIG.system.encryption.key.parse().unwrap(), + } + } +} + +impl Default for StreamConfig { + fn default() -> StreamConfig { + StreamConfig { + path: SERVER_CONFIG.system.stream.path.parse().unwrap(), + } + } +} + +impl Default for TopicConfig { + fn default() -> TopicConfig { + TopicConfig { + path: SERVER_CONFIG.system.topic.path.parse().unwrap(), + max_size: SERVER_CONFIG.system.topic.max_size.parse().unwrap(), + message_expiry: SERVER_CONFIG.system.topic.message_expiry.parse().unwrap(), + } + } +} + +impl Default for PartitionConfig { + fn default() -> PartitionConfig { + PartitionConfig { + path: SERVER_CONFIG.system.partition.path.parse().unwrap(), + size_of_messages_required_to_save: SERVER_CONFIG + .system + .partition + .size_of_messages_required_to_save + .parse() + .unwrap(), + messages_required_to_save: SERVER_CONFIG.system.partition.messages_required_to_save + as u32, + enforce_fsync: SERVER_CONFIG.system.partition.enforce_fsync, + validate_checksum: SERVER_CONFIG.system.partition.validate_checksum, + } + } +} + +impl Default for SegmentConfig { + fn default() -> SegmentConfig { + SegmentConfig { + size: SERVER_CONFIG.system.segment.size.parse().unwrap(), + preallocate: SERVER_CONFIG.system.segment.preallocate, + cache_indexes: SERVER_CONFIG.system.segment.cache_indexes.parse().unwrap(), + archive_expired: SERVER_CONFIG.system.segment.archive_expired, + } + } +} + +impl Default for StateConfig { + fn default() -> StateConfig { + StateConfig { + enforce_fsync: SERVER_CONFIG.system.state.enforce_fsync, + max_file_operation_retries: SERVER_CONFIG.system.state.max_file_operation_retries + as u32, + retry_delay: SERVER_CONFIG.system.state.retry_delay.parse().unwrap(), + } + } +} + +impl Default for MessageDeduplicationConfig { + fn default() -> MessageDeduplicationConfig { + MessageDeduplicationConfig { + enabled: SERVER_CONFIG.system.message_deduplication.enabled, + max_entries: SERVER_CONFIG.system.message_deduplication.max_entries as u64, + expiry: SERVER_CONFIG + .system + .message_deduplication + .expiry + .parse() + .unwrap(), + } + } +} + +impl Default for RecoveryConfig { + fn default() -> RecoveryConfig { + RecoveryConfig { + recreate_missing_state: SERVER_CONFIG.system.recovery.recreate_missing_state, + } + } +} + +impl Default for MemoryPoolConfig { + fn default() -> MemoryPoolConfig { + Self { + enabled: SERVER_CONFIG.system.memory_pool.enabled, + size: SERVER_CONFIG.system.memory_pool.size.parse().unwrap(), + bucket_capacity: SERVER_CONFIG.system.memory_pool.bucket_capacity as u32, + } + } +} + +impl Default for TelemetryConfig { + fn default() -> TelemetryConfig { + TelemetryConfig { + enabled: SERVER_CONFIG.telemetry.enabled, + service_name: SERVER_CONFIG.telemetry.service_name.parse().unwrap(), + logs: TelemetryLogsConfig::default(), + traces: TelemetryTracesConfig::default(), + } + } +} + +impl Default for TelemetryLogsConfig { + fn default() -> TelemetryLogsConfig { + TelemetryLogsConfig { + transport: SERVER_CONFIG.telemetry.logs.transport.parse().unwrap(), + endpoint: SERVER_CONFIG.telemetry.logs.endpoint.parse().unwrap(), + } + } +} + +impl Default for TelemetryTracesConfig { + fn default() -> TelemetryTracesConfig { + TelemetryTracesConfig { + transport: SERVER_CONFIG.telemetry.traces.transport.parse().unwrap(), + endpoint: SERVER_CONFIG.telemetry.traces.endpoint.parse().unwrap(), + } + } +} diff --git a/core/configs/src/common/displays.rs b/core/configs/src/common/displays.rs new file mode 100644 index 0000000000..3d0b432017 --- /dev/null +++ b/core/configs/src/common/displays.rs @@ -0,0 +1,280 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::server::{ + ConsumerGroupConfig, DataMaintenanceConfig, HeartbeatConfig, MessagesMaintenanceConfig, + TelemetryConfig, TelemetryLogsConfig, TelemetryTracesConfig, +}; +use super::system::MessageDeduplicationConfig; +use super::{ + http::{HttpConfig, HttpCorsConfig, HttpJwtConfig, HttpMetricsConfig, HttpTlsConfig}, + server::MessageSaverConfig, + system::{ + CompressionConfig, EncryptionConfig, LoggingConfig, PartitionConfig, SegmentConfig, + StateConfig, StreamConfig, SystemConfig, TopicConfig, + }, +}; +use configs::ConfigEnvMappings; +use std::fmt::{Display, Formatter}; + +impl Display for HttpConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ enabled: {}, address: {}, max_request_size: {}, web_ui: {}, cors: {}, jwt: {}, metrics: {}, tls: {} }}", + self.enabled, + self.address, + self.max_request_size, + self.web_ui, + self.cors, + self.jwt, + self.metrics, + self.tls + ) + } +} + +impl Display for HttpCorsConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ enabled: {}, allowed_methods: {:?}, allowed_origins: {:?}, allowed_headers: {:?}, exposed_headers: {:?}, allow_credentials: {}, allow_private_network: {} }}", + self.enabled, + self.allowed_methods, + self.allowed_origins, + self.allowed_headers, + self.exposed_headers, + self.allow_credentials, + self.allow_private_network + ) + } +} + +impl Display for HttpJwtConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ algorithm: {}, audience: {}, access_token_expiry: {}, use_base64_secret: {} }}", + self.algorithm, self.audience, self.access_token_expiry, self.use_base64_secret + ) + } +} + +impl Display for HttpMetricsConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ enabled: {}, endpoint: {} }}", + self.enabled, self.endpoint + ) + } +} + +impl Display for HttpTlsConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ enabled: {}, cert_file: {}, key_file: {} }}", + self.enabled, self.cert_file, self.key_file + ) + } +} + +impl Display for CompressionConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ allowed_override: {}, default_algorithm: {} }}", + self.allow_override, self.default_algorithm + ) + } +} + +impl Display for DataMaintenanceConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{{ messages: {} }}", self.messages) + } +} + +impl Display for MessagesMaintenanceConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ cleaner_enabled: {}, interval: {} }}", + self.cleaner_enabled, self.interval + ) + } +} + +impl Display for ConsumerGroupConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ rebalancing_timeout: {}, rebalancing_check_interval: {} }}", + self.rebalancing_timeout, self.rebalancing_check_interval + ) + } +} + +impl Display for MessageSaverConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ enabled: {}, enforce_fsync: {}, interval: {} }}", + self.enabled, self.enforce_fsync, self.interval + ) + } +} + +impl Display for HeartbeatConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ enabled: {}, interval: {} }}", + self.enabled, self.interval + ) + } +} + +impl Display for EncryptionConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{{ enabled: {} }}", self.enabled) + } +} + +impl Display for StreamConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{{ path: {} }}", self.path) + } +} + +impl Display for TopicConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ path: {}, max_size: {}, message_expiry: {} }}", + self.path, self.max_size, self.message_expiry + ) + } +} + +impl Display for PartitionConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ path: {}, messages_required_to_save: {}, size_of_messages_required_to_save: {}, enforce_fsync: {}, validate_checksum: {} }}", + self.path, + self.messages_required_to_save, + self.size_of_messages_required_to_save, + self.enforce_fsync, + self.validate_checksum + ) + } +} + +impl Display for MessageDeduplicationConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ enabled: {}, max_entries: {:?}, expiry: {:?} }}", + self.enabled, self.max_entries, self.expiry + ) + } +} + +impl Display for SegmentConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ size_bytes: {}, preallocate: {}, cache_indexes: {}, archive_expired: {} }}", + self.size, self.preallocate, self.cache_indexes, self.archive_expired, + ) + } +} + +impl Display for LoggingConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ path: {}, level: {}, file_enabled: {}, max_file_size: {}, max_total_size: {}, rotation_check_interval: {}, retention: {} }}", + self.path, + self.level, + self.file_enabled, + self.max_file_size.as_human_string_with_zero_as_unlimited(), + self.max_total_size.as_human_string_with_zero_as_unlimited(), + self.rotation_check_interval, + self.retention + ) + } +} + +impl Display for TelemetryConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ enabled: {}, service_name: {}, logs: {}, traces: {} }}", + self.enabled, self.service_name, self.logs, self.traces + ) + } +} + +impl Display for TelemetryLogsConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ transport: {}, endpoint: {} }}", + self.transport, self.endpoint + ) + } +} + +impl Display for StateConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ enforce_fsync: {}, max_file_operation_retries: {}, retry_delay: {} }}", + self.enforce_fsync, self.max_file_operation_retries, self.retry_delay, + ) + } +} + +impl Display for TelemetryTracesConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ transport: {}, endpoint: {} }}", + self.transport, self.endpoint + ) + } +} + +impl Display for SystemConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{{ path: {}, logging: {}, stream: {}, topic: {}, partition: {}, segment: {}, encryption: {}, state: {} }}", + self.path, + self.logging, + self.stream, + self.topic, + self.partition, + self.segment, + self.encryption, + self.state, + ) + } +} diff --git a/core/configs/src/server_config/http.rs b/core/configs/src/common/http.rs similarity index 100% rename from core/configs/src/server_config/http.rs rename to core/configs/src/common/http.rs diff --git a/core/server/src/state/mod.rs b/core/configs/src/common/mod.rs similarity index 70% rename from core/server/src/state/mod.rs rename to core/configs/src/common/mod.rs index bb34bd1082..b14356b0a9 100644 --- a/core/server/src/state/mod.rs +++ b/core/configs/src/common/mod.rs @@ -15,13 +15,16 @@ // specific language governing permissions and limitations // under the License. -pub mod command; -pub mod entry; -pub mod file; -pub mod models; -pub mod system; +//! Config vocabulary shared across the crate: the generic +//! [`system::SystemConfig`], the HTTP section, and the top-level sections +//! that [`crate::server_config::server::ServerConfig`] composes. -pub const COMPONENT: &str = "STATE"; +pub mod cache_indexes; +pub mod defaults; +pub mod displays; +pub mod http; +pub mod server; +pub mod system; +pub mod validators; -pub use command::EntryCommand; -pub use entry::StateEntry; +pub const COMPONENT: &str = "CONFIG"; diff --git a/core/configs/src/common/server.rs b/core/configs/src/common/server.rs new file mode 100644 index 0000000000..214ed085d1 --- /dev/null +++ b/core/configs/src/common/server.rs @@ -0,0 +1,144 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use configs::ConfigEnv; +use iggy_common::{IggyByteSize, IggyDuration}; +use serde::{Deserialize, Serialize}; +use serde_with::DisplayFromStr; +use serde_with::serde_as; +use server_common::MemoryPoolConfigOther; +use server_common::log::{TelemetryEndpointSettings, TelemetrySettings}; + +pub use server_common::log::TelemetryTransport; + +/// Configuration for the memory pool. +#[derive(Debug, Deserialize, Serialize, ConfigEnv)] +pub struct MemoryPoolConfig { + pub enabled: bool, + #[config_env(leaf)] + pub size: IggyByteSize, + pub bucket_capacity: u32, +} + +impl MemoryPoolConfig { + pub fn into_other(&self) -> MemoryPoolConfigOther { + MemoryPoolConfigOther { + enabled: self.enabled, + size: self.size, + bucket_capacity: self.bucket_capacity, + } + } +} + +#[serde_as] +#[derive(Debug, Default, Deserialize, Serialize, Clone, ConfigEnv)] +pub struct DataMaintenanceConfig { + pub messages: MessagesMaintenanceConfig, +} + +#[serde_as] +#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] +pub struct MessagesMaintenanceConfig { + pub cleaner_enabled: bool, + #[config_env(leaf)] + #[serde_as(as = "DisplayFromStr")] + pub interval: IggyDuration, +} + +#[serde_as] +#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] +pub struct MessageSaverConfig { + pub enabled: bool, + pub enforce_fsync: bool, + #[config_env(leaf)] + #[serde_as(as = "DisplayFromStr")] + pub interval: IggyDuration, +} + +#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] +pub struct PersonalAccessTokenConfig { + pub max_tokens_per_user: u32, + pub cleaner: PersonalAccessTokenCleanerConfig, +} + +#[serde_as] +#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] +pub struct PersonalAccessTokenCleanerConfig { + pub enabled: bool, + #[config_env(leaf)] + #[serde_as(as = "DisplayFromStr")] + pub interval: IggyDuration, +} + +#[serde_as] +#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] +pub struct HeartbeatConfig { + pub enabled: bool, + #[config_env(leaf)] + #[serde_as(as = "DisplayFromStr")] + pub interval: IggyDuration, +} + +#[serde_as] +#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] +pub struct ConsumerGroupConfig { + #[config_env(leaf)] + #[serde_as(as = "DisplayFromStr")] + pub rebalancing_timeout: IggyDuration, + #[config_env(leaf)] + #[serde_as(as = "DisplayFromStr")] + pub rebalancing_check_interval: IggyDuration, +} + +#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] +pub struct TelemetryConfig { + pub enabled: bool, + pub service_name: String, + pub logs: TelemetryLogsConfig, + pub traces: TelemetryTracesConfig, +} + +#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] +pub struct TelemetryLogsConfig { + #[config_env(leaf)] + pub transport: TelemetryTransport, + pub endpoint: String, +} + +#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] +pub struct TelemetryTracesConfig { + #[config_env(leaf)] + pub transport: TelemetryTransport, + pub endpoint: String, +} + +impl From<&TelemetryConfig> for TelemetrySettings { + fn from(config: &TelemetryConfig) -> Self { + Self { + enabled: config.enabled, + service_name: config.service_name.clone(), + logs: TelemetryEndpointSettings { + transport: config.logs.transport, + endpoint: config.logs.endpoint.clone(), + }, + traces: TelemetryEndpointSettings { + transport: config.traces.transport, + endpoint: config.traces.endpoint.clone(), + }, + } + } +} diff --git a/core/configs/src/server_config/system.rs b/core/configs/src/common/system.rs similarity index 96% rename from core/configs/src/server_config/system.rs rename to core/configs/src/common/system.rs index 60437a94fb..d72913fc5c 100644 --- a/core/configs/src/server_config/system.rs +++ b/core/configs/src/common/system.rs @@ -17,7 +17,6 @@ use super::cache_indexes::CacheIndexesConfig; use super::server::MemoryPoolConfig; -use super::sharding::ShardingConfig; use configs::{ConfigEnv, ConfigEnvMappings}; use iggy_common::IggyByteSize; use iggy_common::IggyError; @@ -33,13 +32,11 @@ use server_common::log::LoggingSettings; pub const INDEX_EXTENSION: &str = "index"; pub const LOG_EXTENSION: &str = "log"; -// Generic over the sharding config so the legacy server and `server-ng` each -// bind their own `ShardingConfig` (different knob sets, different default -// source) while sharing this whole struct and its path helpers. The default -// type param keeps bare `SystemConfig` meaning the legacy variant, so existing -// callers compile unchanged. +// Generic over the sharding config so every server flavour binds its own +// `ShardingConfig` (different knob sets, different default source) while +// sharing this whole struct and its path helpers. #[derive(Debug, Deserialize, Serialize, ConfigEnv)] -pub struct SystemConfig { +pub struct SystemConfig { pub path: String, pub backup: BackupConfig, pub state: StateConfig, @@ -178,6 +175,8 @@ pub struct RecoveryConfig { pub struct SegmentConfig { #[config_env(leaf)] pub size: IggyByteSize, + #[serde(default)] + pub preallocate: bool, #[config_env(leaf)] pub cache_indexes: CacheIndexesConfig, pub archive_expired: bool, diff --git a/core/configs/src/common/validators.rs b/core/configs/src/common/validators.rs new file mode 100644 index 0000000000..9c7d54c43c --- /dev/null +++ b/core/configs/src/common/validators.rs @@ -0,0 +1,357 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use super::COMPONENT; +use super::server::{ + DataMaintenanceConfig, MessageSaverConfig, MessagesMaintenanceConfig, TelemetryConfig, +}; +use super::server::{MemoryPoolConfig, PersonalAccessTokenConfig}; +use super::system::SegmentConfig; +use super::system::{CompressionConfig, LoggingConfig, PartitionConfig}; +use crate::ConfigurationError; +use cpu_allocation::{CpuAllocation, allowed_cpus}; +use err_trail::ErrContext; +use iggy_common::CompressionAlgorithm; +use iggy_common::Validatable; +use std::thread::available_parallelism; +use tracing::warn; + +/// 1 GiB max segment size. +pub const SEGMENT_MAX_SIZE_BYTES: u64 = 1024 * 1024 * 1024; + +impl Validatable for CompressionConfig { + fn validate(&self) -> Result<(), ConfigurationError> { + let compression_alg = &self.default_algorithm; + if *compression_alg != CompressionAlgorithm::None { + // TODO(numinex): Change this message once server side compression is fully developed. + warn!( + "Server started with server-side compression enabled, using algorithm: {compression_alg}, this feature is not implemented yet!" + ); + } + + Ok(()) + } +} + +impl Validatable for TelemetryConfig { + fn validate(&self) -> Result<(), ConfigurationError> { + if !self.enabled { + return Ok(()); + } + + if self.service_name.trim().is_empty() { + eprintln!("telemetry.service_name cannot be empty when telemetry is enabled"); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + if self.logs.endpoint.is_empty() { + eprintln!("telemetry.logs.endpoint cannot be empty when telemetry is enabled"); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + if self.traces.endpoint.is_empty() { + eprintln!("telemetry.traces.endpoint cannot be empty when telemetry is enabled"); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + Ok(()) + } +} + +impl Validatable for PartitionConfig { + fn validate(&self) -> Result<(), ConfigurationError> { + if self.messages_required_to_save == 0 { + eprintln!("Configured system.partition.messages_required_to_save cannot be 0"); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + Ok(()) + } +} + +impl Validatable for SegmentConfig { + fn validate(&self) -> Result<(), ConfigurationError> { + if self.size > SEGMENT_MAX_SIZE_BYTES { + eprintln!( + "Configured system.segment.size {} B is greater than maximum {} B", + self.size.as_bytes_u64(), + SEGMENT_MAX_SIZE_BYTES + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + if !self.size.as_bytes_u64().is_multiple_of(512) { + eprintln!( + "Configured system.segment.size {} B is not a multiple of 512 B", + self.size.as_bytes_u64() + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + Ok(()) + } +} + +impl Validatable for MessageSaverConfig { + fn validate(&self) -> Result<(), ConfigurationError> { + if self.enabled && self.interval.is_zero() { + eprintln!("message_saver.interval cannot be zero when message_saver is enabled"); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + Ok(()) + } +} + +impl Validatable for DataMaintenanceConfig { + fn validate(&self) -> Result<(), ConfigurationError> { + self.messages.validate().error(|e: &ConfigurationError| { + format!("{COMPONENT} (error: {e}) - failed to validate messages maintenance config") + })?; + Ok(()) + } +} + +impl Validatable for MessagesMaintenanceConfig { + fn validate(&self) -> Result<(), ConfigurationError> { + if self.cleaner_enabled && self.interval.is_zero() { + eprintln!("data_maintenance.messages.interval cannot be zero when cleaner is enabled"); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + Ok(()) + } +} + +impl Validatable for PersonalAccessTokenConfig { + fn validate(&self) -> Result<(), ConfigurationError> { + if self.max_tokens_per_user == 0 { + eprintln!("personal_access_token.max_tokens_per_user cannot be 0"); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + if self.cleaner.enabled && self.cleaner.interval.is_zero() { + eprintln!( + "personal_access_token.cleaner.interval cannot be zero when cleaner is enabled" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + Ok(()) + } +} + +impl Validatable for LoggingConfig { + fn validate(&self) -> Result<(), ConfigurationError> { + if self.level.is_empty() { + eprintln!("system.logging.level is supposed be configured"); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + if self.retention.as_secs() < 1 { + eprintln!( + "Configured system.logging.retention {} is less than minimum 1 second", + self.retention + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + if self.rotation_check_interval.as_secs() < 1 { + eprintln!( + "Configured system.logging.rotation_check_interval {} is less than minimum 1 second", + self.rotation_check_interval + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + let max_total_size_unlimited = self.max_total_size.as_bytes_u64() == 0; + if !max_total_size_unlimited + && self.max_file_size.as_bytes_u64() > self.max_total_size.as_bytes_u64() + { + eprintln!( + "Configured system.logging.max_total_size {} is less than system.logging.max_file_size {}", + self.max_total_size, self.max_file_size + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + Ok(()) + } +} + +impl Validatable for MemoryPoolConfig { + fn validate(&self) -> Result<(), ConfigurationError> { + if self.enabled && self.size == 0 { + eprintln!( + "Configured system.memory_pool.enabled is true and system.memory_pool.size is 0" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + const MIN_POOL_SIZE: u64 = 512 * 1024 * 1024; // 512 MiB + const MIN_BUCKET_CAPACITY: u32 = 128; + const DEFAULT_PAGE_SIZE: u64 = 4096; + + if self.enabled && self.size < MIN_POOL_SIZE { + eprintln!( + "Configured system.memory_pool.size {} B ({} MiB) is less than minimum {} B, ({} MiB)", + self.size.as_bytes_u64(), + self.size.as_bytes_u64() / (1024 * 1024), + MIN_POOL_SIZE, + MIN_POOL_SIZE / (1024 * 1024), + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + if self.enabled && !self.size.as_bytes_u64().is_multiple_of(DEFAULT_PAGE_SIZE) { + eprintln!( + "Configured system.memory_pool.size {} B is not a multiple of default page size {} B", + self.size.as_bytes_u64(), + DEFAULT_PAGE_SIZE + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + if self.enabled && self.bucket_capacity < MIN_BUCKET_CAPACITY { + eprintln!( + "Configured system.memory_pool.buffers {} is less than minimum {}", + self.bucket_capacity, MIN_BUCKET_CAPACITY + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + if self.enabled && !self.bucket_capacity.is_power_of_two() { + eprintln!( + "Configured system.memory_pool.buffers {} is not a power of 2", + self.bucket_capacity + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + Ok(()) + } +} + +/// Validate a [`CpuAllocation`] against the machine's available parallelism +/// and, when pinning, the process affinity mask. +pub(crate) fn validate_cpu_allocation( + cpu_allocation: &CpuAllocation, + pin_cores: bool, +) -> Result<(), ConfigurationError> { + let available_cpus = available_parallelism() + .map_err(|_| { + eprintln!("Failed to detect available CPU cores"); + ConfigurationError::InvalidConfigurationValue + })? + .get(); + + match cpu_allocation { + CpuAllocation::All => Ok(()), + CpuAllocation::Count(count) => { + if *count == 0 { + eprintln!("Invalid sharding configuration: cpu_allocation count cannot be 0"); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if *count > available_cpus { + eprintln!( + "Invalid sharding configuration: cpu_allocation count {count} exceeds available CPU cores {available_cpus}" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + Ok(()) + } + CpuAllocation::Range(start, end) => { + if start >= end { + eprintln!( + "Invalid sharding configuration: cpu_allocation range {start}..{end} is invalid (start must be less than end)" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if *end - *start > available_cpus { + eprintln!( + "Invalid sharding configuration: cpu_allocation range {start}..{end} yields {} shards, exceeding available CPU cores {available_cpus}", + *end - *start + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if !pin_cores { + return Ok(()); + } + let allowed = allowed_cpus(); + if let Some(cpu) = (*start..*end).find(|cpu| !allowed.contains(cpu)) { + eprintln!( + "Invalid sharding configuration: cpu_allocation range {start}..{end} includes CPU {cpu}, which is outside the set of cores allowed for this process (affinity/cpuset mask)" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + Ok(()) + } + // NUMA topology validation requires hwlocality (runtime dep). + // Full NUMA validation happens in shard_allocator at startup. + CpuAllocation::NumaAware(_) => Ok(()), + } +} + +#[cfg(test)] +mod cpu_allocation_tests { + use super::*; + + #[test] + fn inverted_range_is_rejected() { + assert!(validate_cpu_allocation(&CpuAllocation::Range(2, 2), true).is_err()); + } + + #[test] + fn pinned_range_within_allowed_set_is_accepted() { + let first = allowed_cpus()[0]; + assert!(validate_cpu_allocation(&CpuAllocation::Range(first, first + 1), true).is_ok()); + } + + #[test] + fn pinned_range_outside_allowed_set_is_rejected() { + let past_last = allowed_cpus().last().copied().unwrap() + 1; + assert!( + validate_cpu_allocation(&CpuAllocation::Range(past_last, past_last + 1), true).is_err() + ); + } + + #[test] + fn pinned_range_wider_than_parallelism_is_rejected() { + // Under a cgroup CPU quota the affinity mask stays full while + // `available_parallelism` shrinks, so membership alone would + // accept this; the shard-count cap must reject it. + let first = allowed_cpus()[0]; + let available = available_parallelism().unwrap().get(); + assert!( + validate_cpu_allocation(&CpuAllocation::Range(first, first + available + 1), true) + .is_err() + ); + } + + #[test] + fn unpinned_range_is_capped_by_shard_count_not_core_ids() { + // Core ids outside the machine are fine unpinned; only the + // resulting shard count matters. + let outside = 1 << 20; + assert!( + validate_cpu_allocation(&CpuAllocation::Range(outside, outside + 1), false).is_ok() + ); + + let available = available_parallelism().unwrap().get(); + assert!(validate_cpu_allocation(&CpuAllocation::Range(0, available + 1), false).is_err()); + } +} diff --git a/core/configs/src/lib.rs b/core/configs/src/lib.rs index 914729ef04..e9339b2aa6 100644 --- a/core/configs/src/lib.rs +++ b/core/configs/src/lib.rs @@ -17,20 +17,15 @@ extern crate self as configs; +mod common; mod configs_impl; mod server_config; -mod server_ng_config; +pub use common::{COMPONENT, cache_indexes, defaults, displays, http, system, validators}; pub use configs_derive::ConfigEnv; pub use configs_impl::{ ConfigEnvMappings, ConfigProvider, ConfigurationError, ConfigurationType, EnvVarMapping, FileConfigProvider, TypedEnvProvider, parse_env_value_to_json, }; pub use server_config::{ - COMPONENT, cache_indexes, cluster, defaults, displays, http, quic, server, sharding, system, - tcp, validators, websocket, -}; -pub use server_ng_config::{ - COMPONENT_NG, cluster as ng_cluster, message_bus, metadata as ng_metadata, - partition as ng_partition, quic as ng_quic, server_ng, sharding as ng_sharding, tcp as ng_tcp, - websocket as ng_websocket, + cluster, message_bus, metadata, partition, quic, server, sharding, tcp, websocket, }; diff --git a/core/configs/src/server_config/cluster.rs b/core/configs/src/server_config/cluster.rs index af6f26c513..ad767c6b20 100644 --- a/core/configs/src/server_config/cluster.rs +++ b/core/configs/src/server_config/cluster.rs @@ -15,23 +15,257 @@ // specific language governing permissions and limitations // under the License. +//! Cluster schema: node topology plus the VSR consensus tunables. + +use super::defaults::SERVER_CONFIG; +use crate::ConfigurationError; +use crate::http::HttpJwtConfig; use configs::ConfigEnv; +use iggy_common::{IggyDuration, Validatable}; +use ipnet::{IpNet, Ipv4Net}; use serde::{Deserialize, Serialize}; +use serde_with::{DisplayFromStr, serde_as}; +use std::cmp::Reverse; +use std::fmt; +use std::net::{IpAddr, Ipv6Addr, SocketAddr}; +use std::str::FromStr; +use std::time::Duration; + +/// Absolute floor for the backup liveness window, independent of the +/// commit-broadcast rate. The primary signals liveness through its commit +/// broadcast (`commit_broadcast_interval`, 500ms by default); 2s spans several +/// broadcasts, so a single delayed one never elects. The per-config +/// `MIN_HEARTBEAT_TO_COMMIT_BROADCAST_RATIO` check scales the same headroom +/// when the broadcast interval is retuned. +pub const MIN_CLUSTER_HEARTBEAT_TIMEOUT: Duration = Duration::from_secs(2); + +/// The backup liveness window (`heartbeat_timeout`) must span at least this +/// many commit broadcasts (`commit_broadcast_interval`), so one dropped or +/// delayed broadcast never trips a view change on a healthy primary. +const MIN_HEARTBEAT_TO_COMMIT_BROADCAST_RATIO: u32 = 4; + +/// The view-change status backstop (`view_change_status_timeout`) must span at +/// least this many retransmit intervals (`view_change_retransmit_interval`), so +/// a few dropped `StartViewChange` / `DoViewChange` messages retransmit rather +/// than escalating a progressing view change into a fresh cluster-wide election. +const MIN_STATUS_TO_RETRANSMIT_RATIO: u32 = 4; + +/// Default recovering-replica probe-attempt ceiling. Duplicated here rather +/// than imported so `core/configs` keeps off a build-time edge onto +/// `core/consensus` (mirroring [`super::partition`]); `core/server`'s +/// bootstrap static-asserts it equal to `consensus::PROBE_ATTEMPTS_MAX`. +pub const DEFAULT_VIEW_PROBE_ATTEMPTS_MAX: u32 = 5; + +/// Upper bound on `view_probe_attempts_max`. A recovering replica probes once +/// per `request_start_view_retransmit_interval`, so hundreds of attempts would +/// stall the election fallback for minutes on a full-cluster restart; this is a +/// typo guard, not a sizing endorsement. +const MAX_VIEW_PROBE_ATTEMPTS: u32 = 100; + +/// Default per-round repair-serving chunk. Duplicated here rather than imported +/// so `core/configs` keeps off a build-time edge onto `core/shard` (mirroring +/// [`DEFAULT_VIEW_PROBE_ATTEMPTS_MAX`]); `core/server`'s bootstrap +/// static-asserts it equal to `shard::REPAIR_CHUNK_MAX`. +pub const DEFAULT_REPAIR_CHUNK_MAX: usize = 128; + +/// `size_of::()`. Duplicated here for the same reason as +/// [`DEFAULT_REPAIR_CHUNK_MAX`]; `core/server`'s bootstrap static-asserts it +/// against the real header. Used to reject a `[message_bus] max_message_size` +/// too small to carry a single state-transfer chunk. +pub const STATE_CHUNK_HEADER_LEN: u64 = 256; + +/// Upper bound on `repair_chunk_max`. A chunk rides the per-peer bus queue, so +/// the load-bearing rule is `repair_chunk_max < message_bus.peer_queue_capacity` +/// (enforced at the top level); this standalone ceiling is a typo guard. +const MAX_REPAIR_CHUNK_MAX: usize = 1024; + +/// Upper bound on per-node `advertised_addresses` selectors. Must mirror the +/// `#[config_env(max_elements = 16)]` cap on the field: env overrides above +/// the cap do not exist, so a TOML roster exceeding it could never be +/// replicated through the env path. Also bounds the cross-node conflict scan +/// (quadratic in pooled entries) and the per-request longest-prefix walk. +const MAX_ADVERTISED_SELECTORS: usize = 16; + +/// Length floor for the replica-auth PSK, in raw bytes. The 32-byte MAC key +/// is KDF-derived from these bytes at use-site, so any encoding clearing this +/// length is accepted. +const MIN_SHARED_SECRET_LEN: usize = 32; + +/// DNS caps a full name at 255 octets on the wire, which leaves 253 +/// characters of presentation text (RFC 1035). +const MAX_HOSTNAME_LEN: usize = 253; + +/// Per-label limit from RFC 1035. +const MAX_HOSTNAME_LABEL_LEN: usize = 63; + +/// serde fallback for configs written before the field existed; the value +/// itself lives in `core/server/config.toml` like every other default. +fn default_heartbeat_timeout() -> IggyDuration { + SERVER_CONFIG.cluster.heartbeat_timeout.parse().unwrap() +} + +/// serde fallback for configs written before the field existed; the value +/// itself lives in `core/server/config.toml` like every other default. +fn default_commit_broadcast_interval() -> IggyDuration { + SERVER_CONFIG + .cluster + .commit_broadcast_interval + .parse() + .unwrap() +} + +/// serde fallback for configs written before the field existed; the value +/// itself lives in `core/server/config.toml` like every other default. +fn default_prepare_retransmit_interval() -> IggyDuration { + SERVER_CONFIG + .cluster + .prepare_retransmit_interval + .parse() + .unwrap() +} + +/// serde fallback for configs written before the field existed; the value +/// itself lives in `core/server/config.toml` like every other default. +fn default_view_change_retransmit_interval() -> IggyDuration { + SERVER_CONFIG + .cluster + .view_change_retransmit_interval + .parse() + .unwrap() +} + +/// serde fallback for configs written before the field existed; the value +/// itself lives in `core/server/config.toml` like every other default. +fn default_view_change_status_timeout() -> IggyDuration { + SERVER_CONFIG + .cluster + .view_change_status_timeout + .parse() + .unwrap() +} + +/// serde fallback for configs written before the field existed; the value +/// itself lives in `core/server/config.toml` like every other default. +fn default_request_start_view_retransmit_interval() -> IggyDuration { + SERVER_CONFIG + .cluster + .request_start_view_retransmit_interval + .parse() + .unwrap() +} + +/// serde fallback for configs written before the field existed; the value +/// itself lives in `core/server/config.toml` like every other default. +fn default_view_probe_attempts_max() -> u32 { + SERVER_CONFIG.cluster.view_probe_attempts_max as u32 +} + +/// serde fallback for configs written before the field existed; the value +/// itself lives in `core/server/config.toml` like every other default. +fn default_repair_retry_interval() -> IggyDuration { + SERVER_CONFIG.cluster.repair_retry_interval.parse().unwrap() +} + +/// serde fallback for configs written before the field existed; the value +/// itself lives in `core/server/config.toml` like every other default. +fn default_repair_chunk_max() -> usize { + SERVER_CONFIG.cluster.repair_chunk_max as usize +} +#[serde_as] #[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] #[serde(deny_unknown_fields)] pub struct ClusterConfig { pub enabled: bool, pub name: String, + /// Backup-side liveness window for a plane's primary. A replica that + /// sees no primary traffic for this long starts a view change + /// (`normal_heartbeat_timeout`). Raise it on oversubscribed hosts where + /// scheduling stalls fake primary death; sub-`MIN_CLUSTER_HEARTBEAT_TIMEOUT` + /// values (including the `0` / `disabled` / `unlimited` sentinels, which + /// all parse to zero) are rejected at boot. + #[serde(default = "default_heartbeat_timeout")] + #[serde_as(as = "DisplayFromStr")] + #[config_env(leaf)] + pub heartbeat_timeout: IggyDuration, + /// How often the primary broadcasts its commit point to every backup, the + /// cluster's primary-liveness signal. Each broadcast resets the backups' + /// `heartbeat_timeout` window, so that window must span several broadcasts: + /// boot rejects `heartbeat_timeout < MIN_HEARTBEAT_TO_COMMIT_BROADCAST_RATIO + /// * commit_broadcast_interval`. Sizes the consensus `CommitMessage` timer. + /// Zero (and the `0` / `disabled` / `unlimited` sentinels, which all parse + /// to zero) is rejected at boot. + #[serde(default = "default_commit_broadcast_interval")] + #[serde_as(as = "DisplayFromStr")] + #[config_env(leaf)] + pub commit_broadcast_interval: IggyDuration, + /// How often the primary retransmits prepares a backup has not yet acked. + /// Lower recovers faster from a dropped prepare at the cost of replica + /// traffic. Sizes the consensus `Prepare` timer. Zero (and the `0` / + /// `disabled` / `unlimited` sentinels, which all parse to zero) is rejected + /// at boot. + #[serde(default = "default_prepare_retransmit_interval")] + #[serde_as(as = "DisplayFromStr")] + #[config_env(leaf)] + pub prepare_retransmit_interval: IggyDuration, + /// How often a plane retransmits its `StartViewChange` / `DoViewChange` + /// while a view change is running. Lower converges a healthy election + /// faster at the cost of replica traffic. Sizes both consensus view-change + /// retransmit timers, which are deliberately equal. Zero (and the `0` / + /// `disabled` / `unlimited` sentinels, which all parse to zero) is rejected + /// at boot. + #[serde(default = "default_view_change_retransmit_interval")] + #[serde_as(as = "DisplayFromStr")] + #[config_env(leaf)] + pub view_change_retransmit_interval: IggyDuration, + /// Backstop for a stalled view change: one that does not conclude within + /// this window escalates to a fresh cluster-wide election. Must span + /// several `view_change_retransmit_interval`s so a few dropped view-change + /// messages retransmit rather than escalate: boot rejects + /// `view_change_status_timeout < MIN_STATUS_TO_RETRANSMIT_RATIO * + /// view_change_retransmit_interval`. Zero (and the `0` / `disabled` / + /// `unlimited` sentinels, which all parse to zero) is rejected at boot. + #[serde(default = "default_view_change_status_timeout")] + #[serde_as(as = "DisplayFromStr")] + #[config_env(leaf)] + pub view_change_status_timeout: IggyDuration, + /// How often a recovering or view-change backup re-requests the current + /// view's `StartView` from its primary (`RequestStartView`). Sizes the + /// consensus `RequestStartView` timer. Zero (and the `0` / `disabled` / + /// `unlimited` sentinels, which all parse to zero) is rejected at boot. + #[serde(default = "default_request_start_view_retransmit_interval")] + #[serde_as(as = "DisplayFromStr")] + #[config_env(leaf)] + pub request_start_view_retransmit_interval: IggyDuration, + /// How many consecutive unanswered `RequestStartView` probes a recovering + /// replica tolerates before it falls back to an election (a full-cluster + /// restart leaves nobody settled to answer). Must be >= 1 and <= + /// `MAX_VIEW_PROBE_ATTEMPTS`. + #[serde(default = "default_view_probe_attempts_max")] + pub view_probe_attempts_max: u32, + /// How long a stalled journal-repair stream waits before re-requesting its + /// remaining window from the serving peer. Repair frames are + /// fire-and-forget over the lossy bus, so a session with no retry wedges + /// forever on a single dropped frame. Paces both the metadata and + /// partition repair loops. Sizes the retry threshold in consensus ticks. + /// Zero (and the `0` / `disabled` / `unlimited` sentinels, which all parse + /// to zero) is rejected at boot. + #[serde(default = "default_repair_retry_interval")] + #[serde_as(as = "DisplayFromStr")] + #[config_env(leaf)] + pub repair_retry_interval: IggyDuration, + /// Prepares a peer serves per repair round before the requester walks to + /// the next chunk. Each frame rides the per-peer message-bus queue, so this + /// must stay below `message_bus.peer_queue_capacity` or a full round + /// overruns the queue and silently drops frames (enforced at the top + /// level). Applies to both the metadata and partition repair planes. Must + /// be > 0 and <= `MAX_REPAIR_CHUNK_MAX`. + #[serde(default = "default_repair_chunk_max")] + pub repair_chunk_max: usize, /// Full roster of cluster members. Intended to be byte-identical across /// every node so operators ship one config. The running node's identity /// is supplied out-of-band via the `--replica-id` CLI flag, which /// selects the entry in this list that describes the current node. - // - // TODO(hubcio): IGGY-155 `register-replica` CLI (a validated roster - // append) is deferred - it is convenience only over a manual TOML edit, - // and `ClusterConfig::validate` already rejects a malformed roster at - // boot. Add it only if scripted/automated roster edits become a need. #[serde(default)] pub nodes: Vec, /// Replica-to-replica authentication settings (PSK + BLAKE3 handshake). @@ -71,11 +305,25 @@ pub struct ClusterAuthConfig { #[serde(default, skip_serializing)] #[config_env(secret)] pub shared_secret: String, + /// Retiring pre-shared key, accepted for VERIFICATION only during a key + /// rotation window; every MAC this node produces uses [`Self::shared_secret`]. + /// + /// Enables rolling PSK rotation without an auth outage, three rolls: + /// 1. every node gets `shared_secret = old, previous_shared_secret = new`; + /// 2. every node gets `shared_secret = new, previous_shared_secret = old`; + /// 3. every node gets `shared_secret = new` alone, closing the window. + /// + /// Leave empty (default) outside a rotation. Same 32-byte minimum and + /// provisioning rules as `shared_secret` + /// (`IGGY_CLUSTER_AUTH_PREVIOUS_SHARED_SECRET`). + #[serde(default, skip_serializing)] + #[config_env(secret)] + pub previous_shared_secret: String, } /// Replica-to-replica TLS for the consensus (`tcp_replica`) port. /// -/// Mirrors the legacy [`super::tcp::TcpTlsConfig`] shape plus `ca_file`: +/// Mirrors the legacy [`crate::tcp::TcpTlsConfig`] shape plus `ca_file`: /// the replica plane DIALS its peers (a TLS client role the /// client-facing server plane never has), so the dialer needs a trust /// anchor to verify the acceptor's certificate against. @@ -110,9 +358,23 @@ pub struct ClusterTlsConfig { } #[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] +#[serde(deny_unknown_fields)] pub struct ClusterNodeConfig { pub name: String, pub ip: String, + /// Optional client-facing address: a literal IP or a DNS hostname, + /// validated as [`AdvertisedAddress`] at boot. Replica traffic continues + /// to use [`Self::ip`]. + #[serde(default)] + pub advertised_address: Option, + /// Client-network-scoped overrides of [`Self::advertised_address`], + /// resolved by longest-prefix match over the client's IP (see + /// [`AdvertisedAddressSelector`]). Empty by default, so existing configs + /// keep the single catch-all address. At most `MAX_ADVERTISED_SELECTORS` + /// per node (validated), matching the env-override cap below. + #[serde(default)] + #[config_env(max_elements = 16)] + pub advertised_addresses: Vec, /// Numeric replica ID for VSR consensus (0-based). /// /// Must be unique across [`ClusterConfig::nodes`] and strictly less than @@ -121,6 +383,165 @@ pub struct ClusterNodeConfig { pub ports: TransportPorts, } +/// One client-network-scoped advertised address: clients whose IP falls +/// inside `client_cidr` are told `address` instead of the node's catch-all +/// [`ClusterNodeConfig::advertised_address`]. +/// +/// Typical split-network case: the roster `ip` is VPC-private and +/// `advertised_address` is public; a selector with the VPC CIDR keeps +/// in-VPC clients on the private address while everyone else stays on the +/// public one. Selection is longest-prefix match across a node's selectors. +/// Selection sees the transport-level peer address, so clients arriving +/// through a proxy or load balancer match the proxy's network, not their +/// own. +#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] +#[serde(deny_unknown_fields)] +pub struct AdvertisedAddressSelector { + /// Client network this selector matches, in CIDR notation + /// (`10.0.0.0/16`, `2001:db8::/32`). Must parse at boot; duplicate + /// networks within one node are rejected. A v4-mapped v6 network + /// (`::ffff:10.0.0.0/104`) canonicalizes to its v4 form (`10.0.0.0/8`), + /// matching how client IPs canonicalize before matching. + pub client_cidr: String, + /// Address advertised to matching clients: a literal IP or a DNS + /// hostname, validated as [`AdvertisedAddress`] at boot (no port; ports + /// come from [`ClusterNodeConfig::ports`]). + pub address: String, +} + +/// A roster node with its advertised-address selectors and catch-all parsed +/// once, built wherever a roster is assembled for serving clients +/// (listener/shard start). Per-request resolution never re-parses config +/// strings: everything is snapshotted here, so mutating the source config +/// after conversion has no effect on what clients are told. Entries that do +/// not parse are dropped at build time; validation already rejects them +/// whenever the cluster is enabled, and a disabled cluster never consults +/// the roster. +#[derive(Debug, Clone)] +pub struct ResolvedClusterNode { + config: ClusterNodeConfig, + /// Truncated, canonicalized selector networks with their parsed + /// addresses, in declaration order. + selectors: Vec<(IpNet, AdvertisedAddress)>, + /// Parsed catch-all: [`ClusterNodeConfig::advertised_address`], else the + /// roster [`ClusterNodeConfig::ip`]. `None` when the configured value + /// does not parse - a set `advertised_address` never falls through to + /// the private roster ip. + catch_all: Option, + /// Parsed roster [`ClusterNodeConfig::ip`], the replica-plane dial + /// address. `None` when the roster ip is not a literal IP (boot only + /// requires it non-empty); internal forwarding then has no dial target. + replica_ip: Option, +} + +impl From for ResolvedClusterNode { + fn from(config: ClusterNodeConfig) -> Self { + let selectors = config + .advertised_addresses + .iter() + .filter_map(|selector| { + let network = selector.client_cidr.parse::().ok()?; + let address = selector.address.parse::().ok()?; + Some((canonical_ip_net(network.trunc()), address)) + }) + .collect(); + let catch_all = match config.advertised_address.as_deref() { + Some(advertised_address) => advertised_address.parse().ok(), + None => config.ip.parse().ok(), + }; + let replica_ip = config.ip.parse().ok(); + Self { + config, + selectors, + catch_all, + replica_ip, + } + } +} + +impl ResolvedClusterNode { + /// The roster entry this node was built from. Read-only: resolution runs + /// on the boot-parsed snapshot, never on the config strings. + #[must_use] + pub fn config(&self) -> &ClusterNodeConfig { + &self.config + } + + /// The roster `ip` as a dialable address for the replica plane and + /// internal request forwarding. Never routed through the advertised + /// ladder: this is what servers dial, not what clients are told. + #[must_use] + pub fn replica_ip(&self) -> Option { + self.replica_ip + } + + /// The client-facing address for a client connecting from `client_ip`: + /// longest-prefix match over the selector networks, then the parsed + /// catch-all. `None` when no selector matches and the catch-all did not + /// parse; callers choose whether to fail closed (redirect URLs) or to + /// publish [`Self::raw_advertised_fallback`] verbatim (cluster metadata). + #[must_use] + pub fn advertised_for(&self, client_ip: Option) -> Option<&AdvertisedAddress> { + client_ip + .and_then(|client_ip| self.selector_address(client_ip)) + .or(self.catch_all.as_ref()) + } + + /// The catch-all ladder ([`ClusterNodeConfig::advertised_address`], else + /// the roster [`ClusterNodeConfig::ip`]) as configured, unparsed. Cluster + /// metadata publishes this verbatim when [`Self::advertised_for`] finds + /// nothing: the roster `ip` is only validated non-empty, and Docker + /// service names with underscores exist in the wild. + #[must_use] + pub fn raw_advertised_fallback(&self) -> &str { + self.config + .advertised_address + .as_deref() + .unwrap_or(&self.config.ip) + } + + /// Longest-prefix match over the boot-parsed selector networks. The + /// client IP is canonicalized first so a v4-mapped v6 peer + /// (`::ffff:10.0.0.7`, the shape a dual-stack listener reports) matches + /// v4 networks. `min_by_key` keeps the first of equal-length matches, so + /// resolution stays declaration-order deterministic even though a + /// validated config cannot produce two matching networks of equal length + /// (equal-length distinct networks are disjoint, duplicates are + /// rejected). + fn selector_address(&self, client_ip: IpAddr) -> Option<&AdvertisedAddress> { + let client_ip = client_ip.to_canonical(); + self.selectors + .iter() + .filter(|(network, _)| network.contains(&client_ip)) + .min_by_key(|(network, _)| Reverse(network.prefix_len())) + .map(|(_, address)| address) + } +} + +/// Network-side mirror of the `IpAddr::to_canonical` applied to client IPs +/// before matching: a selector network written in v4-mapped v6 form +/// (`::ffff:10.0.0.0/104`) becomes its v4 equivalent (`10.0.0.0/8`), since a +/// canonicalized client could never match the v6 spelling. Prefixes shorter +/// than 96 bits cannot drop the `::ffff:` mapping and stay v6 (they match +/// native v6 clients only). +fn canonical_ip_net(network: IpNet) -> IpNet { + if let IpNet::V6(v6_network) = network + && v6_network.prefix_len() >= 96 + && let IpAddr::V4(v4_address) = v6_network.addr().to_canonical() + && let Ok(v4_network) = Ipv4Net::new(v4_address, v6_network.prefix_len() - 96) + { + return IpNet::V4(v4_network); + } + network +} + +/// Per-node listener ports advertised in the cluster roster. In cluster mode +/// the roster is the single source of ports: every enabled transport needs +/// an explicit per-node port (validated at startup, no fallback to the +/// transport's top-level `address` port). The roster entry's `ip` is the +/// advertised address only: tcp/ws/quic/http bind the interface from their own +/// `address` config, and followers forward HTTP requests to the primary at +/// `ip:http`. #[derive(Debug, Deserialize, Serialize, Clone, Default, ConfigEnv)] pub struct TransportPorts { pub tcp: Option, @@ -131,6 +552,832 @@ pub struct TransportPorts { pub tcp_replica: Option, } +/// A validated client-facing node address: a literal IP or a DNS hostname. +/// +/// Hostnames follow RFC 1123: ASCII letters, digits and hyphens in labels of +/// 1-63 characters that do not start or end with a hyphen, at most +/// [`MAX_HOSTNAME_LEN`] characters total, no port and no trailing dot. Names +/// consisting solely of digits and dots are rejected as malformed IPv4 rather +/// than accepted as hostnames, so `10.0.0.256` fails loudly instead of being +/// handed to DNS. Hostnames normalize to lowercase and IPs to their canonical +/// form ([`IpAddr`]), so textual variants of one address (`Broker.Example.COM`, +/// `2001:DB8::1`, `[2001:db8::1]`) compare equal. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum AdvertisedAddress { + Ip(IpAddr), + Hostname(String), +} + +impl AdvertisedAddress { + /// Render `host:port` for a URL or endpoint listing, bracketing IPv6 + /// hosts (`[::1]:8080`) so the port separator stays unambiguous. + pub fn authority(&self, port: u16) -> String { + match self { + Self::Ip(ip) => SocketAddr::new(*ip, port).to_string(), + Self::Hostname(hostname) => format!("{hostname}:{port}"), + } + } +} + +impl FromStr for AdvertisedAddress { + type Err = AdvertisedAddressError; + + fn from_str(address: &str) -> Result { + if address.is_empty() { + return Err(AdvertisedAddressError::Empty); + } + if let Ok(ip) = address.parse::() { + return Ok(Self::Ip(ip)); + } + // URL-style bracketed IPv6 (`[2001:db8::1]`) is unambiguous; accept + // it and store the inner address. + if let Some(inner) = address + .strip_prefix('[') + .and_then(|rest| rest.strip_suffix(']')) + && let Ok(ip) = inner.parse::() + { + return Ok(Self::Ip(IpAddr::V6(ip))); + } + if let Some((host, port)) = address.rsplit_once(':') { + // `host:port` and `[v6]:port` are the common misconfigurations; + // anything else with a colon can only be a broken IPv6 literal, + // since ':' never appears in a hostname. + let bracketed_host = host.starts_with('[') && host.ends_with(']'); + if !port.is_empty() + && port.bytes().all(|byte| byte.is_ascii_digit()) + && (bracketed_host || !host.contains(':')) + { + return Err(AdvertisedAddressError::PortNotAllowed); + } + return Err(AdvertisedAddressError::MalformedIpv6); + } + if address.len() > MAX_HOSTNAME_LEN { + return Err(AdvertisedAddressError::HostnameTooLong { + length: address.len(), + }); + } + let mut all_labels_numeric = true; + for label in address.split('.') { + if label.is_empty() { + return Err(AdvertisedAddressError::EmptyLabel); + } + if label.len() > MAX_HOSTNAME_LABEL_LEN { + return Err(AdvertisedAddressError::LabelTooLong { + label: label.to_owned(), + }); + } + if label.starts_with('-') || label.ends_with('-') { + return Err(AdvertisedAddressError::LabelHyphen { + label: label.to_owned(), + }); + } + if let Some(character) = label + .chars() + .find(|character| !character.is_ascii_alphanumeric() && *character != '-') + { + return Err(AdvertisedAddressError::InvalidCharacter { character }); + } + all_labels_numeric &= label.bytes().all(|byte| byte.is_ascii_digit()); + } + if all_labels_numeric { + return Err(AdvertisedAddressError::MalformedIpv4); + } + // DNS resolution is case-insensitive; normalizing here makes equality + // (and thus endpoint-conflict detection) case-insensitive too. + Ok(Self::Hostname(address.to_ascii_lowercase())) + } +} + +impl fmt::Display for AdvertisedAddress { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Ip(ip) => write!(formatter, "{ip}"), + Self::Hostname(hostname) => write!(formatter, "{hostname}"), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AdvertisedAddressError { + Empty, + PortNotAllowed, + MalformedIpv4, + MalformedIpv6, + HostnameTooLong { length: usize }, + EmptyLabel, + LabelTooLong { label: String }, + LabelHyphen { label: String }, + InvalidCharacter { character: char }, +} + +impl fmt::Display for AdvertisedAddressError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Empty => write!(formatter, "address cannot be empty"), + Self::PortNotAllowed => write!( + formatter, + "address must not include a port; ports are configured in cluster.nodes.ports" + ), + Self::MalformedIpv4 => write!( + formatter, + "address consists only of digits and dots but is not a valid IPv4 address" + ), + Self::MalformedIpv6 => write!( + formatter, + "address contains ':' but is not a valid IPv6 address, and ':' cannot appear in a hostname" + ), + Self::HostnameTooLong { length } => write!( + formatter, + "hostname is {length} characters long; the limit is {MAX_HOSTNAME_LEN}" + ), + Self::EmptyLabel => write!( + formatter, + "hostname contains an empty label (leading, trailing, or doubled dot)" + ), + Self::LabelTooLong { label } => write!( + formatter, + "hostname label '{label}' exceeds {MAX_HOSTNAME_LABEL_LEN} characters" + ), + Self::LabelHyphen { label } => write!( + formatter, + "hostname label '{label}' cannot start or end with a hyphen" + ), + Self::InvalidCharacter { character } => write!( + formatter, + "character '{character}' is not allowed in a hostname (allowed: ASCII letters, digits, '-', '.')" + ), + } + } +} + +impl std::error::Error for AdvertisedAddressError {} + +/// Whether cluster-wide JWT key material exists: a configured `http.jwt` +/// secret, or the signing key derived from the cluster PSK. When it does, a +/// bearer minted on any node verifies on every node - the invariant +/// follower-to-primary HTTP forwarding depends on. Callers gate `http.enabled` +/// themselves; this covers only the key material. +/// +/// Forwarding targets resolve from the roster (`ip:ports.http`); the config +/// validator unconditionally requires a roster port for every enabled +/// transport, so a forward never dials a node without a declared http port. +pub fn http_forwarding_key_material(jwt: &HttpJwtConfig, cluster: &ClusterConfig) -> bool { + cluster.enabled + && ((cluster.auth.enabled && !cluster.auth.shared_secret.is_empty()) + || !jwt.encoding_secret.is_empty() + || !jwt.decoding_secret.is_empty()) +} + +impl Validatable for ClusterConfig { + fn validate(&self) -> Result<(), ConfigurationError> { + // Ahead of the enabled gate: the top-level rule against + // message_bus.peer_queue_capacity holds repair_chunk_max unconditionally, + // so a single-node config skipping these would be bound by the + // cross-section rule while its own floor and ceiling went unchecked. + if self.repair_chunk_max == 0 { + eprintln!("Invalid cluster configuration: cluster.repair_chunk_max must be > 0"); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if self.repair_chunk_max > MAX_REPAIR_CHUNK_MAX { + eprintln!( + "Invalid cluster configuration: cluster.repair_chunk_max ({}) exceeds the maximum \ + ({MAX_REPAIR_CHUNK_MAX})", + self.repair_chunk_max + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + if !self.enabled { + return Ok(()); + } + + if self.name.trim().is_empty() { + eprintln!("Invalid cluster configuration: cluster name cannot be empty"); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + // `0` / `disabled` / `unlimited` all parse to a zero duration and + // land here too: there is no way to switch the liveness window off. + if self.heartbeat_timeout.get_duration() < MIN_CLUSTER_HEARTBEAT_TIMEOUT { + eprintln!( + "Invalid cluster configuration: cluster.heartbeat_timeout '{}' must be at least {}s \ + (the primary signals liveness through its commit broadcast; a shorter window \ + elects on every scheduling hiccup)", + self.heartbeat_timeout, + MIN_CLUSTER_HEARTBEAT_TIMEOUT.as_secs() + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + // The commit broadcast is the cluster's liveness feed and the prepare + // retransmit its recovery timer; both size consensus timers that have + // to advance. `0` / `disabled` / `unlimited` all collapse to a zero + // duration, which would stall the timer - reject them. + if self.commit_broadcast_interval.get_duration().is_zero() { + eprintln!( + "Invalid cluster configuration: cluster.commit_broadcast_interval must be nonzero \ + (it drives the primary's liveness broadcast)" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if self.prepare_retransmit_interval.get_duration().is_zero() { + eprintln!( + "Invalid cluster configuration: cluster.prepare_retransmit_interval must be \ + nonzero (it drives prepare retransmission)" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + // The liveness window must span several commit broadcasts so a single + // delayed broadcast never trips a view change on a healthy primary. + let min_heartbeat = self + .commit_broadcast_interval + .get_duration() + .saturating_mul(MIN_HEARTBEAT_TO_COMMIT_BROADCAST_RATIO); + if self.heartbeat_timeout.get_duration() < min_heartbeat { + eprintln!( + "Invalid cluster configuration: cluster.heartbeat_timeout '{}' must be at least \ + {}x cluster.commit_broadcast_interval '{}' so the liveness window spans several \ + broadcasts", + self.heartbeat_timeout, + MIN_HEARTBEAT_TO_COMMIT_BROADCAST_RATIO, + self.commit_broadcast_interval + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + // The three view-change timers each size a consensus timer that has to + // advance; `0` / `disabled` / `unlimited` all collapse to zero and + // would stall it - reject them. + if self + .view_change_retransmit_interval + .get_duration() + .is_zero() + { + eprintln!( + "Invalid cluster configuration: cluster.view_change_retransmit_interval must be \ + nonzero (it drives StartViewChange / DoViewChange retransmission)" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if self.view_change_status_timeout.get_duration().is_zero() { + eprintln!( + "Invalid cluster configuration: cluster.view_change_status_timeout must be nonzero \ + (it backstops a stalled view change)" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if self + .request_start_view_retransmit_interval + .get_duration() + .is_zero() + { + eprintln!( + "Invalid cluster configuration: cluster.request_start_view_retransmit_interval \ + must be nonzero (it drives RequestStartView retransmission)" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + // The status backstop must span several retransmits so a few dropped + // view-change messages retransmit rather than escalating a progressing + // view change into a fresh cluster-wide election. + let min_status = self + .view_change_retransmit_interval + .get_duration() + .saturating_mul(MIN_STATUS_TO_RETRANSMIT_RATIO); + if self.view_change_status_timeout.get_duration() < min_status { + eprintln!( + "Invalid cluster configuration: cluster.view_change_status_timeout '{}' must be at \ + least {}x cluster.view_change_retransmit_interval '{}' so a stalled view change \ + retransmits before it escalates to an election", + self.view_change_status_timeout, + MIN_STATUS_TO_RETRANSMIT_RATIO, + self.view_change_retransmit_interval + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + // A recovering replica needs at least one probe before it may give up + // and elect; the ceiling is a typo guard (see MAX_VIEW_PROBE_ATTEMPTS). + if self.view_probe_attempts_max == 0 { + eprintln!( + "Invalid cluster configuration: cluster.view_probe_attempts_max must be >= 1" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if self.view_probe_attempts_max > MAX_VIEW_PROBE_ATTEMPTS { + eprintln!( + "Invalid cluster configuration: cluster.view_probe_attempts_max ({}) exceeds the \ + maximum ({MAX_VIEW_PROBE_ATTEMPTS})", + self.view_probe_attempts_max + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + // The repair retry interval sizes a tick threshold that has to advance; + // `0` / `disabled` / `unlimited` all collapse to zero and would wedge + // every stalled repair stream - reject them. + if self.repair_retry_interval.get_duration().is_zero() { + eprintln!( + "Invalid cluster configuration: cluster.repair_retry_interval must be nonzero \ + (it paces stalled-repair retries)" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + if self.nodes.is_empty() { + eprintln!( + "Invalid cluster configuration: cluster.nodes must contain at least one entry when cluster is enabled" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + // VSR needs every replica to have a stable, unique id strictly + // less than the total replica count. Duplicate ids would split the + // cluster into two replicas claiming the same slot; out-of-range + // ids never win a primary election. Both are unrecoverable at + // runtime - fail fast at startup. + let total_replicas = u8::try_from(self.nodes.len()).map_err(|_| { + eprintln!("Invalid cluster configuration: more than 255 replicas is unsupported"); + ConfigurationError::InvalidConfigurationValue + })?; + + let mut seen_ids = std::collections::HashSet::new(); + let mut seen_names = std::collections::HashSet::new(); + let mut used_endpoints = std::collections::HashSet::new(); + let mut advertised_endpoints: Vec = Vec::new(); + + for node in &self.nodes { + if node.name.trim().is_empty() { + eprintln!("Invalid cluster configuration: node name cannot be empty"); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + if node.ip.trim().is_empty() { + eprintln!( + "Invalid cluster configuration: IP cannot be empty for node '{}'", + node.name + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + if !seen_names.insert(node.name.clone()) { + eprintln!( + "Invalid cluster configuration: duplicate node name '{}' found", + node.name + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + if node.replica_id >= total_replicas { + eprintln!( + "Invalid cluster configuration: replica_id {} for node '{}' must be < total replica count {total_replicas}", + node.replica_id, node.name + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + if !seen_ids.insert(node.replica_id) { + eprintln!( + "Invalid cluster configuration: duplicate replica_id {} (two nodes claim the same slot)", + node.replica_id + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + let client_ports = [ + ("TCP", node.ports.tcp), + ("QUIC", node.ports.quic), + ("HTTP", node.ports.http), + ("WebSocket", node.ports.websocket), + ]; + let replica_port = ("TCP_REPLICA", node.ports.tcp_replica); + + for (name, port_opt) in client_ports.into_iter().chain([replica_port]) { + if let Some(port) = port_opt { + if port == 0 { + eprintln!( + "Invalid cluster configuration: {} port cannot be 0 for node '{}'", + name, node.name + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + let endpoint = format!("{}:{}", node.ip, port); + if !used_endpoints.insert(endpoint.clone()) { + eprintln!( + "Invalid cluster configuration: port conflict - {endpoint} is already bound (node '{}', transport {name})", + node.name + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + } + } + + // An advertised address must parse strictly (IP or RFC 1123 + // hostname): the value is handed verbatim to every client via + // cluster metadata and redirect URLs, so a bad one poisons them + // all. The roster `ip` predates this check and is only validated + // as non-empty (Docker service names with underscores exist in + // the wild), so when it backs the client endpoints an unparsable + // value falls back to raw-string comparison instead of failing + // boot. + let client_address = match node.advertised_address.as_deref() { + Some(advertised_address) => match advertised_address.parse::() { + Ok(address) => Some(address), + Err(error) => { + eprintln!( + "Invalid cluster configuration: advertised_address '{advertised_address}' for node '{}': {error}", + node.name + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + }, + None => node.ip.parse::().ok(), + }; + + if node.advertised_addresses.len() > MAX_ADVERTISED_SELECTORS { + eprintln!( + "Invalid cluster configuration: node '{}' declares {} advertised_addresses \ + selectors, exceeding the maximum ({MAX_ADVERTISED_SELECTORS})", + node.name, + node.advertised_addresses.len() + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + // Selector CIDRs and addresses feed clients the same way the + // catch-all advertised address does, so they get the same strict + // parse. Networks are compared truncated (`10.0.1.0/16` == + // `10.0.0.0/16`) and canonicalized (`::ffff:10.0.0.0/104` == + // `10.0.0.0/8`) since matching truncates and canonicalizes too. + // Parsed before the catch-all enters the conflict pool because + // every entry's effective client set depends on the node's full + // selector list. + let mut selectors = Vec::with_capacity(node.advertised_addresses.len()); + let mut seen_selector_cidrs = std::collections::HashSet::new(); + for selector in &node.advertised_addresses { + let client_cidr = match selector.client_cidr.parse::() { + Ok(client_cidr) => canonical_ip_net(client_cidr.trunc()), + Err(error) => { + eprintln!( + "Invalid cluster configuration: advertised_addresses client_cidr '{}' for node '{}': {error}", + selector.client_cidr, node.name + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + }; + if !seen_selector_cidrs.insert(client_cidr) { + eprintln!( + "Invalid cluster configuration: duplicate advertised_addresses client_cidr '{}' for node '{}'", + selector.client_cidr, node.name + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + let address = match selector.address.parse::() { + Ok(address) => address, + Err(error) => { + eprintln!( + "Invalid cluster configuration: advertised_addresses address '{}' for node '{}': {error}", + selector.address, node.name + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + }; + selectors.push((client_cidr, address)); + } + let selector_ranges: Vec = selectors + .iter() + .map(|(network, _)| ClientAddressRange::from(network)) + .collect(); + + // Endpoint conflicts are checked across every node's selectors + // and catch-all on effective client sets - the clients an entry + // actually wins after this node's longest-prefix match. Two + // nodes may reuse one host:port as long as the winning sets stay + // disjoint (that is the feature, and it includes a per-subnet + // override shadowing the same node's wider selector); a conflict + // means some client wins both entries and would resolve both + // nodes to one endpoint. The catch-all is an implicit + // match-everything-else selector, so it pools the same way. A + // roster ip that fails the strict parse skips the pool: it can + // never equal a parsed host, and two raw ips sharing host:port + // are already rejected by the bind-endpoint check above. + if let Some(address) = &client_address { + let catch_all_clients = EffectiveClients::for_catch_all(&selector_ranges); + for (name, port) in &client_ports { + if let Some(port) = port { + insert_advertised_endpoint( + &mut advertised_endpoints, + AdvertisedEndpoint { + node_name: &node.name, + transport: name, + network: None, + clients: catch_all_clients.clone(), + host: address.clone(), + port: *port, + }, + )?; + } + } + } + + for (selector_index, (client_cidr, address)) in selectors.iter().enumerate() { + let sibling_ranges: Vec = selector_ranges + .iter() + .enumerate() + .filter(|(other_index, _)| *other_index != selector_index) + .map(|(_, range)| *range) + .collect(); + let clients = EffectiveClients::for_selector(client_cidr, &sibling_ranges); + for (name, port) in &client_ports { + if let Some(port) = port { + insert_advertised_endpoint( + &mut advertised_endpoints, + AdvertisedEndpoint { + node_name: &node.name, + transport: name, + network: Some(*client_cidr), + clients: clients.clone(), + host: address.clone(), + port: *port, + }, + )?; + } + } + } + } + + // Replica-auth PSK (only reached when the cluster is enabled; the early + // return above skips these while it is disabled). When auth is enabled + // the key is mandatory; any configured key must clear the length floor - + // a typo guard that fires with auth off too, though only while the + // cluster itself is enabled. + let secret_len = self.auth.shared_secret.len(); + if self.auth.enabled && self.auth.shared_secret.is_empty() { + eprintln!( + "Invalid cluster configuration: cluster.auth.shared_secret must be set when cluster.auth.enabled is true" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if !self.auth.shared_secret.is_empty() && secret_len < MIN_SHARED_SECRET_LEN { + eprintln!( + "Invalid cluster configuration: cluster.auth.shared_secret must be >= {MIN_SHARED_SECRET_LEN} bytes" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + // Rotation window key: same typo guard as the primary, plus a + // distinctness check - a window equal to the primary means the + // operator rolled the config without changing the key, so the + // "rotation" would silently be a no-op. + if !self.auth.previous_shared_secret.is_empty() { + if self.auth.previous_shared_secret.len() < MIN_SHARED_SECRET_LEN { + eprintln!( + "Invalid cluster configuration: cluster.auth.previous_shared_secret must be >= {MIN_SHARED_SECRET_LEN} bytes" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if self.auth.previous_shared_secret == self.auth.shared_secret { + eprintln!( + "Invalid cluster configuration: cluster.auth.previous_shared_secret must differ from cluster.auth.shared_secret (an identical window is a no-op rotation)" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + } + + // Replica TLS. Both cert modes run one-directional TLS (no client + // certificate anywhere), so TLS only authenticates the acceptor to + // the dialer; peer authentication comes solely from the PSK + // handshake. Without it any TLS-capable host could register as a + // replica - require auth in both modes. CA mode (the default) + // additionally needs all three PEM paths: cert/key for this node's + // acceptor side, ca_file as the dialer's trust anchor. + if self.tls.enabled { + if !self.auth.enabled { + eprintln!( + "Invalid cluster configuration: cluster.tls.enabled = true requires cluster.auth.enabled = true (TLS authenticates the acceptor only; the PSK handshake authenticates the peer)" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if !self.tls.self_signed { + for (field, value) in [ + ("cert_file", &self.tls.cert_file), + ("key_file", &self.tls.key_file), + ("ca_file", &self.tls.ca_file), + ] { + if value.trim().is_empty() { + eprintln!( + "Invalid cluster configuration: cluster.tls.{field} must be set when cluster.tls.enabled = true and self_signed = false" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + } + } + } + + Ok(()) + } +} + +/// One advertised client endpoint and the clients it wins, pooled by +/// [`ClusterConfig::validate`] so selectors and catch-all conflict-check +/// against each other. `network: None` is the catch-all (`advertised_address`, +/// or the roster `ip` as fallback); `clients` is the entry's effective set +/// after its node's longest-prefix shadowing. +struct AdvertisedEndpoint<'roster> { + node_name: &'roster str, + transport: &'static str, + network: Option, + clients: EffectiveClients, + host: AdvertisedAddress, + port: u16, +} + +impl AdvertisedEndpoint<'_> { + /// True when some client would resolve both entries to one host:port on + /// two different nodes. Effective client sets already encode each node's + /// longest-prefix shadowing, so nested networks conflict only where the + /// wider entry still wins some client that the other node's entry also + /// wins. Entries of one node never conflict: their effective sets are + /// disjoint by construction. + fn conflicts_with(&self, other: &Self) -> bool { + self.node_name != other.node_name + && self.port == other.port + && self.host == other.host + && self.clients.overlaps(&other.clients) + } + + fn authority(&self) -> String { + self.host.authority(self.port) + } + + fn network_description(&self) -> String { + match self.network { + Some(network) => format!("client_cidr {network}"), + None => "every client network (catch-all)".to_owned(), + } + } +} + +/// The clients an advertised entry actually wins under its node's +/// longest-prefix match, built by [`ClusterConfig::validate`] for the +/// cross-node conflict scan. +#[derive(Clone)] +struct EffectiveClients { + /// Sorted disjoint ranges of winning client addresses. + ranges: Vec, + /// The catch-all also wins clients whose peer address the transport + /// could not produce ([`ResolvedClusterNode::advertised_for`] with no + /// client IP), so two catch-all overlap even when selectors cover both + /// address families. + serves_unknown_peers: bool, +} + +impl EffectiveClients { + /// A selector wins its network minus the sibling networks nested inside + /// it (longer prefixes take the node's LPM). `sibling_ranges` must + /// exclude the selector's own network. + fn for_selector(network: &IpNet, sibling_ranges: &[ClientAddressRange]) -> Self { + Self { + ranges: ClientAddressRange::from(network).subtract_nested(sibling_ranges), + serves_unknown_peers: false, + } + } + + /// The catch-all wins every client no selector matches, in both address + /// families, plus unknown-peer clients. + fn for_catch_all(selector_ranges: &[ClientAddressRange]) -> Self { + let mut ranges = ClientAddressRange::FULL_IPV4.subtract_nested(selector_ranges); + ranges.extend(ClientAddressRange::FULL_IPV6.subtract_nested(selector_ranges)); + Self { + ranges, + serves_unknown_peers: true, + } + } + + fn overlaps(&self, other: &Self) -> bool { + if self.serves_unknown_peers && other.serves_unknown_peers { + return true; + } + self.ranges.iter().any(|range| { + other.ranges.iter().any(|other_range| { + range.is_ipv4 == other_range.is_ipv4 + && range.first <= other_range.last + && other_range.first <= range.last + }) + }) + } +} + +/// Inclusive range of client addresses within one family. Client IPs +/// canonicalize to v4 before matching, so v4 and v6 networks match disjoint +/// client populations and a range never spans families. +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +struct ClientAddressRange { + is_ipv4: bool, + first: u128, + last: u128, +} + +impl ClientAddressRange { + const FULL_IPV4: Self = Self { + is_ipv4: true, + first: 0, + last: u32::MAX as u128, + }; + const FULL_IPV6: Self = Self { + is_ipv4: false, + first: 0, + last: u128::MAX, + }; + + /// `self` minus every range nested inside it, as sorted disjoint + /// leftovers. CIDR networks are nested or disjoint, never partially + /// overlapping, so a range outside `self` is either disjoint from it + /// (subtracts nothing) or contains it (a shorter prefix, which loses + /// LPM and also subtracts nothing). + fn subtract_nested(self, ranges: &[Self]) -> Vec { + let mut nested: Vec = ranges + .iter() + .filter(|range| { + range.is_ipv4 == self.is_ipv4 + && range.first >= self.first + && range.last <= self.last + }) + .copied() + .collect(); + nested.sort_unstable(); + let mut remaining = Vec::new(); + let mut cursor = Some(self.first); + for nested_range in nested { + let Some(next_free) = cursor else { break }; + if nested_range.first > next_free { + remaining.push(Self { + is_ipv4: self.is_ipv4, + first: next_free, + last: nested_range.first - 1, + }); + } + cursor = nested_range + .last + .checked_add(1) + .map(|after| after.max(next_free)); + } + if let Some(next_free) = cursor + && next_free <= self.last + { + remaining.push(Self { + is_ipv4: self.is_ipv4, + first: next_free, + last: self.last, + }); + } + remaining + } +} + +impl From<&IpNet> for ClientAddressRange { + fn from(network: &IpNet) -> Self { + match network { + IpNet::V4(network) => Self { + is_ipv4: true, + first: u128::from(u32::from(network.network())), + last: u128::from(u32::from(network.broadcast())), + }, + IpNet::V6(network) => Self { + is_ipv4: false, + first: u128::from(network.network()), + last: u128::from(network.broadcast()), + }, + } + } +} + +fn insert_advertised_endpoint<'roster>( + advertised_endpoints: &mut Vec>, + endpoint: AdvertisedEndpoint<'roster>, +) -> Result<(), ConfigurationError> { + if let Some(existing) = advertised_endpoints + .iter() + .find(|existing| existing.conflicts_with(&endpoint)) + { + eprintln!( + "Invalid cluster configuration: advertised client endpoint conflict - {} is advertised for {} (node '{}', transport {}) and for {} (node '{}', transport {}); their effective client sets overlap after longest-prefix shadowing, so a client in the overlap would resolve both nodes to one endpoint", + endpoint.authority(), + endpoint.network_description(), + endpoint.node_name, + endpoint.transport, + existing.network_description(), + existing.node_name, + existing.transport, + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + advertised_endpoints.push(endpoint); + Ok(()) +} + #[cfg(test)] mod tests { use super::*; @@ -145,10 +1392,21 @@ mod tests { let config = ClusterConfig { enabled: true, name: "iggy-cluster".to_owned(), + heartbeat_timeout: default_heartbeat_timeout(), + commit_broadcast_interval: default_commit_broadcast_interval(), + prepare_retransmit_interval: default_prepare_retransmit_interval(), + view_change_retransmit_interval: default_view_change_retransmit_interval(), + view_change_status_timeout: default_view_change_status_timeout(), + request_start_view_retransmit_interval: default_request_start_view_retransmit_interval( + ), + view_probe_attempts_max: default_view_probe_attempts_max(), + repair_retry_interval: default_repair_retry_interval(), + repair_chunk_max: default_repair_chunk_max(), nodes: Vec::new(), auth: ClusterAuthConfig { enabled: true, shared_secret: "current-psk-MUST-NOT-be-persisted".to_owned(), + previous_shared_secret: "retiring-psk-MUST-NOT-be-persisted".to_owned(), }, tls: ClusterTlsConfig::default(), }; @@ -162,4 +1420,1184 @@ mod tests { "shared_secret field present in serialized config: {serialized}" ); } + + #[test] + fn cluster_node_rejects_unknown_fields() { + let error = serde_json::from_str::( + r#"{ + "name": "node-0", + "ip": "10.0.0.1", + "advertise_address": "203.0.113.1", + "replica_id": 0, + "ports": {} + }"#, + ) + .expect_err("misspelled advertised_address must be rejected"); + + assert!( + error + .to_string() + .contains("unknown field `advertise_address`"), + "unexpected deserialization error: {error}" + ); + } + + #[test] + fn advertised_addresses_env_expansion_is_capped() { + // The selectors Vec nests inside the nodes Vec, so the derive's + // index ceilings multiply; without the field's max_elements cap the + // default of 256x256 adds ~131k leaked mappings to every boot. + let mappings = ::env_mappings(); + assert!( + mappings + .iter() + .any(|mapping| mapping.env_name.contains("ADVERTISED_ADDRESSES_15_")), + "selector index 15 must stay reachable by env override" + ); + assert!( + !mappings + .iter() + .any(|mapping| mapping.env_name.contains("ADVERTISED_ADDRESSES_16_")), + "selector env expansion must stop at max_elements = 16" + ); + } +} + +#[cfg(test)] +mod advertised_address_tests { + use super::*; + + #[test] + fn parses_ip_literals_to_canonical_form() { + assert_eq!( + "203.0.113.1".parse::(), + Ok(AdvertisedAddress::Ip("203.0.113.1".parse().unwrap())) + ); + for equivalent_address in ["2001:DB8::1", "2001:db8:0:0:0:0:0:1", "[2001:db8::1]"] { + assert_eq!( + equivalent_address.parse::(), + Ok(AdvertisedAddress::Ip("2001:db8::1".parse().unwrap())), + "'{equivalent_address}' must parse to canonical 2001:db8::1" + ); + } + } + + #[test] + fn normalizes_hostname_to_lowercase() { + let address = "Broker-1.Example.COM".parse::(); + assert_eq!( + address, + Ok(AdvertisedAddress::Hostname( + "broker-1.example.com".to_owned() + )) + ); + } + + #[test] + fn authority_brackets_ipv6_hosts_only() { + let cases = [ + ("203.0.113.1", "203.0.113.1:8090"), + ("2001:db8::1", "[2001:db8::1]:8090"), + ("broker-1.example.com", "broker-1.example.com:8090"), + ]; + for (host, expected_authority) in cases { + let address = host.parse::().expect("valid address"); + assert_eq!(address.authority(8090), expected_authority); + } + } + + #[test] + fn rejects_port_suffixes() { + for address_with_port in ["example.com:8090", "10.0.0.1:8090", "[2001:db8::1]:8090"] { + assert_eq!( + address_with_port.parse::(), + Err(AdvertisedAddressError::PortNotAllowed), + "'{address_with_port}' must be rejected as host:port" + ); + } + } + + #[test] + fn rejects_dotted_numeric_strings_as_malformed_ipv4() { + for malformed_ip in ["10.0.0.256", "192.168.1", "12345"] { + assert_eq!( + malformed_ip.parse::(), + Err(AdvertisedAddressError::MalformedIpv4), + "'{malformed_ip}' must not pass as a hostname" + ); + } + } + + #[test] + fn rejects_broken_ipv6_literals() { + for broken_ipv6 in ["2001:db8:::1", "[2001:db8::zz]", "::1::2"] { + assert_eq!( + broken_ipv6.parse::(), + Err(AdvertisedAddressError::MalformedIpv6), + "'{broken_ipv6}' must be rejected as malformed IPv6" + ); + } + } +} + +#[cfg(test)] +mod advertised_for_tests { + use super::*; + + fn node_with_selectors(selectors: Vec) -> ClusterNodeConfig { + ClusterNodeConfig { + name: "node-0".to_owned(), + ip: "10.0.1.5".to_owned(), + advertised_address: Some("203.0.113.10".to_owned()), + advertised_addresses: selectors, + replica_id: 0, + ports: TransportPorts::default(), + } + } + + fn selector(client_cidr: &str, address: &str) -> AdvertisedAddressSelector { + AdvertisedAddressSelector { + client_cidr: client_cidr.to_owned(), + address: address.to_owned(), + } + } + + fn resolved(node: ClusterNodeConfig) -> ResolvedClusterNode { + node.into() + } + + fn ip(address: &str) -> IpAddr { + address.parse().unwrap() + } + + #[test] + fn falls_back_to_advertised_address_without_selectors() { + let node = node_with_selectors(Vec::new()); + assert_eq!( + resolved(node).advertised_for(Some(ip("10.0.0.7"))), + Some(&AdvertisedAddress::Ip(ip("203.0.113.10"))) + ); + } + + #[test] + fn falls_back_to_roster_ip_without_advertised_address() { + let mut node = node_with_selectors(Vec::new()); + node.advertised_address = None; + assert_eq!( + resolved(node).advertised_for(Some(ip("10.0.0.7"))), + Some(&AdvertisedAddress::Ip(ip("10.0.1.5"))) + ); + } + + #[test] + fn is_none_when_no_fallback_parses() { + let mut node = node_with_selectors(Vec::new()); + node.advertised_address = None; + node.ip = "iggy_node".to_owned(); + assert_eq!(resolved(node).advertised_for(Some(ip("10.0.0.7"))), None); + } + + #[test] + fn matching_selector_beats_advertised_address() { + let node = node_with_selectors(vec![selector("10.0.0.0/16", "10.0.1.5")]); + assert_eq!( + resolved(node).advertised_for(Some(ip("10.0.200.7"))), + Some(&AdvertisedAddress::Ip(ip("10.0.1.5"))) + ); + } + + #[test] + fn unmatched_client_falls_back_to_advertised_address() { + let node = node_with_selectors(vec![selector("10.0.0.0/16", "10.0.1.5")]); + assert_eq!( + resolved(node).advertised_for(Some(ip("192.168.0.7"))), + Some(&AdvertisedAddress::Ip(ip("203.0.113.10"))) + ); + } + + #[test] + fn unknown_client_ip_falls_back_to_advertised_address() { + let node = node_with_selectors(vec![selector("10.0.0.0/16", "10.0.1.5")]); + assert_eq!( + resolved(node).advertised_for(None), + Some(&AdvertisedAddress::Ip(ip("203.0.113.10"))) + ); + } + + #[test] + fn longest_prefix_wins_regardless_of_declaration_order() { + let node = resolved(node_with_selectors(vec![ + selector("10.0.0.0/8", "10.255.255.1"), + selector("10.0.0.0/16", "10.0.1.5"), + ])); + assert_eq!( + node.advertised_for(Some(ip("10.0.200.7"))), + Some(&AdvertisedAddress::Ip(ip("10.0.1.5"))), + "the /16 must win over the /8 even though it is declared second" + ); + assert_eq!( + node.advertised_for(Some(ip("10.9.0.7"))), + Some(&AdvertisedAddress::Ip(ip("10.255.255.1"))), + "a client outside the /16 but inside the /8 must match the /8" + ); + } + + #[test] + fn equal_prefix_matches_resolve_deterministically_to_first_declared() { + // No validated config reaches this state: these networks truncate to + // one /16, which validation rejects as a duplicate. Pinned anyway so + // a future relaxation of that rule cannot make resolution + // order-dependent. + let node = node_with_selectors(vec![ + selector("10.0.1.0/16", "10.0.1.5"), + selector("10.0.2.0/16", "10.0.2.5"), + ]); + assert_eq!( + resolved(node).advertised_for(Some(ip("10.0.200.7"))), + Some(&AdvertisedAddress::Ip(ip("10.0.1.5"))) + ); + } + + #[test] + fn v4_mapped_v6_client_matches_v4_cidr() { + // A dual-stack listener reports v4 peers as `::ffff:a.b.c.d`. + let node = node_with_selectors(vec![selector("10.0.0.0/16", "10.0.1.5")]); + assert_eq!( + resolved(node).advertised_for(Some(ip("::ffff:10.0.0.7"))), + Some(&AdvertisedAddress::Ip(ip("10.0.1.5"))) + ); + } + + #[test] + fn v4_mapped_v6_selector_cidr_matches_v4_client() { + // The mirror case: the CIDR side canonicalizes at build, so + // `::ffff:10.0.0.0/104` matches like `10.0.0.0/8` instead of being + // a silently dead selector. + let node = node_with_selectors(vec![selector("::ffff:10.0.0.0/104", "10.0.1.5")]); + assert_eq!( + resolved(node).advertised_for(Some(ip("10.0.0.7"))), + Some(&AdvertisedAddress::Ip(ip("10.0.1.5"))) + ); + } + + #[test] + fn v6_selector_matches_v6_client() { + let node = node_with_selectors(vec![selector("2001:db8::/32", "2001:db8::1")]); + assert_eq!( + resolved(node).advertised_for(Some(ip("2001:db8::7"))), + Some(&AdvertisedAddress::Ip(ip("2001:db8::1"))) + ); + } + + #[test] + fn selector_address_may_be_a_hostname() { + let node = node_with_selectors(vec![selector("10.0.0.0/16", "Broker.Internal.Example")]); + assert_eq!( + resolved(node).advertised_for(Some(ip("10.0.0.7"))), + Some(&AdvertisedAddress::Hostname( + "broker.internal.example".to_owned() + )) + ); + } +} + +#[cfg(test)] +mod cluster_validate_tests { + use super::*; + + fn node(name: &str, id: u8) -> ClusterNodeConfig { + ClusterNodeConfig { + name: name.to_string(), + ip: "127.0.0.1".to_string(), + advertised_address: None, + advertised_addresses: Vec::new(), + replica_id: id, + ports: TransportPorts::default(), + } + } + + fn selector(client_cidr: &str, address: &str) -> AdvertisedAddressSelector { + AdvertisedAddressSelector { + client_cidr: client_cidr.to_owned(), + address: address.to_owned(), + } + } + + fn cfg(nodes: Vec) -> ClusterConfig { + ClusterConfig { + enabled: true, + name: "iggy-cluster".to_string(), + heartbeat_timeout: default_heartbeat_timeout(), + commit_broadcast_interval: default_commit_broadcast_interval(), + prepare_retransmit_interval: default_prepare_retransmit_interval(), + view_change_retransmit_interval: default_view_change_retransmit_interval(), + view_change_status_timeout: default_view_change_status_timeout(), + request_start_view_retransmit_interval: default_request_start_view_retransmit_interval( + ), + view_probe_attempts_max: default_view_probe_attempts_max(), + repair_retry_interval: default_repair_retry_interval(), + repair_chunk_max: default_repair_chunk_max(), + nodes, + auth: ClusterAuthConfig::default(), + tls: ClusterTlsConfig::default(), + } + } + + #[test] + fn validate_rejects_sub_minimum_heartbeat_timeout() { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.heartbeat_timeout = IggyDuration::new(Duration::from_millis(500)); + assert!(c.validate().is_err()); + // The "disabled" / "unlimited" sentinels collapse to zero and must + // be rejected the same way. + c.heartbeat_timeout = IggyDuration::new(Duration::ZERO); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_rejects_zero_commit_broadcast_interval() { + // `0` / `disabled` / `unlimited` all collapse to zero and stall the + // liveness broadcast. + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.commit_broadcast_interval = IggyDuration::new(Duration::ZERO); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_rejects_zero_prepare_retransmit_interval() { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.prepare_retransmit_interval = IggyDuration::new(Duration::ZERO); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_rejects_heartbeat_below_commit_broadcast_ratio() { + // 3s clears the absolute 2s floor but is still < 4x the 1s broadcast, + // so the ratio rule is what rejects here, not the floor. + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.heartbeat_timeout = IggyDuration::new(Duration::from_secs(3)); + c.commit_broadcast_interval = IggyDuration::new(Duration::from_secs(1)); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_accepts_heartbeat_at_commit_broadcast_ratio() { + // Exactly 4x the broadcast (and above the 2s floor) must pass. + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.heartbeat_timeout = IggyDuration::new(Duration::from_secs(4)); + c.commit_broadcast_interval = IggyDuration::new(Duration::from_secs(1)); + assert!(c.validate().is_ok()); + } + + #[test] + fn validate_rejects_zero_view_change_retransmit_interval() { + // `0` / `disabled` / `unlimited` all collapse to zero and stall the + // view-change retransmit timers. + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.view_change_retransmit_interval = IggyDuration::new(Duration::ZERO); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_rejects_zero_view_change_status_timeout() { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.view_change_status_timeout = IggyDuration::new(Duration::ZERO); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_rejects_zero_request_start_view_retransmit_interval() { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.request_start_view_retransmit_interval = IggyDuration::new(Duration::ZERO); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_rejects_view_change_status_below_retransmit_ratio() { + // 3s is nonzero but still < 4x the 1s retransmit, so the ratio rule is + // what rejects here, not the zero check. + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.view_change_retransmit_interval = IggyDuration::new(Duration::from_secs(1)); + c.view_change_status_timeout = IggyDuration::new(Duration::from_secs(3)); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_accepts_view_change_status_at_retransmit_ratio() { + // Exactly 4x the retransmit interval must pass. + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.view_change_retransmit_interval = IggyDuration::new(Duration::from_secs(1)); + c.view_change_status_timeout = IggyDuration::new(Duration::from_secs(4)); + assert!(c.validate().is_ok()); + } + + #[test] + fn validate_rejects_zero_view_probe_attempts_max() { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.view_probe_attempts_max = 0; + assert!(c.validate().is_err()); + } + + #[test] + fn validate_rejects_view_probe_attempts_above_ceiling() { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.view_probe_attempts_max = MAX_VIEW_PROBE_ATTEMPTS + 1; + assert!(c.validate().is_err()); + } + + #[test] + fn validate_accepts_view_probe_attempts_at_ceiling() { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.view_probe_attempts_max = MAX_VIEW_PROBE_ATTEMPTS; + assert!(c.validate().is_ok()); + } + + #[test] + fn validate_rejects_zero_repair_retry_interval() { + // `0` / `disabled` / `unlimited` all collapse to zero and would wedge + // stalled repair streams. + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.repair_retry_interval = IggyDuration::new(Duration::ZERO); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_rejects_zero_repair_chunk_max() { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.repair_chunk_max = 0; + assert!(c.validate().is_err()); + } + + #[test] + fn validate_rejects_repair_chunk_max_above_ceiling() { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.repair_chunk_max = MAX_REPAIR_CHUNK_MAX + 1; + assert!(c.validate().is_err()); + } + + #[test] + fn validate_accepts_repair_chunk_max_at_ceiling() { + // Section-level validate only; the cross-section rule against + // message_bus.peer_queue_capacity lives in the top-level validate. + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.repair_chunk_max = MAX_REPAIR_CHUNK_MAX; + assert!(c.validate().is_ok()); + } + + #[test] + fn validate_rejects_empty_nodes() { + let c = cfg(vec![]); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_rejects_duplicate_replica_ids() { + let c = cfg(vec![node("n1", 0), node("n2", 0)]); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_rejects_duplicate_names() { + let c = cfg(vec![node("n1", 0), node("n1", 1)]); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_rejects_out_of_range_replica_id() { + // 2 nodes total, so id 2 is out of range. + let c = cfg(vec![node("n1", 0), node("n2", 2)]); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_accepts_unique_contiguous_replica_ids() { + let c = cfg(vec![node("n1", 0), node("n2", 1), node("n3", 2)]); + assert!(c.validate().is_ok()); + } + + #[test] + fn validate_skips_checks_when_disabled() { + let mut c = cfg(vec![]); + c.enabled = false; + assert!(c.validate().is_ok()); + } + + // repair_chunk_max is also read by the unconditional top-level check + // against message_bus.peer_queue_capacity, so its own bounds apply with + // the cluster off too. + #[test] + fn validate_rejects_zero_repair_chunk_max_when_disabled() { + let mut c = cfg(vec![]); + c.enabled = false; + c.repair_chunk_max = 0; + assert!(c.validate().is_err()); + } + + #[test] + fn validate_rejects_repair_chunk_max_above_ceiling_when_disabled() { + let mut c = cfg(vec![]); + c.enabled = false; + c.repair_chunk_max = MAX_REPAIR_CHUNK_MAX + 1; + assert!(c.validate().is_err()); + } + + #[test] + fn validate_rejects_duplicate_tcp_replica_port() { + let ports = TransportPorts { + tcp: None, + quic: None, + http: None, + websocket: None, + tcp_replica: Some(9090), + }; + let mut n1 = node("n1", 0); + n1.ports = ports.clone(); + let mut n2 = node("n2", 1); + n2.ports = ports; + let c = cfg(vec![n1, n2]); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_rejects_cross_transport_port_reuse() { + let mut n1 = node("n1", 0); + n1.ports = TransportPorts { + tcp: Some(8090), + quic: None, + http: Some(8090), + websocket: None, + tcp_replica: None, + }; + let c = cfg(vec![n1]); + assert!( + c.validate().is_err(), + "same port on TCP and HTTP of the same node must be rejected" + ); + } + + #[test] + fn validate_accepts_same_port_on_different_ips() { + let mut n1 = node("n1", 0); + n1.ip = "127.0.0.1".to_string(); + n1.ports = TransportPorts { + tcp: Some(8090), + quic: None, + http: None, + websocket: None, + tcp_replica: None, + }; + let mut n2 = node("n2", 1); + n2.ip = "127.0.0.2".to_string(); + n2.ports = TransportPorts { + tcp: Some(8090), + quic: None, + http: None, + websocket: None, + tcp_replica: None, + }; + let c = cfg(vec![n1, n2]); + assert!(c.validate().is_ok()); + } + + #[test] + fn validate_rejects_duplicate_advertised_client_endpoint() { + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_address = Some("203.0.113.1".to_owned()); + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "10.0.0.2".to_owned(); + n2.advertised_address = n1.advertised_address.clone(); + n2.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, n2]).validate().is_err()); + } + + #[test] + fn validate_rejects_equivalent_ipv6_advertised_client_endpoints() { + for equivalent_address in ["2001:DB8::1", "2001:db8:0:0:0:0:0:1", "[2001:db8::1]"] { + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_address = Some("2001:db8::1".to_owned()); + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "10.0.0.2".to_owned(); + n2.advertised_address = Some(equivalent_address.to_owned()); + n2.ports.tcp = Some(8090); + + assert!( + cfg(vec![n1, n2]).validate().is_err(), + "{equivalent_address} must conflict with 2001:db8::1" + ); + } + } + + #[test] + fn validate_rejects_equivalent_ipv6_client_endpoints_from_node_ip() { + let mut n1 = node("n1", 0); + n1.ip = "2001:db8::1".to_owned(); + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "2001:db8:0:0:0:0:0:1".to_owned(); + n2.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, n2]).validate().is_err()); + } + + #[test] + fn validate_accepts_distinct_advertised_client_endpoints() { + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_address = Some("203.0.113.1".to_owned()); + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "10.0.0.2".to_owned(); + n2.advertised_address = Some("203.0.113.2".to_owned()); + n2.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, n2]).validate().is_ok()); + } + + #[test] + fn validate_accepts_hostname_advertised_address() { + let mut n1 = node("n1", 0); + n1.advertised_address = Some("iggy-node-1.example.com".to_owned()); + n1.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, node("n2", 1)]).validate().is_ok()); + } + + #[test] + fn validate_rejects_malformed_advertised_addresses() { + let oversized_label = format!("{}.example.com", "a".repeat(64)); + let oversized_hostname = format!("{}example.com", "a.".repeat(130)); + for advertised_address in [ + "", + " 203.0.113.1", + "10.0.0.256", + "192.168.1", + "example.com:8090", + "[2001:db8::1]:8090", + "2001:db8:::1", + "iggy_node.example.com", + "-node.example.com", + "node-.example.com", + ".example.com", + "example..com", + "example.com.", + "ex\u{e4}mple.com", + oversized_label.as_str(), + oversized_hostname.as_str(), + ] { + let mut n1 = node("n1", 0); + n1.advertised_address = Some(advertised_address.to_owned()); + + assert!( + cfg(vec![n1, node("n2", 1)]).validate().is_err(), + "'{advertised_address}' must be rejected" + ); + } + } + + #[test] + fn validate_rejects_case_variant_hostname_advertised_endpoints() { + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_address = Some("broker.example.com".to_owned()); + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "10.0.0.2".to_owned(); + n2.advertised_address = Some("Broker.Example.COM".to_owned()); + n2.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, n2]).validate().is_err()); + } + + #[test] + fn validate_rejects_node_ip_hostname_clashing_with_advertised_hostname() { + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_address = Some("broker.example.com".to_owned()); + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "broker.example.com".to_owned(); + n2.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, n2]).validate().is_err()); + } + + #[test] + fn validate_accepts_distinct_hostname_advertised_endpoints() { + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_address = Some("broker-1.example.com".to_owned()); + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "10.0.0.2".to_owned(); + n2.advertised_address = Some("broker-2.example.com".to_owned()); + n2.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, n2]).validate().is_ok()); + } + + #[test] + fn validate_accepts_selectors_with_distinct_cidrs() { + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_address = Some("203.0.113.1".to_owned()); + n1.advertised_addresses = vec![ + selector("10.0.0.0/16", "10.0.0.1"), + selector("10.0.0.0/8", "broker-1.internal.example"), + ]; + n1.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, node("n2", 1)]).validate().is_ok()); + } + + #[test] + fn validate_rejects_malformed_selector_cidr() { + for client_cidr in ["10.0.0.0", "10.0.0.0/33", "not-a-cidr", ""] { + let mut n1 = node("n1", 0); + n1.advertised_addresses = vec![selector(client_cidr, "10.0.0.1")]; + + assert!( + cfg(vec![n1, node("n2", 1)]).validate().is_err(), + "client_cidr '{client_cidr}' must be rejected" + ); + } + } + + #[test] + fn validate_rejects_malformed_selector_address() { + for address in ["", "10.0.0.1:8090", "10.0.0.256", "iggy_node"] { + let mut n1 = node("n1", 0); + n1.advertised_addresses = vec![selector("10.0.0.0/16", address)]; + + assert!( + cfg(vec![n1, node("n2", 1)]).validate().is_err(), + "selector address '{address}' must be rejected" + ); + } + } + + #[test] + fn validate_rejects_duplicate_selector_cidr_within_a_node() { + // `10.0.1.0/16` truncates to `10.0.0.0/16`: the two selectors match + // the identical client set, so the second could never win LPM. + let mut n1 = node("n1", 0); + n1.advertised_addresses = vec![ + selector("10.0.0.0/16", "10.0.0.1"), + selector("10.0.1.0/16", "10.0.0.2"), + ]; + + assert!(cfg(vec![n1, node("n2", 1)]).validate().is_err()); + } + + #[test] + fn validate_accepts_selector_count_at_the_cap() { + let mut n1 = node("n1", 0); + n1.advertised_addresses = (0..MAX_ADVERTISED_SELECTORS) + .map(|index| selector(&format!("10.{index}.0.0/16"), &format!("192.0.2.{index}"))) + .collect(); + + assert!(cfg(vec![n1, node("n2", 1)]).validate().is_ok()); + } + + #[test] + fn validate_rejects_selector_count_above_the_cap() { + // The env-override path stops expanding selector indices at the same + // ceiling, so a TOML roster exceeding it could never be replicated + // byte-identically through env vars. + let mut n1 = node("n1", 0); + n1.advertised_addresses = (0..=MAX_ADVERTISED_SELECTORS) + .map(|index| selector(&format!("10.{index}.0.0/16"), &format!("192.0.2.{index}"))) + .collect(); + + assert!(cfg(vec![n1, node("n2", 1)]).validate().is_err()); + } + + #[test] + fn validate_rejects_selector_endpoint_conflict_within_one_cidr() { + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_addresses = vec![selector("10.0.0.0/16", "10.0.7.7")]; + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "10.0.0.2".to_owned(); + n2.advertised_addresses = vec![selector("10.0.0.0/16", "10.0.7.7")]; + n2.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, n2]).validate().is_err()); + } + + #[test] + fn validate_accepts_identical_selector_endpoint_across_different_cidrs() { + // Reusing one host:port across DIFFERENT client networks is the + // feature (e.g. each network NATs the address to its local node). + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_addresses = vec![selector("10.1.0.0/16", "192.0.2.10")]; + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "10.0.0.2".to_owned(); + n2.advertised_addresses = vec![selector("10.2.0.0/16", "192.0.2.10")]; + n2.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, n2]).validate().is_ok()); + } + + #[test] + fn validate_rejects_v4_mapped_v6_selector_cidr_duplicating_its_v4_form() { + // `::ffff:10.0.0.0/104` canonicalizes to `10.0.0.0/8` (matching how + // client IPs canonicalize before LPM), so these two selectors match + // the identical client set. + let mut n1 = node("n1", 0); + n1.advertised_addresses = vec![ + selector("10.0.0.0/8", "10.0.0.1"), + selector("::ffff:10.0.0.0/104", "10.0.0.2"), + ]; + + assert!(cfg(vec![n1, node("n2", 1)]).validate().is_err()); + } + + #[test] + fn validate_rejects_selector_endpoint_clashing_with_another_nodes_catch_all() { + // The catch-all matches every client, so a 10.0.0.0/16 client would + // resolve both nodes to 192.0.2.10:8090. + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_addresses = vec![selector("10.0.0.0/16", "192.0.2.10")]; + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "10.0.0.2".to_owned(); + n2.advertised_address = Some("192.0.2.10".to_owned()); + n2.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, n2]).validate().is_err()); + } + + #[test] + fn validate_rejects_selector_endpoint_clashing_with_another_nodes_roster_ip() { + // Without an advertised_address the roster ip backs the catch-all, + // so the same cross-set conflict applies to it. + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_addresses = vec![selector("10.0.0.0/16", "10.0.0.2")]; + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "10.0.0.2".to_owned(); + n2.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, n2]).validate().is_err()); + } + + #[test] + fn validate_rejects_identical_selector_endpoint_across_nested_cidrs() { + // LPM runs per node, not cluster-wide: n1 has no longer prefix of + // its own shadowing the /16 overlap, so a 10.0.0.0/16 client wins + // n1's /8 and n2's /16, resolving both to 192.0.2.10:8090. + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_addresses = vec![selector("10.0.0.0/8", "192.0.2.10")]; + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "10.0.0.2".to_owned(); + n2.advertised_addresses = vec![selector("10.0.0.0/16", "192.0.2.10")]; + n2.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, n2]).validate().is_err()); + } + + #[test] + fn validate_accepts_nested_cidr_reuse_shadowed_by_same_node_longer_prefix() { + // n1's /16 selector shadows its /8 within 10.0.0.0/16, so n1's /8 + // entry wins only 10.0.0.0/8 minus 10.0.0.0/16 - disjoint from n2's + // /16. No client resolves both nodes to 192.0.2.10:8090. + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_addresses = vec![ + selector("10.0.0.0/8", "192.0.2.10"), + selector("10.0.0.0/16", "192.0.2.20"), + ]; + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "10.0.0.2".to_owned(); + n2.advertised_addresses = vec![selector("10.0.0.0/16", "192.0.2.10")]; + n2.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, n2]).validate().is_ok()); + } + + #[test] + fn validate_rejects_partially_shadowed_nested_cidr_reuse() { + // n1's /24 shadow carves only part of the /16 overlap: a client in + // 10.0.0.0/16 outside 10.0.0.0/24 still wins n1's /8 entry and n2's + // /16 entry, resolving both nodes to 192.0.2.10:8090. + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_addresses = vec![ + selector("10.0.0.0/8", "192.0.2.10"), + selector("10.0.0.0/24", "192.0.2.20"), + ]; + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "10.0.0.2".to_owned(); + n2.advertised_addresses = vec![selector("10.0.0.0/16", "192.0.2.10")]; + n2.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, n2]).validate().is_err()); + } + + #[test] + fn validate_accepts_catch_all_reuse_shadowed_by_same_node_selector() { + // n1's /16 selector shadows its catch-all within 10.0.0.0/16, so + // the catch-all never wins a client inside n2's /24. Without the + // shadow the same pair conflicts (see + // validate_rejects_selector_endpoint_clashing_with_another_nodes_catch_all). + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_address = Some("192.0.2.10".to_owned()); + n1.advertised_addresses = vec![selector("10.0.0.0/16", "192.0.2.20")]; + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "10.0.0.2".to_owned(); + n2.advertised_addresses = vec![selector("10.0.0.0/24", "192.0.2.10")]; + n2.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, n2]).validate().is_ok()); + } + + #[test] + fn validate_accepts_selector_reusing_a_fully_shadowed_catch_all_address() { + // n1's selectors cover both address families, so its catch-all wins + // known peers nowhere; only unknown-peer clients reach it, and they + // never match n2's selector. + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_address = Some("192.0.2.10".to_owned()); + n1.advertised_addresses = vec![ + selector("0.0.0.0/0", "192.0.2.20"), + selector("::/0", "192.0.2.30"), + ]; + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "10.0.0.2".to_owned(); + n2.advertised_addresses = vec![selector("10.0.0.0/16", "192.0.2.10")]; + n2.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, n2]).validate().is_ok()); + } + + #[test] + fn validate_accepts_catch_all_spelling_another_nodes_selector_address_when_self_shadowed() { + // Split-network NAT roster: n2's catch-all spells n1's 10/8 selector + // address, but n2's own 10/8 selector shadows its catch-all inside + // 10/8 (outside it n1 serves its own catch-all), so no client + // resolves both nodes to 192.0.2.10:8090. + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_addresses = vec![selector("10.0.0.0/8", "192.0.2.10")]; + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "10.0.0.2".to_owned(); + n2.advertised_address = Some("192.0.2.10".to_owned()); + n2.advertised_addresses = vec![selector("10.0.0.0/8", "192.0.2.20")]; + n2.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, n2]).validate().is_ok()); + } + + #[test] + fn validate_rejects_duplicate_catch_all_even_when_fully_shadowed() { + // A client whose peer address the transport cannot produce always + // falls to the catch-all, so duplicate catch-all conflict even when + // selectors cover every known network. + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_address = Some("192.0.2.10".to_owned()); + n1.advertised_addresses = vec![ + selector("0.0.0.0/0", "192.0.2.20"), + selector("::/0", "192.0.2.30"), + ]; + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "10.0.0.2".to_owned(); + n2.advertised_address = Some("192.0.2.10".to_owned()); + n2.advertised_addresses = vec![ + selector("0.0.0.0/0", "192.0.2.40"), + selector("::/0", "192.0.2.50"), + ]; + n2.ports.tcp = Some(8090); + + assert!(cfg(vec![n1, n2]).validate().is_err()); + } + + #[test] + fn validate_accepts_selector_reusing_its_own_nodes_catch_all_address() { + // Redundant but harmless: within one node the selector and the + // catch-all cannot resolve a client to two different nodes. + let mut n1 = node("n1", 0); + n1.ip = "10.0.0.1".to_owned(); + n1.advertised_address = Some("192.0.2.10".to_owned()); + n1.advertised_addresses = vec![selector("10.0.0.0/16", "192.0.2.10")]; + n1.ports.tcp = Some(8090); + let mut n2 = node("n2", 1); + n2.ip = "10.0.0.2".to_owned(); + n2.ports.tcp = Some(8091); + + assert!(cfg(vec![n1, n2]).validate().is_ok()); + } + + #[test] + fn validate_rejects_zero_tcp_replica_port() { + let ports = TransportPorts { + tcp: None, + quic: None, + http: None, + websocket: None, + tcp_replica: Some(0), + }; + let mut n1 = node("n1", 0); + n1.ports = ports; + let c = cfg(vec![n1]); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_accepts_empty_secret_when_auth_disabled() { + // Default: no secret, auth off -> legacy mode, must pass. + let c = cfg(vec![node("n1", 0), node("n2", 1)]); + assert!(c.validate().is_ok()); + } + + #[test] + fn validate_rejects_missing_secret_when_auth_enabled() { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.auth.enabled = true; + assert!(c.validate().is_err()); + } + + #[test] + fn validate_rejects_short_secret_when_auth_enabled() { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.auth.enabled = true; + c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN - 1); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_rejects_short_secret_even_when_auth_disabled() { + // Typo guard: a configured-but-short key fails even with auth off. + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN - 1); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_accepts_valid_secret_when_auth_enabled() { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.auth.enabled = true; + c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); + assert!(c.validate().is_ok()); + } + + #[test] + fn validate_accepts_valid_rotation_window() { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.auth.enabled = true; + c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); + c.auth.previous_shared_secret = "b".repeat(MIN_SHARED_SECRET_LEN); + assert!(c.validate().is_ok()); + } + + #[test] + fn validate_rejects_short_previous_secret() { + // Same typo guard as the primary key. + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.auth.enabled = true; + c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); + c.auth.previous_shared_secret = "b".repeat(MIN_SHARED_SECRET_LEN - 1); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_rejects_rotation_window_equal_to_primary() { + // An identical window is a no-op rotation: the operator rolled the + // config without changing the key. + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.auth.enabled = true; + c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); + c.auth.previous_shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); + assert!(c.validate().is_err()); + } + + fn tls_files() -> ClusterTlsConfig { + ClusterTlsConfig { + enabled: true, + self_signed: false, + cert_file: "cert.pem".to_string(), + key_file: "key.pem".to_string(), + ca_file: "ca.pem".to_string(), + } + } + + #[test] + fn validate_rejects_tls_ca_mode_with_missing_files() { + // Auth on so the failure exercises the file check, not the auth gate. + for missing in ["cert_file", "key_file", "ca_file"] { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.auth.enabled = true; + c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); + c.tls = tls_files(); + match missing { + "cert_file" => c.tls.cert_file.clear(), + "key_file" => c.tls.key_file.clear(), + _ => c.tls.ca_file.clear(), + } + assert!(c.validate().is_err(), "missing {missing} must be rejected"); + } + } + + #[test] + fn validate_rejects_tls_self_signed_without_auth() { + // Accept-any certificate without the PSK handshake = MITM-able. + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.tls = ClusterTlsConfig { + enabled: true, + self_signed: true, + ..ClusterTlsConfig::default() + }; + assert!(c.validate().is_err()); + } + + #[test] + fn validate_accepts_tls_self_signed_with_auth() { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.auth.enabled = true; + c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); + c.tls = ClusterTlsConfig { + enabled: true, + self_signed: true, + ..ClusterTlsConfig::default() + }; + assert!(c.validate().is_ok()); + } + + #[test] + fn validate_rejects_tls_ca_mode_without_auth() { + // TLS never authenticates the dialer (no client certificates); + // only the PSK handshake does, so it is mandatory with TLS on. + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.tls = tls_files(); + assert!(c.validate().is_err()); + } + + #[test] + fn validate_accepts_tls_ca_mode_with_auth() { + let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); + c.auth.enabled = true; + c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); + c.tls = tls_files(); + assert!(c.validate().is_ok()); + } } diff --git a/core/configs/src/server_config/defaults.rs b/core/configs/src/server_config/defaults.rs index 155239cd37..33422f752d 100644 --- a/core/configs/src/server_config/defaults.rs +++ b/core/configs/src/server_config/defaults.rs @@ -15,65 +15,167 @@ // specific language governing permissions and limitations // under the License. +//! `Default` impls for the sections this module owns (`tcp`, `websocket`, +//! `quic`, `cluster`, `metadata`, `partition`, `message_bus`), sourced +//! from `core/server/config.toml` via [`SERVER_CONFIG`]. Sections drawn +//! from [`crate::common`] (`http`, `system`, `telemetry`, +//! `consumer_group`, `data_maintenance`, `message_saver`, +//! `personal_access_token`, `heartbeat`) delegate to the `Default` impls +//! in [`crate::common::defaults`]. + use super::cluster::{ ClusterAuthConfig, ClusterConfig, ClusterNodeConfig, ClusterTlsConfig, TransportPorts, }; -use super::http::{HttpConfig, HttpCorsConfig, HttpJwtConfig, HttpMetricsConfig, HttpTlsConfig}; +use super::message_bus::MessageBusConfig; +use super::metadata::MetadataConfig; +use super::partition::PartitionConfig; use super::quic::{QuicCertificateConfig, QuicConfig, QuicSocketConfig}; -use super::server::{ - ConsumerGroupConfig, DataMaintenanceConfig, HeartbeatConfig, MemoryPoolConfig, - MessageSaverConfig, MessagesMaintenanceConfig, PersonalAccessTokenCleanerConfig, - PersonalAccessTokenConfig, ServerConfig, TelemetryConfig, TelemetryLogsConfig, - TelemetryTracesConfig, -}; -use super::system::{ - BackupConfig, CompatibilityConfig, CompressionConfig, EncryptionConfig, LoggingConfig, - MessageDeduplicationConfig, PartitionConfig, RecoveryConfig, RuntimeConfig, SegmentConfig, - StateConfig, StreamConfig, SystemConfig, TopicConfig, -}; -use super::tcp::TcpSocketConfig; -use super::tcp::{TcpConfig, TcpTlsConfig}; +use super::server::ServerSystemConfig; +use super::server::{ExtraConfig, ServerConfig}; +use super::tcp::{TcpConfig, TcpSocketConfig, TcpTlsConfig}; use super::websocket::{WebSocketConfig, WebSocketTlsConfig}; -use configs::ConfigEnvMappings; -use iggy_common::IggyByteSize; -use iggy_common::IggyDuration; +use crate::common::http::HttpConfig; +use crate::common::server::{ + ConsumerGroupConfig, DataMaintenanceConfig, HeartbeatConfig, MessageSaverConfig, + PersonalAccessTokenConfig, TelemetryConfig, +}; use std::sync::Arc; -use std::time::Duration; -static_toml::static_toml! { - // static_toml resolves relative to CARGO_MANIFEST_DIR (core/configs/). - pub static SERVER_CONFIG = include_toml!("../server/config.toml"); -} +// Same embedded TOML the shared sections read; re-exported so sibling +// modules reach it as `super::defaults::SERVER_CONFIG`. +pub use crate::common::defaults::SERVER_CONFIG; impl Default for ServerConfig { fn default() -> ServerConfig { ServerConfig { consumer_group: ConsumerGroupConfig::default(), data_maintenance: DataMaintenanceConfig::default(), + extra: ExtraConfig::default(), heartbeat: HeartbeatConfig::default(), message_saver: MessageSaverConfig::default(), personal_access_token: PersonalAccessTokenConfig::default(), - system: Arc::new(SystemConfig::default()), + system: Arc::new(ServerSystemConfig::default()), quic: QuicConfig::default(), tcp: TcpConfig::default(), websocket: WebSocketConfig::default(), http: HttpConfig::default(), telemetry: TelemetryConfig::default(), cluster: ClusterConfig::default(), + metadata: MetadataConfig::default(), + partition: PartitionConfig::default(), + message_bus: MessageBusConfig::default(), } } } -impl Default for MessagesMaintenanceConfig { - fn default() -> MessagesMaintenanceConfig { - MessagesMaintenanceConfig { - cleaner_enabled: SERVER_CONFIG.data_maintenance.messages.cleaner_enabled, - interval: SERVER_CONFIG - .data_maintenance - .messages - .interval +impl Default for ClusterConfig { + fn default() -> ClusterConfig { + ClusterConfig { + enabled: SERVER_CONFIG.cluster.enabled, + name: SERVER_CONFIG.cluster.name.parse().unwrap(), + heartbeat_timeout: SERVER_CONFIG.cluster.heartbeat_timeout.parse().unwrap(), + commit_broadcast_interval: SERVER_CONFIG + .cluster + .commit_broadcast_interval + .parse() + .unwrap(), + prepare_retransmit_interval: SERVER_CONFIG + .cluster + .prepare_retransmit_interval + .parse() + .unwrap(), + view_change_retransmit_interval: SERVER_CONFIG + .cluster + .view_change_retransmit_interval .parse() .unwrap(), + view_change_status_timeout: SERVER_CONFIG + .cluster + .view_change_status_timeout + .parse() + .unwrap(), + request_start_view_retransmit_interval: SERVER_CONFIG + .cluster + .request_start_view_retransmit_interval + .parse() + .unwrap(), + view_probe_attempts_max: SERVER_CONFIG.cluster.view_probe_attempts_max as u32, + repair_retry_interval: SERVER_CONFIG + .cluster + .repair_retry_interval + .parse() + .unwrap(), + repair_chunk_max: SERVER_CONFIG.cluster.repair_chunk_max as usize, + nodes: SERVER_CONFIG + .cluster + .nodes + .iter() + .map(|node| ClusterNodeConfig { + name: node.name.parse().unwrap(), + ip: node.ip.parse().unwrap(), + advertised_address: None, + advertised_addresses: Vec::new(), + replica_id: u8::try_from(node.replica_id).expect( + "static_toml replica_id must fit in u8 (0..=255); \ + fix core/server/config.toml", + ), + ports: TransportPorts { + tcp: Some(u16::try_from(node.ports.tcp).expect( + "static_toml cluster.nodes.ports.tcp must fit in u16 (0..=65535); \ + fix core/server/config.toml", + )), + quic: Some(u16::try_from(node.ports.quic).expect( + "static_toml cluster.nodes.ports.quic must fit in u16 (0..=65535); \ + fix core/server/config.toml", + )), + http: Some(u16::try_from(node.ports.http).expect( + "static_toml cluster.nodes.ports.http must fit in u16 (0..=65535); \ + fix core/server/config.toml", + )), + websocket: Some(u16::try_from(node.ports.websocket).expect( + "static_toml cluster.nodes.ports.websocket must fit in u16 (0..=65535); \ + fix core/server/config.toml", + )), + tcp_replica: Some(u16::try_from(node.ports.tcp_replica).expect( + "static_toml cluster.nodes.ports.tcp_replica must fit in u16 (0..=65535); \ + fix core/server/config.toml", + )), + }, + }) + .collect(), + auth: ClusterAuthConfig::default(), + tls: ClusterTlsConfig::default(), + } + } +} + +impl Default for MetadataConfig { + fn default() -> MetadataConfig { + // Read from the embedded TOML so the Default impl and the on-disk + // schema cannot drift (same pattern as MessageBusConfig below). + let metadata = &SERVER_CONFIG.metadata; + MetadataConfig { + prepare_queue_depth: metadata.prepare_queue_depth as usize, + journal_slots: metadata.journal_slots as usize, + clients_table_max: metadata.clients_table_max as usize, + } + } +} + +impl Default for PartitionConfig { + fn default() -> PartitionConfig { + // Read from the embedded TOML so the Default impl and the on-disk + // schema cannot drift (same pattern as MetadataConfig above). + let partition = &SERVER_CONFIG.partition; + PartitionConfig { + prepare_queue_depth: partition.prepare_queue_depth as usize, + evicted_ring_capacity: partition.evicted_ring_capacity as usize, + evicted_ring_bytes_max: partition.evicted_ring_bytes_max.parse().unwrap(), + transfer_served_cache_bytes_max: partition + .transfer_served_cache_bytes_max + .parse() + .unwrap(), + transfer_artifact_bytes_max: partition.transfer_artifact_bytes_max.parse().unwrap(), } } } @@ -148,18 +250,22 @@ impl Default for TcpTlsConfig { impl Default for TcpSocketConfig { fn default() -> TcpSocketConfig { TcpSocketConfig { - override_defaults: false, - recv_buffer_size: IggyByteSize::from(100_000_u64), - send_buffer_size: IggyByteSize::from(100_000_u64), - keepalive: false, - nodelay: false, - linger: IggyDuration::new(Duration::new(0, 0)), + override_defaults: SERVER_CONFIG.tcp.socket.override_defaults, + recv_buffer_size: SERVER_CONFIG.tcp.socket.recv_buffer_size.parse().unwrap(), + send_buffer_size: SERVER_CONFIG.tcp.socket.send_buffer_size.parse().unwrap(), + keepalive: SERVER_CONFIG.tcp.socket.keepalive, + nodelay: SERVER_CONFIG.tcp.socket.nodelay, + linger: SERVER_CONFIG.tcp.socket.linger.parse().unwrap(), } } } impl Default for WebSocketConfig { fn default() -> WebSocketConfig { + // The size knobs are optional in the schema (commented-out by + // default), so they map to `None` here when absent; every other + // field comes from the embedded TOML so the Default impl and + // the on-disk schema cannot drift. WebSocketConfig { enabled: SERVER_CONFIG.websocket.enabled, address: SERVER_CONFIG.websocket.address.parse().unwrap(), @@ -168,7 +274,7 @@ impl Default for WebSocketConfig { max_write_buffer_size: None, max_message_size: None, max_frame_size: None, - accept_unmasked_frames: false, + accept_unmasked_frames: SERVER_CONFIG.websocket.accept_unmasked_frames, tls: WebSocketTlsConfig::default(), } } @@ -185,430 +291,20 @@ impl Default for WebSocketTlsConfig { } } -impl Default for HttpConfig { - fn default() -> HttpConfig { - HttpConfig { - enabled: SERVER_CONFIG.http.enabled, - address: SERVER_CONFIG.http.address.parse().unwrap(), - max_request_size: SERVER_CONFIG.http.max_request_size.parse().unwrap(), - web_ui: SERVER_CONFIG.http.web_ui, - cors: HttpCorsConfig::default(), - jwt: HttpJwtConfig::default(), - metrics: HttpMetricsConfig::default(), - tls: HttpTlsConfig::default(), - } - } -} - -impl Default for HttpCorsConfig { - fn default() -> HttpCorsConfig { - HttpCorsConfig { - enabled: SERVER_CONFIG.http.cors.enabled, - allowed_methods: SERVER_CONFIG - .http - .cors - .allowed_methods - .iter() - .map(|s| s.parse().unwrap()) - .collect(), - allowed_origins: SERVER_CONFIG - .http - .cors - .allowed_origins - .iter() - .map(|s| s.parse().unwrap()) - .collect(), - allowed_headers: SERVER_CONFIG - .http - .cors - .allowed_headers - .iter() - .map(|s| s.parse().unwrap()) - .collect(), - exposed_headers: SERVER_CONFIG - .http - .cors - .exposed_headers - .iter() - .map(|s| s.parse().unwrap()) - .collect(), - allow_credentials: SERVER_CONFIG.http.cors.allow_credentials, - allow_private_network: SERVER_CONFIG.http.cors.allow_private_network, - } - } -} - -impl Default for HttpJwtConfig { - fn default() -> HttpJwtConfig { - HttpJwtConfig { - algorithm: SERVER_CONFIG.http.jwt.algorithm.parse().unwrap(), - issuer: SERVER_CONFIG.http.jwt.issuer.parse().unwrap(), - audience: SERVER_CONFIG.http.jwt.audience.parse().unwrap(), - valid_issuers: SERVER_CONFIG - .http - .jwt - .valid_issuers - .iter() - .map(|s| s.parse().unwrap()) - .collect(), - valid_audiences: SERVER_CONFIG - .http - .jwt - .valid_audiences - .iter() - .map(|s| s.parse().unwrap()) - .collect(), - access_token_expiry: SERVER_CONFIG.http.jwt.access_token_expiry.parse().unwrap(), - clock_skew: SERVER_CONFIG.http.jwt.clock_skew.parse().unwrap(), - not_before: SERVER_CONFIG.http.jwt.not_before.parse().unwrap(), - encoding_secret: SERVER_CONFIG.http.jwt.encoding_secret.parse().unwrap(), - decoding_secret: SERVER_CONFIG.http.jwt.decoding_secret.parse().unwrap(), - use_base64_secret: SERVER_CONFIG.http.jwt.use_base_64_secret, - trusted_issuers: None, - } - } -} - -impl Default for HttpMetricsConfig { - fn default() -> HttpMetricsConfig { - HttpMetricsConfig { - enabled: SERVER_CONFIG.http.metrics.enabled, - endpoint: SERVER_CONFIG.http.metrics.endpoint.parse().unwrap(), - } - } -} - -impl Default for HttpTlsConfig { - fn default() -> HttpTlsConfig { - HttpTlsConfig { - enabled: SERVER_CONFIG.http.tls.enabled, - cert_file: SERVER_CONFIG.http.tls.cert_file.parse().unwrap(), - key_file: SERVER_CONFIG.http.tls.key_file.parse().unwrap(), - } - } -} - -impl Default for MessageSaverConfig { - fn default() -> MessageSaverConfig { - MessageSaverConfig { - enabled: SERVER_CONFIG.message_saver.enabled, - enforce_fsync: SERVER_CONFIG.message_saver.enforce_fsync, - interval: SERVER_CONFIG.message_saver.interval.parse().unwrap(), - } - } -} - -impl Default for PersonalAccessTokenConfig { - fn default() -> PersonalAccessTokenConfig { - PersonalAccessTokenConfig { - max_tokens_per_user: SERVER_CONFIG.personal_access_token.max_tokens_per_user as u32, - cleaner: PersonalAccessTokenCleanerConfig::default(), - } - } -} - -impl Default for PersonalAccessTokenCleanerConfig { - fn default() -> PersonalAccessTokenCleanerConfig { - PersonalAccessTokenCleanerConfig { - enabled: SERVER_CONFIG.personal_access_token.cleaner.enabled, - interval: SERVER_CONFIG - .personal_access_token - .cleaner - .interval - .parse() - .unwrap(), - } - } -} - -impl Default for SystemConfig { - fn default() -> Self { - Self { - path: SERVER_CONFIG.system.path.parse().unwrap(), - backup: BackupConfig::default(), - runtime: RuntimeConfig::default(), - logging: LoggingConfig::default(), - stream: StreamConfig::default(), - encryption: EncryptionConfig::default(), - topic: TopicConfig::default(), - partition: PartitionConfig::default(), - segment: SegmentConfig::default(), - state: StateConfig::default(), - compression: CompressionConfig::default(), - message_deduplication: MessageDeduplicationConfig::default(), - recovery: RecoveryConfig::default(), - memory_pool: MemoryPoolConfig::default(), - sharding: S::default(), - } - } -} - -impl Default for BackupConfig { - fn default() -> BackupConfig { - BackupConfig { - path: SERVER_CONFIG.system.backup.path.parse().unwrap(), - compatibility: CompatibilityConfig::default(), - } - } -} - -impl Default for CompatibilityConfig { - fn default() -> Self { - CompatibilityConfig { - path: SERVER_CONFIG - .system - .backup - .compatibility - .path - .parse() - .unwrap(), - } - } -} - -impl Default for HeartbeatConfig { - fn default() -> HeartbeatConfig { - HeartbeatConfig { - enabled: SERVER_CONFIG.heartbeat.enabled, - interval: SERVER_CONFIG.heartbeat.interval.parse().unwrap(), - } - } -} - -impl Default for ConsumerGroupConfig { - fn default() -> ConsumerGroupConfig { - ConsumerGroupConfig { - rebalancing_timeout: SERVER_CONFIG - .consumer_group - .rebalancing_timeout - .parse() - .unwrap(), - rebalancing_check_interval: SERVER_CONFIG - .consumer_group - .rebalancing_check_interval - .parse() - .unwrap(), - } - } -} - -impl Default for RuntimeConfig { - fn default() -> RuntimeConfig { - RuntimeConfig { - path: SERVER_CONFIG.system.runtime.path.parse().unwrap(), - } - } -} - -impl Default for CompressionConfig { - fn default() -> Self { - CompressionConfig { - allow_override: SERVER_CONFIG.system.compression.allow_override, - default_algorithm: SERVER_CONFIG - .system - .compression - .default_algorithm - .parse() - .unwrap(), - } - } -} - -impl Default for LoggingConfig { - fn default() -> LoggingConfig { - LoggingConfig { - path: SERVER_CONFIG.system.logging.path.parse().unwrap(), - level: SERVER_CONFIG.system.logging.level.parse().unwrap(), - file_enabled: SERVER_CONFIG.system.logging.file_enabled, - max_file_size: SERVER_CONFIG.system.logging.max_file_size.parse().unwrap(), - max_total_size: SERVER_CONFIG.system.logging.max_total_size.parse().unwrap(), - rotation_check_interval: SERVER_CONFIG - .system - .logging - .rotation_check_interval - .parse() - .unwrap(), - retention: SERVER_CONFIG.system.logging.retention.parse().unwrap(), - sysinfo_print_interval: SERVER_CONFIG - .system - .logging - .sysinfo_print_interval - .parse() - .unwrap(), - } - } -} - -impl Default for EncryptionConfig { - fn default() -> EncryptionConfig { - EncryptionConfig { - enabled: SERVER_CONFIG.system.encryption.enabled, - key: SERVER_CONFIG.system.encryption.key.parse().unwrap(), - } - } -} - -impl Default for StreamConfig { - fn default() -> StreamConfig { - StreamConfig { - path: SERVER_CONFIG.system.stream.path.parse().unwrap(), - } - } -} - -impl Default for TopicConfig { - fn default() -> TopicConfig { - TopicConfig { - path: SERVER_CONFIG.system.topic.path.parse().unwrap(), - max_size: SERVER_CONFIG.system.topic.max_size.parse().unwrap(), - message_expiry: SERVER_CONFIG.system.topic.message_expiry.parse().unwrap(), - } - } -} - -impl Default for PartitionConfig { - fn default() -> PartitionConfig { - PartitionConfig { - path: SERVER_CONFIG.system.partition.path.parse().unwrap(), - size_of_messages_required_to_save: SERVER_CONFIG - .system - .partition - .size_of_messages_required_to_save - .parse() - .unwrap(), - messages_required_to_save: SERVER_CONFIG.system.partition.messages_required_to_save - as u32, - enforce_fsync: SERVER_CONFIG.system.partition.enforce_fsync, - validate_checksum: SERVER_CONFIG.system.partition.validate_checksum, - } - } -} - -impl Default for SegmentConfig { - fn default() -> SegmentConfig { - SegmentConfig { - size: SERVER_CONFIG.system.segment.size.parse().unwrap(), - cache_indexes: SERVER_CONFIG.system.segment.cache_indexes.parse().unwrap(), - archive_expired: SERVER_CONFIG.system.segment.archive_expired, - } - } -} - -impl Default for StateConfig { - fn default() -> StateConfig { - StateConfig { - enforce_fsync: SERVER_CONFIG.system.state.enforce_fsync, - max_file_operation_retries: SERVER_CONFIG.system.state.max_file_operation_retries - as u32, - retry_delay: SERVER_CONFIG.system.state.retry_delay.parse().unwrap(), - } - } -} - -impl Default for MessageDeduplicationConfig { - fn default() -> MessageDeduplicationConfig { - MessageDeduplicationConfig { - enabled: SERVER_CONFIG.system.message_deduplication.enabled, - max_entries: SERVER_CONFIG.system.message_deduplication.max_entries as u64, - expiry: SERVER_CONFIG - .system - .message_deduplication - .expiry - .parse() - .unwrap(), - } - } -} - -impl Default for RecoveryConfig { - fn default() -> RecoveryConfig { - RecoveryConfig { - recreate_missing_state: SERVER_CONFIG.system.recovery.recreate_missing_state, - } - } -} - -impl Default for MemoryPoolConfig { - fn default() -> MemoryPoolConfig { - Self { - enabled: SERVER_CONFIG.system.memory_pool.enabled, - size: SERVER_CONFIG.system.memory_pool.size.parse().unwrap(), - bucket_capacity: SERVER_CONFIG.system.memory_pool.bucket_capacity as u32, - } - } -} - -impl Default for TelemetryConfig { - fn default() -> TelemetryConfig { - TelemetryConfig { - enabled: SERVER_CONFIG.telemetry.enabled, - service_name: SERVER_CONFIG.telemetry.service_name.parse().unwrap(), - logs: TelemetryLogsConfig::default(), - traces: TelemetryTracesConfig::default(), - } - } -} - -impl Default for TelemetryLogsConfig { - fn default() -> TelemetryLogsConfig { - TelemetryLogsConfig { - transport: SERVER_CONFIG.telemetry.logs.transport.parse().unwrap(), - endpoint: SERVER_CONFIG.telemetry.logs.endpoint.parse().unwrap(), - } - } -} - -impl Default for TelemetryTracesConfig { - fn default() -> TelemetryTracesConfig { - TelemetryTracesConfig { - transport: SERVER_CONFIG.telemetry.traces.transport.parse().unwrap(), - endpoint: SERVER_CONFIG.telemetry.traces.endpoint.parse().unwrap(), - } - } -} - -impl Default for ClusterConfig { - fn default() -> ClusterConfig { - ClusterConfig { - enabled: SERVER_CONFIG.cluster.enabled, - name: SERVER_CONFIG.cluster.name.parse().unwrap(), - nodes: SERVER_CONFIG - .cluster - .nodes - .iter() - .map(|node| ClusterNodeConfig { - name: node.name.parse().unwrap(), - ip: node.ip.parse().unwrap(), - replica_id: u8::try_from(node.replica_id).expect( - "static_toml replica_id must fit in u8 (0..=255); \ - fix core/server/config.toml", - ), - ports: TransportPorts { - tcp: Some(u16::try_from(node.ports.tcp).expect( - "static_toml cluster.nodes.ports.tcp must fit in u16 (0..=65535); \ - fix core/server/config.toml", - )), - quic: Some(u16::try_from(node.ports.quic).expect( - "static_toml cluster.nodes.ports.quic must fit in u16 (0..=65535); \ - fix core/server/config.toml", - )), - http: Some(u16::try_from(node.ports.http).expect( - "static_toml cluster.nodes.ports.http must fit in u16 (0..=65535); \ - fix core/server/config.toml", - )), - websocket: Some(u16::try_from(node.ports.websocket).expect( - "static_toml cluster.nodes.ports.websocket must fit in u16 (0..=65535); \ - fix core/server/config.toml", - )), - tcp_replica: Some(u16::try_from(node.ports.tcp_replica).expect( - "static_toml cluster.nodes.ports.tcp_replica must fit in u16 (0..=65535); \ - fix core/server/config.toml", - )), - }, - }) - .collect(), - auth: ClusterAuthConfig::default(), - tls: ClusterTlsConfig::default(), +impl Default for MessageBusConfig { + fn default() -> MessageBusConfig { + // Read every field from the embedded TOML so the Default impl + // and the on-disk schema cannot drift. Sibling impls in this + // file follow the same pattern. + let bus = &SERVER_CONFIG.message_bus; + MessageBusConfig { + max_batch: bus.max_batch as usize, + max_message_size: bus.max_message_size.parse().unwrap(), + peer_queue_capacity: bus.peer_queue_capacity as usize, + reconnect_period: bus.reconnect_period.parse().unwrap(), + close_peer_timeout: bus.close_peer_timeout.parse().unwrap(), + close_grace: bus.close_grace.parse().unwrap(), + handshake_grace: bus.handshake_grace.parse().unwrap(), } } } diff --git a/core/configs/src/server_config/displays.rs b/core/configs/src/server_config/displays.rs index f7693965d9..2fac643c36 100644 --- a/core/configs/src/server_config/displays.rs +++ b/core/configs/src/server_config/displays.rs @@ -15,258 +15,101 @@ // specific language governing permissions and limitations // under the License. -use super::quic::{QuicCertificateConfig, QuicConfig}; -use super::server::{ - ConsumerGroupConfig, DataMaintenanceConfig, HeartbeatConfig, MessagesMaintenanceConfig, - TelemetryConfig, TelemetryLogsConfig, TelemetryTracesConfig, -}; -use super::system::MessageDeduplicationConfig; -use super::{ - http::{HttpConfig, HttpCorsConfig, HttpJwtConfig, HttpMetricsConfig, HttpTlsConfig}, - server::{MessageSaverConfig, ServerConfig}, - system::{ - CompressionConfig, EncryptionConfig, LoggingConfig, PartitionConfig, SegmentConfig, - StateConfig, StreamConfig, SystemConfig, TopicConfig, - }, - tcp::{TcpConfig, TcpSocketConfig, TcpTlsConfig}, -}; -use configs::ConfigEnvMappings; +//! `Display` impls for the sections this module owns. +//! +//! Sections drawn from [`crate::common`] pick up [`Display`] from +//! [`crate::displays`]; this module only adds the top-level +//! [`ServerConfig`] formatter and the [`MessageBusConfig`] section +//! formatter. + +use super::message_bus::MessageBusConfig; +use super::metadata::MetadataConfig; +use super::partition::PartitionConfig; +use super::quic::{QuicCertificateConfig, QuicConfig, QuicSocketConfig}; +use super::server::{ExtraConfig, NamespaceConfig, ServerConfig}; +use super::tcp::{TcpConfig, TcpSocketConfig, TcpTlsConfig}; use std::fmt::{Display, Formatter}; -impl Display for HttpConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ enabled: {}, address: {}, max_request_size: {}, web_ui: {}, cors: {}, jwt: {}, metrics: {}, tls: {} }}", - self.enabled, - self.address, - self.max_request_size, - self.web_ui, - self.cors, - self.jwt, - self.metrics, - self.tls - ) - } -} - -impl Display for HttpCorsConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ enabled: {}, allowed_methods: {:?}, allowed_origins: {:?}, allowed_headers: {:?}, exposed_headers: {:?}, allow_credentials: {}, allow_private_network: {} }}", - self.enabled, - self.allowed_methods, - self.allowed_origins, - self.allowed_headers, - self.exposed_headers, - self.allow_credentials, - self.allow_private_network - ) - } -} - -impl Display for HttpJwtConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ algorithm: {}, audience: {}, access_token_expiry: {}, use_base64_secret: {} }}", - self.algorithm, self.audience, self.access_token_expiry, self.use_base64_secret - ) - } -} - -impl Display for HttpMetricsConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ enabled: {}, endpoint: {} }}", - self.enabled, self.endpoint - ) - } -} - -impl Display for HttpTlsConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ enabled: {}, cert_file: {}, key_file: {} }}", - self.enabled, self.cert_file, self.key_file - ) - } -} - -impl Display for QuicConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ enabled: {}, address: {}, max_concurrent_bidi_streams: {}, datagram_send_buffer_size: {}, initial_mtu: {}, send_window: {}, receive_window: {}, keep_alive_interval: {}, max_idle_timeout: {}, certificate: {} }}", - self.enabled, - self.address, - self.max_concurrent_bidi_streams, - self.datagram_send_buffer_size, - self.initial_mtu, - self.send_window, - self.receive_window, - self.keep_alive_interval, - self.max_idle_timeout, - self.certificate - ) - } -} - -impl Display for QuicCertificateConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ self_signed: {}, cert_file: {}, key_file: {} }}", - self.self_signed, self.cert_file, self.key_file - ) - } -} - -impl Display for CompressionConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ allowed_override: {}, default_algorithm: {} }}", - self.allow_override, self.default_algorithm - ) - } -} - -impl Display for DataMaintenanceConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{{ messages: {} }}", self.messages) - } -} - -impl Display for MessagesMaintenanceConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ cleaner_enabled: {}, interval: {} }}", - self.cleaner_enabled, self.interval - ) - } -} - impl Display for ServerConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!( f, - "{{ consumer_group: {}, data_maintenance: {}, message_saver: {}, heartbeat: {}, system: {}, quic: {}, tcp: {}, http: {}, telemetry: {} }}", + "{{ consumer_group: {}, data_maintenance: {}, extra: {}, message_saver: {}, \ + heartbeat: {}, system: {}, quic: {}, tcp: {}, http: {}, telemetry: {}, \ + metadata: {}, message_bus: {}, partition: {} }}", self.consumer_group, self.data_maintenance, + self.extra, self.message_saver, self.heartbeat, self.system, self.quic, self.tcp, self.http, - self.telemetry - ) - } -} - -impl Display for ConsumerGroupConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ rebalancing_timeout: {}, rebalancing_check_interval: {} }}", - self.rebalancing_timeout, self.rebalancing_check_interval - ) - } -} - -impl Display for MessageSaverConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ enabled: {}, enforce_fsync: {}, interval: {} }}", - self.enabled, self.enforce_fsync, self.interval - ) - } -} - -impl Display for HeartbeatConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ enabled: {}, interval: {} }}", - self.enabled, self.interval + self.telemetry, + self.metadata, + self.message_bus, + self.partition, ) } } -impl Display for EncryptionConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{{ enabled: {} }}", self.enabled) - } -} - -impl Display for StreamConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{{ path: {} }}", self.path) - } -} - -impl Display for TopicConfig { +impl Display for PartitionConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!( f, - "{{ path: {}, max_size: {}, message_expiry: {} }}", - self.path, self.max_size, self.message_expiry + "{{ prepare_queue_depth: {}, evicted_ring_capacity: {}, \ + evicted_ring_bytes_max: {}, transfer_served_cache_bytes_max: {}, \ + transfer_artifact_bytes_max: {} }}", + self.prepare_queue_depth, + self.evicted_ring_capacity, + self.evicted_ring_bytes_max, + self.transfer_served_cache_bytes_max, + self.transfer_artifact_bytes_max, ) } } -impl Display for PartitionConfig { +impl Display for MetadataConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!( f, - "{{ path: {}, messages_required_to_save: {}, size_of_messages_required_to_save: {}, enforce_fsync: {}, validate_checksum: {} }}", - self.path, - self.messages_required_to_save, - self.size_of_messages_required_to_save, - self.enforce_fsync, - self.validate_checksum + "{{ prepare_queue_depth: {}, journal_slots: {}, clients_table_max: {} }}", + self.prepare_queue_depth, self.journal_slots, self.clients_table_max, ) } } -impl Display for MessageDeduplicationConfig { +impl Display for MessageBusConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!( f, - "{{ enabled: {}, max_entries: {:?}, expiry: {:?} }}", - self.enabled, self.max_entries, self.expiry + "{{ max_batch: {}, max_message_size: {}, peer_queue_capacity: {}, \ + reconnect_period: {}, close_peer_timeout: {}, close_grace: {}, \ + handshake_grace: {} }}", + self.max_batch, + self.max_message_size, + self.peer_queue_capacity, + self.reconnect_period, + self.close_peer_timeout, + self.close_grace, + self.handshake_grace, ) } } -impl Display for SegmentConfig { +impl Display for ExtraConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ size_bytes: {}, cache_indexes: {}, archive_expired: {} }}", - self.size, self.cache_indexes, self.archive_expired, - ) + write!(f, "{{ namespace: {} }}", self.namespace) } } -impl Display for LoggingConfig { +impl Display for NamespaceConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!( f, - "{{ path: {}, level: {}, file_enabled: {}, max_file_size: {}, max_total_size: {}, rotation_check_interval: {}, retention: {} }}", - self.path, - self.level, - self.file_enabled, - self.max_file_size.as_human_string_with_zero_as_unlimited(), - self.max_total_size.as_human_string_with_zero_as_unlimited(), - self.rotation_check_interval, - self.retention + "{{ max_streams: {}, max_topics: {}, max_partitions: {} }}", + self.max_streams, self.max_topics, self.max_partitions ) } } @@ -295,7 +138,7 @@ impl Display for TcpSocketConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!( f, - "{{ override defaults: {}, recv buffer size: {}, send buffer size {}, keepalive: {}, nodelay: {}, linger: {} }}", + "{{ override_defaults: {}, recv_buffer_size: {}, send_buffer_size: {}, keepalive: {}, nodelay: {}, linger: {} }}", self.override_defaults, self.recv_buffer_size, self.send_buffer_size, @@ -306,59 +149,41 @@ impl Display for TcpSocketConfig { } } -impl Display for TelemetryConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ enabled: {}, service_name: {}, logs: {}, traces: {} }}", - self.enabled, self.service_name, self.logs, self.traces - ) - } -} - -impl Display for TelemetryLogsConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ transport: {}, endpoint: {} }}", - self.transport, self.endpoint - ) - } -} - -impl Display for StateConfig { +impl Display for QuicConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!( f, - "{{ enforce_fsync: {}, max_file_operation_retries: {}, retry_delay: {} }}", - self.enforce_fsync, self.max_file_operation_retries, self.retry_delay, + "{{ enabled: {}, address: {}, max_concurrent_bidi_streams: {}, datagram_send_buffer_size: {}, initial_mtu: {}, send_window: {}, receive_window: {}, keep_alive_interval: {}, max_idle_timeout: {}, certificate: {} }}", + self.enabled, + self.address, + self.max_concurrent_bidi_streams, + self.datagram_send_buffer_size, + self.initial_mtu, + self.send_window, + self.receive_window, + self.keep_alive_interval, + self.max_idle_timeout, + self.certificate ) } } -impl Display for TelemetryTracesConfig { +impl Display for QuicCertificateConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!( f, - "{{ transport: {}, endpoint: {} }}", - self.transport, self.endpoint + "{{ self_signed: {}, cert_file: {}, key_file: {} }}", + self.self_signed, self.cert_file, self.key_file ) } } -impl Display for SystemConfig { +impl Display for QuicSocketConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!( f, - "{{ path: {}, logging: {}, stream: {}, topic: {}, partition: {}, segment: {}, encryption: {}, state: {} }}", - self.path, - self.logging, - self.stream, - self.topic, - self.partition, - self.segment, - self.encryption, - self.state, + "{{ override_defaults: {}, recv_buffer_size: {}, send_buffer_size: {}, keepalive: {} }}", + self.override_defaults, self.recv_buffer_size, self.send_buffer_size, self.keepalive ) } } diff --git a/core/configs/src/server_ng_config/message_bus.rs b/core/configs/src/server_config/message_bus.rs similarity index 87% rename from core/configs/src/server_ng_config/message_bus.rs rename to core/configs/src/server_config/message_bus.rs index 3fcd2b1615..91a84d6a01 100644 --- a/core/configs/src/server_ng_config/message_bus.rs +++ b/core/configs/src/server_config/message_bus.rs @@ -29,13 +29,13 @@ //! and `[websocket]` + `[websocket.tls]` respectively. //! - WebSocket frame-layer tuning (buffer sizes, message / frame //! ceilings, unmasked-frame acceptance) lives in `[websocket]` -//! ([`super::websocket::WebSocketConfig`]): the bus IS server-ng's +//! ([`super::websocket::WebSocketConfig`]): the bus IS the server's //! WS / WSS install path, so the listener section carries the frame //! tuning and the runtime folds it into a compio-ws //! `WebSocketConfig` once at bus construction. The //! `websocket.max_*` <= `message_bus.max_message_size` chain is //! enforced as a cross-section check in -//! [`super::server_ng::ServerNgConfig`]'s validator. +//! [`super::server::ServerConfig`]'s validator. //! //! Tunables the bus owns directly: bus-internal abstractions the //! operator does not see anywhere else in the schema (batch sizing, @@ -48,10 +48,10 @@ //! this section. //! //! Construction of the runtime type from this struct happens in the -//! follow-up PR that wires `core/server-ng` to call -//! [`super::server_ng::ServerNgConfig::load`]. +//! follow-up PR that wires `core/server` to call +//! [`super::server::ServerConfig::load`]. -use super::COMPONENT_NG; +use super::COMPONENT; use crate::ConfigurationError; use configs::ConfigEnv; use iggy_common::{IggyByteSize, IggyDuration, Validatable}; @@ -68,7 +68,7 @@ use serde_with::{DisplayFromStr, serde_as}; /// `IggyMessageBus::with_config`. A unit test below pins the literal so /// any future bump on the runtime side surfaces as a configs-build /// failure until both are reconciled. -pub const IOV_MAX_LIMIT_NG: usize = 512; +pub const IOV_MAX_LIMIT: usize = 512; /// Tunables for the message bus that ships consensus traffic between /// replicas and SDK-client traffic between shards. @@ -77,7 +77,7 @@ pub const IOV_MAX_LIMIT_NG: usize = 512; pub struct MessageBusConfig { /// Maximum number of `BusMessage` entries the writer task coalesces /// into a single `writev(2)` call. Higher values amortise syscalls - /// at the cost of tail latency. Capped at [`IOV_MAX_LIMIT_NG`]. + /// at the cost of tail latency. Capped at [`IOV_MAX_LIMIT`]. pub max_batch: usize, /// Wire-level cap on a single framed message. Read-side validator; @@ -127,38 +127,38 @@ pub struct MessageBusConfig { impl Validatable for MessageBusConfig { fn validate(&self) -> Result<(), ConfigurationError> { if self.max_batch == 0 { - eprintln!("{COMPONENT_NG} message_bus.max_batch must be > 0"); + eprintln!("{COMPONENT} message_bus.max_batch must be > 0"); return Err(ConfigurationError::InvalidConfigurationValue); } - if self.max_batch > IOV_MAX_LIMIT_NG { + if self.max_batch > IOV_MAX_LIMIT { eprintln!( - "{COMPONENT_NG} message_bus.max_batch ({}) exceeds IOV_MAX_LIMIT ({IOV_MAX_LIMIT_NG})", + "{COMPONENT} message_bus.max_batch ({}) exceeds IOV_MAX_LIMIT ({IOV_MAX_LIMIT})", self.max_batch ); return Err(ConfigurationError::InvalidConfigurationValue); } if self.peer_queue_capacity == 0 { - eprintln!("{COMPONENT_NG} message_bus.peer_queue_capacity must be > 0"); + eprintln!("{COMPONENT} message_bus.peer_queue_capacity must be > 0"); return Err(ConfigurationError::InvalidConfigurationValue); } if self.max_message_size.as_bytes_u64() == 0 { - eprintln!("{COMPONENT_NG} message_bus.max_message_size must be > 0"); + eprintln!("{COMPONENT} message_bus.max_message_size must be > 0"); return Err(ConfigurationError::InvalidConfigurationValue); } if self.handshake_grace.as_micros() == 0 { - eprintln!("{COMPONENT_NG} message_bus.handshake_grace must be > 0"); + eprintln!("{COMPONENT} message_bus.handshake_grace must be > 0"); return Err(ConfigurationError::InvalidConfigurationValue); } if self.close_grace.as_micros() == 0 { - eprintln!("{COMPONENT_NG} message_bus.close_grace must be > 0"); + eprintln!("{COMPONENT} message_bus.close_grace must be > 0"); return Err(ConfigurationError::InvalidConfigurationValue); } if self.close_peer_timeout.as_micros() == 0 { - eprintln!("{COMPONENT_NG} message_bus.close_peer_timeout must be > 0"); + eprintln!("{COMPONENT} message_bus.close_peer_timeout must be > 0"); return Err(ConfigurationError::InvalidConfigurationValue); } if self.reconnect_period.as_micros() == 0 { - eprintln!("{COMPONENT_NG} message_bus.reconnect_period must be > 0"); + eprintln!("{COMPONENT} message_bus.reconnect_period must be > 0"); return Err(ConfigurationError::InvalidConfigurationValue); } Ok(()) @@ -188,14 +188,14 @@ mod tests { #[test] fn rejects_max_batch_above_iov_max() { let mut c = baseline(); - c.max_batch = IOV_MAX_LIMIT_NG + 1; + c.max_batch = IOV_MAX_LIMIT + 1; assert!(c.validate().is_err()); } #[test] fn accepts_max_batch_at_iov_max() { let mut c = baseline(); - c.max_batch = IOV_MAX_LIMIT_NG; + c.max_batch = IOV_MAX_LIMIT; assert!(c.validate().is_ok()); } @@ -220,7 +220,7 @@ mod tests { /// `core/configs` does not depend on `core/message_bus`. #[test] fn iov_max_limit_matches_runtime_crate() { - assert_eq!(IOV_MAX_LIMIT_NG, 512); + assert_eq!(IOV_MAX_LIMIT, 512); } #[test] diff --git a/core/configs/src/server_ng_config/metadata.rs b/core/configs/src/server_config/metadata.rs similarity index 74% rename from core/configs/src/server_ng_config/metadata.rs rename to core/configs/src/server_config/metadata.rs index f38840f893..2ba8d821fb 100644 --- a/core/configs/src/server_ng_config/metadata.rs +++ b/core/configs/src/server_config/metadata.rs @@ -28,7 +28,7 @@ //! between forced checkpoints) //! - `clients_table_max` -> `consensus::CLIENTS_TABLE_MAX` (the VSR //! client-table slot count; independent of the two above). The -//! server-ng HTTP session cap tracks it at half. +//! HTTP session cap tracks it at half. //! //! The first two interlock through the forced-checkpoint margin //! (`max(64, prepare_queue_depth)` at bootstrap): while a checkpoint @@ -39,11 +39,11 @@ //! The defaults are duplicated literals rather than imports so //! `core/configs` does not grow build-time edges onto `core/consensus` //! and `core/journal` (the runtime crates are the consumers of this -//! config, mirroring the `IOV_MAX_LIMIT_NG` precedent in -//! [`super::message_bus`]). `core/server-ng`'s bootstrap pins these +//! config, mirroring the `IOV_MAX_LIMIT` precedent in +//! [`super::message_bus`]). `core/server`'s bootstrap pins these //! literals against the runtime constants with static asserts. -use super::COMPONENT_NG; +use super::COMPONENT; use crate::ConfigurationError; use configs::ConfigEnv; use iggy_common::Validatable; @@ -60,10 +60,15 @@ pub const DEFAULT_METADATA_JOURNAL_SLOTS: usize = 1024; /// margin is `max(this, prepare_queue_depth)`. pub const METADATA_CHECKPOINT_MARGIN_FLOOR: usize = 64; -/// Upper bound on `prepare_queue_depth`. Every queued prepare pins a -/// full message buffer; four thousand in-flight metadata ops is far past -/// any sane deployment and a likely unit typo. -pub const MAX_METADATA_PREPARE_QUEUE_DEPTH: usize = 4096; +/// Upper bound on `prepare_queue_depth`. +/// +/// Pinned by the view-change wire format, not by memory: a `DoViewChange` carries +/// the sender's uncommitted suffix plus one nack bit and one present bit per entry, +/// each bitset a single `u128` (`consensus::DVC_HEADERS_MAX` = 128). The suffix +/// spans `commit_max..=op`, which this depth bounds, so a deeper queue produces +/// entries the new primary can neither adopt nor prove dead. The reserved head slot +/// leaves room for the head op. +pub const MAX_METADATA_PREPARE_QUEUE_DEPTH: usize = 127; /// Upper bound on `journal_slots`. Each slot costs index memory and every /// checkpoint rewrites the live WAL suffix; a million slots is the sanity @@ -73,7 +78,7 @@ pub const MAX_METADATA_JOURNAL_SLOTS: usize = 1 << 20; /// Mirrors `consensus::CLIENTS_TABLE_MAX`, the VSR client-table slot count. pub const DEFAULT_METADATA_CLIENTS_TABLE_MAX: usize = 8192; -/// Floor on `clients_table_max`. The server-ng HTTP session cap derives as +/// Floor on `clients_table_max`. The HTTP session cap derives as /// `clients_table_max / 2`; below two that floors to zero and HTTP could /// register no sessions at all. pub const MIN_METADATA_CLIENTS_TABLE_MAX: usize = 2; @@ -101,7 +106,7 @@ pub struct MetadataConfig { /// Slot count of the VSR client table: how many distinct clients /// (TCP/QUIC/WS virtual clients and HTTP sessions together) hold live /// session state before the oldest-committed entry is evicted. The - /// server-ng HTTP session cap tracks this at half, so raising it lifts + /// HTTP session cap tracks this at half, so raising it lifts /// both. pub clients_table_max: usize, } @@ -122,19 +127,23 @@ impl MetadataConfig { impl Validatable for MetadataConfig { fn validate(&self) -> Result<(), ConfigurationError> { if self.prepare_queue_depth == 0 { - eprintln!("{COMPONENT_NG} metadata.prepare_queue_depth must be > 0"); + eprintln!("{COMPONENT} metadata.prepare_queue_depth must be > 0"); return Err(ConfigurationError::InvalidConfigurationValue); } if self.prepare_queue_depth > MAX_METADATA_PREPARE_QUEUE_DEPTH { eprintln!( - "{COMPONENT_NG} metadata.prepare_queue_depth ({}) exceeds the maximum ({MAX_METADATA_PREPARE_QUEUE_DEPTH})", + "{COMPONENT} metadata.prepare_queue_depth ({}) exceeds the maximum \ + ({MAX_METADATA_PREPARE_QUEUE_DEPTH}). The ceiling is the view-change wire, not memory: \ + a DoViewChange describes the uncommitted suffix with one bit per op in a u128 \ + bitset, and this depth bounds that suffix. Deeper produces entries a new \ + primary can neither adopt nor prove dead. Lowered from 256; not raisable.", self.prepare_queue_depth ); return Err(ConfigurationError::InvalidConfigurationValue); } if self.journal_slots > MAX_METADATA_JOURNAL_SLOTS { eprintln!( - "{COMPONENT_NG} metadata.journal_slots ({}) exceeds the maximum ({MAX_METADATA_JOURNAL_SLOTS})", + "{COMPONENT} metadata.journal_slots ({}) exceeds the maximum ({MAX_METADATA_JOURNAL_SLOTS})", self.journal_slots ); return Err(ConfigurationError::InvalidConfigurationValue); @@ -146,21 +155,21 @@ impl Validatable for MetadataConfig { let min_slots = 4 * self.checkpoint_margin(); if self.journal_slots < min_slots { eprintln!( - "{COMPONENT_NG} metadata.journal_slots ({}) must be >= 4 * max({METADATA_CHECKPOINT_MARGIN_FLOOR}, prepare_queue_depth) = {min_slots}", + "{COMPONENT} metadata.journal_slots ({}) must be >= 4 * max({METADATA_CHECKPOINT_MARGIN_FLOOR}, prepare_queue_depth) = {min_slots}", self.journal_slots ); return Err(ConfigurationError::InvalidConfigurationValue); } if self.clients_table_max < MIN_METADATA_CLIENTS_TABLE_MAX { eprintln!( - "{COMPONENT_NG} metadata.clients_table_max ({}) must be >= {MIN_METADATA_CLIENTS_TABLE_MAX}", + "{COMPONENT} metadata.clients_table_max ({}) must be >= {MIN_METADATA_CLIENTS_TABLE_MAX}", self.clients_table_max ); return Err(ConfigurationError::InvalidConfigurationValue); } if self.clients_table_max > MAX_METADATA_CLIENTS_TABLE_MAX { eprintln!( - "{COMPONENT_NG} metadata.clients_table_max ({}) exceeds the maximum ({MAX_METADATA_CLIENTS_TABLE_MAX})", + "{COMPONENT} metadata.clients_table_max ({}) exceeds the maximum ({MAX_METADATA_CLIENTS_TABLE_MAX})", self.clients_table_max ); return Err(ConfigurationError::InvalidConfigurationValue); @@ -187,33 +196,52 @@ mod tests { #[test] fn margin_tracks_deep_prepare_queue() { let config = MetadataConfig { - prepare_queue_depth: 256, + prepare_queue_depth: MAX_METADATA_PREPARE_QUEUE_DEPTH, journal_slots: 4096, clients_table_max: DEFAULT_METADATA_CLIENTS_TABLE_MAX, }; assert!(config.validate().is_ok()); - assert_eq!(config.checkpoint_margin(), 256); + assert_eq!(config.checkpoint_margin(), MAX_METADATA_PREPARE_QUEUE_DEPTH); } #[test] fn journal_must_outsize_margin() { - // Deep queue, journal kept at the old default: margin becomes 256, - // 4 * 256 = 1024 == journal_slots, boundary accepted... + // Deepest permitted queue: margin becomes the depth, and the journal + // must hold 4x that. At exactly 4x the boundary is accepted... + let min_slots = 4 * MAX_METADATA_PREPARE_QUEUE_DEPTH; let boundary = MetadataConfig { - prepare_queue_depth: 256, - journal_slots: 1024, + prepare_queue_depth: MAX_METADATA_PREPARE_QUEUE_DEPTH, + journal_slots: min_slots, clients_table_max: DEFAULT_METADATA_CLIENTS_TABLE_MAX, }; assert!(boundary.validate().is_ok()); // ...one slot fewer is refused. let starved = MetadataConfig { - prepare_queue_depth: 256, - journal_slots: 1023, + prepare_queue_depth: MAX_METADATA_PREPARE_QUEUE_DEPTH, + journal_slots: min_slots - 1, clients_table_max: DEFAULT_METADATA_CLIENTS_TABLE_MAX, }; assert!(starved.validate().is_err()); } + #[test] + fn prepare_queue_depth_capped_by_view_change_bitset_width() { + // Not a memory guard: it keeps every uncommitted suffix entry addressable by + // a `u128` bitset in a `DoViewChange`. One past it must be refused, or a view + // change meets an entry it can neither adopt nor prove dead. + let over = MetadataConfig { + prepare_queue_depth: MAX_METADATA_PREPARE_QUEUE_DEPTH + 1, + journal_slots: MAX_METADATA_JOURNAL_SLOTS, + clients_table_max: DEFAULT_METADATA_CLIENTS_TABLE_MAX, + }; + assert!(over.validate().is_err()); + assert_eq!( + MAX_METADATA_PREPARE_QUEUE_DEPTH + 1, + 128, + "cap must leave the head op a slot inside the 128-bit bitset" + ); + } + #[test] fn zero_depth_is_refused() { let config = MetadataConfig { diff --git a/core/configs/src/server_config/mod.rs b/core/configs/src/server_config/mod.rs index e88b2f03eb..718aee5cdd 100644 --- a/core/configs/src/server_config/mod.rs +++ b/core/configs/src/server_config/mod.rs @@ -15,17 +15,23 @@ // specific language governing permissions and limitations // under the License. -pub mod cache_indexes; +//! On-disk config schema for the `iggy-server` binary. +//! +//! Composes the shared section vocabulary from [`crate::common`] with the +//! transport, cluster, metadata and bus sections this server owns. +//! [`server::ServerConfig`] is the root type the bootstrap loads. + pub mod cluster; pub mod defaults; pub mod displays; -pub mod http; +pub mod message_bus; +pub mod metadata; +pub mod partition; pub mod quic; pub mod server; pub mod sharding; -pub mod system; pub mod tcp; pub mod validators; pub mod websocket; -pub const COMPONENT: &str = "CONFIG"; +pub use crate::common::COMPONENT; diff --git a/core/configs/src/server_ng_config/partition.rs b/core/configs/src/server_config/partition.rs similarity index 81% rename from core/configs/src/server_ng_config/partition.rs rename to core/configs/src/server_config/partition.rs index 011e82a9b8..37a5933858 100644 --- a/core/configs/src/server_ng_config/partition.rs +++ b/core/configs/src/server_config/partition.rs @@ -27,18 +27,18 @@ //! (the per-partition journal-repair retention ring's dual ceilings) //! //! Distinct from `[metadata]` (a single, shard-0-global VSR plane) because -//! partition pipelines exist PER PARTITION. The default mirrors the runtime -//! constant so a default deployment is byte-identical; the ceiling is far -//! below metadata's because the request queue (`depth * 2` slots) pins full -//! inbound produce batches, so pinned memory scales with the partition count -//! (see [`MAX_PARTITION_PREPARE_QUEUE_DEPTH`]). +//! partition pipelines exist PER PARTITION: the request queue (`depth * 2` slots) +//! pins full inbound produce batches, so pinned memory scales with the partition +//! count. The default mirrors the runtime constant; the ceiling matches metadata's, +//! since both planes ship the same `DoViewChange` suffix over the same bitsets (see +//! [`MAX_PARTITION_PREPARE_QUEUE_DEPTH`]). //! //! The default is a duplicated literal rather than an import so //! `core/configs` does not grow a build-time edge onto `core/consensus` -//! (mirroring [`super::metadata`]). `core/server-ng`'s bootstrap pins the +//! (mirroring [`super::metadata`]). `core/server`'s bootstrap pins the //! literal against the runtime constant with a static assert. -use super::COMPONENT_NG; +use super::COMPONENT; use crate::ConfigurationError; use configs::ConfigEnv; use iggy_common::{IggyByteSize, Validatable}; @@ -47,13 +47,19 @@ use serde::{Deserialize, Serialize}; /// Mirrors `consensus::PIPELINE_PREPARE_QUEUE_MAX`. pub const DEFAULT_PARTITION_PREPARE_QUEUE_DEPTH: usize = 32; -/// Upper bound on `prepare_queue_depth`. Unlike the single metadata pipeline, -/// a pipeline exists per partition, and each queued request pins a full -/// inbound produce batch (a 4 KiB floor up to megabytes). Worst-case pinned -/// memory therefore scales as `depth * 2 * partition_count * batch_size`, so -/// this ceiling sits far below metadata's 4096: it is a typo guard, not a -/// sizing endorsement. -pub const MAX_PARTITION_PREPARE_QUEUE_DEPTH: usize = 256; +/// Upper bound on `prepare_queue_depth`. +/// +/// Pinned by the view-change wire format, and equal to +/// [`super::metadata::MAX_METADATA_PREPARE_QUEUE_DEPTH`] for that reason: a +/// `DoViewChange` carries the sender's uncommitted suffix spanning `commit..=op` +/// with one nack bit and one present bit per entry, each bitset a single `u128` +/// (`consensus::DVC_HEADERS_MAX` = 128). This depth bounds `op - commit`, so a +/// deeper queue produces entries the new primary can neither adopt nor prove dead. +/// The reserved head slot leaves room for the head op. +/// +/// The memory bound (`depth * 2 * partition_count * batch_size` of pinned produce +/// batches) still holds and is looser, so the wire is what decides. +pub const MAX_PARTITION_PREPARE_QUEUE_DEPTH: usize = 127; /// Mirrors the free const `shard::PARTITION_ARTIFACT_LEN_DEFAULT` (segment /// ceiling plus the one whole batch a segment may close past it). @@ -137,35 +143,39 @@ pub struct PartitionConfig { impl Validatable for PartitionConfig { fn validate(&self) -> Result<(), ConfigurationError> { if self.prepare_queue_depth == 0 { - eprintln!("{COMPONENT_NG} partition.prepare_queue_depth must be > 0"); + eprintln!("{COMPONENT} partition.prepare_queue_depth must be > 0"); return Err(ConfigurationError::InvalidConfigurationValue); } if self.prepare_queue_depth > MAX_PARTITION_PREPARE_QUEUE_DEPTH { eprintln!( - "{COMPONENT_NG} partition.prepare_queue_depth ({}) exceeds the maximum ({MAX_PARTITION_PREPARE_QUEUE_DEPTH})", + "{COMPONENT} partition.prepare_queue_depth ({}) exceeds the maximum \ + ({MAX_PARTITION_PREPARE_QUEUE_DEPTH}). The ceiling is the view-change wire, not memory: \ + a DoViewChange describes the uncommitted suffix with one bit per op in a u128 \ + bitset, and this depth bounds that suffix. Deeper produces entries a new \ + primary can neither adopt nor prove dead. Lowered from 256; not raisable.", self.prepare_queue_depth ); return Err(ConfigurationError::InvalidConfigurationValue); } if self.evicted_ring_capacity == 0 { - eprintln!("{COMPONENT_NG} partition.evicted_ring_capacity must be > 0"); + eprintln!("{COMPONENT} partition.evicted_ring_capacity must be > 0"); return Err(ConfigurationError::InvalidConfigurationValue); } if self.evicted_ring_capacity > MAX_EVICTED_RING_CAPACITY { eprintln!( - "{COMPONENT_NG} partition.evicted_ring_capacity ({}) exceeds the maximum ({MAX_EVICTED_RING_CAPACITY})", + "{COMPONENT} partition.evicted_ring_capacity ({}) exceeds the maximum ({MAX_EVICTED_RING_CAPACITY})", self.evicted_ring_capacity ); return Err(ConfigurationError::InvalidConfigurationValue); } // The FLOOR on `transfer_artifact_bytes_max` cannot live here (it needs // `system.segment.size` and the bus cap); it is enforced in the - // `ServerNgConfig` validator, which is what turns that misconfiguration + // `ServerConfig` validator, which is what turns that misconfiguration // into a boot error instead of a silent per-partition rejoin livelock. let served_cache = self.transfer_served_cache_bytes_max.as_bytes_u64(); if served_cache == 0 || served_cache > MAX_TRANSFER_BYTES { eprintln!( - "{COMPONENT_NG} partition.transfer_served_cache_bytes_max ({served_cache} bytes) \ + "{COMPONENT} partition.transfer_served_cache_bytes_max ({served_cache} bytes) \ must be > 0 and <= {MAX_TRANSFER_BYTES} bytes" ); return Err(ConfigurationError::InvalidConfigurationValue); @@ -173,19 +183,19 @@ impl Validatable for PartitionConfig { let artifact_bytes = self.transfer_artifact_bytes_max.as_bytes_u64(); if artifact_bytes == 0 || artifact_bytes > MAX_TRANSFER_BYTES { eprintln!( - "{COMPONENT_NG} partition.transfer_artifact_bytes_max ({artifact_bytes} bytes) \ + "{COMPONENT} partition.transfer_artifact_bytes_max ({artifact_bytes} bytes) \ must be > 0 and <= {MAX_TRANSFER_BYTES} bytes" ); return Err(ConfigurationError::InvalidConfigurationValue); } let ring_bytes = self.evicted_ring_bytes_max.as_bytes_u64(); if ring_bytes == 0 { - eprintln!("{COMPONENT_NG} partition.evicted_ring_bytes_max must be > 0"); + eprintln!("{COMPONENT} partition.evicted_ring_bytes_max must be > 0"); return Err(ConfigurationError::InvalidConfigurationValue); } if ring_bytes > MAX_EVICTED_RING_BYTES { eprintln!( - "{COMPONENT_NG} partition.evicted_ring_bytes_max ({ring_bytes} bytes) exceeds the maximum ({MAX_EVICTED_RING_BYTES} bytes)" + "{COMPONENT} partition.evicted_ring_bytes_max ({ring_bytes} bytes) exceeds the maximum ({MAX_EVICTED_RING_BYTES} bytes)" ); return Err(ConfigurationError::InvalidConfigurationValue); } diff --git a/core/configs/src/server_config/quic.rs b/core/configs/src/server_config/quic.rs index 502d1bf7a4..b6a52f3077 100644 --- a/core/configs/src/server_config/quic.rs +++ b/core/configs/src/server_config/quic.rs @@ -15,9 +15,12 @@ // specific language governing permissions and limitations // under the License. +//! QUIC listener schema. + +use super::COMPONENT; +use crate::ConfigurationError; use configs::ConfigEnv; -use iggy_common::IggyByteSize; -use iggy_common::IggyDuration; +use iggy_common::{IggyByteSize, IggyDuration, Validatable}; use serde::{Deserialize, Serialize}; use serde_with::DisplayFromStr; use serde_with::serde_as; @@ -62,3 +65,153 @@ pub struct QuicCertificateConfig { pub cert_file: String, pub key_file: String, } + +/// Validates the field range constraints the runtime conversion in +/// `core::message_bus::config::build_quic_tuning` previously enforced +/// via `expect(...)`. Surfacing them here turns boot-time misconfig +/// into a `ConfigurationError` instead of a panic in the bus crate. +impl Validatable for QuicConfig { + fn validate(&self) -> Result<(), ConfigurationError> { + // QUIC requires at least one bidi stream per connection. + if self.max_concurrent_bidi_streams == 0 { + eprintln!("{COMPONENT} quic.max_concurrent_bidi_streams must be >= 1"); + return Err(ConfigurationError::InvalidConfigurationValue); + } + // quinn-proto stores stream counts as u32 internally. + if u32::try_from(self.max_concurrent_bidi_streams).is_err() { + eprintln!( + "{COMPONENT} quic.max_concurrent_bidi_streams ({}) does not fit in u32", + self.max_concurrent_bidi_streams + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + // The datagram send buffer is materialized as a `Vec` of this + // length in compio-quic, so it must fit in `usize` on the target + // platform. + if usize::try_from(self.datagram_send_buffer_size.as_bytes_u64()).is_err() { + eprintln!( + "{COMPONENT} quic.datagram_send_buffer_size ({} bytes) does not fit in usize on this target", + self.datagram_send_buffer_size.as_bytes_u64() + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + // RFC 9000 §14: minimum required MTU is 1200 bytes; quinn stores + // initial_mtu as u16 (max 65535). + let initial_mtu = self.initial_mtu.as_bytes_u64(); + if initial_mtu < 1200 { + eprintln!( + "{COMPONENT} quic.initial_mtu ({initial_mtu}) is below the QUIC minimum of 1200 bytes", + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if u16::try_from(initial_mtu).is_err() { + eprintln!("{COMPONENT} quic.initial_mtu ({initial_mtu}) exceeds u16::MAX (65535)",); + return Err(ConfigurationError::InvalidConfigurationValue); + } + // quinn VarInt for `receive_window` accepts u32; rejecting + // out-of-range values here surfaces a config error rather than + // panicking inside the bus crate's runtime conversion. + if u32::try_from(self.receive_window.as_bytes_u64()).is_err() { + eprintln!( + "{COMPONENT} quic.receive_window ({} bytes) does not fit in u32 (quinn VarInt limit)", + self.receive_window.as_bytes_u64() + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + // `send_window` is u64-sized in QuicTuning, but quinn's VarInt + // protocol-level cap is 2^62 - 1; reject anything above that. + const QUINN_VARINT_MAX: u64 = (1u64 << 62) - 1; + if self.send_window.as_bytes_u64() > QUINN_VARINT_MAX { + eprintln!( + "{COMPONENT} quic.send_window ({} bytes) exceeds quinn VarInt max (2^62 - 1)", + self.send_window.as_bytes_u64() + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn baseline() -> QuicConfig { + QuicConfig { + enabled: false, + address: String::new(), + max_concurrent_bidi_streams: 1, + datagram_send_buffer_size: IggyByteSize::from(100_u64 * 1024), + initial_mtu: IggyByteSize::from(8_u64 * 1024), + send_window: IggyByteSize::from(64_u64 * 1024 * 1024), + receive_window: IggyByteSize::from(64_u64 * 1024 * 1024), + keep_alive_interval: IggyDuration::from(std::time::Duration::from_secs(10)), + max_idle_timeout: IggyDuration::from(std::time::Duration::from_secs(30)), + certificate: QuicCertificateConfig { + self_signed: false, + cert_file: String::new(), + key_file: String::new(), + }, + socket: QuicSocketConfig { + override_defaults: false, + recv_buffer_size: IggyByteSize::from(0_u64), + send_buffer_size: IggyByteSize::from(0_u64), + keepalive: false, + }, + } + } + + #[test] + fn baseline_validates() { + baseline().validate().expect("baseline is valid"); + } + + #[test] + fn rejects_zero_max_concurrent_bidi_streams() { + let mut c = baseline(); + c.max_concurrent_bidi_streams = 0; + assert!(c.validate().is_err()); + } + + #[test] + fn rejects_max_concurrent_bidi_streams_above_u32() { + let mut c = baseline(); + c.max_concurrent_bidi_streams = u64::from(u32::MAX) + 1; + assert!(c.validate().is_err()); + } + + #[test] + fn rejects_initial_mtu_below_qiuc_minimum() { + let mut c = baseline(); + c.initial_mtu = IggyByteSize::from(1199_u64); + assert!(c.validate().is_err()); + } + + #[test] + fn rejects_initial_mtu_above_u16() { + let mut c = baseline(); + c.initial_mtu = IggyByteSize::from(u64::from(u16::MAX) + 1); + assert!(c.validate().is_err()); + } + + #[test] + fn rejects_receive_window_above_u32() { + let mut c = baseline(); + c.receive_window = IggyByteSize::from(u64::from(u32::MAX) + 1); + assert!(c.validate().is_err()); + } + + #[test] + fn rejects_send_window_above_quinn_varint_max() { + let mut c = baseline(); + c.send_window = IggyByteSize::from(1_u64 << 62); + assert!(c.validate().is_err()); + } + + #[test] + fn accepts_initial_mtu_at_quic_minimum() { + let mut c = baseline(); + c.initial_mtu = IggyByteSize::from(1200_u64); + assert!(c.validate().is_ok()); + } +} diff --git a/core/configs/src/server_config/server.rs b/core/configs/src/server_config/server.rs index 2ef47c23ec..4dbd7e81de 100644 --- a/core/configs/src/server_config/server.rs +++ b/core/configs/src/server_config/server.rs @@ -17,161 +17,85 @@ use super::COMPONENT; use super::cluster::ClusterConfig; -use super::http::HttpConfig; +use super::message_bus::MessageBusConfig; +use super::metadata::MetadataConfig; +use super::partition::PartitionConfig; use super::quic::QuicConfig; -use super::system::SystemConfig; use super::tcp::TcpConfig; use super::websocket::WebSocketConfig; use crate::ConfigurationError; +use crate::common::http::HttpConfig; +use crate::common::system::SystemConfig; use configs::{ConfigEnv, ConfigEnvMappings, ConfigProvider, FileConfigProvider, TypedEnvProvider}; use err_trail::ErrContext; use figment::providers::{Format, Toml}; use figment::value::Dict; use figment::{Metadata, Profile, Provider}; -use iggy_common::{IggyByteSize, IggyDuration, Validatable}; +use iggy_common::Validatable; use serde::{Deserialize, Serialize}; -use serde_with::DisplayFromStr; -use serde_with::serde_as; -use server_common::MemoryPoolConfigOther; -use server_common::log::{TelemetryEndpointSettings, TelemetrySettings}; +use server_common::sharding::{MAX_PARTITIONS, MAX_STREAMS, MAX_TOPICS}; use std::env; use std::sync::Arc; -pub use server_common::log::TelemetryTransport; +pub use crate::common::server::{ + ConsumerGroupConfig, DataMaintenanceConfig, HeartbeatConfig, MemoryPoolConfig, + MessageSaverConfig, MessagesMaintenanceConfig, PersonalAccessTokenCleanerConfig, + PersonalAccessTokenConfig, TelemetryConfig, TelemetryLogsConfig, TelemetryTracesConfig, + TelemetryTransport, +}; const DEFAULT_CONFIG_PATH: &str = "core/server/config.toml"; +/// [`SystemConfig`] bound to this crate's own +/// [`super::sharding::ShardingConfig`]. `core/server` names this alias +/// wherever it refers to the system config. +pub type ServerSystemConfig = SystemConfig; + +/// Top-level on-disk config schema for the `iggy-server` binary. +/// +/// Composes the shared section types from [`crate::common`] with the +/// transport, cluster, metadata and [`MessageBusConfig`] sections owned +/// by [`super`]. #[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] #[config_env(prefix = "IGGY_", name = "iggy-server-config")] pub struct ServerConfig { pub consumer_group: ConsumerGroupConfig, pub data_maintenance: DataMaintenanceConfig, + #[serde(default)] + pub extra: ExtraConfig, pub message_saver: MessageSaverConfig, pub personal_access_token: PersonalAccessTokenConfig, pub heartbeat: HeartbeatConfig, - pub system: Arc, + pub system: Arc, pub quic: QuicConfig, pub tcp: TcpConfig, pub http: HttpConfig, pub websocket: WebSocketConfig, pub telemetry: TelemetryConfig, pub cluster: ClusterConfig, + pub metadata: MetadataConfig, + pub partition: PartitionConfig, + pub message_bus: MessageBusConfig, } -/// Configuration for the memory pool. -#[derive(Debug, Deserialize, Serialize, ConfigEnv)] -pub struct MemoryPoolConfig { - pub enabled: bool, - #[config_env(leaf)] - pub size: IggyByteSize, - pub bucket_capacity: u32, -} - -impl MemoryPoolConfig { - pub fn into_other(&self) -> MemoryPoolConfigOther { - MemoryPoolConfigOther { - enabled: self.enabled, - size: self.size, - bucket_capacity: self.bucket_capacity, - } - } -} - -#[serde_as] #[derive(Debug, Default, Deserialize, Serialize, Clone, ConfigEnv)] -pub struct DataMaintenanceConfig { - pub messages: MessagesMaintenanceConfig, -} - -#[serde_as] -#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] -pub struct MessagesMaintenanceConfig { - pub cleaner_enabled: bool, - #[config_env(leaf)] - #[serde_as(as = "DisplayFromStr")] - pub interval: IggyDuration, -} - -#[serde_as] -#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] -pub struct MessageSaverConfig { - pub enabled: bool, - pub enforce_fsync: bool, - #[config_env(leaf)] - #[serde_as(as = "DisplayFromStr")] - pub interval: IggyDuration, -} - -#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] -pub struct PersonalAccessTokenConfig { - pub max_tokens_per_user: u32, - pub cleaner: PersonalAccessTokenCleanerConfig, -} - -#[serde_as] -#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] -pub struct PersonalAccessTokenCleanerConfig { - pub enabled: bool, - #[config_env(leaf)] - #[serde_as(as = "DisplayFromStr")] - pub interval: IggyDuration, -} - -#[serde_as] -#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] -pub struct HeartbeatConfig { - pub enabled: bool, - #[config_env(leaf)] - #[serde_as(as = "DisplayFromStr")] - pub interval: IggyDuration, -} - -#[serde_as] -#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] -pub struct ConsumerGroupConfig { - #[config_env(leaf)] - #[serde_as(as = "DisplayFromStr")] - pub rebalancing_timeout: IggyDuration, - #[config_env(leaf)] - #[serde_as(as = "DisplayFromStr")] - pub rebalancing_check_interval: IggyDuration, -} - -#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] -pub struct TelemetryConfig { - pub enabled: bool, - pub service_name: String, - pub logs: TelemetryLogsConfig, - pub traces: TelemetryTracesConfig, -} - -#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] -pub struct TelemetryLogsConfig { - #[config_env(leaf)] - pub transport: TelemetryTransport, - pub endpoint: String, +pub struct ExtraConfig { + pub namespace: NamespaceConfig, } #[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] -pub struct TelemetryTracesConfig { - #[config_env(leaf)] - pub transport: TelemetryTransport, - pub endpoint: String, +pub struct NamespaceConfig { + pub max_streams: usize, + pub max_topics: usize, + pub max_partitions: usize, } -impl From<&TelemetryConfig> for TelemetrySettings { - fn from(config: &TelemetryConfig) -> Self { +impl Default for NamespaceConfig { + fn default() -> Self { Self { - enabled: config.enabled, - service_name: config.service_name.clone(), - logs: TelemetryEndpointSettings { - transport: config.logs.transport, - endpoint: config.logs.endpoint.clone(), - }, - traces: TelemetryEndpointSettings { - transport: config.traces.transport, - endpoint: config.traces.endpoint.clone(), - }, + max_streams: MAX_STREAMS, + max_topics: MAX_TOPICS, + max_partitions: MAX_PARTITIONS, } } } @@ -179,49 +103,36 @@ impl From<&TelemetryConfig> for TelemetrySettings { impl ServerConfig { /// Load server configuration from file and environment variables. /// - /// Uses compile-time generated env var mappings for unambiguous resolution. + /// The path comes from `IGGY_CONFIG_PATH` or defaults to + /// `core/server/config.toml`; missing on-disk paths fall through + /// to the embedded default TOML; env-var overrides flow through the + /// [`ServerConfigEnvProvider`]; the result is validated before + /// returning. + /// + /// # Errors + /// Returns [`ConfigurationError`] when the config cannot be parsed + /// from the configured source(s) or fails [`Validatable::validate`]. pub async fn load() -> Result { - Self::load_with_path( - DEFAULT_CONFIG_PATH, - include_str!("../../../server/config.toml"), - ) - .await - } - - pub async fn load_with_path( - default_config_path: &str, - default_config: &'static str, - ) -> Result { let config_path = - env::var("IGGY_CONFIG_PATH").unwrap_or_else(|_| default_config_path.to_string()); - let config_provider = - ServerConfig::config_provider_with_default(&config_path, default_config); - let server_config: ServerConfig = - config_provider + env::var("IGGY_CONFIG_PATH").unwrap_or_else(|_| DEFAULT_CONFIG_PATH.to_string()); + let provider = ServerConfig::config_provider(&config_path); + let cfg: ServerConfig = + provider .load_config() .await .error(|e: &configs::ConfigurationError| { - format!("{COMPONENT} (error: {e}) - failed to load config") + format!("{COMPONENT} (error: {e}) - failed to load server config") })?; - server_config - .validate() - .error(|e: &configs::ConfigurationError| { - format!("{COMPONENT} (error: {e}) - failed to validate server config") - })?; - Ok(server_config) + cfg.validate().error(|e: &configs::ConfigurationError| { + format!("{COMPONENT} (error: {e}) - failed to validate server config") + })?; + Ok(cfg) } - /// Create a config provider using compile-time generated env var mappings. + /// Build the file-backed config provider with the embedded default + /// TOML and the type-safe env-var provider attached. pub fn config_provider(config_path: &str) -> FileConfigProvider { - Self::config_provider_with_default(config_path, include_str!("../../../server/config.toml")) - } - - /// Create a config provider using compile-time generated env var mappings. - pub fn config_provider_with_default( - config_path: &str, - default_config: &'static str, - ) -> FileConfigProvider { - let default_config = Toml::string(default_config); + let default_config = Toml::string(include_str!("../../../server/config.toml")); FileConfigProvider::new( config_path.to_string(), ServerConfigEnvProvider::default(), @@ -230,16 +141,16 @@ impl ServerConfig { ) } - /// Returns all valid environment variable names for ServerConfig. + /// All recognised env var names for [`ServerConfig`]. pub fn all_env_var_names() -> Vec<&'static str> { ::all_env_var_names() } } -/// Type-safe environment provider using compile-time generated mappings. +/// Type-safe environment provider for [`ServerConfig`]. /// -/// Uses the `ConfigEnvMappings` trait generated by `#[derive(ConfigEnv)]` -/// to directly look up known environment variable names, eliminating path ambiguity. +/// Uses the [`ConfigEnvMappings`] trait generated by `#[derive(ConfigEnv)]` +/// to look up known env var names directly, eliminating path ambiguity. #[derive(Debug, Clone)] pub struct ServerConfigEnvProvider { provider: TypedEnvProvider, @@ -266,3 +177,47 @@ impl Provider for ServerConfigEnvProvider { }) } } + +#[cfg(test)] +mod tests { + use super::*; + use figment::Figment; + + /// The embedded default TOML deserializes into a fully populated + /// [`ServerConfig`] and passes validation. Exercises the + /// `include_str!` resolution and the deserialization of every + /// section without depending on an async runtime in `dev-deps`. + #[test] + fn embedded_default_toml_deserializes_and_validates() { + let toml_str = include_str!("../../../server/config.toml"); + let cfg: ServerConfig = Figment::new() + .merge(Toml::string(toml_str)) + .extract() + .expect("embedded TOML deserializes"); + cfg.validate().expect("embedded default validates"); + + // Spot-check: defaults match the runtime crate's invariants. + assert_eq!(cfg.message_bus.max_batch, 256); + assert_eq!(cfg.message_bus.peer_queue_capacity, 256); + } + + #[test] + fn default_impl_validates() { + let cfg = ServerConfig::default(); + cfg.validate().expect("Default impl validates"); + } + + #[test] + fn env_prefix_is_iggy() { + assert_eq!(ServerConfig::ENV_PREFIX, "IGGY_"); + } + + #[test] + fn all_env_var_names_include_message_bus_section() { + let names = ServerConfig::all_env_var_names(); + assert!( + names.iter().any(|n| n.starts_with("IGGY_MESSAGE_BUS_")), + "expected at least one IGGY_MESSAGE_BUS_* env var, got: {names:?}" + ); + } +} diff --git a/core/configs/src/server_config/sharding.rs b/core/configs/src/server_config/sharding.rs index 87a411e1fe..b690d45fb1 100644 --- a/core/configs/src/server_config/sharding.rs +++ b/core/configs/src/server_config/sharding.rs @@ -15,21 +15,59 @@ // specific language governing permissions and limitations // under the License. +//! Sharding config: the full thread-per-core + bus surface, with +//! defaults read from the embedded `core/server/config.toml`. + +use iggy_common::IggyDuration; +use iggy_common::Validatable; use serde::{Deserialize, Serialize}; +use serde_with::{DisplayFromStr, serde_as}; +use std::time::Duration; use super::defaults::SERVER_CONFIG; +use crate::ConfigurationError; +use crate::common::validators::validate_cpu_allocation; use configs::ConfigEnv; -// `CpuAllocation`/`NumaConfig` are pure config types and live in their own -// leaf crate so both `configs` and `shard_allocator` can share them without -// pulling each other's heavier dependency trees. Re-exported here to keep the -// `configs::sharding::*` path stable for existing callers. +// Re-exported so callers reach these through `configs::sharding::*` +// alongside the rest of the section. pub use cpu_allocation::{CpuAllocation, NumaConfig}; -/// Sharding config for the legacy `core/server`. That server consumes only -/// `cpu_allocation` and `pin_cores`; the bus / shutdown / reconcile knobs are -/// server-ng concepts and live in [`crate::server_ng_config::sharding`]. +/// Maximum permitted per-shard inbox depth. The channel is allocated +/// up-front per shard, so a runaway value here OOMs the process at boot. +/// `1 << 20` (~1M frames) is several orders of magnitude above any +/// realistic backpressure target and still fits comfortably in process +/// address space. +pub const INBOX_CAPACITY_MAX: usize = 1 << 20; + +/// Hard upper bound on `shutdown_drain_timeout`. A drain that never +/// completes wedges process exit; capping at 10 minutes guarantees the +/// watchdog eventually force-tears the bus even with a pathological +/// config typo. +pub const SHUTDOWN_DRAIN_TIMEOUT_MAX: Duration = Duration::from_secs(600); + +/// Hard upper bound on `shutdown_poll_interval`. A poll interval longer +/// than the drain timeout makes the flag effectively unobservable; cap +/// at 5s so Ctrl-C latency stays bounded regardless of config. +pub const SHUTDOWN_POLL_INTERVAL_MAX: Duration = Duration::from_secs(5); + +/// Hard upper bound on `shutdown_join_timeout`. Comfortably above the +/// drain cap so a full drain always fits inside the join budget, while +/// still guaranteeing process exit against a pathological config typo. +pub const SHUTDOWN_JOIN_TIMEOUT_MAX: Duration = Duration::from_secs(900); + +/// Hard upper bound on `reconcile_periodic_interval`. A tick longer +/// than ~30s makes post-failure recovery latency operator-visible; the +/// cap reins in pathological typos without disturbing reasonable +/// production values. +pub const RECONCILE_PERIODIC_INTERVAL_MAX: Duration = Duration::from_secs(30); + +// Every omitted field falls back to the frozen `Default`, so a partial +// `[system.sharding]` table resolves each key independently instead of +// failing on the first missing one (parity with the legacy type). +#[serde_as] #[derive(Debug, Deserialize, Serialize, ConfigEnv)] +#[serde(default)] pub struct ShardingConfig { #[serde(default)] #[config_env(leaf)] @@ -45,6 +83,60 @@ pub struct ShardingConfig { /// `false` drops both the CPU and memory-node bindings (and logs a /// warning, since NUMA placement without pinning is meaningless). pub pin_cores: bool, + /// Per-shard inter-shard inbox channel capacity. Bounded by design. + /// Drops on full inbox of consensus frames are recovered by VSR + /// retransmit. Drops of cross-shard client Reply frames are terminal: + /// the client never receives the reply (no in-protocol retransmit). + /// Both frame classes share this one channel, so a consensus burst + /// can starve client-reply forwards: size against the worst-case sum + /// of consensus working set + peak client-reply fan-out per shard + /// occurring together. + /// + // TODO(hubcio): split into two priority lanes - one bounded queue for + // consensus frames (drops recovered by VSR retransmit) and one for + // client `Reply` frames (drops terminal, must be sized for worst-case + // fan-out). Current single-channel design is the minimum-viable + // wiring so `frame_drops_total{variant,reason}` surfaces under load + // and yields real numbers to size the split against. + pub inbox_capacity: usize, + /// Wall-clock budget for a single shard's bus drain on shutdown. + /// Drives `IggyMessageBus::shutdown(..)` from the per-shard watchdog + /// and the parallel-join survivor path. Sized larger than typical + /// TCP RTT times in-flight write-batch so writers receive their full + /// last `write_vectored_all` budget before the connection registry + /// force-tears the bus. Slow-fsync hosts may need to extend this past + /// the default; the cap is `SHUTDOWN_DRAIN_TIMEOUT_MAX` so a config + /// typo cannot wedge process exit. + #[serde_as(as = "DisplayFromStr")] + #[config_env(leaf)] + pub shutdown_drain_timeout: IggyDuration, + /// Poll cadence for the cross-thread shutdown flag and for the + /// `await_metadata_bundle` / `broadcast_metadata_bundle` poll loops. + /// Trades off Ctrl-C latency against idle wakeup cost; the default + /// keeps shutdown observably prompt without measurable scheduler + /// overhead. Capped at `SHUTDOWN_POLL_INTERVAL_MAX` so the flag + /// remains effectively observable regardless of config. + #[serde_as(as = "DisplayFromStr")] + #[config_env(leaf)] + pub shutdown_poll_interval: IggyDuration, + /// Hard wall-clock deadline for joining shard threads at process + /// exit. A shard whose pump or listener wedges past this budget is + /// abandoned with an error log instead of blocking exit forever. + /// Must be at least `shutdown_drain_timeout` (abandoning a shard + /// mid-drain would interrupt its WAL fsync / replica drain) and at + /// most [`SHUTDOWN_JOIN_TIMEOUT_MAX`]. + #[serde_as(as = "DisplayFromStr")] + #[config_env(leaf)] + pub shutdown_join_timeout: IggyDuration, + /// Safety-tick cadence for the partition reconciliation loop; the + /// reconciler also wakes immediately on every + /// `LifecycleFrame::MetadataCommitTick` from shard 0. This periodic + /// fallback covers dropped wake-ups (the wake channel is capacity-1) + /// and the initial post-bootstrap convergence window. Values above + /// [`RECONCILE_PERIODIC_INTERVAL_MAX`] are rejected by the validator. + #[serde_as(as = "DisplayFromStr")] + #[config_env(leaf)] + pub reconcile_periodic_interval: IggyDuration, } impl Default for ShardingConfig { @@ -52,6 +144,283 @@ impl Default for ShardingConfig { Self { cpu_allocation: CpuAllocation::default(), pin_cores: SERVER_CONFIG.system.sharding.pin_cores, + inbox_capacity: SERVER_CONFIG.system.sharding.inbox_capacity as usize, + shutdown_drain_timeout: SERVER_CONFIG + .system + .sharding + .shutdown_drain_timeout + .parse() + .unwrap(), + shutdown_poll_interval: SERVER_CONFIG + .system + .sharding + .shutdown_poll_interval + .parse() + .unwrap(), + shutdown_join_timeout: SERVER_CONFIG + .system + .sharding + .shutdown_join_timeout + .parse() + .unwrap(), + reconcile_periodic_interval: SERVER_CONFIG + .system + .sharding + .reconcile_periodic_interval + .parse() + .unwrap(), + } + } +} + +impl Validatable for ShardingConfig { + fn validate(&self) -> Result<(), ConfigurationError> { + if self.inbox_capacity == 0 { + eprintln!( + "Invalid sharding configuration: inbox_capacity must be > 0 (crossfire silently \ + rounds 0 to 1, masking config errors)" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if self.inbox_capacity > INBOX_CAPACITY_MAX { + eprintln!( + "Invalid sharding configuration: inbox_capacity {} exceeds the {} cap (each \ + shard preallocates a channel of this size; oversizing here OOMs the process at \ + boot)", + self.inbox_capacity, INBOX_CAPACITY_MAX + ); + return Err(ConfigurationError::InvalidConfigurationValue); } + + let drain = self.shutdown_drain_timeout.get_duration(); + if drain.is_zero() { + eprintln!( + "Invalid sharding configuration: shutdown_drain_timeout must be > 0 (a zero \ + budget force-tears the bus mid-WAL-fsync on every shutdown)" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if drain > SHUTDOWN_DRAIN_TIMEOUT_MAX { + eprintln!( + "Invalid sharding configuration: shutdown_drain_timeout {:?} exceeds the {:?} \ + cap (an unbounded drain wedges process exit on bus stall)", + drain, SHUTDOWN_DRAIN_TIMEOUT_MAX + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + let poll = self.shutdown_poll_interval.get_duration(); + if poll.is_zero() { + eprintln!( + "Invalid sharding configuration: shutdown_poll_interval must be > 0 (a zero \ + cadence busy-loops every shard's watchdog and metadata-handoff poller)" + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if poll > SHUTDOWN_POLL_INTERVAL_MAX { + eprintln!( + "Invalid sharding configuration: shutdown_poll_interval {:?} exceeds the {:?} \ + cap (a coarse cadence stalls Ctrl-C handling and metadata handoff abort)", + poll, SHUTDOWN_POLL_INTERVAL_MAX + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if poll > drain { + eprintln!( + "Invalid sharding configuration: shutdown_poll_interval {:?} must be <= \ + shutdown_drain_timeout {:?} (a poll cadence coarser than the drain budget makes \ + the shutdown flag effectively unobservable)", + poll, drain + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + let join = self.shutdown_join_timeout.get_duration(); + if join < drain { + eprintln!( + "Invalid sharding configuration: shutdown_join_timeout {:?} must be >= \ + shutdown_drain_timeout {:?} (a join budget shorter than the drain abandons \ + shards mid-drain, interrupting the WAL fsync / replica drain)", + join, drain + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if join > SHUTDOWN_JOIN_TIMEOUT_MAX { + eprintln!( + "Invalid sharding configuration: shutdown_join_timeout {:?} exceeds the {:?} \ + cap (an unbounded join budget wedges process exit on a stuck shard)", + join, SHUTDOWN_JOIN_TIMEOUT_MAX + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + let reconcile = self.reconcile_periodic_interval.get_duration(); + if reconcile.is_zero() { + eprintln!( + "Invalid sharding configuration: reconcile_periodic_interval resolves to zero. \ + Note that \"0\", \"none\", \"unlimited\", and \"disabled\" all parse to zero. The \ + periodic reconcile tick is a safety net for dropped commit-wakes and cannot be \ + turned off; set a positive duration (default \"1s\", max {RECONCILE_PERIODIC_INTERVAL_MAX:?})." + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if reconcile > RECONCILE_PERIODIC_INTERVAL_MAX { + eprintln!( + "Invalid sharding configuration: reconcile_periodic_interval {:?} exceeds the \ + {:?} cap (a long tick makes post-failure convergence latency operator-visible)", + reconcile, RECONCILE_PERIODIC_INTERVAL_MAX + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + + validate_cpu_allocation(&self.cpu_allocation, self.pin_cores) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::server_config::server::ServerConfig; + use figment::Figment; + use figment::providers::{Format, Toml}; + + #[test] + fn defaults_validate() { + assert!(ShardingConfig::default().validate().is_ok()); + } + + #[test] + fn zero_drain_is_rejected() { + let cfg = ShardingConfig { + shutdown_drain_timeout: IggyDuration::new(Duration::ZERO), + ..ShardingConfig::default() + }; + assert!(cfg.validate().is_err()); + } + + #[test] + fn over_cap_drain_is_rejected() { + let cfg = ShardingConfig { + shutdown_drain_timeout: IggyDuration::new( + SHUTDOWN_DRAIN_TIMEOUT_MAX + Duration::from_secs(1), + ), + ..ShardingConfig::default() + }; + assert!(cfg.validate().is_err()); + } + + #[test] + fn zero_poll_is_rejected() { + let cfg = ShardingConfig { + shutdown_poll_interval: IggyDuration::new(Duration::ZERO), + ..ShardingConfig::default() + }; + assert!(cfg.validate().is_err()); + } + + #[test] + fn over_cap_poll_is_rejected() { + let cfg = ShardingConfig { + shutdown_poll_interval: IggyDuration::new( + SHUTDOWN_POLL_INTERVAL_MAX + Duration::from_secs(1), + ), + ..ShardingConfig::default() + }; + assert!(cfg.validate().is_err()); + } + + #[test] + fn poll_greater_than_drain_is_rejected() { + let cfg = ShardingConfig { + shutdown_drain_timeout: IggyDuration::new(Duration::from_millis(20)), + shutdown_poll_interval: IggyDuration::new(Duration::from_millis(50)), + ..ShardingConfig::default() + }; + assert!(cfg.validate().is_err()); + } + + #[test] + fn join_shorter_than_drain_is_rejected() { + // A join budget under the drain would abandon shards mid-drain. + let cfg = ShardingConfig { + shutdown_drain_timeout: IggyDuration::new(Duration::from_secs(10)), + shutdown_join_timeout: IggyDuration::new(Duration::from_secs(5)), + ..ShardingConfig::default() + }; + assert!(cfg.validate().is_err()); + } + + #[test] + fn over_cap_join_is_rejected() { + let cfg = ShardingConfig { + shutdown_join_timeout: IggyDuration::new( + SHUTDOWN_JOIN_TIMEOUT_MAX + Duration::from_secs(1), + ), + ..ShardingConfig::default() + }; + assert!(cfg.validate().is_err()); + } + + #[test] + fn join_equal_to_drain_is_accepted() { + let cfg = ShardingConfig { + shutdown_drain_timeout: IggyDuration::new(Duration::from_secs(10)), + shutdown_join_timeout: IggyDuration::new(Duration::from_secs(10)), + ..ShardingConfig::default() + }; + assert!(cfg.validate().is_ok()); + } + + // Guards the single source of truth: the sharding defaults resolve + // from the embedded TOML, not hard-coded Rust values. + #[test] + fn embedded_toml_resolves_sharding_defaults() { + let toml_str = include_str!("../../../server/config.toml"); + let config: ServerConfig = Figment::new() + .merge(Toml::string(toml_str)) + .extract() + .expect("embedded TOML deserializes"); + config.validate().expect("embedded config validates"); + + let sharding = &config.system.sharding; + assert!(sharding.pin_cores); + assert_eq!(sharding.inbox_capacity, 1024); + assert_eq!(sharding.shutdown_drain_timeout, "10 s".parse().unwrap()); + assert_eq!(sharding.shutdown_poll_interval, "50 ms".parse().unwrap()); + assert_eq!(sharding.shutdown_join_timeout, "30 s".parse().unwrap()); + assert_eq!(sharding.reconcile_periodic_interval, "1 s".parse().unwrap()); + } + + // Extract straight from a raw table (no embedded base layer) so the + // struct-level `#[serde(default)]` is what fills the gaps, not the + // provider's embedded-TOML fallback. + #[test] + fn partial_table_fills_missing_fields_with_frozen_defaults() { + let sharding: ShardingConfig = Figment::new() + .merge(Toml::string("pin_cores = false")) + .extract() + .expect("partial sharding table deserializes"); + + assert!(!sharding.pin_cores); + assert_eq!(sharding.inbox_capacity, 1024); + assert_eq!(sharding.shutdown_drain_timeout, "10 s".parse().unwrap()); + assert_eq!(sharding.shutdown_poll_interval, "50 ms".parse().unwrap()); + assert_eq!(sharding.shutdown_join_timeout, "30 s".parse().unwrap()); + assert_eq!(sharding.reconcile_periodic_interval, "1 s".parse().unwrap()); + } + + #[test] + fn empty_table_yields_all_frozen_defaults() { + let sharding: ShardingConfig = Figment::new() + .merge(Toml::string("")) + .extract() + .expect("empty sharding table deserializes"); + + assert!(sharding.pin_cores); + assert_eq!(sharding.inbox_capacity, 1024); + assert_eq!(sharding.shutdown_drain_timeout, "10 s".parse().unwrap()); + assert_eq!(sharding.shutdown_poll_interval, "50 ms".parse().unwrap()); + assert_eq!(sharding.shutdown_join_timeout, "30 s".parse().unwrap()); + assert_eq!(sharding.reconcile_periodic_interval, "1 s".parse().unwrap()); } } diff --git a/core/configs/src/server_config/tcp.rs b/core/configs/src/server_config/tcp.rs index 7d81f20930..8e0acafdb8 100644 --- a/core/configs/src/server_config/tcp.rs +++ b/core/configs/src/server_config/tcp.rs @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +//! TCP listener schema. + use configs::ConfigEnv; use iggy_common::{IggyByteSize, IggyDuration}; use serde::{Deserialize, Serialize}; diff --git a/core/configs/src/server_config/validators.rs b/core/configs/src/server_config/validators.rs index d35763bc6a..6784ef294e 100644 --- a/core/configs/src/server_config/validators.rs +++ b/core/configs/src/server_config/validators.rs @@ -15,45 +15,27 @@ // specific language governing permissions and limitations // under the License. +//! [`Validatable`] for [`ServerConfig`]. +//! +//! Delegates section by section, including +//! [`super::message_bus::MessageBusConfig::validate`], then applies the +//! cross-section invariants: topic vs segment sizing, JWT gating when +//! HTTP is enabled, and server-default expiry sanity. + use super::COMPONENT; -use super::cluster::ClusterConfig; -use super::server::{ - DataMaintenanceConfig, MessageSaverConfig, MessagesMaintenanceConfig, TelemetryConfig, -}; -use super::server::{MemoryPoolConfig, PersonalAccessTokenConfig, ServerConfig}; -use super::sharding::{CpuAllocation, ShardingConfig}; -use super::system::SegmentConfig; -use super::system::{CompressionConfig, LoggingConfig, PartitionConfig}; +use super::cluster::STATE_CHUNK_HEADER_LEN; +use super::server::{ExtraConfig, NamespaceConfig, ServerConfig}; use crate::ConfigurationError; -use cpu_allocation::allowed_cpus; use err_trail::ErrContext; -use iggy_common::CompressionAlgorithm; -use iggy_common::IggyExpiry; -use iggy_common::MaxTopicSize; -use iggy_common::Validatable; -use std::thread::available_parallelism; +use iggy_common::{IggyExpiry, MaxTopicSize, Validatable}; +use server_common::sharding::IggyNamespace; use tracing::warn; -/// 1 GiB max segment size. Canonical definition; re-exported by core/server streaming. -pub const SEGMENT_MAX_SIZE_BYTES: u64 = 1024 * 1024 * 1024; - -/// Return `Err(reason)` when `alloc` would yield a host-dependent shard -/// count, disqualifying it for cluster mode where every node must derive -/// the same count from its byte-identical config. -/// -/// Deterministic variants (`Count(n)`, `Range(s, e)`, explicit -/// `NumaAware`) return `Ok`. -fn host_dependent_cpu_allocation(alloc: &CpuAllocation) -> Result<(), &'static str> { - match alloc { - CpuAllocation::All => Err("'all' (follows host CPU count)"), - CpuAllocation::NumaAware(numa) if numa.nodes.is_empty() && numa.cores_per_node == 0 => { - Err("'numa:auto' (follows host NUMA topology)") - } - CpuAllocation::Count(_) | CpuAllocation::Range(_, _) | CpuAllocation::NumaAware(_) => { - Ok(()) - } - } -} +/// compio-ws (tungstenite 0.29) `write_buffer_size` default. Used to +/// evaluate the `max_write_buffer_size > write_buffer_size` invariant +/// when the operator leaves `write_buffer_size` unset; keep in sync +/// with the defaults documented in the shipped config.toml. +const WS_DEFAULT_WRITE_BUFFER_SIZE: u64 = 128 * 1024; impl Validatable for ServerConfig { fn validate(&self) -> Result<(), ConfigurationError> { @@ -75,6 +57,9 @@ impl Validatable for ServerConfig { "{COMPONENT} (error: {e}) - failed to validate personal access token config" ) })?; + self.extra.validate().error(|e: &ConfigurationError| { + format!("{COMPONENT} (error: {e}) - failed to validate extra config") + })?; self.system .segment .validate() @@ -99,876 +84,755 @@ impl Validatable for ServerConfig { self.cluster.validate().error(|e: &ConfigurationError| { format!("{COMPONENT} (error: {e}) - failed to validate cluster config") })?; - - // Cluster consensus routing (`calculate_shard_from_consensus_ns`) - // hashes namespaces modulo the local shard count. Every node must - // agree on that count or control-plane messages (StartViewChange, - // DoViewChange, StartView, Commit) route to different shards on - // different nodes, splitting the view-change quorum. - // - // Because `cluster.nodes` is byte-identical across every host, the - // only way shard counts can drift is if `system.sharding.cpu_allocation` - // depends on host topology. We can't prove divergence from a single - // node at load time (peers may be homogeneous), so this is a warning - // rather than a hard error; operators running heterogeneous hardware - // must pin a deterministic value themselves. - if self.cluster.enabled - && let Err(reason) = host_dependent_cpu_allocation(&self.system.sharding.cpu_allocation) - { - warn!( - "cluster.enabled = true with host-dependent system.sharding.cpu_allocation ({reason}); \ - if peers resolve this to different shard counts, view-change quorum will split. \ - Pin a deterministic value (count, explicit range, or explicit numa) on heterogeneous hardware." - ); - } - + self.metadata.validate().error(|e: &ConfigurationError| { + format!("{COMPONENT} (error: {e}) - failed to validate metadata config") + })?; + self.partition.validate().error(|e: &ConfigurationError| { + format!("{COMPONENT} (error: {e}) - failed to validate partition config") + })?; self.system .logging .validate() .error(|e: &ConfigurationError| { format!("{COMPONENT} (error: {e}) - failed to validate logging config") })?; + self.message_saver + .validate() + .error(|e: &ConfigurationError| { + format!("{COMPONENT} (error: {e}) - failed to validate message saver config") + })?; let topic_size = match self.system.topic.max_size { MaxTopicSize::Custom(size) => Ok(size.as_bytes_u64()), MaxTopicSize::Unlimited => Ok(u64::MAX), MaxTopicSize::ServerDefault => { - eprintln!("system.topic.max_size cannot be ServerDefault in server config"); + eprintln!("system.topic.max_size cannot be ServerDefault in the server config"); Err(ConfigurationError::InvalidConfigurationValue) } }?; if let IggyExpiry::ServerDefault = self.system.topic.message_expiry { - eprintln!("system.topic.message_expiry cannot be ServerDefault in server config"); + eprintln!("system.topic.message_expiry cannot be ServerDefault in the server config"); return Err(ConfigurationError::InvalidConfigurationValue); } - if self.http.enabled - && let IggyExpiry::ServerDefault = self.http.jwt.access_token_expiry + // A zero duration encodes to wire value 0, the same value the wire uses + // for ServerDefault, so it would silently collide with that sentinel. + if let IggyExpiry::ExpireDuration(duration) = self.system.topic.message_expiry + && duration.as_micros() == 0 { - eprintln!("http.jwt.access_token_expiry cannot be ServerDefault when HTTP is enabled"); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - if topic_size < self.system.segment.size.as_bytes_u64() { eprintln!( - "system.topic.max_size ({} B) must be >= system.segment.size ({} B)", - topic_size, - self.system.segment.size.as_bytes_u64() - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - Ok(()) - } -} - -impl Validatable for CompressionConfig { - fn validate(&self) -> Result<(), ConfigurationError> { - let compression_alg = &self.default_algorithm; - if *compression_alg != CompressionAlgorithm::None { - // TODO(numinex): Change this message once server side compression is fully developed. - warn!( - "Server started with server-side compression enabled, using algorithm: {compression_alg}, this feature is not implemented yet!" + "system.topic.message_expiry is a zero duration, which collides with the server-default sentinel on the wire; use \"none\" to never expire or a positive duration" ); - } - - Ok(()) - } -} - -impl Validatable for TelemetryConfig { - fn validate(&self) -> Result<(), ConfigurationError> { - if !self.enabled { - return Ok(()); - } - - if self.service_name.trim().is_empty() { - eprintln!("telemetry.service_name cannot be empty when telemetry is enabled"); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - if self.logs.endpoint.is_empty() { - eprintln!("telemetry.logs.endpoint cannot be empty when telemetry is enabled"); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - if self.traces.endpoint.is_empty() { - eprintln!("telemetry.traces.endpoint cannot be empty when telemetry is enabled"); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - Ok(()) - } -} - -impl Validatable for PartitionConfig { - fn validate(&self) -> Result<(), ConfigurationError> { - if self.messages_required_to_save == 0 { - eprintln!("Configured system.partition.messages_required_to_save cannot be 0"); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - Ok(()) - } -} - -impl Validatable for SegmentConfig { - fn validate(&self) -> Result<(), ConfigurationError> { - if self.size > SEGMENT_MAX_SIZE_BYTES { - eprintln!( - "Configured system.segment.size {} B is greater than maximum {} B", - self.size.as_bytes_u64(), - SEGMENT_MAX_SIZE_BYTES - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - if !self.size.as_bytes_u64().is_multiple_of(512) { - eprintln!( - "Configured system.segment.size {} B is not a multiple of 512 B", - self.size.as_bytes_u64() - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - Ok(()) - } -} - -impl Validatable for MessageSaverConfig { - fn validate(&self) -> Result<(), ConfigurationError> { - if self.enabled && self.interval.is_zero() { - eprintln!("message_saver.interval cannot be zero when message_saver is enabled"); return Err(ConfigurationError::InvalidConfigurationValue); } - Ok(()) - } -} - -impl Validatable for DataMaintenanceConfig { - fn validate(&self) -> Result<(), ConfigurationError> { - self.messages.validate().error(|e: &ConfigurationError| { - format!("{COMPONENT} (error: {e}) - failed to validate messages maintenance config") - })?; - Ok(()) - } -} - -impl Validatable for MessagesMaintenanceConfig { - fn validate(&self) -> Result<(), ConfigurationError> { - if self.cleaner_enabled && self.interval.is_zero() { - eprintln!("data_maintenance.messages.interval cannot be zero when cleaner is enabled"); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - Ok(()) - } -} - -impl Validatable for PersonalAccessTokenConfig { - fn validate(&self) -> Result<(), ConfigurationError> { - if self.max_tokens_per_user == 0 { - eprintln!("personal_access_token.max_tokens_per_user cannot be 0"); + if self.http.enabled + && let IggyExpiry::ServerDefault = self.http.jwt.access_token_expiry + { + eprintln!("http.jwt.access_token_expiry cannot be ServerDefault when HTTP is enabled"); return Err(ConfigurationError::InvalidConfigurationValue); } - if self.cleaner.enabled && self.cleaner.interval.is_zero() { + if self.http.enabled + && self.http.tls.enabled + && (self.http.tls.cert_file.is_empty() || self.http.tls.key_file.is_empty()) + { eprintln!( - "personal_access_token.cleaner.interval cannot be zero when cleaner is enabled" + "http.tls.enabled=true requires non-empty http.tls.cert_file and http.tls.key_file" ); return Err(ConfigurationError::InvalidConfigurationValue); } - Ok(()) - } -} - -impl Validatable for LoggingConfig { - fn validate(&self) -> Result<(), ConfigurationError> { - if self.level.is_empty() { - eprintln!("system.logging.level is supposed be configured"); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - if self.retention.as_secs() < 1 { - eprintln!( - "Configured system.logging.retention {} is less than minimum 1 second", - self.retention - ); - return Err(ConfigurationError::InvalidConfigurationValue); + // Cluster mode has no port fallbacks: the roster is the single source + // of listener ports, so every enabled transport needs an explicit + // per-node port. Falling back to the port of a transport's top-level + // `address` would hand two same-host nodes the same socket and fail + // only at bind time, and a portless node would silently degrade every + // follower-to-primary HTTP forward through it to a fail-closed 503. + if self.cluster.enabled { + for node in &self.cluster.nodes { + let required_ports = [ + ("tcp", true, node.ports.tcp), + ("quic", self.quic.enabled, node.ports.quic), + ("http", self.http.enabled, node.ports.http), + ("websocket", self.websocket.enabled, node.ports.websocket), + ("tcp_replica", true, node.ports.tcp_replica), + ]; + for (transport, enabled, port) in required_ports { + if enabled && port.is_none() { + eprintln!( + "cluster node '{}' has no ports.{transport}; cluster mode requires an explicit roster port for every enabled transport", + node.name + ); + return Err(ConfigurationError::InvalidConfigurationValue); + } + } + } } - if self.rotation_check_interval.as_secs() < 1 { + if topic_size < self.system.segment.size.as_bytes_u64() { eprintln!( - "Configured system.logging.rotation_check_interval {} is less than minimum 1 second", - self.rotation_check_interval + "system.topic.max_size ({} B) must be >= system.segment.size ({} B)", + topic_size, + self.system.segment.size.as_bytes_u64() ); return Err(ConfigurationError::InvalidConfigurationValue); } - let max_total_size_unlimited = self.max_total_size.as_bytes_u64() == 0; - if !max_total_size_unlimited - && self.max_file_size.as_bytes_u64() > self.max_total_size.as_bytes_u64() - { + // A received segment artifact can be one whole batch larger than the + // segment cap (rotation checks the cap AFTER appending), and the real + // batch bound is the BUS frame cap -- the server never enforces + // `MAX_PAYLOAD_SIZE`. An artifact ceiling under that floor refuses a + // legal segment, and the manifest check is all-or-nothing, so the + // partition livelocks re-requesting the same segment from every peer at + // the backoff ceiling. Caught here so it is a boot error rather than one + // partition that silently never rejoins. + let artifact_floor = self + .system + .segment + .size + .as_bytes_u64() + .saturating_add(self.message_bus.max_message_size.as_bytes_u64()); + if self.partition.transfer_artifact_bytes_max.as_bytes_u64() < artifact_floor { eprintln!( - "Configured system.logging.max_total_size {} is less than system.logging.max_file_size {}", - self.max_total_size, self.max_file_size + "{COMPONENT} partition.transfer_artifact_bytes_max ({} B) must be at least \ + system.segment.size ({} B) + message_bus.max_message_size ({} B) = \ + {artifact_floor} B: a segment may close one whole batch past its cap, and an \ + artifact ceiling below that refuses a legal segment and livelocks the \ + partition's rejoin", + self.partition.transfer_artifact_bytes_max.as_bytes_u64(), + self.system.segment.size.as_bytes_u64(), + self.message_bus.max_message_size.as_bytes_u64(), ); return Err(ConfigurationError::InvalidConfigurationValue); } - Ok(()) - } -} + self.message_bus + .validate() + .error(|e: &ConfigurationError| { + format!("{COMPONENT} (error: {e}) - failed to validate message_bus config") + })?; -impl Validatable for MemoryPoolConfig { - fn validate(&self) -> Result<(), ConfigurationError> { - if self.enabled && self.size == 0 { + // Repair frames ride the bounded per-peer message-bus queue. A repair + // round of cluster.repair_chunk_max frames that meets or overruns + // message_bus.peer_queue_capacity drops its own tail silently, wedging + // the repair loop into slow retries. Keep the chunk strictly below the + // queue; this also floors peer_queue_capacity, which is otherwise only + // checked for > 0. + if self.cluster.repair_chunk_max >= self.message_bus.peer_queue_capacity { eprintln!( - "Configured system.memory_pool.enabled is true and system.memory_pool.size is 0" + "{COMPONENT} cluster.repair_chunk_max ({}) must be < message_bus.peer_queue_capacity ({}): repair frames ride the per-peer bus queue, so a chunk that fills or overruns it drops frames and wedges repair", + self.cluster.repair_chunk_max, self.message_bus.peer_queue_capacity ); return Err(ConfigurationError::InvalidConfigurationValue); } - const MIN_POOL_SIZE: u64 = 512 * 1024 * 1024; // 512 MiB - const MIN_BUCKET_CAPACITY: u32 = 128; - const DEFAULT_PAGE_SIZE: u64 = 4096; - - if self.enabled && self.size < MIN_POOL_SIZE { + // State-transfer chunks ride the same bus. A cap that cannot carry one + // header plus a byte of payload makes every rejoin that needs a + // transfer impossible, and the failure surfaces only as a replica + // connection tearing down when the frame is rejected on the read side. + let bus_cap = self.message_bus.max_message_size.as_bytes_u64(); + if bus_cap <= STATE_CHUNK_HEADER_LEN { eprintln!( - "Configured system.memory_pool.size {} B ({} MiB) is less than minimum {} B, ({} MiB)", - self.size.as_bytes_u64(), - self.size.as_bytes_u64() / (1024 * 1024), - MIN_POOL_SIZE, - MIN_POOL_SIZE / (1024 * 1024), + "{COMPONENT} message_bus.max_message_size ({bus_cap}) must exceed the {STATE_CHUNK_HEADER_LEN}-byte state-chunk header: state transfer serves artifact chunks over this bus, and a frame above the cap is rejected by the receiving transport, which tears down the whole replica connection" ); return Err(ConfigurationError::InvalidConfigurationValue); } - if self.enabled && !self.size.as_bytes_u64().is_multiple_of(DEFAULT_PAGE_SIZE) { + // WS frame chain: websocket.max_frame_size <= websocket.max_message_size + // <= message_bus.max_message_size. The bus's WS / WSS install path takes + // its frame tuning from [websocket], so a WS ceiling above the bus's own + // frame cap would admit messages the bus read-side validator then tears + // the connection down over. An absent knob defers to the compio-ws + // default (16 MiB frame / 64 MiB message), which satisfies the chain + // against the shipped bus cap in practice. + let bus_max_message_size = self.message_bus.max_message_size.as_bytes_u64(); + if let (Some(frame), Some(message)) = ( + self.websocket.max_frame_size, + self.websocket.max_message_size, + ) && frame.as_bytes_u64() > message.as_bytes_u64() + { eprintln!( - "Configured system.memory_pool.size {} B is not a multiple of default page size {} B", - self.size.as_bytes_u64(), - DEFAULT_PAGE_SIZE + "{COMPONENT} websocket.max_frame_size ({}) exceeds websocket.max_message_size ({})", + frame.as_bytes_u64(), + message.as_bytes_u64() ); return Err(ConfigurationError::InvalidConfigurationValue); } - - if self.enabled && self.bucket_capacity < MIN_BUCKET_CAPACITY { + if let Some(message) = self.websocket.max_message_size + && message.as_bytes_u64() > bus_max_message_size + { eprintln!( - "Configured system.memory_pool.buffers {} is less than minimum {}", - self.bucket_capacity, MIN_BUCKET_CAPACITY + "{COMPONENT} websocket.max_message_size ({}) exceeds message_bus.max_message_size ({})", + message.as_bytes_u64(), + bus_max_message_size ); return Err(ConfigurationError::InvalidConfigurationValue); } - - if self.enabled && !self.bucket_capacity.is_power_of_two() { + if let Some(frame) = self.websocket.max_frame_size + && frame.as_bytes_u64() > bus_max_message_size + { eprintln!( - "Configured system.memory_pool.buffers {} is not a power of 2", - self.bucket_capacity + "{COMPONENT} websocket.max_frame_size ({}) exceeds message_bus.max_message_size ({})", + frame.as_bytes_u64(), + bus_max_message_size ); return Err(ConfigurationError::InvalidConfigurationValue); } - Ok(()) - } -} - -/// Validate a [`CpuAllocation`] against the machine's available parallelism -/// and, when pinning, the process affinity mask. Shared by the legacy and -/// server-ng sharding configs, which both carry these two knobs. -pub(crate) fn validate_cpu_allocation( - cpu_allocation: &CpuAllocation, - pin_cores: bool, -) -> Result<(), ConfigurationError> { - let available_cpus = available_parallelism() - .map_err(|_| { - eprintln!("Failed to detect available CPU cores"); - ConfigurationError::InvalidConfigurationValue - })? - .get(); - - match cpu_allocation { - CpuAllocation::All => Ok(()), - CpuAllocation::Count(count) => { - if *count == 0 { - eprintln!("Invalid sharding configuration: cpu_allocation count cannot be 0"); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if *count > available_cpus { + // "0", "unlimited" and "none" all parse to a zero IggyByteSize. A + // zero WS tunable is never usable: zero message or frame ceilings + // reject every inbound frame, and zero buffers starve the + // compio-ws pipeline. Reject at boot instead of shipping a + // listener that cannot serve a single message. + for (key, size) in [ + ("read_buffer_size", self.websocket.read_buffer_size), + ("write_buffer_size", self.websocket.write_buffer_size), + ( + "max_write_buffer_size", + self.websocket.max_write_buffer_size, + ), + ("max_message_size", self.websocket.max_message_size), + ("max_frame_size", self.websocket.max_frame_size), + ] { + if let Some(size) = size + && size.as_bytes_u64() == 0 + { eprintln!( - "Invalid sharding configuration: cpu_allocation count {count} exceeds available CPU cores {available_cpus}" + "{COMPONENT} websocket.{key} must be non-zero (\"0\", \"unlimited\" and \"none\" all parse to zero)" ); return Err(ConfigurationError::InvalidConfigurationValue); } - Ok(()) } - CpuAllocation::Range(start, end) => { - if start >= end { - eprintln!( - "Invalid sharding configuration: cpu_allocation range {start}..{end} is invalid (start must be less than end)" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if *end - *start > available_cpus { - eprintln!( - "Invalid sharding configuration: cpu_allocation range {start}..{end} yields {} shards, exceeding available CPU cores {available_cpus}", - *end - *start - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if !pin_cores { - return Ok(()); - } - let allowed = allowed_cpus(); - if let Some(cpu) = (*start..*end).find(|cpu| !allowed.contains(cpu)) { + + // tungstenite asserts `max_write_buffer_size > write_buffer_size` + // during connection setup, so a violating pair panics on every + // accepted socket. Enforce the invariant at boot; an unset + // write_buffer_size runs at the compio-ws default. + if let Some(max_write) = self.websocket.max_write_buffer_size { + let write_buffer_size = self + .websocket + .write_buffer_size + .map_or(WS_DEFAULT_WRITE_BUFFER_SIZE, |size| size.as_bytes_u64()); + if max_write.as_bytes_u64() <= write_buffer_size { eprintln!( - "Invalid sharding configuration: cpu_allocation range {start}..{end} includes CPU {cpu}, which is outside the set of cores allowed for this process (affinity/cpuset mask)" + "{COMPONENT} websocket.max_write_buffer_size ({}) must exceed websocket.write_buffer_size ({write_buffer_size})", + max_write.as_bytes_u64() ); return Err(ConfigurationError::InvalidConfigurationValue); } - Ok(()) } - // NUMA topology validation requires hwlocality (runtime dep). - // Full NUMA validation happens in shard_allocator at startup. - CpuAllocation::NumaAware(_) => Ok(()), - } -} - -impl Validatable for ShardingConfig { - fn validate(&self) -> Result<(), ConfigurationError> { - validate_cpu_allocation(&self.cpu_allocation, self.pin_cores) - } -} -/// Length floor for the replica-auth PSK, in raw bytes. The 32-byte MAC key -/// is KDF-derived from these bytes at use-site, so any encoding clearing this -/// length is accepted. -const MIN_SHARED_SECRET_LEN: usize = 32; - -impl Validatable for ClusterConfig { - fn validate(&self) -> Result<(), ConfigurationError> { - if !self.enabled { - return Ok(()); - } + self.quic.validate().error(|e: &ConfigurationError| { + format!("{COMPONENT} (error: {e}) - failed to validate quic config") + })?; - if self.name.trim().is_empty() { - eprintln!("Invalid cluster configuration: cluster name cannot be empty"); + // Both knobs below sit on shared section structs, so the rejects live + // here rather than in those types' own `Validatable` impls. `0` / + // `disabled` / `unlimited` all parse to the same zero duration. + if self + .consumer_group + .rebalancing_timeout + .get_duration() + .is_zero() + { + eprintln!( + "{COMPONENT} consumer_group.rebalancing_timeout must be nonzero: it is the deadline after which a pending revocation completes without the source client committing what it was served, so zero force-transfers every partition on the next reconciler tick and reopens the duplicate-delivery window" + ); return Err(ConfigurationError::InvalidConfigurationValue); } - - if self.nodes.is_empty() { + if self.heartbeat.enabled && self.heartbeat.interval.get_duration().is_zero() { eprintln!( - "Invalid cluster configuration: cluster.nodes must contain at least one entry when cluster is enabled" + "{COMPONENT} heartbeat.interval must be nonzero when heartbeat.enabled: it sizes both the verifier's sleep and the staleness window, so zero spins the verifier and reaps every live session on its first pass" ); return Err(ConfigurationError::InvalidConfigurationValue); } - // VSR needs every replica to have a stable, unique id strictly - // less than the total replica count. Duplicate ids would split the - // cluster into two replicas claiming the same slot; out-of-range - // ids never win a primary election. Both are unrecoverable at - // runtime - fail fast at startup. - let total_replicas = u8::try_from(self.nodes.len()).map_err(|_| { - eprintln!("Invalid cluster configuration: more than 255 replicas is unsupported"); - ConfigurationError::InvalidConfigurationValue - })?; - - let mut seen_ids = std::collections::HashSet::new(); - let mut seen_names = std::collections::HashSet::new(); - let mut used_endpoints = std::collections::HashSet::new(); - - for node in &self.nodes { - if node.name.trim().is_empty() { - eprintln!("Invalid cluster configuration: node name cannot be empty"); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - if node.ip.trim().is_empty() { - eprintln!( - "Invalid cluster configuration: IP cannot be empty for node '{}'", - node.name - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } + reject_unsupported_and_warn_inert(self)?; - if !seen_names.insert(node.name.clone()) { - eprintln!( - "Invalid cluster configuration: duplicate node name '{}' found", - node.name - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } + Ok(()) + } +} - if node.replica_id >= total_replicas { - eprintln!( - "Invalid cluster configuration: replica_id {} for node '{}' must be < total replica count {total_replicas}", - node.replica_id, node.name - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } +/// The server parses the whole config surface but does not yet honor every +/// knob. Make the still-inert ones loud at boot: reject the unsupported +/// features (all off by default, so only a deliberate opt-in trips this) and +/// warn once for tuning knobs the server silently ignores. Warnings fire only +/// when a knob deviates from its [`ServerConfig::default`] baseline, so a +/// pristine config.toml boots without noise. The guard test below pins the +/// compared knobs against drift. +fn reject_unsupported_and_warn_inert(config: &ServerConfig) -> Result<(), ConfigurationError> { + if config.system.message_deduplication.enabled { + eprintln!("system.message_deduplication.enabled is not supported"); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if config.system.segment.archive_expired { + eprintln!("system.segment.archive_expired is not supported"); + return Err(ConfigurationError::InvalidConfigurationValue); + } + if config.system.recovery.recreate_missing_state { + eprintln!("system.recovery.recreate_missing_state is not supported"); + return Err(ConfigurationError::InvalidConfigurationValue); + } - if !seen_ids.insert(node.replica_id) { - eprintln!( - "Invalid cluster configuration: duplicate replica_id {} (two nodes claim the same slot)", - node.replica_id - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } + let defaults = ServerConfig::default(); - let port_list = [ - ("TCP", node.ports.tcp), - ("QUIC", node.ports.quic), - ("HTTP", node.ports.http), - ("WebSocket", node.ports.websocket), - ("TCP_REPLICA", node.ports.tcp_replica), - ]; - - for (name, port_opt) in &port_list { - if let Some(port) = port_opt { - if *port == 0 { - eprintln!( - "Invalid cluster configuration: {} port cannot be 0 for node '{}'", - name, node.name - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - let endpoint = format!("{}:{}", node.ip, port); - if !used_endpoints.insert(endpoint.clone()) { - eprintln!( - "Invalid cluster configuration: port conflict - {endpoint} is already bound (node '{}', transport {name})", - node.name - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - } - } - } + if config.tcp.socket.override_defaults { + warn!("tcp.socket tuning is set but not applied"); + } + if config.quic.socket.override_defaults { + warn!("quic.socket tuning is set but not applied"); + } + if config.tcp.ipv6 { + warn!("tcp.ipv6 is ignored; IPv4 vs IPv6 is decided by the tcp.address string"); + } + if config.tcp.socket_migration != defaults.tcp.socket_migration { + warn!("tcp.socket_migration is not implemented"); + } + if config.system.segment.cache_indexes != defaults.system.segment.cache_indexes { + warn!("system.segment.cache_indexes is not applied"); + } + if config.system.logging.sysinfo_print_interval + != defaults.system.logging.sysinfo_print_interval + { + warn!("system.logging.sysinfo_print_interval is not applied"); + } + if config.system.backup.path != defaults.system.backup.path + || config.system.backup.compatibility.path != defaults.system.backup.compatibility.path + { + warn!("backup is not supported"); + } + // default_algorithm deviation is already warned by the delegated legacy + // CompressionConfig::validate; only allow_override needs a signal here. + if config.system.compression.allow_override != defaults.system.compression.allow_override { + warn!( + "system.compression.allow_override is inert; live compression is per-topic from the request" + ); + } + if config.system.state.enforce_fsync != defaults.system.state.enforce_fsync + || config.system.state.max_file_operation_retries + != defaults.system.state.max_file_operation_retries + || config.system.state.retry_delay != defaults.system.state.retry_delay + { + warn!( + "system.state tuning (enforce_fsync, max_file_operation_retries, retry_delay) is not applied" + ); + } + if config.consumer_group.rebalancing_check_interval + != defaults.consumer_group.rebalancing_check_interval + { + warn!( + "consumer_group.rebalancing_check_interval is not applied; rebalancing cadence uses system.sharding.reconcile_periodic_interval" + ); + } + if config.message_saver.interval != defaults.message_saver.interval + || config.message_saver.enforce_fsync != defaults.message_saver.enforce_fsync + { + warn!("periodic message_saver is not implemented; only shutdown-flush is active"); + } - // Replica-auth PSK (only reached when the cluster is enabled; the early - // return above skips these while it is disabled). When auth is enabled - // the key is mandatory; any configured key must clear the length floor - - // a typo guard that fires with auth off too, though only while the - // cluster itself is enabled. - let secret_len = self.auth.shared_secret.len(); - if self.auth.enabled && self.auth.shared_secret.is_empty() { - eprintln!( - "Invalid cluster configuration: cluster.auth.shared_secret must be set when cluster.auth.enabled is true" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if !self.auth.shared_secret.is_empty() && secret_len < MIN_SHARED_SECRET_LEN { - eprintln!( - "Invalid cluster configuration: cluster.auth.shared_secret must be >= {MIN_SHARED_SECRET_LEN} bytes" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } + Ok(()) +} - // Replica TLS. Both cert modes run one-directional TLS (no client - // certificate anywhere), so TLS only authenticates the acceptor to - // the dialer; peer authentication comes solely from the PSK - // handshake. Without it any TLS-capable host could register as a - // replica - require auth in both modes. CA mode (the default) - // additionally needs all three PEM paths: cert/key for this node's - // acceptor side, ca_file as the dialer's trust anchor. - if self.tls.enabled { - if !self.auth.enabled { - eprintln!( - "Invalid cluster configuration: cluster.tls.enabled = true requires cluster.auth.enabled = true (TLS authenticates the acceptor only; the PSK handshake authenticates the peer)" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if !self.tls.self_signed { - for (field, value) in [ - ("cert_file", &self.tls.cert_file), - ("key_file", &self.tls.key_file), - ("ca_file", &self.tls.ca_file), - ] { - if value.trim().is_empty() { - eprintln!( - "Invalid cluster configuration: cluster.tls.{field} must be set when cluster.tls.enabled = true and self_signed = false" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - } - } - } +impl Validatable for ExtraConfig { + fn validate(&self) -> Result<(), ConfigurationError> { + self.namespace.validate().error(|e: &ConfigurationError| { + format!("{COMPONENT} (error: {e}) - failed to validate namespace config") + })?; + Ok(()) + } +} +impl Validatable for NamespaceConfig { + fn validate(&self) -> Result<(), ConfigurationError> { + IggyNamespace::validate_capacity(self.max_streams, self.max_topics, self.max_partitions) + .map_err(|error| { + eprintln!("extra.namespace is invalid: {error}"); + ConfigurationError::InvalidConfigurationValue + })?; Ok(()) } } #[cfg(test)] -mod cluster_validate_tests { +mod tests { + use super::super::cluster::{ClusterNodeConfig, TransportPorts}; use super::*; - use crate::server_config::cluster::{ - ClusterAuthConfig, ClusterConfig, ClusterNodeConfig, ClusterTlsConfig, TransportPorts, - }; + use figment::Figment; + use figment::providers::{Format, Toml}; - fn node(name: &str, id: u8) -> ClusterNodeConfig { - ClusterNodeConfig { - name: name.to_string(), - ip: "127.0.0.1".to_string(), - replica_id: id, - ports: TransportPorts::default(), - } - } + const DEFAULT_CONFIG: &str = include_str!("../../../server/config.toml"); - fn cfg(nodes: Vec) -> ClusterConfig { - ClusterConfig { - enabled: true, - name: "iggy-cluster".to_string(), - nodes, - auth: ClusterAuthConfig::default(), - tls: ClusterTlsConfig::default(), - } + /// Deep-merge a partial override over the shipped default, mirroring the + /// file-over-embedded layering the runtime loader performs. + fn config_with_override(override_toml: &str) -> ServerConfig { + Figment::new() + .merge(Toml::string(DEFAULT_CONFIG)) + .merge(Toml::string(override_toml)) + .extract() + .expect("config deserializes") } #[test] - fn validate_rejects_empty_nodes() { - let c = cfg(vec![]); - assert!(c.validate().is_err()); + fn given_shipped_default_config_when_validating_should_pass() { + let config: ServerConfig = Figment::new() + .merge(Toml::string(DEFAULT_CONFIG)) + .extract() + .expect("default config deserializes"); + config.validate().expect("pristine config must validate"); } #[test] - fn validate_rejects_duplicate_replica_ids() { - let c = cfg(vec![node("n1", 0), node("n2", 0)]); - assert!(c.validate().is_err()); + fn given_message_deduplication_enabled_when_validating_should_reject() { + let config = config_with_override("[system.message_deduplication]\nenabled = true\n"); + assert!(config.validate().is_err()); } #[test] - fn validate_rejects_duplicate_names() { - let c = cfg(vec![node("n1", 0), node("n1", 1)]); - assert!(c.validate().is_err()); + fn given_web_ui_enabled_when_validating_should_pass() { + let config = config_with_override("[http]\nweb_ui = true\n"); + config + .validate() + .expect("web_ui is served by the server and must validate"); } #[test] - fn validate_rejects_out_of_range_replica_id() { - // 2 nodes total, so id 2 is out of range. - let c = cfg(vec![node("n1", 0), node("n2", 2)]); - assert!(c.validate().is_err()); + fn given_archive_expired_enabled_when_validating_should_reject() { + let config = config_with_override("[system.segment]\narchive_expired = true\n"); + assert!(config.validate().is_err()); } #[test] - fn validate_accepts_unique_contiguous_replica_ids() { - let c = cfg(vec![node("n1", 0), node("n2", 1), node("n3", 2)]); - assert!(c.validate().is_ok()); + fn given_recreate_missing_state_enabled_when_validating_should_reject() { + let config = config_with_override("[system.recovery]\nrecreate_missing_state = true\n"); + assert!(config.validate().is_err()); } #[test] - fn validate_skips_checks_when_disabled() { - let mut c = cfg(vec![]); - c.enabled = false; - assert!(c.validate().is_ok()); + fn given_zero_message_expiry_when_validating_should_reject() { + let config = config_with_override("[system.topic]\nmessage_expiry = \"0s\"\n"); + assert!(config.validate().is_err()); } #[test] - fn validate_rejects_duplicate_tcp_replica_port() { - let ports = TransportPorts { - tcp: None, - quic: None, - http: None, - websocket: None, - tcp_replica: Some(9090), - }; - let mut n1 = node("n1", 0); - n1.ports = ports.clone(); - let mut n2 = node("n2", 1); - n2.ports = ports; - let c = cfg(vec![n1, n2]); - assert!(c.validate().is_err()); + fn given_peer_queue_capacity_not_above_repair_chunk_max_when_validating_should_reject() { + // The default repair_chunk_max (128) must stay strictly below + // peer_queue_capacity; shrinking the queue to the chunk size is the + // silent wedged-repair footgun this cross-section guard closes. + let config = config_with_override("[message_bus]\npeer_queue_capacity = 128\n"); + assert!(config.validate().is_err()); } #[test] - fn validate_rejects_cross_transport_port_reuse() { - let mut n1 = node("n1", 0); - n1.ports = TransportPorts { - tcp: Some(8090), - quic: None, - http: Some(8090), - websocket: None, - tcp_replica: None, - }; - let c = cfg(vec![n1]); - assert!( - c.validate().is_err(), - "same port on TCP and HTTP of the same node must be rejected" - ); + fn given_repair_chunk_max_at_peer_queue_capacity_when_validating_should_reject() { + let config = config_with_override("[cluster]\nrepair_chunk_max = 256\n"); + assert!(config.validate().is_err()); } #[test] - fn validate_accepts_same_port_on_different_ips() { - let mut n1 = node("n1", 0); - n1.ip = "127.0.0.1".to_string(); - n1.ports = TransportPorts { - tcp: Some(8090), - quic: None, - http: None, - websocket: None, - tcp_replica: None, - }; - let mut n2 = node("n2", 1); - n2.ip = "127.0.0.2".to_string(); - n2.ports = TransportPorts { - tcp: Some(8090), - quic: None, - http: None, - websocket: None, - tcp_replica: None, - }; - let c = cfg(vec![n1, n2]); - assert!(c.validate().is_ok()); + fn given_repair_chunk_max_below_peer_queue_capacity_when_validating_should_pass() { + let config = config_with_override("[cluster]\nrepair_chunk_max = 255\n"); + config + .validate() + .expect("a chunk below the peer queue capacity must validate"); } #[test] - fn validate_rejects_zero_tcp_replica_port() { - let ports = TransportPorts { - tcp: None, - quic: None, - http: None, - websocket: None, - tcp_replica: Some(0), - }; - let mut n1 = node("n1", 0); - n1.ports = ports; - let c = cfg(vec![n1]); - assert!(c.validate().is_err()); + fn given_ws_frame_size_above_ws_message_size_when_validating_should_reject() { + let config = config_with_override( + "[websocket]\nmax_message_size = \"1 MiB\"\nmax_frame_size = \"2 MiB\"\n", + ); + assert!(config.validate().is_err()); } + // The shipped bus cap is 64 MiB, so a 128 MiB WS ceiling breaks the chain. #[test] - fn validate_accepts_empty_secret_when_auth_disabled() { - // Default: no secret, auth off -> legacy mode, must pass. - let c = cfg(vec![node("n1", 0), node("n2", 1)]); - assert!(c.validate().is_ok()); + fn given_ws_message_size_above_bus_max_message_size_when_validating_should_reject() { + let config = config_with_override("[websocket]\nmax_message_size = \"128 MiB\"\n"); + assert!(config.validate().is_err()); } #[test] - fn validate_rejects_missing_secret_when_auth_enabled() { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.auth.enabled = true; - assert!(c.validate().is_err()); + fn given_ws_frame_size_above_bus_max_message_size_when_validating_should_reject() { + let config = config_with_override("[websocket]\nmax_frame_size = \"128 MiB\"\n"); + assert!(config.validate().is_err()); } #[test] - fn validate_rejects_short_secret_when_auth_enabled() { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.auth.enabled = true; - c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN - 1); - assert!(c.validate().is_err()); + fn given_ws_frame_chain_in_ascending_order_when_validating_should_pass() { + let config = config_with_override( + "[websocket]\nmax_message_size = \"32 MiB\"\nmax_frame_size = \"16 MiB\"\n", + ); + config + .validate() + .expect("frame <= message <= bus cap must validate"); } + // "unlimited" is not a supported sentinel for the WS size knobs: it + // parses to zero, which as a cap would reject every message. #[test] - fn validate_rejects_short_secret_even_when_auth_disabled() { - // Typo guard: a configured-but-short key fails even with auth off. - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN - 1); - assert!(c.validate().is_err()); + fn given_zero_ws_size_when_validating_should_reject() { + let config = config_with_override("[websocket]\nmax_message_size = \"unlimited\"\n"); + assert!(config.validate().is_err()); } + // tungstenite panics on this pair at connection setup; boot must + // reject it first. #[test] - fn validate_accepts_valid_secret_when_auth_enabled() { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.auth.enabled = true; - c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); - assert!(c.validate().is_ok()); - } - - fn tls_files() -> ClusterTlsConfig { - ClusterTlsConfig { - enabled: true, - self_signed: false, - cert_file: "cert.pem".to_string(), - key_file: "key.pem".to_string(), - ca_file: "ca.pem".to_string(), - } + fn given_max_write_buffer_at_write_buffer_when_validating_should_reject() { + let config = config_with_override( + "[websocket]\nwrite_buffer_size = \"256 KiB\"\nmax_write_buffer_size = \"256 KiB\"\n", + ); + assert!(config.validate().is_err()); } #[test] - fn validate_rejects_tls_ca_mode_with_missing_files() { - // Auth on so the failure exercises the file check, not the auth gate. - for missing in ["cert_file", "key_file", "ca_file"] { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.auth.enabled = true; - c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); - c.tls = tls_files(); - match missing { - "cert_file" => c.tls.cert_file.clear(), - "key_file" => c.tls.key_file.clear(), - _ => c.tls.ca_file.clear(), - } - assert!(c.validate().is_err(), "missing {missing} must be rejected"); - } + fn given_max_write_buffer_below_default_write_buffer_when_validating_should_reject() { + let config = config_with_override("[websocket]\nmax_write_buffer_size = \"64 KiB\"\n"); + assert!(config.validate().is_err()); } #[test] - fn validate_rejects_tls_self_signed_without_auth() { - // Accept-any certificate without the PSK handshake = MITM-able. - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.tls = ClusterTlsConfig { - enabled: true, - self_signed: true, - ..ClusterTlsConfig::default() - }; - assert!(c.validate().is_err()); + fn given_max_write_buffer_above_write_buffer_when_validating_should_pass() { + let config = config_with_override( + "[websocket]\nwrite_buffer_size = \"128 KiB\"\nmax_write_buffer_size = \"1 MiB\"\n", + ); + config + .validate() + .expect("max write buffer above write buffer must validate"); } + // The size knobs are strictly typed; a malformed string must fail + // deserialization at load rather than degrade to the compio-ws default. #[test] - fn validate_accepts_tls_self_signed_with_auth() { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.auth.enabled = true; - c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); - c.tls = ClusterTlsConfig { - enabled: true, - self_signed: true, - ..ClusterTlsConfig::default() - }; - assert!(c.validate().is_ok()); + fn given_malformed_ws_size_string_when_deserializing_should_reject() { + let result: Result = Figment::new() + .merge(Toml::string(DEFAULT_CONFIG)) + .merge(Toml::string( + "[websocket]\nmax_message_size = \"not-a-size\"\n", + )) + .extract(); + assert!( + result.is_err(), + "malformed websocket.max_message_size must fail config load" + ); } + // The shipped config is single-node (cluster.enabled = false), where the + // cross-section rule above is the only repair_chunk_max check that used to + // run; its structural bounds have to hold there too. #[test] - fn validate_rejects_tls_ca_mode_without_auth() { - // TLS never authenticates the dialer (no client certificates); - // only the PSK handshake does, so it is mandatory with TLS on. - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.tls = tls_files(); - assert!(c.validate().is_err()); + fn given_single_node_zero_repair_chunk_max_when_validating_should_reject() { + let config = config_with_override("[cluster]\nrepair_chunk_max = 0\n"); + assert!(config.validate().is_err()); } #[test] - fn validate_accepts_tls_ca_mode_with_auth() { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.auth.enabled = true; - c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); - c.tls = tls_files(); - assert!(c.validate().is_ok()); + fn given_single_node_repair_chunk_max_above_ceiling_when_validating_should_reject() { + // Queue widened past the chunk so the cross-section rule passes and + // only the structural ceiling can reject. + let config = config_with_override( + "[cluster]\nrepair_chunk_max = 2000\n\n[message_bus]\npeer_queue_capacity = 4096\n", + ); + assert!(config.validate().is_err()); } -} - -#[cfg(test)] -mod cluster_shards_count_determinism_tests { - use super::*; - use crate::server_config::sharding::NumaConfig; #[test] - fn all_is_rejected() { - let err = host_dependent_cpu_allocation(&CpuAllocation::All).unwrap_err(); - assert!(err.contains("all")); + fn given_zero_rebalancing_timeout_when_validating_should_reject() { + let config = config_with_override("[consumer_group]\nrebalancing_timeout = \"0\"\n"); + assert!(config.validate().is_err()); } #[test] - fn numa_auto_is_rejected() { - let err = host_dependent_cpu_allocation(&CpuAllocation::NumaAware(NumaConfig::default())) - .unwrap_err(); - assert!(err.contains("numa:auto")); + fn given_disabled_rebalancing_timeout_when_validating_should_reject() { + // "disabled" reads like an opt-out but parses to the same zero + // duration, which force-transfers every revocation instead. + let config = config_with_override("[consumer_group]\nrebalancing_timeout = \"disabled\"\n"); + assert!(config.validate().is_err()); } #[test] - fn count_is_accepted() { - assert!(host_dependent_cpu_allocation(&CpuAllocation::Count(4)).is_ok()); + fn given_zero_heartbeat_interval_when_heartbeat_enabled_should_reject() { + let config = config_with_override("[heartbeat]\nenabled = true\ninterval = \"0\"\n"); + assert!(config.validate().is_err()); } #[test] - fn range_is_accepted() { - assert!(host_dependent_cpu_allocation(&CpuAllocation::Range(0, 4)).is_ok()); + fn given_zero_heartbeat_interval_when_heartbeat_disabled_should_pass() { + let config = config_with_override("[heartbeat]\nenabled = false\ninterval = \"0\"\n"); + config + .validate() + .expect("a disabled heartbeat never reads its interval"); } + /// The warn-helper baseline is [`ServerConfig::default`], but the reused + /// legacy sections source that default from the legacy server config.toml, + /// not this NG file. Pin the knobs the helper compares so any drift between + /// the two config.toml files fails here instead of as a spurious boot warn. #[test] - fn explicit_numa_is_accepted() { - let numa = NumaConfig { - nodes: vec![0, 1], - cores_per_node: 4, - avoid_hyperthread: true, - }; - assert!(host_dependent_cpu_allocation(&CpuAllocation::NumaAware(numa)).is_ok()); + fn given_shipped_ng_config_when_compared_to_default_should_match_warned_knobs() { + let shipped: ServerConfig = Figment::new() + .merge(Toml::string(DEFAULT_CONFIG)) + .extract() + .expect("default config deserializes"); + let defaults = ServerConfig::default(); + + assert_eq!(shipped.tcp.socket_migration, defaults.tcp.socket_migration); + assert_eq!( + shipped.system.segment.cache_indexes, + defaults.system.segment.cache_indexes + ); + assert_eq!( + shipped.system.logging.sysinfo_print_interval, + defaults.system.logging.sysinfo_print_interval + ); + assert_eq!(shipped.system.backup.path, defaults.system.backup.path); + assert_eq!( + shipped.system.backup.compatibility.path, + defaults.system.backup.compatibility.path + ); + assert_eq!( + shipped.system.compression.allow_override, + defaults.system.compression.allow_override + ); + assert_eq!( + shipped.system.state.enforce_fsync, + defaults.system.state.enforce_fsync + ); + assert_eq!( + shipped.system.state.max_file_operation_retries, + defaults.system.state.max_file_operation_retries + ); + assert_eq!( + shipped.system.state.retry_delay, + defaults.system.state.retry_delay + ); + assert_eq!( + shipped.consumer_group.rebalancing_check_interval, + defaults.consumer_group.rebalancing_check_interval + ); + assert_eq!( + shipped.message_saver.interval, + defaults.message_saver.interval + ); + assert_eq!( + shipped.message_saver.enforce_fsync, + defaults.message_saver.enforce_fsync + ); } -} -#[cfg(test)] -mod sharding_cpu_range_tests { - use super::*; + // http.enabled needs a non-ServerDefault JWT expiry to clear the sibling + // check above; ServerConfig::default() already satisfies that. + fn https_config(cert_file: &str, key_file: &str) -> ServerConfig { + let mut cfg = ServerConfig::default(); + cfg.http.enabled = true; + cfg.http.tls.enabled = true; + cfg.http.tls.cert_file = cert_file.to_string(); + cfg.http.tls.key_file = key_file.to_string(); + cfg + } #[test] - fn inverted_range_is_rejected() { - let cfg = ShardingConfig { - cpu_allocation: CpuAllocation::Range(2, 2), - pin_cores: true, - }; + fn validate_rejects_tls_enabled_with_empty_cert_file() { + let cfg = https_config("", "key.pem"); assert!(cfg.validate().is_err()); } #[test] - fn pinned_range_within_allowed_set_is_accepted() { - let first = allowed_cpus()[0]; - let cfg = ShardingConfig { - cpu_allocation: CpuAllocation::Range(first, first + 1), - pin_cores: true, - }; + fn validate_accepts_tls_enabled_with_both_files_set() { + let cfg = https_config("cert.pem", "key.pem"); assert!(cfg.validate().is_ok()); } + fn cluster_node(replica_id: u8, http: Option) -> ClusterNodeConfig { + ClusterNodeConfig { + name: format!("node-{replica_id}"), + ip: "127.0.0.1".to_string(), + advertised_address: None, + advertised_addresses: Vec::new(), + replica_id, + ports: TransportPorts { + tcp: Some(8090 + u16::from(replica_id)), + quic: Some(8080 + u16::from(replica_id)), + http, + websocket: Some(8070 + u16::from(replica_id)), + tcp_replica: Some(9090 + u16::from(replica_id)), + }, + } + } + + fn clustered_http_config(nodes: Vec) -> ServerConfig { + let mut cfg = ServerConfig::default(); + cfg.http.enabled = true; + cfg.cluster.enabled = true; + cfg.cluster.name = "test-cluster".to_string(); + cfg.cluster.nodes = nodes; + cfg + } + + // Keyless cluster+http boots: forwarding degrades to off instead of + // failing the whole server. #[test] - fn pinned_range_outside_allowed_set_is_rejected() { - let past_last = allowed_cpus().last().copied().unwrap() + 1; - let cfg = ShardingConfig { - cpu_allocation: CpuAllocation::Range(past_last, past_last + 1), - pin_cores: true, - }; - assert!(cfg.validate().is_err()); + fn validate_accepts_cluster_http_without_jwt_secret_or_cluster_auth() { + let cfg = clustered_http_config(vec![ + cluster_node(0, Some(3000)), + cluster_node(1, Some(3001)), + ]); + assert!(cfg.validate().is_ok()); } + // Cluster mode has no port fallbacks, so a portless roster node is + // invalid even when forwarding is off (keyless). #[test] - fn pinned_range_wider_than_parallelism_is_rejected() { - // Under a cgroup CPU quota the affinity mask stays full while - // `available_parallelism` shrinks, so membership alone would - // accept this; the shard-count cap must reject it. - let first = allowed_cpus()[0]; - let available = available_parallelism().unwrap().get(); - let cfg = ShardingConfig { - cpu_allocation: CpuAllocation::Range(first, first + available + 1), - pin_cores: true, - }; + fn validate_rejects_keyless_cluster_http_with_portless_roster_node() { + let cfg = clustered_http_config(vec![cluster_node(0, Some(3000)), cluster_node(1, None)]); assert!(cfg.validate().is_err()); } + // The explicit-port rule covers every enabled transport, not just http. #[test] - fn unpinned_range_is_capped_by_shard_count_not_core_ids() { - // Core ids outside the machine are fine unpinned; only the - // resulting shard count matters. - let cfg = ShardingConfig { - cpu_allocation: CpuAllocation::Range(1 << 20, (1 << 20) + 1), - pin_cores: false, - }; - assert!(cfg.validate().is_ok()); - - let available = available_parallelism().unwrap().get(); - let cfg = ShardingConfig { - cpu_allocation: CpuAllocation::Range(0, available + 1), - pin_cores: false, - }; + fn validate_rejects_cluster_node_without_port_for_enabled_quic() { + let mut cfg = clustered_http_config(vec![ + cluster_node(0, Some(3000)), + cluster_node(1, Some(3001)), + ]); + cfg.quic.enabled = true; + cfg.cluster.nodes[1].ports.quic = None; assert!(cfg.validate().is_err()); } -} -#[cfg(test)] -mod sharding_embedded_default_tests { - use super::*; - use figment::Figment; - use figment::providers::{Format, Toml}; + // A disabled transport never binds, so its roster port may stay unset. + #[test] + fn validate_accepts_cluster_node_without_port_for_disabled_quic() { + let mut cfg = clustered_http_config(vec![ + cluster_node(0, Some(3000)), + cluster_node(1, Some(3001)), + ]); + cfg.quic.enabled = false; + cfg.cluster.nodes[1].ports.quic = None; + assert!(cfg.validate().is_ok()); + } - // Guards the single source of truth: the legacy sharding defaults resolve - // from the embedded legacy TOML, not hard-coded Rust values. #[test] - fn legacy_embedded_toml_resolves_sharding_defaults() { - let toml_str = include_str!("../../../server/config.toml"); - let config: ServerConfig = Figment::new() - .merge(Toml::string(toml_str)) - .extract() - .expect("embedded legacy TOML deserializes"); - config.validate().expect("embedded legacy config validates"); + fn validate_accepts_cluster_http_with_configured_jwt_secrets() { + let mut cfg = clustered_http_config(vec![ + cluster_node(0, Some(3000)), + cluster_node(1, Some(3001)), + ]); + cfg.http.jwt.encoding_secret = "0123456789abcdef0123456789abcdef".to_string(); + cfg.http.jwt.decoding_secret = "0123456789abcdef0123456789abcdef".to_string(); + assert!(cfg.validate().is_ok()); + } - assert!(config.system.sharding.pin_cores); + #[test] + fn validate_accepts_cluster_http_with_cluster_auth_as_jwt_key_source() { + let mut cfg = clustered_http_config(vec![ + cluster_node(0, Some(3000)), + cluster_node(1, Some(3001)), + ]); + cfg.cluster.auth.enabled = true; + cfg.cluster.auth.shared_secret = "0123456789abcdef0123456789abcdef".to_string(); + assert!(cfg.validate().is_ok()); } } diff --git a/core/configs/src/server_config/websocket.rs b/core/configs/src/server_config/websocket.rs index 30f7dc5c64..85120daa61 100644 --- a/core/configs/src/server_config/websocket.rs +++ b/core/configs/src/server_config/websocket.rs @@ -15,28 +15,76 @@ // specific language governing permissions and limitations // under the License. +//! WebSocket listener schema. +//! +//! This section is the live frame-tuning source for the WS / WSS +//! plane: the message bus folds the +//! `Option` knobs below into a compio-ws +//! `WebSocketConfig` once at bus construction. The sizes are strictly +//! typed, so a malformed size string fails config load instead of +//! being silently ignored at conversion time. The conversion itself +//! lives in `core/message_bus` because the standalone `tungstenite` +//! dependency and the compio-ws re-export are different major versions +//! with incompatible config types. + use configs::ConfigEnv; use iggy_common::IggyByteSize; use serde::{Deserialize, Serialize}; +use serde_with::{DisplayFromStr, serde_as}; use std::fmt::{Display, Formatter}; -use tungstenite::protocol::WebSocketConfig as TungsteniteConfig; +#[serde_as] #[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] pub struct WebSocketConfig { pub enabled: bool, pub address: String, + + /// Target minimum size of the frame read buffer. `None` keeps the + /// compio-ws default (currently 128 KiB). + #[config_env(leaf)] #[serde(default)] - pub read_buffer_size: Option, + #[serde_as(as = "Option")] + pub read_buffer_size: Option, + + /// Target buffer size for batched writes before compio-ws flushes. + /// `None` keeps the compio-ws default (currently 128 KiB). + #[config_env(leaf)] #[serde(default)] - pub write_buffer_size: Option, + #[serde_as(as = "Option")] + pub write_buffer_size: Option, + + /// Hard ceiling on the write buffer; writes past it error instead + /// of buffering. Must exceed [`Self::write_buffer_size`] by at + /// least one frame. `None` keeps the compio-ws default + /// (unlimited). + #[config_env(leaf)] #[serde(default)] - pub max_write_buffer_size: Option, + #[serde_as(as = "Option")] + pub max_write_buffer_size: Option, + + /// Hard upper bound on a single inbound WebSocket message + /// (post-fragment-reassembly). `None` keeps the compio-ws default + /// (currently 64 MiB). + #[config_env(leaf)] #[serde(default)] - pub max_message_size: Option, + #[serde_as(as = "Option")] + pub max_message_size: Option, + + /// Hard upper bound on a single inbound WebSocket frame + /// (pre-fragment-reassembly). `None` keeps the compio-ws default + /// (currently 16 MiB). + #[config_env(leaf)] #[serde(default)] - pub max_frame_size: Option, + #[serde_as(as = "Option")] + pub max_frame_size: Option, + + /// Whether to accept unmasked frames from clients in violation of + /// RFC 6455 client-to-server framing rules. Strict (`false`) by + /// default; enable only for non-browser test clients that emit + /// unmasked frames. #[serde(default)] pub accept_unmasked_frames: bool, + #[serde(default)] pub tls: WebSocketTlsConfig, } @@ -49,46 +97,6 @@ pub struct WebSocketTlsConfig { pub key_file: String, } -impl WebSocketConfig { - pub fn to_tungstenite_config(&self) -> TungsteniteConfig { - let mut config = TungsteniteConfig::default(); - - if let Some(read_buf_size_str) = &self.read_buffer_size - && let Ok(byte_size) = read_buf_size_str.parse::() - { - config = config.read_buffer_size(byte_size.as_bytes_u64() as usize); - } - - if let Some(write_buf_size_str) = &self.write_buffer_size - && let Ok(byte_size) = write_buf_size_str.parse::() - { - config = config.write_buffer_size(byte_size.as_bytes_u64() as usize); - } - - if let Some(max_write_buf_size_str) = &self.max_write_buffer_size - && let Ok(byte_size) = max_write_buf_size_str.parse::() - { - config = config.max_write_buffer_size(byte_size.as_bytes_u64() as usize); - } - - if let Some(msg_size_str) = &self.max_message_size - && let Ok(byte_size) = msg_size_str.parse::() - { - config = config.max_message_size(Some(byte_size.as_bytes_u64() as usize)); - } - - if let Some(frame_size_str) = &self.max_frame_size - && let Ok(byte_size) = frame_size_str.parse::() - { - config = config.max_frame_size(Some(byte_size.as_bytes_u64() as usize)); - } - - config = config.accept_unmasked_frames(self.accept_unmasked_frames); - - config - } -} - impl Display for WebSocketConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!( diff --git a/core/configs/src/server_ng_config/cluster.rs b/core/configs/src/server_ng_config/cluster.rs deleted file mode 100644 index 9a6b38fc62..0000000000 --- a/core/configs/src/server_ng_config/cluster.rs +++ /dev/null @@ -1,2612 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Server-ng cluster schema. -//! -//! Field shape mirrors the legacy [`crate::cluster::ClusterConfig`] -//! plus the ng-only `heartbeat_timeout` knob; the type is forked into -//! `server_ng_config` so server-ng can evolve its cluster surface -//! (VSR consensus tunables) independently of the legacy server. - -use super::defaults::SERVER_NG_CONFIG; -use crate::ConfigurationError; -use crate::http::HttpJwtConfig; -use configs::ConfigEnv; -use iggy_common::{IggyDuration, Validatable}; -use ipnet::{IpNet, Ipv4Net}; -use serde::{Deserialize, Serialize}; -use serde_with::{DisplayFromStr, serde_as}; -use std::cmp::Reverse; -use std::fmt; -use std::net::{IpAddr, Ipv6Addr, SocketAddr}; -use std::str::FromStr; -use std::time::Duration; - -/// Absolute floor for the backup liveness window, independent of the -/// commit-broadcast rate. The primary signals liveness through its commit -/// broadcast (`commit_broadcast_interval`, 500ms by default); 2s spans several -/// broadcasts, so a single delayed one never elects. The per-config -/// `MIN_HEARTBEAT_TO_COMMIT_BROADCAST_RATIO` check scales the same headroom -/// when the broadcast interval is retuned. -pub const MIN_CLUSTER_HEARTBEAT_TIMEOUT: Duration = Duration::from_secs(2); - -/// The backup liveness window (`heartbeat_timeout`) must span at least this -/// many commit broadcasts (`commit_broadcast_interval`), so one dropped or -/// delayed broadcast never trips a view change on a healthy primary. -const MIN_HEARTBEAT_TO_COMMIT_BROADCAST_RATIO: u32 = 4; - -/// The view-change status backstop (`view_change_status_timeout`) must span at -/// least this many retransmit intervals (`view_change_retransmit_interval`), so -/// a few dropped `StartViewChange` / `DoViewChange` messages retransmit rather -/// than escalating a progressing view change into a fresh cluster-wide election. -const MIN_STATUS_TO_RETRANSMIT_RATIO: u32 = 4; - -/// Default recovering-replica probe-attempt ceiling. Duplicated here rather -/// than imported so `core/configs` keeps off a build-time edge onto -/// `core/consensus` (mirroring [`super::partition`]); `core/server-ng`'s -/// bootstrap static-asserts it equal to `consensus::PROBE_ATTEMPTS_MAX`. -pub const DEFAULT_VIEW_PROBE_ATTEMPTS_MAX: u32 = 5; - -/// Upper bound on `view_probe_attempts_max`. A recovering replica probes once -/// per `request_start_view_retransmit_interval`, so hundreds of attempts would -/// stall the election fallback for minutes on a full-cluster restart; this is a -/// typo guard, not a sizing endorsement. -const MAX_VIEW_PROBE_ATTEMPTS: u32 = 100; - -/// Default per-round repair-serving chunk. Duplicated here rather than imported -/// so `core/configs` keeps off a build-time edge onto `core/shard` (mirroring -/// [`DEFAULT_VIEW_PROBE_ATTEMPTS_MAX`]); `core/server-ng`'s bootstrap -/// static-asserts it equal to `shard::REPAIR_CHUNK_MAX`. -pub const DEFAULT_REPAIR_CHUNK_MAX: usize = 128; - -/// `size_of::()`. Duplicated here for the same reason as -/// [`DEFAULT_REPAIR_CHUNK_MAX`]; `core/server-ng`'s bootstrap static-asserts it -/// against the real header. Used to reject a `[message_bus] max_message_size` -/// too small to carry a single state-transfer chunk. -pub const STATE_CHUNK_HEADER_LEN: u64 = 256; - -/// Upper bound on `repair_chunk_max`. A chunk rides the per-peer bus queue, so -/// the load-bearing rule is `repair_chunk_max < message_bus.peer_queue_capacity` -/// (enforced at the top level); this standalone ceiling is a typo guard. -const MAX_REPAIR_CHUNK_MAX: usize = 1024; - -/// Upper bound on per-node `advertised_addresses` selectors. Must mirror the -/// `#[config_env(max_elements = 16)]` cap on the field: env overrides above -/// the cap do not exist, so a TOML roster exceeding it could never be -/// replicated through the env path. Also bounds the cross-node conflict scan -/// (quadratic in pooled entries) and the per-request longest-prefix walk. -const MAX_ADVERTISED_SELECTORS: usize = 16; - -/// Length floor for the replica-auth PSK, in raw bytes. The 32-byte MAC key -/// is KDF-derived from these bytes at use-site, so any encoding clearing this -/// length is accepted. -const MIN_SHARED_SECRET_LEN: usize = 32; - -/// DNS caps a full name at 255 octets on the wire, which leaves 253 -/// characters of presentation text (RFC 1035). -const MAX_HOSTNAME_LEN: usize = 253; - -/// Per-label limit from RFC 1035. -const MAX_HOSTNAME_LABEL_LEN: usize = 63; - -/// serde fallback for configs written before the field existed; the value -/// itself lives in `core/server-ng/config.toml` like every other default. -fn default_heartbeat_timeout() -> IggyDuration { - SERVER_NG_CONFIG.cluster.heartbeat_timeout.parse().unwrap() -} - -/// serde fallback for configs written before the field existed; the value -/// itself lives in `core/server-ng/config.toml` like every other default. -fn default_commit_broadcast_interval() -> IggyDuration { - SERVER_NG_CONFIG - .cluster - .commit_broadcast_interval - .parse() - .unwrap() -} - -/// serde fallback for configs written before the field existed; the value -/// itself lives in `core/server-ng/config.toml` like every other default. -fn default_prepare_retransmit_interval() -> IggyDuration { - SERVER_NG_CONFIG - .cluster - .prepare_retransmit_interval - .parse() - .unwrap() -} - -/// serde fallback for configs written before the field existed; the value -/// itself lives in `core/server-ng/config.toml` like every other default. -fn default_view_change_retransmit_interval() -> IggyDuration { - SERVER_NG_CONFIG - .cluster - .view_change_retransmit_interval - .parse() - .unwrap() -} - -/// serde fallback for configs written before the field existed; the value -/// itself lives in `core/server-ng/config.toml` like every other default. -fn default_view_change_status_timeout() -> IggyDuration { - SERVER_NG_CONFIG - .cluster - .view_change_status_timeout - .parse() - .unwrap() -} - -/// serde fallback for configs written before the field existed; the value -/// itself lives in `core/server-ng/config.toml` like every other default. -fn default_request_start_view_retransmit_interval() -> IggyDuration { - SERVER_NG_CONFIG - .cluster - .request_start_view_retransmit_interval - .parse() - .unwrap() -} - -/// serde fallback for configs written before the field existed; the value -/// itself lives in `core/server-ng/config.toml` like every other default. -fn default_view_probe_attempts_max() -> u32 { - SERVER_NG_CONFIG.cluster.view_probe_attempts_max as u32 -} - -/// serde fallback for configs written before the field existed; the value -/// itself lives in `core/server-ng/config.toml` like every other default. -fn default_repair_retry_interval() -> IggyDuration { - SERVER_NG_CONFIG - .cluster - .repair_retry_interval - .parse() - .unwrap() -} - -/// serde fallback for configs written before the field existed; the value -/// itself lives in `core/server-ng/config.toml` like every other default. -fn default_repair_chunk_max() -> usize { - SERVER_NG_CONFIG.cluster.repair_chunk_max as usize -} - -#[serde_as] -#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] -#[serde(deny_unknown_fields)] -pub struct ClusterConfig { - pub enabled: bool, - pub name: String, - /// Backup-side liveness window for a plane's primary. A replica that - /// sees no primary traffic for this long starts a view change - /// (`normal_heartbeat_timeout`). Raise it on oversubscribed hosts where - /// scheduling stalls fake primary death; sub-`MIN_CLUSTER_HEARTBEAT_TIMEOUT` - /// values (including the `0` / `disabled` / `unlimited` sentinels, which - /// all parse to zero) are rejected at boot. - #[serde(default = "default_heartbeat_timeout")] - #[serde_as(as = "DisplayFromStr")] - #[config_env(leaf)] - pub heartbeat_timeout: IggyDuration, - /// How often the primary broadcasts its commit point to every backup, the - /// cluster's primary-liveness signal. Each broadcast resets the backups' - /// `heartbeat_timeout` window, so that window must span several broadcasts: - /// boot rejects `heartbeat_timeout < MIN_HEARTBEAT_TO_COMMIT_BROADCAST_RATIO - /// * commit_broadcast_interval`. Sizes the consensus `CommitMessage` timer. - /// Zero (and the `0` / `disabled` / `unlimited` sentinels, which all parse - /// to zero) is rejected at boot. - #[serde(default = "default_commit_broadcast_interval")] - #[serde_as(as = "DisplayFromStr")] - #[config_env(leaf)] - pub commit_broadcast_interval: IggyDuration, - /// How often the primary retransmits prepares a backup has not yet acked. - /// Lower recovers faster from a dropped prepare at the cost of replica - /// traffic. Sizes the consensus `Prepare` timer. Zero (and the `0` / - /// `disabled` / `unlimited` sentinels, which all parse to zero) is rejected - /// at boot. - #[serde(default = "default_prepare_retransmit_interval")] - #[serde_as(as = "DisplayFromStr")] - #[config_env(leaf)] - pub prepare_retransmit_interval: IggyDuration, - /// How often a plane retransmits its `StartViewChange` / `DoViewChange` - /// while a view change is running. Lower converges a healthy election - /// faster at the cost of replica traffic. Sizes both consensus view-change - /// retransmit timers, which are deliberately equal. Zero (and the `0` / - /// `disabled` / `unlimited` sentinels, which all parse to zero) is rejected - /// at boot. - #[serde(default = "default_view_change_retransmit_interval")] - #[serde_as(as = "DisplayFromStr")] - #[config_env(leaf)] - pub view_change_retransmit_interval: IggyDuration, - /// Backstop for a stalled view change: one that does not conclude within - /// this window escalates to a fresh cluster-wide election. Must span - /// several `view_change_retransmit_interval`s so a few dropped view-change - /// messages retransmit rather than escalate: boot rejects - /// `view_change_status_timeout < MIN_STATUS_TO_RETRANSMIT_RATIO * - /// view_change_retransmit_interval`. Zero (and the `0` / `disabled` / - /// `unlimited` sentinels, which all parse to zero) is rejected at boot. - #[serde(default = "default_view_change_status_timeout")] - #[serde_as(as = "DisplayFromStr")] - #[config_env(leaf)] - pub view_change_status_timeout: IggyDuration, - /// How often a recovering or view-change backup re-requests the current - /// view's `StartView` from its primary (`RequestStartView`). Sizes the - /// consensus `RequestStartView` timer. Zero (and the `0` / `disabled` / - /// `unlimited` sentinels, which all parse to zero) is rejected at boot. - #[serde(default = "default_request_start_view_retransmit_interval")] - #[serde_as(as = "DisplayFromStr")] - #[config_env(leaf)] - pub request_start_view_retransmit_interval: IggyDuration, - /// How many consecutive unanswered `RequestStartView` probes a recovering - /// replica tolerates before it falls back to an election (a full-cluster - /// restart leaves nobody settled to answer). Must be >= 1 and <= - /// `MAX_VIEW_PROBE_ATTEMPTS`. - #[serde(default = "default_view_probe_attempts_max")] - pub view_probe_attempts_max: u32, - /// How long a stalled journal-repair stream waits before re-requesting its - /// remaining window from the serving peer. Repair frames are - /// fire-and-forget over the lossy bus, so a session with no retry wedges - /// forever on a single dropped frame. Paces both the metadata and - /// partition repair loops. Sizes the retry threshold in consensus ticks. - /// Zero (and the `0` / `disabled` / `unlimited` sentinels, which all parse - /// to zero) is rejected at boot. - #[serde(default = "default_repair_retry_interval")] - #[serde_as(as = "DisplayFromStr")] - #[config_env(leaf)] - pub repair_retry_interval: IggyDuration, - /// Prepares a peer serves per repair round before the requester walks to - /// the next chunk. Each frame rides the per-peer message-bus queue, so this - /// must stay below `message_bus.peer_queue_capacity` or a full round - /// overruns the queue and silently drops frames (enforced at the top - /// level). Applies to both the metadata and partition repair planes. Must - /// be > 0 and <= `MAX_REPAIR_CHUNK_MAX`. - #[serde(default = "default_repair_chunk_max")] - pub repair_chunk_max: usize, - /// Full roster of cluster members. Intended to be byte-identical across - /// every node so operators ship one config. The running node's identity - /// is supplied out-of-band via the `--replica-id` CLI flag, which - /// selects the entry in this list that describes the current node. - #[serde(default)] - pub nodes: Vec, - /// Replica-to-replica authentication settings (PSK + BLAKE3 handshake). - #[serde(default)] - pub auth: ClusterAuthConfig, - /// Replica-to-replica TLS settings for the consensus (`tcp_replica`) port. - #[serde(default)] - pub tls: ClusterTlsConfig, -} - -/// Replica-to-replica authentication for the consensus (`tcp_replica`) port. -#[derive(Debug, Default, Deserialize, Serialize, Clone, ConfigEnv)] -#[serde(deny_unknown_fields)] -pub struct ClusterAuthConfig { - /// When true, every replica peer must complete the authenticated handshake - /// or be rejected, and [`Self::shared_secret`] is mandatory. When false - /// (default) the replica handshake stays in legacy unauthenticated mode and - /// `shared_secret` is not used for authentication. A configured non-empty - /// `shared_secret` must still meet the 32-byte minimum whenever the cluster - /// is enabled (a short value fails boot even with auth off). - /// - /// Enabling auth is a coordinated-restart change, and not the only one: the - /// consensus `cluster_id` is derived from `ClusterConfig::name` - /// unconditionally, so a mixed-version roster fails to connect regardless of - /// this flag. Flip every node in one restart. - #[serde(default)] - pub enabled: bool, - /// Cluster-wide pre-shared key for replica-to-replica authentication. - /// - /// At least 32 bytes of CSPRNG output, byte-identical across every node. - /// Provisioned out-of-band, normally via `IGGY_CLUSTER_AUTH_SHARED_SECRET` - /// rather than the on-disk config. - // skip_serializing keeps the PSK out of the runtime `current_config.toml` - // (and the `ServerConfig` diagnostic snapshot that cats it). The live - // secret is read from env / on-disk config at boot, never from the - // snapshot, so it must never be persisted there. Deserialize is retained. - #[serde(default, skip_serializing)] - #[config_env(secret)] - pub shared_secret: String, - /// Retiring pre-shared key, accepted for VERIFICATION only during a key - /// rotation window; every MAC this node produces uses [`Self::shared_secret`]. - /// - /// Enables rolling PSK rotation without an auth outage, three rolls: - /// 1. every node gets `shared_secret = old, previous_shared_secret = new`; - /// 2. every node gets `shared_secret = new, previous_shared_secret = old`; - /// 3. every node gets `shared_secret = new` alone, closing the window. - /// - /// Leave empty (default) outside a rotation. Same 32-byte minimum and - /// provisioning rules as `shared_secret` - /// (`IGGY_CLUSTER_AUTH_PREVIOUS_SHARED_SECRET`). - #[serde(default, skip_serializing)] - #[config_env(secret)] - pub previous_shared_secret: String, -} - -/// Replica-to-replica TLS for the consensus (`tcp_replica`) port. -/// -/// Mirrors the legacy [`crate::tcp::TcpTlsConfig`] shape plus `ca_file`: -/// the replica plane DIALS its peers (a TLS client role the -/// client-facing server plane never has), so the dialer needs a trust -/// anchor to verify the acceptor's certificate against. -#[derive(Debug, Default, Deserialize, Serialize, Clone, ConfigEnv)] -#[serde(deny_unknown_fields)] -pub struct ClusterTlsConfig { - /// When true every replica connection is wrapped in TLS (1.3 only) - /// before the replica handshake runs. Requires `cluster.auth.enabled`: - /// TLS carries no client certificates, so it authenticates the - /// acceptor only; the PSK handshake authenticates the peer while TLS - /// supplies confidentiality. Enabling is a coordinated-restart - /// change: a TLS dialer cannot talk to a plaintext acceptor or vice - /// versa. Flip every node in one restart. - #[serde(default)] - pub enabled: bool, - /// When true the node auto-generates a self-signed certificate at - /// boot and the dialer accepts ANY peer certificate. With the - /// default `false`, `cert_file` / `key_file` / `ca_file` are all - /// required. - #[serde(default)] - pub self_signed: bool, - /// PEM certificate chain presented by this node's acceptor side. - #[serde(default)] - pub cert_file: String, - /// PEM private key matching `cert_file`. - #[serde(default)] - pub key_file: String, - /// PEM trust anchor(s) the dialer verifies peer certificates - /// against. Unused when `self_signed` is true. - #[serde(default)] - pub ca_file: String, -} - -#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] -#[serde(deny_unknown_fields)] -pub struct ClusterNodeConfig { - pub name: String, - pub ip: String, - /// Optional client-facing address: a literal IP or a DNS hostname, - /// validated as [`AdvertisedAddress`] at boot. Replica traffic continues - /// to use [`Self::ip`]. - #[serde(default)] - pub advertised_address: Option, - /// Client-network-scoped overrides of [`Self::advertised_address`], - /// resolved by longest-prefix match over the client's IP (see - /// [`AdvertisedAddressSelector`]). Empty by default, so existing configs - /// keep the single catch-all address. At most `MAX_ADVERTISED_SELECTORS` - /// per node (validated), matching the env-override cap below. - #[serde(default)] - #[config_env(max_elements = 16)] - pub advertised_addresses: Vec, - /// Numeric replica ID for VSR consensus (0-based). - /// - /// Must be unique across [`ClusterConfig::nodes`] and strictly less than - /// `nodes.len()`. Validated by [`ClusterConfig::validate`]. - pub replica_id: u8, - pub ports: TransportPorts, -} - -/// One client-network-scoped advertised address: clients whose IP falls -/// inside `client_cidr` are told `address` instead of the node's catch-all -/// [`ClusterNodeConfig::advertised_address`]. -/// -/// Typical split-network case: the roster `ip` is VPC-private and -/// `advertised_address` is public; a selector with the VPC CIDR keeps -/// in-VPC clients on the private address while everyone else stays on the -/// public one. Selection is longest-prefix match across a node's selectors. -/// Selection sees the transport-level peer address, so clients arriving -/// through a proxy or load balancer match the proxy's network, not their -/// own. -#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] -#[serde(deny_unknown_fields)] -pub struct AdvertisedAddressSelector { - /// Client network this selector matches, in CIDR notation - /// (`10.0.0.0/16`, `2001:db8::/32`). Must parse at boot; duplicate - /// networks within one node are rejected. A v4-mapped v6 network - /// (`::ffff:10.0.0.0/104`) canonicalizes to its v4 form (`10.0.0.0/8`), - /// matching how client IPs canonicalize before matching. - pub client_cidr: String, - /// Address advertised to matching clients: a literal IP or a DNS - /// hostname, validated as [`AdvertisedAddress`] at boot (no port; ports - /// come from [`ClusterNodeConfig::ports`]). - pub address: String, -} - -/// A roster node with its advertised-address selectors and catch-all parsed -/// once, built wherever a roster is assembled for serving clients -/// (listener/shard start). Per-request resolution never re-parses config -/// strings: everything is snapshotted here, so mutating the source config -/// after conversion has no effect on what clients are told. Entries that do -/// not parse are dropped at build time; validation already rejects them -/// whenever the cluster is enabled, and a disabled cluster never consults -/// the roster. -#[derive(Debug, Clone)] -pub struct ResolvedClusterNode { - config: ClusterNodeConfig, - /// Truncated, canonicalized selector networks with their parsed - /// addresses, in declaration order. - selectors: Vec<(IpNet, AdvertisedAddress)>, - /// Parsed catch-all: [`ClusterNodeConfig::advertised_address`], else the - /// roster [`ClusterNodeConfig::ip`]. `None` when the configured value - /// does not parse - a set `advertised_address` never falls through to - /// the private roster ip. - catch_all: Option, - /// Parsed roster [`ClusterNodeConfig::ip`], the replica-plane dial - /// address. `None` when the roster ip is not a literal IP (boot only - /// requires it non-empty); internal forwarding then has no dial target. - replica_ip: Option, -} - -impl From for ResolvedClusterNode { - fn from(config: ClusterNodeConfig) -> Self { - let selectors = config - .advertised_addresses - .iter() - .filter_map(|selector| { - let network = selector.client_cidr.parse::().ok()?; - let address = selector.address.parse::().ok()?; - Some((canonical_ip_net(network.trunc()), address)) - }) - .collect(); - let catch_all = match config.advertised_address.as_deref() { - Some(advertised_address) => advertised_address.parse().ok(), - None => config.ip.parse().ok(), - }; - let replica_ip = config.ip.parse().ok(); - Self { - config, - selectors, - catch_all, - replica_ip, - } - } -} - -impl ResolvedClusterNode { - /// The roster entry this node was built from. Read-only: resolution runs - /// on the boot-parsed snapshot, never on the config strings. - #[must_use] - pub fn config(&self) -> &ClusterNodeConfig { - &self.config - } - - /// The roster `ip` as a dialable address for the replica plane and - /// internal request forwarding. Never routed through the advertised - /// ladder: this is what servers dial, not what clients are told. - #[must_use] - pub fn replica_ip(&self) -> Option { - self.replica_ip - } - - /// The client-facing address for a client connecting from `client_ip`: - /// longest-prefix match over the selector networks, then the parsed - /// catch-all. `None` when no selector matches and the catch-all did not - /// parse; callers choose whether to fail closed (redirect URLs) or to - /// publish [`Self::raw_advertised_fallback`] verbatim (cluster metadata). - #[must_use] - pub fn advertised_for(&self, client_ip: Option) -> Option<&AdvertisedAddress> { - client_ip - .and_then(|client_ip| self.selector_address(client_ip)) - .or(self.catch_all.as_ref()) - } - - /// The catch-all ladder ([`ClusterNodeConfig::advertised_address`], else - /// the roster [`ClusterNodeConfig::ip`]) as configured, unparsed. Cluster - /// metadata publishes this verbatim when [`Self::advertised_for`] finds - /// nothing: the roster `ip` is only validated non-empty, and Docker - /// service names with underscores exist in the wild. - #[must_use] - pub fn raw_advertised_fallback(&self) -> &str { - self.config - .advertised_address - .as_deref() - .unwrap_or(&self.config.ip) - } - - /// Longest-prefix match over the boot-parsed selector networks. The - /// client IP is canonicalized first so a v4-mapped v6 peer - /// (`::ffff:10.0.0.7`, the shape a dual-stack listener reports) matches - /// v4 networks. `min_by_key` keeps the first of equal-length matches, so - /// resolution stays declaration-order deterministic even though a - /// validated config cannot produce two matching networks of equal length - /// (equal-length distinct networks are disjoint, duplicates are - /// rejected). - fn selector_address(&self, client_ip: IpAddr) -> Option<&AdvertisedAddress> { - let client_ip = client_ip.to_canonical(); - self.selectors - .iter() - .filter(|(network, _)| network.contains(&client_ip)) - .min_by_key(|(network, _)| Reverse(network.prefix_len())) - .map(|(_, address)| address) - } -} - -/// Network-side mirror of the `IpAddr::to_canonical` applied to client IPs -/// before matching: a selector network written in v4-mapped v6 form -/// (`::ffff:10.0.0.0/104`) becomes its v4 equivalent (`10.0.0.0/8`), since a -/// canonicalized client could never match the v6 spelling. Prefixes shorter -/// than 96 bits cannot drop the `::ffff:` mapping and stay v6 (they match -/// native v6 clients only). -fn canonical_ip_net(network: IpNet) -> IpNet { - if let IpNet::V6(v6_network) = network - && v6_network.prefix_len() >= 96 - && let IpAddr::V4(v4_address) = v6_network.addr().to_canonical() - && let Ok(v4_network) = Ipv4Net::new(v4_address, v6_network.prefix_len() - 96) - { - return IpNet::V4(v4_network); - } - network -} - -/// Per-node listener ports advertised in the cluster roster. In cluster mode -/// the roster is the single source of ports: every enabled transport needs -/// an explicit per-node port (validated at startup, no fallback to the -/// transport's top-level `address` port). The roster entry's `ip` is the -/// advertised address only: tcp/ws/quic/http bind the interface from their own -/// `address` config, and followers forward HTTP requests to the primary at -/// `ip:http`. -#[derive(Debug, Deserialize, Serialize, Clone, Default, ConfigEnv)] -pub struct TransportPorts { - pub tcp: Option, - pub quic: Option, - pub http: Option, - pub websocket: Option, - /// Dedicated port for replica-to-replica consensus traffic. - pub tcp_replica: Option, -} - -/// A validated client-facing node address: a literal IP or a DNS hostname. -/// -/// Hostnames follow RFC 1123: ASCII letters, digits and hyphens in labels of -/// 1-63 characters that do not start or end with a hyphen, at most -/// [`MAX_HOSTNAME_LEN`] characters total, no port and no trailing dot. Names -/// consisting solely of digits and dots are rejected as malformed IPv4 rather -/// than accepted as hostnames, so `10.0.0.256` fails loudly instead of being -/// handed to DNS. Hostnames normalize to lowercase and IPs to their canonical -/// form ([`IpAddr`]), so textual variants of one address (`Broker.Example.COM`, -/// `2001:DB8::1`, `[2001:db8::1]`) compare equal. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub enum AdvertisedAddress { - Ip(IpAddr), - Hostname(String), -} - -impl AdvertisedAddress { - /// Render `host:port` for a URL or endpoint listing, bracketing IPv6 - /// hosts (`[::1]:8080`) so the port separator stays unambiguous. - pub fn authority(&self, port: u16) -> String { - match self { - Self::Ip(ip) => SocketAddr::new(*ip, port).to_string(), - Self::Hostname(hostname) => format!("{hostname}:{port}"), - } - } -} - -impl FromStr for AdvertisedAddress { - type Err = AdvertisedAddressError; - - fn from_str(address: &str) -> Result { - if address.is_empty() { - return Err(AdvertisedAddressError::Empty); - } - if let Ok(ip) = address.parse::() { - return Ok(Self::Ip(ip)); - } - // URL-style bracketed IPv6 (`[2001:db8::1]`) is unambiguous; accept - // it and store the inner address. - if let Some(inner) = address - .strip_prefix('[') - .and_then(|rest| rest.strip_suffix(']')) - && let Ok(ip) = inner.parse::() - { - return Ok(Self::Ip(IpAddr::V6(ip))); - } - if let Some((host, port)) = address.rsplit_once(':') { - // `host:port` and `[v6]:port` are the common misconfigurations; - // anything else with a colon can only be a broken IPv6 literal, - // since ':' never appears in a hostname. - let bracketed_host = host.starts_with('[') && host.ends_with(']'); - if !port.is_empty() - && port.bytes().all(|byte| byte.is_ascii_digit()) - && (bracketed_host || !host.contains(':')) - { - return Err(AdvertisedAddressError::PortNotAllowed); - } - return Err(AdvertisedAddressError::MalformedIpv6); - } - if address.len() > MAX_HOSTNAME_LEN { - return Err(AdvertisedAddressError::HostnameTooLong { - length: address.len(), - }); - } - let mut all_labels_numeric = true; - for label in address.split('.') { - if label.is_empty() { - return Err(AdvertisedAddressError::EmptyLabel); - } - if label.len() > MAX_HOSTNAME_LABEL_LEN { - return Err(AdvertisedAddressError::LabelTooLong { - label: label.to_owned(), - }); - } - if label.starts_with('-') || label.ends_with('-') { - return Err(AdvertisedAddressError::LabelHyphen { - label: label.to_owned(), - }); - } - if let Some(character) = label - .chars() - .find(|character| !character.is_ascii_alphanumeric() && *character != '-') - { - return Err(AdvertisedAddressError::InvalidCharacter { character }); - } - all_labels_numeric &= label.bytes().all(|byte| byte.is_ascii_digit()); - } - if all_labels_numeric { - return Err(AdvertisedAddressError::MalformedIpv4); - } - // DNS resolution is case-insensitive; normalizing here makes equality - // (and thus endpoint-conflict detection) case-insensitive too. - Ok(Self::Hostname(address.to_ascii_lowercase())) - } -} - -impl fmt::Display for AdvertisedAddress { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Ip(ip) => write!(formatter, "{ip}"), - Self::Hostname(hostname) => write!(formatter, "{hostname}"), - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum AdvertisedAddressError { - Empty, - PortNotAllowed, - MalformedIpv4, - MalformedIpv6, - HostnameTooLong { length: usize }, - EmptyLabel, - LabelTooLong { label: String }, - LabelHyphen { label: String }, - InvalidCharacter { character: char }, -} - -impl fmt::Display for AdvertisedAddressError { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Empty => write!(formatter, "address cannot be empty"), - Self::PortNotAllowed => write!( - formatter, - "address must not include a port; ports are configured in cluster.nodes.ports" - ), - Self::MalformedIpv4 => write!( - formatter, - "address consists only of digits and dots but is not a valid IPv4 address" - ), - Self::MalformedIpv6 => write!( - formatter, - "address contains ':' but is not a valid IPv6 address, and ':' cannot appear in a hostname" - ), - Self::HostnameTooLong { length } => write!( - formatter, - "hostname is {length} characters long; the limit is {MAX_HOSTNAME_LEN}" - ), - Self::EmptyLabel => write!( - formatter, - "hostname contains an empty label (leading, trailing, or doubled dot)" - ), - Self::LabelTooLong { label } => write!( - formatter, - "hostname label '{label}' exceeds {MAX_HOSTNAME_LABEL_LEN} characters" - ), - Self::LabelHyphen { label } => write!( - formatter, - "hostname label '{label}' cannot start or end with a hyphen" - ), - Self::InvalidCharacter { character } => write!( - formatter, - "character '{character}' is not allowed in a hostname (allowed: ASCII letters, digits, '-', '.')" - ), - } - } -} - -impl std::error::Error for AdvertisedAddressError {} - -/// Whether cluster-wide JWT key material exists: a configured `http.jwt` -/// secret, or the signing key derived from the cluster PSK. When it does, a -/// bearer minted on any node verifies on every node - the invariant -/// follower-to-primary HTTP forwarding depends on. Callers gate `http.enabled` -/// themselves; this covers only the key material. -/// -/// Forwarding targets resolve from the roster (`ip:ports.http`); the config -/// validator unconditionally requires a roster port for every enabled -/// transport, so a forward never dials a node without a declared http port. -pub fn http_forwarding_key_material(jwt: &HttpJwtConfig, cluster: &ClusterConfig) -> bool { - cluster.enabled - && ((cluster.auth.enabled && !cluster.auth.shared_secret.is_empty()) - || !jwt.encoding_secret.is_empty() - || !jwt.decoding_secret.is_empty()) -} - -impl Validatable for ClusterConfig { - fn validate(&self) -> Result<(), ConfigurationError> { - // Ahead of the enabled gate: the top-level rule against - // message_bus.peer_queue_capacity holds repair_chunk_max unconditionally, - // so a single-node config skipping these would be bound by the - // cross-section rule while its own floor and ceiling went unchecked. - if self.repair_chunk_max == 0 { - eprintln!("Invalid cluster configuration: cluster.repair_chunk_max must be > 0"); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if self.repair_chunk_max > MAX_REPAIR_CHUNK_MAX { - eprintln!( - "Invalid cluster configuration: cluster.repair_chunk_max ({}) exceeds the maximum \ - ({MAX_REPAIR_CHUNK_MAX})", - self.repair_chunk_max - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - if !self.enabled { - return Ok(()); - } - - if self.name.trim().is_empty() { - eprintln!("Invalid cluster configuration: cluster name cannot be empty"); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - // `0` / `disabled` / `unlimited` all parse to a zero duration and - // land here too: there is no way to switch the liveness window off. - if self.heartbeat_timeout.get_duration() < MIN_CLUSTER_HEARTBEAT_TIMEOUT { - eprintln!( - "Invalid cluster configuration: cluster.heartbeat_timeout '{}' must be at least {}s \ - (the primary signals liveness through its commit broadcast; a shorter window \ - elects on every scheduling hiccup)", - self.heartbeat_timeout, - MIN_CLUSTER_HEARTBEAT_TIMEOUT.as_secs() - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - // The commit broadcast is the cluster's liveness feed and the prepare - // retransmit its recovery timer; both size consensus timers that have - // to advance. `0` / `disabled` / `unlimited` all collapse to a zero - // duration, which would stall the timer - reject them. - if self.commit_broadcast_interval.get_duration().is_zero() { - eprintln!( - "Invalid cluster configuration: cluster.commit_broadcast_interval must be nonzero \ - (it drives the primary's liveness broadcast)" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if self.prepare_retransmit_interval.get_duration().is_zero() { - eprintln!( - "Invalid cluster configuration: cluster.prepare_retransmit_interval must be \ - nonzero (it drives prepare retransmission)" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - // The liveness window must span several commit broadcasts so a single - // delayed broadcast never trips a view change on a healthy primary. - let min_heartbeat = self - .commit_broadcast_interval - .get_duration() - .saturating_mul(MIN_HEARTBEAT_TO_COMMIT_BROADCAST_RATIO); - if self.heartbeat_timeout.get_duration() < min_heartbeat { - eprintln!( - "Invalid cluster configuration: cluster.heartbeat_timeout '{}' must be at least \ - {}x cluster.commit_broadcast_interval '{}' so the liveness window spans several \ - broadcasts", - self.heartbeat_timeout, - MIN_HEARTBEAT_TO_COMMIT_BROADCAST_RATIO, - self.commit_broadcast_interval - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - // The three view-change timers each size a consensus timer that has to - // advance; `0` / `disabled` / `unlimited` all collapse to zero and - // would stall it - reject them. - if self - .view_change_retransmit_interval - .get_duration() - .is_zero() - { - eprintln!( - "Invalid cluster configuration: cluster.view_change_retransmit_interval must be \ - nonzero (it drives StartViewChange / DoViewChange retransmission)" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if self.view_change_status_timeout.get_duration().is_zero() { - eprintln!( - "Invalid cluster configuration: cluster.view_change_status_timeout must be nonzero \ - (it backstops a stalled view change)" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if self - .request_start_view_retransmit_interval - .get_duration() - .is_zero() - { - eprintln!( - "Invalid cluster configuration: cluster.request_start_view_retransmit_interval \ - must be nonzero (it drives RequestStartView retransmission)" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - // The status backstop must span several retransmits so a few dropped - // view-change messages retransmit rather than escalating a progressing - // view change into a fresh cluster-wide election. - let min_status = self - .view_change_retransmit_interval - .get_duration() - .saturating_mul(MIN_STATUS_TO_RETRANSMIT_RATIO); - if self.view_change_status_timeout.get_duration() < min_status { - eprintln!( - "Invalid cluster configuration: cluster.view_change_status_timeout '{}' must be at \ - least {}x cluster.view_change_retransmit_interval '{}' so a stalled view change \ - retransmits before it escalates to an election", - self.view_change_status_timeout, - MIN_STATUS_TO_RETRANSMIT_RATIO, - self.view_change_retransmit_interval - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - // A recovering replica needs at least one probe before it may give up - // and elect; the ceiling is a typo guard (see MAX_VIEW_PROBE_ATTEMPTS). - if self.view_probe_attempts_max == 0 { - eprintln!( - "Invalid cluster configuration: cluster.view_probe_attempts_max must be >= 1" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if self.view_probe_attempts_max > MAX_VIEW_PROBE_ATTEMPTS { - eprintln!( - "Invalid cluster configuration: cluster.view_probe_attempts_max ({}) exceeds the \ - maximum ({MAX_VIEW_PROBE_ATTEMPTS})", - self.view_probe_attempts_max - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - // The repair retry interval sizes a tick threshold that has to advance; - // `0` / `disabled` / `unlimited` all collapse to zero and would wedge - // every stalled repair stream - reject them. - if self.repair_retry_interval.get_duration().is_zero() { - eprintln!( - "Invalid cluster configuration: cluster.repair_retry_interval must be nonzero \ - (it paces stalled-repair retries)" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - if self.nodes.is_empty() { - eprintln!( - "Invalid cluster configuration: cluster.nodes must contain at least one entry when cluster is enabled" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - // VSR needs every replica to have a stable, unique id strictly - // less than the total replica count. Duplicate ids would split the - // cluster into two replicas claiming the same slot; out-of-range - // ids never win a primary election. Both are unrecoverable at - // runtime - fail fast at startup. - let total_replicas = u8::try_from(self.nodes.len()).map_err(|_| { - eprintln!("Invalid cluster configuration: more than 255 replicas is unsupported"); - ConfigurationError::InvalidConfigurationValue - })?; - - let mut seen_ids = std::collections::HashSet::new(); - let mut seen_names = std::collections::HashSet::new(); - let mut used_endpoints = std::collections::HashSet::new(); - let mut advertised_endpoints: Vec = Vec::new(); - - for node in &self.nodes { - if node.name.trim().is_empty() { - eprintln!("Invalid cluster configuration: node name cannot be empty"); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - if node.ip.trim().is_empty() { - eprintln!( - "Invalid cluster configuration: IP cannot be empty for node '{}'", - node.name - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - if !seen_names.insert(node.name.clone()) { - eprintln!( - "Invalid cluster configuration: duplicate node name '{}' found", - node.name - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - if node.replica_id >= total_replicas { - eprintln!( - "Invalid cluster configuration: replica_id {} for node '{}' must be < total replica count {total_replicas}", - node.replica_id, node.name - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - if !seen_ids.insert(node.replica_id) { - eprintln!( - "Invalid cluster configuration: duplicate replica_id {} (two nodes claim the same slot)", - node.replica_id - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - let client_ports = [ - ("TCP", node.ports.tcp), - ("QUIC", node.ports.quic), - ("HTTP", node.ports.http), - ("WebSocket", node.ports.websocket), - ]; - let replica_port = ("TCP_REPLICA", node.ports.tcp_replica); - - for (name, port_opt) in client_ports.into_iter().chain([replica_port]) { - if let Some(port) = port_opt { - if port == 0 { - eprintln!( - "Invalid cluster configuration: {} port cannot be 0 for node '{}'", - name, node.name - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - let endpoint = format!("{}:{}", node.ip, port); - if !used_endpoints.insert(endpoint.clone()) { - eprintln!( - "Invalid cluster configuration: port conflict - {endpoint} is already bound (node '{}', transport {name})", - node.name - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - } - } - - // An advertised address must parse strictly (IP or RFC 1123 - // hostname): the value is handed verbatim to every client via - // cluster metadata and redirect URLs, so a bad one poisons them - // all. The roster `ip` predates this check and is only validated - // as non-empty (Docker service names with underscores exist in - // the wild), so when it backs the client endpoints an unparsable - // value falls back to raw-string comparison instead of failing - // boot. - let client_address = match node.advertised_address.as_deref() { - Some(advertised_address) => match advertised_address.parse::() { - Ok(address) => Some(address), - Err(error) => { - eprintln!( - "Invalid cluster configuration: advertised_address '{advertised_address}' for node '{}': {error}", - node.name - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - }, - None => node.ip.parse::().ok(), - }; - - if node.advertised_addresses.len() > MAX_ADVERTISED_SELECTORS { - eprintln!( - "Invalid cluster configuration: node '{}' declares {} advertised_addresses \ - selectors, exceeding the maximum ({MAX_ADVERTISED_SELECTORS})", - node.name, - node.advertised_addresses.len() - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - // Selector CIDRs and addresses feed clients the same way the - // catch-all advertised address does, so they get the same strict - // parse. Networks are compared truncated (`10.0.1.0/16` == - // `10.0.0.0/16`) and canonicalized (`::ffff:10.0.0.0/104` == - // `10.0.0.0/8`) since matching truncates and canonicalizes too. - // Parsed before the catch-all enters the conflict pool because - // every entry's effective client set depends on the node's full - // selector list. - let mut selectors = Vec::with_capacity(node.advertised_addresses.len()); - let mut seen_selector_cidrs = std::collections::HashSet::new(); - for selector in &node.advertised_addresses { - let client_cidr = match selector.client_cidr.parse::() { - Ok(client_cidr) => canonical_ip_net(client_cidr.trunc()), - Err(error) => { - eprintln!( - "Invalid cluster configuration: advertised_addresses client_cidr '{}' for node '{}': {error}", - selector.client_cidr, node.name - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - }; - if !seen_selector_cidrs.insert(client_cidr) { - eprintln!( - "Invalid cluster configuration: duplicate advertised_addresses client_cidr '{}' for node '{}'", - selector.client_cidr, node.name - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - let address = match selector.address.parse::() { - Ok(address) => address, - Err(error) => { - eprintln!( - "Invalid cluster configuration: advertised_addresses address '{}' for node '{}': {error}", - selector.address, node.name - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - }; - selectors.push((client_cidr, address)); - } - let selector_ranges: Vec = selectors - .iter() - .map(|(network, _)| ClientAddressRange::from(network)) - .collect(); - - // Endpoint conflicts are checked across every node's selectors - // and catch-all on effective client sets - the clients an entry - // actually wins after this node's longest-prefix match. Two - // nodes may reuse one host:port as long as the winning sets stay - // disjoint (that is the feature, and it includes a per-subnet - // override shadowing the same node's wider selector); a conflict - // means some client wins both entries and would resolve both - // nodes to one endpoint. The catch-all is an implicit - // match-everything-else selector, so it pools the same way. A - // roster ip that fails the strict parse skips the pool: it can - // never equal a parsed host, and two raw ips sharing host:port - // are already rejected by the bind-endpoint check above. - if let Some(address) = &client_address { - let catch_all_clients = EffectiveClients::for_catch_all(&selector_ranges); - for (name, port) in &client_ports { - if let Some(port) = port { - insert_advertised_endpoint( - &mut advertised_endpoints, - AdvertisedEndpoint { - node_name: &node.name, - transport: name, - network: None, - clients: catch_all_clients.clone(), - host: address.clone(), - port: *port, - }, - )?; - } - } - } - - for (selector_index, (client_cidr, address)) in selectors.iter().enumerate() { - let sibling_ranges: Vec = selector_ranges - .iter() - .enumerate() - .filter(|(other_index, _)| *other_index != selector_index) - .map(|(_, range)| *range) - .collect(); - let clients = EffectiveClients::for_selector(client_cidr, &sibling_ranges); - for (name, port) in &client_ports { - if let Some(port) = port { - insert_advertised_endpoint( - &mut advertised_endpoints, - AdvertisedEndpoint { - node_name: &node.name, - transport: name, - network: Some(*client_cidr), - clients: clients.clone(), - host: address.clone(), - port: *port, - }, - )?; - } - } - } - } - - // Replica-auth PSK (only reached when the cluster is enabled; the early - // return above skips these while it is disabled). When auth is enabled - // the key is mandatory; any configured key must clear the length floor - - // a typo guard that fires with auth off too, though only while the - // cluster itself is enabled. - let secret_len = self.auth.shared_secret.len(); - if self.auth.enabled && self.auth.shared_secret.is_empty() { - eprintln!( - "Invalid cluster configuration: cluster.auth.shared_secret must be set when cluster.auth.enabled is true" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if !self.auth.shared_secret.is_empty() && secret_len < MIN_SHARED_SECRET_LEN { - eprintln!( - "Invalid cluster configuration: cluster.auth.shared_secret must be >= {MIN_SHARED_SECRET_LEN} bytes" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - // Rotation window key: same typo guard as the primary, plus a - // distinctness check - a window equal to the primary means the - // operator rolled the config without changing the key, so the - // "rotation" would silently be a no-op. - if !self.auth.previous_shared_secret.is_empty() { - if self.auth.previous_shared_secret.len() < MIN_SHARED_SECRET_LEN { - eprintln!( - "Invalid cluster configuration: cluster.auth.previous_shared_secret must be >= {MIN_SHARED_SECRET_LEN} bytes" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if self.auth.previous_shared_secret == self.auth.shared_secret { - eprintln!( - "Invalid cluster configuration: cluster.auth.previous_shared_secret must differ from cluster.auth.shared_secret (an identical window is a no-op rotation)" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - } - - // Replica TLS. Both cert modes run one-directional TLS (no client - // certificate anywhere), so TLS only authenticates the acceptor to - // the dialer; peer authentication comes solely from the PSK - // handshake. Without it any TLS-capable host could register as a - // replica - require auth in both modes. CA mode (the default) - // additionally needs all three PEM paths: cert/key for this node's - // acceptor side, ca_file as the dialer's trust anchor. - if self.tls.enabled { - if !self.auth.enabled { - eprintln!( - "Invalid cluster configuration: cluster.tls.enabled = true requires cluster.auth.enabled = true (TLS authenticates the acceptor only; the PSK handshake authenticates the peer)" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if !self.tls.self_signed { - for (field, value) in [ - ("cert_file", &self.tls.cert_file), - ("key_file", &self.tls.key_file), - ("ca_file", &self.tls.ca_file), - ] { - if value.trim().is_empty() { - eprintln!( - "Invalid cluster configuration: cluster.tls.{field} must be set when cluster.tls.enabled = true and self_signed = false" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - } - } - } - - Ok(()) - } -} - -/// One advertised client endpoint and the clients it wins, pooled by -/// [`ClusterConfig::validate`] so selectors and catch-all conflict-check -/// against each other. `network: None` is the catch-all (`advertised_address`, -/// or the roster `ip` as fallback); `clients` is the entry's effective set -/// after its node's longest-prefix shadowing. -struct AdvertisedEndpoint<'roster> { - node_name: &'roster str, - transport: &'static str, - network: Option, - clients: EffectiveClients, - host: AdvertisedAddress, - port: u16, -} - -impl AdvertisedEndpoint<'_> { - /// True when some client would resolve both entries to one host:port on - /// two different nodes. Effective client sets already encode each node's - /// longest-prefix shadowing, so nested networks conflict only where the - /// wider entry still wins some client that the other node's entry also - /// wins. Entries of one node never conflict: their effective sets are - /// disjoint by construction. - fn conflicts_with(&self, other: &Self) -> bool { - self.node_name != other.node_name - && self.port == other.port - && self.host == other.host - && self.clients.overlaps(&other.clients) - } - - fn authority(&self) -> String { - self.host.authority(self.port) - } - - fn network_description(&self) -> String { - match self.network { - Some(network) => format!("client_cidr {network}"), - None => "every client network (catch-all)".to_owned(), - } - } -} - -/// The clients an advertised entry actually wins under its node's -/// longest-prefix match, built by [`ClusterConfig::validate`] for the -/// cross-node conflict scan. -#[derive(Clone)] -struct EffectiveClients { - /// Sorted disjoint ranges of winning client addresses. - ranges: Vec, - /// The catch-all also wins clients whose peer address the transport - /// could not produce ([`ResolvedClusterNode::advertised_for`] with no - /// client IP), so two catch-all overlap even when selectors cover both - /// address families. - serves_unknown_peers: bool, -} - -impl EffectiveClients { - /// A selector wins its network minus the sibling networks nested inside - /// it (longer prefixes take the node's LPM). `sibling_ranges` must - /// exclude the selector's own network. - fn for_selector(network: &IpNet, sibling_ranges: &[ClientAddressRange]) -> Self { - Self { - ranges: ClientAddressRange::from(network).subtract_nested(sibling_ranges), - serves_unknown_peers: false, - } - } - - /// The catch-all wins every client no selector matches, in both address - /// families, plus unknown-peer clients. - fn for_catch_all(selector_ranges: &[ClientAddressRange]) -> Self { - let mut ranges = ClientAddressRange::FULL_IPV4.subtract_nested(selector_ranges); - ranges.extend(ClientAddressRange::FULL_IPV6.subtract_nested(selector_ranges)); - Self { - ranges, - serves_unknown_peers: true, - } - } - - fn overlaps(&self, other: &Self) -> bool { - if self.serves_unknown_peers && other.serves_unknown_peers { - return true; - } - self.ranges.iter().any(|range| { - other.ranges.iter().any(|other_range| { - range.is_ipv4 == other_range.is_ipv4 - && range.first <= other_range.last - && other_range.first <= range.last - }) - }) - } -} - -/// Inclusive range of client addresses within one family. Client IPs -/// canonicalize to v4 before matching, so v4 and v6 networks match disjoint -/// client populations and a range never spans families. -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -struct ClientAddressRange { - is_ipv4: bool, - first: u128, - last: u128, -} - -impl ClientAddressRange { - const FULL_IPV4: Self = Self { - is_ipv4: true, - first: 0, - last: u32::MAX as u128, - }; - const FULL_IPV6: Self = Self { - is_ipv4: false, - first: 0, - last: u128::MAX, - }; - - /// `self` minus every range nested inside it, as sorted disjoint - /// leftovers. CIDR networks are nested or disjoint, never partially - /// overlapping, so a range outside `self` is either disjoint from it - /// (subtracts nothing) or contains it (a shorter prefix, which loses - /// LPM and also subtracts nothing). - fn subtract_nested(self, ranges: &[Self]) -> Vec { - let mut nested: Vec = ranges - .iter() - .filter(|range| { - range.is_ipv4 == self.is_ipv4 - && range.first >= self.first - && range.last <= self.last - }) - .copied() - .collect(); - nested.sort_unstable(); - let mut remaining = Vec::new(); - let mut cursor = Some(self.first); - for nested_range in nested { - let Some(next_free) = cursor else { break }; - if nested_range.first > next_free { - remaining.push(Self { - is_ipv4: self.is_ipv4, - first: next_free, - last: nested_range.first - 1, - }); - } - cursor = nested_range - .last - .checked_add(1) - .map(|after| after.max(next_free)); - } - if let Some(next_free) = cursor - && next_free <= self.last - { - remaining.push(Self { - is_ipv4: self.is_ipv4, - first: next_free, - last: self.last, - }); - } - remaining - } -} - -impl From<&IpNet> for ClientAddressRange { - fn from(network: &IpNet) -> Self { - match network { - IpNet::V4(network) => Self { - is_ipv4: true, - first: u128::from(u32::from(network.network())), - last: u128::from(u32::from(network.broadcast())), - }, - IpNet::V6(network) => Self { - is_ipv4: false, - first: u128::from(network.network()), - last: u128::from(network.broadcast()), - }, - } - } -} - -fn insert_advertised_endpoint<'roster>( - advertised_endpoints: &mut Vec>, - endpoint: AdvertisedEndpoint<'roster>, -) -> Result<(), ConfigurationError> { - if let Some(existing) = advertised_endpoints - .iter() - .find(|existing| existing.conflicts_with(&endpoint)) - { - eprintln!( - "Invalid cluster configuration: advertised client endpoint conflict - {} is advertised for {} (node '{}', transport {}) and for {} (node '{}', transport {}); their effective client sets overlap after longest-prefix shadowing, so a client in the overlap would resolve both nodes to one endpoint", - endpoint.authority(), - endpoint.network_description(), - endpoint.node_name, - endpoint.transport, - existing.network_description(), - existing.node_name, - existing.transport, - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - advertised_endpoints.push(endpoint); - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn shared_secret_is_never_serialized() { - // Regression guard: the runtime current_config.toml (and the - // ServerConfig diagnostic snapshot that cats it) are produced by - // serializing this struct, so the PSK must not survive serialize. - // skip_serializing is format-agnostic, so a JSON dump proves the toml - // path too. - let config = ClusterConfig { - enabled: true, - name: "iggy-cluster".to_owned(), - heartbeat_timeout: default_heartbeat_timeout(), - commit_broadcast_interval: default_commit_broadcast_interval(), - prepare_retransmit_interval: default_prepare_retransmit_interval(), - view_change_retransmit_interval: default_view_change_retransmit_interval(), - view_change_status_timeout: default_view_change_status_timeout(), - request_start_view_retransmit_interval: default_request_start_view_retransmit_interval( - ), - view_probe_attempts_max: default_view_probe_attempts_max(), - repair_retry_interval: default_repair_retry_interval(), - repair_chunk_max: default_repair_chunk_max(), - nodes: Vec::new(), - auth: ClusterAuthConfig { - enabled: true, - shared_secret: "current-psk-MUST-NOT-be-persisted".to_owned(), - previous_shared_secret: "retiring-psk-MUST-NOT-be-persisted".to_owned(), - }, - tls: ClusterTlsConfig::default(), - }; - let serialized = serde_json::to_string(&config).expect("serialize cluster config"); - assert!( - !serialized.contains("MUST-NOT-be-persisted"), - "PSK leaked into serialized config: {serialized}" - ); - assert!( - !serialized.contains("shared_secret"), - "shared_secret field present in serialized config: {serialized}" - ); - } - - #[test] - fn cluster_node_rejects_unknown_fields() { - let error = serde_json::from_str::( - r#"{ - "name": "node-0", - "ip": "10.0.0.1", - "advertise_address": "203.0.113.1", - "replica_id": 0, - "ports": {} - }"#, - ) - .expect_err("misspelled advertised_address must be rejected"); - - assert!( - error - .to_string() - .contains("unknown field `advertise_address`"), - "unexpected deserialization error: {error}" - ); - } - - #[test] - fn advertised_addresses_env_expansion_is_capped() { - // The selectors Vec nests inside the nodes Vec, so the derive's - // index ceilings multiply; without the field's max_elements cap the - // default of 256x256 adds ~131k leaked mappings to every boot. - let mappings = ::env_mappings(); - assert!( - mappings - .iter() - .any(|mapping| mapping.env_name.contains("ADVERTISED_ADDRESSES_15_")), - "selector index 15 must stay reachable by env override" - ); - assert!( - !mappings - .iter() - .any(|mapping| mapping.env_name.contains("ADVERTISED_ADDRESSES_16_")), - "selector env expansion must stop at max_elements = 16" - ); - } -} - -#[cfg(test)] -mod advertised_address_tests { - use super::*; - - #[test] - fn parses_ip_literals_to_canonical_form() { - assert_eq!( - "203.0.113.1".parse::(), - Ok(AdvertisedAddress::Ip("203.0.113.1".parse().unwrap())) - ); - for equivalent_address in ["2001:DB8::1", "2001:db8:0:0:0:0:0:1", "[2001:db8::1]"] { - assert_eq!( - equivalent_address.parse::(), - Ok(AdvertisedAddress::Ip("2001:db8::1".parse().unwrap())), - "'{equivalent_address}' must parse to canonical 2001:db8::1" - ); - } - } - - #[test] - fn normalizes_hostname_to_lowercase() { - let address = "Broker-1.Example.COM".parse::(); - assert_eq!( - address, - Ok(AdvertisedAddress::Hostname( - "broker-1.example.com".to_owned() - )) - ); - } - - #[test] - fn authority_brackets_ipv6_hosts_only() { - let cases = [ - ("203.0.113.1", "203.0.113.1:8090"), - ("2001:db8::1", "[2001:db8::1]:8090"), - ("broker-1.example.com", "broker-1.example.com:8090"), - ]; - for (host, expected_authority) in cases { - let address = host.parse::().expect("valid address"); - assert_eq!(address.authority(8090), expected_authority); - } - } - - #[test] - fn rejects_port_suffixes() { - for address_with_port in ["example.com:8090", "10.0.0.1:8090", "[2001:db8::1]:8090"] { - assert_eq!( - address_with_port.parse::(), - Err(AdvertisedAddressError::PortNotAllowed), - "'{address_with_port}' must be rejected as host:port" - ); - } - } - - #[test] - fn rejects_dotted_numeric_strings_as_malformed_ipv4() { - for malformed_ip in ["10.0.0.256", "192.168.1", "12345"] { - assert_eq!( - malformed_ip.parse::(), - Err(AdvertisedAddressError::MalformedIpv4), - "'{malformed_ip}' must not pass as a hostname" - ); - } - } - - #[test] - fn rejects_broken_ipv6_literals() { - for broken_ipv6 in ["2001:db8:::1", "[2001:db8::zz]", "::1::2"] { - assert_eq!( - broken_ipv6.parse::(), - Err(AdvertisedAddressError::MalformedIpv6), - "'{broken_ipv6}' must be rejected as malformed IPv6" - ); - } - } -} - -#[cfg(test)] -mod advertised_for_tests { - use super::*; - - fn node_with_selectors(selectors: Vec) -> ClusterNodeConfig { - ClusterNodeConfig { - name: "node-0".to_owned(), - ip: "10.0.1.5".to_owned(), - advertised_address: Some("203.0.113.10".to_owned()), - advertised_addresses: selectors, - replica_id: 0, - ports: TransportPorts::default(), - } - } - - fn selector(client_cidr: &str, address: &str) -> AdvertisedAddressSelector { - AdvertisedAddressSelector { - client_cidr: client_cidr.to_owned(), - address: address.to_owned(), - } - } - - fn resolved(node: ClusterNodeConfig) -> ResolvedClusterNode { - node.into() - } - - fn ip(address: &str) -> IpAddr { - address.parse().unwrap() - } - - #[test] - fn falls_back_to_advertised_address_without_selectors() { - let node = node_with_selectors(Vec::new()); - assert_eq!( - resolved(node).advertised_for(Some(ip("10.0.0.7"))), - Some(&AdvertisedAddress::Ip(ip("203.0.113.10"))) - ); - } - - #[test] - fn falls_back_to_roster_ip_without_advertised_address() { - let mut node = node_with_selectors(Vec::new()); - node.advertised_address = None; - assert_eq!( - resolved(node).advertised_for(Some(ip("10.0.0.7"))), - Some(&AdvertisedAddress::Ip(ip("10.0.1.5"))) - ); - } - - #[test] - fn is_none_when_no_fallback_parses() { - let mut node = node_with_selectors(Vec::new()); - node.advertised_address = None; - node.ip = "iggy_node".to_owned(); - assert_eq!(resolved(node).advertised_for(Some(ip("10.0.0.7"))), None); - } - - #[test] - fn matching_selector_beats_advertised_address() { - let node = node_with_selectors(vec![selector("10.0.0.0/16", "10.0.1.5")]); - assert_eq!( - resolved(node).advertised_for(Some(ip("10.0.200.7"))), - Some(&AdvertisedAddress::Ip(ip("10.0.1.5"))) - ); - } - - #[test] - fn unmatched_client_falls_back_to_advertised_address() { - let node = node_with_selectors(vec![selector("10.0.0.0/16", "10.0.1.5")]); - assert_eq!( - resolved(node).advertised_for(Some(ip("192.168.0.7"))), - Some(&AdvertisedAddress::Ip(ip("203.0.113.10"))) - ); - } - - #[test] - fn unknown_client_ip_falls_back_to_advertised_address() { - let node = node_with_selectors(vec![selector("10.0.0.0/16", "10.0.1.5")]); - assert_eq!( - resolved(node).advertised_for(None), - Some(&AdvertisedAddress::Ip(ip("203.0.113.10"))) - ); - } - - #[test] - fn longest_prefix_wins_regardless_of_declaration_order() { - let node = resolved(node_with_selectors(vec![ - selector("10.0.0.0/8", "10.255.255.1"), - selector("10.0.0.0/16", "10.0.1.5"), - ])); - assert_eq!( - node.advertised_for(Some(ip("10.0.200.7"))), - Some(&AdvertisedAddress::Ip(ip("10.0.1.5"))), - "the /16 must win over the /8 even though it is declared second" - ); - assert_eq!( - node.advertised_for(Some(ip("10.9.0.7"))), - Some(&AdvertisedAddress::Ip(ip("10.255.255.1"))), - "a client outside the /16 but inside the /8 must match the /8" - ); - } - - #[test] - fn equal_prefix_matches_resolve_deterministically_to_first_declared() { - // No validated config reaches this state: these networks truncate to - // one /16, which validation rejects as a duplicate. Pinned anyway so - // a future relaxation of that rule cannot make resolution - // order-dependent. - let node = node_with_selectors(vec![ - selector("10.0.1.0/16", "10.0.1.5"), - selector("10.0.2.0/16", "10.0.2.5"), - ]); - assert_eq!( - resolved(node).advertised_for(Some(ip("10.0.200.7"))), - Some(&AdvertisedAddress::Ip(ip("10.0.1.5"))) - ); - } - - #[test] - fn v4_mapped_v6_client_matches_v4_cidr() { - // A dual-stack listener reports v4 peers as `::ffff:a.b.c.d`. - let node = node_with_selectors(vec![selector("10.0.0.0/16", "10.0.1.5")]); - assert_eq!( - resolved(node).advertised_for(Some(ip("::ffff:10.0.0.7"))), - Some(&AdvertisedAddress::Ip(ip("10.0.1.5"))) - ); - } - - #[test] - fn v4_mapped_v6_selector_cidr_matches_v4_client() { - // The mirror case: the CIDR side canonicalizes at build, so - // `::ffff:10.0.0.0/104` matches like `10.0.0.0/8` instead of being - // a silently dead selector. - let node = node_with_selectors(vec![selector("::ffff:10.0.0.0/104", "10.0.1.5")]); - assert_eq!( - resolved(node).advertised_for(Some(ip("10.0.0.7"))), - Some(&AdvertisedAddress::Ip(ip("10.0.1.5"))) - ); - } - - #[test] - fn v6_selector_matches_v6_client() { - let node = node_with_selectors(vec![selector("2001:db8::/32", "2001:db8::1")]); - assert_eq!( - resolved(node).advertised_for(Some(ip("2001:db8::7"))), - Some(&AdvertisedAddress::Ip(ip("2001:db8::1"))) - ); - } - - #[test] - fn selector_address_may_be_a_hostname() { - let node = node_with_selectors(vec![selector("10.0.0.0/16", "Broker.Internal.Example")]); - assert_eq!( - resolved(node).advertised_for(Some(ip("10.0.0.7"))), - Some(&AdvertisedAddress::Hostname( - "broker.internal.example".to_owned() - )) - ); - } -} - -#[cfg(test)] -mod cluster_validate_tests { - use super::*; - - fn node(name: &str, id: u8) -> ClusterNodeConfig { - ClusterNodeConfig { - name: name.to_string(), - ip: "127.0.0.1".to_string(), - advertised_address: None, - advertised_addresses: Vec::new(), - replica_id: id, - ports: TransportPorts::default(), - } - } - - fn selector(client_cidr: &str, address: &str) -> AdvertisedAddressSelector { - AdvertisedAddressSelector { - client_cidr: client_cidr.to_owned(), - address: address.to_owned(), - } - } - - fn cfg(nodes: Vec) -> ClusterConfig { - ClusterConfig { - enabled: true, - name: "iggy-cluster".to_string(), - heartbeat_timeout: default_heartbeat_timeout(), - commit_broadcast_interval: default_commit_broadcast_interval(), - prepare_retransmit_interval: default_prepare_retransmit_interval(), - view_change_retransmit_interval: default_view_change_retransmit_interval(), - view_change_status_timeout: default_view_change_status_timeout(), - request_start_view_retransmit_interval: default_request_start_view_retransmit_interval( - ), - view_probe_attempts_max: default_view_probe_attempts_max(), - repair_retry_interval: default_repair_retry_interval(), - repair_chunk_max: default_repair_chunk_max(), - nodes, - auth: ClusterAuthConfig::default(), - tls: ClusterTlsConfig::default(), - } - } - - #[test] - fn validate_rejects_sub_minimum_heartbeat_timeout() { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.heartbeat_timeout = IggyDuration::new(Duration::from_millis(500)); - assert!(c.validate().is_err()); - // The "disabled" / "unlimited" sentinels collapse to zero and must - // be rejected the same way. - c.heartbeat_timeout = IggyDuration::new(Duration::ZERO); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_rejects_zero_commit_broadcast_interval() { - // `0` / `disabled` / `unlimited` all collapse to zero and stall the - // liveness broadcast. - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.commit_broadcast_interval = IggyDuration::new(Duration::ZERO); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_rejects_zero_prepare_retransmit_interval() { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.prepare_retransmit_interval = IggyDuration::new(Duration::ZERO); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_rejects_heartbeat_below_commit_broadcast_ratio() { - // 3s clears the absolute 2s floor but is still < 4x the 1s broadcast, - // so the ratio rule is what rejects here, not the floor. - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.heartbeat_timeout = IggyDuration::new(Duration::from_secs(3)); - c.commit_broadcast_interval = IggyDuration::new(Duration::from_secs(1)); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_accepts_heartbeat_at_commit_broadcast_ratio() { - // Exactly 4x the broadcast (and above the 2s floor) must pass. - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.heartbeat_timeout = IggyDuration::new(Duration::from_secs(4)); - c.commit_broadcast_interval = IggyDuration::new(Duration::from_secs(1)); - assert!(c.validate().is_ok()); - } - - #[test] - fn validate_rejects_zero_view_change_retransmit_interval() { - // `0` / `disabled` / `unlimited` all collapse to zero and stall the - // view-change retransmit timers. - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.view_change_retransmit_interval = IggyDuration::new(Duration::ZERO); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_rejects_zero_view_change_status_timeout() { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.view_change_status_timeout = IggyDuration::new(Duration::ZERO); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_rejects_zero_request_start_view_retransmit_interval() { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.request_start_view_retransmit_interval = IggyDuration::new(Duration::ZERO); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_rejects_view_change_status_below_retransmit_ratio() { - // 3s is nonzero but still < 4x the 1s retransmit, so the ratio rule is - // what rejects here, not the zero check. - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.view_change_retransmit_interval = IggyDuration::new(Duration::from_secs(1)); - c.view_change_status_timeout = IggyDuration::new(Duration::from_secs(3)); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_accepts_view_change_status_at_retransmit_ratio() { - // Exactly 4x the retransmit interval must pass. - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.view_change_retransmit_interval = IggyDuration::new(Duration::from_secs(1)); - c.view_change_status_timeout = IggyDuration::new(Duration::from_secs(4)); - assert!(c.validate().is_ok()); - } - - #[test] - fn validate_rejects_zero_view_probe_attempts_max() { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.view_probe_attempts_max = 0; - assert!(c.validate().is_err()); - } - - #[test] - fn validate_rejects_view_probe_attempts_above_ceiling() { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.view_probe_attempts_max = MAX_VIEW_PROBE_ATTEMPTS + 1; - assert!(c.validate().is_err()); - } - - #[test] - fn validate_accepts_view_probe_attempts_at_ceiling() { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.view_probe_attempts_max = MAX_VIEW_PROBE_ATTEMPTS; - assert!(c.validate().is_ok()); - } - - #[test] - fn validate_rejects_zero_repair_retry_interval() { - // `0` / `disabled` / `unlimited` all collapse to zero and would wedge - // stalled repair streams. - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.repair_retry_interval = IggyDuration::new(Duration::ZERO); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_rejects_zero_repair_chunk_max() { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.repair_chunk_max = 0; - assert!(c.validate().is_err()); - } - - #[test] - fn validate_rejects_repair_chunk_max_above_ceiling() { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.repair_chunk_max = MAX_REPAIR_CHUNK_MAX + 1; - assert!(c.validate().is_err()); - } - - #[test] - fn validate_accepts_repair_chunk_max_at_ceiling() { - // Section-level validate only; the cross-section rule against - // message_bus.peer_queue_capacity lives in the top-level validate. - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.repair_chunk_max = MAX_REPAIR_CHUNK_MAX; - assert!(c.validate().is_ok()); - } - - #[test] - fn validate_rejects_empty_nodes() { - let c = cfg(vec![]); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_rejects_duplicate_replica_ids() { - let c = cfg(vec![node("n1", 0), node("n2", 0)]); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_rejects_duplicate_names() { - let c = cfg(vec![node("n1", 0), node("n1", 1)]); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_rejects_out_of_range_replica_id() { - // 2 nodes total, so id 2 is out of range. - let c = cfg(vec![node("n1", 0), node("n2", 2)]); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_accepts_unique_contiguous_replica_ids() { - let c = cfg(vec![node("n1", 0), node("n2", 1), node("n3", 2)]); - assert!(c.validate().is_ok()); - } - - #[test] - fn validate_skips_checks_when_disabled() { - let mut c = cfg(vec![]); - c.enabled = false; - assert!(c.validate().is_ok()); - } - - // repair_chunk_max is also read by the unconditional top-level check - // against message_bus.peer_queue_capacity, so its own bounds apply with - // the cluster off too. - #[test] - fn validate_rejects_zero_repair_chunk_max_when_disabled() { - let mut c = cfg(vec![]); - c.enabled = false; - c.repair_chunk_max = 0; - assert!(c.validate().is_err()); - } - - #[test] - fn validate_rejects_repair_chunk_max_above_ceiling_when_disabled() { - let mut c = cfg(vec![]); - c.enabled = false; - c.repair_chunk_max = MAX_REPAIR_CHUNK_MAX + 1; - assert!(c.validate().is_err()); - } - - #[test] - fn validate_rejects_duplicate_tcp_replica_port() { - let ports = TransportPorts { - tcp: None, - quic: None, - http: None, - websocket: None, - tcp_replica: Some(9090), - }; - let mut n1 = node("n1", 0); - n1.ports = ports.clone(); - let mut n2 = node("n2", 1); - n2.ports = ports; - let c = cfg(vec![n1, n2]); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_rejects_cross_transport_port_reuse() { - let mut n1 = node("n1", 0); - n1.ports = TransportPorts { - tcp: Some(8090), - quic: None, - http: Some(8090), - websocket: None, - tcp_replica: None, - }; - let c = cfg(vec![n1]); - assert!( - c.validate().is_err(), - "same port on TCP and HTTP of the same node must be rejected" - ); - } - - #[test] - fn validate_accepts_same_port_on_different_ips() { - let mut n1 = node("n1", 0); - n1.ip = "127.0.0.1".to_string(); - n1.ports = TransportPorts { - tcp: Some(8090), - quic: None, - http: None, - websocket: None, - tcp_replica: None, - }; - let mut n2 = node("n2", 1); - n2.ip = "127.0.0.2".to_string(); - n2.ports = TransportPorts { - tcp: Some(8090), - quic: None, - http: None, - websocket: None, - tcp_replica: None, - }; - let c = cfg(vec![n1, n2]); - assert!(c.validate().is_ok()); - } - - #[test] - fn validate_rejects_duplicate_advertised_client_endpoint() { - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_address = Some("203.0.113.1".to_owned()); - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "10.0.0.2".to_owned(); - n2.advertised_address = n1.advertised_address.clone(); - n2.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, n2]).validate().is_err()); - } - - #[test] - fn validate_rejects_equivalent_ipv6_advertised_client_endpoints() { - for equivalent_address in ["2001:DB8::1", "2001:db8:0:0:0:0:0:1", "[2001:db8::1]"] { - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_address = Some("2001:db8::1".to_owned()); - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "10.0.0.2".to_owned(); - n2.advertised_address = Some(equivalent_address.to_owned()); - n2.ports.tcp = Some(8090); - - assert!( - cfg(vec![n1, n2]).validate().is_err(), - "{equivalent_address} must conflict with 2001:db8::1" - ); - } - } - - #[test] - fn validate_rejects_equivalent_ipv6_client_endpoints_from_node_ip() { - let mut n1 = node("n1", 0); - n1.ip = "2001:db8::1".to_owned(); - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "2001:db8:0:0:0:0:0:1".to_owned(); - n2.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, n2]).validate().is_err()); - } - - #[test] - fn validate_accepts_distinct_advertised_client_endpoints() { - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_address = Some("203.0.113.1".to_owned()); - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "10.0.0.2".to_owned(); - n2.advertised_address = Some("203.0.113.2".to_owned()); - n2.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, n2]).validate().is_ok()); - } - - #[test] - fn validate_accepts_hostname_advertised_address() { - let mut n1 = node("n1", 0); - n1.advertised_address = Some("iggy-node-1.example.com".to_owned()); - n1.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, node("n2", 1)]).validate().is_ok()); - } - - #[test] - fn validate_rejects_malformed_advertised_addresses() { - let oversized_label = format!("{}.example.com", "a".repeat(64)); - let oversized_hostname = format!("{}example.com", "a.".repeat(130)); - for advertised_address in [ - "", - " 203.0.113.1", - "10.0.0.256", - "192.168.1", - "example.com:8090", - "[2001:db8::1]:8090", - "2001:db8:::1", - "iggy_node.example.com", - "-node.example.com", - "node-.example.com", - ".example.com", - "example..com", - "example.com.", - "ex\u{e4}mple.com", - oversized_label.as_str(), - oversized_hostname.as_str(), - ] { - let mut n1 = node("n1", 0); - n1.advertised_address = Some(advertised_address.to_owned()); - - assert!( - cfg(vec![n1, node("n2", 1)]).validate().is_err(), - "'{advertised_address}' must be rejected" - ); - } - } - - #[test] - fn validate_rejects_case_variant_hostname_advertised_endpoints() { - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_address = Some("broker.example.com".to_owned()); - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "10.0.0.2".to_owned(); - n2.advertised_address = Some("Broker.Example.COM".to_owned()); - n2.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, n2]).validate().is_err()); - } - - #[test] - fn validate_rejects_node_ip_hostname_clashing_with_advertised_hostname() { - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_address = Some("broker.example.com".to_owned()); - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "broker.example.com".to_owned(); - n2.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, n2]).validate().is_err()); - } - - #[test] - fn validate_accepts_distinct_hostname_advertised_endpoints() { - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_address = Some("broker-1.example.com".to_owned()); - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "10.0.0.2".to_owned(); - n2.advertised_address = Some("broker-2.example.com".to_owned()); - n2.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, n2]).validate().is_ok()); - } - - #[test] - fn validate_accepts_selectors_with_distinct_cidrs() { - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_address = Some("203.0.113.1".to_owned()); - n1.advertised_addresses = vec![ - selector("10.0.0.0/16", "10.0.0.1"), - selector("10.0.0.0/8", "broker-1.internal.example"), - ]; - n1.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, node("n2", 1)]).validate().is_ok()); - } - - #[test] - fn validate_rejects_malformed_selector_cidr() { - for client_cidr in ["10.0.0.0", "10.0.0.0/33", "not-a-cidr", ""] { - let mut n1 = node("n1", 0); - n1.advertised_addresses = vec![selector(client_cidr, "10.0.0.1")]; - - assert!( - cfg(vec![n1, node("n2", 1)]).validate().is_err(), - "client_cidr '{client_cidr}' must be rejected" - ); - } - } - - #[test] - fn validate_rejects_malformed_selector_address() { - for address in ["", "10.0.0.1:8090", "10.0.0.256", "iggy_node"] { - let mut n1 = node("n1", 0); - n1.advertised_addresses = vec![selector("10.0.0.0/16", address)]; - - assert!( - cfg(vec![n1, node("n2", 1)]).validate().is_err(), - "selector address '{address}' must be rejected" - ); - } - } - - #[test] - fn validate_rejects_duplicate_selector_cidr_within_a_node() { - // `10.0.1.0/16` truncates to `10.0.0.0/16`: the two selectors match - // the identical client set, so the second could never win LPM. - let mut n1 = node("n1", 0); - n1.advertised_addresses = vec![ - selector("10.0.0.0/16", "10.0.0.1"), - selector("10.0.1.0/16", "10.0.0.2"), - ]; - - assert!(cfg(vec![n1, node("n2", 1)]).validate().is_err()); - } - - #[test] - fn validate_accepts_selector_count_at_the_cap() { - let mut n1 = node("n1", 0); - n1.advertised_addresses = (0..MAX_ADVERTISED_SELECTORS) - .map(|index| selector(&format!("10.{index}.0.0/16"), &format!("192.0.2.{index}"))) - .collect(); - - assert!(cfg(vec![n1, node("n2", 1)]).validate().is_ok()); - } - - #[test] - fn validate_rejects_selector_count_above_the_cap() { - // The env-override path stops expanding selector indices at the same - // ceiling, so a TOML roster exceeding it could never be replicated - // byte-identically through env vars. - let mut n1 = node("n1", 0); - n1.advertised_addresses = (0..=MAX_ADVERTISED_SELECTORS) - .map(|index| selector(&format!("10.{index}.0.0/16"), &format!("192.0.2.{index}"))) - .collect(); - - assert!(cfg(vec![n1, node("n2", 1)]).validate().is_err()); - } - - #[test] - fn validate_rejects_selector_endpoint_conflict_within_one_cidr() { - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_addresses = vec![selector("10.0.0.0/16", "10.0.7.7")]; - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "10.0.0.2".to_owned(); - n2.advertised_addresses = vec![selector("10.0.0.0/16", "10.0.7.7")]; - n2.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, n2]).validate().is_err()); - } - - #[test] - fn validate_accepts_identical_selector_endpoint_across_different_cidrs() { - // Reusing one host:port across DIFFERENT client networks is the - // feature (e.g. each network NATs the address to its local node). - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_addresses = vec![selector("10.1.0.0/16", "192.0.2.10")]; - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "10.0.0.2".to_owned(); - n2.advertised_addresses = vec![selector("10.2.0.0/16", "192.0.2.10")]; - n2.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, n2]).validate().is_ok()); - } - - #[test] - fn validate_rejects_v4_mapped_v6_selector_cidr_duplicating_its_v4_form() { - // `::ffff:10.0.0.0/104` canonicalizes to `10.0.0.0/8` (matching how - // client IPs canonicalize before LPM), so these two selectors match - // the identical client set. - let mut n1 = node("n1", 0); - n1.advertised_addresses = vec![ - selector("10.0.0.0/8", "10.0.0.1"), - selector("::ffff:10.0.0.0/104", "10.0.0.2"), - ]; - - assert!(cfg(vec![n1, node("n2", 1)]).validate().is_err()); - } - - #[test] - fn validate_rejects_selector_endpoint_clashing_with_another_nodes_catch_all() { - // The catch-all matches every client, so a 10.0.0.0/16 client would - // resolve both nodes to 192.0.2.10:8090. - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_addresses = vec![selector("10.0.0.0/16", "192.0.2.10")]; - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "10.0.0.2".to_owned(); - n2.advertised_address = Some("192.0.2.10".to_owned()); - n2.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, n2]).validate().is_err()); - } - - #[test] - fn validate_rejects_selector_endpoint_clashing_with_another_nodes_roster_ip() { - // Without an advertised_address the roster ip backs the catch-all, - // so the same cross-set conflict applies to it. - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_addresses = vec![selector("10.0.0.0/16", "10.0.0.2")]; - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "10.0.0.2".to_owned(); - n2.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, n2]).validate().is_err()); - } - - #[test] - fn validate_rejects_identical_selector_endpoint_across_nested_cidrs() { - // LPM runs per node, not cluster-wide: n1 has no longer prefix of - // its own shadowing the /16 overlap, so a 10.0.0.0/16 client wins - // n1's /8 and n2's /16, resolving both to 192.0.2.10:8090. - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_addresses = vec![selector("10.0.0.0/8", "192.0.2.10")]; - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "10.0.0.2".to_owned(); - n2.advertised_addresses = vec![selector("10.0.0.0/16", "192.0.2.10")]; - n2.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, n2]).validate().is_err()); - } - - #[test] - fn validate_accepts_nested_cidr_reuse_shadowed_by_same_node_longer_prefix() { - // n1's /16 selector shadows its /8 within 10.0.0.0/16, so n1's /8 - // entry wins only 10.0.0.0/8 minus 10.0.0.0/16 - disjoint from n2's - // /16. No client resolves both nodes to 192.0.2.10:8090. - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_addresses = vec![ - selector("10.0.0.0/8", "192.0.2.10"), - selector("10.0.0.0/16", "192.0.2.20"), - ]; - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "10.0.0.2".to_owned(); - n2.advertised_addresses = vec![selector("10.0.0.0/16", "192.0.2.10")]; - n2.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, n2]).validate().is_ok()); - } - - #[test] - fn validate_rejects_partially_shadowed_nested_cidr_reuse() { - // n1's /24 shadow carves only part of the /16 overlap: a client in - // 10.0.0.0/16 outside 10.0.0.0/24 still wins n1's /8 entry and n2's - // /16 entry, resolving both nodes to 192.0.2.10:8090. - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_addresses = vec![ - selector("10.0.0.0/8", "192.0.2.10"), - selector("10.0.0.0/24", "192.0.2.20"), - ]; - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "10.0.0.2".to_owned(); - n2.advertised_addresses = vec![selector("10.0.0.0/16", "192.0.2.10")]; - n2.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, n2]).validate().is_err()); - } - - #[test] - fn validate_accepts_catch_all_reuse_shadowed_by_same_node_selector() { - // n1's /16 selector shadows its catch-all within 10.0.0.0/16, so - // the catch-all never wins a client inside n2's /24. Without the - // shadow the same pair conflicts (see - // validate_rejects_selector_endpoint_clashing_with_another_nodes_catch_all). - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_address = Some("192.0.2.10".to_owned()); - n1.advertised_addresses = vec![selector("10.0.0.0/16", "192.0.2.20")]; - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "10.0.0.2".to_owned(); - n2.advertised_addresses = vec![selector("10.0.0.0/24", "192.0.2.10")]; - n2.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, n2]).validate().is_ok()); - } - - #[test] - fn validate_accepts_selector_reusing_a_fully_shadowed_catch_all_address() { - // n1's selectors cover both address families, so its catch-all wins - // known peers nowhere; only unknown-peer clients reach it, and they - // never match n2's selector. - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_address = Some("192.0.2.10".to_owned()); - n1.advertised_addresses = vec![ - selector("0.0.0.0/0", "192.0.2.20"), - selector("::/0", "192.0.2.30"), - ]; - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "10.0.0.2".to_owned(); - n2.advertised_addresses = vec![selector("10.0.0.0/16", "192.0.2.10")]; - n2.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, n2]).validate().is_ok()); - } - - #[test] - fn validate_accepts_catch_all_spelling_another_nodes_selector_address_when_self_shadowed() { - // Split-network NAT roster: n2's catch-all spells n1's 10/8 selector - // address, but n2's own 10/8 selector shadows its catch-all inside - // 10/8 (outside it n1 serves its own catch-all), so no client - // resolves both nodes to 192.0.2.10:8090. - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_addresses = vec![selector("10.0.0.0/8", "192.0.2.10")]; - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "10.0.0.2".to_owned(); - n2.advertised_address = Some("192.0.2.10".to_owned()); - n2.advertised_addresses = vec![selector("10.0.0.0/8", "192.0.2.20")]; - n2.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, n2]).validate().is_ok()); - } - - #[test] - fn validate_rejects_duplicate_catch_all_even_when_fully_shadowed() { - // A client whose peer address the transport cannot produce always - // falls to the catch-all, so duplicate catch-all conflict even when - // selectors cover every known network. - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_address = Some("192.0.2.10".to_owned()); - n1.advertised_addresses = vec![ - selector("0.0.0.0/0", "192.0.2.20"), - selector("::/0", "192.0.2.30"), - ]; - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "10.0.0.2".to_owned(); - n2.advertised_address = Some("192.0.2.10".to_owned()); - n2.advertised_addresses = vec![ - selector("0.0.0.0/0", "192.0.2.40"), - selector("::/0", "192.0.2.50"), - ]; - n2.ports.tcp = Some(8090); - - assert!(cfg(vec![n1, n2]).validate().is_err()); - } - - #[test] - fn validate_accepts_selector_reusing_its_own_nodes_catch_all_address() { - // Redundant but harmless: within one node the selector and the - // catch-all cannot resolve a client to two different nodes. - let mut n1 = node("n1", 0); - n1.ip = "10.0.0.1".to_owned(); - n1.advertised_address = Some("192.0.2.10".to_owned()); - n1.advertised_addresses = vec![selector("10.0.0.0/16", "192.0.2.10")]; - n1.ports.tcp = Some(8090); - let mut n2 = node("n2", 1); - n2.ip = "10.0.0.2".to_owned(); - n2.ports.tcp = Some(8091); - - assert!(cfg(vec![n1, n2]).validate().is_ok()); - } - - #[test] - fn validate_rejects_zero_tcp_replica_port() { - let ports = TransportPorts { - tcp: None, - quic: None, - http: None, - websocket: None, - tcp_replica: Some(0), - }; - let mut n1 = node("n1", 0); - n1.ports = ports; - let c = cfg(vec![n1]); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_accepts_empty_secret_when_auth_disabled() { - // Default: no secret, auth off -> legacy mode, must pass. - let c = cfg(vec![node("n1", 0), node("n2", 1)]); - assert!(c.validate().is_ok()); - } - - #[test] - fn validate_rejects_missing_secret_when_auth_enabled() { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.auth.enabled = true; - assert!(c.validate().is_err()); - } - - #[test] - fn validate_rejects_short_secret_when_auth_enabled() { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.auth.enabled = true; - c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN - 1); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_rejects_short_secret_even_when_auth_disabled() { - // Typo guard: a configured-but-short key fails even with auth off. - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN - 1); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_accepts_valid_secret_when_auth_enabled() { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.auth.enabled = true; - c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); - assert!(c.validate().is_ok()); - } - - #[test] - fn validate_accepts_valid_rotation_window() { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.auth.enabled = true; - c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); - c.auth.previous_shared_secret = "b".repeat(MIN_SHARED_SECRET_LEN); - assert!(c.validate().is_ok()); - } - - #[test] - fn validate_rejects_short_previous_secret() { - // Same typo guard as the primary key. - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.auth.enabled = true; - c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); - c.auth.previous_shared_secret = "b".repeat(MIN_SHARED_SECRET_LEN - 1); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_rejects_rotation_window_equal_to_primary() { - // An identical window is a no-op rotation: the operator rolled the - // config without changing the key. - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.auth.enabled = true; - c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); - c.auth.previous_shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); - assert!(c.validate().is_err()); - } - - fn tls_files() -> ClusterTlsConfig { - ClusterTlsConfig { - enabled: true, - self_signed: false, - cert_file: "cert.pem".to_string(), - key_file: "key.pem".to_string(), - ca_file: "ca.pem".to_string(), - } - } - - #[test] - fn validate_rejects_tls_ca_mode_with_missing_files() { - // Auth on so the failure exercises the file check, not the auth gate. - for missing in ["cert_file", "key_file", "ca_file"] { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.auth.enabled = true; - c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); - c.tls = tls_files(); - match missing { - "cert_file" => c.tls.cert_file.clear(), - "key_file" => c.tls.key_file.clear(), - _ => c.tls.ca_file.clear(), - } - assert!(c.validate().is_err(), "missing {missing} must be rejected"); - } - } - - #[test] - fn validate_rejects_tls_self_signed_without_auth() { - // Accept-any certificate without the PSK handshake = MITM-able. - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.tls = ClusterTlsConfig { - enabled: true, - self_signed: true, - ..ClusterTlsConfig::default() - }; - assert!(c.validate().is_err()); - } - - #[test] - fn validate_accepts_tls_self_signed_with_auth() { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.auth.enabled = true; - c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); - c.tls = ClusterTlsConfig { - enabled: true, - self_signed: true, - ..ClusterTlsConfig::default() - }; - assert!(c.validate().is_ok()); - } - - #[test] - fn validate_rejects_tls_ca_mode_without_auth() { - // TLS never authenticates the dialer (no client certificates); - // only the PSK handshake does, so it is mandatory with TLS on. - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.tls = tls_files(); - assert!(c.validate().is_err()); - } - - #[test] - fn validate_accepts_tls_ca_mode_with_auth() { - let mut c = cfg(vec![node("n1", 0), node("n2", 1)]); - c.auth.enabled = true; - c.auth.shared_secret = "a".repeat(MIN_SHARED_SECRET_LEN); - c.tls = tls_files(); - assert!(c.validate().is_ok()); - } -} diff --git a/core/configs/src/server_ng_config/defaults.rs b/core/configs/src/server_ng_config/defaults.rs deleted file mode 100644 index 40695a63e6..0000000000 --- a/core/configs/src/server_ng_config/defaults.rs +++ /dev/null @@ -1,335 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! `Default` impls for the server-ng config surface. -//! -//! Sections that fork (`tcp`, `websocket`, `quic`, `cluster`, -//! `message_bus`) have their own `Default` impls here, sourced from -//! `core/server-ng/config.toml` via [`SERVER_NG_CONFIG`]. Sections that -//! still reuse legacy types (`http`, `system`, `telemetry`, -//! `consumer_group`, `data_maintenance`, `message_saver`, -//! `personal_access_token`, `heartbeat`) delegate to the legacy -//! `Default` impls; overrides land at the consumer level once -//! [`super::server_ng::ServerNgConfig::load`] is wired into server-ng's -//! bootstrap. - -use super::cluster::{ - ClusterAuthConfig, ClusterConfig, ClusterNodeConfig, ClusterTlsConfig, TransportPorts, -}; -use super::message_bus::MessageBusConfig; -use super::metadata::MetadataConfig; -use super::partition::PartitionConfig; -use super::quic::{QuicCertificateConfig, QuicConfig, QuicSocketConfig}; -use super::server_ng::NgSystemConfig; -use super::server_ng::{ExtraConfig, ServerNgConfig}; -use super::tcp::{TcpConfig, TcpSocketConfig, TcpTlsConfig}; -use super::websocket::{WebSocketConfig, WebSocketTlsConfig}; -use crate::server_config::http::HttpConfig; -use crate::server_config::server::{ - ConsumerGroupConfig, DataMaintenanceConfig, HeartbeatConfig, MessageSaverConfig, - PersonalAccessTokenConfig, TelemetryConfig, -}; -use std::sync::Arc; - -static_toml::static_toml! { - // static_toml resolves relative to CARGO_MANIFEST_DIR (core/configs/). - pub static SERVER_NG_CONFIG = include_toml!("../server-ng/config.toml"); -} - -impl Default for ServerNgConfig { - fn default() -> ServerNgConfig { - ServerNgConfig { - consumer_group: ConsumerGroupConfig::default(), - data_maintenance: DataMaintenanceConfig::default(), - extra: ExtraConfig::default(), - heartbeat: HeartbeatConfig::default(), - message_saver: MessageSaverConfig::default(), - personal_access_token: PersonalAccessTokenConfig::default(), - system: Arc::new(NgSystemConfig::default()), - quic: QuicConfig::default(), - tcp: TcpConfig::default(), - websocket: WebSocketConfig::default(), - http: HttpConfig::default(), - telemetry: TelemetryConfig::default(), - cluster: ClusterConfig::default(), - metadata: MetadataConfig::default(), - partition: PartitionConfig::default(), - message_bus: MessageBusConfig::default(), - } - } -} - -impl Default for ClusterConfig { - fn default() -> ClusterConfig { - ClusterConfig { - enabled: SERVER_NG_CONFIG.cluster.enabled, - name: SERVER_NG_CONFIG.cluster.name.parse().unwrap(), - heartbeat_timeout: SERVER_NG_CONFIG.cluster.heartbeat_timeout.parse().unwrap(), - commit_broadcast_interval: SERVER_NG_CONFIG - .cluster - .commit_broadcast_interval - .parse() - .unwrap(), - prepare_retransmit_interval: SERVER_NG_CONFIG - .cluster - .prepare_retransmit_interval - .parse() - .unwrap(), - view_change_retransmit_interval: SERVER_NG_CONFIG - .cluster - .view_change_retransmit_interval - .parse() - .unwrap(), - view_change_status_timeout: SERVER_NG_CONFIG - .cluster - .view_change_status_timeout - .parse() - .unwrap(), - request_start_view_retransmit_interval: SERVER_NG_CONFIG - .cluster - .request_start_view_retransmit_interval - .parse() - .unwrap(), - view_probe_attempts_max: SERVER_NG_CONFIG.cluster.view_probe_attempts_max as u32, - repair_retry_interval: SERVER_NG_CONFIG - .cluster - .repair_retry_interval - .parse() - .unwrap(), - repair_chunk_max: SERVER_NG_CONFIG.cluster.repair_chunk_max as usize, - nodes: SERVER_NG_CONFIG - .cluster - .nodes - .iter() - .map(|node| ClusterNodeConfig { - name: node.name.parse().unwrap(), - ip: node.ip.parse().unwrap(), - advertised_address: None, - advertised_addresses: Vec::new(), - replica_id: u8::try_from(node.replica_id).expect( - "static_toml replica_id must fit in u8 (0..=255); \ - fix core/server-ng/config.toml", - ), - ports: TransportPorts { - tcp: Some(u16::try_from(node.ports.tcp).expect( - "static_toml cluster.nodes.ports.tcp must fit in u16 (0..=65535); \ - fix core/server-ng/config.toml", - )), - quic: Some(u16::try_from(node.ports.quic).expect( - "static_toml cluster.nodes.ports.quic must fit in u16 (0..=65535); \ - fix core/server-ng/config.toml", - )), - http: Some(u16::try_from(node.ports.http).expect( - "static_toml cluster.nodes.ports.http must fit in u16 (0..=65535); \ - fix core/server-ng/config.toml", - )), - websocket: Some(u16::try_from(node.ports.websocket).expect( - "static_toml cluster.nodes.ports.websocket must fit in u16 (0..=65535); \ - fix core/server-ng/config.toml", - )), - tcp_replica: Some(u16::try_from(node.ports.tcp_replica).expect( - "static_toml cluster.nodes.ports.tcp_replica must fit in u16 (0..=65535); \ - fix core/server-ng/config.toml", - )), - }, - }) - .collect(), - auth: ClusterAuthConfig::default(), - tls: ClusterTlsConfig::default(), - } - } -} - -impl Default for MetadataConfig { - fn default() -> MetadataConfig { - // Read from the embedded TOML so the Default impl and the on-disk - // schema cannot drift (same pattern as MessageBusConfig below). - let metadata = &SERVER_NG_CONFIG.metadata; - MetadataConfig { - prepare_queue_depth: metadata.prepare_queue_depth as usize, - journal_slots: metadata.journal_slots as usize, - clients_table_max: metadata.clients_table_max as usize, - } - } -} - -impl Default for PartitionConfig { - fn default() -> PartitionConfig { - // Read from the embedded TOML so the Default impl and the on-disk - // schema cannot drift (same pattern as MetadataConfig above). - let partition = &SERVER_NG_CONFIG.partition; - PartitionConfig { - prepare_queue_depth: partition.prepare_queue_depth as usize, - evicted_ring_capacity: partition.evicted_ring_capacity as usize, - evicted_ring_bytes_max: partition.evicted_ring_bytes_max.parse().unwrap(), - transfer_served_cache_bytes_max: partition - .transfer_served_cache_bytes_max - .parse() - .unwrap(), - transfer_artifact_bytes_max: partition.transfer_artifact_bytes_max.parse().unwrap(), - } - } -} - -impl Default for QuicConfig { - fn default() -> QuicConfig { - QuicConfig { - enabled: SERVER_NG_CONFIG.quic.enabled, - address: SERVER_NG_CONFIG.quic.address.parse().unwrap(), - max_concurrent_bidi_streams: SERVER_NG_CONFIG.quic.max_concurrent_bidi_streams as u64, - datagram_send_buffer_size: SERVER_NG_CONFIG - .quic - .datagram_send_buffer_size - .parse() - .unwrap(), - initial_mtu: SERVER_NG_CONFIG.quic.initial_mtu.parse().unwrap(), - send_window: SERVER_NG_CONFIG.quic.send_window.parse().unwrap(), - receive_window: SERVER_NG_CONFIG.quic.receive_window.parse().unwrap(), - keep_alive_interval: SERVER_NG_CONFIG.quic.keep_alive_interval.parse().unwrap(), - max_idle_timeout: SERVER_NG_CONFIG.quic.max_idle_timeout.parse().unwrap(), - certificate: QuicCertificateConfig::default(), - socket: QuicSocketConfig::default(), - } - } -} - -impl Default for QuicSocketConfig { - fn default() -> QuicSocketConfig { - QuicSocketConfig { - override_defaults: SERVER_NG_CONFIG.quic.socket.override_defaults, - recv_buffer_size: SERVER_NG_CONFIG - .quic - .socket - .recv_buffer_size - .parse() - .unwrap(), - send_buffer_size: SERVER_NG_CONFIG - .quic - .socket - .send_buffer_size - .parse() - .unwrap(), - keepalive: SERVER_NG_CONFIG.quic.socket.keepalive, - } - } -} - -impl Default for QuicCertificateConfig { - fn default() -> QuicCertificateConfig { - QuicCertificateConfig { - self_signed: SERVER_NG_CONFIG.quic.certificate.self_signed, - cert_file: SERVER_NG_CONFIG.quic.certificate.cert_file.parse().unwrap(), - key_file: SERVER_NG_CONFIG.quic.certificate.key_file.parse().unwrap(), - } - } -} - -impl Default for TcpConfig { - fn default() -> TcpConfig { - TcpConfig { - enabled: SERVER_NG_CONFIG.tcp.enabled, - address: SERVER_NG_CONFIG.tcp.address.parse().unwrap(), - ipv6: SERVER_NG_CONFIG.tcp.ipv_6, - tls: TcpTlsConfig::default(), - socket: TcpSocketConfig::default(), - socket_migration: SERVER_NG_CONFIG.tcp.socket_migration, - } - } -} - -impl Default for TcpTlsConfig { - fn default() -> TcpTlsConfig { - TcpTlsConfig { - enabled: SERVER_NG_CONFIG.tcp.tls.enabled, - self_signed: SERVER_NG_CONFIG.tcp.tls.self_signed, - cert_file: SERVER_NG_CONFIG.tcp.tls.cert_file.parse().unwrap(), - key_file: SERVER_NG_CONFIG.tcp.tls.key_file.parse().unwrap(), - } - } -} - -impl Default for TcpSocketConfig { - fn default() -> TcpSocketConfig { - TcpSocketConfig { - override_defaults: SERVER_NG_CONFIG.tcp.socket.override_defaults, - recv_buffer_size: SERVER_NG_CONFIG - .tcp - .socket - .recv_buffer_size - .parse() - .unwrap(), - send_buffer_size: SERVER_NG_CONFIG - .tcp - .socket - .send_buffer_size - .parse() - .unwrap(), - keepalive: SERVER_NG_CONFIG.tcp.socket.keepalive, - nodelay: SERVER_NG_CONFIG.tcp.socket.nodelay, - linger: SERVER_NG_CONFIG.tcp.socket.linger.parse().unwrap(), - } - } -} - -impl Default for WebSocketConfig { - fn default() -> WebSocketConfig { - // The size knobs are optional in the schema (commented-out by - // default), so they map to `None` here when absent; every other - // field comes from the embedded TOML so the Default impl and - // the on-disk schema cannot drift. - WebSocketConfig { - enabled: SERVER_NG_CONFIG.websocket.enabled, - address: SERVER_NG_CONFIG.websocket.address.parse().unwrap(), - read_buffer_size: None, - write_buffer_size: None, - max_write_buffer_size: None, - max_message_size: None, - max_frame_size: None, - accept_unmasked_frames: SERVER_NG_CONFIG.websocket.accept_unmasked_frames, - tls: WebSocketTlsConfig::default(), - } - } -} - -impl Default for WebSocketTlsConfig { - fn default() -> WebSocketTlsConfig { - WebSocketTlsConfig { - enabled: SERVER_NG_CONFIG.websocket.tls.enabled, - self_signed: SERVER_NG_CONFIG.websocket.tls.self_signed, - cert_file: SERVER_NG_CONFIG.websocket.tls.cert_file.parse().unwrap(), - key_file: SERVER_NG_CONFIG.websocket.tls.key_file.parse().unwrap(), - } - } -} - -impl Default for MessageBusConfig { - fn default() -> MessageBusConfig { - // Read every field from the embedded TOML so the Default impl - // and the on-disk schema cannot drift. Sibling impls in this - // file follow the same pattern. - let bus = &SERVER_NG_CONFIG.message_bus; - MessageBusConfig { - max_batch: bus.max_batch as usize, - max_message_size: bus.max_message_size.parse().unwrap(), - peer_queue_capacity: bus.peer_queue_capacity as usize, - reconnect_period: bus.reconnect_period.parse().unwrap(), - close_peer_timeout: bus.close_peer_timeout.parse().unwrap(), - close_grace: bus.close_grace.parse().unwrap(), - handshake_grace: bus.handshake_grace.parse().unwrap(), - } - } -} diff --git a/core/configs/src/server_ng_config/displays.rs b/core/configs/src/server_ng_config/displays.rs deleted file mode 100644 index ab487129b6..0000000000 --- a/core/configs/src/server_ng_config/displays.rs +++ /dev/null @@ -1,189 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! `Display` impls for the server-ng config surface. -//! -//! Reused section types pick up [`Display`] from -//! [`crate::displays`]; this module only adds the top-level -//! [`ServerNgConfig`] formatter and the new [`MessageBusConfig`] -//! section formatter. - -use super::message_bus::MessageBusConfig; -use super::metadata::MetadataConfig; -use super::partition::PartitionConfig; -use super::quic::{QuicCertificateConfig, QuicConfig, QuicSocketConfig}; -use super::server_ng::{ExtraConfig, NamespaceConfig, ServerNgConfig}; -use super::tcp::{TcpConfig, TcpSocketConfig, TcpTlsConfig}; -use std::fmt::{Display, Formatter}; - -impl Display for ServerNgConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ consumer_group: {}, data_maintenance: {}, extra: {}, message_saver: {}, \ - heartbeat: {}, system: {}, quic: {}, tcp: {}, http: {}, telemetry: {}, \ - metadata: {}, message_bus: {}, partition: {} }}", - self.consumer_group, - self.data_maintenance, - self.extra, - self.message_saver, - self.heartbeat, - self.system, - self.quic, - self.tcp, - self.http, - self.telemetry, - self.metadata, - self.message_bus, - self.partition, - ) - } -} - -impl Display for PartitionConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ prepare_queue_depth: {}, evicted_ring_capacity: {}, \ - evicted_ring_bytes_max: {}, transfer_served_cache_bytes_max: {}, \ - transfer_artifact_bytes_max: {} }}", - self.prepare_queue_depth, - self.evicted_ring_capacity, - self.evicted_ring_bytes_max, - self.transfer_served_cache_bytes_max, - self.transfer_artifact_bytes_max, - ) - } -} - -impl Display for MetadataConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ prepare_queue_depth: {}, journal_slots: {}, clients_table_max: {} }}", - self.prepare_queue_depth, self.journal_slots, self.clients_table_max, - ) - } -} - -impl Display for MessageBusConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ max_batch: {}, max_message_size: {}, peer_queue_capacity: {}, \ - reconnect_period: {}, close_peer_timeout: {}, close_grace: {}, \ - handshake_grace: {} }}", - self.max_batch, - self.max_message_size, - self.peer_queue_capacity, - self.reconnect_period, - self.close_peer_timeout, - self.close_grace, - self.handshake_grace, - ) - } -} - -impl Display for ExtraConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{{ namespace: {} }}", self.namespace) - } -} - -impl Display for NamespaceConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ max_streams: {}, max_topics: {}, max_partitions: {} }}", - self.max_streams, self.max_topics, self.max_partitions - ) - } -} - -impl Display for TcpConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ enabled: {}, address: {}, ipv6: {}, tls: {}, socket: {}, socket_migration: {} }}", - self.enabled, self.address, self.ipv6, self.tls, self.socket, self.socket_migration - ) - } -} - -impl Display for TcpTlsConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ enabled: {}, self_signed: {}, cert_file: {}, key_file: {} }}", - self.enabled, self.self_signed, self.cert_file, self.key_file - ) - } -} - -impl Display for TcpSocketConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ override_defaults: {}, recv_buffer_size: {}, send_buffer_size: {}, keepalive: {}, nodelay: {}, linger: {} }}", - self.override_defaults, - self.recv_buffer_size, - self.send_buffer_size, - self.keepalive, - self.nodelay, - self.linger, - ) - } -} - -impl Display for QuicConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ enabled: {}, address: {}, max_concurrent_bidi_streams: {}, datagram_send_buffer_size: {}, initial_mtu: {}, send_window: {}, receive_window: {}, keep_alive_interval: {}, max_idle_timeout: {}, certificate: {} }}", - self.enabled, - self.address, - self.max_concurrent_bidi_streams, - self.datagram_send_buffer_size, - self.initial_mtu, - self.send_window, - self.receive_window, - self.keep_alive_interval, - self.max_idle_timeout, - self.certificate - ) - } -} - -impl Display for QuicCertificateConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ self_signed: {}, cert_file: {}, key_file: {} }}", - self.self_signed, self.cert_file, self.key_file - ) - } -} - -impl Display for QuicSocketConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ override_defaults: {}, recv_buffer_size: {}, send_buffer_size: {}, keepalive: {} }}", - self.override_defaults, self.recv_buffer_size, self.send_buffer_size, self.keepalive - ) - } -} diff --git a/core/configs/src/server_ng_config/mod.rs b/core/configs/src/server_ng_config/mod.rs deleted file mode 100644 index a1b9b774eb..0000000000 --- a/core/configs/src/server_ng_config/mod.rs +++ /dev/null @@ -1,45 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! On-disk schema for the `server-ng` binary. -//! -//! Mirrors the legacy server-config section surface verbatim -//! (operator-facing schema is unchanged) and adds a `message_bus` section -//! for inter-shard / inter-replica bus tunables previously hardcoded in -//! the `core/message_bus` runtime crate. -//! -//! Scaffolding-only at the time of introduction: the type is defined and -//! loadable but no binary calls [`server_ng::ServerNgConfig::load`]; the -//! wiring PR for `core/server-ng`'s bootstrap and the message_bus crate's -//! runtime type is a separate change. - -pub mod cluster; -pub mod defaults; -pub mod displays; -pub mod message_bus; -pub mod metadata; -pub mod partition; -pub mod quic; -pub mod server_ng; -pub mod sharding; -pub mod tcp; -pub mod validators; -pub mod websocket; - -/// Component tag used in error messages for the server-ng config surface. -/// Mirrors [`crate::COMPONENT`] (`"CONFIG"`). -pub const COMPONENT_NG: &str = "CONFIG_NG"; diff --git a/core/configs/src/server_ng_config/quic.rs b/core/configs/src/server_ng_config/quic.rs deleted file mode 100644 index d86402921c..0000000000 --- a/core/configs/src/server_ng_config/quic.rs +++ /dev/null @@ -1,222 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Server-ng QUIC listener schema. -//! -//! Field shape mirrors the legacy [`crate::quic::QuicConfig`] verbatim; -//! the type is forked into `server_ng_config` so server-ng can evolve -//! its QUIC surface independently of the legacy server. No semantic -//! change at fork time. - -use super::COMPONENT_NG; -use crate::ConfigurationError; -use configs::ConfigEnv; -use iggy_common::{IggyByteSize, IggyDuration, Validatable}; -use serde::{Deserialize, Serialize}; -use serde_with::DisplayFromStr; -use serde_with::serde_as; - -#[serde_as] -#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] -pub struct QuicConfig { - pub enabled: bool, - pub address: String, - pub max_concurrent_bidi_streams: u64, - #[config_env(leaf)] - pub datagram_send_buffer_size: IggyByteSize, - #[config_env(leaf)] - pub initial_mtu: IggyByteSize, - #[config_env(leaf)] - pub send_window: IggyByteSize, - #[config_env(leaf)] - pub receive_window: IggyByteSize, - #[config_env(leaf)] - #[serde_as(as = "DisplayFromStr")] - pub keep_alive_interval: IggyDuration, - #[config_env(leaf)] - #[serde_as(as = "DisplayFromStr")] - pub max_idle_timeout: IggyDuration, - pub certificate: QuicCertificateConfig, - pub socket: QuicSocketConfig, -} - -#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] -pub struct QuicSocketConfig { - pub override_defaults: bool, - #[config_env(leaf)] - pub recv_buffer_size: IggyByteSize, - #[config_env(leaf)] - pub send_buffer_size: IggyByteSize, - pub keepalive: bool, -} - -#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] -pub struct QuicCertificateConfig { - pub self_signed: bool, - pub cert_file: String, - pub key_file: String, -} - -/// Validates the field range constraints the runtime conversion in -/// `core::message_bus::config::build_quic_tuning` previously enforced -/// via `expect(...)`. Surfacing them here turns boot-time misconfig -/// into a `ConfigurationError` instead of a panic in the bus crate. -impl Validatable for QuicConfig { - fn validate(&self) -> Result<(), ConfigurationError> { - // QUIC requires at least one bidi stream per connection. - if self.max_concurrent_bidi_streams == 0 { - eprintln!("{COMPONENT_NG} quic.max_concurrent_bidi_streams must be >= 1"); - return Err(ConfigurationError::InvalidConfigurationValue); - } - // quinn-proto stores stream counts as u32 internally. - if u32::try_from(self.max_concurrent_bidi_streams).is_err() { - eprintln!( - "{COMPONENT_NG} quic.max_concurrent_bidi_streams ({}) does not fit in u32", - self.max_concurrent_bidi_streams - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - // The datagram send buffer is materialized as a `Vec` of this - // length in compio-quic, so it must fit in `usize` on the target - // platform. - if usize::try_from(self.datagram_send_buffer_size.as_bytes_u64()).is_err() { - eprintln!( - "{COMPONENT_NG} quic.datagram_send_buffer_size ({} bytes) does not fit in usize on this target", - self.datagram_send_buffer_size.as_bytes_u64() - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - // RFC 9000 §14: minimum required MTU is 1200 bytes; quinn stores - // initial_mtu as u16 (max 65535). - let initial_mtu = self.initial_mtu.as_bytes_u64(); - if initial_mtu < 1200 { - eprintln!( - "{COMPONENT_NG} quic.initial_mtu ({initial_mtu}) is below the QUIC minimum of 1200 bytes", - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if u16::try_from(initial_mtu).is_err() { - eprintln!("{COMPONENT_NG} quic.initial_mtu ({initial_mtu}) exceeds u16::MAX (65535)",); - return Err(ConfigurationError::InvalidConfigurationValue); - } - // quinn VarInt for `receive_window` accepts u32; rejecting - // out-of-range values here surfaces a config error rather than - // panicking inside the bus crate's runtime conversion. - if u32::try_from(self.receive_window.as_bytes_u64()).is_err() { - eprintln!( - "{COMPONENT_NG} quic.receive_window ({} bytes) does not fit in u32 (quinn VarInt limit)", - self.receive_window.as_bytes_u64() - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - // `send_window` is u64-sized in QuicTuning, but quinn's VarInt - // protocol-level cap is 2^62 - 1; reject anything above that. - const QUINN_VARINT_MAX: u64 = (1u64 << 62) - 1; - if self.send_window.as_bytes_u64() > QUINN_VARINT_MAX { - eprintln!( - "{COMPONENT_NG} quic.send_window ({} bytes) exceeds quinn VarInt max (2^62 - 1)", - self.send_window.as_bytes_u64() - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn baseline() -> QuicConfig { - QuicConfig { - enabled: false, - address: String::new(), - max_concurrent_bidi_streams: 1, - datagram_send_buffer_size: IggyByteSize::from(100_u64 * 1024), - initial_mtu: IggyByteSize::from(8_u64 * 1024), - send_window: IggyByteSize::from(64_u64 * 1024 * 1024), - receive_window: IggyByteSize::from(64_u64 * 1024 * 1024), - keep_alive_interval: IggyDuration::from(std::time::Duration::from_secs(10)), - max_idle_timeout: IggyDuration::from(std::time::Duration::from_secs(30)), - certificate: QuicCertificateConfig { - self_signed: false, - cert_file: String::new(), - key_file: String::new(), - }, - socket: QuicSocketConfig { - override_defaults: false, - recv_buffer_size: IggyByteSize::from(0_u64), - send_buffer_size: IggyByteSize::from(0_u64), - keepalive: false, - }, - } - } - - #[test] - fn baseline_validates() { - baseline().validate().expect("baseline is valid"); - } - - #[test] - fn rejects_zero_max_concurrent_bidi_streams() { - let mut c = baseline(); - c.max_concurrent_bidi_streams = 0; - assert!(c.validate().is_err()); - } - - #[test] - fn rejects_max_concurrent_bidi_streams_above_u32() { - let mut c = baseline(); - c.max_concurrent_bidi_streams = u64::from(u32::MAX) + 1; - assert!(c.validate().is_err()); - } - - #[test] - fn rejects_initial_mtu_below_qiuc_minimum() { - let mut c = baseline(); - c.initial_mtu = IggyByteSize::from(1199_u64); - assert!(c.validate().is_err()); - } - - #[test] - fn rejects_initial_mtu_above_u16() { - let mut c = baseline(); - c.initial_mtu = IggyByteSize::from(u64::from(u16::MAX) + 1); - assert!(c.validate().is_err()); - } - - #[test] - fn rejects_receive_window_above_u32() { - let mut c = baseline(); - c.receive_window = IggyByteSize::from(u64::from(u32::MAX) + 1); - assert!(c.validate().is_err()); - } - - #[test] - fn rejects_send_window_above_quinn_varint_max() { - let mut c = baseline(); - c.send_window = IggyByteSize::from(1_u64 << 62); - assert!(c.validate().is_err()); - } - - #[test] - fn accepts_initial_mtu_at_quic_minimum() { - let mut c = baseline(); - c.initial_mtu = IggyByteSize::from(1200_u64); - assert!(c.validate().is_ok()); - } -} diff --git a/core/configs/src/server_ng_config/server_ng.rs b/core/configs/src/server_ng_config/server_ng.rs deleted file mode 100644 index 3484d5f36c..0000000000 --- a/core/configs/src/server_ng_config/server_ng.rs +++ /dev/null @@ -1,229 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use super::COMPONENT_NG; -use super::cluster::ClusterConfig; -use super::message_bus::MessageBusConfig; -use super::metadata::MetadataConfig; -use super::partition::PartitionConfig; -use super::quic::QuicConfig; -use super::tcp::TcpConfig; -use super::websocket::WebSocketConfig; -use crate::ConfigurationError; -use crate::server_config::http::HttpConfig; -use crate::server_config::server::{ - ConsumerGroupConfig, DataMaintenanceConfig, HeartbeatConfig, MessageSaverConfig, - PersonalAccessTokenConfig, TelemetryConfig, -}; -use crate::server_config::system::SystemConfig; -use configs::{ConfigEnv, ConfigEnvMappings, ConfigProvider, FileConfigProvider, TypedEnvProvider}; -use err_trail::ErrContext; -use figment::providers::{Format, Toml}; -use figment::value::Dict; -use figment::{Metadata, Profile, Provider}; -use iggy_common::Validatable; -use serde::{Deserialize, Serialize}; -use server_common::sharding::{MAX_PARTITIONS, MAX_STREAMS, MAX_TOPICS}; -use std::env; -use std::sync::Arc; - -const DEFAULT_CONFIG_PATH: &str = "core/server-ng/config.toml"; - -/// The `server-ng` flavour of [`SystemConfig`], bound to this crate's own -/// [`super::sharding::ShardingConfig`]. `core/server-ng` names this alias -/// wherever it refers to the system config. -pub type NgSystemConfig = SystemConfig; - -/// Top-level on-disk config schema for the `server-ng` binary. -/// -/// Mirrors the legacy [`crate::server::ServerConfig`] section surface -/// verbatim (operator-facing schema is unchanged) and adds a -/// [`MessageBusConfig`] section for inter-shard / inter-replica bus -/// tunables. -/// -/// Section types are reused directly from the legacy server-config -/// modules; only [`MessageBusConfig`] and this composer are net-new -/// code at the time of introduction. -/// -/// At the time of introduction this type is NOT consumed by -/// `core/server-ng`'s bootstrap. The wiring PR is a separate change. -#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] -#[config_env(prefix = "IGGY_", name = "iggy-server-ng-config")] -pub struct ServerNgConfig { - pub consumer_group: ConsumerGroupConfig, - pub data_maintenance: DataMaintenanceConfig, - #[serde(default)] - pub extra: ExtraConfig, - pub message_saver: MessageSaverConfig, - pub personal_access_token: PersonalAccessTokenConfig, - pub heartbeat: HeartbeatConfig, - pub system: Arc, - pub quic: QuicConfig, - pub tcp: TcpConfig, - pub http: HttpConfig, - pub websocket: WebSocketConfig, - pub telemetry: TelemetryConfig, - pub cluster: ClusterConfig, - pub metadata: MetadataConfig, - pub partition: PartitionConfig, - pub message_bus: MessageBusConfig, -} - -#[derive(Debug, Default, Deserialize, Serialize, Clone, ConfigEnv)] -pub struct ExtraConfig { - pub namespace: NamespaceConfig, -} - -#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] -pub struct NamespaceConfig { - pub max_streams: usize, - pub max_topics: usize, - pub max_partitions: usize, -} - -impl Default for NamespaceConfig { - fn default() -> Self { - Self { - max_streams: MAX_STREAMS, - max_topics: MAX_TOPICS, - max_partitions: MAX_PARTITIONS, - } - } -} - -impl ServerNgConfig { - /// Load server-ng configuration from file and environment variables. - /// - /// Mirrors [`crate::server::ServerConfig::load`]: the path comes - /// from `IGGY_CONFIG_PATH` or defaults to - /// `core/server-ng/config.toml`; missing on-disk paths fall through - /// to the embedded default TOML; env-var overrides flow through the - /// [`ServerNgConfigEnvProvider`]; the result is validated before - /// returning. - /// - /// # Errors - /// Returns [`ConfigurationError`] when the config cannot be parsed - /// from the configured source(s) or fails [`Validatable::validate`]. - pub async fn load() -> Result { - let config_path = - env::var("IGGY_CONFIG_PATH").unwrap_or_else(|_| DEFAULT_CONFIG_PATH.to_string()); - let provider = ServerNgConfig::config_provider(&config_path); - let cfg: ServerNgConfig = - provider - .load_config() - .await - .error(|e: &configs::ConfigurationError| { - format!("{COMPONENT_NG} (error: {e}) - failed to load server-ng config") - })?; - cfg.validate().error(|e: &configs::ConfigurationError| { - format!("{COMPONENT_NG} (error: {e}) - failed to validate server-ng config") - })?; - Ok(cfg) - } - - /// Build the file-backed config provider with the embedded default - /// TOML and the type-safe env-var provider attached. - pub fn config_provider(config_path: &str) -> FileConfigProvider { - let default_config = Toml::string(include_str!("../../../server-ng/config.toml")); - FileConfigProvider::new( - config_path.to_string(), - ServerNgConfigEnvProvider::default(), - true, - Some(default_config), - ) - } - - /// All recognised env var names for [`ServerNgConfig`]. - pub fn all_env_var_names() -> Vec<&'static str> { - ::all_env_var_names() - } -} - -/// Type-safe environment provider for [`ServerNgConfig`]. -/// -/// Uses the [`ConfigEnvMappings`] trait generated by `#[derive(ConfigEnv)]` -/// to look up known env var names directly, eliminating path ambiguity. -#[derive(Debug, Clone)] -pub struct ServerNgConfigEnvProvider { - provider: TypedEnvProvider, -} - -impl Default for ServerNgConfigEnvProvider { - fn default() -> Self { - Self { - provider: TypedEnvProvider::from_config(ServerNgConfig::ENV_PREFIX), - } - } -} - -impl Provider for ServerNgConfigEnvProvider { - fn metadata(&self) -> Metadata { - Metadata::named(ServerNgConfig::ENV_PROVIDER_NAME) - } - - fn data(&self) -> Result, figment::Error> { - self.provider.deserialize().map_err(|e| { - figment::Error::from(format!( - "Cannot deserialize environment variables for server-ng config: {e}" - )) - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use figment::Figment; - - /// The embedded default TOML deserializes into a fully populated - /// [`ServerNgConfig`] and passes validation. Exercises the - /// `include_str!` resolution and the deserialization of every - /// section without depending on an async runtime in `dev-deps`. - #[test] - fn embedded_default_toml_deserializes_and_validates() { - let toml_str = include_str!("../../../server-ng/config.toml"); - let cfg: ServerNgConfig = Figment::new() - .merge(Toml::string(toml_str)) - .extract() - .expect("embedded TOML deserializes"); - cfg.validate().expect("embedded default validates"); - - // Spot-check: defaults match the runtime crate's invariants. - assert_eq!(cfg.message_bus.max_batch, 256); - assert_eq!(cfg.message_bus.peer_queue_capacity, 256); - } - - #[test] - fn default_impl_validates() { - let cfg = ServerNgConfig::default(); - cfg.validate().expect("Default impl validates"); - } - - #[test] - fn env_prefix_is_iggy() { - assert_eq!(ServerNgConfig::ENV_PREFIX, "IGGY_"); - } - - #[test] - fn all_env_var_names_include_message_bus_section() { - let names = ServerNgConfig::all_env_var_names(); - assert!( - names.iter().any(|n| n.starts_with("IGGY_MESSAGE_BUS_")), - "expected at least one IGGY_MESSAGE_BUS_* env var, got: {names:?}" - ); - } -} diff --git a/core/configs/src/server_ng_config/sharding.rs b/core/configs/src/server_ng_config/sharding.rs deleted file mode 100644 index f5c2257d43..0000000000 --- a/core/configs/src/server_ng_config/sharding.rs +++ /dev/null @@ -1,431 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Sharding config for `server-ng`. Forked from the legacy -//! [`crate::server_config::sharding`] because the two servers own different -//! knob sets and different default sources: this type carries the full -//! thread-per-core + bus surface and reads its defaults from the server-ng -//! TOML, while the legacy type keeps only `cpu_allocation` + `pin_cores`. - -use iggy_common::IggyDuration; -use iggy_common::Validatable; -use serde::{Deserialize, Serialize}; -use serde_with::{DisplayFromStr, serde_as}; -use std::time::Duration; - -use super::defaults::SERVER_NG_CONFIG; -use crate::ConfigurationError; -use crate::server_config::validators::validate_cpu_allocation; -use configs::ConfigEnv; - -// Re-exported so the `configs::ng_sharding::*` path mirrors the legacy -// `configs::sharding::*` surface for callers. -pub use cpu_allocation::{CpuAllocation, NumaConfig}; - -/// Maximum permitted per-shard inbox depth. The channel is allocated -/// up-front per shard, so a runaway value here OOMs the process at boot. -/// `1 << 20` (~1M frames) is several orders of magnitude above any -/// realistic backpressure target and still fits comfortably in process -/// address space. -pub const INBOX_CAPACITY_MAX: usize = 1 << 20; - -/// Hard upper bound on `shutdown_drain_timeout`. A drain that never -/// completes wedges process exit; capping at 10 minutes guarantees the -/// watchdog eventually force-tears the bus even with a pathological -/// config typo. -pub const SHUTDOWN_DRAIN_TIMEOUT_MAX: Duration = Duration::from_secs(600); - -/// Hard upper bound on `shutdown_poll_interval`. A poll interval longer -/// than the drain timeout makes the flag effectively unobservable; cap -/// at 5s so Ctrl-C latency stays bounded regardless of config. -pub const SHUTDOWN_POLL_INTERVAL_MAX: Duration = Duration::from_secs(5); - -/// Hard upper bound on `shutdown_join_timeout`. Comfortably above the -/// drain cap so a full drain always fits inside the join budget, while -/// still guaranteeing process exit against a pathological config typo. -pub const SHUTDOWN_JOIN_TIMEOUT_MAX: Duration = Duration::from_secs(900); - -/// Hard upper bound on `reconcile_periodic_interval`. A tick longer -/// than ~30s makes post-failure recovery latency operator-visible; the -/// cap reins in pathological typos without disturbing reasonable -/// production values. -pub const RECONCILE_PERIODIC_INTERVAL_MAX: Duration = Duration::from_secs(30); - -// Every omitted field falls back to the frozen `Default`, so a partial -// `[system.sharding]` table resolves each key independently instead of -// failing on the first missing one (parity with the legacy type). -#[serde_as] -#[derive(Debug, Deserialize, Serialize, ConfigEnv)] -#[serde(default)] -pub struct ShardingConfig { - #[serde(default)] - #[config_env(leaf)] - pub cpu_allocation: CpuAllocation, - /// Whether shard threads are pinned to dedicated CPU cores - /// (`sched_setaffinity`). Pinning maximizes cache locality when this - /// server owns its cores (dedicated host, `numa:` allocations). Set to - /// `false` when the server shares cores with other workloads — e.g. a - /// multi-tenant host slicing CPU via cgroup quotas — where every process - /// pinning to the same low-numbered cores would pile onto one core while - /// the rest sit idle; unpinned shards let the kernel scheduler place - /// threads freely within the allowed set. With a NUMA-aware allocation, - /// `false` drops both the CPU and memory-node bindings (and logs a - /// warning, since NUMA placement without pinning is meaningless). - pub pin_cores: bool, - /// Per-shard inter-shard inbox channel capacity. Bounded by design. - /// Drops on full inbox of consensus frames are recovered by VSR - /// retransmit. Drops of cross-shard client Reply frames are terminal: - /// the client never receives the reply (no in-protocol retransmit). - /// Both frame classes share this one channel, so a consensus burst - /// can starve client-reply forwards: size against the worst-case sum - /// of consensus working set + peak client-reply fan-out per shard - /// occurring together. - /// - // TODO(hubcio): split into two priority lanes - one bounded queue for - // consensus frames (drops recovered by VSR retransmit) and one for - // client `Reply` frames (drops terminal, must be sized for worst-case - // fan-out). Current single-channel design is the minimum-viable - // wiring so `frame_drops_total{variant,reason}` surfaces under load - // and yields real numbers to size the split against. - pub inbox_capacity: usize, - /// Wall-clock budget for a single shard's bus drain on shutdown. - /// Drives `IggyMessageBus::shutdown(..)` from the per-shard watchdog - /// and the parallel-join survivor path. Sized larger than typical - /// TCP RTT times in-flight write-batch so writers receive their full - /// last `write_vectored_all` budget before the connection registry - /// force-tears the bus. Slow-fsync hosts may need to extend this past - /// the default; the cap is `SHUTDOWN_DRAIN_TIMEOUT_MAX` so a config - /// typo cannot wedge process exit. - #[serde_as(as = "DisplayFromStr")] - #[config_env(leaf)] - pub shutdown_drain_timeout: IggyDuration, - /// Poll cadence for the cross-thread shutdown flag and for the - /// `await_metadata_bundle` / `broadcast_metadata_bundle` poll loops. - /// Trades off Ctrl-C latency against idle wakeup cost; the default - /// keeps shutdown observably prompt without measurable scheduler - /// overhead. Capped at `SHUTDOWN_POLL_INTERVAL_MAX` so the flag - /// remains effectively observable regardless of config. - #[serde_as(as = "DisplayFromStr")] - #[config_env(leaf)] - pub shutdown_poll_interval: IggyDuration, - /// Hard wall-clock deadline for joining shard threads at process - /// exit. A shard whose pump or listener wedges past this budget is - /// abandoned with an error log instead of blocking exit forever. - /// Must be at least `shutdown_drain_timeout` (abandoning a shard - /// mid-drain would interrupt its WAL fsync / replica drain) and at - /// most [`SHUTDOWN_JOIN_TIMEOUT_MAX`]. - #[serde_as(as = "DisplayFromStr")] - #[config_env(leaf)] - pub shutdown_join_timeout: IggyDuration, - /// Safety-tick cadence for the partition reconciliation loop; the - /// reconciler also wakes immediately on every - /// `LifecycleFrame::MetadataCommitTick` from shard 0. This periodic - /// fallback covers dropped wake-ups (the wake channel is capacity-1) - /// and the initial post-bootstrap convergence window. Values above - /// [`RECONCILE_PERIODIC_INTERVAL_MAX`] are rejected by the validator. - #[serde_as(as = "DisplayFromStr")] - #[config_env(leaf)] - pub reconcile_periodic_interval: IggyDuration, -} - -impl Default for ShardingConfig { - fn default() -> Self { - Self { - cpu_allocation: CpuAllocation::default(), - pin_cores: SERVER_NG_CONFIG.system.sharding.pin_cores, - inbox_capacity: SERVER_NG_CONFIG.system.sharding.inbox_capacity as usize, - shutdown_drain_timeout: SERVER_NG_CONFIG - .system - .sharding - .shutdown_drain_timeout - .parse() - .unwrap(), - shutdown_poll_interval: SERVER_NG_CONFIG - .system - .sharding - .shutdown_poll_interval - .parse() - .unwrap(), - shutdown_join_timeout: SERVER_NG_CONFIG - .system - .sharding - .shutdown_join_timeout - .parse() - .unwrap(), - reconcile_periodic_interval: SERVER_NG_CONFIG - .system - .sharding - .reconcile_periodic_interval - .parse() - .unwrap(), - } - } -} - -impl Validatable for ShardingConfig { - fn validate(&self) -> Result<(), ConfigurationError> { - if self.inbox_capacity == 0 { - eprintln!( - "Invalid sharding configuration: inbox_capacity must be > 0 (crossfire silently \ - rounds 0 to 1, masking config errors)" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if self.inbox_capacity > INBOX_CAPACITY_MAX { - eprintln!( - "Invalid sharding configuration: inbox_capacity {} exceeds the {} cap (each \ - shard preallocates a channel of this size; oversizing here OOMs the process at \ - boot)", - self.inbox_capacity, INBOX_CAPACITY_MAX - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - let drain = self.shutdown_drain_timeout.get_duration(); - if drain.is_zero() { - eprintln!( - "Invalid sharding configuration: shutdown_drain_timeout must be > 0 (a zero \ - budget force-tears the bus mid-WAL-fsync on every shutdown)" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if drain > SHUTDOWN_DRAIN_TIMEOUT_MAX { - eprintln!( - "Invalid sharding configuration: shutdown_drain_timeout {:?} exceeds the {:?} \ - cap (an unbounded drain wedges process exit on bus stall)", - drain, SHUTDOWN_DRAIN_TIMEOUT_MAX - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - let poll = self.shutdown_poll_interval.get_duration(); - if poll.is_zero() { - eprintln!( - "Invalid sharding configuration: shutdown_poll_interval must be > 0 (a zero \ - cadence busy-loops every shard's watchdog and metadata-handoff poller)" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if poll > SHUTDOWN_POLL_INTERVAL_MAX { - eprintln!( - "Invalid sharding configuration: shutdown_poll_interval {:?} exceeds the {:?} \ - cap (a coarse cadence stalls Ctrl-C handling and metadata handoff abort)", - poll, SHUTDOWN_POLL_INTERVAL_MAX - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if poll > drain { - eprintln!( - "Invalid sharding configuration: shutdown_poll_interval {:?} must be <= \ - shutdown_drain_timeout {:?} (a poll cadence coarser than the drain budget makes \ - the shutdown flag effectively unobservable)", - poll, drain - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - let join = self.shutdown_join_timeout.get_duration(); - if join < drain { - eprintln!( - "Invalid sharding configuration: shutdown_join_timeout {:?} must be >= \ - shutdown_drain_timeout {:?} (a join budget shorter than the drain abandons \ - shards mid-drain, interrupting the WAL fsync / replica drain)", - join, drain - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if join > SHUTDOWN_JOIN_TIMEOUT_MAX { - eprintln!( - "Invalid sharding configuration: shutdown_join_timeout {:?} exceeds the {:?} \ - cap (an unbounded join budget wedges process exit on a stuck shard)", - join, SHUTDOWN_JOIN_TIMEOUT_MAX - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - let reconcile = self.reconcile_periodic_interval.get_duration(); - if reconcile.is_zero() { - eprintln!( - "Invalid sharding configuration: reconcile_periodic_interval resolves to zero. \ - Note that \"0\", \"none\", \"unlimited\", and \"disabled\" all parse to zero. The \ - periodic reconcile tick is a safety net for dropped commit-wakes and cannot be \ - turned off; set a positive duration (default \"1s\", max {RECONCILE_PERIODIC_INTERVAL_MAX:?})." - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if reconcile > RECONCILE_PERIODIC_INTERVAL_MAX { - eprintln!( - "Invalid sharding configuration: reconcile_periodic_interval {:?} exceeds the \ - {:?} cap (a long tick makes post-failure convergence latency operator-visible)", - reconcile, RECONCILE_PERIODIC_INTERVAL_MAX - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - validate_cpu_allocation(&self.cpu_allocation, self.pin_cores) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::server_ng_config::server_ng::ServerNgConfig; - use figment::Figment; - use figment::providers::{Format, Toml}; - - #[test] - fn defaults_validate() { - assert!(ShardingConfig::default().validate().is_ok()); - } - - #[test] - fn zero_drain_is_rejected() { - let cfg = ShardingConfig { - shutdown_drain_timeout: IggyDuration::new(Duration::ZERO), - ..ShardingConfig::default() - }; - assert!(cfg.validate().is_err()); - } - - #[test] - fn over_cap_drain_is_rejected() { - let cfg = ShardingConfig { - shutdown_drain_timeout: IggyDuration::new( - SHUTDOWN_DRAIN_TIMEOUT_MAX + Duration::from_secs(1), - ), - ..ShardingConfig::default() - }; - assert!(cfg.validate().is_err()); - } - - #[test] - fn zero_poll_is_rejected() { - let cfg = ShardingConfig { - shutdown_poll_interval: IggyDuration::new(Duration::ZERO), - ..ShardingConfig::default() - }; - assert!(cfg.validate().is_err()); - } - - #[test] - fn over_cap_poll_is_rejected() { - let cfg = ShardingConfig { - shutdown_poll_interval: IggyDuration::new( - SHUTDOWN_POLL_INTERVAL_MAX + Duration::from_secs(1), - ), - ..ShardingConfig::default() - }; - assert!(cfg.validate().is_err()); - } - - #[test] - fn poll_greater_than_drain_is_rejected() { - let cfg = ShardingConfig { - shutdown_drain_timeout: IggyDuration::new(Duration::from_millis(20)), - shutdown_poll_interval: IggyDuration::new(Duration::from_millis(50)), - ..ShardingConfig::default() - }; - assert!(cfg.validate().is_err()); - } - - #[test] - fn join_shorter_than_drain_is_rejected() { - // A join budget under the drain would abandon shards mid-drain. - let cfg = ShardingConfig { - shutdown_drain_timeout: IggyDuration::new(Duration::from_secs(10)), - shutdown_join_timeout: IggyDuration::new(Duration::from_secs(5)), - ..ShardingConfig::default() - }; - assert!(cfg.validate().is_err()); - } - - #[test] - fn over_cap_join_is_rejected() { - let cfg = ShardingConfig { - shutdown_join_timeout: IggyDuration::new( - SHUTDOWN_JOIN_TIMEOUT_MAX + Duration::from_secs(1), - ), - ..ShardingConfig::default() - }; - assert!(cfg.validate().is_err()); - } - - #[test] - fn join_equal_to_drain_is_accepted() { - let cfg = ShardingConfig { - shutdown_drain_timeout: IggyDuration::new(Duration::from_secs(10)), - shutdown_join_timeout: IggyDuration::new(Duration::from_secs(10)), - ..ShardingConfig::default() - }; - assert!(cfg.validate().is_ok()); - } - - // Guards the single source of truth: the server-ng sharding defaults - // resolve from the embedded server-ng TOML, not hard-coded Rust values. - #[test] - fn ng_embedded_toml_resolves_sharding_defaults() { - let toml_str = include_str!("../../../server-ng/config.toml"); - let config: ServerNgConfig = Figment::new() - .merge(Toml::string(toml_str)) - .extract() - .expect("embedded server-ng TOML deserializes"); - config - .validate() - .expect("embedded server-ng config validates"); - - let sharding = &config.system.sharding; - assert!(sharding.pin_cores); - assert_eq!(sharding.inbox_capacity, 1024); - assert_eq!(sharding.shutdown_drain_timeout, "10 s".parse().unwrap()); - assert_eq!(sharding.shutdown_poll_interval, "50 ms".parse().unwrap()); - assert_eq!(sharding.shutdown_join_timeout, "30 s".parse().unwrap()); - assert_eq!(sharding.reconcile_periodic_interval, "1 s".parse().unwrap()); - } - - // Extract straight from a raw table (no embedded base layer) so the - // struct-level `#[serde(default)]` is what fills the gaps, not the - // provider's embedded-TOML fallback. - #[test] - fn partial_table_fills_missing_fields_with_frozen_defaults() { - let sharding: ShardingConfig = Figment::new() - .merge(Toml::string("pin_cores = false")) - .extract() - .expect("partial sharding table deserializes"); - - assert!(!sharding.pin_cores); - assert_eq!(sharding.inbox_capacity, 1024); - assert_eq!(sharding.shutdown_drain_timeout, "10 s".parse().unwrap()); - assert_eq!(sharding.shutdown_poll_interval, "50 ms".parse().unwrap()); - assert_eq!(sharding.shutdown_join_timeout, "30 s".parse().unwrap()); - assert_eq!(sharding.reconcile_periodic_interval, "1 s".parse().unwrap()); - } - - #[test] - fn empty_table_yields_all_frozen_defaults() { - let sharding: ShardingConfig = Figment::new() - .merge(Toml::string("")) - .extract() - .expect("empty sharding table deserializes"); - - assert!(sharding.pin_cores); - assert_eq!(sharding.inbox_capacity, 1024); - assert_eq!(sharding.shutdown_drain_timeout, "10 s".parse().unwrap()); - assert_eq!(sharding.shutdown_poll_interval, "50 ms".parse().unwrap()); - assert_eq!(sharding.shutdown_join_timeout, "30 s".parse().unwrap()); - assert_eq!(sharding.reconcile_periodic_interval, "1 s".parse().unwrap()); - } -} diff --git a/core/configs/src/server_ng_config/tcp.rs b/core/configs/src/server_ng_config/tcp.rs deleted file mode 100644 index 7fbb34bb06..0000000000 --- a/core/configs/src/server_ng_config/tcp.rs +++ /dev/null @@ -1,62 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Server-ng TCP listener schema. -//! -//! Field shape mirrors the legacy [`crate::tcp::TcpConfig`] verbatim; -//! the type is forked into `server_ng_config` so server-ng can evolve -//! its TCP/TLS surface (additional knobs, removed knobs) independently -//! of the legacy server. No semantic change at fork time. - -use configs::ConfigEnv; -use iggy_common::{IggyByteSize, IggyDuration}; -use serde::{Deserialize, Serialize}; -use serde_with::DisplayFromStr; -use serde_with::serde_as; - -#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] -pub struct TcpConfig { - pub enabled: bool, - pub address: String, - pub ipv6: bool, - pub tls: TcpTlsConfig, - pub socket: TcpSocketConfig, - pub socket_migration: bool, -} - -#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] -pub struct TcpTlsConfig { - pub enabled: bool, - pub self_signed: bool, - pub cert_file: String, - pub key_file: String, -} - -#[serde_as] -#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] -pub struct TcpSocketConfig { - pub override_defaults: bool, - #[config_env(leaf)] - pub recv_buffer_size: IggyByteSize, - #[config_env(leaf)] - pub send_buffer_size: IggyByteSize, - pub keepalive: bool, - pub nodelay: bool, - #[config_env(leaf)] - #[serde_as(as = "DisplayFromStr")] - pub linger: IggyDuration, -} diff --git a/core/configs/src/server_ng_config/validators.rs b/core/configs/src/server_ng_config/validators.rs deleted file mode 100644 index f7408b7d35..0000000000 --- a/core/configs/src/server_ng_config/validators.rs +++ /dev/null @@ -1,861 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! [`Validatable`] for [`ServerNgConfig`]. -//! -//! Mirrors the section-by-section delegation of -//! [`crate::validators`]'s `impl Validatable for ServerConfig`, plus a -//! call into [`super::message_bus::MessageBusConfig::validate`] for the -//! new section. The cross-section invariants (topic vs segment sizing, -//! JWT gating when HTTP is enabled, server-default expiry sanity) are -//! mirrored exactly so server-ng inherits the same boot-time safety -//! net. - -use super::COMPONENT_NG; -use super::cluster::STATE_CHUNK_HEADER_LEN; -use super::server_ng::{ExtraConfig, NamespaceConfig, ServerNgConfig}; -use crate::ConfigurationError; -use err_trail::ErrContext; -use iggy_common::{IggyExpiry, MaxTopicSize, Validatable}; -use server_common::sharding::IggyNamespace; -use tracing::warn; - -/// compio-ws (tungstenite 0.29) `write_buffer_size` default. Used to -/// evaluate the `max_write_buffer_size > write_buffer_size` invariant -/// when the operator leaves `write_buffer_size` unset; keep in sync -/// with the defaults documented in the shipped config.toml. -const WS_DEFAULT_WRITE_BUFFER_SIZE: u64 = 128 * 1024; - -impl Validatable for ServerNgConfig { - fn validate(&self) -> Result<(), ConfigurationError> { - self.system - .memory_pool - .validate() - .error(|e: &ConfigurationError| { - format!("{COMPONENT_NG} (error: {e}) - failed to validate memory pool config") - })?; - self.data_maintenance - .validate() - .error(|e: &ConfigurationError| { - format!("{COMPONENT_NG} (error: {e}) - failed to validate data maintenance config") - })?; - self.personal_access_token - .validate() - .error(|e: &ConfigurationError| { - format!( - "{COMPONENT_NG} (error: {e}) - failed to validate personal access token config" - ) - })?; - self.extra.validate().error(|e: &ConfigurationError| { - format!("{COMPONENT_NG} (error: {e}) - failed to validate extra config") - })?; - self.system - .segment - .validate() - .error(|e: &ConfigurationError| { - format!("{COMPONENT_NG} (error: {e}) - failed to validate segment config") - })?; - self.system - .compression - .validate() - .error(|e: &ConfigurationError| { - format!("{COMPONENT_NG} (error: {e}) - failed to validate compression config") - })?; - self.telemetry.validate().error(|e: &ConfigurationError| { - format!("{COMPONENT_NG} (error: {e}) - failed to validate telemetry config") - })?; - self.system - .sharding - .validate() - .error(|e: &ConfigurationError| { - format!("{COMPONENT_NG} (error: {e}) - failed to validate sharding config") - })?; - self.cluster.validate().error(|e: &ConfigurationError| { - format!("{COMPONENT_NG} (error: {e}) - failed to validate cluster config") - })?; - self.metadata.validate().error(|e: &ConfigurationError| { - format!("{COMPONENT_NG} (error: {e}) - failed to validate metadata config") - })?; - self.partition.validate().error(|e: &ConfigurationError| { - format!("{COMPONENT_NG} (error: {e}) - failed to validate partition config") - })?; - self.system - .logging - .validate() - .error(|e: &ConfigurationError| { - format!("{COMPONENT_NG} (error: {e}) - failed to validate logging config") - })?; - self.message_saver - .validate() - .error(|e: &ConfigurationError| { - format!("{COMPONENT_NG} (error: {e}) - failed to validate message saver config") - })?; - - let topic_size = match self.system.topic.max_size { - MaxTopicSize::Custom(size) => Ok(size.as_bytes_u64()), - MaxTopicSize::Unlimited => Ok(u64::MAX), - MaxTopicSize::ServerDefault => { - eprintln!("system.topic.max_size cannot be ServerDefault in server-ng config"); - Err(ConfigurationError::InvalidConfigurationValue) - } - }?; - - if let IggyExpiry::ServerDefault = self.system.topic.message_expiry { - eprintln!("system.topic.message_expiry cannot be ServerDefault in server-ng config"); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - // A zero duration encodes to wire value 0, the same value the wire uses - // for ServerDefault, so it would silently collide with that sentinel. - if let IggyExpiry::ExpireDuration(duration) = self.system.topic.message_expiry - && duration.as_micros() == 0 - { - eprintln!( - "system.topic.message_expiry is a zero duration, which collides with the server-default sentinel on the wire; use \"none\" to never expire or a positive duration" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - if self.http.enabled - && let IggyExpiry::ServerDefault = self.http.jwt.access_token_expiry - { - eprintln!("http.jwt.access_token_expiry cannot be ServerDefault when HTTP is enabled"); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - if self.http.enabled - && self.http.tls.enabled - && (self.http.tls.cert_file.is_empty() || self.http.tls.key_file.is_empty()) - { - eprintln!( - "http.tls.enabled=true requires non-empty http.tls.cert_file and http.tls.key_file" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - // Cluster mode has no port fallbacks: the roster is the single source - // of listener ports, so every enabled transport needs an explicit - // per-node port. Falling back to the port of a transport's top-level - // `address` would hand two same-host nodes the same socket and fail - // only at bind time, and a portless node would silently degrade every - // follower-to-primary HTTP forward through it to a fail-closed 503. - if self.cluster.enabled { - for node in &self.cluster.nodes { - let required_ports = [ - ("tcp", true, node.ports.tcp), - ("quic", self.quic.enabled, node.ports.quic), - ("http", self.http.enabled, node.ports.http), - ("websocket", self.websocket.enabled, node.ports.websocket), - ("tcp_replica", true, node.ports.tcp_replica), - ]; - for (transport, enabled, port) in required_ports { - if enabled && port.is_none() { - eprintln!( - "cluster node '{}' has no ports.{transport}; cluster mode requires an explicit roster port for every enabled transport", - node.name - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - } - } - } - - if topic_size < self.system.segment.size.as_bytes_u64() { - eprintln!( - "system.topic.max_size ({} B) must be >= system.segment.size ({} B)", - topic_size, - self.system.segment.size.as_bytes_u64() - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - // A received segment artifact can be one whole batch larger than the - // segment cap (rotation checks the cap AFTER appending), and the real - // batch bound is the BUS frame cap -- server-ng never enforces - // `MAX_PAYLOAD_SIZE`. An artifact ceiling under that floor refuses a - // legal segment, and the manifest check is all-or-nothing, so the - // partition livelocks re-requesting the same segment from every peer at - // the backoff ceiling. Caught here so it is a boot error rather than one - // partition that silently never rejoins. - let artifact_floor = self - .system - .segment - .size - .as_bytes_u64() - .saturating_add(self.message_bus.max_message_size.as_bytes_u64()); - if self.partition.transfer_artifact_bytes_max.as_bytes_u64() < artifact_floor { - eprintln!( - "{COMPONENT_NG} partition.transfer_artifact_bytes_max ({} B) must be at least \ - system.segment.size ({} B) + message_bus.max_message_size ({} B) = \ - {artifact_floor} B: a segment may close one whole batch past its cap, and an \ - artifact ceiling below that refuses a legal segment and livelocks the \ - partition's rejoin", - self.partition.transfer_artifact_bytes_max.as_bytes_u64(), - self.system.segment.size.as_bytes_u64(), - self.message_bus.max_message_size.as_bytes_u64(), - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - self.message_bus - .validate() - .error(|e: &ConfigurationError| { - format!("{COMPONENT_NG} (error: {e}) - failed to validate message_bus config") - })?; - - // Repair frames ride the bounded per-peer message-bus queue. A repair - // round of cluster.repair_chunk_max frames that meets or overruns - // message_bus.peer_queue_capacity drops its own tail silently, wedging - // the repair loop into slow retries. Keep the chunk strictly below the - // queue; this also floors peer_queue_capacity, which is otherwise only - // checked for > 0. - if self.cluster.repair_chunk_max >= self.message_bus.peer_queue_capacity { - eprintln!( - "{COMPONENT_NG} cluster.repair_chunk_max ({}) must be < message_bus.peer_queue_capacity ({}): repair frames ride the per-peer bus queue, so a chunk that fills or overruns it drops frames and wedges repair", - self.cluster.repair_chunk_max, self.message_bus.peer_queue_capacity - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - // State-transfer chunks ride the same bus. A cap that cannot carry one - // header plus a byte of payload makes every rejoin that needs a - // transfer impossible, and the failure surfaces only as a replica - // connection tearing down when the frame is rejected on the read side. - let bus_cap = self.message_bus.max_message_size.as_bytes_u64(); - if bus_cap <= STATE_CHUNK_HEADER_LEN { - eprintln!( - "{COMPONENT_NG} message_bus.max_message_size ({bus_cap}) must exceed the {STATE_CHUNK_HEADER_LEN}-byte state-chunk header: state transfer serves artifact chunks over this bus, and a frame above the cap is rejected by the receiving transport, which tears down the whole replica connection" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - // WS frame chain: websocket.max_frame_size <= websocket.max_message_size - // <= message_bus.max_message_size. The bus's WS / WSS install path takes - // its frame tuning from [websocket], so a WS ceiling above the bus's own - // frame cap would admit messages the bus read-side validator then tears - // the connection down over. An absent knob defers to the compio-ws - // default (16 MiB frame / 64 MiB message), which satisfies the chain - // against the shipped bus cap in practice. - let bus_max_message_size = self.message_bus.max_message_size.as_bytes_u64(); - if let (Some(frame), Some(message)) = ( - self.websocket.max_frame_size, - self.websocket.max_message_size, - ) && frame.as_bytes_u64() > message.as_bytes_u64() - { - eprintln!( - "{COMPONENT_NG} websocket.max_frame_size ({}) exceeds websocket.max_message_size ({})", - frame.as_bytes_u64(), - message.as_bytes_u64() - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if let Some(message) = self.websocket.max_message_size - && message.as_bytes_u64() > bus_max_message_size - { - eprintln!( - "{COMPONENT_NG} websocket.max_message_size ({}) exceeds message_bus.max_message_size ({})", - message.as_bytes_u64(), - bus_max_message_size - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if let Some(frame) = self.websocket.max_frame_size - && frame.as_bytes_u64() > bus_max_message_size - { - eprintln!( - "{COMPONENT_NG} websocket.max_frame_size ({}) exceeds message_bus.max_message_size ({})", - frame.as_bytes_u64(), - bus_max_message_size - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - // "0", "unlimited" and "none" all parse to a zero IggyByteSize. A - // zero WS tunable is never usable: zero message or frame ceilings - // reject every inbound frame, and zero buffers starve the - // compio-ws pipeline. Reject at boot instead of shipping a - // listener that cannot serve a single message. - for (key, size) in [ - ("read_buffer_size", self.websocket.read_buffer_size), - ("write_buffer_size", self.websocket.write_buffer_size), - ( - "max_write_buffer_size", - self.websocket.max_write_buffer_size, - ), - ("max_message_size", self.websocket.max_message_size), - ("max_frame_size", self.websocket.max_frame_size), - ] { - if let Some(size) = size - && size.as_bytes_u64() == 0 - { - eprintln!( - "{COMPONENT_NG} websocket.{key} must be non-zero (\"0\", \"unlimited\" and \"none\" all parse to zero)" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - } - - // tungstenite asserts `max_write_buffer_size > write_buffer_size` - // during connection setup, so a violating pair panics on every - // accepted socket. Enforce the invariant at boot; an unset - // write_buffer_size runs at the compio-ws default. - if let Some(max_write) = self.websocket.max_write_buffer_size { - let write_buffer_size = self - .websocket - .write_buffer_size - .map_or(WS_DEFAULT_WRITE_BUFFER_SIZE, |size| size.as_bytes_u64()); - if max_write.as_bytes_u64() <= write_buffer_size { - eprintln!( - "{COMPONENT_NG} websocket.max_write_buffer_size ({}) must exceed websocket.write_buffer_size ({write_buffer_size})", - max_write.as_bytes_u64() - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - } - - self.quic.validate().error(|e: &ConfigurationError| { - format!("{COMPONENT_NG} (error: {e}) - failed to validate quic config") - })?; - - // Both knobs below are live in server-ng but sit on structs the legacy - // server shares, so the rejects live here instead of in a `Validatable` - // impl that would tighten legacy boots too. `0` / `disabled` / - // `unlimited` all parse to the same zero duration. - if self - .consumer_group - .rebalancing_timeout - .get_duration() - .is_zero() - { - eprintln!( - "{COMPONENT_NG} consumer_group.rebalancing_timeout must be nonzero: it is the deadline after which a pending revocation completes without the source client committing what it was served, so zero force-transfers every partition on the next reconciler tick and reopens the duplicate-delivery window" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if self.heartbeat.enabled && self.heartbeat.interval.get_duration().is_zero() { - eprintln!( - "{COMPONENT_NG} heartbeat.interval must be nonzero when heartbeat.enabled: it sizes both the verifier's sleep and the staleness window, so zero spins the verifier and reaps every live session on its first pass" - ); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - reject_unsupported_and_warn_inert(self)?; - - Ok(()) - } -} - -/// server-ng parses the whole legacy config surface but does not yet honor -/// every knob. Make the still-inert ones loud at boot: reject the unsupported -/// features (all off by default, so only a deliberate opt-in trips this) and -/// warn once for tuning knobs server-ng silently ignores. Warnings fire only -/// when a knob deviates from its [`ServerNgConfig::default`] baseline, so a -/// pristine config.toml boots without noise. Baseline caveat: only the tcp/quic -/// fork sections take that default from the ng config.toml. The reused legacy -/// sections (`system.*`, `consumer_group.*`, `message_saver.*`) take theirs from -/// the legacy server config.toml; those match ng's shipped values today but are -/// not schema-locked, so editing such a knob in the ng config.toml could surface -/// a spurious warn. The guard test below pins the compared knobs against drift. -fn reject_unsupported_and_warn_inert(config: &ServerNgConfig) -> Result<(), ConfigurationError> { - if config.system.message_deduplication.enabled { - eprintln!("system.message_deduplication.enabled is not supported in server-ng"); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if config.system.segment.archive_expired { - eprintln!("system.segment.archive_expired is not supported in server-ng"); - return Err(ConfigurationError::InvalidConfigurationValue); - } - if config.system.recovery.recreate_missing_state { - eprintln!("system.recovery.recreate_missing_state is not supported in server-ng"); - return Err(ConfigurationError::InvalidConfigurationValue); - } - - let defaults = ServerNgConfig::default(); - - if config.tcp.socket.override_defaults { - warn!("tcp.socket tuning is set but not applied in server-ng"); - } - if config.quic.socket.override_defaults { - warn!("quic.socket tuning is set but not applied in server-ng"); - } - if config.tcp.ipv6 { - warn!( - "tcp.ipv6 is ignored in server-ng; IPv4 vs IPv6 is decided by the tcp.address string" - ); - } - if config.tcp.socket_migration != defaults.tcp.socket_migration { - warn!("tcp.socket_migration is not implemented in server-ng"); - } - if config.system.partition.validate_checksum != defaults.system.partition.validate_checksum { - warn!( - "system.partition.validate_checksum is not applied in server-ng; nothing verifies checksums on load" - ); - } - if config.system.segment.cache_indexes != defaults.system.segment.cache_indexes { - warn!("system.segment.cache_indexes is not applied in server-ng"); - } - if config.system.logging.sysinfo_print_interval - != defaults.system.logging.sysinfo_print_interval - { - warn!("system.logging.sysinfo_print_interval is not applied in server-ng"); - } - if config.system.backup.path != defaults.system.backup.path - || config.system.backup.compatibility.path != defaults.system.backup.compatibility.path - { - warn!("backup is not supported in server-ng"); - } - // default_algorithm deviation is already warned by the delegated legacy - // CompressionConfig::validate; only allow_override needs a signal here. - if config.system.compression.allow_override != defaults.system.compression.allow_override { - warn!( - "system.compression.allow_override is inert in server-ng; live compression is per-topic from the request" - ); - } - if config.system.state.enforce_fsync != defaults.system.state.enforce_fsync - || config.system.state.max_file_operation_retries - != defaults.system.state.max_file_operation_retries - || config.system.state.retry_delay != defaults.system.state.retry_delay - { - warn!( - "system.state tuning (enforce_fsync, max_file_operation_retries, retry_delay) is not applied in server-ng" - ); - } - if config.consumer_group.rebalancing_check_interval - != defaults.consumer_group.rebalancing_check_interval - { - warn!( - "consumer_group.rebalancing_check_interval is not applied in server-ng; rebalancing cadence uses system.sharding.reconcile_periodic_interval" - ); - } - if config.message_saver.interval != defaults.message_saver.interval - || config.message_saver.enforce_fsync != defaults.message_saver.enforce_fsync - { - warn!( - "periodic message_saver is not implemented in server-ng; only shutdown-flush is active" - ); - } - - Ok(()) -} - -impl Validatable for ExtraConfig { - fn validate(&self) -> Result<(), ConfigurationError> { - self.namespace.validate().error(|e: &ConfigurationError| { - format!("{COMPONENT_NG} (error: {e}) - failed to validate namespace config") - })?; - Ok(()) - } -} - -impl Validatable for NamespaceConfig { - fn validate(&self) -> Result<(), ConfigurationError> { - IggyNamespace::validate_capacity(self.max_streams, self.max_topics, self.max_partitions) - .map_err(|error| { - eprintln!("extra.namespace is invalid: {error}"); - ConfigurationError::InvalidConfigurationValue - })?; - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::super::cluster::{ClusterNodeConfig, TransportPorts}; - use super::*; - use figment::Figment; - use figment::providers::{Format, Toml}; - - const DEFAULT_CONFIG: &str = include_str!("../../../server-ng/config.toml"); - - /// Deep-merge a partial override over the shipped default, mirroring the - /// file-over-embedded layering the runtime loader performs. - fn config_with_override(override_toml: &str) -> ServerNgConfig { - Figment::new() - .merge(Toml::string(DEFAULT_CONFIG)) - .merge(Toml::string(override_toml)) - .extract() - .expect("config deserializes") - } - - #[test] - fn given_shipped_default_config_when_validating_should_pass() { - let config: ServerNgConfig = Figment::new() - .merge(Toml::string(DEFAULT_CONFIG)) - .extract() - .expect("default config deserializes"); - config - .validate() - .expect("pristine server-ng config must validate"); - } - - #[test] - fn given_message_deduplication_enabled_when_validating_should_reject() { - let config = config_with_override("[system.message_deduplication]\nenabled = true\n"); - assert!(config.validate().is_err()); - } - - #[test] - fn given_web_ui_enabled_when_validating_should_pass() { - let config = config_with_override("[http]\nweb_ui = true\n"); - config - .validate() - .expect("web_ui is now served by server-ng and must validate"); - } - - #[test] - fn given_archive_expired_enabled_when_validating_should_reject() { - let config = config_with_override("[system.segment]\narchive_expired = true\n"); - assert!(config.validate().is_err()); - } - - #[test] - fn given_recreate_missing_state_enabled_when_validating_should_reject() { - let config = config_with_override("[system.recovery]\nrecreate_missing_state = true\n"); - assert!(config.validate().is_err()); - } - - #[test] - fn given_zero_message_expiry_when_validating_should_reject() { - let config = config_with_override("[system.topic]\nmessage_expiry = \"0s\"\n"); - assert!(config.validate().is_err()); - } - - #[test] - fn given_peer_queue_capacity_not_above_repair_chunk_max_when_validating_should_reject() { - // The default repair_chunk_max (128) must stay strictly below - // peer_queue_capacity; shrinking the queue to the chunk size is the - // silent wedged-repair footgun this cross-section guard closes. - let config = config_with_override("[message_bus]\npeer_queue_capacity = 128\n"); - assert!(config.validate().is_err()); - } - - #[test] - fn given_repair_chunk_max_at_peer_queue_capacity_when_validating_should_reject() { - let config = config_with_override("[cluster]\nrepair_chunk_max = 256\n"); - assert!(config.validate().is_err()); - } - - #[test] - fn given_repair_chunk_max_below_peer_queue_capacity_when_validating_should_pass() { - let config = config_with_override("[cluster]\nrepair_chunk_max = 255\n"); - config - .validate() - .expect("a chunk below the peer queue capacity must validate"); - } - - #[test] - fn given_ws_frame_size_above_ws_message_size_when_validating_should_reject() { - let config = config_with_override( - "[websocket]\nmax_message_size = \"1 MiB\"\nmax_frame_size = \"2 MiB\"\n", - ); - assert!(config.validate().is_err()); - } - - // The shipped bus cap is 64 MiB, so a 128 MiB WS ceiling breaks the chain. - #[test] - fn given_ws_message_size_above_bus_max_message_size_when_validating_should_reject() { - let config = config_with_override("[websocket]\nmax_message_size = \"128 MiB\"\n"); - assert!(config.validate().is_err()); - } - - #[test] - fn given_ws_frame_size_above_bus_max_message_size_when_validating_should_reject() { - let config = config_with_override("[websocket]\nmax_frame_size = \"128 MiB\"\n"); - assert!(config.validate().is_err()); - } - - #[test] - fn given_ws_frame_chain_in_ascending_order_when_validating_should_pass() { - let config = config_with_override( - "[websocket]\nmax_message_size = \"32 MiB\"\nmax_frame_size = \"16 MiB\"\n", - ); - config - .validate() - .expect("frame <= message <= bus cap must validate"); - } - - // "unlimited" is not a supported sentinel for the WS size knobs: it - // parses to zero, which as a cap would reject every message. - #[test] - fn given_zero_ws_size_when_validating_should_reject() { - let config = config_with_override("[websocket]\nmax_message_size = \"unlimited\"\n"); - assert!(config.validate().is_err()); - } - - // tungstenite panics on this pair at connection setup; boot must - // reject it first. - #[test] - fn given_max_write_buffer_at_write_buffer_when_validating_should_reject() { - let config = config_with_override( - "[websocket]\nwrite_buffer_size = \"256 KiB\"\nmax_write_buffer_size = \"256 KiB\"\n", - ); - assert!(config.validate().is_err()); - } - - #[test] - fn given_max_write_buffer_below_default_write_buffer_when_validating_should_reject() { - let config = config_with_override("[websocket]\nmax_write_buffer_size = \"64 KiB\"\n"); - assert!(config.validate().is_err()); - } - - #[test] - fn given_max_write_buffer_above_write_buffer_when_validating_should_pass() { - let config = config_with_override( - "[websocket]\nwrite_buffer_size = \"128 KiB\"\nmax_write_buffer_size = \"1 MiB\"\n", - ); - config - .validate() - .expect("max write buffer above write buffer must validate"); - } - - // The size knobs are strictly typed; a malformed string must fail - // deserialization at load rather than degrade to the compio-ws default. - #[test] - fn given_malformed_ws_size_string_when_deserializing_should_reject() { - let result: Result = Figment::new() - .merge(Toml::string(DEFAULT_CONFIG)) - .merge(Toml::string( - "[websocket]\nmax_message_size = \"not-a-size\"\n", - )) - .extract(); - assert!( - result.is_err(), - "malformed websocket.max_message_size must fail config load" - ); - } - - // The shipped config is single-node (cluster.enabled = false), where the - // cross-section rule above is the only repair_chunk_max check that used to - // run; its structural bounds have to hold there too. - #[test] - fn given_single_node_zero_repair_chunk_max_when_validating_should_reject() { - let config = config_with_override("[cluster]\nrepair_chunk_max = 0\n"); - assert!(config.validate().is_err()); - } - - #[test] - fn given_single_node_repair_chunk_max_above_ceiling_when_validating_should_reject() { - // Queue widened past the chunk so the cross-section rule passes and - // only the structural ceiling can reject. - let config = config_with_override( - "[cluster]\nrepair_chunk_max = 2000\n\n[message_bus]\npeer_queue_capacity = 4096\n", - ); - assert!(config.validate().is_err()); - } - - #[test] - fn given_zero_rebalancing_timeout_when_validating_should_reject() { - let config = config_with_override("[consumer_group]\nrebalancing_timeout = \"0\"\n"); - assert!(config.validate().is_err()); - } - - #[test] - fn given_disabled_rebalancing_timeout_when_validating_should_reject() { - // "disabled" reads like an opt-out but parses to the same zero - // duration, which force-transfers every revocation instead. - let config = config_with_override("[consumer_group]\nrebalancing_timeout = \"disabled\"\n"); - assert!(config.validate().is_err()); - } - - #[test] - fn given_zero_heartbeat_interval_when_heartbeat_enabled_should_reject() { - let config = config_with_override("[heartbeat]\nenabled = true\ninterval = \"0\"\n"); - assert!(config.validate().is_err()); - } - - #[test] - fn given_zero_heartbeat_interval_when_heartbeat_disabled_should_pass() { - let config = config_with_override("[heartbeat]\nenabled = false\ninterval = \"0\"\n"); - config - .validate() - .expect("a disabled heartbeat never reads its interval"); - } - - /// The warn-helper baseline is [`ServerNgConfig::default`], but the reused - /// legacy sections source that default from the legacy server config.toml, - /// not this NG file. Pin the knobs the helper compares so any drift between - /// the two config.toml files fails here instead of as a spurious boot warn. - #[test] - fn given_shipped_ng_config_when_compared_to_default_should_match_warned_knobs() { - let shipped: ServerNgConfig = Figment::new() - .merge(Toml::string(DEFAULT_CONFIG)) - .extract() - .expect("default config deserializes"); - let defaults = ServerNgConfig::default(); - - assert_eq!(shipped.tcp.socket_migration, defaults.tcp.socket_migration); - assert_eq!( - shipped.system.partition.validate_checksum, - defaults.system.partition.validate_checksum - ); - assert_eq!( - shipped.system.segment.cache_indexes, - defaults.system.segment.cache_indexes - ); - assert_eq!( - shipped.system.logging.sysinfo_print_interval, - defaults.system.logging.sysinfo_print_interval - ); - assert_eq!(shipped.system.backup.path, defaults.system.backup.path); - assert_eq!( - shipped.system.backup.compatibility.path, - defaults.system.backup.compatibility.path - ); - assert_eq!( - shipped.system.compression.allow_override, - defaults.system.compression.allow_override - ); - assert_eq!( - shipped.system.state.enforce_fsync, - defaults.system.state.enforce_fsync - ); - assert_eq!( - shipped.system.state.max_file_operation_retries, - defaults.system.state.max_file_operation_retries - ); - assert_eq!( - shipped.system.state.retry_delay, - defaults.system.state.retry_delay - ); - assert_eq!( - shipped.consumer_group.rebalancing_check_interval, - defaults.consumer_group.rebalancing_check_interval - ); - assert_eq!( - shipped.message_saver.interval, - defaults.message_saver.interval - ); - assert_eq!( - shipped.message_saver.enforce_fsync, - defaults.message_saver.enforce_fsync - ); - } - - // http.enabled needs a non-ServerDefault JWT expiry to clear the sibling - // check above; ServerNgConfig::default() already satisfies that. - fn https_config(cert_file: &str, key_file: &str) -> ServerNgConfig { - let mut cfg = ServerNgConfig::default(); - cfg.http.enabled = true; - cfg.http.tls.enabled = true; - cfg.http.tls.cert_file = cert_file.to_string(); - cfg.http.tls.key_file = key_file.to_string(); - cfg - } - - #[test] - fn validate_rejects_tls_enabled_with_empty_cert_file() { - let cfg = https_config("", "key.pem"); - assert!(cfg.validate().is_err()); - } - - #[test] - fn validate_accepts_tls_enabled_with_both_files_set() { - let cfg = https_config("cert.pem", "key.pem"); - assert!(cfg.validate().is_ok()); - } - - fn cluster_node(replica_id: u8, http: Option) -> ClusterNodeConfig { - ClusterNodeConfig { - name: format!("node-{replica_id}"), - ip: "127.0.0.1".to_string(), - advertised_address: None, - advertised_addresses: Vec::new(), - replica_id, - ports: TransportPorts { - tcp: Some(8090 + u16::from(replica_id)), - quic: Some(8080 + u16::from(replica_id)), - http, - websocket: Some(8070 + u16::from(replica_id)), - tcp_replica: Some(9090 + u16::from(replica_id)), - }, - } - } - - fn clustered_http_config(nodes: Vec) -> ServerNgConfig { - let mut cfg = ServerNgConfig::default(); - cfg.http.enabled = true; - cfg.cluster.enabled = true; - cfg.cluster.name = "test-cluster".to_string(); - cfg.cluster.nodes = nodes; - cfg - } - - // Keyless cluster+http boots: forwarding degrades to off instead of - // failing the whole server. - #[test] - fn validate_accepts_cluster_http_without_jwt_secret_or_cluster_auth() { - let cfg = clustered_http_config(vec![ - cluster_node(0, Some(3000)), - cluster_node(1, Some(3001)), - ]); - assert!(cfg.validate().is_ok()); - } - - // Cluster mode has no port fallbacks, so a portless roster node is - // invalid even when forwarding is off (keyless). - #[test] - fn validate_rejects_keyless_cluster_http_with_portless_roster_node() { - let cfg = clustered_http_config(vec![cluster_node(0, Some(3000)), cluster_node(1, None)]); - assert!(cfg.validate().is_err()); - } - - // The explicit-port rule covers every enabled transport, not just http. - #[test] - fn validate_rejects_cluster_node_without_port_for_enabled_quic() { - let mut cfg = clustered_http_config(vec![ - cluster_node(0, Some(3000)), - cluster_node(1, Some(3001)), - ]); - cfg.quic.enabled = true; - cfg.cluster.nodes[1].ports.quic = None; - assert!(cfg.validate().is_err()); - } - - // A disabled transport never binds, so its roster port may stay unset. - #[test] - fn validate_accepts_cluster_node_without_port_for_disabled_quic() { - let mut cfg = clustered_http_config(vec![ - cluster_node(0, Some(3000)), - cluster_node(1, Some(3001)), - ]); - cfg.quic.enabled = false; - cfg.cluster.nodes[1].ports.quic = None; - assert!(cfg.validate().is_ok()); - } - - #[test] - fn validate_accepts_cluster_http_with_configured_jwt_secrets() { - let mut cfg = clustered_http_config(vec![ - cluster_node(0, Some(3000)), - cluster_node(1, Some(3001)), - ]); - cfg.http.jwt.encoding_secret = "0123456789abcdef0123456789abcdef".to_string(); - cfg.http.jwt.decoding_secret = "0123456789abcdef0123456789abcdef".to_string(); - assert!(cfg.validate().is_ok()); - } - - #[test] - fn validate_accepts_cluster_http_with_cluster_auth_as_jwt_key_source() { - let mut cfg = clustered_http_config(vec![ - cluster_node(0, Some(3000)), - cluster_node(1, Some(3001)), - ]); - cfg.cluster.auth.enabled = true; - cfg.cluster.auth.shared_secret = "0123456789abcdef0123456789abcdef".to_string(); - assert!(cfg.validate().is_ok()); - } -} diff --git a/core/configs/src/server_ng_config/websocket.rs b/core/configs/src/server_ng_config/websocket.rs deleted file mode 100644 index 9bfed9b9ee..0000000000 --- a/core/configs/src/server_ng_config/websocket.rs +++ /dev/null @@ -1,116 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Server-ng WebSocket listener schema. -//! -//! Unlike the legacy [`crate::websocket::WebSocketConfig`] it was -//! forked from, this section is the live frame-tuning source for -//! server-ng's WS / WSS plane: the message bus folds the -//! `Option` knobs below into a compio-ws -//! `WebSocketConfig` once at bus construction. The sizes are strictly -//! typed, so a malformed size string fails config load instead of -//! being silently ignored at conversion time. The conversion itself -//! lives in `core/message_bus` because the standalone `tungstenite` -//! dependency and the compio-ws re-export are different major versions -//! with incompatible config types. - -use configs::ConfigEnv; -use iggy_common::IggyByteSize; -use serde::{Deserialize, Serialize}; -use serde_with::{DisplayFromStr, serde_as}; -use std::fmt::{Display, Formatter}; - -#[serde_as] -#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] -pub struct WebSocketConfig { - pub enabled: bool, - pub address: String, - - /// Target minimum size of the frame read buffer. `None` keeps the - /// compio-ws default (currently 128 KiB). - #[config_env(leaf)] - #[serde(default)] - #[serde_as(as = "Option")] - pub read_buffer_size: Option, - - /// Target buffer size for batched writes before compio-ws flushes. - /// `None` keeps the compio-ws default (currently 128 KiB). - #[config_env(leaf)] - #[serde(default)] - #[serde_as(as = "Option")] - pub write_buffer_size: Option, - - /// Hard ceiling on the write buffer; writes past it error instead - /// of buffering. Must exceed [`Self::write_buffer_size`] by at - /// least one frame. `None` keeps the compio-ws default - /// (unlimited). - #[config_env(leaf)] - #[serde(default)] - #[serde_as(as = "Option")] - pub max_write_buffer_size: Option, - - /// Hard upper bound on a single inbound WebSocket message - /// (post-fragment-reassembly). `None` keeps the compio-ws default - /// (currently 64 MiB). - #[config_env(leaf)] - #[serde(default)] - #[serde_as(as = "Option")] - pub max_message_size: Option, - - /// Hard upper bound on a single inbound WebSocket frame - /// (pre-fragment-reassembly). `None` keeps the compio-ws default - /// (currently 16 MiB). - #[config_env(leaf)] - #[serde(default)] - #[serde_as(as = "Option")] - pub max_frame_size: Option, - - /// Whether to accept unmasked frames from clients in violation of - /// RFC 6455 client-to-server framing rules. Strict (`false`) by - /// default; enable only for non-browser test clients that emit - /// unmasked frames. - #[serde(default)] - pub accept_unmasked_frames: bool, - - #[serde(default)] - pub tls: WebSocketTlsConfig, -} - -#[derive(Debug, Deserialize, Serialize, Clone, ConfigEnv)] -pub struct WebSocketTlsConfig { - pub enabled: bool, - pub self_signed: bool, - pub cert_file: String, - pub key_file: String, -} - -impl Display for WebSocketConfig { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{{ enabled: {}, address: {}, read_buffer_size: {:?}, write_buffer_size: {:?}, max_write_buffer_size: {:?}, max_message_size: {:?}, max_frame_size: {:?}, accept_unmasked_frames: {} }}", - self.enabled, - self.address, - self.read_buffer_size, - self.write_buffer_size, - self.max_write_buffer_size, - self.max_message_size, - self.max_frame_size, - self.accept_unmasked_frames - ) - } -} diff --git a/core/connectors/runtime/Cargo.toml b/core/connectors/runtime/Cargo.toml index 9533516678..c9b7f5a5eb 100644 --- a/core/connectors/runtime/Cargo.toml +++ b/core/connectors/runtime/Cargo.toml @@ -29,9 +29,6 @@ repository = "https://github.com/apache/iggy" readme = "README.md" publish = false -[features] -vsr = ["iggy/vsr"] - [dependencies] async-trait = { workspace = true } axum = { workspace = true } diff --git a/core/consensus/Cargo.toml b/core/consensus/Cargo.toml index 1ba5610c24..342ce604d8 100644 --- a/core/consensus/Cargo.toml +++ b/core/consensus/Cargo.toml @@ -45,6 +45,7 @@ tracing = { workspace = true } twox-hash = { workspace = true } [dev-dependencies] +aligned-vec = { workspace = true } futures = { workspace = true } [lints.clippy] diff --git a/core/consensus/src/client_table.rs b/core/consensus/src/client_table.rs index 4413b8643c..665053f990 100644 --- a/core/consensus/src/client_table.rs +++ b/core/consensus/src/client_table.rs @@ -437,7 +437,7 @@ impl ClientTable { /// Resize the table to `max_clients` slots. Boot-only: reallocating a /// populated table would silently drop live sessions, so this must run - /// before any client registers (server-ng bootstrap applies the configured + /// before any client registers (the server bootstrap applies the configured /// `[metadata] clients_table_max` here). /// /// # Panics @@ -1019,7 +1019,12 @@ impl From for ClientTableWireError { } /// Format tag for [`ClientTable::encode`]; bump on layout change. -pub const CLIENT_TABLE_MAGIC: [u8; 4] = *b"ICT1"; +/// +/// That includes any `ReplyHeader` layout move -- cached replies are embedded +/// as raw wire bytes, so an artifact written under an older header layout must +/// be refused, not silently misread. `ICT2`: `status` sits at offset 216 (the +/// pre-`ICT2` layout carried a `namespace` word before it). +pub const CLIENT_TABLE_MAGIC: [u8; 4] = *b"ICT2"; /// Per-entry fixed fields in the wire encoding: `client(u128) epoch(u64) /// user_id(u32) watermark(u64) watermark_checksum(u128) ring_len(u8)`. diff --git a/core/consensus/src/dvc_merge.rs b/core/consensus/src/dvc_merge.rs new file mode 100644 index 0000000000..19bda15e3c --- /dev/null +++ b/core/consensus/src/dvc_merge.rs @@ -0,0 +1,981 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Merging a `DoViewChange` quorum into the new view's log: for every op that +//! might be uncommitted, does the new view keep it or discard it? +//! +//! Keeping an op that was never committed costs a wasted slot. Discarding one +//! that WAS committed loses acknowledged client data, so the only proof accepted +//! for discarding is a nack quorum: enough replicas stating they never prepared +//! it that a replication quorum provably never formed. Absent that the op is +//! kept, and if no replica offers its body the view does not start. Stalling is +//! visible and recoverable; losing the op is neither. + +#[cfg(test)] +use crate::view_change_quorum::DvcSuffix; +use crate::view_change_quorum::{DvcQuorumArray, StoredDvc, dvc_count, dvc_iter}; +use iggy_binary_protocol::{CHECKSUM_UNSEALED, PrepareHeader}; + +/// Sizes the merge needs from the replica. +#[derive(Debug, Clone, Copy)] +pub struct MergeQuorums { + /// `DoViewChange` messages needed before a view may start. + pub view_change: usize, + /// Nacks needed to prove an op uncommitted, so it may be discarded. + pub nack_prepare: usize, + /// Cluster size, which bounds how many more DVCs could still arrive. + pub replica_count: usize, + /// Cluster-wide pipeline ceiling: an op further than this below a sender's head + /// cannot still be uncommitted, since no node could have kept it in flight. + /// + /// NOT this node's configured depth. The bound applies to a *peer's* head op, + /// and a local depth larger than that peer's manufactures a commit the peer + /// never made. Every node's depth is pinned below `DVC_HEADERS_MAX`, so the + /// ceiling holds for all of them. + pub prepare_queue_ceiling: u64, +} + +/// What the collected DVCs say about starting the view. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum MergeOutcome { + /// Fewer than `view_change` DVCs so far. + AwaitingQuorum, + /// Quorum is in, but some op is neither provably dead nor recoverable and an + /// unreported replica could still settle it. Wait for them. + AwaitingRepair { + /// The op that cannot yet be decided. + undecided_op: u64, + }, + /// Every replica reported and an op is still neither provably dead nor + /// recoverable. No further message changes that: data loss already happened, + /// and truncating here would turn it from detected into silent. + Deadlocked { + /// The op that cannot be decided. + undecided_op: u64, + }, + /// The view can start. + Ready(MergedLog), +} + +/// The log the new primary adopts. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MergedLog { + /// New head op. Below the highest op any canonical sender reported when a + /// nack quorum proved the ops above it dead. + pub op_head: u64, + /// Highest op the quorum proves committed. The merge never discards at or + /// below this. + pub commit_max: u64, + /// Canonical headers for `commit_max..=op_head`, ordered high-to-low op. + /// The new primary installs these over its own log. + pub headers: Vec, + /// Headers non-canonical senders report committed and the canonical chain + /// corroborates. See [`committed_elsewhere`]. + pub committed_elsewhere: Vec, +} + +/// Highest op the quorum proves committed. +/// +/// Three independent lower bounds, because a single sender's view of the commit +/// point can lag arbitrarily while the cluster's cannot: +/// * each sender's own reported commit, +/// * the `commit` its head prepare carries, stamped by that op's primary, +/// * its head minus the cluster-wide pipeline ceiling, since nothing further back +/// than one pipeline can still be in flight. +/// +/// The lowest op in a sender's suffix is deliberately NOT a fourth bound, and +/// re-adding one is a data-loss bug. It would be sound only if the suffix floor +/// were the commit point by construction; here it is computed, and two paths in +/// `build_dvc_suffix` raise it above the sender's commit (ops are 1-based, so +/// commit 0 floors at op 1; the `DVC_HEADERS_MAX` clamp drops the bottom of an +/// over-wide window). Nothing on the wire distinguishes the two. +/// +/// Nothing is lost by omitting it: the snapshot is tagged with the `(op, commit)` +/// the header is stamped with and dropped on a mismatch, so the floor either +/// equals `dvc.commit`, already the first bound, or exceeds it, the unsound case. +#[must_use] +pub fn merge_commit_max(quorum: &DvcQuorumArray, prepare_queue_ceiling: u64) -> u64 { + let mut commit_max = 0; + for dvc in dvc_iter(quorum) { + commit_max = commit_max.max(dvc.commit); + commit_max = commit_max.max(dvc.op.saturating_sub(prepare_queue_ceiling)); + if let Some(head) = dvc.suffix.headers().first() { + commit_max = commit_max.max(head.commit); + } + } + commit_max +} + +/// Highest `log_view` any sender reported. +/// +/// Senders at this `log_view` were in every earlier view change, so their headers +/// already reflect the truncations those views decided. That makes them canonical, +/// and a lower-`log_view` sender disagreeing is evidence against its own header. +fn log_view_canonical(quorum: &DvcQuorumArray) -> Option { + dvc_iter(quorum).map(|dvc| dvc.log_view).max() +} + +/// Per-op tally over the whole quorum. +struct OpVerdict<'a> { + canonical: Option<&'a PrepareHeader>, + /// Senders holding the canonical header AND able to serve its body. + copies: usize, + nacks: usize, + /// Canonical senders disagree about this op, so no header here is trustworthy. + conflict: bool, +} + +/// What the canonical senders say about one op. +struct CanonicalAt<'a> { + header: Option<&'a PrepareHeader>, + /// Two senders at the canonical `log_view` disagree here. + conflict: bool, +} + +/// The canonical header at `op`, and whether the canonical senders agree. +/// +/// Every canonical sender is consulted, not just the first: they were all in +/// normal status in that view and a primary prepares one thing per op, so a +/// disagreement is not a vote but proof that one header is wrong with no way to +/// tell which. The caller treats the op as undecidable. +fn canonical_header_at<'a>(canonical_senders: &[&'a StoredDvc], op: u64) -> CanonicalAt<'a> { + let mut header: Option<&'a PrepareHeader> = None; + let mut conflict = false; + for dvc in canonical_senders { + let Some(index) = dvc.suffix.index_of(dvc.op, op) else { + continue; + }; + let Some(candidate) = dvc.suffix.valid_header_at(index) else { + continue; + }; + match header { + Some(existing) if existing.checksum != candidate.checksum => conflict = true, + Some(_) => {} + None => header = Some(candidate), + } + } + CanonicalAt { header, conflict } +} + +/// Tally every sender's position on `op`. +fn tally_op<'a>( + quorum: &'a DvcQuorumArray, + canonical_senders: &[&'a StoredDvc], + canonical_log_view: u32, + op: u64, +) -> OpVerdict<'a> { + let CanonicalAt { + header: canonical, + conflict, + } = canonical_header_at(canonical_senders, op); + let mut copies = 0; + let mut nacks = 0; + + for dvc in dvc_iter(quorum) { + // The sender's log stops below this op, so it never prepared it. + if dvc.op < op { + nacks += 1; + continue; + } + let Some(index) = dvc.suffix.index_of(dvc.op, op) else { + // The sender said nothing about this op, so it abstains: no nack, no + // copy. Counting silence as a nack is sound only when every DVC carries + // headers, since then falling outside a window means being above it. + // Two cases here are not, and abstaining costs a slower view change + // where nacking costs data. + // + // An empty suffix is not a vote. It comes from a replica with + // nothing uncommitted, or one whose snapshot no longer matches its + // log; reading it as + // agreement with someone else's nack discards a committed op on one real + // nack plus one silence. And with `dvc.op >= op` established above, a + // non-empty suffix not covering `op` puts `op` below the sender's window + // floor, at or below its own commit point, so nacking it is backwards. + // Only the defensive `DVC_HEADERS_MAX` clamp reaches that. + continue; + }; + + let held = dvc.suffix.valid_header_at(index); + if let (Some(held), Some(canonical)) = (held, canonical) + && dvc.suffix.offers_body(index) + && held.checksum == canonical.checksum + { + copies += 1; + } + + if dvc.suffix.nacks(index) { + // Explicit: the sender proves it never prepared this op. + nacks += 1; + } else if let Some(held) = held { + // Only a sender BEHIND the canonical log_view can implicitly nack. + // Without this, corrupting one canonical header in transit turns every + // honest sender's correct header into an implicit nack against the + // garbage: a nack quorum on three replicas. A same-log_view + // disagreement is evidence, not a vote, and goes through `conflict`. + let may_nack_implicitly = dvc.log_view < canonical_log_view; + match canonical { + // Implicit: the sender holds a DIFFERENT prepare, so not this one. + Some(canonical) if may_nack_implicitly && held.checksum != canonical.checksum => { + nacks += 1; + } + // Implicit: no canonical sender holds anything here, so a newer + // view already truncated this op and the sender holds a corpse. + None if may_nack_implicitly => nacks += 1, + _ => {} + } + } + } + + OpVerdict { + canonical, + copies, + nacks, + conflict, + } +} + +/// Collect the canonical headers for `commit_max..=op_head`, high-to-low. +/// +/// Checks the hash chain as it walks: a break means the canonical senders agreed +/// on individual ops but not on one history, which no later step would notice. +fn canonical_headers( + canonical_senders: &[&StoredDvc], + op_head: u64, + commit_max: u64, +) -> Option> { + if op_head == 0 { + return Some(Vec::new()); + } + let floor = commit_max.max(1); + let mut headers = Vec::new(); + let mut child: Option = None; + let mut op = op_head; + loop { + let at = canonical_header_at(canonical_senders, op); + if at.conflict { + return None; + } + let header = *at.header?; + if let Some(child) = child + && child.parent != header.checksum + { + tracing::error!( + op, + child_op = child.op, + "view-change headers do not hash-chain; refusing to install" + ); + return None; + } + child = Some(header); + headers.push(header); + if op <= floor { + break; + } + op -= 1; + } + Some(headers) +} + +/// Headers a non-canonical sender reports committed, corroborated by the canonical +/// chain. +/// +/// Needed because header repair stops at a gap, so an op missing below the new +/// primary's commit point can never be repaired into place. +/// +/// But a sender's `commit` is not proof: `commit_max` advances from the primary's +/// claim without checking the local log matches, and reconcile leaves a divergent +/// entry at or below the applied floor in place. So a replica can honestly report +/// `commit >= N` holding a header at N that never committed. Nothing journals these, +/// but they pin the repair gate: the genuine repaired prepare then mismatches and is +/// discarded, and the view change stalls at N. +/// +/// So each candidate must be the `parent` the entry one op above names, walking down +/// from the canonical window: the canonical senders vouch, not the offering sender. +/// One that chains to nothing is dropped, leaving the op unconstrained for repair +/// rather than pinned to an unconfirmable header. Unsealed on either side passes. +/// +/// `None` when two senders report *different* prepares committed at one op: as in +/// [`canonical_header_at`], undecidable. +fn committed_elsewhere( + quorum: &DvcQuorumArray, + canonical_log_view: u32, + already_installed: &[PrepareHeader], +) -> Option> { + let mut claimed: Vec = Vec::new(); + for dvc in dvc_iter(quorum).filter(|dvc| dvc.log_view < canonical_log_view) { + for (index, header) in dvc.suffix.headers().iter().enumerate() { + if header.op > dvc.commit { + continue; + } + if dvc.suffix.valid_header_at(index).is_none() { + continue; + } + if already_installed + .iter() + .any(|installed| installed.op == header.op) + { + continue; + } + if let Some(queued) = claimed.iter().find(|queued| queued.op == header.op) { + if queued.checksum != header.checksum { + tracing::error!( + op = header.op, + "replicas disagree about the prepare committed at op {}; refusing to \ + choose one to install", + header.op + ); + return None; + } + continue; + } + claimed.push(*header); + } + } + + // Descending, so the entry a candidate must parent is already canonical or + // already corroborated. + claimed.sort_unstable_by_key(|header| std::cmp::Reverse(header.op)); + let mut extra: Vec = Vec::new(); + for header in claimed { + let child = already_installed + .iter() + .chain(extra.iter()) + .find(|candidate| candidate.op == header.op + 1); + let Some(child) = child else { + tracing::warn!( + op = header.op, + "op {} reported committed but nothing in the view's log chains to it; \ + leaving it unconstrained for repair", + header.op + ); + continue; + }; + if child.parent != header.checksum + && child.parent != CHECKSUM_UNSEALED + && header.checksum != CHECKSUM_UNSEALED + { + tracing::warn!( + op = header.op, + child_op = child.op, + "op {} reported committed but the entry above does not name it as parent; \ + dropping the claim", + header.op + ); + continue; + } + extra.push(header); + } + Some(extra) +} + +/// Decide whether the new view can start, and with what log. +/// +/// Walks every op that might be uncommitted, from the proven commit point to the +/// highest a canonical sender reported, stopping at the first proved dead. +#[must_use] +pub fn merge_dvc_quorum(quorum: &DvcQuorumArray, quorums: MergeQuorums) -> MergeOutcome { + let received = dvc_count(quorum); + if received < quorums.view_change { + return MergeOutcome::AwaitingQuorum; + } + + let Some(canonical_log_view) = log_view_canonical(quorum) else { + return MergeOutcome::AwaitingQuorum; + }; + let canonical_senders: Vec<&StoredDvc> = dvc_iter(quorum) + .filter(|dvc| dvc.log_view == canonical_log_view) + .collect(); + debug_assert!( + !canonical_senders.is_empty(), + "the max log_view must be held by at least one sender" + ); + + let commit_max = merge_commit_max(quorum, quorums.prepare_queue_ceiling); + let op_head_max = canonical_senders + .iter() + .map(|dvc| dvc.op) + .max() + .unwrap_or(commit_max) + .max(commit_max); + + if op_head_max == 0 { + // Nothing was ever prepared, so nothing to decide. Ops are 1-based, and + // scanning op 0 reads every sender's absent entry as a nack, manufacturing + // a nack quorum for an op that does not exist. + return MergeOutcome::Ready(MergedLog { + op_head: 0, + commit_max: 0, + headers: Vec::new(), + committed_elsewhere: Vec::new(), + }); + } + + let mut op_head = op_head_max; + // Start at the proven commit point, or op 1 when nothing is committed. The + // commit point is scanned so the adopted log is anchored on a servable header. + let mut op = commit_max.max(1); + while op <= op_head_max { + let verdict = tally_op(quorum, &canonical_senders, canonical_log_view, op); + + if verdict.nacks >= quorums.nack_prepare { + if op <= commit_max { + // A nack quorum for a committed op is impossible under quorum + // intersection, so a peer lied or a bitset is wrong. Refuse the + // view rather than assert, so one bad peer stalls the group instead + // of panicking a node into a restart loop. + tracing::error!( + op, + commit_max, + nacks = verdict.nacks, + nack_quorum = quorums.nack_prepare, + "nack quorum for an op the quorum proves committed; refusing the view" + ); + return MergeOutcome::Deadlocked { undecided_op: op }; + } + op_head = op - 1; + break; + } + + if verdict.conflict { + // Senders all in normal status in the same view disagree about what it + // prepared here. One header is wrong with no way to tell which, so + // refuse rather than pick. + tracing::error!( + op, + log_view = canonical_log_view, + "replicas at the same log_view disagree about op {op}; refusing to choose a \ + canonical header for it" + ); + return if received < quorums.replica_count { + MergeOutcome::AwaitingRepair { undecided_op: op } + } else { + MergeOutcome::Deadlocked { undecided_op: op } + }; + } + + if verdict.canonical.is_none() || verdict.copies == 0 { + // Neither provably dead nor recoverable. An outstanding replica may + // hold the body or supply the deciding nack. + return if received < quorums.replica_count { + MergeOutcome::AwaitingRepair { undecided_op: op } + } else { + tracing::error!( + op, + canonical = verdict.canonical.is_some(), + copies = verdict.copies, + nacks = verdict.nacks, + nack_quorum = quorums.nack_prepare, + "every replica reported and op {op} is neither recoverable nor provably \ + uncommitted; the view cannot start" + ); + MergeOutcome::Deadlocked { undecided_op: op } + }; + } + + op += 1; + } + + debug_assert!(op_head >= commit_max); + let Some(headers) = canonical_headers(&canonical_senders, op_head, commit_max) else { + return MergeOutcome::Deadlocked { + undecided_op: op_head, + }; + }; + let Some(committed_elsewhere) = committed_elsewhere(quorum, canonical_log_view, &headers) + else { + // Two senders disagree about a committed op. Not a repair problem: these + // install without a nack quorum, and no message resolves which is true. + return MergeOutcome::Deadlocked { + undecided_op: op_head, + }; + }; + + MergeOutcome::Ready(MergedLog { + op_head, + commit_max, + headers, + committed_elsewhere, + }) +} + +/// Build a suffix for a sender that holds every op in `commit..=op` with a +/// servable body. Test helper. +#[cfg(test)] +#[must_use] +pub fn suffix_all_present(headers: Vec) -> DvcSuffix { + // `1 << 128` overflows, and a full-width suffix is exactly what the clamp + // produces, so the widest case cannot use the shift. Past the width is left to + // `DvcSuffix::new`, which rejects it by name rather than as an overflow. + let count = u32::try_from(headers.len()).unwrap_or(u32::MAX).min(128); + let mask = u128::MAX.checked_shr(128 - count).unwrap_or(0); + DvcSuffix::new(headers, 0, mask) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::DVC_HEADERS_MAX; + use crate::view_change_quorum::{dvc_blank, dvc_quorum_array_empty, dvc_record}; + use iggy_binary_protocol::{Command2, Operation}; + + /// Three replicas: replication 2, view-change 2, nack 2. + fn quorums_r3() -> MergeQuorums { + MergeQuorums { + view_change: 2, + nack_prepare: 2, + replica_count: 3, + prepare_queue_ceiling: 32, + } + } + + /// A prepare whose checksum derives from its op, so the hash chain connects. + fn prepare(op: u64, view: u32) -> PrepareHeader { + PrepareHeader { + command: Command2::Prepare, + operation: Operation::CreateStream, + op, + view, + checksum: u128::from(op) | (u128::from(view) << 64), + parent: if op <= 1 { + 0 + } else { + u128::from(op - 1) | (u128::from(view) << 64) + }, + // Left at zero so each test drives `commit_max` through the bound it is + // about; `merge_commit_max` honours this field, covered separately below. + commit: 0, + ..Default::default() + } + } + + /// Headers for `low..=high`, ordered high-to-low as a suffix requires. + fn suffix_headers(low: u64, high: u64, view: u32) -> Vec { + (low..=high).rev().map(|op| prepare(op, view)).collect() + } + + fn dvc(replica: u8, log_view: u32, op: u64, commit: u64, suffix: DvcSuffix) -> StoredDvc { + StoredDvc { + replica, + log_view, + op, + commit, + suffix, + } + } + + #[test] + fn given_agreeing_quorum_when_merging_should_adopt_the_shared_head() { + let mut quorum = dvc_quorum_array_empty(); + dvc_record( + &mut quorum, + dvc(0, 1, 5, 3, suffix_all_present(suffix_headers(3, 5, 1))), + ); + dvc_record( + &mut quorum, + dvc(1, 1, 5, 3, suffix_all_present(suffix_headers(3, 5, 1))), + ); + + let MergeOutcome::Ready(log) = merge_dvc_quorum(&quorum, quorums_r3()) else { + panic!("an agreeing quorum must be ready"); + }; + assert_eq!(log.op_head, 5); + assert_eq!(log.commit_max, 3); + assert_eq!( + log.headers.iter().map(|h| h.op).collect::>(), + vec![5, 4, 3], + "headers run high-to-low from the head down to commit_max" + ); + } + + #[test] + fn given_nack_quorum_above_commit_when_merging_should_truncate_to_the_nacked_op() { + // Both survivors hold 1..=3 and never prepared 4, so the head drops to 3. + let mut quorum = dvc_quorum_array_empty(); + let mut headers = suffix_headers(2, 4, 1); + headers[0] = dvc_blank(4); + let nack_op_four = DvcSuffix::new(headers.clone(), 0b001, 0b110); + dvc_record(&mut quorum, dvc(0, 1, 4, 2, nack_op_four.clone())); + dvc_record(&mut quorum, dvc(1, 1, 4, 2, nack_op_four)); + + let MergeOutcome::Ready(log) = merge_dvc_quorum(&quorum, quorums_r3()) else { + panic!("a nack quorum must decide the view"); + }; + assert_eq!(log.op_head, 3, "op 4 is provably uncommitted"); + assert!(log.headers.iter().all(|header| header.op <= 3)); + } + + #[test] + fn given_one_nack_short_of_quorum_when_merging_should_keep_the_op() { + // Replica 0 never saw op 4; replica 1 holds it and can serve it. One nack + // is short of the quorum of 2, so op 4 survives. + let mut quorum = dvc_quorum_array_empty(); + let mut holed = suffix_headers(2, 4, 1); + holed[0] = dvc_blank(4); + dvc_record( + &mut quorum, + dvc(0, 1, 4, 2, DvcSuffix::new(holed, 0b001, 0b110)), + ); + dvc_record( + &mut quorum, + dvc(1, 1, 4, 2, suffix_all_present(suffix_headers(2, 4, 1))), + ); + + let MergeOutcome::Ready(log) = merge_dvc_quorum(&quorum, quorums_r3()) else { + panic!("one nack must not decide the view"); + }; + assert_eq!(log.op_head, 4, "a single nack cannot discard op 4"); + assert!( + log.headers.iter().any(|header| header.op == 4), + "the surviving op must be installed" + ); + } + + #[test] + fn given_committed_op_when_nacked_by_quorum_should_refuse_rather_than_truncate() { + // A nack quorum at or below the proven commit point is impossible under + // quorum intersection. If it appears anyway, refuse; never truncate. + let mut quorum = dvc_quorum_array_empty(); + let blanks = vec![dvc_blank(3)]; + let all_nacked = DvcSuffix::new(blanks, 0b1, 0b0); + dvc_record(&mut quorum, dvc(0, 1, 3, 3, all_nacked.clone())); + dvc_record(&mut quorum, dvc(1, 1, 3, 3, all_nacked)); + + assert_eq!( + merge_dvc_quorum(&quorum, quorums_r3()), + MergeOutcome::Deadlocked { undecided_op: 3 }, + "a committed op must never be truncated" + ); + } + + #[test] + fn given_blank_commit_point_from_every_sender_should_deadlock() { + // The commit point is scanned and may not be discarded, so a sender that + // reports it blank is deferring to a peer. When every sender defers there + // is no peer left and the view cannot start. + // + // Nothing in the merge can rescue this, which is why the senders must not + // produce it: a replica keeps the header at its own commit point through + // compaction (the metadata checkpoint drain stops one op short, a + // partition answers from its evicted ring). + let mut quorum = dvc_quorum_array_empty(); + let blank_at_commit = DvcSuffix::new(vec![dvc_blank(5)], 0, 0); + for replica in 0..3 { + dvc_record(&mut quorum, dvc(replica, 1, 5, 5, blank_at_commit.clone())); + } + + assert_eq!( + merge_dvc_quorum(&quorum, quorums_r3()), + MergeOutcome::Deadlocked { undecided_op: 5 }, + "a blank commit point is neither adoptable nor discardable" + ); + } + + #[test] + fn given_blank_commit_point_from_one_sender_should_adopt_the_peer_header() { + // The same suffix stops being fatal the moment one sender still holds the + // header: that one is canonical and serves the body, and the deferring + // sender neither nacks it nor conflicts with it. + let mut quorum = dvc_quorum_array_empty(); + dvc_record( + &mut quorum, + dvc(0, 1, 5, 5, DvcSuffix::new(vec![dvc_blank(5)], 0, 0)), + ); + dvc_record( + &mut quorum, + dvc(1, 1, 5, 5, suffix_all_present(suffix_headers(5, 5, 1))), + ); + + let MergeOutcome::Ready(log) = merge_dvc_quorum(&quorum, quorums_r3()) else { + panic!("one surviving copy of the commit point is enough to start the view"); + }; + assert_eq!(log.op_head, 5); + assert_eq!(log.commit_max, 5); + } + + #[test] + fn given_header_without_a_servable_body_when_replicas_outstanding_should_await_repair() { + // Both senders have op 4's header, neither can serve its body, and replica 2 + // has not reported. A head whose body nobody holds would wedge the view. + let mut quorum = dvc_quorum_array_empty(); + let headers = suffix_headers(2, 4, 1); + let header_only = DvcSuffix::new(headers, 0, 0b110); + dvc_record(&mut quorum, dvc(0, 1, 4, 2, header_only.clone())); + dvc_record(&mut quorum, dvc(1, 1, 4, 2, header_only)); + + assert_eq!( + merge_dvc_quorum(&quorum, quorums_r3()), + MergeOutcome::AwaitingRepair { undecided_op: 4 } + ); + } + + #[test] + fn given_all_replicas_reported_and_op_undecidable_should_deadlock() { + let mut quorum = dvc_quorum_array_empty(); + let headers = suffix_headers(2, 4, 1); + let header_only = DvcSuffix::new(headers, 0, 0b110); + dvc_record(&mut quorum, dvc(0, 1, 4, 2, header_only.clone())); + dvc_record(&mut quorum, dvc(1, 1, 4, 2, header_only.clone())); + dvc_record(&mut quorum, dvc(2, 1, 4, 2, header_only)); + + assert_eq!( + merge_dvc_quorum(&quorum, quorums_r3()), + MergeOutcome::Deadlocked { undecided_op: 4 }, + "with every replica in, an unrecoverable op stalls the view forever" + ); + } + + #[test] + fn given_lower_log_view_sender_when_merging_should_prefer_the_canonical_log() { + // Replica 1 is at the newer log_view, so its op 4 is canonical and + // replica 0's stale op 4 counts as an implicit nack against itself. + let mut quorum = dvc_quorum_array_empty(); + dvc_record( + &mut quorum, + dvc(0, 1, 4, 2, suffix_all_present(suffix_headers(2, 4, 1))), + ); + dvc_record( + &mut quorum, + dvc(1, 2, 4, 2, suffix_all_present(suffix_headers(2, 4, 2))), + ); + + let MergeOutcome::Ready(log) = merge_dvc_quorum(&quorum, quorums_r3()) else { + panic!("the canonical log must win"); + }; + assert_eq!(log.op_head, 4); + assert!( + log.headers.iter().all(|header| header.view == 2), + "installed headers must come from the canonical log_view" + ); + } + + #[test] + fn given_committed_header_on_a_stale_sender_should_be_installed_anyway() { + // Replica 0 is behind on log_view but reports op 2 committed. The canonical + // window starts at 3, so op 2 is otherwise unreachable across the gap. Its + // header is genuine (a `view` stamp is when the entry was appended, not the + // sender's `log_view`), so op 3 names it as parent. + let mut quorum = dvc_quorum_array_empty(); + dvc_record( + &mut quorum, + dvc(0, 1, 2, 2, suffix_all_present(suffix_headers(2, 2, 2))), + ); + dvc_record( + &mut quorum, + dvc(1, 2, 4, 3, suffix_all_present(suffix_headers(3, 4, 2))), + ); + + let MergeOutcome::Ready(log) = merge_dvc_quorum(&quorum, quorums_r3()) else { + panic!("the view must start"); + }; + assert!( + log.committed_elsewhere.iter().any(|header| header.op == 2), + "a committed header from a stale sender must still be installed" + ); + } + + #[test] + fn given_a_committed_claim_the_canonical_log_does_not_chain_to_should_be_dropped() { + // A replica can honestly report `commit >= 2` while holding a header at 2 + // that never committed. Installing it pins the repair gate: the genuine + // repaired prepare then mismatches, and the view change stalls at op 2. + let mut quorum = dvc_quorum_array_empty(); + let mut impostor = prepare(2, 2); + impostor.checksum ^= 0xFF; + dvc_record( + &mut quorum, + dvc(0, 1, 2, 2, suffix_all_present(vec![impostor])), + ); + dvc_record( + &mut quorum, + dvc(1, 2, 4, 3, suffix_all_present(suffix_headers(3, 4, 2))), + ); + + let MergeOutcome::Ready(log) = merge_dvc_quorum(&quorum, quorums_r3()) else { + panic!("the view must start"); + }; + assert!( + log.committed_elsewhere.is_empty(), + "a claim the canonical log does not chain to must not pin the repair gate" + ); + } + + #[test] + fn given_a_corrupted_canonical_header_should_not_let_honest_senders_nack_it() { + // Transit corruption of one canonical sender's suffix entry must not turn + // every other sender's CORRECT header at that op into an implicit nack + // against the garbage, reaching a nack quorum through the one path that does + // not verify. Senders at the canonical log_view cannot legitimately disagree, + // since a primary prepares one thing per op, so it is evidence, not a vote. + let mut quorum = dvc_quorum_array_empty(); + + let mut corrupted = suffix_headers(2, 4, 1); + corrupted[0].checksum ^= 0xFFFF; + dvc_record(&mut quorum, dvc(0, 1, 4, 2, suffix_all_present(corrupted))); + // Two honest senders at the same log_view holding the real op 4. + dvc_record( + &mut quorum, + dvc(1, 1, 4, 2, suffix_all_present(suffix_headers(2, 4, 1))), + ); + dvc_record( + &mut quorum, + dvc(2, 1, 4, 2, suffix_all_present(suffix_headers(2, 4, 1))), + ); + + let outcome = merge_dvc_quorum(&quorum, quorums_r3()); + if let MergeOutcome::Ready(log) = &outcome { + assert_eq!( + log.op_head, 4, + "op 4 is held by two honest senders and must not be discarded" + ); + } + assert!( + !matches!(&outcome, MergeOutcome::Ready(log) if log.op_head < 4), + "a corrupted canonical header must never authorise truncating op 4, got {outcome:?}" + ); + } + + #[test] + fn given_a_mixed_version_quorum_when_one_upgraded_sender_nacks_should_not_truncate() { + // A silent sender must never count as agreement with someone else's nack. + // Only replica 1 proves it never held op 4; replica 0 sends no suffix and so + // says nothing. One real nack is short of the quorum of two, so op 4 has to + // survive -- reading the empty suffix as a second nack discards it. + let mut quorum = dvc_quorum_array_empty(); + dvc_record(&mut quorum, dvc(0, 1, 4, 2, DvcSuffix::empty())); + let mut holed = suffix_headers(2, 4, 1); + holed[0] = dvc_blank(4); + dvc_record( + &mut quorum, + dvc(1, 1, 4, 2, DvcSuffix::new(holed, 0b001, 0b110)), + ); + dvc_record( + &mut quorum, + dvc(2, 1, 4, 2, suffix_all_present(suffix_headers(2, 4, 1))), + ); + + let MergeOutcome::Ready(log) = merge_dvc_quorum(&quorum, quorums_r3()) else { + panic!("op 4 is recoverable from replica 2, so the view must start"); + }; + assert_eq!( + log.op_head, 4, + "an empty suffix must not stand in for the second nack" + ); + } + + #[test] + fn given_a_clamped_suffix_floor_when_merging_should_not_raise_commit_max() { + // `build_dvc_suffix` clamps a window wider than `DVC_HEADERS_MAX` from below, + // so its floor stops being the sender's commit point with nothing on the wire + // saying so. Reading that floor as a commit point marks every op between the + // real commit and the clamp committed: applied and replied to without a + // replication quorum, and unreachable by later truncation via `Deadlocked`. + let mut quorum = dvc_quorum_array_empty(); + // Sender at op 400, commit 200, whose window clamped to 273..=400. + let clamped = suffix_headers(273, 400, 1); + assert_eq!(clamped.len(), DVC_HEADERS_MAX); + dvc_record( + &mut quorum, + dvc(0, 1, 400, 200, suffix_all_present(clamped.clone())), + ); + dvc_record( + &mut quorum, + dvc(1, 1, 400, 200, suffix_all_present(clamped)), + ); + + // A ceiling wide enough not to bind, so this asserts the floor rule alone. In + // production the ceiling is `PREPARE_QUEUE_CEILING`, which already puts + // `commit_max` at or above any clamped floor. + assert_eq!( + merge_commit_max(&quorum, 1000), + 200, + "a clamped window floor is a scan bound, not a proven commit point" + ); + } + + #[test] + fn given_a_sender_at_commit_zero_when_merging_should_not_commit_op_one() { + // The other floor-raising path: ops are 1-based, so a sender with nothing + // committed still floors its window at op 1. Every sender says commit 0, so + // treating op 1 as committed would ack a client for an unheld op. + let mut quorum = dvc_quorum_array_empty(); + let floored = suffix_headers(1, 1, 1); + dvc_record( + &mut quorum, + dvc(0, 1, 1, 0, suffix_all_present(floored.clone())), + ); + dvc_record(&mut quorum, dvc(1, 1, 1, 0, suffix_all_present(floored))); + + assert_eq!( + merge_commit_max(&quorum, 32), + 0, + "nothing is committed, so the merge must prove nothing committed" + ); + } + + #[test] + fn given_disagreeing_committed_elsewhere_headers_should_refuse_the_view() { + // These headers install unconditionally, with no nack quorum behind them, so + // first-wins is the defect `canonical_header_at` refuses for the canonical + // range: only one of two committed claims can be true. + let mut quorum = dvc_quorum_array_empty(); + // Canonical sender at the higher log_view. Its window starts at the proven + // commit point, so ops below it come only from a stale sender. + dvc_record( + &mut quorum, + dvc(0, 2, 6, 5, suffix_all_present(suffix_headers(5, 6, 2))), + ); + // Two senders behind on log_view but level on op, so no nack and no conflict + // inside the canonical range. They disagree only at op 3, which both report + // committed; `prepare` derives the checksum from `(op, view)`. + dvc_record( + &mut quorum, + dvc(1, 1, 6, 5, suffix_all_present(suffix_headers(3, 6, 2))), + ); + let mut divergent = suffix_headers(3, 6, 2); + *divergent.last_mut().expect("suffix is non-empty") = prepare(3, 5); + dvc_record(&mut quorum, dvc(2, 1, 6, 5, suffix_all_present(divergent))); + + assert!( + matches!( + merge_dvc_quorum(&quorum, quorums_r3()), + MergeOutcome::Deadlocked { .. } + ), + "two committed claims at one op must refuse the view, not pick one" + ); + } + + #[test] + fn given_head_header_claiming_a_higher_commit_should_raise_commit_max() { + // The head prepare's `commit` was stamped by the primary that prepared it, so + // it proves a commit point even when every sender's own tracking lags. + // Without it the merge rescans committed ops and could accept nacks for them. + let mut quorum = dvc_quorum_array_empty(); + let mut headers = suffix_headers(2, 4, 1); + headers[0].commit = 3; + dvc_record( + &mut quorum, + dvc(0, 1, 4, 2, suffix_all_present(headers.clone())), + ); + dvc_record(&mut quorum, dvc(1, 1, 4, 2, suffix_all_present(headers))); + + assert_eq!( + merge_commit_max(&quorum, 32), + 3, + "the head header's commit field is a commit_max lower bound" + ); + } +} diff --git a/core/consensus/src/impls.rs b/core/consensus/src/impls.rs index d23378f154..e55c9c3b9f 100644 --- a/core/consensus/src/impls.rs +++ b/core/consensus/src/impls.rs @@ -18,24 +18,25 @@ use crate::oneshot::{self, Receiver, Sender}; use crate::vsr_timeout::{TimeoutKind, TimeoutManager}; use crate::{ - AckLogEvent, Consensus, ControlActionLogEvent, DvcQuorumArray, IgnoreReason, Pipeline, - PlaneKind, PrepareLogEvent, Project, ReplicaLogContext, SimEventKind, StoredDvc, - ViewChangeLogEvent, ViewChangeReason, VsrState, dvc_count, dvc_max_commit, - dvc_quorum_array_empty, dvc_record, dvc_reset, dvc_select_winner, emit_replica_event, - emit_sim_event, + AckLogEvent, Consensus, ControlActionLogEvent, DvcQuorumArray, DvcSuffix, IgnoreReason, + MergeOutcome, MergeQuorums, MergedLog, Pipeline, PlaneKind, PrepareLogEvent, Project, + ReplicaLogContext, SimEventKind, StoredDvc, ViewChangeLogEvent, ViewChangeReason, VsrState, + dvc_count, dvc_iter, dvc_quorum_array_empty, dvc_record, dvc_reset, dvc_suffix_decode, + emit_replica_event, emit_sim_event, merge_dvc_quorum, seal_prepare_checksum, }; use bit_set::BitSet; use clock::{Clock, IggySystemClock}; use iggy_binary_protocol::{ Command2, ConsensusHeader, DoViewChangeHeader, GenericHeader, PrepareHeader, PrepareOkHeader, - ReplyHeader, RequestHeader, RequestStartViewHeader, StartViewChangeHeader, StartViewHeader, + ReplyHeader, RequestStartViewHeader, RoutedRequestHeader, StartViewChangeHeader, + StartViewHeader, frame_body, }; use iggy_common::IggyTimestamp; use iggy_common::calculate_checksum; use message_bus::IggyMessageBus; use message_bus::MessageBus; use server_common::Message; -use server_common::sharding::{IggyNamespace, METADATA_CONSENSUS_NAMESPACE}; +use server_common::sharding::{IggyNamespace, METADATA_GROUP}; use std::cell::{Cell, RefCell}; use std::collections::VecDeque; use std::rc::Rc; @@ -127,7 +128,7 @@ impl Sequencer for LocalSequencer { /// Default in-flight prepare-queue depth. /// -/// [`LocalPipeline::new`] uses it, and the server-ng config default +/// [`LocalPipeline::new`] uses it, and the server config default /// (`DEFAULT_METADATA_PREPARE_QUEUE_DEPTH`) is static-asserted equal to it at /// bootstrap. Operators raise the running bound via `[metadata] /// prepare_queue_depth`; the pipeline then carries its own capacity (see @@ -146,6 +147,27 @@ pub const PIPELINE_REQUEST_QUEUE_MAX: usize = 64; /// Maximum number of replicas in a cluster. pub const REPLICAS_MAX: usize = 32; +/// Ceiling on [`VsrConsensus::quorum_replication`]. +/// +/// Past three acks marginal durability is small and every extra ack sits on the +/// commit path, so wide clusters spend the difference on the view-change quorum. +pub const QUORUM_REPLICATION_MAX: usize = 3; + +/// Headers a `DoViewChange` may carry, and so the widest uncommitted suffix a +/// view change can reason about. +/// +/// Pinned by the wire: `DoViewChangeHeader`'s nack and present bitsets are one +/// `u128` each, one bit per entry. The suffix spans `commit_max..=op`, bounded by +/// `prepare_queue_max`, so capping that depth here keeps every suffix addressable. +pub const DVC_HEADERS_MAX: usize = 128; + +/// Deepest prepare queue any node in the cluster may be configured with. +/// +/// One less than [`DVC_HEADERS_MAX`]: the suffix spans `commit..=op` and the head +/// needs the reserved slot. Config ceilings and [`LocalPipeline::with_capacities`] +/// both enforce it, so it holds for a peer as well as for this node. +pub const PREPARE_QUEUE_CEILING: usize = DVC_HEADERS_MAX - 1; + /// Unanswered `RequestStartView` probes tolerated before a recovering /// replica gives up waiting for a settled primary and falls back to an /// election (a full-cluster restart leaves nobody able to answer). @@ -242,7 +264,7 @@ impl PipelineEntry { /// Accepted request waiting in `request_queue` for a prepare slot. #[derive(Debug)] pub struct RequestEntry { - pub message: Message, + pub message: Message, // TODO: populate from monotonic clock at push, promote to `pub` for // age-based filtering. Currently `0`; `pub(crate)` blocks sort-on-stub. #[allow(dead_code)] @@ -256,7 +278,7 @@ pub struct RequestEntry { impl RequestEntry { #[must_use] - pub const fn new(message: Message) -> Self { + pub const fn new(message: Message) -> Self { Self { message, received_at: 0, @@ -271,7 +293,7 @@ impl RequestEntry { /// instead of being bounced with a transient error. #[must_use] pub fn with_subscriber( - message: Message, + message: Message, ) -> (Self, Receiver>) { let (sender, receiver) = oneshot::channel(); let entry = Self { @@ -296,7 +318,7 @@ pub struct LocalPipeline { /// Requests awaiting a prepare slot; cap [`Self::request_queue_max`]. request_queue: VecDeque, /// Depth bound for `prepare_queue`; [`PIPELINE_PREPARE_QUEUE_MAX`] - /// unless the operator overrode it (`[metadata]` in the server-ng + /// unless the operator overrode it (`[metadata]` in the server /// config). prepare_queue_max: usize, /// Depth bound for `request_queue`; [`PIPELINE_REQUEST_QUEUE_MAX`] @@ -325,7 +347,8 @@ impl LocalPipeline { /// `SnapshotCoordinator` in `core/metadata`). /// /// # Panics - /// If a depth is zero — a zero-depth pipeline can never admit an op. + /// If a depth is zero, or if the prepare depth would let the uncommitted + /// suffix outgrow what a `DoViewChange` can address. #[must_use] pub fn with_capacities(prepare_queue_max: usize, request_queue_max: usize) -> Self { assert!( @@ -333,6 +356,16 @@ impl LocalPipeline { "pipeline queue depths must be non-zero \ (prepare={prepare_queue_max}, request={request_queue_max})" ); + // Each `DoViewChange` bitset addresses one suffix entry with one bit of a + // `u128`, and the suffix spans `commit..=op`. Deeper, and the builder clamps + // its window from below, leaving undecidable ops and a stalled view change. + // Config ceilings also enforce this; a stall is worth a loud boot. + assert!( + prepare_queue_max < DVC_HEADERS_MAX, + "prepare queue depth {prepare_queue_max} would produce an uncommitted suffix wider \ + than a DoViewChange can address (max {})", + DVC_HEADERS_MAX - 1, + ); Self { prepare_queue: VecDeque::with_capacity(prepare_queue_max), request_queue: VecDeque::with_capacity(request_queue_max), @@ -723,7 +756,7 @@ pub enum CommitOutcome { #[derive(Debug, Clone)] pub enum VsrAction { /// Send `StartViewChange` to all replicas. - SendStartViewChange { view: u32, namespace: u64 }, + SendStartViewChange { view: u32, group: u64 }, /// Send `DoViewChange` to primary. SendDoViewChange { view: u32, @@ -731,13 +764,18 @@ pub enum VsrAction { log_view: u32, op: u64, commit: u64, - namespace: u64, + group: u64, + /// The sender's uncommitted suffix, snapshotted for this view. Carried on + /// the action rather than re-read by the dispatcher so the wire bytes match + /// this replica's own `StoredDvc`: a merge seeing two versions of one + /// sender's suffix could adopt a header no replica holds. + suffix: DvcSuffix, }, /// Broadcast a `RequestStartView` probe (recovering replica asking for /// the current view's `StartView`; only that view's primary answers). /// Stamped with the prober's view so peers can fence stale duplicates /// out of the probed-primary election path. - SendRequestStartView { view: u32, namespace: u64 }, + SendRequestStartView { view: u32, group: u64 }, /// Send `StartView`, as the view's primary. /// /// `incarnation` echoes the requester's nonce when this answers a @@ -754,7 +792,14 @@ pub enum VsrAction { commit: u64, incarnation: u128, target: Option, - namespace: u64, + group: u64, + /// The view's suffix, high-to-low op from `op` down toward `commit`. + /// + /// Lets a backup check the head it is told to adopt against real headers, + /// and gives it canonical checksums to verify repaired bodies against. + /// Empty on the probe-answer path, where the primary reports its own + /// frontier rather than concluding a view change; the backup trusts `op`. + suffix: Vec, }, /// Send `PrepareOK` for each op in `[from_op, to_op]` that is present in the WAL. /// @@ -766,7 +811,7 @@ pub enum VsrAction { from_op: u64, to_op: u64, target: u8, - namespace: u64, + group: u64, }, /// Retransmit uncommitted prepares from the WAL to replicas that haven't acked. /// @@ -793,7 +838,7 @@ pub enum VsrAction { SendCommit { view: u32, commit: u64, - namespace: u64, + group: u64, timestamp_monotonic: u64, }, } @@ -829,7 +874,7 @@ where cluster: u128, replica: u8, replica_count: u8, - namespace: u64, + group: u64, view: Cell, @@ -923,6 +968,29 @@ where /// built-in default. probe_attempts_max: Cell, + /// This replica's own uncommitted suffix, with the `(op, commit)` the journal + /// was at when it was read. + /// + /// Installed by the shard via [`Self::set_local_dvc_suffix`] before any handler + /// that could enter a view change. Snapshotted rather than recomputed per send + /// so a retransmit is byte-identical: a nack is a durable claim about this + /// replica's log, and silently retracting one lets the new primary assemble a + /// quorum that never simultaneously existed. + /// + /// Tagged by `(op, commit)`, not by view, because that is what the suffix + /// describes: a view advance leaves the log alone so the snapshot survives, + /// while anything moving the head or commit point makes the tag mismatch, + /// which reads as no snapshot at all. + local_dvc_suffix: RefCell>, + + /// The log a DVC quorum settled on, parked until this replica's journal can + /// serve all of it. + /// + /// Non-`None` means "primary-elect, repairing": decided but not started, so + /// this replica prepares and announces nothing. Cleared by + /// [`VsrConsensus::start_pending_view`], or by `reset_view_change_state`. + pending_view_log: RefCell>, + /// Tracks DVC messages received (only used by primary candidate) /// Stores metadata; actual log comes from message do_view_change_from_all_replicas: RefCell, @@ -952,7 +1020,7 @@ impl> VsrConsensus { cluster: u128, replica: u8, replica_count: u8, - namespace: u64, + group: u64, message_bus: B, pipeline: P, ) -> Self { @@ -960,7 +1028,7 @@ impl> VsrConsensus { cluster, replica, replica_count, - namespace, + group, message_bus, pipeline, ConsensusClock::system(), @@ -977,7 +1045,7 @@ impl> VsrConsensus { cluster: u128, replica: u8, replica_count: u8, - namespace: u64, + group: u64, message_bus: B, pipeline: P, clock: ConsensusClock, @@ -988,25 +1056,25 @@ impl> VsrConsensus { ); assert!(replica_count >= 1, "need at least 1 replica"); // Consensus-control routing distinguishes metadata frames from - // partition frames by namespace value: metadata uses the sentinel, + // partition frames by the group id: metadata uses the sentinel, // partitions use `IggyNamespace::inner()` which lives strictly - // inside the packed range. A namespace outside both ranges would + // inside the packed range. A group outside both ranges would // route to neither and silently warn-drop on every receiving peer. debug_assert!( - namespace == METADATA_CONSENSUS_NAMESPACE || IggyNamespace::is_packable(namespace), - "VsrConsensus namespace must be METADATA_CONSENSUS_NAMESPACE or a packable \ - IggyNamespace; got {namespace:#x}" + group == METADATA_GROUP || IggyNamespace::is_packable(group), + "VsrConsensus group must be METADATA_GROUP or a packable \ + IggyNamespace; got {group:#x}" ); // TODO: Verify that XOR-based seeding provides sufficient jitter diversity // across groups. Consider using a proper hash (e.g., Murmur3) of - // (replica_id, namespace) for production. - let timeout_seed = u128::from(replica) ^ u128::from(namespace); + // (replica_id, group) for production. + let timeout_seed = u128::from(replica) ^ u128::from(group); let prepare_queue_max = pipeline.prepare_queue_max(); Self { cluster, replica, replica_count, - namespace, + group, view: Cell::new(0), log_view: Cell::new(0), view_durable: Cell::new(0), @@ -1030,6 +1098,8 @@ impl> VsrConsensus { start_view_change_from_all_replicas: RefCell::new(BitSet::with_capacity(REPLICAS_MAX)), probe_attempts: Cell::new(0), probe_attempts_max: Cell::new(PROBE_ATTEMPTS_MAX), + local_dvc_suffix: RefCell::new(None), + pending_view_log: RefCell::new(None), do_view_change_from_all_replicas: RefCell::new(dvc_quorum_array_empty()), do_view_change_quorum: Cell::new(false), sent_own_start_view_change: Cell::new(false), @@ -1249,10 +1319,45 @@ impl> VsrConsensus { (self.replica_count as usize - 1) / 2 } - /// Quorum size = f + 1 = `max_faulty` + 1 + /// Replicas that must ack before an op is committed. + /// + /// Capped at [`QUORUM_REPLICATION_MAX`] to keep a wide cluster's commit path + /// cheap; the view-change quorum grows so the two still sum above the count. + #[must_use] + pub const fn quorum_replication(&self) -> usize { + if self.replica_count == 2 { + // =1 would intersect, but =2 keeps a two-replica cluster durable. + return 2; + } + let half_rounded_up = (self.replica_count as usize).div_ceil(2); + if half_rounded_up < QUORUM_REPLICATION_MAX { + half_rounded_up + } else { + QUORUM_REPLICATION_MAX + } + } + + /// Replicas that must send a `DoViewChange` before a view can start. + /// + /// Pays for the cheaper replication quorum, which is the far hotter path. + #[must_use] + pub const fn quorum_view_change(&self) -> usize { + if self.replica_count == 2 { + // Avoids a single-replica view change special case. + return 2; + } + self.replica_count as usize - self.quorum_replication() + 1 + } + + /// Nacks required to prove an op was never committed, so the new primary may + /// truncate it. + /// + /// Sized so a nack quorum and a replication quorum cannot both exist for one + /// op. This is what makes truncation safe, so it is the one quorum that must + /// never be loosened. #[must_use] - pub const fn quorum(&self) -> usize { - self.max_faulty() + 1 + pub const fn quorum_nack_prepare(&self) -> usize { + self.replica_count as usize - self.quorum_replication() + 1 } /// Highest op locally executed (state machine applied, client table updated). @@ -1431,8 +1536,8 @@ impl> VsrConsensus { } #[must_use] - pub const fn namespace(&self) -> u64 { - self.namespace + pub const fn group(&self) -> u64 { + self.group } #[must_use] @@ -1533,6 +1638,64 @@ impl> VsrConsensus { } } + /// Install this replica's uncommitted-suffix snapshot for the current view. + /// + /// Called by the shard, which owns the journal. Consensus keeps the snapshot + /// rather than deriving it so the copy in this replica's `StoredDvc` and the + /// copy on the wire are the same bytes: a merge seeing two versions of one + /// sender's suffix could adopt a header no replica holds. + /// + /// Installing twice for one view overwrites: the shard refreshes before each + /// handler, and a later suffix is at least as complete (repair only adds). + pub fn set_local_dvc_suffix(&self, suffix: DvcSuffix) { + let (op, commit) = self.local_dvc_suffix_tag(); + *self.local_dvc_suffix.borrow_mut() = Some((op, commit, suffix)); + } + + /// Drop the cached suffix snapshot. + /// + /// The `(op, commit)` tag tracks how far the log reaches, not what it still + /// contains, so a mutation that removes entries without moving either + /// (truncating a diverging uncommitted range) leaves a snapshot reading as + /// current while offering bodies this replica can no longer serve. A peer that + /// picks it as a body source then waits out the whole view change. + /// + /// Call from the mutation site. The next refresh re-reads the journal. + pub fn invalidate_local_dvc_suffix(&self) { + self.local_dvc_suffix.borrow_mut().take(); + } + + /// The `(op, commit)` a snapshot must match to still describe this log. + /// `commit` is clamped to `op` exactly as the outgoing DVC clamps it. + fn local_dvc_suffix_tag(&self) -> (u64, u64) { + let op = self.sequencer.current_sequence(); + (op, self.commit_max.get().min(op)) + } + + /// This replica's suffix snapshot, or an empty one when none matches the log's + /// current head and commit point. Empty is the safe direction: it nacks nothing + /// and offers no bodies, so it can only stall a view change, never authorise a + /// truncation. + #[must_use] + pub fn local_dvc_suffix(&self) -> DvcSuffix { + let tag = self.local_dvc_suffix_tag(); + match &*self.local_dvc_suffix.borrow() { + Some((op, commit, suffix)) if (*op, *commit) == tag => suffix.clone(), + _ => DvcSuffix::empty(), + } + } + + /// True when no snapshot matches the log's current head and commit point, + /// so the shard must read one from the journal before this replica votes. + #[must_use] + pub fn local_dvc_suffix_stale(&self) -> bool { + let tag = self.local_dvc_suffix_tag(); + !matches!( + &*self.local_dvc_suffix.borrow(), + Some((op, commit, _)) if (*op, *commit) == tag + ) + } + /// True when the current `(view, log_view)` is not yet in the superblock, so a /// view-scoped send would advertise a view a crash could lose. The split-brain /// gate: the dispatcher persists first when this holds. `commit_max` is @@ -1615,6 +1778,9 @@ impl> VsrConsensus { self.reset_dvc_quorum(); self.sent_own_start_view_change.set(false); self.sent_own_do_view_change.set(false); + // A merge parked for the superseded view may describe a different log, so + // drop it and let the new attempt re-derive from the DVCs it collects. + self.pending_view_log.borrow_mut().take(); self.loopback_queue.borrow_mut().clear(); let mut pipeline = self.pipeline.borrow_mut(); pipeline.cancel_all_subscribers(); @@ -1696,7 +1862,7 @@ impl> VsrConsensus { if self.state_transfer_stage.get() != StateTransferStage::Idle { tracing::info!( replica = self.replica, - namespace_raw = self.namespace, + namespace_raw = self.group, "view probe exhausted; abandoning state transfer (cluster bootstrap)" ); self.set_state_transfer_stage(StateTransferStage::Idle); @@ -1711,7 +1877,7 @@ impl> VsrConsensus { .reset(TimeoutKind::RequestStartViewMessage); actions.push(VsrAction::SendRequestStartView { view: self.view.get(), - namespace: self.namespace, + group: self.group, }); } } @@ -1721,7 +1887,7 @@ impl> VsrConsensus { .reset(TimeoutKind::RequestStartViewMessage); actions.push(VsrAction::SendRequestStartView { view: self.view.get(), - namespace: self.namespace, + group: self.group, }); } _ => { @@ -1785,7 +1951,7 @@ impl> VsrConsensus { if self.observed_newer_view.get() > self.view.get() { tracing::info!( replica = self.replica, - namespace_raw = self.namespace, + namespace_raw = self.group, view = self.view.get(), observed_newer_view = self.observed_newer_view.get(), "heartbeat timed out behind a newer view; probing to catch up" @@ -1830,7 +1996,7 @@ impl> VsrConsensus { let action = VsrAction::SendStartViewChange { view: new_view, - namespace: self.namespace, + group: self.group, }; emit_sim_event( SimEventKind::ControlMessageScheduled, @@ -1854,7 +2020,7 @@ impl> VsrConsensus { let action = VsrAction::SendStartViewChange { view: self.view.get(), - namespace: self.namespace, + group: self.group, }; emit_sim_event( SimEventKind::ControlMessageScheduled, @@ -1885,16 +2051,13 @@ impl> VsrConsensus { .borrow_mut() .reset(TimeoutKind::DoViewChangeMessage); - let current_op = self.sequencer.current_sequence(); - let action = VsrAction::SendDoViewChange { - view: self.view.get(), - target: self.primary_index(self.view.get()), - log_view: self.log_view.get(), - op: current_op, - // commit_max clamped to op: see `handle_start_view_change`. - commit: self.commit_max.get().min(current_op), - namespace: self.namespace, - }; + // NOT the snapshot the first send used: `build_do_view_change` re-reads + // `local_dvc_suffix()`, whose `(op, commit)` tag can have moved since, in + // which case it answers EMPTY, retracting every nack and body offer already + // sent. Survivable only because `dvc_record` drops a duplicate sender, so the + // candidate keeps the first vote. Allow a retransmit to replace a seated vote + // and this must pin the snapshot instead. + let action = self.build_do_view_change(self.primary_index(self.view.get())); emit_sim_event( SimEventKind::ControlMessageScheduled, &ControlActionLogEvent::from_vsr_action( @@ -1938,7 +2101,7 @@ impl> VsrConsensus { let action = VsrAction::SendStartViewChange { view: next_view, - namespace: self.namespace, + group: self.group, }; emit_sim_event( SimEventKind::ControlMessageScheduled, @@ -2047,7 +2210,7 @@ impl> VsrConsensus { if self.observed_newer_view.get() > self.view.get() { tracing::info!( replica = self.replica, - namespace_raw = self.namespace, + namespace_raw = self.group, view = self.view.get(), observed_newer_view = self.observed_newer_view.get(), "stale primary-by-index behind a newer view; probing to catch up" @@ -2068,7 +2231,7 @@ impl> VsrConsensus { vec![VsrAction::SendCommit { view: self.view.get(), commit: self.commit_min.get(), - namespace: self.namespace, + group: self.group, timestamp_monotonic: ts, }] } @@ -2080,16 +2243,13 @@ impl> VsrConsensus { /// that will be the primary in the new view." /// /// # Panics - /// If `header.namespace` does not match this replica's namespace. + /// If `header.group` does not match this replica's namespace. pub fn handle_start_view_change( &self, plane: PlaneKind, header: &StartViewChangeHeader, ) -> Vec { - assert_eq!( - header.namespace, self.namespace, - "SVC routed to wrong group" - ); + assert_eq!(header.group, self.group, "SVC routed to wrong group"); // A recovering replica is quorum-invisible: it lost (or cannot trust) // its durable state, so it must not vote history into existence. The // election proceeds among the peers; its conclusion reaches this @@ -2140,7 +2300,7 @@ impl> VsrConsensus { // Send our own SVC let action = VsrAction::SendStartViewChange { view: msg_view, - namespace: self.namespace, + group: self.group, }; emit_sim_event( SimEventKind::ControlMessageScheduled, @@ -2166,36 +2326,14 @@ impl> VsrConsensus { let primary_candidate = self.primary_index(self.view.get()); let current_op = self.sequencer.current_sequence(); - // DVC carries commit_max (highest known-committed), not commit_min - // (locally applied). The new primary floors its pipeline rebuild at - // max(commit) across the quorum; only commit_max bounds that range - // to pipeline depth (every replica holds op - commit_max <= depth). - // commit_min can lag far behind and overflow the rebuild. The - // committed-but-unapplied tail (commit_min..commit_max] is replayed - // by the new primary's CommitJournal, not the pipeline. - // - // Clamp to op: a backup learns commit_max from a heartbeat before - // receiving the prepares, so commit_max can exceed its op. The wire - // contract `DoViewChangeHeader::validate` rejects commit > op and - // drops such a DVC (view-change liveness stall). The clamp is - // lossless for the rebuild floor: quorum intersection guarantees - // some sender whose op covers the true commit point carries it, so - // max(commit) across the quorum is unchanged. - let commit = self.commit_max.get().min(current_op); + let commit = self.dvc_commit(); // Start DVC timeout self.timeouts .borrow_mut() .start(TimeoutKind::DoViewChangeMessage); - let action = VsrAction::SendDoViewChange { - view: self.view.get(), - target: primary_candidate, - log_view: self.log_view.get(), - op: current_op, - commit, - namespace: self.namespace, - }; + let action = self.build_do_view_change(primary_candidate); emit_sim_event( SimEventKind::ControlMessageScheduled, &ControlActionLogEvent::from_vsr_action( @@ -2212,15 +2350,19 @@ impl> VsrConsensus { log_view: self.log_view.get(), op: current_op, commit, + suffix: self.local_dvc_suffix(), }; dvc_record( &mut self.do_view_change_from_all_replicas.borrow_mut(), own_dvc, ); - // Check if we now have quorum - if dvc_count(&self.do_view_change_from_all_replicas.borrow()) >= self.quorum() { - self.do_view_change_quorum.set(true); + // `complete_view_change_as_primary` latches only once the merge + // decides, so an undecidable quorum stays open to later DVCs. + if !self.do_view_change_quorum.get() + && dvc_count(&self.do_view_change_from_all_replicas.borrow()) + >= self.quorum_view_change() + { actions.extend(self.complete_view_change_as_primary(plane)); } } @@ -2235,17 +2377,77 @@ impl> VsrConsensus { /// replicas (including itself), it sets its view-number to that in the messages /// and selects as the new log the one contained in the message with the largest v'..." /// + /// The `commit` this replica advertises in a `DoViewChange`. + /// + /// `commit_max`, not `commit_min`: the new primary floors its pipeline rebuild + /// at `max(commit)` across the quorum, and only `commit_max` bounds that range + /// to the pipeline depth. `commit_min` can lag far enough to overflow the + /// rebuild; `CommitJournal` replays the committed-but-unapplied tail instead. + /// + /// Clamped to `op`, since a backup learns `commit_max` from a heartbeat before + /// the prepares and `DoViewChangeHeader::validate` rejects `commit > op`. + /// Lossless for the rebuild floor: quorum intersection guarantees some sender + /// whose head covers the true commit point carries it. + fn dvc_commit(&self) -> u64 { + let op = self.sequencer.current_sequence(); + self.commit_max.get().min(op) + } + + /// Build this replica's `DoViewChange` for the current view. + fn build_do_view_change(&self, target: u8) -> VsrAction { + VsrAction::SendDoViewChange { + view: self.view.get(), + target, + log_view: self.log_view.get(), + op: self.sequencer.current_sequence(), + commit: self.dvc_commit(), + group: self.group, + suffix: self.local_dvc_suffix(), + } + } + + /// Decode a peer's suffix, or `None` to drop the whole `DoViewChange`. + /// + /// A suffix that will not decode makes the numbers untrustworthy too. Dropping + /// the message lets the sender's retransmit try again, rather than seating a + /// vote whose nacks and offered bodies cannot be placed against an op. + fn decode_peer_suffix( + &self, + header: &DoViewChangeHeader, + suffix_body: &[u8], + ) -> Option { + match dvc_suffix_decode( + suffix_body, + header.op, + header.nack_bitset, + header.present_bitset, + ) { + Ok(suffix) => Some(suffix), + Err(error) => { + tracing::warn!( + replica = self.replica, + from_replica = header.replica, + view = header.view, + op = header.op, + "dropping do_view_change with an unreadable suffix: {error}" + ); + None + } + } + } + + /// `suffix_body` is the sender's uncommitted-suffix headers. Empty from a peer + /// unable to snapshot one, which then contributes numbers only. + /// /// # Panics - /// If `header.namespace` does not match this replica's namespace. + /// If `header.group` does not match this replica's namespace. pub fn handle_do_view_change( &self, plane: PlaneKind, header: &DoViewChangeHeader, + suffix_body: &[u8], ) -> Vec { - assert_eq!( - header.namespace, self.namespace, - "DVC routed to wrong group" - ); + assert_eq!(header.group, self.group, "DVC routed to wrong group"); // Quorum-invisible while recovering (see handle_start_view_change): // a recovering replica must not collect DVCs and crown itself. if self.status.get() == Status::Recovering { @@ -2256,6 +2458,9 @@ impl> VsrConsensus { let msg_log_view = header.log_view; let msg_op = header.op; let msg_commit = header.commit; + let Some(msg_suffix) = self.decode_peer_suffix(header, suffix_body) else { + return Vec::new(); + }; // Ignore DVCs for old views if msg_view < self.view.get() { @@ -2297,7 +2502,7 @@ impl> VsrConsensus { // Send our own SVC let action = VsrAction::SendStartViewChange { view: msg_view, - namespace: self.namespace, + group: self.group, }; emit_sim_event( SimEventKind::ControlMessageScheduled, @@ -2332,6 +2537,7 @@ impl> VsrConsensus { log_view: self.log_view.get(), op: current_op, commit, + suffix: self.local_dvc_suffix(), }; dvc_record( &mut self.do_view_change_from_all_replicas.borrow_mut(), @@ -2345,14 +2551,16 @@ impl> VsrConsensus { log_view: msg_log_view, op: msg_op, commit: msg_commit, + suffix: msg_suffix, }; dvc_record(&mut self.do_view_change_from_all_replicas.borrow_mut(), dvc); - // Check if quorum achieved + // `complete_view_change_as_primary` latches only once the merge decides, + // so an undecidable quorum re-merges as each further DVC lands. if !self.do_view_change_quorum.get() - && dvc_count(&self.do_view_change_from_all_replicas.borrow()) >= self.quorum() + && dvc_count(&self.do_view_change_from_all_replicas.borrow()) + >= self.quorum_view_change() { - self.do_view_change_quorum.set(true); actions.extend(self.complete_view_change_as_primary(plane)); } @@ -2373,7 +2581,7 @@ impl> VsrConsensus { pub fn begin_view_probe(&self) { tracing::info!( replica = self.replica, - namespace_raw = self.namespace, + namespace_raw = self.group, "beginning view probe" ); self.status.set(Status::Recovering); @@ -2423,7 +2631,7 @@ impl> VsrConsensus { ); tracing::info!( replica = self.replica, - namespace_raw = self.namespace, + namespace_raw = self.group, ?from, ?to, "state transfer stage" @@ -2449,7 +2657,7 @@ impl> VsrConsensus { header: &RequestStartViewHeader, ) -> Vec { assert_eq!( - header.namespace, self.namespace, + header.group, self.group, "RequestStartView routed to wrong group" ); if self.status.get() != Status::Normal { @@ -2490,7 +2698,10 @@ impl> VsrConsensus { commit: self.commit_max.get(), incarnation: header.incarnation, target: Some(header.replica), - namespace: self.namespace, + // A probe answer reports this primary's settled frontier, not a + // freshly merged log, so there is no canonical suffix to publish. + suffix: Vec::new(), + group: self.group, }] } @@ -2519,6 +2730,44 @@ impl> VsrConsensus { .stop(TimeoutKind::RequestStartViewMessage); } + /// Decide which head to adopt from a `StartView`, and record the view's + /// canonical headers when it carried any. + /// + /// Headers go in `pending_view_log`, not the journal: a journal entry is a + /// header plus its body, and a backup adopting a view usually holds neither. + /// Keeping them lets the repair ingest reject a body that disagrees with what + /// the view decided, which is what makes fetching by op number safe. + /// + /// Falls back to the announced `op` on an empty body (probe answer, stale-view + /// correction). + fn adopt_start_view_suffix(&self, header: &StartViewHeader, suffix_body: &[u8]) -> u64 { + let suffix = match dvc_suffix_decode(suffix_body, header.op, 0, 0) { + Ok(suffix) => suffix, + Err(error) => { + tracing::warn!( + replica = self.replica, + from_replica = header.replica, + view = header.view, + op = header.op, + "start_view suffix did not decode, falling back to the announced op: {error}" + ); + return header.op; + } + }; + let headers = suffix.headers(); + if headers.is_empty() { + return header.op; + } + + *self.pending_view_log.borrow_mut() = Some(MergedLog { + op_head: header.op, + commit_max: header.commit, + headers: headers.to_vec(), + committed_elsewhere: Vec::new(), + }); + header.op + } + /// Handle a received `StartView` message (backups only). /// /// "When other replicas receive the STARTVIEW message, they replace their log @@ -2527,7 +2776,7 @@ impl> VsrConsensus { /// their status to normal, and send `PrepareOK` for any uncommitted ops." /// /// # Panics - /// If `header.namespace` does not match this replica's namespace. + /// If `header.group` does not match this replica's namespace. /// # Client-table maintenance /// /// Backups maintain the client-table during normal operation via @@ -2537,8 +2786,15 @@ impl> VsrConsensus { /// /// Gap: if a backup never received a prepare (lost message), /// `commit_journal` stops at the gap. Requires message repair. - pub fn handle_start_view(&self, plane: PlaneKind, header: &StartViewHeader) -> Vec { - assert_eq!(header.namespace, self.namespace, "SV routed to wrong group"); + /// `suffix_body` is the message body: the view's canonical headers, empty + /// when the announcement carries numbers only. + pub fn handle_start_view( + &self, + plane: PlaneKind, + header: &StartViewHeader, + suffix_body: &[u8], + ) -> Vec { + assert_eq!(header.group, self.group, "SV routed to wrong group"); let from_replica = header.replica; let msg_view = header.view; let msg_op = header.op; @@ -2563,7 +2819,7 @@ impl> VsrConsensus { // incarnation is set (partition plane, tests). // // A zero `header.incarnation` makes no claim either way: it is what an - // unsolicited StartView carries, and what a peer predating the field sends. + // unsolicited StartView carries. // Classifying it stale would have this replica reject a current StartView // from a healthy primary purely because that primary is older, so it falls // through to the view checks that governed before the field existed. @@ -2640,11 +2896,11 @@ impl> VsrConsensus { // Stale pipeline entries from the old view must be discarded self.pipeline.borrow_mut().clear(); - // TODO: StartView should carry uncommitted headers so backup installs - // into WAL and sets op WAL-verified. Today we trust msg_op, correct - // for truncation (sequencer > msg_op) but wrong when behind - // (sequencer < msg_op): gap is unreachable without message repair. - self.sequencer.set_sequence(msg_op); + // Cross-check the announced head against the headers published with it: a + // suffix head disagreeing with `header.op` means an inconsistently built + // frame, and either value leaves this replica chasing an unservable head. + let announced = self.adopt_start_view_suffix(header, suffix_body); + self.sequencer.set_sequence(announced); // Update timeouts for normal backup operation { @@ -2676,7 +2932,7 @@ impl> VsrConsensus { from_op: msg_commit + 1, to_op: msg_op, target: from_replica, - namespace: self.namespace, + group: self.group, }; emit_sim_event( SimEventKind::ControlMessageScheduled, @@ -2700,12 +2956,9 @@ impl> VsrConsensus { /// to prevent old/replayed messages from suppressing view changes. /// /// # Panics - /// If `header.namespace` does not match this replica's namespace. + /// If `header.group` does not match this replica's namespace. pub fn handle_commit(&self, header: &iggy_binary_protocol::CommitHeader) -> CommitOutcome { - assert_eq!( - header.namespace, self.namespace, - "Commit routed to wrong group" - ); + assert_eq!(header.group, self.group, "Commit routed to wrong group"); if self.is_primary() { // A heartbeat from the primary of an OLDER view means that @@ -2783,22 +3036,83 @@ impl> VsrConsensus { /// contains entries for all committed ops it received. /// /// Gap: missing prepares (lost messages) require message repair. + /// + /// Re-entrant, called again for every `DoViewChange` landing while the merge is + /// undecided. Every non-`Ready` outcome leaves this replica untouched, so a + /// re-run costs only the merge. fn complete_view_change_as_primary(&self, plane: PlaneKind) -> Vec { - let dvc_array = self.do_view_change_from_all_replicas.borrow(); + let merged = { + let dvc_array = self.do_view_change_from_all_replicas.borrow(); + merge_dvc_quorum(&dvc_array, self.merge_quorums()) + }; - let Some(winner) = dvc_select_winner(&dvc_array) else { - return Vec::new(); + let merged = match merged { + MergeOutcome::Ready(merged) => merged, + // Every non-ready outcome keeps this replica in `ViewChange` with its + // log untouched. Picking a winner unconditionally and letting the + // pipeline rebuild truncate what it cannot find locally discards + // committed ops; an unavailable cluster that says so is the better + // failure. + // + // None of these latch `do_view_change_quorum`: an undecidable quorum is + // not a decision, and the replicas still to report are what would + // settle it. The flag belongs only where the quorum is decidable. + MergeOutcome::AwaitingQuorum => return Vec::new(), + MergeOutcome::AwaitingRepair { undecided_op } => { + tracing::debug!( + replica = self.replica, + view = self.view.get(), + undecided_op, + "view change waiting on more DoViewChange messages to decide an op" + ); + return Vec::new(); + } + MergeOutcome::Deadlocked { undecided_op } => { + tracing::error!( + replica = self.replica, + view = self.view.get(), + undecided_op, + "view change cannot start: op {undecided_op} is neither recoverable from any \ + replica nor provably uncommitted" + ); + return Vec::new(); + } }; - let new_op = winner.op; - let max_commit = dvc_max_commit(&dvc_array); + // The pipeline must hold the whole uncommitted range, and the merge decides + // that range against a cluster-wide ceiling, so a node configured shallower + // than its peers can be handed a range it cannot rebuild. + // + // Refuse rather than panic: a further DoViewChange can raise `commit_max` + // and shrink the range, and otherwise the status timeout escalates. A panic + // would restart into the same merge. + if merged.op_head.saturating_sub(merged.commit_max) > self.prepare_queue_max as u64 { + tracing::error!( + replica = self.replica, + view = self.view.get(), + commit_max = merged.commit_max, + op_head = merged.op_head, + prepare_queue_max = self.prepare_queue_max, + "view change cannot start: the merged log claims {} in-flight ops, more than this \ + replica's pipeline holds; refusing the view", + merged.op_head - merged.commit_max, + ); + return Vec::new(); + } - // Update state - self.log_view.set(self.view.get()); - self.status.set(Status::Normal); - self.ceded_primaryship.set(false); + // Quorum closed now the merge decided: re-merging after parking could + // produce a different log than the one already being repaired toward. + self.do_view_change_quorum.set(true); + + // The merged log is authoritative but this replica may not hold every body + // yet. Park it, let the shard repair up to it, and `start_pending_view` + // finishes once the journal covers the range. Until then this replica stays + // in `ViewChange` and prepares nothing, so no client op is stamped onto an + // unproven log. `log_view` does NOT advance here; see `start_pending_view`. + let max_commit = merged.commit_max; + let new_op = merged.op_head; self.advance_commit_max(max_commit); - self.sequencer.set_sequence(new_op); + *self.pending_view_log.borrow_mut() = Some(merged); // Stale pipeline entries are invalid in new view; reconciliation // replays from journal. @@ -2818,6 +3132,137 @@ impl> VsrConsensus { // the loopback queue directly. self.loopback_queue.borrow_mut().clear(); + tracing::info!( + replica = self.replica, + view = self.view.get(), + op_head = new_op, + commit_max = max_commit, + "view-change quorum merged; repairing up to the merged log before starting the view" + ); + emit_replica_event( + SimEventKind::ReplicaStateChanged, + &ReplicaLogContext::from_consensus(self, plane), + ); + + // No sends yet: `SendStartView` promises this replica can serve every op in + // the merged log, and a backup adopting the announced head asks it for the + // bodies behind it. + Vec::new() + } + + /// Sizes handed to the DVC merge. + const fn merge_quorums(&self) -> MergeQuorums { + MergeQuorums { + view_change: self.quorum_view_change(), + nack_prepare: self.quorum_nack_prepare(), + replica_count: self.replica_count as usize, + // The cluster-wide ceiling, not `self.prepare_queue_max`: this node's + // config says nothing about how deep a peer's pipeline is. + prepare_queue_ceiling: PREPARE_QUEUE_CEILING as u64, + } + } + + /// The merged log this replica is repairing toward, if a view change is + /// mid-transition. The shard reads it for the op range it must cover before the + /// view can start, and for which peers offered the bodies. + /// + /// Clones two `Vec`. Prefer [`Self::view_log_is_pending`] / + /// [`Self::with_pending_view_log`]; clone only to hold it across an `.await` or + /// across [`Self::start_pending_view`], which takes the cell. + #[must_use] + pub fn pending_view_log(&self) -> Option { + self.pending_view_log.borrow().clone() + } + + /// Whether a merge is parked, without cloning it. + #[must_use] + pub fn view_log_is_pending(&self) -> bool { + self.pending_view_log.borrow().is_some() + } + + /// Read the parked merge in place. The closure must not re-enter consensus: the + /// `RefCell` stays borrowed for its whole body. + pub fn with_pending_view_log(&self, read: impl FnOnce(&MergedLog) -> T) -> Option { + self.pending_view_log.borrow().as_ref().map(read) + } + + /// Replicas that offered a body for `op`, most-recent-log_view first. + /// + /// Only meaningful while a merge is parked. These peers and nobody else: a + /// cleared present bit means the body was never held or cannot be read back, + /// and the view change is blocked on the round-trip. + #[must_use] + pub fn pending_view_body_sources(&self, op: u64) -> Vec { + let quorum = self.do_view_change_from_all_replicas.borrow(); + let mut sources: Vec<(u32, u8)> = dvc_iter(&quorum) + .filter(|dvc| dvc.replica != self.replica) + .filter_map(|dvc| { + let index = dvc.suffix.index_of(dvc.op, op)?; + dvc.suffix + .offers_body(index) + .then_some((dvc.log_view, dvc.replica)) + }) + .collect(); + sources.sort_unstable_by_key(|(log_view, _)| std::cmp::Reverse(*log_view)); + sources.into_iter().map(|(_, replica)| replica).collect() + } + + /// Finish the parked view change: this replica's journal now covers the merged + /// log, so it can serve any op it is about to announce. + /// + /// Called by the shard after repair progress. No-op when nothing is parked. + /// + /// # Panics + /// If the merged uncommitted range exceeds pipeline capacity, which needs a head + /// more than one pipeline depth above the proven commit point. + pub fn start_pending_view(&self, plane: PlaneKind) -> Vec { + // A backup's parked log (the `StartView` suffix) is only what its ingest + // verifies bodies against. It must never take this path: starting the view + // claims the primaryship of a view this replica did not win. + if !self.is_primary_for_view(self.view.get()) { + return Vec::new(); + } + let Some(merged) = self.pending_view_log.borrow_mut().take() else { + return Vec::new(); + }; + let new_op = merged.op_head; + let max_commit = merged.commit_max; + + // The one view-change exit that skips `reset_view_change_state`, so the DVCs + // (a suffix `Vec` per sender, per group led) would be held for the whole + // primaryship. Nothing reads the array after the view starts: a late + // same-view DVC returns at the status gate, a higher-view one resets first. + // + // `dvc_reset`, not `reset_dvc_quorum`: the latter also clears the + // `do_view_change_quorum` latch, which from here means "log decided" and is + // what stops `handle_do_view_change_timeout` retransmitting. + dvc_reset(&mut self.do_view_change_from_all_replicas.borrow_mut()); + self.invalidate_local_dvc_suffix(); + + self.status.set(Status::Normal); + self.ceded_primaryship.set(false); + self.sequencer.set_sequence(new_op); + if let Some(head) = merged.headers.first() { + // Keep the hash chain continuous: the next prepare must chain onto the + // head this view adopted, not onto whatever was appended last. + self.set_last_prepare_checksum(head.checksum); + } + for header in &merged.headers { + self.observe_prepare_timestamp(header.timestamp); + } + // Only now, with the merged head installed above. `log_view` claims "my log + // IS the log this view decided", and it selects the canonical senders of the + // next view change, whose headers outrank everyone else's. + // + // Raising it at merge time breaks that claim for the whole parked window, + // which can end in supersession or a crash (`log_view` is durable): the + // replica then votes as canonical carrying its own stale head, and ops the + // merge decided to keep fall outside the next scan range, discarded with no + // nack required. Merge-time assignment is only truthful where every merged + // header is installed there; parking installs nothing and the repair ingest + // only fills holes, so a parked replica still holds its old view's log. + self.log_view.set(self.view.get()); + // Update timeouts for normal primary operation { let mut timeouts = self.timeouts.borrow_mut(); @@ -2846,7 +3291,9 @@ impl> VsrConsensus { commit: max_commit, incarnation: 0, target: None, - namespace: self.namespace, + group: self.group, + // `merged` was taken out of the parked slot, so hand the headers over. + suffix: merged.headers, }; emit_sim_event( SimEventKind::ControlMessageScheduled, @@ -2864,10 +3311,12 @@ impl> VsrConsensus { // The new primary must rebuild its pipeline from the journal so that // incoming PrepareOk messages can be matched and commits can proceed. if max_commit < new_op { - assert!( + // `complete_view_change_as_primary` already refused a non-fitting + // range. Asserted so the sites cannot drift; this one cannot decline. + debug_assert!( (new_op - max_commit) <= self.prepare_queue_max as u64, "view change: uncommitted range {}..={} ({} ops) exceeds pipeline capacity ({}); \ - DVC winner claims more in-flight ops than the pipeline can hold", + the merged log claims more in-flight ops than the pipeline can hold", max_commit + 1, new_op, new_op - max_commit, @@ -2948,7 +3397,7 @@ impl> VsrConsensus { // Record the ack from this replica let ack_count = entry.add_ack(header.replica); - let quorum = self.quorum(); + let quorum = self.quorum_replication(); let quorum_reached = ack_count >= quorum && !entry.ok_quorum_received; // Check if we've reached quorum @@ -3025,7 +3474,7 @@ impl> VsrConsensus { } } -impl Project, VsrConsensus> for Message +impl Project, VsrConsensus> for Message where B: MessageBus, P: Pipeline, @@ -3048,12 +3497,8 @@ where // stores the same value and the scan verifies it after a crash. The body is // never re-stamped (`restamp_prepare_view` patches only `view`), so this // survives view-change retransmits. The header `checksum` and its `parent` - // chain stay `0`: activating them needs the retransmit path to re-seal a - // re-stamped header, a separate change. Whoever activates it must also - // audit every `set_last_prepare_checksum` caller for cross-plane carry -- - // the repair router in `shard` drops metadata-plane frames it cannot - // journal precisely so one cannot stamp a PARTITION consensus, which is - // inert only while these values are structurally zero. + // chain are sealed too, for both planes, by `seal_prepare_checksum` below; + // they exclude `view`, which is what lets a restamp leave them valid. // // Metadata plane only. A partition produce prepare already carries a verified // `batch_checksum` over the same bytes, so a second full-payload pass is pure @@ -3062,15 +3507,35 @@ where // sealed region before the entry is journaled. Leaving those prepares at `0` // is the designed "nothing to verify" sentinel, so a future durable partition // journal skips verification instead of failing every entry as corrupt. - let checksum_body = if consensus.namespace == METADATA_CONSENSUS_NAMESPACE { - u128::from(calculate_checksum( - &self.as_slice()[size_of::()..], - )) + // + // TODO(consensus): a partition prepare's `checksum` covers its header alone, + // so two at one op with matching header fields are indistinguishable however + // far their batch bytes diverge. The merge then counts a divergent replica as + // holding a servable copy, and the partition repair ingest (no merged-log + // identity gate) short-circuits `verify_prepare_integrity`'s body branch on + // the zero. Two closures, both larger than they look: + // + // 1. The batch checksum, recomputed after `stamp_prepare_for_persistence`. + // But stamping runs per replica after replication and folds `base_offset` + // in, so identity would change at stamp time and the journaled entry would + // no longer match the pipeline entry `handle_prepare_ok` compares. + // 2. The stamp-invariant cover: everything past the 256-byte command header, + // which stamping never touches. Identical on every replica, safe to seal + // here, but costs a produce-path pass and retires the "0 means nothing to + // verify" sentinel that lets an existing WAL replay. + // + // Bounded by `size`, the range every verifier re-reads; the prepare + // inherits it verbatim below. + let checksum_body = if consensus.group == METADATA_GROUP { + u128::from(calculate_checksum(frame_body( + self.as_slice(), + self.header().size, + ))) } else { 0 }; - self.transmute_header(|old, new| { + let prepared = self.transmute_header(|old, new| { *new = PrepareHeader { cluster: consensus.cluster, size: old.size, @@ -3086,12 +3551,12 @@ where op, timestamp, operation: old.operation, - // The GROUP's namespace, never the request's: a client - // RequestHeader carries namespace 0, and journaling that - // would make the stored prepare route to the wrong plane - // when repair later ships it verbatim (live replication - // masked this; repair replay is what broke). - namespace: consensus.namespace, + // The GROUP's own id, never the request's: a routed request + // header can carry group 0, and journaling that would make + // the stored prepare route to the wrong plane when repair + // later ships it verbatim (live replication masked this; + // repair replay is what broke). + group: consensus.group, checksum_body, // Copied verbatim: carries the stamped acting user for client // ops (and the authenticated user on Register), so the in-apply @@ -3099,7 +3564,11 @@ where user_id: old.user_id, ..Default::default() } - }) + }); + // Last, because the checksum covers every other field. Gives the op the + // stable identity the view-change merge compares across replicas; `parent` + // chains it, so the log is hash-linked rather than nominally so. + seal_prepare_checksum(prepared) } } @@ -3126,12 +3595,13 @@ where commit: consensus.commit_max.get(), timestamp: old.timestamp, operation: old.operation, - namespace: old.namespace, + group: old.group, // PrepareOk is header-only; the frame is exactly the header, so // `size` is the header size. size: std::mem::size_of::() as u32, ..Default::default() }; + new.seal(); }) } } @@ -3144,7 +3614,7 @@ where type MessageBus = B; #[rustfmt::skip] // Scuffed formatter. TODO: Make the naming less ambiguous for `Message`. type Message = Message where H: ConsensusHeader; - type RequestHeader = RequestHeader; + type RoutedRequestHeader = RoutedRequestHeader; type ReplicateHeader = PrepareHeader; type AckHeader = PrepareOkHeader; @@ -3182,20 +3652,20 @@ mod request_queue_tests { use super::*; use iggy_binary_protocol::{Command2, Operation}; - fn make_request(client: u128, request_num: u64) -> Message { - let header_size = std::mem::size_of::(); - let mut msg = Message::::new(header_size); - let header = bytemuck::checked::try_from_bytes_mut::( + fn make_request(client: u128, request_num: u64) -> Message { + let header_size = std::mem::size_of::(); + let mut msg = Message::::new(header_size); + let header = bytemuck::checked::try_from_bytes_mut::( &mut msg.as_mut_slice()[..header_size], ) .expect("zeroed bytes are valid"); - *header = RequestHeader { + *header = RoutedRequestHeader { command: Command2::Request, client, session: 1, request: request_num, operation: Operation::SendMessages, - ..RequestHeader::default() + ..RoutedRequestHeader::default() }; msg } @@ -3500,7 +3970,7 @@ mod timestamp_clamp_tests { 1, 0, 1, - METADATA_CONSENSUS_NAMESPACE, + METADATA_GROUP, NoopBus, LocalPipeline::new(), lagging_clock, @@ -3532,7 +4002,7 @@ mod timestamp_clamp_tests { 1, 0, 1, - METADATA_CONSENSUS_NAMESPACE, + METADATA_GROUP, NoopBus, LocalPipeline::new(), leading_clock, @@ -3565,7 +4035,7 @@ mod timestamp_clamp_tests { header.commit = op; header.replica = replica; header.incarnation = incarnation; - header.namespace = METADATA_CONSENSUS_NAMESPACE; + header.group = METADATA_GROUP; header.size = size as u32; msg } @@ -3586,7 +4056,7 @@ mod timestamp_clamp_tests { 1, 0, 3, - METADATA_CONSENSUS_NAMESPACE, + METADATA_GROUP, NoopBus, LocalPipeline::new(), ConsensusClock::system(), @@ -3600,7 +4070,7 @@ mod timestamp_clamp_tests { let stale = make_start_view(1, 4, 1, STALE); assert!( consensus - .handle_start_view(PlaneKind::Metadata, stale.header()) + .handle_start_view(PlaneKind::Metadata, stale.header(), &[]) .is_empty(), "a StartView echoing a previous incarnation must be ignored while recovering" ); @@ -3620,7 +4090,7 @@ mod timestamp_clamp_tests { let fresh = make_start_view(1, 4, 1, CURRENT); assert!( !consensus - .handle_start_view(PlaneKind::Metadata, fresh.header()) + .handle_start_view(PlaneKind::Metadata, fresh.header(), &[]) .is_empty(), "a StartView echoing our current incarnation must be adopted" ); @@ -3652,14 +4122,14 @@ mod timestamp_clamp_tests { LocalPipeline::new(), ConsensusClock::new(Rc::new(FixedClock(100_000))), ); - let header_size = size_of::(); + let header_size = size_of::(); let body = b"produce payload"; - let mut msg = Message::::new(header_size + body.len()); + let mut msg = Message::::new(header_size + body.len()); msg.as_mut_slice()[header_size..].copy_from_slice(body); - let header = bytemuck::checked::try_from_bytes_mut::( + let header = bytemuck::checked::try_from_bytes_mut::( &mut msg.as_mut_slice()[..header_size], ) - .expect("zeroed bytes are a valid RequestHeader"); + .expect("zeroed bytes are a valid RoutedRequestHeader"); header.command = Command2::Request; header.client = 1; header.request = 1; @@ -3669,7 +4139,7 @@ mod timestamp_clamp_tests { }; assert_ne!( - seal(METADATA_CONSENSUS_NAMESPACE), + seal(METADATA_GROUP), 0, "a metadata prepare must be sealed: the WAL scan verifies it after a crash" ); @@ -3698,7 +4168,7 @@ mod timestamp_clamp_tests { 1, 0, 3, - METADATA_CONSENSUS_NAMESPACE, + METADATA_GROUP, NoopBus, LocalPipeline::new(), ConsensusClock::system(), @@ -3712,7 +4182,11 @@ mod timestamp_clamp_tests { // head covers every op it told us was committed. assert!( consensus - .handle_start_view(PlaneKind::Metadata, make_start_view(7, 104, 1, 0).header()) + .handle_start_view( + PlaneKind::Metadata, + make_start_view(7, 104, 1, 0).header(), + &[] + ) .is_empty(), "an equal-view StartView below the commit floor must be skipped" ); @@ -3726,7 +4200,11 @@ mod timestamp_clamp_tests { // Adopt it and drop the discarded suffix. assert!( !consensus - .handle_start_view(PlaneKind::Metadata, make_start_view(7, 105, 1, 0).header()) + .handle_start_view( + PlaneKind::Metadata, + make_start_view(7, 105, 1, 0).header(), + &[] + ) .is_empty(), "an equal-view StartView at or above the commit floor must be adopted, \ even when its head is behind a WAL suffix the view already discarded" @@ -3746,14 +4224,8 @@ mod timestamp_clamp_tests { /// pins the predicate the dispatch sites and the debug tripwire both read. #[test] fn given_view_change_when_needs_superblock_persist_should_track_durability() { - let mut consensus = VsrConsensus::new( - 1, - 0, - 3, - METADATA_CONSENSUS_NAMESPACE, - NoopBus, - LocalPipeline::new(), - ); + let mut consensus = + VsrConsensus::new(1, 0, 3, METADATA_GROUP, NoopBus, LocalPipeline::new()); assert!( !consensus.needs_superblock_persist(), "fresh replica: view == view_durable == 0" @@ -3994,3 +4466,82 @@ mod state_transfer_stage_tests { assert_eq!(consensus.status(), Status::Recovering); } } + +#[cfg(test)] +mod quorum_tests { + //! Pin the three quorum sizes for replica counts 1 through 8. The + //! intersection asserts are the safety properties: replication and + //! view-change quorums must overlap, so a committed op is visible to the + //! next view, and replication and nack quorums must overlap, so an op that + //! may have committed can never gather a nack quorum. + + use super::*; + use crate::LocalPipeline; + use server_common::MESSAGE_ALIGN; + use server_common::iobuf::Frozen; + + struct NoopBus; + + impl MessageBus for NoopBus { + async fn send_to_client( + &self, + _client_id: u128, + _data: Frozen, + ) -> Result<(), message_bus::SendError> { + Ok(()) + } + + async fn send_to_replica( + &self, + _replica: u8, + _data: Frozen, + ) -> Result<(), message_bus::SendError> { + Ok(()) + } + + fn set_connection_lost_fn(&self, _f: message_bus::ConnectionLostFn) {} + fn set_replica_forward_fn(&self, _f: message_bus::ReplicaForwardFn) {} + fn set_client_forward_fn(&self, _f: message_bus::ClientForwardFn) {} + fn track_background(&self, _handle: message_bus::JoinHandle<()>) {} + } + + fn consensus_with_replica_count(replica_count: u8) -> VsrConsensus { + VsrConsensus::new( + 1, + 0, + replica_count, + METADATA_GROUP, + NoopBus, + LocalPipeline::new(), + ) + } + + #[test] + fn given_any_replica_count_when_sizing_quorums_should_intersect() { + for replica_count in 1u8..=REPLICAS_MAX_U8 { + let consensus = consensus_with_replica_count(replica_count); + let count = usize::from(replica_count); + + assert!( + consensus.quorum_replication() + consensus.quorum_view_change() > count, + "replication+view-change must intersect at replica_count={replica_count}" + ); + assert!( + consensus.quorum_nack_prepare() + consensus.quorum_replication() > count, + "nack+replication must intersect at replica_count={replica_count}" + ); + assert!(consensus.quorum_replication() <= count); + assert!(consensus.quorum_view_change() <= count); + assert!(consensus.quorum_nack_prepare() <= count); + } + } + + /// `REPLICAS_MAX` as a `u8` for loop bounds. + const REPLICAS_MAX_U8: u8 = { + assert!(REPLICAS_MAX <= u8::MAX as usize); + #[allow(clippy::cast_possible_truncation)] + { + REPLICAS_MAX as u8 + } + }; +} diff --git a/core/consensus/src/lib.rs b/core/consensus/src/lib.rs index 4d7a53d53e..02cca677b0 100644 --- a/core/consensus/src/lib.rs +++ b/core/consensus/src/lib.rs @@ -27,7 +27,7 @@ pub trait Project { pub trait Pipeline { type Entry; /// Accepted-but-not-yet-prepared client request. For `LocalPipeline`, - /// `RequestEntry` wrapping `Message`. + /// `RequestEntry` wrapping `Message`. type Request; fn push(&mut self, entry: Self::Entry); @@ -93,7 +93,7 @@ pub trait Pipeline { } } -pub type RequestMessage = ::Message<::RequestHeader>; +pub type RequestMessage = ::Message<::RoutedRequestHeader>; pub type ReplicateMessage = ::Message<::ReplicateHeader>; pub type AckMessage = ::Message<::AckHeader>; @@ -102,7 +102,7 @@ pub trait Consensus: Sized { #[rustfmt::skip] // Scuffed formatter. type Message: ConsensusMessage where H: ConsensusHeader; - type RequestHeader: ConsensusHeader; + type RoutedRequestHeader: ConsensusHeader; type ReplicateHeader: ConsensusHeader; type AckHeader: ConsensusHeader; @@ -180,6 +180,9 @@ pub use observability::*; mod view_change_quorum; pub use view_change_quorum::*; + +mod dvc_merge; +pub use dvc_merge::*; mod vsr_state; pub use vsr_state::{VsrState, VsrStateError}; mod vsr_timeout; diff --git a/core/consensus/src/observability.rs b/core/consensus/src/observability.rs index 4cd041bc4f..e5c58326bb 100644 --- a/core/consensus/src/observability.rs +++ b/core/consensus/src/observability.rs @@ -249,7 +249,7 @@ impl ReplicaLogContext { plane, cluster_id: consensus.cluster(), replica_id: consensus.replica(), - namespace: NamespaceLogContext::from_raw(plane, consensus.namespace()), + namespace: NamespaceLogContext::from_raw(plane, consensus.group()), view: consensus.view(), log_view: consensus.log_view(), commit: consensus.commit_max(), diff --git a/core/consensus/src/plane_helpers.rs b/core/consensus/src/plane_helpers.rs index 4515f2ba7d..87f3b65e6e 100644 --- a/core/consensus/src/plane_helpers.rs +++ b/core/consensus/src/plane_helpers.rs @@ -19,10 +19,71 @@ use crate::{ Consensus, IgnoreReason, Pipeline, PipelineEntry, PlaneKind, PrepareOkOutcome, Sequencer, Status, VsrConsensus, }; -use iggy_binary_protocol::{Command2, PrepareHeader, PrepareOkHeader, ReplyHeader, RequestHeader}; +use iggy_binary_protocol::{ + CHECKSUM_UNSEALED, Command2, ConsensusHeader, GenericHeader, PrepareHeader, PrepareOkHeader, + ReplyHeader, RoutedRequestHeader, frame_body, +}; use message_bus::{MessageBus, SendError}; -use server_common::{Message, iobuf::Owned}; -use std::ops::AsyncFnOnce; +use server_common::{ + MESSAGE_ALIGN, Message, + iobuf::{Frozen, Owned}, +}; +use std::{error::Error, fmt, mem::size_of, ops::AsyncFnOnce}; + +/// Failure to route or forward a prepare through the replication chain. +#[derive(Debug)] +#[non_exhaustive] +pub enum ChainReplicationError { + MalformedPrepare, + UnexpectedCommand { command: Command2 }, + CommittedPrepare { op: u64, commit_min: u64 }, + SelfRoute { replica: u8 }, + Transport(SendError), +} + +impl ChainReplicationError { + #[must_use] + pub const fn is_transport(&self) -> bool { + matches!(self, Self::Transport(_)) + } +} + +impl fmt::Display for ChainReplicationError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::MalformedPrepare => formatter.write_str("malformed prepare frame"), + Self::UnexpectedCommand { command } => { + write!(formatter, "expected prepare command, found {command:?}") + } + Self::CommittedPrepare { op, commit_min } => write!( + formatter, + "prepare op {op} is not above committed op {commit_min}" + ), + Self::SelfRoute { replica } => { + write!( + formatter, + "replication chain routes replica {replica} to itself" + ) + } + Self::Transport(error) => error.fmt(formatter), + } + } +} + +impl Error for ChainReplicationError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Transport(error) => Some(error), + _ => None, + } + } +} + +impl From for ChainReplicationError { + fn from(error: SendError) -> Self { + Self::Transport(error) + } +} /// Shared pipeline-first request flow (metadata + partitions). /// @@ -75,43 +136,166 @@ where /// /// # Errors /// -/// Returns `SendError` if the bus fails to deliver to the next replica. +/// Returns an error if the prepare cannot be routed or the bus cannot deliver +/// it to the next replica. /// Callers decide error policy (VSR retransmits from WAL via prepare timeout). -/// -/// # Panics -/// - If `header.command` is not `Command2::Prepare`. -/// - If `header.op <= consensus.commit_min()`. -/// - If the computed next replica equals self. #[allow(clippy::future_not_send)] pub async fn replicate_to_next_in_chain( consensus: &VsrConsensus, message: &Message, -) -> Result<(), SendError> +) -> Result<(), ChainReplicationError> where B: MessageBus, P: Pipeline, { - let header = *message.header(); + let Some(next) = replication_target(consensus, message.header())? else { + return Ok(()); + }; + let frozen = message.deep_copy().into_generic().into_frozen(); + consensus + .message_bus() + .send_to_replica(next, frozen) + .await + .map_err(Into::into) +} - assert_eq!(header.command, Command2::Prepare); - assert!(header.op > consensus.commit_min()); +/// Forward an already validated frozen prepare to the next replica without +/// copying its payload. +/// +/// # Errors +/// +/// Returns an error if the frame is malformed, cannot be routed, or the bus +/// cannot deliver it to the next replica. +#[allow(clippy::future_not_send)] +pub async fn replicate_frozen_to_next_in_chain( + consensus: &VsrConsensus, + message: Frozen, +) -> Result<(), ChainReplicationError> +where + B: MessageBus, + P: Pipeline, +{ + let header = frozen_prepare_header(&message)?; + let Some(next) = replication_target(consensus, &header)? else { + return Ok(()); + }; + consensus + .message_bus() + .send_to_replica(next, message) + .await + .map_err(Into::into) +} + +fn frozen_prepare_header( + message: &Frozen, +) -> Result { + let header_bytes = message + .as_slice() + .get(..size_of::()) + .ok_or(ChainReplicationError::MalformedPrepare)?; + let header = bytemuck::checked::try_from_bytes::(header_bytes) + .copied() + .map_err(|_| ChainReplicationError::MalformedPrepare)?; + header + .validate() + .map_err(|_| ChainReplicationError::MalformedPrepare)?; + let frame_size = + usize::try_from(header.size).map_err(|_| ChainReplicationError::MalformedPrepare)?; + if !(size_of::()..=message.len()).contains(&frame_size) { + return Err(ChainReplicationError::MalformedPrepare); + } + Ok(header) +} + +fn replication_target( + consensus: &VsrConsensus, + header: &PrepareHeader, +) -> Result, ChainReplicationError> +where + B: MessageBus, + P: Pipeline, +{ + if header.command != Command2::Prepare { + return Err(ChainReplicationError::UnexpectedCommand { + command: header.command, + }); + } + let commit_min = consensus.commit_min(); + if header.op <= commit_min { + return Err(ChainReplicationError::CommittedPrepare { + op: header.op, + commit_min, + }); + } let next = (consensus.replica() + 1) % consensus.replica_count(); let primary = consensus.primary_index(header.view); if next == primary { - return Ok(()); + return Ok(None); + } + + if next == consensus.replica() { + return Err(ChainReplicationError::SelfRoute { + replica: consensus.replica(), + }); } + Ok(Some(next)) +} - assert_ne!(next, consensus.replica()); +/// Re-stamp a stored prepare with the current view before retransmission. +/// The prepare identity excludes `view`, so the payload and operation identity +/// remain unchanged. +#[must_use] +pub fn restamp_prepare_view( + stored: Frozen, + view: u32, +) -> Option> { + const VIEW_OFFSET: usize = std::mem::offset_of!(PrepareHeader, view); + + let header = bytemuck::checked::try_from_bytes::( + stored.as_slice().get(..size_of::())?, + ) + .ok()?; + if header.view == view { + return Some(stored); + } - // Chain replication to the next replica is N=1, so the freeze-once - // trick does not apply: the caller has already appended `message` to - // its local journal (durability-before-ack) and kept a reference for - // this forward, so we deep_copy a fresh Frozen here. Future refactor - // could freeze once and share the backing with the journal path. - let frozen = message.deep_copy().into_generic().into_frozen(); - consensus.message_bus().send_to_replica(next, frozen).await + let mut owned = Owned::::copy_from_slice(stored.as_slice()); + owned.as_mut_slice()[VIEW_OFFSET..VIEW_OFFSET + size_of::()] + .copy_from_slice(&view.to_ne_bytes()); + Message::::try_from(owned) + .ok() + .map(Message::into_frozen) +} + +/// Recompute a prepare's integrity fields and report the first that disagrees. +/// +/// Everywhere else `checksum` is an opaque token: the pipeline, the merge, and the +/// repair ingest compare it for equality without asking whether it describes the +/// bytes it arrived with, so a corrupted frame is admitted whenever its flipped +/// value satisfies those comparisons, then journaled and re-served to peers. +/// +/// `frame` is the whole message. The body range comes from [`frame_body`], not the +/// caller, so no ingress point can verify a different span than the producer sealed. +/// [`CHECKSUM_UNSEALED`] skips the partition plane, which carries `batch_checksum` +/// over the same bytes instead. +/// +/// # Errors +/// Returns a static description of which field failed. +pub fn verify_prepare_integrity(header: &PrepareHeader, frame: &[u8]) -> Result<(), &'static str> { + if header.checksum != CHECKSUM_UNSEALED && header.identity_checksum() != header.checksum { + return Err("prepare header does not match its own checksum"); + } + if header.checksum_body != 0 + && u128::from(iggy_common::calculate_checksum(frame_body( + frame, + header.size, + ))) != header.checksum_body + { + return Err("prepare body does not match its checksum"); + } + Ok(()) } /// Shared preflight checks for `on_replicate`. @@ -169,6 +353,23 @@ where Ok(current_op) } +/// Stamp [`PrepareHeader::identity_checksum`] into a freshly built prepare. +/// +/// Call once, after every other field is final: the checksum covers them. +/// `checksum_body` in particular, since that is how the body reaches the value. +/// +/// # Panics +/// If the message is shorter than its own header. +#[must_use] +pub fn seal_prepare_checksum(mut message: Message) -> Message { + let checksum = message.header().identity_checksum(); + let bytes = &mut message.as_mut_slice()[..size_of::()]; + let header = bytemuck::checked::try_from_bytes_mut::(bytes) + .expect("a prepare header round-trips its own bit pattern"); + header.checksum = checksum; + message +} + /// Shared preflight checks for `on_ack`. /// /// # Errors @@ -349,7 +550,6 @@ where timestamp: prepare_header.timestamp, request: prepare_header.request, operation: prepare_header.operation, - namespace: prepare_header.namespace, ..Default::default() }; // `BytesMut` makes no alignment guarantee, so never cast into it. @@ -384,7 +584,7 @@ where #[must_use] #[allow(clippy::cast_possible_truncation)] pub fn build_result_rejection_reply( - request_header: &RequestHeader, + request_header: &RoutedRequestHeader, commit: u64, code: u32, ) -> Message { @@ -412,7 +612,6 @@ pub fn build_result_rejection_reply( timestamp: request_header.timestamp, request: request_header.request, operation: request_header.operation, - namespace: request_header.namespace, ..Default::default() }; buffer[..header_size].copy_from_slice(bytemuck::bytes_of(&header)); @@ -436,7 +635,7 @@ pub fn build_result_rejection_reply( #[allow(clippy::needless_pass_by_value, clippy::cast_possible_truncation)] pub fn build_reply_from_request( consensus: &VsrConsensus, - request_header: &RequestHeader, + request_header: &RoutedRequestHeader, body: bytes::Bytes, ) -> Message where @@ -466,7 +665,6 @@ where timestamp: request_header.timestamp, request: request_header.request, operation: request_header.operation, - namespace: request_header.namespace, ..Default::default() }; buffer[..header_size].copy_from_slice(bytemuck::bytes_of(&header)); @@ -492,7 +690,7 @@ where /// If the constructed message buffer is not valid. pub fn build_deny_reply_from_request( consensus: &VsrConsensus, - request_header: &RequestHeader, + request_header: &RoutedRequestHeader, status: u32, ) -> Message where @@ -511,7 +709,7 @@ where } /// [`build_deny_reply_from_request`] for layers that hold no consensus group -/// for the request's namespace (a shard fencing a frame aimed at a torn-down +/// for the request's group (a shard fencing a frame aimed at a torn-down /// or never-materialised partition). /// /// Replica-stamped fields (`cluster`, `view`, `replica`) echo the request @@ -524,7 +722,7 @@ where #[must_use] #[allow(clippy::cast_possible_truncation)] pub fn build_deny_reply_from_request_header( - request_header: &RequestHeader, + request_header: &RoutedRequestHeader, status: u32, ) -> Message { let header_size = std::mem::size_of::(); @@ -543,7 +741,6 @@ pub fn build_deny_reply_from_request_header( timestamp: request_header.timestamp, request: request_header.request, operation: request_header.operation, - namespace: request_header.namespace, ..Default::default() }; buffer[..header_size].copy_from_slice(bytemuck::bytes_of(&header)); @@ -620,14 +817,18 @@ pub async fn send_prepare_ok( prepare_checksum: header.checksum, request: header.request, operation: header.operation, - namespace: header.namespace, + group: header.group, size: std::mem::size_of::() as u32, ..Default::default() }; - let message: Message = - Message::::new(std::mem::size_of::()) - .transmute_header(|_, new| *new = prepare_ok_header); + let message: Message = Message::::new(std::mem::size_of::< + PrepareOkHeader, + >()) + .transmute_header(|_, new| { + *new = prepare_ok_header; + new.seal(); + }); let primary = consensus.primary_index(consensus.view()); consensus @@ -639,10 +840,18 @@ pub async fn send_prepare_ok( mod tests { use super::*; use crate::{Consensus, LocalPipeline, VsrAction}; + use aligned_vec::{AVec, ConstAlign}; use iggy_binary_protocol::{ConsensusHeader, Operation, StartViewChangeHeader}; + use iggy_common::calculate_checksum; use message_bus::SendError; use server_common::{MESSAGE_ALIGN, iobuf::Frozen}; + /// `PrepareHeader`'s alignment, which every suffix body has to satisfy. + const BODY_ALIGN: usize = align_of::(); + + /// A control-message body, aligned for the headers packed into it. + type Body = AVec>; + #[derive(Debug, Default)] struct NoopBus; @@ -753,7 +962,7 @@ mod tests { new.size = std::mem::size_of::() as u32; new.view = 1; new.replica = 0; - new.namespace = 0; + new.group = 0; }); let actions = consensus.handle_start_view_change(PlaneKind::Metadata, svc.header()); @@ -793,7 +1002,7 @@ mod tests { new.size = std::mem::size_of::() as u32; new.view = 1; new.replica = 0; - new.namespace = 0; + new.group = 0; }); let actions = consensus.handle_start_view_change(PlaneKind::Metadata, svc.header()); @@ -816,7 +1025,7 @@ mod tests { checksum: 0, checksum_body: 0, cluster: 0, - size: 0, + size: std::mem::size_of::() as u32, view: 1, release: 0, command: Command2::DoViewChange, @@ -824,9 +1033,11 @@ mod tests { reserved_frame: [0; 66], op: dvc_op, commit, - namespace: 0, + group: 0, log_view: 0, - reserved: [0; 100], + reserved: [0; 68], + nack_bitset: 0, + present_bitset: 0, }; assert!(header(dvc_commit).validate().is_ok()); assert!( @@ -835,6 +1046,138 @@ mod tests { ); } + #[test] + fn given_restamped_view_when_sealing_should_keep_the_same_identity() { + // `restamp_prepare_view` rewrites `view` on retransmission. If the identity + // moved with it, one op would carry different checksums per receiving view + // and the merge would read them as competing prepares nacking each other. + let base = PrepareHeader { + command: Command2::Prepare, + operation: iggy_binary_protocol::Operation::CreateStream, + op: 9, + view: 4, + client: 11, + request: 2, + timestamp: 1234, + checksum_body: 99, + ..Default::default() + }; + let restamped = PrepareHeader { view: 12, ..base }; + assert_eq!( + base.identity_checksum(), + restamped.identity_checksum(), + "view must not participate in a prepare's identity" + ); + } + + #[test] + fn given_matching_view_when_restamping_should_reuse_frozen_prepare() { + let message = prepare_message(9, 7, 11).transmute_header(|old, new: &mut PrepareHeader| { + *new = old; + new.view = 4; + }); + let frozen = message.into_frozen(); + let original_ptr = frozen.as_slice().as_ptr(); + + let restamped = restamp_prepare_view(frozen, 4).expect("valid prepare"); + let header = bytemuck::checked::try_from_bytes::( + &restamped[..size_of::()], + ) + .expect("restamped prepare header"); + + assert_eq!(restamped.as_slice().as_ptr(), original_ptr); + assert_eq!(header.view, 4); + } + + #[test] + fn given_new_view_when_restamping_should_only_change_view() { + let message = prepare_message(9, 7, 11).transmute_header(|old, new: &mut PrepareHeader| { + *new = old; + new.view = 4; + new.client = 17; + new.request = 23; + }); + let expected_identity = message.header().identity_checksum(); + let expected_op = message.header().op; + let expected_client = message.header().client; + let expected_request = message.header().request; + + let restamped = restamp_prepare_view(message.into_frozen(), 12).expect("valid prepare"); + let header = bytemuck::checked::try_from_bytes::( + &restamped[..size_of::()], + ) + .expect("restamped prepare header"); + + assert_eq!(header.view, 12); + assert_eq!(header.identity_checksum(), expected_identity); + assert_eq!(header.op, expected_op); + assert_eq!(header.client, expected_client); + assert_eq!(header.request, expected_request); + } + + #[test] + fn given_truncated_buffer_when_restamping_should_reject() { + let malformed: Frozen = Owned::::copy_from_slice(&[0]).into(); + + assert!(restamp_prepare_view(malformed, 1).is_none()); + } + + #[test] + fn given_truncated_buffer_when_reading_frozen_prepare_should_reject() { + let malformed: Frozen = Owned::::copy_from_slice(&[0]).into(); + + assert!(matches!( + frozen_prepare_header(&malformed), + Err(ChainReplicationError::MalformedPrepare) + )); + } + + #[test] + fn given_committed_prepare_when_selecting_replication_target_should_reject() { + let consensus = VsrConsensus::new(1, 0, 3, 0, NoopBus, LocalPipeline::new()); + consensus.init(); + let prepare = prepare_message(0, 0, 0); + + assert!(matches!( + replication_target(&consensus, prepare.header()), + Err(ChainReplicationError::CommittedPrepare { + op: 0, + commit_min: 0 + }) + )); + } + + #[test] + fn given_different_prepares_at_one_op_when_sealing_should_differ() { + // The distinction the merge depends on: two prepares at one op number are + // told apart, so a canonical header is distinguishable from a stale one. + let first = PrepareHeader { + command: Command2::Prepare, + operation: iggy_binary_protocol::Operation::CreateStream, + op: 5, + client: 1, + request: 1, + timestamp: 100, + ..Default::default() + }; + let second = PrepareHeader { client: 2, ..first }; + assert_ne!( + first.identity_checksum(), + second.identity_checksum(), + "distinct prepares at the same op must not share an identity" + ); + + let body_differs = PrepareHeader { + checksum_body: 7, + ..first + }; + assert_ne!( + first.identity_checksum(), + body_differs.identity_checksum(), + "the body reaches the identity through checksum_body" + ); + } + #[test] fn loopback_push_and_drain() { let consensus = VsrConsensus::new(1, 0, 3, 0, NoopBus, LocalPipeline::new()); @@ -894,81 +1237,590 @@ mod tests { assert_eq!(typed.header().command, Command2::PrepareOk); } - #[test] - fn loopback_cleared_on_complete_view_change_as_primary() { - use iggy_binary_protocol::{DoViewChangeHeader, StartViewChangeHeader}; + /// A sender's suffix and matching body bytes for a replica that holds every op + /// in `commit..=op` and can serve each body. Checksums derive from `(op, view)` + /// so the hash chain connects, which the merge checks. + fn dvc_with_full_suffix( + replica: u8, + view: u32, + log_view: u32, + op: u64, + commit: u64, + ) -> (iggy_binary_protocol::DoViewChangeHeader, Body) { + dvc_with_suffix(replica, view, log_view, op, commit, None) + } - // 3 replicas, replica 0 is primary for view 0 (and view 3: 3 % 3 = 0). - let consensus = VsrConsensus::new(1, 0, 3, 0, NoopBus, LocalPipeline::new()); - consensus.init(); + /// As [`dvc_with_full_suffix`], but `withhold_body` names one op whose present + /// bit is cleared: header held, body unservable. A quorum where every sender + /// withholds the same op decides nothing yet. + fn dvc_with_suffix( + replica: u8, + view: u32, + log_view: u32, + op: u64, + commit: u64, + withhold_body: Option, + ) -> (iggy_binary_protocol::DoViewChangeHeader, Body) { + use iggy_binary_protocol::DoViewChangeHeader; + + let headers = suffix_headers(commit, op, log_view); + let body = encode_body(&headers); + let mut present = if headers.is_empty() { + 0 + } else { + (1u128 << headers.len()) - 1 + }; + if let Some(withheld) = withhold_body + && let Some(index) = headers.iter().position(|header| header.op == withheld) + { + present &= !(1u128 << index); + } + let header = DoViewChangeHeader { + checksum: 0, + checksum_body: 0, + cluster: 0, + size: u32::try_from(std::mem::size_of::() + body.len()) + .expect("synthetic DVC frame fits u32"), + view, + release: 0, + command: Command2::DoViewChange, + replica, + reserved_frame: [0; 66], + op, + commit, + group: 0, + log_view, + reserved: [0; 68], + nack_bitset: 0, + present_bitset: present, + }; + (header, body) + } - // SVC from replica 1, view 3. Replica 0 advances to view 3 - // (reset_view_change_state clears loopback), records own SVC+DVC and - // replica 1's SVC. DVC quorum needs 2; have 1. - let svc = StartViewChangeHeader { + /// Headers for `low..=high`, high-to-low as a suffix requires, sealed and + /// chained the way a real producer writes them. + /// + /// Built ascending so each `parent` is the previous entry's real identity, then + /// reversed. The decoder recomputes both, so fabricated checksums are rejected + /// before the code under test sees them. + fn suffix_headers(low: u64, high: u64, view: u32) -> Vec { + if high == 0 { + return Vec::new(); + } + let mut parent = 0u128; + let mut ascending = Vec::new(); + for op in low.max(1)..=high { + let mut header = PrepareHeader { + command: Command2::Prepare, + operation: iggy_binary_protocol::Operation::CreateStream, + op, + view, + parent, + // Strictly increasing with op, so the suffix reads decreasing. + timestamp: op, + // Zero so the DVC's own commit drives `commit_max`. + commit: 0, + ..Default::default() + }; + header.checksum = header.identity_checksum(); + parent = header.checksum; + ascending.push(header); + } + ascending.reverse(); + ascending + } + + fn svc_header(replica: u8, view: u32) -> iggy_binary_protocol::StartViewChangeHeader { + iggy_binary_protocol::StartViewChangeHeader { checksum: 0, checksum_body: 0, cluster: 0, - size: 0, - view: 3, + size: u32::try_from(std::mem::size_of::< + iggy_binary_protocol::StartViewChangeHeader, + >()) + .expect("header fits u32"), + view, release: 0, command: Command2::StartViewChange, - replica: 1, + replica, reserved_frame: [0; 66], - namespace: 0, + group: 0, reserved: [0; 120], + } + } + + /// Install the suffix this replica would read from its own journal. + fn install_local_suffix( + consensus: &VsrConsensus, + op: u64, + commit: u64, + log_view: u32, + ) { + let headers = suffix_headers(commit, op, log_view); + consensus.set_local_dvc_suffix(crate::dvc_merge::suffix_all_present(headers)); + } + + #[test] + fn given_an_undecidable_quorum_when_a_later_dvc_decides_it_should_start_the_view() { + // Reaching a view-change quorum is not the same as deciding a log. Latching + // `do_view_change_quorum` at the quorum makes every non-Ready outcome + // terminal: later DoViewChanges are recorded, but the guard that calls the + // merge is already false, so the view burns its status timeout for nothing. + // + // 5 replicas, view_change quorum 3, replica 0 is primary for view 5. + let consensus = VsrConsensus::new(1, 0, 5, 0, NoopBus, LocalPipeline::new()); + consensus.init(); + consensus.restore_commit_state(2, 2); + consensus.sequencer().set_sequence(4); + // This replica holds op 4's header but cannot serve its body. Suffix entries + // run high-to-low, so bit 0 is op 4: clearing it offers ops 3 and 2 only. + let local = suffix_headers(2, 4, 0); + consensus.set_local_dvc_suffix(crate::view_change_quorum::DvcSuffix::new(local, 0, 0b110)); + + let _ = consensus.handle_start_view_change(PlaneKind::Metadata, &svc_header(1, 5)); + + // Two peers report, reaching the quorum of 3. All three hold op 4's header, + // none can serve its body, and two replicas have yet to report. + for replica in [1u8, 2] { + let (dvc, body) = dvc_with_suffix(replica, 5, 0, 4, 2, Some(4)); + let actions = consensus.handle_do_view_change(PlaneKind::Metadata, &dvc, &body); + assert!(actions.is_empty()); + } + assert!( + consensus.pending_view_log().is_none(), + "op 4 is neither servable nor provably dead, so nothing may be parked yet" + ); + + // Replica 3 arrives holding the body: the deciding message, still merged. + let (dvc, body) = dvc_with_full_suffix(3, 5, 0, 4, 2); + let _ = consensus.handle_do_view_change(PlaneKind::Metadata, &dvc, &body); + + let pending = consensus + .pending_view_log() + .expect("the DVC that supplies the missing body must complete the merge"); + assert_eq!(pending.op_head, 4); + assert_eq!(pending.commit_max, 2); + } + + #[test] + fn given_a_sealed_prepare_when_verifying_integrity_should_accept() { + let message = Message::::new(size_of::()).transmute_header( + |_, header: &mut PrepareHeader| { + header.command = Command2::Prepare; + header.op = 7; + header.size = u32::try_from(size_of::()).expect("header fits u32"); + }, + ); + let sealed = seal_prepare_checksum(message); + assert_eq!(verify_prepare_integrity(sealed.header(), &[]), Ok(())); + } + + #[test] + fn given_a_prepare_whose_header_was_altered_when_verifying_should_reject() { + // Downstream compares `checksum` as an opaque token, so without this a frame + // corrupted in transit is journaled and then re-served to peers from the WAL. + let message = Message::::new(size_of::()).transmute_header( + |_, header: &mut PrepareHeader| { + header.command = Command2::Prepare; + header.op = 7; + header.size = u32::try_from(size_of::()).expect("header fits u32"); + }, + ); + let mut sealed = seal_prepare_checksum(message); + let bytes = &mut sealed.as_mut_slice()[..size_of::()]; + let header = bytemuck::checked::try_from_bytes_mut::(bytes) + .expect("a prepare header round-trips its own bit pattern"); + header.op = 8; + assert!(verify_prepare_integrity(&header.clone(), &[]).is_err()); + } + + #[test] + fn given_an_unsealed_prepare_when_verifying_should_abstain() { + // The partition plane leaves `checksum` at `CHECKSUM_UNSEALED` and carries a + // verified `batch_checksum` over the same bytes instead. + let header = PrepareHeader { + command: Command2::Prepare, + op: 7, + ..Default::default() }; - let _ = consensus.handle_start_view_change(PlaneKind::Metadata, &svc); + assert_eq!(header.checksum, CHECKSUM_UNSEALED); + assert_eq!(verify_prepare_integrity(&header, &[]), Ok(())); + } + + /// A frame carrying `body`, with `size` covering exactly header + body and + /// `checksum_body` sealed over it, as the metadata projection does. + /// `trailing` bytes of garbage past the sealed frame, which `size` does not + /// cover. The buffer is `MESSAGE_ALIGN`ed: `PrepareHeader` holds `u128`s, so a + /// `Vec` would only be 16-aligned by the allocator's good graces and miri + /// rejects the cast. + fn sealed_frame(body: &[u8], trailing: usize) -> Owned { + let size = size_of::() + body.len(); + let mut frame = Owned::::zeroed(size + trailing); + let bytes = frame.as_mut_slice(); + bytes[size_of::()..size].copy_from_slice(body); + bytes[size..].fill(0xAA); + let header = bytemuck::checked::from_bytes_mut::( + &mut bytes[..size_of::()], + ); + header.command = Command2::Prepare; + header.op = 7; + header.size = u32::try_from(size).expect("fits u32"); + header.checksum_body = u128::from(calculate_checksum(body)); + frame + } + + fn frame_header(frame: &Owned) -> PrepareHeader { + *bytemuck::checked::from_bytes::( + &frame.as_slice()[..size_of::()], + ) + } + + #[test] + fn given_a_prepare_whose_body_was_altered_when_verifying_should_reject() { + let mut frame = sealed_frame(b"body", 0); + let header = frame_header(&frame); + assert_eq!(verify_prepare_integrity(&header, frame.as_slice()), Ok(())); + + *frame + .as_mut_slice() + .last_mut() + .expect("the frame has a body") ^= 1; + assert!(verify_prepare_integrity(&header, frame.as_slice()).is_err()); + } + + #[test] + fn given_bytes_past_the_frame_size_when_verifying_should_ignore_them() { + // `try_from` accepts a buffer longer than `size` without trimming; hashing to + // the end would reject a correctly sealed prepare and disagree with the WAL scan. + let padded = sealed_frame(b"body", 16); + let header = frame_header(&padded); + assert_eq!( + verify_prepare_integrity(&header, padded.as_slice()), + Ok(()), + "only the bytes `size` covers are the body" + ); + } + + #[test] + fn given_a_size_that_overruns_the_buffer_when_verifying_should_reject() { + // Truncated frame, header still claims the full length: the body it names is + // not there to hash. + let frame = sealed_frame(b"body", 0); + let header = frame_header(&frame); + + let truncated = &frame.as_slice()[..frame.as_slice().len() - 1]; + assert!(verify_prepare_integrity(&header, truncated).is_err()); + } + + #[test] + fn given_a_parked_merge_when_not_yet_started_should_not_advance_log_view() { + // `log_view` claims "my log IS the log this view decided", which is what + // makes a sender canonical next time. Raising it when the merge parks, before + // the merged head is installed, lets a primary-elect that never finishes + // repair vote as canonical carrying its own stale head, and ops the merge + // kept then fall outside the next scan range, dropped with no nack. + let consensus = VsrConsensus::new(1, 0, 3, 0, NoopBus, LocalPipeline::new()); + consensus.init(); + consensus.restore_commit_state(2, 2); + consensus.sequencer().set_sequence(3); + install_local_suffix(&consensus, 3, 2, 0); + assert_eq!(consensus.log_view(), 0); + + let _ = consensus.handle_start_view_change(PlaneKind::Metadata, &svc_header(1, 3)); + let (dvc, body) = dvc_with_full_suffix(2, 3, 0, 3, 2); + let _ = consensus.handle_do_view_change(PlaneKind::Metadata, &dvc, &body); + + assert!(consensus.pending_view_log().is_some(), "the merge parks"); + assert_eq!( + consensus.log_view(), + 0, + "a parked merge has installed nothing, so log_view must still \ + describe the log this replica actually holds" + ); + assert_eq!(consensus.view(), 3, "the view itself did advance"); + + let _ = consensus.start_pending_view(PlaneKind::Metadata); + assert_eq!( + consensus.log_view(), + 3, + "installing the merged head is what earns the log_view claim" + ); + } + + #[test] + fn loopback_cleared_on_complete_view_change_as_primary() { + // 3 replicas, replica 0 is primary for view 0 (and view 3: 3 % 3 = 0). + let consensus = VsrConsensus::new(1, 0, 3, 0, NoopBus, LocalPipeline::new()); + consensus.init(); + consensus.restore_commit_state(2, 2); + consensus.sequencer().set_sequence(3); + install_local_suffix(&consensus, 3, 2, 0); + + // SVC from replica 1, view 3. Replica 0 advances, records own SVC+DVC and + // replica 1's SVC. DVC quorum needs 2; have 1. + let _ = consensus.handle_start_view_change(PlaneKind::Metadata, &svc_header(1, 3)); // Stale loopback queued between SVC and DVC quorum. let stale_msg = Message::::new(std::mem::size_of::()); consensus.push_loopback(stale_msg.into_generic()); - // DVC from replica 2, view 3, quorum, complete_view_change_as_primary fires. - let dvc = DoViewChangeHeader { + // DVC from replica 2 forms the quorum and the merge settles the log. + let (dvc, body) = dvc_with_full_suffix(2, 3, 0, 3, 2); + let actions = consensus.handle_do_view_change(PlaneKind::Metadata, &dvc, &body); + + // Parked: nothing is announced until the journal can serve it. + assert!( + actions.is_empty(), + "a merged view change must announce nothing until repair completes" + ); + let pending = consensus + .pending_view_log() + .expect("a decidable quorum must park a merged log"); + assert_eq!(pending.op_head, 3); + assert_eq!(pending.commit_max, 2); + assert_eq!(consensus.status(), Status::ViewChange); + + // Stale loopback must be cleared. + let mut buf = Vec::new(); + consensus.drain_loopback_into(&mut buf); + assert!( + buf.is_empty(), + "loopback queue must be empty after view change completion" + ); + + // Journal now covers the merged log, so the view starts and announces. + let actions = consensus.start_pending_view(PlaneKind::Metadata); + assert!( + actions + .iter() + .any(|a| matches!(a, crate::VsrAction::SendStartView { .. })), + "expected SendStartView once the view starts" + ); + assert_eq!(consensus.status(), Status::Normal); + assert!( + consensus.pending_view_log().is_none(), + "starting the view must consume the parked log" + ); + } + + /// Refusing to start a view must not be terminal. + /// + /// A parked merge leaves the replica in `ViewChange` announcing nothing if the + /// bodies never arrive, which is the intended trade against losing data but has + /// to stay recoverable: the status timeout fires, escalates, and drops the parked + /// log. Reusing a log merged for a superseded view would leak a truncation + /// decided there into a view that never voted for it. + /// A `StartView` from the view's primary, optionally carrying the view's + /// canonical headers. + fn start_view_with_suffix( + replica: u8, + view: u32, + op: u64, + commit: u64, + with_suffix: bool, + ) -> (iggy_binary_protocol::StartViewHeader, Body) { + use iggy_binary_protocol::StartViewHeader; + + let body = if with_suffix { + encode_body(&suffix_headers(commit, op, view)) + } else { + Body::new(BODY_ALIGN) + }; + let header = StartViewHeader { checksum: 0, checksum_body: 0, cluster: 0, - size: 0, - view: 3, + size: u32::try_from(std::mem::size_of::() + body.len()) + .expect("synthetic StartView fits u32"), + view, release: 0, - command: Command2::DoViewChange, - replica: 2, + command: Command2::StartView, + replica, reserved_frame: [0; 66], - op: 0, - commit: 0, - namespace: 0, - log_view: 0, - reserved: [0; 100], + op, + commit, + group: 0, + reserved: [0; 88], + incarnation: 0, }; - let actions = consensus.handle_do_view_change(PlaneKind::Metadata, &dvc); + (header, body) + } + + /// Encode headers as a control-message body. + /// + /// Aligned, because `dvc_suffix_decode` uses a checked `bytemuck` cast per + /// 256-byte chunk: a `Vec` body reports `MalformedHeader` for entry 0 + /// instead of the failure under test. glibc over-aligns these; Miri does not. + fn encode_body(headers: &[PrepareHeader]) -> Body { + let mut body = Body::with_capacity(BODY_ALIGN, std::mem::size_of_val(headers)); + for header in headers { + body.extend_from_slice(bytemuck::bytes_of(header)); + } + body + } + + #[test] + fn given_a_corrupted_suffix_entry_when_decoding_should_reject_the_frame() { + // The worst failure mode: a flipped bit in a canonical sender's header makes + // it canonical for the view, so honest senders read as disagreeing and can + // reach a nack quorum against a committed op. Recomputing keeps that out. + let mut headers = suffix_headers(2, 4, 1); + headers[0].timestamp ^= 0xFF; + let body = encode_body(&headers); + + let error = crate::dvc_suffix_decode(&body, 4, 0, 0) + .expect_err("a header that does not match its own checksum must be rejected"); + assert_eq!(error, crate::DvcSuffixError::ChecksumMismatch { index: 0 }); + } + + #[test] + fn given_a_broken_suffix_chain_when_decoding_should_reject_the_frame() { + // Well-sealed entries that do not link: a log, not a bag of records. + let mut headers = suffix_headers(2, 4, 1); + headers[0].parent ^= 0xFF; + headers[0].checksum = headers[0].identity_checksum(); + let body = encode_body(&headers); + + let error = crate::dvc_suffix_decode(&body, 4, 0, 0) + .expect_err("a suffix whose entries do not chain must be rejected"); + assert_eq!(error, crate::DvcSuffixError::ChainBreak { index: 1 }); + } + + #[test] + fn given_a_suffix_with_mixed_view_stamps_when_decoding_should_be_accepted() { + // A stitched suffix: a held op keeps the view that delivered it, a repaired + // neighbour carries the view that decided it. Rejecting drops the sender's + // vote forever (the retransmit is byte-identical) and the cluster can fail to + // elect. No re-seal: `identity_checksum` excludes `view`. + let mut headers = suffix_headers(2, 4, 2); + headers[1].view = 1; + let body = encode_body(&headers); + + let suffix = crate::dvc_suffix_decode(&body, 4, 0, 0) + .expect("a stitched suffix with mixed view stamps must decode"); + assert_eq!(suffix.len(), 3); + } + + #[test] + fn given_an_unsealed_suffix_when_decoding_should_be_accepted() { + // The on-disk sentinel: suffixes are read out of the journal, which may hold + // pre-seal entries, and partition-plane prepares are unsealed by construction. + let headers: Vec = suffix_headers(2, 4, 1) + .into_iter() + .map(|mut header| { + header.checksum = 0; + header.parent = 0; + header + }) + .collect(); + let body = encode_body(&headers); + + let suffix = + crate::dvc_suffix_decode(&body, 4, 0, 0).expect("an unsealed suffix must still decode"); + assert_eq!(suffix.len(), 3); + } + + #[test] + fn given_start_view_with_suffix_when_adopted_should_record_the_canonical_headers() { + // The backup keeps the view's headers so its repair ingest can reject a body + // that disagrees with the view's decision, and so a disagreeing local entry + // is reported rather than silently blocking its own repair forever. + let consensus = VsrConsensus::new(1, 0, 3, 0, NoopBus, LocalPipeline::new()); + consensus.init(); + + // Replica 1 is primary for view 1 (1 % 3). + let (header, body) = start_view_with_suffix(1, 1, 5, 3, true); + let actions = consensus.handle_start_view(PlaneKind::Metadata, &header, &body); + + assert!(!actions.is_empty(), "a valid StartView must be adopted"); + assert_eq!(consensus.status(), Status::Normal); + assert_eq!(consensus.sequencer().current_sequence(), 5); + + let recorded = consensus + .pending_view_log() + .expect("an adopted StartView carrying a suffix must record its headers"); + assert_eq!(recorded.op_head, 5); + assert_eq!(recorded.commit_max, 3); + assert_eq!( + recorded.headers.iter().map(|h| h.op).collect::>(), + vec![5, 4, 3], + "headers run high-to-low from the head down to the announced commit" + ); + } + + #[test] + fn given_start_view_without_suffix_when_adopted_should_trust_the_announced_op() { + // Probe answers and stale-view corrections carry numbers only: a backup must + // still adopt, and record nothing it could mistake for the view's decision. + let consensus = VsrConsensus::new(1, 0, 3, 0, NoopBus, LocalPipeline::new()); + consensus.init(); + + let (header, body) = start_view_with_suffix(1, 1, 5, 3, false); + assert!(body.is_empty()); + let actions = consensus.handle_start_view(PlaneKind::Metadata, &header, &body); - // View change complete → SendStartView action. assert!( - actions - .iter() - .any(|a| matches!(a, crate::VsrAction::SendStartView { .. })), - "expected SendStartView after DVC quorum" + !actions.is_empty(), + "a numbers-only StartView must still adopt" + ); + assert_eq!(consensus.sequencer().current_sequence(), 5); + assert!( + consensus.pending_view_log().is_none(), + "no suffix means no canonical headers to verify against" ); + } + + #[test] + fn given_parked_view_change_when_status_timeout_fires_should_escalate_and_drop_merged_log() { + let consensus = VsrConsensus::new(1, 0, 3, 0, NoopBus, LocalPipeline::new()); + consensus.init(); + consensus.restore_commit_state(2, 2); + consensus.sequencer().set_sequence(3); + install_local_suffix(&consensus, 3, 2, 0); + + let _ = consensus.handle_start_view_change(PlaneKind::Metadata, &svc_header(1, 3)); + let (dvc, body) = dvc_with_full_suffix(2, 3, 0, 3, 2); + let _ = consensus.handle_do_view_change(PlaneKind::Metadata, &dvc, &body); + + // Parked: no shard here reports coverage, so the view never starts. + assert!(consensus.pending_view_log().is_some()); + assert_eq!(consensus.status(), Status::ViewChange); + let parked_view = consensus.view(); + + // `VIEW_CHANGE_STATUS_TICKS` is 500; tick past it. Escalation shows as the + // view advancing, since the 50-tick SVC retransmit also emits a send. + let mut escalated = false; + for _ in 0..600 { + let _ = consensus.tick(PlaneKind::Metadata); + if consensus.view() > parked_view { + escalated = true; + break; + } + } - // Stale loopback must be cleared. - let mut buf = Vec::new(); - consensus.drain_loopback_into(&mut buf); assert!( - buf.is_empty(), - "loopback queue must be empty after view change completion" + escalated, + "a parked view change must still escalate on the status timeout" ); + assert!( + consensus.view() > parked_view, + "escalation must advance the view past {parked_view}, got {}", + consensus.view() + ); + assert!( + consensus.pending_view_log().is_none(), + "the superseded merged log must be dropped, not carried into the next view" + ); + assert_eq!(consensus.status(), Status::ViewChange); } - /// A DVC winner may claim an uncommitted range up to the *configured* + /// A merged log may claim an uncommitted range up to the *configured* /// prepare depth. With a pipeline deeper than the default const, the new /// primary schedules the rebuild rather than panicking on the old /// `PIPELINE_PREPARE_QUEUE_MAX` bound. #[test] #[allow(clippy::cast_possible_truncation)] - fn given_view_change_range_above_default_when_complete_as_primary_should_rebuild() { - use iggy_binary_protocol::{DoViewChangeHeader, StartViewChangeHeader}; - + fn given_view_change_range_above_default_when_starting_view_should_rebuild() { let depth = crate::PIPELINE_PREPARE_QUEUE_MAX * 2; // Strictly above the default const, still within the configured depth. let winner_op = (crate::PIPELINE_PREPARE_QUEUE_MAX + 8) as u64; @@ -983,48 +1835,23 @@ mod tests { LocalPipeline::with_capacities(depth, depth * 2), ); consensus.init(); + consensus.sequencer().set_sequence(winner_op); + install_local_suffix(&consensus, winner_op, 1, 0); // SVC from replica 1 moves replica 0 into view 3 and records its own DVC. - let svc = StartViewChangeHeader { - checksum: 0, - checksum_body: 0, - cluster: 0, - size: 0, - view: 3, - release: 0, - command: Command2::StartViewChange, - replica: 1, - reserved_frame: [0; 66], - namespace: 0, - reserved: [0; 120], - }; - let _ = consensus.handle_start_view_change(PlaneKind::Metadata, &svc); + let _ = consensus.handle_start_view_change(PlaneKind::Metadata, &svc_header(1, 3)); - // DVC from replica 2 claims a log head far past commit, forming quorum. - let dvc = DoViewChangeHeader { - checksum: 0, - checksum_body: 0, - cluster: 0, - size: 0, - view: 3, - release: 0, - command: Command2::DoViewChange, - replica: 2, - reserved_frame: [0; 66], - op: winner_op, - commit: 0, - namespace: 0, - log_view: 0, - reserved: [0; 100], - }; - let actions = consensus.handle_do_view_change(PlaneKind::Metadata, &dvc); + // DVC from replica 2 claims the same deep log, forming quorum. + let (dvc, body) = dvc_with_full_suffix(2, 3, 0, winner_op, 1); + let _ = consensus.handle_do_view_change(PlaneKind::Metadata, &dvc, &body); + let actions = consensus.start_pending_view(PlaneKind::Metadata); assert!( actions.iter().any(|action| matches!( action, - VsrAction::RebuildPipeline { from_op: 1, to_op } if *to_op == winner_op + VsrAction::RebuildPipeline { from_op: 2, to_op } if *to_op == winner_op )), - "expected RebuildPipeline over the full uncommitted range" + "expected RebuildPipeline over the uncommitted range, got {actions:?}" ); } @@ -1163,12 +1990,11 @@ mod tests { consensus.init(); consensus.advance_commit_max(4); - let request = RequestHeader { + let request = RoutedRequestHeader { command: Command2::Request, operation: Operation::DeleteConsumerOffset2, client: 42, request: 7, - namespace: 9, ..Default::default() }; let status = 3021; @@ -1181,7 +2007,6 @@ mod tests { assert_eq!(header.commit, 4); assert_eq!(header.client, 42); assert_eq!(header.request, 7); - assert_eq!(header.namespace, 9); assert_eq!(header.operation, Operation::DeleteConsumerOffset2); assert_eq!( header.size as usize, diff --git a/core/consensus/src/view_change_quorum.rs b/core/consensus/src/view_change_quorum.rs index 2eaff15b36..95da481b31 100644 --- a/core/consensus/src/view_change_quorum.rs +++ b/core/consensus/src/view_change_quorum.rs @@ -16,27 +16,201 @@ // under the License. use crate::REPLICAS_MAX; +use iggy_binary_protocol::{ + CHECKSUM_UNSEALED, Command2, ConsensusHeader, DVC_HEADERS_MAX, Operation, PrepareHeader, +}; + +/// Write prepare headers into a control-message body, high-to-low op. +/// +/// Shared by `DoViewChange` and `StartView`, which both carry a suffix as a plain +/// run of 256-byte headers; stating the layout once keeps them from drifting. +/// +/// # Panics +/// When `dst` is not exactly `headers.len()` headers wide. +pub fn encode_prepare_headers(headers: &[PrepareHeader], dst: &mut [u8]) { + let stride = size_of::(); + assert_eq!( + dst.len(), + std::mem::size_of_val(headers), + "control-message body buffer must fit the headers exactly" + ); + for (index, header) in headers.iter().enumerate() { + dst[index * stride..(index + 1) * stride].copy_from_slice(bytemuck::bytes_of(header)); + } +} + +/// Placeholder standing in for a suffix entry the sender does not hold. +/// +/// The suffix stays consecutive so a `(head_op, op)` pair indexes it arithmetically +/// and one bitset bit lines up with one op. A gap is transmitted, not omitted. +/// +/// `Operation::Reserved` is the marker, since no real prepare carries it, and every +/// other field is zero. [`dvc_header_kind`] insists on exactly that, so arbitrary +/// bytes cannot pass as a blank the merge would index. +#[must_use] +pub fn dvc_blank(op: u64) -> PrepareHeader { + PrepareHeader { + command: Command2::Prepare, + operation: Operation::Reserved, + op, + ..Default::default() + } +} + +/// What a suffix slot says about the sender's log at that op. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DvcHeaderKind { + /// No header for this op, or one the sender cannot vouch for. The nack bit is + /// what distinguishes "never prepared" (proof) from "lost it" (no proof). + Blank, + /// A real prepare header the sender holds. + Valid, +} + +/// Classify a suffix slot, by exact comparison against the canonical blank. +#[must_use] +pub fn dvc_header_kind(header: &PrepareHeader) -> DvcHeaderKind { + if *header == dvc_blank(header.op) { + DvcHeaderKind::Blank + } else { + DvcHeaderKind::Valid + } +} + +/// A sender's uncommitted suffix: the headers spanning `commit..=op`, high-to-low, +/// plus one nack bit and one present bit per entry. +/// +/// Index 0 is the head (`StoredDvc::op`); index `i` is op `op - i`. Empty when the +/// sender has nothing uncommitted, or could not snapshot a suffix for this log. +#[derive(Debug, Clone, Default)] +pub struct DvcSuffix { + headers: Vec, + nack_bitset: u128, + present_bitset: u128, +} + +// These all read through the `Vec`, and neither `Vec::len` nor `Deref` is const on +// the pinned toolchain, so clippy's suggestion does not compile. +#[allow(clippy::missing_const_for_fn)] +impl DvcSuffix { + /// # Panics + /// When `headers` exceeds [`DVC_HEADERS_MAX`], or a bitset sets a bit past the + /// suffix. Sender-side programming errors; the same conditions off the wire go + /// through `DoViewChangeHeader::validate`. + #[must_use] + pub fn new(headers: Vec, nack_bitset: u128, present_bitset: u128) -> Self { + assert!( + headers.len() <= DVC_HEADERS_MAX, + "DVC suffix of {} entries exceeds the addressable maximum {DVC_HEADERS_MAX}", + headers.len() + ); + if headers.len() < DVC_HEADERS_MAX { + let beyond = !((1u128 << headers.len()) - 1); + assert_eq!( + nack_bitset & beyond, + 0, + "nack bit set past the {}-entry suffix", + headers.len() + ); + assert_eq!( + present_bitset & beyond, + 0, + "present bit set past the {}-entry suffix", + headers.len() + ); + } + Self { + headers, + nack_bitset, + present_bitset, + } + } + + /// A sender contributing numbers only: no headers, nacks, or offered bodies. + #[must_use] + pub fn empty() -> Self { + Self::default() + } + + #[must_use] + pub fn len(&self) -> usize { + self.headers.len() + } + + #[must_use] + pub fn is_empty(&self) -> bool { + self.headers.is_empty() + } + + #[must_use] + pub fn headers(&self) -> &[PrepareHeader] { + &self.headers + } + + #[must_use] + pub const fn nack_bitset(&self) -> u128 { + self.nack_bitset + } + + #[must_use] + pub const fn present_bitset(&self) -> u128 { + self.present_bitset + } + + /// Bytes the headers occupy on the wire. + #[must_use] + pub fn encoded_len(&self) -> usize { + self.headers.len() * size_of::() + } + + /// Write the headers into a `DoViewChange` body, high-to-low op. + /// + /// Paired with [`dvc_suffix_decode`], so the wire ordering is stated once. + /// + /// # Panics + /// When `dst` is not exactly [`Self::encoded_len`] bytes. + pub fn encode_into(&self, dst: &mut [u8]) { + encode_prepare_headers(&self.headers, dst); + } + + /// Slot index for `op`. `None` when `op` falls outside this sender's window. + #[must_use] + pub fn index_of(&self, head_op: u64, op: u64) -> Option { + let distance = usize::try_from(head_op.checked_sub(op)?).ok()?; + (distance < self.headers.len()).then_some(distance) + } + + /// The header at `index`, or `None` for a blank or out-of-range slot. + #[must_use] + pub fn valid_header_at(&self, index: usize) -> Option<&PrepareHeader> { + let header = self.headers.get(index)?; + matches!(dvc_header_kind(header), DvcHeaderKind::Valid).then_some(header) + } + + /// Whether the sender proves it never prepared the entry at `index`. + #[must_use] + pub fn nacks(&self, index: usize) -> bool { + index < self.headers.len() && self.nack_bitset & (1u128 << index) != 0 + } + + /// Whether the sender can serve the body of the entry at `index`. + #[must_use] + pub fn offers_body(&self, index: usize) -> bool { + index < self.headers.len() && self.present_bitset & (1u128 << index) != 0 + } +} /// Stored information from a `DoViewChange` message. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone)] pub struct StoredDvc { pub replica: u8, /// The view when the replica's status was last normal. pub log_view: u32, pub op: u64, pub commit: u64, -} - -impl StoredDvc { - /// Compare for log selection: highest `log_view`, then highest op. - #[must_use] - pub const fn is_better_than(&self, other: &Self) -> bool { - if self.log_view == other.log_view { - self.op > other.op - } else { - self.log_view > other.log_view - } - } + /// The sender's uncommitted suffix. Empty from a silent sender, which counts + /// toward the quorum and `max(commit)` but neither nacks nor offers bodies. + pub suffix: DvcSuffix, } /// Array type for storing DVC messages from all replicas. @@ -44,12 +218,13 @@ pub type DvcQuorumArray = [Option; REPLICAS_MAX]; /// Create an empty DVC quorum array. #[must_use] -pub const fn dvc_quorum_array_empty() -> DvcQuorumArray { - [None; REPLICAS_MAX] +pub fn dvc_quorum_array_empty() -> DvcQuorumArray { + // `[None; REPLICAS_MAX]` needs `StoredDvc: Copy`, ruled out by the `Vec`. + std::array::from_fn(|_| None) } /// Record a DVC in the array. Returns true if this is a new entry (not duplicate). -pub const fn dvc_record(array: &mut DvcQuorumArray, dvc: StoredDvc) -> bool { +pub fn dvc_record(array: &mut DvcQuorumArray, dvc: StoredDvc) -> bool { let slot = &mut array[dvc.replica as usize]; if slot.is_some() { return false; // Duplicate @@ -64,43 +239,203 @@ pub fn dvc_count(array: &DvcQuorumArray) -> usize { array.iter().filter(|m| m.is_some()).count() } -/// Check if a specific replica has sent a DVC. -#[must_use] -pub fn dvc_has_from(array: &DvcQuorumArray, replica: u8) -> bool { - array.get(replica as usize).is_some_and(Option::is_some) +/// Reset the DVC quorum array. +pub fn dvc_reset(array: &mut DvcQuorumArray) { + *array = dvc_quorum_array_empty(); } -/// Select the winning DVC (best log) from the quorum. -/// Returns the DVC with: highest `log_view`, then highest op. -#[must_use] -pub fn dvc_select_winner(array: &DvcQuorumArray) -> Option<&StoredDvc> { - array - .iter() - .filter_map(|m| m.as_ref()) - .max_by(|a, b| match a.log_view.cmp(&b.log_view) { - std::cmp::Ordering::Equal => a.op.cmp(&b.op), - other => other, - }) +/// Iterator over all stored DVCs. +pub fn dvc_iter(array: &DvcQuorumArray) -> impl Iterator { + array.iter().filter_map(|m| m.as_ref()) } -/// Get the maximum commit number across all DVCs. -#[must_use] -pub fn dvc_max_commit(array: &DvcQuorumArray) -> u64 { - array - .iter() - .filter_map(|m| m.as_ref()) - .map(|dvc| dvc.commit) - .max() - .unwrap_or(0) +/// Why a `DoViewChange` body could not be read as a suffix. +/// +/// Dropped whole rather than partially trusted: the merge indexes arithmetically +/// from the head op, so one bad offset misattributes a header, nack, or body. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DvcSuffixError { + /// Body length is not a whole number of headers. + NotHeaderMultiple { body_len: usize }, + /// More entries than the bitsets can address. + TooManyEntries { count: usize }, + /// An entry is not a valid `PrepareHeader` bit pattern. + MalformedHeader { index: usize }, + /// Entries are not consecutive descending from the head op. + OpOutOfOrder { + index: usize, + expected: u64, + found: u64, + }, + /// A bitset addresses an entry the body does not contain. + BitsetBeyondSuffix { count: usize }, + /// An entry's identity checksum does not match its own contents. + ChecksumMismatch { index: usize }, + /// A lower entry claims a timestamp at or after the entry above it. + TimestampNotDecreasing { index: usize }, + /// Consecutive entries do not hash-chain. + ChainBreak { index: usize }, } -/// Reset the DVC quorum array. -pub const fn dvc_reset(array: &mut DvcQuorumArray) { - *array = dvc_quorum_array_empty(); +impl std::fmt::Display for DvcSuffixError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::NotHeaderMultiple { body_len } => write!( + f, + "do_view_change body of {body_len} bytes is not a whole number of {} -byte headers", + size_of::() + ), + Self::TooManyEntries { count } => write!( + f, + "do_view_change suffix of {count} entries exceeds the maximum {DVC_HEADERS_MAX}" + ), + Self::MalformedHeader { index } => { + write!( + f, + "do_view_change suffix entry {index} is not a prepare header" + ) + } + Self::OpOutOfOrder { + index, + expected, + found, + } => write!( + f, + "do_view_change suffix entry {index} carries op {found}, expected {expected}" + ), + Self::BitsetBeyondSuffix { count } => write!( + f, + "do_view_change bitset addresses an entry past the {count}-entry suffix" + ), + Self::ChecksumMismatch { index } => write!( + f, + "do_view_change suffix entry {index} does not match its own identity checksum" + ), + Self::TimestampNotDecreasing { index } => write!( + f, + "do_view_change suffix entry {index} does not predate the entry above it" + ), + Self::ChainBreak { index } => write!( + f, + "do_view_change suffix entry {index} does not chain to the entry above it" + ), + } + } } -/// Iterator over all stored DVCs. -// TODO: add #[must_use] -- pure iterator query, callers should not ignore. -pub fn dvc_iter(array: &DvcQuorumArray) -> impl Iterator { - array.iter().filter_map(|m| m.as_ref()) +impl std::error::Error for DvcSuffixError {} + +/// Read a suffix out of a `DoViewChange` body. +/// +/// `head_op` is the sender's `header.op`; entries run consecutively down from it, +/// blanks included, so slot `i` is unambiguously op `head_op - i`. An empty body +/// yields an empty suffix, which is what a sender with nothing to describe sends. +/// +/// `body` must carry [`PrepareHeader`]'s alignment: the cast below is checked, so an +/// unaligned body reports every entry as [`DvcSuffixError::MalformedHeader`] instead +/// of what is actually wrong. Real frames clear this because the body starts a whole +/// number of 256-byte headers into an aligned buffer; the debug assert catches a +/// hand-built one. +/// +/// # Errors +/// [`DvcSuffixError`] when the body is not a consecutive run of valid prepare +/// headers descending from `head_op`, or a bitset addresses a missing entry. +pub fn dvc_suffix_decode( + body: &[u8], + head_op: u64, + nack_bitset: u128, + present_bitset: u128, +) -> Result { + debug_assert!( + body.is_empty() + || body + .as_ptr() + .addr() + .is_multiple_of(align_of::()), + "suffix body must be aligned for PrepareHeader" + ); + let header_size = size_of::(); + if !body.len().is_multiple_of(header_size) { + return Err(DvcSuffixError::NotHeaderMultiple { + body_len: body.len(), + }); + } + let count = body.len() / header_size; + if count > DVC_HEADERS_MAX { + return Err(DvcSuffixError::TooManyEntries { count }); + } + if count < DVC_HEADERS_MAX { + let beyond = !((1u128 << count) - 1); + if nack_bitset & beyond != 0 || present_bitset & beyond != 0 { + return Err(DvcSuffixError::BitsetBeyondSuffix { count }); + } + } + + let mut headers = Vec::with_capacity(count); + // The entry above the current one, skipping blanks: high-to-low, so the child. + let mut child: Option = None; + for index in 0..count { + let chunk = &body[index * header_size..(index + 1) * header_size]; + let header = bytemuck::checked::try_from_bytes::(chunk) + .map_err(|_| DvcSuffixError::MalformedHeader { index })?; + // The cast proves the enums, not the reserved regions. A dirty reserved byte + // with `checksum == 0` reads as Valid here (blanks are classified by exact + // struct equality) AND skips the identity recompute below, so it conflicts + // with every honest header at that op and no canonical one can be picked. + header + .validate() + .map_err(|_| DvcSuffixError::MalformedHeader { index })?; + let expected = head_op + .checked_sub(index as u64) + .ok_or(DvcSuffixError::OpOutOfOrder { + index, + expected: 0, + found: header.op, + })?; + if header.op != expected { + return Err(DvcSuffixError::OpOutOfOrder { + index, + expected, + found: header.op, + }); + } + + if matches!(dvc_header_kind(header), DvcHeaderKind::Valid) { + // Recompute rather than trust the field. Otherwise one bit flipped in + // transit becomes a canonical header no replica holds, honest senders + // read as disagreeing, and a corrupted frame turns into a nack quorum + // against a committed op. The on-disk sentinel is skipped: a suffix is read + // out of the journal, which may hold entries a pre-seal build wrote, and + // partition-plane prepares are unsealed by construction. + if header.checksum != CHECKSUM_UNSEALED && header.identity_checksum() != header.checksum + { + return Err(DvcSuffixError::ChecksumMismatch { index }); + } + if let Some(child) = child { + // Timestamps never run forwards down the log, and consecutive entries + // hash-chain. A frame breaking either describes a log that cannot exist. + // + // `view` is NOT checked. Monotone along one log, but a suffix is two: + // `build_dvc_suffix` stitches the journal over the adopted view's + // headers, and `restamp_prepare_view` rewrites `view` in place, so a + // held op keeps whichever view delivered it. A hole below one is enough + // to make an honest sender's suffix look regressed, and rejecting drops + // its vote forever (the retransmit is byte-identical). Nothing reads a + // suffix entry's `view`; the merge ranks by the message's `log_view`. + if header.timestamp >= child.timestamp { + return Err(DvcSuffixError::TimestampNotDecreasing { index }); + } + if header.op + 1 == child.op + && header.checksum != CHECKSUM_UNSEALED + && child.parent != header.checksum + { + return Err(DvcSuffixError::ChainBreak { index }); + } + } + child = Some(*header); + } + headers.push(*header); + } + + Ok(DvcSuffix::new(headers, nack_bitset, present_bitset)) } diff --git a/core/harness_derive/src/attrs.rs b/core/harness_derive/src/attrs.rs index 7a895b2bb1..ee4ff391e2 100644 --- a/core/harness_derive/src/attrs.rs +++ b/core/harness_derive/src/attrs.rs @@ -636,11 +636,8 @@ mod tests { #[test] fn parse_server_executable_path() { - let attrs: IggyTestAttrs = syn::parse_quote!(server(executable_path = "iggy-server-ng")); - assert_eq!( - attrs.server.executable_path.as_deref(), - Some("iggy-server-ng") - ); + let attrs: IggyTestAttrs = syn::parse_quote!(server(executable_path = "iggy-server")); + assert_eq!(attrs.server.executable_path.as_deref(), Some("iggy-server")); } #[test] diff --git a/core/harness_derive/src/codegen.rs b/core/harness_derive/src/codegen.rs index 95e8e987b7..76e2b664ad 100644 --- a/core/harness_derive/src/codegen.rs +++ b/core/harness_derive/src/codegen.rs @@ -125,8 +125,8 @@ fn generate_variants(attrs: &IggyTestAttrs) -> Vec { } /// No transport is gated out of the VSR test matrix. Retained as the single -/// seam where a transport could be excluded from `--features vsr` if one is -/// ever unsupported by the next-gen server again. +/// seam where a transport could be excluded from the matrix if the server +/// ever drops support for one again. fn vsr_transport_cfg(_transport: Transport) -> TokenStream { quote!() } diff --git a/core/integration/Cargo.toml b/core/integration/Cargo.toml index d4d0bdbd53..bb7bdc51b2 100644 --- a/core/integration/Cargo.toml +++ b/core/integration/Cargo.toml @@ -34,7 +34,6 @@ ignored = ["cfg_aliases", "rust-s3"] ci-qemu = [] default = ["login-session"] login-session = ["dep:zbus-secret-service-keyring-store"] -vsr = ["dep:consensus", "dep:journal", "iggy/vsr"] [dependencies] assert_cmd = { workspace = true } @@ -46,9 +45,9 @@ bytes = { workspace = true } compio = { workspace = true } configs = { workspace = true } configs_derive = { workspace = true } -# vsr-only: decode a metadata replica's durable `VsrState` off disk in the -# superblock recovery test. -consensus = { workspace = true, optional = true } +# Decodes a metadata replica's durable `VsrState` off disk in the superblock +# recovery test. +consensus = { workspace = true } ctor = { workspace = true } deltalake = { workspace = true } dtor = { workspace = true } @@ -64,9 +63,9 @@ iggy_common = { workspace = true } # `build_label` function — keeping the test and production label format in lock-step. iggy_connector_doris_sink = { path = "../connectors/sinks/doris_sink" } iggy_connector_sdk = { workspace = true, features = ["api"] } -# vsr-only: locate and decode the on-disk superblock slot files in the recovery -# test (`SLOT_FILE_NAMES`, `decode_slots`). -journal = { workspace = true, optional = true } +# Locates and decodes the on-disk superblock slot files in the recovery test +# (`SLOT_FILE_NAMES`, `decode_slots`). +journal = { workspace = true } jsonwebtoken = { workspace = true } keyring-core = { workspace = true } lazy_static = { workspace = true } @@ -89,7 +88,6 @@ secrecy = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } serial_test = { workspace = true } -server = { workspace = true } sqlx = { workspace = true } sysinfo = { workspace = true } tempfile = { workspace = true } diff --git a/core/integration/src/bench_utils.rs b/core/integration/src/bench_utils.rs index de2de07d1b..dcfbde1c84 100644 --- a/core/integration/src/bench_utils.rs +++ b/core/integration/src/bench_utils.rs @@ -34,10 +34,9 @@ const DEFAULT_NUMBER_OF_STREAMS: u64 = 8; // Generous for a few MB of traffic even in debug builds, and deliberately // UNDER nextest's harness timeout (`.config/nextest.toml` sigkills at // 60s x 5): a longer wait here would never fire, taking the capture dump and -// the `--features vsr` hint below with it. Exists because a protocol mismatch -// (an SDK framing the server does not speak, e.g. a default-features -// iggy-bench against a vsr cluster) hangs both sides silently instead of -// erroring. +// the stale-binary hint below with it. Exists because a stale prebuilt +// iggy-bench speaking an outdated protocol hangs both sides silently +// instead of erroring. const BENCH_WAIT_TIMEOUT: Duration = Duration::from_secs(240); pub fn run_bench_and_wait_for_finish( @@ -156,9 +155,9 @@ pub fn run_bench_and_wait_for_finish( assert!( !timed_out, - "iggy-bench did not finish within {BENCH_WAIT_TIMEOUT:?}; if the server \ - runs in vsr mode, make sure iggy-bench was built with --features vsr \ - (the SDK framing is chosen at compile time)" + "iggy-bench did not finish within {BENCH_WAIT_TIMEOUT:?}; the harness \ + spawns the prebuilt binary, so make sure iggy-bench was rebuilt \ + alongside the server (a stale binary hangs instead of erroring)" ); assert!(status.is_some_and(|status| status.success())); } diff --git a/core/integration/src/harness/config/resolve.rs b/core/integration/src/harness/config/resolve.rs index 99d89f4d30..e5f36f60a4 100644 --- a/core/integration/src/harness/config/resolve.rs +++ b/core/integration/src/harness/config/resolve.rs @@ -17,15 +17,15 @@ //! Runtime validation and resolution of config paths to environment variables. -use configs::ConfigEnvMappings; -use server::configs::server::ServerConfig; +use configs::server::ServerConfig; +use configs::{ConfigEnvMappings, EnvVarMapping}; use std::collections::HashMap; /// Resolve config paths to environment variable names. /// /// Takes a map of dot-notation config paths (e.g., "segment.size") and their values, -/// validates them against the `ServerConfig` mappings, and returns the corresponding -/// environment variable names with values. +/// validates them against the `ServerConfig` mappings, and returns the +/// corresponding environment variable names with values. /// /// # Implicit defaults /// @@ -51,13 +51,8 @@ pub fn resolve_config_paths( path.as_str() }; - let mapping = ServerConfig::find_by_config_path(resolved_path) - .or_else(|| ServerConfig::find_by_config_path(&format!("system.{}", resolved_path))) - // Fields that exist only in the next-gen server's config (e.g. - // `metadata.*`, `cluster.*`, `message_bus.*`). Env names share - // the `IGGY_` prefix, so the resolved variable reaches whichever - // binary the harness spawns; the legacy server ignores unknowns. - .or_else(|| configs::server_ng::ServerNgConfig::find_by_config_path(resolved_path)); + let mapping = find_mapping(resolved_path) + .or_else(|| find_mapping(&format!("system.{}", resolved_path))); match mapping { Some(m) => { @@ -96,24 +91,18 @@ pub fn resolve_config_paths( } // Auto-enable override_defaults when socket settings are customized - if needs_tcp_socket_override - && let Some(m) = ServerConfig::find_by_config_path("tcp.socket.override_defaults") - { + if needs_tcp_socket_override && let Some(m) = find_mapping("tcp.socket.override_defaults") { env_vars .entry(m.env_name.to_string()) .or_insert_with(|| "true".to_string()); } - if needs_quic_socket_override - && let Some(m) = ServerConfig::find_by_config_path("quic.socket.override_defaults") - { + if needs_quic_socket_override && let Some(m) = find_mapping("quic.socket.override_defaults") { env_vars .entry(m.env_name.to_string()) .or_insert_with(|| "true".to_string()); } // Auto-enable encryption when key is set - if needs_encryption_enabled - && let Some(m) = ServerConfig::find_by_config_path("system.encryption.enabled") - { + if needs_encryption_enabled && let Some(m) = find_mapping("system.encryption.enabled") { env_vars .entry(m.env_name.to_string()) .or_insert_with(|| "true".to_string()); @@ -122,6 +111,10 @@ pub fn resolve_config_paths( Ok(env_vars) } +fn find_mapping(path: &str) -> Option<&'static EnvVarMapping> { + ServerConfig::find_by_config_path(path) +} + fn levenshtein(a: &str, b: &str) -> usize { let a_len = a.len(); let b_len = b.len(); @@ -184,7 +177,7 @@ fn find_similar_paths(unknown: &str) -> Vec { }) .collect(); - candidates.sort_by_key(|(_, score)| *score); + candidates.sort_by_key(|(path, score)| (*score, *path)); candidates .into_iter() diff --git a/core/integration/src/harness/config/server.rs b/core/integration/src/harness/config/server.rs index 14c4433c3c..07c7f8e940 100644 --- a/core/integration/src/harness/config/server.rs +++ b/core/integration/src/harness/config/server.rs @@ -54,12 +54,12 @@ mod tests { fn test_server_config_builder() { let config = TestServerConfig::builder() .quic_enabled(false) - .executable_path("iggy-server-ng") + .executable_path("iggy-server") .extra_envs(HashMap::from([("FOO".to_string(), "BAR".to_string())])) .build(); assert!(!config.quic_enabled); - assert_eq!(config.executable_path.as_deref(), Some("iggy-server-ng")); + assert_eq!(config.executable_path.as_deref(), Some("iggy-server")); assert_eq!(config.extra_envs.get("FOO"), Some(&"BAR".to_string())); } diff --git a/core/integration/src/harness/handle/server.rs b/core/integration/src/harness/handle/server.rs index 34c5498072..9e23d077ef 100644 --- a/core/integration/src/harness/handle/server.rs +++ b/core/integration/src/harness/handle/server.rs @@ -92,15 +92,7 @@ impl std::fmt::Debug for ServerHandle { impl ServerHandle { fn default_server_binary() -> &'static str { - #[cfg(feature = "vsr")] - { - "iggy-server-ng" - } - - #[cfg(not(feature = "vsr"))] - { - "iggy-server" - } + "iggy-server" } fn launched_binary(&self) -> String { @@ -837,13 +829,9 @@ impl TestBinary for ServerHandle { // trusts (rcgen self-signed certs share the same subject DN), which // rustls rejects as `BadSignature`. Generate only when absent so all // nodes and clients share one keypair; this also keeps the - // certificate stable across a restart. The legacy single-node - // harness keeps its regenerate-per-start behavior. - #[cfg(feature = "vsr")] + // certificate stable across a restart. let should_generate = !(cert_dir.join("test_cert.pem").exists() && cert_dir.join("test_key.pem").exists()); - #[cfg(not(feature = "vsr"))] - let should_generate = true; if should_generate { generate_test_certificates(cert_dir.to_str().unwrap()).map_err(|e| { TestBinaryError::InvalidState { @@ -895,13 +883,6 @@ impl TestBinary for ServerHandle { } command.envs(&self.envs); - // Legacy clustering elects node 0 externally and requires explicit followers. - // VSR/server-ng elects its own primary and should see symmetric node startup. - #[cfg(not(feature = "vsr"))] - if self.server_id > 0 { - command.arg("--follower"); - } - // `--replica-id` is the single identity input expected by the // server when cluster mode is enabled; all other cluster config is // byte-identical across nodes. Pass the harness's `server_id` diff --git a/core/integration/src/harness/orchestrator/builder.rs b/core/integration/src/harness/orchestrator/builder.rs index c73cd273fa..4eb8b3d78f 100644 --- a/core/integration/src/harness/orchestrator/builder.rs +++ b/core/integration/src/harness/orchestrator/builder.rs @@ -297,7 +297,7 @@ fn build_servers( fn default_cluster_node_count() -> usize { // Suite-wide override: run every test that does not pin `cluster_nodes` // against an N-node cluster (e.g. `IGGY_TEST_CLUSTER_NODES=1` probes the - // whole vsr suite on a single server-ng node). Explicit attrs win. + // whole vsr suite on a single the server node). Explicit attrs win. if let Some(count) = std::env::var("IGGY_TEST_CLUSTER_NODES") .ok() .and_then(|value| value.parse::().ok()) @@ -306,15 +306,7 @@ fn default_cluster_node_count() -> usize { return count; } - #[cfg(feature = "vsr")] - { - 3 - } - - #[cfg(not(feature = "vsr"))] - { - 1 - } + 3 } fn build_cluster_envs( @@ -332,7 +324,6 @@ fn build_cluster_envs( envs.insert("IGGY_CLUSTER_ENABLED".to_string(), "true".to_string()); envs.insert("IGGY_CLUSTER_NAME".to_string(), cluster_name.to_string()); - #[cfg(feature = "vsr")] envs.insert( "IGGY_MESSAGE_BUS_RECONNECT_PERIOD".to_string(), "100ms".to_string(), diff --git a/core/integration/src/harness/orchestrator/harness.rs b/core/integration/src/harness/orchestrator/harness.rs index 4a661578a9..03e8257003 100644 --- a/core/integration/src/harness/orchestrator/harness.rs +++ b/core/integration/src/harness/orchestrator/harness.rs @@ -26,16 +26,12 @@ use crate::harness::handle::{ use crate::harness::traits::{Restartable, TestBinary}; use futures::executor::block_on; use iggy::prelude::{ClientWrapper, IggyClient}; -#[cfg(feature = "vsr")] use iggy_common::Client; use iggy_common::TransportProtocol; use std::path::Path; use std::sync::Arc; -#[cfg(feature = "vsr")] use std::time::{Duration, Instant}; -#[cfg(feature = "vsr")] use tokio::time::{sleep, timeout}; -#[cfg(feature = "vsr")] use tracing::warn; use wiremock::matchers::{method, path}; use wiremock::{Mock, MockServer, ResponseTemplate}; @@ -154,20 +150,13 @@ impl TestHarness { self.jwks_server = Some(mock_server); } - // Legacy single-server harness: plain start, no cluster readiness. - #[cfg(not(feature = "vsr"))] - for server in &mut self.servers { - server.start()?; - } - - // Cluster startup (vsr) can hit a transient replica-handshake blip that + // Cluster startup can hit a transient replica-handshake blip that // leaves the mesh incomplete (a peer link drops mid-handshake, so a // node never reaches "all peers connected"). Rather than fail the whole // test on a startup blip, retry spawn + mesh-readiness a few times, // tearing down and respawning between attempts. `ServerHandle::start` // truncates the captured stdout (`File::create`), so the readiness // log-grep never matches a stale marker from a prior attempt. - #[cfg(feature = "vsr")] { const CLUSTER_STARTUP_ATTEMPTS: usize = 3; for attempt in 1..=CLUSTER_STARTUP_ATTEMPTS { @@ -219,7 +208,6 @@ impl TestHarness { Ok(()) } - #[cfg(feature = "vsr")] async fn wait_for_cluster_ready(&self) -> Result<(), TestBinaryError> { { if self.servers.len() <= 1 { diff --git a/core/integration/tests/cli/message/test_message_flush_command.rs b/core/integration/tests/cli/message/test_message_flush_command.rs index 9658943dbf..6b1dfde463 100644 --- a/core/integration/tests/cli/message/test_message_flush_command.rs +++ b/core/integration/tests/cli/message/test_message_flush_command.rs @@ -29,10 +29,7 @@ use iggy::prelude::Client; use iggy::prelude::Identifier; use iggy::prelude::IggyExpiry; use iggy::prelude::MaxTopicSize; -#[cfg(feature = "vsr")] use predicates::str::contains; -#[cfg(not(feature = "vsr"))] -use predicates::str::diff; use serial_test::parallel; use std::str::FromStr; @@ -150,21 +147,12 @@ impl IggyCmdTestCase for TestMessageFetchCmd { } ); - // server-ng has no on-demand flush primitive: FLUSH_UNSAVED_BUFFER + // The server has no on-demand flush primitive: FLUSH_UNSAVED_BUFFER // surfaces a typed FeatureUnavailable (see flush_vsr.rs), so the CLI // reports a flush problem instead of success. - #[cfg(feature = "vsr")] command_state.failure().stderr(contains(format!( "Problem flushing messages {identification_part}" ))); - - #[cfg(not(feature = "vsr"))] - { - let message = format!( - "Executing flush messages {identification_part}\nFlushed messages {identification_part}\n" - ); - command_state.success().stdout(diff(message)); - } } async fn verify_server_state(&self, client: &dyn Client) { diff --git a/core/integration/tests/cli/stream/test_stream_purge_command.rs b/core/integration/tests/cli/stream/test_stream_purge_command.rs index 0a472120aa..e0e7bace5c 100644 --- a/core/integration/tests/cli/stream/test_stream_purge_command.rs +++ b/core/integration/tests/cli/stream/test_stream_purge_command.rs @@ -115,7 +115,7 @@ impl IggyCmdTestCase for TestStreamPurgeCmd { } async fn verify_server_state(&self, client: &dyn Client) { - // server-ng purge is eventually consistent: the partition reset (and + // The server purge is eventually consistent: the partition reset (and // its stats zeroing) runs in the reconciler after the metadata commit // the purge command awaited. Legacy is synchronous and satisfies this // on the first poll. diff --git a/core/integration/tests/cli/system/test_cli_session_scenario.rs b/core/integration/tests/cli/system/test_cli_session_scenario.rs index 407f40238e..8b7d245af5 100644 --- a/core/integration/tests/cli/system/test_cli_session_scenario.rs +++ b/core/integration/tests/cli/system/test_cli_session_scenario.rs @@ -91,12 +91,12 @@ pub async fn should_be_successful() { iggy_cmd_test .execute_test(TestMeCmd::new( TransportProtocol::Tcp, - Scenario::FailureDueToSessionTimeout(server_address), + Scenario::FailureDueToSessionTimeout, )) .await; // After the session timed out, logging in again with username and password // must recover: the CLI drops the dead session token and recreates it, - // rather than wedging on the expired credential (regression for server-ng, + // rather than wedging on the expired credential (regression for the server, // where the terminal auth failure is opaque and self-heal never happened). iggy_cmd_test .execute_test(TestLoginCmd::new( diff --git a/core/integration/tests/cli/system/test_me_command.rs b/core/integration/tests/cli/system/test_me_command.rs index a0bc269fc9..cb67e457e6 100644 --- a/core/integration/tests/cli/system/test_me_command.rs +++ b/core/integration/tests/cli/system/test_me_command.rs @@ -30,7 +30,7 @@ pub(super) enum Scenario { SuccessWithCredentials, SuccessWithoutCredentials, FailureWithoutCredentials, - FailureDueToSessionTimeout(String), + FailureDueToSessionTimeout, } // Helper trait to add command-specific methods to TransportProtocol @@ -78,7 +78,7 @@ impl IggyCmdTestCase for TestMeCmd { match &self.scenario { Scenario::SuccessWithCredentials => command.with_env_credentials(), Scenario::FailureWithoutCredentials => command.disable_backtrace(), - Scenario::FailureDueToSessionTimeout(_) => command.disable_backtrace(), + Scenario::FailureDueToSessionTimeout => command.disable_backtrace(), _ => command, } } @@ -99,19 +99,12 @@ impl IggyCmdTestCase for TestMeCmd { .failure() .stderr(diff("Error: CommandError(Iggy command line tool error\n\nCaused by:\n Missing iggy server credentials)\n")); } - Scenario::FailureDueToSessionTimeout(server_address) => { - #[cfg(not(feature = "vsr"))] - command_state.failure().stderr(diff(format!("Error: CommandError(Login session expired for Iggy server: {server_address}, please login again or use other authentication method)\n"))); - // server-ng maps an expired/invalid stored session to a generic - // login-with-token failure rather than the legacy "session - // expired" message. - #[cfg(feature = "vsr")] - { - let _ = server_address; - command_state.failure().stderr(diff( - "Error: CommandError(Problem with server login with token)\n", - )); - } + Scenario::FailureDueToSessionTimeout => { + // An expired or invalid stored session surfaces as a generic + // login-with-token failure. + command_state.failure().stderr(diff( + "Error: CommandError(Problem with server login with token)\n", + )); } } } diff --git a/core/integration/tests/cli/topic/test_topic_purge_command.rs b/core/integration/tests/cli/topic/test_topic_purge_command.rs index ab36525ebd..4d1323c6f6 100644 --- a/core/integration/tests/cli/topic/test_topic_purge_command.rs +++ b/core/integration/tests/cli/topic/test_topic_purge_command.rs @@ -141,7 +141,7 @@ impl IggyCmdTestCase for TestTopicPurgeCmd { } async fn verify_server_state(&self, client: &dyn Client) { - // server-ng purge is eventually consistent: the partition reset (and + // The server purge is eventually consistent: the partition reset (and // its stats zeroing) runs in the reconciler after the metadata commit // the purge command awaited. Legacy is synchronous and satisfies this // on the first poll. diff --git a/core/integration/tests/cluster/client_table_restart.rs b/core/integration/tests/cluster/client_table_restart.rs index 4e572eb5c4..d2ea0ea59a 100644 --- a/core/integration/tests/cluster/client_table_restart.rs +++ b/core/integration/tests/cluster/client_table_restart.rs @@ -71,8 +71,6 @@ //! work later settles on an explicit resume handshake, adjust `resume_request` //! to speak it -- but it must stay credential-bearing. -#![cfg(feature = "vsr")] - use bytes::Bytes; use iggy::prelude::*; use iggy_binary_protocol::codec::{WireDecode, WireEncode}; @@ -80,7 +78,6 @@ use iggy_binary_protocol::consensus::{ Command2, Operation, ReplyHeader, RequestHeader, read_size_field, result_code, result_section_len, }; -use iggy_binary_protocol::namespace::METADATA_CONSENSUS_NAMESPACE; use iggy_binary_protocol::requests::streams::CreateStreamRequest; use iggy_binary_protocol::requests::users::LoginRegisterRequest; use iggy_binary_protocol::responses::users::LoginRegisterResponse; @@ -278,10 +275,6 @@ fn request_header( client: CLIENT_ID, session, request, - namespace: match operation { - Operation::Register => METADATA_CONSENSUS_NAMESPACE, - _ => 0, - }, ..Default::default() } } diff --git a/core/integration/tests/cluster/metadata_checkpoint_restart.rs b/core/integration/tests/cluster/metadata_checkpoint_restart.rs index 75ae60f6eb..499b66bae9 100644 --- a/core/integration/tests/cluster/metadata_checkpoint_restart.rs +++ b/core/integration/tests/cluster/metadata_checkpoint_restart.rs @@ -45,8 +45,6 @@ //! the `forced checkpoint completed` markers below pin it rather than trust //! it. -#![cfg(feature = "vsr")] - use super::client_table_restart::{ commit_request, create_stream_payload, register, resume_request, tcp_addr, tcp_addrs, }; @@ -119,8 +117,10 @@ async fn await_checkpoint_on_all_nodes(harness: &TestHarness, generation: usize) // a checkpoint, so the transfer descriptor's `commit_op == snapshot_seq` and // the post-install tail repair has nothing to fetch (`commit_min == // commit_max` skips it). The below-floor retry then proves the reply ring -// rode the transferred table: request 191's reply was minted at op 192, which -// every node drained out of its WAL at that same checkpoint. +// rode the transferred table: request 191's reply was minted at op 192, and +// replay starts at `snapshot_seq + 1`, so no node re-executes it. The +// checkpoint drain keeps op 192's entry as the commit-point header a +// `DoViewChange` needs, but never replays it. #[iggy_harness(cluster_nodes = 3, server(metadata.journal_slots = "256"))] async fn given_drained_journal_when_node_restarts_should_install_snapshot_only( harness: &mut TestHarness, @@ -327,7 +327,7 @@ async fn wait_for_stream(harness: &TestHarness, stream: &str) -> IggyClient { } // Metadata checkpoint-fold recovery across a solo restart, over -// `iggy-server-ng`'s production snapshot and WAL path. +// `iggy-server`'s production snapshot and WAL path. // // Between checkpoints a replica recovers its metadata by replaying the WAL. Once the // WAL fills, the `SnapshotCoordinator` checkpoints: it persists `snapshot.bin`, pairs diff --git a/core/integration/tests/cluster/metadata_state_transfer.rs b/core/integration/tests/cluster/metadata_state_transfer.rs index 2759464125..79d78fd9c7 100644 --- a/core/integration/tests/cluster/metadata_state_transfer.rs +++ b/core/integration/tests/cluster/metadata_state_transfer.rs @@ -32,8 +32,6 @@ //! functional assert (post-restart continuation commits cluster-wide) rides //! on top. -#![cfg(feature = "vsr")] - use super::client_table_restart::{ commit_request, create_stream_payload, register, resume_request, tcp_addr, tcp_addrs, }; diff --git a/core/integration/tests/cluster/multi_shard_partition_convergence.rs b/core/integration/tests/cluster/multi_shard_partition_convergence.rs index a26fbd293f..5379074b94 100644 --- a/core/integration/tests/cluster/multi_shard_partition_convergence.rs +++ b/core/integration/tests/cluster/multi_shard_partition_convergence.rs @@ -48,8 +48,6 @@ //! owner, a park queue that never drains, or a fence that denies forever -- //! since every one of those surfaces as a failed send or a short poll. -#![cfg(feature = "vsr")] - use iggy::prelude::*; use integration::harness::TestHarness; use integration::iggy_harness; diff --git a/core/integration/tests/cluster/partition_state_transfer.rs b/core/integration/tests/cluster/partition_state_transfer.rs index 13413d8866..fbc5c93919 100644 --- a/core/integration/tests/cluster/partition_state_transfer.rs +++ b/core/integration/tests/cluster/partition_state_transfer.rs @@ -27,8 +27,6 @@ //! `RangeEvicted`, the repaired window cannot connect to recovered state, //! and `complete_repair` returns the `FloorRefused` conversion trigger. -#![cfg(feature = "vsr")] - use std::path::{Path, PathBuf}; use std::str::FromStr; use std::time::{Duration, Instant}; @@ -40,7 +38,7 @@ use tokio::time::sleep; const STREAM_NAME: &str = "partition-transfer-stream"; const TOPIC_NAME: &str = "partition-transfer-topic"; -/// server-ng partition ids are 0-based (CreateTopic assigns them from 0). +/// Partition ids are 0-based (CreateTopic assigns them from 0). const PARTITION_ID: u32 = 0; /// Enough batches to push the evicted ring (capacity 64) well past the /// window a rejoiner could repair from op 1. @@ -127,7 +125,8 @@ async fn given_evicted_ring_when_fresh_node_joins_late_should_state_transfer_par await_marker(harness, 2, INSTALL_MARKER).await; // Disk proof on the rejoined node: transferred segment bytes and a - // persisted consumer-offset file (a single LE u64). + // persisted consumer-offset file (leading LE u64 offset, trailing + // checksum; see `partitions::offset_storage::encode_offset_record`). let data_path = harness.node(2).data_path(); // Each transferred batch is at least its 256-byte header; anything below // this floor is a truncated install, not the seeded 200 batches. @@ -147,8 +146,11 @@ async fn given_evicted_ring_when_fresh_node_joins_late_should_state_transfer_par let offsets_file = find_consumer_offset_file(&data_path) .expect("transferred consumer offset file exists on node 2"); let bytes = std::fs::read(&offsets_file).expect("read transferred consumer offset"); + let offset_bytes = bytes + .first_chunk::<8>() + .expect("offset file starts with a u64 offset"); assert_eq!( - u64::from_le_bytes(bytes.as_slice().try_into().expect("offset file is one u64")), + u64::from_le_bytes(*offset_bytes), STORED_CONSUMER_OFFSET, "the stored consumer offset must survive the transfer" ); @@ -610,7 +612,7 @@ fn find_consumer_offset_file(root: &Path) -> Option { path.parent() .and_then(Path::file_name) .is_some_and(|name| name == "consumers") - && std::fs::metadata(path).is_ok_and(|metadata| metadata.len() == 8) + && std::fs::metadata(path).is_ok_and(|metadata| metadata.len() >= 8) }) } diff --git a/core/integration/tests/config_provider/mod.rs b/core/integration/tests/config_provider/mod.rs index b2d2c6d366..32d0d9aec6 100644 --- a/core/integration/tests/config_provider/mod.rs +++ b/core/integration/tests/config_provider/mod.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use configs::server::ServerConfig; use configs::{ConfigEnvMappings, ConfigProvider, TypedEnvProvider}; use configs_derive::ConfigEnv; use figment::providers::{Format, Toml}; @@ -22,7 +23,6 @@ use figment::value::Dict; use figment::{Figment, Provider}; use serde::{Deserialize, Serialize}; use serial_test::serial; -use server::configs::server::ServerConfig; use std::env; use std::path::PathBuf; diff --git a/core/integration/tests/data_integrity/mod.rs b/core/integration/tests/data_integrity/mod.rs index 5a3815d2d0..2418ef39b5 100644 --- a/core/integration/tests/data_integrity/mod.rs +++ b/core/integration/tests/data_integrity/mod.rs @@ -16,29 +16,26 @@ // under the License. // Partially vsr-gated inside the module: the remaining gates cover -// `flush_unsaved_buffer`, which server-ng answers `FeatureUnavailable` and +// `flush_unsaved_buffer`, which the server answers `FeatureUnavailable` and // which the eager-flush server envs replace under vsr. The bench-fill test // itself runs under vsr since PARTITION-plane state transfer landed, but the // harness spawns `iggy-bench` off disk with no cargo build-graph edge, so the -// binary must have been built `--features vsr` or its login hangs on the -// framing mismatch. +// binary must be freshly built or a stale build hangs on login. mod verify_after_server_restart; mod verify_user_login_after_restart; // Not restart-based: it creates a user + PAT, stops the server, and greps the -// data dir for plaintext. No replica catch-up needed, and server-ng hashes the +// data dir for plaintext. No replica catch-up needed, and the server hashes the // password / PAT before either reaches the WAL, so it runs under vsr too. mod verify_no_plaintext_credentials_on_disk; -// The cooperative-rebalance matrix runs under vsr too: it exercises server-ng's -// consumer-group rebalancing (a VSR feature). Green at 95/95. +// The cooperative-rebalance matrix exercises the server's consumer-group +// rebalancing (a VSR capability). Green at 95/95. mod verify_consumer_group_partition_assignment; // Cross-replica on-disk data identity is VSR-only. -#[cfg(feature = "vsr")] mod verify_cluster_replica_data_identical; // Auto-commit offset replication is inherently a multi-node (VSR) property: the // backup only holds the offset if the poll's auto-commit rode consensus. -#[cfg(feature = "vsr")] mod verify_auto_commit_offset_replicates; diff --git a/core/integration/tests/data_integrity/verify_after_server_restart.rs b/core/integration/tests/data_integrity/verify_after_server_restart.rs index 71a6164945..1bed4408e8 100644 --- a/core/integration/tests/data_integrity/verify_after_server_restart.rs +++ b/core/integration/tests/data_integrity/verify_after_server_restart.rs @@ -40,18 +40,16 @@ fn build_server_config(cache_setting: &str) -> TestServerConfig { "IGGY_SYSTEM_SEGMENT_CACHE_INDEXES".to_string(), cache_setting.to_string(), ); - // server-ng flushes on the journal thresholds (no flush primitive), so + // The server flushes on the journal thresholds (no flush primitive), so // force every committed batch straight to disk: the restart asserts // below need everything durable, and the explicit flush calls are // cfg'd out under vsr (`flush_unsaved_buffer` answers // FeatureUnavailable there and is slated for removal). Legacy keeps its // shipped buffered defaults; the flush loops below are its barrier. - #[cfg(feature = "vsr")] extra_envs.insert( "IGGY_SYSTEM_PARTITION_MESSAGES_REQUIRED_TO_SAVE".to_string(), "1".to_string(), ); - #[cfg(feature = "vsr")] extra_envs.insert( "IGGY_SYSTEM_PARTITION_ENFORCE_FSYNC".to_string(), "true".to_string(), @@ -61,20 +59,23 @@ fn build_server_config(cache_setting: &str) -> TestServerConfig { // TODO(numminex) - Move the message generation method from benchmark run to a special method. // -// Under vsr this runs against a 3-node cluster and needs two adaptations: -// the durability barrier is the eager-flush envs in `build_server_config` -// (`flush_unsaved_buffer` answers FeatureUnavailable there, so the explicit -// flush loops are cfg'd out), and `iggy-bench` must be built with -// `--features vsr` because the SDK framing is chosen at compile time. A -// default-features bench binary never completes a frame against server-ng -// and the run trips the bench timeout in `run_bench_and_wait_for_finish`. +// The durability barrier is the eager-flush envs in `build_server_config` +// (`flush_unsaved_buffer` answers FeatureUnavailable on VSR, so there is no +// explicit flush loop), and `iggy-bench` must be freshly built: the harness +// spawns the prebuilt binary, and a stale one never completes a frame +// against the server, tripping the bench timeout in +// `run_bench_and_wait_for_finish`. #[test_matrix( [cache_all(), cache_open_segment(), cache_none()] )] #[tokio::test] #[parallel] async fn should_fill_data_and_verify_after_restart(cache_setting: &'static str) { + // Restart scenarios run single-node: restarting a node in a multi-node + // cluster trips a known partitions-plane view-change stall, tracked + // separately. let mut harness = TestHarness::builder() + .cluster_nodes(1) .server(build_server_config(cache_setting)) .build() .unwrap(); @@ -104,16 +105,6 @@ async fn should_fill_data_and_verify_after_restart(cache_setting: &'static str) let client = harness.tcp_root_client().await.unwrap(); let topic_id = Identifier::numeric(0).unwrap(); - // Durability barrier on the legacy server only; server-ng persists - // eagerly via the config envs and answers FeatureUnavailable here. - #[cfg(not(feature = "vsr"))] - for i in 0..7 { - let stream_id = Identifier::numeric(i).unwrap(); - client - .flush_unsaved_buffer(&stream_id, &topic_id, 0, true) - .await - .unwrap(); - } // Create consumer groups to test persistence let consumer_group_names = ["test-cg-1", "test-cg-2", "test-cg-3"]; @@ -220,19 +211,6 @@ async fn should_fill_data_and_verify_after_restart(cache_setting: &'static str) // Connect and login to server let client = harness.tcp_root_client().await.unwrap(); - // Durability barrier on the legacy server only (see the first loop). - #[cfg(not(feature = "vsr"))] - { - let topic_id = Identifier::numeric(0).unwrap(); - for i in 0..7 { - let stream_id = Identifier::numeric(i).unwrap(); - client - .flush_unsaved_buffer(&stream_id, &topic_id, 0, true) - .await - .unwrap(); - } - } - // Save stats from the second server (should have double the data) let stats = client.get_stats().await.unwrap(); let actual_messages_size_bytes = stats.messages_size_bytes; @@ -325,6 +303,7 @@ async fn should_fill_data_and_verify_after_restart(cache_setting: &'static str) #[parallel] async fn should_handle_resource_deletion_and_restart() { let mut harness = TestHarness::builder() + .cluster_nodes(1) .server(TestServerConfig::default()) .build() .unwrap(); diff --git a/core/integration/tests/data_integrity/verify_auto_commit_offset_replicates.rs b/core/integration/tests/data_integrity/verify_auto_commit_offset_replicates.rs index 6f26dd983f..cb953b1b8c 100644 --- a/core/integration/tests/data_integrity/verify_auto_commit_offset_replicates.rs +++ b/core/integration/tests/data_integrity/verify_auto_commit_offset_replicates.rs @@ -150,8 +150,10 @@ async fn run(harness: &TestHarness) { /// The u64 offset persisted under any `offsets/consumers/` file in a node's /// data dir, or `None` when no such file has been written yet. Walks the tree so /// it is robust to the stream/topic/partition id layout; the test drives exactly -/// one consumer, so at most one such file exists. A zero-length read (persist -/// truncates before writing the 8 bytes) is treated as not-yet-written. +/// one consumer, so at most one such file exists. Reads the leading u64 of the +/// record: the file is offset + trailing checksum (see +/// `partitions::offset_storage::encode_offset_record`), and a shorter read +/// (persist truncates before writing) is treated as not-yet-written. fn read_replicated_consumer_offset(data_dir: &Path) -> Option { let mut stack: Vec = vec![data_dir.to_path_buf()]; while let Some(dir) = stack.pop() { @@ -178,9 +180,9 @@ fn read_replicated_consumer_offset(data_dir: &Path) -> Option { .is_some_and(|name| name == "offsets"); if is_consumer_offset && let Ok(bytes) = std::fs::read(&path) - && let Ok(array) = <[u8; 8]>::try_from(bytes.as_slice()) + && let Some(offset_bytes) = bytes.first_chunk::<8>() { - return Some(u64::from_le_bytes(array)); + return Some(u64::from_le_bytes(*offset_bytes)); } } } diff --git a/core/integration/tests/data_integrity/verify_consumer_group_partition_assignment.rs b/core/integration/tests/data_integrity/verify_consumer_group_partition_assignment.rs index b92c5bcf4d..8fac77b964 100644 --- a/core/integration/tests/data_integrity/verify_consumer_group_partition_assignment.rs +++ b/core/integration/tests/data_integrity/verify_consumer_group_partition_assignment.rs @@ -15,6 +15,25 @@ // specific language governing permissions and limitations // under the License. +//! Consumer-group partition assignment specs. +//! +//! # Why most specs here ask for a 60s server heartbeat +//! +//! `run_heartbeat_verifier` evicts a connection idle past `1.2 x +//! heartbeat.interval`, and a harness client NEVER pings on its own: the SDK's +//! pinger is spawned by `IggyClient::connect`, which the harness builder does +//! not call. A spec that joins a member and then spends its setup elsewhere +//! therefore races its own subject being reaped, and the failure surfaces as a +//! short `members_count` or a `StaleClient` rather than as anything about +//! assignment. At the former 2s interval the 16-consumer spec sat 2.27s into a +//! 2.4s deadline over quic -- under 5% margin. +//! +//! These specs assert ASSIGNMENT, not liveness, so the deadline is pushed out +//! of reach instead of being raced. The two that genuinely drive eviction keep +//! the short interval and are marked as such: they build members with +//! [`create_stale_tcp_client`], whose 1h client-side heartbeat means only the +//! server's verifier can ever remove them. + use iggy::prelude::*; use integration::iggy_harness; use std::collections::HashSet; @@ -60,6 +79,8 @@ async fn create_tcp_client(server_addr: &str) -> IggyClient { #[iggy_harness(server( heartbeat.enabled = true, + // Deliberately short: this spec drives the server's eviction path (see the + // module note), so the verifier must be able to reap a stale member. heartbeat.interval = "2s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true @@ -256,7 +277,7 @@ async fn should_not_duplicate_partition_assignments_after_stale_client_cleanup( #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "2s", + heartbeat.interval = "60s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -334,7 +355,7 @@ async fn should_not_reshuffle_partitions_when_new_member_joins(harness: &TestHar #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "2s", + heartbeat.interval = "60s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -416,7 +437,7 @@ async fn should_skip_revoked_partitions_in_round_robin(harness: &TestHarness) { #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "2s", + heartbeat.interval = "60s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -576,7 +597,7 @@ async fn should_not_lose_messages_with_concurrent_polls_during_partition_add( #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "2s", + heartbeat.interval = "60s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -724,7 +745,7 @@ async fn should_handle_partition_add_then_consumer_disconnect_then_new_join(harn #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "2s", + heartbeat.interval = "60s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -861,7 +882,7 @@ async fn should_handle_partition_delete_while_multiple_consumers_polling(harness #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "2s", + heartbeat.interval = "60s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -1043,7 +1064,7 @@ async fn should_reach_even_distribution_after_multiple_joins(harness: &TestHarne #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "2s", + heartbeat.interval = "60s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -1164,7 +1185,7 @@ async fn should_split_evenly_when_consumer_joins_after_partitions_added(harness: #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "2s", + heartbeat.interval = "60s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -1265,7 +1286,7 @@ async fn should_not_duplicate_messages_when_partitions_added_during_polling(harn #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "2s", + heartbeat.interval = "60s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -1397,7 +1418,7 @@ async fn should_handle_delete_partitions_with_uncommitted_work(harness: &TestHar #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "2s", + heartbeat.interval = "60s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -1535,7 +1556,7 @@ async fn should_handle_rapid_partition_changes_with_active_consumers(harness: &T #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "2s", + heartbeat.interval = "60s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -1609,7 +1630,7 @@ async fn should_rebalance_after_adding_partitions(harness: &TestHarness) { #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "2s", + heartbeat.interval = "60s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -1697,7 +1718,7 @@ async fn should_rebalance_after_deleting_partitions(harness: &TestHarness) { #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "2s", + heartbeat.interval = "60s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -1888,6 +1909,8 @@ async fn should_timeout_revocation(harness: &TestHarness) { #[iggy_harness(server( heartbeat.enabled = true, + // Deliberately short: this spec drives the server's eviction path (see the + // module note), so the verifier must be able to reap a stale member. heartbeat.interval = "2s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true @@ -2309,7 +2332,7 @@ fn assert_balanced_partition_distribution(cg: &ConsumerGroupDetails, expected_to #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "2s", + heartbeat.interval = "60s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -2418,7 +2441,7 @@ async fn should_not_return_same_message_to_two_consumers_during_rebalance(harnes #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "2s", + heartbeat.interval = "60s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -2490,7 +2513,7 @@ async fn should_complete_revocation_on_auto_commit(harness: &TestHarness) { #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "2s", + heartbeat.interval = "60s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -2548,7 +2571,7 @@ async fn should_transfer_never_polled_partitions_immediately(harness: &TestHarne #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "2s", + heartbeat.interval = "60s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -2640,7 +2663,7 @@ async fn should_rebalance_when_member_with_pending_revocation_leaves(harness: &T #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "2s", + heartbeat.interval = "60s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -2798,7 +2821,7 @@ async fn should_not_produce_duplicate_messages_with_sequential_consumer_joins( #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "2s", + heartbeat.interval = "60s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -2907,7 +2930,7 @@ async fn should_wait_for_manual_commit_before_completing_revocation(harness: &Te #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "2s", + heartbeat.interval = "60s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -3000,7 +3023,7 @@ async fn should_redistribute_when_revocation_target_leaves(harness: &TestHarness #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "2s", + heartbeat.interval = "60s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -3100,7 +3123,7 @@ async fn should_distribute_partitions_evenly_with_concurrent_joins(harness: &Tes #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "2s", + heartbeat.interval = "60s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -3206,7 +3229,7 @@ async fn should_not_assign_partition_to_wrong_member_after_slab_reuse(harness: & #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "2s", + heartbeat.interval = "60s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -3337,7 +3360,7 @@ async fn should_not_complete_other_members_revocations_on_leave(harness: &TestHa #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "2s", + heartbeat.interval = "60s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -3430,7 +3453,7 @@ async fn should_distribute_16_partitions_evenly_across_16_consumers(harness: &Te #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "2s", + heartbeat.interval = "60s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -3497,7 +3520,7 @@ async fn should_distribute_excess_evenly_when_multiple_idle_members_join(harness #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "2s", + heartbeat.interval = "60s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -3562,7 +3585,7 @@ async fn should_distribute_remainder_fairly_with_uneven_ratio(harness: &TestHarn #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "2s", + heartbeat.interval = "60s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -3623,7 +3646,7 @@ async fn should_collect_excess_from_multiple_overassigned_members(harness: &Test #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "2s", + heartbeat.interval = "60s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] @@ -3678,7 +3701,7 @@ async fn should_not_starve_any_member_in_large_scale_rebalance(harness: &TestHar #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic], server( heartbeat.enabled = true, - heartbeat.interval = "2s", + heartbeat.interval = "60s", tcp.socket.override_defaults = true, tcp.socket.nodelay = true ))] diff --git a/core/integration/tests/mod.rs b/core/integration/tests/mod.rs index efb0cc8cab..7bdcd52353 100644 --- a/core/integration/tests/mod.rs +++ b/core/integration/tests/mod.rs @@ -30,29 +30,18 @@ use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::util::SubscriberInitExt; use tracing_subscriber::{EnvFilter, fmt}; -// Drives the `iggy` CLI binary against a running server, in both legacy and -// vsr/server-ng modes. Under vsr, single-node and the default 3-node cluster -// both pass. A few cases are mode-split where server-ng diverges from legacy by -// design: flush returns FeatureUnavailable, the session-timeout message -// differs, and purge is eventually consistent so server state is polled. +// Drives the `iggy` CLI binary against a running server. Single-node and the +// default 3-node cluster both pass. mod cli; // Raw-wire spec tests for VSR session continuity across a node restart -// (IGGY-137); the module is vsr-only by construction (file-level cfg). +// (IGGY-137). mod cluster; mod config_provider; mod connectors; -// Runs under vsr; the one gap (`verify_after_server_restart`) is gated inside -// the module: its rejoin window exceeds journal retention (state transfer). mod data_integrity; mod mcp; mod sdk; mod server; -// Unit-tests the legacy `server` crate's state-file machinery directly -// (`server::state::file::FileState` etc.). server-ng replaces that layer with -// the metadata WAL, so this suite is legacy-only by construction, not a gap. -#[cfg(not(feature = "vsr"))] -mod state; -mod storage; lazy_static! { static ref TESTS_FAILED: AtomicBool = AtomicBool::new(false); diff --git a/core/integration/tests/sdk/consumer_group_membership.rs b/core/integration/tests/sdk/consumer_group_membership.rs index 91800cf765..ab07bb3f1f 100644 --- a/core/integration/tests/sdk/consumer_group_membership.rs +++ b/core/integration/tests/sdk/consumer_group_membership.rs @@ -148,7 +148,7 @@ async fn given_group_member_holds_no_partitions_when_group_deleted_should_surfac // End-to-end wire pin for the consumer-group join/leave error ladder. The // metadata STM unit tests pin the committed result codes; this pins that -// server-ng actually emits them over the wire, so a client observes the same +// the server actually emits them over the wire, so a client observes the same // codes the legacy server returns. Binary transports only: the HTTP client // has no join/leave (stateless sessions carry no member identity, the SDK // returns FeatureUnavailable client-side), so the ladder cannot run there. diff --git a/core/integration/tests/sdk/hello_world.rs b/core/integration/tests/sdk/hello_world.rs index c7ad212893..10a646a3e9 100644 --- a/core/integration/tests/sdk/hello_world.rs +++ b/core/integration/tests/sdk/hello_world.rs @@ -24,7 +24,6 @@ async fn hello_world(harness: &TestHarness) { client.ping().await.unwrap(); } -#[cfg(feature = "vsr")] #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic])] async fn hello_world(harness: &TestHarness) { let client = harness.new_client().await.unwrap(); diff --git a/core/integration/tests/sdk/http_refresh.rs b/core/integration/tests/sdk/http_refresh.rs index 98236fb385..e33e2a96ba 100644 --- a/core/integration/tests/sdk/http_refresh.rs +++ b/core/integration/tests/sdk/http_refresh.rs @@ -16,7 +16,7 @@ // under the License. //! End-to-end coverage for the SDK HTTP client's `refresh_access_token` -//! against a live server-ng listener: the reissued token must replace the one +//! against a live the server listener: the reissued token must replace the one //! the client holds and keep it authenticated. use iggy::http::http_client::HttpClient; diff --git a/core/integration/tests/sdk/mod.rs b/core/integration/tests/sdk/mod.rs index ade2ec45c7..a59335337d 100644 --- a/core/integration/tests/sdk/mod.rs +++ b/core/integration/tests/sdk/mod.rs @@ -16,13 +16,10 @@ // under the License. mod consumer_group; -#[cfg(feature = "vsr")] mod consumer_group_membership; mod hello_world; -#[cfg(feature = "vsr")] mod http_refresh; mod producer; -#[cfg(feature = "vsr")] mod protocol_version; mod raw; mod send_confirmation; diff --git a/core/integration/tests/sdk/protocol_version.rs b/core/integration/tests/sdk/protocol_version.rs index b24db05f23..91bdb3bddf 100644 --- a/core/integration/tests/sdk/protocol_version.rs +++ b/core/integration/tests/sdk/protocol_version.rs @@ -22,12 +22,9 @@ //! frame carrying `IncompatibleProtocol` plus the accepted window; a body //! without a decodable prefix with `MalformedLogin` and a zero window. -#![cfg(feature = "vsr")] - use iggy::prelude::*; use iggy_binary_protocol::codec::WireEncode; use iggy_binary_protocol::consensus::{Command2, Operation, RequestHeader}; -use iggy_binary_protocol::namespace::METADATA_CONSENSUS_NAMESPACE; use iggy_binary_protocol::requests::users::LoginRegisterRequest; use iggy_binary_protocol::{ ClientVersionInfo, HEADER_SIZE, IGGY_PROTOCOL_VERSION, IGGY_PROTOCOL_VERSION_MIN, WireName, @@ -88,7 +85,6 @@ async fn assert_login_evicted( client: 0xC0FFEE, session: 0, request: 0, - namespace: METADATA_CONSENSUS_NAMESPACE, ..Default::default() }; diff --git a/core/integration/tests/sdk/raw.rs b/core/integration/tests/sdk/raw.rs index 0bf0049470..29ff474ec0 100644 --- a/core/integration/tests/sdk/raw.rs +++ b/core/integration/tests/sdk/raw.rs @@ -22,16 +22,6 @@ use iggy_binary_protocol::codes::{GET_STATS_CODE, LOGIN_USER_CODE, PING_CODE}; use iggy_binary_protocol::requests::system::{GetStatsRequest, PingRequest}; use integration::iggy_harness; -#[cfg(not(feature = "vsr"))] -#[iggy_harness(test_client_transport = [Tcp, Quic, Http, WebSocket])] -async fn given_authenticated_client_when_sending_raw_request_should_round_trip( - harness: &TestHarness, -) { - let client = harness.root_client().await.unwrap(); - assert_raw_round_trip(&client).await; -} - -#[cfg(feature = "vsr")] #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic])] async fn given_authenticated_client_when_sending_raw_request_should_round_trip( harness: &TestHarness, @@ -92,7 +82,6 @@ async fn assert_raw_round_trip(client: &IggyClient) { // follow-up ping is the point of the case: the server must answer // with a deny frame, not drop the frame and leave the connection // wedged until the read timeout. - #[cfg(feature = "vsr")] { let error = client .send_binary_request(60_000, Bytes::new()) diff --git a/core/integration/tests/sdk/send_confirmation.rs b/core/integration/tests/sdk/send_confirmation.rs index 616ee700a1..61d1c0e7e6 100644 --- a/core/integration/tests/sdk/send_confirmation.rs +++ b/core/integration/tests/sdk/send_confirmation.rs @@ -16,7 +16,7 @@ // under the License. //! Commit confirmations for `SendMessages`: which partition a batch landed in -//! and at which offset. server-ng answers a committed send with a confirmation +//! and at which offset. The server answers a committed send with a confirmation //! payload; the legacy server answers with an empty body, which the SDK reports //! as no confirmations rather than as a decode failure. @@ -28,14 +28,11 @@ const TOPIC_NAME: &str = "confirmation-topic"; const MESSAGES_COUNT: u32 = 10; const PARTITIONS_COUNT: u32 = 3; -// server-ng partition ids are 0-based (CreateTopic assigns them from 0). -#[cfg(feature = "vsr")] +// Partition ids are 0-based (CreateTopic assigns them from 0). const PARTITION_ID: u32 = 0; /// Chunking for the direct producer: `CHUNKS * CHUNK_LENGTH` messages exceed /// one request, so the send is split and every chunk confirms separately. -#[cfg(feature = "vsr")] const CHUNK_LENGTH: u32 = 4; -#[cfg(feature = "vsr")] const CHUNKS: u32 = 3; fn batch(count: u32) -> Vec { @@ -71,7 +68,6 @@ async fn create_stream_and_topic(client: &IggyClient, partitions_count: u32) -> (stream.id, topic.id) } -#[cfg(feature = "vsr")] fn sole_confirmation(response: &SendMessagesResponse) -> &SendMessagesConfirmationResponse { assert_eq!( response.confirmations.len(), @@ -83,7 +79,6 @@ fn sole_confirmation(response: &SendMessagesResponse) -> &SendMessagesConfirmati /// Each transport carries the reply body on its own path, so the full /// confirmation shape is pinned on all three. -#[cfg(feature = "vsr")] #[iggy_harness(test_client_transport = [Tcp, WebSocket, Quic])] async fn given_explicit_partition_when_sending_two_batches_should_confirm_advancing_base_offset( harness: &TestHarness, @@ -128,7 +123,6 @@ async fn given_explicit_partition_when_sending_two_batches_should_confirm_advanc client.logout_user().await.unwrap(); } -#[cfg(feature = "vsr")] #[iggy_harness] async fn given_balanced_partitioning_when_sending_should_confirm_a_partition_of_the_topic( harness: &TestHarness, @@ -167,7 +161,6 @@ async fn given_balanced_partitioning_when_sending_should_confirm_a_partition_of_ client.logout_user().await.unwrap(); } -#[cfg(feature = "vsr")] #[iggy_harness] async fn given_messages_key_partitioning_when_sending_should_confirm_a_partition_of_the_topic( harness: &TestHarness, @@ -200,7 +193,6 @@ async fn given_messages_key_partitioning_when_sending_should_confirm_a_partition client.logout_user().await.unwrap(); } -#[cfg(feature = "vsr")] #[iggy_harness] async fn given_direct_producer_when_send_splits_into_chunks_should_confirm_every_chunk( harness: &TestHarness, @@ -242,27 +234,3 @@ async fn given_direct_producer_when_send_splits_into_chunks_should_confirm_every client.logout_user().await.unwrap(); } - -#[cfg(not(feature = "vsr"))] -#[iggy_harness] -async fn given_legacy_server_when_sending_should_report_no_confirmations(harness: &TestHarness) { - let client = harness.root_client().await.unwrap(); - - create_stream_and_topic(&client, PARTITIONS_COUNT).await; - let mut messages = batch(MESSAGES_COUNT); - let response = client - .send_messages( - &Identifier::named(STREAM_NAME).unwrap(), - &Identifier::named(TOPIC_NAME).unwrap(), - &Partitioning::balanced(), - &mut messages, - ) - .await - .expect("send_messages"); - - assert!( - response.confirmations.is_empty(), - "the legacy server reports no offsets; a synthetic entry would be \ - indistinguishable from a genuine commit at offset 0" - ); -} diff --git a/core/integration/tests/server/a2a_jwt/jwt_tests.rs b/core/integration/tests/server/a2a_jwt/jwt_tests.rs index 4d0b2b1943..1b7d915a9f 100644 --- a/core/integration/tests/server/a2a_jwt/jwt_tests.rs +++ b/core/integration/tests/server/a2a_jwt/jwt_tests.rs @@ -25,7 +25,6 @@ use iggy_common::{StreamClient, UserClient}; use integration::iggy_harness; use jsonwebtoken::{Algorithm, EncodingKey, Header, encode}; use serde::{Deserialize, Serialize}; -use server::http::jwt::json_web_token::Audience; const TEST_ISSUER: &str = "https://test-issuer.com"; const TEST_AUDIENCE: &str = "iggy"; @@ -69,6 +68,16 @@ async fn seed_a2a_user( Ok(()) } +/// The `aud` claim as RFC 7519 allows it on the wire: one string, or an array +/// of them. Untagged serialization emits each shape verbatim, which is what the +/// server's own audience parser reads back. +#[derive(Debug, Serialize, Deserialize)] +#[serde(untagged)] +enum Audience { + Single(String), + Multiple(Vec), +} + /// Test claims structure for JWT tokens /// Supports both single string and array audience per RFC 7519 #[derive(Debug, Serialize, Deserialize)] @@ -96,7 +105,7 @@ fn create_valid_jwt(exp_seconds: u64) -> String { let claims = TestClaims { jti: uuid::Uuid::now_v7().to_string(), iss: TEST_ISSUER.to_string(), - aud: Audience::from(TEST_AUDIENCE), + aud: Audience::Single(TEST_AUDIENCE.to_string()), sub: "external-a2a-user-123".to_string(), exp: now + exp_seconds, iat: now, @@ -116,7 +125,7 @@ fn create_valid_jwt_with_array_aud(exp_seconds: u64) -> String { let claims = TestClaims { jti: uuid::Uuid::now_v7().to_string(), iss: TEST_ISSUER.to_string(), - aud: Audience::from(vec![ + aud: Audience::Multiple(vec![ "some-other-service".to_string(), TEST_AUDIENCE.to_string(), "another-service".to_string(), @@ -140,7 +149,7 @@ fn create_expired_jwt() -> String { let claims = TestClaims { jti: uuid::Uuid::now_v7().to_string(), iss: TEST_ISSUER.to_string(), - aud: Audience::from(TEST_AUDIENCE), + aud: Audience::Single(TEST_AUDIENCE.to_string()), sub: "external-a2a-user-123".to_string(), exp: now.saturating_sub(3600), iat: now.saturating_sub(7200), @@ -160,7 +169,7 @@ fn create_unknown_issuer_jwt() -> String { let claims = TestClaims { jti: uuid::Uuid::now_v7().to_string(), iss: "https://unknown-issuer.com".to_string(), - aud: Audience::from(TEST_AUDIENCE), + aud: Audience::Single(TEST_AUDIENCE.to_string()), sub: "external-a2a-user-123".to_string(), exp: now + 3600, iat: now, @@ -184,7 +193,7 @@ fn create_algorithm_confusion_jwt() -> String { let claims = TestClaims { jti: uuid::Uuid::now_v7().to_string(), iss: TEST_ISSUER.to_string(), - aud: Audience::from(TEST_AUDIENCE), + aud: Audience::Single(TEST_AUDIENCE.to_string()), sub: "external-a2a-user-123".to_string(), exp: now + 3600, iat: now, @@ -206,7 +215,7 @@ fn create_jwt_with_kid(kid: &str) -> String { let claims = TestClaims { jti: uuid::Uuid::now_v7().to_string(), iss: TEST_ISSUER.to_string(), - aud: Audience::from(TEST_AUDIENCE), + aud: Audience::Single(TEST_AUDIENCE.to_string()), sub: "external-a2a-user-123".to_string(), exp: now + 3600, iat: now, diff --git a/core/integration/tests/server/cluster_view_durability_vsr.rs b/core/integration/tests/server/cluster_view_durability_vsr.rs index a2cc801f43..bf59636a90 100644 --- a/core/integration/tests/server/cluster_view_durability_vsr.rs +++ b/core/integration/tests/server/cluster_view_durability_vsr.rs @@ -16,7 +16,7 @@ // under the License. //! Metadata-plane view durability across a real view change and process restarts, -//! over `iggy-server-ng`'s production superblock path. +//! over `iggy-server`'s production superblock path. //! //! The superblock exists so a replica recovers a view it already acted in from its //! OWN disk after a crash, instead of inferring a stale view from the WAL or @@ -41,7 +41,7 @@ //! so recovery is exercised without wedging the cluster below quorum. //! //! vsr-only: a metadata view change has no analog on the single-process legacy -//! server, and the superblock is server-ng's durable consensus record. +//! server, and the superblock is the server's durable consensus record. use std::path::Path; use std::time::{Duration, Instant}; diff --git a/core/integration/tests/server/flush_vsr.rs b/core/integration/tests/server/flush_vsr.rs index 36c840724f..56ff451c32 100644 --- a/core/integration/tests/server/flush_vsr.rs +++ b/core/integration/tests/server/flush_vsr.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -//! Flush contract against server-ng (vsr): the server has no on-demand flush +//! Flush contract against the server (vsr): the server has no on-demand flush //! primitive, so `FLUSH_UNSAVED_BUFFER` must surface a typed //! `FeatureUnavailable` over the SDK rather than the non-replicated catch-all's //! empty-ok, which would fake a durability guarantee. @@ -27,7 +27,7 @@ use iggy::prelude::*; use integration::iggy_harness; -// server-ng partition ids are 0-based (CreateTopic assigns them from 0). +// Partition ids are 0-based (CreateTopic assigns them from 0). const PARTITION_ID: u32 = 0; #[iggy_harness( diff --git a/core/integration/tests/server/general.rs b/core/integration/tests/server/general.rs index c9a5ad6068..d21d118ab4 100644 --- a/core/integration/tests/server/general.rs +++ b/core/integration/tests/server/general.rs @@ -15,8 +15,6 @@ // specific language governing permissions and limitations // under the License. -#[cfg(not(feature = "vsr"))] -use crate::server::scenarios::bench_scenario; use crate::server::scenarios::{ authentication_scenario, consumer_timestamp_polling_scenario, invalid_consumer_offset_scenario, message_headers_scenario, permissions_scenario, snapshot_scenario, @@ -103,23 +101,6 @@ async fn stream_size_validation(harness: &TestHarness) { stream_size_validation_scenario::run(harness).await; } -// Blocked under vsr: pushes 8 MiB through the data plane, which drains the -// in-memory partition journal to disk segments; benchmarks are out of -// scope for the vsr test pass. -#[cfg(not(feature = "vsr"))] -#[iggy_harness( - test_client_transport = [Tcp, Http, Quic, WebSocket], - server( - tcp.socket.override_defaults = true, - tcp.socket.nodelay = true, - quic.max_idle_timeout = "500s", - quic.keep_alive_interval = "15s" - ) -)] -async fn bench(harness: &TestHarness) { - bench_scenario::run(harness).await; -} - #[iggy_harness( test_client_transport = [Tcp, Http, Quic, WebSocket], server( diff --git a/core/integration/tests/server/http_client.rs b/core/integration/tests/server/http_client.rs index e254fbfd7e..7393ffef36 100644 --- a/core/integration/tests/server/http_client.rs +++ b/core/integration/tests/server/http_client.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -//! Shared HTTP transport plumbing for the server-ng REST suites (`http_vsr`, +//! Shared HTTP transport plumbing for the server REST suites (`http_vsr`, //! `http_rbac`): one authenticated `reqwest` session with the login-retry gate //! and the generic verb helpers. Each suite keeps its own request shapes and //! assertions as extension methods on [`HttpClient`], so the wire-contract and @@ -38,7 +38,7 @@ pub const REQUEST_TIMEOUT: Duration = Duration::from_secs(30); /// One authenticated HTTP session against the test server's shard-0 listener: a /// `reqwest` client, the listener base URL, and the bearer to send. The bearer -/// is either a login JWT or a raw personal access token (server-ng resolves +/// is either a login JWT or a raw personal access token (the server resolves /// either on the `Authorization: Bearer` header). pub struct HttpClient { pub client: reqwest::Client, diff --git a/core/integration/tests/server/http_rbac.rs b/core/integration/tests/server/http_rbac.rs index e07fffb5d6..6a113d03c9 100644 --- a/core/integration/tests/server/http_rbac.rs +++ b/core/integration/tests/server/http_rbac.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -//! HTTP wire-contract residue for server-ng's shard-0 REST listener. The RBAC +//! HTTP wire-contract residue for the server's shard-0 REST listener. The RBAC //! authorization matrix itself (who may do what, over every transport) lives in //! the cross-transport `permissions_scenario` suite; this file keeps only what //! the SDK abstracts away and only raw HTTP can show: @@ -42,7 +42,7 @@ use integration::iggy_harness; use reqwest::{Response, StatusCode}; use serde_json::{Value, json}; -// server-ng partition ids are 0-based (CreateTopic assigns them from 0). +// Partition ids are 0-based (CreateTopic assigns them from 0). const PARTITION_ID: u32 = 0; // Explicit consumer id in the poll query (`Consumer::default()` is numeric 0). const CONSUMER_ID: u32 = 1; diff --git a/core/integration/tests/server/http_tls.rs b/core/integration/tests/server/http_tls.rs index f36e4a058d..15e5233f85 100644 --- a/core/integration/tests/server/http_tls.rs +++ b/core/integration/tests/server/http_tls.rs @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -//! End-to-end HTTPS proof for the server-ng shard-0 REST listener. The TLS -//! accept pump ([`server_ng::http::tls`]) and the `start()` HTTPS branch +//! End-to-end HTTPS proof for the server shard-0 REST listener. The TLS +//! accept pump ([`server::http::tls`]) and the `start()` HTTPS branch //! unit-test in isolation; this is the only path that drives a live rustls //! client over the wire and proves the response was actually served over //! HTTP/2, negotiated via ALPN. A failure here (h1 fallback, handshake @@ -71,7 +71,7 @@ fn cert_asset(file: &str) -> PathBuf { .unwrap_or_else(|error| panic!("canonicalize repo cert asset {file}: {error}")) } -/// Boot an iggy-server-ng cluster with `[http.tls]` enabled against the repo +/// Boot an iggy-server cluster with `[http.tls]` enabled against the repo /// loopback cert, then prove a real HTTPS request is served over HTTP/2. Two /// nodes rather than one because the `/cluster/metadata` assertion below /// needs an enabled cluster roster (a single-node harness runs with the @@ -113,7 +113,7 @@ async fn given_http_tls_enabled_when_pinging_should_serve_https_over_http2() { harness .start() .await - .expect("start server-ng cluster with HTTPS enabled"); + .expect("start the server cluster with HTTPS enabled"); let addr = harness .server() diff --git a/core/integration/tests/server/http_vsr.rs b/core/integration/tests/server/http_vsr.rs index 5d6e1e2ea5..ca4acf4c37 100644 --- a/core/integration/tests/server/http_vsr.rs +++ b/core/integration/tests/server/http_vsr.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -//! HTTP data-plane gate for server-ng: produce, poll, and consumer-offset +//! HTTP data-plane gate for the server: produce, poll, and consumer-offset //! routes exercised over raw `reqwest` (not the SDK HTTP client) so the wire //! contract itself is under test - exact status codes, the //! `iggy-durability` header, the body-size cap, and cross-request isolation @@ -36,7 +36,7 @@ use std::str::FromStr; use std::time::{Duration, Instant}; use tokio::time::sleep; -// server-ng partition ids are 0-based (CreateTopic assigns them from 0). +// Partition ids are 0-based (CreateTopic assigns them from 0). const PARTITION_ID: u32 = 0; /// Explicit consumer id shared by the offset store body and the read/delete @@ -826,7 +826,7 @@ async fn given_valid_access_token_when_refreshing_should_issue_working_token_wit "the refreshed token must authenticate" ); - // Stateless by design: server-ng has no replicated revocation list (P3), so + // Stateless by design: the server has no replicated revocation list (P3), so // refreshing never invalidates the token it was minted from - the old token // lives to its natural exp. Deliberate, not a bug; the same posture as // logout, which ends a session without revoking its bearer. diff --git a/core/integration/tests/server/legacy_login_vsr.rs b/core/integration/tests/server/legacy_login_vsr.rs index 4161a4c3b2..9e129e928e 100644 --- a/core/integration/tests/server/legacy_login_vsr.rs +++ b/core/integration/tests/server/legacy_login_vsr.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -//! Legacy login codes against server-ng (vsr). server-ng authenticates only +//! Legacy login codes against the server (vsr). The server authenticates only //! through the Register handshake, so the pre-register `LOGIN_USER` (38) and //! `LOGIN_WITH_PERSONAL_ACCESS_TOKEN` (44) codes -- which the vsr SDK never //! emits (its typed login methods send the register codes, its raw path @@ -27,12 +27,9 @@ //! TCP socket: a header-only non-replicated frame carrying the code in the //! reserved command slot. -#![cfg(feature = "vsr")] - use iggy_binary_protocol::HEADER_SIZE; use iggy_binary_protocol::codes::{LOGIN_USER_CODE, LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE}; use iggy_binary_protocol::consensus::{Command2, Operation, RequestHeader}; -use iggy_binary_protocol::namespace::METADATA_CONSENSUS_NAMESPACE; use integration::harness::TestHarness; use integration::iggy_harness; use std::mem::offset_of; @@ -72,7 +69,6 @@ async fn assert_legacy_login_code_evicted(harness: &TestHarness, code: u32) { client: 0xC0FFEE, session: 0, request: 0, - namespace: METADATA_CONSENSUS_NAMESPACE, ..Default::default() }; // A non-replicated command code travels in the first 4 reserved bytes. diff --git a/core/integration/tests/server/login_credentials_vsr.rs b/core/integration/tests/server/login_credentials_vsr.rs new file mode 100644 index 0000000000..0f05c9beb4 --- /dev/null +++ b/core/integration/tests/server/login_credentials_vsr.rs @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Credential rejections on the register handshake against the server (vsr). +//! +//! A username/password body that fails verification falls through to the PAT +//! decode attempt, so the terminal rejection has to carry the credential +//! failure that actually happened. Reporting the payload shape instead +//! (`MalformedLogin` -> `InvalidFormat`) tells a client its request was +//! malformed when the request was fine and the password was not. + +use iggy::prelude::*; +use integration::iggy_harness; + +#[iggy_harness(test_client_transport = [Tcp])] +async fn given_wrong_password_when_logging_in_should_reject_invalid_credentials( + harness: &TestHarness, +) { + let client = harness + .new_client_for(TransportProtocol::Tcp) + .await + .expect("tcp client"); + + let result = client + .login_user("iggy", "definitely-not-the-password") + .await; + + let expected = IggyError::InvalidCredentials.as_code(); + assert!( + matches!(&result, Err(error) if error.as_code() == expected), + "a wrong password must surface Err(InvalidCredentials), got {result:?}" + ); +} + +#[iggy_harness(test_client_transport = [Tcp])] +async fn given_unknown_username_when_logging_in_should_reject_invalid_credentials( + harness: &TestHarness, +) { + let client = harness + .new_client_for(TransportProtocol::Tcp) + .await + .expect("tcp client"); + + let result = client.login_user("no-such-user", "irrelevant").await; + + // Same rejection as a wrong password: an unknown username must not be + // distinguishable from a bad password. + let expected = IggyError::InvalidCredentials.as_code(); + assert!( + matches!(&result, Err(error) if error.as_code() == expected), + "an unknown username must surface Err(InvalidCredentials), got {result:?}" + ); +} + +#[iggy_harness(test_client_transport = [Tcp])] +async fn given_rejected_login_when_retrying_with_valid_credentials_should_succeed( + harness: &TestHarness, +) { + let client = harness + .new_client_for(TransportProtocol::Tcp) + .await + .expect("tcp client"); + + let rejected = client.login_user("iggy", "wrong").await; + assert!(rejected.is_err(), "the first login must be rejected"); + + // The rejection evicts the session, so a usable client has to reconnect; + // the point is that a bad password does not poison the account. + let retry = harness + .new_client_for(TransportProtocol::Tcp) + .await + .expect("tcp client"); + retry + .login_user("iggy", "iggy") + .await + .expect("valid credentials must still authenticate after a rejection"); +} diff --git a/core/integration/tests/server/mod.rs b/core/integration/tests/server/mod.rs index ed6ce07437..f31ca33a02 100644 --- a/core/integration/tests/server/mod.rs +++ b/core/integration/tests/server/mod.rs @@ -16,63 +16,55 @@ // under the License. // a2a_jwt exercises trusted-issuer (JWKS) tokens; both the legacy verifier and -// server-ng's ported trusted-issuer path verify them. +// the server's ported trusted-issuer path verify them. mod a2a_jwt; mod cg; -// Flush (FLUSH_UNSAVED_BUFFER) has no server-ng primitive; it must deny typed. -#[cfg(feature = "vsr")] +// Flush (FLUSH_UNSAVED_BUFFER) has no the server primitive; it must deny typed. mod flush_vsr; -// Legacy login codes (LOGIN_USER / LOGIN_WITH_PAT) have no server-ng handler; +// Legacy login codes (LOGIN_USER / LOGIN_WITH_PAT) have no the server handler; // they must evict typed (MalformedLogin), not stall or reply empty-ok. -#[cfg(feature = "vsr")] mod legacy_login_vsr; +// A failed credential login must report the credential failure, not the +// payload shape it fell through to. +mod login_credentials_vsr; // Poll addressing + timestamp semantics: typed PartitionNotFound on a bad // partition id, at-or-after timestamp polls. -#[cfg(feature = "vsr")] mod poll_semantics_vsr; // Create-topic static bounds deny typed before consensus. -#[cfg(feature = "vsr")] mod topic_admission_vsr; +// Stats aggregates the cross-shard connected-client count, not a hardcoded 0. +mod stats_vsr; // Purge durability: applied generation survives restart; journal-resident // purged batches stay fenced behind the purge floor. -#[cfg(feature = "vsr")] mod purge_vsr; // Shared HTTP transport plumbing (session + verb helpers) for the raw-HTTP -// server-ng suites below. -#[cfg(feature = "vsr")] +// server suites below. mod http_client; -// Raw-HTTP data-plane contract against server-ng's shard-0 listener. -#[cfg(feature = "vsr")] +// Raw-HTTP data-plane contract against the server's shard-0 listener. mod http_vsr; -// Raw-HTTP wire-contract residue against server-ng (status codes + typed error +// Raw-HTTP wire-contract residue against the server (status codes + typed error // bodies); the RBAC matrix lives in permissions_scenario. -#[cfg(feature = "vsr")] mod http_rbac; -// End-to-end HTTPS: server-ng serves the REST listener over TLS and negotiates +// End-to-end HTTPS: the server serves the REST listener over TLS and negotiates // HTTP/2 via ALPN. -#[cfg(feature = "vsr")] mod http_tls; // Binary GetClusterMetadata must serve the real roster from a VSR cluster. -#[cfg(feature = "vsr")] mod cluster_metadata_vsr; // A metadata view change must persist the advanced view and recover it from disk // across a replica restart. -#[cfg(feature = "vsr")] mod cluster_view_durability_vsr; // A partition view change must persist the advanced view in that group's own // superblock and recover it from disk across a replica restart. -#[cfg(feature = "vsr")] mod partition_view_durability_vsr; // 80-case race matrix with hardcoded HTTP variants (test_matrix bypasses // the harness transport filter). mod concurrent_addition; mod general; -// The per-shard segment cleaner deletes expired / oversize segments from disk -// under both the legacy server and server-ng. +// The per-shard segment cleaner deletes expired / oversize segments from disk. mod message_cleanup; mod message_retrieval; // Server restarts, consumer-group barriers, and DeleteSegments maintenance. -// The full restart matrix (consumer variants included) runs under server-ng: +// The full restart matrix (consumer variants included) runs under the server: // a restarted replica rejoins via the view probe + journal repair. mod purge_delete; mod scenarios; diff --git a/core/integration/tests/server/partition_view_durability_vsr.rs b/core/integration/tests/server/partition_view_durability_vsr.rs index 0071473998..0ea467e5ae 100644 --- a/core/integration/tests/server/partition_view_durability_vsr.rs +++ b/core/integration/tests/server/partition_view_durability_vsr.rs @@ -29,7 +29,7 @@ //! partition data path rides along. //! //! vsr-only: partition consensus groups and their superblocks exist only on -//! `iggy-server-ng`. +//! `iggy-server`. use std::path::{Path, PathBuf}; use std::str::FromStr; @@ -44,7 +44,7 @@ use tokio::time::sleep; const STREAM_NAME: &str = "partition-view-durability-stream"; const TOPIC_NAME: &str = "partition-view-durability-topic"; -/// server-ng partition ids are 0-based (CreateTopic assigns them from 0). +/// Partition ids are 0-based (CreateTopic assigns them from 0). const PARTITION_ID: u32 = 0; const MESSAGES_COUNT: u32 = 10; diff --git a/core/integration/tests/server/poll_semantics_vsr.rs b/core/integration/tests/server/poll_semantics_vsr.rs index 1f48eef36b..9d211e9a6a 100644 --- a/core/integration/tests/server/poll_semantics_vsr.rs +++ b/core/integration/tests/server/poll_semantics_vsr.rs @@ -15,10 +15,12 @@ // specific language governing permissions and limitations // under the License. -//! Poll semantics against server-ng (vsr): a poll aimed at a partition id the +//! Poll semantics against the server (vsr): a poll aimed at a partition id the //! topic does not have must surface a typed `PartitionNotFound`, not an empty -//! poll a consumer would read as end-of-partition; the same addressing error -//! on `get_consumer_offset` must not decode as "no offset stored"; and a +//! poll a consumer would read as end-of-partition; a poll whose stream or +//! topic does not resolve must surface the legacy `StreamIdNotFound` / +//! `TopicIdNotFound` the same way; the partition addressing error on +//! `get_consumer_offset` must not decode as "no offset stored"; and a //! timestamp poll must be at-or-after, including the message stamped exactly at //! the queried timestamp (the timestamp replies report per message). @@ -87,10 +89,72 @@ async fn given_missing_partition_when_polling_should_reject_partition_not_found( assert_eq!(valid.messages.len(), 0, "empty topic polls empty"); } +#[iggy_harness( + test_client_transport = [Tcp], + server(tcp.socket.override_defaults = true, tcp.socket.nodelay = true) +)] +async fn given_missing_stream_when_polling_should_reject_stream_not_found(harness: &TestHarness) { + let client = harness.tcp_root_client().await.expect("tcp root client"); + let stream_id = Identifier::from_str_value("no-such-stream").expect("stream identifier"); + let topic_id = Identifier::from_str_value("no-such-topic").expect("topic identifier"); + + let result = client + .poll_messages( + &stream_id, + &topic_id, + Some(0), + &Consumer::default(), + &PollingStrategy::offset(0), + 1, + false, + ) + .await; + + let expected = IggyError::StreamIdNotFound(Identifier::default()).as_code(); + assert!( + matches!(&result, Err(error) if error.as_code() == expected), + "polling a missing stream must surface Err(StreamIdNotFound), got {result:?}" + ); +} + +#[iggy_harness( + test_client_transport = [Tcp], + server(tcp.socket.override_defaults = true, tcp.socket.nodelay = true) +)] +async fn given_missing_topic_when_polling_should_reject_topic_not_found(harness: &TestHarness) { + let client = harness.tcp_root_client().await.expect("tcp root client"); + client + .create_stream("topicless-stream") + .await + .expect("create stream"); + let stream_id = Identifier::from_str_value("topicless-stream").expect("stream identifier"); + let topic_id = Identifier::from_str_value("no-such-topic").expect("topic identifier"); + + let result = client + .poll_messages( + &stream_id, + &topic_id, + Some(0), + &Consumer::default(), + &PollingStrategy::offset(0), + 1, + false, + ) + .await; + + let expected = + IggyError::TopicIdNotFound(Identifier::default(), Identifier::default()).as_code(); + assert!( + matches!(&result, Err(error) if error.as_code() == expected), + "polling a missing topic of an existing stream must surface Err(TopicIdNotFound), \ + got {result:?}" + ); +} + /// `get_consumer_offset` answered an unknown partition with an empty body, /// which the SDK decodes as `None` - the same value a consumer that simply has /// no stored offset yet gets back, so a client could not tell a typo from a -/// fresh consumer. Legacy swallows this one too; server-ng surfaces the code +/// fresh consumer. Legacy swallows this one too; the server surfaces the code /// the poll path already surfaces for the identical addressing error. #[iggy_harness( test_client_transport = [Tcp], diff --git a/core/integration/tests/server/purge_delete.rs b/core/integration/tests/server/purge_delete.rs index 5b631ae608..c597b9ed52 100644 --- a/core/integration/tests/server/purge_delete.rs +++ b/core/integration/tests/server/purge_delete.rs @@ -19,12 +19,17 @@ use crate::server::scenarios::purge_delete_scenario; use integration::iggy_harness; use test_case::test_matrix; -#[iggy_harness(server( - segment.size = "5KiB", - segment.cache_indexes = ["all", "none", "open_segment"], - partition.messages_required_to_save = "1", - partition.enforce_fsync = "true", -))] +// Restart scenarios run single-node: restarting a node in a multi-node cluster +// trips a known partitions-plane view-change stall, tracked separately. +#[iggy_harness( + cluster_nodes = 1, + server( + segment.size = "5KiB", + segment.cache_indexes = ["all", "none", "open_segment"], + partition.messages_required_to_save = "1", + partition.enforce_fsync = "true", + ) +)] #[test_matrix([restart_off(), restart_on()])] async fn should_delete_segments_and_validate_filesystem( harness: &mut TestHarness, @@ -33,12 +38,15 @@ async fn should_delete_segments_and_validate_filesystem( purge_delete_scenario::run(harness, restart_server).await; } -#[iggy_harness(server( - segment.size = "5KiB", - segment.cache_indexes = ["all", "none", "open_segment"], - partition.messages_required_to_save = "1", - partition.enforce_fsync = "true", -))] +#[iggy_harness( + cluster_nodes = 1, + server( + segment.size = "5KiB", + segment.cache_indexes = ["all", "none", "open_segment"], + partition.messages_required_to_save = "1", + partition.enforce_fsync = "true", + ) +)] #[test_matrix([restart_off(), restart_on()])] async fn should_delete_segments_without_consumers(harness: &mut TestHarness, restart_server: bool) { purge_delete_scenario::run_no_consumers(harness, restart_server).await; @@ -57,12 +65,15 @@ async fn should_delete_segments_with_consumer_group_barrier(harness: &TestHarnes purge_delete_scenario::run_consumer_group_barrier(&client, &data_path).await; } -#[iggy_harness(server( - segment.size = "5KiB", - segment.cache_indexes = ["all", "none", "open_segment"], - partition.messages_required_to_save = "1", - partition.enforce_fsync = "true", -))] +#[iggy_harness( + cluster_nodes = 1, + server( + segment.size = "5KiB", + segment.cache_indexes = ["all", "none", "open_segment"], + partition.messages_required_to_save = "1", + partition.enforce_fsync = "true", + ) +)] #[test_matrix([restart_off(), restart_on()])] async fn should_block_deletion_until_all_consumers_pass_segment( harness: &mut TestHarness, @@ -71,12 +82,15 @@ async fn should_block_deletion_until_all_consumers_pass_segment( purge_delete_scenario::run_multi_consumer_barrier(harness, restart_server).await; } -#[iggy_harness(server( - segment.size = "5KiB", - segment.cache_indexes = ["all", "none", "open_segment"], - partition.messages_required_to_save = "1", - partition.enforce_fsync = "true", -))] +#[iggy_harness( + cluster_nodes = 1, + server( + segment.size = "5KiB", + segment.cache_indexes = ["all", "none", "open_segment"], + partition.messages_required_to_save = "1", + partition.enforce_fsync = "true", + ) +)] // The scenario asserts the exact [0, 7, 14, 21] layout only on the legacy path; // under vsr it verifies the framing-agnostic purge outcome (offsets cleared, // files deleted, partition reset to a single segment at offset 0). diff --git a/core/integration/tests/server/purge_vsr.rs b/core/integration/tests/server/purge_vsr.rs index 5cd8028549..6a5eac2088 100644 --- a/core/integration/tests/server/purge_vsr.rs +++ b/core/integration/tests/server/purge_vsr.rs @@ -15,11 +15,14 @@ // specific language governing permissions and limitations // under the License. -//! server-ng purge durability: the applied purge generation survives a +//! Server purge durability: the applied purge generation survives a //! restart (`purge.gen`), and purged journal-resident batches stay fenced //! behind the purge floor instead of resurfacing through the shutdown flush. +//! Plus read-your-purge: the counters a purge acks are visible to the very +//! next read, without waiting for the reconciler's on-disk reset. use crate::server::scenarios::purge_delete_scenario; +use iggy::prelude::*; use integration::iggy_harness; // Single node: the tests reason about ONE replica's on-disk state across a @@ -49,3 +52,99 @@ async fn given_journal_resident_messages_when_purged_should_not_resurface( ) { purge_delete_scenario::run_resident_purge_no_resurface(harness).await; } + +// No sleep, no poll: the purge acks on commit and the segment prune runs later +// on the reconciler, so the reset of the counters `get_topic` / `get_stream` +// read has to happen in the replicated apply. A retry loop here would pass +// against the pre-apply behavior too. +#[iggy_harness( + test_client_transport = [Tcp], + server(tcp.socket.override_defaults = true, tcp.socket.nodelay = true) +)] +async fn given_purged_topic_when_getting_topic_immediately_should_report_zero_stats( + harness: &TestHarness, +) { + const STREAM: &str = "purge-stats-stream"; + const TOPIC: &str = "purge-stats-topic"; + + let client = harness.tcp_root_client().await.expect("tcp root client"); + client.create_stream(STREAM).await.expect("create stream"); + let stream_id = Identifier::from_str_value(STREAM).expect("stream identifier"); + let topic_id = Identifier::from_str_value(TOPIC).expect("topic identifier"); + client + .create_topic( + &stream_id, + TOPIC, + 1, + CompressionAlgorithm::None, + None, + IggyExpiry::NeverExpire, + MaxTopicSize::ServerDefault, + ) + .await + .expect("create topic"); + + let mut messages: Vec = (0..10) + .map(|index| { + IggyMessage::builder() + .payload(format!("message-{index}").into()) + .build() + .expect("build message") + }) + .collect(); + client + .send_messages( + &stream_id, + &topic_id, + &Partitioning::partition_id(0), + &mut messages, + ) + .await + .expect("send messages"); + + let before = client + .get_topic(&stream_id, &topic_id) + .await + .expect("get topic before purge") + .expect("topic exists before purge"); + assert_eq!( + before.messages_count, 10, + "the send must be counted before the purge, or the assert below proves nothing" + ); + assert!(before.size.as_bytes_u64() > 0); + + client + .purge_topic(&stream_id, &topic_id) + .await + .expect("purge topic"); + + let topic = client + .get_topic(&stream_id, &topic_id) + .await + .expect("get topic after purge") + .expect("purge keeps the topic"); + assert_eq!( + topic.messages_count, 0, + "a read right after the purge ack must not report pre-purge messages" + ); + assert_eq!(topic.size.as_bytes_u64(), 0); + assert_eq!( + topic.partitions.len(), + 1, + "purge keeps the partition, it only empties it" + ); + assert_eq!(topic.partitions[0].messages_count, 0); + assert_eq!(topic.partitions[0].size.as_bytes_u64(), 0); + assert_eq!(topic.partitions[0].current_offset, 0); + + let stream = client + .get_stream(&stream_id) + .await + .expect("get stream after purge") + .expect("purge keeps the stream"); + assert_eq!( + stream.messages_count, 0, + "the stream rollup must drop with its purged topic" + ); + assert_eq!(stream.size.as_bytes_u64(), 0); +} diff --git a/core/integration/tests/server/scenarios/authentication_scenario.rs b/core/integration/tests/server/scenarios/authentication_scenario.rs index e39d6cd6a4..0932e55861 100644 --- a/core/integration/tests/server/scenarios/authentication_scenario.rs +++ b/core/integration/tests/server/scenarios/authentication_scenario.rs @@ -127,17 +127,16 @@ async fn test_all_commands_require_auth(client: &IggyClient) { ) { continue; } - // server-ng serves `GetClusterMetadata` pre-auth so a client can + // The server serves `GetClusterMetadata` pre-auth so a client can // locate the cluster leader before signing in; the legacy server // still auth-gates it. - #[cfg(feature = "vsr")] if code == GET_CLUSTER_METADATA_CODE { continue; } // Stateful - not supported on HTTP. `SYNC_CONSUMER_GROUP` is // SDK-internal (issued during poll partition resolution), with no // top-level client method to invoke unauthenticated here; its auth - // gate is exercised through the server-ng dispatch allowlist instead. + // gate is exercised through the server dispatch allowlist instead. if matches!( code, JOIN_CONSUMER_GROUP_CODE | LEAVE_CONSUMER_GROUP_CODE | SYNC_CONSUMER_GROUP_CODE @@ -153,7 +152,7 @@ async fn test_all_commands_require_auth(client: &IggyClient) { } // v2 consumer-offset ops are registered in the dispatch table for the // consensus/simulator pathway but are not wired into the legacy binary - // server's dispatch. They'll move into server-ng alongside the rest of + // server's dispatch. They'll move into the server alongside the rest of // the v2 surface; re-enable these codes here once that lands. if matches!( code, diff --git a/core/integration/tests/server/scenarios/bench_scenario.rs b/core/integration/tests/server/scenarios/bench_scenario.rs deleted file mode 100644 index b5a333e67c..0000000000 --- a/core/integration/tests/server/scenarios/bench_scenario.rs +++ /dev/null @@ -1,45 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use iggy::prelude::*; -use iggy_common::TransportProtocol; -use integration::bench_utils::run_bench_and_wait_for_finish; -use integration::harness::TestHarness; - -pub async fn run(harness: &TestHarness) { - let transport = harness.transport().expect("Transport not set"); - let server = harness.server(); - let server_addr = match transport { - TransportProtocol::Tcp => server.raw_tcp_addr().expect("TCP address not available"), - TransportProtocol::Http => server - .http_addr() - .expect("HTTP address not available") - .to_string(), - TransportProtocol::Quic => server - .quic_addr() - .expect("QUIC address not available") - .to_string(), - TransportProtocol::WebSocket => server - .websocket_addr() - .expect("WebSocket address not available") - .to_string(), - }; - let data_size = IggyByteSize::from(8 * 1024 * 1024); - - run_bench_and_wait_for_finish(&server_addr, &transport, "pinned-producer", data_size); - run_bench_and_wait_for_finish(&server_addr, &transport, "pinned-consumer", data_size); -} diff --git a/core/integration/tests/server/scenarios/encryption_scenario.rs b/core/integration/tests/server/scenarios/encryption_scenario.rs index 79c1c2e995..4316a1fd90 100644 --- a/core/integration/tests/server/scenarios/encryption_scenario.rs +++ b/core/integration/tests/server/scenarios/encryption_scenario.rs @@ -31,7 +31,11 @@ use test_case::test_matrix; #[tokio::test] #[parallel] async fn should_fill_data_with_headers_and_verify_after_restart_using_api(encryption: bool) { + // Restart scenarios run single-node: restarting a node in a multi-node + // cluster trips a known partitions-plane view-change stall, tracked + // separately. let mut harness = TestHarness::builder() + .cluster_nodes(1) .server(build_server_config(encryption)) .build() .unwrap(); @@ -93,20 +97,9 @@ async fn should_fill_data_with_headers_and_verify_after_restart_using_api(encryp .await .unwrap(); - // server-ng has no flush primitive (FLUSH_UNSAVED_BUFFER denies typed); - // the eager-flush envs in `build_server_config` make every committed - // batch hit disk instead. - #[cfg(not(feature = "vsr"))] - client - .flush_unsaved_buffer( - &Identifier::named(stream_name).unwrap(), - &Identifier::named(topic_name).unwrap(), - 0, - true, - ) - .await - .unwrap(); - + // No flush primitive exists (FLUSH_UNSAVED_BUFFER denies typed); the + // eager-flush envs in `build_server_config` make every committed batch hit + // disk instead. tokio::time::sleep(tokio::time::Duration::from_millis(500)).await; // Verify on-disk encryption of headers and payload @@ -251,20 +244,9 @@ async fn should_fill_data_with_headers_and_verify_after_restart_using_api(encryp .await .unwrap(); - // server-ng has no flush primitive (FLUSH_UNSAVED_BUFFER denies typed); - // the eager-flush envs in `build_server_config` make every committed - // batch hit disk instead. - #[cfg(not(feature = "vsr"))] - client - .flush_unsaved_buffer( - &Identifier::named(stream_name).unwrap(), - &Identifier::named(topic_name).unwrap(), - 0, - true, - ) - .await - .unwrap(); - + // No flush primitive exists (FLUSH_UNSAVED_BUFFER denies typed); the + // eager-flush envs in `build_server_config` make every committed batch hit + // disk instead. tokio::time::sleep(tokio::time::Duration::from_millis(500)).await; let polled = client @@ -498,7 +480,7 @@ fn encryption_disabled() -> bool { fn build_server_config(encryption: bool) -> TestServerConfig { let mut extra_envs = HashMap::new(); - // server-ng flushes on the journal thresholds (no flush primitive), so + // The server flushes on the journal thresholds (no flush primitive), so // force every committed batch straight to disk for the on-disk asserts. extra_envs.insert( "IGGY_SYSTEM_PARTITION_MESSAGES_REQUIRED_TO_SAVE".to_string(), diff --git a/core/integration/tests/server/scenarios/mod.rs b/core/integration/tests/server/scenarios/mod.rs index 5287c30539..0143621e9c 100644 --- a/core/integration/tests/server/scenarios/mod.rs +++ b/core/integration/tests/server/scenarios/mod.rs @@ -16,8 +16,6 @@ // under the License. pub mod authentication_scenario; -#[cfg(not(feature = "vsr"))] -pub mod bench_scenario; pub mod concurrent_produce_consume_scenario; pub mod concurrent_scenario; pub mod consumer_group_auto_commit_reconnection_scenario; @@ -29,7 +27,7 @@ pub mod consumer_group_with_multiple_clients_polling_messages_scenario; pub mod consumer_group_with_single_client_polling_messages_scenario; pub mod consumer_timestamp_polling_scenario; // Cross-protocol PAT visibility (create via HTTP, list via TCP across shards, -// and the reverse). Runs under vsr too: server-ng serves the PAT routes on its +// and the reverse). Runs under vsr too: the server serves the PAT routes on its // shard-0 HTTP listener and the create/delete commit through the metadata STM, // so the token replicates to every shard a TCP client may land on. pub mod cross_protocol_pat_scenario; @@ -79,7 +77,7 @@ const MESSAGES_COUNT: u32 = 1337; /// /// `send_messages` acks at consensus commit while the owning shard applies /// the batch asynchronously (see the materialisation race note at the top -/// of `server-ng/src/partition_reconciler.rs`), so the first read after a +/// of `server/src/partition_reconciler.rs`), so the first read after a /// send burst can observe fewer messages than were acked. Retrying absorbs /// that convergence window without weakening the caller's assertion: real /// message loss still returns short and fails it once the deadline expires. diff --git a/core/integration/tests/server/scenarios/permissions_scenario.rs b/core/integration/tests/server/scenarios/permissions_scenario.rs index 76df9d80f3..cdbb514b07 100644 --- a/core/integration/tests/server/scenarios/permissions_scenario.rs +++ b/core/integration/tests/server/scenarios/permissions_scenario.rs @@ -69,7 +69,7 @@ pub async fn run(harness: &TestHarness) { // Missing resource behavior tests test_missing_resource_behavior(harness, &root_client).await; - // RBAC surface ported from the raw-HTTP server-ng suite (server::http_rbac), + // RBAC surface ported from the raw-HTTP the server suite (server::http_rbac), // asserting the exact typed IggyError over every transport. test_consumer_offset_permissions(harness, &root_client).await; test_change_password_reply_path(harness, &root_client).await; @@ -2474,7 +2474,7 @@ async fn test_consumer_offset_permissions(harness: &TestHarness, root_client: &I IggyError::Unauthorized, "no perms: delete_consumer_offset", ); - // GET is enumeration-safe: legacy answers Ok(None), server-ng denies typed. + // GET is enumeration-safe: legacy answers Ok(None), the server denies typed. assert_unauthorized( client .get_consumer_offset(&consumer, &stream_id, &topic_id, Some(0)) diff --git a/core/integration/tests/server/scenarios/purge_delete_scenario.rs b/core/integration/tests/server/scenarios/purge_delete_scenario.rs index c790aae63f..30cda49ac4 100644 --- a/core/integration/tests/server/scenarios/purge_delete_scenario.rs +++ b/core/integration/tests/server/scenarios/purge_delete_scenario.rs @@ -31,44 +31,22 @@ const PARTITION_ID: u32 = 0; const LOG_EXTENSION: &str = "log"; const INDEX_EXTENSION: &str = "index"; -/// Payload chosen so IGGY_MESSAGE_HEADER_SIZE + payload = 1000B per message on disk. -/// -/// Rotation mechanics (with segment.size = 5KiB = 5120B, messages_required_to_save = 1): -/// `is_full()` checks `size >= 5120` BEFORE persisting the current message. -/// After 6 persisted messages (6000B >= 5120) the next arrival sees is_full=true, -/// gets persisted into the same segment, then rotation fires. -/// Result: 7 messages per sealed segment (7000B on disk). +/// The server persists the actual `SendMessages2` batch framing: a 256-byte +/// command header per append (each send below is a single-message batch) plus +/// a 48-byte per-message header, and a 24-byte sparse index entry per flush +/// (one per message with messages_required_to_save = 1). See +/// `server_common::send_messages2` and `stream_size_validation_scenario`. const PAYLOAD_SIZE: usize = 936; -#[cfg(not(feature = "vsr"))] -const MESSAGE_ON_DISK_SIZE: u64 = IGGY_MESSAGE_HEADER_SIZE as u64 + PAYLOAD_SIZE as u64; -#[cfg(not(feature = "vsr"))] -const INDEX_SIZE_PER_MSG: u64 = INDEX_SIZE as u64; -// server-ng persists the actual `SendMessages2` batch framing: a 256-byte -// command header per append (each send below is a single-message batch) plus -// a 48-byte per-message header, and a 24-byte sparse index entry per flush -// (one per message with messages_required_to_save = 1). See -// `server_common::send_messages2` and `stream_size_validation_scenario`. -#[cfg(feature = "vsr")] const NG_BATCH_HEADER_SIZE: u64 = 256; -#[cfg(feature = "vsr")] const NG_MESSAGE_HEADER_SIZE: u64 = 48; -#[cfg(feature = "vsr")] const MESSAGE_ON_DISK_SIZE: u64 = NG_BATCH_HEADER_SIZE + NG_MESSAGE_HEADER_SIZE + PAYLOAD_SIZE as u64; -#[cfg(feature = "vsr")] const INDEX_SIZE_PER_MSG: u64 = 24; const TOTAL_MESSAGES: u32 = 25; -/// 3 sealed segments (7 msgs each) + 1 active (4 msgs at offsets 21-24). -#[cfg(not(feature = "vsr"))] -const EXPECTED_SEGMENT_OFFSETS: &[u64] = &[0, 7, 14, 21]; -#[cfg(not(feature = "vsr"))] -const MSGS_PER_SEALED_SEGMENT: u64 = 7; /// 5 sealed segments (5 msgs each at 1240B on disk; the post-append size /// check seals at 6200B >= 5KiB) + 1 empty active segment at offset 25. -#[cfg(feature = "vsr")] const EXPECTED_SEGMENT_OFFSETS: &[u64] = &[0, 5, 10, 15, 20, 25]; -#[cfg(feature = "vsr")] const MSGS_PER_SEALED_SEGMENT: u64 = 5; /// Single consumer barrier: oldest-first deletion, barrier advancement, and edge cases. @@ -159,11 +137,29 @@ pub async fn run(harness: &mut TestHarness, restart_server: bool) { // reflect the true partition max (24), not messages_count - 1 (17). { let max_offset = (TOTAL_MESSAGES - 1) as u64; - let offset_info = client - .get_consumer_offset(&consumer, &stream_ident, &topic_ident, Some(PARTITION_ID)) - .await - .unwrap() - .expect("consumer offset must exist after segment deletion"); + // Short poll, not a one-shot read: the restart cells reconnect, and a + // read issued before the SDK settles on the leader can land on a replica + // that has not applied the offset op yet, which answers "no offset" + // rather than redirecting. Measured sub-millisecond on every converging + // run, so 2s is a transient allowance -- an offset that is genuinely + // gone still fails here. + let offset_deadline = std::time::Instant::now() + std::time::Duration::from_secs(2); + let offset_info = loop { + // A read racing the SDK's post-restart re-sign-in answers + // `Unauthenticated`; retry it inside the window like an absent + // offset rather than panicking on the Result. + if let Ok(Some(info)) = client + .get_consumer_offset(&consumer, &stream_ident, &topic_ident, Some(PARTITION_ID)) + .await + { + break info; + } + assert!( + std::time::Instant::now() < offset_deadline, + "consumer offset must exist after segment deletion" + ); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + }; assert_eq!(offset_info.stored_offset, stored_offset); assert_eq!( offset_info.current_offset, @@ -374,18 +370,9 @@ pub async fn run_no_consumers(harness: &mut TestHarness, restart_server: bool) { let partition_path = partition_path(&data_path, stream_id, topic_id); - // server-ng's per-message on-disk framing differs from legacy, so the - // segment boundaries are not hardcodable. Capture the real layout once; - // legacy still verifies it matches the calculated offsets + file sizes. + // Capture the real layout rather than hardcoding boundaries: this path + // only needs a sealed segment plus the active one. let layout = get_sorted_segment_offsets(&partition_path); - #[cfg(not(feature = "vsr"))] - { - assert_eq!( - layout, EXPECTED_SEGMENT_OFFSETS, - "Segment layout must match calculated offsets" - ); - assert_segment_file_sizes(&partition_path, EXPECTED_SEGMENT_OFFSETS); - } assert!( layout.len() >= 2, "expected at least one sealed segment plus the active one, got {layout:?}" @@ -405,11 +392,9 @@ pub async fn run_no_consumers(harness: &mut TestHarness, restart_server: bool) { maybe_restart(harness, restart_server).await; let first_surviving = layout[i + 1]; - // server-ng deletes asynchronously (metadata commit -> reconciler); - // legacy deletes synchronously. Converge before asserting. + // Deletion is asynchronous (metadata commit -> reconciler), so + // converge before asserting. await_segment_layout(&partition_path, &layout[i + 1..]).await; - #[cfg(not(feature = "vsr"))] - assert_segment_file_sizes(&partition_path, &layout[i + 1..]); await_polled_offsets( &client, &stream_ident, @@ -431,8 +416,6 @@ pub async fn run_no_consumers(harness: &mut TestHarness, restart_server: bool) { .unwrap(); await_segment_layout(&partition_path, std::slice::from_ref(&active)).await; - #[cfg(not(feature = "vsr"))] - assert_segment_file_sizes(&partition_path, std::slice::from_ref(&active)); await_polled_offsets( &client, &stream_ident, @@ -865,15 +848,6 @@ pub async fn run_purge_topic(harness: &mut TestHarness, restart_server: bool) { let partition_path = partition_path(&data_path, stream_id, topic_id); - // Exact layout is legacy-framing-specific; the purge outcome asserted below - // (offsets cleared, files deleted, partition reset to a single empty segment - // at offset 0, new messages from offset 0) is framing-agnostic. - #[cfg(not(feature = "vsr"))] - assert_eq!( - get_sorted_segment_offsets(&partition_path), - EXPECTED_SEGMENT_OFFSETS - ); - // --- Store individual consumer offset at 13 --- let consumer = Consumer { kind: ConsumerKind::Consumer, @@ -982,16 +956,15 @@ pub async fn run_purge_topic(harness: &mut TestHarness, restart_server: bool) { let drained_before_restart = is_dir_empty(&consumers_dir) && is_dir_empty(&groups_dir); maybe_restart(harness, restart_server).await; - // server-ng purges asynchronously (metadata commit -> reconciler -> pump); - // legacy purges synchronously. The pump's purge resets the partition to a - // single segment at offset 0 and clears consumer offsets + files in the - // same frame, so converging on the [0] layout means the whole purge landed. - #[cfg(feature = "vsr")] + // Purge is asynchronous (metadata commit -> reconciler -> pump). The + // pump's purge resets the partition to a single segment at offset 0 and + // clears consumer offsets + files in the same frame, so converging on the + // [0] layout means the whole purge landed. await_segment_layout(&partition_path, &[0]).await; // --- Verify consumer offsets cleared (memory + disk) --- - // ZERO tolerance everywhere except one cell: vsr + restart where the kill - // landed mid-purge. There boot plants the [0] layout itself (fencing a torn + // ZERO tolerance everywhere except one cell: restart where the kill landed + // mid-purge. There boot plants the [0] layout itself (fencing a torn // chain, or recovering an already-drained directory) with the offset files // still present, so the layout gate above is satisfied BEFORE the // reconciler's re-purge clears them (the kill preceded the purge.gen @@ -1001,26 +974,41 @@ pub async fn run_purge_topic(harness: &mut TestHarness, restart_server: bool) { // would hide a regression that clears them one frame late. Kept short -- // a client-visible stale offset after purge-then-restart is a real // (bounded) window, not something to paper over with a long tolerance. - let poll_window = if cfg!(feature = "vsr") && restart_server && !drained_before_restart { - std::time::Duration::from_secs(2) + // 5s, not 2s: the re-purge after a restart is floor-bounded by the + // reconciler's 1s PERIODIC tick, not by a wake -- measured at 1.06-1.11s in + // isolation against 0.5-0.8ms for every non-restart cell. 2s left under one + // tick of slack, so metadata repair under load pushed it over. Still a + // bounded window on purpose: widen only with a measurement, and if this + // starts needing more, the wake is missing rather than the budget too small. + let poll_window = if restart_server && !drained_before_restart { + std::time::Duration::from_secs(5) } else { std::time::Duration::ZERO }; let offsets_deadline = std::time::Instant::now() + poll_window; loop { - let consumer_offset = client - .get_consumer_offset(&consumer, &stream_ident, &topic_ident, Some(PARTITION_ID)) - .await - .unwrap(); - let group_offset = client - .get_consumer_offset( + // Errors retry inside the window instead of panicking: the restart cells + // reconnect mid-loop, so the first read after the server comes back can + // answer `Unauthenticated` while the SDK is still re-signing in. A + // transient here is "not converged yet", not a verdict. + let reads = futures::future::join( + client.get_consumer_offset(&consumer, &stream_ident, &topic_ident, Some(PARTITION_ID)), + client.get_consumer_offset( &group_consumer_ref, &stream_ident, &topic_ident, Some(PARTITION_ID), - ) - .await - .unwrap(); + ), + ) + .await; + let (Ok(consumer_offset), Ok(group_offset)) = reads else { + assert!( + std::time::Instant::now() < offsets_deadline, + "consumer offset reads never succeeded after purge: {reads:?}" + ); + tokio::time::sleep(std::time::Duration::from_millis(25)).await; + continue; + }; let consumer_files: Vec<_> = read_dir(&consumers_dir) .map(|e| e.filter_map(|e| e.ok().map(|e| e.file_name())).collect()) .unwrap_or_default(); @@ -1043,7 +1031,7 @@ pub async fn run_purge_topic(harness: &mut TestHarness, restart_server: bool) { } // --- Verify partition reset: single empty segment at offset 0 --- - assert_fresh_empty_partition(&partition_path); + assert_fresh_empty_partition(&partition_path).await; // --- Verify new messages start at offset 0 --- let new_msg_count = 3u32; @@ -1090,15 +1078,12 @@ pub async fn run_purge_topic(harness: &mut TestHarness, restart_server: bool) { /// post-purge data. Without that file a restart re-reads applied=0 against /// the replayed committed generation and silently wipes the new messages on /// its first pass. -#[cfg(feature = "vsr")] pub async fn run_purge_survives_restart(harness: &mut TestHarness) { let client = build_root_client(harness); client.connect().await.unwrap(); - let data_path = harness.server().data_path().to_path_buf(); - - let stream = client.create_stream(STREAM_NAME).await.unwrap(); + client.create_stream(STREAM_NAME).await.unwrap(); let stream_ident = Identifier::named(STREAM_NAME).unwrap(); - let topic = client + client .create_topic( &stream_ident, TOPIC_NAME, @@ -1111,14 +1096,21 @@ pub async fn run_purge_survives_restart(harness: &mut TestHarness) { .await .unwrap(); let topic_ident = Identifier::named(TOPIC_NAME).unwrap(); - let partition_path = partition_path(&data_path, stream.id, topic.id); send_messages(&client, &stream_ident, &topic_ident, 10).await; client .purge_topic(&stream_ident, &topic_ident) .await .unwrap(); - await_segment_layout(&partition_path, &[0]).await; + // The poll going empty is the barrier, not the segment layout: 10 messages + // never rotate the default 1.07 GB segment, so the directory holds one + // `0.log` before AND after the purge and a layout gate on `[0]` passes on + // its first read, before the purge has applied. `purge_topic` returns on + // the metadata commit while the reconciler stages the reset and the pump + // applies it, so an unsynchronized send races that window and is either + // wiped or fenced below the purge floor -- silently, since the offset it + // was acked at never becomes visible. + poll_exactly(&client, &stream_ident, &topic_ident, 0).await; send_messages(&client, &stream_ident, &topic_ident, 3).await; poll_exactly(&client, &stream_ident, &topic_ident, 3).await; @@ -1143,7 +1135,6 @@ pub async fn run_purge_survives_restart(harness: &mut TestHarness) { /// in-memory journal as consensus history, and the graceful-shutdown flush /// walks them again. The purge floor must fence them out of the segment so /// the restart recovers only the post-purge appends. -#[cfg(feature = "vsr")] pub async fn run_resident_purge_no_resurface(harness: &mut TestHarness) { let client = build_root_client(harness); client.connect().await.unwrap(); @@ -1199,7 +1190,6 @@ pub async fn run_resident_purge_no_resurface(harness: &mut TestHarness) { /// messages are served, so an extra resurfaced message fails the count /// instead of being cropped by the poll size. Panics after /// [`POLL_CONVERGENCE_TIMEOUT`] with the last observed count. -#[cfg(feature = "vsr")] async fn poll_exactly( client: &IggyClient, stream_ident: &Identifier, @@ -1260,11 +1250,9 @@ async fn await_stored_offset( /// Wait for the partition's on-disk segment layout to converge to `expected`. /// -/// server-ng's `DeleteSegments` is eventually-consistent: the client call -/// returns after the metadata `TruncatePartition` commit, and the partition -/// reconciler performs the on-disk deletion on its next pass. Legacy deletes -/// synchronously, so it asserts immediately. -#[cfg(feature = "vsr")] +/// `DeleteSegments` is eventually-consistent: the client call returns after +/// the metadata `TruncatePartition` commit, and the partition reconciler +/// performs the on-disk deletion on its next pass. async fn await_segment_layout(partition_path: &str, expected: &[u64]) { for _ in 0..200 { if get_sorted_segment_offsets(partition_path).as_slice() == expected { @@ -1281,11 +1269,9 @@ async fn await_segment_layout(partition_path: &str, expected: &[u64]) { /// Assert the layout stays at `expected` when no deletion must happen. /// -/// The vsr side sleeps past a reconciler pass first, since an erroneous -/// deletion would land asynchronously; legacy deletes synchronously, so an -/// immediate assert suffices. +/// Sleeps past a reconciler pass first, since an erroneous deletion would +/// land asynchronously. async fn assert_layout_stable(partition_path: &str, expected: &[u64]) { - #[cfg(feature = "vsr")] tokio::time::sleep(std::time::Duration::from_millis(1500)).await; assert_eq!( get_sorted_segment_offsets(partition_path).as_slice(), @@ -1294,14 +1280,6 @@ async fn assert_layout_stable(partition_path: &str, expected: &[u64]) { ); } -#[cfg(not(feature = "vsr"))] -async fn await_segment_layout(partition_path: &str, expected: &[u64]) { - assert_eq!( - get_sorted_segment_offsets(partition_path).as_slice(), - expected - ); -} - async fn maybe_restart(harness: &mut TestHarness, restart_server: bool) { if restart_server { harness.restart_server().await.unwrap(); @@ -1400,7 +1378,10 @@ async fn poll_all_offsets( kind: ConsumerKind::Consumer, id: Identifier::numeric(99).unwrap(), }; - let polled = client + // An errored poll reads as "nothing yet" so the caller's retry loop keeps + // going: the restart cells reconnect mid-scenario and the first poll after + // the server returns can answer `Unauthenticated` while the SDK re-signs in. + client .poll_messages( stream_ident, topic_ident, @@ -1411,8 +1392,8 @@ async fn poll_all_offsets( false, ) .await - .unwrap(); - polled.messages.iter().map(|m| m.header.offset).collect() + .map(|polled| polled.messages.iter().map(|m| m.header.offset).collect()) + .unwrap_or_default() } /// Asserts that each segment's `.log` and `.index` files have the exact expected size. @@ -1493,12 +1474,12 @@ fn is_dir_empty(dir: &str) -> bool { /// Asserts the partition directory contains exactly one .log and one .index file at offset 0, /// both with size 0 — the expected state after a full purge or segment reset. -fn assert_fresh_empty_partition(partition_path: &str) { - assert_eq!( - get_sorted_segment_offsets(partition_path), - [0], - "Partition must contain a single segment at offset 0" - ); +/// +/// Awaits the layout rather than reading once: a state-transfer install unlinks +/// the old chain before planting the replacement, so a replica that learns the +/// purge that way exposes a window with no `.log` at all. +async fn assert_fresh_empty_partition(partition_path: &str) { + await_segment_layout(partition_path, &[0]).await; assert_eq!( count_files_with_ext(partition_path, INDEX_EXTENSION), 1, @@ -1524,7 +1505,7 @@ fn assert_fresh_empty_partition(partition_path: &str) { /// /// `get_sorted_segment_offsets` only checks .log files -- this additionally /// verifies that the .index file count matches, catching stale .index files -/// left behind. server-ng unlinks a segment's .log and .index files across +/// left behind. The server unlinks a segment's .log and .index files across /// separate awaits, so a layout that already converged on .log files can /// transiently show one extra .index file. async fn assert_no_orphaned_segment_files(partition_path: &str, expected_count: usize) { diff --git a/core/integration/tests/server/scenarios/reconnect_after_restart_scenario.rs b/core/integration/tests/server/scenarios/reconnect_after_restart_scenario.rs index bc2da26a73..32b4d0cd02 100644 --- a/core/integration/tests/server/scenarios/reconnect_after_restart_scenario.rs +++ b/core/integration/tests/server/scenarios/reconnect_after_restart_scenario.rs @@ -627,7 +627,7 @@ pub async fn run_ring_overflow_rejoin(harness: &mut TestHarness) { // count the overflow stops happening and the test silently passes without // covering the floor path. Fail loud instead. const _: () = assert!( - RING_OVERFLOW_OPS as usize > configs::ng_partition::DEFAULT_EVICTED_RING_CAPACITY, + RING_OVERFLOW_OPS as usize > configs::partition::DEFAULT_EVICTED_RING_CAPACITY, "RING_OVERFLOW_OPS must exceed the default evicted ring capacity, or this scenario no longer exercises RangeEvicted", ); diff --git a/core/integration/tests/server/scenarios/restart_offset_skip_scenario.rs b/core/integration/tests/server/scenarios/restart_offset_skip_scenario.rs index 31378d0b0f..03bf5774f2 100644 --- a/core/integration/tests/server/scenarios/restart_offset_skip_scenario.rs +++ b/core/integration/tests/server/scenarios/restart_offset_skip_scenario.rs @@ -95,17 +95,9 @@ pub async fn run(harness: &mut TestHarness) { .await .unwrap(); - // Explicitly flush to disk, then wait for message_saver to persist. - // server-ng has no flush primitive (FLUSH_UNSAVED_BUFFER denies typed); - // its graceful shutdown flushes the committed journal, which this - // scenario's restart exercises instead. - #[cfg(not(feature = "vsr"))] - setup_client - .flush_unsaved_buffer( - &stream_id, &topic_id, 0, true, // fsync - ) - .await - .unwrap(); + // No flush primitive exists (FLUSH_UNSAVED_BUFFER denies typed); graceful + // shutdown flushes the committed journal, which this scenario's restart + // exercises instead. tokio::time::sleep(Duration::from_secs(2)).await; drop(setup_client); diff --git a/core/integration/tests/server/scenarios/stream_size_validation_scenario.rs b/core/integration/tests/server/scenarios/stream_size_validation_scenario.rs index ffa38a84a7..fb16f39918 100644 --- a/core/integration/tests/server/scenarios/stream_size_validation_scenario.rs +++ b/core/integration/tests/server/scenarios/stream_size_validation_scenario.rs @@ -36,21 +36,14 @@ const T1_NAME: &str = "test-topic-1"; const S2_NAME: &str = "test-stream-2"; const T2_NAME: &str = "test-topic-2"; const MESSAGE_PAYLOAD_SIZE_BYTES: u64 = 57; -#[cfg(not(feature = "vsr"))] -const MSG_SIZE: u64 = IGGY_MESSAGE_HEADER_SIZE as u64 + MESSAGE_PAYLOAD_SIZE_BYTES; // number of bytes in a single message const MSGS_COUNT: u64 = 117; // number of messages in a single topic after one pass of appending -#[cfg(not(feature = "vsr"))] -const MSGS_SIZE: u64 = MSG_SIZE * MSGS_COUNT; // number of bytes in a single topic after one pass of appending -// server-ng accounts the actual on-disk batch framing: one 256-byte +// The server accounts the actual on-disk batch framing: one 256-byte // `SendMessages2` command header per append pass plus a 48-byte per-message // header (`server_common::send_messages2::{COMMAND_HEADER_SIZE, -// MESSAGE_HEADER_SIZE}`), instead of the legacy 64-byte per-message header. -// Each pass below sends all `MSGS_COUNT` messages in one batch. -#[cfg(feature = "vsr")] +// MESSAGE_HEADER_SIZE}`). Each pass below sends all `MSGS_COUNT` messages in +// one batch. const NG_BATCH_HEADER_SIZE: u64 = 256; -#[cfg(feature = "vsr")] const NG_MESSAGE_HEADER_SIZE: u64 = 48; -#[cfg(feature = "vsr")] const MSGS_SIZE: u64 = NG_BATCH_HEADER_SIZE + (NG_MESSAGE_HEADER_SIZE + MESSAGE_PAYLOAD_SIZE_BYTES) * MSGS_COUNT; diff --git a/core/integration/tests/server/scenarios/system_scenario.rs b/core/integration/tests/server/scenarios/system_scenario.rs index 2e11ba4991..5e1b81ba4d 100644 --- a/core/integration/tests/server/scenarios/system_scenario.rs +++ b/core/integration/tests/server/scenarios/system_scenario.rs @@ -289,11 +289,8 @@ pub async fn run(harness: &TestHarness) { assert_eq!(topic.name, TOPIC_NAME); assert_eq!(topic.partitions_count, PARTITIONS_COUNT); assert_eq!(topic.partitions.len(), PARTITIONS_COUNT as usize); - // The exact byte size is framing-specific: legacy counts a 64-byte header - // per message; server-ng counts its on-disk batch framing. - #[cfg(not(feature = "vsr"))] - assert_eq!(topic.size, 100502); - #[cfg(feature = "vsr")] + // The exact byte size tracks the on-disk batch framing, so only its + // presence is asserted here. assert!(topic.size > 0); assert_eq!(topic.messages_count, MESSAGES_COUNT as u64); let topic_partition = topic.partitions.get((PARTITION_ID) as usize).unwrap(); diff --git a/core/integration/tests/server/specific.rs b/core/integration/tests/server/specific.rs index 0c455d045e..d0508a4e9d 100644 --- a/core/integration/tests/server/specific.rs +++ b/core/integration/tests/server/specific.rs @@ -72,21 +72,12 @@ async fn producer_reconnect_after_server_restart(harness: &mut TestHarness) { reconnect_after_restart_scenario::run_producer(harness).await; } -// QUIC stays vsr-gated on an SDK gap: after the restart the QUIC client -// redirects to the new leader, reconnects, and signs in, but the long-lived -// consumer's polls then return nothing for the whole window -- the -// post-reconnect request path wedges (QUIC also lacks the TCP client's -// mid-connection failover). TCP and WebSocket run. -#[cfg_attr(not(feature = "vsr"), iggy_harness( - test_client_transport = [Tcp, WebSocket, Quic], - server( - tcp.socket.override_defaults = true, - tcp.socket.nodelay = true, - quic.max_idle_timeout = "500s", - quic.keep_alive_interval = "15s" - ) -))] -#[cfg_attr(feature = "vsr", iggy_harness( +// QUIC is excluded on an SDK gap: after the restart the QUIC client redirects +// to the new leader, reconnects, and signs in, but the long-lived consumer's +// polls then return nothing for the whole window -- the post-reconnect request +// path wedges (QUIC also lacks the TCP client's mid-connection failover). TCP +// and WebSocket run. +#[iggy_harness( test_client_transport = [Tcp, WebSocket], server( tcp.socket.override_defaults = true, @@ -94,7 +85,7 @@ async fn producer_reconnect_after_server_restart(harness: &mut TestHarness) { quic.max_idle_timeout = "500s", quic.keep_alive_interval = "15s" ) -))] +)] async fn consumer_reconnect_after_server_restart(harness: &mut TestHarness) { reconnect_after_restart_scenario::run_consumer(harness).await; } @@ -107,10 +98,8 @@ async fn single_message_restart_offset_zero(harness: &mut TestHarness) { reconnect_after_restart_scenario::run_single_message_offset_zero_restart(harness).await; } -// Full-cluster restart is vsr-only by construction: it exercises the rejoin -// probe's election fallback across all replicas, which a single-process -// legacy server has no equivalent of (plain restart covers it there). -#[cfg(feature = "vsr")] +// Exercises the rejoin probe's election fallback across all replicas, which a +// plain single-node restart does not reach. #[iggy_harness(server( partition.messages_required_to_save = "1", partition.enforce_fsync = true @@ -119,10 +108,8 @@ async fn full_cluster_restart_recovers_and_serves(harness: &mut TestHarness) { reconnect_after_restart_scenario::run_full_cluster_restart(harness).await; } -// vsr-only: exercises `RangeEvicted` + the commit floor, which only exist -// on the replicated plane (the rejoin window exceeds the peers' evicted -// ring, so journal repair alone cannot cover it). -#[cfg(feature = "vsr")] +// Exercises `RangeEvicted` + the commit floor: the rejoin window exceeds the +// peers' evicted ring, so journal repair alone cannot cover it. #[iggy_harness(server( partition.messages_required_to_save = "1", partition.enforce_fsync = true @@ -171,8 +158,8 @@ async fn restart_offset_skip(harness: &mut TestHarness) { /// Test configuration: /// - 8 producers total (2 per protocol: TCP, HTTP, QUIC, WebSocket) /// - All producers write to the same partition for maximum lock contention -// Concurrency race test: under vsr it runs over the three VSR transports -// (TCP/QUIC/WebSocket -- HTTP/REST carries no VSR framing), legacy runs all four. +// Concurrency race test: runs over the three VSR transports (TCP/QUIC/ +// WebSocket -- HTTP/REST carries no VSR framing). #[iggy_harness(server( segment.size = "512B", message_saver.interval = "1s", diff --git a/core/server/src/shard/tasks/periodic/systemd_watchdog.rs b/core/integration/tests/server/stats_vsr.rs similarity index 51% rename from core/server/src/shard/tasks/periodic/systemd_watchdog.rs rename to core/integration/tests/server/stats_vsr.rs index 29aea68da7..beaa97831f 100644 --- a/core/server/src/shard/tasks/periodic/systemd_watchdog.rs +++ b/core/integration/tests/server/stats_vsr.rs @@ -15,33 +15,25 @@ // specific language governing permissions and limitations // under the License. -use crate::shard::IggyShard; -use crate::shard::systemd; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::info; +//! Stats against the server (vsr): `clients_count` must report the cross-shard +//! connected-client total gathered by the `ListClients` broadcast, not the +//! hardcoded 0 the sync single-shard read used to answer. -pub fn spawn_systemd_watchdog(shard: Rc) { - let Some(timeout) = sd_notify::watchdog_enabled() else { - return; - }; +use iggy::prelude::*; +use integration::iggy_harness; - let interval = timeout / 2; - info!( - "Systemd watchdog enabled, pinging every {}s (timeout: {}s).", - interval.as_secs(), - timeout.as_secs() - ); +#[iggy_harness( + test_client_transport = [Tcp], + server(tcp.socket.override_defaults = true, tcp.socket.nodelay = true) +)] +async fn given_connected_clients_when_getting_stats_should_count_clients(harness: &TestHarness) { + let clients = harness.tcp_root_clients(2).await.expect("tcp root clients"); - shard - .task_registry - .periodic("systemd_watchdog") - .every(interval) - .tick(move |_shutdown| ping_watchdog()) - .spawn(); -} + let stats = clients[0].get_stats().await.expect("get stats"); -async fn ping_watchdog() -> Result<(), IggyError> { - systemd::ping_watchdog(); - Ok(()) + assert_eq!( + stats.clients_count, 2, + "stats must count both connected clients, got {}", + stats.clients_count + ); } diff --git a/core/integration/tests/server/topic_admission_vsr.rs b/core/integration/tests/server/topic_admission_vsr.rs index 8f7c7da157..dac97dd969 100644 --- a/core/integration/tests/server/topic_admission_vsr.rs +++ b/core/integration/tests/server/topic_admission_vsr.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -//! Topic admission and echo semantics against server-ng (vsr). Create bounds +//! Topic admission and echo semantics against the server (vsr). Create bounds //! must be rejected with typed errors before consensus: partitions count above //! `MAX_PARTITIONS_PER_REQUEST` denies with `TooManyPartitions` (for create //! topic, create partitions and delete partitions alike); a custom @@ -23,8 +23,10 @@ //! `InvalidTopicSize`; `ServerDefault` and `Unlimited` sizes pass. Update //! stores `max_topic_size` and `message_expiry` verbatim and gets echo the //! stored value (never the node default frozen at update time), matching -//! legacy wire behavior. Listing topics of a missing stream replies with an -//! empty list, as the legacy server does. +//! legacy wire behavior. Deleting more partitions than the topic has rejects +//! with `InvalidPartitionsCount` as a committed result instead of silently +//! acking a no-op. Listing topics of a missing stream replies with an empty +//! list, as the legacy server does. use std::str::FromStr; @@ -254,6 +256,69 @@ async fn given_out_of_bounds_partitions_count_when_mutating_should_reject_typed( ); } +#[iggy_harness( + test_client_transport = [Tcp], + server(tcp.socket.override_defaults = true, tcp.socket.nodelay = true) +)] +async fn given_over_count_when_deleting_partitions_should_reject_invalid_partitions_count( + harness: &TestHarness, +) { + let client = harness.tcp_root_client().await.expect("tcp root client"); + client + .create_stream("over-count-stream") + .await + .expect("create stream"); + let stream_id = Identifier::from_str_value("over-count-stream").expect("stream identifier"); + create_topic_with( + &client, + &stream_id, + "over-count-topic", + 3, + MaxTopicSize::ServerDefault, + ) + .await + .expect("create topic"); + let topic_id = Identifier::from_str_value("over-count-topic").expect("topic identifier"); + + // Deleting more partitions than the topic has must reject with the legacy + // typed error, not silently no-op and ack. + let invalid_count = IggyError::InvalidPartitionsCount.as_code(); + let result = client.delete_partitions(&stream_id, &topic_id, 4).await; + assert!( + matches!(&result, Err(error) if error.as_code() == invalid_count), + "deleting 4 partitions of a 3-partition topic must deny with \ + InvalidPartitionsCount, got {result:?}" + ); + let topic = client + .get_topic(&stream_id, &topic_id) + .await + .expect("get topic") + .expect("topic exists"); + assert_eq!( + topic.partitions_count, 3, + "the rejected over-count delete must not remove any partition" + ); + + client + .delete_partitions(&stream_id, &topic_id, 3) + .await + .expect("deleting exactly the topic's partition count is accepted"); + let topic = client + .get_topic(&stream_id, &topic_id) + .await + .expect("get topic") + .expect("topic exists"); + assert_eq!(topic.partitions_count, 0, "all partitions are gone"); + + // Same rejection once the topic is already empty (any count exceeds 0). + let result = client.delete_partitions(&stream_id, &topic_id, 1).await; + assert!( + matches!(&result, Err(error) if error.as_code() == invalid_count), + "deleting from a zero-partition topic must deny with \ + InvalidPartitionsCount, got {result:?}" + ); +} + #[iggy_harness( test_client_transport = [Tcp], server(tcp.socket.override_defaults = true, tcp.socket.nodelay = true) diff --git a/core/integration/tests/state/file.rs b/core/integration/tests/state/file.rs deleted file mode 100644 index 2e604625c5..0000000000 --- a/core/integration/tests/state/file.rs +++ /dev/null @@ -1,164 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::state::StateSetup; -use bytes::Bytes; -use iggy_binary_protocol::WireEncode; -use iggy_binary_protocol::WireName; -use iggy_binary_protocol::requests::{streams::CreateStreamRequest, users::CreateUserRequest}; -use server::state::command::EntryCommand; -use server::state::entry::StateEntry; -use server::state::models::{CreateStreamWithId, CreateUserWithId}; - -#[compio::test] -async fn should_be_empty_given_initialized_state() { - let setup = StateSetup::init().await; - let state = setup.state(); - state.init().await.unwrap(); - let entries = state.load_entries().await.unwrap(); - assert!(entries.is_empty()); -} - -#[compio::test] -async fn should_apply_single_entry() { - let setup = StateSetup::init().await; - let state = setup.state(); - state.init().await.unwrap(); - - let user_id = 0; - let command = EntryCommand::CreateUser(CreateUserWithId { - user_id: 1, - command: CreateUserRequest { - username: WireName::new("test").unwrap(), - password: "secret".to_string(), - status: 1, - permissions: None, - }, - }); - let command_bytes = command.to_bytes(); - - state.apply(user_id, &command).await.unwrap(); - - let mut entries = state.load_entries().await.unwrap(); - assert_eq!(entries.len(), 1); - let entry = entries.remove(0); - assert_entry(entry, 0, setup.version(), user_id, command_bytes); -} - -#[compio::test] -async fn should_apply_encrypted_entry() { - let setup = StateSetup::init_with_encryptor().await; - let state = setup.state(); - state.init().await.unwrap(); - - let user_id = 0; - let command = EntryCommand::CreateUser(CreateUserWithId { - user_id: 1, - command: CreateUserRequest { - username: WireName::new("test").unwrap(), - password: "secret".to_string(), - status: 1, - permissions: None, - }, - }); - let command_bytes = command.to_bytes(); - - state.apply(user_id, &command).await.unwrap(); - - let mut entries = state.load_entries().await.unwrap(); - assert_eq!(entries.len(), 1); - let entry = entries.remove(0); - assert_entry(entry, 0, setup.version(), user_id, command_bytes); -} - -#[compio::test] -async fn should_apply_multiple_entries() { - let setup = StateSetup::init().await; - let state = setup.state(); - let entries = state.init().await.unwrap(); - - assert!(entries.is_empty()); - assert_eq!(state.current_index(), 0); - assert_eq!(state.entries_count(), 0); - assert_eq!(state.term(), 0); - - let first_user_id = 0; // Root user - let created_user_id = 1; // First created user - let create_user = EntryCommand::CreateUser(CreateUserWithId { - user_id: created_user_id, - command: CreateUserRequest { - username: WireName::new("test").unwrap(), - password: "secret".to_string(), - status: 1, - permissions: None, - }, - }); - let create_user_bytes = create_user.to_bytes(); - - state.apply(first_user_id, &create_user).await.unwrap(); - - assert_eq!(state.current_index(), 0); - assert_eq!(state.entries_count(), 1); - - let second_user_id = 1; - let stream_id = 1; - let create_stream = EntryCommand::CreateStream(CreateStreamWithId { - stream_id, - command: CreateStreamRequest { - name: WireName::new("test").unwrap(), - }, - }); - let create_stream_bytes = create_stream.to_bytes(); - - state.apply(second_user_id, &create_stream).await.unwrap(); - - assert_eq!(state.current_index(), 1); - assert_eq!(state.entries_count(), 2); - - let mut entries = state.load_entries().await.unwrap(); - assert_eq!(entries.len(), 2); - - let create_user_entry = entries.remove(0); - assert_entry( - create_user_entry, - 0, - setup.version(), - first_user_id, - create_user_bytes, - ); - - let create_stream_entry = entries.remove(0); - assert_entry( - create_stream_entry, - 1, - setup.version(), - second_user_id, - create_stream_bytes, - ); -} - -fn assert_entry(entry: StateEntry, index: u64, version: u32, user_id: u32, command: Bytes) { - assert_eq!(entry.index, index); - assert_eq!(entry.term, 0); - assert_eq!(entry.version, version); - assert_eq!(entry.flags, 0); - assert!(entry.checksum > 0); - assert!(entry.timestamp.as_micros() > 0); - assert_eq!(entry.user_id, user_id); - assert_eq!(entry.command, command); - assert!(entry.context.is_empty()); -} diff --git a/core/integration/tests/state/mod.rs b/core/integration/tests/state/mod.rs deleted file mode 100644 index b6623bcba3..0000000000 --- a/core/integration/tests/state/mod.rs +++ /dev/null @@ -1,92 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use compio::fs::create_dir; -use iggy::prelude::{Aes256GcmEncryptor, EncryptorKind}; -use iggy_common::SemanticVersion; -use server::state::file::FileState; -use server::streaming::persistence::persister::{FileWithSyncPersister, PersisterKind}; -use server::streaming::utils::file::overwrite; -use std::str::FromStr; -use std::sync::Arc; -use std::sync::atomic::{AtomicU32, AtomicU64}; -use uuid::Uuid; - -mod file; -mod system; - -pub struct StateSetup { - directory_path: String, - state: FileState, - version: u32, -} - -impl StateSetup { - pub async fn init() -> StateSetup { - StateSetup::create(None).await - } - - pub async fn init_with_encryptor() -> StateSetup { - StateSetup::create(Some(&[1; 32])).await - } - - pub async fn create(encryption_key: Option<&[u8]>) -> StateSetup { - let directory_path = format!("state_{}", Uuid::now_v7().to_u128_le()); - let messages_file_path = format!("{directory_path}/log"); - create_dir(&directory_path).await.unwrap(); - overwrite(&messages_file_path).await.unwrap(); - - let version = SemanticVersion::from_str("1.2.3").unwrap(); - let persister = PersisterKind::FileWithSync(FileWithSyncPersister {}); - let encryptor = encryption_key - .map(|key| EncryptorKind::Aes256Gcm(Aes256GcmEncryptor::new(key).unwrap())); - let state_current_index = Arc::new(AtomicU64::new(0)); - let state_entries_count = Arc::new(AtomicU64::new(0)); - let state_current_leader = Arc::new(AtomicU32::new(0)); - let state_term = Arc::new(AtomicU64::new(0)); - let state = FileState::new( - &messages_file_path, - &version, - Arc::new(persister), - encryptor, - state_current_index, - state_entries_count, - state_current_leader, - state_term, - ); - - Self { - directory_path, - state, - version: version.get_numeric_version().unwrap(), - } - } - - pub fn state(&self) -> &FileState { - &self.state - } - - pub fn version(&self) -> u32 { - self.version - } -} - -impl Drop for StateSetup { - fn drop(&mut self) { - std::fs::remove_dir_all(&self.directory_path).unwrap(); - } -} diff --git a/core/integration/tests/state/system.rs b/core/integration/tests/state/system.rs deleted file mode 100644 index f6491fcd25..0000000000 --- a/core/integration/tests/state/system.rs +++ /dev/null @@ -1,212 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::state::StateSetup; -use iggy_binary_protocol::requests::{ - consumer_groups::CreateConsumerGroupRequest, partitions::CreatePartitionsRequest, - personal_access_tokens::CreatePersonalAccessTokenRequest, streams::CreateStreamRequest, - streams::DeleteStreamRequest, topics::CreateTopicRequest, users::CreateUserRequest, -}; -use iggy_binary_protocol::{WireIdentifier, WireName}; -use server::state::command::EntryCommand; -use server::state::models::{ - CreateConsumerGroupWithId, CreatePersonalAccessTokenWithHash, CreateStreamWithId, - CreateTopicWithId, CreateUserWithId, -}; -use server::state::system::SystemState; - -#[compio::test] -async fn should_be_initialized_based_on_state_entries() { - let setup = StateSetup::init().await; - let state = setup.state(); - state.init().await.unwrap(); - - let user_id = 0; - let create_user = CreateUserRequest { - username: WireName::new("user").unwrap(), - password: "secret".to_string(), - status: 1, // Active - permissions: None, - }; - - let stream1_id = 1u32; - let create_stream1 = CreateStreamRequest { - name: WireName::new("stream1").unwrap(), - }; - - let topic1_id = 1u32; - let create_topic1 = CreateTopicRequest { - stream_id: WireIdentifier::numeric(stream1_id), - partitions_count: 1, - compression_algorithm: 1, // None compression - message_expiry: 0, // NeverExpire - max_topic_size: 0, // ServerDefault - replication_factor: 0, // None - name: WireName::new("topic1").unwrap(), - }; - - let stream2_id = 2u32; - let create_stream2 = CreateStreamRequest { - name: WireName::new("stream2").unwrap(), - }; - - let topic2_id = 2u32; - let create_topic2 = CreateTopicRequest { - stream_id: WireIdentifier::numeric(stream2_id), - partitions_count: 1, - compression_algorithm: 1, - message_expiry: 0, - max_topic_size: 0, - replication_factor: 0, - name: WireName::new("topic2").unwrap(), - }; - - let create_partitions = CreatePartitionsRequest { - stream_id: WireIdentifier::numeric(stream1_id), - topic_id: WireIdentifier::numeric(topic1_id), - partitions_count: 2, - }; - - let delete_stream = DeleteStreamRequest { - stream_id: WireIdentifier::numeric(stream2_id), - }; - - let create_personal_access_token = CreatePersonalAccessTokenWithHash { - command: CreatePersonalAccessTokenRequest { - name: WireName::new("test").unwrap(), - expiry: 0, // NeverExpire - }, - hash: "hash".to_string(), - }; - - let create_consumer_group = CreateConsumerGroupRequest { - stream_id: WireIdentifier::numeric(stream1_id), - topic_id: WireIdentifier::numeric(topic1_id), - name: WireName::new("test").unwrap(), - }; - - let group_id = 1u32; - - state - .apply( - user_id, - &EntryCommand::CreateUser(CreateUserWithId { - user_id, - command: create_user, - }), - ) - .await - .unwrap(); - state - .apply( - user_id, - &EntryCommand::CreateStream(CreateStreamWithId { - stream_id: stream1_id, - command: create_stream1, - }), - ) - .await - .unwrap(); - state - .apply( - user_id, - &EntryCommand::CreateTopic(CreateTopicWithId { - topic_id: topic1_id, - command: create_topic1, - }), - ) - .await - .unwrap(); - state - .apply( - user_id, - &EntryCommand::CreateStream(CreateStreamWithId { - stream_id: stream2_id, - command: create_stream2, - }), - ) - .await - .unwrap(); - state - .apply( - user_id, - &EntryCommand::CreateTopic(CreateTopicWithId { - topic_id: topic2_id, - command: create_topic2, - }), - ) - .await - .unwrap(); - state - .apply(user_id, &EntryCommand::CreatePartitions(create_partitions)) - .await - .unwrap(); - state - .apply(user_id, &EntryCommand::DeleteStream(delete_stream)) - .await - .unwrap(); - state - .apply( - user_id, - &EntryCommand::CreatePersonalAccessToken(create_personal_access_token), - ) - .await - .unwrap(); - state - .apply( - user_id, - &EntryCommand::CreateConsumerGroup(CreateConsumerGroupWithId { - group_id, - command: create_consumer_group, - }), - ) - .await - .unwrap(); - - let entries = state.load_entries().await.unwrap(); - assert_eq!(entries.len(), 9); - - let mut system = SystemState::init(entries).await.unwrap(); - - assert_eq!(system.users.len(), 1); - let mut user = system.users.remove(&user_id).unwrap(); - assert_eq!(user.id, user_id); - assert_eq!(user.username, "user"); - assert_eq!(user.password_hash, "secret"); - assert_eq!(user.personal_access_tokens.len(), 1); - - let personal_access_token = user.personal_access_tokens.remove("test").unwrap(); - assert_eq!(personal_access_token.token_hash, "hash"); - assert_eq!(personal_access_token.name, "test"); - - assert_eq!(system.streams.len(), 1); - let mut stream = system.streams.remove(&stream1_id).unwrap(); - assert_eq!(stream.id, stream1_id); - assert_eq!(stream.name, "stream1"); - assert_eq!(stream.topics.len(), 1); - - let mut topic = stream.topics.remove(&topic1_id).unwrap(); - assert_eq!(topic.id, topic1_id); - assert_eq!(topic.name, "topic1"); - assert_eq!(topic.partitions.len(), 3); - - assert_eq!(topic.consumer_groups.len(), 1); - let consumer_group = topic.consumer_groups.remove(&group_id).unwrap(); - - assert_eq!(consumer_group.id, group_id); - assert_eq!(consumer_group.name, "test"); -} diff --git a/core/integration/tests/storage/consumer_offsets.rs b/core/integration/tests/storage/consumer_offsets.rs deleted file mode 100644 index f0e36f2233..0000000000 --- a/core/integration/tests/storage/consumer_offsets.rs +++ /dev/null @@ -1,179 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use iggy_common::{ConsumerKind, IggyError}; -use server::streaming::partitions::storage::{load_consumer_group_offsets, load_consumer_offsets}; -use std::path::Path; -use std::sync::atomic::Ordering; - -fn write_offset_file(dir: &Path, name: &str, offset: u64) { - std::fs::write(dir.join(name), offset.to_le_bytes()).unwrap(); -} - -#[test] -fn load_consumer_offsets_valid_files() { - let dir = tempfile::tempdir().unwrap(); - write_offset_file(dir.path(), "1", 100); - write_offset_file(dir.path(), "2", 200); - write_offset_file(dir.path(), "3", 300); - - let offsets = load_consumer_offsets(dir.path().to_str().unwrap()).unwrap(); - - assert_eq!(offsets.len(), 3); - assert_eq!(offsets[0].consumer_id, 1); - assert_eq!(offsets[0].offset.load(Ordering::Relaxed), 100); - assert_eq!(offsets[0].kind, ConsumerKind::Consumer); - assert_eq!(offsets[1].consumer_id, 2); - assert_eq!(offsets[1].offset.load(Ordering::Relaxed), 200); - assert_eq!(offsets[2].consumer_id, 3); - assert_eq!(offsets[2].offset.load(Ordering::Relaxed), 300); -} - -#[test] -fn load_consumer_offsets_skips_non_numeric_files() { - let dir = tempfile::tempdir().unwrap(); - write_offset_file(dir.path(), ".DS_Store", 0); - write_offset_file(dir.path(), "backup.bak", 0); - write_offset_file(dir.path(), "1", 42); - - let offsets = load_consumer_offsets(dir.path().to_str().unwrap()).unwrap(); - - assert_eq!(offsets.len(), 1); - assert_eq!(offsets[0].consumer_id, 1); - assert_eq!(offsets[0].offset.load(Ordering::Relaxed), 42); -} - -#[test] -fn load_consumer_offsets_skips_truncated_files() { - let dir = tempfile::tempdir().unwrap(); - std::fs::write(dir.path().join("1"), [0u8; 3]).unwrap(); - std::fs::write(dir.path().join("2"), []).unwrap(); - write_offset_file(dir.path(), "3", 500); - - let offsets = load_consumer_offsets(dir.path().to_str().unwrap()).unwrap(); - - assert_eq!(offsets.len(), 1); - assert_eq!(offsets[0].consumer_id, 3); - assert_eq!(offsets[0].offset.load(Ordering::Relaxed), 500); -} - -#[test] -fn load_consumer_offsets_empty_dir() { - let dir = tempfile::tempdir().unwrap(); - - let offsets = load_consumer_offsets(dir.path().to_str().unwrap()).unwrap(); - - assert!(offsets.is_empty()); -} - -#[test] -fn load_consumer_offsets_skips_directories() { - let dir = tempfile::tempdir().unwrap(); - std::fs::create_dir(dir.path().join("123")).unwrap(); - write_offset_file(dir.path(), "1", 77); - - let offsets = load_consumer_offsets(dir.path().to_str().unwrap()).unwrap(); - - assert_eq!(offsets.len(), 1); - assert_eq!(offsets[0].consumer_id, 1); - assert_eq!(offsets[0].offset.load(Ordering::Relaxed), 77); -} - -#[test] -fn load_consumer_offsets_nonexistent_dir() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().to_str().unwrap().to_string(); - drop(dir); - - let result = load_consumer_offsets(&path); - - assert!(result.is_err()); - assert!(matches!( - result.unwrap_err(), - IggyError::CannotReadConsumerOffsets(_) - )); -} - -#[test] -fn load_consumer_group_offsets_valid_files() { - let dir = tempfile::tempdir().unwrap(); - write_offset_file(dir.path(), "1", 500); - write_offset_file(dir.path(), "2", 600); - - let offsets = load_consumer_group_offsets(dir.path().to_str().unwrap()).unwrap(); - - assert_eq!(offsets.len(), 2); - for (group_id, offset) in &offsets { - assert_eq!(offset.kind, ConsumerKind::ConsumerGroup); - assert_eq!(offset.consumer_id, group_id.0 as u32); - } - let ids: Vec = offsets.iter().map(|(_, co)| co.consumer_id).collect(); - assert!(ids.contains(&1)); - assert!(ids.contains(&2)); -} - -#[test] -fn load_consumer_group_offsets_skips_non_numeric_files() { - let dir = tempfile::tempdir().unwrap(); - write_offset_file(dir.path(), ".DS_Store", 0); - write_offset_file(dir.path(), "notes.txt", 0); - write_offset_file(dir.path(), "5", 999); - - let offsets = load_consumer_group_offsets(dir.path().to_str().unwrap()).unwrap(); - - assert_eq!(offsets.len(), 1); - assert_eq!(offsets[0].0.0, 5); - assert_eq!(offsets[0].1.consumer_id, 5); - assert_eq!(offsets[0].1.offset.load(Ordering::Relaxed), 999); -} - -#[test] -fn load_consumer_group_offsets_skips_truncated_files() { - let dir = tempfile::tempdir().unwrap(); - std::fs::write(dir.path().join("1"), [0u8; 4]).unwrap(); - write_offset_file(dir.path(), "2", 750); - - let offsets = load_consumer_group_offsets(dir.path().to_str().unwrap()).unwrap(); - - assert_eq!(offsets.len(), 1); - assert_eq!(offsets[0].0.0, 2); - assert_eq!(offsets[0].1.offset.load(Ordering::Relaxed), 750); -} - -#[test] -fn load_consumer_group_offsets_empty_dir() { - let dir = tempfile::tempdir().unwrap(); - - let offsets = load_consumer_group_offsets(dir.path().to_str().unwrap()).unwrap(); - - assert!(offsets.is_empty()); -} - -#[test] -fn load_consumer_group_offsets_nonexistent_dir() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().to_str().unwrap().to_string(); - drop(dir); - - let result = load_consumer_group_offsets(&path); - - assert!(result.is_err()); - assert!(matches!( - result.unwrap_err(), - IggyError::CannotReadConsumerOffsets(_) - )); -} diff --git a/core/integration/tests/storage/mod.rs b/core/integration/tests/storage/mod.rs deleted file mode 100644 index 1c05a5a3b0..0000000000 --- a/core/integration/tests/storage/mod.rs +++ /dev/null @@ -1,18 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -mod consumer_offsets; diff --git a/core/journal/src/lib.rs b/core/journal/src/lib.rs index 9de342b6c1..1d223d7e6c 100644 --- a/core/journal/src/lib.rs +++ b/core/journal/src/lib.rs @@ -46,6 +46,25 @@ where None } + /// Remove every entry at or above `from_op`, returning how many went, and + /// leave the snapshot watermark where it is. + /// + /// Not `drain` with a different range: `drain` advances the watermark past what + /// it removed, which would mark the removed ops evictable when a suffix + /// truncation needs them refillable. + /// + /// Required, not defaulted: an `Unsupported` default hides a missing impl until + /// mid-view-change, where the caller can only wedge or start a view over a log it + /// cannot serve. + /// + /// # Errors + /// I/O error if the rewrite fails. + fn truncate_from(&self, from_op: u64) -> impl Future>; + + /// Highest op the index holds. Not derivable from [`Self::header`]: a caller + /// looking for a suffix ABOVE some op has no bound to probe up to. + fn last_op(&self) -> Option; + /// Remove entries with ops in `ops` from the journal, /// returning the removed entries sorted by op. /// diff --git a/core/journal/src/prepare_journal.rs b/core/journal/src/prepare_journal.rs index 3f4f332ccd..9e665db836 100644 --- a/core/journal/src/prepare_journal.rs +++ b/core/journal/src/prepare_journal.rs @@ -18,7 +18,7 @@ use crate::file_storage::FileStorage; use crate::{Journal, JournalHandle}; use compio::io::AsyncWriteAtExt; -use iggy_binary_protocol::consensus::{Command2, PrepareHeader}; +use iggy_binary_protocol::consensus::{CHECKSUM_UNSEALED, Command2, PrepareHeader}; use server_common::{MESSAGE_ALIGN, Message, iobuf::Owned}; use std::cell::{Cell, OnceCell, Ref, RefCell}; use std::fmt; @@ -66,7 +66,7 @@ pub(crate) const SLOT_COUNT: usize = 1024; /// Default in-memory index size, in slots. Overridable per journal via /// [`PrepareJournal::open_with_slots`] (operator knob: `[metadata] -/// journal_slots` in the server-ng config). +/// journal_slots` in the server config). pub const DEFAULT_SLOT_COUNT: usize = SLOT_COUNT; /// Error type for journal operations. @@ -391,7 +391,7 @@ impl PrepareJournal { /// `slot_count` bounds how many committed-but-unsnapshotted entries the /// journal holds before a forced checkpoint must reclaim WAL space; the /// caller owns keeping it above its checkpoint margin + prepare-queue - /// depth (validated at config load for the server-ng `[metadata]` knob). + /// depth (validated at config load for the server `[metadata]` knob). /// /// # Errors /// Returns `JournalError::Io` if the WAL file cannot be opened or read, @@ -412,7 +412,7 @@ impl PrepareJournal { Self::scan(storage, snapshot_op, slot_count).await } - #[allow(clippy::future_not_send)] + #[allow(clippy::future_not_send, clippy::too_many_lines)] async fn scan( storage: FileStorage, snapshot_op: u64, @@ -423,6 +423,8 @@ impl PrepareJournal { let mut offsets: Vec> = vec![None; slot_count]; let mut last_op: Option = None; let mut unsealed_entries: u64 = 0; + // Previous entry's `(op, checksum)`, for the parent-chain check. + let mut chain_previous: Option<(u64, u128)> = None; let mut pos: u64 = 0; let mut header_buf = vec![0u8; HEADER_SIZE]; // Reused 16-aligned scratch (PrepareHeader has u128 fields). Avoids @@ -477,14 +479,57 @@ impl PrepareJournal { // verify against and is skipped, not rejected: see // [`CHECKSUM_BODY_UNSEALED`]. // - // TODO(wal-integrity): the header `checksum` and its `parent` chain stay - // unverified, since the producer does not seal them yet (blocked on - // re-sealing re-stamped retransmits), so a bit-flip in a - // structurally-valid header field slips through. Recovery derives - // `commit_watermark = max(header.commit)`, so a flipped `commit` makes it - // apply prepared-but-uncommitted ops as committed, the very ops a view - // change may have truncated cluster-wide, diverging this replica. Seal - // and verify the header checksum + parent chain. + // The header's own integrity field is checked first, since a flipped + // header field is the more dangerous of the two: recovery derives + // `commit_watermark = max(header.commit)`, so a corrupted `commit` applies + // uncommitted ops as committed, diverging from the group. `size` and `op` + // are equally load-bearing for the scan itself. + if header.checksum != CHECKSUM_UNSEALED && header.identity_checksum() != header.checksum + { + if pos + entry_size < file_len { + return Err(JournalError::Io(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "interior WAL corruption at pos {pos} (op {}, operation {:?}): \ + prepare header checksum mismatch with {} bytes of entries \ + following; refusing to truncate and discard the committed suffix", + header.op, + header.operation, + file_len - (pos + entry_size), + ), + ))); + } + truncate_or_fail(&storage, pos, "prepare header checksum mismatch at tail").await?; + break; + } + + // The hash chain, checked only where meaningful: consecutive ops with both + // ends sealed. A gap means compaction dropped the predecessor, and an + // unsealed end has nothing to chain from, so neither is evidence of damage. + if let Some((previous_op, previous_checksum)) = chain_previous + && previous_op + 1 == header.op + && previous_checksum != CHECKSUM_UNSEALED + && header.parent != previous_checksum + { + if pos + entry_size < file_len { + return Err(JournalError::Io(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "interior WAL corruption at pos {pos}: op {} does not chain to op \ + {previous_op} (parent {} != {previous_checksum}) with {} bytes of \ + entries following; refusing to truncate and discard the committed \ + suffix", + header.op, + header.parent, + file_len - (pos + entry_size), + ), + ))); + } + truncate_or_fail(&storage, pos, "prepare parent chain break at tail").await?; + break; + } + chain_previous = Some((header.op, header.checksum)); + if header.checksum_body == CHECKSUM_BODY_UNSEALED { // Skip the body read too, so a WAL written entirely by a pre-sealing // build scans without touching its payload. @@ -697,6 +742,128 @@ impl PrepareJournal { clippy::future_not_send )] impl Journal for PrepareJournal { + fn last_op(&self) -> Option { + self.last_op.get() + } + + /// Remove every entry at or above `from_op`, leaving the snapshot floor where + /// it is. Returns how many entries went. + /// + /// Deliberately not `drain`, which compacts a committed prefix and advances + /// `snapshot_op` past its range. Doing that to a suffix would declare everything + /// below the head snapshotted, letting `append` evict live entries repair cannot + /// put back, when those ops are exactly the ones that must stay refillable. + /// + /// For the one caller that needs it: a backup whose uncommitted entries disagree + /// with the log a view change settled on. They cannot be corrected in place, and + /// journal repair skips their ops as already-present, so dropping them is what + /// lets the primary's retransmission refill the range. + /// + /// # Errors + /// I/O error if the rewrite fails. Past the rename the journal is poisoned on any + /// failure, as in `drain`: serving a pre-truncation offset or appending at a stale + /// `write_offset` is worse than a hard stop. `from_op` must be at least 1. + async fn truncate_from(&self, from_op: u64) -> io::Result { + if let Some(state) = self.poisoned.get() { + return Err(Self::poisoned_io_error(state)); + } + if from_op == 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "truncate_from: ops are 1-based, so 0 would discard the whole journal", + )); + } + // Shares the drain guard: both rewrite the same WAL through the same tmp + // path, so letting them overlap would race the swap. + if self.drain_in_flight.replace(true) { + return Err(io::Error::new( + io::ErrorKind::ResourceBusy, + "drain or truncate already in flight: concurrent rewrites would race the WAL", + )); + } + let _guard = DrainInFlightGuard(&self.drain_in_flight); + + let mut removed = 0usize; + let mut live: Vec<(PrepareHeader, u64)> = Vec::new(); + { + let headers = self.headers.borrow(); + let offsets = self.offsets.borrow(); + for slot in 0..self.slot_count { + if let (Some(header), Some(offset)) = (&headers[slot], offsets[slot]) { + if header.op >= from_op { + removed += 1; + } else { + live.push((*header, offset)); + } + } + } + } + if removed == 0 { + return Ok(0); + } + live.sort_unstable_by_key(|(header, _)| header.op); + + let wal_path = self.storage.path(); + let tmp_path = wal_path.with_extension("wal.tmp"); + let tmp_guard = TmpFileGuard::new(tmp_path.clone()); + { + let mut tmp = compio::fs::File::create(&tmp_path).await?; + let mut write_pos: u64 = 0; + for (header, old_offset) in &live { + let size = header.size as usize; + let buf = vec![0u8; size]; + let buf = self.storage.read_at(*old_offset, buf).await?; + let (result, _buf) = tmp.write_all_at(buf, write_pos).await.into(); + result?; + write_pos += size as u64; + } + tmp.sync_all().await?; + } + + // COMMIT POINT, same as `drain`: past the rename the on-disk WAL is the new + // one while the in-memory index still describes the old layout, so every + // fallible step below poisons rather than serving stale offsets. + compio::fs::rename(&tmp_path, wal_path).await?; + tmp_guard.defuse(); + + if let Some(parent) = wal_path.parent() { + let dir = match compio::fs::File::open(parent).await { + Ok(dir) => dir, + Err(error) => { + return Err(self.poison("truncate_from: open parent dir for fsync", error)); + } + }; + if let Err(error) = dir.sync_all().await { + return Err(self.poison("truncate_from: parent dir fsync", error)); + } + } + if let Err(error) = self.storage.reopen().await { + return Err(self.poison("truncate_from: storage reopen after rename", error)); + } + + // `snapshot_op` is deliberately untouched. See the doc comment. + let mut headers = self.headers.borrow_mut(); + let mut offsets = self.offsets.borrow_mut(); + let mut pos: u64 = 0; + for (header, _) in &live { + let slot = slot_for_op(header.op, self.slot_count); + offsets[slot] = Some(pos); + pos += u64::from(header.size); + } + for slot in 0..self.slot_count { + if let Some(header) = &headers[slot] + && header.op >= from_op + { + headers[slot] = None; + offsets[slot] = None; + } + } + // Unlike a prefix drain, removing a suffix moves the head. + self.last_op.set(live.last().map(|(header, _)| header.op)); + + Ok(removed) + } + type Header = PrepareHeader; type Entry = Message; type HeaderRef<'a> = Ref<'a, PrepareHeader>; @@ -1047,6 +1214,257 @@ mod tests { Message::try_from(buffer).unwrap() } + /// A prepare with both integrity fields sealed and its parent chained, as a live + /// producer writes them. `make_entry` leaves `checksum` zero, read as unsealed. + fn make_identity_sealed_prepare( + op: u64, + body_size: usize, + parent: u128, + ) -> Message { + let mut message = make_prepare(op, body_size); + let bytes = message.as_mut_slice(); + let header = bytemuck::checked::from_bytes_mut::(&mut bytes[..HEADER_SIZE]); + header.parent = parent; + header.view = 1; + let checksum = header.identity_checksum(); + header.checksum = checksum; + message + } + + /// Byte offset of `field_offset` within the entry for `op`, at a fixed stride. + const fn header_field_offset(op: u64, body_size: usize, field_offset: usize) -> usize { + (op as usize - 1) * (HEADER_SIZE + body_size) + field_offset + } + + #[compio::test] + async fn truncate_from_removes_the_suffix_and_keeps_the_snapshot_floor() { + // The property that makes this not-a-drain: the floor must not move, or the + // removed ops become evictable and repair can never put them back. + let dir = tempdir().unwrap(); + let path = dir.path().join("journal.wal"); + let journal = PrepareJournal::open(&path, 0).await.unwrap(); + for op in 1..=5u64 { + journal + .append(make_prepare(op, 64).deep_copy()) + .await + .unwrap(); + } + assert_eq!(journal.last_op(), Some(5)); + let floor_before = journal.snapshot_op(); + + let removed = journal.truncate_from(3).await.unwrap(); + assert_eq!(removed, 3, "ops 3, 4 and 5 must go"); + assert_eq!( + journal.snapshot_op(), + floor_before, + "truncating a suffix must not advance the snapshot floor" + ); + assert_eq!( + journal.last_op(), + Some(2), + "the head follows the truncation" + ); + for op in 1..=2u64 { + assert!( + journal.header(op as usize).is_some(), + "op {op} must survive" + ); + } + for op in 3..=5u64 { + assert!( + journal.header(op as usize).is_none(), + "op {op} must be gone" + ); + } + } + + #[compio::test] + async fn truncate_from_leaves_a_refillable_range() { + // The whole point: after truncation the ops can be appended again. A raised + // floor would either reject that or silently evict a live entry. + let dir = tempdir().unwrap(); + let path = dir.path().join("journal.wal"); + let journal = PrepareJournal::open(&path, 0).await.unwrap(); + for op in 1..=4u64 { + journal + .append(make_prepare(op, 64).deep_copy()) + .await + .unwrap(); + } + journal.truncate_from(3).await.unwrap(); + + for op in 3..=4u64 { + journal + .append(make_prepare(op, 64).deep_copy()) + .await + .expect("a truncated op must be appendable again"); + } + assert_eq!(journal.last_op(), Some(4)); + assert!(journal.header(3).is_some()); + assert!(journal.header(4).is_some()); + } + + #[compio::test] + async fn truncate_from_survives_reopen() { + // The rewrite has to be durable, not just reflected in the index. + const BODY: usize = 64; + let dir = tempdir().unwrap(); + let path = dir.path().join("journal.wal"); + { + let journal = PrepareJournal::open(&path, 0).await.unwrap(); + let mut parent = 0u128; + for op in 1..=4u64 { + let entry = make_identity_sealed_prepare(op, BODY, parent); + parent = entry.header().checksum; + journal.append(entry.deep_copy()).await.unwrap(); + } + assert_eq!(journal.truncate_from(3).await.unwrap(), 2); + } + let journal = PrepareJournal::open(&path, 0).await.unwrap(); + assert_eq!( + journal.last_op(), + Some(2), + "the truncation must be on disk, not only in the index" + ); + assert!(journal.header(3).is_none()); + } + + #[compio::test] + async fn scan_accepts_a_sealed_and_chained_wal() { + // Everything below only means something if the happy path still opens. + const BODY: usize = 64; + let dir = tempdir().unwrap(); + let path = dir.path().join("journal.wal"); + { + let journal = PrepareJournal::open(&path, 0).await.unwrap(); + let mut parent = 0u128; + for op in 1..=3u64 { + let entry = make_identity_sealed_prepare(op, BODY, parent); + parent = entry.header().checksum; + journal.append(entry.deep_copy()).await.unwrap(); + } + } + let journal = PrepareJournal::open(&path, 0).await.unwrap(); + assert_eq!(journal.last_op(), Some(3)); + assert_eq!( + journal.unsealed_entry_count(), + 0, + "sealed entries must not be counted as unsealed" + ); + } + + #[compio::test] + async fn scan_truncates_tail_entry_with_header_checksum_mismatch() { + // A flipped `commit` leaves the header structurally valid, so only the identity + // checksum catches it. Recovery derives its watermark from `max(header.commit)`, + // so an undetected flip applies prepared-but-uncommitted ops as committed. + const BODY: usize = 64; + let dir = tempdir().unwrap(); + let path = dir.path().join("journal.wal"); + { + let journal = PrepareJournal::open(&path, 0).await.unwrap(); + let first = make_identity_sealed_prepare(1, BODY, 0); + let parent = first.header().checksum; + journal.append(first.deep_copy()).await.unwrap(); + journal + .append(make_identity_sealed_prepare(2, BODY, parent).deep_copy()) + .await + .unwrap(); + } + + let commit_offset = + header_field_offset(2, BODY, std::mem::offset_of!(PrepareHeader, commit)); + let mut bytes = std::fs::read(&path).unwrap(); + bytes[commit_offset] ^= 0xFF; + std::fs::write(&path, &bytes).unwrap(); + + let journal = PrepareJournal::open(&path, 0).await.unwrap(); + assert_eq!( + journal.last_op(), + Some(1), + "a header-checksum mismatch on the tail entry must truncate it" + ); + assert!(journal.header(2).is_none()); + } + + #[compio::test] + async fn scan_refuses_boot_on_interior_header_checksum_mismatch() { + const BODY: usize = 64; + let dir = tempdir().unwrap(); + let path = dir.path().join("journal.wal"); + { + let journal = PrepareJournal::open(&path, 0).await.unwrap(); + let mut parent = 0u128; + for op in 1..=3u64 { + let entry = make_identity_sealed_prepare(op, BODY, parent); + parent = entry.header().checksum; + journal.append(entry.deep_copy()).await.unwrap(); + } + } + + let commit_offset = + header_field_offset(2, BODY, std::mem::offset_of!(PrepareHeader, commit)); + let mut bytes = std::fs::read(&path).unwrap(); + bytes[commit_offset] ^= 0xFF; + std::fs::write(&path, &bytes).unwrap(); + + let error = PrepareJournal::open(&path, 0).await.unwrap_err(); + let message = error.to_string(); + assert!( + message.contains("interior WAL corruption"), + "an interior header flip must refuse boot rather than discard the \ + committed suffix, got: {message}" + ); + } + + #[compio::test] + async fn scan_detects_a_parent_chain_break() { + // Both entries are individually well sealed; only the link is wrong. Catching + // this is what makes the log a chain rather than a bag of valid records. + const BODY: usize = 64; + let dir = tempdir().unwrap(); + let path = dir.path().join("journal.wal"); + { + let journal = PrepareJournal::open(&path, 0).await.unwrap(); + journal + .append(make_identity_sealed_prepare(1, BODY, 0).deep_copy()) + .await + .unwrap(); + // Op 2 chains to a parent that is not op 1's checksum. + journal + .append(make_identity_sealed_prepare(2, BODY, 0xDEAD_BEEF).deep_copy()) + .await + .unwrap(); + } + + let journal = PrepareJournal::open(&path, 0).await.unwrap(); + assert_eq!( + journal.last_op(), + Some(1), + "op 2 does not chain to op 1 and must be truncated as a torn tail" + ); + } + + #[compio::test] + async fn scan_skips_verification_for_unsealed_entries() { + // A WAL from a pre-sealing build must still open: `checksum` reads as the + // unsealed sentinel, so neither the identity nor the chain is checked. + const BODY: usize = 32; + let dir = tempdir().unwrap(); + let path = dir.path().join("journal.wal"); + { + let journal = PrepareJournal::open(&path, 0).await.unwrap(); + for op in 1..=2u64 { + journal + .append(make_unsealed_prepare(op, BODY).deep_copy()) + .await + .unwrap(); + } + } + let journal = PrepareJournal::open(&path, 0).await.unwrap(); + assert_eq!(journal.last_op(), Some(2)); + } + #[compio::test] async fn scan_truncates_entry_with_body_checksum_mismatch() { let dir = tempdir().unwrap(); diff --git a/core/message_bus/src/client_listener/quic.rs b/core/message_bus/src/client_listener/quic.rs index 63cb6ac78c..923eb0b5b7 100644 --- a/core/message_bus/src/client_listener/quic.rs +++ b/core/message_bus/src/client_listener/quic.rs @@ -22,7 +22,7 @@ //! invoking the supplied callback, so the callback receives a //! ready-for-traffic [`compio_quic::Connection`] plus its //! `(SendStream, RecvStream)` pair. No ALPN is advertised; protocol -//! version is validated by the caller (server-ng) inside the LOGIN +//! version is validated by the caller (the server) inside the LOGIN //! command. //! //! 0-RTT data is refused at accept time diff --git a/core/message_bus/src/config.rs b/core/message_bus/src/config.rs index 1d30a0420f..c31d14672d 100644 --- a/core/message_bus/src/config.rs +++ b/core/message_bus/src/config.rs @@ -18,7 +18,7 @@ //! Runtime tunables for the message bus. //! //! Single source of truth for these knobs is the on-disk schema -//! [`configs::server_ng::ServerNgConfig`]. The bus consumes that +//! [`configs::server::ServerConfig`]. The bus consumes that //! schema at construction (see [`crate::IggyMessageBus::with_config`]) //! and converts the schema-typed fields //! ([`iggy_common::IggyDuration`] / [`iggy_common::IggyByteSize`]) @@ -27,10 +27,10 @@ //! //! The WebSocket frame-layer config the bus consumes lives under the //! schema's `[websocket]` block (buffer sizes, message / frame -//! ceilings, unmasked-frame acceptance): the bus IS server-ng's +//! ceilings, unmasked-frame acceptance): the bus IS the server's //! WS / WSS install path, so the listener section carries the frame -//! tuning (see `configs::ng_websocket`). -//! [`From<&ServerNgConfig> for MessageBusConfig`](MessageBusConfig) +//! tuning (see `configs::websocket`). +//! [`From<&ServerConfig> for MessageBusConfig`](MessageBusConfig) //! folds that section into [`WebSocketConfig`] once at boot. //! //! Liveness detection is NOT done via TCP keepalive on the bus: SDK @@ -39,17 +39,17 @@ //! than by `SO_KEEPALIVE`. //! //! Neither plane is authenticated at the bus layer: identity and -//! credential checks belong to the caller (`core/server-ng`) via +//! credential checks belong to the caller (`core/server`) via //! `LOGIN_*` commands. This struct therefore carries no secret / //! token-source state. pub use compio::ws::tungstenite::protocol::WebSocketConfig; -use configs::server_ng::ServerNgConfig; +use configs::server::ServerConfig; use std::time::Duration; /// Pre-converted QUIC transport tuning derived from -/// [`ServerNgConfig::quic`](configs::ng_quic::QuicConfig). +/// [`ServerConfig::quic`](configs::quic::QuicConfig). /// /// Threaded into [`crate::transports::quic::transport_config_from`] at /// every bind site so the schema's `[quic]` block actually drives @@ -109,8 +109,8 @@ pub const IOV_MAX_LIMIT: usize = 512; /// Pre-converted runtime tunables in effect on a `IggyMessageBus` /// instance. /// -/// Built from a fully-validated [`ServerNgConfig`] via -/// [`From<&ServerNgConfig>`] at boot. All fields are runtime-typed +/// Built from a fully-validated [`ServerConfig`] via +/// [`From<&ServerConfig>`] at boot. All fields are runtime-typed /// (`Duration`, `usize`, `tungstenite::WebSocketConfig`) so hot paths /// read them directly without `.get_duration()` / `.as_bytes_u64()` /// conversion. @@ -177,9 +177,9 @@ pub struct MessageBusConfig { /// Threaded into `compio_ws::accept_async_with_config` on the WS /// install path and into `WssTransportConn::ws_handshake` for WSS. /// Built once at boot by `build_ws_config` (see the - /// [`From<&ServerNgConfig> for MessageBusConfig`](MessageBusConfig) impl below) + /// [`From<&ServerConfig> for MessageBusConfig`](MessageBusConfig) impl below) /// from the schema's `[websocket]` section, the live frame-tuning - /// source for server-ng's WS plane. + /// source for the server's WS plane. /// /// The [`WebSocketConfig`] type is re-exported from `compio_ws`'s /// vendored `tungstenite` so callers do not need a direct dep on @@ -187,23 +187,23 @@ pub struct MessageBusConfig { pub ws_config: WebSocketConfig, /// QUIC transport tuning, pre-converted from - /// [`ServerNgConfig::quic`](configs::ng_quic::QuicConfig) at boot. + /// [`ServerConfig::quic`](configs::quic::QuicConfig) at boot. pub quic: QuicTuning, } -impl From<&ServerNgConfig> for MessageBusConfig { - fn from(cfg: &ServerNgConfig) -> Self { +impl From<&ServerConfig> for MessageBusConfig { + fn from(cfg: &ServerConfig) -> Self { let bus = &cfg.message_bus; - // Production load goes through `ServerNgConfig::validate()`, which + // Production load goes through `ServerConfig::validate()`, which // already exercises `bus.validate()`. This debug-assert catches - // direct callers (tests, simulators) that build a `ServerNgConfig` + // direct callers (tests, simulators) that build a `ServerConfig` // by hand and forget to validate before converting. debug_assert!( >::validate(bus) .is_ok(), - "MessageBusConfig::from(&ServerNgConfig) called on an unvalidated bus config", + "MessageBusConfig::from(&ServerConfig) called on an unvalidated bus config", ); Self { max_batch: bus.max_batch, @@ -225,7 +225,7 @@ impl From<&ServerNgConfig> for MessageBusConfig { } } -/// Convert the schema's [`configs::ng_quic::QuicConfig`] +/// Convert the schema's [`configs::quic::QuicConfig`] /// (`IggyByteSize` / `IggyDuration` typed) into the runtime /// [`QuicTuning`] (plain integer / `Duration` fields). /// @@ -235,7 +235,7 @@ impl From<&ServerNgConfig> for MessageBusConfig { /// `unwrap_or` arms below are still bounded saturations that keep /// the build unconditionally infallible if a future caller skips /// validation in dev / test code. -fn build_quic_tuning(quic: &configs::ng_quic::QuicConfig) -> QuicTuning { +fn build_quic_tuning(quic: &configs::quic::QuicConfig) -> QuicTuning { QuicTuning { max_concurrent_bidi_streams: u32::try_from(quic.max_concurrent_bidi_streams) .unwrap_or(u32::MAX), @@ -251,12 +251,12 @@ fn build_quic_tuning(quic: &configs::ng_quic::QuicConfig) -> QuicTuning { impl Default for QuicTuning { /// Mirrors the `[quic]` defaults in - /// `core/server-ng/config.toml`: 64 MiB send/receive windows, + /// `core/server/config.toml`: 64 MiB send/receive windows, /// 30 s idle timeout, 10 s keep-alive, 8 KiB initial MTU, 100 KiB /// datagram send buffer, single bidi stream per peer. /// /// Intended for tests and direct callers; production builds - /// derive the field from [`ServerNgConfig`] so the values stay in + /// derive the field from [`ServerConfig`] so the values stay in /// lock-step with the on-disk schema. fn default() -> Self { Self { @@ -283,7 +283,7 @@ impl Default for QuicTuning { /// Conversion to `usize` saturates on platforms where `IggyByteSize` /// would overflow, but on supported targets `usize` is at least 32 /// bits, so saturation is unreachable in practice. -fn build_ws_config(websocket: &configs::ng_websocket::WebSocketConfig) -> WebSocketConfig { +fn build_ws_config(websocket: &configs::websocket::WebSocketConfig) -> WebSocketConfig { let mut ws = WebSocketConfig::default(); if let Some(sz) = websocket.read_buffer_size { ws = ws.read_buffer_size(byte_size_to_usize(sz)); @@ -311,7 +311,7 @@ fn byte_size_to_usize(sz: iggy_common::IggyByteSize) -> usize { impl Default for MessageBusConfig { fn default() -> Self { - Self::from(&ServerNgConfig::default()) + Self::from(&ServerConfig::default()) } } @@ -321,13 +321,13 @@ mod tests { /// `QuicTuning::default()` carries hand-coded literals that must /// match the schema-derived path through - /// `From<&ServerNgConfig> for MessageBusConfig`. If the embedded + /// `From<&ServerConfig> for MessageBusConfig`. If the embedded /// TOML or the literals drift, every test that uses /// `QuicTuning::default()` (e.g. `quic_client_roundtrip`) silently /// observes different bytes than production. Pin both sides here. #[test] fn quic_tuning_default_matches_schema() { - let schema_quic = MessageBusConfig::from(&ServerNgConfig::default()).quic; + let schema_quic = MessageBusConfig::from(&ServerConfig::default()).quic; let literal = QuicTuning::default(); assert_eq!( diff --git a/core/message_bus/src/installer/conn_info.rs b/core/message_bus/src/installer/conn_info.rs index 8584d86698..10cf776d45 100644 --- a/core/message_bus/src/installer/conn_info.rs +++ b/core/message_bus/src/installer/conn_info.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -//! Per-client transport metadata exposed to the caller (`server-ng`). +//! Per-client transport metadata exposed to the caller (`the server`). //! //! Constructed by the listener / install path on shard 0 with whatever //! is known at accept time (`client_id`, `peer_addr`, `transport`) and diff --git a/core/message_bus/src/installer/mod.rs b/core/message_bus/src/installer/mod.rs index 3a04c2ccaa..641fd74e6f 100644 --- a/core/message_bus/src/installer/mod.rs +++ b/core/message_bus/src/installer/mod.rs @@ -109,7 +109,7 @@ pub trait ConnectionInstaller { /// handshake, then installs WS reader / writer tasks via /// [`install_client_ws`] on success. On handshake failure /// the fd is closed by dropping the wrapping `TcpStream`. No - /// subprotocol negotiation: the caller (server-ng) gates command + /// subprotocol negotiation: the caller (the server) gates command /// access via the LOGIN allowlist. fn install_client_ws_fd(&self, fd: DupedFd, meta: ClientConnMeta, on_request: RequestHandler); diff --git a/core/message_bus/src/installer/wss.rs b/core/message_bus/src/installer/wss.rs index 32db26cb4c..d65f66fd34 100644 --- a/core/message_bus/src/installer/wss.rs +++ b/core/message_bus/src/installer/wss.rs @@ -37,7 +37,7 @@ use tracing::warn; /// then run inside the transport's `run` body on the per-connection /// install task; the install path stays thin. No subprotocol /// negotiation: client identity is established post-handshake by the -/// LOGIN command on the caller (server-ng). +/// LOGIN command on the caller (the server). /// /// `TCP_NODELAY` is applied pre-handshake for symmetry with /// [`super::tcp_tls::install_client_tcp_tls`]. `SO_KEEPALIVE` is diff --git a/core/message_bus/src/lib.rs b/core/message_bus/src/lib.rs index 7f3c84a654..644591110a 100644 --- a/core/message_bus/src/lib.rs +++ b/core/message_bus/src/lib.rs @@ -27,13 +27,13 @@ //! - **SDK-client plane**: ephemeral client connections. Available //! transports: TCP, TCP-TLS, WebSocket, WSS, QUIC. Each request //! carries a `(client: u128, request: u64)` pair in `RequestHeader`; -//! downstream consumers in `core/server-ng` are free to use it for +//! downstream consumers in `core/server` are free to use it for //! tracing, idempotency, or correlation. //! //! # Auth //! //! Neither plane is authenticated at the bus layer. Both connect first -//! and let the caller (`core/server-ng`) gate command access via +//! and let the caller (`core/server`) gate command access via //! application-level LOGIN commands: //! //! - SDK-client plane: `LOGIN_USER` / `LOGIN_WITH_PERSONAL_ACCESS_TOKEN`, @@ -101,7 +101,7 @@ pub use lifecycle::{ pub use transports::tls::TlsServerCredentials; pub use compio::runtime::JoinHandle; -use configs::server_ng::ServerNgConfig; +use configs::server::ServerConfig; use iggy_binary_protocol::{GenericHeader, ReplyHeader}; use server_common::{MESSAGE_ALIGN, Message, iobuf::Frozen}; use std::array; @@ -144,7 +144,7 @@ pub const OWNER_NONE: u16 = u16::MAX; /// Shared atomic owner table mapping `replica_id` to `owning_shard_id`. /// -/// One Arc-cloned instance is allocated per server-ng process at +/// One Arc-cloned instance is allocated per the server process at /// bootstrap and shared across every shard's [`IggyMessageBus`]. The owning shard /// stamps its id into a slot when an inbound replica connection passes /// the registry-insert race; the same shard CAS-clears the slot when @@ -723,7 +723,7 @@ pub struct IggyMessageBus { impl IggyMessageBus { /// Construct a bus with default tunables (derived from - /// [`ServerNgConfig::default`]). + /// [`ServerConfig::default`]). #[must_use] pub fn new(shard_id: u16) -> Self { Self::with_tunables(shard_id, MessageBusConfig::default()) @@ -743,10 +743,10 @@ impl IggyMessageBus { Self::with_tunables(shard_id, cfg) } - /// Construct a bus from the validated server-ng schema. + /// Construct a bus from the validated the server schema. /// /// Production constructor: takes a fully-validated - /// [`ServerNgConfig`] and derives the runtime [`MessageBusConfig`] + /// [`ServerConfig`] and derives the runtime [`MessageBusConfig`] /// internally. Field conversions ([`iggy_common::IggyDuration`] -> [`Duration`], /// [`iggy_common::IggyByteSize`] -> `usize`, schema WS knobs -> /// tungstenite [`WebSocketConfig`]) happen once here so hot paths @@ -759,19 +759,19 @@ impl IggyMessageBus { /// surfaces operator misconfiguration loudly rather than letting /// every `writev` fail silently with `EMSGSIZE` once traffic starts. #[must_use] - pub fn with_config(shard_id: u16, cfg: &ServerNgConfig) -> Self { + pub fn with_config(shard_id: u16, cfg: &ServerConfig) -> Self { Self::with_tunables(shard_id, MessageBusConfig::from(cfg)) } - /// Production constructor for multi-shard server-ng: same as + /// Production constructor for multi-shard the server: same as /// [`Self::with_config`] but takes a pre-allocated /// [`ReplicaOwnerTable`] Arc. Bootstrap allocates one table per - /// server-ng process and clones the Arc into every shard so all + /// server process and clones the Arc into every shard so all /// buses see the same atomic slots. #[must_use] pub fn with_config_and_owner_table( shard_id: u16, - cfg: &ServerNgConfig, + cfg: &ServerConfig, owner_table: Arc, ) -> Self { Self::with_tunables_and_owner_table(shard_id, MessageBusConfig::from(cfg), owner_table) @@ -781,7 +781,7 @@ impl IggyMessageBus { /// /// Used by the public constructors above and by tests that need to /// patch a single field on the derived [`MessageBusConfig`] without - /// round-tripping through [`ServerNgConfig`]. + /// round-tripping through [`ServerConfig`]. /// /// # Panics /// diff --git a/core/message_bus/src/replica/io.rs b/core/message_bus/src/replica/io.rs index d36ec4d18f..290db8e2d2 100644 --- a/core/message_bus/src/replica/io.rs +++ b/core/message_bus/src/replica/io.rs @@ -59,7 +59,7 @@ use crate::{ /// The cert chain is the leaf-first sequence rustls expects; the key /// is the server's private key in DER form. Tests use rcgen to mint a /// throwaway pair; production callers load real PKI material via -/// `core/server-ng`'s `[quic.certificate]` config section. +/// `core/server`'s `[quic.certificate]` config section. pub struct QuicServerCredentials { pub cert_chain: Vec>, pub key_der: PrivateKeyDer<'static>, @@ -331,7 +331,7 @@ pub async fn start_on_shard_zero( /// Leaves the WS, QUIC, TCP-TLS, and WSS listener slots unconfigured /// (`None`). Convenience entry for TCP-only deployments and existing /// tests; prefer the full [`start_on_shard_zero`] in production where -/// the optional planes come from `core/server-ng`'s config. +/// the optional planes come from `core/server`'s config. /// /// # Errors /// diff --git a/core/message_bus/src/transports/quic.rs b/core/message_bus/src/transports/quic.rs index 8fdf49e830..df571c9941 100644 --- a/core/message_bus/src/transports/quic.rs +++ b/core/message_bus/src/transports/quic.rs @@ -527,7 +527,7 @@ impl TransportConn for QuicTransportConn { /// Applies [`transport_config_from`] and disables 0-RTT; otherwise /// inherits upstream defaults including `migration: true`. No ALPN is /// advertised; protocol-version validation lives in the LOGIN command -/// on the caller (server-ng). +/// on the caller (the server). /// /// # Errors /// diff --git a/core/message_bus/src/transports/tls/mod.rs b/core/message_bus/src/transports/tls/mod.rs index c92ca70de4..888f9358e6 100644 --- a/core/message_bus/src/transports/tls/mod.rs +++ b/core/message_bus/src/transports/tls/mod.rs @@ -29,7 +29,7 @@ //! `ServerConfig::builder()` / `ClientConfig::builder()` call. The //! workspace pins `rustls = { features = ["ring"] }`, so the //! provider is `rustls::crypto::ring::default_provider()`. Multiple -//! call sites (transports, tests, server-ng bootstrap) may attempt to +//! call sites (transports, tests, the server bootstrap) may attempt to //! install it; the wrapper here is idempotent and safe under races //! between concurrent first-use sites. @@ -245,7 +245,7 @@ impl ServerCertVerifier for AcceptAnyServerCert { /// rustls 0.23 requires a default provider before any /// `ServerConfig::builder()` or `ClientConfig::builder()` call. This /// helper centralises the install so transports, tests, and -/// server-ng bootstrap converge on the same provider without +/// the server bootstrap converge on the same provider without /// per-call-site `let _ = ...install_default()`. pub fn install_default_crypto_provider() { // Race-safe: rustls's `install_default` returns Err if a provider diff --git a/core/message_bus/tests/ws_client_roundtrip.rs b/core/message_bus/tests/ws_client_roundtrip.rs index 88214becfd..a66f5a621f 100644 --- a/core/message_bus/tests/ws_client_roundtrip.rs +++ b/core/message_bus/tests/ws_client_roundtrip.rs @@ -26,7 +26,7 @@ //! server's `bus.send_to_client` reply lands on the client's reader. //! Verifies the full bidirectional plane through the reader / writer //! two-task split. Pre-LOGIN command gating is the caller's -//! responsibility (server-ng) and is not exercised here. +//! responsibility (the server) and is not exercised here. mod common; @@ -123,7 +123,7 @@ async fn handshake_succeeds_and_round_trip_completes() { async fn handshake_succeeds_without_subprotocol_header() { // Without the subprotocol gate, a client that sends NO // Sec-WebSocket-Protocol header must still complete the upgrade. - // Pre-LOGIN command gating is enforced by the caller (server-ng), + // Pre-LOGIN command gating is enforced by the caller (the server), // not at the bus layer. let bus = Rc::new(IggyMessageBus::new(0)); let on_request: RequestHandler = Rc::new(|_, _| {}); diff --git a/core/metadata/Cargo.toml b/core/metadata/Cargo.toml index 681ff73ead..cd83798bc6 100644 --- a/core/metadata/Cargo.toml +++ b/core/metadata/Cargo.toml @@ -32,7 +32,7 @@ publish = false # Simulator-only seed helper (`Streams::seed_single_partition`): applies # CreateStream + CreateTopic straight onto the writer STM, bypassing metadata # consensus, so the deterministic dispatch shell can resolve a namespace -# without driving the reconciler. A `-p iggy-server-ng` build excludes it; +# without driving the reconciler. A `-p iggy-server` build excludes it; # `cargo build --workspace` unifies features so the shared `metadata` unit # compiles it in when the simulator requests it. No production caller. simulator = [] diff --git a/core/metadata/src/impls/metadata.rs b/core/metadata/src/impls/metadata.rs index 9643493fa8..c4bb620e08 100644 --- a/core/metadata/src/impls/metadata.rs +++ b/core/metadata/src/impls/metadata.rs @@ -34,6 +34,7 @@ use consensus::{ is_caught_up_primary, panic_if_hash_chain_would_break_in_same_view, peek_committable_head, pipeline_prepare_common, register_preflight, replicate_preflight, replicate_to_next_in_chain, request_preflight, send_eviction_to_client, send_prepare_ok as send_prepare_ok_common, + verify_prepare_integrity, }; use iggy_binary_protocol::WireIdentifier; use iggy_binary_protocol::primitives::partition_assignment::CreatedPartitionAssignment; @@ -43,7 +44,7 @@ use iggy_binary_protocol::requests::topics::CreateTopicRequest as WireCreateTopi use iggy_binary_protocol::requests::topics::CreateTopicWithAssignmentsRequest as PersistedCreateTopicRequest; use iggy_binary_protocol::{ Command2, ConsensusHeader, EvictionReason, GenericHeader, Operation, PrepareHeader, - PrepareOkHeader, ReplyHeader, RequestHeader, WireDecode, WireEncode, WireName, + PrepareOkHeader, ReplyHeader, RoutedRequestHeader, WireDecode, WireEncode, WireName, }; use iggy_common::IggyError; use iggy_common::UserId; @@ -414,17 +415,29 @@ impl SnapshotCoordinator { Ok(checksum) } - /// Drain the snapshotted prefix `0..=last_op` to reclaim WAL space. Runs only - /// after the pairing is durable (see [`Self::persist_snapshot`]). + /// Drain the snapshotted prefix below `last_op` to reclaim WAL space. Runs + /// only after the pairing is durable (see [`Self::persist_snapshot`]). + /// + /// `last_op` itself is retained, one entry the snapshot has already + /// superseded. It is this replica's commit point, and a `DoViewChange` + /// carries a header for every op from there up. Draining it inclusively + /// leaves that entry blank, and blank at the commit point is the one slot + /// the merge can neither adopt nor discard: a quorum of senders that all + /// checkpointed at the same op deadlocks the view change + /// (`dvc_merge::merge_dvc_quorum`). Reclaiming one more entry is not worth + /// a group that cannot elect. #[allow(clippy::future_not_send)] async fn drain( &self, journal: &J, last_op: u64, ) -> Result<(), SnapshotError> { + let Some(drain_to) = last_op.checked_sub(1) else { + return Ok(()); + }; journal .handle() - .drain(0..=last_op) + .drain(0..=drain_to) .await .map_err(SnapshotError::Io)?; Ok(()) @@ -628,7 +641,7 @@ pub fn apply_committed_prepare( /// Late-bound callback invoked after every committed op on shard 0's metadata /// commit path (via `gated_apply`, including a gated no-op). /// -/// Wired by server-ng bootstrap once the metadata bundle has broadcast; +/// Wired by the server bootstrap once the metadata bundle has broadcast; /// receives the committed [`Operation`] so the recipient can filter (the /// partition reconciliation loop only cares about partition-shaped /// events). Wrapped in [`RefCell`] for late binding; the per-shard @@ -647,7 +660,7 @@ pub struct IggyMetadata { /// the WAL at all. They receive a `MetadataHandoff::Waiter` factory /// bundle from shard 0 over the bootstrap broadcast channel and /// reconstruct `mux_stm` from the in-memory snapshot it carries (see - /// `server-ng/src/bootstrap.rs` `await_metadata_bundle` / + /// `server/src/bootstrap.rs` `await_metadata_bundle` / /// `broadcast_metadata_bundle`). pub journal: Option, /// `Some` on shard 0, `None` on other shards. @@ -715,7 +728,7 @@ pub struct IggyMetadata { /// after `gated_apply` returns (including a gated `Unauthorized` no-op that /// never reaches [`crate::stm::StateMachine::update`]) in both /// [`Plane::on_ack`] and [`Self::commit_journal`]. `None` until - /// [`Self::set_commit_notifier`] runs (server-ng bootstrap on shard + /// [`Self::set_commit_notifier`] runs (the server bootstrap on shard /// 0 sets it; peer shards and tests leave it `None`). commit_notifier: RefCell>, /// Resolved byte value for `MaxTopicSize::ServerDefault` (`0` on the @@ -933,7 +946,10 @@ where Error = iggy_common::IggyError, >, { - async fn on_request(&self, message: as Consensus>::Message) { + async fn on_request( + &self, + message: as Consensus>::Message, + ) { let Some(consensus) = require_shard_zero(self.consensus.as_ref(), "on_request", "consensus") else { @@ -1038,6 +1054,23 @@ where let header = *message.header(); + // Before anything trusts `checksum` as an identity token, and before the WAL + // takes the bytes. Every live prepare travels this path: unverified, a frame + // corrupted between primary and backup is journaled as-is and re-served to + // peers, which the interior-corruption boot refusal turns into an unbootable + // node on the next restart. + if let Err(reason) = verify_prepare_integrity(&header, message.as_slice()) { + warn!( + target: "iggy.metadata.diag", + plane = "metadata", + replica_id = consensus.replica(), + view = consensus.view(), + op = header.op, + "discarding prepare: {reason}" + ); + return; + } + let current_op = match replicate_preflight(consensus, &header) { Ok(current_op) => current_op, Err(reason) => { @@ -1863,7 +1896,7 @@ where } let request = build_register_request_message(consensus, client_id, user_id); - // Wire path runs `RequestHeader::validate` at network boundary; + // Wire path runs `RoutedRequestHeader::validate` at network boundary; // in-process skips it. debug_assert pins drift. debug_assert!( { @@ -1965,7 +1998,7 @@ where /// commit. fn answer_preflight( consensus: &VsrConsensus, - request_header: &RequestHeader, + request_header: &RoutedRequestHeader, outcome: PreflightOutcome, ) -> Option, MetadataSubmitError>> { let client_id = request_header.client; @@ -2278,10 +2311,10 @@ where // Build the prepare directly so the `client = 0` header skips the // client-header validation in `prepare_request` / `Project::project` // (the in-process path `build_prepare_message` documents). - let header = RequestHeader { + let header = RoutedRequestHeader { client: 0, - namespace: server_common::sharding::METADATA_CONSENSUS_NAMESPACE, - ..RequestHeader::default() + group: server_common::sharding::METADATA_GROUP, + ..RoutedRequestHeader::default() }; let prepare = build_prepare_message( consensus, @@ -2324,7 +2357,7 @@ where #[allow(clippy::future_not_send)] pub async fn submit_request_in_process( &self, - message: Message, + message: Message, ) -> Result, MetadataSubmitError> { let request_header = *message.header(); let client_id = request_header.client; @@ -3091,7 +3124,7 @@ where return; }; // Serialize whole checkpoints against each other. In-process metadata submits - // each run on their own spawned task (`bus.spawn` in server-ng's metadata submit + // each run on their own spawned task (`bus.spawn` in the server's metadata submit // handler), so at the checkpoint margin two can enter here concurrently; without // this lock they would run concurrent `persist_snapshot`s over the single // `snapshot.bin` and concurrently `drain` the WAL, which rewrites through a @@ -3200,7 +3233,7 @@ where #[allow(clippy::too_many_lines)] fn prepare_request( &self, - mut message: Message, + mut message: Message, ) -> Result, iggy_common::IggyError> { let consensus = self.consensus.as_ref().unwrap(); let operation = message.header().operation; @@ -3231,8 +3264,8 @@ where if let Some(acting_user_id) = resolve_acting_user_id(operation, client_id, &self.client_table)? { - let request_header = bytemuck::checked::from_bytes_mut::( - &mut message.as_mut_slice()[..size_of::()], + let request_header = bytemuck::checked::from_bytes_mut::( + &mut message.as_mut_slice()[..size_of::()], ); request_header.user_id = acting_user_id; } @@ -3245,7 +3278,7 @@ where // authz gate. The default arm projects the mutated buffer directly and // is order-independent. let header = *message.header(); - let body = &message.as_slice()[size_of::()..header.size as usize]; + let body = &message.as_slice()[size_of::()..header.size as usize]; match header.operation { Operation::CreateTopic => { @@ -3478,36 +3511,36 @@ where } } -/// In-process Register `Message`. Mirrors +/// In-process Register `Message`. Mirrors /// `SimClient::register`: `session=0`, `request=0` per -/// [`RequestHeader::validate`]; empty body. +/// [`RoutedRequestHeader::validate`]; empty body. /// /// `cluster` + `view` from `consensus` for self-consistency before /// `Project::project` overwrites. `release = 0` matches wire today; both /// paths should switch to `consensus.release()` once /// `ClientReleaseTooLow/TooHigh` lands. /// -/// Buffer is `size_of::()`; `prepare_request` transmutes into +/// Buffer is `size_of::()`; `prepare_request` transmutes into /// `PrepareHeader` (also 256 bytes), no realloc. fn build_register_request_message( consensus: &VsrConsensus, client_id: u128, user_id: u32, -) -> Message +) -> Message where B: MessageBus, P: Pipeline, { - let header_size = size_of::(); - let mut msg = Message::::new(header_size); - let header = bytemuck::checked::try_from_bytes_mut::( + let header_size = size_of::(); + let mut msg = Message::::new(header_size); + let header = bytemuck::checked::try_from_bytes_mut::( &mut msg.as_mut_slice()[..header_size], ) - .expect("zeroed bytes are a valid RequestHeader"); - *header = RequestHeader { + .expect("zeroed bytes are a valid RoutedRequestHeader"); + *header = RoutedRequestHeader { command: Command2::Request, operation: Operation::Register, - size: u32::try_from(header_size).expect("RequestHeader size fits u32"), + size: u32::try_from(header_size).expect("RoutedRequestHeader size fits u32"), cluster: consensus.cluster(), view: consensus.view(), release: 0, @@ -3520,8 +3553,8 @@ where // prepare is re-routed on each peer by namespace; a `0` here would // hash to a non-zero shard with no metadata consensus and be // silently dropped (see `shard::router::route_typed`). - namespace: server_common::sharding::METADATA_CONSENSUS_NAMESPACE, - ..RequestHeader::default() + group: server_common::sharding::METADATA_GROUP, + ..RoutedRequestHeader::default() }; msg } @@ -3531,21 +3564,21 @@ fn build_logout_request_message( client_id: u128, session: u64, request: u64, -) -> Message +) -> Message where B: MessageBus, P: Pipeline, { - let header_size = size_of::(); - let mut msg = Message::::new(header_size); - let header = bytemuck::checked::try_from_bytes_mut::( + let header_size = size_of::(); + let mut msg = Message::::new(header_size); + let header = bytemuck::checked::try_from_bytes_mut::( &mut msg.as_mut_slice()[..header_size], ) - .expect("zeroed bytes are a valid RequestHeader"); - *header = RequestHeader { + .expect("zeroed bytes are a valid RoutedRequestHeader"); + *header = RoutedRequestHeader { command: Command2::Request, operation: Operation::Logout, - size: u32::try_from(header_size).expect("RequestHeader size fits u32"), + size: u32::try_from(header_size).expect("RoutedRequestHeader size fits u32"), cluster: consensus.cluster(), view: consensus.view(), release: 0, @@ -3553,8 +3586,8 @@ where session, request, // Metadata consensus group (see `build_register_request_message`). - namespace: server_common::sharding::METADATA_CONSENSUS_NAMESPACE, - ..RequestHeader::default() + group: server_common::sharding::METADATA_GROUP, + ..RoutedRequestHeader::default() }; msg } @@ -3564,21 +3597,21 @@ fn build_complete_revocation_request_message( client_id: u128, request: u64, body: &[u8], -) -> Message +) -> Message where B: MessageBus, P: Pipeline, { - let header_size = size_of::(); + let header_size = size_of::(); let total = header_size + body.len(); - let mut msg = Message::::new(total); + let mut msg = Message::::new(total); { let slice = msg.as_mut_slice(); slice[header_size..total].copy_from_slice(body); let header = - bytemuck::checked::try_from_bytes_mut::(&mut slice[..header_size]) - .expect("zeroed bytes are a valid RequestHeader"); - *header = RequestHeader { + bytemuck::checked::try_from_bytes_mut::(&mut slice[..header_size]) + .expect("zeroed bytes are a valid RoutedRequestHeader"); + *header = RoutedRequestHeader { command: Command2::Request, operation: Operation::CompleteConsumerGroupRevocation, size: u32::try_from(total).expect("request size fits u32"), @@ -3590,8 +3623,8 @@ where // there is no real session (the commit path skips reply-caching). session: 1, request, - namespace: server_common::sharding::METADATA_CONSENSUS_NAMESPACE, - ..RequestHeader::default() + group: server_common::sharding::METADATA_GROUP, + ..RoutedRequestHeader::default() }; } msg @@ -3614,14 +3647,14 @@ where /// a few fixed-width fields, so this cannot happen in practice. #[must_use] pub fn build_truncate_partition_client_message( - template: &RequestHeader, + template: &RoutedRequestHeader, client_id: u128, session: u64, stream_id: u32, topic_id: u32, partition_id: u32, up_to_offset: u64, -) -> Message { +) -> Message { build_truncate_partition_client_message_with_identifiers( template, client_id, @@ -3645,14 +3678,14 @@ pub fn build_truncate_partition_client_message( /// a few small fields, so this cannot happen in practice. #[must_use] pub fn build_truncate_partition_client_message_with_identifiers( - template: &RequestHeader, + template: &RoutedRequestHeader, client_id: u128, session: u64, stream_id: WireIdentifier, topic_id: WireIdentifier, partition_id: u32, up_to_offset: u64, -) -> Message { +) -> Message { let body = TruncatePartitionRequest { stream_id, topic_id, @@ -3660,16 +3693,16 @@ pub fn build_truncate_partition_client_message_with_identifiers( up_to_offset, } .to_bytes(); - let header_size = size_of::(); + let header_size = size_of::(); let total = header_size + body.len(); - let mut msg = Message::::new(total); + let mut msg = Message::::new(total); { let slice = msg.as_mut_slice(); slice[header_size..total].copy_from_slice(&body); let header = - bytemuck::checked::try_from_bytes_mut::(&mut slice[..header_size]) - .expect("zeroed bytes are a valid RequestHeader"); - *header = RequestHeader { + bytemuck::checked::try_from_bytes_mut::(&mut slice[..header_size]) + .expect("zeroed bytes are a valid RoutedRequestHeader"); + *header = RoutedRequestHeader { command: Command2::Request, operation: Operation::TruncatePartition, size: u32::try_from(total).expect("request size fits u32"), @@ -3679,8 +3712,8 @@ pub fn build_truncate_partition_client_message_with_identifiers( client: client_id, session, request: template.request, - namespace: server_common::sharding::METADATA_CONSENSUS_NAMESPACE, - ..RequestHeader::default() + group: server_common::sharding::METADATA_GROUP, + ..RoutedRequestHeader::default() }; } msg @@ -3688,7 +3721,7 @@ pub fn build_truncate_partition_client_message_with_identifiers( fn build_prepare_message( consensus: &VsrConsensus, - request: &RequestHeader, + request: &RoutedRequestHeader, operation: Operation, body: &[u8], ) -> Message @@ -3735,7 +3768,7 @@ where operation, // The group's namespace, never the request's: clients send 0, and a // journaled 0 mis-routes the entry when repair replays it verbatim. - namespace: consensus.namespace(), + group: consensus.group(), // Carry the acting user id so the in-apply RBAC gate sees the same // identity on every replica. The default projection copies it (see // `Project::project`); this helper builds prepares for the ops it @@ -3752,7 +3785,12 @@ where ..Default::default() }; - prepare + // Last, because the identity checksum covers every other field. Same contract as + // the wire path in `Project::project`; skipping it would leave the rewritten + // prepares (CreateTopic/CreatePartitions assignments, the UpdateTopic default-size + // rewrite, the PAT-cleaner delete) as the only ops the merge cannot tell apart + // from a competing prepare. + consensus::seal_prepare_checksum(prepare) } /// Eviction reason for a request `prepare_request` rejected as structurally @@ -3766,7 +3804,7 @@ const fn eviction_reason_for_invalid(operation: Operation) -> EvictionReason { } /// Resolve the acting user id to stamp into a client op's replicated -/// `RequestHeader`, so the in-apply RBAC gate (`crate::stm::authz`) reads the +/// `RoutedRequestHeader`, so the in-apply RBAC gate (`crate::stm::authz`) reads the /// same identity on every replica (WAL replay has no session table). /// /// - `Ok(Some(id))`: overwrite the header's `user_id` with the committed @@ -3843,7 +3881,7 @@ fn log_commit_reply_outcome(outcome: CommitReply, client_id: u128, op: u64) { /// A cached REJECTION replays untouched: it carries no secret, so serving it /// is both safe and useful. fn unreplayable_secret_refusal( - request_header: &RequestHeader, + request_header: &RoutedRequestHeader, cached: &Frozen<{ server_common::MESSAGE_ALIGN }>, commit: u64, client_id: u128, @@ -4132,7 +4170,7 @@ mod tests { 1, 0, 1, - server_common::sharding::METADATA_CONSENSUS_NAMESPACE, + server_common::sharding::METADATA_GROUP, NoopBus, LocalPipeline::new(), ); @@ -4239,7 +4277,7 @@ mod tests { 1, 0, 1, - server_common::sharding::METADATA_CONSENSUS_NAMESPACE, + server_common::sharding::METADATA_GROUP, NoopBus, LocalPipeline::new(), ); @@ -4247,7 +4285,7 @@ mod tests { IggyMetadata::new(Some(consensus), None, None, None, TestMux::default(), None) } - fn create_topic_request(client: u128, wire_user_id: u32) -> Message { + fn create_topic_request(client: u128, wire_user_id: u32) -> Message { let body = CreateTopicRequest { stream_id: WireIdentifier::numeric(1), partitions_count: 1, @@ -4258,15 +4296,15 @@ mod tests { name: WireName::new("t").unwrap(), } .to_bytes(); - let header_size = size_of::(); + let header_size = size_of::(); let total = header_size + body.len(); - let mut message = Message::::new(total); + let mut message = Message::::new(total); { let slice = message.as_mut_slice(); slice[header_size..total].copy_from_slice(&body); let header = - bytemuck::checked::from_bytes_mut::(&mut slice[..header_size]); - *header = RequestHeader { + bytemuck::checked::from_bytes_mut::(&mut slice[..header_size]); + *header = RoutedRequestHeader { command: Command2::Request, operation: Operation::CreateTopic, size: u32::try_from(total).unwrap(), @@ -4274,7 +4312,7 @@ mod tests { session: 1, request: 1, user_id: wire_user_id, - namespace: server_common::sharding::METADATA_CONSENSUS_NAMESPACE, + group: server_common::sharding::METADATA_GROUP, ..Default::default() }; } @@ -4402,7 +4440,7 @@ mod tests { 1, 0, 1, - server_common::sharding::METADATA_CONSENSUS_NAMESPACE, + server_common::sharding::METADATA_GROUP, NoopBus, LocalPipeline::new(), ); @@ -4497,39 +4535,43 @@ mod tests { reply } - fn pat_create_request(client: u128, request: u64) -> Message { - let header_size = size_of::(); - let mut message = Message::::new(header_size); - let header = bytemuck::checked::from_bytes_mut::( + fn pat_create_request(client: u128, request: u64) -> Message { + let header_size = size_of::(); + let mut message = Message::::new(header_size); + let header = bytemuck::checked::from_bytes_mut::( &mut message.as_mut_slice()[..header_size], ); - *header = RequestHeader { + *header = RoutedRequestHeader { command: Command2::Request, operation: Operation::CreatePersonalAccessToken, size: u32::try_from(header_size).unwrap(), client, session: 1, request, - namespace: server_common::sharding::METADATA_CONSENSUS_NAMESPACE, + group: server_common::sharding::METADATA_GROUP, ..Default::default() }; message } - fn create_stream_request(client: u128, request: u64, name: &str) -> Message { + fn create_stream_request( + client: u128, + request: u64, + name: &str, + ) -> Message { let body = iggy_binary_protocol::requests::streams::CreateStreamRequest { name: WireName::new(name).unwrap(), } .to_bytes(); - let header_size = size_of::(); + let header_size = size_of::(); let total = header_size + body.len(); - let mut message = Message::::new(total); + let mut message = Message::::new(total); { let slice = message.as_mut_slice(); slice[header_size..total].copy_from_slice(&body); let header = - bytemuck::checked::from_bytes_mut::(&mut slice[..header_size]); - *header = RequestHeader { + bytemuck::checked::from_bytes_mut::(&mut slice[..header_size]); + *header = RoutedRequestHeader { command: Command2::Request, operation: Operation::CreateStream, size: u32::try_from(total).unwrap(), @@ -4537,7 +4579,7 @@ mod tests { session: 1, request, user_id: 0, - namespace: server_common::sharding::METADATA_CONSENSUS_NAMESPACE, + group: server_common::sharding::METADATA_GROUP, ..Default::default() }; } @@ -4579,7 +4621,7 @@ mod tests { 1, 0, 1, - server_common::sharding::METADATA_CONSENSUS_NAMESPACE, + server_common::sharding::METADATA_GROUP, StallBus::default(), LocalPipeline::new(), ); @@ -4688,6 +4730,87 @@ mod tests { ); } + /// A checkpoint reclaims the WAL prefix the snapshot supersedes, but must + /// stop one op short of the checkpoint op itself. + /// + /// That op is the replica's commit point, and its `DoViewChange` suffix is + /// floored there. The merge scans the commit point and may not discard it, + /// so a sender with no header to put there is deferring to a peer; when + /// every sender has checkpointed at the same op the view change deadlocks + /// (`dvc_merge::merge_dvc_quorum`). Checkpoints fire on local journal + /// occupancy, which is symmetric across replicas seeing the same ops, so + /// "every sender" is the ordinary case, not a coincidence. + #[compio::test] + async fn checkpoint_drain_retains_the_commit_point_header() { + const CLIENT: u128 = 1; + const SESSION: u64 = 1; + const ACTING_USER: u32 = 7; + const OPS: u64 = 5; + const CHECKPOINT_OP: u64 = 3; + + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(dir.path().join(crate::impls::METADATA_DIR)).unwrap(); + let journal = + journal::prepare_journal::PrepareJournal::open(&dir.path().join("journal.wal"), 0) + .await + .unwrap(); + let consensus = VsrConsensus::new( + 1, + 0, + 1, + server_common::sharding::METADATA_GROUP, + NoopBus, + LocalPipeline::new(), + ); + consensus.init(); + let md: IggyMetadata<_, journal::prepare_journal::PrepareJournal, (), TestMux> = + IggyMetadata::new( + Some(consensus), + Some(journal), + None, + None, + TestMux::default(), + Some(dir.path().to_path_buf()), + ); + let consensus = md.consensus.as_ref().unwrap(); + md.client_table.borrow_mut().commit_register( + CLIENT, + ACTING_USER, + register_reply(CLIENT, SESSION), + ); + + for op in 1..=OPS { + let prepare = md + .prepare_request(create_stream_request(CLIENT, op, &format!("s{op}"))) + .expect("CreateStream is client-allowed"); + consensus.pipeline_message(PlaneKind::Metadata, &prepare); + md.on_replicate(prepare).await; + } + + let journal = md.journal.as_ref().unwrap(); + md.coordinator + .as_ref() + .expect("data_dir present arms the coordinator") + .drain(journal, CHECKPOINT_OP) + .await + .expect("drain the snapshotted prefix"); + + let header_at = |op: u64| journal.header(usize::try_from(op).expect("test ops fit usize")); + for op in 1..CHECKPOINT_OP { + assert!( + header_at(op).is_none(), + "op {op} is below the checkpoint and must be reclaimed" + ); + } + assert!( + header_at(CHECKPOINT_OP).is_some(), + "the checkpoint op is the commit point and must stay describable in a DVC" + ); + for op in CHECKPOINT_OP + 1..=OPS { + assert!(header_at(op).is_some(), "op {op} was never snapshotted"); + } + } + /// Reproduces the single-node "metadata prepare queue is full" wedge /// /// `checkpoint_if_needed` runs inside `on_replicate`, once per submit. @@ -4729,7 +4852,7 @@ mod tests { 1, 0, 1, - server_common::sharding::METADATA_CONSENSUS_NAMESPACE, + server_common::sharding::METADATA_GROUP, NoopBus, LocalPipeline::new(), ); @@ -4876,7 +4999,7 @@ mod tests { 1, 0, 1, - server_common::sharding::METADATA_CONSENSUS_NAMESPACE, + server_common::sharding::METADATA_GROUP, NoopBus, LocalPipeline::new(), ); @@ -5001,7 +5124,7 @@ mod tests { 1, 0, 1, - server_common::sharding::METADATA_CONSENSUS_NAMESPACE, + server_common::sharding::METADATA_GROUP, NoopBus, LocalPipeline::new(), ); @@ -5137,7 +5260,7 @@ mod tests { 1, 0, 1, - server_common::sharding::METADATA_CONSENSUS_NAMESPACE, + server_common::sharding::METADATA_GROUP, NoopBus, LocalPipeline::new(), ); diff --git a/core/metadata/src/impls/recovery.rs b/core/metadata/src/impls/recovery.rs index 286d286e20..3b2e009afd 100644 --- a/core/metadata/src/impls/recovery.rs +++ b/core/metadata/src/impls/recovery.rs @@ -23,8 +23,9 @@ use consensus::{ ClientTable, ClientTableDecodeError, VsrState, VsrStateError, build_reply_message, build_reply_message_with, }; -use iggy_binary_protocol::consensus::{Operation, PrepareHeader}; +use iggy_binary_protocol::consensus::{CHECKSUM_UNSEALED, Operation, PrepareHeader}; use iggy_common::IggyError; +use journal::Journal as _; use journal::prepare_journal::{JournalError, PrepareJournal}; use journal::superblock::{ PingPongSuperblock, SLOT_FILE_NAMES, SuperblockContents, SuperblockStore, @@ -307,6 +308,13 @@ pub struct RecoveredMetadata { /// they stay journal-only until the recovered primary re-replicates them /// (or a backup sees the commit point advance past them). pub last_journaled_op: Option, + /// First op replay could not connect to its predecessor, `None` when the + /// replayed range is one unbroken chain. + /// + /// `Some(op)` means entries at and above `op` were truncated and must come back + /// from the cluster. `last_journaled_op` stops below it, which keeps the restored + /// head, the re-pipeline range, and the recovery barrier honest. + pub chain_break_op: Option, } /// Recover metadata state from disk. @@ -545,10 +553,35 @@ where let mut last_applied_op: Option = None; let mut last_journaled_op: Option = None; + let mut chain_break_op: Option = None; + let mut previous: Option = None; for header in &headers_to_replay { - // TODO: Check hash chain integrity against `previous_header`. On a - // same-view break, stop replay here and mark the remaining entries for - // repair via VSR instead of panicking. + // Stop at the first op that does not connect to the one before it. Applying + // across a hole replays effects onto a state machine that never saw the + // missing op, and nothing downstream re-checks it. + // + // The WAL scan does not cover this: it only fires on CONSECUTIVE ops with + // both ends sealed, so a gap reaches here. The first replayed op is exempt, + // since a snapshot records no checksum for its parent to chain to. + if let Some(previous) = previous { + let gap = previous.op + 1 != header.op; + let broken_chain = previous.checksum != CHECKSUM_UNSEALED + && header.checksum != CHECKSUM_UNSEALED + && header.parent != previous.checksum; + if gap || broken_chain { + tracing::error!( + op = header.op, + previous_op = previous.op, + gap, + broken_chain, + "metadata WAL does not connect at this op; stopping replay and dropping the \ + suffix for VSR repair" + ); + chain_break_op = Some(header.op); + break; + } + } + previous = Some(*header); last_journaled_op = Some(header.op); if header.op > commit_watermark { @@ -626,6 +659,22 @@ where last_applied_op = Some(header.op); } + // `truncate_from`, never `drain`: the removed ops must stay refillable, so the + // snapshot watermark stays put. Leaving them resident would make `append` refuse + // the slot, failing repair on exactly the ops it exists to fix. + if let Some(break_op) = chain_break_op { + let removed = journal + .truncate_from(break_op) + .await + .map_err(RecoveryError::Io)?; + tracing::warn!( + break_op, + removed, + last_journaled_op, + "dropped the disconnected metadata WAL suffix; the cluster re-supplies these ops" + ); + } + Ok(RecoveredMetadata { journal, snapshot, @@ -636,6 +685,7 @@ where client_table, last_applied_op, last_journaled_op, + chain_break_op, }) } @@ -976,6 +1026,130 @@ mod tests { assert_eq!(recovered.journal.last_op(), Some(3)); } + /// A prepare sealed the way a live primary seals one: `parent` chains to the + /// previous op's identity and `checksum` is that identity. + fn make_chained_prepare(op: u64, commit: u64, parent: u128) -> Message { + let mut message = make_prepare_with_commit(op, commit, 32); + let header = bytemuck::checked::from_bytes_mut::( + &mut message.as_mut_slice()[..HEADER_SIZE], + ); + header.parent = parent; + let checksum = header.identity_checksum(); + header.checksum = checksum; + message + } + + #[compio::test] + async fn recover_stops_at_a_gap_and_drops_the_disconnected_suffix() { + // Ops 1-3 then 5: op 4 never landed. Replaying 5 over a state machine that + // never saw 4 diverges silently, and the WAL scan waves this through -- + // its chain check only fires on CONSECUTIVE ops, since a gap is also what + // ordinary compaction leaves behind. + let dir = tempdir().unwrap(); + let metadata_dir = dir.path().join("metadata"); + std::fs::create_dir_all(&metadata_dir).unwrap(); + + { + let journal = PrepareJournal::open(&metadata_dir.join("journal.wal"), 0) + .await + .unwrap(); + for op in 1..=3u64 { + journal + .append(make_prepare_with_commit(op, op, 32)) + .await + .unwrap(); + } + journal + .append(make_prepare_with_commit(5, 5, 32)) + .await + .unwrap(); + journal.storage_ref().fsync().await.unwrap(); + } + + let recovered = recover::( + dir.path(), + CLUSTERED, + journal::prepare_journal::DEFAULT_SLOT_COUNT, + CLIENTS_TABLE_MAX, + |_| {}, + ) + .await + .unwrap(); + + assert_eq!(recovered.chain_break_op, Some(5)); + assert_eq!( + recovered.last_applied_op, + Some(3), + "op 5 must not apply across the hole at op 4" + ); + assert_eq!( + recovered.last_journaled_op, + Some(3), + "the restored head stops below the break, so nothing re-pipelines it" + ); + assert_eq!( + recovered.journal.last_op(), + Some(3), + "the disconnected entry is dropped so repair can journal the cluster's op 5" + ); + assert_eq!( + recovered.journal.snapshot_op(), + 0, + "truncating a suffix must leave the watermark, or the ops stop being refillable" + ); + } + + #[compio::test] + async fn recover_stops_at_a_broken_chain_between_consecutive_ops() { + // Consecutive and sealed on both ends, but op 3 names a parent that is not + // op 2: a fork left by a crash mid view change. Ops are appended out of + // ascending file order so the scan's own chain check does not fire first. + let dir = tempdir().unwrap(); + let metadata_dir = dir.path().join("metadata"); + std::fs::create_dir_all(&metadata_dir).unwrap(); + + { + let journal = PrepareJournal::open(&metadata_dir.join("journal.wal"), 0) + .await + .unwrap(); + let first = make_chained_prepare(1, 1, 0); + let first_checksum = first.header().checksum; + journal.append(first).await.unwrap(); + let second = make_chained_prepare(2, 2, first_checksum); + journal.append(second).await.unwrap(); + // Parent of a prepare that is not op 2. + journal + .append(make_chained_prepare(3, 3, 0xdead_beef)) + .await + .unwrap(); + journal.storage_ref().fsync().await.unwrap(); + } + + let recovered = recover::( + dir.path(), + CLUSTERED, + journal::prepare_journal::DEFAULT_SLOT_COUNT, + CLIENTS_TABLE_MAX, + |_| {}, + ) + .await; + + // The WAL scan reaches this first and refuses boot: consecutive ops, both + // sealed, chain broken, with no entry after it is only a tail. Either + // outcome is a refusal to apply the fork; what must never happen is a + // clean recovery that replayed op 3. + match recovered { + Err(RecoveryError::Journal(_) | RecoveryError::Io(_)) => {} + Ok(recovered) => { + assert!( + recovered.last_applied_op < Some(3), + "op 3 forks the chain and must not be applied" + ); + } + Err(other) => panic!("unexpected recovery error: {other:?}"), + } + } + #[compio::test] async fn recover_applies_only_the_committed_prefix() { let dir = tempdir().unwrap(); diff --git a/core/metadata/src/stm/result.rs b/core/metadata/src/stm/result.rs index 67a0a1383d..6a610bf3a5 100644 --- a/core/metadata/src/stm/result.rs +++ b/core/metadata/src/stm/result.rs @@ -183,6 +183,7 @@ result_enum!(CreatePartitionsResult { result_enum!(DeletePartitionsResult { StreamNotFound = 1009, TopicNotFound = 2010, + InvalidPartitionsCount = 2019, }); // `TruncatePartition` is the committed form of a client `DeleteSegments`; an // unresolvable target commits as a rejection so the request sequence stays diff --git a/core/metadata/src/stm/snapshot.rs b/core/metadata/src/stm/snapshot.rs index ca28a30a4a..e9fadca173 100644 --- a/core/metadata/src/stm/snapshot.rs +++ b/core/metadata/src/stm/snapshot.rs @@ -53,6 +53,10 @@ pub enum SnapshotError { commit_op: u64, table_frontier: u64, }, + /// The snapshot was written under a different format version. Refuse it + /// rather than reinterpret embedded raw bytes (the client table's cached + /// replies are wire `ReplyHeader` frames) under the wrong layout. + UnsupportedVersion { found: u32, supported: u32 }, } /// Stage at which snapshot persistence failed. @@ -104,6 +108,13 @@ impl fmt::Display for SnapshotError { commit_op {commit_op}, table frontier {table_frontier}" ) } + Self::UnsupportedVersion { found, supported } => { + write!( + f, + "unsupported metadata snapshot version {found}; this build reads only \ + version {supported}" + ) + } } } } @@ -116,7 +127,8 @@ impl std::error::Error for SnapshotError { Self::Io(e) | Self::Persist { source: e, .. } => Some(e), Self::ChecksumMismatch { .. } | Self::Truncated { .. } - | Self::IncoherentManifest { .. } => None, + | Self::IncoherentManifest { .. } + | Self::UnsupportedVersion { .. } => None, } } } @@ -138,10 +150,17 @@ impl From for SnapshotError { /// replicas with identical state must serialize identically. Regression guards: /// `stream::tests::populated_streams_snapshot_reencode_is_byte_stable` and /// `impls::metadata::tests::populated_snapshot_reencode_and_checksum_are_stable`. +/// Current [`MetadataSnapshot::version`]. Bump whenever the serialized form +/// changes meaning without changing shape -- in particular the client table's +/// cached replies, which are embedded as raw `ReplyHeader` wire bytes msgpack +/// cannot introspect. Version 2: `status` sits at reply-header offset 216 +/// (version 1 carried a `namespace` word before it). +pub const METADATA_SNAPSHOT_VERSION: u32 = 2; + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct MetadataSnapshot { - /// Snapshot format version for forward/backward compatibility. - /// TODO(krishvishal): Properly handle versioning for snapshot. This is a placeholder for now. + /// Snapshot format version; [`MetadataSnapshot::decode`] refuses any other + /// value (see [`METADATA_SNAPSHOT_VERSION`]). pub version: u32, /// Timestamp when the snapshot was created (microseconds since epoch). pub created_at: u64, @@ -171,7 +190,7 @@ impl MetadataSnapshot { #[must_use] pub const fn new(sequence_number: u64) -> Self { Self { - version: 1, + version: METADATA_SNAPSHOT_VERSION, // Deterministic placeholder. The real creation time is stamped by // `Snapshot::create` from the consensus-injected clock (see // `VsrConsensus::clock_realtime_micros`) so a replayed simulator @@ -196,9 +215,18 @@ impl MetadataSnapshot { /// Decode a snapshot from msgpack bytes. /// /// # Errors - /// Returns `SnapshotError::Deserialize` if msgpack deserialization fails. + /// Returns `SnapshotError::Deserialize` if msgpack deserialization fails, + /// or `SnapshotError::UnsupportedVersion` if the snapshot was written + /// under a different format version. pub fn decode(bytes: &[u8]) -> Result { - rmp_serde::from_slice(bytes).map_err(SnapshotError::Deserialize) + let snapshot: Self = rmp_serde::from_slice(bytes).map_err(SnapshotError::Deserialize)?; + if snapshot.version != METADATA_SNAPSHOT_VERSION { + return Err(SnapshotError::UnsupportedVersion { + found: snapshot.version, + supported: METADATA_SNAPSHOT_VERSION, + }); + } + Ok(snapshot) } } @@ -440,6 +468,25 @@ mod tests { assert!(decoded.client_table.is_none()); } + // The client table's cached replies are embedded as raw `ReplyHeader` + // wire bytes msgpack cannot introspect, so a snapshot from a different + // format version must be refused, never reinterpreted under the current + // header layout. + #[test] + fn decode_refuses_a_snapshot_from_another_format_version() { + let mut snapshot = MetadataSnapshot::new(42); + snapshot.version = METADATA_SNAPSHOT_VERSION - 1; + + let encoded = snapshot.encode().unwrap(); + assert!(matches!( + MetadataSnapshot::decode(&encoded), + Err(SnapshotError::UnsupportedVersion { + found, + supported: METADATA_SNAPSHOT_VERSION, + }) if found == METADATA_SNAPSHOT_VERSION - 1 + )); + } + #[test] fn roundtrip_with_data() { let ts = IggyTimestamp::from(1_694_968_446_131_680_u64); diff --git a/core/metadata/src/stm/stream.rs b/core/metadata/src/stm/stream.rs index 0a97ddc8e2..e4dfc1b12c 100644 --- a/core/metadata/src/stm/stream.rs +++ b/core/metadata/src/stm/stream.rs @@ -183,7 +183,7 @@ pub struct Topic { /// key (keyed by group id) can't be inherited by a recreated group. /// /// Ceiling: the partition-plane offset key is `u32`, so a group id must stay - /// within `u32::MAX` (the wire rewrite in `server-ng` clamps past-ceiling + /// within `u32::MAX` (the wire rewrite in `the server` clamps past-ceiling /// ids to `u32::MAX` rather than panic). ~4 billion group creates on a /// single topic is unreachable in practice, but the cap is real -- past it /// clamped wire ids all collide on `u32::MAX`, including with a live @@ -362,7 +362,22 @@ impl Stream { pub struct StatsRegistry { streams: std::sync::Mutex>>, topics: std::sync::Mutex>>, - partitions: std::sync::Mutex>>, + partitions: std::sync::Mutex>, +} + +/// Shared partition counters plus the purge generation they were last reset for. +#[derive(Debug)] +struct PartitionEntry { + stats: Arc, + /// Highest [`Partition::purge_generation`] this entry's counters were reset + /// for, the registry's mirror of the partition plane's + /// `applied_purge_generation` gate. + /// + /// Load-bearing: an apply runs on BOTH left-right buffers and the second run + /// is deferred to the next metadata publish, which can be long after the + /// purge acked. Counters are shared side state (one `Arc` across buffers), + /// so an ungated second reset would wipe messages sent since the purge. + purged_generation: u64, } impl StatsRegistry { @@ -407,7 +422,11 @@ impl StatsRegistry { .lock() .expect("stats registry mutex poisoned") .entry((stream_id, topic_id, partition_id)) - .or_insert_with(|| Arc::new(PartitionStats::new(parent))) + .or_insert_with(|| PartitionEntry { + stats: Arc::new(PartitionStats::new(parent)), + purged_generation: 0, + }) + .stats .clone() } @@ -426,7 +445,63 @@ impl StatsRegistry { .lock() .expect("stats registry mutex poisoned") .get(&(stream_id, topic_id, partition_id)) - .cloned() + .map(|entry| entry.stats.clone()) + } + + /// Reset the counters of every partition a purge just advanced, so a client + /// that reads right after the ack sees the purge instead of pre-purge + /// totals. The on-disk reset stays async (the reconciler resets each + /// partition on every replica once it observes the committed generation); + /// this only moves the counters to the shape that reset converges on. + /// + /// Reset, never decrement: `zero_out_all` swaps in 0 and rolls each parent + /// back by exactly what it swapped out, so a replayed purge entry over an + /// already-zeroed registry cannot underflow a parent total. The generation + /// gate on top makes the replay a no-op outright. + /// + /// The entry is created when missing so the gate is recorded even for a + /// partition this node has not materialized yet. A fresh entry holds no + /// segment, and `ensure_initial_segment` counts the one it plants, hence + /// the segment is restored only for a partition that already had storage -- + /// inventing one here would double-count against that later bump. + // The guard spans a read-modify-write of one entry (check the gate, stamp + // it, take the `Arc`), so it cannot collapse into the single chained + // expression the drop-tightening lint asks for. + #[allow(clippy::significant_drop_tightening)] + fn reset_purged_partitions( + &self, + stream_id: usize, + topic_id: usize, + parent: &Arc, + partitions: &[Partition], + ) { + for partition in partitions { + // Guard dropped before the counters move: `zero_out_all` cascades a + // rollback into the parent topic and stream totals, which the + // registry map has no part in. + let stats = { + let mut entries = self + .partitions + .lock() + .expect("stats registry mutex poisoned"); + let entry = entries + .entry((stream_id, topic_id, partition.id)) + .or_insert_with(|| PartitionEntry { + stats: Arc::new(PartitionStats::new(Arc::clone(parent))), + purged_generation: 0, + }); + if entry.purged_generation >= partition.purge_generation { + continue; + } + entry.purged_generation = partition.purge_generation; + entry.stats.clone() + }; + let had_storage = stats.segments_count_inconsistent() > 0; + stats.zero_out_all(); + if had_storage { + stats.increment_segments_count(1); + } + } } fn remove_stream(&self, id: usize) { @@ -1508,28 +1583,31 @@ impl StateHandler for PurgeStreamRequest { type State = StreamsInner; fn apply(&self, state: &mut StreamsInner, _timestamp: IggyTimestamp) -> ApplyReply { // Stream purge = topic purge over every topic in the stream: advance - // each partition's monotonic purge generation and clear the delete - // watermark; every replica's reconciler observes the committed - // generation and resets the partition to a single empty segment at - // offset 0 with cleared offsets (see `PurgeTopicRequest`). Metadata - // shape stays intact. - let advanced = { - let Some(stream_id) = state.resolve_stream_id(&self.stream_id) else { - return ApplyReply::err(PurgeStreamResult::StreamNotFound); - }; - let Some(stream) = state.items.get_mut(stream_id) else { - return ApplyReply::err(PurgeStreamResult::StreamNotFound); - }; - let mut advanced = false; - for (_, topic) in &mut stream.topics { - for partition in &mut topic.partitions { - partition.purge_generation = partition.purge_generation.wrapping_add(1); - partition.deleted_up_to_offset = 0; - advanced = true; - } - } - advanced + // each partition's monotonic purge generation, clear the delete + // watermark, and reset the partition counters; every replica's + // reconciler observes the committed generation and resets the partition + // to a single empty segment at offset 0 with cleared offsets (see + // `PurgeTopicRequest`). Metadata shape stays intact. + let Some(stream_id) = state.resolve_stream_id(&self.stream_id) else { + return ApplyReply::err(PurgeStreamResult::StreamNotFound); }; + let Some(stream) = state.items.get_mut(stream_id) else { + return ApplyReply::err(PurgeStreamResult::StreamNotFound); + }; + let mut advanced = false; + for (topic_id, topic) in &mut stream.topics { + for partition in &mut topic.partitions { + partition.purge_generation = partition.purge_generation.wrapping_add(1); + partition.deleted_up_to_offset = 0; + advanced = true; + } + state.stats_registry.reset_purged_partitions( + stream_id, + topic_id, + &topic.stats, + &topic.partitions, + ); + } if advanced { state.revision = state.revision.wrapping_add(1); } @@ -1772,25 +1850,34 @@ impl StateHandler for PurgeTopicRequest { // offsets at 0 and drops the consumer-offset barrier that bounded the // trim, and the reconciler re-stages any nonzero watermark on every // pass -- a surviving one would delete post-purge segments. - let advanced = { - let Some(stream_id) = state.resolve_stream_id(&self.stream_id) else { - return ApplyReply::err(PurgeTopicResult::StreamNotFound); - }; - let Some(topic_id) = state.resolve_topic_id(stream_id, &self.topic_id) else { - return ApplyReply::err(PurgeTopicResult::TopicNotFound); - }; - let Some(stream) = state.items.get_mut(stream_id) else { - return ApplyReply::err(PurgeTopicResult::StreamNotFound); - }; - let Some(topic) = stream.topics.get_mut(topic_id) else { - return ApplyReply::err(PurgeTopicResult::TopicNotFound); - }; - for partition in &mut topic.partitions { - partition.purge_generation = partition.purge_generation.wrapping_add(1); - partition.deleted_up_to_offset = 0; - } - !topic.partitions.is_empty() + // + // The shared partition counters are reset here too: they are read back + // by `get_topic` / `get_stream` on any node that applied this commit, + // and leaving them until the reconciler runs makes a purge ack followed + // by a read report pre-purge totals. + let Some(stream_id) = state.resolve_stream_id(&self.stream_id) else { + return ApplyReply::err(PurgeTopicResult::StreamNotFound); }; + let Some(topic_id) = state.resolve_topic_id(stream_id, &self.topic_id) else { + return ApplyReply::err(PurgeTopicResult::TopicNotFound); + }; + let Some(stream) = state.items.get_mut(stream_id) else { + return ApplyReply::err(PurgeTopicResult::StreamNotFound); + }; + let Some(topic) = stream.topics.get_mut(topic_id) else { + return ApplyReply::err(PurgeTopicResult::TopicNotFound); + }; + for partition in &mut topic.partitions { + partition.purge_generation = partition.purge_generation.wrapping_add(1); + partition.deleted_up_to_offset = 0; + } + let advanced = !topic.partitions.is_empty(); + state.stats_registry.reset_purged_partitions( + stream_id, + topic_id, + &topic.stats, + &topic.partitions, + ); if advanced { state.revision = state.revision.wrapping_add(1); } @@ -1892,8 +1979,12 @@ impl StateHandler for DeletePartitionsRequest { }; let count_to_delete = self.partitions_count as usize; - let did_delete = count_to_delete > 0 && count_to_delete <= topic.partitions.len(); - if did_delete { + if count_to_delete > topic.partitions.len() { + return ApplyReply::err(DeletePartitionsResult::InvalidPartitionsCount); + } + // Zero count is rejected pre-consensus; a replayed legacy entry still + // applies as the historical ok no-op. + if count_to_delete > 0 { let retained = topic.partitions.len() - count_to_delete; topic.partitions.truncate(retained); // Members assigned the removed partitions must give them up. @@ -1903,8 +1994,6 @@ impl StateHandler for DeletePartitionsRequest { state .stats_registry .remove_partitions_from(stream_id, topic_id, retained); - } - if did_delete { state.revision = state.revision.wrapping_add(1); } ApplyReply::ok(Bytes::new()) @@ -2465,6 +2554,63 @@ mod tests { assert!(apply.body.is_empty()); } + /// Over-count deletes were acked ok as a silent no-op; they must commit the + /// legacy `InvalidPartitionsCount` rejection. Zero stays an ok no-op at the + /// apply (rejected pre-consensus; a replayed entry keeps its historical ack). + #[test] + fn given_delete_partitions_counts_when_applied_should_reject_over_count() { + let cases: &[(u32, u32, u32, usize)] = &[ + // (partitions in topic, count to delete, expected code, remaining) + ( + 3, + 4, + u32::from(DeletePartitionsResult::InvalidPartitionsCount), + 3, + ), + ( + 0, + 1, + u32::from(DeletePartitionsResult::InvalidPartitionsCount), + 0, + ), + (3, 0, 0, 3), + (3, 3, 0, 0), + (3, 2, 0, 1), + ]; + for &(partitions_count, count_to_delete, expected_code, expected_remaining) in cases { + let mut inner = StreamsInner::new(); + create_stream(&mut inner, "stream"); + let create_topic = CreateTopicWithAssignmentsRequest { + request: make_topic_request(0, partitions_count, "topic"), + partitions: (0..partitions_count) + .map(|partition_id| CreatedPartitionAssignment { + partition_id, + consensus_group_id: 1, + }) + .collect(), + }; + let _ = StateHandler::apply(&create_topic, &mut inner, IggyTimestamp::now()); + + let delete = DeletePartitionsRequest { + stream_id: WireIdentifier::numeric(0), + topic_id: WireIdentifier::numeric(0), + partitions_count: count_to_delete, + }; + let apply = StateHandler::apply(&delete, &mut inner, IggyTimestamp::now()); + + assert_eq!( + apply.code, expected_code, + "deleting {count_to_delete} of {partitions_count} partitions" + ); + assert!(apply.body.is_empty()); + assert_eq!( + inner.items[0].topics[0].partitions.len(), + expected_remaining, + "deleting {count_to_delete} of {partitions_count} partitions" + ); + } + } + #[test] fn given_live_stream_when_apply_purge_stream_should_return_ok_with_empty_body() { let mut inner = StreamsInner::new(); @@ -2541,6 +2687,181 @@ mod tests { ); } + /// A purge acks on commit while the on-disk reset waits for the reconciler, + /// so the counters `get_topic` / `get_stream` read must move in the apply or + /// a read right after the ack reports pre-purge totals. + #[test] + fn given_counted_partition_when_apply_purge_topic_should_zero_the_scope() { + let mut inner = inner_with_registered_partition(); + let stats = inner.stats_registry.partition_get(0, 0, 0).expect("stats"); + stats.increment_segments_count(1); + stats.increment_messages_count(7); + stats.increment_size_bytes(512); + stats.set_current_offset(6); + assert_eq!( + inner.items[0].topics[0].stats.messages_count_inconsistent(), + 7, + "partition counters must roll up before the purge, or the test proves nothing" + ); + + let purge = PurgeTopicRequest { + stream_id: WireIdentifier::numeric(0), + topic_id: WireIdentifier::numeric(0), + }; + let apply = StateHandler::apply(&purge, &mut inner, IggyTimestamp::now()); + assert_eq!(apply.code, 0); + + assert_eq!(stats.messages_count_inconsistent(), 0); + assert_eq!(stats.size_bytes_inconsistent(), 0); + assert_eq!(stats.current_offset(), 0); + assert_eq!( + stats.segments_count_inconsistent(), + 1, + "a purged partition keeps the one empty segment the reset lands on" + ); + let topic_stats = &inner.items[0].topics[0].stats; + assert_eq!(topic_stats.messages_count_inconsistent(), 0); + assert_eq!(topic_stats.size_bytes_inconsistent(), 0); + let stream_stats = &inner.items[0].stats; + assert_eq!(stream_stats.messages_count_inconsistent(), 0); + assert_eq!(stream_stats.size_bytes_inconsistent(), 0); + } + + /// A stream purge walks every topic, so every topic's partitions must reset, + /// not just the first one. + #[test] + fn given_counted_partitions_when_apply_purge_stream_should_zero_every_topic() { + let mut inner = inner_with_registered_partition(); + let create_topic = CreateTopicWithAssignmentsRequest { + request: make_topic_request(0, 1, "metrics"), + partitions: vec![CreatedPartitionAssignment { + partition_id: 0, + consensus_group_id: 2, + }], + }; + let _ = StateHandler::apply(&create_topic, &mut inner, IggyTimestamp::now()); + let second_topic_stats = inner.items[0].topics[1].stats.clone(); + inner.stats_registry.partition(0, 1, 0, second_topic_stats); + + let counters: Vec> = (0..2) + .map(|topic_id| { + let stats = inner + .stats_registry + .partition_get(0, topic_id, 0) + .expect("stats"); + stats.increment_segments_count(1); + stats.increment_messages_count(9); + stats.increment_size_bytes(64); + stats + }) + .collect(); + assert_eq!(inner.items[0].stats.messages_count_inconsistent(), 18); + + let purge = PurgeStreamRequest { + stream_id: WireIdentifier::numeric(0), + }; + let apply = StateHandler::apply(&purge, &mut inner, IggyTimestamp::now()); + assert_eq!(apply.code, 0); + + for stats in &counters { + assert_eq!(stats.messages_count_inconsistent(), 0); + assert_eq!(stats.size_bytes_inconsistent(), 0); + assert_eq!(stats.segments_count_inconsistent(), 1); + } + assert_eq!(inner.items[0].stats.messages_count_inconsistent(), 0); + assert_eq!(inner.items[0].stats.size_bytes_inconsistent(), 0); + } + + /// The left-right buffers absorb every op twice and the second absorb is + /// deferred to the next metadata publish, which can land long after the + /// purge acked. Counters are shared side state, so the deferred replay must + /// leave post-purge traffic alone -- and must not decrement a parent total + /// it already rolled back. + #[test] + fn given_purged_buffer_when_other_buffer_replays_purge_should_keep_new_counters() { + let mut first = inner_with_registered_partition(); + let mut second = first.clone(); + let stats = first.stats_registry.partition_get(0, 0, 0).expect("stats"); + stats.increment_segments_count(1); + stats.increment_messages_count(10); + stats.increment_size_bytes(320); + + let purge = PurgeTopicRequest { + stream_id: WireIdentifier::numeric(0), + topic_id: WireIdentifier::numeric(0), + }; + let _ = StateHandler::apply(&purge, &mut first, IggyTimestamp::now()); + assert_eq!(stats.messages_count_inconsistent(), 0); + + // Sent after the ack, before the deferred absorb on the other buffer. + stats.increment_messages_count(4); + stats.increment_size_bytes(128); + + let _ = StateHandler::apply(&purge, &mut second, IggyTimestamp::now()); + assert_eq!( + second.items[0].topics[0].partitions[0].purge_generation, 1, + "the replay computes the same generation, so the gate is what stops it" + ); + assert_eq!( + stats.messages_count_inconsistent(), + 4, + "the deferred replay must not wipe post-purge counters" + ); + assert_eq!(stats.size_bytes_inconsistent(), 128); + let topic_stats = first.items[0].topics[0].stats.clone(); + assert_eq!( + topic_stats.messages_count_inconsistent(), + 4, + "a second rollback of the same total would underflow the parent" + ); + assert_eq!(topic_stats.size_bytes_inconsistent(), 128); + + // A genuinely new purge still resets: the gate is per generation. + let _ = StateHandler::apply(&purge, &mut first, IggyTimestamp::now()); + assert_eq!(stats.messages_count_inconsistent(), 0); + assert_eq!(topic_stats.messages_count_inconsistent(), 0); + } + + /// Boot replays the metadata WAL before any partition materializes, so the + /// purge has no counters to reset -- but it must still record the gate, or + /// the deferred second absorb wipes whatever the partition loaded since. + #[test] + fn given_unmaterialized_partition_when_apply_purge_should_gate_the_replay() { + let mut inner = StreamsInner::new(); + create_stream(&mut inner, "alpha"); + let create_topic = CreateTopicWithAssignmentsRequest { + request: make_topic_request(0, 1, "logs"), + partitions: vec![CreatedPartitionAssignment { + partition_id: 0, + consensus_group_id: 1, + }], + }; + let _ = StateHandler::apply(&create_topic, &mut inner, IggyTimestamp::now()); + let mut replay = inner.clone(); + + let purge = PurgeTopicRequest { + stream_id: WireIdentifier::numeric(0), + topic_id: WireIdentifier::numeric(0), + }; + let _ = StateHandler::apply(&purge, &mut inner, IggyTimestamp::now()); + + // The data plane materializes the partition afterwards and counts what + // it plants; the purge must not have invented a segment for it. + let topic_stats = inner.items[0].topics[0].stats.clone(); + let stats = inner.stats_registry.partition(0, 0, 0, topic_stats); + assert_eq!(stats.segments_count_inconsistent(), 0); + stats.increment_segments_count(1); + stats.increment_messages_count(5); + + let _ = StateHandler::apply(&purge, &mut replay, IggyTimestamp::now()); + assert_eq!( + stats.messages_count_inconsistent(), + 5, + "the gate recorded at apply must survive into the partition's entry" + ); + assert_eq!(stats.segments_count_inconsistent(), 1); + } + #[test] fn given_missing_topic_when_apply_purge_topic_should_return_topic_not_found() { let mut inner = StreamsInner::new(); diff --git a/core/metadata/src/stm/user.rs b/core/metadata/src/stm/user.rs index 18b7a2e567..7dffbeb8ef 100644 --- a/core/metadata/src/stm/user.rs +++ b/core/metadata/src/stm/user.rs @@ -267,7 +267,7 @@ impl Users { "root username length {length} outside {MIN_USERNAME_LENGTH}..={MAX_USERNAME_LENGTH}; fix IGGY_ROOT_USERNAME" ); - // Boot-only invariant: server-ng calls this before listeners and + // Boot-only invariant: the server calls this before listeners and // consensus traffic start, on shard 0 initialization. The read/apply // split cannot race another user creation in that phase. let username = WireName::new(username).expect("root username must be valid"); @@ -571,7 +571,7 @@ impl StateHandler for ChangePasswordRequest { }; // An empty `new_password` is the primary's signal that the caller's - // current password did not match (see server-ng + // current password did not match (see the server // `verify_and_rewrite_change_password`): the accept path always // replicates a non-empty Argon2 hash, so this is unambiguous. Rejecting // here (rather than denying pre-consensus) commits the op as a no-op, diff --git a/core/partitions/Cargo.toml b/core/partitions/Cargo.toml index 0acc918ba1..4e29b1a3b6 100644 --- a/core/partitions/Cargo.toml +++ b/core/partitions/Cargo.toml @@ -32,7 +32,7 @@ publish = false # Simulator-only detector hook (`IggyPartitions::hold_borrow_across_await`): # deliberately holds a `with_partition` borrow across an `.await` so the # dispatch shell can prove its borrow-across-await detector. A -# `-p iggy-server-ng` build excludes it; `cargo build --workspace` unifies +# `-p iggy-server` build excludes it; `cargo build --workspace` unifies # features so the shared `partitions` unit compiles it in when the simulator # requests it. No production caller. simulator = [] @@ -48,6 +48,7 @@ iggy_binary_protocol = { workspace = true } iggy_common = { workspace = true } journal = { workspace = true } message_bus = { workspace = true } +nix = { workspace = true } papaya = { workspace = true } ringbuffer = { workspace = true } server_common = { workspace = true } @@ -55,6 +56,9 @@ smallvec = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } +[dev-dependencies] +tempfile = { workspace = true } + [lints.clippy] enum_glob_use = "deny" pedantic = "deny" diff --git a/core/partitions/src/iggy_partition.rs b/core/partitions/src/iggy_partition.rs index 148c4078b9..e5ed479a19 100644 --- a/core/partitions/src/iggy_partition.rs +++ b/core/partitions/src/iggy_partition.rs @@ -40,8 +40,9 @@ use consensus::{ ReplicaLogContext, RequestLogEvent, Sequencer, SimEventKind, VsrConsensus, ack_preflight, ack_quorum_reached, build_deny_reply_from_request, build_reply_from_request, build_reply_message, drain_committable_prefix, emit_namespace_progress_event, - emit_partition_diag, emit_sim_event, fence_old_prepare_by_commit, replicate_preflight, - replicate_to_next_in_chain, send_prepare_ok as send_prepare_ok_common, + emit_partition_diag, emit_sim_event, fence_old_prepare_by_commit, + replicate_frozen_to_next_in_chain, replicate_preflight, restamp_prepare_view, + send_prepare_ok as send_prepare_ok_common, verify_prepare_integrity, }; use iggy_binary_protocol::requests::consumer_offsets::{ DeleteConsumerOffset2Request, DeleteConsumerOffsetRequest, StoreConsumerOffset2Request, @@ -53,7 +54,7 @@ use iggy_binary_protocol::responses::messages::{ use iggy_binary_protocol::{ AckLevel, GenericHeader, Operation, PrepareHeader, WireDecode, WireEncode, WireIdentifier, }; -use iggy_binary_protocol::{PrepareOkHeader, RequestHeader}; +use iggy_binary_protocol::{PrepareOkHeader, RoutedRequestHeader}; use iggy_common::{ ConsumerGroupId, ConsumerGroupOffsets, ConsumerKind, ConsumerOffset, ConsumerOffsets, IggyByteSize, IggyError, IggyExpiry, IggyTimestamp, PartitionStats, PollingKind, @@ -69,8 +70,8 @@ use server_common::{ MESSAGE_ALIGN, Message, SegmentStorage, iobuf::{Frozen, Owned}, send_messages2::{ - ChecksumMode, convert_request_message, decode_prepare_slice, decode_prepare_slice_trusted, - stamp_prepare_for_persistence, verify_received_send_messages, + ChecksumMode, SendMessages2Header, convert_request_message, decode_prepare_slice, + decode_prepare_slice_trusted, stamp_prepare_for_persistence, }, sharding::IggyNamespace, }; @@ -278,7 +279,7 @@ where // are the ones diagnostics actually key on. fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("IggyPartition") - .field("namespace", &self.consensus.namespace()) + .field("namespace", &self.consensus.group()) .field("offset", &self.offset) .field("dirty_offset", &self.dirty_offset) .field("should_increment_offset", &self.should_increment_offset) @@ -292,12 +293,12 @@ where } /// Post-preflight dispatch in `on_request`: replicate via VSR or take the -/// `NoAck` leader-local fast path. `RequestHeader` is boxed to avoid the +/// `NoAck` leader-local fast path. `RoutedRequestHeader` is boxed to avoid the /// 277-byte inline variant tripping clippy's `large_enum_variant`. enum Disposition { Replicate(Message), NoAck { - request_header: Box, + request_header: Box, kind: ConsumerKind, consumer_id: u32, offset: Option, @@ -718,7 +719,7 @@ where target: "iggy.partitions.diag", plane = "partitions", replica_id = self.consensus.replica(), - namespace_raw = self.consensus.namespace(), + namespace_raw = self.consensus.group(), view = state.view, log_view = state.log_view, superblock_write_failures = failures, @@ -763,14 +764,31 @@ where return; } tracing::info!( - namespace_raw = self.consensus().namespace(), + namespace_raw = self.consensus().group(), offset_frontier = frontier, "restored partition offset frontier from its superblock" ); self.offset.store(recovered_end, Ordering::Release); self.dirty_offset.store(recovered_end, Ordering::Relaxed); self.should_increment_offset = true; - self.stats.set_current_offset(recovered_end); + } + + /// Copy this incarnation's offset counter into the shared + /// [`PartitionStats`], making it the value readers (offset validation, + /// `get_topic`, `get_stats`) see. + /// + /// Called from [`IggyPartitions::insert`](crate::IggyPartitions::insert) + /// only: when the instance BECOMES the addressable one, never while + /// building it. The stats registry keys on the namespace, not the + /// incarnation, so every build of a namespace holds the same `Arc` as + /// whatever is already serving it -- and a build is not guaranteed to be + /// adopted. Seeding from the build instead leaves a zeroed `current_offset` + /// on the live incarnation, which then rejects every + /// `store_consumer_offset` above 0 with `InvalidOffset` until the next send + /// re-seeds it. + pub(crate) fn publish_current_offset(&self) { + self.stats + .set_current_offset(self.offset.load(Ordering::Acquire)); } /// The next message offset this replica will mint, `0` while the offset @@ -1298,7 +1316,7 @@ where #[allow(clippy::future_not_send)] async fn apply_consumer_offset_no_ack( &self, - request_header: Box, + request_header: Box, kind: ConsumerKind, consumer_id: u32, offset: Option, @@ -1416,6 +1434,7 @@ where &mut self, consumer: PollingConsumer, args: &PollingArgs, + validate_checksum: bool, ) -> PollPlan { // Reads the durable commit frontier (`self.offset`, stored only on // commit). Also used below as the poll's high-water bound: this function @@ -1527,6 +1546,7 @@ where segments, start_position, namespace_raw: self.namespace().inner(), + validate_checksum, }; // Snapshot the resident journal tail now (on the pump, under the // borrow) so the straddle splice runs off-task on owned data with no @@ -1613,68 +1633,9 @@ where &mut self, message: Message, ) -> Result { - let header = *message.header(); - if header.operation != Operation::SendMessages { - return Err(IggyError::CannotAppendMessage); - } - - let dirty_offset = if self.should_increment_offset { - self.dirty_offset.load(Ordering::Relaxed) + 1 - } else { - 0 - }; - - // Reuse the prepare's monotonic timestamp, assigned once by the primary - // in `project()` (`next_monotonic_timestamp`) and replicated verbatim to - // every backup. Sourcing it here instead of a fresh local `now()` makes - // the persisted `base_timestamp` (and the `batch_checksum` derived from - // it) byte-identical across replicas; a local `now()` diverges per node. - let batch_timestamp = header.timestamp; - let (message, batch, batch_messages_count) = - stamp_prepare_for_persistence(message, dirty_offset, batch_timestamp) - .map_err(|_| IggyError::CannotAppendMessage)?; - - if batch_messages_count == 0 { - return Ok(AppendResult::new(0, 0, 0)); - } - - let batch_messages_size = - u64::try_from(batch.total_size()).map_err(|_| IggyError::CannotAppendMessage)?; - - let last_dirty_offset = dirty_offset + u64::from(batch_messages_count) - 1; - - if !self.should_increment_offset { - self.should_increment_offset = true; - } - self.dirty_offset - .store(last_dirty_offset, Ordering::Relaxed); - - let segment_index = self.log.segments().len() - 1; - let current_position = self.log.segments()[segment_index].current_position; - self.log.segments_mut()[segment_index].current_position = current_position - .checked_add(batch_messages_size) - .ok_or(IggyError::CannotAppendMessage)?; - - let journal = self.log.journal_mut(); - journal.info.messages_count += batch_messages_count; - journal.info.size += IggyByteSize::from(batch_messages_size); - journal.info.current_offset = last_dirty_offset; - if journal.info.first_timestamp == 0 { - journal.info.first_timestamp = batch.base_timestamp; - } - journal.info.end_timestamp = batch.base_timestamp; - journal.info.max_timestamp = journal.info.max_timestamp.max(batch.base_timestamp); - journal - .inner - .append(message.into_frozen()) + self.stamp_and_append_messages(message) .await - .map_err(|_| IggyError::CannotAppendMessage)?; - - Ok(AppendResult::new( - dirty_offset, - last_dirty_offset, - batch_messages_count, - )) + .map(|journaled| journaled.result) } #[allow(clippy::cast_possible_truncation)] @@ -1716,9 +1677,40 @@ where B: MessageBus, SB: SuperblockStore, { + async fn stamp_and_append_messages( + &mut self, + message: Message, + ) -> Result { + let header = *message.header(); + if header.operation != Operation::SendMessages { + return Err(IggyError::CannotAppendMessage); + } + + let dirty_offset = if self.should_increment_offset { + self.dirty_offset + .load(Ordering::Relaxed) + .checked_add(1) + .ok_or(IggyError::CannotAppendMessage)? + } else { + 0 + }; + + // Reuse the prepare's monotonic timestamp, assigned once by the primary + // in `project()` (`next_monotonic_timestamp`) and replicated verbatim to + // every backup. Sourcing it here instead of a fresh local `now()` makes + // the persisted `base_timestamp` (and the `batch_checksum` derived from + // it) byte-identical across replicas. A local `now()` diverges per node. + let batch_timestamp = header.timestamp; + let (message, batch, batch_messages_count) = + stamp_prepare_for_persistence(message, dirty_offset, batch_timestamp) + .map_err(|_| IggyError::CannotAppendMessage)?; + + debug_assert_eq!(batch.message_count, batch_messages_count); + self.append_stamped_messages(message, batch).await + } #[must_use] fn namespace(&self) -> IggyNamespace { - IggyNamespace::from_raw(self.consensus.namespace()) + IggyNamespace::from_raw(self.consensus.group()) } fn partition_dir(&self) -> Option { @@ -1820,9 +1812,9 @@ where /// Panics if called when this partition's consensus instance is not the /// primary, is not in normal status, or is currently syncing. #[allow(clippy::future_not_send, clippy::too_many_lines)] - pub async fn on_request(&mut self, message: Message) { + pub async fn on_request(&mut self, message: Message) { self.clear_pending_consumer_offset_commits_if_view_changed(); - let namespace = IggyNamespace::from_raw(message.header().namespace); + let namespace = IggyNamespace::from_raw(message.header().group); let client_id = message.header().client; let request = message.header().request; @@ -2096,6 +2088,22 @@ where pub async fn on_replicate(&mut self, message: Message) { self.clear_pending_consumer_offset_commits_if_view_changed(); let header = *message.header(); + // Same reason as the metadata plane: `checksum` is compared as an opaque token + // downstream, so a corrupted frame passes whenever its flipped value satisfies + // those comparisons. + if let Err(reason) = verify_prepare_integrity(&header, message.as_slice()) { + emit_partition_diag( + tracing::Level::WARN, + &PartitionDiagEvent::new( + ReplicaLogContext::from_consensus(self.consensus(), PlaneKind::Partitions), + "discarding prepare that failed its own integrity check", + ) + .with_operation(header.operation) + .with_op(header.op) + .with_reason(reason), + ); + return; + } let current_op = { let consensus = self.consensus(); match replicate_preflight(consensus, &header) { @@ -2147,11 +2155,53 @@ where .with_operation(header.operation) .with_op(header.op), ); - let clone_for_forward = message.clone(); - let consensus = self.consensus(); - if let Err(error) = replicate_to_next_in_chain(consensus, &clone_for_forward).await { + let Some(journaled) = self.log.journal().inner.repair_entry(header.op) else { + emit_partition_diag( + tracing::Level::ERROR, + &PartitionDiagEvent::new( + self.diag_ctx(), + "journal header exists without matching prepare bytes", + ) + .with_operation(header.operation) + .with_op(header.op), + ); + return; + }; + if !journaled_prepare_matches_retransmit(&journaled, &message) { emit_partition_diag( tracing::Level::WARN, + &PartitionDiagEvent::new( + self.diag_ctx(), + "rejecting retransmitted prepare that differs from the journaled entry", + ) + .with_operation(header.operation) + .with_op(header.op), + ); + return; + } + let Some(frozen_for_forward) = restamp_prepare_view(journaled, header.view) else { + emit_partition_diag( + tracing::Level::ERROR, + &PartitionDiagEvent::new( + self.diag_ctx(), + "failed to restamp journaled prepare for retransmission", + ) + .with_operation(header.operation) + .with_op(header.op), + ); + return; + }; + let consensus = self.consensus(); + if let Err(error) = + replicate_frozen_to_next_in_chain(consensus, frozen_for_forward).await + { + let is_transport_error = error.is_transport(); + emit_partition_diag( + if is_transport_error { + tracing::Level::WARN + } else { + tracing::Level::ERROR + }, &PartitionDiagEvent::new( self.diag_ctx(), "failed to re-forward retransmitted prepare to next in chain", @@ -2160,6 +2210,9 @@ where .with_op(header.op) .with_error(error.to_string()), ); + if !is_transport_error { + return; + } } self.send_prepare_ok(&header).await; return; @@ -2235,78 +2288,63 @@ where ); } } - // First blob-integrity check on the replicated path. The consensus - // layer never validates the body (PrepareHeader integrity fields are - // inert zeros) and the batch checksum is recomputed locally at stamp, - // so a follower must verify each message's stamp-invariant per-message - // checksum before journaling transit bytes. Follower-only: the primary - // (and single-node self-replicate) produced these bytes and already - // checked the client batch at ingest, so they must not pay this pass. - // Fail closed on mismatch - drop without journaling, forwarding, or - // acking; the primary retransmits on prepare-timeout. - if is_backup - && header.operation == Operation::SendMessages - && let Err(error) = verify_received_send_messages(message.as_slice()) - { - emit_partition_diag( - tracing::Level::WARN, - &PartitionDiagEvent::new( - self.diag_ctx(), - "rejecting replicated send_messages: per-message checksum mismatch", - ) - .with_operation(header.operation) - .with_op(header.op) - .with_error(error.to_string()), - ); - return; - } - - // Durability-before-ack: clone for chain-replicate, forward only - // AFTER apply_replicated_operation persists. Forward-first would - // give downstream an op whose WAL entry we never wrote, that violates - // tail-ahead-of-head. Clone is cheap (Arc bumps in common case). - let clone_for_forward = message.clone(); - let replicated_result = self.apply_replicated_operation(message).await; - if replicated_result.is_ok() { - let consensus = self.consensus(); - // Backup only: advance sequencer + checksum after journal append. - // Pre-advance on failing apply would leave consensus claiming op N - // while journal has nothing; retransmit of N would silently drop - // as is_old_prepare (header.op <= current_sequence). Primary must - // NOT re-set here: push_prepare_entry already advanced, and a - // sibling request pipelined during the apply await would be - // rewound to a stale op + parent, projecting a duplicate next. - if is_backup { - consensus.sequencer().set_sequence(header.op); - consensus.set_last_prepare_checksum(header.checksum); - consensus.observe_prepare_timestamp(header.timestamp); - } - if let Err(error) = replicate_to_next_in_chain(consensus, &clone_for_forward).await { + // Forward only after apply_replicated_operation journals the prepare. + // The journal and network share the frozen allocation, so the bytes + // retained for repair are exactly the bytes sent downstream. + let replicated_result = if is_backup && header.operation == Operation::SendMessages { + self.append_received_send_messages_to_journal(message).await + } else { + self.apply_replicated_operation(message).await + }; + let frozen_for_forward = match replicated_result { + Ok(frozen) => frozen, + Err(error) => { emit_partition_diag( tracing::Level::WARN, &PartitionDiagEvent::new( self.diag_ctx(), - "failed to replicate prepare to next in chain", + "failed to apply replicated partition operation", ) .with_operation(header.operation) .with_op(header.op) .with_error(error.to_string()), ); + return; } - } + }; - if let Err(error) = replicated_result { + let consensus = self.consensus(); + // Backup only: advance sequencer + checksum after journal append. + // Pre-advance on failing apply would leave consensus claiming op N + // while the journal has nothing. Retransmit of N would silently drop + // as is_old_prepare (header.op <= current_sequence). The primary does + // not re-set here because push_prepare_entry already advanced it. A + // sibling request pipelined during the apply await would otherwise be + // rewound to a stale op + parent, projecting a duplicate next. + if is_backup { + consensus.sequencer().set_sequence(header.op); + consensus.set_last_prepare_checksum(header.checksum); + consensus.observe_prepare_timestamp(header.timestamp); + } + if let Err(error) = replicate_frozen_to_next_in_chain(consensus, frozen_for_forward).await { + let is_transport_error = error.is_transport(); emit_partition_diag( - tracing::Level::WARN, + if is_transport_error { + tracing::Level::WARN + } else { + tracing::Level::ERROR + }, &PartitionDiagEvent::new( self.diag_ctx(), - "failed to apply replicated partition operation", + "failed to replicate prepare to next in chain", ) .with_operation(header.operation) .with_op(header.op) .with_error(error.to_string()), ); - return; + if !is_transport_error { + return; + } } { @@ -2435,14 +2473,14 @@ where async fn apply_replicated_operation( &mut self, message: Message, - ) -> Result<(), IggyError> { + ) -> Result, IggyError> { let header = *message.header(); let replica_id = self.consensus.replica(); - let namespace_raw = self.consensus.namespace(); + let namespace_raw = self.consensus.group(); match header.operation { Operation::SendMessages => { - self.append_send_messages_to_journal(message).await?; + let frozen = self.append_send_messages_to_journal(message).await?; debug!( target: "iggy.partitions.diag", plane = "partitions", @@ -2452,7 +2490,7 @@ where operation = ?header.operation, "replicated send_messages appended to partition journal" ); - Ok(()) + Ok(frozen) } Operation::StoreConsumerOffset | Operation::DeleteConsumerOffset @@ -2473,10 +2511,11 @@ where // the `journal.info` accounting: it counts SendMessages // batches for segment-commit thresholds, which do not // apply to offset ops. + let frozen = message.into_frozen(); self.log .journal() .inner - .append(message.clone().into_frozen()) + .append(frozen.clone()) .await .map_err(|_| IggyError::CannotAppendMessage)?; @@ -2508,7 +2547,7 @@ where offset = ?offset, "replicated consumer offset journaled and staged" ); - Ok(()) + Ok(frozen) } _ => { warn!( @@ -2520,7 +2559,7 @@ where operation = ?header.operation, "unexpected replicated partition operation" ); - Ok(()) + Err(IggyError::InvalidCommand) } } } @@ -2528,10 +2567,190 @@ where async fn append_send_messages_to_journal( &mut self, message: Message, - ) -> Result<(), IggyError> { + ) -> Result, IggyError> { let write_lock = self.write_lock.clone(); let _guard = write_lock.lock().await; - self.append_messages(message).await.map(|_| ()) + self.stamp_and_append_messages(message) + .await + .map(|journaled| journaled.prepare) + } + + async fn append_received_send_messages_to_journal( + &mut self, + message: Message, + ) -> Result, IggyError> { + let write_lock = self.write_lock.clone(); + let _guard = write_lock.lock().await; + let header = *message.header(); + if header.operation != Operation::SendMessages { + return Err(IggyError::CannotAppendMessage); + } + let validated = decode_prepare_slice(message.as_slice())?.header; + if validated.message_count == 0 { + return Err(IggyError::InvalidCommand); + } + let expected_offset = if self.should_increment_offset { + self.dirty_offset + .load(Ordering::Relaxed) + .checked_add(1) + .ok_or(IggyError::CannotAppendMessage)? + } else { + 0 + }; + if (validated.base_offset, validated.base_timestamp) != (expected_offset, header.timestamp) + { + return Err(IggyError::CannotAppendMessage); + } + self.append_stamped_messages(message, validated) + .await + .map(|journaled| journaled.prepare) + } + + async fn append_stamped_messages( + &mut self, + message: Message, + batch: SendMessages2Header, + ) -> Result { + let batch_messages_count = batch.message_count; + if batch_messages_count == 0 { + return Err(IggyError::CannotAppendMessage); + } + + let batch_messages_size = + u64::try_from(batch.total_size()).map_err(|_| IggyError::CannotAppendMessage)?; + let last_dirty_offset = batch + .base_offset + .checked_add(u64::from(batch_messages_count) - 1) + .ok_or(IggyError::CannotAppendMessage)?; + + let segment_index = self.log.segments().len() - 1; + let current_position = self.log.segments()[segment_index].current_position; + let next_position = current_position + .checked_add(batch_messages_size) + .ok_or(IggyError::CannotAppendMessage)?; + + let mut journal_info = self.log.journal().info; + journal_info.messages_count = journal_info + .messages_count + .checked_add(batch_messages_count) + .ok_or(IggyError::CannotAppendMessage)?; + journal_info.size = IggyByteSize::from( + journal_info + .size + .as_bytes_u64() + .checked_add(batch_messages_size) + .ok_or(IggyError::CannotAppendMessage)?, + ); + journal_info.current_offset = last_dirty_offset; + if journal_info.first_timestamp == 0 { + journal_info.first_timestamp = batch.base_timestamp; + } + journal_info.end_timestamp = batch.base_timestamp; + journal_info.max_timestamp = journal_info.max_timestamp.max(batch.base_timestamp); + + let frozen = message.into_frozen(); + self.log + .journal() + .inner + .append(frozen.clone()) + .await + .map_err(|_| IggyError::CannotAppendMessage)?; + + self.should_increment_offset = true; + self.dirty_offset + .store(last_dirty_offset, Ordering::Relaxed); + self.log.segments_mut()[segment_index].current_position = next_position; + self.log.journal_mut().info = journal_info; + + Ok(JournaledMessages { + result: AppendResult::new(batch.base_offset, last_dirty_offset, batch_messages_count), + prepare: frozen, + }) + } + + /// Drop an uncommitted view-divergent suffix and restore every append cursor + /// from the retained prefix as one write-locked operation. + /// + /// # Errors + /// + /// Returns an error if a retained batch is invalid, the restored segment + /// position overflows, or the journal cannot truncate the suffix. + pub async fn truncate_uncommitted_from(&mut self, from_op: u64) -> Result { + let write_lock = self.write_lock.clone(); + let _guard = write_lock.lock().await; + + let mut entries = self.log.journal().inner.resident_entries(); + entries.sort_unstable_by_key(peek_op); + let mut retained_info = JournalInfo::default(); + let mut retained_next_offset = 0; + let mut rewind_next_offset = None; + for entry in &entries { + if peek_operation(entry) != Operation::SendMessages { + continue; + } + let batch = decode_prepare_slice_trusted(entry.as_slice()) + .map_err(|_| IggyError::InvalidCommand)?; + if batch.message_count() == 0 { + continue; + } + if peek_op(entry) >= from_op { + rewind_next_offset = Some( + rewind_next_offset.map_or(batch.header.base_offset, |offset: u64| { + offset.min(batch.header.base_offset) + }), + ); + continue; + } + accumulate_committed_info( + &mut retained_info, + batch.header.base_offset, + batch.header.base_timestamp, + batch.header.total_size() as u64, + batch.message_count(), + ); + retained_next_offset = retained_next_offset.max( + batch + .header + .base_offset + .saturating_add(u64::from(batch.message_count())), + ); + } + + let active = self.log.active_segment(); + let active_size = active.size.as_bytes_u64(); + let durable_next_offset = if active_size == 0 { + active.start_offset + } else { + active.end_offset.saturating_add(1) + }; + let minimum_next_offset = durable_next_offset + .max(retained_next_offset) + .max( + self.recovered_durable_offset + .map_or(0, |offset| offset.saturating_add(1)), + ) + .max(self.installed_frontier.unwrap_or(0)); + let restored_position = active_size + .checked_add(retained_info.size.as_bytes_u64()) + .ok_or(IggyError::CannotAppendMessage)?; + let removed = self + .log + .journal() + .inner + .truncate_from(from_op) + .await + .map_err(|_| IggyError::CannotAppendMessage)?; + + self.log.journal_mut().info = retained_info; + self.log.active_segment_mut().current_position = restored_position; + if let Some(next_offset) = rewind_next_offset { + let next_offset = next_offset.max(minimum_next_offset); + self.dirty_offset + .store(next_offset.saturating_sub(1), Ordering::Relaxed); + self.should_increment_offset = next_offset > 0; + } + self.consensus.invalidate_local_dvc_suffix(); + Ok(removed) } async fn commit_messages(&mut self, config: &PartitionsConfig) -> Result<(), IggyError> { @@ -2862,7 +3081,7 @@ where send_client_replies: bool, ) { let replica_id = self.consensus.replica(); - let namespace_raw = self.consensus.namespace(); + let namespace_raw = self.consensus.group(); let drained_count = drained.len(); if let (Some(first), Some(last)) = (drained.first(), drained.last()) { debug!( @@ -2944,7 +3163,7 @@ where if send_client_replies && !is_auto_commit_client(prepare_header.client) { let body = match prepare_header.operation { Operation::SendMessages => { - send_messages_reply_body(prepare_header.namespace, batch_stats) + send_messages_reply_body(prepare_header.group, batch_stats) } operation => committed_reply_body(operation), }; @@ -3162,13 +3381,13 @@ where fn parse_consumer_offset_request( operation: Operation, - message: &Message, + message: &Message, ) -> Result<(ConsumerKind, u32, Option, AckLevel), IggyError> { let total_size = usize::try_from(message.header().size).map_err(|_| IggyError::InvalidCommand)?; let body = message .as_slice() - .get(std::mem::size_of::()..total_size) + .get(std::mem::size_of::()..total_size) .ok_or(IggyError::InvalidCommand)?; Self::parse_consumer_offset_payload(operation, body) } @@ -3179,7 +3398,7 @@ where /// so nothing replicates. async fn send_partition_deny_or_log( consensus: &VsrConsensus, - header: &RequestHeader, + header: &RoutedRequestHeader, status: u32, send_fail_label: &'static str, ) { @@ -3502,6 +3721,7 @@ where messages_size_bytes, config.enforce_fsync, false, + config.preallocate_segments.then_some(config.segment_size), ) .await .map_err(|_| IggyError::CannotCreateSegmentLogFile(messages_path.clone()))?, @@ -3753,6 +3973,7 @@ where messages_size_bytes, config.enforce_fsync, false, + config.preallocate_segments.then_some(config.segment_size), ) .await .map_err(|_| IggyError::CannotCreateSegmentLogFile(messages_path.clone()))?, @@ -3982,7 +4203,18 @@ where guard.clear(); paths }; - for path in consumer_paths.into_iter().chain(group_paths) { + // Sweep the directories too, not just the map-derived paths: a purge is a + // full reset, and an offset file the live map never held -- a pre-purge + // op re-persisted by journal repair on a restarted replica -- would + // otherwise survive for boot to hydrate back. + let strayed = + crate::state_transfer::strayed_offset_files(self.consumer_offsets_path.as_deref(), &[]) + .into_iter() + .chain(crate::state_transfer::strayed_offset_files( + self.consumer_group_offsets_path.as_deref(), + &[], + )); + for path in consumer_paths.into_iter().chain(group_paths).chain(strayed) { let _ = delete_persisted_offset(&path).await; } // Directory fsync so those unlinks stick, mirroring the install path: a @@ -4155,7 +4387,7 @@ where Err(error) => Err(error), } } else { - self.apply_replicated_operation(message).await + self.apply_replicated_operation(message).await.map(|_| ()) }; if let Err(error) = applied { warn!( @@ -4344,8 +4576,7 @@ where let op = message.header().op; let (base_offset, base_timestamp, total_size, message_count) = { - let batch = - decode_prepare_slice(message.as_slice()).map_err(|_| IggyError::InvalidCommand)?; + let batch = decode_prepare_slice(message.as_slice())?; ( batch.header.base_offset, batch.header.base_timestamp, @@ -4354,7 +4585,7 @@ where ) }; if message_count == 0 { - return Ok(None); + return Err(IggyError::InvalidCommand); } // Purge floor: the same fence every other journal-apply path honors. A @@ -4374,33 +4605,49 @@ where return Ok(None); } - let last_offset = base_offset + u64::from(message_count) - 1; - - self.should_increment_offset = true; + let last_offset = base_offset + .checked_add(u64::from(message_count) - 1) + .ok_or(IggyError::CannotAppendMessage)?; let dirty = self.dirty_offset.load(Ordering::Relaxed); - self.dirty_offset - .store(dirty.max(last_offset), Ordering::Relaxed); let segment_index = self.log.segments().len() - 1; let current_position = self.log.segments()[segment_index].current_position; - self.log.segments_mut()[segment_index].current_position = current_position + let next_position = current_position .checked_add(total_size) .ok_or(IggyError::CannotAppendMessage)?; - let journal = self.log.journal_mut(); - journal.info.messages_count += message_count; - journal.info.size += IggyByteSize::from(total_size); - journal.info.current_offset = last_offset; - if journal.info.first_timestamp == 0 { - journal.info.first_timestamp = base_timestamp; + let mut journal_info = self.log.journal().info; + journal_info.messages_count = journal_info + .messages_count + .checked_add(message_count) + .ok_or(IggyError::CannotAppendMessage)?; + journal_info.size = IggyByteSize::from( + journal_info + .size + .as_bytes_u64() + .checked_add(total_size) + .ok_or(IggyError::CannotAppendMessage)?, + ); + journal_info.current_offset = last_offset; + if journal_info.first_timestamp == 0 { + journal_info.first_timestamp = base_timestamp; } - journal.info.end_timestamp = base_timestamp; - journal.info.max_timestamp = journal.info.max_timestamp.max(base_timestamp); - journal + journal_info.end_timestamp = base_timestamp; + journal_info.max_timestamp = journal_info.max_timestamp.max(base_timestamp); + + let frozen = message.into_frozen(); + self.log + .journal() .inner - .append(message.into_frozen()) + .append(frozen) .await .map_err(|_| IggyError::CannotAppendMessage)?; + + self.should_increment_offset = true; + self.dirty_offset + .store(dirty.max(last_offset), Ordering::Relaxed); + self.log.segments_mut()[segment_index].current_position = next_position; + self.log.journal_mut().info = journal_info; Ok(Some(base_offset)) } @@ -4477,6 +4724,30 @@ fn peek_op(entry: &Frozen<4096>) -> u64 { .op } +/// Match a retransmit against immutable, validated journal bytes. Only the view +/// may change, so exact body equality replaces another checksum pass. +fn journaled_prepare_matches_retransmit( + journaled: &Frozen<4096>, + incoming: &Message, +) -> bool { + const VIEW_OFFSET: usize = std::mem::offset_of!(PrepareHeader, view); + + let stored = journaled.as_slice(); + let received = incoming.as_slice(); + let header_size = std::mem::size_of::(); + if stored.len() != received.len() || stored.len() < header_size { + return false; + } + + let view_end = VIEW_OFFSET + std::mem::size_of::(); + if stored[..VIEW_OFFSET] != received[..VIEW_OFFSET] + || stored[view_end..header_size] != received[view_end..header_size] + { + return false; + } + stored[header_size..] == received[header_size..] +} + /// Success reply body for a committed partition op other than `SendMessages` /// (which confirms its offsets through [`send_messages_reply_body`]). /// @@ -4541,6 +4812,11 @@ struct CommittedBatchStats { size_bytes: u64, } +struct JournaledMessages { + result: AppendResult, + prepare: Frozen<4096>, +} + impl CommittedBatchStats { /// Offset of the batch's last message. The batch carries a contiguous /// offset run, and the sole constructor rejects an empty one, so the @@ -5076,7 +5352,7 @@ mod tests { client_id: u128, request_id: u64, consumer_id: u32, - ) -> Message { + ) -> Message { let body = DeleteConsumerOffset2Request { consumer: WireConsumer::consumer(WireIdentifier::Numeric(consumer_id)), stream_id: WireIdentifier::Numeric(1), @@ -5085,17 +5361,17 @@ mod tests { ack: AckLevel::Quorum, } .to_bytes(); - let header_size = std::mem::size_of::(); + let header_size = std::mem::size_of::(); let total = header_size + body.len(); - let mut message = Message::::new(total); + let mut message = Message::::new(total); message.as_mut_slice()[header_size..].copy_from_slice(&body); - message.transmute_header(|_, header: &mut RequestHeader| { + message.transmute_header(|_, header: &mut RoutedRequestHeader| { header.command = Command2::Request; header.operation = Operation::DeleteConsumerOffset2; header.client = client_id; header.session = 1; header.request = request_id; - header.namespace = IggyNamespace::new(1, 1, 0).inner(); + header.group = IggyNamespace::new(1, 1, 0).inner(); header.size = u32::try_from(total).expect("request size fits u32"); }) } @@ -5187,7 +5463,10 @@ mod tests { let path = format!("{dir}/{consumer_id}"); let read_disk = |p: &str| -> u64 { let bytes = std::fs::read(p).expect("offset file exists"); - u64::from_le_bytes(bytes.try_into().expect("offset file is 8 bytes")) + match crate::offset_storage::decode_offset_record(&bytes) { + crate::offset_storage::OffsetRecord::Value { offset, .. } => offset, + other => panic!("offset file must hold a readable value, got {other:?}"), + } }; // Reordered auto-commits: the later op (109) trails the earlier (114). @@ -5270,7 +5549,10 @@ mod tests { let path = format!("{dir}/{consumer_id}"); let read_disk = |p: &str| -> u64 { let bytes = std::fs::read(p).expect("offset file exists"); - u64::from_le_bytes(bytes.try_into().expect("offset file is 8 bytes")) + match crate::offset_storage::decode_offset_record(&bytes) { + crate::offset_storage::OffsetRecord::Value { offset, .. } => offset, + other => panic!("offset file must hold a readable value, got {other:?}"), + } }; // Simulate the previous process run: the file already holds 114. @@ -5433,6 +5715,7 @@ mod tests { // open exhausts retries -> the walk must fault-close before segment two. let plan = DiskReadPlan { partition_dir: PartitionDirResolution::Resolved(partition_dir), + validate_checksum: true, segments: vec![ DiskSegment { start_offset: 0, @@ -5518,6 +5801,7 @@ mod tests { let plan = DiskReadPlan { partition_dir: PartitionDirResolution::Resolved(partition_dir), + validate_checksum: true, segments: vec![ DiskSegment { start_offset: 0, @@ -5550,6 +5834,77 @@ mod tests { let _ = std::fs::remove_dir_all(&dir); } + /// A segment whose bytes decode cleanly but do not match their own + /// `batch_checksum`: bit rot at rest, not a torn write. Unverified, the batch is + /// served and a consumer reads data provably not what was written. + /// + /// Detection only, per the operator knob: the poll fails closed and reports, with + /// no attempt to repair. + #[compio::test] + async fn read_disk_faults_closed_on_batch_checksum_mismatch() { + let namespace = IggyNamespace::new(1, 1, 0); + let dir = std::env::temp_dir().join(format!( + "iggy-read-disk-bitrot-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system clock after epoch") + .as_nanos(), + )); + compio::fs::create_dir_all(&dir) + .await + .expect("create temp partition dir"); + let partition_dir = dir.to_string_lossy().into_owned(); + + // Structurally valid with one payload byte flipped, so every length and + // offset still decodes and only the checksum disagrees. + let mut record = build_segment_record(namespace, 0); + let last = record.len() - 1; + record[last] ^= 0x01; + let record_len = record.len() as u64; + let path = format!("{partition_dir}/{:0>20}.log", 0u64); + { + let mut file = compio::fs::File::create(&path) + .await + .expect("create segment file"); + let (written, _) = file.write_all_at(record, 0).await.into(); + written.expect("write segment record"); + file.sync_all().await.expect("flush segment file"); + } + + let plan = |validate_checksum| DiskReadPlan { + partition_dir: PartitionDirResolution::Resolved(partition_dir.clone()), + validate_checksum, + segments: vec![DiskSegment { + start_offset: 0, + persisted: record_len, + read_state: None, + }], + start_position: 0, + namespace_raw: namespace.inner(), + }; + let query = MessageLookup::Offset { + offset: 0, + count: 10, + ceiling: u64::MAX, + }; + + let outcome = plan(true).read_disk(query).await; + assert!( + matches!(outcome, DiskReadOutcome::Faulted), + "a batch that fails its own checksum must fault-close" + ); + + // What the opt-out costs. The shipped default is `true` because of it. + let outcome = plan(false).read_disk(query).await; + assert!( + matches!(outcome, DiskReadOutcome::Matched { .. }), + "verification off is an explicit opt-out: the corrupt batch is served" + ); + + let _ = std::fs::remove_dir_all(&dir); + } + /// A simulated (file-less) partition has no segment files by design, so a /// disk poll with no dir must stay `Empty`: the caller then serves the /// resident journal tier, the sim's only tier. @@ -5564,6 +5919,7 @@ mod tests { }], start_position: 0, namespace_raw: IggyNamespace::new(1, 1, 0).inner(), + validate_checksum: true, }; let outcome = plan @@ -5594,6 +5950,7 @@ mod tests { }], start_position: 0, namespace_raw: IggyNamespace::new(1, 1, 0).inner(), + validate_checksum: true, }; let outcome = plan @@ -5652,6 +6009,7 @@ mod tests { let plan = DiskReadPlan { partition_dir: PartitionDirResolution::Resolved(partition_dir.clone()), + validate_checksum: true, segments: vec![DiskSegment { start_offset: 0, persisted: record_len, @@ -5682,6 +6040,7 @@ mod tests { let plan = DiskReadPlan { partition_dir: PartitionDirResolution::Resolved(partition_dir.clone()), + validate_checksum: true, segments: vec![DiskSegment { start_offset: 0, persisted: record_len, @@ -5741,6 +6100,7 @@ mod tests { let handle = SealedSegmentHandle::default(); let plan = DiskReadPlan { partition_dir: PartitionDirResolution::Resolved(partition_dir.clone()), + validate_checksum: true, segments: vec![DiskSegment { start_offset: 0, persisted: record_len, @@ -5824,6 +6184,7 @@ mod tests { let handle = SealedSegmentHandle::default(); let plan = DiskReadPlan { partition_dir: PartitionDirResolution::Resolved(partition_dir.clone()), + validate_checksum: true, segments: vec![DiskSegment { start_offset: 0, persisted: log_len, @@ -5922,6 +6283,7 @@ mod tests { let handle = SealedSegmentHandle::default(); let plan = DiskReadPlan { partition_dir: PartitionDirResolution::Resolved(partition_dir.clone()), + validate_checksum: true, segments: vec![DiskSegment { start_offset: 0, persisted: log_len, @@ -5999,6 +6361,7 @@ mod tests { let handle = Rc::clone(&partition.log.sealed_read_state()[0]); let plan = DiskReadPlan { partition_dir: PartitionDirResolution::Resolved(partition_dir.clone()), + validate_checksum: true, segments: vec![DiskSegment { start_offset: 0, persisted: record_len, @@ -6046,6 +6409,7 @@ mod tests { // unlinked pre-purge inode. let resumed = DiskReadPlan { partition_dir: PartitionDirResolution::Resolved(partition_dir.clone()), + validate_checksum: true, segments: vec![DiskSegment { start_offset: 0, persisted: record_len, @@ -6074,7 +6438,9 @@ mod tests { messages_required_to_save: 1, size_of_messages_required_to_save: IggyByteSize::from(1024 * 1024), enforce_fsync: false, + validate_checksum: true, segment_size: IggyByteSize::from(1024 * 1024), + preallocate_segments: false, encryptor: None, } } @@ -6426,6 +6792,50 @@ mod tests { let _ = std::fs::remove_dir_all(&partition_dir); } + /// A replica that missed the purge entirely: the metadata plane has it + /// committed, this replica never applied it, so its frontier still measures + /// the PRE-purge offset space. The reset offer is the only thing that can + /// converge it, and journal repair cannot bridge the floor the purge moved, + /// so refusing it strands the replica on pre-purge data for good. + /// + /// Distinguished from the lagging-origin case above by `next_offset == 0`: + /// nothing has been appended since the purge, so there is no post-purge + /// data for the offer to rewind. + #[compio::test] + async fn given_replica_that_missed_the_purge_when_offered_the_reset_should_install() { + let partition_dir = transfer_fence_dir("missed-purge-reset").await; + let mut partition = test_partition(); + partition.set_partition_dir(partition_dir.clone()); + partition.should_increment_offset = true; + partition.offset.store(99, Ordering::Release); + assert_eq!( + partition.applied_purge_generation(), + 0, + "a replica that missed the purge has not recorded its generation" + ); + + let reset = crate::state_transfer::ConsumerOffsetsWire { + purge_generation: 1, + next_offset: 0, + consumers: Vec::new(), + groups: Vec::new(), + }; + let installed = partition + .install_state_transfer(&repair_config(), 12, Vec::new(), &reset.encode(), 1) + .await; + + assert!( + !matches!( + installed, + Err(crate::state_transfer::PartitionInstallError::OfferRewindsDurableData { .. }) + ), + "the reset for a purge this replica never applied must pass the rewind \ + fence, got {installed:?}" + ); + + let _ = std::fs::remove_dir_all(&partition_dir); + } + /// Primary-by-index at view 0 with nothing committed refuses to serve: an /// empty group is trivially "caught up", so this gate is the only thing /// separating a real primary from a phantom whose directory vanished, whose @@ -6674,7 +7084,7 @@ mod purge_floor_tests { header.operation = Operation::SendMessages; header.op = op; header.timestamp = op; - header.namespace = namespace.inner(); + header.group = namespace.inner(); header.size = u32::try_from(total).expect("prepare size fits u32"); }); partition @@ -6709,7 +7119,7 @@ mod purge_floor_tests { header.command = Command2::Prepare; header.operation = Operation::StoreConsumerOffset2; header.op = op; - header.namespace = IggyNamespace::new(1, 1, 0).inner(); + header.group = IggyNamespace::new(1, 1, 0).inner(); header.size = u32::try_from(total).expect("prepare size fits u32"); }); partition @@ -7039,7 +7449,7 @@ mod purge_floor_tests { header.command = Command2::Prepare; header.operation = Operation::SendMessages; header.op = 1; - header.namespace = namespace.inner(); + header.group = namespace.inner(); header.size = u32::try_from(total).expect("prepare size fits u32"); }); diff --git a/core/partitions/src/iggy_partitions.rs b/core/partitions/src/iggy_partitions.rs index a25aed657b..45c0df4b53 100644 --- a/core/partitions/src/iggy_partitions.rs +++ b/core/partitions/src/iggy_partitions.rs @@ -23,7 +23,7 @@ use crate::{IggyPartition, Partition, PollingArgs, PollingConsumer}; use ahash::AHashSet; use consensus::{Consensus, Plane, PlaneIdentity, VsrConsensus}; use iggy_binary_protocol::{ - Command2, ConsensusHeader, Operation, PrepareHeader, PrepareOkHeader, RequestHeader, + Command2, ConsensusHeader, Operation, PrepareHeader, PrepareOkHeader, RoutedRequestHeader, }; use journal::superblock::{PingPongSuperblock, SuperblockStore}; use message_bus::MessageBus; @@ -200,6 +200,12 @@ where /// Insert a new partition and return its local index. /// + /// Insertion is the moment a build becomes the addressable incarnation, + /// so this is also where its offset counter is published into the shared + /// `PartitionStats` ([`IggyPartition::publish_current_offset`]). No + /// earlier point is safe: a build that is never adopted must leave the + /// live incarnation's counters alone. + /// /// # Safety discipline (compiler cannot enforce) /// /// Must only be called from the shard's pump task (i.e. inside @@ -220,6 +226,7 @@ where 0, "IggyPartitions::insert while a with_partition borrow is live" ); + partition.publish_current_offset(); // Safety: pump-only invariant, caller responsibility. let partitions = unsafe { &mut *self.partitions.get() }; let local_idx = LocalIdx::new(partitions.len()); @@ -352,7 +359,7 @@ where // by the moved partition's namespace key in O(1); the previous // linear value-scan turned bulk DeleteStream into O(K²) on the // pump task, stalling client traffic for ~10k-partition topics. - let moved_ns = IggyNamespace::from_raw(partitions[idx].consensus().namespace()); + let moved_ns = IggyNamespace::from_raw(partitions[idx].consensus().group()); let entry = self.namespace_map_mut().get_mut(&moved_ns).expect( "IggyPartitions invariant: swapped-in partition missing namespace_to_local entry", ); @@ -412,8 +419,10 @@ where // `build_poll_plan` touches the partition's sealed-read-handle LRU, so it // needs `&mut`. Sound on the pump: it is fully synchronous (no `.await` // inside), so no sibling task can realloc the partitions vec under it. + // Read the knob first: the `&mut` borrow below covers `self.config` too. + let validate_checksum = self.config.validate_checksum; let partition = self.get_mut_by_ns(namespace)?; - Some(partition.build_poll_plan(consumer, args)) + Some(partition.build_poll_plan(consumer, args, validate_checksum)) } /// Read a consumer's stored offset + the partition commit offset. Fully @@ -496,8 +505,11 @@ where B: MessageBus, SB: SuperblockStore, { - async fn on_request(&self, message: as Consensus>::Message) { - let namespace = IggyNamespace::from_raw(message.header().namespace); + async fn on_request( + &self, + message: as Consensus>::Message, + ) { + let namespace = IggyNamespace::from_raw(message.header().group); if self.is_tombstoned(&namespace) { warn!( target: "iggy.partitions.diag", @@ -550,20 +562,20 @@ where } async fn on_replicate(&self, message: as Consensus>::Message) { - let namespace = IggyNamespace::from_raw(message.header().namespace); - if self.is_tombstoned(&namespace) { + let group = IggyNamespace::from_raw(message.header().group); + if self.is_tombstoned(&group) { warn!( target: "iggy.partitions.diag", - namespace_raw = namespace.inner(), + namespace_raw = group.inner(), "dropping prepare: namespace tombstoned" ); return; } - let Some(partition) = self.get_mut_by_ns(&namespace) else { + let Some(partition) = self.get_mut_by_ns(&group) else { warn!( target: "iggy.partitions.diag", plane = "partitions", - namespace_raw = namespace.inner(), + namespace_raw = group.inner(), op = message.header().op, operation = ?message.header().operation, "partition not initialized for namespace" @@ -575,21 +587,21 @@ where #[allow(clippy::too_many_lines)] async fn on_ack(&self, message: as Consensus>::Message) { - let namespace = IggyNamespace::from_raw(message.header().namespace); - if self.is_tombstoned(&namespace) { + let group = IggyNamespace::from_raw(message.header().group); + if self.is_tombstoned(&group) { warn!( target: "iggy.partitions.diag", - namespace_raw = namespace.inner(), + namespace_raw = group.inner(), "dropping prepare-ok: namespace tombstoned" ); return; } let config = self.config.clone(); - let Some(partition) = self.get_mut_by_ns(&namespace) else { + let Some(partition) = self.get_mut_by_ns(&group) else { warn!( target: "iggy.partitions.diag", plane = "partitions", - namespace_raw = namespace.inner(), + namespace_raw = group.inner(), op = message.header().op, "partition not initialized for namespace" ); @@ -654,6 +666,28 @@ mod tests { ) } + /// `build_partition` for a replicated group. The replica count is what + /// decides whether the journal retains evicted entries for repair, so a + /// single-replica partition cannot exercise anything that reads the ring. + fn build_replicated_partition() -> IggyPartition { + let namespace = IggyNamespace::new(1, 1, 0); + let consensus = VsrConsensus::new( + TEST_CLUSTER, + 0, + 3, + namespace.inner(), + IggyMessageBus::new(0), + LocalPipeline::new(), + ); + consensus.init(); + IggyPartition::with_in_memory_storage( + Arc::new(PartitionStats::default()), + consensus, + IggyByteSize::from(1024 * 1024), + false, + ) + } + /// One-message `SendMessages` journal entry stamped at `op` / `base_offset`. /// Reuses the production blob builder + checksum stamping so the entry /// decodes through `decode_prepare_slice` and indexes into `offset_to_op`, @@ -796,6 +830,67 @@ mod tests { ); } + /// A flush evicts the committed prefix up to and INCLUDING `commit_max`, so + /// a caught-up replica keeps no resident header at its own commit point. The + /// `DoViewChange` suffix is floored there and cannot nack it, so reading the + /// resident headers alone sends the commit point out blank, which a quorum of + /// senders turns into a view change that never starts. + /// + /// The entry is still servable (`repair_entry` answers from the evicted + /// ring), so the suffix reads through `repair_header`, over the same range. + #[compio::test] + async fn evicted_commit_point_still_answers_for_the_view_change_suffix() { + let namespace = IggyNamespace::new(1, 1, 0); + let partition = build_replicated_partition(); + + for offset in 0..=2u64 { + partition + .log + .journal() + .inner + .append(build_send_messages_entry(namespace, offset + 1, offset)) + .await + .expect("append journal entry"); + } + + let commit_max = 3; + let prefix = partition.log.journal().inner.committed_prefix(commit_max); + assert_eq!(prefix.len(), 3, "the whole log is committed and flushable"); + partition + .log + .journal() + .inner + .evict_prefix(prefix.len()) + .await; + + assert!( + partition + .log + .journal() + .inner + .header_by_op(commit_max) + .is_none(), + "the flush evicted the commit point from the resident headers", + ); + assert!( + partition + .log + .journal() + .inner + .repair_entry(commit_max) + .is_some(), + "yet the entry is still servable from the evicted ring", + ); + + let header = partition + .log + .journal() + .inner + .repair_header(commit_max) + .expect("the commit point must stay describable for the DVC suffix"); + assert_eq!(header.op, commit_max); + } + /// The resident journal holds replicated-but-uncommitted prepares ahead of /// the commit frontier. A poll must clamp at `ceiling` (the commit offset) /// so it never returns a dirty read of view-change-rollbackable data, even diff --git a/core/partitions/src/journal.rs b/core/partitions/src/journal.rs index f29a951f27..689e2b727f 100644 --- a/core/partitions/src/journal.rs +++ b/core/partitions/src/journal.rs @@ -25,6 +25,7 @@ use std::io; use std::{ cell::{Cell, UnsafeCell}, collections::{BTreeMap, HashMap, VecDeque}, + ops::RangeInclusive, }; use tracing::warn; @@ -188,11 +189,11 @@ where /// Running byte total of the buffers held by `evicted_ring`. evicted_ring_bytes: Cell, /// Entry-count ceiling for `evicted_ring`. Defaults to - /// [`EVICTED_RING_CAPACITY`]; server-ng overrides it from config at + /// [`EVICTED_RING_CAPACITY`]; the server overrides it from config at /// partition build. evicted_ring_capacity: Cell, /// Byte ceiling for `evicted_ring`. Defaults to - /// [`EVICTED_RING_BYTES_MAX`]; server-ng overrides it from config at + /// [`EVICTED_RING_BYTES_MAX`]; the server overrides it from config at /// partition build. evicted_ring_bytes_max: Cell, /// Single-replica groups have nobody to repair; retaining evicted @@ -367,6 +368,64 @@ impl PartitionJournal { .map(|(_, entry)| entry.clone()) } + /// The header at `op`, over exactly the range [`Self::repair_entry`] serves. + /// + /// NOT [`Self::header_by_op`], which reads the resident headers alone. The + /// committed prefix is evicted from those the moment its bytes reach a + /// segment, up to and including `commit_max`, so a `DoViewChange` built off + /// the resident headers reports its own commit point blank. The merge scans + /// the commit point and cannot discard it, so a quorum of such senders is + /// undecidable and the view never starts (`dvc_merge::merge_dvc_quorum`). + /// The entry is still servable from the evicted ring, which is what makes + /// the blank wrong rather than merely pessimistic. + /// + /// The ring drops from the front, so the highest evicted op -- the commit + /// point of the last flush -- is the last thing it forgets. + pub fn repair_header(&self, op: u64) -> Option { + if let Some(header) = self.header_by_op(op) { + return Some(header); + } + let ring = unsafe { &*self.evicted_ring.get() }; + let (_, entry) = ring.iter().find(|(ring_op, _)| *ring_op == op)?; + let header_bytes = entry.as_slice().get(..PREPARE_HEADER_SIZE)?; + bytemuck::checked::try_from_bytes::(header_bytes) + .ok() + .copied() + } + + /// Every repairable header with an op in `ops`, in ONE pass over the resident + /// headers and ONE over the evicted ring. + /// + /// [`Self::repair_header`] is two linear scans, so probing it per op costs + /// O(window x (headers + ring)), and the `DoViewChange` suffix build does + /// exactly that, up to `DVC_HEADERS_MAX` probes, on every SVC/DVC arrival and + /// non-Normal tick, on the pump. Result size is bounded by what the journal + /// holds, not by the width of `ops`. Resident wins over ring, as `repair_header` + /// probes. + #[must_use] + pub fn repair_headers_in(&self, ops: RangeInclusive) -> BTreeMap { + let mut found = BTreeMap::new(); + { + let headers = unsafe { &*self.headers.get() }; + for header in headers.iter().filter(|header| ops.contains(&header.op)) { + found.insert(header.op, *header); + } + } + let ring = unsafe { &*self.evicted_ring.get() }; + for (op, entry) in ring.iter().filter(|(op, _)| ops.contains(op)) { + if found.contains_key(op) { + continue; + } + let Some(header_bytes) = entry.as_slice().get(..PREPARE_HEADER_SIZE) else { + continue; + }; + if let Ok(header) = bytemuck::checked::try_from_bytes::(header_bytes) { + found.insert(*op, *header); + } + } + found + } + /// Oldest op this journal can still serve for repair (ring front, else /// resident head), or `None` when it holds nothing at all. pub fn repair_retained_from(&self) -> Option { @@ -908,6 +967,59 @@ impl Journal for PartitionJournal Option { self.bytes_by_op(header.op).await } + + /// Appends are in op order and every rewrite preserves it, so the tail header + /// carries the highest op. + fn last_op(&self) -> Option { + let headers = unsafe { &*self.headers.get() }; + headers.last().map(|header| header.op) + } + + /// Drop every entry at or above `from_op`, rebuilding the indexes. Same + /// drain-and-re-append shape as `evict_prefix`, from the other end and retaining + /// nothing: `append` has no slot-collision check here, so a superseded entry left + /// in place sits beside the new view's prepare at the same op and + /// `committed_prefix`, which walks positionally, flushes the stale one. + /// + /// Dropped entries do NOT enter the evicted repair ring: it answers repair for + /// committed ops, and these are ones the view just decided against. + async fn truncate_from(&self, from_op: u64) -> io::Result { + if from_op == 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "truncate_from: ops are 1-based, so 0 would discard the whole journal", + )); + } + let all_entries = { + let inner = unsafe { &*self.inner.get() }; + inner.storage.drain() + }; + // Positional against `headers` until the clear below (see the length-lock + // invariant on `append_with_meta`), so the ops are captured first. + let ops: Vec = { + let headers = unsafe { &*self.headers.get() }; + headers.iter().map(|header| header.op).collect() + }; + { + unsafe { &mut *self.headers.get() }.clear(); + unsafe { &mut *self.op_to_storage_offset.get() }.clear(); + unsafe { &mut *self.offset_to_op.get() }.clear(); + unsafe { &mut *self.timestamp_to_op.get() }.clear(); + } + + let mut removed = 0usize; + for (op, entry) in ops.into_iter().zip(all_entries) { + if op >= from_op { + removed += 1; + continue; + } + // Replays bytes this journal already accepted once, so it cannot fail. + self.append_with_meta(entry) + .await + .expect("re-appending a retained journal entry must not fail"); + } + Ok(removed) + } } pub fn select_batch_slice( @@ -1164,6 +1276,74 @@ mod tests { ); } + #[compio::test] + async fn truncate_from_drops_the_suffix_and_keeps_the_prefix_readable() { + let journal = PartitionJournal::::default(); + for op in 1..=5 { + journal + .append(build_prepare(op, HEADER_SIZE + 16).into_frozen()) + .await + .expect("append"); + } + + let removed = journal.truncate_from(4).await.expect("truncate"); + assert_eq!(removed, 2, "ops 4 and 5 must go"); + assert_eq!(journal.last_op(), Some(3)); + for op in 1..=3u64 { + let header = journal + .header_by_op(op) + .expect("a retained op must survive"); + assert!( + journal.entry(&header).await.is_some(), + "a retained entry must still read back after the rewrite" + ); + } + for op in 4..=5u64 { + assert!(journal.header_by_op(op).is_none(), "op {op} must be gone"); + } + + // The point of dropping them: the primary's retransmission refills the range. + journal + .append(build_prepare(4, HEADER_SIZE + 16).into_frozen()) + .await + .expect("a truncated op must be appendable again"); + assert_eq!(journal.last_op(), Some(4)); + } + + #[compio::test] + async fn repair_headers_in_serves_the_commit_point_from_the_evicted_ring() { + // Blank AT the commit point is the one slot a merge can neither adopt nor + // discard, so a quorum that all flushed there deadlocks. A flushed replica has + // no resident header there, so the ring must answer. + let journal = PartitionJournal::::default(); + for op in 1..=4 { + journal + .append(build_prepare(op, HEADER_SIZE + 16).into_frozen()) + .await + .expect("append"); + } + // `commit_messages` evicts the committed prefix inclusively, so the commit + // point's own resident header goes with it. + journal.evict_prefix(2).await; + assert!( + journal.header_by_op(2).is_none(), + "the resident header at the commit point is gone after the flush" + ); + + let window = journal.repair_headers_in(2..=4); + assert!( + window.contains_key(&2), + "the commit point must still be describable, or the view change deadlocks" + ); + for op in 3..=4u64 { + assert!(window.contains_key(&op), "op {op} is resident and in range"); + } + assert!( + !window.contains_key(&1), + "ops outside the window must not be reported" + ); + } + #[compio::test] async fn committed_prefix_reads_then_evict_retains_uncommitted_tail() { // A backup journals ops ahead of the commit frontier. Reading the diff --git a/core/partitions/src/lib.rs b/core/partitions/src/lib.rs index 75b4c70ca8..2b62402cf6 100644 --- a/core/partitions/src/lib.rs +++ b/core/partitions/src/lib.rs @@ -25,7 +25,7 @@ mod iggy_partitions; mod journal; mod log; mod messages_writer; -mod offset_storage; +pub mod offset_storage; mod poll_plan; mod segment; pub mod state_transfer; diff --git a/core/partitions/src/log.rs b/core/partitions/src/log.rs index 34d5b93075..574b37108f 100644 --- a/core/partitions/src/log.rs +++ b/core/partitions/src/log.rs @@ -116,6 +116,14 @@ where ) -> impl Future>> { self.inner.drain(ops) } + + fn truncate_from(&self, from_op: u64) -> impl Future> { + self.inner.truncate_from(from_op) + } + + fn last_op(&self) -> Option { + self.inner.last_op() + } } impl Default for JournalState { diff --git a/core/partitions/src/messages_writer.rs b/core/partitions/src/messages_writer.rs index f6ad70cc18..cf6fc33650 100644 --- a/core/partitions/src/messages_writer.rs +++ b/core/partitions/src/messages_writer.rs @@ -21,11 +21,16 @@ use compio::{ }; use iggy_common::{IggyByteSize, IggyError}; use server_common::iobuf::Frozen; +#[cfg(target_os = "linux")] +use std::os::fd::AsFd; use std::{ rc::Rc, sync::atomic::{AtomicU64, Ordering}, }; -use tracing::error; +use tracing::{error, warn}; + +#[cfg(target_os = "linux")] +use nix::fcntl::{FallocateFlags, fallocate}; const MAX_IOV_COUNT: usize = 1024; @@ -48,6 +53,7 @@ impl MessagesWriter { messages_size_bytes: Rc, fsync: bool, file_exists: bool, + preallocate_size: Option, ) -> Result { let mut opts = OpenOptions::new(); opts.write(true); @@ -59,6 +65,13 @@ impl MessagesWriter { .await .map_err(|_| IggyError::CannotReadFile)?; + if let Some(preallocate_size) = preallocate_size { + #[cfg(target_os = "linux")] + preallocate_file(&file, file_path, preallocate_size.as_bytes_u64()).await; + #[cfg(not(target_os = "linux"))] + preallocate_file(&file, file_path, preallocate_size.as_bytes_u64()); + } + if file_exists { file.sync_all() .await @@ -148,6 +161,58 @@ impl MessagesWriter { } } +#[cfg(target_os = "linux")] +async fn preallocate_file(file: &File, file_path: &str, len: u64) { + let Ok(len) = i64::try_from(len) else { + warn!( + target: "iggy.partitions.storage", + file = file_path, + preallocate_len = len, + "file preallocation size is unsupported, using buffered allocation" + ); + return; + }; + + let file = match file.as_fd().try_clone_to_owned() { + Ok(file) => file, + Err(error) => { + warn!( + target: "iggy.partitions.storage", + file = file_path, + preallocate_len = len, + %error, + "file descriptor duplication failed, using buffered allocation" + ); + return; + } + }; + + // Remote filesystems can make fallocate block. The duplicated descriptor + // lets the blocking pool reserve extents without stalling the shard thread. + let result = compio::runtime::spawn_blocking(move || { + fallocate(file, FallocateFlags::FALLOC_FL_KEEP_SIZE, 0, len) + }) + .await; + if let Err(error) = result { + warn!( + target: "iggy.partitions.storage", + file = file_path, + preallocate_len = len, + %error, + "file preallocation failed, using buffered allocation" + ); + } +} + +#[cfg(not(target_os = "linux"))] +fn preallocate_file(_file: &File, file_path: &str, _len: u64) { + warn!( + target: "iggy.partitions.storage", + file = file_path, + "file preallocation is unavailable on this platform, using buffered allocation" + ); +} + async fn write_frozen_chunked( file: &File, file_path: &str, @@ -178,3 +243,25 @@ async fn write_frozen_chunked( Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[compio::test] + async fn preallocated_file_keeps_logical_length() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("segment.log"); + let writer = MessagesWriter::new( + path.to_str().unwrap(), + Rc::new(AtomicU64::new(0)), + false, + false, + Some(IggyByteSize::from(1024 * 1024_u64)), + ) + .await + .unwrap(); + + assert_eq!(writer.file.metadata().await.unwrap().len(), 0); + } +} diff --git a/core/partitions/src/offset_storage.rs b/core/partitions/src/offset_storage.rs index c3377d824b..618f987eb1 100644 --- a/core/partitions/src/offset_storage.rs +++ b/core/partitions/src/offset_storage.rs @@ -17,23 +17,97 @@ use compio::{ fs::{OpenOptions, create_dir_all, remove_file}, - io::{AsyncReadAtExt, AsyncWriteAtExt}, + io::{AsyncReadAt, AsyncReadAtExt, AsyncWriteAtExt}, }; -use iggy_common::IggyError; +use iggy_common::{IggyError, calculate_checksum}; use std::path::Path; use tracing::warn; const OFFSET_SIZE: usize = core::mem::size_of::(); +const CHECKSUM_SIZE: usize = core::mem::size_of::(); + +/// Bytes a consumer-offset file holds: the offset, then a checksum over it. +/// +/// The offset is a consumer cursor reloaded unchanged on every restart, so a +/// flipped bit silently rewinds the consumer into redelivery or skips it forward. +pub const OFFSET_RECORD_SIZE: usize = OFFSET_SIZE + CHECKSUM_SIZE; /// Per-partition file recording the purge generation this replica last applied -/// locally, in the partition dir beside the segments it fences. Two LE u64s: -/// the applied generation, then the `created_revision` of the partition -/// incarnation it was applied for. +/// locally, in the partition dir beside the segments it fences. +/// +/// Two LE u64s: the applied generation, then the `created_revision` of the +/// partition incarnation it was applied for. pub const PURGE_GENERATION_FILE: &str = "purge.gen"; /// `[generation][created_revision]`, both LE u64. const PURGE_GENERATION_RECORD_SIZE: usize = 2 * OFFSET_SIZE; +/// What a consumer-offset file was found to hold. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OffsetRecord { + /// A usable offset. `checksummed` is false for a bare offset predating the + /// checksum, read as-is and upgraded by the next write. + Value { offset: u64, checksummed: bool }, + /// Shorter than the value: a crash between `persist_offset`'s truncate and write. + Torn, + /// The checksum does not describe the value stored beside it. + Corrupt { + offset: u64, + expected: u64, + found: u64, + }, +} + +/// Encode a consumer offset for persistence. +#[must_use] +pub fn encode_offset_record(offset: u64) -> [u8; OFFSET_RECORD_SIZE] { + let mut record = [0u8; OFFSET_RECORD_SIZE]; + record[..OFFSET_SIZE].copy_from_slice(&offset.to_le_bytes()); + let checksum = calculate_checksum(&record[..OFFSET_SIZE]); + record[OFFSET_SIZE..].copy_from_slice(&checksum.to_le_bytes()); + record +} + +/// Decode whatever a consumer-offset file contained. +/// +/// A file of exactly one offset predates the checksum and is accepted. A partly +/// written checksum region reads as the bare offset for the same reason: the record +/// is written in one call, so the low bytes are the complete new value. +#[must_use] +pub fn decode_offset_record(bytes: &[u8]) -> OffsetRecord { + let Some(value) = bytes.first_chunk::() else { + return OffsetRecord::Torn; + }; + let offset = u64::from_le_bytes(*value); + let Some(stored) = bytes + .get(OFFSET_SIZE..) + .and_then(<[u8]>::first_chunk::) + else { + return OffsetRecord::Value { + offset, + checksummed: false, + }; + }; + let found = u64::from_le_bytes(*stored); + let expected = calculate_checksum(value); + if found == expected { + OffsetRecord::Value { + offset, + checksummed: true, + } + } else { + OffsetRecord::Corrupt { + offset, + expected, + found, + } + } +} + +/// Overwrite a consumer-offset file with `offset` and a checksum over it. +/// +/// # Errors +/// [`IggyError`] when the directory, file, or write cannot be created or completed. pub async fn persist_offset(path: &str, offset: u64, enforce_fsync: bool) -> Result<(), IggyError> { // No `exists()` probe first: that is a BLOCKING `std::path` stat on the pump // in front of every write, which serialises a batched fan-out on stats @@ -52,8 +126,7 @@ pub async fn persist_offset(path: &str, offset: u64, enforce_fsync: bool) -> Res .open(path) .await .map_err(|_| IggyError::CannotOpenConsumerOffsetsFile(path.to_owned()))?; - let buf = offset.to_le_bytes(); - file.write_all_at(buf, 0) + file.write_all_at(encode_offset_record(offset), 0) .await .0 .map_err(|_| IggyError::CannotWriteToFile)?; @@ -67,25 +140,52 @@ pub async fn persist_offset(path: &str, offset: u64, enforce_fsync: bool) -> Res Ok(()) } -/// Monotone counterpart of [`persist_offset`] for a server auto-commit op: -/// folds `max(current_on_disk, offset)` and returns the value now on disk, -/// skipping the write when the file already holds it. Disk-tier polls -/// replicate their auto-committed offsets in IO-completion order, so a -/// committed op can carry a lower offset than an earlier one; a plain -/// overwrite would leave the file rewound and a restart would reload the -/// stale value and re-deliver. The on-disk value is committed-only (this path -/// never writes the eager serving map), so the fold is identical on every -/// replica applying the same op order. +/// Monotone counterpart of [`persist_offset`] for a server auto-commit op. +/// +/// Folds `max(current_on_disk, offset)` and returns the value now on disk, skipping +/// the write when the file already holds it. Disk-tier polls replicate their +/// auto-committed offsets in IO-completion order, so a committed op can carry a lower +/// offset than an earlier one, and a plain overwrite would leave the file rewound for +/// a restart to reload and re-deliver. The on-disk value is committed-only, so the +/// fold is identical on every replica applying the same op order. +/// +/// The read makes this the cold-key path only: once the caller's persisted-offset +/// tracker knows the file's value, warm commits persist with a blind +/// [`persist_offset`] and skip covered offsets without reading. /// -/// The read makes this the cold-key path only: once the caller's -/// persisted-offset tracker knows the file's value, warm commits persist with -/// a blind [`persist_offset`] and skip covered offsets without any file read. +/// A file that fails its checksum folds as absent and is overwritten. Its value is +/// untrusted and the boot loader already discarded it, so nothing is preserved by +/// refusing, and refusing is not survivable: the caller reads a failed commit as +/// divergence and aborts the shard, and the key is cold on every boot, so one damaged +/// file aborts every boot. Redelivery is within at-least-once. +/// +/// # Errors +/// [`IggyError`] when the file cannot be read or written. pub async fn persist_offset_max( path: &str, offset: u64, enforce_fsync: bool, ) -> Result { - let on_disk = read_persisted_offset(path).await?; + let on_disk = match read_offset_record(path).await? { + Some(OffsetRecord::Value { offset, .. }) => Some(offset), + Some(OffsetRecord::Corrupt { + offset: stored, + expected, + found, + }) => { + tracing::error!( + path, + stored, + expected, + found, + fold_to = offset, + "consumer offset file failed its checksum; overwriting it with the committed \ + offset. This consumer may see redelivery." + ); + None + } + Some(OffsetRecord::Torn) | None => None, + }; let effective = on_disk.map_or(offset, |current| current.max(offset)); if on_disk != Some(effective) { persist_offset(path, effective, enforce_fsync).await?; @@ -94,10 +194,11 @@ pub async fn persist_offset_max( } /// Durably record the purge generation a partition has locally applied, keyed -/// to the incarnation (`created_revision`) it was applied for. Truncate+write -/// like [`persist_offset`] but ALWAYS data-synced, regardless of the -/// consumer-offset fsync knob: purges are rare, the record is 16 bytes, and a -/// generation lost from the page cache in a crash makes the reconciler +/// to the incarnation (`created_revision`) it was applied for. +/// +/// Truncate+write like [`persist_offset`] but ALWAYS data-synced, regardless of +/// the consumer-offset fsync knob: purges are rare, the record is 16 bytes, and +/// a generation lost from the page cache in a crash makes the reconciler /// re-purge on restart, wiping messages appended after the purge. A failure /// leaves the previous record on disk so the caller keeps its in-memory /// applied generation old and retries. @@ -196,32 +297,29 @@ pub async fn read_purge_generation(path: &str, created_revision: u64) -> Result< Ok(generation) } -/// Read a single persisted consumer offset. `None` if the file is absent or -/// torn (shorter than 8 bytes): a crash between `persist_offset`'s truncate -/// and write leaves a short file, and the boot-time loader already skips such -/// files, so the commit-path reader must agree or a torn file turns every -/// later commit-apply into an error. Real I/O errors still propagate: mapping -/// them to `None` would silently rewind a valid higher offset. -async fn read_persisted_offset(path: &str) -> Result, IggyError> { - if !Path::new(path).exists() { +/// Read whatever a consumer-offset file holds. `None` only when absent; a short file +/// reports [`OffsetRecord::Torn`] and the caller folds it as the boot loader does. +/// +/// Real I/O errors propagate: unlike a failed checksum, an unreadable file may still +/// hold an intact cursor, and folding it as absent would rewind the consumer. +async fn read_offset_record(path: &str) -> Result, IggyError> { + // Absence answered by the open, not a `Path::exists()` probe: that is a BLOCKING + // stat on the pump before every cold-key commit (see `persist_offset`). + let file = match OpenOptions::new().read(true).open(path).await { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(_) => return Err(IggyError::CannotOpenConsumerOffsetsFile(path.to_owned())), + }; + // One short read, not `read_exact` twice: `decode_offset_record` classifies any + // length, so the returned count separates a legacy 8-byte file from a full + // record. `..read` matters: the zero padding would otherwise decode as a + // checksum and the legacy file as `Corrupt`. + let compio::BufResult(read, buf) = file.read_at(vec![0u8; OFFSET_RECORD_SIZE], 0).await; + let read = read.map_err(|_| IggyError::CannotReadConsumerOffsets(path.to_owned()))?; + if read == 0 { return Ok(None); } - let file = OpenOptions::new() - .read(true) - .open(path) - .await - .map_err(|_| IggyError::CannotOpenConsumerOffsetsFile(path.to_owned()))?; - let buf = vec![0u8; OFFSET_SIZE]; - let compio::BufResult(read, buf) = file.read_exact_at(buf, 0).await; - match read { - Ok(()) => {} - Err(error) if error.kind() == std::io::ErrorKind::UnexpectedEof => return Ok(None), - Err(_) => return Err(IggyError::CannotReadConsumerOffsets(path.to_owned())), - } - let bytes: [u8; OFFSET_SIZE] = buf - .try_into() - .map_err(|_| IggyError::CannotReadConsumerOffsets(path.to_owned()))?; - Ok(Some(u64::from_le_bytes(bytes))) + Ok(Some(decode_offset_record(&buf[..read]))) } /// Unlink a persisted consumer-offset file. A no-op if the file is absent. @@ -256,52 +354,158 @@ mod tests { dir } + #[test] + fn offset_record_round_trips() { + let record = encode_offset_record(114); + assert_eq!(record.len(), OFFSET_RECORD_SIZE); + assert_eq!( + decode_offset_record(&record), + OffsetRecord::Value { + offset: 114, + checksummed: true + } + ); + } + + #[test] + fn offset_record_accepts_a_bare_value_written_before_the_checksum() { + assert_eq!( + decode_offset_record(&114u64.to_le_bytes()), + OffsetRecord::Value { + offset: 114, + checksummed: false + } + ); + } + + #[test] + fn offset_record_rejects_either_half_flipped() { + // The point of the checksum: a flipped bit rewinds a consumer into redelivery + // or skips it forward, and nothing ever notices. + let mut value_flipped = encode_offset_record(114); + value_flipped[0] ^= 0x01; + assert!(matches!( + decode_offset_record(&value_flipped), + OffsetRecord::Corrupt { offset: 115, .. } + )); + + let mut checksum_flipped = encode_offset_record(114); + checksum_flipped[OFFSET_SIZE] ^= 0x01; + assert!(matches!( + decode_offset_record(&checksum_flipped), + OffsetRecord::Corrupt { offset: 114, .. } + )); + } + + #[test] + fn offset_record_partly_written_is_torn_below_the_value_and_bare_above_it() { + assert_eq!(decode_offset_record(&[]), OffsetRecord::Torn); + assert_eq!(decode_offset_record(&[0xAB; 7]), OffsetRecord::Torn); + + // One `write_all_at` writes the whole record, so a torn tail keeps the value. + let record = encode_offset_record(114); + assert_eq!( + decode_offset_record(&record[..OFFSET_SIZE + 3]), + OffsetRecord::Value { + offset: 114, + checksummed: false + } + ); + } + + #[compio::test] + async fn read_offset_record_reports_a_corrupt_file_as_corrupt() { + let dir = unique_temp_dir(); + let path = dir.join("42").to_string_lossy().into_owned(); + + persist_offset(&path, 114, false).await.expect("persist"); + let mut bytes = std::fs::read(&path).expect("offset file exists"); + bytes[0] ^= 0x01; + std::fs::write(&path, &bytes).expect("corrupt the file"); + + let result = read_offset_record(&path).await; + assert!( + matches!(result, Ok(Some(OffsetRecord::Corrupt { .. }))), + "a corrupt cursor must be distinguishable from an absent one, got {result:?}" + ); + + let _ = std::fs::remove_dir_all(&dir); + } + + #[compio::test] + async fn read_offset_record_reads_a_legacy_bare_value() { + let dir = unique_temp_dir(); + let path = dir.join("42").to_string_lossy().into_owned(); + std::fs::write(&path, 114u64.to_le_bytes()).expect("write legacy file"); + + let read = read_offset_record(&path).await.expect("legacy file"); + assert_eq!( + read, + Some(OffsetRecord::Value { + offset: 114, + checksummed: false + }) + ); + + let _ = std::fs::remove_dir_all(&dir); + } + #[compio::test] - async fn read_persisted_offset_absent_file_is_none() { + async fn read_offset_record_absent_file_is_none() { let dir = unique_temp_dir(); let path = dir.join("42").to_string_lossy().into_owned(); - let read = read_persisted_offset(&path).await.expect("absent file"); + let read = read_offset_record(&path).await.expect("absent file"); assert_eq!(read, None); let _ = std::fs::remove_dir_all(&dir); } #[compio::test] - async fn read_persisted_offset_round_trips_persisted_value() { + async fn read_offset_record_round_trips_persisted_value() { let dir = unique_temp_dir(); let path = dir.join("42").to_string_lossy().into_owned(); persist_offset(&path, 114, false).await.expect("persist"); - let read = read_persisted_offset(&path).await.expect("valid file"); - assert_eq!(read, Some(114)); + let read = read_offset_record(&path).await.expect("valid file"); + assert_eq!( + read, + Some(OffsetRecord::Value { + offset: 114, + checksummed: true + }) + ); let _ = std::fs::remove_dir_all(&dir); } #[compio::test] - async fn read_persisted_offset_torn_file_is_none_not_error() { + async fn read_offset_record_torn_file_is_torn_not_error() { let dir = unique_temp_dir(); let path = dir.join("42").to_string_lossy().into_owned(); std::fs::write(&path, [0xAB, 0xCD, 0xEF]).expect("write torn file"); - let read = read_persisted_offset(&path) + let read = read_offset_record(&path) .await .expect("torn file must not error the commit path"); - assert_eq!(read, None, "short read maps to None like the boot loader"); + assert_eq!( + read, + Some(OffsetRecord::Torn), + "a short file folds as absent, like the boot loader, but is not silence" + ); let _ = std::fs::remove_dir_all(&dir); } #[compio::test] - async fn read_persisted_offset_real_io_error_propagates() { + async fn read_offset_record_real_io_error_propagates() { // A directory opens read-only but every read fails with EISDIR: a real // I/O error, not a short read. It must surface as Err, never as None // (a blanket None would silently rewind a valid higher offset). let dir = unique_temp_dir(); let path = dir.to_string_lossy().into_owned(); - let result = read_persisted_offset(&path).await; + let result = read_offset_record(&path).await; assert!( matches!(result, Err(IggyError::CannotReadConsumerOffsets(_))), "real I/O error must propagate, got {result:?}", @@ -402,8 +606,48 @@ mod tests { persist_offset_max(&path, 7, false) .await .expect("torn file folds as absent"); - let read = read_persisted_offset(&path).await.expect("repaired file"); - assert_eq!(read, Some(7)); + let read = read_offset_record(&path).await.expect("repaired file"); + assert_eq!( + read, + Some(OffsetRecord::Value { + offset: 7, + checksummed: true + }) + ); + + let _ = std::fs::remove_dir_all(&dir); + } + + /// The crash-loop guard: a fold against a failed-checksum file must repair it, + /// not fail the commit. The caller aborts the shard on a failed commit, and the + /// key is cold on every boot, so the abort would repeat. + #[compio::test] + async fn persist_offset_max_overwrites_a_corrupt_file_instead_of_failing() { + let dir = unique_temp_dir(); + let path = dir.join("42").to_string_lossy().into_owned(); + + persist_offset(&path, 114, false).await.expect("persist"); + let mut bytes = std::fs::read(&path).expect("offset file exists"); + bytes[0] ^= 0x01; + std::fs::write(&path, &bytes).expect("corrupt the file"); + + let folded = persist_offset_max(&path, 7, false) + .await + .expect("a corrupt file must not fail the commit"); + assert_eq!( + folded, 7, + "the untrusted stored value must not win the fold" + ); + + let read = read_offset_record(&path).await.expect("repaired file"); + assert_eq!( + read, + Some(OffsetRecord::Value { + offset: 7, + checksummed: true + }), + "the corrupt file must be repaired in place, not left to trip the next commit" + ); let _ = std::fs::remove_dir_all(&dir); } diff --git a/core/partitions/src/poll_plan.rs b/core/partitions/src/poll_plan.rs index b618cd84ab..928222d124 100644 --- a/core/partitions/src/poll_plan.rs +++ b/core/partitions/src/poll_plan.rs @@ -39,13 +39,13 @@ use iggy_common::{ ConsumerGroupId, ConsumerGroupOffsets, ConsumerKind, ConsumerOffset, ConsumerOffsets, IggyError, }; use server_common::iobuf::{Frozen, Owned}; -use server_common::send_messages2::{COMMAND_HEADER_SIZE, decode_batch_slice}; +use server_common::send_messages2::{BatchIntegrity, COMMAND_HEADER_SIZE, decode_batch_slice_with}; use std::cell::{Cell, RefCell}; use std::hash::Hash; use std::rc::Rc; use std::sync::Arc; use std::sync::atomic::Ordering; -use tracing::warn; +use tracing::{error, warn}; /// Byte cap for materializing a sealed segment's sparse index into its shared /// read-state handle. Index density is one entry per flush: at the default @@ -120,6 +120,9 @@ pub struct DiskReadPlan { pub(crate) segments: Vec, pub(crate) start_position: u64, pub(crate) namespace_raw: u64, + /// Whether to verify each batch's `batch_checksum` against the bytes read. + /// Detection only; a mismatch fails the poll closed and repairs nothing. + pub(crate) validate_checksum: bool, } pub struct DiskSegment { @@ -283,12 +286,18 @@ impl PollPlan { crate::journal::select_resident(&resident_tail.entries, query) .unwrap_or_else(|| (PollFragments::new(), None)) } - // Disk read stopped on an IO fault. Fail-closed: return an empty - // poll WITHOUT the journal-forward fallback. Falling forward - // here would splice the next resident op over the unreadable run - // and silently skip live data; the fault instead surfaces as a - // visibly stuck consumer that recovers on a later poll once the - // segment reads again. + // Disk read stopped on a fault. Fail-closed: return an empty poll + // WITHOUT the journal-forward fallback. Falling forward here would + // splice the next resident op over the unreadable run and silently + // skip live data. + // + // TODO(partitions): the poll reply has no error channel, so this + // reaches the consumer as an ordinary empty poll. Fair for a transient + // IO fault, wrong for a batch that failed its own checksum: data + // damaged at rest never reads again, so the consumer waits forever. + // Surfacing it needs a status on the poll reply, an SDK-visible change + // on every client. Until then the ERROR in `walk_disk_chunk` is the + // only signal, and it is server-side only. DiskReadOutcome::Faulted => (PollFragments::new(), None), // Straddle: continue past the last disk match into the resident // tail (gate + race argument live on `straddle_continuation`). @@ -533,14 +542,27 @@ impl DiskReadPlan { faulted = true; break 'walk; }; - let consumed = walk_disk_chunk( + let ChunkWalk { consumed, corrupt } = walk_disk_chunk( &chunk, query, count, &mut matched, &mut fragments, &mut last_matching_offset, + if self.validate_checksum { + BatchIntegrity::Verify + } else { + BatchIntegrity::LayoutOnly + }, + self.namespace_raw, ); + if corrupt { + // A batch that does not match its own checksum. Fail closed like + // an IO fault: serving it hands a consumer data provably not what + // was written, and skipping ahead punches a silent gap. + faulted = true; + break 'walk; + } if consumed == 0 { if (len as u64) >= persisted - position { // The whole remainder fit yet no complete batch @@ -883,6 +905,7 @@ pub fn upsert_offset_max( /// chunk, pushing matching fragments. Returns bytes consumed: the start /// of the first batch that did not fully fit in the chunk (the caller /// re-reads from there), or the chunk end when everything decoded. +#[allow(clippy::too_many_arguments)] fn walk_disk_chunk( chunk: &Frozen<4096>, query: MessageLookup, @@ -890,15 +913,37 @@ fn walk_disk_chunk( matched: &mut u32, fragments: &mut PollFragments<4096>, last_matching_offset: &mut Option, -) -> usize { + integrity: BatchIntegrity, + namespace_raw: u64, +) -> ChunkWalk { let bytes: &[u8] = chunk; let mut cursor = 0usize; while *matched < count && cursor + COMMAND_HEADER_SIZE <= bytes.len() { - let Ok(batch) = decode_batch_slice(&bytes[cursor..]) else { - // Incomplete tail batch (or corrupt data): hand the position - // back so the caller can re-read or bail. - break; + let batch = match decode_batch_slice_with(&bytes[cursor..], integrity) { + Ok(batch) => batch, + Err(IggyError::InvalidBatchChecksum(found, expected, base_offset)) => { + // Distinguished from the incomplete-tail case below: this batch is + // entirely present and fails its own checksum, so it is damaged at rest. + error!( + target: "iggy.partitions.diag", + plane = "partitions", + namespace_raw, + base_offset, + expected, + found, + position = cursor, + "disk poll: batch checksum mismatch; segment is corrupt at rest" + ); + return ChunkWalk { + consumed: cursor.min(bytes.len()), + corrupt: true, + }; + } + Err(_) => { + // Incomplete tail batch: hand the position back to re-read or bail. + break; + } }; let total_size = batch.header.total_size(); @@ -919,7 +964,17 @@ fn walk_disk_chunk( cursor += total_size; } - cursor.min(bytes.len()) + ChunkWalk { + consumed: cursor.min(bytes.len()), + corrupt: false, + } +} + +/// How far [`walk_disk_chunk`] got, and whether it stopped on corruption rather +/// than on a batch that simply did not fit in the chunk. +struct ChunkWalk { + consumed: usize, + corrupt: bool, } #[cfg(test)] diff --git a/core/partitions/src/state_transfer.rs b/core/partitions/src/state_transfer.rs index 22bb8cd861..fd9e17e6f4 100644 --- a/core/partitions/src/state_transfer.rs +++ b/core/partitions/src/state_transfer.rs @@ -1512,16 +1512,15 @@ where .retain(|start_offset, _| live.contains_key(start_offset)); } - // Second phantom gate, on the BUILT offer rather than on `commit_max`. - // The gate above keys on `commit_max() == 0`, which a replica that lifted - // its commit floor through an offsets-only repair window clears while - // still holding zero bytes; such a replica passes `is_caught_up_primary` - // and would hand a data-holding peer an empty chain at frontier 0, - // making it unlink its own. An offer with no segments AND no offset - // space is indistinguishable from that phantom, and a group genuinely - // in that state has nothing worth transferring anyway. + // An empty chain at frontier 0 tells the receiver to unlink its own, so + // serve it only when a recorded purge says the emptiness is the truth. + // `install_state_transfer`'s `purge_advances` check re-decides that + // against the metadata plane and refuses the rest. let offsets_wire = self.offsets_wire_snapshot(); - if segments.is_empty() && offsets_wire.next_offset == 0 { + if segments.is_empty() + && offsets_wire.next_offset == 0 + && offsets_wire.purge_generation == 0 + { return Err(PartitionTransferUnavailable::NothingCommitted); } let offsets_bytes = Rc::new(offsets_wire.encode()); @@ -1986,7 +1985,15 @@ where // purge disables the `OfferRewindsDurableData` refusal below -- the // one guard standing between an offer that rewinds this replica's // offset space and its durable data. - let purge_advances = offsets_wire.purge_generation > committed_purge_generation; + // Second disjunct: this replica has NOT applied the committed purge, so + // its frontier still measures the pre-purge offset space and cannot be + // compared against a post-purge offer. Restricted to `next_offset == 0` + // -- the state a purge leaves before anything is appended -- so an + // origin that merely lags within the same purge era still fails the + // fence rather than rewinding this replica's durable post-purge data. + let purge_advances = offsets_wire.purge_generation > committed_purge_generation + || (self.applied_purge_generation < committed_purge_generation + && offsets_wire.next_offset == 0); let local_next_offset = self.offset_frontier(); if !purge_advances && local_next_offset > 0 && offsets_wire.next_offset < local_next_offset { @@ -2094,7 +2101,7 @@ where tracing::error!( target: "iggy.partitions.diag", plane = "partitions", - namespace_raw = self.consensus().namespace(), + namespace_raw = self.consensus().group(), frontier = self.offset_frontier(), "state-transfer install could not record the installed offset frontier; \ the durable record stays at the pre-swap claim until the next view change" @@ -2148,7 +2155,7 @@ where // Unlink the old segment chain oldest-first (a crash mid-loop leaves // the NEWEST suffix, which is contiguous) and drop the in-memory // vectors in lockstep, exactly as `purge` does. - let namespace_raw = self.consensus().namespace(); + let namespace_raw = self.consensus().group(); while let Some((_, mut storage)) = self.log.retire_front() { let (messages_path, index_path) = storage.segment_and_index_paths(); let _ = storage.shutdown(); @@ -2324,6 +2331,7 @@ where messages_w.size_counter(), config.enforce_fsync, true, + config.preallocate_segments.then_some(config.segment_size), ) .await .map_err(|source| PartitionInstallError::SegmentOpen { @@ -2373,7 +2381,7 @@ where // keys are minted from u32 wire ids, so assert it. let old_consumer_paths: Vec = { let guard = self.consumer_offsets.pin(); - let paths = guard + let mut paths: Vec = guard .iter() .filter_map(|(key, _)| { let narrowed = u32::try_from(*key).ok(); @@ -2382,11 +2390,20 @@ where }) .collect(); guard.clear(); + // The map is not the whole truth about what is on disk: a repaired + // pre-purge offset op persists a file this incarnation never held, + // and a purged origin offering `next_offset = 0` drops every + // incoming entry, so a map-only sweep leaves the old table for boot + // to resurrect. + paths.extend(strayed_offset_files( + self.consumer_offsets_path.as_deref(), + &offsets_wire.consumers, + )); paths }; let old_group_paths: Vec = { let guard = self.consumer_group_offsets.pin(); - let paths = guard + let mut paths: Vec = guard .iter() .filter_map(|(key, _)| { let narrowed = u32::try_from(key.0).ok(); @@ -2400,6 +2417,10 @@ where }) .collect(); guard.clear(); + paths.extend(strayed_offset_files( + self.consumer_group_offsets_path.as_deref(), + &offsets_wire.groups, + )); paths }; for path in old_consumer_paths.into_iter().chain(old_group_paths) { @@ -2413,7 +2434,7 @@ where tracing::warn!( target: "iggy.partitions.diag", plane = "partitions", - namespace_raw = self.consensus().namespace(), + namespace_raw = self.consensus().group(), path = %path, %error, "failed to unlink a superseded consumer-offset file during install" @@ -2576,7 +2597,7 @@ where tracing::warn!( target: "iggy.partitions.diag", plane = "partitions", - namespace_raw = self.consensus().namespace(), + namespace_raw = self.consensus().group(), purge_generation = offsets_wire.purge_generation, %error, "state-transfer install could not record the offered purge \ @@ -2680,7 +2701,7 @@ where tracing::error!( target: "iggy.partitions.diag", plane = "partitions", - namespace_raw = self.consensus().namespace(), + namespace_raw = self.consensus().group(), partition_dir, %error, "converge sweep cannot list the partition directory" @@ -2694,7 +2715,7 @@ where Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} Err(error) => { warn_unlink( - self.consensus().namespace(), + self.consensus().group(), &path.display().to_string(), &error, ); @@ -2997,6 +3018,35 @@ pub fn offered_purge_generation(offsets_bytes: &[u8]) -> u64 { .unwrap_or_default() } +/// Offset files under `dir` whose id is absent from `incoming`. +/// +/// The install's own map cannot name these: a pre-purge offset op replayed by +/// journal repair persists a file this incarnation never held, and a purged +/// origin offers `next_offset = 0`, which drops every incoming entry. Left +/// behind, boot hydrates them back. +/// +/// A file whose name is not a bare u32 is left alone rather than guessed at: +/// every offset file is named by its id, so anything else is not ours. +pub(crate) fn strayed_offset_files(dir: Option<&str>, incoming: &[(u32, u64)]) -> Vec { + let Some(dir) = dir else { + return Vec::new(); + }; + let Ok(entries) = std::fs::read_dir(dir) else { + return Vec::new(); + }; + entries + .filter_map(Result::ok) + .filter_map(|entry| { + let name = entry.file_name().into_string().ok()?; + let id: u32 = name.parse().ok()?; + incoming + .iter() + .all(|(incoming_id, _)| *incoming_id != id) + .then(|| format!("{dir}/{name}")) + }) + .collect() +} + /// Stamp over every `SEGMENT_LOG` entry of a manifest, keying /// [`ReuseScanMemo`]. Equal digests mean the two offers expect byte-identical /// staged files, so a scan already done for one answers the other; the offsets diff --git a/core/partitions/src/types.rs b/core/partitions/src/types.rs index f052ff75f0..5bc67b8e7d 100644 --- a/core/partitions/src/types.rs +++ b/core/partitions/src/types.rs @@ -269,8 +269,17 @@ pub struct PartitionsConfig { pub size_of_messages_required_to_save: IggyByteSize, /// Whether to enforce fsync after writes. pub enforce_fsync: bool, + /// Whether a disk poll verifies each batch's `batch_checksum` against the bytes + /// it just read. + /// + /// Detection only: a mismatch fails the poll closed and is reported, with no + /// attempt to repair. The alternative is serving bytes provably not the ones + /// written, which reads to a consumer as ordinary data. + pub validate_checksum: bool, /// Maximum size of a single segment before rotation. pub segment_size: IggyByteSize, + /// Whether local message files reserve the configured segment size on open. + pub preallocate_segments: bool, /// Server-side at-rest encryption. Applied ONCE, on the primary at /// ingestion, so the ciphertext replicates verbatim: every replica /// journals, acks, and persists identical bytes (checksums and the diff --git a/core/sdk/Cargo.toml b/core/sdk/Cargo.toml index e5562b27f6..398680d768 100644 --- a/core/sdk/Cargo.toml +++ b/core/sdk/Cargo.toml @@ -29,9 +29,6 @@ documentation = "https://iggy.apache.org/docs" repository = "https://github.com/apache/iggy" readme = "README.md" -[features] -vsr = ["iggy_common/vsr"] - [dependencies] async-broadcast = { workspace = true } async-dropper = { workspace = true } diff --git a/core/sdk/src/clients/client.rs b/core/sdk/src/clients/client.rs index dd7f91fd98..e91a601a5d 100644 --- a/core/sdk/src/clients/client.rs +++ b/core/sdk/src/clients/client.rs @@ -39,8 +39,9 @@ use iggy_common::locking::{IggyRwLock, IggyRwLockFn}; use iggy_common::{BinaryTransport, Client, HttpMethod, SystemClient}; use iggy_common::{ConnectionStringUtils, DiagnosticEvent, Partitioner, TransportProtocol}; use std::fmt::Debug; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use tokio::spawn; +use tokio::task::JoinHandle; use tokio::time::sleep; use tracing::log::warn; use tracing::{debug, error, info}; @@ -64,6 +65,7 @@ pub struct IggyClient { pub(crate) client: IggyRwLock, partitioner: Option>, pub(crate) encryptor: Option>, + heartbeat_handle: Mutex>>, } impl Default for IggyClient { @@ -92,6 +94,7 @@ impl IggyClient { client, partitioner: None, encryptor: None, + heartbeat_handle: Mutex::new(None), } } @@ -131,6 +134,7 @@ impl IggyClient { client, partitioner, encryptor, + heartbeat_handle: Mutex::new(None), } } @@ -236,6 +240,19 @@ impl IggyClient { } } +impl Drop for IggyClient { + fn drop(&mut self) { + let heartbeat_handle = self + .heartbeat_handle + .get_mut() + .unwrap_or_else(|error| error.into_inner()) + .take(); + if let Some(handle) = heartbeat_handle { + handle.abort(); + } + } +} + #[async_trait] impl Client for IggyClient { async fn connect(&self) -> Result<(), IggyError> { @@ -246,8 +263,20 @@ impl Client for IggyClient { heartbeat_interval = client.heartbeat_interval().await; } + let mut heartbeat_handle = self + .heartbeat_handle + .lock() + .unwrap_or_else(|error| error.into_inner()); + if heartbeat_handle + .as_ref() + .is_some_and(|handle| !handle.is_finished()) + { + return Ok(()); + } + + drop(heartbeat_handle.take()); let client = self.client.clone(); - spawn(async move { + *heartbeat_handle = Some(spawn(async move { loop { debug!("Sending the heartbeat..."); if let Err(error) = client.read().await.ping().await { @@ -268,7 +297,7 @@ impl Client for IggyClient { } sleep(heartbeat_interval.get_duration()).await } - }); + })); Ok(()) } diff --git a/core/sdk/src/http/http_client.rs b/core/sdk/src/http/http_client.rs index 5cd4bb4133..f0c7510196 100644 --- a/core/sdk/src/http/http_client.rs +++ b/core/sdk/src/http/http_client.rs @@ -307,7 +307,7 @@ impl HttpClient { /// - Legacy server: one-shot. The presented token is revoked as it is /// consumed, so a concurrent in-flight request still carrying the old /// token may fail with 401. - /// - server-ng: stateless. The old token stays valid until its natural + /// - the server: stateless. The old token stays valid until its natural /// expiry; refreshing never revokes it. pub async fn refresh_access_token(&self) -> Result { // Release the read guard before `set_token_from_identity` takes the diff --git a/core/sdk/src/lib.rs b/core/sdk/src/lib.rs index 50deca0e51..fd8b22efb2 100644 --- a/core/sdk/src/lib.rs +++ b/core/sdk/src/lib.rs @@ -27,11 +27,9 @@ pub mod quic; pub mod session; pub mod stream_builder; pub mod tcp; -#[cfg(feature = "vsr")] mod vsr; pub mod websocket; /// Rust SDK version sent in the login-register version prefix; must be this /// crate's version, see `VsrSessionControl::sdk_version`. -#[cfg(feature = "vsr")] pub(crate) const SDK_VERSION: &str = env!("CARGO_PKG_VERSION"); diff --git a/core/sdk/src/prelude.rs b/core/sdk/src/prelude.rs index ee0dc06d64..51b34e34ab 100644 --- a/core/sdk/src/prelude.rs +++ b/core/sdk/src/prelude.rs @@ -51,18 +51,18 @@ pub use iggy_common::{ Aes256GcmEncryptor, Args, ArgsOptional, AutoLogin, CacheMetrics, CacheMetricsKey, ClientError, ClientInfoDetails, ClusterMetadata, ClusterNode, ClusterNodeRole, ClusterNodeStatus, CompressionAlgorithm, Consumer, ConsumerGroup, ConsumerGroupDetails, ConsumerGroupMember, - ConsumerKind, EncryptorKind, GlobalPermissions, HeaderField, HeaderKey, HeaderKind, - HeaderValue, HttpClientConfig, HttpClientConfigBuilder, HttpMethod, IdKind, Identifier, - IdentityInfo, IggyByteSize, IggyDuration, IggyError, IggyExpiry, IggyIndexView, IggyMessage, - IggyMessageHeader, IggyMessageHeaderView, IggyMessageView, IggyMessageViewIterator, - IggyTimestamp, MaxTopicSize, Partition, Partitioner, Partitioning, Permissions, - PersonalAccessTokenExpiry, PollMessages, PolledMessages, PollingKind, PollingStrategy, - QuicClientConfig, QuicClientConfigBuilder, QuicClientReconnectionConfig, SendMessages, - SendMessagesConfirmationResponse, SendMessagesResponse, Sizeable, SnapshotCompression, Stats, - Stream, StreamDetails, StreamPermissions, SystemSnapshotType, TcpClientConfig, - TcpClientConfigBuilder, TcpClientReconnectionConfig, Topic, TopicDetails, TopicPermissions, - TransportEndpoints, TransportProtocol, UserId, UserInfo, UserInfoDetails, UserStatus, - Validatable, WebSocketClientConfig, WebSocketClientConfigBuilder, + ConsumerKind, Credentials, EncryptorKind, GlobalPermissions, HeaderField, HeaderKey, + HeaderKind, HeaderValue, HttpClientConfig, HttpClientConfigBuilder, HttpMethod, IdKind, + Identifier, IdentityInfo, IggyByteSize, IggyDuration, IggyError, IggyExpiry, IggyIndexView, + IggyMessage, IggyMessageHeader, IggyMessageHeaderView, IggyMessageView, + IggyMessageViewIterator, IggyTimestamp, MaxTopicSize, Partition, Partitioner, Partitioning, + Permissions, PersonalAccessTokenExpiry, PollMessages, PolledMessages, PollingKind, + PollingStrategy, QuicClientConfig, QuicClientConfigBuilder, QuicClientReconnectionConfig, + SendMessages, SendMessagesConfirmationResponse, SendMessagesResponse, Sizeable, + SnapshotCompression, Stats, Stream, StreamDetails, StreamPermissions, SystemSnapshotType, + TcpClientConfig, TcpClientConfigBuilder, TcpClientReconnectionConfig, Topic, TopicDetails, + TopicPermissions, TransportEndpoints, TransportProtocol, UserId, UserInfo, UserInfoDetails, + UserStatus, Validatable, WebSocketClientConfig, WebSocketClientConfigBuilder, WebSocketClientReconnectionConfig, defaults, locking, }; pub use iggy_common::{ diff --git a/core/sdk/src/quic/quic_client.rs b/core/sdk/src/quic/quic_client.rs index 9a731f6082..7bf52ee660 100644 --- a/core/sdk/src/quic/quic_client.rs +++ b/core/sdk/src/quic/quic_client.rs @@ -17,9 +17,7 @@ use crate::leader_aware::{LeaderRedirectionState, check_and_redirect_to_leader}; use crate::prelude::AutoLogin; -#[cfg(feature = "vsr")] use crate::session::ConsensusSession; -#[cfg(feature = "vsr")] use iggy_common::VsrSessionControl as _; use iggy_common::{BinaryClient, BinaryTransport, Client, PersonalAccessTokenClient, UserClient}; @@ -28,7 +26,6 @@ use crate::quic::skip_server_verification::SkipServerVerification; use async_broadcast::{Receiver, Sender, broadcast}; use async_trait::async_trait; use bytes::Bytes; -#[cfg(feature = "vsr")] use iggy_binary_protocol::codes::{LOGIN_REGISTER_CODE, LOGIN_REGISTER_WITH_PAT_CODE}; use iggy_common::{ ClientState, ConnectionString, ConnectionStringUtils, Credentials, DiagnosticEvent, @@ -41,17 +38,12 @@ use secrecy::ExposeSecret; use std::net::{SocketAddr, ToSocketAddrs}; use std::str::FromStr; use std::sync::Arc; -#[cfg(feature = "vsr")] use std::sync::Mutex as StdMutex; use std::time::Duration; use tokio::sync::Mutex; use tokio::time::sleep; use tracing::{error, info, trace, warn}; -#[cfg(not(feature = "vsr"))] -const REQUEST_INITIAL_BYTES_LENGTH: usize = 4; -#[cfg(not(feature = "vsr"))] -const RESPONSE_INITIAL_BYTES_LENGTH: usize = 8; const NAME: &str = "Iggy"; /// Bound on how long a single QUIC request waits for its response, mirroring the @@ -68,7 +60,6 @@ const RESPONSE_READ_TIMEOUT: Duration = Duration::from_secs(30); /// view-change cancel). Unlike a silent timeout, this reply arrives promptly, so /// a short pause keeps the replay from spinning while the primary catches up. /// Bounded overall by `RESPONSE_READ_TIMEOUT`. -#[cfg(feature = "vsr")] const NOT_READY_RETRY_INTERVAL: Duration = Duration::from_millis(50); /// QUIC client for interacting with the Iggy API. @@ -82,13 +73,10 @@ pub struct QuicClient { pub(crate) connected_at: Mutex>, leader_redirection_state: Mutex, pub(crate) current_server_address: Mutex, - #[cfg(feature = "vsr")] // See `core/sdk/src/tcp/tcp_client.rs` for the `tokio::sync::Mutex` -> // `std::sync::Mutex` rationale (pure-CPU critical section). consensus_session: Arc>, - #[cfg(feature = "vsr")] skip_auto_login_once: Mutex, - #[cfg(feature = "vsr")] consumer_group_state: Arc, } @@ -160,7 +148,6 @@ impl BinaryTransport for QuicClient { return Err(IggyError::Disconnected); } - #[cfg(feature = "vsr")] if matches!(self.config.auto_login, AutoLogin::Disabled) && !is_login_register_code(code) { // Without auto-login a reconnect cannot re-establish the session, so // non-login requests are not recovered here - their transient replay @@ -172,9 +159,7 @@ impl BinaryTransport for QuicClient { } self.disconnect().await?; - #[cfg(feature = "vsr")] let skip_auto_login = is_login_register_code(code); - #[cfg(feature = "vsr")] if skip_auto_login { *self.skip_auto_login_once.lock().await = true; } @@ -184,7 +169,6 @@ impl BinaryTransport for QuicClient { server_address, self.config.client_address ); let reconnect = self.connect().await; - #[cfg(feature = "vsr")] if skip_auto_login && reconnect.is_err() { *self.skip_auto_login_once.lock().await = false; } @@ -196,16 +180,13 @@ impl BinaryTransport for QuicClient { self.config.heartbeat_interval } - #[cfg(feature = "vsr")] fn consumer_group_state(&self) -> Arc { Arc::clone(&self.consumer_group_state) } } -#[cfg(feature = "vsr")] impl iggy_common::VsrSessionSealed for QuicClient {} -#[cfg(feature = "vsr")] #[async_trait::async_trait] impl iggy_common::VsrSessionControl for QuicClient { async fn bind_vsr_session(&self, session: u64) -> Result<(), IggyError> { @@ -302,11 +283,8 @@ impl QuicClient { connected_at: Mutex::new(None), leader_redirection_state: Mutex::new(LeaderRedirectionState::new()), current_server_address: Mutex::new(server_address), - #[cfg(feature = "vsr")] consensus_session: Arc::new(StdMutex::new(ConsensusSession::new())), - #[cfg(feature = "vsr")] skip_auto_login_once: Mutex::new(false), - #[cfg(feature = "vsr")] consumer_group_state: Arc::new(iggy_common::ConsumerGroupClientState::new()), }) } @@ -341,43 +319,7 @@ impl QuicClient { return Err(IggyError::EmptyResponse); } - #[cfg(feature = "vsr")] - { - crate::vsr::decode_response(Bytes::from(buffer)) - } - - #[cfg(not(feature = "vsr"))] - { - let status = u32::from_le_bytes( - buffer[..4] - .try_into() - .map_err(|_| IggyError::InvalidNumberEncoding)?, - ); - if status != 0 { - error!( - "Received an invalid response with status: {} ({}).", - status, - IggyError::from_code_as_string(status) - ); - - return Err(IggyError::from_code(status)); - } - - let length = u32::from_le_bytes( - buffer[4..RESPONSE_INITIAL_BYTES_LENGTH] - .try_into() - .map_err(|_| IggyError::InvalidNumberEncoding)?, - ); - trace!("Status: OK. Response length: {}", length); - if length <= 1 { - return Ok(Bytes::new()); - } - - Ok(Bytes::copy_from_slice( - &buffer[RESPONSE_INITIAL_BYTES_LENGTH - ..RESPONSE_INITIAL_BYTES_LENGTH + length as usize], - )) - } + crate::vsr::decode_response(Bytes::from(buffer)) } async fn connect(&self) -> Result<(), IggyError> { @@ -492,7 +434,6 @@ impl QuicClient { self.connected_at.lock().await.replace(now); self.publish_event(DiagnosticEvent::Connected).await; - #[cfg(feature = "vsr")] let skip_auto_login = { let mut guard = self.skip_auto_login_once.lock().await; std::mem::take(&mut *guard) @@ -505,20 +446,11 @@ impl QuicClient { // Leadership still matters without auto-login: the caller // signs in manually, and a login against a non-leader // replays for its whole read timeout. `GetClusterMetadata` - // is sessionless and pre-auth on server-ng, so the check - // works on the unauthenticated connection. vsr-only: the - // legacy server auth-gates cluster metadata, so this check - // would bounce `Unauthenticated` into the reconnect path - // and recurse back into `connect`. - #[cfg(feature = "vsr")] - { - self.handle_leader_redirection().await? - } - #[cfg(not(feature = "vsr"))] - false + // is sessionless and pre-auth, so the check works on the + // unauthenticated connection. + self.handle_leader_redirection().await? } AutoLogin::Enabled(credentials) => { - #[cfg(feature = "vsr")] if skip_auto_login { info!("Skipping automatic sign-in for a retried login/register request."); false @@ -557,35 +489,6 @@ impl QuicClient { } } - self.handle_leader_redirection().await? - } - #[cfg(not(feature = "vsr"))] - { - info!( - "{NAME} client: {} is signing in...", - self.config.client_address - ); - self.set_state(ClientState::Authenticating).await; - match credentials { - Credentials::UsernamePassword(username, password) => { - self.login_user(username, password.expose_secret()).await?; - self.publish_event(DiagnosticEvent::SignedIn).await; - info!( - "{NAME} client: {} has signed in with the user credentials, username: {username}", - self.config.client_address - ); - } - Credentials::PersonalAccessToken(token) => { - self.login_with_personal_access_token(token.expose_secret()) - .await?; - self.publish_event(DiagnosticEvent::SignedIn).await; - info!( - "{NAME} client: {} has signed in with a personal access token.", - self.config.client_address - ); - } - } - self.handle_leader_redirection().await? } } @@ -648,7 +551,6 @@ impl QuicClient { } self.endpoint.wait_idle().await; - #[cfg(feature = "vsr")] self.reset_vsr_session().await?; self.set_state(ClientState::Shutdown).await; self.publish_event(DiagnosticEvent::Shutdown).await; @@ -668,7 +570,6 @@ impl QuicClient { self.set_state(ClientState::Disconnected).await; self.connection.lock().await.take(); self.endpoint.wait_idle().await; - #[cfg(feature = "vsr")] self.reset_vsr_session().await?; self.publish_event(DiagnosticEvent::Disconnected).await; let now = IggyTimestamp::now(); @@ -704,7 +605,6 @@ impl QuicClient { let connection = self.connection.clone(); let response_buffer_size = self.config.response_buffer_size; - #[cfg(feature = "vsr")] let consensus_session = self.consensus_session.clone(); // SAFETY: we run code holding the `connection` lock in a task so we can't be cancelled while holding the lock. tokio::spawn(async move { @@ -714,9 +614,7 @@ impl QuicClient { return Err(IggyError::NotConnected); }; - #[cfg(feature = "vsr")] - { - let (request_header, request_size) = { + let (request_header, request_size) = { let mut consensus_session = consensus_session .lock() .expect("consensus session mutex poisoned"); @@ -794,42 +692,6 @@ impl QuicClient { Err(error) => return Err(error), } } - } - - #[cfg(not(feature = "vsr"))] - { - let payload_length = payload.len() + REQUEST_INITIAL_BYTES_LENGTH; - let (mut send, mut recv) = connection.open_bi().await.map_err(|error| { - error!("Failed to open a bidirectional stream: {error}"); - IggyError::QuicError - })?; - trace!("Sending a QUIC request with code: {code}"); - send.write_all(&(payload_length as u32).to_le_bytes()) - .await - .map_err(|error| { - error!("Failed to write payload length: {error}"); - IggyError::QuicError - })?; - send.write_all(&code.to_le_bytes()).await.map_err(|error| { - error!("Failed to write payload code: {error}"); - IggyError::QuicError - })?; - send.write_all(&payload).await.map_err(|error| { - error!("Failed to write payload: {error}"); - IggyError::QuicError - })?; - send.finish().map_err(|error| { - error!("Failed to finish sending data: {error}"); - IggyError::QuicError - })?; - trace!("Sent a QUIC request with code: {code}, waiting for a response..."); - QuicClient::handle_response( - &mut recv, - response_buffer_size as usize, - RESPONSE_READ_TIMEOUT, - ) - .await - } }) .await .map_err(|e| { @@ -839,7 +701,6 @@ impl QuicClient { } } -#[cfg(feature = "vsr")] const fn is_login_register_code(code: u32) -> bool { matches!(code, LOGIN_REGISTER_CODE | LOGIN_REGISTER_WITH_PAT_CODE) } diff --git a/core/sdk/src/session.rs b/core/sdk/src/session.rs index ac56c0b337..a26e101e00 100644 --- a/core/sdk/src/session.rs +++ b/core/sdk/src/session.rs @@ -24,7 +24,7 @@ //! The SDK tracks the `(client_id, session)` pair and a monotonically //! increasing `request_id` counter. These values populate the consensus //! headers (`RequestHeader.client`, `.session`, `.request`) when the -//! transport sends requests through server-ng. +//! transport sends requests through the server. //! //! ## Lifecycle //! diff --git a/core/sdk/src/tcp/tcp_client.rs b/core/sdk/src/tcp/tcp_client.rs index f827b9f55b..d68e3d698e 100644 --- a/core/sdk/src/tcp/tcp_client.rs +++ b/core/sdk/src/tcp/tcp_client.rs @@ -18,21 +18,14 @@ use crate::leader_aware::{LeaderRedirectionState, check_and_redirect_to_leader}; use crate::prelude::Client; use crate::prelude::TcpClientConfig; -#[cfg(feature = "vsr")] use crate::session::ConsensusSession; use crate::tcp::tcp_connection_stream::TcpConnectionStream; use crate::tcp::tcp_connection_stream_kind::ConnectionStreamKind; use crate::tcp::tcp_tls_connection_stream::TcpTlsConnectionStream; use async_broadcast::{Receiver, Sender, broadcast}; use async_trait::async_trait; -#[cfg(not(feature = "vsr"))] -use bytes::BufMut; use bytes::{Bytes, BytesMut}; -#[cfg(feature = "vsr")] use iggy_binary_protocol::codes::{LOGIN_REGISTER_CODE, LOGIN_REGISTER_WITH_PAT_CODE}; -#[cfg(not(feature = "vsr"))] -use iggy_common::IggyErrorDiscriminants; -#[cfg(feature = "vsr")] use iggy_common::VsrSessionControl as _; use iggy_common::{ AutoLogin, ClientState, ConnectionString, ConnectionStringUtils, Credentials, DiagnosticEvent, @@ -44,7 +37,6 @@ use secrecy::ExposeSecret; use std::net::SocketAddr; use std::str::FromStr; use std::sync::Arc; -#[cfg(feature = "vsr")] use std::sync::Mutex as StdMutex; use tokio::net::TcpStream; use tokio::sync::Mutex; @@ -52,16 +44,11 @@ use tokio::time::sleep; use tokio_rustls::{TlsConnector, TlsStream}; use tracing::{error, info, trace, warn}; -#[cfg(not(feature = "vsr"))] -const REQUEST_INITIAL_BYTES_LENGTH: usize = 4; -#[cfg(not(feature = "vsr"))] -const RESPONSE_INITIAL_BYTES_LENGTH: usize = 8; const NAME: &str = "Iggy"; /// Upper bound for awaiting a reply on the lockstep VSR connection. Far /// beyond any healthy round-trip; only trips when the server loses the /// reply entirely (e.g. stalled replication quorum), which would otherwise /// hold the stream lock forever and wedge the client. -#[cfg(feature = "vsr")] const RESPONSE_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); /// Backoff before replaying a request the server answered with an explicit @@ -69,7 +56,6 @@ const RESPONSE_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_sec /// view-change cancel). The reply arrives promptly, so a short pause keeps the /// replay from spinning while the primary catches up. Bounded by /// `RESPONSE_READ_TIMEOUT`. -#[cfg(feature = "vsr")] const NOT_READY_RETRY_INTERVAL: std::time::Duration = std::time::Duration::from_millis(50); /// How long a request replays `TransientNotCommitted` on the SAME connection @@ -78,7 +64,6 @@ const NOT_READY_RETRY_INTERVAL: std::time::Duration = std::time::Duration::from_ /// forever, so replaying alone never recovers; periodically consult the /// roster and fail over to the leader. Bounded by `RESPONSE_READ_TIMEOUT` /// overall. -#[cfg(feature = "vsr")] const TRANSIENT_FAILOVER_CHECK_INTERVAL: std::time::Duration = std::time::Duration::from_secs(2); /// TCP client for interacting with the Iggy API. @@ -93,15 +78,12 @@ pub struct TcpClient { pub(crate) connected_at: Mutex>, leader_redirection_state: Mutex, pub(crate) current_server_address: Mutex, - #[cfg(feature = "vsr")] // `std::sync::Mutex` (not `tokio::sync::Mutex`): the critical section // is `encode_request_header`, which is pure CPU and never awaits. The // tokio variant would pay a waker alloc + internal semaphore on // contention with zero correctness benefit. consensus_session: Arc>, - #[cfg(feature = "vsr")] skip_auto_login_once: Mutex, - #[cfg(feature = "vsr")] consumer_group_state: Arc, } @@ -171,21 +153,18 @@ impl BinaryTransport for TcpClient { return Err(IggyError::Disconnected); } - #[cfg(feature = "vsr")] if matches!(self.config.auto_login, AutoLogin::Disabled) && !is_login_register_code(code) { // Without auto-login a reconnect cannot re-establish the session, // so non-login requests fail fast. Login/register itself is the // exception: the server stays deliberately silent on transient - // register failures (server-ng `surface_login_failure`) and + // register failures (the server `surface_login_failure`) and // relies on the client timing out and replaying the request. return Err(error); } self.disconnect().await?; - #[cfg(feature = "vsr")] let skip_auto_login = is_login_register_code(code); - #[cfg(feature = "vsr")] if skip_auto_login { *self.skip_auto_login_once.lock().await = true; } @@ -200,7 +179,6 @@ impl BinaryTransport for TcpClient { } let reconnect = self.connect().await; - #[cfg(feature = "vsr")] if skip_auto_login && reconnect.is_err() { *self.skip_auto_login_once.lock().await = false; } @@ -212,16 +190,13 @@ impl BinaryTransport for TcpClient { self.config.heartbeat_interval } - #[cfg(feature = "vsr")] fn consumer_group_state(&self) -> Arc { Arc::clone(&self.consumer_group_state) } } -#[cfg(feature = "vsr")] impl iggy_common::VsrSessionSealed for TcpClient {} -#[cfg(feature = "vsr")] #[async_trait::async_trait] impl iggy_common::VsrSessionControl for TcpClient { async fn bind_vsr_session(&self, session: u64) -> Result<(), IggyError> { @@ -311,56 +286,12 @@ impl TcpClient { connected_at: Mutex::new(None), leader_redirection_state: Mutex::new(LeaderRedirectionState::new()), current_server_address: Mutex::new(server_address), - #[cfg(feature = "vsr")] consensus_session: Arc::new(StdMutex::new(ConsensusSession::new())), - #[cfg(feature = "vsr")] skip_auto_login_once: Mutex::new(false), - #[cfg(feature = "vsr")] consumer_group_state: Arc::new(iggy_common::ConsumerGroupClientState::new()), }) } - #[cfg(not(feature = "vsr"))] - async fn handle_response( - status: u32, - length: u32, - stream: &mut ConnectionStreamKind, - ) -> Result { - if status != 0 { - // TEMP: See https://github.com/apache/iggy/pull/604 for context. - if status == IggyErrorDiscriminants::TopicNameAlreadyExists as u32 - || status == IggyErrorDiscriminants::StreamNameAlreadyExists as u32 - || status == IggyErrorDiscriminants::UserAlreadyExists as u32 - || status == IggyErrorDiscriminants::PersonalAccessTokenAlreadyExists as u32 - || status == IggyErrorDiscriminants::ConsumerGroupNameAlreadyExists as u32 - { - tracing::debug!( - "Received a server resource already exists response: {} ({})", - status, - IggyError::from_code_as_string(status) - ) - } else { - error!( - "Received an invalid response with status: {} ({}).", - status, - IggyError::from_code_as_string(status), - ); - } - - return Err(IggyError::from_code(status)); - } - - trace!("Status: OK. Response length: {}", length); - if length <= 1 { - return Ok(Bytes::new()); - } - - let mut response_buffer = BytesMut::with_capacity(length as usize); - response_buffer.put_bytes(0, length as usize); - stream.read(&mut response_buffer).await?; - Ok(response_buffer.freeze()) - } - async fn connect(&self) -> Result<(), IggyError> { loop { match self.get_state().await { @@ -542,7 +473,6 @@ impl TcpClient { self.set_state(ClientState::Connected).await; self.connected_at.lock().await.replace(now); self.publish_event(DiagnosticEvent::Connected).await; - #[cfg(feature = "vsr")] let skip_auto_login = { let mut guard = self.skip_auto_login_once.lock().await; std::mem::take(&mut *guard) @@ -555,20 +485,11 @@ impl TcpClient { // Leadership still matters without auto-login: the caller // signs in manually, and a login against a non-leader // replays for its whole read timeout. `GetClusterMetadata` - // is sessionless and pre-auth on server-ng, so the check - // works on the unauthenticated connection. vsr-only: the - // legacy server auth-gates cluster metadata, so this check - // would bounce `Unauthenticated` into the reconnect path - // and recurse back into `connect`. - #[cfg(feature = "vsr")] - { - self.handle_leader_redirection().await? - } - #[cfg(not(feature = "vsr"))] - false + // is sessionless and pre-auth, so the check works on the + // unauthenticated connection. + self.handle_leader_redirection().await? } AutoLogin::Enabled(credentials) => { - #[cfg(feature = "vsr")] if skip_auto_login { info!("Skipping automatic sign-in for a retried login/register request."); false @@ -600,28 +521,6 @@ impl TcpClient { } } - self.handle_leader_redirection().await? - } - #[cfg(not(feature = "vsr"))] - { - info!("{NAME} client: {client_address} is signing in..."); - self.set_state(ClientState::Authenticating).await; - match credentials { - Credentials::UsernamePassword(username, password) => { - self.login_user(username, password.expose_secret()).await?; - info!( - "{NAME} client: {client_address} has signed in with the user credentials, username: {username}", - ); - } - Credentials::PersonalAccessToken(token) => { - self.login_with_personal_access_token(token.expose_secret()) - .await?; - info!( - "{NAME} client: {client_address} has signed in with a personal access token.", - ); - } - } - self.handle_leader_redirection().await? } } @@ -681,7 +580,6 @@ impl TcpClient { info!("{NAME} client: {client_address} is disconnecting from server..."); self.set_state(ClientState::Disconnected).await; self.stream.lock().await.take(); - #[cfg(feature = "vsr")] self.reset_vsr_session().await?; self.publish_event(DiagnosticEvent::Disconnected).await; let now = IggyTimestamp::now(); @@ -700,7 +598,6 @@ impl TcpClient { if let Some(mut stream) = stream { stream.shutdown().await?; } - #[cfg(feature = "vsr")] self.reset_vsr_session().await?; self.set_state(ClientState::Shutdown).await; self.publish_event(DiagnosticEvent::Shutdown).await; @@ -725,126 +622,62 @@ impl TcpClient { _ => {} } - #[cfg(feature = "vsr")] - { - // One overall deadline bounds the request across transient replays - // AND leader failovers, matching the previous single-connection - // budget. Login/register replays stay on this connection for the - // whole budget: the connect flow owns leader redirection for the - // sign-in handshake, and reconnecting from underneath it would - // recurse. - let overall_deadline = tokio::time::Instant::now() + RESPONSE_READ_TIMEOUT; - let mut preencoded = None; - loop { - let transient_deadline = if is_login_register_code(code) { - overall_deadline - } else { - overall_deadline - .min(tokio::time::Instant::now() + TRANSIENT_FAILOVER_CHECK_INTERVAL) - }; - let (header, result) = self - .send_raw_vsr_attempt( - code, - payload.clone(), - preencoded, - transient_deadline, - overall_deadline, - ) - .await; - match result { - Err(IggyError::TransientNotAccepted) - if tokio::time::Instant::now() < overall_deadline - && !is_login_register_code(code) => - { - // The server explicitly did NOT admit the request, so - // re-issuing it -- same id on this session, or a fresh - // id under a new session after a failover -- cannot - // double-apply. Keep the encoded id for same-session - // replays; a redirect re-registers, so the id is - // re-encoded under the new session. - // (`TransientNotCommitted` never reaches this branch: - // its outcome is unknown, so the attempt loop replays - // it same-session for the whole budget and then the - // error propagates to the caller.) - preencoded = header; - if let Ok(true) = self.handle_leader_redirection().await { - self.connect().await?; - preencoded = None; - } - } - Err(IggyError::Disconnected) => { - // Reply stream state is unknown (timed out or torn - // mid-frame); a late reply would desync framing for the - // next request, so drop the connection and let callers - // reconnect. - self.stream.lock().await.take(); - self.set_state(ClientState::Disconnected).await; - return Err(IggyError::Disconnected); + // One overall deadline bounds the request across transient replays + // AND leader failovers, matching the previous single-connection + // budget. Login/register replays stay on this connection for the + // whole budget: the connect flow owns leader redirection for the + // sign-in handshake, and reconnecting from underneath it would + // recurse. + let overall_deadline = tokio::time::Instant::now() + RESPONSE_READ_TIMEOUT; + let mut preencoded = None; + loop { + let transient_deadline = if is_login_register_code(code) { + overall_deadline + } else { + overall_deadline + .min(tokio::time::Instant::now() + TRANSIENT_FAILOVER_CHECK_INTERVAL) + }; + let (header, result) = self + .send_raw_vsr_attempt( + code, + payload.clone(), + preencoded, + transient_deadline, + overall_deadline, + ) + .await; + match result { + Err(IggyError::TransientNotAccepted) + if tokio::time::Instant::now() < overall_deadline + && !is_login_register_code(code) => + { + // The server explicitly did NOT admit the request, so + // re-issuing it -- same id on this session, or a fresh + // id under a new session after a failover -- cannot + // double-apply. Keep the encoded id for same-session + // replays; a redirect re-registers, so the id is + // re-encoded under the new session. + // (`TransientNotCommitted` never reaches this branch: + // its outcome is unknown, so the attempt loop replays + // it same-session for the whole budget and then the + // error propagates to the caller.) + preencoded = header; + if let Ok(true) = self.handle_leader_redirection().await { + self.connect().await?; + preencoded = None; } - other => return other, } - } - } - - #[cfg(not(feature = "vsr"))] - { - let stream = self.stream.clone(); - // SAFETY: we run code holding the `stream` lock in a task so we can't be cancelled while holding the lock. - let result = tokio::spawn(async move { - let mut stream = stream.lock().await; - if let Some(stream) = stream.as_mut() { - let payload_length = payload.len() + REQUEST_INITIAL_BYTES_LENGTH; - trace!("Sending a TCP request of size {payload_length} with code: {code}"); - stream.write(&(payload_length as u32).to_le_bytes()).await?; - stream.write(&code.to_le_bytes()).await?; - stream.write(&payload).await?; - stream.flush().await?; - trace!("Sent a TCP request with code: {code}, waiting for a response..."); - let mut response_buffer = [0u8; RESPONSE_INITIAL_BYTES_LENGTH]; - let read_bytes = stream.read(&mut response_buffer).await.map_err(|error| { - error!( - "Failed to read response for TCP request with code: {code}: {error}", - code = code, - error = error - ); - IggyError::Disconnected - })?; - - if read_bytes != RESPONSE_INITIAL_BYTES_LENGTH { - error!("Received an invalid or empty response."); - return Err(IggyError::EmptyResponse); - } - - let status = u32::from_le_bytes( - response_buffer[..4] - .try_into() - .map_err(|_| IggyError::InvalidNumberEncoding)?, - ); - let length = u32::from_le_bytes( - response_buffer[4..] - .try_into() - .map_err(|_| IggyError::InvalidNumberEncoding)?, - ); - return TcpClient::handle_response(status, length, stream).await; + Err(IggyError::Disconnected) => { + // Reply stream state is unknown (timed out or torn + // mid-frame); a late reply would desync framing for the + // next request, so drop the connection and let callers + // reconnect. + self.stream.lock().await.take(); + self.set_state(ClientState::Disconnected).await; + return Err(IggyError::Disconnected); } - - error!("Cannot send data. Client is not connected."); - Err(IggyError::NotConnected) - }) - .await - .map_err(|e| { - error!("Task execution failed during TCP request: {}", e); - IggyError::TcpError - })?; - - if matches!(result, Err(IggyError::Disconnected)) { - // Reply stream state is unknown (timed out or torn mid-frame); - // a late reply would desync framing for the next request, so - // drop the connection and let callers reconnect. - self.stream.lock().await.take(); - self.set_state(ClientState::Disconnected).await; + other => return other, } - result } } @@ -855,7 +688,6 @@ impl TcpClient { /// full request budget -- so a short transient window cannot tear down a /// connection that is merely slow to reply. Returns the header used so the /// caller can replay the same id on a later attempt. - #[cfg(feature = "vsr")] async fn send_raw_vsr_attempt( &self, code: u32, @@ -1026,7 +858,6 @@ impl TcpClient { } } -#[cfg(feature = "vsr")] const fn is_login_register_code(code: u32) -> bool { matches!(code, LOGIN_REGISTER_CODE | LOGIN_REGISTER_WITH_PAT_CODE) } @@ -1037,26 +868,6 @@ const fn is_login_register_code(code: u32) -> bool { #[cfg(test)] mod tests { use super::*; - #[cfg(not(feature = "vsr"))] - use tokio::io::AsyncWriteExt; - #[cfg(not(feature = "vsr"))] - use tokio::net::TcpListener; - - #[cfg(not(feature = "vsr"))] - async fn make_dummy_stream(data: &[u8]) -> ConnectionStreamKind { - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = listener.local_addr().unwrap(); - - let data = data.to_vec(); - tokio::spawn(async move { - let (mut server_side, _) = listener.accept().await.unwrap(); - server_side.write_all(&data).await.unwrap(); - }); - - let client = tokio::net::TcpStream::connect(addr).await.unwrap(); - let client_addr = client.local_addr().unwrap(); - ConnectionStreamKind::Tcp(TcpConnectionStream::new(client_addr, client)) - } #[test] fn should_fail_with_empty_connection_string() { @@ -1325,44 +1136,4 @@ mod tests { IggyDuration::from_str("5s").unwrap() ); } - - #[cfg(not(feature = "vsr"))] - #[tokio::test] - async fn should_return_error_when_status_is_non_zero() { - let mut stream = make_dummy_stream(&[1u8; 10]).await; - let tcp_client = TcpClient::handle_response(1, 0, &mut stream).await; - assert!(tcp_client.is_err()); - } - - #[cfg(not(feature = "vsr"))] - #[tokio::test] - async fn should_return_ok_when_status_is_zero() { - let mut stream = make_dummy_stream(&[1u8; 10]).await; - let tcp_client = TcpClient::handle_response(0, 0, &mut stream).await; - assert!(tcp_client.is_ok()); - } - - #[cfg(not(feature = "vsr"))] - #[tokio::test] - async fn should_return_ok_when_length_is_less_than_data() { - let mut stream = make_dummy_stream(&[1u8; 10]).await; - let tcp_client = TcpClient::handle_response(0, 5, &mut stream).await; - assert!(tcp_client.is_ok()); - } - - #[cfg(not(feature = "vsr"))] - #[tokio::test] - async fn should_return_ok_when_length_is_equal_to_one() { - let mut stream = make_dummy_stream(&[1u8; 10]).await; - let tcp_client = TcpClient::handle_response(0, 1, &mut stream).await; - assert_eq!(tcp_client.unwrap(), Bytes::new()); - } - - #[cfg(not(feature = "vsr"))] - #[tokio::test] - async fn should_return_err_when_length_exceeds_data() { - let mut stream = make_dummy_stream(&[1u8; 10]).await; - let tcp_client = TcpClient::handle_response(0, 50, &mut stream).await; - assert!(tcp_client.is_err()); - } } diff --git a/core/sdk/src/tcp/tcp_connection_stream.rs b/core/sdk/src/tcp/tcp_connection_stream.rs index 299e0fb289..1d87d38161 100644 --- a/core/sdk/src/tcp/tcp_connection_stream.rs +++ b/core/sdk/src/tcp/tcp_connection_stream.rs @@ -17,7 +17,6 @@ use crate::tcp::tcp_stream::ConnectionStream; use async_trait::async_trait; -#[cfg(feature = "vsr")] use bytes::BytesMut; use iggy_common::IggyError; use std::net::SocketAddr; @@ -43,7 +42,6 @@ impl TcpConnectionStream { } } - #[cfg(feature = "vsr")] pub async fn read_buf(&mut self, buf: &mut BytesMut, len: usize) -> Result<(), IggyError> { let target_len = buf.len() + len; while buf.len() < target_len { diff --git a/core/sdk/src/tcp/tcp_connection_stream_kind.rs b/core/sdk/src/tcp/tcp_connection_stream_kind.rs index 2617ea5987..ce51a7d6c6 100644 --- a/core/sdk/src/tcp/tcp_connection_stream_kind.rs +++ b/core/sdk/src/tcp/tcp_connection_stream_kind.rs @@ -18,7 +18,6 @@ use crate::tcp::tcp_connection_stream::TcpConnectionStream; use crate::tcp::tcp_stream::ConnectionStream; use crate::tcp::tcp_tls_connection_stream::TcpTlsConnectionStream; -#[cfg(feature = "vsr")] use bytes::BytesMut; use iggy_common::IggyError; @@ -37,7 +36,6 @@ impl ConnectionStreamKind { } } - #[cfg(feature = "vsr")] pub async fn read_buf(&mut self, buf: &mut BytesMut, len: usize) -> Result<(), IggyError> { match self { Self::Tcp(c) => c.read_buf(buf, len).await, diff --git a/core/sdk/src/tcp/tcp_tls_connection_stream.rs b/core/sdk/src/tcp/tcp_tls_connection_stream.rs index b01ddf1df6..a49a2a6ec1 100644 --- a/core/sdk/src/tcp/tcp_tls_connection_stream.rs +++ b/core/sdk/src/tcp/tcp_tls_connection_stream.rs @@ -17,7 +17,6 @@ use crate::tcp::tcp_stream::ConnectionStream; use async_trait::async_trait; -#[cfg(feature = "vsr")] use bytes::BytesMut; use iggy_common::IggyError; use std::net::SocketAddr; @@ -40,7 +39,6 @@ impl TcpTlsConnectionStream { } } - #[cfg(feature = "vsr")] pub async fn read_buf(&mut self, buf: &mut BytesMut, len: usize) -> Result<(), IggyError> { let target_len = buf.len() + len; while buf.len() < target_len { diff --git a/core/sdk/src/vsr.rs b/core/sdk/src/vsr.rs index 48fd28e314..e0c0efcbdc 100644 --- a/core/sdk/src/vsr.rs +++ b/core/sdk/src/vsr.rs @@ -17,28 +17,14 @@ use crate::session::ConsensusSession; use bytes::{BufMut, Bytes, BytesMut}; -use iggy_binary_protocol::codec::WireDecode; use iggy_binary_protocol::codes::{ - DELETE_CONSUMER_OFFSET_2_CODE, DELETE_CONSUMER_OFFSET_CODE, DELETE_SEGMENTS_CODE, - LOGIN_REGISTER_CODE, LOGIN_REGISTER_WITH_PAT_CODE, LOGOUT_USER_CODE, SEND_MESSAGES_CODE, - STORE_CONSUMER_OFFSET_2_CODE, STORE_CONSUMER_OFFSET_CODE, + LOGIN_REGISTER_CODE, LOGIN_REGISTER_WITH_PAT_CODE, LOGOUT_USER_CODE, }; use iggy_binary_protocol::consensus::{ Command2, EvictionHeader, EvictionReason, GenericHeader, HEADER_SIZE, Operation, ReplyHeader, RequestHeader, read_size_field, result_code, result_section_len, }; -use iggy_binary_protocol::namespace::{ - MAX_PARTITIONS, MAX_STREAMS, MAX_TOPICS, METADATA_CONSENSUS_NAMESPACE, PARTITION_MASK, - PARTITION_SHIFT, STREAM_MASK, STREAM_SHIFT, TOPIC_MASK, TOPIC_SHIFT, -}; -use iggy_binary_protocol::requests::consumer_offsets::{ - DeleteConsumerOffset2Request, DeleteConsumerOffsetRequest, StoreConsumerOffset2Request, - StoreConsumerOffsetRequest, -}; -use iggy_binary_protocol::requests::messages::SendMessagesHeader; -use iggy_binary_protocol::requests::segments::DeleteSegmentsRequest; -use iggy_binary_protocol::{WireIdentifier, WirePartitioning}; -use iggy_common::{IggyError, eviction_reason_to_error}; +use iggy_common::{IggyError, calculate_checksum, eviction_reason_to_error}; const NON_REPLICATED_CODE_RANGE: std::ops::Range = 0..4; @@ -125,12 +111,20 @@ pub(crate) fn encode_request_header( } } }; - let namespace = namespace_for_request(code, payload, operation)?; + // Stamped only for ops the server's `ClientTable` dedups. Partition ops are + // at-least-once with no reply cache to poison, and theirs are the large payloads, + // already covered client-side by `batch_checksum` over the same bytes. + // NonReplicated ops bypass dedup too. + let request_checksum = if operation.is_partition() || operation == Operation::NonReplicated { + 0 + } else { + u128::from(calculate_checksum(payload)) + }; let total_size = HEADER_SIZE .checked_add(payload.len()) .ok_or(IggyError::InvalidConfiguration)?; let size = u32::try_from(total_size).map_err(|_| IggyError::InvalidConfiguration)?; - let mut reserved = [0; 52]; + let mut reserved = [0; 60]; if operation == Operation::NonReplicated { reserved[NON_REPLICATED_CODE_RANGE].copy_from_slice(&code.to_le_bytes()); } @@ -141,7 +135,11 @@ pub(crate) fn encode_request_header( client: session.client_id(), request: request_id, session: session_id, - namespace, + // Lets the client table tell a genuine retry from a `request` number reused + // for different arguments. Zero means unstamped, which is what an SDK + // predating this sends. A server that rewrites the body (PAT, password) + // carries it through untouched, so it keeps describing what the client sent. + request_checksum, // Zeroed: the field is "informational" -- the server copies it into // `ReplyHeader.timestamp` for RTT but nothing else reads it. Paying // a `clock_gettime` syscall per encoded request (formerly held the @@ -345,131 +343,11 @@ fn read_window_field(header_bytes: &[u8; HEADER_SIZE], offset: usize) -> u32 { u32::from_le_bytes(value) } -fn namespace_for_request( - code: u32, - payload: &Bytes, - operation: Operation, -) -> Result { - // Control-plane requests target the metadata replica (shard 0). The - // router's `route_typed` only short-circuits to shard 0 when the - // namespace value equals `METADATA_CONSENSUS_NAMESPACE`; sending plain - // `0` falls into `route_consensus_control` which hashes the namespace - // and lands a Register on a peer shard whose `submit_register_in_process` - // panics ("consensus only exists on shard 0"). - if operation == Operation::Register || operation == Operation::Logout { - return Ok(METADATA_CONSENSUS_NAMESPACE); - } - if operation == Operation::NonReplicated || operation.is_metadata() { - return Ok(0); - } - - let namespace = match code { - SEND_MESSAGES_CODE => { - if payload.len() < 4 { - return Err(IggyError::InvalidCommand); - } - let metadata_length = u32::from_le_bytes( - payload[..4] - .try_into() - .map_err(|_| IggyError::InvalidNumberEncoding)?, - ) as usize; - if payload.len() < 4 + metadata_length { - return Err(IggyError::InvalidCommand); - } - let header = SendMessagesHeader::decode_from(&payload[4..4 + metadata_length]) - .map_err(|_| IggyError::InvalidCommand)?; - namespace_from_partitioning(&header.stream_id, &header.topic_id, &header.partitioning)? - } - STORE_CONSUMER_OFFSET_CODE => { - let request = StoreConsumerOffsetRequest::decode_from(payload) - .map_err(|_| IggyError::InvalidCommand)?; - namespace_from_partition(&request.stream_id, &request.topic_id, request.partition_id)? - } - DELETE_CONSUMER_OFFSET_CODE => { - let request = DeleteConsumerOffsetRequest::decode_from(payload) - .map_err(|_| IggyError::InvalidCommand)?; - namespace_from_partition(&request.stream_id, &request.topic_id, request.partition_id)? - } - STORE_CONSUMER_OFFSET_2_CODE => { - let request = StoreConsumerOffset2Request::decode_from(payload) - .map_err(|_| IggyError::InvalidCommand)?; - namespace_from_partition(&request.stream_id, &request.topic_id, request.partition_id)? - } - DELETE_CONSUMER_OFFSET_2_CODE => { - let request = DeleteConsumerOffset2Request::decode_from(payload) - .map_err(|_| IggyError::InvalidCommand)?; - namespace_from_partition(&request.stream_id, &request.topic_id, request.partition_id)? - } - DELETE_SEGMENTS_CODE => { - let request = DeleteSegmentsRequest::decode_from(payload) - .map_err(|_| IggyError::InvalidCommand)?; - namespace_from_partition( - &request.stream_id, - &request.topic_id, - Some(request.partition_id), - )? - } - _ => return Err(IggyError::FeatureUnavailable), - }; - - Ok(namespace) -} - -fn namespace_from_partitioning( - stream_id: &WireIdentifier, - topic_id: &WireIdentifier, - partitioning: &WirePartitioning, -) -> Result { - let WirePartitioning::PartitionId(partition_id) = partitioning else { - return Err(IggyError::FeatureUnavailable); - }; - namespace_from_partition(stream_id, topic_id, Some(*partition_id)) -} - -fn namespace_from_partition( - stream_id: &WireIdentifier, - topic_id: &WireIdentifier, - partition_id: Option, -) -> Result { - let partition_id = partition_id.ok_or(IggyError::InvalidIdentifier)?; - let Some(stream_id) = stream_id.as_u32() else { - return Ok(0); - }; - let Some(topic_id) = topic_id.as_u32() else { - return Ok(0); - }; - validate_namespace_field(stream_id, MAX_STREAMS)?; - validate_namespace_field(topic_id, MAX_TOPICS)?; - validate_namespace_field(partition_id, MAX_PARTITIONS)?; - Ok(pack_namespace( - stream_id as usize, - topic_id as usize, - partition_id as usize, - )) -} - -fn validate_namespace_field(value: u32, exclusive_max: usize) -> Result<(), IggyError> { - let value = usize::try_from(value).map_err(|_| IggyError::InvalidIdentifier)?; - if value >= exclusive_max { - return Err(IggyError::InvalidIdentifier); - } - Ok(()) -} - -fn pack_namespace(stream_id: usize, topic_id: usize, partition_id: usize) -> u64 { - ((stream_id as u64) & STREAM_MASK) << STREAM_SHIFT - | ((topic_id as u64) & TOPIC_MASK) << TOPIC_SHIFT - | ((partition_id as u64) & PARTITION_MASK) << PARTITION_SHIFT -} - #[cfg(test)] mod tests { use super::*; use crate::session::ConsensusSession; - use iggy_binary_protocol::codes::{ - CREATE_STREAM_CODE, GET_STREAM_CODE, LOGOUT_USER_CODE, PING_CODE, - }; - use iggy_binary_protocol::requests::messages::SendMessagesHeader; + use iggy_binary_protocol::codes::{CREATE_STREAM_CODE, GET_STREAM_CODE, PING_CODE}; use iggy_binary_protocol::requests::streams::CreateStreamRequest; use iggy_binary_protocol::requests::users::LoginRegisterRequest; use iggy_binary_protocol::version::IGGY_PROTOCOL_VERSION; @@ -480,34 +358,6 @@ mod tests { *bytemuck::checked::try_from_bytes::(&bytes[..HEADER_SIZE]).unwrap() } - #[test] - fn register_request_uses_zero_request_and_session() { - let mut session = ConsensusSession::with_client_id(7); - let request = LoginRegisterRequest { - version_info: ClientVersionInfo { - protocol_version: IGGY_PROTOCOL_VERSION, - sdk_name: WireName::new("rust-sdk").unwrap(), - sdk_version: WireName::new("1.0.0").unwrap(), - }, - username: WireName::new("admin").unwrap(), - password: SecretString::from("secret"), - client_context: None, - }; - - let bytes = - encode_contiguous_request(&mut session, LOGIN_REGISTER_CODE, &request.to_bytes()) - .unwrap(); - let header = decode_request_header(&bytes); - - assert_eq!(header.operation, Operation::Register); - assert_eq!(header.request, 0); - assert_eq!(header.session, 0); - assert_eq!(header.client, 7); - // Register is routed to the metadata replica (shard 0). The router's - // namespace==METADATA short-circuit needs the sentinel, not 0. - assert_eq!(header.namespace, METADATA_CONSENSUS_NAMESPACE); - } - #[test] fn second_register_on_bound_session_re_arms_instead_of_panicking() { let request = LoginRegisterRequest { @@ -633,7 +483,27 @@ mod tests { assert_eq!(decode_request_header(&first).request, 1); assert_eq!(decode_request_header(&second).request, 2); assert_eq!(decode_request_header(&second).session, 99); - assert_eq!(decode_request_header(&second).namespace, 0); + } + + #[test] + fn request_checksum_is_stamped_only_for_deduped_operations() { + // The stamp exists to stop a reused `request` number returning the wrong + // cached reply, so it is worth its hashing pass only where `ClientTable` + // dedups. Partition payloads are the large ones and carry `batch_checksum` + // over the same bytes already; hashing them again is pure cost. + let mut session = ConsensusSession::with_client_id(42); + session.bind(99); + let payload = Bytes::from_static(b"payload"); + + let deduped = + encode_contiguous_request(&mut session, CREATE_STREAM_CODE, &payload).unwrap(); + assert_eq!( + decode_request_header(&deduped).request_checksum, + u128::from(calculate_checksum(&payload)), + ); + + let ping = encode_contiguous_request(&mut session, PING_CODE, &Bytes::new()).unwrap(); + assert_eq!(decode_request_header(&ping).request_checksum, 0); } #[test] @@ -653,7 +523,6 @@ mod tests { PING_CODE ); assert_eq!(header.session, 99); - assert_eq!(header.namespace, 0); } #[test] @@ -667,9 +536,6 @@ mod tests { assert_eq!(header.operation, Operation::Logout); assert_eq!(header.request, 1); assert_eq!(header.session, 99); - // Logout, like Register, is routed to shard 0 via the metadata - // sentinel rather than namespace 0. - assert_eq!(header.namespace, METADATA_CONSENSUS_NAMESPACE); } #[test] @@ -740,50 +606,6 @@ mod tests { } } - #[test] - fn namespace_defers_named_identifiers_to_server_resolution() { - let stream = WireIdentifier::named("stream").unwrap(); - let topic = WireIdentifier::numeric(1); - let namespace = namespace_from_partition(&stream, &topic, Some(0)).unwrap(); - assert_eq!(namespace, 0); - } - - #[test] - fn namespace_rejects_out_of_range_fields() { - let stream = WireIdentifier::numeric(MAX_STREAMS as u32); - let topic = WireIdentifier::numeric(1); - let err = namespace_from_partition(&stream, &topic, Some(0)).unwrap_err(); - assert!(matches!(err, IggyError::InvalidIdentifier)); - - let stream = WireIdentifier::numeric(1); - let partition_id = u32::try_from(MAX_PARTITIONS).unwrap(); - let err = namespace_from_partition(&stream, &topic, Some(partition_id)).unwrap_err(); - assert!(matches!(err, IggyError::InvalidIdentifier)); - } - - #[test] - fn send_messages_with_numeric_partition_builds_namespace() { - let header = SendMessagesHeader { - stream_id: WireIdentifier::numeric(2), - topic_id: WireIdentifier::numeric(3), - partitioning: WirePartitioning::PartitionId(4), - messages_count: 0, - }; - let mut payload = BytesMut::new(); - payload.put_u32_le(header.metadata_length() as u32); - header.encode(&mut payload); - - let namespace = namespace_for_request( - SEND_MESSAGES_CODE, - &payload.freeze(), - Operation::SendMessages, - ) - .unwrap(); - assert_eq!((namespace >> STREAM_SHIFT) & STREAM_MASK, 2); - assert_eq!((namespace >> TOPIC_SHIFT) & TOPIC_MASK, 3); - assert_eq!((namespace >> PARTITION_SHIFT) & PARTITION_MASK, 4); - } - #[test] fn metadata_success_reply_strips_result_section_and_returns_payload() { let mut body = BytesMut::new(); diff --git a/core/sdk/src/websocket/websocket_client.rs b/core/sdk/src/websocket/websocket_client.rs index 537d68de22..514ab6d21a 100644 --- a/core/sdk/src/websocket/websocket_client.rs +++ b/core/sdk/src/websocket/websocket_client.rs @@ -16,7 +16,6 @@ // under the License. use crate::leader_aware::{LeaderRedirectionState, check_and_redirect_to_leader}; -#[cfg(feature = "vsr")] use crate::session::ConsensusSession; use crate::websocket::websocket_connection_stream::WebSocketConnectionStream; use crate::websocket::websocket_stream_kind::WebSocketStreamKind; @@ -27,13 +26,7 @@ use crate::prelude::Client; use async_broadcast::{Receiver, Sender, broadcast}; use async_trait::async_trait; use bytes::Bytes; -#[cfg(not(feature = "vsr"))] -use bytes::{BufMut, BytesMut}; -#[cfg(feature = "vsr")] use iggy_binary_protocol::codes::{LOGIN_REGISTER_CODE, LOGIN_REGISTER_WITH_PAT_CODE}; -#[cfg(not(feature = "vsr"))] -use iggy_common::IggyErrorDiscriminants; -#[cfg(feature = "vsr")] use iggy_common::VsrSessionControl as _; use iggy_common::{ AutoLogin, ClientState, ConnectionString, Credentials, DiagnosticEvent, IggyDuration, @@ -43,7 +36,6 @@ use iggy_common::{BinaryClient, BinaryTransport, PersonalAccessTokenClient, User use secrecy::ExposeSecret; use std::net::SocketAddr; use std::sync::Arc; -#[cfg(feature = "vsr")] use std::sync::Mutex as StdMutex; use tokio::net::TcpStream; use tokio::sync::Mutex; @@ -54,16 +46,11 @@ use tokio_tungstenite::{ }; use tracing::{debug, error, info, trace, warn}; -#[cfg(not(feature = "vsr"))] -const REQUEST_INITIAL_BYTES_LENGTH: usize = 4; -#[cfg(not(feature = "vsr"))] -const RESPONSE_INITIAL_BYTES_LENGTH: usize = 8; const NAME: &str = "WebSocket"; /// Bound on how long a single VSR reply read may block. The connection is /// lockstep and the read runs in the caller's task while holding the stream /// lock, so an unanswered read (lost server reply) would wedge every later /// request on this client forever. On expiry the stream is dropped. -#[cfg(feature = "vsr")] const RESPONSE_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); /// Backoff before replaying a request the server answered with an explicit @@ -71,7 +58,6 @@ const RESPONSE_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_sec /// view-change cancel). The reply arrives promptly, so a short pause keeps the /// replay from spinning while the primary catches up. Bounded by /// `RESPONSE_READ_TIMEOUT`. -#[cfg(feature = "vsr")] const NOT_READY_RETRY_INTERVAL: std::time::Duration = std::time::Duration::from_millis(50); #[derive(Debug)] @@ -84,13 +70,10 @@ pub struct WebSocketClient { pub(crate) connected_at: Mutex>, leader_redirection_state: Mutex, pub(crate) current_server_address: Mutex, - #[cfg(feature = "vsr")] // See `core/sdk/src/tcp/tcp_client.rs` for the `tokio::sync::Mutex` -> // `std::sync::Mutex` rationale (pure-CPU critical section). consensus_session: Arc>, - #[cfg(feature = "vsr")] skip_auto_login_once: Mutex, - #[cfg(feature = "vsr")] consumer_group_state: Arc, } @@ -162,16 +145,13 @@ impl BinaryTransport for WebSocketClient { return Err(IggyError::Disconnected); } - #[cfg(feature = "vsr")] if matches!(self.config.auto_login, AutoLogin::Disabled) { return Err(error); } self.disconnect().await?; - #[cfg(feature = "vsr")] let skip_auto_login = is_login_register_code(code); - #[cfg(feature = "vsr")] if skip_auto_login { *self.skip_auto_login_once.lock().await = true; } @@ -185,7 +165,6 @@ impl BinaryTransport for WebSocketClient { } let reconnect = self.connect().await; - #[cfg(feature = "vsr")] if skip_auto_login && reconnect.is_err() { *self.skip_auto_login_once.lock().await = false; } @@ -197,16 +176,13 @@ impl BinaryTransport for WebSocketClient { self.config.heartbeat_interval } - #[cfg(feature = "vsr")] fn consumer_group_state(&self) -> Arc { Arc::clone(&self.consumer_group_state) } } -#[cfg(feature = "vsr")] impl iggy_common::VsrSessionSealed for WebSocketClient {} -#[cfg(feature = "vsr")] #[async_trait::async_trait] impl iggy_common::VsrSessionControl for WebSocketClient { async fn bind_vsr_session(&self, session: u64) -> Result<(), IggyError> { @@ -255,11 +231,8 @@ impl WebSocketClient { connected_at: Mutex::new(None), leader_redirection_state: Mutex::new(LeaderRedirectionState::new()), current_server_address: Mutex::new(server_address), - #[cfg(feature = "vsr")] consensus_session: Arc::new(StdMutex::new(ConsensusSession::new())), - #[cfg(feature = "vsr")] skip_auto_login_once: Mutex::new(false), - #[cfg(feature = "vsr")] consumer_group_state: Arc::new(iggy_common::ConsumerGroupClientState::new()), }) } @@ -536,25 +509,15 @@ impl WebSocketClient { match &self.config.auto_login { // Leadership still matters without auto-login: the caller signs in // manually, and a login against a non-leader replays for its whole - // read timeout. `GetClusterMetadata` is sessionless and pre-auth on - // server-ng, so the check works on the unauthenticated connection. - // vsr-only: the legacy server auth-gates cluster metadata, so this - // check would bounce `Unauthenticated` into the reconnect path and - // recurse back into `connect`. - #[cfg(feature = "vsr")] + // read timeout. `GetClusterMetadata` is sessionless and pre-auth, + // so the check works on the unauthenticated connection. AutoLogin::Disabled => self.handle_leader_redirection().await, - #[cfg(not(feature = "vsr"))] - AutoLogin::Disabled => Ok(false), AutoLogin::Enabled(_) => { // Check leadership BEFORE signing in: register/login are // consensus ops a backup answers with a transient frame, so // signing in against a non-leader replays for the whole read // timeout instead of failing over. `GetClusterMetadata` is - // sessionless and pre-auth on server-ng. vsr-only: the legacy - // server auth-gates cluster metadata, so this pre-login check - // would bounce `Unauthenticated` into the reconnect path and - // recurse back into `connect`. - #[cfg(feature = "vsr")] + // sessionless and pre-auth. if self.handle_leader_redirection().await? { return Ok(true); } @@ -601,7 +564,6 @@ impl WebSocketClient { async fn auto_login(&self) -> Result<(), IggyError> { let client_address = self.get_client_address_value().await; - #[cfg(feature = "vsr")] let skip_auto_login = { let mut guard = self.skip_auto_login_once.lock().await; std::mem::take(&mut *guard) @@ -613,7 +575,6 @@ impl WebSocketClient { Ok(()) } AutoLogin::Enabled(credentials) => { - #[cfg(feature = "vsr")] if skip_auto_login { info!("Skipping automatic sign-in for a retried login/register request."); return Ok(()); @@ -651,7 +612,6 @@ impl WebSocketClient { self.set_state(ClientState::Disconnected).await; self.stream.lock().await.take(); - #[cfg(feature = "vsr")] self.reset_vsr_session().await?; self.publish_event(DiagnosticEvent::Disconnected).await; @@ -675,7 +635,6 @@ impl WebSocketClient { let _ = stream.shutdown().await; } - #[cfg(feature = "vsr")] self.reset_vsr_session().await?; self.set_state(ClientState::Shutdown).await; self.publish_event(DiagnosticEvent::Shutdown).await; @@ -706,7 +665,6 @@ impl WebSocketClient { return Err(IggyError::NotConnected); } - #[cfg(feature = "vsr")] { // Encode the request ONCE: `next_request_id` advances here, so a // transient replay must reuse the same id for the server's dedup. @@ -785,87 +743,9 @@ impl WebSocketClient { } } } - - #[cfg(not(feature = "vsr"))] - { - let stream = stream_guard.as_mut().ok_or_else(|| { - trace!("Cannot send data. Client is not connected."); - IggyError::NotConnected - })?; - let payload_length = payload.len() + REQUEST_INITIAL_BYTES_LENGTH; - let mut request = - BytesMut::with_capacity(4 + REQUEST_INITIAL_BYTES_LENGTH + payload.len()); - request.put_u32_le(payload_length as u32); - request.put_u32_le(code); - request.put_slice(&payload); - trace!( - "Sending {NAME} message with code: {}, payload size: {} bytes", - code, - payload.len() - ); - stream.write(&request).await?; - stream.flush().await?; - - let mut response_initial_buffer = vec![0u8; RESPONSE_INITIAL_BYTES_LENGTH]; - stream.read(&mut response_initial_buffer).await?; - - let status = u32::from_le_bytes([ - response_initial_buffer[0], - response_initial_buffer[1], - response_initial_buffer[2], - response_initial_buffer[3], - ]); - - let length = u32::from_le_bytes([ - response_initial_buffer[4], - response_initial_buffer[5], - response_initial_buffer[6], - response_initial_buffer[7], - ]) as usize; - - trace!( - "Received {NAME} response status: {}, length: {} bytes", - status, length - ); - - if status != 0 { - // TEMP: See https://github.com/apache/iggy/pull/604 for context. - if status == IggyErrorDiscriminants::TopicNameAlreadyExists as u32 - || status == IggyErrorDiscriminants::StreamNameAlreadyExists as u32 - || status == IggyErrorDiscriminants::UserAlreadyExists as u32 - || status == IggyErrorDiscriminants::PersonalAccessTokenAlreadyExists as u32 - || status == IggyErrorDiscriminants::ConsumerGroupNameAlreadyExists as u32 - { - debug!( - "Received a server resource already exists response: {} ({})", - status, - IggyError::from_code_as_string(status) - ) - } else { - error!( - "Received an invalid response with status: {} ({}).", - status, - IggyError::from_code_as_string(status), - ); - } - - return Err(IggyError::from_code(status)); - } - - if length == 0 { - return Ok(Bytes::new()); - } - - let mut response_buffer = vec![0u8; length]; - stream.read(&mut response_buffer).await?; - - trace!("Received {NAME} response payload, size: {} bytes", length); - Ok(Bytes::from(response_buffer)) - } } } -#[cfg(feature = "vsr")] const fn is_login_register_code(code: u32) -> bool { matches!(code, LOGIN_REGISTER_CODE | LOGIN_REGISTER_WITH_PAT_CODE) } diff --git a/core/server-ng/.dockerignore b/core/server-ng/.dockerignore deleted file mode 100644 index e2fdd5f0dd..0000000000 --- a/core/server-ng/.dockerignore +++ /dev/null @@ -1,30 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -.config -.github -/assets -/local_data -/licenses -/certs -/helm -/web -Dockerfile -docker-compose.yml -.dockerignore -.git -.gitignore diff --git a/core/server-ng/Cargo.toml b/core/server-ng/Cargo.toml deleted file mode 100644 index 4b66dc09df..0000000000 --- a/core/server-ng/Cargo.toml +++ /dev/null @@ -1,195 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[package] -name = "server-ng" -version = "0.9.0-edge.2" -edition = "2024" -license = "Apache-2.0" -publish = false - -[package.metadata.cargo-udeps.ignore] -normal = ["tracing-appender"] - -[package.metadata.cargo-machete] -ignored = [ - "ahash", - "anyhow", - "argon2", - "async-channel", - "async_zip", - "axum", - "axum-server", - "bytes", - "chrono", - "ctrlc", - "cyper", - "cyper-axum", - "dashmap", - "err_trail", - "error_set", - "figlet-rs", - "hash32", - "human-repr", - "hwlocality", - "jsonwebtoken", - "left-right", - "mimalloc", - "mime_guess", - "nix", - "opentelemetry", - "opentelemetry-appender-tracing", - "opentelemetry-otlp", - "opentelemetry-semantic-conventions", - "opentelemetry_sdk", - "papaya", - "rand", - "ringbuffer", - "rmp-serde", - "rolling-file", - "rust-embed", - "rustls", - "rustls-pemfile", - "send_wrapper", - "serde", - "slab", - "socket2", - "strum", - "sysinfo", - "tempfile", - "tracing-appender", - "tracing-opentelemetry", - "ulid", - "uuid", - "vergen-git2", -] - -[[bin]] -name = "iggy-server-ng" -path = "src/main.rs" - -[features] -default = ["mimalloc", "iggy-web"] -disable-mimalloc = [] -mimalloc = ["dep:mimalloc"] -iggy-web = ["dep:rust-embed", "dep:mime_guess"] -vsr = ["iggy_common/vsr"] - -[dependencies] -ahash = { workspace = true } -argon2 = { workspace = true } -async-channel = { workspace = true } -async_zip = { workspace = true } -axum = { workspace = true } -axum-server = { workspace = true } -blake3 = { workspace = true } -bytemuck = { workspace = true } -bytes = { workspace = true } -chrono = { workspace = true } -clap = { workspace = true } -compio = { workspace = true } -configs = { workspace = true } -consensus = { workspace = true } -crossfire = { workspace = true } -ctrlc = { workspace = true } -cyper = { workspace = true } -cyper-axum = { workspace = true } -cyper-core = { workspace = true } -dashmap = { workspace = true } -dotenvy = { workspace = true } -err_trail = { workspace = true } -error_set = { workspace = true } -figlet-rs = { workspace = true } -fs2 = { workspace = true } -futures = { workspace = true } -hash32 = { workspace = true } -human-repr = { workspace = true } -hyper = { workspace = true } -hyper-util = { workspace = true } -iggy_binary_protocol = { workspace = true } -iggy_common = { workspace = true } -journal = { workspace = true } -jsonwebtoken = { workspace = true } -left-right = { workspace = true } -message_bus = { workspace = true } -metadata = { workspace = true } -mimalloc = { workspace = true, optional = true } -mime_guess = { workspace = true, optional = true } -nix = { workspace = true } -opentelemetry = { workspace = true } -opentelemetry-appender-tracing = { workspace = true } -opentelemetry-otlp = { workspace = true } -opentelemetry-semantic-conventions = { workspace = true } -opentelemetry_sdk = { workspace = true } -papaya = { workspace = true } -partitions = { workspace = true } -prometheus-client = { workspace = true } -rand = { workspace = true } -ringbuffer = { workspace = true } -rmp-serde = { workspace = true } -rolling-file = { workspace = true } -rust-embed = { workspace = true, optional = true } -rustls = { workspace = true } -rustls-pemfile = { workspace = true } -secrecy = { workspace = true } -send_wrapper = { workspace = true } -serde = { workspace = true } -serde_json = { workspace = true } -server_common = { workspace = true } -shard = { workspace = true } -shard_allocator = { workspace = true } -slab = { workspace = true } -socket2 = { workspace = true } -strum = { workspace = true } -sysinfo = { workspace = true } -system_stats = { workspace = true } -tempfile = { workspace = true } -thiserror = { workspace = true } -tokio = { workspace = true } -toml = { workspace = true } -tower-http = { workspace = true } -tracing = { workspace = true } -tracing-appender = { workspace = true } -tracing-opentelemetry = { workspace = true } -ulid = { workspace = true } -uuid = { workspace = true } - -[target.'cfg(not(target_env = "musl"))'.dependencies] -hwlocality = { workspace = true } - -[target.'cfg(target_env = "musl")'.dependencies] -hwlocality = { workspace = true, features = ["vendored"] } - -[build-dependencies] -vergen-git2 = { workspace = true } - -[dev-dependencies] -assert_cmd = { workspace = true } -bytemuck = { workspace = true } -# Reconciler unit tests assert on `ShardMetrics` snapshots and -# `IggyShard::parked_frame_count`, gated to test/simulator so they cannot grow -# production callers. `shard`'s own `cfg(test)` is false when compiled as our -# dependency, so the feature is how those accessors become visible. The resolver -# keeps a dev-dependency's features out of non-test targets, so a production -# build still links `shard` without `simulator`. -shard = { workspace = true, features = ["simulator"] } -tokio = { workspace = true, features = ["full", "test-util"] } - -[lints.clippy] -enum_glob_use = "deny" -pedantic = "deny" -nursery = "warn" diff --git a/core/server-ng/Dockerfile b/core/server-ng/Dockerfile deleted file mode 100644 index a1bb1a91c7..0000000000 --- a/core/server-ng/Dockerfile +++ /dev/null @@ -1,179 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Apache Iggy (Incubating) is an effort undergoing incubation at the Apache -# Software Foundation (ASF), sponsored by the Apache Incubator PMC. -# -# Incubation is required of all newly accepted projects until a further review -# indicates that the infrastructure, communications, and decision making -# process have stabilized in a manner consistent with other successful ASF -# projects. -# -# While incubation status is not necessarily a reflection of the completeness -# or stability of the code, it does indicate that the project has yet to be -# fully endorsed by the ASF. - -ARG RUST_VERSION=1.97.1 -ARG ALPINE_VERSION=3.23 - -# ── from-source path ───────────────────────────────────────────────────────── -FROM --platform=$BUILDPLATFORM lukemathwalker/cargo-chef:latest-rust-${RUST_VERSION}-alpine${ALPINE_VERSION} AS chef -WORKDIR /app -RUN apk add --no-cache musl-dev pkgconfig - -FROM --platform=$BUILDPLATFORM chef AS planner -COPY . . -RUN cargo chef prepare --recipe-path recipe.json - -FROM --platform=$BUILDPLATFORM chef AS builder -ARG PROFILE=release -ARG TARGETPLATFORM -ARG LIBC=musl -ARG IGGY_CI_BUILD -ENV IGGY_CI_BUILD=${IGGY_CI_BUILD} - -RUN apk add --no-cache zig make autoconf automake libtool pkgconfig hwloc-dev xz-dev xz-static nodejs npm && \ - cargo install cargo-zigbuild --version '=0.22.1' --locked - -COPY rust-toolchain.toml rust-toolchain.toml - -RUN rustup target add \ - x86_64-unknown-linux-musl \ - aarch64-unknown-linux-musl \ - x86_64-unknown-linux-gnu \ - aarch64-unknown-linux-gnu - -# Allow pkg-config to work in cross-compilation mode (needed for hwlocality-sys) -ENV PKG_CONFIG_ALLOW_CROSS=1 - -COPY --from=planner /app/recipe.json recipe.json - -RUN --mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARGETPLATFORM}-${LIBC} \ - --mount=type=cache,target=/usr/local/cargo/git,id=cargo-git-${TARGETPLATFORM}-${LIBC} \ - --mount=type=cache,target=/app/target,id=cargo-target-${TARGETPLATFORM}-${LIBC} \ - case "$TARGETPLATFORM:$LIBC" in \ - "linux/amd64:musl") RUST_TARGET="x86_64-unknown-linux-musl" ;; \ - "linux/arm64:musl") RUST_TARGET="aarch64-unknown-linux-musl" ;; \ - "linux/amd64:glibc") RUST_TARGET="x86_64-unknown-linux-gnu" ;; \ - "linux/arm64:glibc") RUST_TARGET="aarch64-unknown-linux-gnu" ;; \ - *) echo "Unsupported $TARGETPLATFORM/$LIBC" && exit 1 ;; \ - esac && \ - if [ "$PROFILE" = "debug" ]; then \ - cargo chef cook --recipe-path recipe.json --target ${RUST_TARGET} --zigbuild \ - --features vsr -p server-ng -p iggy-cli; \ - else \ - cargo chef cook --recipe-path recipe.json --target ${RUST_TARGET} --zigbuild --release \ - --features vsr -p server-ng -p iggy-cli; \ - fi - -COPY . . - -# Build Web UI static files for embedding -RUN npm --prefix web ci && npm --prefix web run build:static - -RUN --mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARGETPLATFORM}-${LIBC} \ - --mount=type=cache,target=/usr/local/cargo/git,id=cargo-git-${TARGETPLATFORM}-${LIBC} \ - --mount=type=cache,target=/app/target,id=cargo-target-${TARGETPLATFORM}-${LIBC} \ - case "$TARGETPLATFORM:$LIBC" in \ - "linux/amd64:musl") RUST_TARGET="x86_64-unknown-linux-musl" ;; \ - "linux/arm64:musl") RUST_TARGET="aarch64-unknown-linux-musl" ;; \ - "linux/amd64:glibc") RUST_TARGET="x86_64-unknown-linux-gnu" ;; \ - "linux/arm64:glibc") RUST_TARGET="aarch64-unknown-linux-gnu" ;; \ - *) echo "Unsupported $TARGETPLATFORM/$LIBC" && exit 1 ;; \ - esac && \ - if [ "$PROFILE" = "debug" ]; then \ - cargo zigbuild --locked --target ${RUST_TARGET} --features vsr --bin iggy-server-ng --bin iggy && \ - cp /app/target/${RUST_TARGET}/debug/iggy-server-ng /app/iggy-server-ng && \ - cp /app/target/${RUST_TARGET}/debug/iggy /app/iggy; \ - else \ - cargo zigbuild --locked --target ${RUST_TARGET} --features vsr --bin iggy-server-ng --bin iggy --release && \ - cp /app/target/${RUST_TARGET}/release/iggy-server-ng /app/iggy-server-ng && \ - cp /app/target/${RUST_TARGET}/release/iggy /app/iggy; \ - fi - -# ── prebuilt path (FAST) ────────────────────────────────────────────────────── -FROM debian:trixie-slim AS prebuilt -WORKDIR /out -ARG PREBUILT_IGGY_SERVER_NG -ARG PREBUILT_IGGY_CLI -COPY ${PREBUILT_IGGY_SERVER_NG} /out/iggy-server-ng -COPY ${PREBUILT_IGGY_CLI} /out/iggy -RUN chmod +x /out/iggy-server-ng /out/iggy - -# ── final images ────────────────────────────────────────────────────────────── -FROM debian:trixie-slim AS runtime-prebuilt -ARG TARGETPLATFORM -ARG PREBUILT_IGGY_SERVER_NG -ARG PREBUILT_IGGY_CLI -WORKDIR /app -RUN apt-get update && apt-get install -y \ - libhwloc-dev \ - libudev-dev \ - pkg-config \ - && rm -rf /var/lib/apt/lists/* -COPY --from=prebuilt /out/iggy-server-ng /usr/local/bin/iggy-server-ng -COPY --from=prebuilt /out/iggy /usr/local/bin/iggy -RUN echo "═══════════════════════════════════════════════════════════════" && \ - echo " IGGY SERVER-NG BUILD SUMMARY " && \ - echo "═══════════════════════════════════════════════════════════════" && \ - echo "Build Type: PREBUILT BINARIES" && \ - echo "Platform: ${TARGETPLATFORM:-linux/amd64}" && \ - echo "Source Path: ${PREBUILT_IGGY_SERVER_NG:-not specified}" && \ - echo "Binary Info:" && \ - (command -v file >/dev/null 2>&1 && file /usr/local/bin/iggy-server-ng | sed 's/^/ /' || \ - echo " $(ldd /usr/local/bin/iggy-server-ng 2>&1 | head -1)") && \ - echo "Binary Size:" && \ - ls -lh /usr/local/bin/iggy-server-ng /usr/local/bin/iggy | awk '{print " " $9 ": " $5}' && \ - echo "Build Date: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" && \ - echo "Container Base: debian:trixie-slim" && \ - echo "═══════════════════════════════════════════════════════════════" -ENTRYPOINT ["iggy-server-ng"] - -FROM debian:trixie-slim AS runtime -ARG TARGETPLATFORM -ARG PROFILE=release -ARG LIBC=musl -WORKDIR /app -RUN apt-get update && apt-get install -y \ - libhwloc15 \ - libudev1 \ - pkg-config \ - && rm -rf /var/lib/apt/lists/* -COPY --from=builder /app/iggy-server-ng /usr/local/bin/iggy-server-ng -COPY --from=builder /app/iggy /usr/local/bin/iggy -RUN echo "═══════════════════════════════════════════════════════════════" && \ - echo " IGGY SERVER-NG BUILD SUMMARY " && \ - echo "═══════════════════════════════════════════════════════════════" && \ - echo "Build Type: FROM SOURCE" && \ - echo "Platform: ${TARGETPLATFORM:-linux/amd64}" && \ - echo "Profile: ${PROFILE}" && \ - echo "Libc: ${LIBC}" && \ - case "${TARGETPLATFORM:-linux/amd64}:${LIBC}" in \ - "linux/amd64:musl") echo "Target: x86_64-unknown-linux-musl" ;; \ - "linux/arm64:musl") echo "Target: aarch64-unknown-linux-musl" ;; \ - "linux/amd64:glibc") echo "Target: x86_64-unknown-linux-gnu" ;; \ - "linux/arm64:glibc") echo "Target: aarch64-unknown-linux-gnu" ;; \ - *) echo "Target: unknown" ;; \ - esac && \ - echo "Binary Info:" && \ - (command -v file >/dev/null 2>&1 && file /usr/local/bin/iggy-server-ng | sed 's/^/ /' || \ - echo " $(ldd /usr/local/bin/iggy-server-ng 2>&1 | head -1)") && \ - echo "Binary Size:" && \ - ls -lh /usr/local/bin/iggy-server-ng /usr/local/bin/iggy | awk '{print " " $9 ": " $5}' && \ - echo "Build Date: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" && \ - echo "═══════════════════════════════════════════════════════════════" -ENTRYPOINT ["iggy-server-ng"] diff --git a/core/server-ng/build.rs b/core/server-ng/build.rs deleted file mode 100644 index 4d07885ebb..0000000000 --- a/core/server-ng/build.rs +++ /dev/null @@ -1,86 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::path::PathBuf; -use std::{env, error}; -use vergen_git2::{Build, Cargo, Emitter, Git2, Rustc, Sysinfo}; - -const WEB_ASSETS_PATH: &str = "web/build/static"; -const WEB_INDEX_FILE: &str = "web/build/static/index.html"; - -fn main() -> Result<(), Box> { - verify_web_assets_if_enabled(); - emit_vergen_instructions()?; - Ok(()) -} - -/// Returns the workspace root (iggy/), two levels up from core/server-ng. -fn workspace_root() -> PathBuf { - PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()) - .parent() - .and_then(|p| p.parent()) - .expect("server-ng crate must be at core/server-ng within workspace") - .to_path_buf() -} - -fn emit_vergen_instructions() -> Result<(), Box> { - if option_env!("IGGY_CI_BUILD") != Some("true") { - println!("cargo:info=Skipping vergen because IGGY_CI_BUILD is not set to 'true'"); - return Ok(()); - } - - Emitter::default() - .add_instructions(&Build::all_build())? - .add_instructions(&Cargo::all_cargo())? - .add_instructions(&Git2::all_git())? - .add_instructions(&Rustc::all_rustc())? - .add_instructions(&Sysinfo::all_sysinfo())? - .emit()?; - - let configs_path = workspace_root() - .join("core/configs") - .canonicalize() - .unwrap_or_else(|e| panic!("Failed to canonicalize configs path: {e}")); - - println!("cargo:rerun-if-changed={}", configs_path.display()); - Ok(()) -} - -fn verify_web_assets_if_enabled() { - if env::var("CARGO_FEATURE_IGGY_WEB").is_err() { - return; - } - - let assets_dir = workspace_root().join(WEB_ASSETS_PATH); - let index_file = workspace_root().join(WEB_INDEX_FILE); - - println!("cargo:rerun-if-changed={}", assets_dir.display()); - - if !assets_dir.exists() || !index_file.exists() { - println!( - "cargo:info=Web UI assets not found at {}. \ - To build them, run: npm --prefix web ci && npm --prefix web run build:static", - assets_dir.display() - ); - return; - } - - println!( - "cargo:info=Web UI assets verified at {}", - assets_dir.display() - ); -} diff --git a/core/server-ng/config.toml b/core/server-ng/config.toml deleted file mode 100644 index 10a529930a..0000000000 --- a/core/server-ng/config.toml +++ /dev/null @@ -1,1027 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Configuration for consumer group cooperative partition rebalancing. -[consumer_group] -# Maximum time a partition can remain in pending revocation before being force-transferred to the target member. -rebalancing_timeout = "30s" -# How often the periodic checker scans for timed-out pending revocations. -# TODO(hubcio): inert in server-ng, which paces the scan from -# system.sharding.reconcile_periodic_interval instead. Boot warns when set. -rebalancing_check_interval = "5s" - -[data_maintenance.messages] -# Enables or disables the expired message cleaner process. -cleaner_enabled = false - -# Interval for running the message cleaner. -interval = "1 m" - -# HTTP server configuration -[http] -# Determines if the HTTP server is active. -# `true` enables the server, allowing it to handle HTTP requests. -# `false` disables the server, preventing it from handling HTTP requests. -# In cluster mode, followers forward control-plane requests (streams, topics, -# users, ...) to the current primary when a cluster-wide JWT key exists (see -# http.jwt / cluster.auth below). -# TODO: forwarding does not cover the partition-plane APIs yet - message -# produce and consumer-offset writes are never forwarded and must reach the -# partition's primary node directly (message polls read locally on any node). -enabled = true - -# Specifies the network address and port for the HTTP server. -# The format is "HOST:PORT". For example, "127.0.0.1:3000" listens on localhost only on port 3000. -# In cluster mode the HOST still picks the bind interface, while the port -# comes from this node's cluster.nodes ports.http entry. -address = "127.0.0.1:3000" - -# Maximum size of the request body in bytes. For security reasons, the default limit is 2 MB. -max_request_size = "2 MB" - -# Enables the embedded Web UI dashboard at '/ui'. -# When set to `true` and the server is compiled with the 'iggy-web' feature, -# the Svelte dashboard will be served at the '/ui' endpoint, providing a -# browser-based interface for managing streams, topics, and viewing messages. -# If the server is compiled without 'iggy-web' feature and this is set to `true`, -# a warning will be logged at startup but the server will continue to run. -# `true` enables the embedded Web UI (requires server built with 'iggy-web' feature). -# `false` disables the embedded Web UI (default). -web_ui = false - -# Configuration for Cross-Origin Resource Sharing (CORS). -[http.cors] -# Controls whether CORS is enabled for the HTTP server. -# `true` allows handling cross-origin requests with specified rules. -# `false` blocks cross-origin requests, enhancing security. -enabled = true - -# Specifies which HTTP methods are allowed when CORS is enabled. -# For example, ["GET", "POST"] would allow only GET and POST requests. -allowed_methods = ["GET", "POST", "PUT", "DELETE"] - -# Defines which origins are permitted to make cross-origin requests. -# An asterisk "*" as the first entry allows all origins (any entries after it -# are ignored); "*" in any other position fails the config. Specific domains -# can be listed to restrict access. -allowed_origins = ["*"] - -# Lists allowed headers that can be used in CORS requests. -# For example, ["content-type"] permits only the content-type header. -allowed_headers = ["content-type", "authorization"] - -# Headers that browsers are allowed to access in CORS responses. -# `iggy-view` carries the current VSR view number; exposing it lets browser -# clients read it on cross-origin responses. -exposed_headers = ["iggy-view"] - -# Determines if credentials like cookies or HTTP auth can be included in CORS requests. -# `true` allows credentials to be included, useful for authenticated sessions; -# it requires explicit (non-wildcard) allowed_origins, allowed_headers, and -# exposed_headers. -# `false` prevents credentials, enhancing privacy and security. -allow_credentials = false - -# Allows or blocks requests from private networks in CORS. -# `true` permits requests from private networks. -# `false` disallows such requests, providing additional security. -allow_private_network = false - -# JWT (JSON Web Token) configuration for HTTP. -[http.jwt] -# Specifies the algorithm used for signing JWTs. -# For example, "HS256" indicates HMAC with SHA-256. -algorithm = "HS256" - -# The issuer of the JWT, typically a URL or an identifier of the issuing entity. -issuer = "iggy.apache.org" - -# Intended audience for the JWT, usually the recipient or system intended to process the token. -audience = "iggy.apache.org" - -# Lists valid issuers for JWT validation to ensure tokens are from trusted sources. -valid_issuers = ["iggy.apache.org"] - -# Lists valid audiences for JWT validation to confirm tokens are for the intended recipient. -valid_audiences = ["iggy.apache.org"] - -# Expiry time for access tokens. -access_token_expiry = "1 h" - -# Tolerance for timing discrepancies during token validation. -clock_skew = "5 s" - -# Time before which the token should not be considered valid. -not_before = "0 s" - -# Secret key for encoding JWTs. -# If left empty, a secure random secret will be generated on each server start. -# In cluster mode a configured secret (identical on every node) makes bearers -# valid cluster-wide and activates follower-to-primary HTTP forwarding; with -# cluster.auth enabled the key is instead derived from the shared PSK. Without -# either, tokens are node-local and forwarding stays disabled. -encoding_secret = "" - -# Secret key for decoding JWTs. -# If left empty, a secure random secret will be generated on each server start. -decoding_secret = "" - -# Indicates if the secret key is base64 encoded. -# `true` means the secret is base64 encoded. -# `false` means the secret is in plain text. -use_base64_secret = false - -# Trusted issuers for A2A (Application-to-Application) authentication. Opt-in: -# with none configured the listener accepts only self-issued HS256 tokens. -# `issuer`, `audience` and `jwks_url` are required per entry; `user_id` is -# optional but defaults to 0 (root), which is rejected - set it to the non-zero -# iggy user every token from that issuer is remapped onto. -# -# Operational note: enabling an issuer opens an outbound JWKS fetch that is -# reachable before a token's signature is verified - a token naming this issuer -# with an unknown key id can trigger a fetch to `jwks_url`. The target is fixed -# (not attacker-chosen); concurrent misses coalesce onto one fetch and repeats -# are rate-limited to at most one outbound request per issuer per short window, -# so an unknown-key-id flood cannot amplify. The same window bounds how quickly a -# freshly rotated signing key is picked up. -# [[http.jwt.trusted_issuers]] -# issuer = "test-issuer" -# jwks_url = "http://127.0.0.1:8081/.well-known/jwks.json" -# audience = "iggy.apache.org" -# user_id = 1 - -# Metrics configuration for HTTP. -[http.metrics] -# Enable or disable the metrics endpoint. -# `true` makes metrics available at the specified endpoint. -# `false` disables metrics collection. -enabled = true - -# Specifies the endpoint for accessing metrics, e.g., "/metrics". -endpoint = "/metrics" - -# TLS (Transport Layer Security) configuration for HTTP. -[http.tls] -# Controls the use of TLS for encrypted HTTP connections. -# `true` enables TLS, enhancing security. -# `false` disables TLS, which may be appropriate in secure internal networks. -enabled = false - -# Path to the TLS certificate file. -cert_file = "core/certs/iggy_cert.pem" - -# Path to the TLS key file. -key_file = "core/certs/iggy_key.pem" - -# TCP server configuration. -[tcp] -# Determines if the TCP server is active. -# `true` enables the TCP server for handling TCP connections. -# `false` disables it, preventing any TCP communication. -enabled = true - -# Defines the network address and port for the TCP server. -# For example, "127.0.0.1:8090" listens on localhost only on port 8090. -address = "127.0.0.1:8090" - -# Enable TCP socket migration across shards. -# TODO(hubcio): inert in server-ng, not implemented. Boot warns when set. -socket_migration = true - -# Whether to use ipv4 or ipv6 -# TODO(hubcio): inert in server-ng, which takes the family from the -# tcp.address string. Boot warns when set. -ipv6 = false - -# TLS configuration for the TCP server. -[tcp.tls] -# Enables or disables TLS for TCP connections. -# `true` secures TCP connections with TLS. -# `false` leaves TCP connections unencrypted. -enabled = false - -# Enables or disables self-signed certificate generation. -# `true` generates a self-signed certificate if cert files don't exist. -# `false` requires certificate files to exist at the specified paths. -self_signed = true - -# Path to the TLS certificate file. -cert_file = "core/certs/iggy_cert.pem" - -# Path to the TLS key file. -key_file = "core/certs/iggy_key.pem" - -# Configuration for the TCP socket -# TODO(hubcio): the whole section is inert in server-ng, which leaves the OS -# defaults in place. Boot warns when override_defaults is set. -[tcp.socket] -# Whether to overwrite the OS-default socket parameters -override_defaults = false - -# SO_RCVBUF: maximum size of the receive buffer, can be clamped by the OS -recv_buffer_size = "100 KB" - -# SO_SNDBUF: maximum size of the send buffer, can be clamped by the OS -send_buffer_size = "100 KB" - -# SO_KEEPALIVE: whether to regularly send a keepalive packet maintaining the connection -keepalive = false - -# TCP_NODELAY: enable/disable the Nagle algorithm which buffers data before sending segments -nodelay = false - -# SO_LINGER: delay to wait for while data is being transmitted before closing the socket after a -# close or shutdown call has been received -linger = "0 s" - -# QUIC protocol configuration. -[quic] -# Controls whether the QUIC server is enabled. -# `true` enables QUIC for fast, secure connections. -# `false` disables QUIC, possibly for compatibility or simplicity. -enabled = true - -# Network address and port for the QUIC server. -# For example, "127.0.0.1:8080" binds to localhost on port 8080. -address = "127.0.0.1:8080" - -# Maximum number of simultaneous bidirectional streams per QUIC -# connection. The message bus opens exactly one bidi stream per peer -# (no multiplexing), so any value above 1 is wasted on extra -# preallocated quinn-proto state. -max_concurrent_bidi_streams = 1 - -# Size of the buffer for sending datagrams in QUIC. Binary-aligned to -# match QuicTuning::default() (102_400 bytes) and the rest of this [quic] -# block, which uses MiB throughout (`send_window`, `receive_window`). -datagram_send_buffer_size = "100 KiB" - -# Initial Maximum Transmission Unit (MTU) for QUIC connections. Binary -# units for the same reason as `datagram_send_buffer_size`. -initial_mtu = "8 KiB" - -# Send-flow window per connection. Sized to fit a single max-size -# framed message (`message_bus.max_message_size`) without -# head-of-line wait. -send_window = "64 MiB" - -# Receive-flow window per connection. Symmetric with `send_window`. -receive_window = "64 MiB" - -# Interval for sending QUIC keep-alive PINGs. One third of -# `max_idle_timeout` so up to two consecutive losses fit before the -# idle timer closes the connection. Set to "0 s" to disable. -keep_alive_interval = "10 s" - -# Maximum idle time before a QUIC connection is closed. Set to -# "0 s" to disable (not recommended). -max_idle_timeout = "30 s" - -# QUIC certificate configuration. -[quic.certificate] -# Indicates whether the QUIC certificate is self-signed. -# `true` for self-signed certificates, often used in internal or testing environments. -# `false` for certificates issued by a certificate authority, common in production. -self_signed = true - -# Path to the QUIC TLS certificate file. -cert_file = "core/certs/iggy_cert.pem" - -# Path to the QUIC TLS key file. -key_file = "core/certs/iggy_key.pem" - -# Configuration for the QUIC socket -# TODO(hubcio): the whole section is inert in server-ng, which leaves the OS -# defaults in place. Boot warns when override_defaults is set. -[quic.socket] -# Whether to override the OS-default socket parameters -override_defaults = false - -# SO_RCVBUF: maximum size of the receive buffer, can be clamped by the OS -recv_buffer_size = "64 KB" - -# SO_SNDBUF: maximum size of the send buffer, can be clamped by the OS -send_buffer_size = "64 KB" - -# SO_KEEPALIVE: whether to regularly send a keepalive packet maintaining the connection -keepalive = false - -# Message saver configuration. -[message_saver] -# Enables or disables the background process for saving buffered data to disk. -# `true` ensures data is periodically written to disk. -# `false` turns off automatic saving, relying on other triggers for data persistence. -enabled = true - -# Controls whether data saving is synchronous (enforce fsync) or asynchronous. -# `true` for synchronous saving, ensuring data integrity at the cost of performance. -# `false` for asynchronous saving, improving performance but with delayed data writing. -# TODO(hubcio): inert in server-ng, which only flushes on shutdown and has no -# periodic saver to configure. Boot warns when set. -enforce_fsync = true - -# Interval for running the message saver. -# TODO(hubcio): inert in server-ng, see enforce_fsync above. Boot warns when set. -interval = "30 s" - -# Personal access token configuration. -[personal_access_token] -# Sets the maximum number of active tokens allowed per user. -max_tokens_per_user = 100 - -# Personal access token cleaner configuration. -[personal_access_token.cleaner] -# Enables or disables the token cleaner process. -# `true` activates periodic token cleaning. -# `false` disables it, tokens remain active until manually revoked or expired. -enabled = true - -# Interval for running the token cleaner. -interval = "1 m" - -# Heartbeat configuration -[heartbeat] -# Enables or disables the client heartbeat verification process. -enabled = false -# Interval for expected client heartbeats -interval = "5 s" - -# OpenTelemetry configuration -[telemetry] -# Enables or disables telemetry. -enabled = false -# Service name for telemetry. -service_name = "iggy" - -# OpenTelemetry logs configuration -[telemetry.logs] -# Transport for sending logs. Options: "grpc", "http". -transport = "grpc" -# Endpoint for sending logs. -endpoint = "http://localhost:7281/v1/logs" - -# OpenTelemetry traces configuration -[telemetry.traces] -# Transport for sending traces. Options: "grpc", "http". -transport = "grpc" -# Endpoint for sending traces. -endpoint = "http://localhost:7281/v1/traces" - -# System configuration. -[system] -# Base path for system data storage. -path = "local_data" - -# Backup configuration -# TODO(hubcio): backup is not supported in server-ng; both paths below are -# inert. Boot warns when either is set. -[system.backup] -# Path for storing backup. -path = "backup" - -# Compatibility conversion configuration -[system.backup.compatibility] -# Subpath of the backup directory where converted segment data is stored after compatibility conversion. -path = "compatibility" - -# TODO(hubcio): the three tunables below are inert in server-ng, whose state -# writes do not go through the legacy retrying file layer. Boot warns when any -# is set. -[system.state] -# Determines whether to enforce file synchronization on state updates (boolean). -# `true` ensures immediate writing of data to disk for durability. -# `false` allows the OS to manage write operations, which can improve performance. -enforce_fsync = false - -# Maximum number of retries for a failed file operation (e.g., append, overwrite). -# This defines how many times the system will attempt the operation before failing. -max_file_operation_retries = 1 - -# Delay between retries in case of a failed file operation. -# This helps to avoid immediate repeated attempts and can reduce load. -retry_delay = "1 s" - -# Runtime configuration. -[system.runtime] -# Path for storing runtime data. -# Specifies the directory where any runtime data is stored, relative to `system.path`. -path = "runtime" - -# Logging configuration. -[system.logging] -# Path for storing log files. -path = "logs" - -# Log filtering directive using the same syntax as the RUST_LOG environment variable. -# Supports simple levels ("trace", "debug", "info", "warn", "error", "off" or "none") -# as well as complex directives like "warn,server=debug,iggy=trace". -# Note: RUST_LOG environment variable always takes precedence over this setting. -level = "info" - -# Whether to write logs to file. When false, logs are only written to stdout. -# When enabled, logs are stored in {system.path}/{system.logging.path} (default: local_data/logs). -file_enabled = true - -# Maximum size of a single log file before rotation occurs. When a log -# file reaches this size, it will be rotated (closed and a new file -# created). This setting works together with max_total_size to control -# log storage. You can set it to 0 to enable unlimited size of single -# log, but all logs will be written to a single file, thus disabling -# log rotation. Please configure 0 with caution, esp. RUST_LOG > debug -max_file_size = "500 MB" - -# Maximum total size of all log files. When this size is reached, -# the oldest log files will be deleted first. Set it to 0 to allow -# an unlimited number of archived logs. This does not disable time -# based log rotation or per-log-file size limits. -max_total_size = "4 GB" - -# Time interval for checking log rotation status. Avoid less than 1s. -rotation_check_interval = "1 h" - -# Time to retain log files before deletion. Avoid less than 1s, too. -retention = "7 days" - -# Interval for printing system information to the log. -# TODO(hubcio): inert in server-ng, which has no sysinfo printer. Boot warns -# when set. -sysinfo_print_interval = "10 s" - -# Encryption configuration -[system.encryption] -# Determines whether server-side data encryption for the messages payloads and state commands is enabled (boolean). -# `true` enables encryption for stored data using AES-256-GCM. -# `false` means data is stored without encryption. -enabled = false - -# The encryption key used when encryption is enabled (string). -# Should be a 32 bytes length key, provided as a base64 encoded string. -# This key is required and used only if encryption is enabled. -key = "" - -# Compression configuration -[system.compression] -# Allows overriding the default compression algorithm per data segment (boolean). -# `true` permits different compression algorithms for individual segments. -# `false` means all data segments use the default compression algorithm. -# TODO(hubcio): inert in server-ng, where live compression is already per-topic -# from the request. Boot warns when set. -allow_override = false - -# The default compression algorithm used for data storage (string). -# "none" indicates no compression, other values can specify different algorithms. -default_algorithm = "none" - -# Stream configuration -[system.stream] -# Path for storing stream-related data (string). -# Specifies the directory where stream data is stored, relative to `system.path`. -path = "streams" - -# Topic configuration - default settings for new topics -[system.topic] -# Path for storing topic-related data, relative to `stream.path`. -path = "topics" - -# Messages can be deleted based on two independent policies: -# 1. Size-based: delete oldest segments when topic exceeds max_size -# 2. Time-based: delete segments older than message_expiry -# Both can be active simultaneously. Per-topic overrides via CreateTopic/UpdateTopic. - -# Maximum topic size before oldest segments are deleted. -# "unlimited" or "0" = no size limit (messages kept indefinitely). -# When 90% of this limit is reached, oldest segments are removed to make room. -# Applies to sealed segments only (active segment is protected). -# Example: "10 GiB" -max_size = "unlimited" - -# Maximum age of messages before segments are deleted. -# "none" = no time limit (messages kept indefinitely). -# Applies to sealed segments only (active segment is protected). -# Example: "7 days", "2 days 4 hours 15 minutes" -message_expiry = "none" - -# Partition configuration -[system.partition] -# Path for storing partition-related data (string). -# Specifies the directory where partition data is stored, relative to `topic.path`. -path = "partitions" - -# Determines whether to enforce file synchronization on partition updates (boolean). -# `true` ensures immediate writing of data to disk for durability. -# `false` allows the OS to manage write operations, which can improve performance. -enforce_fsync = false - -# Enables checksum validation for data integrity (boolean). -# TODO(hubcio): inert in server-ng, which never verifies checksums on load -# whatever this says - `true` buys no corruption guard here. Boot warns when -# set. -validate_checksum = false - -# The count threshold of buffered messages before triggering a save to disk. -# Together with `size_of_messages_required_to_save` it defines the threshold. -# This is a soft limit - actual count may be higher depending on last batch size. -# Minimum value is 1. -messages_required_to_save = 1024 - -# The size threshold of buffered messages before triggering a save to disk. -# Together with `messages_required_to_save` it defines the threshold. -# This is a soft limit - actual size may be higher depending on last batch size. -size_of_messages_required_to_save = "1 MiB" - -# Segment configuration -[system.segment] -# Defines the soft limit for the size of a storage segment. -# When a segment reaches this size, a new segment is created for subsequent data. -# Example: if `size` is set "1GiB", the actual segment size may be 1GiB + the size of remaining messages in received batch. -# Maximum size is 1 GiB. Size has to be a multiple of 512 B. -size = "1 GiB" - -# Configures whether expired segments are archived (boolean) or just deleted without archiving. -# Unsupported in server-ng: setting this to `true` aborts boot. -archive_expired = false - -# Controls whether to cache indexes (time and positional) for segment access. -# Possible values: -# - "true" or "all": keeps indexes in memory, speeding up data retrieval at the cost of memory -# - "open_segment": keeps indexes in memory only for the currently open segment -# - "false" or "none": reads indexes from disk, which can conserve memory at the cost of access speed -# TODO(hubcio): inert in server-ng, which picks its own index residency. Boot -# warns when set. -cache_indexes = "open_segment" - -# Message deduplication configuration -[system.message_deduplication] -# Controls whether message deduplication is enabled (boolean). -# `true` activates deduplication, ignoring messages with duplicate IDs. -# `false` treats each message as unique, even if IDs are duplicated. -# Unsupported in server-ng: setting this to `true` aborts boot. -enabled = false -# Maximum number of ID entries in the deduplication cache (u64). -max_entries = 10000 -# Maximum age of ID entries in the deduplication cache in human-readable format. -expiry = "1 m" - -# Recovery configuration in case of lost data -[system.recovery] -# Controls whether streams/topics/partitions should be recreated if the expected data for existing state is missing (boolean). -# Unsupported in server-ng: setting this to `true` aborts boot. -recreate_missing_state = false - -# Memory pool configuration -[system.memory_pool] -# Enables or disables the memory pool (boolean). -# `true` enables the memory pool. -# `false` disables the memory pool. -enabled = true - -# Size of the memory pool (string). -# Example: "512 MiB" or "1 GiB". -# This defines the maximum, total memory allocated for the memory pool. -# Note: This number has to be multiplication of 4096 (default linux page size). -# Minimum size is 512 MiB due to internal implementation details. -size = "4 GiB" - -# Maximum number of buffers in each bucket (u32). -# There are 32 buckets in the memory pool. Each bucket can hold up to this number of buffers -# and holds different buffer sizes, from 256 B to 512 MiB. -# Note: This number has to be a power of 2. Minimum value is 128 due to internal implementation details. -bucket_capacity = 8192 - -# Cluster configuration -[cluster] -# Enables or disables cluster mode (boolean). -# When enabled, this node will participate in the cluster and coordinate with other nodes. -enabled = false - -# Unique cluster name (string). -# All nodes in the same cluster must share the same name. -# This prevents accidental cross-cluster communication. -name = "iggy-cluster" - -# Backup-side liveness window for a consensus plane's primary (duration). -# A replica that sees no primary traffic for this long starts a view change. -# Raise it on oversubscribed hosts where scheduling stalls fake primary -# death. Must be at least "2s" and at least 4x commit_broadcast_interval: the -# primary signals liveness through its commit broadcast, and the window must -# span several broadcasts so one delayed broadcast never trips an election. -heartbeat_timeout = "5s" - -# How often the primary broadcasts its commit point to every backup (duration). -# This is the cluster's liveness signal: each broadcast resets every backup's -# heartbeat_timeout window and carries the latest commit point forward. Must be -# nonzero and, with heartbeat_timeout, satisfy heartbeat_timeout >= 4x this -# value. Drives the consensus CommitMessage timer. -commit_broadcast_interval = "500ms" - -# How often the primary retransmits prepares that backups have not yet acked -# (duration). Lower values recover faster from a dropped prepare at the cost of -# more replica traffic; must be nonzero. Drives the consensus Prepare timer. -prepare_retransmit_interval = "250ms" - -# How often a replica retransmits its StartViewChange / DoViewChange while a -# view change is in progress (duration). Lower values converge a healthy -# election faster at the cost of more replica traffic; must be nonzero. Drives -# both consensus view-change retransmit timers. -view_change_retransmit_interval = "500ms" - -# Backstop for a stalled view change (duration): one that does not conclude -# within this window escalates to a fresh cluster-wide election. Must be nonzero -# and at least 4x view_change_retransmit_interval, so a few dropped view-change -# messages retransmit rather than prematurely escalate. -view_change_status_timeout = "5s" - -# How often a recovering or view-change backup re-requests the current view's -# StartView from its primary (duration); must be nonzero. Drives the consensus -# RequestStartView timer. -request_start_view_retransmit_interval = "1s" - -# How many consecutive unanswered RequestStartView probes a recovering replica -# tolerates before falling back to an election (integer). A full-cluster restart -# leaves nobody settled to answer, so the replica elects on its recovered log. -# Must be between 1 and 100. -view_probe_attempts_max = 5 - -# How long a stalled journal-repair stream waits before re-requesting its -# remaining window from the serving peer (duration). Repair frames are -# fire-and-forget over the lossy bus, so a session with no retry wedges forever -# on a single dropped frame. Paces both the metadata and partition repair loops; -# must be nonzero. -repair_retry_interval = "1s" - -# Prepares a peer serves per repair round before the requester walks to the next -# chunk (integer). Each frame rides the per-peer message-bus queue, so this must -# stay strictly below message_bus.peer_queue_capacity or a full round overruns -# the queue and drops frames. Must be > 0 and <= 1024. -repair_chunk_max = 128 - -# Replica-to-replica authentication (PSK + BLAKE3 keyed-MAC handshake). -[cluster.auth] -# When true, every replica peer must complete the authenticated handshake or be -# rejected, and shared_secret becomes mandatory. Off by default = legacy -# unauthenticated replica traffic. Enabling it is a coordinated-restart change. -# With http enabled and no http.jwt secrets configured, the PSK also becomes -# the JWT key source, making bearers valid cluster-wide and activating -# follower-to-primary HTTP forwarding. -enabled = false - -# Cluster-wide pre-shared key, >= 32 bytes of CSPRNG output, byte-identical on -# every node. Prefer the IGGY_CLUSTER_AUTH_SHARED_SECRET env var (masked in -# logs, never persisted) over storing it on disk. Ignored when enabled = false. -shared_secret = "" - -# Retiring pre-shared key, accepted for verification only during a rolling key -# rotation (this node keeps signing with shared_secret). Rotate in three rolls: -# 1) shared_secret = old + previous_shared_secret = new on every node, -# 2) shared_secret = new + previous_shared_secret = old on every node, -# 3) shared_secret = new alone. Leave empty outside a rotation. Same length -# floor and env-var preference as shared_secret -# (IGGY_CLUSTER_AUTH_PREVIOUS_SHARED_SECRET). -previous_shared_secret = "" - -# Replica-to-replica TLS for the consensus (tcp_replica) port. -[cluster.tls] -# When true every replica connection is wrapped in TLS 1.3 (ALPN -# "iggy-replica") before the replica handshake runs. Requires -# cluster.auth.enabled: TLS carries no client certificates, so it -# authenticates the acceptor only; the PSK handshake authenticates the -# peer, TLS supplies confidentiality. Off by default = plaintext replica -# traffic. Enabling it is a coordinated-restart change: a TLS dialer -# cannot talk to a plaintext acceptor or vice versa. -enabled = false - -# When true the node auto-generates a self-signed certificate at boot and -# the dialer accepts ANY peer certificate. When false (default), -# cert_file / key_file / ca_file are all required. -self_signed = false - -# PEM certificate chain presented by this node's acceptor side. -cert_file = "" - -# PEM private key matching cert_file. -key_file = "" - -# PEM trust anchor(s) the dialer verifies peer certificates against. -# Unused when self_signed = true. -ca_file = "" - -# Full roster of cluster members. Byte-identical on every node. The running -# node's identity is resolved at launch from the '--replica-id ' CLI -# flag, which selects the entry in this list that describes the current -# node. All other entries are remote peers. -# -# 'ip' is the node's roster address. Replica-to-replica traffic and -# follower-to-primary HTTP forwarding use it. It is not the bind interface for -# tcp/quic/http/websocket, which comes from each transport's own 'address' -# setting above; the roster supplies those transports their port only. A -# cluster spread across hosts therefore needs each transport's 'address' set to -# '0.0.0.0' or the routable NIC; the defaults below listen on loopback only, -# and a bind that cannot serve the advertised 'ip' is warned about at startup. -# -# Each node may also set 'advertised_address': the client-facing address -# handed out in cluster metadata and leader redirects. Set it when 'ip' is -# a private replica-network address unreachable by clients (Docker, -# Kubernetes, NAT). Accepts a literal IPv4/IPv6 address or a DNS hostname -# (RFC 1123: ASCII letters, digits, '-' and '.'; no port, no trailing dot). -# When unset, clients receive 'ip'. -# -# When different client networks need different addresses (a public -# 'advertised_address' would route in-VPC clients out through the public -# side), add per-network 'advertised_addresses' selectors: clients whose -# peer IP falls inside 'client_cidr' are handed 'address' instead of the -# catch-all. 'address' takes the same forms as 'advertised_address' -# (literal IP or RFC 1123 hostname, never a port - ports always come from -# 'ports'). At most 16 selectors per node; boot also rejects duplicate -# 'client_cidr' entries on one node (compared truncated, so '10.0.1.0/16' -# duplicates '10.0.0.0/16') and any two nodes advertising one host:port -# to overlapping client sets - reusing a host:port across nodes is legal -# only when no client would resolve both nodes to it. -# -# The longest matching prefix wins; clients matching no selector fall -# back to 'advertised_address', then 'ip'. Matching is per address -# family: '0.0.0.0/0' matches no IPv6 client and '::/0' matches no IPv4 -# client, so covering both families takes one selector per family (or the -# catch-all). IPv4-mapped IPv6 CIDRs ('::ffff:10.0.0.0/104') match like -# their IPv4 form only at prefix length 96 or longer; shorter ones match -# native IPv6 clients only. Matching sees the transport-level peer -# address, so clients behind a proxy or load balancer match the proxy's -# network, not their own. -# -# Every 'address' must be routable from inside its own 'client_cidr': -# leader-aware SDK clients redial whatever address metadata advertises, -# so a selector pointing at a host its own clients cannot reach strands -# them mid-redirect. Prefer literal IPs over hostnames - the SDKs differ -# in how they compare an advertised hostname against the address they -# dialed, and a mismatch costs a reconnect on every fresh connect. -# -# Note for rolling upgrades: older server binaries reject a TOML config -# containing 'advertised_addresses' but silently ignore the equivalent -# 'IGGY_CLUSTER_NODES_*_ADVERTISED_ADDRESSES_*' env vars; either way, -# upgrade every binary first, then add selectors. Mid-upgrade, an env-var -# roster would serve selector addresses from upgraded nodes and the -# catch-all from the rest. -# -# [[cluster.nodes]] -# name = "iggy-node-1" -# ip = "10.0.1.5" # replica plane + last-resort fallback -# advertised_address = "203.0.113.10" # catch-all for unmatched clients -# replica_id = 0 -# ports = { tcp = 8090, http = 3000, tcp_replica = 9090 } -# -# [[cluster.nodes.advertised_addresses]] -# client_cidr = "10.0.0.0/16" # in-VPC clients stay private -# address = "10.0.1.5" -# -# In cluster mode, 'ports' is the single source of listener ports: every -# enabled transport needs an explicit per-node port, otherwise the server -# refuses to start. -[[cluster.nodes]] -name = "iggy-node-1" -ip = "127.0.0.1" -replica_id = 0 -ports = { tcp = 8090, quic = 8080, http = 3000, websocket = 8092, tcp_replica = 9090 } - -[[cluster.nodes]] -name = "iggy-node-2" -ip = "127.0.0.1" -replica_id = 1 -ports = { tcp = 8091, quic = 8081, http = 3001, websocket = 8093, tcp_replica = 9091 } - -# Example additional node (commented out). tcp skips 8092-8094: those are the -# websocket ports of the three nodes, which collide once nodes share a host. -# [[cluster.nodes]] -# name = "iggy-node-3" -# ip = "192.168.1.100" -# advertised_address = "iggy-node-3.example.com" -# replica_id = 2 -# ports = { tcp = 8095, quic = 8082, http = 3002, websocket = 8094, tcp_replica = 9092 } - -# Sharding configuration -[system.sharding] -# CPU allocation - controls the number of shards and their CPU affinity. -# Possible values: -# - "all": Use all available CPU cores (default) -# - numeric value (e.g. 4): Use 4 shards (4 threads pinned to cores 0, 1, 2, 3) -# - range (e.g. "5..8"): Use 3 shards with affinity to cores 5, 6, 7 -# - numa settings: -# + "numa:auto": Use all available numa node, cores -# + "numa:nodes=0,1;cores=4;no_ht=true": Use NUMA node 0 and 1, each nodes use 4 cores, and no hyperthreads -cpu_allocation = "numa:auto" - -# Whether shard threads are pinned to dedicated CPU cores (default: true). -# Pinned cores are drawn from the process's allowed CPU set (affinity/cpuset -# mask), so the server cooperates with systemd `AllowedCPUs=` and container -# cpusets. Set to false when the server shares cores with other workloads -# (e.g. a multi-tenant host slicing CPU via cgroup quotas): unpinned shards -# let the kernel scheduler place threads freely instead of piling every -# process onto the same low-numbered cores. -pin_cores = true - -# Per-shard inter-shard inbox capacity. Bounded by design: consensus-frame -# drops recover via VSR retransmit, but cross-shard client-reply drops are -# terminal. Size for the worst-case sum of both: the consensus working set -# (~ the prepare queue depth of the planes the shard hosts - [metadata] on -# shard 0, [partition] elsewhere - times replica_count times directions) plus -# peak client-reply fan-out per shard. Both depths are tunable, so raising -# either raises the capacity needed here. -inbox_capacity = 1024 - -# Wall-clock budget for a single shard's bus drain on shutdown. Drives -# the per-shard watchdog and the parallel-join survivor path; sized -# larger than typical TCP RTT times in-flight write-batch so writers -# receive their full last `write_vectored_all` budget before the -# connection registry force-tears the bus. Slow-fsync hosts may need -# to extend this past the default. -shutdown_drain_timeout = "10 s" - -# Poll cadence for the cross-thread shutdown flag and for the -# metadata-handoff loops. Trades off Ctrl-C latency against idle wakeup -# cost; the default keeps shutdown observably prompt without measurable -# scheduler overhead. Must be less than or equal to shutdown_drain_timeout. -shutdown_poll_interval = "50 ms" - -# Hard wall-clock deadline for joining shard threads at process exit. A -# shard whose pump or listener wedges past this budget is abandoned with -# an error log instead of blocking exit forever. Must be at least -# shutdown_drain_timeout, or shards would be abandoned mid-drain. -shutdown_join_timeout = "30 s" - -# Safety-tick cadence for the partition reconciliation loop. The reconciler -# also wakes on every metadata commit from shard 0, so this only covers -# dropped wake-ups and the initial post-bootstrap convergence window. -reconcile_periodic_interval = "1 s" - -# WebSocket listener configuration. The frame-tuning knobs below are the -# live source for server-ng's WS / WSS plane; they are folded into a -# compio-ws WebSocketConfig once at bus construction. Each size knob is -# optional: commenting it out keeps the compio-ws (tungstenite) default -# noted next to it. A malformed size string fails config load. -[websocket] -enabled = true -address = "127.0.0.1:8092" - -# Target minimum size of the frame read buffer. compio-ws default: "128 KiB". -# read_buffer_size = "128 KiB" - -# Target buffer size for batched writes before flush. compio-ws -# default: "128 KiB". -# write_buffer_size = "128 KiB" - -# Hard ceiling on the write buffer; writes past it error instead of -# buffering, so it must exceed write_buffer_size by at least one message. -# compio-ws default: unlimited. -# max_write_buffer_size = "128 MiB" - -# Hard upper bound on a single inbound WebSocket message -# (post-fragment-reassembly). Must not exceed message_bus.max_message_size. -# compio-ws default: "64 MiB". -# max_message_size = "64 MiB" - -# Hard upper bound on a single inbound WebSocket frame -# (pre-fragment-reassembly). Must not exceed max_message_size. -# compio-ws default: "16 MiB". -# max_frame_size = "16 MiB" - -# Whether to accept unmasked frames from clients in violation of -# RFC 6455 client-to-server framing rules. Strict (false) by default. -accept_unmasked_frames = false - -[websocket.tls] -enabled = false -self_signed = true -cert_file = "core/certs/iggy_cert.pem" -key_file = "core/certs/iggy_key.pem" - -# Metadata consensus plane tunables (shard 0's VSR replica: users, -# streams, topics, sessions). Size these together: a deeper prepare queue -# admits more concurrent in-flight metadata ops (e.g. login storms), and -# the journal must hold enough slots that a forced checkpoint (triggered -# when remaining slots fall to the checkpoint margin, which itself is -# max(64, prepare_queue_depth)) stays rare. Validation enforces -# journal_slots >= 4 * max(64, prepare_queue_depth). -[metadata] -# Depth of the metadata prepare queue: how many uncommitted metadata ops -# may be in flight at once. Submits beyond it are rejected with the -# transient "metadata prepare queue is full" and retried by the SDK. -prepare_queue_depth = 32 - -# Size of the metadata WAL's in-memory index, in slots (one committed but -# not-yet-snapshotted op per slot). Larger values buy more headroom -# between forced checkpoints at the cost of memory and bigger WAL -# rewrites per checkpoint. -journal_slots = 1024 - -# Slot count of the VSR client table: how many distinct clients (TCP/QUIC/WS -# virtual clients and HTTP sessions together) hold live session state at once. -# When full, the client whose last commit is oldest is evicted and its next -# request re-registers. The HTTP session cap tracks this at half, so raising -# it lifts both. Must be between 2 and 65536. -clients_table_max = 8192 - -# Per-partition consensus plane tunables. Unlike [metadata] (one shard-0 -# plane), a pipeline exists per partition, so raising this multiplies pinned -# request-buffer memory by the partition count. Keep it modest. -[partition] -# Depth of a partition's prepare queue: how many uncommitted produce / -# consumer-offset ops may be in flight at once for that partition. Submits past -# it spill into a request queue of twice this depth; once both are full the -# server drops the request without a reply and the client retries on its own -# request timeout. Must be > 0 and <= 256. -prepare_queue_depth = 32 - -# Entries the evicted ring retains per multi-replica partition for journal -# repair after a peer rejoins. Larger widens the window a restarting peer can be -# served from the ring before falling back to bulk sync, at the cost of pinned -# memory per partition. Must be > 0 and <= 65536. Single-replica partitions -# retain nothing regardless. -evicted_ring_capacity = 4096 - -# Byte ceiling for the evicted ring per partition; whichever ring cap (this or -# evicted_ring_capacity) trips first evicts. Bounds the ring memory a burst of -# large batches can pin. Must be > 0 and <= "256 MiB". -evicted_ring_bytes_max = "16 MiB" - -# Byte budget for segment payloads a SERVING shard keeps resident to answer -# state-transfer chunk requests. PER SHARD, and shard count defaults to core -# count, so the process-wide high-water is this times the core count on top of -# page cache -- keep that product in mind before raising it. The default is a -# FIXED 2176 MiB: two sealed segments at the SHIPPED system.segment.size of -# 1 GiB, each of which can close one whole message_bus.max_message_size past -# its target, which is why it is not 2 GiB. It does not track your segment -# size. How many groups this shard serves at once IS derived from yours: -# floor(this / max(partition.transfer_artifact_bytes_max, -# system.segment.size + 64 MiB)), minimum one. So raising either that knob or -# system.segment.size without raising this lowers concurrency and can take it -# to one, serialising rejoins, and nothing at boot warns about it. -# Below one segment a single rejoining node thrashes the cache by itself and -# every miss re-reads and re-hashes a whole segment to serve one 256 KiB chunk. -# Running under the budget costs re-reads, not failures. -# Must be > 0 and <= "64 GiB". -transfer_served_cache_bytes_max = "2176 MiB" - -# Alloc ceiling for ONE received state-transfer artifact, per shard. The -# receiver holds it resident through verify, walk and staging write, and up to -# four transfers run at once. MUST cover system.segment.size plus -# message_bus.max_message_size (a segment may close one whole batch past its -# cap): under that, a legal segment is refused, the whole manifest with it, and -# the partition livelocks re-requesting it from every peer. Boot validates the -# floor. Raising this above the floor for headroom also DIVIDES the serving -# concurrency derived from transfer_served_cache_bytes_max above, so raise that -# in step. Must be > 0 and <= "64 GiB". -transfer_artifact_bytes_max = "1088 MiB" - -# Message bus configuration. -# Tunables for the inter-shard / inter-replica internal bus that ships -# consensus traffic between replicas and SDK-client traffic between -# shards. These knobs are consensus-liveness-critical (max_batch gates -# throughput under backpressure). Defaults match -# core::message_bus::config::MessageBusConfig::default(). - -[message_bus] -# Maximum number of BusMessage entries coalesced into a single writev(2) -# call. Hard upper bound: IOV_MAX/2 = 512 on Linux. -max_batch = 256 - -# Wire-level cap on a single framed message. -max_message_size = "64 MiB" - -# Bound on the per-peer mpsc queue. The writer task drains; the -# send_to_* path enqueues. -peer_queue_capacity = 256 - -# Interval between outbound reconnect attempts to peers with peer_id > self_id. -reconnect_period = "5 s" - -# Timeout for per-peer close drain (flush writer, tear down reader) -# before force-cancellation. -close_peer_timeout = "2 s" - -# Wall-clock bound on a single stream.shutdown() / ws.close() in the -# safe-shutdown sequence of the TLS-family transports. -close_grace = "2 s" - -# Wall-clock bound on a single connection's handshake phase. Threaded -# into compio::time::timeout(handshake_grace, ...) at each accept site -# (TCP-TLS rustls accept, WS HTTP-Upgrade, WSS combined TLS+WS, QUIC -# connecting.await + accept_bi.await) so a slowloris peer cannot pin -# per-conn channels + registry slot + spawned task indefinitely. -handshake_grace = "10 s" - -[extra.namespace] -max_streams = 4096 -max_topics = 4096 -max_partitions = 1_000_000 diff --git a/core/server-ng/server.http b/core/server-ng/server.http deleted file mode 100644 index b0be45ca82..0000000000 --- a/core/server-ng/server.http +++ /dev/null @@ -1,369 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -@url = http://localhost:3000 -@stream_id = 0 -@topic_id = 0 -@partition_id = 0 -@consumer_group_id = 1 -@consumer_id = 1 -@client_id = 1 -@partition_id_payload_base64 = AAAAAA== -@message_1_payload_base64 = aGVsbG8= -@message_2_payload_base64 = d29ybGQ= -@header_key_1_base64 = a2V5XzE= -@header_value_1_base64 = dmFsdWUgMQ== -@header_key_2_base64 = Kg== -@header_value_2_base64 = AAAA -@root_username = iggy -@root_password = iggy -@user1_username = user1 -@user1_password = secret -@access_token = secret -@root_id = 0 -@user1_id = 1 -@pat_name = dev_token -@pat_raw_token = secret - -### -GET {{url}}/ping - -### -POST {{url}}/users/login -Content-Type: application/json - -{ - "username": "{{root_username}}", - "password": "{{root_password}}" -} - -### -POST {{url}}/personal-access-tokens/login -Content-Type: application/json - -{ - "token": "{{pat_raw_token}}" -} - -### -GET {{url}}/stats -Authorization: Bearer {{access_token}} - -### -POST {{url}}/snapshot -Authorization: Bearer {{access_token}} -Content-Type: application/json - -{ - "compression": "Deflated", - "snapshot_types": ["All"] -} - -### -GET {{url}}/cluster/metadata -Authorization: Bearer {{access_token}} - -### -GET {{url}}/clients -Authorization: Bearer {{access_token}} - -### -GET {{url}}/clients/{{client_id}} -Authorization: Bearer {{access_token}} - -### -DELETE {{url}}/users/logout -Authorization: Bearer {{access_token}} - -### -POST {{url}}/users -Authorization: Bearer {{access_token}} -Content-Type: application/json - -{ - "username": "{{user1_username}}", - "password": "{{user1_password}}", - "status": "active", - "permissions": null -} - -### -GET {{url}}/users -Authorization: Bearer {{access_token}} - -### -GET {{url}}/users/{{user1_id}} -Authorization: Bearer {{access_token}} - -### -PUT {{url}}/users/{{user1_id}} -Authorization: Bearer {{access_token}} -Content-Type: application/json - -{ - "username": "{{user1_username}}", - "status": "active", - "permissions": null -} - -### -PUT {{url}}/users/{{user1_id}}/password -Authorization: Bearer {{access_token}} -Content-Type: application/json - -{ - "current_password": "{{user1_password}}", - "new_password": "secret1" -} - -### -PUT {{url}}/users/{{user1_id}}/permissions -Authorization: Bearer {{access_token}} -Content-Type: application/json - -{ - "permissions": { - "global": { - "manage_servers": false, - "read_servers": true, - "manage_users": true, - "read_users": true, - "manage_streams": false, - "read_streams": true, - "manage_topics": false, - "read_topics": true, - "poll_messages": true, - "send_messages": true - }, - "streams": { - "0": { - "manage_stream": false, - "read_stream": true, - "manage_topics": false, - "read_topics": true, - "poll_messages": true, - "send_messages": true, - "topics": { - "0": { - "manage_topic": false, - "read_topic": true, - "poll_messages": true, - "send_messages": true - } - } - } - } - } -} - - -### -DELETE {{url}}/users/{{user1_id}} -Authorization: Bearer {{access_token}} - -### -GET {{url}}/personal-access-tokens -Authorization: Bearer {{access_token}} - -### -POST {{url}}/personal-access-tokens -Authorization: Bearer {{access_token}} -Content-Type: application/json - -{ - "name": "{{pat_name}}", - "expiry": 1000 -} - -### -DELETE {{url}}/personal-access-tokens/{{pat_name}} -Authorization: Bearer {{access_token}} - -### -GET {{url}}/streams -Authorization: Bearer {{access_token}} - -### -GET {{url}}/streams/{{stream_id}} -Authorization: Bearer {{access_token}} - -### -POST {{url}}/streams -Authorization: Bearer {{access_token}} -Content-Type: application/json - -{ - "name": "stream1" -} - -### -PUT {{url}}/streams/{{stream_id}} -Authorization: Bearer {{access_token}} -Content-Type: application/json - -{ - "name": "stream1" -} - -### -DELETE {{url}}/streams/{{stream_id}} -Authorization: Bearer {{access_token}} - -### -DELETE {{url}}/streams/{{stream_id}}/purge -Authorization: Bearer {{access_token}} - -### -GET {{url}}/streams/{{stream_id}}/topics -Authorization: Bearer {{access_token}} - -### -GET {{url}}/streams/{{stream_id}}/topics/{{topic_id}} -Authorization: Bearer {{access_token}} - -### -POST {{url}}/streams/{{stream_id}}/topics -Authorization: Bearer {{access_token}} -Content-Type: application/json - -{ - "name": "topic1", - "partitions_count": 1, - "compression_algorithm": "none", - "max_topic_size": 0, - "message_expiry": 0 -} - -### -PUT {{url}}/streams/{{stream_id}}/topics/{{topic_id}} -Authorization: Bearer {{access_token}} -Content-Type: application/json - -{ - "name": "topic1", - "compression_algorithm": "none", - "max_topic_size": 0, - "message_expiry": 0 -} - -### -DELETE {{url}}/streams/{{stream_id}}/topics/{{topic_id}} -Authorization: Bearer {{access_token}} - -### -DELETE {{url}}/streams/{{stream_id}}/topics/{{topic_id}}/purge -Authorization: Bearer {{access_token}} - -### -POST {{url}}/streams/{{stream_id}}/topics/{{topic_id}}/partitions -Authorization: Bearer {{access_token}} -Content-Type: application/json - -{ - "partitions_count": 3 -} - -### -DELETE {{url}}/streams/{{stream_id}}/topics/{{topic_id}}/partitions?partitions_count=1 -Authorization: Bearer {{access_token}} - -### -### Delete segments -DELETE {{url}}/streams/{{stream_id}}/topics/{{topic_id}}/partitions/{{partition_id}}?segments_count=3 -Authorization: Bearer {{access_token}} - -### -POST {{url}}/streams/{{stream_id}}/topics/{{topic_id}}/messages -Authorization: Bearer {{access_token}} -Content-Type: application/json - -{ - "partitioning": { - "kind": "partition_id", - "value": "{{partition_id_payload_base64}}" - }, - "messages": [{ - "id": 0, - "payload": "{{message_1_payload_base64}}" - }, { - "id": 0, - "payload": "{{message_2_payload_base64}}", - "user_headers": [{ - "key": { - "kind": "string", - "value": "{{header_key_1_base64}}" - }, - "value": { - "kind": "string", - "value": "{{header_value_1_base64}}" - } - }, { - "key": { - "kind": "uint32", - "value": "{{header_key_2_base64}}" - }, - "value": { - "kind": "int32", - "value": "{{header_value_2_base64}}" - } - }] - }] -} - -### -GET {{url}}/streams/{{stream_id}}/topics/{{topic_id}}/messages?consumer_id={{consumer_id}}&partition_id={{partition_id}}&kind=offset&value=0&count=10&auto_commit=false -Authorization: Bearer {{access_token}} - -### -PUT {{url}}/streams/{{stream_id}}/topics/{{topic_id}}/consumer-offsets -Authorization: Bearer {{access_token}} -Content-Type: application/json - -{ - "consumer_id": {{consumer_id}}, - "partition_id": {{partition_id}}, - "offset": 1 -} - -### -GET {{url}}/streams/{{stream_id}}/topics/{{topic_id}}/consumer-offsets?consumer_id={{consumer_id}}&partition_id={{partition_id}} -Authorization: Bearer {{access_token}} - -### -DELETE {{url}}/streams/{{stream_id}}/topics/{{topic_id}}/consumer-offsets/{{consumer_id}}?partition_id={{partition_id}} -Authorization: Bearer {{access_token}} - -### -GET {{url}}/streams/{{stream_id}}/topics/{{topic_id}}/consumer-groups -Authorization: Bearer {{access_token}} - -### -GET {{url}}/streams/{{stream_id}}/topics/{{topic_id}}/consumer-groups/{{consumer_group_id}} -Authorization: Bearer {{access_token}} - -### -POST {{url}}/streams/{{stream_id}}/topics/{{topic_id}}/consumer-groups -Authorization: Bearer {{access_token}} -Content-Type: application/json - -{ - "name": "consumer_group_1" -} - -### -DELETE {{url}}/streams/{{stream_id}}/topics/{{topic_id}}/consumer-groups/{{consumer_group_id}} -Authorization: Bearer {{access_token}} diff --git a/core/server-ng/src/bootstrap.rs b/core/server-ng/src/bootstrap.rs deleted file mode 100644 index ad1671b201..0000000000 --- a/core/server-ng/src/bootstrap.rs +++ /dev/null @@ -1,4385 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::auth::warm_dummy_password_hash; -use crate::cluster_meta::ClusterRoster; -use crate::config_writer::write_current_config; -use crate::dispatch::{ - make_client_request_handler, make_deferred_client_request_handler, - make_deferred_replica_message_handler, make_list_clients_handler, make_metadata_submit_handler, - make_partition_read_handler, -}; -use crate::http; -use crate::partition_helpers::{ - build_partition_fresh, configure_consumer_offsets, ensure_initial_segment, - open_partition_superblock, restore_partition_view, validate_namespace_bounds, -}; -use crate::segment_recovery::{RecoveredSegment, load_persisted_segments}; -use crate::server_error::{ServerNgError, ShardJoinFailure, ShardJoinFailureKind}; -use crate::session_manager::SessionManager; -use configs::ng_sharding::{ - INBOX_CAPACITY_MAX, SHUTDOWN_DRAIN_TIMEOUT_MAX, SHUTDOWN_POLL_INTERVAL_MAX, -}; -use configs::server_ng::{NgSystemConfig, ServerNgConfig}; -use consensus::{ - ClientTable, LocalPipeline, MetadataHandle, PartitionsHandle, PipelineEntry, Sequencer, - VsrConsensus, -}; -// `try_send` / `try_recv` resolve through these traits on `MAsyncTx` / -// `MAsyncRx`; the metadata-handoff loops below depend on the -// non-blocking variants for cancel-safe shutdown polling. -use consensus::VsrState; -use crossfire::{AsyncRxTrait, AsyncTxTrait}; -use iggy_binary_protocol::{Operation, PrepareHeader}; -use iggy_common::defaults::{ - DEFAULT_ROOT_USERNAME, MAX_PASSWORD_LENGTH, MAX_USERNAME_LENGTH, MIN_PASSWORD_LENGTH, - MIN_USERNAME_LENGTH, -}; -use iggy_common::{Aes256GcmEncryptor, EncryptorKind, IggyByteSize, PartitionStats, variadic}; -use journal::prepare_journal::PrepareJournal; -use journal::superblock::{PingPongSuperblock, SuperblockStore}; -use journal::{Journal, JournalHandle}; -use message_bus::client_listener::{self, RequestHandler}; -use message_bus::installer; -use message_bus::installer::conn_info::{ClientConnMeta, ClientTransportKind}; -use message_bus::replica::auth::{self, ReplicaAuth}; -use message_bus::replica::handshake::{ReplicaHandshakeCtx, ReplicaTlsCtx}; -use message_bus::replica::io as replica_io; -use message_bus::replica::listener::{self as replica_listener, MessageHandler}; -use message_bus::transports::quic::server_config_with_cert; -use message_bus::transports::tls::{ - AcceptAnyServerCert, REPLICA_ALPN, TlsServerCredentials, install_default_crypto_provider, - load_ca_pem, load_pem, self_signed_for_loopback, -}; -use message_bus::{ - AcceptedClientFn, AcceptedQuicClientFn, AcceptedReplicaFn, AcceptedTlsClientFn, - AcceptedWsClientFn, AcceptedWssClientFn, ConnectionInstaller, DialedReplicaFn, IggyMessageBus, - MAX_INFLIGHT_REPLICA_HANDSHAKES, MessageBus, ReplicaOwnerTable, connector, -}; -use metadata::IggyMetadata; -use metadata::MuxStateMachine; -use metadata::ReplicaIdentity; -use metadata::impls::metadata::{IggySnapshot, StreamsFrontend}; -use metadata::impls::recovery::recover; -use metadata::stm::mux::WithFactory; -use metadata::stm::snapshot::Snapshot; -use metadata::stm::stream::{Partition, Streams}; -use metadata::stm::user::Users; -use partitions::{ - IggyIndexWriter, IggyPartition, IggyPartitions, MessagesWriter, PartitionsConfig, -}; -use rustls::pki_types::ServerName; -use server_common::Message; -use server_common::bootstrap::create_directories; -use server_common::crypto; -use server_common::executor::create_shard_executor; -use server_common::log::{Logging, LoggingSettings, TelemetrySettings}; -use server_common::sharding::{IggyNamespace, PartitionLocation, ShardId}; -use shard::builder::IggyShardBuilder; -use shard::metrics::{ShardMetrics, frame_drop_reason, frame_drop_variant}; -use shard::shards_table::{PapayaShardsTable, ShardsTable, calculate_shard_assignment}; -use shard::{ - CoordinatorConfig, IggyShard, LifecycleFrame, ListClientsHandler, MetadataSubmitHandler, - PartitionConsensusConfig, PartitionReadHandler, Receiver as ShardReceiver, ShardFrame, - ShardIdentity, TaggedSender, channel, shard_mesh_channels, -}; -use shard_allocator::{ShardAllocator, ShardInfo}; -use std::cell::RefCell; -use std::collections::HashMap; -use std::env; -use std::net::{IpAddr, SocketAddr}; -use std::path::{Path, PathBuf}; -use std::rc::{Rc, Weak}; -use std::sync::Arc; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::thread; -use std::time::{Duration, Instant}; -use tracing::{error, info, warn}; - -const SHARD_REPLICA_ID: u8 = 0; - -pub const IGGY_ROOT_USERNAME_ENV: &str = "IGGY_ROOT_USERNAME"; -pub const IGGY_ROOT_PASSWORD_ENV: &str = "IGGY_ROOT_PASSWORD"; - -type ServerNgMuxStateMachine = MuxStateMachine; - -/// Cross-thread bundle carrying one `ReadHandleFactory` per metadata -/// state. Shard 0 mints one after `recover()` and broadcasts a clone to -/// every peer shard; each peer rebuilds a reader-mode -/// [`ServerNgMuxStateMachine`] on its own runtime, skipping the WAL. -type ServerNgMetadataBundle = ::Bundle; - -pub(crate) type ServerNgMetadata = IggyMetadata< - VsrConsensus>, - PrepareJournal, - IggySnapshot, - ServerNgMuxStateMachine, ->; - -/// The shard type the dispatch layer is generic over. -/// -/// `B`/`MJ`/`S`/`SB` are free; the metadata state machine (`M`) and shards -/// table (`T`) are pinned, being identical in production and the simulator. -/// Production instantiates it as [`ServerNgShard`], defaulting `SB` to the -/// on-disk [`PingPongSuperblock`]; the simulator supplies its own -/// `B`/`MJ`/`S`/`SB`. -pub type ShellShard = - IggyShard; - -/// Late-bound self-reference the deferred dispatch handlers upgrade per frame. -pub type ShellShardHandle = - Rc>>>>; - -/// Bus bounds the dispatch/pump path needs (matches `run_message_pump`). -/// Blanket-impl'd, so it is only shorthand for the four underlying bounds. -pub trait ShellBus: MessageBus + ConnectionInstaller + Clone + 'static {} -impl ShellBus for B {} - -/// The five dispatch handlers a shard is built with, plus the -/// [`SessionManager`] the request-plane pair shares. -/// -/// Both production ([`build_shard_for_thread`]) and the simulator's shell -/// mode construct these through [`wire_shell_handlers`], so the request -/// plane is wired one way. The simulator's shell-off fast path uses -/// [`ShellHandlers::noop`] instead. -pub struct ShellHandlers { - pub on_replica_message: MessageHandler, - pub on_client_request: RequestHandler, - pub on_metadata_submit: MetadataSubmitHandler, - pub on_list_clients: ListClientsHandler, - pub on_partition_read: PartitionReadHandler, - /// Bound by the client-request handler, read by the get-clients - /// handler; the caller keeps it to reach locally-homed sessions. - pub sessions: Rc>, -} - -impl ShellHandlers { - /// Inert handlers for the shell-off fast path: every callback is a - /// no-op over an empty [`SessionManager`]. Behaviorally identical to - /// hand-written no-op closures, so a caller can keep one destructure - /// site across both toggle states. - #[must_use] - pub fn noop() -> Self { - Self { - on_replica_message: Rc::new(|_, _| {}), - on_client_request: Rc::new(|_, _| {}), - on_metadata_submit: Rc::new(|_| {}), - on_list_clients: Rc::new(|_| {}), - on_partition_read: Rc::new(|_, _, _| {}), - sessions: Rc::new(RefCell::new(SessionManager::new())), - } - } -} - -/// Build the deferred dispatch handlers for `shard_handle` against `bus`. -/// -/// They share one fresh [`SessionManager`]. The caller must set the weak -/// self-reference in `shard_handle` once the shard is built, so the -/// handlers can upgrade it per frame. -pub fn wire_shell_handlers( - bus: &B, - shard_handle: &ShellShardHandle, - system_config: Arc, - max_tokens_per_user: u32, -) -> ShellHandlers -where - B: ShellBus, - MJ: JournalHandle + 'static, - MJ::Target: Journal, Header = PrepareHeader>, - S: 'static, - SB: SuperblockStore + 'static, -{ - let sessions = Rc::new(RefCell::new(SessionManager::new())); - ShellHandlers { - on_replica_message: make_deferred_replica_message_handler(shard_handle), - on_client_request: make_deferred_client_request_handler( - bus, - shard_handle, - &sessions, - system_config, - max_tokens_per_user, - ), - on_metadata_submit: make_metadata_submit_handler(shard_handle), - on_list_clients: make_list_clients_handler(&sessions), - on_partition_read: make_partition_read_handler(shard_handle), - sessions, - } -} - -pub type ServerNgShard = ShellShard, PrepareJournal, IggySnapshot>; - -/// Result of a multi-shard bootstrap. -/// -/// Carries the cross-thread shutdown flag and one OS-thread `JoinHandle` -/// per shard. The caller flips the flag via [`Self::install_ctrlc_handler`] -/// and then drains every shard via [`Self::join_all`], bounded by -/// `join_timeout` (`system.sharding.shutdown_join_timeout`). -pub struct ShardHandles { - shutdown_flag: Arc, - shard_threads: Vec<(u16, thread::JoinHandle>)>, - join_timeout: Duration, -} - -impl ShardHandles { - /// Install a SIGINT/Ctrl-C handler that flips the shutdown flag on - /// the first signal. A second signal is logged but otherwise - /// ignored so an in-flight WAL fsync or replica drain runs to - /// completion. - /// - /// # Errors - /// - /// Returns the underlying `ctrlc::Error` if the handler cannot be - /// installed (typically because another handler already owns the - /// signal). - pub fn install_ctrlc_handler(&self) -> Result<(), ctrlc::Error> { - let flag = Arc::clone(&self.shutdown_flag); - ctrlc::set_handler(move || { - if flag.swap(true, Ordering::Relaxed) { - // Second Ctrl-C: leave the shutdown machinery to drain. - // Refusing to abort here keeps the WAL fsync / replica - // drain from being interrupted mid-frame. - warn!("second Ctrl-C ignored; server is already shutting down"); - } else { - info!("Ctrl-C received; signalling server shutdown"); - } - }) - } - - /// Drain every shard thread. This is the main thread's park for the - /// server's whole lifetime, so shards are awaited WITHOUT any time - /// bound while the server runs; the `shutdown_join_timeout` clock - /// only starts once the cross-thread shutdown flag flips (Ctrl-C or - /// a shard failure). Each shard's outcome is logged (`info` on clean - /// exit, `error` on Err, panic, or wedge). If any shard failed, - /// returns every failure together as - /// [`ServerNgError::ShardJoinFailures`] so the operator sees the - /// full set rather than just the first. - /// - /// A shard whose thread is still running when the post-shutdown - /// deadline passes is abandoned (its `JoinHandle` dropped, the OS - /// thread left to die with the process) and reported as - /// [`ShardJoinFailureKind::Wedged`]: a wedged pump or listener must - /// not block process exit forever. - /// - /// # Errors - /// - /// Returns [`ServerNgError::ShardJoinFailures`] if any shard - /// returned a `Result::Err`, panicked, or wedged past the deadline. - /// The variant carries every per-shard failure in shard-id order so - /// the caller does not need to read the trace log to discover - /// late-failing shards. - pub fn join_all(self) -> Result<(), ServerNgError> { - let mut failures: Vec = Vec::new(); - // Armed on the first poll that observes the shutdown flag, shared - // across all shards: one budget covers the whole drain, not one - // budget per shard. - let mut deadline: Option = None; - // Shards run thread-per-core with compio's blocking fallback pool - // disabled, so an io_uring opcode the kernel lacks aborts every shard - // with the same panic. Surface the actionable diagnostic once. - let mut io_uring_diagnostic_shown = false; - for (shard_id, handle) in self.shard_threads { - let Some(joined) = join_until_shutdown_deadline( - handle, - &self.shutdown_flag, - self.join_timeout, - &mut deadline, - ) else { - error!( - shard_id, - waited = ?self.join_timeout, - "shard thread still running at the shutdown join deadline; abandoning it" - ); - failures.push(ShardJoinFailure { - shard_id, - kind: ShardJoinFailureKind::Wedged { - waited: self.join_timeout, - }, - }); - continue; - }; - match joined { - Ok(Ok(())) => { - info!(shard_id, "shard thread exited cleanly"); - } - Ok(Err(error)) => { - error!(shard_id, error = %error, "shard thread returned error"); - failures.push(ShardJoinFailure { - shard_id, - kind: ShardJoinFailureKind::Error(Box::new(error)), - }); - } - Err(panic_payload) => { - let message = panic_payload_to_string(&*panic_payload); - error!(shard_id, message = %message, "shard thread panicked"); - if !io_uring_diagnostic_shown - && message - .contains(server_common::diagnostics::ASYNCIFY_POOL_DISABLED_PANIC_MSG) - { - server_common::diagnostics::print_incomplete_io_uring_ops_info(); - io_uring_diagnostic_shown = true; - } - failures.push(ShardJoinFailure { - shard_id, - kind: ShardJoinFailureKind::Panic { message }, - }); - } - } - } - if failures.is_empty() { - Ok(()) - } else { - Err(ServerNgError::ShardJoinFailures { failures }) - } - } -} - -/// Poll cadence for the bounded shard joins. Coarse enough to cost -/// nothing during a normal drain, fine enough that exit latency past -/// the last shard's return stays imperceptible. -const JOIN_POLL_INTERVAL: Duration = Duration::from_millis(25); - -/// Join `handle`, waiting indefinitely while the server runs. The -/// `join_timeout` clock starts only when `shutdown_flag` is observed set -/// (arming the caller-shared `deadline` once, so all shards drain under -/// ONE budget); a running server parked here for hours must never be -/// mistaken for a wedged shard. `None` means the thread was still -/// running at the post-shutdown deadline and the handle was dropped -/// (the OS thread keeps running detached; process exit reaps it). -/// `JoinHandle` has no timed join, so this polls `is_finished` at -/// [`JOIN_POLL_INTERVAL`]; the closing `join()` on a finished thread -/// returns immediately. -fn join_until_shutdown_deadline( - handle: thread::JoinHandle>, - shutdown_flag: &AtomicBool, - join_timeout: Duration, - deadline: &mut Option, -) -> Option>> { - while !handle.is_finished() { - if deadline.is_none() && shutdown_flag.load(Ordering::Relaxed) { - *deadline = Some(Instant::now() + join_timeout); - } - if let Some(deadline) = deadline - && Instant::now() >= *deadline - { - return None; - } - thread::sleep(JOIN_POLL_INTERVAL); - } - Some(handle.join()) -} - -/// Best-effort extraction of the panic message from a -/// `Box` returned by `JoinHandle::join`. Tries the two -/// payload shapes the standard library guarantees (`&'static str` and -/// `String`) and falls back to a placeholder so the panic still surfaces -/// in the error chain. -fn panic_payload_to_string(payload: &(dyn std::any::Any + Send)) -> String { - if let Some(s) = payload.downcast_ref::<&'static str>() { - return (*s).to_string(); - } - if let Some(s) = payload.downcast_ref::() { - return s.clone(); - } - "".to_string() -} - -/// Joins survivor shard threads after a partial-spawn failure, bounded -/// by the same `shutdown_join_timeout` budget as the normal exit path. -/// -/// Polls every survivor's `is_finished` in one loop instead of spawning -/// per-survivor joiner threads: the likely OS state on this path is -/// `pthread_create` EAGAIN (the parent spawn just failed with it), so -/// nothing here may create threads, and polling drains all survivors in -/// parallel anyway. A survivor still running at the deadline is -/// abandoned with an error log so the failed bootstrap can surface its -/// spawn error instead of hanging on a wedged shard. -fn join_partial_shard_survivors( - shard_threads: Vec<(u16, thread::JoinHandle>)>, - join_timeout: Duration, -) { - let deadline = Instant::now() + join_timeout; - let mut remaining = shard_threads; - loop { - let mut still_running = Vec::with_capacity(remaining.len()); - for (shard_id, survivor) in remaining { - if survivor.is_finished() { - let _ = survivor.join(); - info!(shard_id, "survivor shard thread drained"); - } else { - still_running.push((shard_id, survivor)); - } - } - remaining = still_running; - if remaining.is_empty() || Instant::now() >= deadline { - break; - } - thread::sleep(JOIN_POLL_INTERVAL); - } - for (shard_id, _survivor) in remaining { - error!( - shard_id, - waited = ?join_timeout, - "survivor shard thread still running at the shutdown join deadline; abandoning it" - ); - } -} - -/// Flips the cross-thread shutdown flag on `Drop` unless disarmed. -/// -/// A shard thread that exits via an error `?` or a panic unwind would -/// otherwise leave sibling shards parked forever on `bus.token().wait()`: -/// their watchdogs never observe the flag and the bus has no -/// `Drop`-triggered shutdown. Arming this for the whole thread body makes -/// every non-clean exit drive sibling-shard teardown. Disarmed only on a -/// clean `Ok(())`. -struct ShutdownOnDrop { - flag: Arc, - armed: bool, -} - -impl ShutdownOnDrop { - const fn new(flag: Arc) -> Self { - Self { flag, armed: true } - } - - const fn disarm(&mut self) { - self.armed = false; - } -} - -impl Drop for ShutdownOnDrop { - fn drop(&mut self) { - if self.armed { - self.flag.store(true, Ordering::Relaxed); - } - } -} - -/// Shard-local end of the metadata bundle handoff. -/// -/// Shard 0 owns the WAL writer and runs `recover()` to build the only -/// `WriteHandle`-bearing [`ServerNgMuxStateMachine`]. It then mints a -/// [`ServerNgMetadataBundle`] (a tuple of `Send + Sync` -/// `ReadHandleFactory`s) and pushes one clone per peer onto `bundle_tx`. -/// Every other shard receives the bundle and rebuilds a reader-mode -/// `MuxStateMachine` on its own runtime - no WAL access, no replay, no -/// `RecoverySync` two-phase fence. The old phase-2 WAL fence is gone -/// because peers no longer scan the WAL. They do still scan live shared -/// metadata to load their on-disk partitions, so a separate listener -/// fence is still required - see [`BootstrapBarrier`]. -/// -/// The channel is bounded to the peer count so shard 0's `send` never -/// blocks beyond a peer drain. A peer that dies before recv drops its -/// `bundle_rx`, so shard 0's `send` eventually sees a disconnected -/// channel; the cross-thread shutdown flag drives every waiter out of -/// its `recv` loop if shard 0 panics before broadcasting. -enum MetadataHandoff { - Owner { - bundle_tx: crossfire::MAsyncTx>, - }, - Waiter { - bundle_rx: crossfire::MAsyncRx>, - }, -} - -/// Reverse handshake to [`MetadataHandoff`]: gates shard 0's client -/// listeners until every peer has loaded its on-disk partitions. -/// -/// Peers build their owned-partition set from live shared metadata and -/// load each segment from disk in `build_shard_for_thread`. If shard 0 -/// opened listeners the instant `broadcast_metadata_bundle` returned -/// (peers have only *received* the bundle, not *loaded* partitions), a -/// client could create a partition before a peer's load scan finished. -/// That freshly committed partition would surface in the peer's scan -/// with no segment dir on disk yet, and `load_partition`'s `walk_dir` -/// would fail with `CannotReadPartitions`, aborting the whole node. A -/// partition created after boot must take the runtime reconciler path -/// (which creates its dir), never the bootstrap load path. -/// -/// Shard 0 (`Owner`) drains one signal per peer before binding -/// listeners; each peer (`Waiter`) sends one once its load completes. -/// The cross-thread shutdown flag drives both sides out of their poll -/// loop if any shard dies mid-boot. -enum BootstrapBarrier { - Owner { - ready_rx: crossfire::MAsyncRx>, - }, - Waiter { - ready_tx: crossfire::MAsyncTx>, - }, -} - -struct TcpTopology { - /// Domain-separation cluster id derived from `cluster.name`; threaded to - /// every consensus instance and the replica handshake so frames agree. - cluster_id: u128, - self_replica_id: u8, - replica_count: u8, - client_listen_addr: SocketAddr, - replica_listen_addr: Option, - ws_listen_addr: Option, - quic_listen_addr: Option, - http_listen_addr: Option, - tcp_tls_listen_addr: Option, - peers: Vec<(u8, SocketAddr)>, -} - -struct LocalClientAcceptFns { - tcp: AcceptedClientFn, - ws: AcceptedWsClientFn, - quic: AcceptedQuicClientFn, - tcp_tls: AcceptedTlsClientFn, - wss: AcceptedWssClientFn, -} - -#[derive(Default)] -struct BoundClientListeners { - tcp: Option, - tcp_tls: Option, - ws: Option, - quic: Option, -} - -/// Load config, prepare directories, and complete late logging init. -/// -/// # Errors -/// -/// Returns an error if config loading, directory preparation, or logging -/// setup fails. -pub async fn load_config(logging: &mut Logging) -> Result { - let config = ServerNgConfig::load() - .await - .map_err(ServerNgError::Config)?; - create_directories(&config.system).await.map_err(|source| { - error!( - system_path = %config.system.get_system_path(), - error = %source, - "failed to prepare server-ng directories" - ); - source - })?; - logging - .late_init( - config.system.get_system_path(), - &LoggingSettings::from(&config.system.logging), - &TelemetrySettings::from(&config.telemetry), - ) - .map_err(ServerNgError::Logging)?; - - Ok(config) -} - -/// Resolve the operator's `cpu_allocation` into concrete shard -/// assignments plus the checked `u16` shard count. -/// -/// Shard ids index `ReplicaOwnerTable` slots as `u16`. `OWNER_NONE` -/// (`u16::MAX`) is reserved as the empty-slot sentinel, so a server -/// configured with `u16::MAX` shards would mint a shard id that -/// collides with the sentinel and an owner-table lookup could never -/// tell that shard apart from an unowned slot. Reject at boot so the -/// invariant is held by the type system, not by hoping the operator -/// never configures 65535 cores worth of shards. -fn resolve_shard_assignments( - sharding: &configs::ng_sharding::ShardingConfig, -) -> Result<(Vec, u16), ServerNgError> { - let allocator = ShardAllocator::new(&sharding.cpu_allocation, sharding.pin_cores) - .map_err(ServerNgError::ShardAllocator)?; - let assignments = allocator - .to_shard_assignments() - .map_err(ServerNgError::ShardAllocator)?; - if assignments.is_empty() { - return Err(ServerNgError::ShardsCountZero); - } - match u16::try_from(assignments.len()) { - Ok(count) if count < message_bus::OWNER_NONE => Ok((assignments, count)), - _ => Err(ServerNgError::ShardsCountOverflow { - count: assignments.len(), - }), - } -} - -/// Re-validate the runtime sharding knobs that the per-shard runtime -/// consumes directly. Mirrors `ShardingConfig::validate` so a caller -/// that built the config without running it (e.g. tests, embedded -/// usage) cannot OOM at boot or wedge process exit with an out-of-range -/// value. -fn validate_sharding_runtime_knobs( - sharding: &configs::ng_sharding::ShardingConfig, -) -> Result<(), ServerNgError> { - let inbox_capacity = sharding.inbox_capacity; - if inbox_capacity == 0 || inbox_capacity > INBOX_CAPACITY_MAX { - return Err(ServerNgError::InvalidInboxCapacity { - value: inbox_capacity, - max: INBOX_CAPACITY_MAX, - }); - } - let drain_timeout = sharding.shutdown_drain_timeout.get_duration(); - if drain_timeout.is_zero() || drain_timeout > SHUTDOWN_DRAIN_TIMEOUT_MAX { - return Err(ServerNgError::InvalidShutdownDrainTimeout { - value: drain_timeout, - max: SHUTDOWN_DRAIN_TIMEOUT_MAX, - }); - } - let poll_interval = sharding.shutdown_poll_interval.get_duration(); - if poll_interval.is_zero() || poll_interval > SHUTDOWN_POLL_INTERVAL_MAX { - return Err(ServerNgError::InvalidShutdownPollInterval { - value: poll_interval, - max: SHUTDOWN_POLL_INTERVAL_MAX, - }); - } - // Ordering: a poll cadence coarser than the drain budget makes the - // cross-thread shutdown flag effectively unobservable during teardown. - if poll_interval > drain_timeout { - return Err(ServerNgError::ShutdownPollExceedsDrain { - poll: poll_interval, - drain: drain_timeout, - }); - } - Ok(()) -} - -/// Spawn the multi-shard `server-ng` runtime. -/// -/// Resolves shard count + CPU affinities from -/// `system.sharding.cpu_allocation`, builds canonical-ordered -/// `(senders, inboxes)` channels, and spawns one OS thread per shard. -/// -/// Each thread pins itself (`nix::sched::sched_setaffinity` on Linux via -/// [`ShardInfo::bind_cpu`]), binds memory to its NUMA node when -/// configured, builds a fresh `compio::runtime::Runtime` (one -/// `io_uring` instance per shard), and runs `shard_main` inside it. -/// -/// Returns [`ShardHandles`] containing the cross-thread shutdown flag -/// and the per-shard `JoinHandle`s. The caller (`main.rs`) installs a -/// `ctrlc` handler that flips the flag, then `.join()`s every handle. -/// -/// # Errors -/// -/// Returns an error if shard allocation fails, the inbox capacity is -/// invalid, or any OS thread fails to spawn. Per-shard recovery / -/// listener / consensus failures surface through the per-thread `Result` -/// the caller observes on `.join()`. -/// -/// # Panics -/// -/// Panics if [`shard_mesh_channels`] returns an inbox slot already -/// consumed - a bootstrap programming error that would only fire if this -/// function were called twice with the same inboxes. -#[allow(clippy::too_many_lines)] -pub fn bootstrap( - config: ServerNgConfig, - current_replica_id: Option, -) -> Result { - warm_dummy_password_hash(); - // The sync GetStats read path has no access to server config, so capture - // the data directory here for its disk-usage reporting. - crate::responses::init_stats_data_path(config.system.get_system_path().into()); - let (assignments, total_shards) = resolve_shard_assignments(&config.system.sharding)?; - let shards_count = assignments.len(); - - // Re-check the full valid range, not just the zero floor: a caller - // that built the config without running `ShardingConfig::validate` - // would otherwise OOM at boot allocating an oversized inbox channel, - // busy-loop every shutdown watchdog on a zero poll cadence, or wedge - // process exit on an unbounded drain budget. - let inbox_capacity = config.system.sharding.inbox_capacity; - validate_sharding_runtime_knobs(&config.system.sharding)?; - - let (senders, mut inboxes) = shard_mesh_channels(total_shards, inbox_capacity); - let shutdown_flag = Arc::new(AtomicBool::new(false)); - let config = Arc::new(config); - // One owner table per server process, Arc-cloned into every shard's bus so - // any shard's bus reads the same atomic slots that the owning - // shard's installer / disconnect path writes. - let owner_table = Arc::new(ReplicaOwnerTable::new()); - - // Single-shot bundle handoff (see `MetadataHandoff`): shard 0 sends - // one cloned `ServerNgMetadataBundle` per peer; each peer drains - // exactly one. Bounded to the peer count so shard 0's broadcast - // never blocks past a peer drain. A single-shard deployment (zero - // peers) still needs a non-zero capacity, so clamp up explicitly - // rather than relying on crossfire's internal cap=0 -> 1 promotion. - // If a peer dies before recv, shard 0's `send` eventually sees a - // disconnected channel; the cross-thread shutdown flag drives every - // waiter out of its recv loop if shard 0 panics before broadcasting. - let metadata_peers = shards_count.saturating_sub(1).max(1); - let (metadata_bundle_tx, metadata_bundle_rx) = - crossfire::mpmc::bounded_async::(metadata_peers); - - // Reverse barrier (see `BootstrapBarrier`): every peer sends one - // signal once it finishes loading its on-disk partitions; shard 0 - // drains them all before binding listeners. Bounded to the peer - // count so a sender never blocks (each peer sends exactly once). - let (ready_tx, ready_rx) = crossfire::mpmc::bounded_async::(metadata_peers); - - let mut shard_threads: Vec<(u16, thread::JoinHandle>)> = - Vec::with_capacity(shards_count); - // Shared metadata-group view: written by shard 0's publisher task, read by - // every shard's cluster-metadata roster so leader marking works off-shard. - let metadata_view = Arc::new(AtomicU64::new(crate::cluster_meta::METADATA_VIEW_UNKNOWN)); - for (idx, assignment) in assignments.into_iter().enumerate() { - #[allow(clippy::cast_possible_truncation)] - let shard_id = idx as u16; - let inbox = inboxes[idx] - .take() - .expect("shard_mesh_channels populates every inbox slot exactly once"); - let senders_for_shard = senders.clone(); - let config_for_shard = Arc::clone(&config); - let shutdown_flag_for_shard = Arc::clone(&shutdown_flag); - let owner_table_for_shard = Arc::clone(&owner_table); - let metadata_handoff_for_shard = if shard_id == 0 { - MetadataHandoff::Owner { - bundle_tx: metadata_bundle_tx.clone(), - } - } else { - MetadataHandoff::Waiter { - bundle_rx: metadata_bundle_rx.clone(), - } - }; - let barrier_for_shard = if shard_id == 0 { - BootstrapBarrier::Owner { - ready_rx: ready_rx.clone(), - } - } else { - BootstrapBarrier::Waiter { - ready_tx: ready_tx.clone(), - } - }; - - let metadata_view_for_shard = Arc::clone(&metadata_view); - let handle = match thread::Builder::new() - .name(format!("shard-{shard_id}")) - .spawn(move || -> Result<(), ServerNgError> { - run_shard_thread( - shard_id, - total_shards, - current_replica_id, - assignment, - senders_for_shard, - inbox, - config_for_shard, - shutdown_flag_for_shard, - metadata_handoff_for_shard, - barrier_for_shard, - owner_table_for_shard, - metadata_view_for_shard, - ) - }) { - Ok(handle) => handle, - Err(source) => { - // Signal every shard already spawned before propagating, so - // their watchdog loops drive `bus.shutdown(...)` and the - // process can exit instead of hanging on stuck OS threads. - shutdown_flag.store(true, Ordering::Relaxed); - // Drop bootstrap's own channel clones before joining - // survivors. Otherwise a peer waiting on `bundle_rx.recv` - // would never observe the sender side disconnecting and - // would hang until the shutdown watchdog kicks the bus. - drop(metadata_bundle_tx); - drop(metadata_bundle_rx); - drop(ready_tx); - drop(ready_rx); - join_partial_shard_survivors( - shard_threads, - config.system.sharding.shutdown_join_timeout.get_duration(), - ); - return Err(ServerNgError::ShardSpawnFailed { shard_id, source }); - } - }; - shard_threads.push((shard_id, handle)); - } - - // Drop bootstrap's own channel clones now that every shard owns its - // half. Keeping them on bootstrap's stack would deadlock a peer - // whose `bundle_rx.recv` only completes once every sender - // disconnects. - drop(metadata_bundle_tx); - drop(metadata_bundle_rx); - drop(ready_tx); - drop(ready_rx); - - info!( - shards_count, - "server-ng bootstrap dispatched; awaiting shard runtimes" - ); - - Ok(ShardHandles { - shutdown_flag, - shard_threads, - join_timeout: config.system.sharding.shutdown_join_timeout.get_duration(), - }) -} - -/// Per-shard OS thread entry. Pins CPU + memory, builds the compio -/// runtime, and `block_on`s `shard_main`. -#[allow(clippy::needless_pass_by_value, clippy::too_many_arguments)] -fn run_shard_thread( - shard_id: u16, - total_shards: u16, - replica_id: Option, - assignment: ShardInfo, - senders: Vec, - inbox: ShardReceiver, - config: Arc, - shutdown_flag: Arc, - metadata_handoff: MetadataHandoff, - barrier: BootstrapBarrier, - owner_table: Arc, - metadata_view: Arc, -) -> Result<(), ServerNgError> { - // Armed for the whole thread body: a post-spawn error `?` or a panic - // unwind here must flip `shutdown_flag` so sibling watchdogs drive - // their bus shutdown instead of parking forever on `bus.token().wait()`. - let mut shutdown_guard = ShutdownOnDrop::new(Arc::clone(&shutdown_flag)); - - assignment - .bind_cpu() - .map_err(|source| ServerNgError::CpuAffinityFailed { shard_id, source })?; - assignment - .bind_memory() - .map_err(|source| ServerNgError::MemoryAffinityFailed { shard_id, source })?; - - // `enrich_runtime_create_error` folds the io_uring remediation (raise - // `ulimit -l`, unblock seccomp, kernel-flag floor) into the error, so the - // guidance survives into the shard-join failure report instead of only - // stderr. Multi-shard boxes exhaust RLIMIT_MEMLOCK on per-shard rings - // before the bootstrap runtime does, so this path needs it most. - let runtime = create_shard_executor().map_err(|source| { - let source = server_common::diagnostics::enrich_runtime_create_error(source); - ServerNgError::ShardRuntimeCreateFailed { shard_id, source } - })?; - - let result = runtime.block_on(async move { - // `shard_main`'s future grows past clippy's `large_futures` cap - // (it ferries the metadata handoff, bus, builders, and inflight - // I/O in one state machine). Heap-pin it so the top-level - // `block_on` future stays small; one allocation per startup buys - // the stack budget back. - Box::pin(shard_main( - shard_id, - total_shards, - replica_id, - senders, - inbox, - &config, - shutdown_flag, - metadata_handoff, - barrier, - owner_table, - metadata_view, - )) - .await - }); - - if result.is_ok() { - shutdown_guard.disarm(); - } - result -} - -/// Per-shard async lifecycle. Builds the bus, recovers metadata, -/// constructs the `IggyShard` for this shard's slice of partitions, -/// wires listeners on shard 0, and runs the message pump until -/// shutdown. -#[allow(clippy::too_many_arguments, clippy::too_many_lines)] -async fn shard_main( - shard_id: u16, - total_shards: u16, - replica_id: Option, - senders: Vec, - inbox: ShardReceiver, - config: &ServerNgConfig, - shutdown_flag: Arc, - metadata_handoff: MetadataHandoff, - barrier: BootstrapBarrier, - owner_table: Arc, - metadata_view: Arc, -) -> Result<(), ServerNgError> { - let topology = resolve_tcp_topology(config, replica_id)?; - let bus = Rc::new(IggyMessageBus::with_config_and_owner_table( - shard_id, - config, - owner_table, - )); - // Every shard can own a delegated replica connection, so every - // shard's bus needs the handshake identity (the handshake itself - // runs on the owning shard, not on shard 0). - bus.set_replica_handshake_ctx(ReplicaHandshakeCtx { - cluster_id: topology.cluster_id, - self_id: topology.self_replica_id, - replica_count: topology.replica_count, - auth: load_replica_auth(config).map(Rc::new), - tls: load_replica_tls_ctx(config, &topology)?.map(Rc::new), - }); - - let drain_timeout = config.system.sharding.shutdown_drain_timeout.get_duration(); - let poll_interval = config.system.sharding.shutdown_poll_interval.get_duration(); - - let shutdown_flag_for_handoff = Arc::clone(&shutdown_flag); - spawn_shutdown_watchdog(Rc::clone(&bus), shutdown_flag, drain_timeout, poll_interval); - - // Metadata bootstrap is single-writer: shard 0 owns the WAL and the - // only `WriteHandle`-bearing `MuxStateMachine`. Peer shards receive - // a `ReadHandleFactory` bundle on the inter-thread channel and - // rebuild a reader-mode `MuxStateMachine` on their own runtime - no - // WAL access, no replay. Writes still funnel through shard 0's - // metadata VSR; per-commit `publish()` (in `WriteCell::apply`) - // bounds reader staleness to one op. - let data_dir = Path::new(&config.system.path); - let (mux_stm, owner_state) = match metadata_handoff { - MetadataHandoff::Owner { bundle_tx } => { - // Root is created locally at boot (never journaled), so replay - // must start from the same baseline or every WAL-created user - // shifts one slab id and root is lost after the first restart. - let recovered = recover::( - data_dir, - ReplicaIdentity { - cluster: topology.cluster_id, - replica_id: topology.self_replica_id, - replica_count: topology.replica_count, - }, - config.metadata.journal_slots, - config.metadata.clients_table_max, - |mux_stm| { - ensure_default_root_user(mux_stm); - }, - ) - .await - .map_err(ServerNgError::MetadataRecovery)?; - validate_cluster_root_bootstrap(config, &recovered.mux_stm)?; - ensure_default_root_user(&recovered.mux_stm); - // The factory bundle hands every peer a read handle over the - // same `Inner`, so `Arc` (and the parent - // `Arc`) is shared across all shards. Zero the - // snapshot totals here, once, before any peer can observe the - // bundle. Per-shard `load_partition` deltas in - // `build_shard_for_thread` then race only against other - // atomic adds, never against a concurrent `swap(0)` that - // would mistake an in-flight delta for the snapshot total - // and decrement the parent `StreamStats` by it. - let () = recovered.mux_stm.streams().read(|inner| { - for (_, stream) in &inner.items { - for (_, topic) in &stream.topics { - topic.stats.zero_out_all(); - } - } - }); - broadcast_metadata_bundle( - shard_id, - &bundle_tx, - recovered.mux_stm.factory_bundle(), - total_shards.saturating_sub(1), - &shutdown_flag_for_handoff, - poll_interval, - ) - .await?; - ( - recovered.mux_stm, - Some(RecoveredOwnerState { - journal: recovered.journal, - snapshot: recovered.snapshot, - last_applied_op: recovered.last_applied_op, - last_journaled_op: recovered.last_journaled_op, - client_table: recovered.client_table, - superblock: recovered.superblock, - recovered_state: recovered.recovered_state, - snapshot_checkpoint: recovered.snapshot_checkpoint, - }), - ) - } - MetadataHandoff::Waiter { bundle_rx } => { - let bundle = await_metadata_bundle( - shard_id, - &bundle_rx, - &shutdown_flag_for_handoff, - poll_interval, - ) - .await?; - (ServerNgMuxStateMachine::from_factory_bundle(bundle), None) - } - }; - - // Metadata consensus + journal + snapshot live only on shard 0. - // `IggyShard::tick_metadata` short-circuits when `consensus.is_none()`, - // so peer shards have no caller that reads `journal` or `snapshot`. - let ( - metadata_consensus, - journal_for_metadata, - snapshot_for_metadata, - superblock_for_metadata, - checkpoint_seed, - recovered_client_table, - ) = if let Some(owner) = owner_state { - // `recover()` already opened the superblock, read `recovered_state`, and - // verified the on-disk snapshot against its checkpoint pairing BEFORE decoding - // it. Reuse that superblock rather than re-opening it, which would fork the - // ping-pong sequence counter. Consensus recovers its true (view, log_view) - // from `recovered_state` instead of inferring a stale view from the WAL. - let consensus = restore_metadata_consensus(&owner, &topology, config, Rc::clone(&bus)); - let superblock = Rc::new(owner.superblock); - ( - Some(consensus), - Some(owner.journal), - owner.snapshot, - Some(superblock), - owner.snapshot_checkpoint, - Some(owner.client_table), - ) - } else { - (None, None, None, None, (0, 0), None) - }; - let metadata = ServerNgMetadata::new( - metadata_consensus, - journal_for_metadata, - snapshot_for_metadata, - superblock_for_metadata, - mux_stm, - Some(PathBuf::from(&config.system.path)), - ); - // Size the VSR client table before listeners bind and any client registers. - // Must precede the recovered-table install below: the setter rebuilds the - // table from scratch, so running it afterwards would drop every resumed - // session (and trip its empty-table assert). - metadata.set_clients_table_max(config.metadata.clients_table_max); - // Reinstall the sessions recovery restored from the checkpoint and the WAL - // suffix, so a rebooted node dedups retries and admits continuations from - // clients that kept their identity across the restart (IGGY-137). Recovery - // sized this table from the same config value, so the install preserves the - // configured cap. - if let Some(client_table) = recovered_client_table { - // Refusal (a client registered before this ran) keeps the live table - // and is logged by the callee; boot continues either way. - let _ = metadata.install_client_table(client_table); - } - // Seed the coordinator's last-checkpoint pairing so the first post-boot - // view-change superblock write records the real (checkpoint_op, checksum) - // instead of (0, 0). No-op on peer shards, which have no coordinator. - metadata.seed_checkpoint_ref(checkpoint_seed.0, checkpoint_seed.1); - // Shard 0's copy resolves the `ServerDefault` sentinels (max topic size and - // message expiry) at create admission; responses echo stored values verbatim. - metadata.set_default_max_topic_size(config.system.topic.max_size.as_bytes_u64()); - metadata.set_default_message_expiry(u64::from(config.system.topic.message_expiry)); - // Keep the forced-checkpoint margin >= the configured prepare-queue - // depth: ops already pipelined while a checkpoint runs append into that - // margin (config validation keeps journal_slots >= 4x this). - metadata.set_checkpoint_margin(config.metadata.checkpoint_margin()); - - let shard_metrics = ShardMetrics::for_shard(); - // Notifier install deferred until after tick handler wires below. - let senders_for_notifier = senders.clone(); - let metrics_for_notifier = shard_metrics.clone(); - // Heap-pin like `shard_main` above: the builder future carries the whole - // shard construction state machine and outgrew clippy's `large_futures` - // cap; one allocation per shard startup. - let (shard, sessions) = Box::pin(build_shard_for_thread( - shard_id, - total_shards, - config, - &topology, - metadata, - Rc::clone(&bus), - senders, - inbox, - shard_metrics, - Arc::clone(&metadata_view), - )) - .await?; - - // Shard 0 owns the metadata consensus; publish its view so every shard's - // cluster-metadata read (and the SDK's leader discovery) marks the live - // primary. Detached: dies with this shard's runtime at process exit. - if shard_id == 0 { - let publisher_shard = Rc::clone(&shard); - let publisher_view = Arc::clone(&metadata_view); - compio::runtime::spawn(async move { - loop { - if let Some(consensus) = publisher_shard.plane.metadata().consensus.as_ref() { - // While this replica declines its recovered view's - // primaryship, that view must not reach the roster: the - // delegated shards would compute a leader that never - // heartbeats. Publish "unknown" until the election - // resolves the role. - let published = if consensus.has_ceded_primaryship() - && consensus.primary_index(consensus.view()) == consensus.replica() - { - crate::cluster_meta::METADATA_VIEW_UNKNOWN - } else { - u64::from(consensus.view()) - }; - publisher_view.store(published, Ordering::Relaxed); - } - compio::time::sleep(std::time::Duration::from_millis(100)).await; - } - }) - .detach(); - } - - info!( - shard = shard_id, - partitions = shard.plane.partitions().len(), - "server-ng shard initialized" - ); - - // Re-check the cross-thread shutdown flag here, *before* spawning the - // message pump. A sibling shard may have failed in the window between - // the metadata broadcast and this point; gating before spawn keeps the - // bus' `background_tasks` vec empty on the shutdown path. Spawn-then- - // check would leave `bus.track_background(pump_handle)` registering a - // `JoinHandle` that only `bus.shutdown()` drains, but the watchdog - // driving `bus.shutdown()` is `.detach()`'d (see TODO at - // `spawn_shutdown_watchdog`) and may not be scheduled before this - // function returns `Ok(())` and the compio runtime drops, cancelling - // the pump mid-`write_vectored_all`. - // - // Without this gate shard 0 would also still open TCP/QUIC/WS - // listeners for a server that is already tearing down, briefly - // accepting connections that immediately get torn by the watchdog. - if shutdown_flag_for_handoff.load(Ordering::Relaxed) { - return Ok(()); - } - - // Tick handler must install before the notifier so early commits - // do not broadcast ticks whose handler slot is still `None`. - let (reconcile_wake_tx, reconcile_wake_rx) = channel::<()>(1); - let (reconcile_stop_tx, reconcile_stop_rx) = channel::<()>(1); - crate::partition_reconciler::install_tick_handler(&shard, reconcile_wake_tx); - - // Only shard 0 commits metadata. - if shard_id == 0 { - let notifier = make_metadata_commit_notifier(senders_for_notifier, metrics_for_notifier); - shard.plane.metadata().set_commit_notifier(Some(notifier)); - } else { - drop(senders_for_notifier); - drop(metrics_for_notifier); - } - - // The pump task also drives the consensus timer tick (heartbeats, prepare - // retransmit, view-change timeouts) as a select! arm, serialized with frame - // processing - see `run_message_pump`. - let (stop_tx, stop_rx) = channel(1); - let pump_shard = Rc::clone(&shard); - // Owned and awaited by shard_main at exit, NOT `track_background`: the - // background drain runs inside `bus.shutdown()`, which the Ctrl-C path - // never drives (the watchdog stands down when the token fires), so a - // tracked pump would be cancelled by runtime teardown mid final-flush - // and every graceful shutdown would silently drop the committed journal - // tail that had not hit a flush threshold yet. - let mut pump_handle = Some(compio::runtime::spawn(async move { - pump_shard.run_message_pump(stop_rx).await; - })); - - let reconciler_ctx = Rc::new(crate::partition_reconciler::ReconcilerCtx::new( - Rc::clone(&shard), - total_shards, - Rc::new(config.clone()), - topology.cluster_id, - topology.self_replica_id, - topology.replica_count, - )); - let reconcile_periodic = config - .system - .sharding - .reconcile_periodic_interval - .get_duration(); - let reconciler_handle = compio::runtime::spawn({ - let ctx = Rc::clone(&reconciler_ctx); - async move { - crate::partition_reconciler::run_reconciler( - ctx, - reconcile_wake_rx, - reconcile_stop_rx, - reconcile_periodic, - ) - .await; - } - }); - bus.track_background(reconciler_handle); - - // Per-shard heartbeat verifier: evicts connections that stop pinging, - // releasing their consumer-group membership. Gated on config so a - // deployment without heartbeats never reaps live sessions. - let heartbeat_stop_tx = if config.heartbeat.enabled { - let (hb_stop_tx, hb_stop_rx) = channel::<()>(1); - let hb_shard = Rc::clone(&shard); - let hb_sessions = Rc::clone(&sessions); - let hb_interval = config.heartbeat.interval.get_duration(); - let hb_handle = compio::runtime::spawn(async move { - crate::dispatch::run_heartbeat_verifier(hb_shard, hb_sessions, hb_interval, hb_stop_rx) - .await; - }); - bus.track_background(hb_handle); - Some(hb_stop_tx) - } else { - None - }; - // Expired-PAT cleaner: shard 0 only (it owns the metadata consensus - // group) and only when enabled. Each pass no-ops unless this node is - // the caught-up metadata primary, so the delete is proposed once and - // replicated to every replica. - let pat_cleaner_stop = if shard_id == 0 && config.personal_access_token.cleaner.enabled { - let (cleaner_stop_tx, cleaner_stop_rx) = channel(1); - let cleaner_shard = Rc::clone(&shard); - let interval = config.personal_access_token.cleaner.interval.get_duration(); - let cleaner_handle = compio::runtime::spawn(async move { - crate::personal_access_token_cleaner::run_pat_cleaner( - cleaner_shard, - cleaner_stop_rx, - interval, - ) - .await; - }); - bus.track_background(cleaner_handle); - Some(cleaner_stop_tx) - } else { - None - }; - - // Segment cleaner: runs on every shard (each replica trims its own log, - // primary and backup alike). Local and unreplicated; gated by the shared - // data-maintenance config. - let segment_cleaner_stop = if config.data_maintenance.messages.cleaner_enabled { - let (stop_tx, stop_rx) = channel(1); - let cleaner_shard = Rc::clone(&shard); - let interval = config.data_maintenance.messages.interval.get_duration(); - let cleaner_handle = compio::runtime::spawn(async move { - crate::segment_cleaner::run_segment_cleaner(cleaner_shard, stop_rx, interval).await; - }); - bus.track_background(cleaner_handle); - Some(stop_tx) - } else { - None - }; - - // Listener fence (see `BootstrapBarrier`). Peers still scan live - // shared metadata and load their on-disk partitions in - // `build_shard_for_thread`; the factory-bundle handoff only proves - // they *received* the bundle, not that they finished loading. Shard - // 0 must not accept client traffic until every peer's load scan is - // done, otherwise a partition created by the first client surfaces - // in a still-running scan with no segment dir on disk and aborts the - // node with `CannotReadPartitions`. By this point every shard has - // also spawned its pump + reconciler, so a partition created after - // the fence takes the runtime reconciler path on its owning shard. - match barrier { - BootstrapBarrier::Owner { ready_rx } => { - await_bootstrap_complete( - &ready_rx, - usize::from(total_shards.saturating_sub(1)), - &shutdown_flag_for_handoff, - poll_interval, - ) - .await?; - } - BootstrapBarrier::Waiter { ready_tx } => { - signal_bootstrap_complete( - shard_id, - &ready_tx, - &shutdown_flag_for_handoff, - poll_interval, - ) - .await?; - } - } - - // Listeners (replica + every client transport) bind on shard 0 only. - // Shard 0's coordinator round-robins inbound TCP/WS connections to - // peer shards via fd-transfer. QUIC and TCP-TLS clients terminate - // locally on shard 0 (their per-connection state is non-portable - - // see `LifecycleFrame::ClientWsConnectionSetup` rustdoc). - if shard_id == 0 { - let coord = shard - .coordinator() - .expect("shard 0 always has a coordinator attached by the builder"); - // Reseed the client-id minter above every recovered entry before any - // listener accepts. The counter is per process; the table it must not - // collide with was rebuilt from the previous boot's WAL. Keyed by view - // so a later promotion refolds the table (the minting path calls the - // same method, see `HttpInner::register_session_once`). - let boot_view = shard - .plane - .metadata() - .consensus - .as_ref() - .map_or(0, consensus::VsrConsensus::view); - coord.seed_client_sequence( - boot_view, - shard.plane.metadata().client_table.borrow().client_ids(), - ); - let on_client_request = make_client_request_handler( - &shard, - &sessions, - Arc::clone(&config.system), - config.personal_access_token.max_tokens_per_user, - ); - let (accepted_replica, dialed_replica) = - make_replica_delegation_fns(Rc::clone(&coord), &bus); - let accepted_client = make_shard_zero_client_accept_fns(coord, &bus, on_client_request); - - if let Err(error) = start_tcp_runtime( - &shard, - config, - &topology, - accepted_replica, - dialed_replica, - accepted_client, - ) - .await - { - let _ = stop_tx.try_send(()); - let _ = reconcile_stop_tx.try_send(()); - if let Some(tx) = &heartbeat_stop_tx { - let _ = tx.try_send(()); - } - if let Some(cleaner_stop_tx) = &pat_cleaner_stop { - let _ = cleaner_stop_tx.try_send(()); - } - if let Some(tx) = &segment_cleaner_stop { - let _ = tx.try_send(()); - } - await_pump_drain(pump_handle.take(), config, shard_id).await; - return Err(error); - } - } - - bus.token().wait().await; - let _ = stop_tx.try_send(()); - let _ = reconcile_stop_tx.try_send(()); - if let Some(tx) = &heartbeat_stop_tx { - let _ = tx.try_send(()); - } - if let Some(cleaner_stop_tx) = &pat_cleaner_stop { - let _ = cleaner_stop_tx.try_send(()); - } - if let Some(tx) = &segment_cleaner_stop { - let _ = tx.try_send(()); - } - - await_pump_drain(pump_handle.take(), config, shard_id).await; - - info!(shard = shard_id, "server-ng shard exited cleanly"); - Ok(()) -} - -/// Await the message pump's completion before the shard returns: its -/// post-loop work includes the final flush of every committed journal to -/// segment storage, and returning first drops the compio runtime, which -/// cancels that flush at its next await point. -async fn await_pump_drain( - pump_handle: Option>, - config: &ServerNgConfig, - shard_id: u16, -) { - let Some(pump_handle) = pump_handle else { - return; - }; - let drain_budget = config.system.sharding.shutdown_drain_timeout.get_duration(); - if compio::time::timeout(drain_budget, pump_handle) - .await - .is_err() - { - warn!( - shard = shard_id, - "message pump did not drain within the shutdown budget; \ - committed journal tail may not have flushed" - ); - } -} - -/// Block until shard 0 broadcasts the metadata factory bundle, or the -/// cross-thread shutdown flag flips. Polled in a `poll_interval` loop -/// so a shard 0 that panics before it broadcasts cannot strand peer -/// shards: the shutdown path flips the flag, every waiter observes it -/// on the next tick, and the server tears down instead of hanging. -/// -/// Uses `try_recv` + sleep rather than `timeout(recv())`. Crossfire 3.x -/// documents `recv()` as cancellation-safe (no leak/deadlock) but does -/// not guarantee atomicity for the dropped future's result; `try_recv` -/// keeps each tick fully synchronous and side-effect-free, so the -/// shutdown poll cadence cannot ambiguously consume a bundle. -async fn await_metadata_bundle( - shard_id: u16, - bundle_rx: &crossfire::MAsyncRx>, - shutdown_flag: &Arc, - poll_interval: Duration, -) -> Result { - loop { - match bundle_rx.try_recv() { - Ok(bundle) => return Ok(bundle), - Err(crossfire::TryRecvError::Disconnected) => { - return Err(ServerNgError::MetadataHandoffAborted { shard_id }); - } - Err(crossfire::TryRecvError::Empty) => { - if shutdown_flag.load(Ordering::Relaxed) { - return Err(ServerNgError::MetadataHandoffAborted { shard_id }); - } - compio::time::sleep(poll_interval).await; - } - } - } -} - -/// Push `peers` cloned bundles onto `bundle_tx`, polling each send in a -/// `poll_interval` loop so the cross-thread shutdown flag can interrupt -/// a stalled handoff. Symmetric to [`await_metadata_bundle`]: shutdown -/// observed mid-handshake aborts cleanly rather than stalling on a -/// `send` future that can no longer make progress. -/// -/// Uses `try_send` + sleep rather than `timeout(send())`. Crossfire 3.x -/// documents `send()` as cancellation-safe in the leak/deadlock sense -/// but explicitly warns the true result is unknown when `SendFuture` is -/// dropped on cancellation. For a retry loop that re-clones on every -/// tick that would risk publishing the same bundle twice, stuffing the -/// bounded channel past `peers` and stranding a follow-up `send`. -/// `try_send` returns the bundle back inside `TrySendError::Full`, so -/// the loop reuses it instead of re-cloning when the channel is full. -async fn broadcast_metadata_bundle( - shard_id: u16, - bundle_tx: &crossfire::MAsyncTx>, - bundle: ServerNgMetadataBundle, - peers: u16, - shutdown_flag: &Arc, - poll_interval: Duration, -) -> Result<(), ServerNgError> { - for _ in 0..peers { - let mut pending = bundle.clone(); - loop { - match bundle_tx.try_send(pending) { - Ok(()) => break, - Err(crossfire::TrySendError::Disconnected(_)) => { - // Every peer dropped its `bundle_rx` before recv. Shard - // 0 must not silently continue past handoff: it would - // bind listeners and commit consensus state for a - // cluster whose peers are gone. Propagate the abort so - // `shard_main` short-circuits before further side - // effects; `shutdown_flag` will flip via the normal - // teardown path. - return Err(ServerNgError::MetadataHandoffAborted { shard_id }); - } - Err(crossfire::TrySendError::Full(returned)) => { - if shutdown_flag.load(Ordering::Relaxed) { - return Err(ServerNgError::MetadataHandoffAborted { shard_id }); - } - pending = returned; - compio::time::sleep(poll_interval).await; - } - } - } - } - Ok(()) -} - -/// Peer side of [`BootstrapBarrier`]: tell shard 0 this shard finished -/// loading its on-disk partitions. Mirrors [`broadcast_metadata_bundle`]'s -/// `try_send`-or-shutdown poll loop so a sibling failure (which flips the -/// shutdown flag) drives this out instead of stranding it on a full -/// channel. The channel is sized to the peer count and each peer sends -/// exactly once, so `Full` is not expected; the branch only keeps the -/// loop interruptible. -async fn signal_bootstrap_complete( - shard_id: u16, - ready_tx: &crossfire::MAsyncTx>, - shutdown_flag: &Arc, - poll_interval: Duration, -) -> Result<(), ServerNgError> { - let mut pending = shard_id; - loop { - match ready_tx.try_send(pending) { - Ok(()) => return Ok(()), - Err(crossfire::TrySendError::Disconnected(_)) => { - // Shard 0 dropped its `ready_rx` before draining (it - // aborted before binding listeners). Propagate so this - // shard short-circuits; the shutdown flag flips via the - // normal teardown path. - return Err(ServerNgError::MetadataHandoffAborted { shard_id }); - } - Err(crossfire::TrySendError::Full(returned)) => { - if shutdown_flag.load(Ordering::Relaxed) { - return Err(ServerNgError::MetadataHandoffAborted { shard_id }); - } - pending = returned; - compio::time::sleep(poll_interval).await; - } - } - } -} - -/// Owner side of [`BootstrapBarrier`]: drain one ready signal per peer -/// before shard 0 binds listeners. Polls the shutdown flag so a peer that -/// dies mid-load (flipping the flag) aborts the wait instead of hanging on -/// a signal that will never arrive. A single shard (`peers == 0`) returns -/// immediately. -async fn await_bootstrap_complete( - ready_rx: &crossfire::MAsyncRx>, - peers: usize, - shutdown_flag: &Arc, - poll_interval: Duration, -) -> Result<(), ServerNgError> { - let mut remaining = peers; - while remaining > 0 { - match ready_rx.try_recv() { - Ok(_shard_id) => remaining -= 1, - Err(crossfire::TryRecvError::Disconnected) => { - return Err(ServerNgError::ShardBootstrapBarrierAborted { remaining }); - } - Err(crossfire::TryRecvError::Empty) => { - if shutdown_flag.load(Ordering::Relaxed) { - return Err(ServerNgError::ShardBootstrapBarrierAborted { remaining }); - } - compio::time::sleep(poll_interval).await; - } - } - } - Ok(()) -} - -/// Spawn a per-shard polling task that watches the cross-thread shutdown -/// flag and triggers this shard's bus shutdown on transition. The flag -/// is the only Send signal we have; the bus' shutdown machinery is -/// `!Send` (`Rc>` + per-shard `async_channel`), so it must be -/// triggered from within the runtime that owns the bus. -#[allow(clippy::needless_pass_by_value)] -fn spawn_shutdown_watchdog( - bus: Rc, - shutdown_flag: Arc, - drain_timeout: Duration, - poll_interval: Duration, -) { - let bus_for_task = Rc::clone(&bus); - let bus_token = bus.token(); - let watchdog = compio::runtime::spawn(async move { - loop { - if shutdown_flag.load(Ordering::Relaxed) { - break; - } - if bus_token.is_triggered() { - // Bus shutdown was driven from elsewhere (e.g. internal - // failure path). Watchdog has nothing left to do. - return; - } - compio::time::sleep(poll_interval).await; - } - let _ = bus_for_task.shutdown(drain_timeout).await; - }); - // TODO(hubcio): `.detach()` races bus shutdown: when `bus.token()` is - // triggered, `shard_main` returns and the runtime drops the watchdog - // mid-`bus.shutdown()`, truncating in-flight `ClientForwardFailed` - // replies (terminal per `SendError` docs). Cannot use - // `bus.track_background(watchdog)` here because the watchdog itself - // drives `bus.shutdown()`, and the bg-drain loop in `shutdown()` - // would re-enter awaiting the watchdog's own pending shutdown call - // (self-deadlock). Fix: extract a `core/task_registry` crate mirroring - // `core/server`'s task-tracking mechanism, share it between the bus - // and server-ng so background tasks can be reaped without coupling - // to the bus shutdown order. - watchdog.detach(); -} - -/// Copy the configured cluster roster plus this node's own client ports into -/// the shared [`ClusterRoster`] so the binary `GetClusterMetadata` read serves -/// the real topology. `self_*` back only the cluster-disabled self-synthesis -/// and carry the requested listener ports from the resolved topology, not the -/// bound ones (a `:0` wildcard is reported as 0). -fn build_cluster_roster( - config: &ServerNgConfig, - topology: &TcpTopology, - metadata_view: Arc, -) -> ClusterRoster { - ClusterRoster { - enabled: config.cluster.enabled, - name: config.cluster.name.clone(), - nodes: config - .cluster - .nodes - .iter() - .cloned() - .map(Into::into) - .collect(), - self_ip: topology.client_listen_addr.ip().to_string(), - self_ports: configs::ng_cluster::TransportPorts { - tcp: Some(topology.client_listen_addr.port()), - quic: topology.quic_listen_addr.map(|addr| addr.port()), - http: topology.http_listen_addr.map(|addr| addr.port()), - websocket: topology.ws_listen_addr.map(|addr| addr.port()), - tcp_replica: None, - }, - metadata_view, - } -} - -#[allow(clippy::too_many_arguments, clippy::too_many_lines)] -async fn build_shard_for_thread( - shard_id: u16, - total_shards: u16, - config: &ServerNgConfig, - topology: &TcpTopology, - metadata: ServerNgMetadata, - bus: Rc, - senders: Vec, - inbox: ShardReceiver, - metrics: ShardMetrics, - metadata_view: Arc, -) -> Result<(Rc, Rc>), ServerNgError> { - let shard_local_id = ShardId::new(shard_id); - let total_partitions = metadata.mux_stm.streams().read(|inner| { - inner - .items - .iter() - .map(|(_, stream)| { - stream - .topics - .iter() - .map(|(_, topic)| topic.partitions.len()) - .sum::() - }) - .sum::() - }); - - // IggyPartitions holds only the partitions owned by this shard - // (see the filter below at insert time), so the server-wide total - // is an N-fold overshoot. `ceil(total / shards) * 2` is a coarse - // upper bound that absorbs hash skew without paying the full - // multiplier. PapayaShardsTable below stays sized to the server-wide - // total because every shard routes every namespace. - let owned_partitions_capacity = total_partitions - .div_ceil(usize::from(total_shards).max(1)) - .saturating_mul(2); - // At-rest encryption: built once per shard from the shared config; the - // ingestion path encrypts on the primary and the poll reply decrypts. - // A bad key fails the boot rather than silently serving plaintext. - let encryptor = if config.system.encryption.enabled { - let aes = Aes256GcmEncryptor::from_base64_key(&config.system.encryption.key) - .map_err(|error| ServerNgError::Iggy(Box::new(error)))?; - Some(Arc::new(EncryptorKind::Aes256Gcm(aes))) - } else { - None - }; - let partitions = IggyPartitions::with_capacity( - shard_local_id, - PartitionsConfig { - messages_required_to_save: config.system.partition.messages_required_to_save, - size_of_messages_required_to_save: config - .system - .partition - .size_of_messages_required_to_save, - enforce_fsync: config.system.partition.enforce_fsync, - segment_size: config.system.segment.size, - encryptor, - }, - owned_partitions_capacity, - ); - let shards_table = PapayaShardsTable::with_capacity(total_partitions); - - // Stream-filter inside the `read()` closure: only partitions owned by - // this shard need the heavy (`Arc` + `Partition`) clones - // for the async `load_partition` below. Non-owning entries are pushed - // straight into `shards_table` here, so no Vec scales with the - // server-wide partition count. - let owned = metadata.mux_stm.streams().read(|inner| { - let mut owned = Vec::with_capacity(owned_partitions_capacity); - for (_, stream) in &inner.items { - for (topic_id, topic) in &stream.topics { - for partition in &topic.partitions { - let namespace = IggyNamespace::new(stream.id, topic_id, partition.id); - let owning_shard = - calculate_shard_assignment(&namespace, u32::from(total_shards)); - if owning_shard == shard_id { - // Shared per-partition stats from the registry: the - // same `Arc` backs every shard's `get_topic` reply. - let stats = inner.stats_registry.partition( - stream.id, - topic_id, - partition.id, - topic.stats.clone(), - ); - owned.push((stream.id, topic_id, stats, partition.clone())); - } else { - shards_table.insert( - namespace, - PartitionLocation::new( - ShardId::new(owning_shard), - partition.created_revision, - ), - ); - } - } - } - } - owned - }); - - // Snapshot totals were zeroed once on shard 0 before the factory - // bundle was broadcast (see `MetadataHandoff::Owner`). All shards - // here only add their per-partition deltas, so the shared - // `Arc` atomics race only against other atomic adds. - for (stream_id, topic_id, partition_stats, partition_metadata) in owned { - validate_namespace_bounds(config, stream_id, topic_id, partition_metadata.id)?; - let namespace = IggyNamespace::new(stream_id, topic_id, partition_metadata.id); - let partition = match load_partition( - config, - namespace, - Arc::clone(&partition_stats), - &partition_metadata, - topology.cluster_id, - topology.self_replica_id, - topology.replica_count, - Rc::clone(&bus), - ) - .await - { - Ok(partition) => partition, - // ONE damaged local chain must not take the node down. The shapes - // this refuses are exactly what a failed state-transfer quarantine - // leaves behind, so fence that group the same way the runtime path - // does -- move its segment files aside, keeping the superblock so it - // cannot re-enter view 0 -- and materialise it fresh. The ordinary - // rejoin path (repair, then state transfer on a refused floor) - // recovers its data from a peer. - Err(ServerNgError::PartitionChainRefused { dir, reason, .. }) => { - let partition_dir = dir.to_string_lossy().into_owned(); - error!( - stream_id, - topic_id, - partition_id = partition_metadata.id, - partition_dir, - %reason, - "refusing the recovered segment chain; fencing this partition and \ - rebuilding it empty for the rejoin path" - ); - match partitions::state_transfer::quarantine_segment_files(&partition_dir).await { - Ok(fenced_dir) => error!( - stream_id, - topic_id, - partition_id = partition_metadata.id, - fenced_dir, - "quarantined the refused segment files; they are kept for inspection" - ), - Err(error) => { - // NOT rebuilt: `build_partition_fresh` reaches - // `ensure_initial_segment`, which opens segment 0 with - // `file_exists = false` and TRUNCATES whatever the - // failed quarantine left behind. The likeliest failures - // (suffix cap exhausted, `create_dir_all`) move zero - // files, so rebuilding would destroy the oldest segment - // on the first attempt while the higher-offset survivors - // keep refusing every boot -- a loop that never - // terminates and eats the chain one segment at a time. - // Tombstone instead: the namespace stays unmaterialised - // and unrouted, the reconciler backs off, and an - // operator still has every byte. - error!( - stream_id, - topic_id, - partition_id = partition_metadata.id, - partition_dir, - %error, - "failed to quarantine the refused segment files; leaving this \ - partition tombstoned rather than rebuilding over them" - ); - partition_stats.zero_out_all(); - partitions.tombstone(namespace); - continue; - } - } - // The refused load already folded its segment counts in. - partition_stats.zero_out_all(); - build_partition_fresh( - config, - namespace, - partition_stats, - partition_metadata.created_revision, - topology.cluster_id, - topology.self_replica_id, - topology.replica_count, - Rc::clone(&bus), - ) - .await? - } - // An untrustworthy superblock fences ONE group, not the node. The - // segment files stay exactly where they are -- unlike a refused - // chain, the data on disk is not the thing in doubt -- so there is - // nothing to quarantine and nothing to rebuild: rebuilding fresh - // would hand this replica a view-0 identity while a record it - // cannot read says otherwise. Tombstoned, the namespace stays - // unmaterialised and unrouted, the reconciler backs off, and an - // operator has every byte plus a message naming the directory. - Err( - error @ (ServerNgError::PartitionSuperblockIo { .. } - | ServerNgError::PartitionSuperblockVersionUnknown { .. } - | ServerNgError::PartitionSuperblockUnverifiable { .. } - | ServerNgError::PartitionSuperblockUndecodable { .. } - | ServerNgError::PartitionSuperblockIdentityMismatch { .. }), - ) => { - error!( - stream_id, - topic_id, - partition_id = partition_metadata.id, - %error, - "cannot trust this partition's durable consensus state; tombstoning the \ - partition and continuing to boot the rest of the shard" - ); - partition_stats.zero_out_all(); - partitions.tombstone(namespace); - continue; - } - Err(error) => return Err(error), - }; - partitions.insert(namespace, partition); - shards_table.insert( - namespace, - PartitionLocation::new(ShardId::new(shard_id), partition_metadata.created_revision), - ); - } - - let shard_handle = Rc::new(RefCell::new(None)); - // Same wiring path as the simulator's shell mode: one per-shard - // SessionManager shared by the client-request handler (binds sessions) - // and the get_clients handler (reads them). It also carries this shard's - // cluster roster for the pre-auth GetClusterMetadata read. - let ShellHandlers { - on_replica_message, - on_client_request, - on_metadata_submit, - on_list_clients, - on_partition_read, - sessions, - } = wire_shell_handlers( - &bus, - &shard_handle, - Arc::clone(&config.system), - config.personal_access_token.max_tokens_per_user, - ); - sessions - .borrow_mut() - .set_cluster_roster(Rc::new(build_cluster_roster( - config, - topology, - metadata_view, - ))); - let shard_name = format!("server-ng-shard-{shard_id}"); - let built = IggyShardBuilder::new( - ShardIdentity::new(shard_id, shard_name), - Rc::clone(&bus), - on_replica_message, - on_client_request, - on_metadata_submit, - on_list_clients, - on_partition_read, - metadata, - partitions, - senders, - inbox, - shards_table, - PartitionConsensusConfig::new( - topology.cluster_id, - shard::ReplicaTopology::new(topology.self_replica_id, topology.replica_count), - Rc::clone(&bus), - ), - CoordinatorConfig::default(), - metrics, - ) - .build() - .map_err(ServerNgError::ShardConstruction)?; - - let shard = Rc::new(built.shard); - // Repair pacing is shared by both planes' repair loops, so it is a - // per-shard tunable set once here rather than per consensus group. - shard.set_repair_retry_ticks(repair_retry_ticks(config)); - shard.set_served_segment_cache_bytes_max( - config - .partition - .transfer_served_cache_bytes_max - .as_bytes_u64(), - ); - shard.set_partition_artifact_len_max( - config.partition.transfer_artifact_bytes_max.as_bytes_u64(), - ); - shard.set_repair_chunk_max(config.cluster.repair_chunk_max as u64); - // Bounds a served state-transfer chunk. A frame above the bus ceiling is - // rejected by the RECEIVING transport, which tears the replica connection - // down rather than dropping one message. - shard.set_bus_max_message_size( - usize::try_from(config.message_bus.max_message_size.as_bytes_u64()).unwrap_or(usize::MAX), - ); - *shard_handle.borrow_mut() = Some(Rc::downgrade(&shard)); - Ok((shard, sessions)) -} - -// Pin the configs-crate default literals (duplicated there to avoid a -// build-time edge onto the runtime crates) against the runtime constants, -// mirroring the message_bus IOV_MAX pin. A drift on either side fails this -// crate's build until both are reconciled. -const _: () = assert!( - configs::ng_metadata::DEFAULT_METADATA_PREPARE_QUEUE_DEPTH - == consensus::PIPELINE_PREPARE_QUEUE_MAX -); -const _: () = assert!( - configs::ng_metadata::DEFAULT_METADATA_JOURNAL_SLOTS - == journal::prepare_journal::DEFAULT_SLOT_COUNT -); -const _: () = assert!( - configs::ng_partition::DEFAULT_PARTITION_PREPARE_QUEUE_DEPTH - == consensus::PIPELINE_PREPARE_QUEUE_MAX -); -const _: () = assert!( - configs::ng_metadata::DEFAULT_METADATA_CLIENTS_TABLE_MAX == consensus::CLIENTS_TABLE_MAX -); -const _: () = - assert!(configs::ng_cluster::DEFAULT_VIEW_PROBE_ATTEMPTS_MAX == consensus::PROBE_ATTEMPTS_MAX); -const _: () = assert!( - configs::ng_partition::DEFAULT_EVICTED_RING_CAPACITY == partitions::EVICTED_RING_CAPACITY -); -const _: () = assert!( - configs::ng_partition::DEFAULT_EVICTED_RING_BYTES_MAX == partitions::EVICTED_RING_BYTES_MAX -); -const _: () = assert!( - configs::ng_partition::DEFAULT_TRANSFER_ARTIFACT_BYTES_MAX - == shard::PARTITION_ARTIFACT_LEN_DEFAULT -); -const _: () = assert!( - configs::ng_partition::DEFAULT_TRANSFER_SERVED_CACHE_BYTES_MAX - == shard::SERVED_SEGMENT_CACHE_BYTES_DEFAULT -); -const _: () = - assert!(configs::ng_cluster::DEFAULT_REPAIR_CHUNK_MAX as u64 == shard::REPAIR_CHUNK_MAX); -const _: () = assert!( - configs::ng_cluster::STATE_CHUNK_HEADER_LEN - == size_of::() as u64 -); -/// Convert a consensus-timer interval to whole ticks, floored at one tick so a -/// sub-tick value still fires and saturated on overflow. -fn duration_to_ticks(interval: Duration) -> u64 { - let ticks = interval.as_millis() / shard::CONSENSUS_TICK_INTERVAL.as_millis(); - u64::try_from(ticks.max(1)).unwrap_or(u64::MAX) -} - -/// `[cluster] heartbeat_timeout` in consensus ticks. Every consensus group -/// (metadata and per-partition planes alike) gets the same window: the failure -/// it guards against - a primary that stopped heartbeating - is host-level, not -/// per-plane. -pub(crate) fn cluster_heartbeat_ticks(config: &ServerNgConfig) -> u64 { - duration_to_ticks(config.cluster.heartbeat_timeout.get_duration()) -} - -/// Floor for the post-restart read-recovery deadline (see -/// [`recovery_barrier_deadline`]). At and below the 5s default heartbeat the -/// worst-case recovery is dominated by the heartbeat-independent term - the -/// `ViewChangeStatus` backstop plus election ceremony and suffix recommit, -/// empirically ~7s - so the scaled value must never fall under this or a -/// fast-heartbeat cluster would 503 legitimate reads mid-recovery. The backstop -/// is the configurable `[cluster] view_change_status_timeout`; raising it past -/// its 5s default is why `recovery_barrier_deadline` scales that knob in too -/// rather than leaning on this floor to cover it. -const RECOVERY_BARRIER_DEADLINE_FLOOR: Duration = Duration::from_secs(15); - -/// Safety factor applied to each scaled term of the recovery deadline: a slower -/// heartbeat stretches election and suffix recommit proportionally, and a wider -/// status backstop stretches the ceremony it bounds. 3x reproduces the -/// empirically chosen 15s margin at the shared 5s default (3 x 5s = 15s) and -/// holds that factor as either knob grows. -const RECOVERY_BARRIER_MULTIPLIER: u32 = 3; - -/// How long the post-restart read path waits for the recovered WAL suffix to -/// re-commit before failing loud (retryable 503): the largest of the fixed -/// floor, a `[cluster] heartbeat_timeout`-scaled window, and a -/// `[cluster] view_change_status_timeout`-scaled window. Both knobs feed it -/// because either, raised far past its default, stretches worst-case recovery -/// past the fixed floor; see `await_recovery_barrier` for the read-side wait. -pub(crate) fn recovery_barrier_deadline( - heartbeat: Duration, - view_change_status: Duration, -) -> Duration { - // saturating: neither timeout has a config ceiling, plain `*` panics - heartbeat - .saturating_mul(RECOVERY_BARRIER_MULTIPLIER) - .max(view_change_status.saturating_mul(RECOVERY_BARRIER_MULTIPLIER)) - .max(RECOVERY_BARRIER_DEADLINE_FLOOR) -} - -/// `[cluster] commit_broadcast_interval` in consensus ticks: how often the -/// primary broadcasts its commit point, the cluster's liveness feed. Applied -/// to every consensus group, matching `cluster_heartbeat_ticks`. -pub(crate) fn commit_broadcast_ticks(config: &ServerNgConfig) -> u64 { - duration_to_ticks(config.cluster.commit_broadcast_interval.get_duration()) -} - -/// `[cluster] prepare_retransmit_interval` in consensus ticks: how often the -/// primary retransmits un-acked prepares. Applied to every consensus group, -/// matching `cluster_heartbeat_ticks`. -pub(crate) fn prepare_retransmit_ticks(config: &ServerNgConfig) -> u64 { - duration_to_ticks(config.cluster.prepare_retransmit_interval.get_duration()) -} - -/// `[cluster] view_change_retransmit_interval` in consensus ticks: how often a -/// replica retransmits its `StartViewChange` / `DoViewChange` during a view -/// change. Applied to every consensus group, matching `cluster_heartbeat_ticks`. -pub(crate) fn view_change_retransmit_ticks(config: &ServerNgConfig) -> u64 { - duration_to_ticks( - config - .cluster - .view_change_retransmit_interval - .get_duration(), - ) -} - -/// `[cluster] view_change_status_timeout` in consensus ticks: the stalled -/// view-change backstop before escalating to a fresh election. Applied to every -/// consensus group, matching `cluster_heartbeat_ticks`. -pub(crate) fn view_change_status_ticks(config: &ServerNgConfig) -> u64 { - duration_to_ticks(config.cluster.view_change_status_timeout.get_duration()) -} - -/// `[cluster] request_start_view_retransmit_interval` in consensus ticks: how -/// often a recovering or view-change backup re-requests the current `StartView`. -/// Applied to every consensus group, matching `cluster_heartbeat_ticks`. -pub(crate) fn request_start_view_ticks(config: &ServerNgConfig) -> u64 { - duration_to_ticks( - config - .cluster - .request_start_view_retransmit_interval - .get_duration(), - ) -} - -/// `[cluster] repair_retry_interval` in consensus ticks: how long a stalled -/// journal-repair stream waits before re-requesting its window. Both planes' -/// repair loops share it, so it is applied once per shard (not per consensus -/// group). Clamped to `u32`, the width of the session idle-tick counter. -pub(crate) fn repair_retry_ticks(config: &ServerNgConfig) -> u32 { - u32::try_from(duration_to_ticks( - config.cluster.repair_retry_interval.get_duration(), - )) - .unwrap_or(u32::MAX) -} - -/// Shard 0's half of a metadata recovery: everything [`recover`] produced except the -/// state machine, which every shard receives through the factory bundle. -/// -/// Named rather than a positional tuple: the fields are same-typed `Option`s and -/// `(u64, u128)` pairs that a reorder would silently rebind, and one of them decides -/// what view the replica boots into. -struct RecoveredOwnerState { - journal: PrepareJournal, - snapshot: Option, - last_applied_op: Option, - last_journaled_op: Option, - client_table: ClientTable, - superblock: PingPongSuperblock, - recovered_state: Option, - snapshot_checkpoint: (u64, u128), -} - -/// Rebuild metadata consensus from what recovery read off this replica's own disk. -/// -/// Takes the recovery result, topology and config whole rather than the dozen-plus -/// scalars it needs from them: most were `u64` tick counts, where a misordered -/// argument type-checks and mistunes a timeout silently. -fn restore_metadata_consensus( - owner: &RecoveredOwnerState, - topology: &TcpTopology, - config: &ServerNgConfig, - bus: Rc, -) -> VsrConsensus> { - let journal = &owner.journal; - let replica_count = topology.replica_count; - let recovered_state = owner.recovered_state; - let snapshot_floor = owner - .snapshot - .as_ref() - .map_or(0, IggySnapshot::sequence_number); - let commit_watermark = owner.last_applied_op.unwrap_or(snapshot_floor); - let restored_op = owner.last_journaled_op.unwrap_or(snapshot_floor); - let recovery_deadline = recovery_barrier_deadline( - config.cluster.heartbeat_timeout.get_duration(), - config.cluster.view_change_status_timeout.get_duration(), - ); - let prepare_queue_depth = config.metadata.prepare_queue_depth; - - let mut consensus = VsrConsensus::new( - topology.cluster_id, - topology.self_replica_id, - replica_count, - server_common::sharding::METADATA_CONSENSUS_NAMESPACE, - bus, - // Request queue keeps the stock 2x ratio over the prepare queue - // (32 -> 64 at defaults): buffered requests are cheap relative to - // in-flight prepares and drain as prepares commit. - LocalPipeline::with_capacities(prepare_queue_depth, prepare_queue_depth * 2), - ); - consensus.set_normal_heartbeat_ticks(cluster_heartbeat_ticks(config)); - consensus.set_commit_message_ticks(commit_broadcast_ticks(config)); - consensus.set_prepare_ticks(prepare_retransmit_ticks(config)); - consensus.set_view_change_retransmit_ticks(view_change_retransmit_ticks(config)); - consensus.set_view_change_status_ticks(view_change_status_ticks(config)); - consensus.set_request_start_view_ticks(request_start_view_ticks(config)); - consensus.set_probe_attempts_max(config.cluster.view_probe_attempts_max); - // Fresh random incarnation each boot, so a StartView addressed to a previous - // incarnation still in flight is ignored (`handle_start_view` guard). `| 1` - // guarantees the non-zero the guard treats as set. The deterministic simulator - // overrides this with a seed-derived value bumped per restart. - consensus.set_incarnation(rand::random::() | 1); - - let last_header = journal - .last_op() - .and_then(|op| usize::try_from(op).ok()) - .and_then(|op| journal.header(op).map(|header| *header)); - // View and log_view come from the durable superblock when present. A present but - // unreadable superblock already refused boot in `recover()`, so reaching the - // `else` means it is genuinely absent: a fresh node, or one that took writes but - // never checkpointed or changed view. There, inferring the view from the last WAL - // prepare is safe, since the persist-before-send gate guarantees this replica - // never externalized a view beyond what a re-probe re-derives, and it re-probes - // as a backup below. log_view cannot be inferred and stays 0 until the next - // superblock write. - if let Some(state) = recovered_state { - consensus.set_view(state.view); - consensus.set_log_view(state.log_view); - consensus.mark_superblock_durable(state.view, state.log_view); - } else if let Some(header) = last_header { - consensus.set_view(header.view); - } - - // On a RESTART in a cluster, rejoin as a quorum-invisible backup and - // probe for the current view (`RequestStartView`): the view's primary - // answers with a `StartView`, the replica adopts it as a backup, and - // journal repair fills any WAL gap. A probing replica never resumes - // primaryship -- if this replica IS the current primary-by-index, its - // probe makes the backups elect past it. - // The probe re-broadcasts on its timeout, so it needs no live mesh at - // boot. A FRESH boot keeps the plain init: the cluster needs its view-0 - // primary to exist, and a single-replica cluster has no peer to ask. - // - // Prior life is EITHER a non-empty WAL or a recovered superblock. A view - // change persists without touching the WAL, so a replica that changed - // view before its first metadata write comes back with a non-zero view - // and an empty journal; gating on the WAL alone would `init()` it into - // `Status::Normal` as primary for a view the cluster may have moved past, - // with `ceded_primaryship` false and no probe to correct it. - if replica_count > 1 && (restored_op > 0 || recovered_state.is_some()) { - consensus.init_as_backup(); - consensus.begin_view_probe(); - // Restart in a cluster: replace snapshot-shaped metadata state - // (snapshot + client table) from the live primary the probe finds, - // then journal-repair the tail. If the probe exhausts instead -- - // full-cluster bootstrap, nobody live to fetch from -- the election - // fallback clears the stage and this local recovery stands. - consensus.begin_state_transfer_await(); - } else { - consensus.init(); - } - consensus.sequencer().set_sequence(restored_op); - // A SOLO replica's durable journal head IS its commit point: quorum is - // 1-of-1, so an entry commits the instant it is durable, and the acks - // the cluster ceremony below would wait on cannot topologically exist. - // The embedded watermark is structurally one op stale (the commit point - // is only ever written down inside the NEXT entry), so trusting it solo - // manufactures an "uncommitted" suffix that provably committed and - // wedges the recovery barrier forever. - let commit_watermark = if replica_count == 1 { - restored_op - } else { - commit_watermark - }; - // The commit point is restored from the WAL's embedded watermark (each - // journaled prepare carries the primary's commit at send time), NOT from - // the journal head: journaled does not imply committed, and claiming - // commit for the un-quorum'd tail both risks split-brain on a later view - // change and starves the tail of re-replication (it would live in no - // pipeline). The suffix `(commit_watermark, restored_op]` is re-pipelined - // below when this replica is the recovered view's primary. - // - // TODO(hubcio): the watermark is a lower bound (the last entry stamps - // the commit point as of its send). Persisting an explicit (view, - // commit_op) watermark on the commit path would tighten recovery and - // allow refusing boot on an excessive gap; a backup that recovered a - // LONGER tail than the cluster's primary still needs uncommitted-suffix - // truncation when conflicting ops arrive (message repair milestone). - consensus.restore_commit_state(commit_watermark, commit_watermark); - if let Some(header) = last_header { - consensus.set_last_prepare_checksum(header.checksum); - consensus.observe_prepare_timestamp(header.timestamp); - } - - // The WAL's tail past the watermark is prepared-but-not-provably-committed - // state. Until the cluster confirms it (re-pipelined below on a resumed - // primary; via StartView adoption + the local commit walk on a rejoined - // backup), serving reads would show pre-restart state that clients already - // saw acked -- gate them on the barrier regardless of role. If the suffix - // never re-commits cluster-wide, the read path fails loud with a retryable - // 503 once the paired deadline expires (`await_recovery_barrier`). - if commit_watermark < restored_op { - consensus.set_recovery_barrier(restored_op); - consensus.set_recovery_deadline(recovery_deadline); - } - - // Re-pipeline the prepared-but-uncommitted suffix so the primary's - // retransmit machinery re-replicates it and quorum can (re-)commit it. - // A backup's suffix stays journal-only: the primary's traffic either - // confirms it (re-forward + re-ack path) or supersedes it. - if consensus.is_primary() - && !consensus.has_ceded_primaryship() - && commit_watermark < restored_op - { - info!( - commit_watermark, - restored_op, "re-pipelining recovered uncommitted metadata suffix" - ); - let mut pipeline = consensus.pipeline().borrow_mut(); - #[allow(clippy::cast_possible_truncation)] - for op in (commit_watermark + 1)..=restored_op { - let Some(header) = journal.header(op as usize) else { - warn!( - op, - "recovered journal suffix has a gap; stopping re-pipeline" - ); - break; - }; - let mut entry = PipelineEntry::new(*header); - entry.add_ack(topology.self_replica_id); - pipeline.push(entry); - } - } - - consensus -} - -#[allow(clippy::too_many_arguments)] -async fn load_partition( - config: &ServerNgConfig, - namespace: IggyNamespace, - stats: Arc, - partition_metadata: &Partition, - cluster_id: u128, - self_replica_id: u8, - replica_count: u8, - bus: Rc, -) -> Result>, ServerNgError> { - let stream_id = namespace.stream_id(); - let topic_id = namespace.topic_id(); - let partition_id = namespace.partition_id(); - // Request queue holds 2x the prepare depth (buffered requests drain as - // prepares commit); depth is the per-partition `[partition]` knob. - let prepare_queue_depth = config.partition.prepare_queue_depth; - let mut consensus = VsrConsensus::new( - cluster_id, - self_replica_id, - replica_count, - namespace.inner(), - bus, - LocalPipeline::with_capacities(prepare_queue_depth, prepare_queue_depth * 2), - ); - consensus.set_normal_heartbeat_ticks(cluster_heartbeat_ticks(config)); - consensus.set_commit_message_ticks(commit_broadcast_ticks(config)); - consensus.set_prepare_ticks(prepare_retransmit_ticks(config)); - consensus.set_view_change_retransmit_ticks(view_change_retransmit_ticks(config)); - consensus.set_view_change_status_ticks(view_change_status_ticks(config)); - consensus.set_request_start_view_ticks(request_start_view_ticks(config)); - consensus.set_probe_attempts_max(config.cluster.view_probe_attempts_max); - - // (view, log_view) come from the group's durable superblock when present; - // a present but unverifiable record already refused boot inside - // `open_partition_superblock`. Restored BEFORE choosing how to join, so - // the backup probe below never advertises a view older than the recorded - // one. - let partition_dir = config - .system - .get_partition_path(stream_id, topic_id, partition_id); - let (superblock, recovered_state) = open_partition_superblock( - &partition_dir, - ReplicaIdentity { - cluster: cluster_id, - replica_id: self_replica_id, - replica_count, - }, - ) - .await?; - if let Some(state) = recovered_state.as_ref() { - restore_partition_view(&mut consensus, state); - } - - // A recovered partition lost its journal state with the process: the - // partition journal is in-memory and segments carry no op numbers, so - // this replica cannot know the group's (op, commit) even when the - // superblock restored its view. In a cluster it boots as a - // quorum-invisible backup and probes for the current view - // (`RequestStartView`): the view's primary answers with a `StartView`, - // journal repair fills the rejoin window, and the commit floor settles - // at the serving peer's retention point. The probe re-broadcasts on its - // timeout, so it needs no live mesh at boot. Single-replica groups - // have no peer to ask and keep the plain init. - if replica_count > 1 { - consensus.init_as_backup(); - consensus.begin_view_probe(); - } else { - consensus.init(); - } - - // No prepare-timestamp floor is restored here: the partition consensus - // journal is non-durable today, so there is no persisted head to observe - // (unlike `restore_metadata_consensus`, which observes its restored head). - // When PartitionJournal becomes durable (the milestone named in the - // multi-shard wiring commit body), observe the restored head and the max - // recovered message timestamp here, or an NTP rewind across a restart could - // regress persisted `base_timestamp`. - - let recovered_segments = - load_persisted_segments(config, stream_id, topic_id, partition_id, &stats) - .await - .map_err(|source| { - error!( - stream_id, - topic_id, - partition_id, - error = %source, - "failed to load partition log during server-ng bootstrap" - ); - source - })?; - - let mut partition = IggyPartition::new(stats.clone(), consensus); - partition.set_superblock(superblock, recovered_state.as_ref()); - // Recovered partitions honor the same config-surfaced ring ceilings as the - // fresh-create path (build_partition_fresh). Retention is already off for - // single-replica groups, so this only sizes the multi-replica ring. - partition.log.journal().inner.set_ring_caps( - config.partition.evicted_ring_capacity, - config.partition.evicted_ring_bytes_max.as_bytes_u64(), - ); - partition.set_partition_dir(partition_dir); - // Before the hydrate: the durable record is keyed by incarnation, so a - // `purge.gen` left behind by a previous life of this namespace reads 0. - partition.set_created_revision(partition_metadata.created_revision); - partition.hydrate_applied_purge_generation().await?; - hydrate_partition_log( - &mut partition, - config, - stream_id, - topic_id, - partition_id, - recovered_segments, - ) - .await?; - - let sized_end = partition - .log - .segments() - .iter() - .filter(|segment| segment.size > IggyByteSize::default()) - .map(|segment| segment.end_offset) - .max(); - // An empty chain whose segment is named for a nonzero offset is the - // shape a state-transfer install (or its converge) plants at the group - // frontier after the origin GC'd everything: the file name carries the - // frontier, and re-minting offsets from 0 here would fork this - // replica's batch stamps from the rest of the group after a restart. - let empty_frontier = partition - .log - .segments() - .iter() - .map(|segment| segment.start_offset) - .max() - .filter(|&start| sized_end.is_none() && start > 0); - let current_offset = sized_end.or_else(|| empty_frontier.map(|start| start - 1)); - partition.created_at = partition_metadata.created_at; - partition.recovered_durable_offset = sized_end; - // The OFFSET COUNTER is restored from that file name (above), but the - // `installed_frontier` CLAIM deliberately is not: the claim says "everything - // below me is represented here", and `converge_to_empty_after_failed_install` - // refuses to make it when staged segments were dropped -- yet a converge - // plants exactly the same empty `{frontier:020}.log` a legitimate empty - // install does, so boot provably cannot tell them apart. Re-deriving it here - // would hand the refused claim back: the repair floor stand-in would accept a - // commit floor over ops this replica holds zero bytes for, and the replica - // would pass the serve gate and offer that emptiness onward, making a peer - // unlink its own chain. Leaving it `None` costs one spurious full - // re-transfer on the legitimate empty-install restart; a false caught-up - // claim is not recoverable. A durable home for the frontier (the partition - // superblock already reserves a field) is what would settle it properly. - let counter = current_offset.unwrap_or(0); - partition.offset.store(counter, Ordering::Release); - partition.dirty_offset.store(counter, Ordering::Relaxed); - partition.should_increment_offset = current_offset.is_some(); - partition.stats.set_current_offset(counter); - // The durable frontier is a LOWER BOUND on top of what the segments proved: - // it is the only carrier left when the segments that named the frontier are - // gone (an all-GC'd origin's install, a crash inside the swap window), and - // taking the max means real recovered data always wins. - partition.restore_offset_frontier(recovered_state.as_ref()); - let current_offset = partition.offset.load(Ordering::Acquire); - - configure_consumer_offsets(&mut partition, config, namespace, current_offset)?; - ensure_initial_segment(&mut partition, config, stream_id, topic_id, partition_id).await?; - - Ok(partition) -} - -async fn hydrate_partition_log( - partition: &mut IggyPartition>, - config: &ServerNgConfig, - stream_id: usize, - topic_id: usize, - partition_id: usize, - recovered_segments: Vec, -) -> Result<(), ServerNgError> { - for RecoveredSegment { segment, storage } in recovered_segments { - partition - .log - .add_persisted_segment(segment, storage, None, None); - } - - if let Some(active_index) = partition.log.segments().len().checked_sub(1) { - let storage = &partition.log.storages()[active_index]; - if let ( - Some(messages_reader), - Some(index_reader), - Some(storage_messages_writer), - Some(storage_index_writer), - ) = ( - storage.messages_reader.as_ref(), - storage.index_reader.as_ref(), - storage.messages_writer.as_ref(), - storage.index_writer.as_ref(), - ) { - let index_path = index_reader.path(); - // Share the storage's size counters: the readers bound reads by - // these atomics, so a writer with a private counter persists bytes - // the readers never learn about. - let messages_size_counter = storage_messages_writer.size_counter(); - let index_size_counter = storage_index_writer.size_counter(); - partition.log.messages_writers_mut()[active_index] = Some(Rc::new( - MessagesWriter::new( - &messages_reader.path(), - messages_size_counter, - config.system.partition.enforce_fsync, - true, - ) - .await - .map_err(|source| { - error!( - stream_id, - topic_id, - partition_id, - path = %messages_reader.path(), - error = %source, - "failed to initialize persisted messages writer" - ); - source - })?, - )); - partition.log.index_writers_mut()[active_index] = Some(Rc::new( - IggyIndexWriter::new( - &index_path, - index_size_counter, - config.system.partition.enforce_fsync, - true, - ) - .await - .map_err(|source| { - error!( - stream_id, - topic_id, - partition_id, - path = %index_path, - error = %source, - "failed to initialize persisted sparse index writer" - ); - source - })?, - )); - } - } - - Ok(()) -} - -fn resolve_tcp_topology( - config: &ServerNgConfig, - current_replica_id: Option, -) -> Result { - let default_client_addr = parse_socket_addr("tcp.address", &config.tcp.address)?; - let default_ws_addr = resolve_optional_listener_addr( - config.websocket.enabled, - "websocket.address", - &config.websocket.address, - )?; - let default_quic_addr = - resolve_optional_listener_addr(config.quic.enabled, "quic.address", &config.quic.address)?; - let default_http_addr = - resolve_optional_listener_addr(config.http.enabled, "http.address", &config.http.address)?; - if !config.cluster.enabled { - if let Some(replica_id) = current_replica_id - && replica_id != SHARD_REPLICA_ID - { - return Err(ServerNgError::ReplicaIdRequiresCluster { - supplied: replica_id, - default: SHARD_REPLICA_ID, - }); - } - return Ok(TcpTopology { - cluster_id: auth::cluster_domain_id(&config.cluster.name), - // Keep parity with the current server binary and the integration - // harness: `--replica-id 0` may be passed unconditionally in - // single-node mode; any other id is rejected above so the WAL - // cannot commit under an identity that will later disagree with - // a cluster.nodes[] entry. - self_replica_id: SHARD_REPLICA_ID, - replica_count: 1, - client_listen_addr: default_client_addr, - replica_listen_addr: Some(SocketAddr::new(default_client_addr.ip(), 0)), - ws_listen_addr: default_ws_addr, - quic_listen_addr: default_quic_addr, - http_listen_addr: default_http_addr, - tcp_tls_listen_addr: config.tcp.tls.enabled.then_some(default_client_addr), - peers: Vec::new(), - }); - } - - let self_replica_id = current_replica_id.ok_or(ServerNgError::MissingReplicaId)?; - - let self_node = config - .cluster - .nodes - .iter() - .find(|node| node.replica_id == self_replica_id) - .ok_or(ServerNgError::ClusterNodeNotFound { - replica_id: self_replica_id, - })?; - let replica_count = u8::try_from(config.cluster.nodes.len()).map_err(|_| { - ServerNgError::ClusterReplicaCountTooLarge { - count: config.cluster.nodes.len(), - } - })?; - let ClusterClientAddrs { - client: client_listen_addr, - ws: ws_listen_addr, - quic: quic_listen_addr, - http: http_listen_addr, - } = resolve_cluster_client_addrs( - self_node, - default_client_addr, - default_ws_addr, - default_quic_addr, - default_http_addr, - )?; - let replica_port = self_node - .ports - .tcp_replica - .ok_or(ServerNgError::ClusterPortMissing { - transport: "tcp_replica", - replica_id: self_node.replica_id, - })?; - let replica_listen_addr = Some(socket_addr_from_parts( - "cluster.nodes[*].ports.tcp_replica", - &self_node.ip, - replica_port, - )?); - let peers = resolve_cluster_replica_peers(&config.cluster.nodes, self_replica_id)?; - - Ok(TcpTopology { - cluster_id: auth::cluster_domain_id(&config.cluster.name), - self_replica_id, - replica_count, - client_listen_addr, - replica_listen_addr, - ws_listen_addr, - quic_listen_addr, - http_listen_addr, - tcp_tls_listen_addr: config.tcp.tls.enabled.then_some(client_listen_addr), - peers, - }) -} - -fn resolve_optional_listener_addr( - enabled: bool, - context: &'static str, - address: &str, -) -> Result, ServerNgError> { - if enabled { - return Ok(Some(parse_socket_addr(context, address)?)); - } - Ok(None) -} - -/// Client-facing listener addresses resolved for this cluster node. Each port -/// comes from the node's roster entry; there is no fallback to the top-level -/// listener port, an enabled transport without a roster port refuses to boot. -/// Every transport keeps the bind interface from its own `address` config: the -/// roster ip is advertised, not bound. -struct ClusterClientAddrs { - client: SocketAddr, - ws: Option, - quic: Option, - http: Option, -} - -fn resolve_cluster_client_addrs( - self_node: &configs::ng_cluster::ClusterNodeConfig, - default_tcp_addr: SocketAddr, - default_ws_addr: Option, - default_quic_addr: Option, - default_http_addr: Option, -) -> Result { - let client_port = self_node - .ports - .tcp - .ok_or(ServerNgError::ClusterPortMissing { - transport: "tcp", - replica_id: self_node.replica_id, - })?; - let client = - merge_roster_port_with_bind_ip("tcp", &self_node.ip, default_tcp_addr, client_port); - let ws = resolve_cluster_optional_addr(self_node, "websocket", default_ws_addr, |ports| { - ports.websocket - })?; - let quic = - resolve_cluster_optional_addr(self_node, "quic", default_quic_addr, |ports| ports.quic)?; - let http = - resolve_cluster_optional_addr(self_node, "http", default_http_addr, |ports| ports.http)?; - Ok(ClusterClientAddrs { - client, - ws, - quic, - http, - }) -} - -fn resolve_cluster_optional_addr( - self_node: &configs::ng_cluster::ClusterNodeConfig, - transport: &'static str, - default_addr: Option, - port_selector: impl Fn(&configs::ng_cluster::TransportPorts) -> Option, -) -> Result, ServerNgError> { - let Some(default_addr) = default_addr else { - return Ok(None); - }; - // No fallback to the top-level port: two same-host nodes leaving the same - // transport port unset would race for one socket. Either the roster is - // explicit or the server refuses to boot. - let port = port_selector(&self_node.ports).ok_or(ServerNgError::ClusterPortMissing { - transport, - replica_id: self_node.replica_id, - })?; - Ok(Some(merge_roster_port_with_bind_ip( - transport, - &self_node.ip, - default_addr, - port, - ))) -} - -/// Combine the roster-supplied `port` with the bind interface the transport's -/// own `address` config asked for. -/// -/// The roster ip is what the cluster advertises (metadata, follower-to-primary -/// HTTP forwarding targets); the transport's own `address` decides the bind -/// interface. Merging keeps a loopback-only `127.0.0.1` private and a -/// `0.0.0.0` wide in cluster mode instead of silently rebinding to the roster -/// interface, which would strand every co-located dialer (sidecars, health -/// probes, on-host consumers) on `ECONNREFUSED`. -fn merge_roster_port_with_bind_ip( - transport: &'static str, - roster_ip: &str, - bind_addr: SocketAddr, - port: u16, -) -> SocketAddr { - let listen_addr = SocketAddr::new(bind_addr.ip(), port); - if roster_ip_unreachable_from_bind_addr(roster_ip, listen_addr) { - warn!( - "{transport} listener binds {listen_addr} but the roster advertises {roster_ip}:{port}; \ - peers and clients dialing the advertised endpoint may not reach this node" - ); - } - listen_addr -} - -/// Whether a dialer aiming at the advertised roster ip misses `listen_addr`. An -/// unspecified bind covers every interface, and a roster ip that parses as -/// neither IPv4 nor IPv6 (a DNS name, say) can resolve to the bound interface, -/// so both cases stay quiet. -fn roster_ip_unreachable_from_bind_addr(roster_ip: &str, listen_addr: SocketAddr) -> bool { - !listen_addr.ip().is_unspecified() - && roster_ip - .parse::() - .is_ok_and(|parsed| parsed != listen_addr.ip()) -} - -fn resolve_cluster_replica_peers( - nodes: &[configs::ng_cluster::ClusterNodeConfig], - self_replica_id: u8, -) -> Result, ServerNgError> { - let mut peers = Vec::with_capacity(nodes.len().saturating_sub(1)); - for node in nodes { - if node.replica_id == self_replica_id { - continue; - } - let replica_port = node - .ports - .tcp_replica - .ok_or(ServerNgError::ClusterPortMissing { - transport: "tcp_replica", - replica_id: node.replica_id, - })?; - peers.push(( - node.replica_id, - socket_addr_from_parts("cluster.nodes[*].ports.tcp_replica", &node.ip, replica_port)?, - )); - } - Ok(peers) -} - -async fn start_tcp_runtime( - shard: &Rc, - config: &ServerNgConfig, - topology: &TcpTopology, - accepted_replica: AcceptedReplicaFn, - dialed_replica: DialedReplicaFn, - accepted_clients: LocalClientAcceptFns, -) -> Result<(), ServerNgError> { - if config.tcp.enabled && !config.tcp.tls.enabled { - start_via_replica_io( - shard, - config, - topology, - accepted_replica, - dialed_replica, - accepted_clients, - ) - .await?; - } else { - start_manual_runtime( - shard, - config, - topology, - accepted_replica, - dialed_replica, - accepted_clients, - ) - .await?; - } - - // HTTP is served over TCP but sits outside the replica_io / manual client - // reactor, so it binds independently. Shard-0 gating comes from the sole - // caller of this function. - if let Some(http_addr) = topology.http_listen_addr { - let self_ports = configs::ng_cluster::TransportPorts { - tcp: config - .tcp - .enabled - .then(|| topology.client_listen_addr.port()), - quic: topology.quic_listen_addr.map(|addr| addr.port()), - websocket: topology.ws_listen_addr.map(|addr| addr.port()), - ..Default::default() - }; - http::start( - shard, - http_addr, - &config.http, - config.metadata.clients_table_max, - config.personal_access_token.max_tokens_per_user, - &config.cluster, - Arc::clone(&config.system), - self_ports, - ) - .await?; - } - - Ok(()) -} - -// ws/wss bindings intentionally mirror the transport names (same convention as -// `replica_io::start_on_shard_zero`). -#[allow(clippy::similar_names)] -async fn start_via_replica_io( - shard: &Rc, - config: &ServerNgConfig, - topology: &TcpTopology, - accepted_replica: AcceptedReplicaFn, - dialed_replica: DialedReplicaFn, - accepted_clients: LocalClientAcceptFns, -) -> Result<(), ServerNgError> { - let replica_addr = topology - .replica_listen_addr - .expect("topology must include replica listener address"); - let quic_credentials = topology - .quic_listen_addr - .is_some() - .then(|| load_quic_server_credentials(config)) - .transpose()?; - let tcp_tls_credentials = topology - .tcp_tls_listen_addr - .is_some() - .then(|| load_tcp_tls_server_credentials(config)) - .transpose()?; - // `websocket.tls.enabled` upgrades the websocket address to a WSS - // listener; the plain-WS listener must NOT also bind it (one port, one - // handshake kind -- a plain upgrade parser fed a TLS ClientHello rejects - // every connection with an httparse error). - let wss_enabled = config.websocket.tls.enabled; - let ws_listen_addr = (!wss_enabled).then_some(topology.ws_listen_addr).flatten(); - let wss_listen_addr = wss_enabled.then_some(topology.ws_listen_addr).flatten(); - let wss_credentials = wss_listen_addr - .is_some() - .then(|| load_wss_server_credentials(config)) - .transpose()?; - - let LocalClientAcceptFns { - tcp, - ws, - quic, - tcp_tls, - wss, - } = accepted_clients; - - let bound = replica_io::start_on_shard_zero( - &shard.bus, - replica_addr, - topology.client_listen_addr, - ws_listen_addr, - topology.quic_listen_addr, - quic_credentials, - topology.tcp_tls_listen_addr, - tcp_tls_credentials, - wss_listen_addr, - wss_credentials, - topology.self_replica_id, - topology.peers.clone(), - accepted_replica, - dialed_replica, - tcp, - ws_listen_addr.map(|_| ws), - topology.quic_listen_addr.map(|_| quic), - topology.tcp_tls_listen_addr.map(|_| tcp_tls), - wss_listen_addr.map(|_| wss), - shard.bus.config().reconnect_period, - ) - .await - .map_err(|source| { - error!( - replica_addr = %replica_addr, - client_addr = %topology.client_listen_addr, - error = %source, - "failed to start server-ng listeners via replica_io" - ); - source - })?; - let Some(bound) = bound else { - return Ok(()); - }; - - write_current_config( - config, - Some(topology.self_replica_id), - Some(bound.client), - config.cluster.enabled.then_some(bound.replica), - bound.tcp_tls, - bound.quic, - // The WSS listener occupies the configured websocket address slot. - bound.wss.or(bound.ws), - ) - .await?; - if config.cluster.enabled { - info!( - shard = shard.id, - replica = %bound.replica, - tcp = %bound.client, - tcp_tls = ?bound.tcp_tls, - ws = ?bound.ws, - quic = ?bound.quic, - "server-ng listeners started" - ); - } else { - info!( - shard = shard.id, - tcp = %bound.client, - tcp_tls = ?bound.tcp_tls, - ws = ?bound.ws, - quic = ?bound.quic, - "server-ng client listeners started" - ); - } - - Ok(()) -} - -async fn start_manual_runtime( - shard: &Rc, - config: &ServerNgConfig, - topology: &TcpTopology, - accepted_replica: AcceptedReplicaFn, - dialed_replica: DialedReplicaFn, - accepted_clients: LocalClientAcceptFns, -) -> Result<(), ServerNgError> { - let bound_replica = if config.cluster.enabled { - let replica_addr = topology - .replica_listen_addr - .expect("cluster-enabled topology must include replica listener address"); - let (replica_listener, bound_addr) = - replica_listener::bind(replica_addr) - .await - .map_err(|source| { - error!( - replica_addr = %replica_addr, - error = %source, - "failed to bind replica listener" - ); - source - })?; - let token = shard.bus.token(); - let replica_handle = compio::runtime::spawn(async move { - replica_listener::run(replica_listener, token, accepted_replica).await; - }); - shard.bus.track_background(replica_handle); - connector::start( - &shard.bus, - topology.self_replica_id, - topology.peers.clone(), - dialed_replica, - shard.bus.config().reconnect_period, - ) - .await; - Some(bound_addr) - } else { - None - }; - - let bound_clients = start_client_listeners(shard, config, topology, &accepted_clients).await?; - write_current_config( - config, - Some(topology.self_replica_id), - bound_clients.tcp, - bound_replica, - bound_clients.tcp_tls, - bound_clients.quic, - bound_clients.ws, - ) - .await?; - - if config.cluster.enabled { - info!( - shard = shard.id, - replica = ?bound_replica, - tcp = ?bound_clients.tcp, - tcp_tls = ?bound_clients.tcp_tls, - ws = ?bound_clients.ws, - quic = ?bound_clients.quic, - "server-ng listeners started" - ); - } else { - info!( - shard = shard.id, - tcp = ?bound_clients.tcp, - tcp_tls = ?bound_clients.tcp_tls, - ws = ?bound_clients.ws, - quic = ?bound_clients.quic, - "server-ng client listeners started" - ); - } - - Ok(()) -} - -fn ensure_default_root_user(mux_stm: &ServerNgMuxStateMachine) { - if !mux_stm.users().read(|users| users.items.is_empty()) { - return; - } - - let (username, password_hash) = create_root_credentials(); - mux_stm.users().ensure_root_user(&username, &password_hash); -} - -/// Resolve the root user credentials from `IGGY_ROOT_USERNAME` / -/// `IGGY_ROOT_PASSWORD`, falling back to the default username with a -/// generated password (printed to stdout, mirroring the legacy server). -/// -/// Returns `(username, password_hash)`; the plaintext password never -/// leaves this function. -fn create_root_credentials() -> (String, String) { - let mut username = env::var(IGGY_ROOT_USERNAME_ENV); - let mut password = env::var(IGGY_ROOT_PASSWORD_ENV); - assert_eq!( - username.is_ok(), - password.is_ok(), - "When providing the custom root user credentials, both username and password must be set." - ); - if username.is_ok() && password.is_ok() { - info!("Using the custom root user credentials."); - } else { - info!("Using the default root user credentials..."); - username = Ok(DEFAULT_ROOT_USERNAME.to_string()); - let generated_password = crypto::generate_secret(20..40); - println!("Generated root user password: {generated_password}"); - password = Ok(generated_password); - } - - let username = username.expect("Root username is not set."); - let password = password.expect("Root password is not set."); - assert!( - !username.is_empty() && !password.is_empty(), - "Root user credentials cannot be empty." - ); - assert!( - username.len() >= MIN_USERNAME_LENGTH, - "Root username is too short." - ); - assert!( - username.len() <= MAX_USERNAME_LENGTH, - "Root username is too long." - ); - assert!( - password.len() >= MIN_PASSWORD_LENGTH, - "Root password is too short." - ); - assert!( - password.len() <= MAX_PASSWORD_LENGTH, - "Root password is too long." - ); - - (username, crypto::hash_password(&password)) -} - -fn validate_cluster_root_bootstrap( - config: &ServerNgConfig, - mux_stm: &ServerNgMuxStateMachine, -) -> Result<(), ServerNgError> { - if !config.cluster.enabled || !mux_stm.users().read(|users| users.items.is_empty()) { - return Ok(()); - } - - if env::var(IGGY_ROOT_USERNAME_ENV).is_ok() && env::var(IGGY_ROOT_PASSWORD_ENV).is_ok() { - return Ok(()); - } - - Err(ServerNgError::ClusterRootCredentialsRequired { - username_env: IGGY_ROOT_USERNAME_ENV, - password_env: IGGY_ROOT_PASSWORD_ENV, - }) -} - -/// Replica delegation callbacks for shard 0's listener and connector. -/// -/// Inbound: acquire a slot in the shard-0-global in-flight handshake cap -/// (drop the connection when full), then blind-delegate the raw fd -/// through the coordinator's round-robin. The fd lands on the target -/// shard's inbox as a [`shard::LifecycleFrame::ReplicaInboundSetup`] -/// frame; the owning shard runs the acceptor handshake and acks the -/// slot back. A failed delegation releases the slot immediately. -/// -/// Outbound: delegate the dialed fd as -/// [`shard::LifecycleFrame::ReplicaOutboundSetup`] and mark the peer -/// dial-pending so the reconnect sweep skips it until the owning -/// shard's handshake outcome arrives (or the entry expires). -fn make_replica_delegation_fns( - coord: Rc, - bus: &Rc, -) -> (AcceptedReplicaFn, DialedReplicaFn) { - let inbound_bus = Rc::clone(bus); - let inbound_coord = Rc::clone(&coord); - let accepted: AcceptedReplicaFn = Rc::new(move |stream| { - let Some(slot) = inbound_bus.try_acquire_replica_handshake_slot() else { - warn!( - cap = MAX_INFLIGHT_REPLICA_HANDSHAKES, - "replica handshake in-flight cap reached; dropping inbound" - ); - return; - }; - match inbound_coord.delegate_replica_inbound(stream, slot) { - Ok(target) => { - info!(slot, target, "inbound replica connection delegated"); - } - Err(error) => { - inbound_bus.release_replica_handshake_slot(slot); - warn!( - error = ?error, - "delegate_replica_inbound failed; dropping inbound replica connection" - ); - } - } - }); - - let outbound_bus = Rc::clone(bus); - let dialed: DialedReplicaFn = - Rc::new( - move |stream, peer_id| match coord.delegate_replica_outbound(stream, peer_id) { - Ok(target) => { - outbound_bus.mark_dial_pending(peer_id); - info!(peer_id, target, "outbound replica connection delegated"); - } - Err(error) => { - warn!( - peer_id, - error = ?error, - "delegate_replica_outbound failed; dropping dialed replica connection" - ); - } - }, - ); - - (accepted, dialed) -} - -/// Shard-0 client accept callbacks. TCP and WS clients are delegated via -/// the coordinator (round-robin to peer shards); QUIC and TCP-TLS install -/// locally on shard 0 because their per-connection state is not portable -/// across shards (`compio_quic` endpoint binds one UDP socket; rustls TLS -/// state ties to the post-handshake reactor). -// ws/wss bindings intentionally mirror the transport names (same convention as -// `replica_io::start_on_shard_zero`). -#[allow(clippy::similar_names)] -fn make_shard_zero_client_accept_fns( - coord: Rc, - bus: &Rc, - on_request: RequestHandler, -) -> LocalClientAcceptFns { - let quic_bus = Rc::clone(bus); - let tcp_tls_bus = Rc::clone(bus); - let wss_bus = Rc::clone(bus); - let quic_request = on_request.clone(); - let wss_request = on_request.clone(); - let tcp_tls_request = on_request; - - let tcp_coord = Rc::clone(&coord); - let tcp = Rc::new(move |stream| match tcp_coord.delegate_client(stream) { - Ok(client_id) => info!(client_id, "TCP client delegated"), - Err(error) => warn!(error = ?error, "delegate_client failed; dropping TCP client"), - }); - - let ws_coord = Rc::clone(&coord); - let ws = Rc::new(move |stream| match ws_coord.delegate_ws_client(stream) { - Ok(client_id) => info!(client_id, "WS client delegated"), - Err(error) => warn!(error = ?error, "delegate_ws_client failed; dropping WS client"), - }); - - // QUIC and TCP-TLS terminate locally on shard 0 but mint their client - // ids through the coordinator's `client_seq`, the same counter the - // delegated TCP/WS path uses. A separate counter here would let a - // shard-0-local id collide with a delegated id that round-robined to - // shard 0 (both encode target shard 0) in shard 0's connection - // registry. - let quic_coord = Rc::clone(&coord); - let quic = Rc::new(move |accepted: message_bus::AcceptedQuicConn| { - let meta = mint_client_meta(&quic_coord, accepted.peer_addr(), ClientTransportKind::Quic); - installer::install_client_quic(&quic_bus, meta, accepted, quic_request.clone()); - }); - - let tcp_tls_coord = Rc::clone(&coord); - let tcp_tls = Rc::new(move |stream, tls_config| { - let Some(meta) = - client_meta_from_stream(&stream, &tcp_tls_coord, ClientTransportKind::TcpTls) - else { - return; - }; - installer::install_client_tcp_tls( - &tcp_tls_bus, - meta, - stream, - tls_config, - tcp_tls_request.clone(), - ); - }); - - // WSS terminates locally on shard 0 like TCP-TLS (rustls state is not - // serialisable across the delegate path), minting ids through the same - // coordinator counter. - let wss_coord = coord; - let wss = Rc::new(move |stream, tls_config| { - let Some(meta) = client_meta_from_stream(&stream, &wss_coord, ClientTransportKind::Wss) - else { - return; - }; - installer::install_client_wss(&wss_bus, meta, stream, tls_config, wss_request.clone()); - }); - - LocalClientAcceptFns { - tcp, - ws, - quic, - tcp_tls, - wss, - } -} - -fn client_meta_from_stream( - stream: &compio::net::TcpStream, - coord: &shard::coordinator::ShardZeroCoordinator, - transport: ClientTransportKind, -) -> Option { - let peer_addr = match stream.peer_addr() { - Ok(peer_addr) => peer_addr, - Err(error) => { - warn!(error = %error, "dropping accepted client with unknown peer address"); - return None; - } - }; - Some(mint_client_meta(coord, peer_addr, transport)) -} - -fn mint_client_meta( - coord: &shard::coordinator::ShardZeroCoordinator, - peer_addr: SocketAddr, - transport: ClientTransportKind, -) -> ClientConnMeta { - ClientConnMeta::new(coord.mint_shard_zero_client_id(), peer_addr, transport) -} - -async fn start_client_listeners( - shard: &Rc, - config: &ServerNgConfig, - topology: &TcpTopology, - accepted_clients: &LocalClientAcceptFns, -) -> Result { - let mut bound = BoundClientListeners::default(); - - if config.tcp.enabled && !config.tcp.tls.enabled { - let (listener, bound_addr) = client_listener::tcp::bind(topology.client_listen_addr) - .await - .map_err(|source| { - error!( - addr = %topology.client_listen_addr, - error = %source, - "failed to bind TCP client listener" - ); - source - })?; - let token = shard.bus.token(); - let accepted_client = accepted_clients.tcp.clone(); - let client_handle = compio::runtime::spawn(async move { - client_listener::tcp::run(listener, token, accepted_client).await; - }); - shard.bus.track_background(client_handle); - bound.tcp = Some(bound_addr); - } - - if let Some(ws_addr) = topology.ws_listen_addr { - bound.ws = Some(start_websocket_listener(shard, config, ws_addr, accepted_clients).await?); - } - - if let Some(quic_addr) = topology.quic_listen_addr { - install_default_crypto_provider(); - let credentials = load_quic_server_credentials(config)?; - let server_config = server_config_with_cert( - credentials.cert_chain, - credentials.key_der, - &shard.bus.config().quic, - ) - .map_err(|e| { - let source = - iggy_common::IggyError::IoError(format!("QUIC server config build failed: {e}")); - error!(addr = %quic_addr, error = %source, "failed to build QUIC server config"); - source - })?; - let (endpoint, bound_addr) = client_listener::quic::bind(quic_addr, server_config) - .map_err(|source| { - error!(addr = %quic_addr, error = %source, "failed to bind QUIC listener"); - source - })?; - let token = shard.bus.token(); - let handshake_grace = shard.bus.config().handshake_grace; - let accepted_quic = accepted_clients.quic.clone(); - let quic_handle = compio::runtime::spawn(async move { - client_listener::quic::run(endpoint, token, accepted_quic, handshake_grace).await; - }); - shard.bus.track_background(quic_handle); - bound.quic = Some(bound_addr); - } - - if config.tcp.enabled && config.tcp.tls.enabled { - let credentials = load_tcp_tls_server_credentials(config)?; - let (listener, tls_config, bound_addr) = - client_listener::tcp_tls::bind(topology.client_listen_addr, credentials).map_err( - |source| { - error!( - addr = %topology.client_listen_addr, - error = %source, - "failed to bind TCP TLS listener" - ); - source - }, - )?; - let token = shard.bus.token(); - let accepted_tls = accepted_clients.tcp_tls.clone(); - let tls_handle = compio::runtime::spawn(async move { - client_listener::tcp_tls::run(listener, tls_config, token, accepted_tls).await; - }); - shard.bus.track_background(tls_handle); - bound.tcp_tls = Some(bound_addr); - } - - Ok(bound) -} - -/// Build the replica auth context from cluster config. Returns `None` when the -/// cluster or replica auth is disabled, keeping the handshake in legacy mode. -/// Only the derived MAC keys are carried onward in [`ReplicaAuth`]; the raw -/// secrets (masked in config logs via `config_env(secret)`) are read here only -/// to derive them. A non-empty `previous_shared_secret` opens the verify-only -/// rotation acceptance window (see the [`ReplicaAuth`] rustdoc for the rolling -/// rotation procedure). `ClusterConfig::validate` guarantees a non-empty -/// secret whenever both `cluster.enabled` and `cluster.auth.enabled` are set -/// (validate early-returns `Ok` while `cluster.enabled` is false). -fn load_replica_auth(config: &ServerNgConfig) -> Option { - if !config.cluster.enabled || !config.cluster.auth.enabled { - return None; - } - let auth = ReplicaAuth::new(config.cluster.auth.shared_secret.as_bytes()); - let previous_shared_secret = &config.cluster.auth.previous_shared_secret; - if previous_shared_secret.is_empty() { - return Some(auth); - } - Some(auth.with_previous_secret(previous_shared_secret.as_bytes())) -} - -/// Build the replica TLS context from cluster config. Returns `None` when -/// the cluster or replica TLS is disabled. Every shard calls this once at -/// boot: CA mode re-reads the same PEM files per shard; self-signed mode -/// mints a per-shard throwaway certificate. Neither mode carries client -/// certificates, so TLS authenticates the acceptor only; peer -/// authentication comes from the PSK handshake (`ClusterConfig::validate` -/// enforces `cluster.auth.enabled` whenever `cluster.tls.enabled`). -/// -/// Both rustls configs are TLS 1.3 only with the [`REPLICA_ALPN`] -/// protocol pinned. The dialer's SNI / certificate-verify name for each -/// peer is the roster entry's `ip` field (a hostname or IP literal, the -/// same string the connector dials). -fn load_replica_tls_ctx( - config: &ServerNgConfig, - topology: &TcpTopology, -) -> Result, ServerNgError> { - let tls = &config.cluster.tls; - if !config.cluster.enabled || !tls.enabled { - return Ok(None); - } - install_default_crypto_provider(); - let credential_error = |source: std::io::Error| ServerNgError::ListenerCredentials { - transport: "cluster.tls", - source, - }; - - let credentials = if tls.self_signed { - let san = config - .cluster - .nodes - .iter() - .find(|node| node.replica_id == topology.self_replica_id) - .map(|node| node.ip.as_str()) - .ok_or_else(|| { - credential_error(std::io::Error::other(format!( - "replica id {} not present in cluster.nodes", - topology.self_replica_id - ))) - })?; - let (cert_chain, key_der) = server_common::generate_self_signed_certificate(san) - .map_err(|error| credential_error(std::io::Error::other(error.to_string())))?; - TlsServerCredentials { - cert_chain, - key_der, - } - } else { - load_pem(Path::new(&tls.cert_file), Path::new(&tls.key_file)).map_err(credential_error)? - }; - - let mut server = - rustls::ServerConfig::builder_with_protocol_versions(&[&rustls::version::TLS13]) - .with_no_client_auth() - .with_single_cert(credentials.cert_chain, credentials.key_der) - .map_err(|error| { - credential_error(std::io::Error::other(format!( - "replica TLS server config rejected credentials: {error}" - ))) - })?; - server.alpn_protocols = vec![REPLICA_ALPN.to_vec()]; - - let client_builder = - rustls::ClientConfig::builder_with_protocol_versions(&[&rustls::version::TLS13]); - let mut client = if tls.self_signed { - client_builder - .dangerous() - .with_custom_certificate_verifier(Arc::new(AcceptAnyServerCert)) - .with_no_client_auth() - } else { - let roots = load_ca_pem(Path::new(&tls.ca_file)).map_err(credential_error)?; - client_builder - .with_root_certificates(Arc::new(roots)) - .with_no_client_auth() - }; - client.alpn_protocols = vec![REPLICA_ALPN.to_vec()]; - - // Keyed by replica id, never by roster position: sparse ids (dynamic - // replica join) would make a positional lookup verify against another - // peer's SNI name. - let peer_names = config - .cluster - .nodes - .iter() - .map(|node| { - let name = ServerName::try_from(node.ip.clone()).map_err(|error| { - credential_error(std::io::Error::new( - std::io::ErrorKind::InvalidInput, - format!( - "cluster node '{}' ip '{}' is not a valid TLS server name: {error}", - node.name, node.ip - ), - )) - })?; - Ok((node.replica_id, name)) - }) - .collect::, ServerNgError>>()?; - - Ok(Some(ReplicaTlsCtx { - server: Arc::new(server), - client: Arc::new(client), - peer_names, - })) -} - -fn load_tcp_tls_server_credentials( - config: &ServerNgConfig, -) -> Result { - let tls = &config.tcp.tls; - if tls.self_signed && !Path::new(&tls.cert_file).exists() { - return Ok(self_signed_for_loopback()); - } - - load_pem(Path::new(&tls.cert_file), Path::new(&tls.key_file)).map_err(|source| { - ServerNgError::ListenerCredentials { - transport: "tcp.tls", - source, - } - }) -} - -/// Bind the websocket client listener on `ws_addr`: WSS when -/// `websocket.tls.enabled` (the plain-WS accept loop must not also bind the -/// port -- a plain upgrade parser fed a TLS `ClientHello` rejects every -/// connection with an httparse error), plain WS otherwise. -async fn start_websocket_listener( - shard: &Rc, - config: &ServerNgConfig, - ws_addr: SocketAddr, - accepted_clients: &LocalClientAcceptFns, -) -> Result { - if config.websocket.tls.enabled { - let credentials = load_wss_server_credentials(config)?; - let (listener, tls_config, bound_addr) = client_listener::wss::bind(ws_addr, credentials) - .map_err(|source| { - error!(addr = %ws_addr, error = %source, "failed to bind WSS listener"); - source - })?; - let token = shard.bus.token(); - let accepted_wss = accepted_clients.wss.clone(); - let wss_handle = compio::runtime::spawn(async move { - client_listener::wss::run(listener, tls_config, token, accepted_wss).await; - }); - shard.bus.track_background(wss_handle); - Ok(bound_addr) - } else { - let (listener, bound_addr) = - client_listener::ws::bind(ws_addr).await.map_err(|source| { - error!(addr = %ws_addr, error = %source, "failed to bind websocket listener"); - source - })?; - let token = shard.bus.token(); - let accepted_ws = accepted_clients.ws.clone(); - let ws_handle = compio::runtime::spawn(async move { - client_listener::ws::run(listener, token, accepted_ws).await; - }); - shard.bus.track_background(ws_handle); - Ok(bound_addr) - } -} - -fn load_wss_server_credentials( - config: &ServerNgConfig, -) -> Result { - let tls = &config.websocket.tls; - if tls.self_signed && !Path::new(&tls.cert_file).exists() { - return Ok(self_signed_for_loopback()); - } - - load_pem(Path::new(&tls.cert_file), Path::new(&tls.key_file)).map_err(|source| { - ServerNgError::ListenerCredentials { - transport: "websocket.tls", - source, - } - }) -} - -fn load_quic_server_credentials( - config: &ServerNgConfig, -) -> Result { - let certificate = &config.quic.certificate; - if certificate.self_signed { - let (cert_chain, key_der) = server_common::generate_self_signed_certificate("localhost") - .map_err(|error| ServerNgError::ListenerCredentials { - transport: "quic", - source: std::io::Error::other(error.to_string()), - })?; - return Ok(replica_io::QuicServerCredentials { - cert_chain, - key_der, - }); - } - - let credentials = load_pem( - Path::new(&certificate.cert_file), - Path::new(&certificate.key_file), - ) - .map_err(|source| ServerNgError::ListenerCredentials { - transport: "quic", - source, - })?; - Ok(replica_io::QuicServerCredentials { - cert_chain: credentials.cert_chain, - key_der: credentials.key_der, - }) -} - -fn parse_socket_addr(context: &'static str, address: &str) -> Result { - address - .parse() - .map_err(|source| ServerNgError::SocketAddressParse { - context, - address: address.to_string(), - source, - }) -} - -fn socket_addr_from_parts( - context: &'static str, - host: &str, - port: u16, -) -> Result { - let ip = host - .parse::() - .map_err(|source| ServerNgError::SocketAddressParse { - context, - address: format!("{host}:{port}"), - source, - })?; - Ok(SocketAddr::new(ip, port)) -} - -/// Build the closure that broadcasts a -/// [`LifecycleFrame::MetadataCommitTick`] to every shard's inbox after a -/// partition-shaped metadata operation commits on shard 0. -/// -/// The receiver-side partition reconciliation loop listens for these -/// wake-ups; coalescing is intentional, so `Full` is recorded as a metric -/// and dropped (the periodic tick recovers). Installed via -/// [`metadata::IggyMetadata::set_commit_notifier`] on shard 0 only, the -/// sole writer of the metadata state machine. -fn make_metadata_commit_notifier( - senders: Vec, - metrics: ShardMetrics, -) -> metadata::CommitNotifier { - Rc::new(move |operation: Operation| { - if !operation_triggers_partition_reconcile(operation) { - return; - } - for sender in &senders { - let frame = ShardFrame::lifecycle(LifecycleFrame::MetadataCommitTick); - match sender.try_send(frame) { - Ok(()) => {} - Err(crossfire::TrySendError::Full(_)) => { - metrics.record_frame_drop( - frame_drop_variant::METADATA_COMMIT_TICK, - frame_drop_reason::FULL, - ); - } - Err(crossfire::TrySendError::Disconnected(_)) => { - metrics.record_frame_drop( - frame_drop_variant::METADATA_COMMIT_TICK, - frame_drop_reason::DISCONNECTED, - ); - } - } - } - }) -} - -/// Filter at the broadcast site, keeping unrelated ops off the SDK reply -/// path. Any new partition-shape op must be added here. -/// -/// The bare `CreateTopic` / `CreatePartitions` arms are unreachable: the -/// leader's prepare-builder in `IggyMetadata` rewrites both into their -/// `*WithAssignments` form, stamping each partition's `consensus_group_id` -/// before journaling, so a committed prepare only ever carries the -/// assignment-bearing variant. Kept as defense-in-depth against a future -/// commit path that emits a bare op. -/// -/// "Partition-shape" is not only the partition SET: the purge and truncate -/// ops leave the set intact but advance per-partition state (purge -/// generation, delete watermark) that only the reconciler enforces on disk. -/// Omitting them defers the on-disk effect to the periodic safety tick, -/// stretching a purge's client-visible tail to a full -/// `reconcile_periodic_interval`. `DeleteSegments` is absent by design: the -/// leader rewrites it into `TruncatePartition` before journaling, so no -/// commit ever carries it. -const fn operation_triggers_partition_reconcile(op: Operation) -> bool { - matches!( - op, - Operation::CreateTopic - | Operation::CreateTopicWithAssignments - | Operation::CreatePartitions - | Operation::CreatePartitionsWithAssignments - | Operation::DeleteTopic - | Operation::DeleteStream - | Operation::DeletePartitions - | Operation::PurgeStream - | Operation::PurgeTopic - | Operation::TruncatePartition - ) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn default_cluster_heartbeat_timeout_matches_consensus_constant() { - // The config default lives in core/server-ng/config.toml (a string, - // so no static assert can pin it); keep it in lockstep with the - // built-in the simulator and un-configured replicas run on. - let config_default = configs::ng_cluster::ClusterConfig::default() - .heartbeat_timeout - .get_duration() - .as_millis(); - let built_in = u128::from(consensus::TimeoutManager::NORMAL_HEARTBEAT_TICKS) - * shard::CONSENSUS_TICK_INTERVAL.as_millis(); - assert_eq!( - config_default, built_in, - "[cluster] heartbeat_timeout default drifted from \ - TimeoutManager::NORMAL_HEARTBEAT_TICKS" - ); - } - - #[test] - fn reconciler_driven_ops_broadcast_a_commit_tick() { - // These commit without touching the partition set, so nothing else - // signals the reconciler: `reconcile_partition_purges` and - // `reconcile_segment_truncations` are the only code that turns them - // into on-disk effect, and they run only when a pass runs. Dropping - // one from the filter silently downgrades it to the periodic tick. - for op in [ - Operation::PurgeStream, - Operation::PurgeTopic, - Operation::TruncatePartition, - ] { - assert!( - operation_triggers_partition_reconcile(op), - "{op:?} is enforced by the reconciler and must wake it on commit" - ); - } - assert!( - !operation_triggers_partition_reconcile(Operation::CreateUser), - "ops with no partition-shape effect must stay off the broadcast" - ); - } - - #[test] - fn recovery_barrier_deadline_holds_the_floor_for_small_heartbeats() { - // Below the 5s default the heartbeat-independent recovery term (~7s of - // ViewChangeStatus backstop plus ceremony) dominates, so the floor - // governs however small the heartbeat is; 3 x 5s lands exactly on it. - // A default-sized status backstop stays on the floor, not above it. - assert_eq!( - recovery_barrier_deadline(Duration::from_secs(1), Duration::from_secs(5)), - RECOVERY_BARRIER_DEADLINE_FLOOR - ); - assert_eq!( - recovery_barrier_deadline(Duration::from_secs(5), Duration::from_secs(5)), - RECOVERY_BARRIER_DEADLINE_FLOOR - ); - } - - #[test] - fn recovery_barrier_deadline_scales_past_the_floor_for_large_heartbeats() { - // Once 3 x heartbeat clears the floor the scaled window governs, so a - // slow-heartbeat cluster is not failed 503 before its longer recovery - // can finish. A default-sized status backstop stays under it. - assert_eq!( - recovery_barrier_deadline(Duration::from_secs(10), Duration::from_secs(5)), - Duration::from_secs(30) - ); - assert_eq!( - recovery_barrier_deadline(Duration::from_secs(15), Duration::from_secs(5)), - Duration::from_secs(45) - ); - } - - #[test] - fn recovery_barrier_deadline_scales_with_the_status_backstop() { - // A raised view-change status backstop stretches worst-case recovery - // even when the heartbeat stays fast, so the deadline must track it or - // post-restart reads 503 before a slow election settles. - assert_eq!( - recovery_barrier_deadline(Duration::from_secs(1), Duration::from_secs(10)), - Duration::from_secs(30) - ); - } - - #[test] - fn recovery_barrier_deadline_at_config_defaults_matches_the_floor() { - // Folding the status term in must not move the stock deadline: at the - // shared 5s defaults each scaled term lands exactly on the 15s floor, - // so an un-tuned cluster keeps its pre-existing recovery window. - let cluster = configs::ng_cluster::ClusterConfig::default(); - assert_eq!( - recovery_barrier_deadline( - cluster.heartbeat_timeout.get_duration(), - cluster.view_change_status_timeout.get_duration(), - ), - RECOVERY_BARRIER_DEADLINE_FLOOR - ); - } - - #[test] - fn recovery_barrier_deadline_saturates_instead_of_panicking() { - // Neither timeout has a config ceiling, so both multiplies must - // saturate rather than abort boot on an absurd parseable value. - assert_eq!( - recovery_barrier_deadline(Duration::MAX, Duration::from_secs(5)), - Duration::MAX - ); - assert_eq!( - recovery_barrier_deadline(Duration::from_secs(5), Duration::MAX), - Duration::MAX - ); - } - - #[test] - fn default_commit_broadcast_interval_matches_consensus_constant() { - // The config default lives in core/server-ng/config.toml (a string, - // so no static assert can pin it); keep it in lockstep with the - // built-in the simulator and un-configured replicas run on. - let config_default = configs::ng_cluster::ClusterConfig::default() - .commit_broadcast_interval - .get_duration() - .as_millis(); - let built_in = u128::from(consensus::TimeoutManager::COMMIT_MESSAGE_TICKS) - * shard::CONSENSUS_TICK_INTERVAL.as_millis(); - assert_eq!( - config_default, built_in, - "[cluster] commit_broadcast_interval default drifted from \ - TimeoutManager::COMMIT_MESSAGE_TICKS" - ); - } - - #[test] - fn default_prepare_retransmit_interval_matches_consensus_constant() { - // The config default lives in core/server-ng/config.toml (a string, - // so no static assert can pin it); keep it in lockstep with the - // built-in the simulator and un-configured replicas run on. - let config_default = configs::ng_cluster::ClusterConfig::default() - .prepare_retransmit_interval - .get_duration() - .as_millis(); - let built_in = u128::from(consensus::TimeoutManager::PREPARE_TICKS) - * shard::CONSENSUS_TICK_INTERVAL.as_millis(); - assert_eq!( - config_default, built_in, - "[cluster] prepare_retransmit_interval default drifted from \ - TimeoutManager::PREPARE_TICKS" - ); - } - - #[test] - fn default_partition_prepare_queue_depth_matches_consensus_constant() { - // The config default lives in core/server-ng/config.toml and flows - // through PartitionConfig::default(); keep the embedded value in - // lockstep with the pipeline depth LocalPipeline::new() (the simulator - // and tests) runs on, so a default deployment is byte-identical. - let config_default = configs::ng_partition::PartitionConfig::default().prepare_queue_depth; - assert_eq!( - config_default, - consensus::PIPELINE_PREPARE_QUEUE_MAX, - "[partition] prepare_queue_depth default drifted from \ - consensus::PIPELINE_PREPARE_QUEUE_MAX" - ); - } - - #[test] - fn default_view_change_retransmit_interval_matches_consensus_constant() { - // The config default lives in core/server-ng/config.toml (a string, so - // no static assert can pin it). One knob drives both view-change - // retransmit timers, which are equal by design, so pin it against both. - let config_default = configs::ng_cluster::ClusterConfig::default() - .view_change_retransmit_interval - .get_duration() - .as_millis(); - let start_view_change = - u128::from(consensus::TimeoutManager::START_VIEW_CHANGE_MESSAGE_TICKS) - * shard::CONSENSUS_TICK_INTERVAL.as_millis(); - let do_view_change = u128::from(consensus::TimeoutManager::DO_VIEW_CHANGE_MESSAGE_TICKS) - * shard::CONSENSUS_TICK_INTERVAL.as_millis(); - assert_eq!( - config_default, start_view_change, - "[cluster] view_change_retransmit_interval default drifted from \ - TimeoutManager::START_VIEW_CHANGE_MESSAGE_TICKS" - ); - assert_eq!( - config_default, do_view_change, - "[cluster] view_change_retransmit_interval default drifted from \ - TimeoutManager::DO_VIEW_CHANGE_MESSAGE_TICKS" - ); - } - - #[test] - fn default_view_change_status_timeout_matches_consensus_constant() { - // The config default lives in core/server-ng/config.toml (a string, so - // no static assert can pin it); keep it in lockstep with the built-in - // the simulator and un-configured replicas run on. - let config_default = configs::ng_cluster::ClusterConfig::default() - .view_change_status_timeout - .get_duration() - .as_millis(); - let built_in = u128::from(consensus::TimeoutManager::VIEW_CHANGE_STATUS_TICKS) - * shard::CONSENSUS_TICK_INTERVAL.as_millis(); - assert_eq!( - config_default, built_in, - "[cluster] view_change_status_timeout default drifted from \ - TimeoutManager::VIEW_CHANGE_STATUS_TICKS" - ); - } - - #[test] - fn default_request_start_view_retransmit_interval_matches_consensus_constant() { - // The config default lives in core/server-ng/config.toml (a string, so - // no static assert can pin it); keep it in lockstep with the built-in - // the simulator and un-configured replicas run on. - let config_default = configs::ng_cluster::ClusterConfig::default() - .request_start_view_retransmit_interval - .get_duration() - .as_millis(); - let built_in = u128::from(consensus::TimeoutManager::REQUEST_START_VIEW_MESSAGE_TICKS) - * shard::CONSENSUS_TICK_INTERVAL.as_millis(); - assert_eq!( - config_default, built_in, - "[cluster] request_start_view_retransmit_interval default drifted from \ - TimeoutManager::REQUEST_START_VIEW_MESSAGE_TICKS" - ); - } - - #[test] - fn default_view_probe_attempts_max_matches_consensus_constant() { - // Belt and suspenders with the static assert above: that pins the - // duplicated configs-crate literal, this pins the shipped config.toml - // value the simulator and un-configured replicas run on. - let config_default = configs::ng_cluster::ClusterConfig::default().view_probe_attempts_max; - assert_eq!( - config_default, - consensus::PROBE_ATTEMPTS_MAX, - "[cluster] view_probe_attempts_max default drifted from \ - consensus::PROBE_ATTEMPTS_MAX" - ); - } - - #[test] - fn default_repair_retry_interval_matches_partitions_constant() { - // The config default lives in core/server-ng/config.toml (a string, so - // no static assert can pin it); keep it in lockstep with the built-in - // the simulator and un-configured replicas run on. - let config_default = configs::ng_cluster::ClusterConfig::default() - .repair_retry_interval - .get_duration() - .as_millis(); - let built_in = - u128::from(partitions::REPAIR_RETRY_TICKS) * shard::CONSENSUS_TICK_INTERVAL.as_millis(); - assert_eq!( - config_default, built_in, - "[cluster] repair_retry_interval default drifted from \ - partitions::REPAIR_RETRY_TICKS" - ); - } - - #[test] - fn default_repair_chunk_max_matches_shard_constant() { - // Belt and suspenders with the static assert above: that pins the - // duplicated configs-crate literal, this pins the shipped config.toml - // value the simulator and un-configured replicas run on. - let config_default = configs::ng_cluster::ClusterConfig::default().repair_chunk_max; - assert_eq!( - config_default as u64, - shard::REPAIR_CHUNK_MAX, - "[cluster] repair_chunk_max default drifted from shard::REPAIR_CHUNK_MAX" - ); - } - - #[test] - fn default_evicted_ring_capacity_matches_partitions_constant() { - // Belt and suspenders with the static assert above; this pins the - // shipped config.toml value. - let config_default = - configs::ng_partition::PartitionConfig::default().evicted_ring_capacity; - assert_eq!( - config_default, - partitions::EVICTED_RING_CAPACITY, - "[partition] evicted_ring_capacity default drifted from \ - partitions::EVICTED_RING_CAPACITY" - ); - } - - #[test] - fn default_evicted_ring_bytes_max_matches_partitions_constant() { - // Belt and suspenders with the static assert above; this pins the - // shipped config.toml value. - let config_default = configs::ng_partition::PartitionConfig::default() - .evicted_ring_bytes_max - .as_bytes_u64(); - assert_eq!( - config_default, - partitions::EVICTED_RING_BYTES_MAX, - "[partition] evicted_ring_bytes_max default drifted from \ - partitions::EVICTED_RING_BYTES_MAX" - ); - } - - #[test] - fn shutdown_on_drop_armed_flips_flag() { - let flag = Arc::new(AtomicBool::new(false)); - drop(ShutdownOnDrop::new(Arc::clone(&flag))); - assert!( - flag.load(Ordering::Relaxed), - "an armed guard must flip the flag on drop (covers the error `?` \ - and panic-unwind exit paths of run_shard_thread)" - ); - } - - #[test] - fn shutdown_on_drop_disarmed_leaves_flag() { - let flag = Arc::new(AtomicBool::new(false)); - let mut guard = ShutdownOnDrop::new(Arc::clone(&flag)); - guard.disarm(); - drop(guard); - assert!( - !flag.load(Ordering::Relaxed), - "a disarmed guard must not flip the flag (clean `Ok(())` exit)" - ); - } - - const TEST_POLL_INTERVAL: Duration = Duration::from_millis(50); - - #[compio::test] - async fn broadcast_metadata_bundle_returns_immediately_with_no_peers() { - // Single-shard deployment: shard 0 has no peers to fan out to, - // so the handoff must complete without ever calling `send`. - let (bundle_tx, _bundle_rx) = crossfire::mpmc::bounded_async::(0); - let flag = Arc::new(AtomicBool::new(false)); - let mux = ServerNgMuxStateMachine::default(); - broadcast_metadata_bundle( - 0, - &bundle_tx, - mux.factory_bundle(), - 0, - &flag, - TEST_POLL_INTERVAL, - ) - .await - .expect("zero peers must not block shard 0"); - } - - #[compio::test] - async fn metadata_bundle_round_trips_through_channel() { - // End-to-end: shard 0 mints a bundle, a peer receives it on - // another runtime, and `from_factory_bundle` constructs a - // reader-mode mux that observes shard 0's writes via the same - // LeftRight pair. - let peers = 1u16; - let (bundle_tx, bundle_rx) = - crossfire::mpmc::bounded_async::(usize::from(peers)); - let flag = Arc::new(AtomicBool::new(false)); - - let owner = ServerNgMuxStateMachine::default(); - let bundle = owner.factory_bundle(); - broadcast_metadata_bundle(0, &bundle_tx, bundle, peers, &flag, TEST_POLL_INTERVAL) - .await - .expect("broadcast must succeed with one peer drained"); - - let received = await_metadata_bundle(1, &bundle_rx, &flag, TEST_POLL_INTERVAL) - .await - .expect("peer must receive the broadcast bundle"); - let _peer_mux = ServerNgMuxStateMachine::from_factory_bundle(received); - } - - #[compio::test] - async fn broadcast_metadata_bundle_aborts_when_peers_drop_rx() { - // Shard 0 drives handoff but every peer's `bundle_rx` was dropped - // before recv. Silently returning Ok would commit listener binds - // and consensus init for a cluster whose peers are gone; the - // broadcast must surface the disconnect so `shard_main` aborts. - let (bundle_tx, bundle_rx) = crossfire::mpmc::bounded_async::(0); - drop(bundle_rx); - let flag = Arc::new(AtomicBool::new(false)); - let mux = ServerNgMuxStateMachine::default(); - - let err = broadcast_metadata_bundle( - 0, - &bundle_tx, - mux.factory_bundle(), - 3, - &flag, - TEST_POLL_INTERVAL, - ) - .await - .expect_err("dropped rx must surface as MetadataHandoffAborted"); - assert!( - matches!(err, ServerNgError::MetadataHandoffAborted { shard_id: 0 }), - "expected MetadataHandoffAborted, got {err:?}" - ); - } - - #[compio::test] - async fn await_metadata_bundle_aborts_when_owner_drops_without_sending() { - let (bundle_tx, bundle_rx) = crossfire::mpmc::bounded_async::(1); - let flag = Arc::new(AtomicBool::new(false)); - - // Shard 0 dies before broadcasting; the peer must observe the - // disconnect and abort instead of hanging forever. - drop(bundle_tx); - - let err = await_metadata_bundle(1, &bundle_rx, &flag, TEST_POLL_INTERVAL) - .await - .expect_err("a peer whose owner never sends must abort"); - assert!( - matches!(err, ServerNgError::MetadataHandoffAborted { shard_id: 1 }), - "expected MetadataHandoffAborted, got {err:?}" - ); - } - - #[compio::test] - async fn await_metadata_bundle_aborts_on_shutdown_flag() { - // compio 0.19 `JoinHandle` yields `Result`; the - // `ResumeUnwind` impl re-raises a task panic and maps cancellation - // to `None`. - use compio::runtime::ResumeUnwind; - - let (_bundle_tx, bundle_rx) = crossfire::mpmc::bounded_async::(1); - let flag = Arc::new(AtomicBool::new(false)); - - let waiter = compio::runtime::spawn({ - let flag = Arc::clone(&flag); - async move { await_metadata_bundle(1, &bundle_rx, &flag, TEST_POLL_INTERVAL).await } - }); - - // Owner has not sent yet, but shutdown was requested; the peer - // must exit via the flag poll instead of hanging. - compio::time::sleep(TEST_POLL_INTERVAL / 2).await; - flag.store(true, Ordering::Relaxed); - - let err = waiter - .await - .resume_unwind() - .expect("waiter task was cancelled") - .expect_err("shutdown flag must abort the bundle wait"); - assert!( - matches!(err, ServerNgError::MetadataHandoffAborted { shard_id: 1 }), - "expected MetadataHandoffAborted on shutdown, got {err:?}" - ); - } - - #[compio::test] - async fn await_bootstrap_complete_returns_immediately_for_single_shard() { - // A single-shard server has no peers to wait on; the owner barrier - // must not block when `peers == 0`. - let (_ready_tx, ready_rx) = crossfire::mpmc::bounded_async::(1); - let flag = Arc::new(AtomicBool::new(false)); - await_bootstrap_complete(&ready_rx, 0, &flag, TEST_POLL_INTERVAL) - .await - .expect("single-shard server must not block on the barrier"); - } - - #[compio::test] - async fn await_bootstrap_complete_drains_every_peer_signal() { - // Two peers report load-complete; shard 0 drains both, then proceeds - // to bind listeners. - let (ready_tx, ready_rx) = crossfire::mpmc::bounded_async::(2); - let flag = Arc::new(AtomicBool::new(false)); - signal_bootstrap_complete(1, &ready_tx, &flag, TEST_POLL_INTERVAL) - .await - .expect("peer 1 must signal load-complete"); - signal_bootstrap_complete(2, &ready_tx, &flag, TEST_POLL_INTERVAL) - .await - .expect("peer 2 must signal load-complete"); - await_bootstrap_complete(&ready_rx, 2, &flag, TEST_POLL_INTERVAL) - .await - .expect("owner must drain both peer signals"); - } - - #[compio::test] - async fn await_bootstrap_complete_aborts_on_shutdown_flag() { - use compio::runtime::ResumeUnwind; - - // `_ready_tx` is held so the channel is not disconnected: the owner - // must exit via the shutdown flag, not a dropped sender. - let (_ready_tx, ready_rx) = crossfire::mpmc::bounded_async::(1); - let flag = Arc::new(AtomicBool::new(false)); - - let owner = compio::runtime::spawn({ - let flag = Arc::clone(&flag); - async move { await_bootstrap_complete(&ready_rx, 1, &flag, TEST_POLL_INTERVAL).await } - }); - - // The peer never signals, but a sibling failure flips the flag; the - // owner must abort instead of hanging before listeners. - compio::time::sleep(TEST_POLL_INTERVAL / 2).await; - flag.store(true, Ordering::Relaxed); - - let err = owner - .await - .resume_unwind() - .expect("owner task was cancelled") - .expect_err("shutdown flag must abort the barrier wait"); - assert!( - matches!( - err, - ServerNgError::ShardBootstrapBarrierAborted { remaining: 1 } - ), - "expected ShardBootstrapBarrierAborted, got {err:?}" - ); - } - - #[compio::test] - async fn signal_bootstrap_complete_aborts_when_owner_drops_rx() { - // Shard 0 aborted before draining and dropped its receiver; a peer's - // signal must surface the disconnect instead of stranding. - let (ready_tx, ready_rx) = crossfire::mpmc::bounded_async::(1); - let flag = Arc::new(AtomicBool::new(false)); - drop(ready_rx); - - let err = signal_bootstrap_complete(2, &ready_tx, &flag, TEST_POLL_INTERVAL) - .await - .expect_err("dropped rx must surface as an abort"); - assert!( - matches!(err, ServerNgError::MetadataHandoffAborted { shard_id: 2 }), - "expected MetadataHandoffAborted, got {err:?}" - ); - } - - fn cluster_node(ip: &str, http: Option) -> configs::ng_cluster::ClusterNodeConfig { - cluster_node_with_ports(ip, Some(18070), http) - } - - fn cluster_node_with_ports( - ip: &str, - tcp: Option, - http: Option, - ) -> configs::ng_cluster::ClusterNodeConfig { - configs::ng_cluster::ClusterNodeConfig { - name: "node".to_owned(), - ip: ip.to_owned(), - advertised_address: None, - advertised_addresses: Vec::new(), - replica_id: 0, - ports: configs::ng_cluster::TransportPorts { - tcp, - http, - ..Default::default() - }, - } - } - - fn addr(value: &str) -> SocketAddr { - value.parse().expect("valid socket address literal") - } - - #[test] - fn cluster_http_addr_takes_port_from_roster() { - // A byte-identical top-level [http].address is shared across nodes on - // one host; the per-node roster port is the only port source so each - // node binds a distinct HTTP socket. - let node = cluster_node("127.0.0.1", Some(18090)); - let addrs = resolve_cluster_client_addrs( - &node, - addr("127.0.0.1:8090"), - None, - None, - Some(addr("127.0.0.1:3000")), - ) - .expect("cluster address resolution must succeed"); - assert_eq!(addrs.http, Some(addr("127.0.0.1:18090"))); - } - - #[test] - fn cluster_http_addr_merges_config_ip_with_roster_port() { - // Docker/Helm bind `0.0.0.0` and probe loopback; the roster ip is - // only the advertised address. Cluster mode must keep the configured - // interface and take just the port from the roster. - let node = cluster_node("10.0.0.5", Some(18090)); - let addrs = resolve_cluster_client_addrs( - &node, - addr("0.0.0.0:8090"), - None, - None, - Some(addr("0.0.0.0:3000")), - ) - .expect("cluster address resolution must succeed"); - assert_eq!(addrs.http, Some(addr("0.0.0.0:18090"))); - } - - #[test] - fn cluster_http_addr_requires_roster_port_for_enabled_transport() { - // No fallback to the top-level port: a silent default could collide - // with another same-host node, so a missing roster port for an - // enabled transport must refuse to boot. - let node = cluster_node("10.0.0.5", None); - let result = resolve_cluster_client_addrs( - &node, - addr("127.0.0.1:8090"), - None, - None, - Some(addr("127.0.0.1:3000")), - ); - assert!(matches!( - result, - Err(ServerNgError::ClusterPortMissing { - transport: "http", - replica_id: 0, - }) - )); - } - - #[test] - fn cluster_http_addr_is_none_when_http_disabled() { - // http.enabled = false collapses default_http_addr to None; no roster - // port can revive a listener the operator turned off. - let node = cluster_node("127.0.0.1", Some(18090)); - let addrs = resolve_cluster_client_addrs(&node, addr("127.0.0.1:8090"), None, None, None) - .expect("cluster address resolution must succeed"); - assert_eq!(addrs.http, None); - } - - /// Regression: the shutdown-join deadline must arm at SHUTDOWN, not - /// at boot. The original bound measured from `join_all` entry, so any - /// healthy server outliving `shutdown_join_timeout` (30s default) was - /// abandoned as "wedged" and the process exited - every BDD run died - /// at t+30s while the test container was still compiling. - #[test] - fn join_waits_unbounded_while_the_server_runs() { - let shutdown_flag = AtomicBool::new(false); - // Thread outlives a deliberately tiny join budget; with the flag - // clear the budget must never even arm. - let handle = thread::spawn(|| -> Result<(), ServerNgError> { - thread::sleep(Duration::from_millis(300)); - Ok(()) - }); - let mut deadline = None; - let joined = join_until_shutdown_deadline( - handle, - &shutdown_flag, - Duration::from_millis(20), - &mut deadline, - ); - assert!( - matches!(joined, Some(Ok(Ok(())))), - "a running server must be awaited indefinitely, not abandoned as wedged" - ); - assert!( - deadline.is_none(), - "the join deadline must not arm before the shutdown flag flips" - ); - } - - #[test] - fn join_abandons_a_wedged_shard_after_the_shutdown_deadline() { - let shutdown_flag = AtomicBool::new(true); - // Never finishes: stands in for a wedged pump. The thread leaks - // into the test process, which exits right after. - let handle = thread::spawn(|| -> Result<(), ServerNgError> { - loop { - thread::sleep(Duration::from_secs(1)); - } - }); - let mut deadline = None; - let joined = join_until_shutdown_deadline( - handle, - &shutdown_flag, - Duration::from_millis(100), - &mut deadline, - ); - assert!( - joined.is_none(), - "a shard still running past the post-shutdown budget must be abandoned" - ); - assert!(deadline.is_some(), "the deadline arms once the flag is set"); - } - - #[test] - fn cluster_tcp_addr_takes_port_from_roster() { - // Same rule as the other transports: the roster owns the port so - // same-host nodes sharing one [tcp].address still bind distinct - // sockets. - let node = cluster_node("127.0.0.1", None); - let addrs = resolve_cluster_client_addrs(&node, addr("127.0.0.1:8090"), None, None, None) - .expect("cluster address resolution must succeed"); - assert_eq!(addrs.client, addr("127.0.0.1:18070")); - } - - #[test] - fn cluster_tcp_addr_merges_config_ip_with_roster_port() { - // The roster ip is advertised, not bound. Binding it directly would - // strand every co-located dialer (sidecars, health probes, on-host - // consumers) that reaches this node over loopback. - let node = cluster_node("10.0.0.5", None); - let addrs = resolve_cluster_client_addrs(&node, addr("0.0.0.0:8090"), None, None, None) - .expect("cluster address resolution must succeed"); - assert_eq!(addrs.client, addr("0.0.0.0:18070")); - } - - #[test] - fn cluster_tcp_addr_requires_roster_port() { - // tcp is always enabled in cluster mode, so a roster entry without a - // tcp port refuses to boot rather than falling back to [tcp].address. - let node = cluster_node_with_ports("10.0.0.5", None, None); - let result = resolve_cluster_client_addrs(&node, addr("127.0.0.1:8090"), None, None, None); - assert!(matches!( - result, - Err(ServerNgError::ClusterPortMissing { - transport: "tcp", - replica_id: 0, - }) - )); - } - - #[test] - fn cluster_tcp_addr_keeps_loopback_bind_and_warns_on_roster_mismatch() { - // A loopback [tcp].address under a routable roster ip is honoured - // as configured; remote peers cannot reach it, so the mismatch is - // warned about instead of silently rebinding. - let node = cluster_node("10.0.0.5", None); - let addrs = resolve_cluster_client_addrs(&node, addr("127.0.0.1:8090"), None, None, None) - .expect("cluster address resolution must succeed"); - assert_eq!(addrs.client, addr("127.0.0.1:18070")); - assert!(roster_ip_unreachable_from_bind_addr(&node.ip, addrs.client)); - } - - #[test] - fn roster_mismatch_warning_is_silent_for_wildcard_and_hostname_rosters() { - // A wildcard bind covers the roster interface, and a DNS roster entry - // can resolve to the bound one; neither is a misconfiguration. - assert!(!roster_ip_unreachable_from_bind_addr( - "10.0.0.5", - addr("0.0.0.0:18070") - )); - assert!(!roster_ip_unreachable_from_bind_addr( - "node-1.example.com", - addr("127.0.0.1:18070") - )); - assert!(!roster_ip_unreachable_from_bind_addr( - "10.0.0.5", - addr("10.0.0.5:18070") - )); - } -} diff --git a/core/server-ng/src/http/error.rs b/core/server-ng/src/http/error.rs deleted file mode 100644 index e5f9a18448..0000000000 --- a/core/server-ng/src/http/error.rs +++ /dev/null @@ -1,818 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! HTTP rejection types and hand-built error responses: the auth / write / -//! read / partition-write error enums, their `IntoResponse` renderings, the -//! `?consistency=` and `?ack=` query DTOs, and the primary-redirect helpers. - -use std::net::{IpAddr, SocketAddr}; - -use axum::Json; -use axum::http::header::{LOCATION, RETRY_AFTER}; -use axum::http::{HeaderValue, StatusCode}; -use axum::response::{IntoResponse, Response}; -use configs::ng_cluster::ResolvedClusterNode; -use iggy_binary_protocol::Operation; -use iggy_common::IggyError; -use serde::{Deserialize, Serialize}; -use thiserror::Error; -use tracing::error; - -use crate::cluster_meta::ClusterRoster; - -#[derive(Debug, Error)] -pub(in crate::http) enum CustomError { - #[error(transparent)] - Error(#[from] IggyError), - #[error("Resource not found")] - ResourceNotFound, -} - -#[derive(Debug, Serialize)] -pub(in crate::http) struct ErrorResponse { - /// Two conventions by construction: the `IggyError` numeric code - /// (`IggyError::as_code`) when the error wraps one (via [`Self::from_error`]), - /// or the HTTP status code for the hand-built HTTP-layer errors (429/503/504 - /// and the 404 not-found fallback) that carry no underlying `IggyError`. - pub id: u32, - pub code: String, - pub reason: String, - pub field: Option, -} - -impl IntoResponse for CustomError { - fn into_response(self) -> Response { - match self { - Self::Error(error) => { - error!("There was an error: {error}"); - let status_code = match error { - IggyError::StreamIdNotFound(_) - | IggyError::TopicIdNotFound(_, _) - | IggyError::PartitionNotFound(_, _, _) - | IggyError::SegmentNotFound - | IggyError::ClientNotFound(_) - | IggyError::ConsumerGroupIdNotFound(_, _) - | IggyError::ConsumerGroupNameNotFound(_, _) - | IggyError::ConsumerGroupMemberNotFound(_, _, _) - | IggyError::ConsumerOffsetNotFound(_) - | IggyError::ResourceNotFound(_) => StatusCode::NOT_FOUND, - IggyError::Unauthenticated - | IggyError::AccessTokenMissing - | IggyError::InvalidAccessToken - | IggyError::InvalidPersonalAccessToken => StatusCode::UNAUTHORIZED, - IggyError::Unauthorized => StatusCode::FORBIDDEN, - // The pre-consensus retry frame: reaching this render - // means the write path's replay budget is exhausted and - // the op never committed - a transient server condition, - // retryable like the other cannot-commit-right-now 503s - // (see `service_unavailable`), never a caller error. - IggyError::TransientNotCommitted | IggyError::TransientNotAccepted => { - StatusCode::SERVICE_UNAVAILABLE - } - _ => StatusCode::BAD_REQUEST, - }; - let response = - (status_code, Json(ErrorResponse::from_error(&error))).into_response(); - // Transient 503s are retryable, so the advisory Retry-After hint - // rides along, matching the other transient 503 bodies - // (`service_unavailable`, `server_busy`). - if matches!( - error, - IggyError::TransientNotCommitted | IggyError::TransientNotAccepted - ) { - with_retry_after(response) - } else { - response - } - } - Self::ResourceNotFound => ( - StatusCode::NOT_FOUND, - Json(ErrorResponse { - id: 404, - code: "not_found".to_string(), - reason: "Resource not found".to_string(), - field: None, - }), - ) - .into_response(), - } - } -} - -impl ErrorResponse { - pub fn from_error(error: &IggyError) -> Self { - Self { - id: error.as_code(), - code: error.as_string().to_string(), - reason: error.to_string(), - field: match error { - IggyError::StreamIdNotFound(_) | IggyError::InvalidStreamId => { - Some("stream_id".to_string()) - } - IggyError::TopicIdNotFound(_, _) | IggyError::InvalidTopicId => { - Some("topic_id".to_string()) - } - IggyError::PartitionNotFound(_, _, _) => Some("partition_id".to_string()), - IggyError::SegmentNotFound => Some("segment_id".to_string()), - IggyError::ClientNotFound(_) => Some("client_id".to_string()), - IggyError::InvalidStreamName - | IggyError::StreamNameAlreadyExists(_) - | IggyError::InvalidTopicName - | IggyError::TopicNameAlreadyExists(_, _) - | IggyError::ConsumerGroupNameAlreadyExists(_, _) - | IggyError::PersonalAccessTokenAlreadyExists(_, _) => Some("name".to_string()), - IggyError::InvalidOffset(_) => Some("offset".to_string()), - IggyError::InvalidConsumerGroupId => Some("consumer_group_id".to_string()), - IggyError::UserAlreadyExists => Some("username".to_string()), - _ => None, - }, - } - } -} - -/// Rejection for protected routes. -/// -/// Two failure classes get two statuses: a missing, invalid, or expired -/// credential is the caller's fault (401, rendered as the JSON `ErrorResponse` -/// body every other route error uses), while a VSR session that cannot be -/// established right now is a transient server condition (503) and must never -/// masquerade as an auth failure. -pub(in crate::http) enum AuthError { - Unauthenticated(IggyError), - /// The Register provably never entered the consensus pipeline (not - /// primary, not caught up, or the prepare queue was full), so the request - /// is safe to re-issue anywhere. Rendered with the `TransientNotAccepted` - /// body so a forwarding follower recognizes it as retryable against a - /// re-resolved primary; a plain client sees the same retryable 503 either - /// way. - SessionNotAccepted, - SessionUnavailable, - /// The `client_id` this gateway minted already has a committed session - /// owned by a DIFFERENT user, so the Register was refused terminally. - /// - /// Distinct from [`Self::SessionUnavailable`] because the status code is - /// the whole point: 503 is about the most auto-retried status there is and - /// no foreign SDK special-cases it, so rendering a permanent, deterministic - /// refusal as 503 hands the caller's HTTP stack a retry loop it can never - /// escape. 409 says the id is taken and stops it. - SessionIdOwnedByAnotherUser, - /// The minted `client_id` already had a committed session for this SAME - /// user, so the Register rebound onto it instead of creating one. Internal - /// to the mint retry in `register_session` and never rendered: the caller - /// mints a different id. Present as a variant so the retry cannot confuse - /// it with a terminal cross-user refusal. - SessionIdTaken, -} - -impl From for AuthError { - fn from(error: IggyError) -> Self { - Self::Unauthenticated(error) - } -} - -impl IntoResponse for AuthError { - fn into_response(self) -> Response { - match self { - // Render 401 through the shared `IggyError -> CustomError` map so it - // carries the same JSON `ErrorResponse` body as every other ng error. - // The legacy server's protected-route 401 comes from a bare-status - // JWT middleware (empty body), so this is deliberately richer, not - // byte-identical to legacy. - Self::Unauthenticated(error) => CustomError::from(error).into_response(), - Self::SessionNotAccepted => { - CustomError::from(IggyError::TransientNotAccepted).into_response() - } - // A fresh session could not be established: the Register was - // canceled with its commit outcome unknown, or the session table - // is at its cap (half `[metadata] clients_table_max`) and refused - // the fresh registration. Transient server condition -> 503, - // retryable by the CLIENT only (a forwarder must not re-issue an - // unknown-outcome Register under this node's session budget on - // the caller's behalf). - // `SessionIdTaken` only escapes the mint retry when every attempt - // collided, which means the minter is wrong rather than unlucky -- - // same unknown-outcome answer as a canceled Register. - Self::SessionUnavailable | Self::SessionIdTaken => service_unavailable(), - // Terminal: retrying cannot change the answer, and admitting it - // would run this caller's replicated ops under the entry owner's - // authority. - Self::SessionIdOwnedByAnotherUser => ( - StatusCode::CONFLICT, - Json(ErrorResponse::from_error(&IggyError::InvalidClientId)), - ) - .into_response(), - } - } -} - -/// Rejection for an authenticated control-plane write (`POST /streams` and the -/// writes that follow it). -/// -/// Same two-class split as [`AuthError`], for the same reasons: a caller-side -/// validation failure or a committed business rejection (e.g. a duplicate -/// stream name) renders through the legacy `IggyError -> CustomError` map so -/// SDK error bodies stay byte-identical, while a write that cannot commit right -/// now is a transient server condition (503) and must never surface as a -/// business error or, worse, a 200 with a stale body. -pub(in crate::http) enum WriteError { - Rejected(IggyError), - /// The VSR session was evicted (its client slot was reclaimed cluster-side, - /// e.g. LRU-evicted from the full client table). Renders identically to a - /// terminal `Rejected` (401 -> re-authenticate), but is a distinct variant - /// so the submit path can drop the dead session entry and let the caller's - /// next request re-register cleanly instead of 401-looping on it. - Evicted(IggyError), - Unavailable, -} - -impl IntoResponse for WriteError { - fn into_response(self) -> Response { - match self { - Self::Rejected(error) | Self::Evicted(error) => { - CustomError::from(error).into_response() - } - Self::Unavailable => service_unavailable(), - } - } -} - -/// Rejection for a data-plane partition write (`POST .../messages` produce and -/// the `PUT`/`DELETE .../consumer-offsets` writes). -/// -/// Split differently from [`WriteError`] because the partition plane replies -/// carry no committed error code: a pre-dispatch gate failure is an -/// empty-bodied reply that names itself only in the header (see -/// [`classify_partition_reply`]), and an unanswered write is a distinct -/// outcome the caller must treat as unknown rather than failed. -#[derive(Debug)] -pub(in crate::http) enum PartitionWriteError { - /// Caller-side rejection (bad identifier, oversized batch, an authorization - /// denial), a typed pre-commit deny from the partition plane - /// (`ReplyHeader.status`), or a malformed reply frame, rendered through the - /// legacy `IggyError -> status` map for SDK-identical bodies. - Rejected(IggyError), - /// Backstop for a status-0 reply carrying `op` 0: an ack with no commit - /// number behind it, for a write that never reached the partition plane. - /// Routing failures name themselves through `ReplyHeader.status`, so this - /// shape is left to a peer that still answers a non-committing op this - /// way. Rendered as the legacy 404 body: the alternative is grading a - /// write that never happened as a success. - NotFound, - /// The in-process reply slot could not be installed. Transient server - /// condition -> the shared 503, retryable. - Unavailable, - /// This session is already at [`MAX_IN_FLIGHT_WRITES_PER_SESSION`] - /// awaited writes. 429: the caller's own concurrency is the problem, so - /// it must drain its outstanding writes before submitting more. - TooManyInFlight, - /// Shard 0 is already at [`MAX_IN_FLIGHT_WRITES_GLOBAL`] awaited writes - /// across all sessions. 503 with its own code (distinct from the shared - /// consensus-unavailable body) so an operator can tell admission shedding - /// from a consensus outage. - ServerBusy, - /// No committed reply within [`PARTITION_WRITE_REPLY_TIMEOUT`], or the - /// session's reply target was torn down mid-wait. 504: the commit may - /// still land (at-least-once), so this is a hard "outcome unknown", not a - /// failure the server may transparently retry. Carries the write's - /// operation so the 504 body names which write kind timed out. - Timeout(Operation), -} - -impl IntoResponse for PartitionWriteError { - fn into_response(self) -> Response { - match self { - Self::Rejected(error) => CustomError::from(error).into_response(), - Self::NotFound => CustomError::ResourceNotFound.into_response(), - Self::Unavailable => service_unavailable(), - Self::TooManyInFlight => too_many_in_flight_response(), - Self::ServerBusy => server_busy_response(), - Self::Timeout(operation) => partition_write_timeout_response(operation), - } - } -} - -/// 504 body for a partition write whose commit outcome is unknown, coded per -/// write kind so a caller can tell a produce timeout from an offset-write -/// timeout. Shaped like every other HTTP error (`ErrorResponse`) so clients -/// parse one error schema. -fn partition_write_timeout_response(operation: Operation) -> Response { - let (code, reason) = match operation { - Operation::SendMessages => ( - "produce_timeout", - "produce was not acknowledged in time; the write may still commit", - ), - _ => ( - "offset_write_timeout", - "consumer-offset write was not acknowledged in time; the write may still commit", - ), - }; - gateway_timeout_response(code, reason) -} - -/// Advisory `Retry-After` seconds for the shed / transient 429 and 503 -/// responses. One second: admission shedding, a briefly unavailable consensus -/// group, and a linearizable-read-on-follower all typically clear well within -/// it, and a small hint keeps a backing-off client responsive. -const RETRY_AFTER_SECONDS: u64 = 1; - -/// Attach the advisory [`RETRY_AFTER_SECONDS`] hint to a retryable 429/503. -pub(in crate::http) fn with_retry_after(mut response: Response) -> Response { - response - .headers_mut() - .insert(RETRY_AFTER, HeaderValue::from(RETRY_AFTER_SECONDS)); - response -} - -/// Render an `ErrorResponse` body for `status`, tagged with `code` / `reason` -/// and no field, so every hand-built HTTP error the routes return parses as the -/// one error schema clients already handle. -pub(in crate::http) fn error_response(status: StatusCode, code: &str, reason: &str) -> Response { - ( - status, - Json(ErrorResponse { - id: status.as_u16().into(), - code: code.to_owned(), - reason: reason.to_owned(), - field: None, - }), - ) - .into_response() -} - -/// Shared 504 rendering for an in-band request the partition plane did not -/// answer in time, shaped like every other HTTP error (`ErrorResponse`) so -/// clients parse one error schema. Consumed by the partition-write reply wait, -/// the partition reads ([`ReadError::Timeout`]), and the forward attempt bound. -pub(in crate::http) fn gateway_timeout_response(code: &str, reason: &str) -> Response { - error_response(StatusCode::GATEWAY_TIMEOUT, code, reason) -} - -/// The shared 503 body for a request that could not commit right now: no -/// caught-up primary, a full pipeline, or a view-change cancel. Retryable, and -/// rendered with the `CannotEstablishConnection` code the SDKs treat as a -/// connection-level retry rather than a terminal error. -fn service_unavailable() -> Response { - with_retry_after( - ( - StatusCode::SERVICE_UNAVAILABLE, - Json(ErrorResponse::from_error( - &IggyError::CannotEstablishConnection, - )), - ) - .into_response(), - ) -} - -/// 429 for a session at [`MAX_IN_FLIGHT_WRITES_PER_SESSION`] awaited partition -/// writes. Shaped like every other HTTP error (`ErrorResponse`) so clients -/// parse one error schema; the remedy is the caller's own: let outstanding -/// writes finish, then retry. -fn too_many_in_flight_response() -> Response { - with_retry_after(error_response( - StatusCode::TOO_MANY_REQUESTS, - "too_many_in_flight_writes", - "session reached its in-flight write cap; await outstanding writes and retry", - )) -} - -/// 503 for shard 0 at [`MAX_IN_FLIGHT_WRITES_GLOBAL`] awaited partition writes -/// across all sessions. A distinct `server_busy` code (unlike the shared -/// consensus-unavailable 503) so admission shedding is tellable from a -/// consensus outage; retry with backoff. -fn server_busy_response() -> Response { - with_retry_after(error_response( - StatusCode::SERVICE_UNAVAILABLE, - "server_busy", - "shard is at its in-flight write budget; retry with backoff", - )) -} - -/// Read consistency selected by the `?consistency=` query param. -/// -/// `serializable` (the default) serves from this node's local metadata STM: -/// correct and consensus-free, but may trail the primary by the replication -/// delay. `linearizable` demands the freshest committed state and is honored -/// only on the primary; a follower redirects (307) to the primary when its HTTP -/// address resolves from the roster, else fails closed to 503 (see -/// [`read_local`]). -#[derive(Clone, Copy, Default, PartialEq, Eq, Deserialize)] -#[serde(rename_all = "lowercase")] -pub(in crate::http) enum Consistency { - #[default] - Serializable, - Linearizable, -} - -/// `?consistency=` query wrapper. An absent param defaults to -/// [`Consistency::Serializable`]; an unrecognized value is a 400 (axum `Query`). -#[derive(Default, Deserialize)] -pub(in crate::http) struct ConsistencyQuery { - #[serde(default)] - pub(in crate::http) consistency: Consistency, -} - -/// Produce acknowledgement selected by the `?ack=` query param. -/// -/// `replicated` (the default) answers 201 only after the partition group's -/// quorum commit. `none` is fire-and-forget: the request is validated, -/// dispatched, and answered 202 immediately; the commit still happens, but its -/// reply is shed at the bus (no reply slot is installed). -#[derive(Clone, Copy, Default, PartialEq, Eq, Deserialize)] -#[serde(rename_all = "lowercase")] -pub(in crate::http) enum ProduceAck { - #[default] - Replicated, - None, -} - -/// `?ack=` query wrapper. An absent param defaults to -/// [`ProduceAck::Replicated`]; an unrecognized value is a 400 (axum `Query`). -#[derive(Default, Deserialize)] -pub(in crate::http) struct ProduceQuery { - #[serde(default)] - pub(in crate::http) ack: ProduceAck, -} - -/// Rejection for an authenticated read route (`GET /streams`, -/// `GET /streams/{id}`, and the reads that follow). -pub(in crate::http) enum ReadError { - /// Caller-side or STM rejection (bad identifier, unsupported op, or an - /// authorization denial) graded through the legacy `IggyError -> status` - /// map so SDK error bodies stay byte-identical. - Rejected(IggyError), - /// Requested entity is absent -> 404 with the legacy not-found body. - NotFound, - /// A linearizable read reached a follower and the primary's HTTP address was - /// not resolvable from the roster. Fail-closed 503, retryable against the - /// leader (see [`not_primary_response`]). - NotPrimary, - /// A linearizable read reached a follower and the current VSR primary's HTTP - /// address resolved: 307 to that address carrying the original path and - /// query, so the caller re-issues the read against the leader (see - /// [`primary_redirect_response`]). - RedirectToPrimary(String), - /// The post-restart read-recovery barrier expired with the recovered WAL - /// suffix still uncommitted: serving now could show state that rolls back - /// history a client already saw acked. Fail-closed 503 via the shared - /// [`service_unavailable`] body, retryable once the cluster re-commits the - /// suffix. - RecoveryIncomplete, - /// A partition read (poll / consumer-offset) got no reply from the owning - /// shard within the mesh budget. 504 like a produce timeout: the outcome is - /// unknown (the abandoned read may still be running), so the caller retries. - Timeout, -} - -impl IntoResponse for ReadError { - fn into_response(self) -> Response { - match self { - Self::Rejected(error) => CustomError::from(error).into_response(), - // Reuse the legacy 404 body so a missing stream renders exactly as - // the legacy server's `CustomError::ResourceNotFound` does. - Self::NotFound => CustomError::ResourceNotFound.into_response(), - Self::NotPrimary => not_primary_response(), - Self::RedirectToPrimary(location) => primary_redirect_response(&location), - Self::RecoveryIncomplete => service_unavailable(), - Self::Timeout => gateway_timeout_response( - "partition_read_timeout", - "the partition owner did not answer the read in time; retry", - ), - } - } -} - -/// The 503 fail-closed body for a linearizable read that reached a follower -/// whose primary HTTP address could not be resolved (absent consensus, a roster -/// with no node at the primary index, or a port-less node). The resolvable case -/// is a 307 via [`primary_redirect_response`] instead. Rendered as an -/// `ErrorResponse` so the body shape matches every other HTTP error; the caller -/// retries against the leader. -fn not_primary_response() -> Response { - with_retry_after(error_response( - StatusCode::SERVICE_UNAVAILABLE, - "not_primary", - "linearizable read requires the primary; retry against the leader", - )) -} - -/// 307 Temporary Redirect to the current VSR primary for a linearizable read -/// that reached a follower. `Location` is the primary's HTTP base plus the -/// original path and query, so the caller re-issues the identical read against -/// the leader. Dormant on a single node (always primary) and followed by no SDK -/// yet. A `Location` that is not a valid header value falls back to the 503. -fn primary_redirect_response(location: &str) -> Response { - HeaderValue::from_str(location).map_or_else( - |_| not_primary_response(), - |value| { - let mut response = StatusCode::TEMPORARY_REDIRECT.into_response(); - response.headers_mut().insert(LOCATION, value); - response - }, - ) -} - -/// Build the `Location` for a 307 redirect of a linearizable read to the VSR -/// primary: `://:`. The scheme is the -/// redirecting node's own listener scheme (uniform cluster HTTP config, same -/// assumption the forward hop makes). `client_ip` is the redirected client's -/// peer address, so the `Location` host comes from the primary's -/// per-client-network selectors when one matches. `None` when the primary does -/// not resolve from the roster, so the caller fails closed to a 503 rather -/// than pointing at an unreachable target. Pure (no consensus or axum -/// dependency) so the redirect target is unit-tested in isolation. -pub(in crate::http) fn primary_redirect_location( - roster: &ClusterRoster, - primary_index: u8, - scheme: &str, - path_and_query: &str, - client_ip: Option, -) -> Option { - let authority = primary_advertised_http_authority(roster, primary_index, client_ip)?; - Some(format!("{scheme}://{authority}{path_and_query}")) -} - -/// Resolve the VSR primary's HTTP socket from the static roster: the node -/// whose `replica_id` equals `primary_index`, its `ports.http`, and its -/// private roster `ip` (parsed once at roster build). Internal replica -/// forwarding uses this address; it must never route through -/// [`ResolvedClusterNode::advertised_for`], which picks client-facing hosts. -pub(in crate::http) fn primary_http_socket( - roster: &ClusterRoster, - primary_index: u8, -) -> Option { - let (node, http_port) = primary_node(roster, primary_index)?; - Some(SocketAddr::new(node.replica_ip()?, http_port)) -} - -/// Resolve the client-facing HTTP authority (`host:port`) for a redirect -/// through [`ResolvedClusterNode::advertised_for`]: a client-network selector -/// match first, then the catch-all advertised address, then the private -/// roster IP as the compatibility fallback. `AdvertisedAddress::authority` -/// brackets IPv6 hosts and passes hostnames through, so the redirect URL -/// stays valid. This is the fail-closed caller: a host that is neither a -/// valid IP nor a valid hostname yields `None` and the redirect becomes a -/// 503 rather than a `Location` pointing at an unparsable target (cluster -/// metadata makes the opposite choice and publishes such a host verbatim). -fn primary_advertised_http_authority( - roster: &ClusterRoster, - primary_index: u8, - client_ip: Option, -) -> Option { - let (node, http_port) = primary_node(roster, primary_index)?; - let address = node.advertised_for(client_ip)?; - Some(address.authority(http_port)) -} - -fn primary_node(roster: &ClusterRoster, primary_index: u8) -> Option<(&ResolvedClusterNode, u16)> { - let node = roster - .nodes - .iter() - .find(|node| node.config().replica_id == primary_index)?; - let http_port = node.config().ports.http?; - Some((node, http_port)) -} - -#[cfg(test)] -mod tests { - use super::*; - - use configs::ng_cluster::{ClusterNodeConfig, TransportPorts}; - - const READ_PATH: &str = "/streams?consistency=linearizable"; - fn node(replica_id: u8, ip: &str, http: Option) -> ClusterNodeConfig { - ClusterNodeConfig { - name: format!("node-{replica_id}"), - ip: ip.to_owned(), - advertised_address: None, - advertised_addresses: Vec::new(), - replica_id, - ports: TransportPorts { - tcp: None, - quic: None, - http, - websocket: None, - tcp_replica: None, - }, - } - } - - fn roster(nodes: Vec) -> ClusterRoster { - ClusterRoster { - enabled: true, - name: "test-cluster".to_owned(), - nodes: nodes.into_iter().map(Into::into).collect(), - self_ip: "127.0.0.1".to_owned(), - self_ports: TransportPorts::default(), - metadata_view: std::sync::Arc::new(std::sync::atomic::AtomicU64::new( - crate::cluster_meta::METADATA_VIEW_UNKNOWN, - )), - } - } - - #[test] - fn primary_redirect_location_targets_primary_http_addr_with_path_passthrough() { - let roster = roster(vec![ - node(0, "10.0.0.1", Some(8080)), - node(1, "10.0.0.2", Some(8090)), - ]); - assert_eq!( - primary_redirect_location(&roster, 1, "http", READ_PATH, None), - Some("http://10.0.0.2:8090/streams?consistency=linearizable".to_owned()) - ); - } - - #[test] - fn primary_redirect_location_uses_the_listener_scheme() { - let roster = roster(vec![node(0, "10.0.0.1", Some(8080))]); - assert_eq!( - primary_redirect_location(&roster, 0, "https", READ_PATH, None), - Some("https://10.0.0.1:8080/streams?consistency=linearizable".to_owned()) - ); - } - - #[test] - fn primary_redirect_location_is_none_when_no_node_matches_primary_index() { - let roster = roster(vec![node(0, "10.0.0.1", Some(8080))]); - assert_eq!( - primary_redirect_location(&roster, 2, "http", READ_PATH, None), - None - ); - } - - #[test] - fn primary_redirect_location_is_none_when_primary_has_no_http_port() { - let roster = roster(vec![node(0, "10.0.0.1", None)]); - assert_eq!( - primary_redirect_location(&roster, 0, "http", READ_PATH, None), - None - ); - } - - #[test] - fn primary_redirect_location_is_none_for_empty_roster() { - let roster = roster(Vec::new()); - assert_eq!( - primary_redirect_location(&roster, 0, "http", READ_PATH, None), - None - ); - } - - #[test] - fn primary_redirect_location_brackets_ipv6_host() { - let roster = roster(vec![node(0, "::1", Some(8080))]); - assert_eq!( - primary_redirect_location(&roster, 0, "http", READ_PATH, None), - Some("http://[::1]:8080/streams?consistency=linearizable".to_owned()) - ); - } - - #[test] - fn primary_redirect_location_uses_advertised_address() { - let mut primary = node(0, "10.0.0.1", Some(8080)); - primary.advertised_address = Some("2001:db8::1".to_owned()); - let roster = roster(vec![primary]); - - assert_eq!( - primary_redirect_location(&roster, 0, "https", READ_PATH, None), - Some("https://[2001:db8::1]:8080/streams?consistency=linearizable".to_owned()) - ); - } - - #[test] - fn primary_redirect_location_uses_advertised_hostname() { - let mut primary = node(0, "10.0.0.1", Some(8080)); - primary.advertised_address = Some("broker-1.example.com".to_owned()); - let roster = roster(vec![primary]); - - assert_eq!( - primary_redirect_location(&roster, 0, "https", READ_PATH, None), - Some("https://broker-1.example.com:8080/streams?consistency=linearizable".to_owned()) - ); - } - - #[test] - fn primary_redirect_location_uses_the_selector_address_for_a_matching_client() { - let mut primary = node(0, "10.0.0.1", Some(8080)); - primary.advertised_address = Some("203.0.113.1".to_owned()); - primary.advertised_addresses = vec![configs::ng_cluster::AdvertisedAddressSelector { - client_cidr: "10.0.0.0/16".to_owned(), - address: "10.0.0.1".to_owned(), - }]; - let roster = roster(vec![primary]); - - assert_eq!( - primary_redirect_location( - &roster, - 0, - "https", - READ_PATH, - Some("10.0.9.9".parse().unwrap()) - ), - Some("https://10.0.0.1:8080/streams?consistency=linearizable".to_owned()), - "an in-network client must be redirected to the selector address" - ); - assert_eq!( - primary_redirect_location( - &roster, - 0, - "https", - READ_PATH, - Some("198.51.100.7".parse().unwrap()) - ), - Some("https://203.0.113.1:8080/streams?consistency=linearizable".to_owned()), - "an out-of-network client must stay on the catch-all address" - ); - } - - #[test] - fn primary_http_socket_uses_private_roster_ip() { - let mut primary = node(0, "10.0.0.1", Some(8080)); - primary.advertised_address = Some("203.0.113.1".to_owned()); - let roster = roster(vec![primary]); - - assert_eq!( - primary_http_socket(&roster, 0), - Some("10.0.0.1:8080".parse().expect("valid socket address")) - ); - } - - #[test] - fn transient_not_committed_renders_503_with_retry_after() { - let response = CustomError::from(IggyError::TransientNotCommitted).into_response(); - assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); - assert!(response.headers().contains_key(RETRY_AFTER)); - } - - #[test] - fn transient_not_accepted_renders_503_with_retry_after() { - let response = CustomError::from(IggyError::TransientNotAccepted).into_response(); - assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); - assert!(response.headers().contains_key(RETRY_AFTER)); - } - - #[test] - fn business_error_renders_without_retry_after() { - let response = CustomError::from(IggyError::UserAlreadyExists).into_response(); - assert_eq!(response.status(), StatusCode::BAD_REQUEST); - assert!(!response.headers().contains_key(RETRY_AFTER)); - } - - // The ownership refusal is permanent and deterministic. Rendering it as - // 503 would hand the caller's HTTP stack a retry loop it can never escape - // (no foreign SDK special-cases 503), so the status is load-bearing. - #[test] - fn owned_client_id_renders_as_terminal_conflict() { - let response = AuthError::SessionIdOwnedByAnotherUser.into_response(); - assert_eq!(response.status(), StatusCode::CONFLICT); - assert!( - response.headers().get(RETRY_AFTER).is_none(), - "a terminal refusal must not advertise a retry" - ); - } - - // Its siblings stay retryable, so the split is visible in one place. - #[test] - fn unknown_outcome_registers_stay_retryable() { - for error in [AuthError::SessionUnavailable, AuthError::SessionNotAccepted] { - let status = error.into_response().status(); - assert!( - status.is_server_error(), - "an unknown commit outcome must stay retryable, got {status}" - ); - } - } - - #[test] - fn recovery_incomplete_renders_retryable_503_like_not_primary() { - // Barrier expiry must render as the shared retryable 503: the same - // status and Retry-After hint as the not-primary 503, so an SDK treats - // it as a connection-level retry rather than a terminal error. - let recovery = ReadError::RecoveryIncomplete.into_response(); - assert_eq!(recovery.status(), StatusCode::SERVICE_UNAVAILABLE); - assert_eq!( - recovery.headers().get(RETRY_AFTER), - Some(&HeaderValue::from(RETRY_AFTER_SECONDS)) - ); - - let not_primary = ReadError::NotPrimary.into_response(); - assert_eq!(recovery.status(), not_primary.status()); - assert_eq!( - recovery.headers().get(RETRY_AFTER), - not_primary.headers().get(RETRY_AFTER) - ); - } -} diff --git a/core/server-ng/src/http/metrics.rs b/core/server-ng/src/http/metrics.rs deleted file mode 100644 index bb18a8a9b9..0000000000 --- a/core/server-ng/src/http/metrics.rs +++ /dev/null @@ -1,297 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! The `[http.metrics]` scrape surface: the legacy-parity metric registry -//! (entity gauges plus the request counter), its public scrape handler, and -//! the config gate deciding whether the route is mounted. - -use axum::extract::State; -use configs::http::HttpMetricsConfig; -use consensus::MetadataHandle; -use iggy_common::IggyError; -use metadata::impls::metadata::StreamsFrontend; -use prometheus_client::encoding::text::encode; -use prometheus_client::metrics::counter::Counter; -use prometheus_client::metrics::gauge::Gauge; -use prometheus_client::registry::Registry; -use send_wrapper::SendWrapper; -use tracing::error; - -use crate::http::state::HttpState; - -/// The legacy server's metric set, registered under the same names and help -/// texts so existing dashboards and alerts keep working unchanged. -/// -/// Unlike the legacy server, the entity gauges are not counted at mutation -/// sites: [`get_metrics`] samples the live state on every scrape, so a gauge -/// can never drift from the state it describes. -pub(in crate::http) struct HttpMetrics { - registry: Registry, - http_requests: Counter, - streams: Gauge, - topics: Gauge, - partitions: Gauge, - segments: Gauge, - messages: Gauge, - users: Gauge, - clients: Gauge, -} - -impl HttpMetrics { - pub(in crate::http) fn init() -> Self { - let mut registry = Registry::default(); - let http_requests = Counter::default(); - let streams = Gauge::default(); - let topics = Gauge::default(); - let partitions = Gauge::default(); - let segments = Gauge::default(); - let messages = Gauge::default(); - let users = Gauge::default(); - let clients = Gauge::default(); - registry.register( - "http_requests", - "total count of http_requests", - http_requests.clone(), - ); - registry.register("streams", "total count of streams", streams.clone()); - registry.register("topics", "total count of topics", topics.clone()); - registry.register( - "partitions", - "total count of partitions", - partitions.clone(), - ); - registry.register("segments", "total count of segments", segments.clone()); - registry.register("messages", "total count of messages", messages.clone()); - registry.register("users", "total count of users", users.clone()); - registry.register("clients", "total count of clients", clients.clone()); - Self { - registry, - http_requests, - streams, - topics, - partitions, - segments, - messages, - users, - clients, - } - } - - /// Handle for the router's request-counting layer. The counter is - /// `Arc`-backed, so bumping the clone bumps the registered metric. - pub(in crate::http) fn request_counter(&self) -> Counter { - self.http_requests.clone() - } - - fn formatted_output(&self) -> String { - let mut buffer = String::new(); - if let Err(error) = encode(&mut buffer, &self.registry) { - error!(%error, "failed to encode metrics"); - } - buffer - } -} - -/// Resolve the configured scrape path: `None` when `[http.metrics]` is -/// disabled, so the route is never mounted and the endpoint answers 404. -/// -/// axum's `Router::route` panics on a path without a leading `/`, so an -/// enabled endpoint missing one is rejected as a configuration error before -/// the router is assembled. -/// -/// # Errors -/// -/// Returns [`IggyError::InvalidConfiguration`] when metrics are enabled and -/// the endpoint does not start with `/`. -pub(in crate::http) fn validated_endpoint( - config: &HttpMetricsConfig, -) -> Result, IggyError> { - if !config.enabled { - return Ok(None); - } - if !config.endpoint.starts_with('/') { - error!( - endpoint = %config.endpoint, - "invalid http.metrics.endpoint: the path must start with '/'" - ); - return Err(IggyError::InvalidConfiguration); - } - Ok(Some(config.endpoint.clone())) -} - -/// `GET `: the metric set in prometheus text -/// exposition. Public - reached without proving a credential, exactly like the -/// legacy endpoint. -/// -/// The entity gauges sample the same reads `/stats` serves: the metadata STM -/// stream and user maps plus the stats-registry rollups, whose partition-plane -/// increments are relaxed, so scraped values are approximate while writes are -/// in flight. The clients count scatter-gathers the per-shard session managers -/// exactly like `GET /clients` and turns partial when a shard misses the reply -/// deadline. -pub(in crate::http) async fn get_metrics(State(state): State) -> String { - let (streams_count, topics_count, partitions_count, segments_count, messages_count) = state - .shard - .plane - .metadata() - .mux_stm - .streams() - .read(|streams| { - let mut topics_count = 0u64; - let mut partitions_count = 0u64; - let mut segments_count = 0u64; - let mut messages_count = 0u64; - for (_, stream) in &streams.items { - topics_count = topics_count.saturating_add(stream.topics.len() as u64); - segments_count = segments_count - .saturating_add(u64::from(stream.stats.segments_count_inconsistent())); - messages_count = - messages_count.saturating_add(stream.stats.messages_count_inconsistent()); - for (_, topic) in &stream.topics { - partitions_count = - partitions_count.saturating_add(topic.partitions.len() as u64); - } - } - ( - streams.items.len() as u64, - topics_count, - partitions_count, - segments_count, - messages_count, - ) - }); - let users_count = state - .shard - .plane - .metadata() - .mux_stm - .users() - .read(|users| users.items.len() as u64); - let clients_count = SendWrapper::new(state.shard.list_all_clients()).await.len() as u64; - - let metrics = &state.metrics; - metrics.streams.set(gauge_value(streams_count)); - metrics.topics.set(gauge_value(topics_count)); - metrics.partitions.set(gauge_value(partitions_count)); - metrics.segments.set(gauge_value(segments_count)); - metrics.messages.set(gauge_value(messages_count)); - metrics.users.set(gauge_value(users_count)); - metrics.clients.set(gauge_value(clients_count)); - metrics.formatted_output() -} - -/// Clamp a count into the gauge's `i64` domain; only `messages` can pass -/// `i64::MAX` even in theory, the rest are bounded far below it. -fn gauge_value(count: u64) -> i64 { - i64::try_from(count).unwrap_or(i64::MAX) -} - -#[cfg(test)] -mod tests { - use super::*; - - const PARITY_METRIC_NAMES: [&str; 8] = [ - "http_requests", - "streams", - "topics", - "partitions", - "segments", - "messages", - "users", - "clients", - ]; - - fn metrics_config(enabled: bool, endpoint: &str) -> HttpMetricsConfig { - HttpMetricsConfig { - enabled, - endpoint: endpoint.to_owned(), - } - } - - #[test] - fn formatted_output_exposes_every_parity_metric() { - let metrics = HttpMetrics::init(); - let output = metrics.formatted_output(); - for name in PARITY_METRIC_NAMES { - assert!( - output.contains(&format!("# TYPE {name} ")), - "metric {name} missing from exposition:\n{output}" - ); - } - assert!( - output.ends_with("# EOF\n"), - "missing exposition trailer:\n{output}" - ); - } - - #[test] - fn scraped_values_land_in_the_exposition() { - let metrics = HttpMetrics::init(); - metrics.streams.set(1); - metrics.topics.set(2); - metrics.partitions.set(3); - metrics.segments.set(4); - metrics.messages.set(5); - metrics.users.set(6); - metrics.clients.set(7); - metrics.request_counter().inc(); - let output = metrics.formatted_output(); - for line in [ - "streams 1", - "topics 2", - "partitions 3", - "segments 4", - "messages 5", - "users 6", - "clients 7", - "http_requests_total 1", - ] { - assert!( - output.contains(&format!("\n{line}\n")), - "expected `{line}` in exposition:\n{output}" - ); - } - } - - #[test] - fn gauge_value_clamps_past_i64_range() { - assert_eq!(gauge_value(42), 42); - assert_eq!(gauge_value(u64::MAX), i64::MAX); - } - - #[test] - fn validated_endpoint_disabled_yields_none() { - assert!(matches!( - validated_endpoint(&metrics_config(false, "/metrics")), - Ok(None) - )); - } - - #[test] - fn validated_endpoint_returns_enabled_path() { - let endpoint = validated_endpoint(&metrics_config(true, "/metrics")).unwrap(); - assert_eq!(endpoint.as_deref(), Some("/metrics")); - } - - #[test] - fn validated_endpoint_rejects_missing_leading_slash() { - assert!(matches!( - validated_endpoint(&metrics_config(true, "metrics")), - Err(IggyError::InvalidConfiguration) - )); - } -} diff --git a/core/server-ng/src/lib.rs b/core/server-ng/src/lib.rs deleted file mode 100644 index 1b0f56b769..0000000000 --- a/core/server-ng/src/lib.rs +++ /dev/null @@ -1,47 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#![allow(clippy::future_not_send)] - -use iggy_common::SemanticVersion; - -pub const VERSION: &str = env!("CARGO_PKG_VERSION"); -pub const SEMANTIC_VERSION: SemanticVersion = SemanticVersion::parse_const(VERSION); - -pub mod auth; -pub mod bootstrap; -pub(crate) mod cluster_meta; -pub mod config_writer; -pub mod consumer_group; -pub mod dispatch; -pub(crate) mod http; -pub mod login_register; -pub(crate) mod offset_recovery; -pub mod partition_helpers; -pub mod partition_reconciler; -pub mod pat; -pub(crate) mod personal_access_token_cleaner; -pub mod responses; -pub(crate) mod segment_cleaner; -pub(crate) mod segment_recovery; -pub mod server_error; -pub mod session_manager; -pub(crate) mod snapshot; -pub mod users; -#[cfg(feature = "iggy-web")] -pub(crate) mod web; -pub mod wire; diff --git a/core/server-ng/src/main.rs b/core/server-ng/src/main.rs deleted file mode 100644 index 404a4496f2..0000000000 --- a/core/server-ng/src/main.rs +++ /dev/null @@ -1,89 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#![allow(clippy::future_not_send)] - -mod args; - -use args::Args; -use clap::Parser; -use server_ng::bootstrap::{bootstrap, load_config}; -use server_ng::server_error::ServerNgError; -use system_stats::capture_allowed_cpus; -use tracing::{error, info}; - -fn main() -> Result<(), ServerNgError> { - // Before shard threads pin themselves: a pinned capture sees one core. - capture_allowed_cpus(); - - let bootstrap_runtime = match server_common::create_shard_executor() { - Ok(rt) => rt, - Err(e) => { - let e = server_common::diagnostics::enrich_runtime_create_error(e); - panic!("Cannot create server-ng bootstrap executor: {e}"); - } - }; - - // Bootstrap on a temporary runtime: parse args, init logging, load - // config, init the memory pool. Then drop the runtime and spawn the - // per-shard runtimes - each shard thread builds its OWN - // `compio::runtime::Runtime` via `create_shard_executor`, pinned to - // its CPU. - let bootstrap_result: Result< - ( - configs::server_ng::ServerNgConfig, - Option, - server_common::log::Logging, - ), - ServerNgError, - > = bootstrap_runtime.block_on(async { - let args = Args::parse(); - if let Ok(env_path) = std::env::var("IGGY_ENV_PATH") { - let _ = dotenvy::from_path(&env_path); - } else { - let _ = dotenvy::dotenv(); - } - - let mut logging = server_common::log::Logging::new(server_ng::VERSION); - logging.early_init(); - server_common::print_build_info!(server_ng::VERSION); - - let config = load_config(&mut logging).await?; - server_common::MemoryPool::init_pool(&config.system.memory_pool.into_other()); - - Ok((config, args.replica_id, logging)) - }); - // `_logging` owns the tracing appender worker guards; it must outlive the - // shard threads or every log line after bootstrap is silently dropped. - let (config, replica_id, _logging) = bootstrap_result?; - drop(bootstrap_runtime); - - let shards = bootstrap(config, replica_id)?; - if let Err(error) = shards.install_ctrlc_handler() { - // Without a working SIGINT handler the server has no way to - // observe an operator Ctrl-C and the shutdown flag would never - // flip, leaving shard threads parked indefinitely. Fail fast - // rather than boot into an un-killable state. - error!(error = %error, "failed to install Ctrl-C handler; aborting boot"); - std::process::exit(1); - } - - info!("server-ng running; waiting on shard threads"); - shards.join_all()?; - info!("server-ng shutdown complete"); - Ok(()) -} diff --git a/core/server-ng/src/server_error.rs b/core/server-ng/src/server_error.rs deleted file mode 100644 index 49581a190d..0000000000 --- a/core/server-ng/src/server_error.rs +++ /dev/null @@ -1,391 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use consensus::VsrStateError; -use metadata::impls::recovery::RecoveryError; -use server_common::log::LogError; -use shard::ShardCtorError; -use shard_allocator::ShardingError; -use std::path::PathBuf; -use thiserror::Error; - -#[derive(Debug, Error)] -#[non_exhaustive] -pub enum ServerNgError { - #[error(transparent)] - Iggy(Box), - #[error("failed to load server-ng config")] - Config(#[source] configs::ConfigurationError), - #[error("failed to allocate shards from sharding.cpu_allocation")] - ShardAllocator(#[source] ShardingError), - #[error("failed to bind shard {shard_id} to its CPU set")] - CpuAffinityFailed { - shard_id: u16, - #[source] - source: ShardingError, - }, - #[error("failed to bind shard {shard_id} memory to its NUMA node")] - MemoryAffinityFailed { - shard_id: u16, - #[source] - source: ShardingError, - }, - #[error("failed to spawn OS thread for shard {shard_id}")] - ShardSpawnFailed { - shard_id: u16, - #[source] - source: std::io::Error, - }, - // `{source}` is deliberately part of the Display text: the shard-join - // failure report and `%error` log fields print Display only, and the - // source carries the io_uring remediation folded in by - // `server_common::diagnostics::enrich_runtime_create_error`. - #[error("failed to create io_uring runtime for shard {shard_id}: {source}")] - ShardRuntimeCreateFailed { - shard_id: u16, - #[source] - source: std::io::Error, - }, - #[error( - "shard allocator produced zero shards; server must run at least one \ - shard (check [system.sharding] cpu_allocation)" - )] - ShardsCountZero, - #[error( - "computed shards_count = {count} exceeds the maximum of {} shards per \ - server; shard ids must fit in u16 and stay below the OWNER_NONE \ - sentinel", - message_bus::OWNER_NONE - 1 - )] - ShardsCountOverflow { count: usize }, - #[error("system.sharding.inbox_capacity must be in 1..={max}; got {value}")] - InvalidInboxCapacity { value: usize, max: usize }, - #[error("system.sharding.shutdown_drain_timeout must be in (0, {max:?}]; got {value:?}")] - InvalidShutdownDrainTimeout { - value: std::time::Duration, - max: std::time::Duration, - }, - #[error("system.sharding.shutdown_poll_interval must be in (0, {max:?}]; got {value:?}")] - InvalidShutdownPollInterval { - value: std::time::Duration, - max: std::time::Duration, - }, - #[error( - "system.sharding.shutdown_poll_interval ({poll:?}) must be <= \ - shutdown_drain_timeout ({drain:?})" - )] - ShutdownPollExceedsDrain { - poll: std::time::Duration, - drain: std::time::Duration, - }, - #[error("failed to serialize current server-ng config")] - CurrentConfigSerialize(#[source] toml::ser::Error), - #[error("failed to write current server-ng config at {path}")] - CurrentConfigWrite { - path: String, - #[source] - source: std::io::Error, - }, - #[error("failed to initialize server-ng logging")] - Logging(#[source] LogError), - #[error("failed to recover metadata snapshot and journal")] - MetadataRecovery(#[source] RecoveryError), - #[error("failed to open partition superblock at {dir}")] - PartitionSuperblockIo { - dir: PathBuf, - #[source] - source: std::io::Error, - }, - // Quarantines the one partition rather than treating the group as fresh or - // reading through to a superseded view: mirrors the metadata plane's - // `RecoveryError::SuperblockUnreadable` policy, minus the boot refusal, - // because one unreadable partition directory must not strand every healthy - // group on the shard. - #[error( - "partition superblock at {dir} is present but its format version \ - {version} is unrecognized by this build (a downgrade, or a corrupt \ - version field)" - )] - PartitionSuperblockVersionUnknown { dir: PathBuf, version: u16 }, - #[error( - "partition superblock at {dir} is present but a copy holds bytes that \ - do not verify (bit-rot or a checksum failure), so its latest \ - generation cannot be established" - )] - PartitionSuperblockUnverifiable { dir: PathBuf }, - #[error( - "partition superblock at {dir} was checksum-clean but did not decode; \ - tombstoning this partition rather than inferring a stale view" - )] - PartitionSuperblockUndecodable { - dir: PathBuf, - #[source] - source: VsrStateError, - }, - #[error( - "partition superblock at {dir} belongs to a different {field}: expected \ - {expected}, found {found}; a copied or misplaced data directory, or the \ - cluster was resized without reconfiguration" - )] - PartitionSuperblockIdentityMismatch { - dir: PathBuf, - field: metadata::IdentityField, - expected: u128, - found: u128, - }, - // Per-partition, not fatal: the boot path fences this one group (quarantines - // its segment files and materialises it fresh) instead of taking the node - // down for one damaged local chain. The shapes it reports are exactly what a - // failed state-transfer quarantine leaves behind, and the rebuild recovers - // the data from a peer. - #[error( - "partition {stream_id}/{topic_id}/{partition_id} at {dir} recovered an \ - unusable segment chain: {reason}" - )] - PartitionChainRefused { - dir: PathBuf, - stream_id: usize, - topic_id: usize, - partition_id: usize, - reason: PartitionChainRefusal, - }, - #[error( - "shard {shard_id} aborted while waiting for shard-0 to broadcast the metadata \ - factory bundle; shard 0 dropped its sender (most likely it failed to recover)" - )] - MetadataHandoffAborted { shard_id: u16 }, - #[error( - "shard 0 aborted before binding listeners with {remaining} peer shard(s) still loading \ - their on-disk partitions; a peer most likely failed during bootstrap (shutdown flag set)" - )] - ShardBootstrapBarrierAborted { remaining: usize }, - #[error("failed to parse {context} socket address '{address}'")] - SocketAddressParse { - context: &'static str, - address: String, - #[source] - source: std::net::AddrParseError, - }, - #[error("cluster enabled but no node is configured for replica {replica_id}")] - ClusterNodeNotFound { replica_id: u8 }, - #[error("cluster node count {count} exceeds supported u8 replica count")] - ClusterReplicaCountTooLarge { count: usize }, - #[error("cluster mode requires --replica-id to identify the current node")] - MissingReplicaId, - #[error( - "--replica-id {supplied} was passed with cluster.enabled=false; the WAL would commit \ - under replica {default} which permanently fixes this node's identity. Either set \ - cluster.enabled=true with a matching nodes[] entry, or drop --replica-id" - )] - ReplicaIdRequiresCluster { supplied: u8, default: u8 }, - #[error( - "cluster node for replica {replica_id} is missing ports.{transport}; cluster mode \ - requires an explicit roster port for every enabled transport" - )] - ClusterPortMissing { - transport: &'static str, - replica_id: u8, - }, - #[error( - "cluster bootstrap with empty metadata requires both {username_env} and {password_env} to be set before server-ng can create the root user deterministically" - )] - ClusterRootCredentialsRequired { - username_env: &'static str, - password_env: &'static str, - }, - #[error( - "recovered segment for stream {stream_id}, topic {topic_id}, partition {partition_id} at start_offset {start_offset} has message/index divergence (messages_size={messages_size_bytes}, indexed_size={indexed_size_bytes}, end_offset={end_offset}); recovery aborted before opening listeners. Restore the partition from a healthy replica or snapshot, or move the segment aside for offline repair before restarting." - )] - RecoveredSegmentSizeDivergence { - stream_id: usize, - topic_id: usize, - partition_id: usize, - start_offset: u64, - end_offset: u64, - messages_size_bytes: u64, - indexed_size_bytes: u64, - }, - #[error( - "failed to load persisted {consumer_kind} offsets for stream {stream_id}, topic {topic_id}, partition {partition_id} from {path}" - )] - ConsumerOffsetsLoad { - consumer_kind: &'static str, - stream_id: usize, - topic_id: usize, - partition_id: usize, - path: String, - #[source] - source: Box, - }, - #[error( - "recovered namespace stream {stream_id}, topic {topic_id}, partition {partition_id} exceeds configured limits (max_streams={max_streams}, max_topics={max_topics}, max_partitions={max_partitions})" - )] - RecoveredNamespaceOutOfBounds { - stream_id: usize, - topic_id: usize, - partition_id: usize, - max_streams: usize, - max_topics: usize, - max_partitions: usize, - }, - #[error("failed to load {transport} listener credentials")] - ListenerCredentials { - transport: &'static str, - #[source] - source: std::io::Error, - }, - #[error("failed to build the HTTP forward client: {reason}")] - HttpForwardClient { reason: String }, - #[error("failed to construct IggyShard from bootstrap inputs")] - ShardConstruction(#[source] ShardCtorError), - #[error("{} shard thread(s) failed: {}", failures.len(), format_shard_failures(failures))] - ShardJoinFailures { failures: Vec }, -} - -/// Why a recovered segment chain cannot be served. -/// -/// Both shapes mean the same thing operationally -- the local files do not form -/// a chain this replica can serve -- but they are distinguished because they -/// point at different causes: an empty non-tail segment is a failed rebuild's -/// orphan pairing, a hole is a stray or half-unlinked file. -#[derive(Debug)] -pub enum PartitionChainRefusal { - EmptyNonTailSegment { - empty_start: u64, - next_start: u64, - }, - Hole { - previous_start: u64, - previous_end: u64, - next_start: u64, - }, -} - -impl std::fmt::Display for PartitionChainRefusal { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::EmptyNonTailSegment { - empty_start, - next_start, - } => write!( - f, - "segment {empty_start} is empty yet {next_start} follows it, so the \ - chain cannot be served past it" - ), - Self::Hole { - previous_start, - previous_end, - next_start, - } => write!( - f, - "segment {previous_start} ends at offset {previous_end} but the next \ - starts at {next_start}, leaving a hole" - ), - } - } -} - -/// Per-shard outcome captured by [`crate::bootstrap::ShardHandles::join_all`] -/// when a shard either returned `Err` or panicked. -/// -/// Bundled into [`ServerNgError::ShardJoinFailures`] so the operator sees -/// every failing shard rather than only the first one, which previously -/// lived in the trace log alone. -#[derive(Debug)] -pub struct ShardJoinFailure { - pub shard_id: u16, - pub kind: ShardJoinFailureKind, -} - -#[derive(Debug)] -pub enum ShardJoinFailureKind { - Error(Box), - Panic { - message: String, - }, - /// The shard thread never finished inside `shutdown_join_timeout` - /// and was abandoned so process exit is not blocked forever. - Wedged { - waited: std::time::Duration, - }, -} - -fn format_shard_failures(failures: &[ShardJoinFailure]) -> String { - use std::fmt::Write as _; - let mut out = String::new(); - for (idx, failure) in failures.iter().enumerate() { - if idx > 0 { - out.push_str("; "); - } - match &failure.kind { - ShardJoinFailureKind::Error(err) => { - let _ = write!(out, "shard {} -> {err}", failure.shard_id); - } - ShardJoinFailureKind::Panic { message } => { - let _ = write!(out, "shard {} panicked: {message}", failure.shard_id); - } - ShardJoinFailureKind::Wedged { waited } => { - let _ = write!( - out, - "shard {} wedged: thread still running after {waited:?}, abandoned", - failure.shard_id - ); - } - } - } - out -} - -impl From for ServerNgError { - fn from(source: iggy_common::IggyError) -> Self { - Self::Iggy(Box::new(source)) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn shard_join_failures_display_aggregates_all_entries() { - let failures = vec![ - ShardJoinFailure { - shard_id: 0, - kind: ShardJoinFailureKind::Error(Box::new(ServerNgError::MissingReplicaId)), - }, - ShardJoinFailure { - shard_id: 2, - kind: ShardJoinFailureKind::Panic { - message: "boom".to_string(), - }, - }, - ]; - let rendered = ServerNgError::ShardJoinFailures { failures }.to_string(); - assert!( - rendered.starts_with("2 shard thread(s) failed:"), - "expected count prefix, got {rendered}" - ); - assert!( - rendered.contains("shard 0 ->"), - "shard 0 entry missing: {rendered}" - ); - assert!( - rendered.contains("shard 2 panicked: boom"), - "shard 2 panic entry missing: {rendered}" - ); - } -} diff --git a/core/server-ng/src/wire.rs b/core/server-ng/src/wire.rs deleted file mode 100644 index e7a047089b..0000000000 --- a/core/server-ng/src/wire.rs +++ /dev/null @@ -1,75 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Leaf wire helpers shared by the request-handling modules. -//! -//! Request-body slicing, the `usize -> u32` wire conversion, and the -//! transport-kind discriminant mapping. - -use bytes::Bytes; -use iggy_binary_protocol::RequestHeader; -use iggy_common::IggyError; -use message_bus::installer::conn_info::ClientTransportKind; -use server_common::Message; - -pub(crate) fn request_body(request: &Message) -> &[u8] { - &request.as_slice()[std::mem::size_of::()..request.header().size as usize] -} - -/// Map the transport kind to the legacy wire discriminant -/// (`1=TCP, 2=QUIC, 4=WebSocket`); TLS variants report their base -/// transport. `ClientTransportKind` is `#[non_exhaustive]`, so any other -/// (TCP, TCP-TLS, or a future) variant falls back to TCP. -pub(crate) const fn transport_kind_to_wire(kind: ClientTransportKind) -> u8 { - match kind { - ClientTransportKind::Quic => 2, - ClientTransportKind::Ws | ClientTransportKind::Wss => 4, - _ => 1, - } -} - -pub(crate) fn usize_to_u32(value: usize) -> Result { - u32::try_from(value).map_err(|_| IggyError::InvalidIdentifier) -} - -/// Rebuild a request message with `body` replacing the original payload, -/// preserving the header (and fixing `size`). Used by the primary-side -/// request rewrites that swap a secret-bearing wire body for the -/// hash-carrying replicated body before consensus. -pub(crate) fn rewrite_request_body( - request: &Message, - body: &Bytes, -) -> Result, IggyError> { - let total_size = std::mem::size_of::() - .checked_add(body.len()) - .ok_or(IggyError::InvalidConfiguration)?; - let size = u32::try_from(total_size).map_err(|_| IggyError::InvalidConfiguration)?; - let mut rewritten = Message::::new(total_size); - let header = bytemuck::checked::try_from_bytes_mut::( - &mut rewritten.as_mut_slice()[..std::mem::size_of::()], - ) - .expect("zeroed bytes are a valid request header"); - *header = *request.header(); - header.size = size; - rewritten.as_mut_slice()[std::mem::size_of::()..].copy_from_slice(body); - // TODO(vsr): the body changed but `request_checksum` / `checksum` / - // `checksum_body` were copied verbatim from the original header. Safe - // today because the SDK initializes `request_checksum` to 0 and the - // server does not validate it; the moment integrity checking lands, - // recompute these here (or zero them and re-sign in a follow-up step). - Ok(rewritten) -} diff --git a/core/server/Cargo.toml b/core/server/Cargo.toml index 2c29e13b30..e36b0d1dd8 100644 --- a/core/server/Cargo.toml +++ b/core/server/Cargo.toml @@ -17,13 +17,66 @@ [package] name = "server" -version = "0.8.2-edge.1" +version = "0.9.0-edge.2" edition = "2024" license = "Apache-2.0" publish = false +[package.metadata.cargo-udeps.ignore] +normal = ["tracing-appender"] + [package.metadata.cargo-machete] -ignored = ["vergen-git2"] +ignored = [ + "ahash", + "anyhow", + "argon2", + "async-channel", + "async_zip", + "axum", + "axum-server", + "bytes", + "chrono", + "ctrlc", + "cyper", + "cyper-axum", + "dashmap", + "err_trail", + "error_set", + "figlet-rs", + "hash32", + "human-repr", + "hwlocality", + "jsonwebtoken", + "left-right", + "mimalloc", + "mime_guess", + "nix", + "opentelemetry", + "opentelemetry-appender-tracing", + "opentelemetry-otlp", + "opentelemetry-semantic-conventions", + "opentelemetry_sdk", + "papaya", + "rand", + "ringbuffer", + "rmp-serde", + "rolling-file", + "rust-embed", + "rustls", + "rustls-pemfile", + "send_wrapper", + "serde", + "slab", + "socket2", + "strum", + "sysinfo", + "tempfile", + "tracing-appender", + "tracing-opentelemetry", + "ulid", + "uuid", + "vergen-git2", +] [[bin]] name = "iggy-server" @@ -38,40 +91,57 @@ systemd = ["dep:sd-notify"] [dependencies] ahash = { workspace = true } -anyhow = { workspace = true } +argon2 = { workspace = true } async-channel = { workspace = true } async_zip = { workspace = true } axum = { workspace = true } axum-server = { workspace = true } +blake3 = { workspace = true } +bytemuck = { workspace = true } bytes = { workspace = true } chrono = { workspace = true } clap = { workspace = true } compio = { workspace = true } configs = { workspace = true } +consensus = { workspace = true } +crossfire = { workspace = true } ctrlc = { workspace = true } cyper = { workspace = true } cyper-axum = { workspace = true } +cyper-core = { workspace = true } dashmap = { workspace = true } dotenvy = { workspace = true } err_trail = { workspace = true } error_set = { workspace = true } figlet-rs = { workspace = true } -flume = { workspace = true } fs2 = { workspace = true } futures = { workspace = true } hash32 = { workspace = true } human-repr = { workspace = true } +hyper = { workspace = true } +hyper-util = { workspace = true } iggy_binary_protocol = { workspace = true } iggy_common = { workspace = true } +journal = { workspace = true } jsonwebtoken = { workspace = true } left-right = { workspace = true } +message_bus = { workspace = true } +metadata = { workspace = true } mimalloc = { workspace = true, optional = true } mime_guess = { workspace = true, optional = true } nix = { workspace = true } +opentelemetry = { workspace = true } +opentelemetry-appender-tracing = { workspace = true } +opentelemetry-otlp = { workspace = true } +opentelemetry-semantic-conventions = { workspace = true } +opentelemetry_sdk = { workspace = true } papaya = { workspace = true } +partitions = { workspace = true } prometheus-client = { workspace = true } +rand = { workspace = true } ringbuffer = { workspace = true } rmp-serde = { workspace = true } +rolling-file = { workspace = true } rust-embed = { workspace = true, optional = true } rustls = { workspace = true } rustls-pemfile = { workspace = true } @@ -81,6 +151,7 @@ send_wrapper = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } server_common = { workspace = true } +shard = { workspace = true } shard_allocator = { workspace = true } slab = { workspace = true } socket2 = { workspace = true } @@ -93,8 +164,33 @@ tokio = { workspace = true } toml = { workspace = true } tower-http = { workspace = true } tracing = { workspace = true } +tracing-appender = { workspace = true } +tracing-opentelemetry = { workspace = true } ulid = { workspace = true } uuid = { workspace = true } +[target.'cfg(not(target_env = "musl"))'.dependencies] +hwlocality = { workspace = true } + +[target.'cfg(target_env = "musl")'.dependencies] +hwlocality = { workspace = true, features = ["vendored"] } + [build-dependencies] vergen-git2 = { workspace = true } + +[dev-dependencies] +assert_cmd = { workspace = true } +bytemuck = { workspace = true } +# Reconciler unit tests assert on `ShardMetrics` snapshots and +# `IggyShard::parked_frame_count`, gated to test/simulator so they cannot grow +# production callers. `shard`'s own `cfg(test)` is false when compiled as our +# dependency, so the feature is how those accessors become visible. The resolver +# keeps a dev-dependency's features out of non-test targets, so a production +# build still links `shard` without `simulator`. +shard = { workspace = true, features = ["simulator"] } +tokio = { workspace = true, features = ["full", "test-util"] } + +[lints.clippy] +enum_glob_use = "deny" +pedantic = "deny" +nursery = "warn" diff --git a/core/server/Dockerfile b/core/server/Dockerfile index b2dac2d582..cbeaaa0a5c 100644 --- a/core/server/Dockerfile +++ b/core/server/Dockerfile @@ -29,6 +29,7 @@ ARG RUST_VERSION=1.97.1 ARG ALPINE_VERSION=3.23 + # ── from-source path ───────────────────────────────────────────────────────── FROM --platform=$BUILDPLATFORM lukemathwalker/cargo-chef:latest-rust-${RUST_VERSION}-alpine${ALPINE_VERSION} AS chef WORKDIR /app @@ -154,12 +155,12 @@ RUN apt-get update && apt-get install -y \ libudev-dev \ pkg-config \ && rm -rf /var/lib/apt/lists/* -COPY --from=prebuilt /out/iggy-server /usr/local/bin/iggy-server -COPY --from=prebuilt /out/iggy /usr/local/bin/iggy -COPY --from=license-gen /app/LICENSE-binary /usr/share/doc/iggy/LICENSE-binary +COPY --from=prebuilt /out/iggy-server /usr/local/bin/iggy-server +COPY --from=prebuilt /out/iggy /usr/local/bin/iggy +COPY --from=license-gen /app/LICENSE-binary /usr/share/doc/iggy/LICENSE-binary COPY LICENSE NOTICE /usr/share/doc/iggy/ RUN echo "═══════════════════════════════════════════════════════════════" && \ - echo " IGGY SERVER BUILD SUMMARY " && \ + echo " IGGY SERVER BUILD SUMMARY " && \ echo "═══════════════════════════════════════════════════════════════" && \ echo "Build Type: PREBUILT BINARIES" && \ echo "Platform: ${TARGETPLATFORM:-linux/amd64}" && \ @@ -189,7 +190,7 @@ COPY --from=builder /app/iggy /usr/local/bin/iggy COPY --from=builder /app/LICENSE-binary /usr/share/doc/iggy/LICENSE-binary COPY LICENSE NOTICE /usr/share/doc/iggy/ RUN echo "═══════════════════════════════════════════════════════════════" && \ - echo " IGGY SERVER BUILD SUMMARY " && \ + echo " IGGY SERVER BUILD SUMMARY " && \ echo "═══════════════════════════════════════════════════════════════" && \ echo "Build Type: FROM SOURCE" && \ echo "Platform: ${TARGETPLATFORM:-linux/amd64}" && \ diff --git a/core/server/README.md b/core/server/README.md index fd4f8cea89..f0150b32eb 100644 --- a/core/server/README.md +++ b/core/server/README.md @@ -1,19 +1,46 @@ # Apache Iggy Server -This is the core server component of Apache Iggy. You can run it directly with `cargo run --bin iggy-server --release` or use the Docker image `apache/iggy:latest` (the `edge` tag is for the latest development version). +The core server component of Apache Iggy: a persistent, append-only log for message streaming. It runs thread-per-core and shared-nothing on `io_uring` (through `compio`), and commits every write through Viewstamped Replication (VSR), so the same binary serves a standalone node and a multi-node cluster. -The configuration file is located at [core/server/config.toml](https://github.com/apache/iggy/blob/master/core/server/config.toml). You can customize the server settings by modifying this file or by using environment variables e.g. `IGGY_TCP_ADDRESS=0.0.0.0:8090`. +Clients connect over TCP (custom binary protocol), QUIC, WebSocket, or the HTTP REST API. + +## Running + +```sh +cargo run --bin iggy-server --release +``` + +The Docker image `apache/iggy:latest` ships the server together with the CLI; the `edge` tag tracks the latest development build. + +To run one node of a cluster, pass its replica ID from the `cluster.nodes` roster: + +```sh +cargo run --bin iggy-server --release -- --replica-id 0 +``` + +`--replica-id` is the only command line argument; everything else is configuration. + +## Configuration + +Settings are read from [config.toml](config.toml), resolved relative to the working directory. Set `IGGY_CONFIG_PATH` to load a different file. + +Any single value can be overridden with an `IGGY_`-prefixed environment variable that mirrors the TOML path: + +```sh +IGGY_TCP_ADDRESS=0.0.0.0:8090 IGGY_HTTP_ENABLED=false cargo run --bin iggy-server +``` + +Cluster membership, quorum and replica addressing live under `[cluster]`. ## Systemd integration -Build with the `systemd` feature to enable systemd readiness and watchdog notifications: +Build with the `systemd` feature to enable readiness and watchdog notifications: ```sh cargo build --bin iggy-server --release --features systemd ``` -The server will notify systemd when it is ready and then periodically send -watchdog messages at half the configured `WatchdogSec` interval for the unit. +The server sends `READY=1` only after every enabled transport is bound and accepting, so a unit ordered after it can dial as soon as it is notified. When the unit sets `WatchdogSec=`, the server pings `WATCHDOG=1` at half that interval. On shutdown it sends `STOPPING=1`, which stops a long drain from counting against the watchdog. ![Server](../../assets/server.png) diff --git a/core/server/config.toml b/core/server/config.toml index b1609e7a3e..418d4e9fd3 100644 --- a/core/server/config.toml +++ b/core/server/config.toml @@ -20,6 +20,8 @@ # Maximum time a partition can remain in pending revocation before being force-transferred to the target member. rebalancing_timeout = "30s" # How often the periodic checker scans for timed-out pending revocations. +# TODO(hubcio): inert in the server, which paces the scan from +# system.sharding.reconcile_periodic_interval instead. Boot warns when set. rebalancing_check_interval = "5s" [data_maintenance.messages] @@ -34,10 +36,18 @@ interval = "1 m" # Determines if the HTTP server is active. # `true` enables the server, allowing it to handle HTTP requests. # `false` disables the server, preventing it from handling HTTP requests. +# In cluster mode, followers forward control-plane requests (streams, topics, +# users, ...) to the current primary when a cluster-wide JWT key exists (see +# http.jwt / cluster.auth below). +# TODO: forwarding does not cover the partition-plane APIs yet - message +# produce and consumer-offset writes are never forwarded and must reach the +# partition's primary node directly (message polls read locally on any node). enabled = true # Specifies the network address and port for the HTTP server. # The format is "HOST:PORT". For example, "127.0.0.1:3000" listens on localhost only on port 3000. +# In cluster mode the HOST still picks the bind interface, while the port +# comes from this node's cluster.nodes ports.http entry. address = "127.0.0.1:3000" # Maximum size of the request body in bytes. For security reasons, the default limit is 2 MB. @@ -65,7 +75,9 @@ enabled = true allowed_methods = ["GET", "POST", "PUT", "DELETE"] # Defines which origins are permitted to make cross-origin requests. -# An asterisk "*" allows all origins. Specific domains can be listed to restrict access. +# An asterisk "*" as the first entry allows all origins (any entries after it +# are ignored); "*" in any other position fails the config. Specific domains +# can be listed to restrict access. allowed_origins = ["*"] # Lists allowed headers that can be used in CORS requests. @@ -73,11 +85,14 @@ allowed_origins = ["*"] allowed_headers = ["content-type", "authorization"] # Headers that browsers are allowed to access in CORS responses. -# An empty array means no additional headers are exposed to browsers. -exposed_headers = [""] +# `iggy-view` carries the current VSR view number; exposing it lets browser +# clients read it on cross-origin responses. +exposed_headers = ["iggy-view"] # Determines if credentials like cookies or HTTP auth can be included in CORS requests. -# `true` allows credentials to be included, useful for authenticated sessions. +# `true` allows credentials to be included, useful for authenticated sessions; +# it requires explicit (non-wildcard) allowed_origins, allowed_headers, and +# exposed_headers. # `false` prevents credentials, enhancing privacy and security. allow_credentials = false @@ -115,6 +130,10 @@ not_before = "0 s" # Secret key for encoding JWTs. # If left empty, a secure random secret will be generated on each server start. +# In cluster mode a configured secret (identical on every node) makes bearers +# valid cluster-wide and activates follower-to-primary HTTP forwarding; with +# cluster.auth enabled the key is instead derived from the shared PSK. Without +# either, tokens are node-local and forwarding stays disabled. encoding_secret = "" # Secret key for decoding JWTs. @@ -180,9 +199,12 @@ enabled = true address = "127.0.0.1:8090" # Enable TCP socket migration across shards. +# TODO(hubcio): inert in the server, not implemented. Boot warns when set. socket_migration = true # Whether to use ipv4 or ipv6 +# TODO(hubcio): inert in the server, which takes the family from the +# tcp.address string. Boot warns when set. ipv6 = false # TLS configuration for the TCP server. @@ -204,6 +226,8 @@ cert_file = "core/certs/iggy_cert.pem" key_file = "core/certs/iggy_key.pem" # Configuration for the TCP socket +# TODO(hubcio): the whole section is inert in the server, which leaves the OS +# defaults in place. Boot warns when override_defaults is set. [tcp.socket] # Whether to overwrite the OS-default socket parameters override_defaults = false @@ -235,26 +259,37 @@ enabled = true # For example, "127.0.0.1:8080" binds to localhost on port 8080. address = "127.0.0.1:8080" -# Maximum number of simultaneous bidirectional streams in QUIC. -max_concurrent_bidi_streams = 10_000 +# Maximum number of simultaneous bidirectional streams per QUIC +# connection. The message bus opens exactly one bidi stream per peer +# (no multiplexing), so any value above 1 is wasted on extra +# preallocated quinn-proto state. +max_concurrent_bidi_streams = 1 -# Size of the buffer for sending datagrams in QUIC. -datagram_send_buffer_size = "100 KB" +# Size of the buffer for sending datagrams in QUIC. Binary-aligned to +# match QuicTuning::default() (102_400 bytes) and the rest of this [quic] +# block, which uses MiB throughout (`send_window`, `receive_window`). +datagram_send_buffer_size = "100 KiB" -# Initial Maximum Transmission Unit (MTU) for QUIC connections. -initial_mtu = "8 KB" +# Initial Maximum Transmission Unit (MTU) for QUIC connections. Binary +# units for the same reason as `datagram_send_buffer_size`. +initial_mtu = "8 KiB" -# Size of the sending window in QUIC, controlling data flow. -send_window = "100 KB" +# Send-flow window per connection. Sized to fit a single max-size +# framed message (`message_bus.max_message_size`) without +# head-of-line wait. +send_window = "64 MiB" -# Size of the receiving window in QUIC, controlling data flow. -receive_window = "100 KB" +# Receive-flow window per connection. Symmetric with `send_window`. +receive_window = "64 MiB" -# Interval for sending keep-alive messages in QUIC. -keep_alive_interval = "5 s" +# Interval for sending QUIC keep-alive PINGs. One third of +# `max_idle_timeout` so up to two consecutive losses fit before the +# idle timer closes the connection. Set to "0 s" to disable. +keep_alive_interval = "10 s" -# Maximum idle time before a QUIC connection is closed. -max_idle_timeout = "10 s" +# Maximum idle time before a QUIC connection is closed. Set to +# "0 s" to disable (not recommended). +max_idle_timeout = "30 s" # QUIC certificate configuration. [quic.certificate] @@ -270,6 +305,8 @@ cert_file = "core/certs/iggy_cert.pem" key_file = "core/certs/iggy_key.pem" # Configuration for the QUIC socket +# TODO(hubcio): the whole section is inert in the server, which leaves the OS +# defaults in place. Boot warns when override_defaults is set. [quic.socket] # Whether to override the OS-default socket parameters override_defaults = false @@ -293,9 +330,12 @@ enabled = true # Controls whether data saving is synchronous (enforce fsync) or asynchronous. # `true` for synchronous saving, ensuring data integrity at the cost of performance. # `false` for asynchronous saving, improving performance but with delayed data writing. +# TODO(hubcio): inert in the server, which only flushes on shutdown and has no +# periodic saver to configure. Boot warns when set. enforce_fsync = true # Interval for running the message saver. +# TODO(hubcio): inert in the server, see enforce_fsync above. Boot warns when set. interval = "30 s" # Personal access token configuration. @@ -347,6 +387,8 @@ endpoint = "http://localhost:7281/v1/traces" path = "local_data" # Backup configuration +# TODO(hubcio): backup is not supported; both paths below are +# inert. Boot warns when either is set. [system.backup] # Path for storing backup. path = "backup" @@ -356,6 +398,9 @@ path = "backup" # Subpath of the backup directory where converted segment data is stored after compatibility conversion. path = "compatibility" +# TODO(hubcio): the three tunables below are inert in the server, whose state +# writes do not go through the legacy retrying file layer. Boot warns when any +# is set. [system.state] # Determines whether to enforce file synchronization on state updates (boolean). # `true` ensures immediate writing of data to disk for durability. @@ -412,6 +457,8 @@ rotation_check_interval = "1 h" retention = "7 days" # Interval for printing system information to the log. +# TODO(hubcio): inert in the server, which has no sysinfo printer. Boot warns +# when set. sysinfo_print_interval = "10 s" # Encryption configuration @@ -428,15 +475,15 @@ key = "" # Compression configuration [system.compression] -# Reserved for future server-side compression support; use "none" today. -# For manual compression, store the algorithm in message headers. # Allows overriding the default compression algorithm per data segment (boolean). -# `true` will permit different compression algorithms for individual segments. -# `false` will make all data segments use the default compression algorithm. +# `true` permits different compression algorithms for individual segments. +# `false` means all data segments use the default compression algorithm. +# TODO(hubcio): inert in the server, where live compression is already per-topic +# from the request. Boot warns when set. allow_override = false # The default compression algorithm used for data storage (string). -# "none" indicates no compression. Other values are reserved for future support. +# "none" indicates no compression, other values can specify different algorithms. default_algorithm = "none" # Stream configuration @@ -480,9 +527,12 @@ path = "partitions" enforce_fsync = false # Enables checksum validation for data integrity (boolean). -# `true` activates CRC checks when loading data, guarding against corruption. -# `false` skips these checks for faster loading at the risk of undetected corruption. -validate_checksum = false +# `true` re-hashes every batch a disk poll reads and fails the poll closed on a +# mismatch, so a segment damaged at rest is reported instead of served. +# `false` skips the re-hash and serves whatever decodes, which hands a consumer +# bytes provably not the ones written. Only turn it off with a corruption guard +# somewhere else in the stack. +validate_checksum = true # The count threshold of buffered messages before triggering a save to disk. # Together with `size_of_messages_required_to_save` it defines the threshold. @@ -502,8 +552,11 @@ size_of_messages_required_to_save = "1 MiB" # Example: if `size` is set "1GiB", the actual segment size may be 1GiB + the size of remaining messages in received batch. # Maximum size is 1 GiB. Size has to be a multiple of 512 B. size = "1 GiB" +# Reserves segment space in advance when supported by the local filesystem. +preallocate = true # Configures whether expired segments are archived (boolean) or just deleted without archiving. +# Unsupported: setting this to `true` aborts boot. archive_expired = false # Controls whether to cache indexes (time and positional) for segment access. @@ -511,6 +564,8 @@ archive_expired = false # - "true" or "all": keeps indexes in memory, speeding up data retrieval at the cost of memory # - "open_segment": keeps indexes in memory only for the currently open segment # - "false" or "none": reads indexes from disk, which can conserve memory at the cost of access speed +# TODO(hubcio): inert in the server, which picks its own index residency. Boot +# warns when set. cache_indexes = "open_segment" # Message deduplication configuration @@ -518,6 +573,7 @@ cache_indexes = "open_segment" # Controls whether message deduplication is enabled (boolean). # `true` activates deduplication, ignoring messages with duplicate IDs. # `false` treats each message as unique, even if IDs are duplicated. +# Unsupported: setting this to `true` aborts boot. enabled = false # Maximum number of ID entries in the deduplication cache (u64). max_entries = 10000 @@ -527,6 +583,7 @@ expiry = "1 m" # Recovery configuration in case of lost data [system.recovery] # Controls whether streams/topics/partitions should be recreated if the expected data for existing state is missing (boolean). +# Unsupported: setting this to `true` aborts boot. recreate_missing_state = false # Memory pool configuration @@ -560,26 +617,187 @@ enabled = false # This prevents accidental cross-cluster communication. name = "iggy-cluster" -# Full roster of cluster members. This list must be byte-identical on every -# node so operators can ship a single config.toml. The running node's -# identity is resolved at launch from the '--replica-id ' CLI flag, -# which selects the entry in this list that describes the current node. -# All other entries are treated as remote peers. +# Backup-side liveness window for a consensus plane's primary (duration). +# A replica that sees no primary traffic for this long starts a view change. +# Raise it on oversubscribed hosts where scheduling stalls fake primary +# death. Must be at least "2s" and at least 4x commit_broadcast_interval: the +# primary signals liveness through its commit broadcast, and the window must +# span several broadcasts so one delayed broadcast never trips an election. +heartbeat_timeout = "5s" + +# How often the primary broadcasts its commit point to every backup (duration). +# This is the cluster's liveness signal: each broadcast resets every backup's +# heartbeat_timeout window and carries the latest commit point forward. Must be +# nonzero and, with heartbeat_timeout, satisfy heartbeat_timeout >= 4x this +# value. Drives the consensus CommitMessage timer. +commit_broadcast_interval = "500ms" + +# How often the primary retransmits prepares that backups have not yet acked +# (duration). Lower values recover faster from a dropped prepare at the cost of +# more replica traffic; must be nonzero. Drives the consensus Prepare timer. +prepare_retransmit_interval = "250ms" + +# How often a replica retransmits its StartViewChange / DoViewChange while a +# view change is in progress (duration). Lower values converge a healthy +# election faster at the cost of more replica traffic; must be nonzero. Drives +# both consensus view-change retransmit timers. +view_change_retransmit_interval = "500ms" + +# Backstop for a stalled view change (duration): one that does not conclude +# within this window escalates to a fresh cluster-wide election. Must be nonzero +# and at least 4x view_change_retransmit_interval, so a few dropped view-change +# messages retransmit rather than prematurely escalate. +view_change_status_timeout = "5s" + +# How often a recovering or view-change backup re-requests the current view's +# StartView from its primary (duration); must be nonzero. Drives the consensus +# RequestStartView timer. +request_start_view_retransmit_interval = "1s" + +# How many consecutive unanswered RequestStartView probes a recovering replica +# tolerates before falling back to an election (integer). A full-cluster restart +# leaves nobody settled to answer, so the replica elects on its recovered log. +# Must be between 1 and 100. +view_probe_attempts_max = 5 + +# How long a stalled journal-repair stream waits before re-requesting its +# remaining window from the serving peer (duration). Repair frames are +# fire-and-forget over the lossy bus, so a session with no retry wedges forever +# on a single dropped frame. Paces both the metadata and partition repair loops; +# must be nonzero. +repair_retry_interval = "1s" + +# Prepares a peer serves per repair round before the requester walks to the next +# chunk (integer). Each frame rides the per-peer message-bus queue, so this must +# stay strictly below message_bus.peer_queue_capacity or a full round overruns +# the queue and drops frames. Must be > 0 and <= 1024. +repair_chunk_max = 128 + +# Replica-to-replica authentication (PSK + BLAKE3 keyed-MAC handshake). +[cluster.auth] +# When true, every replica peer must complete the authenticated handshake or be +# rejected, and shared_secret becomes mandatory. Off by default = legacy +# unauthenticated replica traffic. Enabling it is a coordinated-restart change. +# With http enabled and no http.jwt secrets configured, the PSK also becomes +# the JWT key source, making bearers valid cluster-wide and activating +# follower-to-primary HTTP forwarding. +enabled = false + +# Cluster-wide pre-shared key, >= 32 bytes of CSPRNG output, byte-identical on +# every node. Prefer the IGGY_CLUSTER_AUTH_SHARED_SECRET env var (masked in +# logs, never persisted) over storing it on disk. Ignored when enabled = false. +shared_secret = "" + +# Retiring pre-shared key, accepted for verification only during a rolling key +# rotation (this node keeps signing with shared_secret). Rotate in three rolls: +# 1) shared_secret = old + previous_shared_secret = new on every node, +# 2) shared_secret = new + previous_shared_secret = old on every node, +# 3) shared_secret = new alone. Leave empty outside a rotation. Same length +# floor and env-var preference as shared_secret +# (IGGY_CLUSTER_AUTH_PREVIOUS_SHARED_SECRET). +previous_shared_secret = "" + +# Replica-to-replica TLS for the consensus (tcp_replica) port. +[cluster.tls] +# When true every replica connection is wrapped in TLS 1.3 (ALPN +# "iggy-replica") before the replica handshake runs. Requires +# cluster.auth.enabled: TLS carries no client certificates, so it +# authenticates the acceptor only; the PSK handshake authenticates the +# peer, TLS supplies confidentiality. Off by default = plaintext replica +# traffic. Enabling it is a coordinated-restart change: a TLS dialer +# cannot talk to a plaintext acceptor or vice versa. +enabled = false + +# When true the node auto-generates a self-signed certificate at boot and +# the dialer accepts ANY peer certificate. When false (default), +# cert_file / key_file / ca_file are all required. +self_signed = false + +# PEM certificate chain presented by this node's acceptor side. +cert_file = "" + +# PEM private key matching cert_file. +key_file = "" + +# PEM trust anchor(s) the dialer verifies peer certificates against. +# Unused when self_signed = true. +ca_file = "" + +# Full roster of cluster members. Byte-identical on every node. The running +# node's identity is resolved at launch from the '--replica-id ' CLI +# flag, which selects the entry in this list that describes the current +# node. All other entries are remote peers. +# +# 'ip' is the node's roster address. Replica-to-replica traffic and +# follower-to-primary HTTP forwarding use it. It is not the bind interface for +# tcp/quic/http/websocket, which comes from each transport's own 'address' +# setting above; the roster supplies those transports their port only. A +# cluster spread across hosts therefore needs each transport's 'address' set to +# '0.0.0.0' or the routable NIC; the defaults below listen on loopback only, +# and a bind that cannot serve the advertised 'ip' is warned about at startup. +# +# Each node may also set 'advertised_address': the client-facing address +# handed out in cluster metadata and leader redirects. Set it when 'ip' is +# a private replica-network address unreachable by clients (Docker, +# Kubernetes, NAT). Accepts a literal IPv4/IPv6 address or a DNS hostname +# (RFC 1123: ASCII letters, digits, '-' and '.'; no port, no trailing dot). +# When unset, clients receive 'ip'. +# +# When different client networks need different addresses (a public +# 'advertised_address' would route in-VPC clients out through the public +# side), add per-network 'advertised_addresses' selectors: clients whose +# peer IP falls inside 'client_cidr' are handed 'address' instead of the +# catch-all. 'address' takes the same forms as 'advertised_address' +# (literal IP or RFC 1123 hostname, never a port - ports always come from +# 'ports'). At most 16 selectors per node; boot also rejects duplicate +# 'client_cidr' entries on one node (compared truncated, so '10.0.1.0/16' +# duplicates '10.0.0.0/16') and any two nodes advertising one host:port +# to overlapping client sets - reusing a host:port across nodes is legal +# only when no client would resolve both nodes to it. +# +# The longest matching prefix wins; clients matching no selector fall +# back to 'advertised_address', then 'ip'. Matching is per address +# family: '0.0.0.0/0' matches no IPv6 client and '::/0' matches no IPv4 +# client, so covering both families takes one selector per family (or the +# catch-all). IPv4-mapped IPv6 CIDRs ('::ffff:10.0.0.0/104') match like +# their IPv4 form only at prefix length 96 or longer; shorter ones match +# native IPv6 clients only. Matching sees the transport-level peer +# address, so clients behind a proxy or load balancer match the proxy's +# network, not their own. # -# Requirements (enforced at startup): -# - replica_id values must be unique and strictly less than nodes.len(), -# - replica_id for this node must be set via `--replica-id ` command line argument -# - node names and IPs must be non-empty -# - (ip, port) pairs must be unique across the list +# Every 'address' must be routable from inside its own 'client_cidr': +# leader-aware SDK clients redial whatever address metadata advertises, +# so a selector pointing at a host its own clients cannot reach strands +# them mid-redirect. Prefer literal IPs over hostnames - the SDKs differ +# in how they compare an advertised hostname against the address they +# dialed, and a mismatch costs a reconnect on every fresh connect. # -# Each field in 'ports' is optional. If omitted, the transport's primary -# port (tcp.address, quic.address, etc.) configured on the running node is -# used as the fallback when computing peer endpoints for cluster metadata. +# Note for rolling upgrades: older server binaries reject a TOML config +# containing 'advertised_addresses' but silently ignore the equivalent +# 'IGGY_CLUSTER_NODES_*_ADVERTISED_ADDRESSES_*' env vars; either way, +# upgrade every binary first, then add selectors. Mid-upgrade, an env-var +# roster would serve selector addresses from upgraded nodes and the +# catch-all from the rest. +# +# [[cluster.nodes]] +# name = "iggy-node-1" +# ip = "10.0.1.5" # replica plane + last-resort fallback +# advertised_address = "203.0.113.10" # catch-all for unmatched clients +# replica_id = 0 +# ports = { tcp = 8090, http = 3000, tcp_replica = 9090 } +# +# [[cluster.nodes.advertised_addresses]] +# client_cidr = "10.0.0.0/16" # in-VPC clients stay private +# address = "10.0.1.5" +# +# In cluster mode, 'ports' is the single source of listener ports: every +# enabled transport needs an explicit per-node port, otherwise the server +# refuses to start. [[cluster.nodes]] name = "iggy-node-1" ip = "127.0.0.1" replica_id = 0 -ports = { tcp = 8090, quic = 8080, http = 3000, websocket = 8070, tcp_replica = 9090 } +ports = { tcp = 8090, quic = 8080, http = 3000, websocket = 8092, tcp_replica = 9090 } [[cluster.nodes]] name = "iggy-node-2" @@ -587,12 +805,14 @@ ip = "127.0.0.1" replica_id = 1 ports = { tcp = 8091, quic = 8081, http = 3001, websocket = 8093, tcp_replica = 9091 } -# Example additional node (commented out): +# Example additional node (commented out). tcp skips 8092-8094: those are the +# websocket ports of the three nodes, which collide once nodes share a host. # [[cluster.nodes]] # name = "iggy-node-3" # ip = "192.168.1.100" +# advertised_address = "iggy-node-3.example.com" # replica_id = 2 -# ports = { tcp = 8092, http = 3002 } +# ports = { tcp = 8095, quic = 8082, http = 3002, websocket = 8094, tcp_replica = 9092 } # Sharding configuration [system.sharding] @@ -605,6 +825,7 @@ ports = { tcp = 8091, quic = 8081, http = 3001, websocket = 8093, tcp_replica = # + "numa:auto": Use all available numa node, cores # + "numa:nodes=0,1;cores=4;no_ht=true": Use NUMA node 0 and 1, each nodes use 4 cores, and no hyperthreads cpu_allocation = "numa:auto" + # Whether shard threads are pinned to dedicated CPU cores (default: true). # Pinned cores are drawn from the process's allowed CPU set (affinity/cpuset # mask), so the server cooperates with systemd `AllowedCPUs=` and container @@ -614,12 +835,202 @@ cpu_allocation = "numa:auto" # process onto the same low-numbered cores. pin_cores = true +# Per-shard inter-shard inbox capacity. Bounded by design: consensus-frame +# drops recover via VSR retransmit, but cross-shard client-reply drops are +# terminal. Size for the worst-case sum of both: the consensus working set +# (~ the prepare queue depth of the planes the shard hosts - [metadata] on +# shard 0, [partition] elsewhere - times replica_count times directions) plus +# peak client-reply fan-out per shard. Both depths are tunable, so raising +# either raises the capacity needed here. +inbox_capacity = 1024 + +# Wall-clock budget for a single shard's bus drain on shutdown. Drives +# the per-shard watchdog and the parallel-join survivor path; sized +# larger than typical TCP RTT times in-flight write-batch so writers +# receive their full last `write_vectored_all` budget before the +# connection registry force-tears the bus. Slow-fsync hosts may need +# to extend this past the default. +shutdown_drain_timeout = "10 s" + +# Poll cadence for the cross-thread shutdown flag and for the +# metadata-handoff loops. Trades off Ctrl-C latency against idle wakeup +# cost; the default keeps shutdown observably prompt without measurable +# scheduler overhead. Must be less than or equal to shutdown_drain_timeout. +shutdown_poll_interval = "50 ms" + +# Hard wall-clock deadline for joining shard threads at process exit. A +# shard whose pump or listener wedges past this budget is abandoned with +# an error log instead of blocking exit forever. Must be at least +# shutdown_drain_timeout, or shards would be abandoned mid-drain. +shutdown_join_timeout = "30 s" + +# Safety-tick cadence for the partition reconciliation loop. The reconciler +# also wakes on every metadata commit from shard 0, so this only covers +# dropped wake-ups and the initial post-bootstrap convergence window. +reconcile_periodic_interval = "1 s" + +# WebSocket listener configuration. The frame-tuning knobs below are the +# live source for the server's WS / WSS plane; they are folded into a +# compio-ws WebSocketConfig once at bus construction. Each size knob is +# optional: commenting it out keeps the compio-ws (tungstenite) default +# noted next to it. A malformed size string fails config load. [websocket] enabled = true address = "127.0.0.1:8092" +# Target minimum size of the frame read buffer. compio-ws default: "128 KiB". +# read_buffer_size = "128 KiB" + +# Target buffer size for batched writes before flush. compio-ws +# default: "128 KiB". +# write_buffer_size = "128 KiB" + +# Hard ceiling on the write buffer; writes past it error instead of +# buffering, so it must exceed write_buffer_size by at least one message. +# compio-ws default: unlimited. +# max_write_buffer_size = "128 MiB" + +# Hard upper bound on a single inbound WebSocket message +# (post-fragment-reassembly). Must not exceed message_bus.max_message_size. +# compio-ws default: "64 MiB". +# max_message_size = "64 MiB" + +# Hard upper bound on a single inbound WebSocket frame +# (pre-fragment-reassembly). Must not exceed max_message_size. +# compio-ws default: "16 MiB". +# max_frame_size = "16 MiB" + +# Whether to accept unmasked frames from clients in violation of +# RFC 6455 client-to-server framing rules. Strict (false) by default. +accept_unmasked_frames = false + [websocket.tls] enabled = false self_signed = true cert_file = "core/certs/iggy_cert.pem" key_file = "core/certs/iggy_key.pem" + +# Metadata consensus plane tunables (shard 0's VSR replica: users, +# streams, topics, sessions). Size these together: a deeper prepare queue +# admits more concurrent in-flight metadata ops (e.g. login storms), and +# the journal must hold enough slots that a forced checkpoint (triggered +# when remaining slots fall to the checkpoint margin, which itself is +# max(64, prepare_queue_depth)) stays rare. Validation enforces +# journal_slots >= 4 * max(64, prepare_queue_depth). +[metadata] +# Depth of the metadata prepare queue: how many uncommitted metadata ops +# may be in flight at once. Submits beyond it are rejected with the +# transient "metadata prepare queue is full" and retried by the SDK. +# Capped at 127 by the view-change wire format: a DoViewChange describes the +# uncommitted suffix with one nack bit and one present bit per entry in a u128 each, +# so a deeper queue produces entries a view change can neither adopt nor prove dead. +prepare_queue_depth = 32 + +# Size of the metadata WAL's in-memory index, in slots (one committed but +# not-yet-snapshotted op per slot). Larger values buy more headroom +# between forced checkpoints at the cost of memory and bigger WAL +# rewrites per checkpoint. +journal_slots = 1024 + +# Slot count of the VSR client table: how many distinct clients (TCP/QUIC/WS +# virtual clients and HTTP sessions together) hold live session state at once. +# When full, the client whose last commit is oldest is evicted and its next +# request re-registers. The HTTP session cap tracks this at half, so raising +# it lifts both. Must be between 2 and 65536. +clients_table_max = 8192 + +# Per-partition consensus plane tunables. Unlike [metadata] (one shard-0 +# plane), a pipeline exists per partition, so raising this multiplies pinned +# request-buffer memory by the partition count. Keep it modest. +[partition] +# Depth of a partition's prepare queue: how many uncommitted produce / +# consumer-offset ops may be in flight at once for that partition. Submits past +# it spill into a request queue of twice this depth; once both are full the +# server drops the request without a reply and the client retries on its own +# request timeout. Must be > 0 and <= 127: the ceiling is the view-change wire, not +# memory. A DoViewChange describes the uncommitted suffix with one bit per op in a +# u128 bitset, and this depth bounds that suffix. +prepare_queue_depth = 32 + +# Entries the evicted ring retains per multi-replica partition for journal +# repair after a peer rejoins. Larger widens the window a restarting peer can be +# served from the ring before falling back to bulk sync, at the cost of pinned +# memory per partition. Must be > 0 and <= 65536. Single-replica partitions +# retain nothing regardless. +evicted_ring_capacity = 4096 + +# Byte ceiling for the evicted ring per partition; whichever ring cap (this or +# evicted_ring_capacity) trips first evicts. Bounds the ring memory a burst of +# large batches can pin. Must be > 0 and <= "256 MiB". +evicted_ring_bytes_max = "16 MiB" + +# Byte budget for segment payloads a SERVING shard keeps resident to answer +# state-transfer chunk requests. PER SHARD, and shard count defaults to core +# count, so the process-wide high-water is this times the core count on top of +# page cache -- keep that product in mind before raising it. The default is a +# FIXED 2176 MiB: two sealed segments at the SHIPPED system.segment.size of +# 1 GiB, each of which can close one whole message_bus.max_message_size past +# its target, which is why it is not 2 GiB. It does not track your segment +# size. How many groups this shard serves at once IS derived from yours: +# floor(this / max(partition.transfer_artifact_bytes_max, +# system.segment.size + 64 MiB)), minimum one. So raising either that knob or +# system.segment.size without raising this lowers concurrency and can take it +# to one, serialising rejoins, and nothing at boot warns about it. +# Below one segment a single rejoining node thrashes the cache by itself and +# every miss re-reads and re-hashes a whole segment to serve one 256 KiB chunk. +# Running under the budget costs re-reads, not failures. +# Must be > 0 and <= "64 GiB". +transfer_served_cache_bytes_max = "2176 MiB" + +# Alloc ceiling for ONE received state-transfer artifact, per shard. The +# receiver holds it resident through verify, walk and staging write, and up to +# four transfers run at once. MUST cover system.segment.size plus +# message_bus.max_message_size (a segment may close one whole batch past its +# cap): under that, a legal segment is refused, the whole manifest with it, and +# the partition livelocks re-requesting it from every peer. Boot validates the +# floor. Raising this above the floor for headroom also DIVIDES the serving +# concurrency derived from transfer_served_cache_bytes_max above, so raise that +# in step. Must be > 0 and <= "64 GiB". +transfer_artifact_bytes_max = "1088 MiB" + +# Message bus configuration. +# Tunables for the inter-shard / inter-replica internal bus that ships +# consensus traffic between replicas and SDK-client traffic between +# shards. These knobs are consensus-liveness-critical (max_batch gates +# throughput under backpressure). Defaults match +# core::message_bus::config::MessageBusConfig::default(). + +[message_bus] +# Maximum number of BusMessage entries coalesced into a single writev(2) +# call. Hard upper bound: IOV_MAX/2 = 512 on Linux. +max_batch = 256 + +# Wire-level cap on a single framed message. +max_message_size = "64 MiB" + +# Bound on the per-peer mpsc queue. The writer task drains; the +# send_to_* path enqueues. +peer_queue_capacity = 256 + +# Interval between outbound reconnect attempts to peers with peer_id > self_id. +reconnect_period = "5 s" + +# Timeout for per-peer close drain (flush writer, tear down reader) +# before force-cancellation. +close_peer_timeout = "2 s" + +# Wall-clock bound on a single stream.shutdown() / ws.close() in the +# safe-shutdown sequence of the TLS-family transports. +close_grace = "2 s" + +# Wall-clock bound on a single connection's handshake phase. Threaded +# into compio::time::timeout(handshake_grace, ...) at each accept site +# (TCP-TLS rustls accept, WS HTTP-Upgrade, WSS combined TLS+WS, QUIC +# connecting.await + accept_bi.await) so a slowloris peer cannot pin +# per-conn channels + registry slot + spawned task indefinitely. +handshake_grace = "10 s" + +[extra.namespace] +max_streams = 4096 +max_topics = 4096 +max_partitions = 1_000_000 diff --git a/core/server/server.http b/core/server/server.http index 451bca8dd0..b0be45ca82 100644 --- a/core/server/server.http +++ b/core/server/server.http @@ -34,14 +34,11 @@ @user1_username = user1 @user1_password = secret @access_token = secret -@root_id = 1 -@user1_id = 2 +@root_id = 0 +@user1_id = 1 @pat_name = dev_token @pat_raw_token = secret -### -GET {{url}} - ### GET {{url}}/ping @@ -63,11 +60,18 @@ Content-Type: application/json } ### -GET {{url}}/metrics +GET {{url}}/stats +Authorization: Bearer {{access_token}} ### -GET {{url}}/stats +POST {{url}}/snapshot Authorization: Bearer {{access_token}} +Content-Type: application/json + +{ + "compression": "Deflated", + "snapshot_types": ["All"] +} ### GET {{url}}/cluster/metadata @@ -81,14 +85,6 @@ Authorization: Bearer {{access_token}} GET {{url}}/clients/{{client_id}} Authorization: Bearer {{access_token}} -### -POST {{url}}/users/refresh-token -Content-Type: application/json - -{ - "token": "{{access_token}}" -} - ### DELETE {{url}}/users/logout Authorization: Bearer {{access_token}} @@ -154,7 +150,7 @@ Content-Type: application/json "send_messages": true }, "streams": { - "1": { + "0": { "manage_stream": false, "read_stream": true, "manage_topics": false, @@ -162,7 +158,7 @@ Content-Type: application/json "poll_messages": true, "send_messages": true, "topics": { - "1": { + "0": { "manage_topic": false, "read_topic": true, "poll_messages": true, @@ -193,9 +189,6 @@ Content-Type: application/json "expiry": 1000 } -### - - ### DELETE {{url}}/personal-access-tokens/{{pat_name}} Authorization: Bearer {{access_token}} @@ -290,7 +283,7 @@ Authorization: Bearer {{access_token}} ### ### Delete segments -DELETE {{url}}/streams/1/topics/1/partitions/1?segments_count=3 +DELETE {{url}}/streams/{{stream_id}}/topics/{{topic_id}}/partitions/{{partition_id}}?segments_count=3 Authorization: Bearer {{access_token}} ### diff --git a/core/server/src/args.rs b/core/server/src/args.rs index ad63aed19c..665a11c85c 100644 --- a/core/server/src/args.rs +++ b/core/server/src/args.rs @@ -22,10 +22,11 @@ use clap::Parser; author = "Apache Iggy (Incubating)", version, about = "Apache Iggy: Hyper-Efficient Message Streaming at Laser Speed", - long_about = r#"Apache Iggy (Incubating) - A persistent message streaming platform written in Rust + long_about = r#"Apache Iggy (Incubating) - a persistent message streaming platform written in Rust -Apache Iggy is a high-performance message streaming platform that supports QUIC, TCP, and HTTP -transport protocols, capable of processing millions of messages per second with low latency. +Iggy stores every stream in a replicated log kept consistent by Viewstamped +Replication. One binary serves both the single-node and the clustered +deployment; the loaded configuration decides which one you get. WEBSITE: https://iggy.apache.org @@ -37,102 +38,89 @@ DOCUMENTATION: https://iggy.apache.org/docs CONFIGURATION: - The server uses a TOML configuration file. By default, it looks for 'core/server/config.toml' - in the current working directory. You can override this with the IGGY_CONFIG_PATH environment - variable or use the --config-provider flag. + The server reads a TOML configuration file, by default 'core/server/config.toml' + resolved against the current working directory. Point IGGY_CONFIG_PATH at + another file to override it. Examples: - iggy-server # Uses default file provider (core/server/config.toml) - iggy-server --config-provider file # Explicitly use file provider - IGGY_CONFIG_PATH=custom.toml iggy-server # Use custom config file path + iggy-server # Default config file + IGGY_CONFIG_PATH=custom.toml iggy-server # Custom config file path ENVIRONMENT VARIABLES: - Any configuration value can be overridden using environment variables with the IGGY_ prefix. - Use underscores to separate nested configuration keys (e.g., IGGY_TCP_ADDRESS=127.0.0.1:8090). + Any configuration value can be overridden with an IGGY_ prefixed variable; + underscores separate the nested keys (IGGY_TCP_ADDRESS sets [tcp] address). + A '.env' file in the working directory is loaded during startup, or the one + named by IGGY_ENV_PATH. Common examples: - IGGY_TCP_ADDRESS=0.0.0.0:8090 # Override TCP server address - IGGY_HTTP_ENABLED=true # Enable HTTP transport - IGGY_SYSTEM_PATH=/data/iggy # Set data storage path - IGGY_SYSTEM_LOGGING_LEVEL=debug # Set log level to debug + IGGY_SYSTEM_PATH=/data/iggy # Data directory + IGGY_TCP_ADDRESS=0.0.0.0:8090 # TCP listener address + IGGY_HTTP_ADDRESS=0.0.0.0:3000 # HTTP listener address + IGGY_SYSTEM_LOGGING_LEVEL=debug # Log level + IGGY_ROOT_USERNAME=iggy # Root user, set with the password + IGGY_ROOT_PASSWORD=secret # Root password, set with the username TRANSPORT PROTOCOLS: - - TCP (binary protocol): High-performance, low-latency (default: 127.0.0.1:8090) - - QUIC: Modern UDP-based protocol with built-in encryption (default: 127.0.0.1:8080) - - HTTP: RESTful API for web integration (default: 127.0.0.1:3000, disabled by default) + - TCP (binary protocol) (default: 127.0.0.1:8090) + - QUIC (default: 127.0.0.1:8080) + - WebSocket (default: 127.0.0.1:8092) + - HTTP (REST API) (default: 127.0.0.1:3000) GETTING STARTED: - 1. Start the server: iggy-server - 2. Install CLI: cargo install iggy-cli - 3. Create a stream: iggy stream create my-stream - 4. Create a topic: iggy topic create my-stream my-topic 1 none - 5. Send messages: echo "Hello, Iggy!" | iggy message send my-stream my-topic + 1. Start the server: iggy-server --fresh --with-default-root-credentials + 2. Install the CLI: cargo install iggy-cli + 3. Create a stream: iggy stream create my-stream + 4. Create a topic: iggy topic create my-stream my-topic 1 none + 5. Send messages: echo "Hello, Iggy!" | iggy message send my-stream my-topic + +CLUSTER: + Every node runs the same configuration file with cluster.enabled = true and + is told apart only by --replica-id, which selects its own cluster.nodes entry: + + iggy-server --replica-id 0 For more information, visit: https://iggy.apache.org/docs/introduction/getting-started/"# )] +// These doc comments are rendered verbatim as `--help` output, so environment +// variable names and paths must stay unquoted rather than wear rustdoc backticks. +#[allow(clippy::doc_markdown)] pub struct Args { - /// Configuration provider type + /// Remove the system path before starting (WARNING: THIS WILL DELETE ALL DATA!) /// - /// Currently only 'file' provider is supported, which loads configuration from a TOML file. - /// The file path can be specified via IGGY_CONFIG_PATH environment variable. - #[arg(short, long, default_value = "file", verbatim_doc_comment)] - pub config_provider: String, - - /// Remove system path before starting (WARNING: THIS WILL DELETE ALL DATA!) + /// Deletes the configured system data directory ('local_data' by default, + /// see IGGY_SYSTEM_PATH) before the server boots, so it starts on empty + /// state. Intended for clean development setups and testing. /// - /// This flag will completely remove the system data directory (local_data by default) - /// before starting the server. Use this for clean development setups or testing. + /// In cluster mode this wipes THIS replica only; it rejoins and refills by + /// state transfer from the others. Wiping a quorum at the same time destroys + /// committed data, and a service unit file carrying --fresh re-transfers the + /// whole dataset on every restart. /// /// Examples: - /// iggy-server --fresh # Start with fresh data directory - /// iggy-server -f # Short form + /// iggy-server --fresh # Start with a fresh data directory + /// iggy-server -f # Short form #[arg(short, long, default_value_t = false, verbatim_doc_comment)] pub fresh: bool, /// Use default root credentials (INSECURE - FOR DEVELOPMENT ONLY!) /// - /// When this flag is set, the root user will be created with username 'iggy' - /// and password 'iggy' if it doesn't exist. If the root user already exists, - /// this flag has no effect. - /// - /// This flag is equivalent to setting IGGY_ROOT_USERNAME=iggy and IGGY_ROOT_PASSWORD=iggy, - /// but environment variables take precedence over this flag. + /// Sets IGGY_ROOT_USERNAME and IGGY_ROOT_PASSWORD to 'iggy' unless they are + /// already present in the environment, so the flag is equivalent to + /// exporting both by hand and the environment always takes precedence. /// - /// WARNING: This is insecure and should only be used for development and testing! + /// Only the first creation of the root user reads these values. On an + /// existing data directory the stored root user is recovered as it is and + /// the flag has no effect. /// /// Examples: - /// iggy-server --with-default-root-credentials # Use 'iggy/iggy' as root credentials + /// iggy-server --with-default-root-credentials # Root logs in as iggy/iggy #[arg(long, default_value_t = false, verbatim_doc_comment)] pub with_default_root_credentials: bool, - /// Run server as a follower node (FOR TESTING LEADER REDIRECTION) - /// - /// When this flag is set, the server will report itself as a follower node - /// in cluster metadata responses. This is useful for testing leader-aware - /// client connections and redirection logic. - /// - /// The server will return cluster metadata showing this server as a follower node. - /// - /// Examples: - /// iggy-server # Run as leader (default) - /// iggy-server --follower # Run as follower - /// IGGY_TCP_ADDRESS=127.0.0.1:8091 iggy-server --follower # Follower on port 8091 - #[arg(long, default_value_t = false, verbatim_doc_comment)] - pub follower: bool, - /// Identifies this node within `cluster.nodes` by its replica ID. /// /// Required when `cluster.enabled = true`. The value must match exactly - /// one `cluster.nodes[*].replica_id` entry in the loaded configuration; - /// that entry describes the current node, and all other entries are - /// treated as remote peers. - /// - /// Supplying the identity on the command line lets operators ship a - /// single byte-identical `config.toml` to every node in the cluster - /// and differ only in this CLI flag. - /// - /// Examples: - /// iggy-server --replica-id 0 # This node is replica 0 + /// one `cluster.nodes[*].replica_id` entry in the loaded configuration. #[arg(long, verbatim_doc_comment)] pub replica_id: Option, } diff --git a/core/server-ng/src/auth.rs b/core/server/src/auth.rs similarity index 97% rename from core/server-ng/src/auth.rs rename to core/server/src/auth.rs index 74a38444cc..a491eb9288 100644 --- a/core/server-ng/src/auth.rs +++ b/core/server/src/auth.rs @@ -20,7 +20,7 @@ //! Verifies password + PAT credentials locally, then runs the consensus //! `Register` proposal on the metadata owner; terminal failures are //! surfaced as typed `Eviction` frames, transient ones as -//! `TransientNotCommitted` replay hints. +//! `TransientNotAccepted` replay hints. use crate::bootstrap::{ShellBus, ShellShard}; use crate::dispatch::{send_login_eviction, submit_register_on_owner}; @@ -29,7 +29,7 @@ use crate::responses::{build_login_register_reply, current_metadata_commit}; use crate::session_manager::{ClientSdkInfo, SessionManager}; use consensus::{MetadataHandle, build_result_rejection_reply}; use iggy_binary_protocol::PrepareHeader; -use iggy_binary_protocol::{ClientVersionInfo, EvictionReason, RequestHeader}; +use iggy_binary_protocol::{ClientVersionInfo, EvictionReason, RoutedRequestHeader}; use iggy_common::defaults::{ MAX_PASSWORD_LENGTH, MAX_USERNAME_LENGTH, MIN_PASSWORD_LENGTH, MIN_USERNAME_LENGTH, }; @@ -183,7 +183,7 @@ pub(crate) async fn complete_login_register( sessions: &Rc>, transport_client_id: u128, vsr_client_id: u128, - request_header: &RequestHeader, + request_header: &RoutedRequestHeader, user_id: u32, client_version: &ClientVersionInfo, ) -> Result<(), LoginRegisterError> @@ -291,7 +291,7 @@ where pub(crate) async fn surface_login_failure( shard: &Rc>, transport_client_id: u128, - request_header: &RequestHeader, + request_header: &RoutedRequestHeader, error: &LoginRegisterError, ) where B: ShellBus, @@ -310,7 +310,7 @@ pub(crate) async fn surface_login_failure( .await; } else { // Transient consensus failure (not-caught-up / not-primary / pipeline - // full): send the explicit `TransientNotCommitted` frame instead of + // full): send the explicit `TransientNotAccepted` frame instead of // staying silent, so the SDK replays the login immediately rather than // waiting out its read-timeout. Same contract as a transient metadata // request -- nothing committed, so the replayed Register is idempotent. @@ -318,7 +318,7 @@ pub(crate) async fn surface_login_failure( } } -/// Result-framed `TransientNotCommitted` Reply on a transient (non-terminal) +/// Result-framed `TransientNotAccepted` Reply on a transient (non-terminal) /// failed Register. The SDK decodes the nonzero result code and replays the /// same login on the same connection. Only call for transient errors -- see /// [`surface_login_failure`]. @@ -326,7 +326,7 @@ pub(crate) async fn surface_login_failure( async fn send_login_transient_reply( shard: &Rc>, transport_client_id: u128, - request_header: &RequestHeader, + request_header: &RoutedRequestHeader, ) where B: ShellBus, MJ: JournalHandle + 'static, diff --git a/core/server/src/binary/dispatch.rs b/core/server/src/binary/dispatch.rs deleted file mode 100644 index 2225a1a48f..0000000000 --- a/core/server/src/binary/dispatch.rs +++ /dev/null @@ -1,456 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::handlers; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::streaming::session::Session; -use bytes::BytesMut; -use iggy_binary_protocol::RequestFrame; -use iggy_binary_protocol::codec::WireDecode; -use iggy_binary_protocol::codes::*; -use iggy_binary_protocol::requests::consumer_groups::*; -use iggy_binary_protocol::requests::consumer_offsets::*; -use iggy_binary_protocol::requests::messages::*; -use iggy_binary_protocol::requests::partitions::*; -use iggy_binary_protocol::requests::personal_access_tokens::*; -use iggy_binary_protocol::requests::segments::*; -use iggy_binary_protocol::requests::streams::*; -use iggy_binary_protocol::requests::system::*; -use iggy_binary_protocol::requests::topics::*; -use iggy_binary_protocol::requests::users::*; -use iggy_common::{Consumer, ConsumerKind, Identifier, IggyError, PollingKind, PollingStrategy}; -use std::rc::Rc; -use tracing::{error, warn}; - -/// Result of handling a command. Most commands return `Finished`. -/// `SendMessages` may migrate the TCP connection to another shard. -pub enum HandlerResult { - Finished, - Migrated { to_shard: u16 }, -} - -/// Read the full payload from the sender into a buffer. -pub async fn read_payload(sender: &mut SenderKind, length: u32) -> Result { - if length > MAX_CONTROL_FRAME_PAYLOAD { - return Err(IggyError::InvalidCommand); - } - let mut buffer = BytesMut::with_capacity(length as usize); - // SAFETY: when length > 0, sender.read() fills exactly `length` bytes - // before returning Ok. On error the buffer is dropped without being read. - // When length == 0, set_len(0) is a no-op (no uninitialized bytes exposed). - unsafe { - buffer.set_len(length as usize); - } - if length > 0 { - let (result, buf) = sender.read(buffer).await; - result?; - buffer = buf; - } - Ok(buffer) -} - -fn decode(payload: &[u8]) -> Result { - use iggy_binary_protocol::error::WireError; - - let (val, consumed) = T::decode(payload).map_err(|e| { - warn!("wire decode error: {e}"); - match e { - WireError::PayloadTooLarge { .. } => IggyError::InvalidSizeBytes, - WireError::Validation(_) => IggyError::InvalidFormat, - _ => IggyError::InvalidCommand, - } - })?; - if consumed != payload.len() { - warn!( - "wire decode: {} trailing bytes (consumed {consumed}, payload {})", - payload.len() - consumed, - payload.len() - ); - } - Ok(val) -} - -/// Convert a `WireIdentifier` to the domain `Identifier`. -pub fn wire_id_to_identifier( - wire: &iggy_binary_protocol::WireIdentifier, -) -> Result { - match wire { - iggy_binary_protocol::WireIdentifier::Numeric(id) => Identifier::numeric(*id), - iggy_binary_protocol::WireIdentifier::String(name) => Identifier::named(name.as_str()), - } -} - -/// Convert a `WireConsumer` to the domain `Consumer`. -pub fn wire_consumer_to_consumer( - wire: &iggy_binary_protocol::WireConsumer, -) -> Result { - let id = wire_id_to_identifier(&wire.id)?; - let kind = ConsumerKind::from_code(wire.kind)?; - Ok(Consumer { kind, id }) -} - -/// Convert a `WirePollingStrategy` to the domain `PollingStrategy`. -pub fn wire_polling_to_strategy( - wire: &iggy_binary_protocol::WirePollingStrategy, -) -> Result { - Ok(PollingStrategy { - kind: PollingKind::from_code(wire.kind)?, - value: wire.value, - }) -} - -/// Maximum payload size for control-plane commands (non-SendMessages). -/// Prevents OOM from malicious clients sending `length = u32::MAX`. -/// SendMessages has its own size validation via `total_payload_size` checks. -pub const MAX_CONTROL_FRAME_PAYLOAD: u32 = 10 * 1024 * 1024; // 10 MB - -/// Dispatch a SendMessages command with staged socket reads (zero-copy path). -/// -/// Called by transport layers when the command code is `SEND_MESSAGES_CODE`. -/// The handler reads metadata, indexes, and messages directly from the socket -/// into separate `PooledBuffer`s for zero-copy partition append. -pub async fn dispatch_send_messages( - sender: &mut SenderKind, - payload_length: u32, - session: &Session, - shard: &Rc, -) -> Result { - handlers::messages::send_messages_handler::handle_send_messages( - sender, - payload_length, - session, - shard, - ) - .await -} - -/// Central command dispatch for a decoded request frame. -/// -/// Transport layers read the 8-byte header, validate via -/// `RequestFrame::payload_length()`, read the full payload, construct a -/// `RequestFrame::from_parts(code, frame.payload)`, and pass it here. -/// -/// SendMessages is handled separately via `dispatch_send_messages()`. -#[allow(clippy::too_many_lines)] -pub async fn dispatch( - frame: RequestFrame<'_>, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - match frame.code { - // System - PING_CODE => { - handlers::system::ping_handler::handle_ping(sender, session, shard).await - } - GET_STATS_CODE => { - handlers::system::get_stats_handler::handle_get_stats(sender, session, shard).await - } - GET_ME_CODE => { - handlers::system::get_me_handler::handle_get_me(sender, session, shard).await - } - GET_CLIENT_CODE => { - let req: GetClientRequest = decode(frame.payload)?; - handlers::system::get_client_handler::handle_get_client(req, sender, session, shard) - .await - } - GET_CLIENTS_CODE => { - handlers::system::get_clients_handler::handle_get_clients(sender, session, shard).await - } - GET_SNAPSHOT_FILE_CODE => { - let req: GetSnapshotRequest = decode(frame.payload)?; - handlers::system::get_snapshot_handler::handle_get_snapshot( - req, sender, session, shard, - ) - .await - } - GET_CLUSTER_METADATA_CODE => { - handlers::cluster::get_cluster_metadata_handler::handle_get_cluster_metadata( - sender, session, shard, - ) - .await - } - - // Streams - GET_STREAM_CODE => { - let req: GetStreamRequest = decode(frame.payload)?; - handlers::streams::get_stream_handler::handle_get_stream(req, sender, session, shard) - .await - } - GET_STREAMS_CODE => { - handlers::streams::get_streams_handler::handle_get_streams(sender, session, shard) - .await - } - CREATE_STREAM_CODE => { - let req: CreateStreamRequest = decode(frame.payload)?; - handlers::streams::create_stream_handler::handle_create_stream( - req, sender, session, shard, - ) - .await - } - DELETE_STREAM_CODE => { - let req: DeleteStreamRequest = decode(frame.payload)?; - handlers::streams::delete_stream_handler::handle_delete_stream( - req, sender, session, shard, - ) - .await - } - UPDATE_STREAM_CODE => { - let req: UpdateStreamRequest = decode(frame.payload)?; - handlers::streams::update_stream_handler::handle_update_stream( - req, sender, session, shard, - ) - .await - } - PURGE_STREAM_CODE => { - let req: PurgeStreamRequest = decode(frame.payload)?; - handlers::streams::purge_stream_handler::handle_purge_stream( - req, sender, session, shard, - ) - .await - } - - // Topics - GET_TOPIC_CODE => { - let req: GetTopicRequest = decode(frame.payload)?; - handlers::topics::get_topic_handler::handle_get_topic(req, sender, session, shard) - .await - } - GET_TOPICS_CODE => { - let req: GetTopicsRequest = decode(frame.payload)?; - handlers::topics::get_topics_handler::handle_get_topics(req, sender, session, shard) - .await - } - CREATE_TOPIC_CODE => { - let req: CreateTopicRequest = decode(frame.payload)?; - handlers::topics::create_topic_handler::handle_create_topic( - req, sender, session, shard, - ) - .await - } - DELETE_TOPIC_CODE => { - let req: DeleteTopicRequest = decode(frame.payload)?; - handlers::topics::delete_topic_handler::handle_delete_topic( - req, sender, session, shard, - ) - .await - } - UPDATE_TOPIC_CODE => { - let req: UpdateTopicRequest = decode(frame.payload)?; - handlers::topics::update_topic_handler::handle_update_topic( - req, sender, session, shard, - ) - .await - } - PURGE_TOPIC_CODE => { - let req: PurgeTopicRequest = decode(frame.payload)?; - handlers::topics::purge_topic_handler::handle_purge_topic( - req, sender, session, shard, - ) - .await - } - - // Partitions - CREATE_PARTITIONS_CODE => { - let req: CreatePartitionsRequest = decode(frame.payload)?; - handlers::partitions::create_partitions_handler::handle_create_partitions( - req, sender, session, shard, - ) - .await - } - DELETE_PARTITIONS_CODE => { - let req: DeletePartitionsRequest = decode(frame.payload)?; - handlers::partitions::delete_partitions_handler::handle_delete_partitions( - req, sender, session, shard, - ) - .await - } - - // Segments - DELETE_SEGMENTS_CODE => { - let req: DeleteSegmentsRequest = decode(frame.payload)?; - handlers::segments::delete_segments_handler::handle_delete_segments( - req, sender, session, shard, - ) - .await - } - - // Messages (PollMessages + FlushUnsavedBuffer; SendMessages handled above) - POLL_MESSAGES_CODE => { - let req: PollMessagesRequest = decode(frame.payload)?; - handlers::messages::poll_messages_handler::handle_poll_messages( - req, sender, session, shard, - ) - .await - } - FLUSH_UNSAVED_BUFFER_CODE => { - let req: FlushUnsavedBufferRequest = decode(frame.payload)?; - handlers::messages::flush_unsaved_buffer_handler::handle_flush_unsaved_buffer( - req, sender, session, shard, - ) - .await - } - - // Consumer Offsets - GET_CONSUMER_OFFSET_CODE => { - let req: GetConsumerOffsetRequest = decode(frame.payload)?; - handlers::consumer_offsets::get_consumer_offset_handler::handle_get_consumer_offset( - req, sender, session, shard, - ) - .await - } - STORE_CONSUMER_OFFSET_CODE => { - let req: StoreConsumerOffsetRequest = decode(frame.payload)?; - handlers::consumer_offsets::store_consumer_offset_handler::handle_store_consumer_offset( - req, sender, session, shard, - ) - .await - } - DELETE_CONSUMER_OFFSET_CODE => { - let req: DeleteConsumerOffsetRequest = decode(frame.payload)?; - handlers::consumer_offsets::delete_consumer_offset_handler::handle_delete_consumer_offset( - req, sender, session, shard, - ) - .await - } - - // Consumer Groups - GET_CONSUMER_GROUP_CODE => { - let req: GetConsumerGroupRequest = decode(frame.payload)?; - handlers::consumer_groups::get_consumer_group_handler::handle_get_consumer_group( - req, sender, session, shard, - ) - .await - } - GET_CONSUMER_GROUPS_CODE => { - let req: GetConsumerGroupsRequest = decode(frame.payload)?; - handlers::consumer_groups::get_consumer_groups_handler::handle_get_consumer_groups( - req, sender, session, shard, - ) - .await - } - CREATE_CONSUMER_GROUP_CODE => { - let req: CreateConsumerGroupRequest = decode(frame.payload)?; - handlers::consumer_groups::create_consumer_group_handler::handle_create_consumer_group( - req, sender, session, shard, - ) - .await - } - DELETE_CONSUMER_GROUP_CODE => { - let req: DeleteConsumerGroupRequest = decode(frame.payload)?; - handlers::consumer_groups::delete_consumer_group_handler::handle_delete_consumer_group( - req, sender, session, shard, - ) - .await - } - JOIN_CONSUMER_GROUP_CODE => { - let req: JoinConsumerGroupRequest = decode(frame.payload)?; - handlers::consumer_groups::join_consumer_group_handler::handle_join_consumer_group( - req, sender, session, shard, - ) - .await - } - LEAVE_CONSUMER_GROUP_CODE => { - let req: LeaveConsumerGroupRequest = decode(frame.payload)?; - handlers::consumer_groups::leave_consumer_group_handler::handle_leave_consumer_group( - req, sender, session, shard, - ) - .await - } - - // Users - GET_USER_CODE => { - let req: GetUserRequest = decode(frame.payload)?; - handlers::users::get_user_handler::handle_get_user(req, sender, session, shard).await - } - GET_USERS_CODE => { - handlers::users::get_users_handler::handle_get_users(sender, session, shard).await - } - CREATE_USER_CODE => { - let req: CreateUserRequest = decode(frame.payload)?; - handlers::users::create_user_handler::handle_create_user(req, sender, session, shard) - .await - } - DELETE_USER_CODE => { - let req: DeleteUserRequest = decode(frame.payload)?; - handlers::users::delete_user_handler::handle_delete_user(req, sender, session, shard) - .await - } - UPDATE_USER_CODE => { - let req: UpdateUserRequest = decode(frame.payload)?; - handlers::users::update_user_handler::handle_update_user(req, sender, session, shard) - .await - } - UPDATE_PERMISSIONS_CODE => { - let req: UpdatePermissionsRequest = decode(frame.payload)?; - handlers::users::update_permissions_handler::handle_update_permissions( - req, sender, session, shard, - ) - .await - } - CHANGE_PASSWORD_CODE => { - let req: ChangePasswordRequest = decode(frame.payload)?; - handlers::users::change_password_handler::handle_change_password( - req, sender, session, shard, - ) - .await - } - LOGIN_USER_CODE => { - let req: LoginUserRequest = decode(frame.payload)?; - handlers::users::login_user_handler::handle_login_user(req, sender, session, shard) - .await - } - LOGOUT_USER_CODE => { - handlers::users::logout_user_handler::handle_logout_user(sender, session, shard).await - } - - // Personal Access Tokens - GET_PERSONAL_ACCESS_TOKENS_CODE => { - handlers::personal_access_tokens::get_personal_access_tokens_handler::handle_get_personal_access_tokens( - sender, session, shard, - ) - .await - } - CREATE_PERSONAL_ACCESS_TOKEN_CODE => { - let req: CreatePersonalAccessTokenRequest = decode(frame.payload)?; - handlers::personal_access_tokens::create_personal_access_token_handler::handle_create_personal_access_token( - req, sender, session, shard, - ) - .await - } - DELETE_PERSONAL_ACCESS_TOKEN_CODE => { - let req: DeletePersonalAccessTokenRequest = decode(frame.payload)?; - handlers::personal_access_tokens::delete_personal_access_token_handler::handle_delete_personal_access_token( - req, sender, session, shard, - ) - .await - } - LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE => { - let req: LoginWithPersonalAccessTokenRequest = decode(frame.payload)?; - handlers::personal_access_tokens::login_with_personal_access_token_handler::handle_login_with_personal_access_token( - req, sender, session, shard, - ) - .await - } - - _ => { - error!("Unknown command code: {}", frame.code); - Err(IggyError::InvalidCommand) - } - } -} diff --git a/core/server/src/binary/handlers/cluster/get_cluster_metadata_handler.rs b/core/server/src/binary/handlers/cluster/get_cluster_metadata_handler.rs deleted file mode 100644 index 7e32ae8247..0000000000 --- a/core/server/src/binary/handlers/cluster/get_cluster_metadata_handler.rs +++ /dev/null @@ -1,60 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::HandlerResult; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::streaming::session::Session; -use iggy_binary_protocol::codec::WireEncode; -use iggy_binary_protocol::responses::system::get_cluster_metadata::{ - ClusterMetadataResponse, ClusterNodeResponse, -}; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::{debug, instrument}; - -#[instrument(skip_all, name = "trace_get_cluster_metadata", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] -pub async fn handle_get_cluster_metadata( - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - debug!("session: {session}, command: get_cluster_metadata"); - shard.ensure_authenticated(session)?; - - let cluster_metadata = shard.get_cluster_metadata(); - - let response = ClusterMetadataResponse { - name: cluster_metadata.name, - nodes: cluster_metadata - .nodes - .into_iter() - .map(|node| ClusterNodeResponse { - name: node.name, - ip: node.ip, - tcp_port: node.endpoints.tcp, - quic_port: node.endpoints.quic, - http_port: node.endpoints.http, - websocket_port: node.endpoints.websocket, - role: node.role as u8, - status: node.status as u8, - }) - .collect(), - }; - sender.send_ok_response(&response.to_bytes()).await?; - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/cluster/mod.rs b/core/server/src/binary/handlers/cluster/mod.rs deleted file mode 100644 index 46214f179c..0000000000 --- a/core/server/src/binary/handlers/cluster/mod.rs +++ /dev/null @@ -1,18 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod get_cluster_metadata_handler; diff --git a/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs deleted file mode 100644 index 5fce66baaf..0000000000 --- a/core/server/src/binary/handlers/consumer_groups/create_consumer_group_handler.rs +++ /dev/null @@ -1,67 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::HandlerResult; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::shard::transmission::frame::ShardResponse; -use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; -use crate::streaming::session::Session; -use iggy_binary_protocol::WireName; -use iggy_binary_protocol::codec::WireEncode; -use iggy_binary_protocol::requests::consumer_groups::CreateConsumerGroupRequest; -use iggy_binary_protocol::responses::consumer_groups::ConsumerGroupResponse; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::{debug, instrument}; - -#[instrument(skip_all, name = "trace_create_consumer_group", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] -pub async fn handle_create_consumer_group( - req: CreateConsumerGroupRequest, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - debug!( - "session: {session}, command: create_consumer_group, stream_id: {:?}, topic_id: {:?}, name: {}", - req.stream_id, - req.topic_id, - req.name.as_str() - ); - shard.ensure_authenticated(session)?; - - let request = ShardRequest::control_plane(ShardRequestPayload::CreateConsumerGroupRequest { - user_id: session.get_user_id(), - command: req, - }); - - match shard.send_to_control_plane(request).await? { - ShardResponse::CreateConsumerGroupResponse(data) => { - let response = ConsumerGroupResponse { - id: data.id, - partitions_count: data.partitions_count, - members_count: 0, - name: WireName::new(data.name.as_ref()).map_err(|_| IggyError::InvalidCommand)?, - }; - sender.send_ok_response(&response.to_bytes()).await?; - } - ShardResponse::ErrorResponse(err) => return Err(err), - _ => unreachable!("Expected CreateConsumerGroupResponse"), - } - - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/consumer_groups/delete_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/delete_consumer_group_handler.rs deleted file mode 100644 index 059fd87301..0000000000 --- a/core/server/src/binary/handlers/consumer_groups/delete_consumer_group_handler.rs +++ /dev/null @@ -1,56 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::HandlerResult; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::shard::transmission::frame::ShardResponse; -use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; -use crate::streaming::session::Session; -use iggy_binary_protocol::requests::consumer_groups::DeleteConsumerGroupRequest; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::{debug, instrument}; - -#[instrument(skip_all, name = "trace_delete_consumer_group", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] -pub async fn handle_delete_consumer_group( - req: DeleteConsumerGroupRequest, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - debug!( - "session: {session}, command: delete_consumer_group, stream_id: {:?}, topic_id: {:?}, group_id: {:?}", - req.stream_id, req.topic_id, req.group_id - ); - shard.ensure_authenticated(session)?; - - let request = ShardRequest::control_plane(ShardRequestPayload::DeleteConsumerGroupRequest { - user_id: session.get_user_id(), - command: req, - }); - - match shard.send_to_control_plane(request).await? { - ShardResponse::DeleteConsumerGroupResponse => { - sender.send_empty_ok_response().await?; - } - ShardResponse::ErrorResponse(err) => return Err(err), - _ => unreachable!("Expected DeleteConsumerGroupResponse"), - } - - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/consumer_groups/get_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/get_consumer_group_handler.rs deleted file mode 100644 index 00ff8f58e3..0000000000 --- a/core/server/src/binary/handlers/consumer_groups/get_consumer_group_handler.rs +++ /dev/null @@ -1,78 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::{HandlerResult, wire_id_to_identifier}; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::streaming::session::Session; -use iggy_binary_protocol::WireName; -use iggy_binary_protocol::codec::WireEncode; -use iggy_binary_protocol::requests::consumer_groups::GetConsumerGroupRequest; -use iggy_binary_protocol::responses::consumer_groups::{ - ConsumerGroupDetailsResponse, ConsumerGroupMemberResponse, ConsumerGroupResponse, -}; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::debug; - -pub async fn handle_get_consumer_group( - req: GetConsumerGroupRequest, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - let stream_id = wire_id_to_identifier(&req.stream_id)?; - let topic_id = wire_id_to_identifier(&req.topic_id)?; - let group_id = wire_id_to_identifier(&req.group_id)?; - debug!( - "session: {session}, command: get_consumer_group, stream_id: {stream_id}, topic_id: {topic_id}, group_id: {group_id}" - ); - shard.ensure_authenticated(session)?; - - let Some(consumer_group) = shard.metadata.query_consumer_group( - session.get_user_id(), - &stream_id, - &topic_id, - &group_id, - )? - else { - sender.send_empty_ok_response().await?; - return Ok(HandlerResult::Finished); - }; - - let members: Vec = consumer_group - .members - .iter() - .map(|(_, member)| ConsumerGroupMemberResponse { - id: member.id as u32, - partitions_count: member.partitions.len() as u32, - partitions: member.partitions.iter().map(|&p| p as u32).collect(), - }) - .collect(); - let response = ConsumerGroupDetailsResponse { - group: ConsumerGroupResponse { - id: consumer_group.id as u32, - partitions_count: consumer_group.partitions.len() as u32, - members_count: consumer_group.members.len() as u32, - name: WireName::new(consumer_group.name.as_ref()) - .map_err(|_| IggyError::InvalidCommand)?, - }, - members, - }; - sender.send_ok_response(&response.to_bytes()).await?; - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/consumer_groups/get_consumer_groups_handler.rs b/core/server/src/binary/handlers/consumer_groups/get_consumer_groups_handler.rs deleted file mode 100644 index 5fec457f72..0000000000 --- a/core/server/src/binary/handlers/consumer_groups/get_consumer_groups_handler.rs +++ /dev/null @@ -1,69 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::{HandlerResult, wire_id_to_identifier}; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::streaming::session::Session; -use bytes::Bytes; -use iggy_binary_protocol::WireName; -use iggy_binary_protocol::codec::WireEncode; -use iggy_binary_protocol::requests::consumer_groups::GetConsumerGroupsRequest; -use iggy_binary_protocol::responses::consumer_groups::{ - ConsumerGroupResponse, GetConsumerGroupsResponse, -}; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::debug; - -pub async fn handle_get_consumer_groups( - req: GetConsumerGroupsRequest, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - let stream_id = wire_id_to_identifier(&req.stream_id)?; - let topic_id = wire_id_to_identifier(&req.topic_id)?; - debug!( - "session: {session}, command: get_consumer_groups, stream_id: {stream_id}, topic_id: {topic_id}" - ); - shard.ensure_authenticated(session)?; - - let Some(consumer_groups) = - shard - .metadata - .query_consumer_groups(session.get_user_id(), &stream_id, &topic_id)? - else { - sender.send_ok_response(&Bytes::new()).await?; - return Ok(HandlerResult::Finished); - }; - - let groups: Vec = consumer_groups - .iter() - .map(|cg| { - Ok(ConsumerGroupResponse { - id: cg.id as u32, - partitions_count: cg.partitions.len() as u32, - members_count: cg.members.len() as u32, - name: WireName::new(cg.name.as_ref()).map_err(|_| IggyError::InvalidCommand)?, - }) - }) - .collect::>()?; - let response = GetConsumerGroupsResponse { groups }; - sender.send_ok_response(&response.to_bytes()).await?; - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/consumer_groups/join_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/join_consumer_group_handler.rs deleted file mode 100644 index 658952f342..0000000000 --- a/core/server/src/binary/handlers/consumer_groups/join_consumer_group_handler.rs +++ /dev/null @@ -1,57 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::HandlerResult; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::shard::transmission::frame::ShardResponse; -use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; -use crate::streaming::session::Session; -use iggy_binary_protocol::requests::consumer_groups::JoinConsumerGroupRequest; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::{debug, instrument}; - -#[instrument(skip_all, name = "trace_join_consumer_group", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] -pub async fn handle_join_consumer_group( - req: JoinConsumerGroupRequest, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - debug!( - "session: {session}, command: join_consumer_group, stream_id: {:?}, topic_id: {:?}, group_id: {:?}", - req.stream_id, req.topic_id, req.group_id - ); - shard.ensure_authenticated(session)?; - - let request = ShardRequest::control_plane(ShardRequestPayload::JoinConsumerGroupRequest { - user_id: session.get_user_id(), - client_id: session.client_id, - command: req, - }); - - match shard.send_to_control_plane(request).await? { - ShardResponse::JoinConsumerGroupResponse => { - sender.send_empty_ok_response().await?; - } - ShardResponse::ErrorResponse(err) => return Err(err), - _ => unreachable!("Expected JoinConsumerGroupResponse"), - } - - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/consumer_groups/leave_consumer_group_handler.rs b/core/server/src/binary/handlers/consumer_groups/leave_consumer_group_handler.rs deleted file mode 100644 index 3caf382646..0000000000 --- a/core/server/src/binary/handlers/consumer_groups/leave_consumer_group_handler.rs +++ /dev/null @@ -1,57 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::HandlerResult; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::shard::transmission::frame::ShardResponse; -use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; -use crate::streaming::session::Session; -use iggy_binary_protocol::requests::consumer_groups::LeaveConsumerGroupRequest; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::{debug, instrument}; - -#[instrument(skip_all, name = "trace_leave_consumer_group", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] -pub async fn handle_leave_consumer_group( - req: LeaveConsumerGroupRequest, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - debug!( - "session: {session}, command: leave_consumer_group, stream_id: {:?}, topic_id: {:?}, group_id: {:?}", - req.stream_id, req.topic_id, req.group_id - ); - shard.ensure_authenticated(session)?; - - let request = ShardRequest::control_plane(ShardRequestPayload::LeaveConsumerGroupRequest { - user_id: session.get_user_id(), - client_id: session.client_id, - command: req, - }); - - match shard.send_to_control_plane(request).await? { - ShardResponse::LeaveConsumerGroupResponse => { - sender.send_empty_ok_response().await?; - } - ShardResponse::ErrorResponse(err) => return Err(err), - _ => unreachable!("Expected LeaveConsumerGroupResponse"), - } - - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/consumer_groups/mod.rs b/core/server/src/binary/handlers/consumer_groups/mod.rs deleted file mode 100644 index dbefde591e..0000000000 --- a/core/server/src/binary/handlers/consumer_groups/mod.rs +++ /dev/null @@ -1,25 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod create_consumer_group_handler; -pub mod delete_consumer_group_handler; -pub mod get_consumer_group_handler; -pub mod get_consumer_groups_handler; -pub mod join_consumer_group_handler; -pub mod leave_consumer_group_handler; - -pub const COMPONENT: &str = "CONSUMER_GROUP_HANDLER"; diff --git a/core/server/src/binary/handlers/consumer_offsets/delete_consumer_offset_handler.rs b/core/server/src/binary/handlers/consumer_offsets/delete_consumer_offset_handler.rs deleted file mode 100644 index 8684567267..0000000000 --- a/core/server/src/binary/handlers/consumer_offsets/delete_consumer_offset_handler.rs +++ /dev/null @@ -1,56 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::{HandlerResult, wire_consumer_to_consumer, wire_id_to_identifier}; -use crate::binary::handlers::consumer_offsets::COMPONENT; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::streaming::session::Session; -use err_trail::ErrContext; -use iggy_binary_protocol::requests::consumer_offsets::DeleteConsumerOffsetRequest; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::debug; - -pub async fn handle_delete_consumer_offset( - req: DeleteConsumerOffsetRequest, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - let consumer = wire_consumer_to_consumer(&req.consumer)?; - let stream_id = wire_id_to_identifier(&req.stream_id)?; - let topic_id = wire_id_to_identifier(&req.topic_id)?; - debug!( - "session: {session}, command: delete_consumer_offset, stream_id: {stream_id}, topic_id: {topic_id}, partition_id: {:?}", - req.partition_id - ); - shard.ensure_authenticated(session)?; - let topic = shard.resolve_topic_for_delete_consumer_offset( - session.get_user_id(), - &stream_id, - &topic_id, - )?; - shard - .delete_consumer_offset(session.client_id, consumer, topic, req.partition_id) - .await - .error(|e: &IggyError| format!("{COMPONENT} (error: {e}) - failed to delete consumer offset for topic with ID: {} in stream with ID: {} partition ID: {:#?}, session: {}", - topic_id, stream_id, req.partition_id, session - ))?; - sender.send_empty_ok_response().await?; - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/consumer_offsets/get_consumer_offset_handler.rs b/core/server/src/binary/handlers/consumer_offsets/get_consumer_offset_handler.rs deleted file mode 100644 index 7a8ff64047..0000000000 --- a/core/server/src/binary/handlers/consumer_offsets/get_consumer_offset_handler.rs +++ /dev/null @@ -1,79 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::{HandlerResult, wire_consumer_to_consumer, wire_id_to_identifier}; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::shard::transmission::message::ResolvedTopic; -use crate::streaming::session::Session; -use iggy_binary_protocol::codec::WireEncode; -use iggy_binary_protocol::requests::consumer_offsets::GetConsumerOffsetRequest; -use iggy_binary_protocol::responses::consumer_offsets::ConsumerOffsetResponse; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::debug; - -pub async fn handle_get_consumer_offset( - req: GetConsumerOffsetRequest, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - let consumer = wire_consumer_to_consumer(&req.consumer)?; - let stream_id = wire_id_to_identifier(&req.stream_id)?; - let topic_id = wire_id_to_identifier(&req.topic_id)?; - debug!( - "session: {session}, command: get_consumer_offset, stream_id: {stream_id}, topic_id: {topic_id}, partition_id: {:?}", - req.partition_id - ); - shard.ensure_authenticated(session)?; - - let Some(resolved) = - shard - .metadata - .resolve_for_consumer_offset(session.get_user_id(), &stream_id, &topic_id)? - else { - sender.send_empty_ok_response().await?; - return Ok(HandlerResult::Finished); - }; - - let topic = ResolvedTopic { - stream_id: resolved.stream_id, - topic_id: resolved.topic_id, - }; - - let Ok(offset) = shard - .get_consumer_offset(session.client_id, consumer, topic, req.partition_id) - .await - else { - sender.send_empty_ok_response().await?; - return Ok(HandlerResult::Finished); - }; - - let Some(offset) = offset else { - sender.send_empty_ok_response().await?; - return Ok(HandlerResult::Finished); - }; - - let response = ConsumerOffsetResponse { - partition_id: offset.partition_id, - current_offset: offset.current_offset, - stored_offset: offset.stored_offset, - }; - sender.send_ok_response(&response.to_bytes()).await?; - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/consumer_offsets/mod.rs b/core/server/src/binary/handlers/consumer_offsets/mod.rs deleted file mode 100644 index 922fd459ab..0000000000 --- a/core/server/src/binary/handlers/consumer_offsets/mod.rs +++ /dev/null @@ -1,22 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod delete_consumer_offset_handler; -pub mod get_consumer_offset_handler; -pub mod store_consumer_offset_handler; - -pub const COMPONENT: &str = "CONSUMER_OFFSET_HANDLER"; diff --git a/core/server/src/binary/handlers/consumer_offsets/store_consumer_offset_handler.rs b/core/server/src/binary/handlers/consumer_offsets/store_consumer_offset_handler.rs deleted file mode 100644 index bc2d265bc5..0000000000 --- a/core/server/src/binary/handlers/consumer_offsets/store_consumer_offset_handler.rs +++ /dev/null @@ -1,63 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::rc::Rc; - -use crate::binary::dispatch::{HandlerResult, wire_consumer_to_consumer, wire_id_to_identifier}; -use crate::binary::handlers::consumer_offsets::COMPONENT; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::streaming::session::Session; -use err_trail::ErrContext; -use iggy_binary_protocol::requests::consumer_offsets::StoreConsumerOffsetRequest; -use iggy_common::IggyError; -use tracing::debug; - -pub async fn handle_store_consumer_offset( - req: StoreConsumerOffsetRequest, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - let consumer = wire_consumer_to_consumer(&req.consumer)?; - let stream_id = wire_id_to_identifier(&req.stream_id)?; - let topic_id = wire_id_to_identifier(&req.topic_id)?; - debug!( - "session: {session}, command: store_consumer_offset, stream_id: {stream_id}, topic_id: {topic_id}, partition_id: {:?}, offset: {}", - req.partition_id, req.offset - ); - shard.ensure_authenticated(session)?; - let topic = shard.resolve_topic_for_store_consumer_offset( - session.get_user_id(), - &stream_id, - &topic_id, - )?; - shard - .store_consumer_offset( - session.client_id, - consumer, - topic, - req.partition_id, - req.offset, - ) - .await - .error(|e: &IggyError| format!("{COMPONENT} (error: {e}) - failed to store consumer offset for stream_id: {}, topic_id: {}, partition_id: {:?}, offset: {}, session: {}", - stream_id, topic_id, req.partition_id, req.offset, session - ))?; - sender.send_empty_ok_response().await?; - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/messages/flush_unsaved_buffer_handler.rs b/core/server/src/binary/handlers/messages/flush_unsaved_buffer_handler.rs deleted file mode 100644 index 0d78ed907a..0000000000 --- a/core/server/src/binary/handlers/messages/flush_unsaved_buffer_handler.rs +++ /dev/null @@ -1,65 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::{HandlerResult, wire_id_to_identifier}; -use crate::binary::handlers::messages::COMPONENT; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::shard::transmission::message::ResolvedPartition; -use crate::streaming::session::Session; -use err_trail::ErrContext; -use iggy_binary_protocol::requests::messages::FlushUnsavedBufferRequest; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::{debug, instrument}; - -#[instrument(skip_all, name = "trace_flush_unsaved_buffer", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id, iggy_partition_id = req.partition_id, iggy_fsync = req.fsync))] -pub async fn handle_flush_unsaved_buffer( - req: FlushUnsavedBufferRequest, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - let stream_id = wire_id_to_identifier(&req.stream_id)?; - let topic_id = wire_id_to_identifier(&req.topic_id)?; - let partition_id = req.partition_id; - let fsync = req.fsync; - debug!( - "session: {session}, command: flush_unsaved_buffer, stream_id: {stream_id}, topic_id: {topic_id}, partition_id: {partition_id}, fsync: {fsync}" - ); - shard.ensure_authenticated(session)?; - - let user_id = session.get_user_id(); - let topic = shard.resolve_topic(&stream_id, &topic_id)?; - let partition = ResolvedPartition { - stream_id: topic.stream_id, - topic_id: topic.topic_id, - partition_id: partition_id as usize, - }; - - shard - .flush_unsaved_buffer(user_id, partition, fsync) - .await - .error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - failed to flush unsaved buffer for stream_id: {}, topic_id: {}, partition_id: {}, session: {}", - stream_id, topic_id, partition_id, session - ) - })?; - sender.send_empty_ok_response().await?; - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/messages/mod.rs b/core/server/src/binary/handlers/messages/mod.rs deleted file mode 100644 index 64571728aa..0000000000 --- a/core/server/src/binary/handlers/messages/mod.rs +++ /dev/null @@ -1,22 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod flush_unsaved_buffer_handler; -pub mod poll_messages_handler; -pub mod send_messages_handler; - -pub const COMPONENT: &str = "MESSAGE_HANDLER"; diff --git a/core/server/src/binary/handlers/messages/poll_messages_handler.rs b/core/server/src/binary/handlers/messages/poll_messages_handler.rs deleted file mode 100644 index 6007020b7b..0000000000 --- a/core/server/src/binary/handlers/messages/poll_messages_handler.rs +++ /dev/null @@ -1,87 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::{ - HandlerResult, wire_consumer_to_consumer, wire_id_to_identifier, wire_polling_to_strategy, -}; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::shard::system::messages::PollingArgs; -use crate::streaming::session::Session; -use iggy_binary_protocol::requests::messages::PollMessagesRequest; -use iggy_common::IggyError; -use server_common::PooledBuffer; -use std::rc::Rc; -use tracing::{debug, trace}; - -pub async fn handle_poll_messages( - req: PollMessagesRequest, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - let consumer = wire_consumer_to_consumer(&req.consumer)?; - let stream_id = wire_id_to_identifier(&req.stream_id)?; - let topic_id = wire_id_to_identifier(&req.topic_id)?; - let strategy = wire_polling_to_strategy(&req.strategy)?; - let partition_id = req.partition_id; - let count = req.count; - let auto_commit = req.auto_commit; - - debug!( - "session: {session}, command: poll_messages, stream_id: {stream_id}, topic_id: {topic_id}, partition_id: {partition_id:?}" - ); - shard.ensure_authenticated(session)?; - - let args = PollingArgs::new(strategy, count, auto_commit); - - let user_id = session.get_user_id(); - let client_id = session.client_id; - let topic = shard.resolve_topic_for_poll(user_id, &stream_id, &topic_id)?; - let (metadata, mut batch) = shard - .poll_messages(client_id, topic, consumer, partition_id, args) - .await?; - - let response_length = 4 + 8 + 4 + batch.size(); - let response_length_bytes = response_length.to_le_bytes(); - - let mut bufs = Vec::with_capacity(batch.containers_count() + 3); - let mut partition_id_buf = PooledBuffer::with_capacity(4); - let mut current_offset_buf = PooledBuffer::with_capacity(8); - let mut count_buf = PooledBuffer::with_capacity(4); - partition_id_buf.put_u32_le(metadata.partition_id); - current_offset_buf.put_u64_le(metadata.current_offset); - count_buf.put_u32_le(batch.count()); - - bufs.push(partition_id_buf); - bufs.push(current_offset_buf); - bufs.push(count_buf); - - batch.iter_mut().for_each(|m| { - bufs.push(m.take_messages()); - }); - trace!( - "Sending {} messages to client ({} bytes) to client", - batch.count(), - response_length - ); - - sender - .send_ok_response_vectored(&response_length_bytes, bufs) - .await?; - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/messages/send_messages_handler.rs b/core/server/src/binary/handlers/messages/send_messages_handler.rs deleted file mode 100644 index aabce4fd77..0000000000 --- a/core/server/src/binary/handlers/messages/send_messages_handler.rs +++ /dev/null @@ -1,203 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::HandlerResult; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::shard::transmission::message::{ResolvedPartition, ShardRequest, ShardRequestPayload}; -use crate::streaming::segments::{IggyIndexesMut, IggyMessagesBatchMut}; -use crate::streaming::session::Session; -use crate::streaming::topics; -use compio::buf::{IntoInner as _, IoBuf}; -use iggy_common::Identifier; -use iggy_common::Sizeable; -use iggy_common::{INDEX_SIZE, PartitioningKind}; -use iggy_common::{IggyError, Partitioning, Validatable}; -use server_common::PooledBuffer; -use server_common::sharding::IggyNamespace; -use std::rc::Rc; -use tracing::{debug, error, info, instrument}; - -#[instrument(skip_all, name = "trace_send_messages", fields( - iggy_user_id = session.get_user_id(), - iggy_client_id = session.client_id, -))] -pub async fn handle_send_messages( - sender: &mut SenderKind, - length: u32, - session: &Session, - shard: &Rc, -) -> Result { - shard.ensure_authenticated(session)?; - // `length` is the payload size (frame length minus the 4-byte code, - // already subtracted by the transport layer before calling dispatch). - let total_payload_size = length as usize; - let metadata_len_field_size = std::mem::size_of::(); - - let metadata_length_buffer = PooledBuffer::with_capacity(4); - let (result, metadata_len_buf) = sender.read(metadata_length_buffer.slice(0..4)).await; - let metadata_len_buf = metadata_len_buf.into_inner(); - result?; - let metadata_size = u32::from_le_bytes( - metadata_len_buf[..] - .try_into() - .map_err(|_| IggyError::InvalidNumberEncoding)?, - ); - if metadata_size as usize > total_payload_size { - return Err(IggyError::InvalidCommand); - } - - let metadata_buffer = PooledBuffer::with_capacity(metadata_size as usize); - let (result, metadata_buf) = sender - .read(metadata_buffer.slice(0..metadata_size as usize)) - .await; - result?; - let metadata_buf = metadata_buf.into_inner(); - - let mut element_size = 0; - - let stream_id = Identifier::from_raw_bytes(&metadata_buf)?; - element_size += stream_id.get_size_bytes().as_bytes_usize(); - - let topic_id = Identifier::from_raw_bytes( - metadata_buf - .get(element_size..) - .ok_or(IggyError::InvalidCommand)?, - )?; - element_size += topic_id.get_size_bytes().as_bytes_usize(); - - let partitioning = Partitioning::from_raw_bytes( - metadata_buf - .get(element_size..) - .ok_or(IggyError::InvalidCommand)?, - )?; - element_size += partitioning.get_size_bytes().as_bytes_usize(); - - let messages_count = u32::from_le_bytes( - metadata_buf - .get(element_size..element_size + 4) - .ok_or(IggyError::InvalidCommand)? - .try_into() - .map_err(|_| IggyError::InvalidNumberEncoding)?, - ); - let indexes_size = (messages_count as usize) - .checked_mul(INDEX_SIZE) - .ok_or(IggyError::InvalidCommand)?; - if indexes_size > total_payload_size { - return Err(IggyError::InvalidCommand); - } - - let indexes_buffer = PooledBuffer::with_capacity(indexes_size); - let (result, indexes_buffer) = sender.read(indexes_buffer.slice(0..indexes_size)).await; - result?; - let indexes_buffer = indexes_buffer.into_inner(); - - let messages_size = total_payload_size - .checked_sub(metadata_size as usize) - .and_then(|s| s.checked_sub(indexes_size)) - .and_then(|s| s.checked_sub(metadata_len_field_size)) - .ok_or(IggyError::InvalidCommand)?; - let messages_buffer = PooledBuffer::with_capacity(messages_size); - let (result, messages_buffer) = sender.read(messages_buffer.slice(0..messages_size)).await; - result?; - let messages_buffer = messages_buffer.into_inner(); - - let indexes = IggyIndexesMut::from_bytes(indexes_buffer, 0); - let batch = IggyMessagesBatchMut::from_indexes_and_messages(indexes, messages_buffer); - batch.validate()?; - - let topic = shard.resolve_topic_for_append(session.get_user_id(), &stream_id, &topic_id)?; - - let partition_id = match partitioning.kind { - PartitioningKind::Balanced => shard - .metadata - .get_next_partition_id(topic.stream_id, topic.topic_id) - .ok_or(IggyError::TopicIdNotFound( - stream_id.clone(), - topic_id.clone(), - ))?, - PartitioningKind::PartitionId => u32::from_le_bytes( - partitioning - .value - .get(..4) - .ok_or(IggyError::InvalidCommand)? - .try_into() - .map_err(|_| IggyError::InvalidNumberEncoding)?, - ) as usize, - PartitioningKind::MessagesKey => { - let partitions_count = shard - .metadata - .partitions_count(topic.stream_id, topic.topic_id); - topics::helpers::calculate_partition_id_by_messages_key_hash( - partitions_count, - &partitioning.value, - ) - } - }; - - let namespace = IggyNamespace::new(topic.stream_id, topic.topic_id, partition_id); - let user_id = session.get_user_id(); - let unsupported_socket_transfer = matches!( - partitioning.kind, - PartitioningKind::Balanced | PartitioningKind::MessagesKey - ); - let enabled_socket_migration = shard.config.tcp.socket_migration; - - if enabled_socket_migration - && !(session.is_migrated() || unsupported_socket_transfer) - && let Some(target_shard) = shard.find_shard(&namespace) - && target_shard.id != shard.id - { - debug!( - "TCP wrong shared detected: migrating from_shard {}, to_shard {}", - shard.id, target_shard.id - ); - - if let Some(fd) = sender.take_and_migrate_tcp() { - let payload = ShardRequestPayload::SocketTransfer { - fd, - from_shard: shard.id, - client_id: session.client_id, - user_id, - address: session.ip_address, - initial_data: batch, - }; - - let request = ShardRequest::data_plane(namespace, payload); - - if let Err(e) = shard.send_to_data_plane(request).await { - error!("transfer socket to another shard failed, drop connection. {e:?}"); - return Ok(HandlerResult::Finished); - } - - info!("Sending socket transfer to shard {}", target_shard.id); - return Ok(HandlerResult::Migrated { - to_shard: target_shard.id, - }); - } - } - - let partition = ResolvedPartition { - stream_id: topic.stream_id, - topic_id: topic.topic_id, - partition_id, - }; - shard.append_messages(partition, batch).await?; - - sender.send_empty_ok_response().await?; - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/mod.rs b/core/server/src/binary/handlers/mod.rs deleted file mode 100644 index 8a7a7beaf2..0000000000 --- a/core/server/src/binary/handlers/mod.rs +++ /dev/null @@ -1,28 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod cluster; -pub mod consumer_groups; -pub mod consumer_offsets; -pub mod messages; -pub mod partitions; -pub mod personal_access_tokens; -pub mod segments; -pub mod streams; -pub mod system; -pub mod topics; -pub mod users; diff --git a/core/server/src/binary/handlers/partitions/create_partitions_handler.rs b/core/server/src/binary/handlers/partitions/create_partitions_handler.rs deleted file mode 100644 index 3e9d249dd9..0000000000 --- a/core/server/src/binary/handlers/partitions/create_partitions_handler.rs +++ /dev/null @@ -1,61 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::HandlerResult; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::shard::transmission::frame::ShardResponse; -use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; -use crate::streaming::session::Session; -use iggy_binary_protocol::MAX_PARTITIONS_PER_REQUEST; -use iggy_binary_protocol::requests::partitions::CreatePartitionsRequest; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::{debug, instrument}; - -#[instrument(skip_all, name = "trace_create_partitions", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] -pub async fn handle_create_partitions( - req: CreatePartitionsRequest, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - debug!( - "session: {session}, command: create_partitions, stream_id: {:?}, topic_id: {:?}", - req.stream_id, req.topic_id - ); - shard.ensure_authenticated(session)?; - - if !(1..=MAX_PARTITIONS_PER_REQUEST).contains(&req.partitions_count) { - return Err(IggyError::TooManyPartitions); - } - - let request = ShardRequest::control_plane(ShardRequestPayload::CreatePartitionsRequest { - user_id: session.get_user_id(), - command: req, - }); - - match shard.send_to_control_plane(request).await? { - ShardResponse::CreatePartitionsResponse => { - sender.send_empty_ok_response().await?; - } - ShardResponse::ErrorResponse(err) => return Err(err), - _ => unreachable!("Expected CreatePartitionsResponse"), - } - - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/partitions/delete_partitions_handler.rs b/core/server/src/binary/handlers/partitions/delete_partitions_handler.rs deleted file mode 100644 index 90d9836139..0000000000 --- a/core/server/src/binary/handlers/partitions/delete_partitions_handler.rs +++ /dev/null @@ -1,60 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::HandlerResult; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::shard::transmission::frame::ShardResponse; -use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; -use crate::streaming::session::Session; -use iggy_binary_protocol::requests::partitions::DeletePartitionsRequest; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::{debug, instrument}; - -#[instrument(skip_all, name = "trace_delete_partitions", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] -pub async fn handle_delete_partitions( - req: DeletePartitionsRequest, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - debug!( - "session: {session}, command: delete_partitions, stream_id: {:?}, topic_id: {:?}", - req.stream_id, req.topic_id - ); - shard.ensure_authenticated(session)?; - - if req.partitions_count == 0 { - return Err(IggyError::TooManyPartitions); - } - - let request = ShardRequest::control_plane(ShardRequestPayload::DeletePartitionsRequest { - user_id: session.get_user_id(), - command: req, - }); - - match shard.send_to_control_plane(request).await? { - ShardResponse::DeletePartitionsResponse => { - sender.send_empty_ok_response().await?; - } - ShardResponse::ErrorResponse(err) => return Err(err), - _ => unreachable!("Expected DeletePartitionsResponse"), - } - - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/partitions/mod.rs b/core/server/src/binary/handlers/partitions/mod.rs deleted file mode 100644 index 38e9d65215..0000000000 --- a/core/server/src/binary/handlers/partitions/mod.rs +++ /dev/null @@ -1,21 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod create_partitions_handler; -pub mod delete_partitions_handler; - -pub const COMPONENT: &str = "PARTITIONS_HANDLER"; diff --git a/core/server/src/binary/handlers/personal_access_tokens/create_personal_access_token_handler.rs b/core/server/src/binary/handlers/personal_access_tokens/create_personal_access_token_handler.rs deleted file mode 100644 index 9058f61908..0000000000 --- a/core/server/src/binary/handlers/personal_access_tokens/create_personal_access_token_handler.rs +++ /dev/null @@ -1,73 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::HandlerResult; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::shard::transmission::frame::ShardResponse; -use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; -use crate::streaming::session::Session; -use iggy_binary_protocol::WireName; -use iggy_binary_protocol::codec::WireEncode; -use iggy_binary_protocol::requests::personal_access_tokens::CreatePersonalAccessTokenRequest; -use iggy_binary_protocol::responses::personal_access_tokens::RawPersonalAccessTokenResponse; -use iggy_common::IggyError; -use iggy_common::defaults::{ - MAX_PERSONAL_ACCESS_TOKEN_NAME_LENGTH, MIN_PERSONAL_ACCESS_TOKEN_NAME_LENGTH, -}; -use std::rc::Rc; -use tracing::{debug, instrument}; - -#[instrument(skip_all, name = "trace_create_personal_access_token", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] -pub async fn handle_create_personal_access_token( - req: CreatePersonalAccessTokenRequest, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - debug!( - "session: {session}, command: create_personal_access_token, name: {}", - req.name.as_str() - ); - shard.ensure_authenticated(session)?; - - let name_len = req.name.as_str().len(); - if !(MIN_PERSONAL_ACCESS_TOKEN_NAME_LENGTH..=MAX_PERSONAL_ACCESS_TOKEN_NAME_LENGTH) - .contains(&name_len) - { - return Err(IggyError::InvalidPersonalAccessTokenName); - } - - let request = - ShardRequest::control_plane(ShardRequestPayload::CreatePersonalAccessTokenRequest { - user_id: session.get_user_id(), - command: req, - }); - - match shard.send_to_control_plane(request).await? { - ShardResponse::CreatePersonalAccessTokenResponse(_, token) => { - let response = RawPersonalAccessTokenResponse { - token: WireName::new(token).map_err(|_| IggyError::InvalidCommand)?, - }; - sender.send_ok_response(&response.to_bytes()).await?; - } - ShardResponse::ErrorResponse(err) => return Err(err), - _ => unreachable!("Expected CreatePersonalAccessTokenResponse"), - } - - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/personal_access_tokens/delete_personal_access_token_handler.rs b/core/server/src/binary/handlers/personal_access_tokens/delete_personal_access_token_handler.rs deleted file mode 100644 index 24f536e77a..0000000000 --- a/core/server/src/binary/handlers/personal_access_tokens/delete_personal_access_token_handler.rs +++ /dev/null @@ -1,57 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::HandlerResult; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::shard::transmission::frame::ShardResponse; -use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; -use crate::streaming::session::Session; -use iggy_binary_protocol::requests::personal_access_tokens::DeletePersonalAccessTokenRequest; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::{debug, instrument}; - -#[instrument(skip_all, name = "trace_delete_personal_access_token", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] -pub async fn handle_delete_personal_access_token( - req: DeletePersonalAccessTokenRequest, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - debug!( - "session: {session}, command: delete_personal_access_token, name: {}", - req.name.as_str() - ); - shard.ensure_authenticated(session)?; - - let request = - ShardRequest::control_plane(ShardRequestPayload::DeletePersonalAccessTokenRequest { - user_id: session.get_user_id(), - command: req, - }); - - match shard.send_to_control_plane(request).await? { - ShardResponse::DeletePersonalAccessTokenResponse => { - sender.send_empty_ok_response().await?; - } - ShardResponse::ErrorResponse(err) => return Err(err), - _ => unreachable!("Expected DeletePersonalAccessTokenResponse"), - } - - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/personal_access_tokens/get_personal_access_tokens_handler.rs b/core/server/src/binary/handlers/personal_access_tokens/get_personal_access_tokens_handler.rs deleted file mode 100644 index 26af78fadb..0000000000 --- a/core/server/src/binary/handlers/personal_access_tokens/get_personal_access_tokens_handler.rs +++ /dev/null @@ -1,60 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::HandlerResult; -use crate::binary::handlers::personal_access_tokens::COMPONENT; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::streaming::session::Session; -use err_trail::ErrContext; -use iggy_binary_protocol::WireName; -use iggy_binary_protocol::codec::WireEncode; -use iggy_binary_protocol::responses::personal_access_tokens::{ - GetPersonalAccessTokensResponse, PersonalAccessTokenResponse, -}; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::debug; - -pub async fn handle_get_personal_access_tokens( - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - debug!("session: {session}, command: get_personal_access_tokens"); - shard.ensure_authenticated(session)?; - let personal_access_tokens = shard - .get_personal_access_tokens(session.get_user_id()) - .error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - failed to get personal access tokens for user: {}", - session.get_user_id() - ) - })?; - let tokens: Vec = personal_access_tokens - .iter() - .map(|pat| { - Ok(PersonalAccessTokenResponse { - name: WireName::new(pat.name.as_ref()).map_err(|_| IggyError::InvalidCommand)?, - expiry_at: pat.expiry_at.map_or(0, |e| e.as_micros()), - }) - }) - .collect::>()?; - let response = GetPersonalAccessTokensResponse { tokens }; - sender.send_ok_response(&response.to_bytes()).await?; - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/personal_access_tokens/login_with_personal_access_token_handler.rs b/core/server/src/binary/handlers/personal_access_tokens/login_with_personal_access_token_handler.rs deleted file mode 100644 index 622410de7a..0000000000 --- a/core/server/src/binary/handlers/personal_access_tokens/login_with_personal_access_token_handler.rs +++ /dev/null @@ -1,51 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::HandlerResult; -use crate::binary::handlers::personal_access_tokens::COMPONENT; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::streaming::session::Session; -use err_trail::ErrContext; -use iggy_binary_protocol::codec::WireEncode; -use iggy_binary_protocol::requests::personal_access_tokens::LoginWithPersonalAccessTokenRequest; -use iggy_binary_protocol::responses::users::IdentityResponse; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::{debug, instrument}; - -#[instrument(skip_all, name = "trace_login_with_personal_access_token", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] -pub async fn handle_login_with_personal_access_token( - req: LoginWithPersonalAccessTokenRequest, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - debug!("session: {session}, command: login_with_personal_access_token"); - let token = req.token.as_str(); - - let user = shard - .login_with_personal_access_token(token, Some(session)) - .error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - failed to login with personal access token, session: {session}", - ) - })?; - let response = IdentityResponse { user_id: user.id }; - sender.send_ok_response(&response.to_bytes()).await?; - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/personal_access_tokens/mod.rs b/core/server/src/binary/handlers/personal_access_tokens/mod.rs deleted file mode 100644 index 137de0531f..0000000000 --- a/core/server/src/binary/handlers/personal_access_tokens/mod.rs +++ /dev/null @@ -1,23 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod create_personal_access_token_handler; -pub mod delete_personal_access_token_handler; -pub mod get_personal_access_tokens_handler; -pub mod login_with_personal_access_token_handler; - -pub const COMPONENT: &str = "PERSONAL_ACCESS_TOKEN_HANDLER"; diff --git a/core/server/src/binary/handlers/segments/delete_segments_handler.rs b/core/server/src/binary/handlers/segments/delete_segments_handler.rs deleted file mode 100644 index 75b2b1c837..0000000000 --- a/core/server/src/binary/handlers/segments/delete_segments_handler.rs +++ /dev/null @@ -1,76 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::{HandlerResult, wire_id_to_identifier}; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::shard::transmission::frame::ShardResponse; -use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; -use crate::streaming::session::Session; -use iggy_binary_protocol::requests::segments::DeleteSegmentsRequest; -use iggy_common::IggyError; -use server_common::sharding::IggyNamespace; -use std::rc::Rc; -use tracing::{debug, instrument}; - -#[instrument(skip_all, name = "trace_delete_segments", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] -pub async fn handle_delete_segments( - req: DeleteSegmentsRequest, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - let stream_id = wire_id_to_identifier(&req.stream_id)?; - let topic_id = wire_id_to_identifier(&req.topic_id)?; - debug!( - "session: {session}, command: delete_segments, stream_id: {stream_id}, topic_id: {topic_id}" - ); - shard.ensure_authenticated(session)?; - - let partition_id = req.partition_id as usize; - let segments_count = req.segments_count; - - let partition = shard.resolve_partition_for_delete_segments( - session.get_user_id(), - &stream_id, - &topic_id, - partition_id, - )?; - - let namespace = IggyNamespace::new( - partition.stream_id, - partition.topic_id, - partition.partition_id, - ); - let payload = ShardRequestPayload::DeleteSegments { segments_count }; - let request = ShardRequest::data_plane(namespace, payload); - - match shard.send_to_data_plane(request).await? { - ShardResponse::DeleteSegments { - deleted_segments, - deleted_messages, - } => { - shard.metrics.decrement_segments(deleted_segments as u32); - shard.metrics.decrement_messages(deleted_messages); - sender.send_empty_ok_response().await?; - } - ShardResponse::ErrorResponse(err) => return Err(err), - _ => unreachable!("Expected DeleteSegments"), - } - - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/segments/mod.rs b/core/server/src/binary/handlers/segments/mod.rs deleted file mode 100644 index ce45de03bb..0000000000 --- a/core/server/src/binary/handlers/segments/mod.rs +++ /dev/null @@ -1,18 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod delete_segments_handler; diff --git a/core/server/src/binary/handlers/streams/create_stream_handler.rs b/core/server/src/binary/handlers/streams/create_stream_handler.rs deleted file mode 100644 index c25f6f11d6..0000000000 --- a/core/server/src/binary/handlers/streams/create_stream_handler.rs +++ /dev/null @@ -1,67 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::HandlerResult; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::shard::transmission::frame::ShardResponse; -use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; -use crate::streaming::session::Session; -use iggy_binary_protocol::WireName; -use iggy_binary_protocol::codec::WireEncode; -use iggy_binary_protocol::requests::streams::CreateStreamRequest; -use iggy_binary_protocol::responses::streams::StreamResponse; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::{debug, instrument}; - -#[instrument(skip_all, name = "trace_create_stream", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] -pub async fn handle_create_stream( - req: CreateStreamRequest, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - debug!( - "session: {session}, command: create_stream, name: {}", - req.name.as_str() - ); - shard.ensure_authenticated(session)?; - - let request = ShardRequest::control_plane(ShardRequestPayload::CreateStreamRequest { - user_id: session.get_user_id(), - command: req, - }); - - match shard.send_to_control_plane(request).await? { - ShardResponse::CreateStreamResponse(data) => { - let response = StreamResponse { - id: data.id, - created_at: data.created_at.into(), - topics_count: 0, - size_bytes: 0, - messages_count: 0, - name: WireName::new(data.name.as_ref()).map_err(|_| IggyError::InvalidCommand)?, - }; - sender.send_ok_response(&response.to_bytes()).await?; - } - ShardResponse::ErrorResponse(err) => return Err(err), - _ => unreachable!("Expected CreateStreamResponse"), - } - - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/streams/delete_stream_handler.rs b/core/server/src/binary/handlers/streams/delete_stream_handler.rs deleted file mode 100644 index 967c042810..0000000000 --- a/core/server/src/binary/handlers/streams/delete_stream_handler.rs +++ /dev/null @@ -1,56 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::HandlerResult; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::shard::transmission::frame::ShardResponse; -use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; -use crate::streaming::session::Session; -use iggy_binary_protocol::requests::streams::DeleteStreamRequest; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::{debug, instrument}; - -#[instrument(skip_all, name = "trace_delete_stream", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] -pub async fn handle_delete_stream( - req: DeleteStreamRequest, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - debug!( - "session: {session}, command: delete_stream, stream_id: {:?}", - req.stream_id - ); - shard.ensure_authenticated(session)?; - - let request = ShardRequest::control_plane(ShardRequestPayload::DeleteStreamRequest { - user_id: session.get_user_id(), - command: req, - }); - - match shard.send_to_control_plane(request).await? { - ShardResponse::DeleteStreamResponse => { - sender.send_empty_ok_response().await?; - } - ShardResponse::ErrorResponse(err) => return Err(err), - _ => unreachable!("Expected DeleteStreamResponse"), - } - - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/streams/get_stream_handler.rs b/core/server/src/binary/handlers/streams/get_stream_handler.rs deleted file mode 100644 index 74551a8cb3..0000000000 --- a/core/server/src/binary/handlers/streams/get_stream_handler.rs +++ /dev/null @@ -1,121 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::{HandlerResult, wire_id_to_identifier}; -use crate::metadata::{StreamMeta, TopicMeta}; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::streaming::session::Session; -use iggy_binary_protocol::WireName; -use iggy_binary_protocol::codec::WireEncode; -use iggy_binary_protocol::requests::streams::GetStreamRequest; -use iggy_binary_protocol::responses::streams::StreamResponse; -use iggy_binary_protocol::responses::streams::get_stream::{GetStreamResponse, TopicHeader}; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::debug; - -pub async fn handle_get_stream( - req: GetStreamRequest, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - debug!( - "session: {session}, command: get_stream, stream_id: {:?}", - req.stream_id - ); - shard.ensure_authenticated(session)?; - - let stream_id = wire_id_to_identifier(&req.stream_id)?; - - let Some(stream) = shard - .metadata - .query_stream(session.get_user_id(), &stream_id)? - else { - sender.send_empty_ok_response().await?; - return Ok(HandlerResult::Finished); - }; - - let response = build_get_stream_response(&stream)?; - sender.send_ok_response(&response.to_bytes()).await?; - Ok(HandlerResult::Finished) -} - -pub(crate) fn compute_stream_stats(stream: &StreamMeta) -> (u64, u64) { - let mut size = 0u64; - let mut messages = 0u64; - for (_, topic) in stream.topics.iter() { - for partition in topic.partitions.iter() { - size += partition.stats.size_bytes_inconsistent(); - messages += partition.stats.messages_count_inconsistent(); - } - } - (size, messages) -} - -fn compute_topic_stats(topic: &TopicMeta) -> (u64, u64) { - let mut size = 0u64; - let mut messages = 0u64; - for partition in topic.partitions.iter() { - size += partition.stats.size_bytes_inconsistent(); - messages += partition.stats.messages_count_inconsistent(); - } - (size, messages) -} - -pub(crate) fn build_topic_header(topic: &TopicMeta) -> Result { - let (size, messages) = compute_topic_stats(topic); - Ok(TopicHeader { - id: topic.id as u32, - created_at: topic.created_at.into(), - partitions_count: topic.partitions.len() as u32, - message_expiry: topic.message_expiry.into(), - compression_algorithm: topic.compression_algorithm.as_code(), - max_topic_size: topic.max_topic_size.into(), - replication_factor: topic.replication_factor, - size_bytes: size, - messages_count: messages, - name: WireName::new(topic.name.as_ref()).map_err(|_| IggyError::InvalidCommand)?, - }) -} - -fn build_get_stream_response(stream: &StreamMeta) -> Result { - let mut topic_ids: Vec<_> = stream.topics.iter().map(|(k, _)| k).collect(); - topic_ids.sort_unstable(); - - let (total_size, total_messages) = compute_stream_stats(stream); - - let mut topics = Vec::with_capacity(topic_ids.len()); - for &topic_id in &topic_ids { - if let Some(topic) = stream.topics.get(topic_id) { - topics.push(build_topic_header(topic)?); - } - } - - Ok(GetStreamResponse { - stream: StreamResponse { - id: stream.id as u32, - created_at: stream.created_at.into(), - topics_count: topic_ids.len() as u32, - size_bytes: total_size, - messages_count: total_messages, - name: WireName::new(stream.name.as_ref()).map_err(|_| IggyError::InvalidCommand)?, - }, - topics, - }) -} diff --git a/core/server/src/binary/handlers/streams/get_streams_handler.rs b/core/server/src/binary/handlers/streams/get_streams_handler.rs deleted file mode 100644 index 26d3264c09..0000000000 --- a/core/server/src/binary/handlers/streams/get_streams_handler.rs +++ /dev/null @@ -1,62 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use super::get_stream_handler::compute_stream_stats; -use crate::binary::dispatch::HandlerResult; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::streaming::session::Session; -use iggy_binary_protocol::WireName; -use iggy_binary_protocol::codec::WireEncode; -use iggy_binary_protocol::responses::streams::StreamResponse; -use iggy_binary_protocol::responses::streams::get_streams::GetStreamsResponse; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::debug; - -pub async fn handle_get_streams( - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - debug!("session: {session}, command: get_streams"); - shard.ensure_authenticated(session)?; - - let streams = shard.metadata.query_streams(session.get_user_id())?; - - let mut sorted: Vec<_> = streams.iter().collect(); - sorted.sort_by_key(|s| s.id); - - let mut wire_streams = Vec::with_capacity(sorted.len()); - for stream in sorted { - let (total_size, total_messages) = compute_stream_stats(stream); - wire_streams.push(StreamResponse { - id: stream.id as u32, - created_at: stream.created_at.into(), - topics_count: stream.topics.len() as u32, - size_bytes: total_size, - messages_count: total_messages, - name: WireName::new(stream.name.as_ref()).map_err(|_| IggyError::InvalidCommand)?, - }); - } - - let response = GetStreamsResponse { - streams: wire_streams, - }; - sender.send_ok_response(&response.to_bytes()).await?; - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/streams/mod.rs b/core/server/src/binary/handlers/streams/mod.rs deleted file mode 100644 index 9fd1a45d47..0000000000 --- a/core/server/src/binary/handlers/streams/mod.rs +++ /dev/null @@ -1,25 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod create_stream_handler; -pub mod delete_stream_handler; -pub mod get_stream_handler; -pub mod get_streams_handler; -pub mod purge_stream_handler; -pub mod update_stream_handler; - -pub const COMPONENT: &str = "STREAM_HANDLER"; diff --git a/core/server/src/binary/handlers/streams/purge_stream_handler.rs b/core/server/src/binary/handlers/streams/purge_stream_handler.rs deleted file mode 100644 index 7806bc8250..0000000000 --- a/core/server/src/binary/handlers/streams/purge_stream_handler.rs +++ /dev/null @@ -1,56 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::HandlerResult; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::shard::transmission::frame::ShardResponse; -use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; -use crate::streaming::session::Session; -use iggy_binary_protocol::requests::streams::PurgeStreamRequest; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::{debug, instrument}; - -#[instrument(skip_all, name = "trace_purge_stream", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] -pub async fn handle_purge_stream( - req: PurgeStreamRequest, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - debug!( - "session: {session}, command: purge_stream, stream_id: {:?}", - req.stream_id - ); - shard.ensure_authenticated(session)?; - - let request = ShardRequest::control_plane(ShardRequestPayload::PurgeStreamRequest { - user_id: session.get_user_id(), - command: req, - }); - - match shard.send_to_control_plane(request).await? { - ShardResponse::PurgeStreamResponse => { - sender.send_empty_ok_response().await?; - } - ShardResponse::ErrorResponse(err) => return Err(err), - _ => unreachable!("Expected PurgeStreamResponse"), - } - - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/streams/update_stream_handler.rs b/core/server/src/binary/handlers/streams/update_stream_handler.rs deleted file mode 100644 index 193d9d85aa..0000000000 --- a/core/server/src/binary/handlers/streams/update_stream_handler.rs +++ /dev/null @@ -1,56 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::HandlerResult; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::shard::transmission::frame::ShardResponse; -use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; -use crate::streaming::session::Session; -use iggy_binary_protocol::requests::streams::UpdateStreamRequest; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::{debug, instrument}; - -#[instrument(skip_all, name = "trace_update_stream", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] -pub async fn handle_update_stream( - req: UpdateStreamRequest, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - debug!( - "session: {session}, command: update_stream, stream_id: {:?}", - req.stream_id - ); - shard.ensure_authenticated(session)?; - - let request = ShardRequest::control_plane(ShardRequestPayload::UpdateStreamRequest { - user_id: session.get_user_id(), - command: req, - }); - - match shard.send_to_control_plane(request).await? { - ShardResponse::UpdateStreamResponse => { - sender.send_empty_ok_response().await?; - } - ShardResponse::ErrorResponse(err) => return Err(err), - _ => unreachable!("Expected UpdateStreamResponse"), - } - - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/system/get_client_handler.rs b/core/server/src/binary/handlers/system/get_client_handler.rs deleted file mode 100644 index a0456dd9b9..0000000000 --- a/core/server/src/binary/handlers/system/get_client_handler.rs +++ /dev/null @@ -1,54 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use super::get_me_handler::build_client_details_response; -use crate::binary::dispatch::HandlerResult; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::streaming::session::Session; -use iggy_binary_protocol::codec::WireEncode; -use iggy_binary_protocol::requests::system::GetClientRequest; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::debug; - -pub async fn handle_get_client( - req: GetClientRequest, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - debug!( - "session: {session}, command: get_client, client_id: {}", - req.client_id - ); - shard.ensure_authenticated(session)?; - shard.metadata.perm_get_client(session.get_user_id())?; - - if req.client_id == 0 { - return Err(IggyError::InvalidClientId); - } - - let Some(client) = shard.get_client(req.client_id) else { - sender.send_empty_ok_response().await?; - return Ok(HandlerResult::Finished); - }; - - let response = build_client_details_response(&client); - sender.send_ok_response(&response.to_bytes()).await?; - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/system/get_clients_handler.rs b/core/server/src/binary/handlers/system/get_clients_handler.rs deleted file mode 100644 index 1743012a02..0000000000 --- a/core/server/src/binary/handlers/system/get_clients_handler.rs +++ /dev/null @@ -1,44 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use super::get_me_handler::build_client_response; -use crate::binary::dispatch::HandlerResult; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::streaming::session::Session; -use iggy_binary_protocol::codec::WireEncode; -use iggy_binary_protocol::responses::clients::GetClientsResponse; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::debug; - -pub async fn handle_get_clients( - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - debug!("session: {session}, command: get_clients"); - shard.ensure_authenticated(session)?; - shard.metadata.perm_get_clients(session.get_user_id())?; - - let clients = shard.get_clients(); - let response = GetClientsResponse { - clients: clients.iter().map(build_client_response).collect(), - }; - sender.send_ok_response(&response.to_bytes()).await?; - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/system/get_me_handler.rs b/core/server/src/binary/handlers/system/get_me_handler.rs deleted file mode 100644 index 44e332cf7d..0000000000 --- a/core/server/src/binary/handlers/system/get_me_handler.rs +++ /dev/null @@ -1,77 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::HandlerResult; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::streaming::clients::client_manager::Client; -use crate::streaming::session::Session; -use iggy_binary_protocol::codec::WireEncode; -use iggy_binary_protocol::responses::clients::{ - ClientDetailsResponse, ClientResponse, ConsumerGroupInfoResponse, -}; -use iggy_common::IggyError; -use std::rc::Rc; - -pub async fn handle_get_me( - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - shard.ensure_authenticated(session)?; - let Some(client) = shard.get_client(session.client_id) else { - return Err(IggyError::ClientNotFound(session.client_id)); - }; - - let response = build_client_details_response(&client); - sender.send_ok_response(&response.to_bytes()).await?; - Ok(HandlerResult::Finished) -} - -pub(crate) fn build_client_details_response(client: &Client) -> ClientDetailsResponse { - ClientDetailsResponse { - client: build_client_response(client), - consumer_groups: client - .consumer_groups - .iter() - .map(|cg| ConsumerGroupInfoResponse { - stream_id: cg.stream_id, - topic_id: cg.topic_id, - group_id: cg.group_id, - }) - .collect(), - } -} - -pub(crate) fn build_client_response(client: &Client) -> ClientResponse { - ClientResponse { - client_id: client.session.client_id, - user_id: client.user_id.unwrap_or(u32::MAX), - transport: transport_to_u8(&client.transport), - address: client.session.ip_address.to_string(), - consumer_groups_count: client.consumer_groups.len() as u32, - } -} - -pub(crate) fn transport_to_u8(transport: &iggy_common::TransportProtocol) -> u8 { - match transport { - iggy_common::TransportProtocol::Tcp => 1, - iggy_common::TransportProtocol::Quic => 2, - iggy_common::TransportProtocol::Http => 3, - iggy_common::TransportProtocol::WebSocket => 4, - } -} diff --git a/core/server/src/binary/handlers/system/get_snapshot_handler.rs b/core/server/src/binary/handlers/system/get_snapshot_handler.rs deleted file mode 100644 index 132613f6a5..0000000000 --- a/core/server/src/binary/handlers/system/get_snapshot_handler.rs +++ /dev/null @@ -1,53 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::HandlerResult; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::streaming::session::Session; -use bytes::Bytes; -use iggy_binary_protocol::requests::system::GetSnapshotRequest; -use iggy_common::{IggyError, SnapshotCompression, SystemSnapshotType}; -use std::rc::Rc; -use tracing::debug; - -pub async fn handle_get_snapshot( - req: GetSnapshotRequest, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - debug!("session: {session}, command: get_snapshot"); - shard.ensure_authenticated(session)?; - shard.metadata.perm_get_snapshot(session.get_user_id())?; - - let compression = SnapshotCompression::from_code(req.compression)?; - let snapshot_types: Vec = req - .snapshot_types - .iter() - .map(|&code| SystemSnapshotType::from_code(code)) - .collect::>()?; - - if snapshot_types.contains(&SystemSnapshotType::All) && snapshot_types.len() > 1 { - return Err(IggyError::InvalidCommand); - } - - let snapshot = shard.get_snapshot(compression, &snapshot_types).await?; - let bytes = Bytes::copy_from_slice(&snapshot.0); - sender.send_ok_response(&bytes).await?; - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/system/get_stats_handler.rs b/core/server/src/binary/handlers/system/get_stats_handler.rs deleted file mode 100644 index 96ffca68a7..0000000000 --- a/core/server/src/binary/handlers/system/get_stats_handler.rs +++ /dev/null @@ -1,93 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::HandlerResult; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::shard::transmission::frame::ShardResponse; -use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; -use crate::streaming::session::Session; -use iggy_binary_protocol::codec::WireEncode; -use iggy_binary_protocol::responses::system::get_stats::{CacheMetricEntry, StatsResponse}; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::debug; - -pub async fn handle_get_stats( - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - debug!("session: {session}, command: get_stats"); - shard.ensure_authenticated(session)?; - shard.metadata.perm_get_stats(session.get_user_id())?; - - let request = ShardRequest::control_plane(ShardRequestPayload::GetStats { - user_id: session.get_user_id(), - }); - - match shard.send_to_control_plane(request).await? { - ShardResponse::GetStatsResponse(stats) => { - let response = StatsResponse { - process_id: stats.process_id, - cpu_usage: stats.cpu_usage, - total_cpu_usage: stats.total_cpu_usage, - memory_usage: stats.memory_usage.as_bytes_u64(), - total_memory: stats.total_memory.as_bytes_u64(), - available_memory: stats.available_memory.as_bytes_u64(), - run_time: stats.run_time.into(), - start_time: stats.start_time.into(), - read_bytes: stats.read_bytes.as_bytes_u64(), - written_bytes: stats.written_bytes.as_bytes_u64(), - messages_size_bytes: stats.messages_size_bytes.as_bytes_u64(), - streams_count: stats.streams_count, - topics_count: stats.topics_count, - partitions_count: stats.partitions_count, - segments_count: stats.segments_count, - messages_count: stats.messages_count, - clients_count: stats.clients_count, - consumer_groups_count: stats.consumer_groups_count, - hostname: stats.hostname, - os_name: stats.os_name, - os_version: stats.os_version, - kernel_version: stats.kernel_version, - iggy_server_version: stats.iggy_server_version, - iggy_server_semver: stats.iggy_server_semver, - cache_metrics: stats - .cache_metrics - .iter() - .map(|(key, metrics)| CacheMetricEntry { - stream_id: key.stream_id, - topic_id: key.topic_id, - partition_id: key.partition_id, - hits: metrics.hits, - misses: metrics.misses, - hit_ratio: metrics.hit_ratio, - }) - .collect(), - threads_count: stats.threads_count, - free_disk_space: stats.free_disk_space.as_bytes_u64(), - total_disk_space: stats.total_disk_space.as_bytes_u64(), - }; - sender.send_ok_response(&response.to_bytes()).await?; - } - ShardResponse::ErrorResponse(err) => return Err(err), - _ => unreachable!("Expected GetStatsResponse"), - } - - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/system/mod.rs b/core/server/src/binary/handlers/system/mod.rs deleted file mode 100644 index 065260f2d2..0000000000 --- a/core/server/src/binary/handlers/system/mod.rs +++ /dev/null @@ -1,25 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod get_client_handler; -pub mod get_clients_handler; -pub mod get_me_handler; -pub mod get_snapshot_handler; -pub mod get_stats_handler; -pub mod ping_handler; - -pub const COMPONENT: &str = "SYSTEM_HANDLER"; diff --git a/core/server/src/binary/handlers/system/ping_handler.rs b/core/server/src/binary/handlers/system/ping_handler.rs deleted file mode 100644 index 93d5e0ae58..0000000000 --- a/core/server/src/binary/handlers/system/ping_handler.rs +++ /dev/null @@ -1,41 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::HandlerResult; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::streaming::session::Session; -use iggy_common::IggyError; -use iggy_common::IggyTimestamp; -use std::rc::Rc; -use tracing::debug; - -pub async fn handle_ping( - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - debug!("session: {session}, command: ping"); - if let Some(mut client) = shard.client_manager.try_get_client_mut(session.client_id) { - let now = IggyTimestamp::now(); - client.last_heartbeat = now; - debug!("Updated last heartbeat to: {now} for session: {session}"); - } - - sender.send_empty_ok_response().await?; - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/topics/create_topic_handler.rs b/core/server/src/binary/handlers/topics/create_topic_handler.rs deleted file mode 100644 index 402fb02ba5..0000000000 --- a/core/server/src/binary/handlers/topics/create_topic_handler.rs +++ /dev/null @@ -1,103 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::HandlerResult; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::shard::transmission::frame::ShardResponse; -use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; -use crate::streaming::session::Session; -use bytes::BytesMut; -use iggy_binary_protocol::MAX_PARTITIONS_PER_REQUEST; -use iggy_binary_protocol::WireName; -use iggy_binary_protocol::codec::WireEncode; -use iggy_binary_protocol::requests::topics::CreateTopicRequest; -use iggy_binary_protocol::responses::streams::get_stream::TopicHeader; -use iggy_binary_protocol::responses::topics::get_topic::PartitionResponse; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::{debug, instrument}; - -#[instrument(skip_all, name = "trace_create_topic", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] -pub async fn handle_create_topic( - req: CreateTopicRequest, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - debug!( - "session: {session}, command: create_topic, stream_id: {:?}, name: {}", - req.stream_id, - req.name.as_str() - ); - shard.ensure_authenticated(session)?; - - if req.partitions_count > MAX_PARTITIONS_PER_REQUEST { - return Err(IggyError::TooManyPartitions); - } - - let request = ShardRequest::control_plane(ShardRequestPayload::CreateTopicRequest { - user_id: session.get_user_id(), - command: req, - }); - - match shard.send_to_control_plane(request).await? { - ShardResponse::CreateTopicResponse(data) => { - let header = TopicHeader { - id: data.id, - created_at: data.created_at.into(), - partitions_count: data.partitions.len() as u32, - message_expiry: data.message_expiry.into(), - compression_algorithm: data.compression_algorithm.as_code(), - max_topic_size: data.max_topic_size.into(), - replication_factor: data.replication_factor, - size_bytes: 0, - messages_count: 0, - name: WireName::new(data.name.as_ref()).map_err(|_| IggyError::InvalidCommand)?, - }; - let partitions: Vec = data - .partitions - .iter() - .map(|p| PartitionResponse { - id: p.id as u32, - created_at: p.created_at.into(), - segments_count: 0, - current_offset: 0, - size_bytes: 0, - messages_count: 0, - }) - .collect(); - - let mut buf = BytesMut::with_capacity( - header.encoded_size() - + partitions - .iter() - .map(WireEncode::encoded_size) - .sum::(), - ); - header.encode(&mut buf); - for partition in &partitions { - partition.encode(&mut buf); - } - sender.send_ok_response(&buf.freeze()).await?; - } - ShardResponse::ErrorResponse(err) => return Err(err), - _ => unreachable!("Expected CreateTopicResponse"), - } - - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/topics/delete_topic_handler.rs b/core/server/src/binary/handlers/topics/delete_topic_handler.rs deleted file mode 100644 index d66dfbf010..0000000000 --- a/core/server/src/binary/handlers/topics/delete_topic_handler.rs +++ /dev/null @@ -1,56 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::HandlerResult; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::shard::transmission::frame::ShardResponse; -use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; -use crate::streaming::session::Session; -use iggy_binary_protocol::requests::topics::DeleteTopicRequest; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::{debug, instrument}; - -#[instrument(skip_all, name = "trace_delete_topic", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] -pub async fn handle_delete_topic( - req: DeleteTopicRequest, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - debug!( - "session: {session}, command: delete_topic, stream_id: {:?}, topic_id: {:?}", - req.stream_id, req.topic_id - ); - shard.ensure_authenticated(session)?; - - let request = ShardRequest::control_plane(ShardRequestPayload::DeleteTopicRequest { - user_id: session.get_user_id(), - command: req, - }); - - match shard.send_to_control_plane(request).await? { - ShardResponse::DeleteTopicResponse => { - sender.send_empty_ok_response().await?; - } - ShardResponse::ErrorResponse(err) => return Err(err), - _ => unreachable!("Expected DeleteTopicResponse"), - } - - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/topics/get_topic_handler.rs b/core/server/src/binary/handlers/topics/get_topic_handler.rs deleted file mode 100644 index 7632d25065..0000000000 --- a/core/server/src/binary/handlers/topics/get_topic_handler.rs +++ /dev/null @@ -1,76 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::{HandlerResult, wire_id_to_identifier}; -use crate::binary::handlers::streams::get_stream_handler::build_topic_header; -use crate::metadata::TopicMeta; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::streaming::session::Session; -use iggy_binary_protocol::codec::WireEncode; -use iggy_binary_protocol::requests::topics::GetTopicRequest; -use iggy_binary_protocol::responses::topics::get_topic::{GetTopicResponse, PartitionResponse}; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::debug; - -pub async fn handle_get_topic( - req: GetTopicRequest, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - let stream_id = wire_id_to_identifier(&req.stream_id)?; - let topic_id = wire_id_to_identifier(&req.topic_id)?; - debug!("session: {session}, command: get_topic, stream_id: {stream_id}, topic_id: {topic_id}"); - shard.ensure_authenticated(session)?; - - let Some(topic) = shard - .metadata - .query_topic(session.get_user_id(), &stream_id, &topic_id)? - else { - sender.send_empty_ok_response().await?; - return Ok(HandlerResult::Finished); - }; - - let response = build_get_topic_response(&topic)?; - sender.send_ok_response(&response.to_bytes()).await?; - Ok(HandlerResult::Finished) -} - -fn build_get_topic_response(topic: &TopicMeta) -> Result { - let header = build_topic_header(topic)?; - - let partitions: Vec = topic - .partitions - .iter() - .enumerate() - .map(|(partition_id, partition)| PartitionResponse { - id: partition_id as u32, - created_at: partition.created_at.into(), - segments_count: partition.stats.segments_count_inconsistent(), - current_offset: partition.stats.current_offset(), - size_bytes: partition.stats.size_bytes_inconsistent(), - messages_count: partition.stats.messages_count_inconsistent(), - }) - .collect(); - - Ok(GetTopicResponse { - topic: header, - partitions, - }) -} diff --git a/core/server/src/binary/handlers/topics/get_topics_handler.rs b/core/server/src/binary/handlers/topics/get_topics_handler.rs deleted file mode 100644 index 32434ef485..0000000000 --- a/core/server/src/binary/handlers/topics/get_topics_handler.rs +++ /dev/null @@ -1,61 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::{HandlerResult, wire_id_to_identifier}; -use crate::binary::handlers::streams::get_stream_handler::build_topic_header; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::streaming::session::Session; -use iggy_binary_protocol::codec::WireEncode; -use iggy_binary_protocol::requests::topics::GetTopicsRequest; -use iggy_binary_protocol::responses::topics::get_topics::GetTopicsResponse; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::debug; - -pub async fn handle_get_topics( - req: GetTopicsRequest, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - let stream_id = wire_id_to_identifier(&req.stream_id)?; - debug!("session: {session}, command: get_topics, stream_id: {stream_id}"); - shard.ensure_authenticated(session)?; - - let Some(topics) = shard - .metadata - .query_topics(session.get_user_id(), &stream_id)? - else { - sender.send_empty_ok_response().await?; - return Ok(HandlerResult::Finished); - }; - - let mut sorted: Vec<_> = topics.iter().collect(); - sorted.sort_by_key(|t| t.id); - - let mut wire_topics = Vec::with_capacity(sorted.len()); - for topic in sorted { - wire_topics.push(build_topic_header(topic)?); - } - - let response = GetTopicsResponse { - topics: wire_topics, - }; - sender.send_ok_response(&response.to_bytes()).await?; - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/topics/mod.rs b/core/server/src/binary/handlers/topics/mod.rs deleted file mode 100644 index 023a96e103..0000000000 --- a/core/server/src/binary/handlers/topics/mod.rs +++ /dev/null @@ -1,25 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod create_topic_handler; -pub mod delete_topic_handler; -pub mod get_topic_handler; -pub mod get_topics_handler; -pub mod purge_topic_handler; -pub mod update_topic_handler; - -pub const COMPONENT: &str = "TOPIC_HANDLER"; diff --git a/core/server/src/binary/handlers/topics/purge_topic_handler.rs b/core/server/src/binary/handlers/topics/purge_topic_handler.rs deleted file mode 100644 index 8b2171e272..0000000000 --- a/core/server/src/binary/handlers/topics/purge_topic_handler.rs +++ /dev/null @@ -1,56 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::HandlerResult; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::shard::transmission::frame::ShardResponse; -use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; -use crate::streaming::session::Session; -use iggy_binary_protocol::requests::topics::PurgeTopicRequest; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::{debug, instrument}; - -#[instrument(skip_all, name = "trace_purge_topic", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] -pub async fn handle_purge_topic( - req: PurgeTopicRequest, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - debug!( - "session: {session}, command: purge_topic, stream_id: {:?}, topic_id: {:?}", - req.stream_id, req.topic_id - ); - shard.ensure_authenticated(session)?; - - let request = ShardRequest::control_plane(ShardRequestPayload::PurgeTopicRequest { - user_id: session.get_user_id(), - command: req, - }); - - match shard.send_to_control_plane(request).await? { - ShardResponse::PurgeTopicResponse => { - sender.send_empty_ok_response().await?; - } - ShardResponse::ErrorResponse(err) => return Err(err), - _ => unreachable!("Expected PurgeTopicResponse"), - } - - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/topics/update_topic_handler.rs b/core/server/src/binary/handlers/topics/update_topic_handler.rs deleted file mode 100644 index 4559fe3469..0000000000 --- a/core/server/src/binary/handlers/topics/update_topic_handler.rs +++ /dev/null @@ -1,56 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::HandlerResult; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::shard::transmission::frame::ShardResponse; -use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; -use crate::streaming::session::Session; -use iggy_binary_protocol::requests::topics::UpdateTopicRequest; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::{debug, instrument}; - -#[instrument(skip_all, name = "trace_update_topic", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] -pub async fn handle_update_topic( - req: UpdateTopicRequest, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - debug!( - "session: {session}, command: update_topic, stream_id: {:?}, topic_id: {:?}", - req.stream_id, req.topic_id - ); - shard.ensure_authenticated(session)?; - - let request = ShardRequest::control_plane(ShardRequestPayload::UpdateTopicRequest { - user_id: session.get_user_id(), - command: req, - }); - - match shard.send_to_control_plane(request).await? { - ShardResponse::UpdateTopicResponse => { - sender.send_empty_ok_response().await?; - } - ShardResponse::ErrorResponse(err) => return Err(err), - _ => unreachable!("Expected UpdateTopicResponse"), - } - - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/users/change_password_handler.rs b/core/server/src/binary/handlers/users/change_password_handler.rs deleted file mode 100644 index c2799f6a27..0000000000 --- a/core/server/src/binary/handlers/users/change_password_handler.rs +++ /dev/null @@ -1,66 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::HandlerResult; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::shard::transmission::frame::ShardResponse; -use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; -use crate::streaming::session::Session; -use iggy_binary_protocol::requests::users::ChangePasswordRequest; -use iggy_common::IggyError; -use iggy_common::defaults::{MAX_PASSWORD_LENGTH, MIN_PASSWORD_LENGTH}; -use std::rc::Rc; -use tracing::{debug, instrument}; - -#[instrument(skip_all, name = "trace_change_password", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] -pub async fn handle_change_password( - req: ChangePasswordRequest, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - debug!( - "session: {session}, command: change_password, user_id: {:?}", - req.user_id - ); - shard.ensure_authenticated(session)?; - - let current_len = req.current_password.len(); - if !(MIN_PASSWORD_LENGTH..=MAX_PASSWORD_LENGTH).contains(¤t_len) { - return Err(IggyError::InvalidPassword); - } - let new_len = req.new_password.len(); - if !(MIN_PASSWORD_LENGTH..=MAX_PASSWORD_LENGTH).contains(&new_len) { - return Err(IggyError::InvalidPassword); - } - - let request = ShardRequest::control_plane(ShardRequestPayload::ChangePasswordRequest { - user_id: session.get_user_id(), - command: req, - }); - - match shard.send_to_control_plane(request).await? { - ShardResponse::ChangePasswordResponse => { - sender.send_empty_ok_response().await?; - } - ShardResponse::ErrorResponse(err) => return Err(err), - _ => unreachable!("Expected ChangePasswordResponse"), - } - - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/users/create_user_handler.rs b/core/server/src/binary/handlers/users/create_user_handler.rs deleted file mode 100644 index d76ef0504e..0000000000 --- a/core/server/src/binary/handlers/users/create_user_handler.rs +++ /dev/null @@ -1,83 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::HandlerResult; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::shard::transmission::frame::ShardResponse; -use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; -use crate::streaming::session::Session; -use iggy_binary_protocol::WireName; -use iggy_binary_protocol::codec::WireEncode; -use iggy_binary_protocol::requests::users::CreateUserRequest; -use iggy_binary_protocol::responses::users::{UserDetailsResponse, UserResponse}; -use iggy_common::IggyError; -use iggy_common::defaults::{ - MAX_PASSWORD_LENGTH, MAX_USERNAME_LENGTH, MIN_PASSWORD_LENGTH, MIN_USERNAME_LENGTH, -}; -use iggy_common::wire_conversions::permissions_to_wire; -use std::rc::Rc; -use tracing::{debug, instrument}; - -#[instrument(skip_all, name = "trace_create_user", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] -pub async fn handle_create_user( - req: CreateUserRequest, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - debug!( - "session: {session}, command: create_user, username: {}", - req.username.as_str() - ); - shard.ensure_authenticated(session)?; - shard.metadata.perm_create_user(session.get_user_id())?; - - let username_len = req.username.as_str().len(); - if !(MIN_USERNAME_LENGTH..=MAX_USERNAME_LENGTH).contains(&username_len) { - return Err(IggyError::InvalidUsername); - } - let password_len = req.password.len(); - if !(MIN_PASSWORD_LENGTH..=MAX_PASSWORD_LENGTH).contains(&password_len) { - return Err(IggyError::InvalidPassword); - } - - let request = ShardRequest::control_plane(ShardRequestPayload::CreateUserRequest { - user_id: session.get_user_id(), - command: req, - }); - - match shard.send_to_control_plane(request).await? { - ShardResponse::CreateUserResponse(user) => { - let response = UserDetailsResponse { - user: UserResponse { - id: user.id, - created_at: user.created_at.as_micros(), - status: user.status.as_code(), - username: WireName::new(&user.username) - .map_err(|_| IggyError::InvalidCommand)?, - }, - permissions: user.permissions.as_ref().map(permissions_to_wire), - }; - sender.send_ok_response(&response.to_bytes()).await?; - } - ShardResponse::ErrorResponse(err) => return Err(err), - _ => unreachable!("Expected CreateUserResponse"), - } - - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/users/delete_user_handler.rs b/core/server/src/binary/handlers/users/delete_user_handler.rs deleted file mode 100644 index e7dce1c9d7..0000000000 --- a/core/server/src/binary/handlers/users/delete_user_handler.rs +++ /dev/null @@ -1,57 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::HandlerResult; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::shard::transmission::frame::ShardResponse; -use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; -use crate::streaming::session::Session; -use iggy_binary_protocol::requests::users::DeleteUserRequest; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::{debug, instrument}; - -#[instrument(skip_all, name = "trace_delete_user", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] -pub async fn handle_delete_user( - req: DeleteUserRequest, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - debug!( - "session: {session}, command: delete_user, user_id: {:?}", - req.user_id - ); - shard.ensure_authenticated(session)?; - shard.metadata.perm_delete_user(session.get_user_id())?; - - let request = ShardRequest::control_plane(ShardRequestPayload::DeleteUserRequest { - user_id: session.get_user_id(), - command: req, - }); - - match shard.send_to_control_plane(request).await? { - ShardResponse::DeleteUserResponse(_) => { - sender.send_empty_ok_response().await?; - } - ShardResponse::ErrorResponse(err) => return Err(err), - _ => unreachable!("Expected DeleteUserResponse"), - } - - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/users/get_user_handler.rs b/core/server/src/binary/handlers/users/get_user_handler.rs deleted file mode 100644 index 8e243d93ad..0000000000 --- a/core/server/src/binary/handlers/users/get_user_handler.rs +++ /dev/null @@ -1,62 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::{HandlerResult, wire_id_to_identifier}; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::streaming::session::Session; -use iggy_binary_protocol::WireName; -use iggy_binary_protocol::codec::WireEncode; -use iggy_binary_protocol::requests::users::GetUserRequest; -use iggy_binary_protocol::responses::users::{UserDetailsResponse, UserResponse}; -use iggy_common::IggyError; -use iggy_common::wire_conversions::permissions_to_wire; -use std::rc::Rc; -use tracing::debug; - -pub async fn handle_get_user( - req: GetUserRequest, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - debug!( - "session: {session}, command: get_user, user_id: {:?}", - req.user_id - ); - shard.ensure_authenticated(session)?; - - let user_id = wire_id_to_identifier(&req.user_id)?; - - let Some(user) = shard.metadata.query_user(session.get_user_id(), &user_id)? else { - sender.send_empty_ok_response().await?; - return Ok(HandlerResult::Finished); - }; - - let response = UserDetailsResponse { - user: UserResponse { - id: user.id, - created_at: user.created_at.as_micros(), - status: user.status.as_code(), - username: WireName::new(user.username.as_ref()) - .map_err(|_| IggyError::InvalidCommand)?, - }, - permissions: user.permissions.as_deref().map(permissions_to_wire), - }; - sender.send_ok_response(&response.to_bytes()).await?; - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/users/get_users_handler.rs b/core/server/src/binary/handlers/users/get_users_handler.rs deleted file mode 100644 index 99fa31372f..0000000000 --- a/core/server/src/binary/handlers/users/get_users_handler.rs +++ /dev/null @@ -1,53 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::HandlerResult; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::streaming::session::Session; -use iggy_binary_protocol::WireName; -use iggy_binary_protocol::codec::WireEncode; -use iggy_binary_protocol::responses::users::{GetUsersResponse, UserResponse}; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::debug; - -pub async fn handle_get_users( - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - debug!("session: {session}, command: get_users"); - shard.ensure_authenticated(session)?; - - let users = shard.metadata.query_users(session.get_user_id())?; - let wire_users: Vec = users - .iter() - .map(|u| { - Ok(UserResponse { - id: u.id, - created_at: u.created_at.as_micros(), - status: u.status.as_code(), - username: WireName::new(u.username.as_ref()) - .map_err(|_| IggyError::InvalidCommand)?, - }) - }) - .collect::>()?; - let response = GetUsersResponse { users: wire_users }; - sender.send_ok_response(&response.to_bytes()).await?; - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/users/login_user_handler.rs b/core/server/src/binary/handlers/users/login_user_handler.rs deleted file mode 100644 index 22d0943f84..0000000000 --- a/core/server/src/binary/handlers/users/login_user_handler.rs +++ /dev/null @@ -1,71 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::HandlerResult; -use crate::binary::handlers::users::COMPONENT; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::streaming::session::Session; -use err_trail::ErrContext; -use iggy_binary_protocol::codec::WireEncode; -use iggy_binary_protocol::requests::users::LoginUserRequest; -use iggy_binary_protocol::responses::users::IdentityResponse; -use iggy_common::IggyError; -use iggy_common::defaults::{ - MAX_PASSWORD_LENGTH, MAX_USERNAME_LENGTH, MIN_PASSWORD_LENGTH, MIN_USERNAME_LENGTH, -}; -use std::rc::Rc; -use tracing::{debug, info, instrument, warn}; - -#[instrument(skip_all, name = "trace_login_user", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] -pub async fn handle_login_user( - req: LoginUserRequest, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - if shard.is_shutting_down() { - warn!("Rejecting login request during shutdown"); - return Err(IggyError::Disconnected); - } - - let username = req.username.as_str(); - let username_len = username.len(); - if !(MIN_USERNAME_LENGTH..=MAX_USERNAME_LENGTH).contains(&username_len) { - return Err(IggyError::InvalidUsername); - } - let password_len = req.password.len(); - if !(MIN_PASSWORD_LENGTH..=MAX_PASSWORD_LENGTH).contains(&password_len) { - return Err(IggyError::InvalidPassword); - } - - debug!("session: {session}, command: login_user, username: {username}"); - - info!("Logging in user: {username} ..."); - let user = shard - .login_user(username, &req.password, Some(session)) - .error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - failed to login user with name: {username}, session: {session}", - ) - })?; - info!("Logged in user: {username} with ID: {}.", user.id); - - let response = IdentityResponse { user_id: user.id }; - sender.send_ok_response(&response.to_bytes()).await?; - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/users/logout_user_handler.rs b/core/server/src/binary/handlers/users/logout_user_handler.rs deleted file mode 100644 index 8113f7bc7d..0000000000 --- a/core/server/src/binary/handlers/users/logout_user_handler.rs +++ /dev/null @@ -1,44 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::HandlerResult; -use crate::binary::handlers::users::COMPONENT; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::streaming::session::Session; -use err_trail::ErrContext; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::{debug, info, instrument}; - -#[instrument(skip_all, name = "trace_logout_user", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] -pub async fn handle_logout_user( - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - debug!("session: {session}, command: logout_user"); - shard.ensure_authenticated(session)?; - info!("Logging out user with ID: {}...", session.get_user_id()); - shard.logout_user(session).error(|e: &IggyError| { - format!("{COMPONENT} (error: {e}) - failed to logout user, session: {session}") - })?; - info!("Logged out user with ID: {}.", session.get_user_id()); - session.clear_user_id(); - sender.send_empty_ok_response().await?; - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/users/mod.rs b/core/server/src/binary/handlers/users/mod.rs deleted file mode 100644 index df933d49a2..0000000000 --- a/core/server/src/binary/handlers/users/mod.rs +++ /dev/null @@ -1,28 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod change_password_handler; -pub mod create_user_handler; -pub mod delete_user_handler; -pub mod get_user_handler; -pub mod get_users_handler; -pub mod login_user_handler; -pub mod logout_user_handler; -pub mod update_permissions_handler; -pub mod update_user_handler; - -pub const COMPONENT: &str = "USER_HANDLER"; diff --git a/core/server/src/binary/handlers/users/update_permissions_handler.rs b/core/server/src/binary/handlers/users/update_permissions_handler.rs deleted file mode 100644 index 74426704ef..0000000000 --- a/core/server/src/binary/handlers/users/update_permissions_handler.rs +++ /dev/null @@ -1,59 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::HandlerResult; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::shard::transmission::frame::ShardResponse; -use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; -use crate::streaming::session::Session; -use iggy_binary_protocol::requests::users::UpdatePermissionsRequest; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::{debug, instrument}; - -#[instrument(skip_all, name = "trace_update_permissions", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] -pub async fn handle_update_permissions( - req: UpdatePermissionsRequest, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - debug!( - "session: {session}, command: update_permissions, user_id: {:?}", - req.user_id - ); - shard.ensure_authenticated(session)?; - shard - .metadata - .perm_update_permissions(session.get_user_id())?; - - let request = ShardRequest::control_plane(ShardRequestPayload::UpdatePermissionsRequest { - user_id: session.get_user_id(), - command: req, - }); - - match shard.send_to_control_plane(request).await? { - ShardResponse::UpdatePermissionsResponse => { - sender.send_empty_ok_response().await?; - } - ShardResponse::ErrorResponse(err) => return Err(err), - _ => unreachable!("Expected UpdatePermissionsResponse"), - } - - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/handlers/users/update_user_handler.rs b/core/server/src/binary/handlers/users/update_user_handler.rs deleted file mode 100644 index 96d655fbe7..0000000000 --- a/core/server/src/binary/handlers/users/update_user_handler.rs +++ /dev/null @@ -1,65 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::HandlerResult; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::shard::transmission::frame::ShardResponse; -use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; -use crate::streaming::session::Session; -use iggy_binary_protocol::requests::users::UpdateUserRequest; -use iggy_common::IggyError; -use iggy_common::defaults::{MAX_USERNAME_LENGTH, MIN_USERNAME_LENGTH}; -use std::rc::Rc; -use tracing::{debug, instrument}; - -#[instrument(skip_all, name = "trace_update_user", fields(iggy_user_id = session.get_user_id(), iggy_client_id = session.client_id))] -pub async fn handle_update_user( - req: UpdateUserRequest, - sender: &mut SenderKind, - session: &Session, - shard: &Rc, -) -> Result { - debug!( - "session: {session}, command: update_user, user_id: {:?}", - req.user_id - ); - shard.ensure_authenticated(session)?; - shard.metadata.perm_update_user(session.get_user_id())?; - - if let Some(ref username) = req.username { - let username_len = username.as_str().len(); - if !(MIN_USERNAME_LENGTH..=MAX_USERNAME_LENGTH).contains(&username_len) { - return Err(IggyError::InvalidUsername); - } - } - - let request = ShardRequest::control_plane(ShardRequestPayload::UpdateUserRequest { - user_id: session.get_user_id(), - command: req, - }); - - match shard.send_to_control_plane(request).await? { - ShardResponse::UpdateUserResponse(_) => { - sender.send_empty_ok_response().await?; - } - ShardResponse::ErrorResponse(err) => return Err(err), - _ => unreachable!("Expected UpdateUserResponse"), - } - - Ok(HandlerResult::Finished) -} diff --git a/core/server/src/binary/mod.rs b/core/server/src/binary/mod.rs deleted file mode 100644 index 8f9f22b20b..0000000000 --- a/core/server/src/binary/mod.rs +++ /dev/null @@ -1,21 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod dispatch; -pub mod handlers; - -pub const COMPONENT: &str = "BINARY"; diff --git a/core/server/src/bootstrap.rs b/core/server/src/bootstrap.rs index eb42c0539e..15e290532f 100644 --- a/core/server/src/bootstrap.rs +++ b/core/server/src/bootstrap.rs @@ -15,602 +15,4597 @@ // specific language governing permissions and limitations // under the License. -use crate::{ - IGGY_ROOT_PASSWORD_ENV, IGGY_ROOT_USERNAME_ENV, - compat::index_rebuilding::index_rebuilder::IndexRebuilder, - configs::{ - cache_indexes::CacheIndexesConfig, - server::ServerConfig, - system::{INDEX_EXTENSION, LOG_EXTENSION, SystemConfig}, - }, - io::fs_utils::{self, DirEntry}, - metadata::{ConsumerGroupMeta, MetadataWriter, PartitionMeta, StreamMeta, TopicMeta, UserMeta}, - server_error::ServerError, - shard::{ - system::info::SystemInfo, - transmission::{ - connector::{ShardConnector, StopSender}, - frame::ShardFrame, - }, - }, - state::system::{StreamState, TopicState, UserState}, - streaming::{ - partitions::{ - consumer_group_offsets::ConsumerGroupOffsets, consumer_offsets::ConsumerOffsets, - journal::MemoryMessageJournal, log::SegmentedLog, - }, - persistence::persister::{FilePersister, FileWithSyncPersister, PersisterKind}, - segments::{Segment, storage::Storage}, - stats::{PartitionStats, StreamStats, TopicStats}, - storage::SystemStorage, - users::user::User, - utils::crypto, - }, +use crate::auth::warm_dummy_password_hash; +use crate::cluster_meta::ClusterRoster; +use crate::config_writer::write_current_config; +use crate::dispatch::{ + make_client_request_handler, make_deferred_client_request_handler, + make_deferred_replica_message_handler, make_list_clients_handler, make_metadata_submit_handler, + make_partition_read_handler, }; -use err_trail::ErrContext; -use iggy_common::SemanticVersion; -use iggy_common::{ - IggyByteSize, IggyError, PersonalAccessToken, - defaults::{ - DEFAULT_ROOT_USERNAME, MAX_PASSWORD_LENGTH, MAX_USERNAME_LENGTH, MIN_PASSWORD_LENGTH, - MIN_USERNAME_LENGTH, - }, +use crate::http; +use crate::partition_helpers::{ + build_partition_fresh, configure_consumer_offsets, ensure_initial_segment, + open_partition_superblock, restore_partition_view, validate_namespace_bounds, }; -use shard_allocator::ShardInfo; -use slab::Slab; -use std::{env, sync::Arc}; -use tracing::{info, warn}; - -pub fn create_shard_connections( - shard_assignment: &[ShardInfo], -) -> (Vec>, Vec<(u16, StopSender)>) { - // Create connectors with sequential IDs (0, 1, 2, ...) regardless of CPU core numbers - let connectors: Vec> = shard_assignment - .iter() - .enumerate() - .map(|(idx, _assignment)| { - // let cpu_id = assignment.cpu_set.iter().next().unwrap_or(&idx); - ShardConnector::new(idx as u16) - }) - .collect(); +use crate::segment_recovery::{RecoveredSegment, load_persisted_segments}; +use crate::server_error::{ServerError, ShardJoinFailure, ShardJoinFailureKind}; +use crate::session_manager::SessionManager; +use configs::server::{ServerConfig, ServerSystemConfig}; +use configs::sharding::{ + INBOX_CAPACITY_MAX, SHUTDOWN_DRAIN_TIMEOUT_MAX, SHUTDOWN_POLL_INTERVAL_MAX, +}; +use consensus::{ + ClientTable, LocalPipeline, MetadataHandle, PartitionsHandle, PipelineEntry, Sequencer, + VsrConsensus, +}; +// `try_send` / `try_recv` resolve through these traits on `MAsyncTx` / +// `MAsyncRx`; the metadata-handoff loops below depend on the +// non-blocking variants for cancel-safe shutdown polling. +use consensus::VsrState; +use crossfire::{AsyncRxTrait, AsyncTxTrait}; +use iggy_binary_protocol::{Operation, PrepareHeader}; +use iggy_common::defaults::{ + DEFAULT_ROOT_PASSWORD, DEFAULT_ROOT_USERNAME, MAX_PASSWORD_LENGTH, MAX_USERNAME_LENGTH, + MIN_PASSWORD_LENGTH, MIN_USERNAME_LENGTH, +}; +use iggy_common::{Aes256GcmEncryptor, EncryptorKind, IggyByteSize, PartitionStats, variadic}; +use journal::prepare_journal::PrepareJournal; +use journal::superblock::{PingPongSuperblock, SuperblockStore}; +use journal::{Journal, JournalHandle}; +use message_bus::client_listener::{self, RequestHandler}; +use message_bus::installer; +use message_bus::installer::conn_info::{ClientConnMeta, ClientTransportKind}; +use message_bus::replica::auth::{self, ReplicaAuth}; +use message_bus::replica::handshake::{ReplicaHandshakeCtx, ReplicaTlsCtx}; +use message_bus::replica::io as replica_io; +use message_bus::replica::listener::{self as replica_listener, MessageHandler}; +use message_bus::transports::quic::server_config_with_cert; +use message_bus::transports::tls::{ + AcceptAnyServerCert, REPLICA_ALPN, TlsServerCredentials, install_default_crypto_provider, + load_ca_pem, load_pem, self_signed_for_loopback, +}; +use message_bus::{ + AcceptedClientFn, AcceptedQuicClientFn, AcceptedReplicaFn, AcceptedTlsClientFn, + AcceptedWsClientFn, AcceptedWssClientFn, ConnectionInstaller, DialedReplicaFn, IggyMessageBus, + MAX_INFLIGHT_REPLICA_HANDSHAKES, MessageBus, ReplicaOwnerTable, connector, +}; +use metadata::IggyMetadata; +use metadata::MuxStateMachine; +use metadata::ReplicaIdentity; +use metadata::impls::metadata::{IggySnapshot, StreamsFrontend}; +use metadata::impls::recovery::recover; +use metadata::stm::mux::WithFactory; +use metadata::stm::snapshot::Snapshot; +use metadata::stm::stream::{Partition, Streams}; +use metadata::stm::user::Users; +use partitions::{ + IggyIndexWriter, IggyPartition, IggyPartitions, MessagesWriter, PartitionsConfig, +}; +use rustls::pki_types::ServerName; +use server_common::Message; +use server_common::bootstrap::create_directories; +use server_common::crypto; +use server_common::executor::create_shard_executor; +use server_common::fs_utils::remove_dir_all; +use server_common::log::{Logging, LoggingSettings, TelemetrySettings}; +use server_common::sharding::{IggyNamespace, PartitionLocation, ShardId}; +use shard::builder::IggyShardBuilder; +use shard::metrics::{ShardMetrics, frame_drop_reason, frame_drop_variant}; +use shard::shards_table::{PapayaShardsTable, ShardsTable, calculate_shard_assignment}; +use shard::{ + CoordinatorConfig, IggyShard, LifecycleFrame, ListClientsHandler, MetadataSubmitHandler, + PartitionConsensusConfig, PartitionReadHandler, Receiver as ShardReceiver, ShardFrame, + ShardIdentity, TaggedSender, channel, shard_mesh_channels, +}; +use shard_allocator::{ShardAllocator, ShardInfo}; +use std::cell::RefCell; +use std::collections::HashMap; +use std::env; +use std::net::{IpAddr, SocketAddr}; +use std::path::{Path, PathBuf}; +use std::rc::{Rc, Weak}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::thread; +use std::time::{Duration, Instant}; +use tracing::{error, info, warn}; - let shutdown_handles = connectors - .iter() - .map(|conn| (conn.id, conn.stop_sender.clone())) - .collect(); +const SHARD_REPLICA_ID: u8 = 0; - (connectors, shutdown_handles) -} +pub const IGGY_ROOT_USERNAME_ENV: &str = "IGGY_ROOT_USERNAME"; +pub const IGGY_ROOT_PASSWORD_ENV: &str = "IGGY_ROOT_PASSWORD"; -pub async fn load_config() -> Result { - let config = ServerConfig::load().await?; - Ok(config) -} +type ServerMuxStateMachine = MuxStateMachine; -pub fn create_root_user() -> User { - let mut username = env::var(IGGY_ROOT_USERNAME_ENV); - let mut password = env::var(IGGY_ROOT_PASSWORD_ENV); - assert_eq!( - username.is_ok(), - password.is_ok(), - "When providing the custom root user credentials, both username and password must be set." - ); - if username.is_ok() && password.is_ok() { - info!("Using the custom root user credentials."); - } else { - info!("Using the default root user credentials..."); - username = Ok(DEFAULT_ROOT_USERNAME.to_string()); - let generated_password = crypto::generate_secret(20..40); - println!("Generated root user password: {generated_password}"); - password = Ok(generated_password); - } - - let username = username.expect("Root username is not set."); - let password = password.expect("Root password is not set."); - assert!( - !username.is_empty() && !password.is_empty(), - "Root user credentials cannot be empty." - ); - assert!( - username.len() >= MIN_USERNAME_LENGTH, - "Root username is too short." - ); - assert!( - username.len() <= MAX_USERNAME_LENGTH, - "Root username is too long." - ); - assert!( - password.len() >= MIN_PASSWORD_LENGTH, - "Root password is too short." - ); - assert!( - password.len() <= MAX_PASSWORD_LENGTH, - "Root password is too long." - ); +/// Cross-thread bundle carrying one `ReadHandleFactory` per metadata +/// state. Shard 0 mints one after `recover()` and broadcasts a clone to +/// every peer shard; each peer rebuilds a reader-mode +/// [`ServerMuxStateMachine`] on its own runtime, skipping the WAL. +type ServerMetadataBundle = ::Bundle; + +pub(crate) type ServerMetadata = IggyMetadata< + VsrConsensus>, + PrepareJournal, + IggySnapshot, + ServerMuxStateMachine, +>; - User::root(&username, &password) +/// The shard type the dispatch layer is generic over. +/// +/// `B`/`MJ`/`S`/`SB` are free; the metadata state machine (`M`) and shards +/// table (`T`) are pinned, being identical in production and the simulator. +/// Production instantiates it as [`ServerShard`], defaulting `SB` to the +/// on-disk [`PingPongSuperblock`]; the simulator supplies its own +/// `B`/`MJ`/`S`/`SB`. +pub type ShellShard = + IggyShard; + +/// Late-bound self-reference the deferred dispatch handlers upgrade per frame. +pub type ShellShardHandle = + Rc>>>>; + +/// Bus bounds the dispatch/pump path needs (matches `run_message_pump`). +/// Blanket-impl'd, so it is only shorthand for the four underlying bounds. +pub trait ShellBus: MessageBus + ConnectionInstaller + Clone + 'static {} +impl ShellBus for B {} + +/// The five dispatch handlers a shard is built with, plus the +/// [`SessionManager`] the request-plane pair shares. +/// +/// Both production ([`build_shard_for_thread`]) and the simulator's shell +/// mode construct these through [`wire_shell_handlers`], so the request +/// plane is wired one way. The simulator's shell-off fast path uses +/// [`ShellHandlers::noop`] instead. +pub struct ShellHandlers { + pub on_replica_message: MessageHandler, + pub on_client_request: RequestHandler, + pub on_metadata_submit: MetadataSubmitHandler, + pub on_list_clients: ListClientsHandler, + pub on_partition_read: PartitionReadHandler, + /// Bound by the client-request handler, read by the get-clients + /// handler; the caller keeps it to reach locally-homed sessions. + pub sessions: Rc>, } -pub use server_common::{create_directories, create_shard_executor}; +impl ShellHandlers { + /// Inert handlers for the shell-off fast path: every callback is a + /// no-op over an empty [`SessionManager`]. Behaviorally identical to + /// hand-written no-op closures, so a caller can keep one destructure + /// site across both toggle states. + #[must_use] + pub fn noop() -> Self { + Self { + on_replica_message: Rc::new(|_, _| {}), + on_client_request: Rc::new(|_, _| {}), + on_metadata_submit: Rc::new(|_| {}), + on_list_clients: Rc::new(|_| {}), + on_partition_read: Rc::new(|_, _, _| {}), + sessions: Rc::new(RefCell::new(SessionManager::new())), + } + } +} -pub fn resolve_persister(enforce_fsync: bool) -> Arc { - match enforce_fsync { - true => Arc::new(PersisterKind::FileWithSync(FileWithSyncPersister)), - false => Arc::new(PersisterKind::File(FilePersister)), +/// Build the deferred dispatch handlers for `shard_handle` against `bus`. +/// +/// They share one fresh [`SessionManager`]. The caller must set the weak +/// self-reference in `shard_handle` once the shard is built, so the +/// handlers can upgrade it per frame. +pub fn wire_shell_handlers( + bus: &B, + shard_handle: &ShellShardHandle, + system_config: Arc, + max_tokens_per_user: u32, +) -> ShellHandlers +where + B: ShellBus, + MJ: JournalHandle + 'static, + MJ::Target: Journal, Header = PrepareHeader>, + S: 'static, + SB: SuperblockStore + 'static, +{ + let sessions = Rc::new(RefCell::new(SessionManager::new())); + ShellHandlers { + on_replica_message: make_deferred_replica_message_handler(shard_handle), + on_client_request: make_deferred_client_request_handler( + bus, + shard_handle, + &sessions, + system_config, + max_tokens_per_user, + ), + on_metadata_submit: make_metadata_submit_handler(shard_handle), + on_list_clients: make_list_clients_handler(&sessions), + on_partition_read: make_partition_read_handler(shard_handle), + sessions, } } -pub async fn update_system_info( - storage: &SystemStorage, - system_info: &mut SystemInfo, - version: &SemanticVersion, -) -> Result<(), IggyError> { - system_info.update_version(version); - storage.info.save(system_info).await?; - Ok(()) +pub type ServerShard = ShellShard, PrepareJournal, IggySnapshot>; + +/// Result of a multi-shard bootstrap. +/// +/// Carries the cross-thread shutdown flag and one OS-thread `JoinHandle` +/// per shard. The caller flips the flag via [`Self::install_ctrlc_handler`] +/// and then drains every shard via [`Self::join_all`], bounded by +/// `join_timeout` (`system.sharding.shutdown_join_timeout`). +pub struct ShardHandles { + shutdown_flag: Arc, + shard_threads: Vec<(u16, thread::JoinHandle>)>, + join_timeout: Duration, } -async fn collect_log_files(partition_path: &str) -> Result, IggyError> { - let dir_entries = fs_utils::walk_dir(&partition_path) - .await - .map_err(|_| IggyError::CannotReadPartitions)?; - let mut log_files = Vec::new(); - for entry in dir_entries { - if entry.is_dir { - continue; - } +impl ShardHandles { + /// Install a SIGINT/Ctrl-C handler that flips the shutdown flag on + /// the first signal. A second signal is logged but otherwise + /// ignored so an in-flight WAL fsync or replica drain runs to + /// completion. + /// + /// # Errors + /// + /// Returns the underlying `ctrlc::Error` if the handler cannot be + /// installed (typically because another handler already owns the + /// signal). + pub fn install_ctrlc_handler(&self) -> Result<(), ctrlc::Error> { + let flag = Arc::clone(&self.shutdown_flag); + ctrlc::set_handler(move || { + if flag.swap(true, Ordering::Relaxed) { + // Second Ctrl-C: leave the shutdown machinery to drain. + // Refusing to abort here keeps the WAL fsync / replica + // drain from being interrupted mid-frame. + warn!("second Ctrl-C ignored; server is already shutting down"); + } else { + info!("Ctrl-C received; signalling server shutdown"); + } + }) + } - let extension = entry.path.extension(); - if extension.is_none() || extension.unwrap() != LOG_EXTENSION { - continue; + /// Drain every shard thread. This is the main thread's park for the + /// server's whole lifetime, so shards are awaited WITHOUT any time + /// bound while the server runs; the `shutdown_join_timeout` clock + /// only starts once the cross-thread shutdown flag flips (Ctrl-C or + /// a shard failure). Each shard's outcome is logged (`info` on clean + /// exit, `error` on Err, panic, or wedge). If any shard failed, + /// returns every failure together as + /// [`ServerError::ShardJoinFailures`] so the operator sees the + /// full set rather than just the first. + /// + /// A shard whose thread is still running when the post-shutdown + /// deadline passes is abandoned (its `JoinHandle` dropped, the OS + /// thread left to die with the process) and reported as + /// [`ShardJoinFailureKind::Wedged`]: a wedged pump or listener must + /// not block process exit forever. + /// + /// # Errors + /// + /// Returns [`ServerError::ShardJoinFailures`] if any shard + /// returned a `Result::Err`, panicked, or wedged past the deadline. + /// The variant carries every per-shard failure in shard-id order so + /// the caller does not need to read the trace log to discover + /// late-failing shards. + pub fn join_all(self) -> Result<(), ServerError> { + let mut failures: Vec = Vec::new(); + // Armed on the first poll that observes the shutdown flag, shared + // across all shards: one budget covers the whole drain, not one + // budget per shard. + let mut deadline: Option = None; + // Shards run thread-per-core with compio's blocking fallback pool + // disabled, so an io_uring opcode the kernel lacks aborts every shard + // with the same panic. Surface the actionable diagnostic once. + let mut io_uring_diagnostic_shown = false; + for (shard_id, handle) in self.shard_threads { + let Some(joined) = join_until_shutdown_deadline( + handle, + &self.shutdown_flag, + self.join_timeout, + &mut deadline, + ) else { + error!( + shard_id, + waited = ?self.join_timeout, + "shard thread still running at the shutdown join deadline; abandoning it" + ); + failures.push(ShardJoinFailure { + shard_id, + kind: ShardJoinFailureKind::Wedged { + waited: self.join_timeout, + }, + }); + continue; + }; + match joined { + Ok(Ok(())) => { + info!(shard_id, "shard thread exited cleanly"); + } + Ok(Err(error)) => { + error!(shard_id, error = %error, "shard thread returned error"); + failures.push(ShardJoinFailure { + shard_id, + kind: ShardJoinFailureKind::Error(Box::new(error)), + }); + } + Err(panic_payload) => { + let message = panic_payload_to_string(&*panic_payload); + error!(shard_id, message = %message, "shard thread panicked"); + if !io_uring_diagnostic_shown + && message + .contains(server_common::diagnostics::ASYNCIFY_POOL_DISABLED_PANIC_MSG) + { + server_common::diagnostics::print_incomplete_io_uring_ops_info(); + io_uring_diagnostic_shown = true; + } + failures.push(ShardJoinFailure { + shard_id, + kind: ShardJoinFailureKind::Panic { message }, + }); + } + } + } + if failures.is_empty() { + Ok(()) + } else { + Err(ServerError::ShardJoinFailures { failures }) } + } +} - log_files.push(entry); +/// Poll cadence for the bounded shard joins. Coarse enough to cost +/// nothing during a normal drain, fine enough that exit latency past +/// the last shard's return stays imperceptible. +const JOIN_POLL_INTERVAL: Duration = Duration::from_millis(25); + +/// Join `handle`, waiting indefinitely while the server runs. The +/// `join_timeout` clock starts only when `shutdown_flag` is observed set +/// (arming the caller-shared `deadline` once, so all shards drain under +/// ONE budget); a running server parked here for hours must never be +/// mistaken for a wedged shard. `None` means the thread was still +/// running at the post-shutdown deadline and the handle was dropped +/// (the OS thread keeps running detached; process exit reaps it). +/// `JoinHandle` has no timed join, so this polls `is_finished` at +/// [`JOIN_POLL_INTERVAL`]; the closing `join()` on a finished thread +/// returns immediately. +fn join_until_shutdown_deadline( + handle: thread::JoinHandle>, + shutdown_flag: &AtomicBool, + join_timeout: Duration, + deadline: &mut Option, +) -> Option>> { + while !handle.is_finished() { + if deadline.is_none() && shutdown_flag.load(Ordering::Relaxed) { + *deadline = Some(Instant::now() + join_timeout); + } + if let Some(deadline) = deadline + && Instant::now() >= *deadline + { + return None; + } + thread::sleep(JOIN_POLL_INTERVAL); } + Some(handle.join()) +} - Ok(log_files) +/// Best-effort extraction of the panic message from a +/// `Box` returned by `JoinHandle::join`. Tries the two +/// payload shapes the standard library guarantees (`&'static str` and +/// `String`) and falls back to a placeholder so the panic still surfaces +/// in the error chain. +fn panic_payload_to_string(payload: &(dyn std::any::Any + Send)) -> String { + if let Some(s) = payload.downcast_ref::<&'static str>() { + return (*s).to_string(); + } + if let Some(s) = payload.downcast_ref::() { + return s.clone(); + } + "".to_string() } -pub async fn load_segments( - config: &SystemConfig, - stream_id: usize, - topic_id: usize, - partition_id: usize, - partition_path: String, - stats: Arc, -) -> Result, IggyError> { - let mut log_files = collect_log_files(&partition_path).await?; - log_files.sort_by(|a, b| a.path.file_name().cmp(&b.path.file_name())); - let mut log = SegmentedLog::new(MemoryMessageJournal::empty()); - for entry in log_files { - let log_file_name = entry - .path - .file_stem() - .unwrap() - .to_string_lossy() - .to_string(); - - let start_offset = log_file_name.parse::().unwrap(); - - let messages_file_path = format!("{}/{}.{}", partition_path, log_file_name, LOG_EXTENSION); - let index_file_path = format!("{}/{}.{}", partition_path, log_file_name, INDEX_EXTENSION); - - async fn try_exists(path: &str) -> Result { - match compio::fs::metadata(path).await { - Ok(_) => Ok(true), - Err(err) => match err.kind() { - std::io::ErrorKind::NotFound => Ok(false), - _ => Err(err), - }, +/// Joins survivor shard threads after a partial-spawn failure, bounded +/// by the same `shutdown_join_timeout` budget as the normal exit path. +/// +/// Polls every survivor's `is_finished` in one loop instead of spawning +/// per-survivor joiner threads: the likely OS state on this path is +/// `pthread_create` EAGAIN (the parent spawn just failed with it), so +/// nothing here may create threads, and polling drains all survivors in +/// parallel anyway. A survivor still running at the deadline is +/// abandoned with an error log so the failed bootstrap can surface its +/// spawn error instead of hanging on a wedged shard. +fn join_partial_shard_survivors( + shard_threads: Vec<(u16, thread::JoinHandle>)>, + join_timeout: Duration, +) { + let deadline = Instant::now() + join_timeout; + let mut remaining = shard_threads; + loop { + let mut still_running = Vec::with_capacity(remaining.len()); + for (shard_id, survivor) in remaining { + if survivor.is_finished() { + let _ = survivor.join(); + info!(shard_id, "survivor shard thread drained"); + } else { + still_running.push((shard_id, survivor)); } } - - let index_path_exists = try_exists(&index_file_path).await.unwrap(); - let index_cache_enabled = matches!( - config.segment.cache_indexes, - CacheIndexesConfig::All | CacheIndexesConfig::OpenSegment + remaining = still_running; + if remaining.is_empty() || Instant::now() >= deadline { + break; + } + thread::sleep(JOIN_POLL_INTERVAL); + } + for (shard_id, _survivor) in remaining { + error!( + shard_id, + waited = ?join_timeout, + "survivor shard thread still running at the shutdown join deadline; abandoning it" ); + } +} - if index_cache_enabled && !index_path_exists { - warn!( - "Index at path {} does not exist, rebuilding it based on {}...", - index_file_path, messages_file_path - ); - let now = std::time::Instant::now(); - let index_rebuilder = IndexRebuilder::new( - messages_file_path.clone(), - index_file_path.clone(), - start_offset, - ); - index_rebuilder.rebuild().await.unwrap_or_else(|e| { - panic!( - "Failed to rebuild index for partition with ID: {} for stream with ID: {} and topic with ID: {}. Error: {e}", - partition_id, stream_id, topic_id, - ) - }); - info!( - "Rebuilding index for path {} finished, it took {} ms", - index_file_path, - now.elapsed().as_millis() - ); +/// Flips the cross-thread shutdown flag on `Drop` unless disarmed. +/// +/// A shard thread that exits via an error `?` or a panic unwind would +/// otherwise leave sibling shards parked forever on `bus.token().wait()`: +/// their watchdogs never observe the flag and the bus has no +/// `Drop`-triggered shutdown. Arming this for the whole thread body makes +/// every non-clean exit drive sibling-shard teardown. Disarmed only on a +/// clean `Ok(())`. +struct ShutdownOnDrop { + flag: Arc, + armed: bool, +} + +impl ShutdownOnDrop { + const fn new(flag: Arc) -> Self { + Self { flag, armed: true } + } + + const fn disarm(&mut self) { + self.armed = false; + } +} + +impl Drop for ShutdownOnDrop { + fn drop(&mut self) { + if self.armed { + self.flag.store(true, Ordering::Relaxed); } + } +} - let messages_metadata = compio::fs::metadata(&messages_file_path) - .await - .map_err(|_| IggyError::CannotReadPartitions)?; - let messages_size = messages_metadata.len() as u32; +/// Shard-local end of the metadata bundle handoff. +/// +/// Shard 0 owns the WAL writer and runs `recover()` to build the only +/// `WriteHandle`-bearing [`ServerMuxStateMachine`]. It then mints a +/// [`ServerMetadataBundle`] (a tuple of `Send + Sync` +/// `ReadHandleFactory`s) and pushes one clone per peer onto `bundle_tx`. +/// Every other shard receives the bundle and rebuilds a reader-mode +/// `MuxStateMachine` on its own runtime - no WAL access, no replay, no +/// `RecoverySync` two-phase fence. The old phase-2 WAL fence is gone +/// because peers no longer scan the WAL. They do still scan live shared +/// metadata to load their on-disk partitions, so a separate listener +/// fence is still required - see [`BootstrapBarrier`]. +/// +/// The channel is bounded to the peer count so shard 0's `send` never +/// blocks beyond a peer drain. A peer that dies before recv drops its +/// `bundle_rx`, so shard 0's `send` eventually sees a disconnected +/// channel; the cross-thread shutdown flag drives every waiter out of +/// its `recv` loop if shard 0 panics before broadcasting. +enum MetadataHandoff { + Owner { + bundle_tx: crossfire::MAsyncTx>, + }, + Waiter { + bundle_rx: crossfire::MAsyncRx>, + }, +} - let index_size = match compio::fs::metadata(&index_file_path).await { - Ok(metadata) => metadata.len() as u32, - Err(_) => 0, // Default to 0 if index file doesn't exist - }; +/// Reverse handshake to [`MetadataHandoff`]: gates shard 0's client +/// listeners until every peer has loaded its on-disk partitions. +/// +/// Peers build their owned-partition set from live shared metadata and +/// load each segment from disk in `build_shard_for_thread`. If shard 0 +/// opened listeners the instant `broadcast_metadata_bundle` returned +/// (peers have only *received* the bundle, not *loaded* partitions), a +/// client could create a partition before a peer's load scan finished. +/// That freshly committed partition would surface in the peer's scan +/// with no segment dir on disk yet, and `load_partition`'s `walk_dir` +/// would fail with `CannotReadPartitions`, aborting the whole node. A +/// partition created after boot must take the runtime reconciler path +/// (which creates its dir), never the bootstrap load path. +/// +/// Shard 0 (`Owner`) drains one signal per peer before binding +/// listeners; each peer (`Waiter`) sends one once its load completes. +/// The cross-thread shutdown flag drives both sides out of their poll +/// loop if any shard dies mid-boot. +enum BootstrapBarrier { + Owner { + ready_rx: crossfire::MAsyncRx>, + }, + Waiter { + ready_tx: crossfire::MAsyncTx>, + }, +} + +struct TcpTopology { + /// Domain-separation cluster id derived from `cluster.name`; threaded to + /// every consensus instance and the replica handshake so frames agree. + cluster_id: u128, + self_replica_id: u8, + replica_count: u8, + client_listen_addr: SocketAddr, + replica_listen_addr: Option, + ws_listen_addr: Option, + quic_listen_addr: Option, + http_listen_addr: Option, + tcp_tls_listen_addr: Option, + peers: Vec<(u8, SocketAddr)>, +} + +struct LocalClientAcceptFns { + tcp: AcceptedClientFn, + ws: AcceptedWsClientFn, + quic: AcceptedQuicClientFn, + tcp_tls: AcceptedTlsClientFn, + wss: AcceptedWssClientFn, +} + +#[derive(Default)] +struct BoundClientListeners { + tcp: Option, + tcp_tls: Option, + ws: Option, + quic: Option, +} + +/// Load the server configuration from the active config provider. +/// +/// # Errors +/// +/// Returns an error if the configuration cannot be read or parsed. +pub async fn load_config() -> Result { + ServerConfig::load().await.map_err(ServerError::Config) +} - let storage = Storage::new( - &messages_file_path, - &index_file_path, - messages_size as u64, - index_size as u64, - config.partition.enforce_fsync, - config.partition.enforce_fsync, - true, +/// Prepare the on-disk layout the server boots from and complete late +/// logging init. +/// +/// `fresh` wipes the system path first: `late_init` opens a rolling +/// appender under `{system_path}/logs` and `create_directories` +/// materialises exactly what the wipe is meant to remove, so both have to +/// run after it. +/// +/// # Errors +/// +/// Returns an error if the wipe, directory preparation, or logging setup +/// fails. +pub async fn prepare_runtime_dirs( + config: &ServerConfig, + logging: &mut Logging, + fresh: bool, +) -> Result<(), ServerError> { + if fresh { + wipe_system_path(config).await?; + } + create_directories(&config.system).await.map_err(|source| { + error!( + system_path = %config.system.get_system_path(), + error = %source, + "failed to prepare server directories" + ); + source + })?; + logging + .late_init( + config.system.get_system_path(), + &LoggingSettings::from(&config.system.logging), + &TelemetrySettings::from(&config.telemetry), ) - .await?; + .map_err(ServerError::Logging)?; - let loaded_indexes = { - storage. - index_reader - .as_ref() - .unwrap() - .load_all_indexes_from_disk() - .await - .error(|e: &IggyError| format!("Failed to load indexes during startup for stream ID: {}, topic ID: {}, partition_id: {}, {e}", stream_id, topic_id, partition_id)) - .map_err(|_| IggyError::CannotReadFile)? - }; + Ok(()) +} + +/// Delete the configured system path so the server boots on empty state. +async fn wipe_system_path(config: &ServerConfig) -> Result<(), ServerError> { + let path = config.system.get_system_path(); + // `system.path` is relative by default and IGGY_SYSTEM_PATH-overridable, + // so report what is actually about to be deleted, not what was configured. + let resolved = std::path::absolute(&path).unwrap_or_else(|_| PathBuf::from(&path)); + + if config.cluster.enabled { + warn!( + path = %resolved.display(), + "--fresh wipes only this replica, which then refills from the cluster by \ + state transfer; wiping a quorum at once destroys committed data, and a \ + service unit file carrying --fresh re-transfers everything on every restart" + ); + } + + if !Path::new(&path).exists() { + info!(path = %resolved.display(), "--fresh: system path does not exist, nothing to remove"); + return Ok(()); + } + + warn!(path = %resolved.display(), "--fresh: removing the system path, ALL local data will be deleted"); + // A half-removed directory is worse than no removal at all: the surviving + // superblock and snapshot no longer pair up, and boot would report the + // leftovers as a durability violation rather than as a failed wipe. + remove_dir_all(&path) + .await + .map_err(|source| ServerError::FreshWipeFailed { + path: resolved, + source, + }) +} + +/// Resolve the operator's `cpu_allocation` into concrete shard +/// assignments plus the checked `u16` shard count. +/// +/// Shard ids index `ReplicaOwnerTable` slots as `u16`. `OWNER_NONE` +/// (`u16::MAX`) is reserved as the empty-slot sentinel, so a server +/// configured with `u16::MAX` shards would mint a shard id that +/// collides with the sentinel and an owner-table lookup could never +/// tell that shard apart from an unowned slot. Reject at boot so the +/// invariant is held by the type system, not by hoping the operator +/// never configures 65535 cores worth of shards. +fn resolve_shard_assignments( + sharding: &configs::sharding::ShardingConfig, +) -> Result<(Vec, u16), ServerError> { + let allocator = ShardAllocator::new(&sharding.cpu_allocation, sharding.pin_cores) + .map_err(ServerError::ShardAllocator)?; + let assignments = allocator + .to_shard_assignments() + .map_err(ServerError::ShardAllocator)?; + if assignments.is_empty() { + return Err(ServerError::ShardsCountZero); + } + match u16::try_from(assignments.len()) { + Ok(count) if count < message_bus::OWNER_NONE => Ok((assignments, count)), + _ => Err(ServerError::ShardsCountOverflow { + count: assignments.len(), + }), + } +} + +/// Re-validate the runtime sharding knobs that the per-shard runtime +/// consumes directly. Mirrors `ShardingConfig::validate` so a caller +/// that built the config without running it (e.g. tests, embedded +/// usage) cannot OOM at boot or wedge process exit with an out-of-range +/// value. +fn validate_sharding_runtime_knobs( + sharding: &configs::sharding::ShardingConfig, +) -> Result<(), ServerError> { + let inbox_capacity = sharding.inbox_capacity; + if inbox_capacity == 0 || inbox_capacity > INBOX_CAPACITY_MAX { + return Err(ServerError::InvalidInboxCapacity { + value: inbox_capacity, + max: INBOX_CAPACITY_MAX, + }); + } + let drain_timeout = sharding.shutdown_drain_timeout.get_duration(); + if drain_timeout.is_zero() || drain_timeout > SHUTDOWN_DRAIN_TIMEOUT_MAX { + return Err(ServerError::InvalidShutdownDrainTimeout { + value: drain_timeout, + max: SHUTDOWN_DRAIN_TIMEOUT_MAX, + }); + } + let poll_interval = sharding.shutdown_poll_interval.get_duration(); + if poll_interval.is_zero() || poll_interval > SHUTDOWN_POLL_INTERVAL_MAX { + return Err(ServerError::InvalidShutdownPollInterval { + value: poll_interval, + max: SHUTDOWN_POLL_INTERVAL_MAX, + }); + } + // Ordering: a poll cadence coarser than the drain budget makes the + // cross-thread shutdown flag effectively unobservable during teardown. + if poll_interval > drain_timeout { + return Err(ServerError::ShutdownPollExceedsDrain { + poll: poll_interval, + drain: drain_timeout, + }); + } + Ok(()) +} + +/// Spawn the multi-shard `server` runtime. +/// +/// Resolves shard count + CPU affinities from +/// `system.sharding.cpu_allocation`, builds canonical-ordered +/// `(senders, inboxes)` channels, and spawns one OS thread per shard. +/// +/// Each thread pins itself (`nix::sched::sched_setaffinity` on Linux via +/// [`ShardInfo::bind_cpu`]), binds memory to its NUMA node when +/// configured, builds a fresh `compio::runtime::Runtime` (one +/// `io_uring` instance per shard), and runs `shard_main` inside it. +/// +/// Returns [`ShardHandles`] containing the cross-thread shutdown flag +/// and the per-shard `JoinHandle`s. The caller (`main.rs`) installs a +/// `ctrlc` handler that flips the flag, then `.join()`s every handle. +/// +/// # Errors +/// +/// Returns an error if shard allocation fails, the inbox capacity is +/// invalid, or any OS thread fails to spawn. Per-shard recovery / +/// listener / consensus failures surface through the per-thread `Result` +/// the caller observes on `.join()`. +/// +/// # Panics +/// +/// Panics if [`shard_mesh_channels`] returns an inbox slot already +/// consumed - a bootstrap programming error that would only fire if this +/// function were called twice with the same inboxes. +#[allow(clippy::too_many_lines)] +pub fn bootstrap( + config: ServerConfig, + current_replica_id: Option, +) -> Result { + validate_root_credentials_env(&config)?; + warm_dummy_password_hash(); + // The sync GetStats read path has no access to server config, so capture + // the data directory here for its disk-usage reporting. + crate::responses::init_stats_data_path(config.system.get_system_path().into()); + let (assignments, total_shards) = resolve_shard_assignments(&config.system.sharding)?; + let shards_count = assignments.len(); + + // Re-check the full valid range, not just the zero floor: a caller + // that built the config without running `ShardingConfig::validate` + // would otherwise OOM at boot allocating an oversized inbox channel, + // busy-loop every shutdown watchdog on a zero poll cadence, or wedge + // process exit on an unbounded drain budget. + let inbox_capacity = config.system.sharding.inbox_capacity; + validate_sharding_runtime_knobs(&config.system.sharding)?; + + let (senders, mut inboxes) = shard_mesh_channels(total_shards, inbox_capacity); + let shutdown_flag = Arc::new(AtomicBool::new(false)); + let config = Arc::new(config); + // One owner table per server process, Arc-cloned into every shard's bus so + // any shard's bus reads the same atomic slots that the owning + // shard's installer / disconnect path writes. + let owner_table = Arc::new(ReplicaOwnerTable::new()); - let end_offset = if loaded_indexes.count() == 0 { - start_offset + // Single-shot bundle handoff (see `MetadataHandoff`): shard 0 sends + // one cloned `ServerMetadataBundle` per peer; each peer drains + // exactly one. Bounded to the peer count so shard 0's broadcast + // never blocks past a peer drain. A single-shard deployment (zero + // peers) still needs a non-zero capacity, so clamp up explicitly + // rather than relying on crossfire's internal cap=0 -> 1 promotion. + // If a peer dies before recv, shard 0's `send` eventually sees a + // disconnected channel; the cross-thread shutdown flag drives every + // waiter out of its recv loop if shard 0 panics before broadcasting. + let metadata_peers = shards_count.saturating_sub(1).max(1); + let (metadata_bundle_tx, metadata_bundle_rx) = + crossfire::mpmc::bounded_async::(metadata_peers); + + // Reverse barrier (see `BootstrapBarrier`): every peer sends one + // signal once it finishes loading its on-disk partitions; shard 0 + // drains them all before binding listeners. Bounded to the peer + // count so a sender never blocks (each peer sends exactly once). + let (ready_tx, ready_rx) = crossfire::mpmc::bounded_async::(metadata_peers); + + let mut shard_threads: Vec<(u16, thread::JoinHandle>)> = + Vec::with_capacity(shards_count); + // Shared metadata-group view: written by shard 0's publisher task, read by + // every shard's cluster-metadata roster so leader marking works off-shard. + let metadata_view = Arc::new(AtomicU64::new(crate::cluster_meta::METADATA_VIEW_UNKNOWN)); + for (idx, assignment) in assignments.into_iter().enumerate() { + #[allow(clippy::cast_possible_truncation)] + let shard_id = idx as u16; + let inbox = inboxes[idx] + .take() + .expect("shard_mesh_channels populates every inbox slot exactly once"); + let senders_for_shard = senders.clone(); + let config_for_shard = Arc::clone(&config); + let shutdown_flag_for_shard = Arc::clone(&shutdown_flag); + let owner_table_for_shard = Arc::clone(&owner_table); + let metadata_handoff_for_shard = if shard_id == 0 { + MetadataHandoff::Owner { + bundle_tx: metadata_bundle_tx.clone(), + } } else { - let last_index_offset = loaded_indexes.last().unwrap().offset() as u64; - start_offset + last_index_offset + MetadataHandoff::Waiter { + bundle_rx: metadata_bundle_rx.clone(), + } }; - - let (start_timestamp, end_timestamp) = if loaded_indexes.count() == 0 { - (0, 0) + let barrier_for_shard = if shard_id == 0 { + BootstrapBarrier::Owner { + ready_rx: ready_rx.clone(), + } } else { - ( - loaded_indexes.get(0).unwrap().timestamp(), - loaded_indexes.last().unwrap().timestamp(), - ) + BootstrapBarrier::Waiter { + ready_tx: ready_tx.clone(), + } }; - let mut segment = Segment::new(start_offset, config.segment.size); - - segment.start_timestamp = start_timestamp; - segment.end_timestamp = end_timestamp; - segment.end_offset = end_offset; - segment.size = IggyByteSize::from(messages_size as u64); - // At segment load, set the current position to the size of the segment (No data is buffered yet). - segment.current_position = segment.size.as_bytes_u32(); - segment.sealed = true; // Persisted segments are assumed to be sealed - - if config.partition.validate_checksum { - info!( - "Validating checksum for segment at offset {} in stream ID: {}, topic ID: {}, partition ID: {}", - start_offset, stream_id, topic_id, partition_id - ); - let messages_count = loaded_indexes.count() as u32; - if messages_count > 0 { - const BATCH_COUNT: u32 = 10000; - let mut current_relative_offset = 0u32; - let mut processed_count = 0u32; - - while processed_count < messages_count { - let remaining_count = messages_count - processed_count; - let batch_count = std::cmp::min(BATCH_COUNT, remaining_count); - let batch_indexes = loaded_indexes - .slice_by_offset(current_relative_offset, batch_count) - .unwrap(); - - let messages_reader = storage.messages_reader.as_ref().unwrap(); - match messages_reader.load_messages_from_disk(batch_indexes).await { - Ok(messages_batch) => { - if let Err(e) = messages_batch.validate_checksums() { - return Err(IggyError::CannotReadPartitions).error(|_: &IggyError| { - format!( - "Failed to validate message checksum for segment at offset {} in stream ID: {}, topic ID: {}, partition ID: {}, error: {}", - start_offset, stream_id, topic_id, partition_id, e - ) - }); - } - processed_count += messages_batch.count(); - current_relative_offset += batch_count; - } - Err(e) => { - return Err(e).error(|_: &IggyError| { - format!( - "Failed to load messages from disk for checksum validation at offset {} in stream ID: {}, topic ID: {}, partition ID: {}", - start_offset, stream_id, topic_id, partition_id - ) - }); - } - } - } - info!( - "Checksum validation completed for segment at offset {}", - start_offset + let metadata_view_for_shard = Arc::clone(&metadata_view); + let handle = match thread::Builder::new() + .name(format!("shard-{shard_id}")) + .spawn(move || -> Result<(), ServerError> { + run_shard_thread( + shard_id, + total_shards, + current_replica_id, + assignment, + senders_for_shard, + inbox, + config_for_shard, + shutdown_flag_for_shard, + metadata_handoff_for_shard, + barrier_for_shard, + owner_table_for_shard, + metadata_view_for_shard, + ) + }) { + Ok(handle) => handle, + Err(source) => { + // Signal every shard already spawned before propagating, so + // their watchdog loops drive `bus.shutdown(...)` and the + // process can exit instead of hanging on stuck OS threads. + shutdown_flag.store(true, Ordering::Relaxed); + // Drop bootstrap's own channel clones before joining + // survivors. Otherwise a peer waiting on `bundle_rx.recv` + // would never observe the sender side disconnecting and + // would hang until the shutdown watchdog kicks the bus. + drop(metadata_bundle_tx); + drop(metadata_bundle_rx); + drop(ready_tx); + drop(ready_rx); + join_partial_shard_survivors( + shard_threads, + config.system.sharding.shutdown_join_timeout.get_duration(), ); + return Err(ServerError::ShardSpawnFailed { shard_id, source }); } - } + }; + shard_threads.push((shard_id, handle)); + } - log.add_persisted_segment(segment, storage); + // Drop bootstrap's own channel clones now that every shard owns its + // half. Keeping them on bootstrap's stack would deadlock a peer + // whose `bundle_rx.recv` only completes once every sender + // disconnects. + drop(metadata_bundle_tx); + drop(metadata_bundle_rx); + drop(ready_tx); + drop(ready_rx); - stats.increment_segments_count(1); + info!( + shards_count, + "server bootstrap dispatched; awaiting shard runtimes" + ); - stats.increment_size_bytes(messages_size as u64); + Ok(ShardHandles { + shutdown_flag, + shard_threads, + join_timeout: config.system.sharding.shutdown_join_timeout.get_duration(), + }) +} - let messages_count = if end_offset > start_offset { - (end_offset - start_offset + 1) as u64 - } else if messages_size > 0 { - loaded_indexes.count() as u64 - } else { - 0 - }; +/// Per-shard OS thread entry. Pins CPU + memory, builds the compio +/// runtime, and `block_on`s `shard_main`. +#[allow(clippy::needless_pass_by_value, clippy::too_many_arguments)] +fn run_shard_thread( + shard_id: u16, + total_shards: u16, + replica_id: Option, + assignment: ShardInfo, + senders: Vec, + inbox: ShardReceiver, + config: Arc, + shutdown_flag: Arc, + metadata_handoff: MetadataHandoff, + barrier: BootstrapBarrier, + owner_table: Arc, + metadata_view: Arc, +) -> Result<(), ServerError> { + // Armed for the whole thread body: a post-spawn error `?` or a panic + // unwind here must flip `shutdown_flag` so sibling watchdogs drive + // their bus shutdown instead of parking forever on `bus.token().wait()`. + let mut shutdown_guard = ShutdownOnDrop::new(Arc::clone(&shutdown_flag)); - if messages_count > 0 { - stats.increment_messages_count(messages_count); - } + assignment + .bind_cpu() + .map_err(|source| ServerError::CpuAffinityFailed { shard_id, source })?; + assignment + .bind_memory() + .map_err(|source| ServerError::MemoryAffinityFailed { shard_id, source })?; - let should_cache_indexes = match config.segment.cache_indexes { - CacheIndexesConfig::All => true, - CacheIndexesConfig::OpenSegment => false, - CacheIndexesConfig::None => false, - }; + // `enrich_runtime_create_error` folds the io_uring remediation (raise + // `ulimit -l`, unblock seccomp, kernel-flag floor) into the error, so the + // guidance survives into the shard-join failure report instead of only + // stderr. Multi-shard boxes exhaust RLIMIT_MEMLOCK on per-shard rings + // before the bootstrap runtime does, so this path needs it most. + let runtime = create_shard_executor().map_err(|source| { + let source = server_common::diagnostics::enrich_runtime_create_error(source); + ServerError::ShardRuntimeCreateFailed { shard_id, source } + })?; - if should_cache_indexes { - let segment_index = log.segments().len() - 1; - log.set_segment_indexes(segment_index, loaded_indexes); - } - } + let result = runtime.block_on(async move { + // `shard_main`'s future grows past clippy's `large_futures` cap + // (it ferries the metadata handoff, bus, builders, and inflight + // I/O in one state machine). Heap-pin it so the top-level + // `block_on` future stays small; one allocation per startup buys + // the stack budget back. + Box::pin(shard_main( + shard_id, + total_shards, + replica_id, + senders, + inbox, + &config, + shutdown_flag, + metadata_handoff, + barrier, + owner_table, + metadata_view, + )) + .await + }); - // The last segment is the active one and must remain unsealed for writes - if log.has_segments() { - log.segments_mut().last_mut().unwrap().sealed = false; + if result.is_ok() { + shutdown_guard.disarm(); } + result +} - if matches!( - config.segment.cache_indexes, - CacheIndexesConfig::OpenSegment - ) && log.has_segments() - { - let segments_count = log.segments().len(); - if segments_count > 0 { - let last_storage = log.storages().last().unwrap(); - match last_storage.index_reader.as_ref() { - Some(index_reader) => { - if let Ok(loaded_indexes) = index_reader.load_all_indexes_from_disk().await { - log.set_segment_indexes(segments_count - 1, loaded_indexes); +/// Per-shard async lifecycle. Builds the bus, recovers metadata, +/// constructs the `IggyShard` for this shard's slice of partitions, +/// wires listeners on shard 0, and runs the message pump until +/// shutdown. +#[allow(clippy::too_many_arguments, clippy::too_many_lines)] +async fn shard_main( + shard_id: u16, + total_shards: u16, + replica_id: Option, + senders: Vec, + inbox: ShardReceiver, + config: &ServerConfig, + shutdown_flag: Arc, + metadata_handoff: MetadataHandoff, + barrier: BootstrapBarrier, + owner_table: Arc, + metadata_view: Arc, +) -> Result<(), ServerError> { + let topology = resolve_tcp_topology(config, replica_id)?; + let bus = Rc::new(IggyMessageBus::with_config_and_owner_table( + shard_id, + config, + owner_table, + )); + // Every shard can own a delegated replica connection, so every + // shard's bus needs the handshake identity (the handshake itself + // runs on the owning shard, not on shard 0). + bus.set_replica_handshake_ctx(ReplicaHandshakeCtx { + cluster_id: topology.cluster_id, + self_id: topology.self_replica_id, + replica_count: topology.replica_count, + auth: load_replica_auth(config).map(Rc::new), + tls: load_replica_tls_ctx(config, &topology)?.map(Rc::new), + }); + + let drain_timeout = config.system.sharding.shutdown_drain_timeout.get_duration(); + let poll_interval = config.system.sharding.shutdown_poll_interval.get_duration(); + + let shutdown_flag_for_handoff = Arc::clone(&shutdown_flag); + spawn_shutdown_watchdog(Rc::clone(&bus), shutdown_flag, drain_timeout, poll_interval); + + // Metadata bootstrap is single-writer: shard 0 owns the WAL and the + // only `WriteHandle`-bearing `MuxStateMachine`. Peer shards receive + // a `ReadHandleFactory` bundle on the inter-thread channel and + // rebuild a reader-mode `MuxStateMachine` on their own runtime - no + // WAL access, no replay. Writes still funnel through shard 0's + // metadata VSR; per-commit `publish()` (in `WriteCell::apply`) + // bounds reader staleness to one op. + let data_dir = Path::new(&config.system.path); + let (mux_stm, owner_state) = match metadata_handoff { + MetadataHandoff::Owner { bundle_tx } => { + // Root is created locally at boot (never journaled), so replay + // must start from the same baseline or every WAL-created user + // shifts one slab id and root is lost after the first restart. + let recovered = recover::( + data_dir, + ReplicaIdentity { + cluster: topology.cluster_id, + replica_id: topology.self_replica_id, + replica_count: topology.replica_count, + }, + config.metadata.journal_slots, + config.metadata.clients_table_max, + |mux_stm| { + ensure_default_root_user(mux_stm); + }, + ) + .await + .map_err(ServerError::MetadataRecovery)?; + ensure_default_root_user(&recovered.mux_stm); + // The factory bundle hands every peer a read handle over the + // same `Inner`, so `Arc` (and the parent + // `Arc`) is shared across all shards. Zero the + // snapshot totals here, once, before any peer can observe the + // bundle. Per-shard `load_partition` deltas in + // `build_shard_for_thread` then race only against other + // atomic adds, never against a concurrent `swap(0)` that + // would mistake an in-flight delta for the snapshot total + // and decrement the parent `StreamStats` by it. + let () = recovered.mux_stm.streams().read(|inner| { + for (_, stream) in &inner.items { + for (_, topic) in &stream.topics { + topic.stats.zero_out_all(); } } - None => { - warn!("Index reader not available for last segment in OpenSegment mode"); - } - } + }); + broadcast_metadata_bundle( + shard_id, + &bundle_tx, + recovered.mux_stm.factory_bundle(), + total_shards.saturating_sub(1), + &shutdown_flag_for_handoff, + poll_interval, + ) + .await?; + ( + recovered.mux_stm, + Some(RecoveredOwnerState { + journal: recovered.journal, + snapshot: recovered.snapshot, + last_applied_op: recovered.last_applied_op, + last_journaled_op: recovered.last_journaled_op, + client_table: recovered.client_table, + superblock: recovered.superblock, + recovered_state: recovered.recovered_state, + snapshot_checkpoint: recovered.snapshot_checkpoint, + }), + ) + } + MetadataHandoff::Waiter { bundle_rx } => { + let bundle = await_metadata_bundle( + shard_id, + &bundle_rx, + &shutdown_flag_for_handoff, + poll_interval, + ) + .await?; + (ServerMuxStateMachine::from_factory_bundle(bundle), None) } + }; + + // Metadata consensus + journal + snapshot live only on shard 0. + // `IggyShard::tick_metadata` short-circuits when `consensus.is_none()`, + // so peer shards have no caller that reads `journal` or `snapshot`. + let ( + metadata_consensus, + journal_for_metadata, + snapshot_for_metadata, + superblock_for_metadata, + checkpoint_seed, + recovered_client_table, + ) = if let Some(owner) = owner_state { + // `recover()` already opened the superblock, read `recovered_state`, and + // verified the on-disk snapshot against its checkpoint pairing BEFORE decoding + // it. Reuse that superblock rather than re-opening it, which would fork the + // ping-pong sequence counter. Consensus recovers its true (view, log_view) + // from `recovered_state` instead of inferring a stale view from the WAL. + let consensus = restore_metadata_consensus(&owner, &topology, config, Rc::clone(&bus)); + let superblock = Rc::new(owner.superblock); + ( + Some(consensus), + Some(owner.journal), + owner.snapshot, + Some(superblock), + owner.snapshot_checkpoint, + Some(owner.client_table), + ) + } else { + (None, None, None, None, (0, 0), None) + }; + let metadata = ServerMetadata::new( + metadata_consensus, + journal_for_metadata, + snapshot_for_metadata, + superblock_for_metadata, + mux_stm, + Some(PathBuf::from(&config.system.path)), + ); + // Size the VSR client table before listeners bind and any client registers. + // Must precede the recovered-table install below: the setter rebuilds the + // table from scratch, so running it afterwards would drop every resumed + // session (and trip its empty-table assert). + metadata.set_clients_table_max(config.metadata.clients_table_max); + // Reinstall the sessions recovery restored from the checkpoint and the WAL + // suffix, so a rebooted node dedups retries and admits continuations from + // clients that kept their identity across the restart (IGGY-137). Recovery + // sized this table from the same config value, so the install preserves the + // configured cap. + if let Some(client_table) = recovered_client_table { + // Refusal (a client registered before this ran) keeps the live table + // and is logged by the callee; boot continues either way. + let _ = metadata.install_client_table(client_table); } + // Seed the coordinator's last-checkpoint pairing so the first post-boot + // view-change superblock write records the real (checkpoint_op, checksum) + // instead of (0, 0). No-op on peer shards, which have no coordinator. + metadata.seed_checkpoint_ref(checkpoint_seed.0, checkpoint_seed.1); + // Shard 0's copy resolves the `ServerDefault` sentinels (max topic size and + // message expiry) at create admission; responses echo stored values verbatim. + metadata.set_default_max_topic_size(config.system.topic.max_size.as_bytes_u64()); + metadata.set_default_message_expiry(u64::from(config.system.topic.message_expiry)); + // Keep the forced-checkpoint margin >= the configured prepare-queue + // depth: ops already pipelined while a checkpoint runs append into that + // margin (config validation keeps journal_slots >= 4x this). + metadata.set_checkpoint_margin(config.metadata.checkpoint_margin()); - Ok(log) -} - -/// Builds `InnerMetadata` from persisted user and stream state. -pub fn build_inner_metadata( - users_state: impl IntoIterator, - streams_state: impl IntoIterator, -) -> crate::metadata::InnerMetadata { - use crate::metadata::InnerMetadata; - use std::sync::atomic::AtomicUsize; - - let mut user_entries = Vec::new(); - let mut user_index = ahash::AHashMap::default(); - let mut personal_access_tokens: ahash::AHashMap< - u32, - ahash::AHashMap, PersonalAccessToken>, - > = ahash::AHashMap::default(); - let mut users_count = 0; - let mut pats_count = 0; - - for UserState { - id, - username, - password_hash, - status, - created_at, - permissions, - personal_access_tokens: user_pats, - } in users_state - { - let username_arc: Arc = Arc::from(username.as_str()); - let user_meta = UserMeta { - id, - username: username_arc.clone(), - password_hash: Arc::from(password_hash.as_str()), - status, - permissions: permissions.map(Arc::new), - created_at, - }; - user_entries.push((id as usize, user_meta)); - user_index.insert(username_arc, id); - - if !user_pats.is_empty() { - let user_pat_map: ahash::AHashMap, PersonalAccessToken> = user_pats - .into_values() - .map(|token| { - let pat = PersonalAccessToken::raw( - id, - &token.name, - &token.token_hash, - token.expiry_at, - ); - (Arc::from(token.token_hash.as_str()), pat) - }) - .collect(); - pats_count += user_pat_map.len(); - personal_access_tokens.insert(id, user_pat_map); - } + let shard_metrics = ShardMetrics::for_shard(); + // Notifier install deferred until after tick handler wires below. + let senders_for_notifier = senders.clone(); + let metrics_for_notifier = shard_metrics.clone(); + // Heap-pin like `shard_main` above: the builder future carries the whole + // shard construction state machine and outgrew clippy's `large_futures` + // cap; one allocation per shard startup. + let (shard, sessions) = Box::pin(build_shard_for_thread( + shard_id, + total_shards, + config, + &topology, + metadata, + Rc::clone(&bus), + senders, + inbox, + shard_metrics, + Arc::clone(&metadata_view), + )) + .await?; - users_count += 1; + // Shard 0 owns the metadata consensus; publish its view so every shard's + // cluster-metadata read (and the SDK's leader discovery) marks the live + // primary. Detached: dies with this shard's runtime at process exit. + if shard_id == 0 { + let publisher_shard = Rc::clone(&shard); + let publisher_view = Arc::clone(&metadata_view); + compio::runtime::spawn(async move { + loop { + if let Some(consensus) = publisher_shard.plane.metadata().consensus.as_ref() { + // While this replica declines its recovered view's + // primaryship, that view must not reach the roster: the + // delegated shards would compute a leader that never + // heartbeats. Publish "unknown" until the election + // resolves the role. + let published = if consensus.has_ceded_primaryship() + && consensus.primary_index(consensus.view()) == consensus.replica() + { + crate::cluster_meta::METADATA_VIEW_UNKNOWN + } else { + u64::from(consensus.view()) + }; + publisher_view.store(published, Ordering::Relaxed); + } + compio::time::sleep(std::time::Duration::from_millis(100)).await; + } + }) + .detach(); } + info!( - "Building metadata: {} users, {} personal access tokens", - users_count, pats_count + shard = shard_id, + partitions = shard.plane.partitions().len(), + "server shard initialized" ); - let mut stream_entries = Vec::new(); - let mut stream_index = ahash::AHashMap::default(); - let mut streams_count = 0; - let mut topics_count = 0; - let mut partitions_count = 0; - let mut consumer_groups_count = 0; - - for StreamState { - name, - created_at, - id, - topics, - } in streams_state - { - info!( - "Building stream with ID: {}, name: {} metadata...", - id, name - ); - let stream_id = id as usize; - let stream_name: Arc = Arc::from(name.as_str()); - - let stream_stats = Arc::new(StreamStats::default()); - - let mut topic_entries = Vec::new(); - let mut topic_index = ahash::AHashMap::default(); - - for TopicState { - id, - name, - created_at, - compression_algorithm, - message_expiry, - max_topic_size, - replication_factor, - consumer_groups, - partitions, - } in topics.into_values() - { - info!("Building topic with ID: {}, name: {} metadata...", id, name); - let topic_id = id as usize; - let topic_name: Arc = Arc::from(name.as_str()); - - let topic_stats = Arc::new(TopicStats::new(stream_stats.clone())); - - let mut partition_entries = Vec::new(); - let mut partition_ids = Vec::new(); - - for partition_state in partitions.into_values() { - let partition_id = partition_state.id as usize; - partition_ids.push(partition_id); - - let partition_stats = Arc::new(PartitionStats::new(topic_stats.clone())); - let partition_meta = PartitionMeta { - id: partition_id, - created_at: partition_state.created_at, - revision_id: 0, - stats: partition_stats, - consumer_offsets: Arc::new(ConsumerOffsets::with_capacity(0)), - consumer_group_offsets: Arc::new(ConsumerGroupOffsets::with_capacity(0)), - last_polled_offsets: Arc::new(papaya::HashMap::new()), - }; - partition_entries.push((partition_id, partition_meta)); - partitions_count += 1; - } + // Re-check the cross-thread shutdown flag here, *before* spawning the + // message pump. A sibling shard may have failed in the window between + // the metadata broadcast and this point; gating before spawn keeps the + // bus' `background_tasks` vec empty on the shutdown path. Spawn-then- + // check would leave `bus.track_background(pump_handle)` registering a + // `JoinHandle` that only `bus.shutdown()` drains, but the watchdog + // driving `bus.shutdown()` is `.detach()`'d (see TODO at + // `spawn_shutdown_watchdog`) and may not be scheduled before this + // function returns `Ok(())` and the compio runtime drops, cancelling + // the pump mid-`write_vectored_all`. + // + // Without this gate shard 0 would also still open TCP/QUIC/WS + // listeners for a server that is already tearing down, briefly + // accepting connections that immediately get torn by the watchdog. + if shutdown_flag_for_handoff.load(Ordering::Relaxed) { + return Ok(()); + } + + // Tick handler must install before the notifier so early commits + // do not broadcast ticks whose handler slot is still `None`. + let (reconcile_wake_tx, reconcile_wake_rx) = channel::<()>(1); + let (reconcile_stop_tx, reconcile_stop_rx) = channel::<()>(1); + crate::partition_reconciler::install_tick_handler(&shard, reconcile_wake_tx); + + // Only shard 0 commits metadata. + if shard_id == 0 { + let notifier = make_metadata_commit_notifier(senders_for_notifier, metrics_for_notifier); + shard.plane.metadata().set_commit_notifier(Some(notifier)); + } else { + drop(senders_for_notifier); + drop(metrics_for_notifier); + } - partition_ids.sort_unstable(); + // The pump task also drives the consensus timer tick (heartbeats, prepare + // retransmit, view-change timeouts) as a select! arm, serialized with frame + // processing - see `run_message_pump`. + let (stop_tx, stop_rx) = channel(1); + let pump_shard = Rc::clone(&shard); + // Owned and awaited by shard_main at exit, NOT `track_background`: the + // background drain runs inside `bus.shutdown()`, which the Ctrl-C path + // never drives (the watchdog stands down when the token fires), so a + // tracked pump would be cancelled by runtime teardown mid final-flush + // and every graceful shutdown would silently drop the committed journal + // tail that had not hit a flush threshold yet. + let mut pump_handle = Some(compio::runtime::spawn(async move { + pump_shard.run_message_pump(stop_rx).await; + })); - let mut cg_entries = Vec::new(); - let mut cg_index = ahash::AHashMap::default(); + let reconciler_ctx = Rc::new(crate::partition_reconciler::ReconcilerCtx::new( + Rc::clone(&shard), + total_shards, + Rc::new(config.clone()), + topology.cluster_id, + topology.self_replica_id, + topology.replica_count, + )); + let reconcile_periodic = config + .system + .sharding + .reconcile_periodic_interval + .get_duration(); + let reconciler_handle = compio::runtime::spawn({ + let ctx = Rc::clone(&reconciler_ctx); + async move { + crate::partition_reconciler::run_reconciler( + ctx, + reconcile_wake_rx, + reconcile_stop_rx, + reconcile_periodic, + ) + .await; + } + }); + bus.track_background(reconciler_handle); - for cg_state in consumer_groups.into_values() { - info!( - "Building consumer group with ID: {}, name: {} for topic with ID: {} metadata...", - cg_state.id, cg_state.name, topic_id - ); - let group_id = cg_state.id as usize; - let group_name: Arc = Arc::from(cg_state.name.as_str()); - let cg_meta = ConsumerGroupMeta { - id: group_id, - name: group_name.clone(), - partitions: partition_ids.clone(), - members: Slab::new(), - }; - cg_entries.push((group_id, cg_meta)); - cg_index.insert(group_name, group_id); - consumer_groups_count += 1; + // Per-shard heartbeat verifier: evicts connections that stop pinging, + // releasing their consumer-group membership. Gated on config so a + // deployment without heartbeats never reaps live sessions. + let heartbeat_stop_tx = if config.heartbeat.enabled { + let (hb_stop_tx, hb_stop_rx) = channel::<()>(1); + let hb_shard = Rc::clone(&shard); + let hb_sessions = Rc::clone(&sessions); + let hb_interval = config.heartbeat.interval.get_duration(); + let hb_handle = compio::runtime::spawn(async move { + crate::dispatch::run_heartbeat_verifier(hb_shard, hb_sessions, hb_interval, hb_stop_rx) + .await; + }); + bus.track_background(hb_handle); + Some(hb_stop_tx) + } else { + None + }; + // Expired-PAT cleaner: shard 0 only (it owns the metadata consensus + // group) and only when enabled. Each pass no-ops unless this node is + // the caught-up metadata primary, so the delete is proposed once and + // replicated to every replica. + let pat_cleaner_stop = if shard_id == 0 && config.personal_access_token.cleaner.enabled { + let (cleaner_stop_tx, cleaner_stop_rx) = channel(1); + let cleaner_shard = Rc::clone(&shard); + let interval = config.personal_access_token.cleaner.interval.get_duration(); + let cleaner_handle = compio::runtime::spawn(async move { + crate::personal_access_token_cleaner::run_pat_cleaner( + cleaner_shard, + cleaner_stop_rx, + interval, + ) + .await; + }); + bus.track_background(cleaner_handle); + Some(cleaner_stop_tx) + } else { + None + }; + + // Segment cleaner: runs on every shard (each replica trims its own log, + // primary and backup alike). Local and unreplicated; gated by the shared + // data-maintenance config. + let segment_cleaner_stop = if config.data_maintenance.messages.cleaner_enabled { + let (stop_tx, stop_rx) = channel(1); + let cleaner_shard = Rc::clone(&shard); + let interval = config.data_maintenance.messages.interval.get_duration(); + let cleaner_handle = compio::runtime::spawn(async move { + crate::segment_cleaner::run_segment_cleaner(cleaner_shard, stop_rx, interval).await; + }); + bus.track_background(cleaner_handle); + Some(stop_tx) + } else { + None + }; + + // One keep-alive per process, so shard 0 owns it. Started before the + // listeners bind: systemd counts `WatchdogSec=` from unit start, not from + // `READY=1`, so a slow recovery must not look like a hang. + #[cfg(feature = "systemd")] + if shard_id == 0 { + crate::systemd::spawn_watchdog(&bus); + } + + // Listener fence (see `BootstrapBarrier`). Peers still scan live + // shared metadata and load their on-disk partitions in + // `build_shard_for_thread`; the factory-bundle handoff only proves + // they *received* the bundle, not that they finished loading. Shard + // 0 must not accept client traffic until every peer's load scan is + // done, otherwise a partition created by the first client surfaces + // in a still-running scan with no segment dir on disk and aborts the + // node with `CannotReadPartitions`. By this point every shard has + // also spawned its pump + reconciler, so a partition created after + // the fence takes the runtime reconciler path on its owning shard. + match barrier { + BootstrapBarrier::Owner { ready_rx } => { + await_bootstrap_complete( + &ready_rx, + usize::from(total_shards.saturating_sub(1)), + &shutdown_flag_for_handoff, + poll_interval, + ) + .await?; + } + BootstrapBarrier::Waiter { ready_tx } => { + signal_bootstrap_complete( + shard_id, + &ready_tx, + &shutdown_flag_for_handoff, + poll_interval, + ) + .await?; + } + } + + // Listeners (replica + every client transport) bind on shard 0 only. + // Shard 0's coordinator round-robins inbound TCP/WS connections to + // peer shards via fd-transfer. QUIC and TCP-TLS clients terminate + // locally on shard 0 (their per-connection state is non-portable - + // see `LifecycleFrame::ClientWsConnectionSetup` rustdoc). + if shard_id == 0 { + let coord = shard + .coordinator() + .expect("shard 0 always has a coordinator attached by the builder"); + // Reseed the client-id minter above every recovered entry before any + // listener accepts. The counter is per process; the table it must not + // collide with was rebuilt from the previous boot's WAL. Keyed by view + // so a later promotion refolds the table (the minting path calls the + // same method, see `HttpInner::register_session_once`). + let boot_view = shard + .plane + .metadata() + .consensus + .as_ref() + .map_or(0, consensus::VsrConsensus::view); + coord.seed_client_sequence( + boot_view, + shard.plane.metadata().client_table.borrow().client_ids(), + ); + let on_client_request = make_client_request_handler( + &shard, + &sessions, + Arc::clone(&config.system), + config.personal_access_token.max_tokens_per_user, + ); + let (accepted_replica, dialed_replica) = + make_replica_delegation_fns(Rc::clone(&coord), &bus); + let accepted_client = make_shard_zero_client_accept_fns(coord, &bus, on_client_request); + + if let Err(error) = start_tcp_runtime( + &shard, + config, + &topology, + accepted_replica, + dialed_replica, + accepted_client, + ) + .await + { + let _ = stop_tx.try_send(()); + let _ = reconcile_stop_tx.try_send(()); + if let Some(tx) = &heartbeat_stop_tx { + let _ = tx.try_send(()); + } + if let Some(cleaner_stop_tx) = &pat_cleaner_stop { + let _ = cleaner_stop_tx.try_send(()); + } + if let Some(tx) = &segment_cleaner_stop { + let _ = tx.try_send(()); } + await_pump_drain(pump_handle.take(), config, shard_id).await; + return Err(error); + } - let topic_meta = TopicMeta { - id: topic_id, - name: topic_name.clone(), - created_at, - message_expiry, - compression_algorithm, - max_topic_size, - replication_factor: replication_factor.unwrap_or(1), - stats: topic_stats, - partitions: partition_entries.into_iter().map(|(_, p)| p).collect(), - consumer_groups: cg_entries.into_iter().collect(), - consumer_group_index: cg_index, - round_robin_counter: Arc::new(AtomicUsize::new(0)), - }; - topic_entries.push((topic_id, topic_meta)); - topic_index.insert(topic_name, topic_id); - topics_count += 1; - } - - let stream_meta = StreamMeta { - id: stream_id, - name: stream_name.clone(), - created_at, - stats: stream_stats, - topics: topic_entries.into_iter().collect(), - topic_index, - }; - stream_entries.push((stream_id, stream_meta)); - stream_index.insert(stream_name, stream_id); - streams_count += 1; + // Every enabled client transport is bound and accepting by here, so + // this is the first point at which a unit ordered after us may dial. + #[cfg(feature = "systemd")] + crate::systemd::notify_ready(); } - info!( - "Built metadata: {} streams, {} topics, {} partitions, {} consumer groups", - streams_count, topics_count, partitions_count, consumer_groups_count - ); + bus.token().wait().await; + #[cfg(feature = "systemd")] + if shard_id == 0 { + crate::systemd::notify_stopping(); + } + let _ = stop_tx.try_send(()); + let _ = reconcile_stop_tx.try_send(()); + if let Some(tx) = &heartbeat_stop_tx { + let _ = tx.try_send(()); + } + if let Some(cleaner_stop_tx) = &pat_cleaner_stop { + let _ = cleaner_stop_tx.try_send(()); + } + if let Some(tx) = &segment_cleaner_stop { + let _ = tx.try_send(()); + } + + await_pump_drain(pump_handle.take(), config, shard_id).await; + + info!(shard = shard_id, "server shard exited cleanly"); + Ok(()) +} - InnerMetadata { - streams: stream_entries.into_iter().collect(), - users: user_entries.into_iter().collect(), - stream_index, - user_index, - personal_access_tokens, - users_global_permissions: Default::default(), - users_stream_permissions: Default::default(), - users_can_poll_all_streams: Default::default(), - users_can_send_all_streams: Default::default(), - users_can_poll_stream: Default::default(), - users_can_send_stream: Default::default(), - } -} - -/// Loads all metadata from persisted state into metadata writer. -/// Single atomic initialization via `Initialize` operation. -pub fn load_metadata( - users_state: impl IntoIterator, - streams_state: impl IntoIterator, - writer: &mut MetadataWriter, +/// Await the message pump's completion before the shard returns: its +/// post-loop work includes the final flush of every committed journal to +/// segment storage, and returning first drops the compio runtime, which +/// cancels that flush at its next await point. +async fn await_pump_drain( + pump_handle: Option>, + config: &ServerConfig, + shard_id: u16, ) { - let inner = build_inner_metadata(users_state, streams_state); - writer.initialize(inner); + let Some(pump_handle) = pump_handle else { + return; + }; + let drain_budget = config.system.sharding.shutdown_drain_timeout.get_duration(); + if compio::time::timeout(drain_budget, pump_handle) + .await + .is_err() + { + warn!( + shard = shard_id, + "message pump did not drain within the shutdown budget; \ + committed journal tail may not have flushed" + ); + } +} + +/// Block until shard 0 broadcasts the metadata factory bundle, or the +/// cross-thread shutdown flag flips. Polled in a `poll_interval` loop +/// so a shard 0 that panics before it broadcasts cannot strand peer +/// shards: the shutdown path flips the flag, every waiter observes it +/// on the next tick, and the server tears down instead of hanging. +/// +/// Uses `try_recv` + sleep rather than `timeout(recv())`. Crossfire 3.x +/// documents `recv()` as cancellation-safe (no leak/deadlock) but does +/// not guarantee atomicity for the dropped future's result; `try_recv` +/// keeps each tick fully synchronous and side-effect-free, so the +/// shutdown poll cadence cannot ambiguously consume a bundle. +async fn await_metadata_bundle( + shard_id: u16, + bundle_rx: &crossfire::MAsyncRx>, + shutdown_flag: &Arc, + poll_interval: Duration, +) -> Result { + loop { + match bundle_rx.try_recv() { + Ok(bundle) => return Ok(bundle), + Err(crossfire::TryRecvError::Disconnected) => { + return Err(ServerError::MetadataHandoffAborted { shard_id }); + } + Err(crossfire::TryRecvError::Empty) => { + if shutdown_flag.load(Ordering::Relaxed) { + return Err(ServerError::MetadataHandoffAborted { shard_id }); + } + compio::time::sleep(poll_interval).await; + } + } + } +} + +/// Push `peers` cloned bundles onto `bundle_tx`, polling each send in a +/// `poll_interval` loop so the cross-thread shutdown flag can interrupt +/// a stalled handoff. Symmetric to [`await_metadata_bundle`]: shutdown +/// observed mid-handshake aborts cleanly rather than stalling on a +/// `send` future that can no longer make progress. +/// +/// Uses `try_send` + sleep rather than `timeout(send())`. Crossfire 3.x +/// documents `send()` as cancellation-safe in the leak/deadlock sense +/// but explicitly warns the true result is unknown when `SendFuture` is +/// dropped on cancellation. For a retry loop that re-clones on every +/// tick that would risk publishing the same bundle twice, stuffing the +/// bounded channel past `peers` and stranding a follow-up `send`. +/// `try_send` returns the bundle back inside `TrySendError::Full`, so +/// the loop reuses it instead of re-cloning when the channel is full. +async fn broadcast_metadata_bundle( + shard_id: u16, + bundle_tx: &crossfire::MAsyncTx>, + bundle: ServerMetadataBundle, + peers: u16, + shutdown_flag: &Arc, + poll_interval: Duration, +) -> Result<(), ServerError> { + for _ in 0..peers { + let mut pending = bundle.clone(); + loop { + match bundle_tx.try_send(pending) { + Ok(()) => break, + Err(crossfire::TrySendError::Disconnected(_)) => { + // Every peer dropped its `bundle_rx` before recv. Shard + // 0 must not silently continue past handoff: it would + // bind listeners and commit consensus state for a + // cluster whose peers are gone. Propagate the abort so + // `shard_main` short-circuits before further side + // effects; `shutdown_flag` will flip via the normal + // teardown path. + return Err(ServerError::MetadataHandoffAborted { shard_id }); + } + Err(crossfire::TrySendError::Full(returned)) => { + if shutdown_flag.load(Ordering::Relaxed) { + return Err(ServerError::MetadataHandoffAborted { shard_id }); + } + pending = returned; + compio::time::sleep(poll_interval).await; + } + } + } + } + Ok(()) +} + +/// Peer side of [`BootstrapBarrier`]: tell shard 0 this shard finished +/// loading its on-disk partitions. Mirrors [`broadcast_metadata_bundle`]'s +/// `try_send`-or-shutdown poll loop so a sibling failure (which flips the +/// shutdown flag) drives this out instead of stranding it on a full +/// channel. The channel is sized to the peer count and each peer sends +/// exactly once, so `Full` is not expected; the branch only keeps the +/// loop interruptible. +async fn signal_bootstrap_complete( + shard_id: u16, + ready_tx: &crossfire::MAsyncTx>, + shutdown_flag: &Arc, + poll_interval: Duration, +) -> Result<(), ServerError> { + let mut pending = shard_id; + loop { + match ready_tx.try_send(pending) { + Ok(()) => return Ok(()), + Err(crossfire::TrySendError::Disconnected(_)) => { + // Shard 0 dropped its `ready_rx` before draining (it + // aborted before binding listeners). Propagate so this + // shard short-circuits; the shutdown flag flips via the + // normal teardown path. + return Err(ServerError::MetadataHandoffAborted { shard_id }); + } + Err(crossfire::TrySendError::Full(returned)) => { + if shutdown_flag.load(Ordering::Relaxed) { + return Err(ServerError::MetadataHandoffAborted { shard_id }); + } + pending = returned; + compio::time::sleep(poll_interval).await; + } + } + } +} + +/// Owner side of [`BootstrapBarrier`]: drain one ready signal per peer +/// before shard 0 binds listeners. Polls the shutdown flag so a peer that +/// dies mid-load (flipping the flag) aborts the wait instead of hanging on +/// a signal that will never arrive. A single shard (`peers == 0`) returns +/// immediately. +async fn await_bootstrap_complete( + ready_rx: &crossfire::MAsyncRx>, + peers: usize, + shutdown_flag: &Arc, + poll_interval: Duration, +) -> Result<(), ServerError> { + let mut remaining = peers; + while remaining > 0 { + match ready_rx.try_recv() { + Ok(_shard_id) => remaining -= 1, + Err(crossfire::TryRecvError::Disconnected) => { + return Err(ServerError::ShardBootstrapBarrierAborted { remaining }); + } + Err(crossfire::TryRecvError::Empty) => { + if shutdown_flag.load(Ordering::Relaxed) { + return Err(ServerError::ShardBootstrapBarrierAborted { remaining }); + } + compio::time::sleep(poll_interval).await; + } + } + } + Ok(()) +} + +/// Spawn a per-shard polling task that watches the cross-thread shutdown +/// flag and triggers this shard's bus shutdown on transition. The flag +/// is the only Send signal we have; the bus' shutdown machinery is +/// `!Send` (`Rc>` + per-shard `async_channel`), so it must be +/// triggered from within the runtime that owns the bus. +#[allow(clippy::needless_pass_by_value)] +fn spawn_shutdown_watchdog( + bus: Rc, + shutdown_flag: Arc, + drain_timeout: Duration, + poll_interval: Duration, +) { + let bus_for_task = Rc::clone(&bus); + let bus_token = bus.token(); + let watchdog = compio::runtime::spawn(async move { + loop { + if shutdown_flag.load(Ordering::Relaxed) { + break; + } + if bus_token.is_triggered() { + // Bus shutdown was driven from elsewhere (e.g. internal + // failure path). Watchdog has nothing left to do. + return; + } + compio::time::sleep(poll_interval).await; + } + let _ = bus_for_task.shutdown(drain_timeout).await; + }); + // TODO(hubcio): `.detach()` races bus shutdown: when `bus.token()` is + // triggered, `shard_main` returns and the runtime drops the watchdog + // mid-`bus.shutdown()`, truncating in-flight `ClientForwardFailed` + // replies (terminal per `SendError` docs). Cannot use + // `bus.track_background(watchdog)` here because the watchdog itself + // drives `bus.shutdown()`, and the bg-drain loop in `shutdown()` + // would re-enter awaiting the watchdog's own pending shutdown call + // (self-deadlock). Fix: extract a `core/task_registry` crate mirroring + // `core/server`'s task-tracking mechanism, share it between the bus + // and server so background tasks can be reaped without coupling + // to the bus shutdown order. + watchdog.detach(); +} + +/// Copy the configured cluster roster plus this node's own client ports into +/// the shared [`ClusterRoster`] so the binary `GetClusterMetadata` read serves +/// the real topology. `self_*` back only the cluster-disabled self-synthesis +/// and carry the requested listener ports from the resolved topology, not the +/// bound ones (a `:0` wildcard is reported as 0). +fn build_cluster_roster( + config: &ServerConfig, + topology: &TcpTopology, + metadata_view: Arc, +) -> ClusterRoster { + ClusterRoster { + enabled: config.cluster.enabled, + name: config.cluster.name.clone(), + nodes: config + .cluster + .nodes + .iter() + .cloned() + .map(Into::into) + .collect(), + self_ip: topology.client_listen_addr.ip().to_string(), + self_ports: configs::cluster::TransportPorts { + tcp: Some(topology.client_listen_addr.port()), + quic: topology.quic_listen_addr.map(|addr| addr.port()), + http: topology.http_listen_addr.map(|addr| addr.port()), + websocket: topology.ws_listen_addr.map(|addr| addr.port()), + tcp_replica: None, + }, + metadata_view, + } +} + +#[allow(clippy::too_many_arguments, clippy::too_many_lines)] +async fn build_shard_for_thread( + shard_id: u16, + total_shards: u16, + config: &ServerConfig, + topology: &TcpTopology, + metadata: ServerMetadata, + bus: Rc, + senders: Vec, + inbox: ShardReceiver, + metrics: ShardMetrics, + metadata_view: Arc, +) -> Result<(Rc, Rc>), ServerError> { + let shard_local_id = ShardId::new(shard_id); + let total_partitions = metadata.mux_stm.streams().read(|inner| { + inner + .items + .iter() + .map(|(_, stream)| { + stream + .topics + .iter() + .map(|(_, topic)| topic.partitions.len()) + .sum::() + }) + .sum::() + }); + + // IggyPartitions holds only the partitions owned by this shard + // (see the filter below at insert time), so the server-wide total + // is an N-fold overshoot. `ceil(total / shards) * 2` is a coarse + // upper bound that absorbs hash skew without paying the full + // multiplier. PapayaShardsTable below stays sized to the server-wide + // total because every shard routes every namespace. + let owned_partitions_capacity = total_partitions + .div_ceil(usize::from(total_shards).max(1)) + .saturating_mul(2); + // At-rest encryption: built once per shard from the shared config; the + // ingestion path encrypts on the primary and the poll reply decrypts. + // A bad key fails the boot rather than silently serving plaintext. + let encryptor = if config.system.encryption.enabled { + let aes = Aes256GcmEncryptor::from_base64_key(&config.system.encryption.key) + .map_err(|error| ServerError::Iggy(Box::new(error)))?; + Some(Arc::new(EncryptorKind::Aes256Gcm(aes))) + } else { + None + }; + let partitions = IggyPartitions::with_capacity( + shard_local_id, + PartitionsConfig { + messages_required_to_save: config.system.partition.messages_required_to_save, + size_of_messages_required_to_save: config + .system + .partition + .size_of_messages_required_to_save, + enforce_fsync: config.system.partition.enforce_fsync, + validate_checksum: config.system.partition.validate_checksum, + segment_size: config.system.segment.size, + preallocate_segments: config.system.segment.preallocate, + encryptor, + }, + owned_partitions_capacity, + ); + let shards_table = PapayaShardsTable::with_capacity(total_partitions); + + // Stream-filter inside the `read()` closure: only partitions owned by + // this shard need the heavy (`Arc` + `Partition`) clones + // for the async `load_partition` below. Non-owning entries are pushed + // straight into `shards_table` here, so no Vec scales with the + // server-wide partition count. + let owned = metadata.mux_stm.streams().read(|inner| { + let mut owned = Vec::with_capacity(owned_partitions_capacity); + for (_, stream) in &inner.items { + for (topic_id, topic) in &stream.topics { + for partition in &topic.partitions { + let namespace = IggyNamespace::new(stream.id, topic_id, partition.id); + let owning_shard = + calculate_shard_assignment(&namespace, u32::from(total_shards)); + if owning_shard == shard_id { + // Shared per-partition stats from the registry: the + // same `Arc` backs every shard's `get_topic` reply. + let stats = inner.stats_registry.partition( + stream.id, + topic_id, + partition.id, + topic.stats.clone(), + ); + owned.push((stream.id, topic_id, stats, partition.clone())); + } else { + shards_table.insert( + namespace, + PartitionLocation::new( + ShardId::new(owning_shard), + partition.created_revision, + ), + ); + } + } + } + } + owned + }); + + // Snapshot totals were zeroed once on shard 0 before the factory + // bundle was broadcast (see `MetadataHandoff::Owner`). All shards + // here only add their per-partition deltas, so the shared + // `Arc` atomics race only against other atomic adds. + for (stream_id, topic_id, partition_stats, partition_metadata) in owned { + validate_namespace_bounds(config, stream_id, topic_id, partition_metadata.id)?; + let namespace = IggyNamespace::new(stream_id, topic_id, partition_metadata.id); + let partition = match load_partition( + config, + namespace, + Arc::clone(&partition_stats), + &partition_metadata, + topology.cluster_id, + topology.self_replica_id, + topology.replica_count, + Rc::clone(&bus), + ) + .await + { + Ok(partition) => partition, + // ONE damaged local chain must not take the node down. The shapes + // this refuses are exactly what a failed state-transfer quarantine + // leaves behind, so fence that group the same way the runtime path + // does -- move its segment files aside, keeping the superblock so it + // cannot re-enter view 0 -- and materialise it fresh. The ordinary + // rejoin path (repair, then state transfer on a refused floor) + // recovers its data from a peer. + Err(ServerError::PartitionChainRefused { dir, reason, .. }) => { + let partition_dir = dir.to_string_lossy().into_owned(); + error!( + stream_id, + topic_id, + partition_id = partition_metadata.id, + partition_dir, + %reason, + "refusing the recovered segment chain; fencing this partition and \ + rebuilding it empty for the rejoin path" + ); + match partitions::state_transfer::quarantine_segment_files(&partition_dir).await { + Ok(fenced_dir) => error!( + stream_id, + topic_id, + partition_id = partition_metadata.id, + fenced_dir, + "quarantined the refused segment files; they are kept for inspection" + ), + Err(error) => { + // NOT rebuilt: `build_partition_fresh` reaches + // `ensure_initial_segment`, which opens segment 0 with + // `file_exists = false` and TRUNCATES whatever the + // failed quarantine left behind. The likeliest failures + // (suffix cap exhausted, `create_dir_all`) move zero + // files, so rebuilding would destroy the oldest segment + // on the first attempt while the higher-offset survivors + // keep refusing every boot -- a loop that never + // terminates and eats the chain one segment at a time. + // Tombstone instead: the namespace stays unmaterialised + // and unrouted, the reconciler backs off, and an + // operator still has every byte. + error!( + stream_id, + topic_id, + partition_id = partition_metadata.id, + partition_dir, + %error, + "failed to quarantine the refused segment files; leaving this \ + partition tombstoned rather than rebuilding over them" + ); + partition_stats.zero_out_all(); + partitions.tombstone(namespace); + continue; + } + } + // The refused load already folded its segment counts in. + partition_stats.zero_out_all(); + build_partition_fresh( + config, + namespace, + partition_stats, + partition_metadata.created_revision, + topology.cluster_id, + topology.self_replica_id, + topology.replica_count, + Rc::clone(&bus), + ) + .await? + } + // An untrustworthy superblock fences ONE group, not the node. The + // segment files stay exactly where they are -- unlike a refused + // chain, the data on disk is not the thing in doubt -- so there is + // nothing to quarantine and nothing to rebuild: rebuilding fresh + // would hand this replica a view-0 identity while a record it + // cannot read says otherwise. Tombstoned, the namespace stays + // unmaterialised and unrouted, the reconciler backs off, and an + // operator has every byte plus a message naming the directory. + Err( + error @ (ServerError::PartitionSuperblockIo { .. } + | ServerError::PartitionSuperblockVersionUnknown { .. } + | ServerError::PartitionSuperblockUnverifiable { .. } + | ServerError::PartitionSuperblockUndecodable { .. } + | ServerError::PartitionSuperblockIdentityMismatch { .. }), + ) => { + error!( + stream_id, + topic_id, + partition_id = partition_metadata.id, + %error, + "cannot trust this partition's durable consensus state; tombstoning the \ + partition and continuing to boot the rest of the shard" + ); + partition_stats.zero_out_all(); + partitions.tombstone(namespace); + continue; + } + Err(error) => return Err(error), + }; + partitions.insert(namespace, partition); + shards_table.insert( + namespace, + PartitionLocation::new(ShardId::new(shard_id), partition_metadata.created_revision), + ); + } + + let shard_handle = Rc::new(RefCell::new(None)); + // Same wiring path as the simulator's shell mode: one per-shard + // SessionManager shared by the client-request handler (binds sessions) + // and the get_clients handler (reads them). It also carries this shard's + // cluster roster for the pre-auth GetClusterMetadata read. + let ShellHandlers { + on_replica_message, + on_client_request, + on_metadata_submit, + on_list_clients, + on_partition_read, + sessions, + } = wire_shell_handlers( + &bus, + &shard_handle, + Arc::clone(&config.system), + config.personal_access_token.max_tokens_per_user, + ); + sessions + .borrow_mut() + .set_cluster_roster(Rc::new(build_cluster_roster( + config, + topology, + metadata_view, + ))); + let shard_name = format!("server-shard-{shard_id}"); + let built = IggyShardBuilder::new( + ShardIdentity::new(shard_id, shard_name), + Rc::clone(&bus), + on_replica_message, + on_client_request, + on_metadata_submit, + on_list_clients, + on_partition_read, + metadata, + partitions, + senders, + inbox, + shards_table, + PartitionConsensusConfig::new( + topology.cluster_id, + shard::ReplicaTopology::new(topology.self_replica_id, topology.replica_count), + Rc::clone(&bus), + ), + CoordinatorConfig::default(), + metrics, + ) + .build() + .map_err(ServerError::ShardConstruction)?; + + let shard = Rc::new(built.shard); + // Repair pacing is shared by both planes' repair loops, so it is a + // per-shard tunable set once here rather than per consensus group. + shard.set_repair_retry_ticks(repair_retry_ticks(config)); + shard.set_served_segment_cache_bytes_max( + config + .partition + .transfer_served_cache_bytes_max + .as_bytes_u64(), + ); + shard.set_partition_artifact_len_max( + config.partition.transfer_artifact_bytes_max.as_bytes_u64(), + ); + shard.set_repair_chunk_max(config.cluster.repair_chunk_max as u64); + // Bounds a served state-transfer chunk. A frame above the bus ceiling is + // rejected by the RECEIVING transport, which tears the replica connection + // down rather than dropping one message. + shard.set_bus_max_message_size( + usize::try_from(config.message_bus.max_message_size.as_bytes_u64()).unwrap_or(usize::MAX), + ); + *shard_handle.borrow_mut() = Some(Rc::downgrade(&shard)); + Ok((shard, sessions)) +} + +// Pin the configs-crate default literals (duplicated there to avoid a +// build-time edge onto the runtime crates) against the runtime constants, +// mirroring the message_bus IOV_MAX pin. A drift on either side fails this +// crate's build until both are reconciled. +const _: () = assert!( + configs::metadata::DEFAULT_METADATA_PREPARE_QUEUE_DEPTH + == consensus::PIPELINE_PREPARE_QUEUE_MAX +); +const _: () = assert!( + configs::metadata::DEFAULT_METADATA_JOURNAL_SLOTS + == journal::prepare_journal::DEFAULT_SLOT_COUNT +); +const _: () = assert!( + configs::partition::DEFAULT_PARTITION_PREPARE_QUEUE_DEPTH + == consensus::PIPELINE_PREPARE_QUEUE_MAX +); +const _: () = + assert!(configs::metadata::DEFAULT_METADATA_CLIENTS_TABLE_MAX == consensus::CLIENTS_TABLE_MAX); +const _: () = + assert!(configs::cluster::DEFAULT_VIEW_PROBE_ATTEMPTS_MAX == consensus::PROBE_ATTEMPTS_MAX); +const _: () = + assert!(configs::partition::DEFAULT_EVICTED_RING_CAPACITY == partitions::EVICTED_RING_CAPACITY); +const _: () = assert!( + configs::partition::DEFAULT_EVICTED_RING_BYTES_MAX == partitions::EVICTED_RING_BYTES_MAX +); +const _: () = assert!( + configs::partition::DEFAULT_TRANSFER_ARTIFACT_BYTES_MAX + == shard::PARTITION_ARTIFACT_LEN_DEFAULT +); +const _: () = assert!( + configs::partition::DEFAULT_TRANSFER_SERVED_CACHE_BYTES_MAX + == shard::SERVED_SEGMENT_CACHE_BYTES_DEFAULT +); +const _: () = assert!(configs::cluster::DEFAULT_REPAIR_CHUNK_MAX as u64 == shard::REPAIR_CHUNK_MAX); +const _: () = assert!( + configs::cluster::STATE_CHUNK_HEADER_LEN + == size_of::() as u64 +); +// Both prepare-queue ceilings are pinned by the view-change wire, not by memory: a +// `DoViewChange` carries the sender's suffix spanning `commit..=op` with one nack +// bit and one present bit per entry, each bitset a single `u128`. The depth bounds +// `op - commit`, so a depth at or above `DVC_HEADERS_MAX` produces entries the new +// primary can neither adopt nor prove dead. Strictly less than, because the head op +// needs the reserved slot. +const _: () = + assert!(configs::metadata::MAX_METADATA_PREPARE_QUEUE_DEPTH < consensus::DVC_HEADERS_MAX); +const _: () = + assert!(configs::partition::MAX_PARTITION_PREPARE_QUEUE_DEPTH < consensus::DVC_HEADERS_MAX); +// `DVC_HEADERS_MAX` is a bare literal in both the wire crate, which sizes the +// bitsets, and the consensus crate, which cannot depend on it the other way around. +// Same u128, so a drift lets one side address entries the other cannot. +const _: () = + assert!(consensus::DVC_HEADERS_MAX == iggy_binary_protocol::consensus::DVC_HEADERS_MAX); +const _: () = assert!(consensus::DVC_HEADERS_MAX == u128::BITS as usize); +/// Convert a consensus-timer interval to whole ticks, floored at one tick so a +/// sub-tick value still fires and saturated on overflow. +fn duration_to_ticks(interval: Duration) -> u64 { + let ticks = interval.as_millis() / shard::CONSENSUS_TICK_INTERVAL.as_millis(); + u64::try_from(ticks.max(1)).unwrap_or(u64::MAX) +} + +/// `[cluster] heartbeat_timeout` in consensus ticks. Every consensus group +/// (metadata and per-partition planes alike) gets the same window: the failure +/// it guards against - a primary that stopped heartbeating - is host-level, not +/// per-plane. +pub(crate) fn cluster_heartbeat_ticks(config: &ServerConfig) -> u64 { + duration_to_ticks(config.cluster.heartbeat_timeout.get_duration()) +} + +/// Floor for the post-restart read-recovery deadline (see +/// [`recovery_barrier_deadline`]). At and below the 5s default heartbeat the +/// worst-case recovery is dominated by the heartbeat-independent term - the +/// `ViewChangeStatus` backstop plus election ceremony and suffix recommit, +/// empirically ~7s - so the scaled value must never fall under this or a +/// fast-heartbeat cluster would 503 legitimate reads mid-recovery. The backstop +/// is the configurable `[cluster] view_change_status_timeout`; raising it past +/// its 5s default is why `recovery_barrier_deadline` scales that knob in too +/// rather than leaning on this floor to cover it. +const RECOVERY_BARRIER_DEADLINE_FLOOR: Duration = Duration::from_secs(15); + +/// Safety factor applied to each scaled term of the recovery deadline: a slower +/// heartbeat stretches election and suffix recommit proportionally, and a wider +/// status backstop stretches the ceremony it bounds. 3x reproduces the +/// empirically chosen 15s margin at the shared 5s default (3 x 5s = 15s) and +/// holds that factor as either knob grows. +const RECOVERY_BARRIER_MULTIPLIER: u32 = 3; + +/// How long the post-restart read path waits for the recovered WAL suffix to +/// re-commit before failing loud (retryable 503): the largest of the fixed +/// floor, a `[cluster] heartbeat_timeout`-scaled window, and a +/// `[cluster] view_change_status_timeout`-scaled window. Both knobs feed it +/// because either, raised far past its default, stretches worst-case recovery +/// past the fixed floor; see `await_recovery_barrier` for the read-side wait. +pub(crate) fn recovery_barrier_deadline( + heartbeat: Duration, + view_change_status: Duration, +) -> Duration { + // saturating: neither timeout has a config ceiling, plain `*` panics + heartbeat + .saturating_mul(RECOVERY_BARRIER_MULTIPLIER) + .max(view_change_status.saturating_mul(RECOVERY_BARRIER_MULTIPLIER)) + .max(RECOVERY_BARRIER_DEADLINE_FLOOR) +} + +/// `[cluster] commit_broadcast_interval` in consensus ticks: how often the +/// primary broadcasts its commit point, the cluster's liveness feed. Applied +/// to every consensus group, matching `cluster_heartbeat_ticks`. +pub(crate) fn commit_broadcast_ticks(config: &ServerConfig) -> u64 { + duration_to_ticks(config.cluster.commit_broadcast_interval.get_duration()) +} + +/// `[cluster] prepare_retransmit_interval` in consensus ticks: how often the +/// primary retransmits un-acked prepares. Applied to every consensus group, +/// matching `cluster_heartbeat_ticks`. +pub(crate) fn prepare_retransmit_ticks(config: &ServerConfig) -> u64 { + duration_to_ticks(config.cluster.prepare_retransmit_interval.get_duration()) +} + +/// `[cluster] view_change_retransmit_interval` in consensus ticks: how often a +/// replica retransmits its `StartViewChange` / `DoViewChange` during a view +/// change. Applied to every consensus group, matching `cluster_heartbeat_ticks`. +pub(crate) fn view_change_retransmit_ticks(config: &ServerConfig) -> u64 { + duration_to_ticks( + config + .cluster + .view_change_retransmit_interval + .get_duration(), + ) +} + +/// `[cluster] view_change_status_timeout` in consensus ticks: the stalled +/// view-change backstop before escalating to a fresh election. Applied to every +/// consensus group, matching `cluster_heartbeat_ticks`. +pub(crate) fn view_change_status_ticks(config: &ServerConfig) -> u64 { + duration_to_ticks(config.cluster.view_change_status_timeout.get_duration()) +} + +/// `[cluster] request_start_view_retransmit_interval` in consensus ticks: how +/// often a recovering or view-change backup re-requests the current `StartView`. +/// Applied to every consensus group, matching `cluster_heartbeat_ticks`. +pub(crate) fn request_start_view_ticks(config: &ServerConfig) -> u64 { + duration_to_ticks( + config + .cluster + .request_start_view_retransmit_interval + .get_duration(), + ) +} + +/// `[cluster] repair_retry_interval` in consensus ticks: how long a stalled +/// journal-repair stream waits before re-requesting its window. Both planes' +/// repair loops share it, so it is applied once per shard (not per consensus +/// group). Clamped to `u32`, the width of the session idle-tick counter. +pub(crate) fn repair_retry_ticks(config: &ServerConfig) -> u32 { + u32::try_from(duration_to_ticks( + config.cluster.repair_retry_interval.get_duration(), + )) + .unwrap_or(u32::MAX) +} + +/// Shard 0's half of a metadata recovery: everything [`recover`] produced except the +/// state machine, which every shard receives through the factory bundle. +/// +/// Named rather than a positional tuple: the fields are same-typed `Option`s and +/// `(u64, u128)` pairs that a reorder would silently rebind, and one of them decides +/// what view the replica boots into. +struct RecoveredOwnerState { + journal: PrepareJournal, + snapshot: Option, + last_applied_op: Option, + last_journaled_op: Option, + client_table: ClientTable, + superblock: PingPongSuperblock, + recovered_state: Option, + snapshot_checkpoint: (u64, u128), +} + +/// Rebuild metadata consensus from what recovery read off this replica's own disk. +/// +/// Takes the recovery result, topology and config whole rather than the dozen-plus +/// scalars it needs from them: most were `u64` tick counts, where a misordered +/// argument type-checks and mistunes a timeout silently. +fn restore_metadata_consensus( + owner: &RecoveredOwnerState, + topology: &TcpTopology, + config: &ServerConfig, + bus: Rc, +) -> VsrConsensus> { + let journal = &owner.journal; + let replica_count = topology.replica_count; + let recovered_state = owner.recovered_state; + let snapshot_floor = owner + .snapshot + .as_ref() + .map_or(0, IggySnapshot::sequence_number); + let commit_watermark = owner.last_applied_op.unwrap_or(snapshot_floor); + let restored_op = owner.last_journaled_op.unwrap_or(snapshot_floor); + let recovery_deadline = recovery_barrier_deadline( + config.cluster.heartbeat_timeout.get_duration(), + config.cluster.view_change_status_timeout.get_duration(), + ); + let prepare_queue_depth = config.metadata.prepare_queue_depth; + + let mut consensus = VsrConsensus::new( + topology.cluster_id, + topology.self_replica_id, + replica_count, + server_common::sharding::METADATA_GROUP, + bus, + // Request queue keeps the stock 2x ratio over the prepare queue + // (32 -> 64 at defaults): buffered requests are cheap relative to + // in-flight prepares and drain as prepares commit. + LocalPipeline::with_capacities(prepare_queue_depth, prepare_queue_depth * 2), + ); + consensus.set_normal_heartbeat_ticks(cluster_heartbeat_ticks(config)); + consensus.set_commit_message_ticks(commit_broadcast_ticks(config)); + consensus.set_prepare_ticks(prepare_retransmit_ticks(config)); + consensus.set_view_change_retransmit_ticks(view_change_retransmit_ticks(config)); + consensus.set_view_change_status_ticks(view_change_status_ticks(config)); + consensus.set_request_start_view_ticks(request_start_view_ticks(config)); + consensus.set_probe_attempts_max(config.cluster.view_probe_attempts_max); + // Fresh random incarnation each boot, so a StartView addressed to a previous + // incarnation still in flight is ignored (`handle_start_view` guard). `| 1` + // guarantees the non-zero the guard treats as set. The deterministic simulator + // overrides this with a seed-derived value bumped per restart. + consensus.set_incarnation(rand::random::() | 1); + + let last_header = journal + .last_op() + .and_then(|op| usize::try_from(op).ok()) + .and_then(|op| journal.header(op).map(|header| *header)); + // View and log_view come from the durable superblock when present. A present but + // unreadable superblock already refused boot in `recover()`, so reaching the + // `else` means it is genuinely absent: a fresh node, or one that took writes but + // never checkpointed or changed view. There, inferring the view from the last WAL + // prepare is safe, since the persist-before-send gate guarantees this replica + // never externalized a view beyond what a re-probe re-derives, and it re-probes + // as a backup below. log_view cannot be inferred and stays 0 until the next + // superblock write. + if let Some(state) = recovered_state { + consensus.set_view(state.view); + consensus.set_log_view(state.log_view); + consensus.mark_superblock_durable(state.view, state.log_view); + } else if let Some(header) = last_header { + consensus.set_view(header.view); + } + + // On a RESTART in a cluster, rejoin as a quorum-invisible backup and + // probe for the current view (`RequestStartView`): the view's primary + // answers with a `StartView`, the replica adopts it as a backup, and + // journal repair fills any WAL gap. A probing replica never resumes + // primaryship -- if this replica IS the current primary-by-index, its + // probe makes the backups elect past it. + // The probe re-broadcasts on its timeout, so it needs no live mesh at + // boot. A FRESH boot keeps the plain init: the cluster needs its view-0 + // primary to exist, and a single-replica cluster has no peer to ask. + // + // Prior life is EITHER a non-empty WAL or a recovered superblock. A view + // change persists without touching the WAL, so a replica that changed + // view before its first metadata write comes back with a non-zero view + // and an empty journal; gating on the WAL alone would `init()` it into + // `Status::Normal` as primary for a view the cluster may have moved past, + // with `ceded_primaryship` false and no probe to correct it. + if replica_count > 1 && (restored_op > 0 || recovered_state.is_some()) { + consensus.init_as_backup(); + consensus.begin_view_probe(); + // Restart in a cluster: replace snapshot-shaped metadata state + // (snapshot + client table) from the live primary the probe finds, + // then journal-repair the tail. If the probe exhausts instead -- + // full-cluster bootstrap, nobody live to fetch from -- the election + // fallback clears the stage and this local recovery stands. + consensus.begin_state_transfer_await(); + } else { + consensus.init(); + } + consensus.sequencer().set_sequence(restored_op); + // A SOLO replica's durable journal head IS its commit point: quorum is + // 1-of-1, so an entry commits the instant it is durable, and the acks + // the cluster ceremony below would wait on cannot topologically exist. + // The embedded watermark is structurally one op stale (the commit point + // is only ever written down inside the NEXT entry), so trusting it solo + // manufactures an "uncommitted" suffix that provably committed and + // wedges the recovery barrier forever. + let commit_watermark = if replica_count == 1 { + restored_op + } else { + commit_watermark + }; + // The commit point is restored from the WAL's embedded watermark (each + // journaled prepare carries the primary's commit at send time), NOT from + // the journal head: journaled does not imply committed, and claiming + // commit for the un-quorum'd tail both risks split-brain on a later view + // change and starves the tail of re-replication (it would live in no + // pipeline). The suffix `(commit_watermark, restored_op]` is re-pipelined + // below when this replica is the recovered view's primary. + // + // TODO(hubcio): the watermark is a lower bound (the last entry stamps + // the commit point as of its send). Persisting an explicit (view, + // commit_op) watermark on the commit path would tighten recovery and + // allow refusing boot on an excessive gap; a backup that recovered a + // LONGER tail than the cluster's primary still needs uncommitted-suffix + // truncation when conflicting ops arrive (message repair milestone). + consensus.restore_commit_state(commit_watermark, commit_watermark); + if let Some(header) = last_header { + consensus.set_last_prepare_checksum(header.checksum); + consensus.observe_prepare_timestamp(header.timestamp); + } + + // The WAL's tail past the watermark is prepared-but-not-provably-committed + // state. Until the cluster confirms it (re-pipelined below on a resumed + // primary; via StartView adoption + the local commit walk on a rejoined + // backup), serving reads would show pre-restart state that clients already + // saw acked -- gate them on the barrier regardless of role. If the suffix + // never re-commits cluster-wide, the read path fails loud with a retryable + // 503 once the paired deadline expires (`await_recovery_barrier`). + if commit_watermark < restored_op { + consensus.set_recovery_barrier(restored_op); + consensus.set_recovery_deadline(recovery_deadline); + } + + // Re-pipeline the prepared-but-uncommitted suffix so the primary's + // retransmit machinery re-replicates it and quorum can (re-)commit it. + // A backup's suffix stays journal-only: the primary's traffic either + // confirms it (re-forward + re-ack path) or supersedes it. + if consensus.is_primary() + && !consensus.has_ceded_primaryship() + && commit_watermark < restored_op + { + info!( + commit_watermark, + restored_op, "re-pipelining recovered uncommitted metadata suffix" + ); + let mut pipeline = consensus.pipeline().borrow_mut(); + #[allow(clippy::cast_possible_truncation)] + for op in (commit_watermark + 1)..=restored_op { + let Some(header) = journal.header(op as usize) else { + warn!( + op, + "recovered journal suffix has a gap; stopping re-pipeline" + ); + break; + }; + let mut entry = PipelineEntry::new(*header); + entry.add_ack(topology.self_replica_id); + pipeline.push(entry); + } + } + + consensus +} + +#[allow(clippy::too_many_arguments)] +async fn load_partition( + config: &ServerConfig, + namespace: IggyNamespace, + stats: Arc, + partition_metadata: &Partition, + cluster_id: u128, + self_replica_id: u8, + replica_count: u8, + bus: Rc, +) -> Result>, ServerError> { + let stream_id = namespace.stream_id(); + let topic_id = namespace.topic_id(); + let partition_id = namespace.partition_id(); + // Request queue holds 2x the prepare depth (buffered requests drain as + // prepares commit); depth is the per-partition `[partition]` knob. + let prepare_queue_depth = config.partition.prepare_queue_depth; + let mut consensus = VsrConsensus::new( + cluster_id, + self_replica_id, + replica_count, + namespace.inner(), + bus, + LocalPipeline::with_capacities(prepare_queue_depth, prepare_queue_depth * 2), + ); + consensus.set_normal_heartbeat_ticks(cluster_heartbeat_ticks(config)); + consensus.set_commit_message_ticks(commit_broadcast_ticks(config)); + consensus.set_prepare_ticks(prepare_retransmit_ticks(config)); + consensus.set_view_change_retransmit_ticks(view_change_retransmit_ticks(config)); + consensus.set_view_change_status_ticks(view_change_status_ticks(config)); + consensus.set_request_start_view_ticks(request_start_view_ticks(config)); + consensus.set_probe_attempts_max(config.cluster.view_probe_attempts_max); + + // (view, log_view) come from the group's durable superblock when present; + // a present but unverifiable record already refused boot inside + // `open_partition_superblock`. Restored BEFORE choosing how to join, so + // the backup probe below never advertises a view older than the recorded + // one. + let partition_dir = config + .system + .get_partition_path(stream_id, topic_id, partition_id); + let (superblock, recovered_state) = open_partition_superblock( + &partition_dir, + ReplicaIdentity { + cluster: cluster_id, + replica_id: self_replica_id, + replica_count, + }, + ) + .await?; + if let Some(state) = recovered_state.as_ref() { + restore_partition_view(&mut consensus, state); + } + + // A recovered partition lost its journal state with the process: the + // partition journal is in-memory and segments carry no op numbers, so + // this replica cannot know the group's (op, commit) even when the + // superblock restored its view. In a cluster it boots as a + // quorum-invisible backup and probes for the current view + // (`RequestStartView`): the view's primary answers with a `StartView`, + // journal repair fills the rejoin window, and the commit floor settles + // at the serving peer's retention point. The probe re-broadcasts on its + // timeout, so it needs no live mesh at boot. Single-replica groups + // have no peer to ask and keep the plain init. + if replica_count > 1 { + consensus.init_as_backup(); + consensus.begin_view_probe(); + } else { + consensus.init(); + } + + // No prepare-timestamp floor is restored here: the partition consensus + // journal is non-durable today, so there is no persisted head to observe + // (unlike `restore_metadata_consensus`, which observes its restored head). + // When PartitionJournal becomes durable (the milestone named in the + // multi-shard wiring commit body), observe the restored head and the max + // recovered message timestamp here, or an NTP rewind across a restart could + // regress persisted `base_timestamp`. + + let recovered_segments = + load_persisted_segments(config, stream_id, topic_id, partition_id, &stats) + .await + .map_err(|source| { + error!( + stream_id, + topic_id, + partition_id, + error = %source, + "failed to load partition log during server bootstrap" + ); + source + })?; + + let mut partition = IggyPartition::new(stats.clone(), consensus); + partition.set_superblock(superblock, recovered_state.as_ref()); + // Recovered partitions honor the same config-surfaced ring ceilings as the + // fresh-create path (build_partition_fresh). Retention is already off for + // single-replica groups, so this only sizes the multi-replica ring. + partition.log.journal().inner.set_ring_caps( + config.partition.evicted_ring_capacity, + config.partition.evicted_ring_bytes_max.as_bytes_u64(), + ); + partition.set_partition_dir(partition_dir); + // Before the hydrate: the durable record is keyed by incarnation, so a + // `purge.gen` left behind by a previous life of this namespace reads 0. + partition.set_created_revision(partition_metadata.created_revision); + partition.hydrate_applied_purge_generation().await?; + hydrate_partition_log( + &mut partition, + config, + stream_id, + topic_id, + partition_id, + recovered_segments, + ) + .await?; + + let sized_end = partition + .log + .segments() + .iter() + .filter(|segment| segment.size > IggyByteSize::default()) + .map(|segment| segment.end_offset) + .max(); + // An empty chain whose segment is named for a nonzero offset is the + // shape a state-transfer install (or its converge) plants at the group + // frontier after the origin GC'd everything: the file name carries the + // frontier, and re-minting offsets from 0 here would fork this + // replica's batch stamps from the rest of the group after a restart. + let empty_frontier = partition + .log + .segments() + .iter() + .map(|segment| segment.start_offset) + .max() + .filter(|&start| sized_end.is_none() && start > 0); + let current_offset = sized_end.or_else(|| empty_frontier.map(|start| start - 1)); + partition.created_at = partition_metadata.created_at; + partition.recovered_durable_offset = sized_end; + // The OFFSET COUNTER is restored from that file name (above), but the + // `installed_frontier` CLAIM deliberately is not: the claim says "everything + // below me is represented here", and `converge_to_empty_after_failed_install` + // refuses to make it when staged segments were dropped -- yet a converge + // plants exactly the same empty `{frontier:020}.log` a legitimate empty + // install does, so boot provably cannot tell them apart. Re-deriving it here + // would hand the refused claim back: the repair floor stand-in would accept a + // commit floor over ops this replica holds zero bytes for, and the replica + // would pass the serve gate and offer that emptiness onward, making a peer + // unlink its own chain. Leaving it `None` costs one spurious full + // re-transfer on the legitimate empty-install restart; a false caught-up + // claim is not recoverable. A durable home for the frontier (the partition + // superblock already reserves a field) is what would settle it properly. + let counter = current_offset.unwrap_or(0); + partition.offset.store(counter, Ordering::Release); + partition.dirty_offset.store(counter, Ordering::Relaxed); + partition.should_increment_offset = current_offset.is_some(); + // The durable frontier is a LOWER BOUND on top of what the segments proved: + // it is the only carrier left when the segments that named the frontier are + // gone (an all-GC'd origin's install, a crash inside the swap window), and + // taking the max means real recovered data always wins. + partition.restore_offset_frontier(recovered_state.as_ref()); + let current_offset = partition.offset.load(Ordering::Acquire); + + configure_consumer_offsets(&mut partition, config, namespace, current_offset)?; + ensure_initial_segment(&mut partition, config, stream_id, topic_id, partition_id).await?; + + Ok(partition) +} + +async fn hydrate_partition_log( + partition: &mut IggyPartition>, + config: &ServerConfig, + stream_id: usize, + topic_id: usize, + partition_id: usize, + recovered_segments: Vec, +) -> Result<(), ServerError> { + for RecoveredSegment { segment, storage } in recovered_segments { + partition + .log + .add_persisted_segment(segment, storage, None, None); + } + + if let Some(active_index) = partition.log.segments().len().checked_sub(1) { + let storage = &partition.log.storages()[active_index]; + if let ( + Some(messages_reader), + Some(index_reader), + Some(storage_messages_writer), + Some(storage_index_writer), + ) = ( + storage.messages_reader.as_ref(), + storage.index_reader.as_ref(), + storage.messages_writer.as_ref(), + storage.index_writer.as_ref(), + ) { + let index_path = index_reader.path(); + // Share the storage's size counters: the readers bound reads by + // these atomics, so a writer with a private counter persists bytes + // the readers never learn about. + let messages_size_counter = storage_messages_writer.size_counter(); + let index_size_counter = storage_index_writer.size_counter(); + partition.log.messages_writers_mut()[active_index] = Some(Rc::new( + MessagesWriter::new( + &messages_reader.path(), + messages_size_counter, + config.system.partition.enforce_fsync, + true, + config + .system + .segment + .preallocate + .then_some(config.system.segment.size), + ) + .await + .map_err(|source| { + error!( + stream_id, + topic_id, + partition_id, + path = %messages_reader.path(), + error = %source, + "failed to initialize persisted messages writer" + ); + source + })?, + )); + partition.log.index_writers_mut()[active_index] = Some(Rc::new( + IggyIndexWriter::new( + &index_path, + index_size_counter, + config.system.partition.enforce_fsync, + true, + ) + .await + .map_err(|source| { + error!( + stream_id, + topic_id, + partition_id, + path = %index_path, + error = %source, + "failed to initialize persisted sparse index writer" + ); + source + })?, + )); + } + } + + Ok(()) +} + +fn resolve_tcp_topology( + config: &ServerConfig, + current_replica_id: Option, +) -> Result { + let default_client_addr = parse_socket_addr("tcp.address", &config.tcp.address)?; + let default_ws_addr = resolve_optional_listener_addr( + config.websocket.enabled, + "websocket.address", + &config.websocket.address, + )?; + let default_quic_addr = + resolve_optional_listener_addr(config.quic.enabled, "quic.address", &config.quic.address)?; + let default_http_addr = + resolve_optional_listener_addr(config.http.enabled, "http.address", &config.http.address)?; + if !config.cluster.enabled { + if let Some(replica_id) = current_replica_id + && replica_id != SHARD_REPLICA_ID + { + return Err(ServerError::ReplicaIdRequiresCluster { + supplied: replica_id, + default: SHARD_REPLICA_ID, + }); + } + return Ok(TcpTopology { + cluster_id: auth::cluster_domain_id(&config.cluster.name), + // Keep parity with the current server binary and the integration + // harness: `--replica-id 0` may be passed unconditionally in + // single-node mode; any other id is rejected above so the WAL + // cannot commit under an identity that will later disagree with + // a cluster.nodes[] entry. + self_replica_id: SHARD_REPLICA_ID, + replica_count: 1, + client_listen_addr: default_client_addr, + replica_listen_addr: Some(SocketAddr::new(default_client_addr.ip(), 0)), + ws_listen_addr: default_ws_addr, + quic_listen_addr: default_quic_addr, + http_listen_addr: default_http_addr, + tcp_tls_listen_addr: config.tcp.tls.enabled.then_some(default_client_addr), + peers: Vec::new(), + }); + } + + let self_replica_id = current_replica_id.ok_or(ServerError::MissingReplicaId)?; + + let self_node = config + .cluster + .nodes + .iter() + .find(|node| node.replica_id == self_replica_id) + .ok_or(ServerError::ClusterNodeNotFound { + replica_id: self_replica_id, + })?; + let replica_count = u8::try_from(config.cluster.nodes.len()).map_err(|_| { + ServerError::ClusterReplicaCountTooLarge { + count: config.cluster.nodes.len(), + } + })?; + let ClusterClientAddrs { + client: client_listen_addr, + ws: ws_listen_addr, + quic: quic_listen_addr, + http: http_listen_addr, + } = resolve_cluster_client_addrs( + self_node, + default_client_addr, + default_ws_addr, + default_quic_addr, + default_http_addr, + )?; + let replica_port = self_node + .ports + .tcp_replica + .ok_or(ServerError::ClusterPortMissing { + transport: "tcp_replica", + replica_id: self_node.replica_id, + })?; + let replica_listen_addr = Some(socket_addr_from_parts( + "cluster.nodes[*].ports.tcp_replica", + &self_node.ip, + replica_port, + )?); + let peers = resolve_cluster_replica_peers(&config.cluster.nodes, self_replica_id)?; + + Ok(TcpTopology { + cluster_id: auth::cluster_domain_id(&config.cluster.name), + self_replica_id, + replica_count, + client_listen_addr, + replica_listen_addr, + ws_listen_addr, + quic_listen_addr, + http_listen_addr, + tcp_tls_listen_addr: config.tcp.tls.enabled.then_some(client_listen_addr), + peers, + }) +} + +fn resolve_optional_listener_addr( + enabled: bool, + context: &'static str, + address: &str, +) -> Result, ServerError> { + if enabled { + return Ok(Some(parse_socket_addr(context, address)?)); + } + Ok(None) +} + +/// Client-facing listener addresses resolved for this cluster node. Each port +/// comes from the node's roster entry; there is no fallback to the top-level +/// listener port, an enabled transport without a roster port refuses to boot. +/// Every transport keeps the bind interface from its own `address` config: the +/// roster ip is advertised, not bound. +struct ClusterClientAddrs { + client: SocketAddr, + ws: Option, + quic: Option, + http: Option, +} + +fn resolve_cluster_client_addrs( + self_node: &configs::cluster::ClusterNodeConfig, + default_tcp_addr: SocketAddr, + default_ws_addr: Option, + default_quic_addr: Option, + default_http_addr: Option, +) -> Result { + let client_port = self_node.ports.tcp.ok_or(ServerError::ClusterPortMissing { + transport: "tcp", + replica_id: self_node.replica_id, + })?; + let client = + merge_roster_port_with_bind_ip("tcp", &self_node.ip, default_tcp_addr, client_port); + let ws = resolve_cluster_optional_addr(self_node, "websocket", default_ws_addr, |ports| { + ports.websocket + })?; + let quic = + resolve_cluster_optional_addr(self_node, "quic", default_quic_addr, |ports| ports.quic)?; + let http = + resolve_cluster_optional_addr(self_node, "http", default_http_addr, |ports| ports.http)?; + Ok(ClusterClientAddrs { + client, + ws, + quic, + http, + }) +} + +fn resolve_cluster_optional_addr( + self_node: &configs::cluster::ClusterNodeConfig, + transport: &'static str, + default_addr: Option, + port_selector: impl Fn(&configs::cluster::TransportPorts) -> Option, +) -> Result, ServerError> { + let Some(default_addr) = default_addr else { + return Ok(None); + }; + // No fallback to the top-level port: two same-host nodes leaving the same + // transport port unset would race for one socket. Either the roster is + // explicit or the server refuses to boot. + let port = port_selector(&self_node.ports).ok_or(ServerError::ClusterPortMissing { + transport, + replica_id: self_node.replica_id, + })?; + Ok(Some(merge_roster_port_with_bind_ip( + transport, + &self_node.ip, + default_addr, + port, + ))) +} + +/// Combine the roster-supplied `port` with the bind interface the transport's +/// own `address` config asked for. +/// +/// The roster ip is what the cluster advertises (metadata, follower-to-primary +/// HTTP forwarding targets); the transport's own `address` decides the bind +/// interface. Merging keeps a loopback-only `127.0.0.1` private and a +/// `0.0.0.0` wide in cluster mode instead of silently rebinding to the roster +/// interface, which would strand every co-located dialer (sidecars, health +/// probes, on-host consumers) on `ECONNREFUSED`. +fn merge_roster_port_with_bind_ip( + transport: &'static str, + roster_ip: &str, + bind_addr: SocketAddr, + port: u16, +) -> SocketAddr { + let listen_addr = SocketAddr::new(bind_addr.ip(), port); + if roster_ip_unreachable_from_bind_addr(roster_ip, listen_addr) { + warn!( + "{transport} listener binds {listen_addr} but the roster advertises {roster_ip}:{port}; \ + peers and clients dialing the advertised endpoint may not reach this node" + ); + } + listen_addr +} + +/// Whether a dialer aiming at the advertised roster ip misses `listen_addr`. An +/// unspecified bind covers every interface, and a roster ip that parses as +/// neither IPv4 nor IPv6 (a DNS name, say) can resolve to the bound interface, +/// so both cases stay quiet. +fn roster_ip_unreachable_from_bind_addr(roster_ip: &str, listen_addr: SocketAddr) -> bool { + !listen_addr.ip().is_unspecified() + && roster_ip + .parse::() + .is_ok_and(|parsed| parsed != listen_addr.ip()) +} + +fn resolve_cluster_replica_peers( + nodes: &[configs::cluster::ClusterNodeConfig], + self_replica_id: u8, +) -> Result, ServerError> { + let mut peers = Vec::with_capacity(nodes.len().saturating_sub(1)); + for node in nodes { + if node.replica_id == self_replica_id { + continue; + } + let replica_port = node + .ports + .tcp_replica + .ok_or(ServerError::ClusterPortMissing { + transport: "tcp_replica", + replica_id: node.replica_id, + })?; + peers.push(( + node.replica_id, + socket_addr_from_parts("cluster.nodes[*].ports.tcp_replica", &node.ip, replica_port)?, + )); + } + Ok(peers) +} + +async fn start_tcp_runtime( + shard: &Rc, + config: &ServerConfig, + topology: &TcpTopology, + accepted_replica: AcceptedReplicaFn, + dialed_replica: DialedReplicaFn, + accepted_clients: LocalClientAcceptFns, +) -> Result<(), ServerError> { + if config.tcp.enabled && !config.tcp.tls.enabled { + start_via_replica_io( + shard, + config, + topology, + accepted_replica, + dialed_replica, + accepted_clients, + ) + .await?; + } else { + start_manual_runtime( + shard, + config, + topology, + accepted_replica, + dialed_replica, + accepted_clients, + ) + .await?; + } + + // HTTP is served over TCP but sits outside the replica_io / manual client + // reactor, so it binds independently. Shard-0 gating comes from the sole + // caller of this function. + if let Some(http_addr) = topology.http_listen_addr { + let self_ports = configs::cluster::TransportPorts { + tcp: config + .tcp + .enabled + .then(|| topology.client_listen_addr.port()), + quic: topology.quic_listen_addr.map(|addr| addr.port()), + websocket: topology.ws_listen_addr.map(|addr| addr.port()), + ..Default::default() + }; + http::start( + shard, + http_addr, + &config.http, + config.metadata.clients_table_max, + config.personal_access_token.max_tokens_per_user, + &config.cluster, + Arc::clone(&config.system), + self_ports, + ) + .await?; + } + + Ok(()) +} + +// ws/wss bindings intentionally mirror the transport names (same convention as +// `replica_io::start_on_shard_zero`). +#[allow(clippy::similar_names)] +async fn start_via_replica_io( + shard: &Rc, + config: &ServerConfig, + topology: &TcpTopology, + accepted_replica: AcceptedReplicaFn, + dialed_replica: DialedReplicaFn, + accepted_clients: LocalClientAcceptFns, +) -> Result<(), ServerError> { + let replica_addr = topology + .replica_listen_addr + .expect("topology must include replica listener address"); + let quic_credentials = topology + .quic_listen_addr + .is_some() + .then(|| load_quic_server_credentials(config)) + .transpose()?; + let tcp_tls_credentials = topology + .tcp_tls_listen_addr + .is_some() + .then(|| load_tcp_tls_server_credentials(config)) + .transpose()?; + // `websocket.tls.enabled` upgrades the websocket address to a WSS + // listener; the plain-WS listener must NOT also bind it (one port, one + // handshake kind -- a plain upgrade parser fed a TLS ClientHello rejects + // every connection with an httparse error). + let wss_enabled = config.websocket.tls.enabled; + let ws_listen_addr = (!wss_enabled).then_some(topology.ws_listen_addr).flatten(); + let wss_listen_addr = wss_enabled.then_some(topology.ws_listen_addr).flatten(); + let wss_credentials = wss_listen_addr + .is_some() + .then(|| load_wss_server_credentials(config)) + .transpose()?; + + let LocalClientAcceptFns { + tcp, + ws, + quic, + tcp_tls, + wss, + } = accepted_clients; + + let bound = replica_io::start_on_shard_zero( + &shard.bus, + replica_addr, + topology.client_listen_addr, + ws_listen_addr, + topology.quic_listen_addr, + quic_credentials, + topology.tcp_tls_listen_addr, + tcp_tls_credentials, + wss_listen_addr, + wss_credentials, + topology.self_replica_id, + topology.peers.clone(), + accepted_replica, + dialed_replica, + tcp, + ws_listen_addr.map(|_| ws), + topology.quic_listen_addr.map(|_| quic), + topology.tcp_tls_listen_addr.map(|_| tcp_tls), + wss_listen_addr.map(|_| wss), + shard.bus.config().reconnect_period, + ) + .await + .map_err(|source| { + error!( + replica_addr = %replica_addr, + client_addr = %topology.client_listen_addr, + error = %source, + "failed to start server listeners via replica_io" + ); + source + })?; + let Some(bound) = bound else { + return Ok(()); + }; + + write_current_config( + config, + Some(topology.self_replica_id), + Some(bound.client), + config.cluster.enabled.then_some(bound.replica), + bound.tcp_tls, + bound.quic, + // The WSS listener occupies the configured websocket address slot. + bound.wss.or(bound.ws), + ) + .await?; + if config.cluster.enabled { + info!( + shard = shard.id, + replica = %bound.replica, + tcp = %bound.client, + tcp_tls = ?bound.tcp_tls, + ws = ?bound.ws, + quic = ?bound.quic, + "server listeners started" + ); + } else { + info!( + shard = shard.id, + tcp = %bound.client, + tcp_tls = ?bound.tcp_tls, + ws = ?bound.ws, + quic = ?bound.quic, + "server client listeners started" + ); + } + + Ok(()) +} + +async fn start_manual_runtime( + shard: &Rc, + config: &ServerConfig, + topology: &TcpTopology, + accepted_replica: AcceptedReplicaFn, + dialed_replica: DialedReplicaFn, + accepted_clients: LocalClientAcceptFns, +) -> Result<(), ServerError> { + let bound_replica = if config.cluster.enabled { + let replica_addr = topology + .replica_listen_addr + .expect("cluster-enabled topology must include replica listener address"); + let (replica_listener, bound_addr) = + replica_listener::bind(replica_addr) + .await + .map_err(|source| { + error!( + replica_addr = %replica_addr, + error = %source, + "failed to bind replica listener" + ); + source + })?; + let token = shard.bus.token(); + let replica_handle = compio::runtime::spawn(async move { + replica_listener::run(replica_listener, token, accepted_replica).await; + }); + shard.bus.track_background(replica_handle); + connector::start( + &shard.bus, + topology.self_replica_id, + topology.peers.clone(), + dialed_replica, + shard.bus.config().reconnect_period, + ) + .await; + Some(bound_addr) + } else { + None + }; + + let bound_clients = start_client_listeners(shard, config, topology, &accepted_clients).await?; + write_current_config( + config, + Some(topology.self_replica_id), + bound_clients.tcp, + bound_replica, + bound_clients.tcp_tls, + bound_clients.quic, + bound_clients.ws, + ) + .await?; + + if config.cluster.enabled { + info!( + shard = shard.id, + replica = ?bound_replica, + tcp = ?bound_clients.tcp, + tcp_tls = ?bound_clients.tcp_tls, + ws = ?bound_clients.ws, + quic = ?bound_clients.quic, + "server listeners started" + ); + } else { + info!( + shard = shard.id, + tcp = ?bound_clients.tcp, + tcp_tls = ?bound_clients.tcp_tls, + ws = ?bound_clients.ws, + quic = ?bound_clients.quic, + "server client listeners started" + ); + } + + Ok(()) +} + +fn ensure_default_root_user(mux_stm: &ServerMuxStateMachine) { + if !mux_stm.users().read(|users| users.items.is_empty()) { + return; + } + + let (username, password_hash) = create_root_credentials(); + mux_stm.users().ensure_root_user(&username, &password_hash); +} + +/// Apply `--with-default-root-credentials`. +/// +/// Fills in whichever of [`IGGY_ROOT_USERNAME_ENV`] / +/// [`IGGY_ROOT_PASSWORD_ENV`] the operator did not export, so the flag is +/// exactly the sugar for setting both by hand and the environment keeps +/// winning over it. +/// +/// # Safety +/// +/// Mutates the process environment, so the caller must still be +/// single-threaded. +pub unsafe fn apply_default_root_credentials(enabled: bool) { + if !enabled { + return; + } + + let username_set = env::var(IGGY_ROOT_USERNAME_ENV).is_ok(); + let password_set = env::var(IGGY_ROOT_PASSWORD_ENV).is_ok(); + if username_set && password_set { + warn!( + "--with-default-root-credentials ignored: {IGGY_ROOT_USERNAME_ENV} and \ + {IGGY_ROOT_PASSWORD_ENV} are already set" + ); + return; + } + + // SAFETY: single-threaded caller, per this function's contract. + unsafe { + if !username_set { + env::set_var(IGGY_ROOT_USERNAME_ENV, DEFAULT_ROOT_USERNAME); + } + if !password_set { + env::set_var(IGGY_ROOT_PASSWORD_ENV, DEFAULT_ROOT_PASSWORD); + } + } + warn!( + "--with-default-root-credentials: a newly created root user will use the \ + well-known development credentials; INSECURE outside development" + ); +} + +/// Resolve the root user credentials from `IGGY_ROOT_USERNAME` / +/// `IGGY_ROOT_PASSWORD`, falling back to the default username with a +/// generated password. +/// +/// Returns `(username, password_hash)`; the plaintext password never +/// leaves this function. +fn create_root_credentials() -> (String, String) { + if let Some((username, password)) = root_credentials_from_env() { + info!("Using the custom root user credentials."); + return (username, crypto::hash_password(&password)); + } + + info!("Using the default root user credentials..."); + let password = crypto::generate_secret(20..40); + // Through tracing, not stdout: this is the only time the operator can read + // the password, so it has to reach the log file too. + warn!("Generated root user password: {password}"); + ( + DEFAULT_ROOT_USERNAME.to_string(), + crypto::hash_password(&password), + ) +} + +/// The credentials the operator supplied, `None` when neither variable is +/// set. A half-set pair never reaches here: [`validate_root_credentials`] +/// rejects it at boot. +fn root_credentials_from_env() -> Option<(String, String)> { + match ( + env::var(IGGY_ROOT_USERNAME_ENV), + env::var(IGGY_ROOT_PASSWORD_ENV), + ) { + (Ok(username), Ok(password)) => Some((username, password)), + _ => None, + } +} + +/// Reject root-credential misconfiguration before any shard thread exists. +/// +/// Shard 0 seeds the root user from inside `recover`'s baseline closure, +/// which cannot fail, so every operator-facing check has to run here or it +/// would have to panic a shard thread instead. +fn validate_root_credentials_env(config: &ServerConfig) -> Result<(), ServerError> { + // `recover` creates the metadata directory, so its absence is what tells a + // first cluster boot (root must come out identical on every replica, hence + // explicit credentials) apart from a restart that recovers the root user it + // already stored. `--fresh` has already wiped by this point, so a wiped + // replica is correctly treated as a first boot. + let fresh_cluster = config.cluster.enabled + && !Path::new(&config.system.path) + .join(metadata::impls::METADATA_DIR) + .exists(); + + validate_root_credentials( + fresh_cluster, + env::var(IGGY_ROOT_USERNAME_ENV).ok().as_deref(), + env::var(IGGY_ROOT_PASSWORD_ENV).ok().as_deref(), + ) +} + +fn validate_root_credentials( + explicit_required: bool, + username: Option<&str>, + password: Option<&str>, +) -> Result<(), ServerError> { + match (username, password) { + (Some(username), Some(password)) => { + validate_credential_length( + IGGY_ROOT_USERNAME_ENV, + username, + MIN_USERNAME_LENGTH, + MAX_USERNAME_LENGTH, + )?; + validate_credential_length( + IGGY_ROOT_PASSWORD_ENV, + password, + MIN_PASSWORD_LENGTH, + MAX_PASSWORD_LENGTH, + ) + } + (Some(_), None) => Err(ServerError::RootCredentialsIncomplete { + provided_env: IGGY_ROOT_USERNAME_ENV, + missing_env: IGGY_ROOT_PASSWORD_ENV, + }), + (None, Some(_)) => Err(ServerError::RootCredentialsIncomplete { + provided_env: IGGY_ROOT_PASSWORD_ENV, + missing_env: IGGY_ROOT_USERNAME_ENV, + }), + (None, None) if explicit_required => Err(ServerError::ClusterRootCredentialsRequired { + username_env: IGGY_ROOT_USERNAME_ENV, + password_env: IGGY_ROOT_PASSWORD_ENV, + }), + (None, None) => Ok(()), + } +} + +fn validate_credential_length( + env_name: &'static str, + value: &str, + min: usize, + max: usize, +) -> Result<(), ServerError> { + if (min..=max).contains(&value.len()) { + Ok(()) + } else { + Err(ServerError::RootCredentialLength { + env_name, + length: value.len(), + min, + max, + }) + } +} + +/// Replica delegation callbacks for shard 0's listener and connector. +/// +/// Inbound: acquire a slot in the shard-0-global in-flight handshake cap +/// (drop the connection when full), then blind-delegate the raw fd +/// through the coordinator's round-robin. The fd lands on the target +/// shard's inbox as a [`shard::LifecycleFrame::ReplicaInboundSetup`] +/// frame; the owning shard runs the acceptor handshake and acks the +/// slot back. A failed delegation releases the slot immediately. +/// +/// Outbound: delegate the dialed fd as +/// [`shard::LifecycleFrame::ReplicaOutboundSetup`] and mark the peer +/// dial-pending so the reconnect sweep skips it until the owning +/// shard's handshake outcome arrives (or the entry expires). +fn make_replica_delegation_fns( + coord: Rc, + bus: &Rc, +) -> (AcceptedReplicaFn, DialedReplicaFn) { + let inbound_bus = Rc::clone(bus); + let inbound_coord = Rc::clone(&coord); + let accepted: AcceptedReplicaFn = Rc::new(move |stream| { + let Some(slot) = inbound_bus.try_acquire_replica_handshake_slot() else { + warn!( + cap = MAX_INFLIGHT_REPLICA_HANDSHAKES, + "replica handshake in-flight cap reached; dropping inbound" + ); + return; + }; + match inbound_coord.delegate_replica_inbound(stream, slot) { + Ok(target) => { + info!(slot, target, "inbound replica connection delegated"); + } + Err(error) => { + inbound_bus.release_replica_handshake_slot(slot); + warn!( + error = ?error, + "delegate_replica_inbound failed; dropping inbound replica connection" + ); + } + } + }); + + let outbound_bus = Rc::clone(bus); + let dialed: DialedReplicaFn = + Rc::new( + move |stream, peer_id| match coord.delegate_replica_outbound(stream, peer_id) { + Ok(target) => { + outbound_bus.mark_dial_pending(peer_id); + info!(peer_id, target, "outbound replica connection delegated"); + } + Err(error) => { + warn!( + peer_id, + error = ?error, + "delegate_replica_outbound failed; dropping dialed replica connection" + ); + } + }, + ); + + (accepted, dialed) +} + +/// Shard-0 client accept callbacks. TCP and WS clients are delegated via +/// the coordinator (round-robin to peer shards); QUIC and TCP-TLS install +/// locally on shard 0 because their per-connection state is not portable +/// across shards (`compio_quic` endpoint binds one UDP socket; rustls TLS +/// state ties to the post-handshake reactor). +// ws/wss bindings intentionally mirror the transport names (same convention as +// `replica_io::start_on_shard_zero`). +#[allow(clippy::similar_names)] +fn make_shard_zero_client_accept_fns( + coord: Rc, + bus: &Rc, + on_request: RequestHandler, +) -> LocalClientAcceptFns { + let quic_bus = Rc::clone(bus); + let tcp_tls_bus = Rc::clone(bus); + let wss_bus = Rc::clone(bus); + let quic_request = on_request.clone(); + let wss_request = on_request.clone(); + let tcp_tls_request = on_request; + + let tcp_coord = Rc::clone(&coord); + let tcp = Rc::new(move |stream| match tcp_coord.delegate_client(stream) { + Ok(client_id) => info!(client_id, "TCP client delegated"), + Err(error) => warn!(error = ?error, "delegate_client failed; dropping TCP client"), + }); + + let ws_coord = Rc::clone(&coord); + let ws = Rc::new(move |stream| match ws_coord.delegate_ws_client(stream) { + Ok(client_id) => info!(client_id, "WS client delegated"), + Err(error) => warn!(error = ?error, "delegate_ws_client failed; dropping WS client"), + }); + + // QUIC and TCP-TLS terminate locally on shard 0 but mint their client + // ids through the coordinator's `client_seq`, the same counter the + // delegated TCP/WS path uses. A separate counter here would let a + // shard-0-local id collide with a delegated id that round-robined to + // shard 0 (both encode target shard 0) in shard 0's connection + // registry. + let quic_coord = Rc::clone(&coord); + let quic = Rc::new(move |accepted: message_bus::AcceptedQuicConn| { + let meta = mint_client_meta(&quic_coord, accepted.peer_addr(), ClientTransportKind::Quic); + installer::install_client_quic(&quic_bus, meta, accepted, quic_request.clone()); + }); + + let tcp_tls_coord = Rc::clone(&coord); + let tcp_tls = Rc::new(move |stream, tls_config| { + let Some(meta) = + client_meta_from_stream(&stream, &tcp_tls_coord, ClientTransportKind::TcpTls) + else { + return; + }; + installer::install_client_tcp_tls( + &tcp_tls_bus, + meta, + stream, + tls_config, + tcp_tls_request.clone(), + ); + }); + + // WSS terminates locally on shard 0 like TCP-TLS (rustls state is not + // serialisable across the delegate path), minting ids through the same + // coordinator counter. + let wss_coord = coord; + let wss = Rc::new(move |stream, tls_config| { + let Some(meta) = client_meta_from_stream(&stream, &wss_coord, ClientTransportKind::Wss) + else { + return; + }; + installer::install_client_wss(&wss_bus, meta, stream, tls_config, wss_request.clone()); + }); + + LocalClientAcceptFns { + tcp, + ws, + quic, + tcp_tls, + wss, + } +} + +fn client_meta_from_stream( + stream: &compio::net::TcpStream, + coord: &shard::coordinator::ShardZeroCoordinator, + transport: ClientTransportKind, +) -> Option { + let peer_addr = match stream.peer_addr() { + Ok(peer_addr) => peer_addr, + Err(error) => { + warn!(error = %error, "dropping accepted client with unknown peer address"); + return None; + } + }; + Some(mint_client_meta(coord, peer_addr, transport)) +} + +fn mint_client_meta( + coord: &shard::coordinator::ShardZeroCoordinator, + peer_addr: SocketAddr, + transport: ClientTransportKind, +) -> ClientConnMeta { + ClientConnMeta::new(coord.mint_shard_zero_client_id(), peer_addr, transport) +} + +async fn start_client_listeners( + shard: &Rc, + config: &ServerConfig, + topology: &TcpTopology, + accepted_clients: &LocalClientAcceptFns, +) -> Result { + let mut bound = BoundClientListeners::default(); + + if config.tcp.enabled && !config.tcp.tls.enabled { + let (listener, bound_addr) = client_listener::tcp::bind(topology.client_listen_addr) + .await + .map_err(|source| { + error!( + addr = %topology.client_listen_addr, + error = %source, + "failed to bind TCP client listener" + ); + source + })?; + let token = shard.bus.token(); + let accepted_client = accepted_clients.tcp.clone(); + let client_handle = compio::runtime::spawn(async move { + client_listener::tcp::run(listener, token, accepted_client).await; + }); + shard.bus.track_background(client_handle); + bound.tcp = Some(bound_addr); + } + + if let Some(ws_addr) = topology.ws_listen_addr { + bound.ws = Some(start_websocket_listener(shard, config, ws_addr, accepted_clients).await?); + } + + if let Some(quic_addr) = topology.quic_listen_addr { + install_default_crypto_provider(); + let credentials = load_quic_server_credentials(config)?; + let server_config = server_config_with_cert( + credentials.cert_chain, + credentials.key_der, + &shard.bus.config().quic, + ) + .map_err(|e| { + let source = + iggy_common::IggyError::IoError(format!("QUIC server config build failed: {e}")); + error!(addr = %quic_addr, error = %source, "failed to build QUIC server config"); + source + })?; + let (endpoint, bound_addr) = client_listener::quic::bind(quic_addr, server_config) + .map_err(|source| { + error!(addr = %quic_addr, error = %source, "failed to bind QUIC listener"); + source + })?; + let token = shard.bus.token(); + let handshake_grace = shard.bus.config().handshake_grace; + let accepted_quic = accepted_clients.quic.clone(); + let quic_handle = compio::runtime::spawn(async move { + client_listener::quic::run(endpoint, token, accepted_quic, handshake_grace).await; + }); + shard.bus.track_background(quic_handle); + bound.quic = Some(bound_addr); + } + + if config.tcp.enabled && config.tcp.tls.enabled { + let credentials = load_tcp_tls_server_credentials(config)?; + let (listener, tls_config, bound_addr) = + client_listener::tcp_tls::bind(topology.client_listen_addr, credentials).map_err( + |source| { + error!( + addr = %topology.client_listen_addr, + error = %source, + "failed to bind TCP TLS listener" + ); + source + }, + )?; + let token = shard.bus.token(); + let accepted_tls = accepted_clients.tcp_tls.clone(); + let tls_handle = compio::runtime::spawn(async move { + client_listener::tcp_tls::run(listener, tls_config, token, accepted_tls).await; + }); + shard.bus.track_background(tls_handle); + bound.tcp_tls = Some(bound_addr); + } + + Ok(bound) +} + +/// Build the replica auth context from cluster config. Returns `None` when the +/// cluster or replica auth is disabled, keeping the handshake in legacy mode. +/// Only the derived MAC keys are carried onward in [`ReplicaAuth`]; the raw +/// secrets (masked in config logs via `config_env(secret)`) are read here only +/// to derive them. A non-empty `previous_shared_secret` opens the verify-only +/// rotation acceptance window (see the [`ReplicaAuth`] rustdoc for the rolling +/// rotation procedure). `ClusterConfig::validate` guarantees a non-empty +/// secret whenever both `cluster.enabled` and `cluster.auth.enabled` are set +/// (validate early-returns `Ok` while `cluster.enabled` is false). +fn load_replica_auth(config: &ServerConfig) -> Option { + if !config.cluster.enabled || !config.cluster.auth.enabled { + return None; + } + let auth = ReplicaAuth::new(config.cluster.auth.shared_secret.as_bytes()); + let previous_shared_secret = &config.cluster.auth.previous_shared_secret; + if previous_shared_secret.is_empty() { + return Some(auth); + } + Some(auth.with_previous_secret(previous_shared_secret.as_bytes())) +} + +/// Build the replica TLS context from cluster config. Returns `None` when +/// the cluster or replica TLS is disabled. Every shard calls this once at +/// boot: CA mode re-reads the same PEM files per shard; self-signed mode +/// mints a per-shard throwaway certificate. Neither mode carries client +/// certificates, so TLS authenticates the acceptor only; peer +/// authentication comes from the PSK handshake (`ClusterConfig::validate` +/// enforces `cluster.auth.enabled` whenever `cluster.tls.enabled`). +/// +/// Both rustls configs are TLS 1.3 only with the [`REPLICA_ALPN`] +/// protocol pinned. The dialer's SNI / certificate-verify name for each +/// peer is the roster entry's `ip` field (a hostname or IP literal, the +/// same string the connector dials). +fn load_replica_tls_ctx( + config: &ServerConfig, + topology: &TcpTopology, +) -> Result, ServerError> { + let tls = &config.cluster.tls; + if !config.cluster.enabled || !tls.enabled { + return Ok(None); + } + install_default_crypto_provider(); + let credential_error = |source: std::io::Error| ServerError::ListenerCredentials { + transport: "cluster.tls", + source, + }; + + let credentials = if tls.self_signed { + let san = config + .cluster + .nodes + .iter() + .find(|node| node.replica_id == topology.self_replica_id) + .map(|node| node.ip.as_str()) + .ok_or_else(|| { + credential_error(std::io::Error::other(format!( + "replica id {} not present in cluster.nodes", + topology.self_replica_id + ))) + })?; + let (cert_chain, key_der) = server_common::generate_self_signed_certificate(san) + .map_err(|error| credential_error(std::io::Error::other(error.to_string())))?; + TlsServerCredentials { + cert_chain, + key_der, + } + } else { + load_pem(Path::new(&tls.cert_file), Path::new(&tls.key_file)).map_err(credential_error)? + }; + + let mut server = + rustls::ServerConfig::builder_with_protocol_versions(&[&rustls::version::TLS13]) + .with_no_client_auth() + .with_single_cert(credentials.cert_chain, credentials.key_der) + .map_err(|error| { + credential_error(std::io::Error::other(format!( + "replica TLS server config rejected credentials: {error}" + ))) + })?; + server.alpn_protocols = vec![REPLICA_ALPN.to_vec()]; + + let client_builder = + rustls::ClientConfig::builder_with_protocol_versions(&[&rustls::version::TLS13]); + let mut client = if tls.self_signed { + client_builder + .dangerous() + .with_custom_certificate_verifier(Arc::new(AcceptAnyServerCert)) + .with_no_client_auth() + } else { + let roots = load_ca_pem(Path::new(&tls.ca_file)).map_err(credential_error)?; + client_builder + .with_root_certificates(Arc::new(roots)) + .with_no_client_auth() + }; + client.alpn_protocols = vec![REPLICA_ALPN.to_vec()]; + + // Keyed by replica id, never by roster position: sparse ids (dynamic + // replica join) would make a positional lookup verify against another + // peer's SNI name. + let peer_names = config + .cluster + .nodes + .iter() + .map(|node| { + let name = ServerName::try_from(node.ip.clone()).map_err(|error| { + credential_error(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!( + "cluster node '{}' ip '{}' is not a valid TLS server name: {error}", + node.name, node.ip + ), + )) + })?; + Ok((node.replica_id, name)) + }) + .collect::, ServerError>>()?; + + Ok(Some(ReplicaTlsCtx { + server: Arc::new(server), + client: Arc::new(client), + peer_names, + })) +} + +fn load_tcp_tls_server_credentials( + config: &ServerConfig, +) -> Result { + let tls = &config.tcp.tls; + if tls.self_signed && !Path::new(&tls.cert_file).exists() { + return Ok(self_signed_for_loopback()); + } + + load_pem(Path::new(&tls.cert_file), Path::new(&tls.key_file)).map_err(|source| { + ServerError::ListenerCredentials { + transport: "tcp.tls", + source, + } + }) +} + +/// Bind the websocket client listener on `ws_addr`: WSS when +/// `websocket.tls.enabled` (the plain-WS accept loop must not also bind the +/// port -- a plain upgrade parser fed a TLS `ClientHello` rejects every +/// connection with an httparse error), plain WS otherwise. +async fn start_websocket_listener( + shard: &Rc, + config: &ServerConfig, + ws_addr: SocketAddr, + accepted_clients: &LocalClientAcceptFns, +) -> Result { + if config.websocket.tls.enabled { + let credentials = load_wss_server_credentials(config)?; + let (listener, tls_config, bound_addr) = client_listener::wss::bind(ws_addr, credentials) + .map_err(|source| { + error!(addr = %ws_addr, error = %source, "failed to bind WSS listener"); + source + })?; + let token = shard.bus.token(); + let accepted_wss = accepted_clients.wss.clone(); + let wss_handle = compio::runtime::spawn(async move { + client_listener::wss::run(listener, tls_config, token, accepted_wss).await; + }); + shard.bus.track_background(wss_handle); + Ok(bound_addr) + } else { + let (listener, bound_addr) = + client_listener::ws::bind(ws_addr).await.map_err(|source| { + error!(addr = %ws_addr, error = %source, "failed to bind websocket listener"); + source + })?; + let token = shard.bus.token(); + let accepted_ws = accepted_clients.ws.clone(); + let ws_handle = compio::runtime::spawn(async move { + client_listener::ws::run(listener, token, accepted_ws).await; + }); + shard.bus.track_background(ws_handle); + Ok(bound_addr) + } +} + +fn load_wss_server_credentials(config: &ServerConfig) -> Result { + let tls = &config.websocket.tls; + if tls.self_signed && !Path::new(&tls.cert_file).exists() { + return Ok(self_signed_for_loopback()); + } + + load_pem(Path::new(&tls.cert_file), Path::new(&tls.key_file)).map_err(|source| { + ServerError::ListenerCredentials { + transport: "websocket.tls", + source, + } + }) +} + +fn load_quic_server_credentials( + config: &ServerConfig, +) -> Result { + let certificate = &config.quic.certificate; + if certificate.self_signed { + let (cert_chain, key_der) = server_common::generate_self_signed_certificate("localhost") + .map_err(|error| ServerError::ListenerCredentials { + transport: "quic", + source: std::io::Error::other(error.to_string()), + })?; + return Ok(replica_io::QuicServerCredentials { + cert_chain, + key_der, + }); + } + + let credentials = load_pem( + Path::new(&certificate.cert_file), + Path::new(&certificate.key_file), + ) + .map_err(|source| ServerError::ListenerCredentials { + transport: "quic", + source, + })?; + Ok(replica_io::QuicServerCredentials { + cert_chain: credentials.cert_chain, + key_der: credentials.key_der, + }) +} + +fn parse_socket_addr(context: &'static str, address: &str) -> Result { + address + .parse() + .map_err(|source| ServerError::SocketAddressParse { + context, + address: address.to_string(), + source, + }) +} + +fn socket_addr_from_parts( + context: &'static str, + host: &str, + port: u16, +) -> Result { + let ip = host + .parse::() + .map_err(|source| ServerError::SocketAddressParse { + context, + address: format!("{host}:{port}"), + source, + })?; + Ok(SocketAddr::new(ip, port)) +} + +/// Build the closure that broadcasts a +/// [`LifecycleFrame::MetadataCommitTick`] to every shard's inbox after a +/// partition-shaped metadata operation commits on shard 0. +/// +/// The receiver-side partition reconciliation loop listens for these +/// wake-ups; coalescing is intentional, so `Full` is recorded as a metric +/// and dropped (the periodic tick recovers). Installed via +/// [`metadata::IggyMetadata::set_commit_notifier`] on shard 0 only, the +/// sole writer of the metadata state machine. +fn make_metadata_commit_notifier( + senders: Vec, + metrics: ShardMetrics, +) -> metadata::CommitNotifier { + Rc::new(move |operation: Operation| { + if !operation_triggers_partition_reconcile(operation) { + return; + } + for sender in &senders { + let frame = ShardFrame::lifecycle(LifecycleFrame::MetadataCommitTick); + match sender.try_send(frame) { + Ok(()) => {} + Err(crossfire::TrySendError::Full(_)) => { + metrics.record_frame_drop( + frame_drop_variant::METADATA_COMMIT_TICK, + frame_drop_reason::FULL, + ); + } + Err(crossfire::TrySendError::Disconnected(_)) => { + metrics.record_frame_drop( + frame_drop_variant::METADATA_COMMIT_TICK, + frame_drop_reason::DISCONNECTED, + ); + } + } + } + }) +} + +/// Filter at the broadcast site, keeping unrelated ops off the SDK reply +/// path. Any new partition-shape op must be added here. +/// +/// The bare `CreateTopic` / `CreatePartitions` arms are unreachable: the +/// leader's prepare-builder in `IggyMetadata` rewrites both into their +/// `*WithAssignments` form, stamping each partition's `consensus_group_id` +/// before journaling, so a committed prepare only ever carries the +/// assignment-bearing variant. Kept as defense-in-depth against a future +/// commit path that emits a bare op. +/// +/// "Partition-shape" is not only the partition SET: the purge and truncate +/// ops leave the set intact but advance per-partition state (purge +/// generation, delete watermark) that only the reconciler enforces on disk. +/// Omitting them defers the on-disk effect to the periodic safety tick, +/// stretching a purge's client-visible tail to a full +/// `reconcile_periodic_interval`. `DeleteSegments` is absent by design: the +/// leader rewrites it into `TruncatePartition` before journaling, so no +/// commit ever carries it. +const fn operation_triggers_partition_reconcile(op: Operation) -> bool { + matches!( + op, + Operation::CreateTopic + | Operation::CreateTopicWithAssignments + | Operation::CreatePartitions + | Operation::CreatePartitionsWithAssignments + | Operation::DeleteTopic + | Operation::DeleteStream + | Operation::DeletePartitions + | Operation::PurgeStream + | Operation::PurgeTopic + | Operation::TruncatePartition + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fresh_cluster_bootstrap_requires_explicit_root_credentials() { + assert!(matches!( + validate_root_credentials(true, None, None), + Err(ServerError::ClusterRootCredentialsRequired { + username_env: IGGY_ROOT_USERNAME_ENV, + password_env: IGGY_ROOT_PASSWORD_ENV, + }) + )); + validate_root_credentials(true, Some("root"), Some("secret")) + .expect("both credentials supplied must satisfy the fresh-cluster guard"); + } + + #[test] + fn single_node_bootstrap_generates_root_credentials_when_unset() { + validate_root_credentials(false, None, None) + .expect("a single node mints its own root password"); + } + + #[test] + fn half_set_root_credentials_are_rejected_in_both_directions() { + assert!(matches!( + validate_root_credentials(false, Some("root"), None), + Err(ServerError::RootCredentialsIncomplete { + provided_env: IGGY_ROOT_USERNAME_ENV, + missing_env: IGGY_ROOT_PASSWORD_ENV, + }) + )); + assert!(matches!( + validate_root_credentials(false, None, Some("secret")), + Err(ServerError::RootCredentialsIncomplete { + provided_env: IGGY_ROOT_PASSWORD_ENV, + missing_env: IGGY_ROOT_USERNAME_ENV, + }) + )); + } + + #[test] + fn out_of_range_root_credentials_are_rejected() { + assert!(matches!( + validate_root_credentials(false, Some(""), Some("secret")), + Err(ServerError::RootCredentialLength { + env_name: IGGY_ROOT_USERNAME_ENV, + length: 0, + .. + }) + )); + let too_long = "x".repeat(MAX_PASSWORD_LENGTH + 1); + assert!(matches!( + validate_root_credentials(false, Some("root"), Some(&too_long)), + Err(ServerError::RootCredentialLength { + env_name: IGGY_ROOT_PASSWORD_ENV, + .. + }) + )); + } + + #[test] + fn default_cluster_heartbeat_timeout_matches_consensus_constant() { + // The config default lives in core/server/config.toml (a string, + // so no static assert can pin it); keep it in lockstep with the + // built-in the simulator and un-configured replicas run on. + let config_default = configs::cluster::ClusterConfig::default() + .heartbeat_timeout + .get_duration() + .as_millis(); + let built_in = u128::from(consensus::TimeoutManager::NORMAL_HEARTBEAT_TICKS) + * shard::CONSENSUS_TICK_INTERVAL.as_millis(); + assert_eq!( + config_default, built_in, + "[cluster] heartbeat_timeout default drifted from \ + TimeoutManager::NORMAL_HEARTBEAT_TICKS" + ); + } + + #[test] + fn reconciler_driven_ops_broadcast_a_commit_tick() { + // These commit without touching the partition set, so nothing else + // signals the reconciler: `reconcile_partition_purges` and + // `reconcile_segment_truncations` are the only code that turns them + // into on-disk effect, and they run only when a pass runs. Dropping + // one from the filter silently downgrades it to the periodic tick. + for op in [ + Operation::PurgeStream, + Operation::PurgeTopic, + Operation::TruncatePartition, + ] { + assert!( + operation_triggers_partition_reconcile(op), + "{op:?} is enforced by the reconciler and must wake it on commit" + ); + } + assert!( + !operation_triggers_partition_reconcile(Operation::CreateUser), + "ops with no partition-shape effect must stay off the broadcast" + ); + } + + #[test] + fn recovery_barrier_deadline_holds_the_floor_for_small_heartbeats() { + // Below the 5s default the heartbeat-independent recovery term (~7s of + // ViewChangeStatus backstop plus ceremony) dominates, so the floor + // governs however small the heartbeat is; 3 x 5s lands exactly on it. + // A default-sized status backstop stays on the floor, not above it. + assert_eq!( + recovery_barrier_deadline(Duration::from_secs(1), Duration::from_secs(5)), + RECOVERY_BARRIER_DEADLINE_FLOOR + ); + assert_eq!( + recovery_barrier_deadline(Duration::from_secs(5), Duration::from_secs(5)), + RECOVERY_BARRIER_DEADLINE_FLOOR + ); + } + + #[test] + fn recovery_barrier_deadline_scales_past_the_floor_for_large_heartbeats() { + // Once 3 x heartbeat clears the floor the scaled window governs, so a + // slow-heartbeat cluster is not failed 503 before its longer recovery + // can finish. A default-sized status backstop stays under it. + assert_eq!( + recovery_barrier_deadline(Duration::from_secs(10), Duration::from_secs(5)), + Duration::from_secs(30) + ); + assert_eq!( + recovery_barrier_deadline(Duration::from_secs(15), Duration::from_secs(5)), + Duration::from_secs(45) + ); + } + + #[test] + fn recovery_barrier_deadline_scales_with_the_status_backstop() { + // A raised view-change status backstop stretches worst-case recovery + // even when the heartbeat stays fast, so the deadline must track it or + // post-restart reads 503 before a slow election settles. + assert_eq!( + recovery_barrier_deadline(Duration::from_secs(1), Duration::from_secs(10)), + Duration::from_secs(30) + ); + } + + #[test] + fn recovery_barrier_deadline_at_config_defaults_matches_the_floor() { + // Folding the status term in must not move the stock deadline: at the + // shared 5s defaults each scaled term lands exactly on the 15s floor, + // so an un-tuned cluster keeps its pre-existing recovery window. + let cluster = configs::cluster::ClusterConfig::default(); + assert_eq!( + recovery_barrier_deadline( + cluster.heartbeat_timeout.get_duration(), + cluster.view_change_status_timeout.get_duration(), + ), + RECOVERY_BARRIER_DEADLINE_FLOOR + ); + } + + #[test] + fn recovery_barrier_deadline_saturates_instead_of_panicking() { + // Neither timeout has a config ceiling, so both multiplies must + // saturate rather than abort boot on an absurd parseable value. + assert_eq!( + recovery_barrier_deadline(Duration::MAX, Duration::from_secs(5)), + Duration::MAX + ); + assert_eq!( + recovery_barrier_deadline(Duration::from_secs(5), Duration::MAX), + Duration::MAX + ); + } + + #[test] + fn default_commit_broadcast_interval_matches_consensus_constant() { + // The config default lives in core/server/config.toml (a string, + // so no static assert can pin it); keep it in lockstep with the + // built-in the simulator and un-configured replicas run on. + let config_default = configs::cluster::ClusterConfig::default() + .commit_broadcast_interval + .get_duration() + .as_millis(); + let built_in = u128::from(consensus::TimeoutManager::COMMIT_MESSAGE_TICKS) + * shard::CONSENSUS_TICK_INTERVAL.as_millis(); + assert_eq!( + config_default, built_in, + "[cluster] commit_broadcast_interval default drifted from \ + TimeoutManager::COMMIT_MESSAGE_TICKS" + ); + } + + #[test] + fn default_prepare_retransmit_interval_matches_consensus_constant() { + // The config default lives in core/server/config.toml (a string, + // so no static assert can pin it); keep it in lockstep with the + // built-in the simulator and un-configured replicas run on. + let config_default = configs::cluster::ClusterConfig::default() + .prepare_retransmit_interval + .get_duration() + .as_millis(); + let built_in = u128::from(consensus::TimeoutManager::PREPARE_TICKS) + * shard::CONSENSUS_TICK_INTERVAL.as_millis(); + assert_eq!( + config_default, built_in, + "[cluster] prepare_retransmit_interval default drifted from \ + TimeoutManager::PREPARE_TICKS" + ); + } + + #[test] + fn default_partition_prepare_queue_depth_matches_consensus_constant() { + // The config default lives in core/server/config.toml and flows + // through PartitionConfig::default(); keep the embedded value in + // lockstep with the pipeline depth LocalPipeline::new() (the simulator + // and tests) runs on, so a default deployment is byte-identical. + let config_default = configs::partition::PartitionConfig::default().prepare_queue_depth; + assert_eq!( + config_default, + consensus::PIPELINE_PREPARE_QUEUE_MAX, + "[partition] prepare_queue_depth default drifted from \ + consensus::PIPELINE_PREPARE_QUEUE_MAX" + ); + } + + #[test] + fn default_view_change_retransmit_interval_matches_consensus_constant() { + // The config default lives in core/server/config.toml (a string, so + // no static assert can pin it). One knob drives both view-change + // retransmit timers, which are equal by design, so pin it against both. + let config_default = configs::cluster::ClusterConfig::default() + .view_change_retransmit_interval + .get_duration() + .as_millis(); + let start_view_change = + u128::from(consensus::TimeoutManager::START_VIEW_CHANGE_MESSAGE_TICKS) + * shard::CONSENSUS_TICK_INTERVAL.as_millis(); + let do_view_change = u128::from(consensus::TimeoutManager::DO_VIEW_CHANGE_MESSAGE_TICKS) + * shard::CONSENSUS_TICK_INTERVAL.as_millis(); + assert_eq!( + config_default, start_view_change, + "[cluster] view_change_retransmit_interval default drifted from \ + TimeoutManager::START_VIEW_CHANGE_MESSAGE_TICKS" + ); + assert_eq!( + config_default, do_view_change, + "[cluster] view_change_retransmit_interval default drifted from \ + TimeoutManager::DO_VIEW_CHANGE_MESSAGE_TICKS" + ); + } + + #[test] + fn default_view_change_status_timeout_matches_consensus_constant() { + // The config default lives in core/server/config.toml (a string, so + // no static assert can pin it); keep it in lockstep with the built-in + // the simulator and un-configured replicas run on. + let config_default = configs::cluster::ClusterConfig::default() + .view_change_status_timeout + .get_duration() + .as_millis(); + let built_in = u128::from(consensus::TimeoutManager::VIEW_CHANGE_STATUS_TICKS) + * shard::CONSENSUS_TICK_INTERVAL.as_millis(); + assert_eq!( + config_default, built_in, + "[cluster] view_change_status_timeout default drifted from \ + TimeoutManager::VIEW_CHANGE_STATUS_TICKS" + ); + } + + #[test] + fn default_request_start_view_retransmit_interval_matches_consensus_constant() { + // The config default lives in core/server/config.toml (a string, so + // no static assert can pin it); keep it in lockstep with the built-in + // the simulator and un-configured replicas run on. + let config_default = configs::cluster::ClusterConfig::default() + .request_start_view_retransmit_interval + .get_duration() + .as_millis(); + let built_in = u128::from(consensus::TimeoutManager::REQUEST_START_VIEW_MESSAGE_TICKS) + * shard::CONSENSUS_TICK_INTERVAL.as_millis(); + assert_eq!( + config_default, built_in, + "[cluster] request_start_view_retransmit_interval default drifted from \ + TimeoutManager::REQUEST_START_VIEW_MESSAGE_TICKS" + ); + } + + #[test] + fn default_view_probe_attempts_max_matches_consensus_constant() { + // Belt and suspenders with the static assert above: that pins the + // duplicated configs-crate literal, this pins the shipped config.toml + // value the simulator and un-configured replicas run on. + let config_default = configs::cluster::ClusterConfig::default().view_probe_attempts_max; + assert_eq!( + config_default, + consensus::PROBE_ATTEMPTS_MAX, + "[cluster] view_probe_attempts_max default drifted from \ + consensus::PROBE_ATTEMPTS_MAX" + ); + } + + #[test] + fn default_repair_retry_interval_matches_partitions_constant() { + // The config default lives in core/server/config.toml (a string, so + // no static assert can pin it); keep it in lockstep with the built-in + // the simulator and un-configured replicas run on. + let config_default = configs::cluster::ClusterConfig::default() + .repair_retry_interval + .get_duration() + .as_millis(); + let built_in = + u128::from(partitions::REPAIR_RETRY_TICKS) * shard::CONSENSUS_TICK_INTERVAL.as_millis(); + assert_eq!( + config_default, built_in, + "[cluster] repair_retry_interval default drifted from \ + partitions::REPAIR_RETRY_TICKS" + ); + } + + #[test] + fn default_repair_chunk_max_matches_shard_constant() { + // Belt and suspenders with the static assert above: that pins the + // duplicated configs-crate literal, this pins the shipped config.toml + // value the simulator and un-configured replicas run on. + let config_default = configs::cluster::ClusterConfig::default().repair_chunk_max; + assert_eq!( + config_default as u64, + shard::REPAIR_CHUNK_MAX, + "[cluster] repair_chunk_max default drifted from shard::REPAIR_CHUNK_MAX" + ); + } + + #[test] + fn default_evicted_ring_capacity_matches_partitions_constant() { + // Belt and suspenders with the static assert above; this pins the + // shipped config.toml value. + let config_default = configs::partition::PartitionConfig::default().evicted_ring_capacity; + assert_eq!( + config_default, + partitions::EVICTED_RING_CAPACITY, + "[partition] evicted_ring_capacity default drifted from \ + partitions::EVICTED_RING_CAPACITY" + ); + } + + #[test] + fn default_evicted_ring_bytes_max_matches_partitions_constant() { + // Belt and suspenders with the static assert above; this pins the + // shipped config.toml value. + let config_default = configs::partition::PartitionConfig::default() + .evicted_ring_bytes_max + .as_bytes_u64(); + assert_eq!( + config_default, + partitions::EVICTED_RING_BYTES_MAX, + "[partition] evicted_ring_bytes_max default drifted from \ + partitions::EVICTED_RING_BYTES_MAX" + ); + } + + #[test] + fn shutdown_on_drop_armed_flips_flag() { + let flag = Arc::new(AtomicBool::new(false)); + drop(ShutdownOnDrop::new(Arc::clone(&flag))); + assert!( + flag.load(Ordering::Relaxed), + "an armed guard must flip the flag on drop (covers the error `?` \ + and panic-unwind exit paths of run_shard_thread)" + ); + } + + #[test] + fn shutdown_on_drop_disarmed_leaves_flag() { + let flag = Arc::new(AtomicBool::new(false)); + let mut guard = ShutdownOnDrop::new(Arc::clone(&flag)); + guard.disarm(); + drop(guard); + assert!( + !flag.load(Ordering::Relaxed), + "a disarmed guard must not flip the flag (clean `Ok(())` exit)" + ); + } + + const TEST_POLL_INTERVAL: Duration = Duration::from_millis(50); + + #[compio::test] + async fn broadcast_metadata_bundle_returns_immediately_with_no_peers() { + // Single-shard deployment: shard 0 has no peers to fan out to, + // so the handoff must complete without ever calling `send`. + let (bundle_tx, _bundle_rx) = crossfire::mpmc::bounded_async::(0); + let flag = Arc::new(AtomicBool::new(false)); + let mux = ServerMuxStateMachine::default(); + broadcast_metadata_bundle( + 0, + &bundle_tx, + mux.factory_bundle(), + 0, + &flag, + TEST_POLL_INTERVAL, + ) + .await + .expect("zero peers must not block shard 0"); + } + + #[compio::test] + async fn metadata_bundle_round_trips_through_channel() { + // End-to-end: shard 0 mints a bundle, a peer receives it on + // another runtime, and `from_factory_bundle` constructs a + // reader-mode mux that observes shard 0's writes via the same + // LeftRight pair. + let peers = 1u16; + let (bundle_tx, bundle_rx) = + crossfire::mpmc::bounded_async::(usize::from(peers)); + let flag = Arc::new(AtomicBool::new(false)); + + let owner = ServerMuxStateMachine::default(); + let bundle = owner.factory_bundle(); + broadcast_metadata_bundle(0, &bundle_tx, bundle, peers, &flag, TEST_POLL_INTERVAL) + .await + .expect("broadcast must succeed with one peer drained"); + + let received = await_metadata_bundle(1, &bundle_rx, &flag, TEST_POLL_INTERVAL) + .await + .expect("peer must receive the broadcast bundle"); + let _peer_mux = ServerMuxStateMachine::from_factory_bundle(received); + } + + #[compio::test] + async fn broadcast_metadata_bundle_aborts_when_peers_drop_rx() { + // Shard 0 drives handoff but every peer's `bundle_rx` was dropped + // before recv. Silently returning Ok would commit listener binds + // and consensus init for a cluster whose peers are gone; the + // broadcast must surface the disconnect so `shard_main` aborts. + let (bundle_tx, bundle_rx) = crossfire::mpmc::bounded_async::(0); + drop(bundle_rx); + let flag = Arc::new(AtomicBool::new(false)); + let mux = ServerMuxStateMachine::default(); + + let err = broadcast_metadata_bundle( + 0, + &bundle_tx, + mux.factory_bundle(), + 3, + &flag, + TEST_POLL_INTERVAL, + ) + .await + .expect_err("dropped rx must surface as MetadataHandoffAborted"); + assert!( + matches!(err, ServerError::MetadataHandoffAborted { shard_id: 0 }), + "expected MetadataHandoffAborted, got {err:?}" + ); + } + + #[compio::test] + async fn await_metadata_bundle_aborts_when_owner_drops_without_sending() { + let (bundle_tx, bundle_rx) = crossfire::mpmc::bounded_async::(1); + let flag = Arc::new(AtomicBool::new(false)); + + // Shard 0 dies before broadcasting; the peer must observe the + // disconnect and abort instead of hanging forever. + drop(bundle_tx); + + let err = await_metadata_bundle(1, &bundle_rx, &flag, TEST_POLL_INTERVAL) + .await + .expect_err("a peer whose owner never sends must abort"); + assert!( + matches!(err, ServerError::MetadataHandoffAborted { shard_id: 1 }), + "expected MetadataHandoffAborted, got {err:?}" + ); + } + + #[compio::test] + async fn await_metadata_bundle_aborts_on_shutdown_flag() { + // compio 0.19 `JoinHandle` yields `Result`; the + // `ResumeUnwind` impl re-raises a task panic and maps cancellation + // to `None`. + use compio::runtime::ResumeUnwind; + + let (_bundle_tx, bundle_rx) = crossfire::mpmc::bounded_async::(1); + let flag = Arc::new(AtomicBool::new(false)); + + let waiter = compio::runtime::spawn({ + let flag = Arc::clone(&flag); + async move { await_metadata_bundle(1, &bundle_rx, &flag, TEST_POLL_INTERVAL).await } + }); + + // Owner has not sent yet, but shutdown was requested; the peer + // must exit via the flag poll instead of hanging. + compio::time::sleep(TEST_POLL_INTERVAL / 2).await; + flag.store(true, Ordering::Relaxed); + + let err = waiter + .await + .resume_unwind() + .expect("waiter task was cancelled") + .expect_err("shutdown flag must abort the bundle wait"); + assert!( + matches!(err, ServerError::MetadataHandoffAborted { shard_id: 1 }), + "expected MetadataHandoffAborted on shutdown, got {err:?}" + ); + } + + #[compio::test] + async fn await_bootstrap_complete_returns_immediately_for_single_shard() { + // A single-shard server has no peers to wait on; the owner barrier + // must not block when `peers == 0`. + let (_ready_tx, ready_rx) = crossfire::mpmc::bounded_async::(1); + let flag = Arc::new(AtomicBool::new(false)); + await_bootstrap_complete(&ready_rx, 0, &flag, TEST_POLL_INTERVAL) + .await + .expect("single-shard server must not block on the barrier"); + } + + #[compio::test] + async fn await_bootstrap_complete_drains_every_peer_signal() { + // Two peers report load-complete; shard 0 drains both, then proceeds + // to bind listeners. + let (ready_tx, ready_rx) = crossfire::mpmc::bounded_async::(2); + let flag = Arc::new(AtomicBool::new(false)); + signal_bootstrap_complete(1, &ready_tx, &flag, TEST_POLL_INTERVAL) + .await + .expect("peer 1 must signal load-complete"); + signal_bootstrap_complete(2, &ready_tx, &flag, TEST_POLL_INTERVAL) + .await + .expect("peer 2 must signal load-complete"); + await_bootstrap_complete(&ready_rx, 2, &flag, TEST_POLL_INTERVAL) + .await + .expect("owner must drain both peer signals"); + } + + #[compio::test] + async fn await_bootstrap_complete_aborts_on_shutdown_flag() { + use compio::runtime::ResumeUnwind; + + // `_ready_tx` is held so the channel is not disconnected: the owner + // must exit via the shutdown flag, not a dropped sender. + let (_ready_tx, ready_rx) = crossfire::mpmc::bounded_async::(1); + let flag = Arc::new(AtomicBool::new(false)); + + let owner = compio::runtime::spawn({ + let flag = Arc::clone(&flag); + async move { await_bootstrap_complete(&ready_rx, 1, &flag, TEST_POLL_INTERVAL).await } + }); + + // The peer never signals, but a sibling failure flips the flag; the + // owner must abort instead of hanging before listeners. + compio::time::sleep(TEST_POLL_INTERVAL / 2).await; + flag.store(true, Ordering::Relaxed); + + let err = owner + .await + .resume_unwind() + .expect("owner task was cancelled") + .expect_err("shutdown flag must abort the barrier wait"); + assert!( + matches!( + err, + ServerError::ShardBootstrapBarrierAborted { remaining: 1 } + ), + "expected ShardBootstrapBarrierAborted, got {err:?}" + ); + } + + #[compio::test] + async fn signal_bootstrap_complete_aborts_when_owner_drops_rx() { + // Shard 0 aborted before draining and dropped its receiver; a peer's + // signal must surface the disconnect instead of stranding. + let (ready_tx, ready_rx) = crossfire::mpmc::bounded_async::(1); + let flag = Arc::new(AtomicBool::new(false)); + drop(ready_rx); + + let err = signal_bootstrap_complete(2, &ready_tx, &flag, TEST_POLL_INTERVAL) + .await + .expect_err("dropped rx must surface as an abort"); + assert!( + matches!(err, ServerError::MetadataHandoffAborted { shard_id: 2 }), + "expected MetadataHandoffAborted, got {err:?}" + ); + } + + fn cluster_node(ip: &str, http: Option) -> configs::cluster::ClusterNodeConfig { + cluster_node_with_ports(ip, Some(18070), http) + } + + fn cluster_node_with_ports( + ip: &str, + tcp: Option, + http: Option, + ) -> configs::cluster::ClusterNodeConfig { + configs::cluster::ClusterNodeConfig { + name: "node".to_owned(), + ip: ip.to_owned(), + advertised_address: None, + advertised_addresses: Vec::new(), + replica_id: 0, + ports: configs::cluster::TransportPorts { + tcp, + http, + ..Default::default() + }, + } + } + + fn addr(value: &str) -> SocketAddr { + value.parse().expect("valid socket address literal") + } + + #[test] + fn cluster_http_addr_takes_port_from_roster() { + // A byte-identical top-level [http].address is shared across nodes on + // one host; the per-node roster port is the only port source so each + // node binds a distinct HTTP socket. + let node = cluster_node("127.0.0.1", Some(18090)); + let addrs = resolve_cluster_client_addrs( + &node, + addr("127.0.0.1:8090"), + None, + None, + Some(addr("127.0.0.1:3000")), + ) + .expect("cluster address resolution must succeed"); + assert_eq!(addrs.http, Some(addr("127.0.0.1:18090"))); + } + + #[test] + fn cluster_http_addr_merges_config_ip_with_roster_port() { + // Docker/Helm bind `0.0.0.0` and probe loopback; the roster ip is + // only the advertised address. Cluster mode must keep the configured + // interface and take just the port from the roster. + let node = cluster_node("10.0.0.5", Some(18090)); + let addrs = resolve_cluster_client_addrs( + &node, + addr("0.0.0.0:8090"), + None, + None, + Some(addr("0.0.0.0:3000")), + ) + .expect("cluster address resolution must succeed"); + assert_eq!(addrs.http, Some(addr("0.0.0.0:18090"))); + } + + #[test] + fn cluster_http_addr_requires_roster_port_for_enabled_transport() { + // No fallback to the top-level port: a silent default could collide + // with another same-host node, so a missing roster port for an + // enabled transport must refuse to boot. + let node = cluster_node("10.0.0.5", None); + let result = resolve_cluster_client_addrs( + &node, + addr("127.0.0.1:8090"), + None, + None, + Some(addr("127.0.0.1:3000")), + ); + assert!(matches!( + result, + Err(ServerError::ClusterPortMissing { + transport: "http", + replica_id: 0, + }) + )); + } + + #[test] + fn cluster_http_addr_is_none_when_http_disabled() { + // http.enabled = false collapses default_http_addr to None; no roster + // port can revive a listener the operator turned off. + let node = cluster_node("127.0.0.1", Some(18090)); + let addrs = resolve_cluster_client_addrs(&node, addr("127.0.0.1:8090"), None, None, None) + .expect("cluster address resolution must succeed"); + assert_eq!(addrs.http, None); + } + + /// Regression: the shutdown-join deadline must arm at SHUTDOWN, not + /// at boot. The original bound measured from `join_all` entry, so any + /// healthy server outliving `shutdown_join_timeout` (30s default) was + /// abandoned as "wedged" and the process exited - every BDD run died + /// at t+30s while the test container was still compiling. + #[test] + fn join_waits_unbounded_while_the_server_runs() { + let shutdown_flag = AtomicBool::new(false); + // Thread outlives a deliberately tiny join budget; with the flag + // clear the budget must never even arm. + let handle = thread::spawn(|| -> Result<(), ServerError> { + thread::sleep(Duration::from_millis(300)); + Ok(()) + }); + let mut deadline = None; + let joined = join_until_shutdown_deadline( + handle, + &shutdown_flag, + Duration::from_millis(20), + &mut deadline, + ); + assert!( + matches!(joined, Some(Ok(Ok(())))), + "a running server must be awaited indefinitely, not abandoned as wedged" + ); + assert!( + deadline.is_none(), + "the join deadline must not arm before the shutdown flag flips" + ); + } + + #[test] + fn join_abandons_a_wedged_shard_after_the_shutdown_deadline() { + let shutdown_flag = AtomicBool::new(true); + // Never finishes: stands in for a wedged pump. The thread leaks + // into the test process, which exits right after. + let handle = thread::spawn(|| -> Result<(), ServerError> { + loop { + thread::sleep(Duration::from_secs(1)); + } + }); + let mut deadline = None; + let joined = join_until_shutdown_deadline( + handle, + &shutdown_flag, + Duration::from_millis(100), + &mut deadline, + ); + assert!( + joined.is_none(), + "a shard still running past the post-shutdown budget must be abandoned" + ); + assert!(deadline.is_some(), "the deadline arms once the flag is set"); + } + + #[test] + fn cluster_tcp_addr_takes_port_from_roster() { + // Same rule as the other transports: the roster owns the port so + // same-host nodes sharing one [tcp].address still bind distinct + // sockets. + let node = cluster_node("127.0.0.1", None); + let addrs = resolve_cluster_client_addrs(&node, addr("127.0.0.1:8090"), None, None, None) + .expect("cluster address resolution must succeed"); + assert_eq!(addrs.client, addr("127.0.0.1:18070")); + } + + #[test] + fn cluster_tcp_addr_merges_config_ip_with_roster_port() { + // The roster ip is advertised, not bound. Binding it directly would + // strand every co-located dialer (sidecars, health probes, on-host + // consumers) that reaches this node over loopback. + let node = cluster_node("10.0.0.5", None); + let addrs = resolve_cluster_client_addrs(&node, addr("0.0.0.0:8090"), None, None, None) + .expect("cluster address resolution must succeed"); + assert_eq!(addrs.client, addr("0.0.0.0:18070")); + } + + #[test] + fn cluster_tcp_addr_requires_roster_port() { + // tcp is always enabled in cluster mode, so a roster entry without a + // tcp port refuses to boot rather than falling back to [tcp].address. + let node = cluster_node_with_ports("10.0.0.5", None, None); + let result = resolve_cluster_client_addrs(&node, addr("127.0.0.1:8090"), None, None, None); + assert!(matches!( + result, + Err(ServerError::ClusterPortMissing { + transport: "tcp", + replica_id: 0, + }) + )); + } + + #[test] + fn cluster_tcp_addr_keeps_loopback_bind_and_warns_on_roster_mismatch() { + // A loopback [tcp].address under a routable roster ip is honoured + // as configured; remote peers cannot reach it, so the mismatch is + // warned about instead of silently rebinding. + let node = cluster_node("10.0.0.5", None); + let addrs = resolve_cluster_client_addrs(&node, addr("127.0.0.1:8090"), None, None, None) + .expect("cluster address resolution must succeed"); + assert_eq!(addrs.client, addr("127.0.0.1:18070")); + assert!(roster_ip_unreachable_from_bind_addr(&node.ip, addrs.client)); + } + + #[test] + fn roster_mismatch_warning_is_silent_for_wildcard_and_hostname_rosters() { + // A wildcard bind covers the roster interface, and a DNS roster entry + // can resolve to the bound one; neither is a misconfiguration. + assert!(!roster_ip_unreachable_from_bind_addr( + "10.0.0.5", + addr("0.0.0.0:18070") + )); + assert!(!roster_ip_unreachable_from_bind_addr( + "node-1.example.com", + addr("127.0.0.1:18070") + )); + assert!(!roster_ip_unreachable_from_bind_addr( + "10.0.0.5", + addr("10.0.0.5:18070") + )); + } } diff --git a/core/server-ng/src/cluster_meta.rs b/core/server/src/cluster_meta.rs similarity index 98% rename from core/server-ng/src/cluster_meta.rs rename to core/server/src/cluster_meta.rs index 10d17b83db..7372d3e597 100644 --- a/core/server-ng/src/cluster_meta.rs +++ b/core/server/src/cluster_meta.rs @@ -28,7 +28,7 @@ //! leader, but the full roster is still returned). The self-synthesized single //! node is the cluster-disabled fallback, shared by both callers. -use configs::ng_cluster::{ResolvedClusterNode, TransportPorts}; +use configs::cluster::{ResolvedClusterNode, TransportPorts}; use iggy_common::{ ClusterMetadata, ClusterNode, ClusterNodeRole, ClusterNodeStatus, TransportEndpoints, }; @@ -181,7 +181,7 @@ fn ports_to_endpoints(ports: &TransportPorts) -> TransportEndpoints { mod tests { use super::*; - use configs::ng_cluster::{AdvertisedAddressSelector, ClusterNodeConfig}; + use configs::cluster::{AdvertisedAddressSelector, ClusterNodeConfig}; fn node_config(advertised_address: Option) -> ClusterNodeConfig { ClusterNodeConfig { diff --git a/core/server/src/compat/index_rebuilding/index_rebuilder.rs b/core/server/src/compat/index_rebuilding/index_rebuilder.rs deleted file mode 100644 index c36c53b63f..0000000000 --- a/core/server/src/compat/index_rebuilding/index_rebuilder.rs +++ /dev/null @@ -1,118 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::server_error::CompatError; -use crate::streaming::utils::file; -use compio::{ - fs::File, - io::{AsyncBufRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, BufReader, BufWriter}, -}; -use iggy_common::{IGGY_MESSAGE_HEADER_SIZE, IggyMessageHeader}; - -pub struct IndexRebuilder { - pub messages_file_path: String, - pub index_path: String, - pub start_offset: u64, -} - -impl IndexRebuilder { - pub fn new(messages_file_path: String, index_path: String, start_offset: u64) -> Self { - Self { - messages_file_path, - index_path, - start_offset, - } - } - - async fn read_message_header( - reader: &mut BufReader>, - ) -> Result { - let buf = [0u8; IGGY_MESSAGE_HEADER_SIZE]; - let (result, buf) = reader.read_exact(Box::new(buf)).await.into(); - result?; - IggyMessageHeader::from_raw_bytes(&*buf) - .map_err(|_| std::io::Error::from(std::io::ErrorKind::InvalidData)) - } - - async fn write_index_entry( - writer: &mut BufWriter>, - header: &IggyMessageHeader, - position: usize, - start_offset: u64, - ) -> Result<(), CompatError> { - // Write offset (4 bytes) - base_offset + last_offset_delta - start_offset - let offset = start_offset - header.offset; - debug_assert!(offset <= u32::MAX as u64); - let (result, _) = writer - .write_all(Box::new(offset.to_le_bytes())) - .await - .into(); - result?; - - // Write position (4 bytes) - let (result, _) = writer - .write_all(Box::new(position.to_le_bytes())) - .await - .into(); - result?; - - // Write timestamp (8 bytes) - let (result, _) = writer - .write_all(Box::new(header.timestamp.to_le_bytes())) - .await - .into(); - result?; - - Ok(()) - } - - pub async fn rebuild(&self) -> Result<(), CompatError> { - let read_cursor = std::io::Cursor::new(file::open(&self.messages_file_path).await?); - let write_cursor = std::io::Cursor::new(file::overwrite(&self.index_path).await?); - let mut reader = BufReader::new(read_cursor); - let mut writer = BufWriter::new(write_cursor); - let mut position = 0; - let mut next_position; - - loop { - match Self::read_message_header(&mut reader).await { - Ok(header) => { - next_position = position - + IGGY_MESSAGE_HEADER_SIZE - + header.payload_length as usize - + header.user_headers_length as usize; - - Self::write_index_entry(&mut writer, &header, position, self.start_offset) - .await?; - - // Skip message payload and headers - reader.consume( - header.payload_length as usize + header.user_headers_length as usize, - ); - - // Update position for next iteration - position = next_position; - } - Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => break, - Err(e) => return Err(e.into()), - } - } - - writer.flush().await?; - Ok(()) - } -} diff --git a/core/server/src/compat/index_rebuilding/mod.rs b/core/server/src/compat/index_rebuilding/mod.rs deleted file mode 100644 index 6bc85fd6a3..0000000000 --- a/core/server/src/compat/index_rebuilding/mod.rs +++ /dev/null @@ -1,18 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod index_rebuilder; diff --git a/core/server/src/compat/mod.rs b/core/server/src/compat/mod.rs deleted file mode 100644 index acfe367fbc..0000000000 --- a/core/server/src/compat/mod.rs +++ /dev/null @@ -1,18 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod index_rebuilding; diff --git a/core/server-ng/src/config_writer.rs b/core/server/src/config_writer.rs similarity index 87% rename from core/server-ng/src/config_writer.rs rename to core/server/src/config_writer.rs index 1d2602e82d..92d4fb2864 100644 --- a/core/server-ng/src/config_writer.rs +++ b/core/server/src/config_writer.rs @@ -15,10 +15,10 @@ // specific language governing permissions and limitations // under the License. -use crate::server_error::ServerNgError; +use crate::server_error::ServerError; use compio::fs::OpenOptions; use compio::io::AsyncWriteAtExt; -use configs::server_ng::ServerNgConfig; +use configs::server::ServerConfig; use std::net::SocketAddr; /// Write the runtime `current_config.toml` file with the effective bound ports. @@ -28,14 +28,14 @@ use std::net::SocketAddr; /// Returns an error if the config cannot be serialized or if the runtime /// config file cannot be written and synced. pub async fn write_current_config( - config: &ServerNgConfig, + config: &ServerConfig, current_replica_id: Option, bound_tcp: Option, bound_replica: Option, bound_tcp_tls: Option, bound_quic: Option, bound_websocket: Option, -) -> Result<(), ServerNgError> { +) -> Result<(), ServerError> { let mut current_config = config.clone(); if let Some(bound_client_tcp) = bound_tcp_tls.or(bound_tcp) { @@ -59,7 +59,7 @@ pub async fn write_current_config( .nodes .iter_mut() .find(|node| node.replica_id == replica_id) - .ok_or(ServerNgError::ClusterNodeNotFound { replica_id })?; + .ok_or(ServerError::ClusterNodeNotFound { replica_id })?; if let Some(bound_client_tcp) = bound_tcp_tls.or(bound_tcp) { node.ports.tcp = Some(bound_client_tcp.port()); } @@ -76,8 +76,7 @@ pub async fn write_current_config( let runtime_path = current_config.system.get_runtime_path(); let config_path = format!("{runtime_path}/current_config.toml"); - let content = - toml::to_string(¤t_config).map_err(ServerNgError::CurrentConfigSerialize)?; + let content = toml::to_string(¤t_config).map_err(ServerError::CurrentConfigSerialize)?; let mut file = OpenOptions::new() .write(true) @@ -85,7 +84,7 @@ pub async fn write_current_config( .truncate(true) .open(&config_path) .await - .map_err(|source| ServerNgError::CurrentConfigWrite { + .map_err(|source| ServerError::CurrentConfigWrite { path: config_path.clone(), source, })?; @@ -93,14 +92,14 @@ pub async fn write_current_config( file.write_all_at(content.into_bytes(), 0) .await .0 - .map_err(|source| ServerNgError::CurrentConfigWrite { + .map_err(|source| ServerError::CurrentConfigWrite { path: config_path.clone(), source, })?; file.sync_all() .await - .map_err(|source| ServerNgError::CurrentConfigWrite { + .map_err(|source| ServerError::CurrentConfigWrite { path: config_path, source, })?; diff --git a/core/server/src/configs.rs b/core/server/src/configs.rs deleted file mode 100644 index 8f5c7821e1..0000000000 --- a/core/server/src/configs.rs +++ /dev/null @@ -1,21 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub use configs::{ - COMPONENT, cache_indexes, cluster, defaults, displays, http, quic, server, sharding, system, - tcp, validators, websocket, -}; diff --git a/core/server-ng/src/consumer_group.rs b/core/server/src/consumer_group.rs similarity index 97% rename from core/server-ng/src/consumer_group.rs rename to core/server/src/consumer_group.rs index 0347136444..fd93e35261 100644 --- a/core/server-ng/src/consumer_group.rs +++ b/core/server/src/consumer_group.rs @@ -40,7 +40,7 @@ use iggy_binary_protocol::requests::consumer_offsets::{ DeleteConsumerOffset2Request, DeleteConsumerOffsetRequest, StoreConsumerOffset2Request, StoreConsumerOffsetRequest, }; -use iggy_binary_protocol::{KIND_CONSUMER_GROUP, Operation, RequestHeader, WireIdentifier}; +use iggy_binary_protocol::{KIND_CONSUMER_GROUP, Operation, RoutedRequestHeader, WireIdentifier}; use iggy_common::IggyError; use journal::superblock::SuperblockStore; use journal::{Journal, JournalHandle}; @@ -63,8 +63,8 @@ use std::rc::Rc; /// join. Every other operation passes through. pub(crate) async fn maybe_rewrite_consumer_group_request( shard: &Rc>, - request: Message, -) -> Result, IggyError> + request: Message, +) -> Result, IggyError> where B: ShellBus, MJ: JournalHandle + 'static, @@ -208,8 +208,8 @@ where #[allow(clippy::cast_possible_truncation)] pub(crate) fn maybe_rewrite_consumer_offset_request( shard: &Rc>, - request: Message, -) -> Result, IggyError> + request: Message, +) -> Result, IggyError> where B: ShellBus, MJ: JournalHandle + 'static, diff --git a/core/server/src/diagnostics.rs b/core/server/src/diagnostics.rs deleted file mode 100644 index d33d55a82e..0000000000 --- a/core/server/src/diagnostics.rs +++ /dev/null @@ -1,22 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub use server_common::diagnostics::ASYNCIFY_POOL_DISABLED_PANIC_MSG; -pub use server_common::diagnostics::print_incomplete_io_uring_ops_info; -pub use server_common::diagnostics::print_invalid_io_uring_args_info; -pub use server_common::diagnostics::print_io_uring_permission_info; -pub use server_common::diagnostics::print_locked_memory_limit_info; diff --git a/core/server-ng/src/dispatch.rs b/core/server/src/dispatch.rs similarity index 95% rename from core/server-ng/src/dispatch.rs rename to core/server/src/dispatch.rs index 9114d938de..aa198b8d4a 100644 --- a/core/server-ng/src/dispatch.rs +++ b/core/server/src/dispatch.rs @@ -49,9 +49,9 @@ use crate::responses::{ use crate::session_manager::SessionManager; use crate::snapshot; use crate::users::maybe_rewrite_user_password_request; -use crate::wire::{request_body, usize_to_u32}; +use crate::wire::{request_body, usize_to_u32, verify_request_checksum}; use bytes::Bytes; -use configs::server_ng::NgSystemConfig; +use configs::server::ServerSystemConfig; use consensus::{ Consensus, EvictionContext, MetadataHandle, PartitionsHandle, build_eviction_message, build_incompatible_protocol_eviction_message, build_result_rejection_reply, @@ -59,8 +59,9 @@ use consensus::{ use iggy_binary_protocol::PrepareHeader; use iggy_binary_protocol::codes::{ GET_CLIENT_CODE, GET_CLIENTS_CODE, GET_CLUSTER_METADATA_CODE, GET_CONSUMER_OFFSET_CODE, - GET_ME_CODE, GET_PERSONAL_ACCESS_TOKENS_CODE, GET_SNAPSHOT_FILE_CODE, LOGIN_USER_CODE, - LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE, PING_CODE, POLL_MESSAGES_CODE, SYNC_CONSUMER_GROUP_CODE, + GET_ME_CODE, GET_PERSONAL_ACCESS_TOKENS_CODE, GET_SNAPSHOT_FILE_CODE, GET_STATS_CODE, + LOGIN_USER_CODE, LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE, PING_CODE, POLL_MESSAGES_CODE, + SYNC_CONSUMER_GROUP_CODE, }; use iggy_binary_protocol::primitives::consumer::WireConsumer; use iggy_binary_protocol::primitives::polling_strategy::WirePollingStrategy; @@ -85,7 +86,7 @@ use iggy_binary_protocol::responses::system::get_snapshot::GetSnapshotResponse; use iggy_binary_protocol::{ AckLevel, ClientVersionInfo, Command2, EvictionReason, GenericHeader, HEADER_SIZE, KIND_CONSUMER_GROUP, MAX_PARTITIONS_PER_REQUEST, Operation, ProtocolVersion, RequestHeader, - WireDecode, WireEncode, WireIdentifier, is_protocol_compatible, + RoutedRequestHeader, WireDecode, WireEncode, WireIdentifier, is_protocol_compatible, }; use iggy_common::{ IggyError, MaxTopicSize, PollingStrategy, SnapshotCompression, SystemSnapshotType, @@ -123,7 +124,7 @@ pub(crate) type ActiveClientRequests = Rc>>; pub(crate) fn make_client_request_handler( shard: &Rc>, sessions: &Rc>, - system_config: Arc, + system_config: Arc, max_tokens_per_user: u32, ) -> RequestHandler where @@ -401,7 +402,7 @@ fn submit_auto_commit( fn build_auto_commit_request( namespace: IggyNamespace, applied: &AutoCommitApplied, -) -> Result, IggyError> { +) -> Result, IggyError> { let request = StoreConsumerOffset2Request { consumer: WireConsumer { kind: applied.kind.as_code(), @@ -414,26 +415,28 @@ fn build_auto_commit_request( ack: AckLevel::Quorum, }; let body = request.to_bytes(); - let header_size = std::mem::size_of::(); + let header_size = std::mem::size_of::(); let total_size = header_size + body.len(); let size = u32::try_from(total_size).map_err(|_| IggyError::InvalidConfiguration)?; - let mut message = Message::::new(total_size); + let mut message = Message::::new(total_size); message.as_mut_slice()[header_size..].copy_from_slice(&body); - Ok(message.transmute_header(|_, header: &mut RequestHeader| { - *header = RequestHeader { - command: Command2::Request, - operation: Operation::StoreConsumerOffset2, - size, - client: AUTO_COMMIT_CLIENT_ID, - // The partition plane is sessionless (no `ClientTable` dedup); a - // nonzero session + request just satisfy the wire header - // validation. - session: 1, - request: 1, - namespace: namespace.inner(), - ..Default::default() - }; - })) + Ok( + message.transmute_header(|_, header: &mut RoutedRequestHeader| { + *header = RoutedRequestHeader { + command: Command2::Request, + operation: Operation::StoreConsumerOffset2, + size, + client: AUTO_COMMIT_CLIENT_ID, + // The partition plane is sessionless (no `ClientTable` dedup); a + // nonzero session + request just satisfy the wire header + // validation. + session: 1, + request: 1, + group: namespace.inner(), + ..Default::default() + }; + }), + ) } pub(crate) fn make_deferred_replica_message_handler( @@ -458,7 +461,7 @@ pub(crate) fn make_deferred_client_request_handler( bus: &B, shard_handle: &ShellShardHandle, sessions: &Rc>, - system_config: Arc, + system_config: Arc, max_tokens_per_user: u32, ) -> RequestHandler where @@ -568,7 +571,7 @@ where let _ = reply.try_send(commit); } shard::MetadataSubmit::ClientRequest { request, reply } => { - let committed = match request.try_into_typed::() { + let committed = match request.try_into_typed::() { Ok(typed) => shard .plane .metadata() @@ -651,7 +654,7 @@ where fn enqueue_client_request( shard: Rc>, sessions: Rc>, - system_config: Arc, + system_config: Arc, max_tokens_per_user: u32, queues: ClientRequestQueues, active: ActiveClientRequests, @@ -692,7 +695,7 @@ fn enqueue_client_request( async fn drain_client_requests( shard: Rc>, sessions: Rc>, - system_config: Arc, + system_config: Arc, max_tokens_per_user: u32, queues: ClientRequestQueues, active: ActiveClientRequests, @@ -777,7 +780,7 @@ pub(crate) const fn validate_partitions_change_count( /// `ServerDefault` is exempt from the size floor (it resolves against server /// config at admission, matching legacy); `Unlimited` passes numerically. pub(crate) fn validate_topic_bounds( - system_config: &NgSystemConfig, + system_config: &ServerSystemConfig, partitions_count: u32, max_topic_size: MaxTopicSize, ) -> Result<(), IggyError> { @@ -800,7 +803,7 @@ pub(crate) fn validate_topic_bounds( #[allow(clippy::future_not_send)] async fn send_pre_consensus_deny( shard: &Rc>, - header: &RequestHeader, + header: &RoutedRequestHeader, transport_client_id: u128, error: &IggyError, context: &'static str, @@ -838,7 +841,7 @@ async fn send_pre_consensus_deny( async fn handle_client_request( shard: &Rc>, sessions: &Rc>, - system_config: &Arc, + system_config: &Arc, max_tokens_per_user: u32, transport_client_id: u128, message: Message, @@ -860,6 +863,42 @@ async fn handle_client_request( return; } }; + // Promote to the server-internal routed shape at the boundary: the + // client wire carries no group (it is derived -- plane from `operation`, + // partition target from the payload), so it starts unset here and the + // resolution sites below stamp it before anything routes on it. + let request = request.into_routed(); + + // The last point that still sees the body the CLIENT sent; every rewrite below + // substitutes server-chosen bytes and carries the stamp through unchanged. + if let Err(error) = verify_request_checksum(&request) { + warn!( + transport_client_id, + operation = ?request.header().operation, + request = request.header().request, + "dropping client request whose body does not match its own checksum" + ); + let commit = current_metadata_commit(shard); + let reply = build_deny_reply( + request.header(), + transport_client_id, + 0, + commit, + error.as_code(), + ); + if let Err(send_error) = shard + .bus + .send_to_client(transport_client_id, reply.into_generic().into_frozen()) + .await + { + warn!( + transport_client_id, + error = %send_error, + "failed to send request-checksum deny reply" + ); + } + return; + } ensure_transport_connection(shard, sessions, transport_client_id); @@ -881,7 +920,7 @@ async fn handle_client_request( // MUST go through Register first, which binds the acting user the // per-op authz gates resolve. let nr_code = u32::from_le_bytes(request.header().reserved[..4].try_into().unwrap()); - // Legacy (pre-register) login codes. server-ng authenticates only via + // Legacy (pre-register) login codes. The server authenticates only via // the Register handshake (LOGIN_REGISTER / LOGIN_REGISTER_WITH_PAT, // Operation::Register); the vsr SDK funnels both logins there and never // emits these. Reject them uniformly with a typed MalformedLogin (the @@ -896,7 +935,7 @@ async fn handle_client_request( warn!( transport_client_id, code = nr_code, - "rejecting legacy login code; server-ng requires the register handshake" + "rejecting legacy login code; server requires the register handshake" ); send_login_eviction( shard, @@ -982,8 +1021,10 @@ async fn handle_client_request( return; } - let request = request.transmute_header(|header, new_header: &mut RequestHeader| { + let request = request.transmute_header(|header, new_header: &mut RoutedRequestHeader| { *new_header = header; + // Metadata-plane ops route by operation: stamp the sentinel group. + new_header.group = server_common::sharding::METADATA_GROUP; // `bound` is always Some here (unbound transports early-return above); // this sets the consensus client id + session for the replicated op. if let Some((bound_client_id, bound_session)) = bound { @@ -1132,7 +1173,7 @@ async fn handle_get_personal_access_tokens( shard: &Rc>, sessions: &Rc>, transport_client_id: u128, - request: &Message, + request: &Message, ) where B: ShellBus, MJ: JournalHandle + 'static, @@ -1159,7 +1200,7 @@ async fn handle_get_me( shard: &Rc>, sessions: &Rc>, transport_client_id: u128, - request: &Message, + request: &Message, ) where B: ShellBus, MJ: JournalHandle + 'static, @@ -1198,7 +1239,7 @@ async fn handle_get_me( #[allow(clippy::future_not_send)] pub(crate) async fn dispatch_partition_request( shard: &Rc>, - request: Message, + request: Message, vsr_client_id: u128, bound_session: u64, transport_client_id: u128, @@ -1319,9 +1360,9 @@ pub(crate) async fn dispatch_partition_request( return; } }; - let request = request.transmute_header(|header, new_header: &mut RequestHeader| { + let request = request.transmute_header(|header, new_header: &mut RoutedRequestHeader| { *new_header = header; - new_header.namespace = namespace; + new_header.group = namespace; new_header.client = transport_client_id; // Header validation requires `session > 0 && request > 0` for // non-register ops. The partition plane itself is sessionless @@ -1338,9 +1379,9 @@ pub(crate) async fn dispatch_partition_request( async fn handle_non_replicated_request( shard: &Rc>, sessions: &Rc>, - system_config: &Arc, + system_config: &Arc, transport_client_id: u128, - request: Message, + request: Message, ) where B: ShellBus, MJ: JournalHandle + 'static, @@ -1495,7 +1536,7 @@ async fn handle_default_non_replicated( shard: &Rc>, transport_client_id: u128, code: u32, - request: &Message, + request: &Message, user_id: Option, roster: &ClusterRoster, client_ip: Option, @@ -1513,6 +1554,13 @@ async fn handle_default_non_replicated( send_non_replicated_deny(shard, request, transport_client_id, error.as_code()).await; return; } + // Stats is the one default read with an async input: the cross-shard + // connected-client gather. Run it here so the shared builder stays sync. + let clients_count = if code == GET_STATS_CODE { + u32::try_from(shard.list_all_clients().await.len()).unwrap_or(u32::MAX) + } else { + 0 + }; match build_non_replicated_response( shard, code, @@ -1520,6 +1568,7 @@ async fn handle_default_non_replicated( user_id, roster, client_ip, + clients_count, ) { Ok(response) => { let commit = current_metadata_commit(shard); @@ -1565,9 +1614,9 @@ async fn handle_default_non_replicated( #[allow(clippy::future_not_send)] async fn handle_get_snapshot( shard: &Rc>, - system_config: &Arc, + system_config: &Arc, transport_client_id: u128, - request: &Message, + request: &Message, user_id: Option, ) where B: ShellBus, @@ -1647,7 +1696,7 @@ fn decode_get_snapshot( #[allow(clippy::future_not_send)] async fn send_non_replicated_bytes( shard: &Rc>, - request: &Message, + request: &Message, transport_client_id: u128, bytes: Bytes, label: &'static str, @@ -1840,7 +1889,7 @@ async fn evict_stale_client( async fn handle_poll_messages( shard: &Rc>, transport_client_id: u128, - request: &Message, + request: &Message, user_id: Option, ) where B: ShellBus, @@ -1912,14 +1961,19 @@ async fn handle_poll_messages( } } Err(error) => { - // A partition id that does not exist in a resolvable topic is a + // A stream, topic, or partition id that does not resolve is a // client addressing error and must surface as a typed rejection, // not an empty poll a consumer would read as end-of-partition. - if matches!(error, IggyError::PartitionNotFound(..)) { + if matches!( + error, + IggyError::PartitionNotFound(..) + | IggyError::StreamIdNotFound(_) + | IggyError::TopicIdNotFound(..) + ) { warn!( transport_client_id, error = %error, - "poll_messages rejected: partition not found" + "poll_messages rejected: target not found" ); send_non_replicated_deny(shard, request, transport_client_id, error.as_code()) .await; @@ -1956,7 +2010,7 @@ async fn handle_poll_messages( async fn handle_get_consumer_offset( shard: &Rc>, transport_client_id: u128, - request: &Message, + request: &Message, user_id: Option, ) where B: ShellBus, @@ -2043,7 +2097,7 @@ async fn handle_get_consumer_offset( async fn handle_sync_consumer_group( shard: &Rc>, transport_client_id: u128, - request: &Message, + request: &Message, ) where B: ShellBus, MJ: JournalHandle + 'static, @@ -2097,7 +2151,7 @@ async fn handle_sync_consumer_group( async fn send_empty_partition_reply( shard: &Rc>, transport_client_id: u128, - request_header: &RequestHeader, + request_header: &RoutedRequestHeader, ) where B: ShellBus, MJ: JournalHandle + 'static, @@ -2424,7 +2478,7 @@ async fn handle_delete_segments_request( shard: &Rc>, transport_client_id: u128, bound: Option<(u128, u64)>, - request: &Message, + request: &Message, ) where B: ShellBus, MJ: JournalHandle + 'static, @@ -2554,11 +2608,11 @@ async fn handle_delete_segments_request( #[allow(clippy::cast_possible_truncation)] pub(crate) async fn resolve_delete_segments_truncate( shard: &Rc>, - template: &RequestHeader, + template: &RoutedRequestHeader, client_id: u128, session: u64, body: &[u8], -) -> Result, IggyError> +) -> Result, IggyError> where B: ShellBus, MJ: JournalHandle + 'static, @@ -2719,7 +2773,7 @@ fn submit_disconnect_logout( #[allow(clippy::future_not_send)] pub(crate) async fn submit_client_request_on_owner( shard: &Rc>, - request: Message, + request: Message, ) -> Option> where B: ShellBus, @@ -2749,7 +2803,7 @@ async fn handle_logout_request( shard: &Rc>, sessions: &Rc>, transport_client_id: u128, - request: Message, + request: Message, ) where B: ShellBus, MJ: JournalHandle + 'static, @@ -2853,7 +2907,7 @@ async fn handle_login_register_request( shard: &Rc>, sessions: &Rc>, transport_client_id: u128, - request: Message, + request: Message, ) where B: ShellBus, MJ: JournalHandle + 'static, @@ -2904,6 +2958,7 @@ async fn handle_login_register_request( } let body_tail = &body[prefix_len..]; + let mut credentials_rejected = false; if let Ok((wire_request, _)) = LoginRegisterRequest::decode_after_prefix(version_info.clone(), body_tail) { @@ -2933,8 +2988,10 @@ async fn handle_login_register_request( Err(LoginRegisterError::InvalidCredentials) => { // Fall through to PAT attempt so a credential payload that // collides with a valid PAT payload shape still gets a - // chance; if PAT also rejects, the final fall-through emits - // the empty-reply failure path below. + // chance. A password-shaped body rarely parses as a PAT + // body, so remember the rejection: the final fall-through + // must surface InvalidCredentials, not MalformedLogin. + credentials_rejected = true; } Err(error) => { warn!(transport_client_id, error = %error, "login/register failed"); @@ -2982,6 +3039,21 @@ async fn handle_login_register_request( } } + if credentials_rejected { + warn!( + transport_client_id, + "rejecting register request: invalid credentials" + ); + send_login_eviction( + shard, + transport_client_id, + request.header().client, + EvictionReason::InvalidCredentials, + ) + .await; + return; + } + warn!( transport_client_id, "rejecting register request with unsupported payload shape" @@ -3200,16 +3272,16 @@ mod tests { session: u64, request: u64, body: &[u8], - ) -> Message { - let header_size = size_of::(); + ) -> Message { + let header_size = size_of::(); let total = header_size + body.len(); - let mut message = Message::::new(total); + let mut message = Message::::new(total); { let slice = message.as_mut_slice(); slice[header_size..total].copy_from_slice(body); let header = - bytemuck::checked::from_bytes_mut::(&mut slice[..header_size]); - *header = RequestHeader { + bytemuck::checked::from_bytes_mut::(&mut slice[..header_size]); + *header = RoutedRequestHeader { command: Command2::Request, operation, size: u32::try_from(total).expect("test request fits u32"), @@ -3217,7 +3289,7 @@ mod tests { session, request, user_id: 0, - namespace: server_common::sharding::METADATA_CONSENSUS_NAMESPACE, + group: server_common::sharding::METADATA_GROUP, ..Default::default() }; } @@ -3250,12 +3322,13 @@ mod tests { client, request, user_id: 0, - checksum: 42, - namespace: server_common::sharding::METADATA_CONSENSUS_NAMESPACE, + group: server_common::sharding::METADATA_GROUP, ..Default::default() }; } - message + // A real identity, not a placeholder: `on_replicate` recomputes it before the + // prepare reaches the WAL, so an arbitrary value reads as transit corruption. + consensus::seal_prepare_checksum(message) } /// Regression test for the production failure chain "CLI stream @@ -3300,7 +3373,7 @@ mod tests { 1, 0, 1, - server_common::sharding::METADATA_CONSENSUS_NAMESPACE, + server_common::sharding::METADATA_GROUP, bus.clone(), LocalPipeline::new(), ); @@ -3319,7 +3392,9 @@ mod tests { messages_required_to_save: 1, size_of_messages_required_to_save: iggy_common::IggyByteSize::from(1024_u64), enforce_fsync: false, + validate_checksum: true, segment_size: iggy_common::IggyByteSize::from(1_048_576_u64), + preallocate_segments: false, encryptor: None, }, ); @@ -3437,7 +3512,9 @@ mod tests { messages_required_to_save: 1, size_of_messages_required_to_save: iggy_common::IggyByteSize::from(1024_u64), enforce_fsync: false, + validate_checksum: true, segment_size: iggy_common::IggyByteSize::from(1_048_576_u64), + preallocate_segments: false, encryptor: None, }, ); @@ -3560,7 +3637,9 @@ mod tests { messages_required_to_save: 1, size_of_messages_required_to_save: iggy_common::IggyByteSize::from(1024_u64), enforce_fsync: false, + validate_checksum: true, segment_size: iggy_common::IggyByteSize::from(1_048_576_u64), + preallocate_segments: false, encryptor: None, }, ); @@ -3577,9 +3656,9 @@ mod tests { shard.plane.partitions().tombstone(namespace); let request = request_message(Operation::SendMessages, TRANSPORT, SESSION, 1, &[]) - .transmute_header(|header, new_header: &mut RequestHeader| { + .transmute_header(|header, new_header: &mut RoutedRequestHeader| { *new_header = header; - new_header.namespace = namespace.inner(); + new_header.group = namespace.inner(); }); shard.on_message(request.into_generic()).await; @@ -3622,7 +3701,9 @@ mod tests { messages_required_to_save: 1, size_of_messages_required_to_save: iggy_common::IggyByteSize::from(1024_u64), enforce_fsync: false, + validate_checksum: true, segment_size: iggy_common::IggyByteSize::from(1_048_576_u64), + preallocate_segments: false, encryptor: None, }, ); @@ -3651,9 +3732,9 @@ mod tests { let namespace = IggyNamespace::new(0, 0, 0); let request = request_message(Operation::SendMessages, TRANSPORT, SESSION, 1, &[]) - .transmute_header(|header, new_header: &mut RequestHeader| { + .transmute_header(|header, new_header: &mut RoutedRequestHeader| { *new_header = header; - new_header.namespace = namespace.inner(); + new_header.group = namespace.inner(); }); // Namespace neither materialised nor tombstoned: the frame parks. shard.on_message(request.into_generic()).await; @@ -3689,7 +3770,7 @@ mod tests { #[test] fn create_topic_bounds_deny_pre_consensus() { - let config = NgSystemConfig::default(); + let config = ServerSystemConfig::default(); let segment_size = config.segment.size.as_bytes_u64(); assert!(segment_size > 0, "default segment size must be nonzero"); diff --git a/core/server-ng/src/dispatch/authz.rs b/core/server/src/dispatch/authz.rs similarity index 98% rename from core/server-ng/src/dispatch/authz.rs rename to core/server/src/dispatch/authz.rs index 08ed5cac12..afd8cc011c 100644 --- a/core/server-ng/src/dispatch/authz.rs +++ b/core/server/src/dispatch/authz.rs @@ -38,7 +38,9 @@ use iggy_binary_protocol::requests::consumer_groups::{ }; use iggy_binary_protocol::requests::streams::GetStreamRequest; use iggy_binary_protocol::requests::topics::{GetTopicRequest, GetTopicsRequest}; -use iggy_binary_protocol::{Operation, PrepareHeader, RequestHeader, WireDecode, WireIdentifier}; +use iggy_binary_protocol::{ + Operation, PrepareHeader, RoutedRequestHeader, WireDecode, WireIdentifier, +}; use iggy_common::IggyError; use journal::superblock::SuperblockStore; use journal::{Journal, JournalHandle}; @@ -139,7 +141,7 @@ where pub(super) async fn send_partition_deny_reply( shard: &Rc>, transport_client_id: u128, - request_header: &RequestHeader, + request_header: &RoutedRequestHeader, status: u32, ) where B: ShellBus, @@ -299,7 +301,7 @@ where #[allow(clippy::future_not_send)] pub(super) async fn send_non_replicated_deny( shard: &Rc>, - request: &Message, + request: &Message, transport_client_id: u128, status: u32, ) where diff --git a/core/server-ng/src/http.rs b/core/server/src/http.rs similarity index 98% rename from core/server-ng/src/http.rs rename to core/server/src/http.rs index 82b3fd997b..d3e10ca58a 100644 --- a/core/server-ng/src/http.rs +++ b/core/server/src/http.rs @@ -55,16 +55,16 @@ use axum::middleware::{Next, from_fn, from_fn_with_state}; use axum::response::Response; use axum::routing::{delete, get, post, put}; use compio::net::TcpListener; +use configs::cluster::{ClusterConfig, TransportPorts, http_forwarding_key_material}; use configs::http::{HttpConfig, HttpCorsConfig}; -use configs::ng_cluster::{ClusterConfig, TransportPorts, http_forwarding_key_material}; -use configs::server_ng::NgSystemConfig; +use configs::server::ServerSystemConfig; use iggy_common::IggyError; use message_bus::client_listener; use send_wrapper::SendWrapper; use tower_http::cors::{AllowOrigin, CorsLayer}; use tracing::{error, info, warn}; -use crate::bootstrap::ServerNgShard; +use crate::bootstrap::ServerShard; use crate::cluster_meta::ClusterRoster; use crate::http::handlers::{ change_password, create_cg, create_partitions, create_pat, create_stream, create_topic, @@ -79,7 +79,7 @@ use crate::http::handlers::{ use crate::http::jwt::JwtManager; use crate::http::session::RegistrationBarrier; use crate::http::state::{HttpInner, HttpState, insert_view_header}; -use crate::server_error::ServerNgError; +use crate::server_error::ServerError; /// Bind the shard-0 HTTP listener and spawn the `cyper-axum` serve loop as a /// background task on shard 0's compio runtime. Serves HTTPS when @@ -91,20 +91,20 @@ use crate::server_error::ServerNgError; /// /// # Errors /// -/// Returns [`ServerNgError`] if the JWT manager cannot be built from +/// Returns [`ServerError`] if the JWT manager cannot be built from /// `http_config.jwt`, the `[http.cors]` config is invalid, the `[http.tls]` /// credentials cannot be loaded, or the listener cannot bind to `addr`. #[allow(clippy::too_many_arguments)] pub async fn start( - shard: &Rc, + shard: &Rc, addr: SocketAddr, http_config: &HttpConfig, clients_table_max: usize, max_tokens_per_user: u32, cluster: &ClusterConfig, - system_config: Arc, + system_config: Arc, self_ports: TransportPorts, -) -> Result<(), ServerNgError> { +) -> Result<(), ServerError> { // In cluster mode with no configured JWT secret the signing key derives // from the cluster PSK, so a bearer minted on any node verifies on every // node - the invariant follower-to-primary forwarding depends on. @@ -184,11 +184,11 @@ pub async fn start( shard.bus.token(), ); shard.bus.track_background(pump); - info!(address = %bound_addr, "server-ng HTTPS listener started"); + info!(address = %bound_addr, "server HTTPS listener started"); let handle = compio::runtime::spawn(tls::serve(connections, router, shard.bus.token())); shard.bus.track_background(handle); } else { - info!(address = %bound_addr, "server-ng HTTP listener started"); + info!(address = %bound_addr, "server HTTP listener started"); let shutdown = shard.bus.token(); let handle = compio::runtime::spawn(async move { if let Err(error) = cyper_axum::serve( @@ -198,7 +198,7 @@ pub async fn start( .with_graceful_shutdown(async move { shutdown.wait().await }) .await { - error!(%error, "server-ng HTTP listener terminated with error"); + error!(%error, "server HTTP listener terminated with error"); } }); shard.bus.track_background(handle); diff --git a/core/server-ng/src/http/admission.rs b/core/server/src/http/admission.rs similarity index 100% rename from core/server-ng/src/http/admission.rs rename to core/server/src/http/admission.rs diff --git a/core/server/src/http/consumer_groups.rs b/core/server/src/http/consumer_groups.rs deleted file mode 100644 index 9ef910a81d..0000000000 --- a/core/server/src/http/consumer_groups.rs +++ /dev/null @@ -1,176 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::http::error::CustomError; -use crate::http::jwt::json_web_token::Identity; -use crate::http::mapper; -use crate::http::shared::AppState; -use crate::shard::transmission::frame::ShardResponse; -use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; -use axum::debug_handler; -use axum::extract::{Path, State}; -use axum::http::StatusCode; -use axum::routing::get; -use axum::{Extension, Json, Router}; -use iggy_binary_protocol::WireName; -use iggy_binary_protocol::requests::consumer_groups::{ - CreateConsumerGroupRequest as WireCreateConsumerGroup, - DeleteConsumerGroupRequest as WireDeleteConsumerGroup, -}; -use iggy_common::Identifier; -use iggy_common::Validatable; -use iggy_common::create_consumer_group::CreateConsumerGroup; -use iggy_common::wire_conversions::identifier_to_wire; -use iggy_common::{ConsumerGroup, ConsumerGroupDetails, IggyError}; -use std::sync::Arc; -use tracing::instrument; - -pub fn router(state: Arc) -> Router { - Router::new() - .route( - "/streams/{stream_id}/topics/{topic_id}/consumer-groups", - get(get_consumer_groups).post(create_consumer_group), - ) - .route( - "/streams/{stream_id}/topics/{topic_id}/consumer-groups/{group_id}", - get(get_consumer_group).delete(delete_consumer_group), - ) - .with_state(state) -} - -async fn get_consumer_group( - State(state): State>, - Extension(identity): Extension, - Path((stream_id, topic_id, group_id)): Path<(String, String, String)>, -) -> Result, CustomError> { - let identifier_stream_id = Identifier::from_str_value(&stream_id)?; - let identifier_topic_id = Identifier::from_str_value(&topic_id)?; - let identifier_group_id = Identifier::from_str_value(&group_id)?; - - let shard = state.shard.shard(); - let group = shard.resolve_consumer_group( - &identifier_stream_id, - &identifier_topic_id, - &identifier_group_id, - )?; - - shard - .metadata - .perm_get_consumer_group(identity.user_id, group.stream_id, group.topic_id)?; - - let cg_meta = shard - .metadata - .get_consumer_group(group.stream_id, group.topic_id, group.group_id) - .ok_or(CustomError::ResourceNotFound)?; - - let consumer_group = mapper::map_consumer_group_details_from_metadata(&cg_meta); - - Ok(Json(consumer_group)) -} - -async fn get_consumer_groups( - State(state): State>, - Extension(identity): Extension, - Path((stream_id, topic_id)): Path<(String, String)>, -) -> Result>, CustomError> { - let identifier_stream_id = Identifier::from_str_value(&stream_id)?; - let identifier_topic_id = Identifier::from_str_value(&topic_id)?; - - let shard = state.shard.shard(); - let topic = shard.resolve_topic(&identifier_stream_id, &identifier_topic_id)?; - - shard - .metadata - .perm_get_consumer_groups(identity.user_id, topic.stream_id, topic.topic_id)?; - - let topic_meta = shard - .metadata - .get_topic(topic.stream_id, topic.topic_id) - .ok_or(CustomError::ResourceNotFound)?; - let consumer_groups = mapper::map_consumer_groups_from_metadata(&topic_meta); - - Ok(Json(consumer_groups)) -} - -#[debug_handler] -#[instrument(skip_all, name = "trace_create_consumer_group", fields(iggy_user_id = identity.user_id, iggy_stream_id = stream_id, iggy_topic_id = topic_id))] -async fn create_consumer_group( - State(state): State>, - Extension(identity): Extension, - Path((stream_id, topic_id)): Path<(String, String)>, - Json(mut command): Json, -) -> Result<(StatusCode, Json), CustomError> { - command.stream_id = Identifier::from_str_value(&stream_id)?; - command.topic_id = Identifier::from_str_value(&topic_id)?; - command.validate()?; - - let shard = state.shard.shard(); - let topic = shard.resolve_topic(&command.stream_id, &command.topic_id)?; - - let wire_command = WireCreateConsumerGroup { - stream_id: identifier_to_wire(&command.stream_id)?, - topic_id: identifier_to_wire(&command.topic_id)?, - name: WireName::new(&command.name).map_err(|_| IggyError::InvalidConsumerGroupName)?, - }; - let request = ShardRequest::control_plane(ShardRequestPayload::CreateConsumerGroupRequest { - user_id: identity.user_id, - command: wire_command, - }); - - match state.shard.send_to_control_plane(request).await? { - ShardResponse::CreateConsumerGroupResponse(data) => { - let cg_meta = state - .shard - .shard() - .metadata - .get_consumer_group(topic.stream_id, topic.topic_id, data.id as usize) - .expect("Consumer group must exist after creation"); - let consumer_group_details = mapper::map_consumer_group_details_from_metadata(&cg_meta); - Ok((StatusCode::CREATED, Json(consumer_group_details))) - } - ShardResponse::ErrorResponse(err) => Err(err.into()), - _ => unreachable!("Expected CreateConsumerGroupResponse"), - } -} - -#[debug_handler] -#[instrument(skip_all, name = "trace_delete_consumer_group", fields(iggy_user_id = identity.user_id, iggy_stream_id = stream_id, iggy_topic_id = topic_id, iggy_group_id = group_id))] -async fn delete_consumer_group( - State(state): State>, - Extension(identity): Extension, - Path((stream_id, topic_id, group_id)): Path<(String, String, String)>, -) -> Result { - let stream_id = Identifier::from_str_value(&stream_id)?; - let topic_id = Identifier::from_str_value(&topic_id)?; - let group_id = Identifier::from_str_value(&group_id)?; - - let wire_command = WireDeleteConsumerGroup { - stream_id: identifier_to_wire(&stream_id)?, - topic_id: identifier_to_wire(&topic_id)?, - group_id: identifier_to_wire(&group_id)?, - }; - let request = ShardRequest::control_plane(ShardRequestPayload::DeleteConsumerGroupRequest { - user_id: identity.user_id, - command: wire_command, - }); - - match state.shard.send_to_control_plane(request).await? { - ShardResponse::DeleteConsumerGroupResponse => Ok(StatusCode::NO_CONTENT), - ShardResponse::ErrorResponse(err) => Err(err.into()), - _ => unreachable!("Expected DeleteConsumerGroupResponse"), - } -} diff --git a/core/server/src/http/consumer_offsets.rs b/core/server/src/http/consumer_offsets.rs deleted file mode 100644 index 1d856a7311..0000000000 --- a/core/server/src/http/consumer_offsets.rs +++ /dev/null @@ -1,150 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::http::COMPONENT; -use crate::http::error::CustomError; -use crate::http::jwt::json_web_token::Identity; -use crate::http::shared::AppState; -use axum::debug_handler; -use axum::extract::{Path, Query, State}; -use axum::http::StatusCode; -use axum::routing::{delete, get}; -use axum::{Extension, Json, Router}; -use err_trail::ErrContext; -use iggy_common::Consumer; -use iggy_common::ConsumerOffsetInfo; -use iggy_common::Identifier; -use iggy_common::IggyError; -use iggy_common::delete_consumer_offset::DeleteConsumerOffset; -use iggy_common::get_consumer_offset::GetConsumerOffset; -use iggy_common::store_consumer_offset::StoreConsumerOffset; -use std::sync::Arc; - -pub fn router(state: Arc) -> Router { - Router::new() - .route( - "/streams/{stream_id}/topics/{topic_id}/consumer-offsets", - get(get_consumer_offset).put(store_consumer_offset), - ) - .route( - "/streams/{stream_id}/topics/{topic_id}/consumer-offsets/{consumer_id}", - delete(delete_consumer_offset), - ) - .with_state(state) -} - -#[debug_handler] -async fn get_consumer_offset( - State(state): State>, - Extension(identity): Extension, - Path((stream_id, topic_id)): Path<(String, String)>, - query: Query, -) -> Result, CustomError> { - let stream_id = Identifier::from_str_value(&stream_id)?; - let topic_id = Identifier::from_str_value(&topic_id)?; - - let shard = state.shard.shard(); - let topic = shard.resolve_topic(&stream_id, &topic_id)?; - shard - .metadata - .perm_get_consumer_offset(identity.user_id, topic.stream_id, topic.topic_id)?; - - let consumer = Consumer::new(query.consumer.id.clone()); - let Ok(offset) = state - .shard - .get_consumer_offset( - 0, // HTTP uses client_id 0 as it doesn't have persistent sessions - consumer, - &stream_id, - &topic_id, - query.partition_id, - ) - .await - else { - return Err(CustomError::ResourceNotFound); - }; - - let Some(offset) = offset else { - return Err(CustomError::ResourceNotFound); - }; - - Ok(Json(offset)) -} - -#[debug_handler] -async fn store_consumer_offset( - State(state): State>, - Extension(identity): Extension, - Path((stream_id, topic_id)): Path<(String, String)>, - Json(body): Json, -) -> Result { - let stream_id = Identifier::from_str_value(&stream_id)?; - let topic_id = Identifier::from_str_value(&topic_id)?; - - let shard = state.shard.shard(); - let topic = shard.resolve_topic(&stream_id, &topic_id)?; - shard - .metadata - .perm_store_consumer_offset(identity.user_id, topic.stream_id, topic.topic_id)?; - - let consumer = Consumer::new(body.consumer.id); - state.shard - .store_consumer_offset( - 0, // HTTP uses client_id 0 as it doesn't have persistent sessions - consumer, - &stream_id, - &topic_id, - body.partition_id, - body.offset, - ) - .await - .error(|e: &IggyError| format!("{COMPONENT} (error: {e}) - failed to store consumer offset, stream ID: {stream_id}, topic ID: {topic_id}, partition ID: {:?}", body.partition_id))?; - Ok(StatusCode::NO_CONTENT) -} - -#[debug_handler] -async fn delete_consumer_offset( - State(state): State>, - Extension(identity): Extension, - Path((stream_id, topic_id, consumer_id)): Path<(String, String, String)>, - query: Query, -) -> Result { - let stream_id_ident = Identifier::from_str_value(&stream_id)?; - let topic_id_ident = Identifier::from_str_value(&topic_id)?; - - let shard = state.shard.shard(); - let topic = shard.resolve_topic(&stream_id_ident, &topic_id_ident)?; - shard.metadata.perm_delete_consumer_offset( - identity.user_id, - topic.stream_id, - topic.topic_id, - )?; - - let consumer = Consumer::new(consumer_id.try_into()?); - state - .shard - .delete_consumer_offset( - 0, // HTTP uses client_id 0 as it doesn't have persistent sessions - consumer, - &stream_id_ident, - &topic_id_ident, - query.partition_id, - ) - .await - .error(|e: &IggyError| format!("{COMPONENT} (error: {e}) - failed to delete consumer offset, stream ID: {}, topic ID: {}, partition ID: {:?}", stream_id, topic_id, query.partition_id))?; - Ok(StatusCode::NO_CONTENT) -} diff --git a/core/server/src/http/diagnostics.rs b/core/server/src/http/diagnostics.rs deleted file mode 100644 index 56566cc99a..0000000000 --- a/core/server/src/http/diagnostics.rs +++ /dev/null @@ -1,68 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::http::http_server::CompioSocketAddr; -use crate::http::shared::RequestDetails; -use crate::streaming::utils::random_id; -use axum::body::Body; -use axum::{ - extract::ConnectInfo, - http::{Request, StatusCode}, - middleware::Next, - response::Response, -}; -use std::time::Instant; -use tracing::{debug, error}; - -pub async fn request_diagnostics( - ConnectInfo(ip_address): ConnectInfo, - mut request: Request, - next: Next, -) -> Result { - let request_id = random_id::get_ulid(); - let path_and_query = request - .uri() - .path_and_query() - .map(|p| p.as_str()) - .unwrap_or("/"); - let ip_address = ip_address.0; - debug!( - "Processing a request {} {} with ID: {request_id} from client with IP address: {ip_address}...", - request.method(), - path_and_query, - ); - request.extensions_mut().insert(RequestDetails { - request_id, - ip_address, - }); - let now = Instant::now(); - let result = Ok(next.run(request).await); - if let Ok(response) = &result { - let status = response.status(); - if status != StatusCode::NOT_FOUND && status >= StatusCode::BAD_REQUEST { - error!( - "Returning an invalid status code: {status}, IP address: {ip_address}, request ID: {request_id}" - ); - } - } - let elapsed = now.elapsed(); - debug!( - "Processed a request with ID: {request_id} from client with IP address: {ip_address} in {} ms.", - elapsed.as_millis() - ); - result -} diff --git a/core/server/src/http/error.rs b/core/server/src/http/error.rs index 1c422c0a0d..45ef05315c 100644 --- a/core/server/src/http/error.rs +++ b/core/server/src/http/error.rs @@ -15,16 +15,27 @@ // specific language governing permissions and limitations // under the License. +//! HTTP rejection types and hand-built error responses: the auth / write / +//! read / partition-write error enums, their `IntoResponse` renderings, the +//! `?consistency=` and `?ack=` query DTOs, and the primary-redirect helpers. + +use std::net::{IpAddr, SocketAddr}; + use axum::Json; -use axum::http::StatusCode; +use axum::http::header::{LOCATION, RETRY_AFTER}; +use axum::http::{HeaderValue, StatusCode}; use axum::response::{IntoResponse, Response}; +use configs::cluster::ResolvedClusterNode; +use iggy_binary_protocol::Operation; use iggy_common::IggyError; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use thiserror::Error; use tracing::error; +use crate::cluster_meta::ClusterRoster; + #[derive(Debug, Error)] -pub enum CustomError { +pub(in crate::http) enum CustomError { #[error(transparent)] Error(#[from] IggyError), #[error("Resource not found")] @@ -32,7 +43,11 @@ pub enum CustomError { } #[derive(Debug, Serialize)] -pub struct ErrorResponse { +pub(in crate::http) struct ErrorResponse { + /// Two conventions by construction: the `IggyError` numeric code + /// (`IggyError::as_code`) when the error wraps one (via [`Self::from_error`]), + /// or the HTTP status code for the hand-built HTTP-layer errors (429/503/504 + /// and the 404 not-found fallback) that carry no underlying `IggyError`. pub id: u32, pub code: String, pub reason: String, @@ -42,29 +57,49 @@ pub struct ErrorResponse { impl IntoResponse for CustomError { fn into_response(self) -> Response { match self { - CustomError::Error(error) => { + Self::Error(error) => { error!("There was an error: {error}"); let status_code = match error { - IggyError::StreamIdNotFound(_) => StatusCode::NOT_FOUND, - IggyError::TopicIdNotFound(_, _) => StatusCode::NOT_FOUND, - IggyError::PartitionNotFound(_, _, _) => StatusCode::NOT_FOUND, - IggyError::SegmentNotFound => StatusCode::NOT_FOUND, - IggyError::ClientNotFound(_) => StatusCode::NOT_FOUND, - IggyError::ConsumerGroupIdNotFound(_, _) => StatusCode::NOT_FOUND, - IggyError::ConsumerGroupNameNotFound(_, _) => StatusCode::NOT_FOUND, - IggyError::ConsumerGroupMemberNotFound(_, _, _) => StatusCode::NOT_FOUND, - IggyError::ConsumerOffsetNotFound(_) => StatusCode::NOT_FOUND, - IggyError::ResourceNotFound(_) => StatusCode::NOT_FOUND, - IggyError::Unauthenticated => StatusCode::UNAUTHORIZED, - IggyError::AccessTokenMissing => StatusCode::UNAUTHORIZED, - IggyError::InvalidAccessToken => StatusCode::UNAUTHORIZED, - IggyError::InvalidPersonalAccessToken => StatusCode::UNAUTHORIZED, + IggyError::StreamIdNotFound(_) + | IggyError::TopicIdNotFound(_, _) + | IggyError::PartitionNotFound(_, _, _) + | IggyError::SegmentNotFound + | IggyError::ClientNotFound(_) + | IggyError::ConsumerGroupIdNotFound(_, _) + | IggyError::ConsumerGroupNameNotFound(_, _) + | IggyError::ConsumerGroupMemberNotFound(_, _, _) + | IggyError::ConsumerOffsetNotFound(_) + | IggyError::ResourceNotFound(_) => StatusCode::NOT_FOUND, + IggyError::Unauthenticated + | IggyError::AccessTokenMissing + | IggyError::InvalidAccessToken + | IggyError::InvalidPersonalAccessToken => StatusCode::UNAUTHORIZED, IggyError::Unauthorized => StatusCode::FORBIDDEN, + // The pre-consensus retry frame: reaching this render + // means the write path's replay budget is exhausted and + // the op never committed - a transient server condition, + // retryable like the other cannot-commit-right-now 503s + // (see `service_unavailable`), never a caller error. + IggyError::TransientNotCommitted | IggyError::TransientNotAccepted => { + StatusCode::SERVICE_UNAVAILABLE + } _ => StatusCode::BAD_REQUEST, }; - (status_code, Json(ErrorResponse::from_error(error))) + let response = + (status_code, Json(ErrorResponse::from_error(&error))).into_response(); + // Transient 503s are retryable, so the advisory Retry-After hint + // rides along, matching the other transient 503 bodies + // (`service_unavailable`, `server_busy`). + if matches!( + error, + IggyError::TransientNotCommitted | IggyError::TransientNotAccepted + ) { + with_retry_after(response) + } else { + response + } } - CustomError::ResourceNotFound => ( + Self::ResourceNotFound => ( StatusCode::NOT_FOUND, Json(ErrorResponse { id: 404, @@ -72,37 +107,712 @@ impl IntoResponse for CustomError { reason: "Resource not found".to_string(), field: None, }), - ), + ) + .into_response(), } - .into_response() } } impl ErrorResponse { - pub fn from_error(error: IggyError) -> Self { - ErrorResponse { + pub fn from_error(error: &IggyError) -> Self { + Self { id: error.as_code(), code: error.as_string().to_string(), reason: error.to_string(), field: match error { - IggyError::StreamIdNotFound(_) => Some("stream_id".to_string()), - IggyError::TopicIdNotFound(_, _) => Some("topic_id".to_string()), + IggyError::StreamIdNotFound(_) | IggyError::InvalidStreamId => { + Some("stream_id".to_string()) + } + IggyError::TopicIdNotFound(_, _) | IggyError::InvalidTopicId => { + Some("topic_id".to_string()) + } IggyError::PartitionNotFound(_, _, _) => Some("partition_id".to_string()), IggyError::SegmentNotFound => Some("segment_id".to_string()), IggyError::ClientNotFound(_) => Some("client_id".to_string()), - IggyError::InvalidStreamName => Some("name".to_string()), - IggyError::StreamNameAlreadyExists(_) => Some("name".to_string()), - IggyError::InvalidTopicName => Some("name".to_string()), - IggyError::TopicNameAlreadyExists(_, _) => Some("name".to_string()), - IggyError::InvalidStreamId => Some("stream_id".to_string()), - IggyError::InvalidTopicId => Some("topic_id".to_string()), + IggyError::InvalidStreamName + | IggyError::StreamNameAlreadyExists(_) + | IggyError::InvalidTopicName + | IggyError::TopicNameAlreadyExists(_, _) + | IggyError::ConsumerGroupNameAlreadyExists(_, _) + | IggyError::PersonalAccessTokenAlreadyExists(_, _) => Some("name".to_string()), IggyError::InvalidOffset(_) => Some("offset".to_string()), IggyError::InvalidConsumerGroupId => Some("consumer_group_id".to_string()), - IggyError::ConsumerGroupNameAlreadyExists(_, _) => Some("name".to_string()), IggyError::UserAlreadyExists => Some("username".to_string()), - IggyError::PersonalAccessTokenAlreadyExists(_, _) => Some("name".to_string()), _ => None, }, } } } + +/// Rejection for protected routes. +/// +/// Two failure classes get two statuses: a missing, invalid, or expired +/// credential is the caller's fault (401, rendered as the JSON `ErrorResponse` +/// body every other route error uses), while a VSR session that cannot be +/// established right now is a transient server condition (503) and must never +/// masquerade as an auth failure. +pub(in crate::http) enum AuthError { + Unauthenticated(IggyError), + /// The Register provably never entered the consensus pipeline (not + /// primary, not caught up, or the prepare queue was full), so the request + /// is safe to re-issue anywhere. Rendered with the `TransientNotAccepted` + /// body so a forwarding follower recognizes it as retryable against a + /// re-resolved primary; a plain client sees the same retryable 503 either + /// way. + SessionNotAccepted, + SessionUnavailable, + /// The `client_id` this gateway minted already has a committed session + /// owned by a DIFFERENT user, so the Register was refused terminally. + /// + /// Distinct from [`Self::SessionUnavailable`] because the status code is + /// the whole point: 503 is about the most auto-retried status there is and + /// no foreign SDK special-cases it, so rendering a permanent, deterministic + /// refusal as 503 hands the caller's HTTP stack a retry loop it can never + /// escape. 409 says the id is taken and stops it. + SessionIdOwnedByAnotherUser, + /// The minted `client_id` already had a committed session for this SAME + /// user, so the Register rebound onto it instead of creating one. Internal + /// to the mint retry in `register_session` and never rendered: the caller + /// mints a different id. Present as a variant so the retry cannot confuse + /// it with a terminal cross-user refusal. + SessionIdTaken, +} + +impl From for AuthError { + fn from(error: IggyError) -> Self { + Self::Unauthenticated(error) + } +} + +impl IntoResponse for AuthError { + fn into_response(self) -> Response { + match self { + // Render 401 through the shared `IggyError -> CustomError` map so it + // carries the same JSON `ErrorResponse` body as every other ng error. + // The legacy server's protected-route 401 comes from a bare-status + // JWT middleware (empty body), so this is deliberately richer, not + // byte-identical to legacy. + Self::Unauthenticated(error) => CustomError::from(error).into_response(), + Self::SessionNotAccepted => { + CustomError::from(IggyError::TransientNotAccepted).into_response() + } + // A fresh session could not be established: the Register was + // canceled with its commit outcome unknown, or the session table + // is at its cap (half `[metadata] clients_table_max`) and refused + // the fresh registration. Transient server condition -> 503, + // retryable by the CLIENT only (a forwarder must not re-issue an + // unknown-outcome Register under this node's session budget on + // the caller's behalf). + // `SessionIdTaken` only escapes the mint retry when every attempt + // collided, which means the minter is wrong rather than unlucky -- + // same unknown-outcome answer as a canceled Register. + Self::SessionUnavailable | Self::SessionIdTaken => service_unavailable(), + // Terminal: retrying cannot change the answer, and admitting it + // would run this caller's replicated ops under the entry owner's + // authority. + Self::SessionIdOwnedByAnotherUser => ( + StatusCode::CONFLICT, + Json(ErrorResponse::from_error(&IggyError::InvalidClientId)), + ) + .into_response(), + } + } +} + +/// Rejection for an authenticated control-plane write (`POST /streams` and the +/// writes that follow it). +/// +/// Same two-class split as [`AuthError`], for the same reasons: a caller-side +/// validation failure or a committed business rejection (e.g. a duplicate +/// stream name) renders through the legacy `IggyError -> CustomError` map so +/// SDK error bodies stay byte-identical, while a write that cannot commit right +/// now is a transient server condition (503) and must never surface as a +/// business error or, worse, a 200 with a stale body. +pub(in crate::http) enum WriteError { + Rejected(IggyError), + /// The VSR session was evicted (its client slot was reclaimed cluster-side, + /// e.g. LRU-evicted from the full client table). Renders identically to a + /// terminal `Rejected` (401 -> re-authenticate), but is a distinct variant + /// so the submit path can drop the dead session entry and let the caller's + /// next request re-register cleanly instead of 401-looping on it. + Evicted(IggyError), + Unavailable, +} + +impl IntoResponse for WriteError { + fn into_response(self) -> Response { + match self { + Self::Rejected(error) | Self::Evicted(error) => { + CustomError::from(error).into_response() + } + Self::Unavailable => service_unavailable(), + } + } +} + +/// Rejection for a data-plane partition write (`POST .../messages` produce and +/// the `PUT`/`DELETE .../consumer-offsets` writes). +/// +/// Split differently from [`WriteError`] because the partition plane replies +/// carry no committed error code: a pre-dispatch gate failure is an +/// empty-bodied reply that names itself only in the header (see +/// [`classify_partition_reply`]), and an unanswered write is a distinct +/// outcome the caller must treat as unknown rather than failed. +#[derive(Debug)] +pub(in crate::http) enum PartitionWriteError { + /// Caller-side rejection (bad identifier, oversized batch, an authorization + /// denial), a typed pre-commit deny from the partition plane + /// (`ReplyHeader.status`), or a malformed reply frame, rendered through the + /// legacy `IggyError -> status` map for SDK-identical bodies. + Rejected(IggyError), + /// Backstop for a status-0 reply carrying `op` 0: an ack with no commit + /// number behind it, for a write that never reached the partition plane. + /// Routing failures name themselves through `ReplyHeader.status`, so this + /// shape is left to a peer that still answers a non-committing op this + /// way. Rendered as the legacy 404 body: the alternative is grading a + /// write that never happened as a success. + NotFound, + /// The in-process reply slot could not be installed. Transient server + /// condition -> the shared 503, retryable. + Unavailable, + /// This session is already at [`MAX_IN_FLIGHT_WRITES_PER_SESSION`] + /// awaited writes. 429: the caller's own concurrency is the problem, so + /// it must drain its outstanding writes before submitting more. + TooManyInFlight, + /// Shard 0 is already at [`MAX_IN_FLIGHT_WRITES_GLOBAL`] awaited writes + /// across all sessions. 503 with its own code (distinct from the shared + /// consensus-unavailable body) so an operator can tell admission shedding + /// from a consensus outage. + ServerBusy, + /// No committed reply within [`PARTITION_WRITE_REPLY_TIMEOUT`], or the + /// session's reply target was torn down mid-wait. 504: the commit may + /// still land (at-least-once), so this is a hard "outcome unknown", not a + /// failure the server may transparently retry. Carries the write's + /// operation so the 504 body names which write kind timed out. + Timeout(Operation), +} + +impl IntoResponse for PartitionWriteError { + fn into_response(self) -> Response { + match self { + Self::Rejected(error) => CustomError::from(error).into_response(), + Self::NotFound => CustomError::ResourceNotFound.into_response(), + Self::Unavailable => service_unavailable(), + Self::TooManyInFlight => too_many_in_flight_response(), + Self::ServerBusy => server_busy_response(), + Self::Timeout(operation) => partition_write_timeout_response(operation), + } + } +} + +/// 504 body for a partition write whose commit outcome is unknown, coded per +/// write kind so a caller can tell a produce timeout from an offset-write +/// timeout. Shaped like every other HTTP error (`ErrorResponse`) so clients +/// parse one error schema. +fn partition_write_timeout_response(operation: Operation) -> Response { + let (code, reason) = match operation { + Operation::SendMessages => ( + "produce_timeout", + "produce was not acknowledged in time; the write may still commit", + ), + _ => ( + "offset_write_timeout", + "consumer-offset write was not acknowledged in time; the write may still commit", + ), + }; + gateway_timeout_response(code, reason) +} + +/// Advisory `Retry-After` seconds for the shed / transient 429 and 503 +/// responses. One second: admission shedding, a briefly unavailable consensus +/// group, and a linearizable-read-on-follower all typically clear well within +/// it, and a small hint keeps a backing-off client responsive. +const RETRY_AFTER_SECONDS: u64 = 1; + +/// Attach the advisory [`RETRY_AFTER_SECONDS`] hint to a retryable 429/503. +pub(in crate::http) fn with_retry_after(mut response: Response) -> Response { + response + .headers_mut() + .insert(RETRY_AFTER, HeaderValue::from(RETRY_AFTER_SECONDS)); + response +} + +/// Render an `ErrorResponse` body for `status`, tagged with `code` / `reason` +/// and no field, so every hand-built HTTP error the routes return parses as the +/// one error schema clients already handle. +pub(in crate::http) fn error_response(status: StatusCode, code: &str, reason: &str) -> Response { + ( + status, + Json(ErrorResponse { + id: status.as_u16().into(), + code: code.to_owned(), + reason: reason.to_owned(), + field: None, + }), + ) + .into_response() +} + +/// Shared 504 rendering for an in-band request the partition plane did not +/// answer in time, shaped like every other HTTP error (`ErrorResponse`) so +/// clients parse one error schema. Consumed by the partition-write reply wait, +/// the partition reads ([`ReadError::Timeout`]), and the forward attempt bound. +pub(in crate::http) fn gateway_timeout_response(code: &str, reason: &str) -> Response { + error_response(StatusCode::GATEWAY_TIMEOUT, code, reason) +} + +/// The shared 503 body for a request that could not commit right now: no +/// caught-up primary, a full pipeline, or a view-change cancel. Retryable, and +/// rendered with the `CannotEstablishConnection` code the SDKs treat as a +/// connection-level retry rather than a terminal error. +fn service_unavailable() -> Response { + with_retry_after( + ( + StatusCode::SERVICE_UNAVAILABLE, + Json(ErrorResponse::from_error( + &IggyError::CannotEstablishConnection, + )), + ) + .into_response(), + ) +} + +/// 429 for a session at [`MAX_IN_FLIGHT_WRITES_PER_SESSION`] awaited partition +/// writes. Shaped like every other HTTP error (`ErrorResponse`) so clients +/// parse one error schema; the remedy is the caller's own: let outstanding +/// writes finish, then retry. +fn too_many_in_flight_response() -> Response { + with_retry_after(error_response( + StatusCode::TOO_MANY_REQUESTS, + "too_many_in_flight_writes", + "session reached its in-flight write cap; await outstanding writes and retry", + )) +} + +/// 503 for shard 0 at [`MAX_IN_FLIGHT_WRITES_GLOBAL`] awaited partition writes +/// across all sessions. A distinct `server_busy` code (unlike the shared +/// consensus-unavailable 503) so admission shedding is tellable from a +/// consensus outage; retry with backoff. +fn server_busy_response() -> Response { + with_retry_after(error_response( + StatusCode::SERVICE_UNAVAILABLE, + "server_busy", + "shard is at its in-flight write budget; retry with backoff", + )) +} + +/// Read consistency selected by the `?consistency=` query param. +/// +/// `serializable` (the default) serves from this node's local metadata STM: +/// correct and consensus-free, but may trail the primary by the replication +/// delay. `linearizable` demands the freshest committed state and is honored +/// only on the primary; a follower redirects (307) to the primary when its HTTP +/// address resolves from the roster, else fails closed to 503 (see +/// [`read_local`]). +#[derive(Clone, Copy, Default, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "lowercase")] +pub(in crate::http) enum Consistency { + #[default] + Serializable, + Linearizable, +} + +/// `?consistency=` query wrapper. An absent param defaults to +/// [`Consistency::Serializable`]; an unrecognized value is a 400 (axum `Query`). +#[derive(Default, Deserialize)] +pub(in crate::http) struct ConsistencyQuery { + #[serde(default)] + pub(in crate::http) consistency: Consistency, +} + +/// Produce acknowledgement selected by the `?ack=` query param. +/// +/// `replicated` (the default) answers 201 only after the partition group's +/// quorum commit. `none` is fire-and-forget: the request is validated, +/// dispatched, and answered 202 immediately; the commit still happens, but its +/// reply is shed at the bus (no reply slot is installed). +#[derive(Clone, Copy, Default, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "lowercase")] +pub(in crate::http) enum ProduceAck { + #[default] + Replicated, + None, +} + +/// `?ack=` query wrapper. An absent param defaults to +/// [`ProduceAck::Replicated`]; an unrecognized value is a 400 (axum `Query`). +#[derive(Default, Deserialize)] +pub(in crate::http) struct ProduceQuery { + #[serde(default)] + pub(in crate::http) ack: ProduceAck, +} + +/// Rejection for an authenticated read route (`GET /streams`, +/// `GET /streams/{id}`, and the reads that follow). +pub(in crate::http) enum ReadError { + /// Caller-side or STM rejection (bad identifier, unsupported op, or an + /// authorization denial) graded through the legacy `IggyError -> status` + /// map so SDK error bodies stay byte-identical. + Rejected(IggyError), + /// Requested entity is absent -> 404 with the legacy not-found body. + NotFound, + /// A linearizable read reached a follower and the primary's HTTP address was + /// not resolvable from the roster. Fail-closed 503, retryable against the + /// leader (see [`not_primary_response`]). + NotPrimary, + /// A linearizable read reached a follower and the current VSR primary's HTTP + /// address resolved: 307 to that address carrying the original path and + /// query, so the caller re-issues the read against the leader (see + /// [`primary_redirect_response`]). + RedirectToPrimary(String), + /// The post-restart read-recovery barrier expired with the recovered WAL + /// suffix still uncommitted: serving now could show state that rolls back + /// history a client already saw acked. Fail-closed 503 via the shared + /// [`service_unavailable`] body, retryable once the cluster re-commits the + /// suffix. + RecoveryIncomplete, + /// A partition read (poll / consumer-offset) got no reply from the owning + /// shard within the mesh budget. 504 like a produce timeout: the outcome is + /// unknown (the abandoned read may still be running), so the caller retries. + Timeout, +} + +impl IntoResponse for ReadError { + fn into_response(self) -> Response { + match self { + Self::Rejected(error) => CustomError::from(error).into_response(), + // Reuse the legacy 404 body so a missing stream renders exactly as + // the legacy server's `CustomError::ResourceNotFound` does. + Self::NotFound => CustomError::ResourceNotFound.into_response(), + Self::NotPrimary => not_primary_response(), + Self::RedirectToPrimary(location) => primary_redirect_response(&location), + Self::RecoveryIncomplete => service_unavailable(), + Self::Timeout => gateway_timeout_response( + "partition_read_timeout", + "the partition owner did not answer the read in time; retry", + ), + } + } +} + +/// The 503 fail-closed body for a linearizable read that reached a follower +/// whose primary HTTP address could not be resolved (absent consensus, a roster +/// with no node at the primary index, or a port-less node). The resolvable case +/// is a 307 via [`primary_redirect_response`] instead. Rendered as an +/// `ErrorResponse` so the body shape matches every other HTTP error; the caller +/// retries against the leader. +fn not_primary_response() -> Response { + with_retry_after(error_response( + StatusCode::SERVICE_UNAVAILABLE, + "not_primary", + "linearizable read requires the primary; retry against the leader", + )) +} + +/// 307 Temporary Redirect to the current VSR primary for a linearizable read +/// that reached a follower. `Location` is the primary's HTTP base plus the +/// original path and query, so the caller re-issues the identical read against +/// the leader. Dormant on a single node (always primary) and followed by no SDK +/// yet. A `Location` that is not a valid header value falls back to the 503. +fn primary_redirect_response(location: &str) -> Response { + HeaderValue::from_str(location).map_or_else( + |_| not_primary_response(), + |value| { + let mut response = StatusCode::TEMPORARY_REDIRECT.into_response(); + response.headers_mut().insert(LOCATION, value); + response + }, + ) +} + +/// Build the `Location` for a 307 redirect of a linearizable read to the VSR +/// primary: `://:`. The scheme is the +/// redirecting node's own listener scheme (uniform cluster HTTP config, same +/// assumption the forward hop makes). `client_ip` is the redirected client's +/// peer address, so the `Location` host comes from the primary's +/// per-client-network selectors when one matches. `None` when the primary does +/// not resolve from the roster, so the caller fails closed to a 503 rather +/// than pointing at an unreachable target. Pure (no consensus or axum +/// dependency) so the redirect target is unit-tested in isolation. +pub(in crate::http) fn primary_redirect_location( + roster: &ClusterRoster, + primary_index: u8, + scheme: &str, + path_and_query: &str, + client_ip: Option, +) -> Option { + let authority = primary_advertised_http_authority(roster, primary_index, client_ip)?; + Some(format!("{scheme}://{authority}{path_and_query}")) +} + +/// Resolve the VSR primary's HTTP socket from the static roster: the node +/// whose `replica_id` equals `primary_index`, its `ports.http`, and its +/// private roster `ip` (parsed once at roster build). Internal replica +/// forwarding uses this address; it must never route through +/// [`ResolvedClusterNode::advertised_for`], which picks client-facing hosts. +pub(in crate::http) fn primary_http_socket( + roster: &ClusterRoster, + primary_index: u8, +) -> Option { + let (node, http_port) = primary_node(roster, primary_index)?; + Some(SocketAddr::new(node.replica_ip()?, http_port)) +} + +/// Resolve the client-facing HTTP authority (`host:port`) for a redirect +/// through [`ResolvedClusterNode::advertised_for`]: a client-network selector +/// match first, then the catch-all advertised address, then the private +/// roster IP as the compatibility fallback. `AdvertisedAddress::authority` +/// brackets IPv6 hosts and passes hostnames through, so the redirect URL +/// stays valid. This is the fail-closed caller: a host that is neither a +/// valid IP nor a valid hostname yields `None` and the redirect becomes a +/// 503 rather than a `Location` pointing at an unparsable target (cluster +/// metadata makes the opposite choice and publishes such a host verbatim). +fn primary_advertised_http_authority( + roster: &ClusterRoster, + primary_index: u8, + client_ip: Option, +) -> Option { + let (node, http_port) = primary_node(roster, primary_index)?; + let address = node.advertised_for(client_ip)?; + Some(address.authority(http_port)) +} + +fn primary_node(roster: &ClusterRoster, primary_index: u8) -> Option<(&ResolvedClusterNode, u16)> { + let node = roster + .nodes + .iter() + .find(|node| node.config().replica_id == primary_index)?; + let http_port = node.config().ports.http?; + Some((node, http_port)) +} + +#[cfg(test)] +mod tests { + use super::*; + + use configs::cluster::{ClusterNodeConfig, TransportPorts}; + + const READ_PATH: &str = "/streams?consistency=linearizable"; + fn node(replica_id: u8, ip: &str, http: Option) -> ClusterNodeConfig { + ClusterNodeConfig { + name: format!("node-{replica_id}"), + ip: ip.to_owned(), + advertised_address: None, + advertised_addresses: Vec::new(), + replica_id, + ports: TransportPorts { + tcp: None, + quic: None, + http, + websocket: None, + tcp_replica: None, + }, + } + } + + fn roster(nodes: Vec) -> ClusterRoster { + ClusterRoster { + enabled: true, + name: "test-cluster".to_owned(), + nodes: nodes.into_iter().map(Into::into).collect(), + self_ip: "127.0.0.1".to_owned(), + self_ports: TransportPorts::default(), + metadata_view: std::sync::Arc::new(std::sync::atomic::AtomicU64::new( + crate::cluster_meta::METADATA_VIEW_UNKNOWN, + )), + } + } + + #[test] + fn primary_redirect_location_targets_primary_http_addr_with_path_passthrough() { + let roster = roster(vec![ + node(0, "10.0.0.1", Some(8080)), + node(1, "10.0.0.2", Some(8090)), + ]); + assert_eq!( + primary_redirect_location(&roster, 1, "http", READ_PATH, None), + Some("http://10.0.0.2:8090/streams?consistency=linearizable".to_owned()) + ); + } + + #[test] + fn primary_redirect_location_uses_the_listener_scheme() { + let roster = roster(vec![node(0, "10.0.0.1", Some(8080))]); + assert_eq!( + primary_redirect_location(&roster, 0, "https", READ_PATH, None), + Some("https://10.0.0.1:8080/streams?consistency=linearizable".to_owned()) + ); + } + + #[test] + fn primary_redirect_location_is_none_when_no_node_matches_primary_index() { + let roster = roster(vec![node(0, "10.0.0.1", Some(8080))]); + assert_eq!( + primary_redirect_location(&roster, 2, "http", READ_PATH, None), + None + ); + } + + #[test] + fn primary_redirect_location_is_none_when_primary_has_no_http_port() { + let roster = roster(vec![node(0, "10.0.0.1", None)]); + assert_eq!( + primary_redirect_location(&roster, 0, "http", READ_PATH, None), + None + ); + } + + #[test] + fn primary_redirect_location_is_none_for_empty_roster() { + let roster = roster(Vec::new()); + assert_eq!( + primary_redirect_location(&roster, 0, "http", READ_PATH, None), + None + ); + } + + #[test] + fn primary_redirect_location_brackets_ipv6_host() { + let roster = roster(vec![node(0, "::1", Some(8080))]); + assert_eq!( + primary_redirect_location(&roster, 0, "http", READ_PATH, None), + Some("http://[::1]:8080/streams?consistency=linearizable".to_owned()) + ); + } + + #[test] + fn primary_redirect_location_uses_advertised_address() { + let mut primary = node(0, "10.0.0.1", Some(8080)); + primary.advertised_address = Some("2001:db8::1".to_owned()); + let roster = roster(vec![primary]); + + assert_eq!( + primary_redirect_location(&roster, 0, "https", READ_PATH, None), + Some("https://[2001:db8::1]:8080/streams?consistency=linearizable".to_owned()) + ); + } + + #[test] + fn primary_redirect_location_uses_advertised_hostname() { + let mut primary = node(0, "10.0.0.1", Some(8080)); + primary.advertised_address = Some("broker-1.example.com".to_owned()); + let roster = roster(vec![primary]); + + assert_eq!( + primary_redirect_location(&roster, 0, "https", READ_PATH, None), + Some("https://broker-1.example.com:8080/streams?consistency=linearizable".to_owned()) + ); + } + + #[test] + fn primary_redirect_location_uses_the_selector_address_for_a_matching_client() { + let mut primary = node(0, "10.0.0.1", Some(8080)); + primary.advertised_address = Some("203.0.113.1".to_owned()); + primary.advertised_addresses = vec![configs::cluster::AdvertisedAddressSelector { + client_cidr: "10.0.0.0/16".to_owned(), + address: "10.0.0.1".to_owned(), + }]; + let roster = roster(vec![primary]); + + assert_eq!( + primary_redirect_location( + &roster, + 0, + "https", + READ_PATH, + Some("10.0.9.9".parse().unwrap()) + ), + Some("https://10.0.0.1:8080/streams?consistency=linearizable".to_owned()), + "an in-network client must be redirected to the selector address" + ); + assert_eq!( + primary_redirect_location( + &roster, + 0, + "https", + READ_PATH, + Some("198.51.100.7".parse().unwrap()) + ), + Some("https://203.0.113.1:8080/streams?consistency=linearizable".to_owned()), + "an out-of-network client must stay on the catch-all address" + ); + } + + #[test] + fn primary_http_socket_uses_private_roster_ip() { + let mut primary = node(0, "10.0.0.1", Some(8080)); + primary.advertised_address = Some("203.0.113.1".to_owned()); + let roster = roster(vec![primary]); + + assert_eq!( + primary_http_socket(&roster, 0), + Some("10.0.0.1:8080".parse().expect("valid socket address")) + ); + } + + #[test] + fn transient_not_committed_renders_503_with_retry_after() { + let response = CustomError::from(IggyError::TransientNotCommitted).into_response(); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert!(response.headers().contains_key(RETRY_AFTER)); + } + + #[test] + fn transient_not_accepted_renders_503_with_retry_after() { + let response = CustomError::from(IggyError::TransientNotAccepted).into_response(); + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert!(response.headers().contains_key(RETRY_AFTER)); + } + + #[test] + fn business_error_renders_without_retry_after() { + let response = CustomError::from(IggyError::UserAlreadyExists).into_response(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + assert!(!response.headers().contains_key(RETRY_AFTER)); + } + + // The ownership refusal is permanent and deterministic. Rendering it as + // 503 would hand the caller's HTTP stack a retry loop it can never escape + // (no foreign SDK special-cases 503), so the status is load-bearing. + #[test] + fn owned_client_id_renders_as_terminal_conflict() { + let response = AuthError::SessionIdOwnedByAnotherUser.into_response(); + assert_eq!(response.status(), StatusCode::CONFLICT); + assert!( + response.headers().get(RETRY_AFTER).is_none(), + "a terminal refusal must not advertise a retry" + ); + } + + // Its siblings stay retryable, so the split is visible in one place. + #[test] + fn unknown_outcome_registers_stay_retryable() { + for error in [AuthError::SessionUnavailable, AuthError::SessionNotAccepted] { + let status = error.into_response().status(); + assert!( + status.is_server_error(), + "an unknown commit outcome must stay retryable, got {status}" + ); + } + } + + #[test] + fn recovery_incomplete_renders_retryable_503_like_not_primary() { + // Barrier expiry must render as the shared retryable 503: the same + // status and Retry-After hint as the not-primary 503, so an SDK treats + // it as a connection-level retry rather than a terminal error. + let recovery = ReadError::RecoveryIncomplete.into_response(); + assert_eq!(recovery.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!( + recovery.headers().get(RETRY_AFTER), + Some(&HeaderValue::from(RETRY_AFTER_SECONDS)) + ); + + let not_primary = ReadError::NotPrimary.into_response(); + assert_eq!(recovery.status(), not_primary.status()); + assert_eq!( + recovery.headers().get(RETRY_AFTER), + not_primary.headers().get(RETRY_AFTER) + ); + } +} diff --git a/core/server-ng/src/http/extractor.rs b/core/server/src/http/extractor.rs similarity index 100% rename from core/server-ng/src/http/extractor.rs rename to core/server/src/http/extractor.rs diff --git a/core/server-ng/src/http/forward.rs b/core/server/src/http/forward.rs similarity index 98% rename from core/server-ng/src/http/forward.rs rename to core/server/src/http/forward.rs index 08f8608eba..5e4572f701 100644 --- a/core/server-ng/src/http/forward.rs +++ b/core/server/src/http/forward.rs @@ -84,7 +84,7 @@ use crate::http::error::{ }; use crate::http::extractor::{bearer_token, resolve_credential}; use crate::http::state::{HttpInner, VIEW_HEADER}; -use crate::server_error::ServerNgError; +use crate::server_error::ServerError; /// Marker stamped on every forwarded request. Loop guard only: a node that is /// not primary and sees it answers the transient 503 instead of forwarding @@ -166,14 +166,14 @@ pub(in crate::http) struct ForwardState { /// /// # Errors /// -/// [`ServerNgError::ListenerCredentials`] when TLS is enabled but the PEM -/// files cannot be loaded, [`ServerNgError::HttpForwardClient`] when the +/// [`ServerError::ListenerCredentials`] when TLS is enabled but the PEM +/// files cannot be loaded, [`ServerError::HttpForwardClient`] when the /// outbound client cannot be built. pub(in crate::http) fn build_forward_state( tls: &HttpTlsConfig, body_limit: usize, active: bool, -) -> Result { +) -> Result { // Unconditional: cyper's rustls connector resolves the process default // provider even when the client only ever dials plain http. install_default_crypto_provider(); @@ -184,7 +184,7 @@ pub(in crate::http) fn build_forward_state( let (builder, scheme) = if tls.enabled { let credentials = load_pem(Path::new(&tls.cert_file), Path::new(&tls.key_file)).map_err(|source| { - ServerNgError::ListenerCredentials { + ServerError::ListenerCredentials { transport: "http.tls", source, } @@ -192,7 +192,7 @@ pub(in crate::http) fn build_forward_state( // load_pem guarantees a non-empty chain; this is the no-panic // path for the unreachable empty case. let pinned = credentials.cert_chain.into_iter().next().ok_or_else(|| { - ServerNgError::HttpForwardClient { + ServerError::HttpForwardClient { reason: "TLS certificate chain is empty".to_string(), } })?; @@ -218,7 +218,7 @@ pub(in crate::http) fn build_forward_state( }; let client = builder .build() - .map_err(|source| ServerNgError::HttpForwardClient { + .map_err(|source| ServerError::HttpForwardClient { reason: source.to_string(), })?; Ok(ForwardState { diff --git a/core/server-ng/src/http/handlers.rs b/core/server/src/http/handlers.rs similarity index 99% rename from core/server-ng/src/http/handlers.rs rename to core/server/src/http/handlers.rs index 0dc160fe29..cc5d501c56 100644 --- a/core/server-ng/src/http/handlers.rs +++ b/core/server/src/http/handlers.rs @@ -218,7 +218,7 @@ pub(in crate::http) struct RefreshToken { /// `POST /users/refresh-token`: re-issue an access token from a still-valid one, /// answering the same `IdentityInfo` shape as login. /// -/// Stateless by design: server-ng has no replicated revocation list (the P3 +/// Stateless by design: server has no replicated revocation list (the P3 /// roadmap item), so refreshing cannot invalidate the presented token - it stays /// valid until its own `exp`, the same posture as logout ending a session /// without revoking its bearer. Per-node revocation would be false security in a @@ -691,7 +691,7 @@ pub(in crate::http) async fn get_client( /// `StreamDetails` JSON the legacy server returns. /// /// The accepted body is name-only (`{"name": ...}`), matching the legacy -/// request; server-ng's wire `CreateStreamRequest` is likewise name-only and +/// request; server's wire `CreateStreamRequest` is likewise name-only and /// auto-assigns the id, so there is no client-supplied stream id to honor. pub(in crate::http) async fn create_stream( State(state): State, diff --git a/core/server/src/http/http_server.rs b/core/server/src/http/http_server.rs deleted file mode 100644 index a085b75441..0000000000 --- a/core/server/src/http/http_server.rs +++ /dev/null @@ -1,399 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::configs::http::{HttpConfig, HttpCorsConfig}; -use crate::http::diagnostics::request_diagnostics; -use crate::http::http_shard_wrapper::HttpSafeShard; -use crate::http::jwt::jwt_manager::JwtManager; -use crate::http::jwt::middleware::jwt_auth; -use crate::http::metrics::metrics; -use crate::http::shared::AppState; -use crate::http::*; -use crate::shard::IggyShard; -use crate::shard::task_registry::ShutdownToken; -use crate::shard::tasks::periodic::spawn_jwt_token_cleaner; -use crate::shard::transmission::event::ShardEvent; -use crate::streaming::persistence::persister::PersisterKind; -use crate::streaming::utils::crypto; -use axum::extract::DefaultBodyLimit; -use axum::extract::connect_info::Connected; -use axum::http::Method; -use axum::{Router, middleware}; -use axum_server::tls_rustls::RustlsConfig; -use compio::net::TcpListener; -use err_trail::ErrContext; -use iggy_common::IggyError; -use iggy_common::TransportProtocol; -use socket2::{Domain, Protocol, Socket, Type}; -use std::net::SocketAddr; -use std::path::PathBuf; -use std::rc::Rc; -use std::sync::Arc; -use tower_http::cors::{AllowOrigin, CorsLayer}; -use tracing::{error, info, warn}; - -#[derive(Debug, Clone, Copy)] -pub struct CompioSocketAddr(pub SocketAddr); - -impl From for CompioSocketAddr { - fn from(addr: SocketAddr) -> Self { - CompioSocketAddr(addr) - } -} - -impl From for SocketAddr { - fn from(addr: CompioSocketAddr) -> Self { - addr.0 - } -} - -impl<'a> Connected> for CompioSocketAddr { - fn connect_info(target: cyper_axum::IncomingStream<'a, TcpListener>) -> Self { - let addr = *target.remote_addr(); - CompioSocketAddr(addr) - } -} - -/// Starts the HTTP API server. -/// Returns the address the server is listening on. -pub async fn start_http_server( - config: HttpConfig, - persister: Arc, - shard: Rc, - shutdown: ShutdownToken, -) -> Result<(), IggyError> { - if shard.id != 0 { - info!( - "HTTP server disabled for shard {} (only runs on shard 0)", - shard.id - ); - panic!("HTTP server only runs on shard 0"); - } - - let api_name = if config.tls.enabled { - "HTTP API (TLS)" - } else { - "HTTP API" - }; - - let app_state = build_app_state(&config, persister, shard.clone()).await; - let mut app = Router::new() - .merge(system::router(app_state.clone(), &config.metrics)) - .merge(personal_access_tokens::router(app_state.clone())) - .merge(users::router(app_state.clone())) - .merge(streams::router(app_state.clone())) - .merge(topics::router(app_state.clone())) - .merge(consumer_groups::router(app_state.clone())) - .merge(consumer_offsets::router(app_state.clone())) - .merge(partitions::router(app_state.clone())) - .merge(segments::router(app_state.clone())) - .merge(messages::router(app_state.clone())) - .layer(DefaultBodyLimit::max( - config.max_request_size.as_bytes_u64() as usize, - )) - .layer(middleware::from_fn_with_state(app_state.clone(), jwt_auth)); - - if config.cors.enabled { - app = app.layer(configure_cors(config.cors)?); - } - - if config.metrics.enabled { - app = app.layer(middleware::from_fn_with_state(app_state.clone(), metrics)); - } - - spawn_jwt_token_cleaner(shard.clone(), app_state.clone()); - - app = app.layer(middleware::from_fn(request_diagnostics)); - - #[cfg(feature = "iggy-web")] - if config.web_ui { - app = app.merge(web::router()); - info!("Web UI enabled at /ui"); - } - - #[cfg(not(feature = "iggy-web"))] - if config.web_ui { - tracing::warn!( - "Web UI is enabled in configuration (http.web_ui = true) but the server \ - was not compiled with 'iggy-web' feature. The Web UI will not be available. \ - To enable it, rebuild the server with: cargo build --features iggy-web" - ); - } - - if !config.tls.enabled { - let bind_addr: SocketAddr = config - .address - .parse() - .unwrap_or_else(|_| panic!("Failed to parse HTTP address {}", config.address)); - let listener = crate::tcp::bind_reuseport_listener(bind_addr, true, None) - .await - .unwrap_or_else(|_| panic!("Failed to bind to HTTP address {}", config.address)); - let address = listener - .local_addr() - .expect("Failed to get local address for HTTP server"); - info!("Started {api_name} on: {address}"); - - // Notify shard about the bound address - let event = ShardEvent::AddressBound { - protocol: TransportProtocol::Http, - address, - }; - - crate::shard::handlers::handle_event(&shard, event) - .await - .ok(); - - let service = app.into_make_service_with_connect_info::(); - - let shutdown_token = shutdown.clone(); - let result = cyper_axum::serve(listener, service) - .with_graceful_shutdown(async move { shutdown_token.wait().await }) - .await; - - match result { - Ok(()) => { - info!("{api_name} shut down gracefully"); - Ok(()) - } - Err(error) => { - error!("{api_name} server error: {}", error); - Err(IggyError::CannotBindToSocket(format!("HTTP: {}", error))) - } - } - } else { - let tls_config = RustlsConfig::from_pem_file( - PathBuf::from(config.tls.cert_file), - PathBuf::from(config.tls.key_file), - ) - .await - .unwrap(); - - let addr: SocketAddr = config - .address - .parse() - .unwrap_or_else(|e| panic!("Invalid HTTPS address '{}': {e}", config.address)); - let domain = if addr.is_ipv6() { - Domain::IPV6 - } else { - Domain::IPV4 - }; - let socket = Socket::new(domain, Type::STREAM, Some(Protocol::TCP)) - .unwrap_or_else(|e| panic!("Failed to create HTTPS socket: {e}")); - socket - .set_reuse_address(true) - .unwrap_or_else(|e| panic!("Failed to set SO_REUSEADDR: {e}")); - #[cfg(unix)] - socket - .set_reuse_port(true) - .unwrap_or_else(|e| panic!("Failed to set SO_REUSEPORT: {e}")); - socket - .bind(&addr.into()) - .unwrap_or_else(|e| panic!("Failed to bind to HTTPS address {}: {e}", config.address)); - socket - .listen(128) - .unwrap_or_else(|e| panic!("Failed to listen on HTTPS: {e}")); - let listener: std::net::TcpListener = socket.into(); - listener - .set_nonblocking(true) - .expect("Failed to set TLS listener to non-blocking"); - let address = listener - .local_addr() - .expect("Failed to get local address for HTTPS / TLS server"); - - info!("Started {api_name} on: {address}"); - - // Notify shard about the bound address - use crate::shard::transmission::event::ShardEvent; - use iggy_common::TransportProtocol; - let event = ShardEvent::AddressBound { - protocol: TransportProtocol::Http, - address, - }; - - crate::shard::handlers::handle_event(&shard, event) - .await - .ok(); - - let service = app.into_make_service_with_connect_info::(); - let handle = axum_server::Handle::new(); - let shutdown_handle = handle.clone(); - let api_name_for_task = api_name; - shard - .task_registry - .oneshot("http_shutdown_listener") - .critical(false) - .run(move |shutdown: ShutdownToken| async move { - shutdown.wait().await; - info!("Initiating graceful shutdown for {api_name_for_task}"); - shutdown_handle.graceful_shutdown(None); - Ok(()) - }) - .spawn(); - - let server = axum_server::from_tcp_rustls(listener, tls_config) - .map_err(|err| IggyError::HttpError(err.to_string()))? - .handle(handle); - match server.serve(service).await { - Ok(()) => { - info!("{api_name} shut down gracefully"); - Ok(()) - } - Err(error) => { - error!("Failed to start {api_name} server, error: {}", error); - Err(IggyError::CannotBindToSocket(format!("HTTPS: {}", error))) - } - } - } -} - -async fn build_app_state( - config: &HttpConfig, - persister: Arc, - shard: Rc, -) -> Arc { - let tokens_path; - { - tokens_path = shard.config.system.get_state_tokens_path(); - } - - let mut jwt_config = config.jwt.clone(); - let encoding_empty = jwt_config.encoding_secret.is_empty(); - let decoding_empty = jwt_config.decoding_secret.is_empty(); - match (encoding_empty, decoding_empty) { - (true, true) => { - let secret = crypto::generate_secret(32..64); - let redacted: String = secret.chars().take(3).collect(); - warn!( - "JWT encoding and decoding secrets are not configured - generated a random secret: {redacted}***. JWT tokens will be invalidated on server restart. Set 'encoding_secret' and 'decoding_secret' in the config to use persistent secrets." - ); - jwt_config.encoding_secret = secret.clone(); - jwt_config.decoding_secret = secret; - } - (true, false) => { - warn!( - "JWT encoding secret is not configured but decoding secret is set - using decoding secret for both. Set 'encoding_secret' in the config to avoid this warning." - ); - jwt_config.encoding_secret = jwt_config.decoding_secret.clone(); - } - (false, true) => { - warn!( - "JWT decoding secret is not configured but encoding secret is set - using encoding secret for both. Set 'decoding_secret' in the config to avoid this warning." - ); - jwt_config.decoding_secret = jwt_config.encoding_secret.clone(); - } - (false, false) => { - if jwt_config.encoding_secret != jwt_config.decoding_secret - && jwt_config.algorithm.starts_with("HS") - { - warn!( - "JWT encoding and decoding secrets are different but algorithm is {} (HMAC) - both secrets must be identical for symmetric algorithms.", - jwt_config.algorithm - ); - } - } - } - - let jwt_manager = match JwtManager::from_config(persister, &tokens_path, &jwt_config) { - Ok(manager) => manager, - Err(error) => panic!("Failed to initialize JWT manager: {error}"), - }; - if let Err(error) = jwt_manager.load_revoked_tokens().await { - panic!("Failed to load revoked access tokens: {error}"); - } - - Arc::new(AppState { - jwt_manager, - shard: HttpSafeShard::new(shard), - }) -} - -fn configure_cors(config: HttpCorsConfig) -> Result { - let allowed_origins = match config.allowed_origins { - ref origins if origins.is_empty() => AllowOrigin::default(), - ref origins if origins.first().unwrap() == "*" => AllowOrigin::any(), - origins => { - let parsed: Result, _> = origins - .iter() - .filter(|s| !s.trim().is_empty()) - .map(|s| { - s.parse() - .error(|e: &axum::http::header::InvalidHeaderValue| { - format!("Invalid CORS origin '{s}': {e}") - }) - .map_err(|_| IggyError::InvalidConfiguration) - }) - .collect(); - AllowOrigin::list(parsed?) - } - }; - - let allowed_headers: Result, _> = config - .allowed_headers - .iter() - .filter(|s| !s.trim().is_empty()) - .map(|s| { - s.parse() - .error(|e: &axum::http::header::InvalidHeaderName| { - format!("Invalid CORS header '{s}': {e}") - }) - .map_err(|_| IggyError::InvalidConfiguration) - }) - .collect(); - let allowed_headers = allowed_headers?; - - let exposed_headers: Result, _> = config - .exposed_headers - .iter() - .filter(|s| !s.trim().is_empty()) - .map(|s| { - s.parse() - .error(|e: &axum::http::header::InvalidHeaderName| { - format!("Invalid CORS exposed header '{s}': {e}") - }) - .map_err(|_| IggyError::InvalidConfiguration) - }) - .collect(); - let exposed_headers = exposed_headers?; - - let allowed_methods: Result, _> = config - .allowed_methods - .iter() - .filter(|s| !s.trim().is_empty()) - .map(|s| match s.to_uppercase().as_str() { - "GET" => Ok(Method::GET), - "POST" => Ok(Method::POST), - "PUT" => Ok(Method::PUT), - "DELETE" => Ok(Method::DELETE), - "HEAD" => Ok(Method::HEAD), - "OPTIONS" => Ok(Method::OPTIONS), - "CONNECT" => Ok(Method::CONNECT), - "PATCH" => Ok(Method::PATCH), - "TRACE" => Ok(Method::TRACE), - _ => Err(IggyError::InvalidConfiguration) - .error(|_: &IggyError| format!("Invalid HTTP method in CORS config: '{s}'")), - }) - .collect(); - let allowed_methods = allowed_methods?; - - Ok(CorsLayer::new() - .allow_methods(allowed_methods) - .allow_origin(allowed_origins) - .allow_headers(allowed_headers) - .expose_headers(exposed_headers) - .allow_credentials(config.allow_credentials) - .allow_private_network(config.allow_private_network)) -} diff --git a/core/server/src/http/http_shard_wrapper.rs b/core/server/src/http/http_shard_wrapper.rs deleted file mode 100644 index 9952e4de78..0000000000 --- a/core/server/src/http/http_shard_wrapper.rs +++ /dev/null @@ -1,232 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::rc::Rc; - -use iggy_common::{ - Consumer, ConsumerOffsetInfo, Identifier, IggyError, Partitioning, PartitioningKind, -}; -use send_wrapper::SendWrapper; - -use crate::shard::system::messages::PollingArgs; -use crate::shard::transmission::frame::ShardResponse; -use crate::shard::transmission::message::ShardRequest; -use crate::streaming::segments::{IggyMessagesBatchMut, IggyMessagesBatchSet}; -use crate::streaming::topics; -use crate::streaming::users::user::User; -use crate::{shard::IggyShard, streaming::session::Session}; -use iggy_common::IggyPollMetadata; - -/// Wrapper around IggyShard for HTTP handlers. -/// -/// Provides three categories of access: -/// 1. Control-plane mutations via `send_to_control_plane()` (routed through message pump) -/// 2. Read-only metadata access via `shard()` (direct, same-thread safe) -/// 3. Data-plane operations (poll/append messages, consumer offsets) -/// -/// # Safety -/// This wrapper is safe because: -/// 1. HTTP server runs on shard 0's single thread (compio model) -/// 2. All operations are confined to that thread -/// 3. The underlying IggyShard is never accessed from multiple threads -pub struct HttpSafeShard { - inner: Rc, -} - -// Safety: HttpSafeShard is only used in HTTP handlers on shard 0's thread. -// All operations are confined to the thread that created the IggyShard instance. -// The underlying IggyShard contains RefCell and Rc types that are not thread-safe, -// but they are never accessed across threads in the HTTP server context with -// compio's single-threaded model. -unsafe impl Send for HttpSafeShard {} -unsafe impl Sync for HttpSafeShard {} - -impl HttpSafeShard { - pub fn new(shard: Rc) -> Self { - Self { inner: shard } - } - - /// Direct access to shard for read-only operations and auth. - pub fn shard(&self) -> &IggyShard { - &self.inner - } - - /// Route control-plane mutations through the message pump. - pub async fn send_to_control_plane( - &self, - request: ShardRequest, - ) -> Result { - let future = SendWrapper::new(self.inner.send_to_control_plane(request)); - future.await - } - - // === Data-plane operations (message polling/appending) === - - pub async fn get_consumer_offset( - &self, - client_id: u32, - consumer: Consumer, - stream_id: &Identifier, - topic_id: &Identifier, - partition_id: Option, - ) -> Result, IggyError> { - let topic = self.shard().resolve_topic(stream_id, topic_id)?; - let future = SendWrapper::new(self.shard().get_consumer_offset( - client_id, - consumer, - topic, - partition_id, - )); - future.await - } - - pub async fn store_consumer_offset( - &self, - client_id: u32, - consumer: Consumer, - stream_id: &Identifier, - topic_id: &Identifier, - partition_id: Option, - offset: u64, - ) -> Result<(), IggyError> { - let topic = self.shard().resolve_topic(stream_id, topic_id)?; - let future = SendWrapper::new(self.shard().store_consumer_offset( - client_id, - consumer, - topic, - partition_id, - offset, - )); - let _result = future.await?; - Ok(()) - } - - pub async fn delete_consumer_offset( - &self, - client_id: u32, - consumer: Consumer, - stream_id: &Identifier, - topic_id: &Identifier, - partition_id: Option, - ) -> Result<(), IggyError> { - let topic = self.shard().resolve_topic(stream_id, topic_id)?; - let future = SendWrapper::new(self.shard().delete_consumer_offset( - client_id, - consumer, - topic, - partition_id, - )); - let _result = future.await?; - Ok(()) - } - - #[allow(clippy::too_many_arguments)] - pub async fn poll_messages( - &self, - client_id: u32, - user_id: u32, - stream_id: Identifier, - topic_id: Identifier, - consumer: Consumer, - maybe_partition_id: Option, - args: PollingArgs, - ) -> Result<(IggyPollMetadata, IggyMessagesBatchSet), IggyError> { - let topic = self - .shard() - .resolve_topic_for_poll(user_id, &stream_id, &topic_id)?; - let future = SendWrapper::new(self.shard().poll_messages( - client_id, - topic, - consumer.clone(), - maybe_partition_id, - args, - )); - - future.await - } - - pub async fn append_messages( - &self, - user_id: u32, - stream_id: Identifier, - topic_id: Identifier, - partitioning: &Partitioning, - batch: IggyMessagesBatchMut, - ) -> Result<(), IggyError> { - use crate::shard::transmission::message::ResolvedPartition; - - let topic = self - .shard() - .resolve_topic_for_append(user_id, &stream_id, &topic_id)?; - let partition_id = match partitioning.kind { - PartitioningKind::Balanced => self - .shard() - .metadata - .get_next_partition_id(topic.stream_id, topic.topic_id) - .ok_or(IggyError::TopicIdNotFound(stream_id, topic_id))?, - PartitioningKind::PartitionId => u32::from_le_bytes( - partitioning - .value - .get(..4) - .ok_or(IggyError::InvalidCommand)? - .try_into() - .map_err(|_| IggyError::InvalidNumberEncoding)?, - ) as usize, - PartitioningKind::MessagesKey => { - let partitions_count = self - .shard() - .metadata - .partitions_count(topic.stream_id, topic.topic_id); - topics::helpers::calculate_partition_id_by_messages_key_hash( - partitions_count, - &partitioning.value, - ) - } - }; - - let partition = ResolvedPartition { - stream_id: topic.stream_id, - topic_id: topic.topic_id, - partition_id, - }; - - let future = SendWrapper::new(self.shard().append_messages(partition, batch)); - future.await - } - - pub fn login_user( - &self, - username: &str, - password: &str, - session: Option<&Session>, - ) -> Result { - self.shard().login_user(username, password, session) - } - - pub fn logout_user(&self, session: &Session) -> Result<(), IggyError> { - self.shard().logout_user(session) - } - - pub fn login_with_personal_access_token( - &self, - token: &str, - session: Option<&Session>, - ) -> Result { - self.shard() - .login_with_personal_access_token(token, session) - } -} diff --git a/core/server-ng/src/http/jwks.rs b/core/server/src/http/jwks.rs similarity index 100% rename from core/server-ng/src/http/jwks.rs rename to core/server/src/http/jwks.rs diff --git a/core/server-ng/src/http/jwt.rs b/core/server/src/http/jwt.rs similarity index 98% rename from core/server-ng/src/http/jwt.rs rename to core/server/src/http/jwt.rs index c5053e1470..29fae7e437 100644 --- a/core/server-ng/src/http/jwt.rs +++ b/core/server/src/http/jwt.rs @@ -17,7 +17,7 @@ //! Minimal JWT issuer/verifier for the shard-0 HTTP listener. //! -//! Ported from the legacy `server::http::jwt::jwt_manager::JwtManager`, reduced +//! Ported from the legacy server implementation's `JwtManager`, reduced //! to the issue + verify half: no revoked-token persistence. Self-issued HS256 //! is the common path; with `[[http.jwt.trusted_issuers]]` configured, `decode` //! also verifies external RS256/EC tokens against the issuer's JWKS @@ -54,7 +54,11 @@ const GENERATED_SECRET_LEN: Range = 32..64; /// with this fallback a PSK compromise also yields the token-signing key, and /// rotating the PSK invalidates all bearers; operators who want the domains /// decoupled configure an explicit `http.jwt` secret, which always wins. -const JWT_KEY_CONTEXT: &str = "apache-iggy server-ng http-jwt v1 psk->hs256-key"; +/// +/// Frozen string: it is KDF domain separation, not a label. Editing it derives +/// a different signing key and invalidates every bearer issued before the +/// change. +const JWT_KEY_CONTEXT: &str = "apache-iggy server http-jwt v1 psk->hs256-key"; /// Expiry stamp used for a non-expiring token: far enough out to never trip /// `exp` validation, small enough to fit `u32`. Mirrors the legacy server. diff --git a/core/server/src/http/jwt/json_web_token.rs b/core/server/src/http/jwt/json_web_token.rs deleted file mode 100644 index c8c95ecf72..0000000000 --- a/core/server/src/http/jwt/json_web_token.rs +++ /dev/null @@ -1,155 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use iggy_common::UserId; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; -use std::net::SocketAddr; -use std::{fmt, fmt::Display}; - -#[derive(Debug, Clone)] -pub struct Identity { - pub token_id: String, - pub token_expiry: u64, - pub user_id: UserId, - pub ip_address: SocketAddr, -} - -#[derive(Debug, Clone)] -pub enum Audience { - Single(String), - Multiple(Vec), -} - -impl Audience { - pub fn contains(&self, audience: &str) -> bool { - match self { - Audience::Single(aud) => aud == audience, - Audience::Multiple(auds) => auds.iter().any(|a| a == audience), - } - } -} - -impl Display for Audience { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Audience::Single(aud) => f.write_str(aud), - Audience::Multiple(auds) => f.write_str(&auds.join(",")), - } - } -} - -impl From for Audience { - fn from(aud: String) -> Self { - Audience::Single(aud) - } -} - -impl From<&str> for Audience { - fn from(aud: &str) -> Self { - Audience::Single(aud.to_string()) - } -} - -impl From> for Audience { - fn from(auds: Vec) -> Self { - if auds.len() == 1 { - Audience::Single(auds.into_iter().next().unwrap()) - } else { - Audience::Multiple(auds) - } - } -} - -impl Serialize for Audience { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - match self { - Audience::Single(aud) => serializer.serialize_str(aud), - Audience::Multiple(auds) => auds.serialize(serializer), - } - } -} - -impl<'de> Deserialize<'de> for Audience { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - struct AudienceVisitor; - - impl<'de> serde::de::Visitor<'de> for AudienceVisitor { - type Value = Audience; - - fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { - formatter.write_str("a string or an array of strings") - } - - fn visit_str(self, value: &str) -> Result - where - E: serde::de::Error, - { - Ok(Audience::Single(value.to_string())) - } - - fn visit_string(self, value: String) -> Result - where - E: serde::de::Error, - { - Ok(Audience::Single(value)) - } - - fn visit_seq(self, mut seq: A) -> Result - where - A: serde::de::SeqAccess<'de>, - { - let mut auds = Vec::new(); - while let Some(aud) = seq.next_element::()? { - auds.push(aud); - } - Ok(Audience::Multiple(auds)) - } - } - - deserializer.deserialize_any(AudienceVisitor) - } -} - -#[derive(Debug, Serialize, Deserialize, Clone)] -pub struct JwtClaims { - pub jti: String, - pub iss: String, - pub aud: Audience, - pub sub: String, - pub iat: u64, - pub exp: u64, - pub nbf: u64, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct RevokedAccessToken { - pub id: String, - pub expiry: u64, -} - -#[derive(Debug)] -pub struct GeneratedToken { - pub user_id: UserId, - pub access_token: String, - pub access_token_expiry: u64, -} diff --git a/core/server/src/http/jwt/jwks.rs b/core/server/src/http/jwt/jwks.rs deleted file mode 100644 index 0932e5ed44..0000000000 --- a/core/server/src/http/jwt/jwks.rs +++ /dev/null @@ -1,357 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::hash::Hash; -use std::sync::Arc; -use std::time::{Duration, Instant}; - -use dashmap::DashMap; -use iggy_common::IggyError; -use jsonwebtoken::DecodingKey; -use serde::Deserialize; -use strum::{Display, EnumString}; -use tokio::sync::Mutex; - -/// Minimum wall-clock gap between outbound JWKS fetches for one trusted issuer. -/// Inside this window a cache miss is authoritative: the issuer's key set was -/// just read, so an absent `kid` is genuinely absent and no fetch is issued. -/// This bounds a pre-auth caller replaying unknown `kid`s to at most one -/// outbound request per issuer per window; the trade is that a freshly rotated -/// key is only discoverable once the window elapses. -const JWKS_REFRESH_MIN_INTERVAL: Duration = Duration::from_secs(10); - -thread_local! { - // cyper 0.9's `Client` is `!Send`/`!Sync` (`Rc`-backed) and `new()` - // now returns a `Result`, so it can no longer be a global `OnceLock`. - // compio is thread-per-core; keep one client per thread. The `Rc` - // inner makes cloning cheap, so callers take an owned handle. - static HTTP_CLIENT: cyper::Client = - cyper::Client::new().expect("failed to build cyper HTTP client for JWKS"); -} - -fn get_http_client() -> cyper::Client { - HTTP_CLIENT.with(cyper::Client::clone) -} - -/// JWK key type enumeration -#[derive(Debug, Clone, Copy, Display, EnumString, Deserialize, PartialEq, Eq)] -#[strum(serialize_all = "UPPERCASE")] -#[serde(rename_all = "UPPERCASE")] -enum JwkKeyType { - /// RSA key type - #[strum(serialize = "RSA")] - Rsa, - /// EC (Elliptic Curve) key type - #[strum(serialize = "EC")] - Ec, -} - -/// EC curve type enumeration -#[derive(Debug, Clone, Copy, Display, EnumString, Deserialize, PartialEq, Eq)] -#[strum(serialize_all = "UPPERCASE")] -#[serde(rename_all = "UPPERCASE")] -enum EcCurve { - /// P-256 curve - #[strum(serialize = "P-256")] - P256, - /// P-384 curve - #[strum(serialize = "P-384")] - P384, - /// P-521 curve - #[strum(serialize = "P-521")] - P521, -} - -#[derive(Debug, Deserialize)] -struct Jwk { - kty: JwkKeyType, - kid: Option, - n: Option, - e: Option, - x: Option, - y: Option, - crv: Option, -} - -#[derive(Debug, Deserialize)] -struct JwkSet { - keys: Vec, -} - -#[derive(Debug, Clone, Hash, Eq, PartialEq)] -struct CacheKey { - issuer: String, - kid: String, -} - -#[derive(Debug, Clone)] -pub struct JwksClient { - cache: DashMap, - /// Per-issuer single-flight and rate-limit guard. The async mutex serialises - /// refresh attempts for one issuer so a burst of concurrent misses collapses - /// onto a single fetch; the inner instant is when that issuer was last - /// fetched and gates [`JWKS_REFRESH_MIN_INTERVAL`]. Keyed only by issuer - /// (operator-configured, bounded), never by the attacker-controlled `kid`. - refresh_guards: DashMap>>>, -} - -impl Default for JwksClient { - fn default() -> Self { - Self { - cache: DashMap::new(), - refresh_guards: DashMap::new(), - } - } -} - -impl JwksClient { - /// Resolve the decoding key for `{issuer, kid}`, fetching and caching the - /// issuer's JWKS on a cache miss. - /// - /// A cache miss is served under a per-issuer guard: concurrent misses fetch - /// once, and within [`JWKS_REFRESH_MIN_INTERVAL`] of the last fetch a miss is - /// treated as a known-absent `kid` and rejected without an outbound request. - /// This keeps an unauthenticated caller replaying unknown `kid`s from - /// amplifying into unbounded fetches against the issuer's JWKS endpoint. - // The per-issuer guard is deliberately held across the JWKS fetch: that hold - // is what serialises concurrent misses onto a single outbound request. Drop- - // tightening would release it before the await and defeat the single-flight. - #[allow(clippy::significant_drop_tightening)] - pub async fn get_key(&self, issuer: &str, jwks_url: &str, kid: &str) -> Option { - let cache_key = CacheKey { - issuer: issuer.to_string(), - kid: kid.to_string(), - }; - - // Positive-cache fast path: no lock, no fetch. - if let Some(key) = self.cache.get(&cache_key) { - return Some(key.clone()); - } - - // Take the per-issuer guard so concurrent misses serialise onto one - // fetch. Clone the Arc out and drop the DashMap entry lock before the - // await, so no shard lock is held across the network I/O. - let entry = self.refresh_guards.entry(issuer.to_string()).or_default(); - let guard = entry.value().clone(); - drop(entry); - let mut last_fetch = guard.lock().await; - - // A prior holder of the guard may have populated our kid while we waited. - if let Some(key) = self.cache.get(&cache_key) { - return Some(key.clone()); - } - - // Inside the refresh window the last fetch's key set still stands, so a - // miss here means the kid is genuinely absent: reject without touching - // the network. One per-issuer timestamp both negative-caches unknown kids - // and rate-limits outbound fetches, with no attacker-keyed state. - if let Some(fetched_at) = *last_fetch - && fetched_at.elapsed() < JWKS_REFRESH_MIN_INTERVAL - { - return None; - } - - // Stale or first contact: fetch. Record the attempt up front so a failing - // issuer is rate-limited too, not re-hit on every miss. - *last_fetch = Some(Instant::now()); - if self.refresh_keys(issuer, jwks_url).await.is_err() { - return None; - } - - self.cache.get(&cache_key).map(|entry| entry.clone()) - } - - async fn refresh_keys(&self, issuer: &str, jwks_url: &str) -> Result<(), IggyError> { - // The cyper client is `!Send` since 0.9; callers reached from axum - // middleware wrap this future in `SendWrapper` (see - // `http::jwt::middleware::jwt_auth`), so it's free to await cyper - // directly here. - let client = get_http_client(); - let request = client - .get(jwks_url) - .map_err(|e| IggyError::CannotFetchJwks(format!("Failed to build request: {}", e)))? - .build(); - let response = client - .execute(request) - .await - .map_err(|e| IggyError::CannotFetchJwks(format!("HTTP request failed: {}", e)))?; - - let body = response.text().await.map_err(|e| { - IggyError::CannotFetchJwks(format!("Failed to read response body: {}", e)) - })?; - - let jwks: JwkSet = serde_json::from_str(&body) - .map_err(|e| IggyError::CannotFetchJwks(format!("Failed to parse JWKS: {}", e)))?; - - // Collect all current kids from the JWKS response - let current_kids: std::collections::HashSet = - jwks.keys.iter().filter_map(|key| key.kid.clone()).collect(); - - // Remove cached keys for this issuer that are no longer in the JWKS response - // Security fix: Clean up revoked/rotated keys to prevent accepting tokens signed with old keys - let keys_to_remove: Vec = self - .cache - .iter() - .filter(|entry| { - entry.key().issuer == issuer && !current_kids.contains(&entry.key().kid) - }) - .map(|entry| entry.key().clone()) - .collect(); - - for key in keys_to_remove { - self.cache.remove(&key); - } - - for key in jwks.keys { - if let Some(kid) = key.kid { - let decoding_key: DecodingKey = match key.kty { - JwkKeyType::Rsa => { - if let (Some(n), Some(e)) = (key.n.as_deref(), key.e.as_deref()) { - DecodingKey::from_rsa_components(n, e).map_err(|e| { - IggyError::CannotFetchJwks(format!("Invalid RSA key: {}", e)) - })? - } else { - continue; - } - } - JwkKeyType::Ec => { - if let (Some(x), Some(y), Some(crv_str)) = - (key.x.as_deref(), key.y.as_deref(), key.crv.as_deref()) - { - if let Ok(_curve) = crv_str.parse::() { - DecodingKey::from_ec_components(x, y).map_err(|e| { - IggyError::CannotFetchJwks(format!("Invalid EC key: {}", e)) - })? - } else { - continue; - } - } else { - continue; - } - } - }; - - let cache_key = CacheKey { - issuer: issuer.to_string(), - kid, - }; - self.cache.insert(cache_key, decoding_key); - } - } - - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use jsonwebtoken::DecodingKey; - - const TEST_ISSUER: &str = "https://test-issuer.com"; - const TEST_KID: &str = "test-key"; - - fn create_test_decoding_key() -> DecodingKey { - // Use HMAC secret to create a simple test DecodingKey - // Note: This is only for testing cache logic, not a real RSA/EC key - DecodingKey::from_secret(b"test-secret-key-for-cache-testing-only") - } - - #[test] - fn test_cache_key_equality() { - let key1 = CacheKey { - issuer: TEST_ISSUER.to_string(), - kid: TEST_KID.to_string(), - }; - let key2 = CacheKey { - issuer: TEST_ISSUER.to_string(), - kid: TEST_KID.to_string(), - }; - assert_eq!(key1, key2); - } - - #[test] - fn test_cache_key_different_issuer() { - let key1 = CacheKey { - issuer: "issuer1".to_string(), - kid: TEST_KID.to_string(), - }; - let key2 = CacheKey { - issuer: "issuer2".to_string(), - kid: TEST_KID.to_string(), - }; - assert_ne!(key1, key2); - } - - #[test] - fn test_cache_key_different_kid() { - let key1 = CacheKey { - issuer: TEST_ISSUER.to_string(), - kid: "kid1".to_string(), - }; - let key2 = CacheKey { - issuer: TEST_ISSUER.to_string(), - kid: "kid2".to_string(), - }; - assert_ne!(key1, key2); - } - - #[test] - fn test_jwks_client_default() { - let client = JwksClient::default(); - assert!(client.cache.is_empty()); - } - - #[test] - fn test_cache_insert_and_get() { - let client = JwksClient::default(); - let cache_key = CacheKey { - issuer: TEST_ISSUER.to_string(), - kid: TEST_KID.to_string(), - }; - let decoding_key = create_test_decoding_key(); - - client.cache.insert(cache_key.clone(), decoding_key.clone()); - - let cached = client.cache.get(&cache_key); - assert!(cached.is_some()); - } - - #[test] - fn test_cache_multiple_keys() { - let client = JwksClient::default(); - - let key1 = CacheKey { - issuer: "issuer1".to_string(), - kid: "kid1".to_string(), - }; - let key2 = CacheKey { - issuer: "issuer2".to_string(), - kid: "kid2".to_string(), - }; - - let decoding_key1 = create_test_decoding_key(); - let decoding_key2 = create_test_decoding_key(); - - client.cache.insert(key1.clone(), decoding_key1); - client.cache.insert(key2.clone(), decoding_key2); - - assert_eq!(client.cache.len(), 2); - assert!(client.cache.get(&key1).is_some()); - assert!(client.cache.get(&key2).is_some()); - } -} diff --git a/core/server/src/http/jwt/jwt_manager.rs b/core/server/src/http/jwt/jwt_manager.rs deleted file mode 100644 index 2c40416be9..0000000000 --- a/core/server/src/http/jwt/jwt_manager.rs +++ /dev/null @@ -1,457 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::configs::http::{HttpJwtConfig, TrustedIssuerConfig}; -use crate::http::jwt::COMPONENT; -use crate::http::jwt::json_web_token::{Audience, GeneratedToken, JwtClaims, RevokedAccessToken}; -use crate::http::jwt::jwks::JwksClient; -use crate::http::jwt::storage::TokenStorage; -use crate::streaming::persistence::persister::PersisterKind; -use ahash::AHashMap; -use err_trail::ErrContext; -use iggy_common::IggyDuration; -use iggy_common::IggyError; -use iggy_common::IggyExpiry; -use iggy_common::IggyTimestamp; -use iggy_common::UserId; -use iggy_common::locking::IggyRwLock; -use iggy_common::locking::IggyRwLockFn; -use jsonwebtoken::{Algorithm, DecodingKey, EncodingKey, Header, TokenData, Validation, encode}; -use std::collections::HashMap; -use std::sync::Arc; -use tracing::{debug, error, info}; - -pub struct IssuerOptions { - pub issuer: String, - pub audience: String, - pub access_token_expiry: IggyExpiry, - pub not_before: IggyDuration, - pub key: EncodingKey, - pub algorithm: Algorithm, -} - -pub struct ValidatorOptions { - pub valid_audiences: Vec, - pub valid_issuers: Vec, - pub clock_skew: IggyDuration, - pub key: DecodingKey, -} - -pub struct JwtManager { - issuer: IssuerOptions, - validator: ValidatorOptions, - tokens_storage: TokenStorage, - revoked_tokens: IggyRwLock>, - validations: AHashMap, - jwks_client: JwksClient, - trusted_issuer: HashMap, -} - -impl JwtManager { - pub fn new( - persister: Arc, - path: &str, - issuer: IssuerOptions, - validator: ValidatorOptions, - ) -> Result { - let validation = JwtManager::create_validation( - issuer.algorithm, - &validator.valid_issuers, - &validator.valid_audiences, - validator.clock_skew, - ); - - Ok(Self { - validations: vec![(issuer.algorithm, validation)].into_iter().collect(), - issuer, - validator, - tokens_storage: TokenStorage::new(persister, path), - revoked_tokens: IggyRwLock::new(AHashMap::new()), - jwks_client: JwksClient::default(), - trusted_issuer: HashMap::new(), - }) - } - - pub fn from_config( - persister: Arc, - path: &str, - config: &HttpJwtConfig, - ) -> Result { - let algorithm = config.get_algorithm()?; - let issuer = IssuerOptions { - issuer: config.issuer.clone(), - audience: config.audience.clone(), - access_token_expiry: config.access_token_expiry, - not_before: config.not_before, - key: config.get_encoding_key().error(|e: &IggyError| { - format!("{COMPONENT} (error: {e}) - failed to get encoding key") - })?, - algorithm, - }; - let validator = ValidatorOptions { - valid_audiences: config.valid_audiences.clone(), - valid_issuers: config.valid_issuers.clone(), - clock_skew: config.clock_skew, - key: config.get_decoding_key().error(|e: &IggyError| { - format!("{COMPONENT} (error: {e}) - failed to get decoding key") - })?, - }; - let mut manager = JwtManager::new(persister, path, issuer, validator)?; - - if let Some(trusted_issuers) = config.trusted_issuers.as_ref() { - for issuer_config in trusted_issuers { - let normalized_issuer = normalize_issuer_url(&issuer_config.issuer); - manager - .trusted_issuer - .insert(normalized_issuer, issuer_config.clone()); - } - } - - Ok(manager) - } - - fn create_validation( - algorithm: Algorithm, - issuers: &[String], - audiences: &[String], - clock_skew: IggyDuration, - ) -> Validation { - let mut validator = Validation::new(algorithm); - validator.set_issuer(issuers); - validator.set_audience(audiences); - validator.leeway = clock_skew.as_secs() as u64; - validator - } - - pub async fn load_revoked_tokens(&self) -> Result<(), IggyError> { - let revoked_tokens = self.tokens_storage.load_all_revoked_access_tokens().await?; - let mut tokens = self.revoked_tokens.write().await; - for token in revoked_tokens { - tokens.insert(token.id, token.expiry); - } - Ok(()) - } - - pub async fn delete_expired_revoked_tokens(&self, now: u64) -> Result<(), IggyError> { - let mut tokens_to_delete = Vec::new(); - let revoked_tokens = self.revoked_tokens.read().await; - for (id, expiry) in revoked_tokens.iter() { - if expiry <= &now { - tokens_to_delete.push(id.to_string()); - } - } - drop(revoked_tokens); - - debug!( - "Found {} expired revoked access tokens to delete.", - tokens_to_delete.len() - ); - if tokens_to_delete.is_empty() { - return Ok(()); - } - - debug!( - "Deleting {} expired revoked access tokens...", - tokens_to_delete.len() - ); - self.tokens_storage - .delete_revoked_access_tokens(&tokens_to_delete) - .await - .error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - failed to delete revoked access tokens, IDs {tokens_to_delete:?}" - ) - })?; - let mut revoked_tokens = self.revoked_tokens.write().await; - for id in tokens_to_delete { - revoked_tokens.remove(&id); - info!("Deleted expired revoked access token with ID: {id}") - } - Ok(()) - } - - pub fn generate(&self, user_id: UserId) -> Result { - let header = Header::new(self.issuer.algorithm); - let now = IggyTimestamp::now().to_secs(); - let iat = now; - let exp = iat - + (match self.issuer.access_token_expiry { - IggyExpiry::NeverExpire => 1_000_000_000, - IggyExpiry::ServerDefault => 0, // This is not a case, as the server default is not allowed here - IggyExpiry::ExpireDuration(duration) => duration.as_secs(), - }) as u64; - let nbf = iat + self.issuer.not_before.as_secs() as u64; - let claims = JwtClaims { - jti: uuid::Uuid::now_v7().to_string(), - sub: user_id.to_string(), - aud: Audience::from(self.issuer.audience.clone()), - iss: self.issuer.issuer.to_string(), - iat, - exp, - nbf, - }; - - let access_token = encode::(&header, &claims, &self.issuer.key); - if let Err(e) = access_token { - error!("Cannot generate JWT token. Error: {e}"); - return Err(IggyError::CannotGenerateJwt); - } - - Ok(GeneratedToken { - user_id, - access_token: access_token.unwrap(), - access_token_expiry: exp, - }) - } - - // The access token can be refreshed only once and if it is not expired - pub async fn refresh_token(&self, token: &str) -> Result { - if token.is_empty() { - return Err(IggyError::InvalidAccessToken); - } - - let token_header = - jsonwebtoken::decode_header(token).map_err(|_| IggyError::InvalidAccessToken)?; - let jwt_claims = self.decode(token, token_header.alg).await?; - - // Security fix: Reject A2A tokens from external trusted issuers - // A2A tokens should not be refreshable - they have their own lifecycle - let normalized_iss = normalize_issuer_url(&jwt_claims.claims.iss); - if self.trusted_issuer.contains_key(&normalized_iss) { - error!( - "Cannot refresh A2A token from external issuer: {}", - jwt_claims.claims.iss - ); - return Err(IggyError::InvalidAccessToken); - } - - let id = jwt_claims.claims.jti; - let expiry = jwt_claims.claims.exp; - if self - .revoked_tokens - .write() - .await - .insert(id.clone(), expiry) - .is_some() - { - return Err(IggyError::InvalidAccessToken); - } - - self.tokens_storage - .save_revoked_access_token(&RevokedAccessToken { - id: id.clone(), - expiry, - }) - .await - .error(|e: &IggyError| { - format!("{COMPONENT} (error: {e}) - failed to save revoked access token: {id}") - })?; - let user_id = jwt_claims - .claims - .sub - .parse::() - .map_err(|_| IggyError::InvalidAccessToken)?; - self.generate(user_id) - } - - pub async fn decode( - &self, - token: &str, - algorithm: Algorithm, - ) -> Result, IggyError> { - let validation = self.validations.get(&algorithm); - let kid = jsonwebtoken::decode_header(token).ok().and_then(|h| h.kid); - - // try to decode using JWKS if it's a trusted issuer - let insecure = match jsonwebtoken::dangerous::insecure_decode::(token) { - Ok(claims) => claims, - Err(_) => { - error!("Failed to decode JWT insecurely"); - return self.decode_with_fallback(token, validation, algorithm); - } - }; - - let normalized_iss = normalize_issuer_url(&insecure.claims.iss); - let config = match self.trusted_issuer.get(&normalized_iss) { - Some(config) => config, - None => { - debug!("No trusted issuer found for: {}", insecure.claims.iss); - return self.decode_with_fallback(token, validation, algorithm); - } - }; - - if config.user_id == 0 { - error!( - "A2A token cannot map to root user (user_id = 0) for issuer: {}", - config.issuer - ); - return Err(IggyError::Unauthenticated); - } - - let kid_str = match kid.as_deref() { - Some(kid) => kid, - None => { - error!("No kid found in JWT header"); - return self.decode_with_fallback(token, validation, algorithm); - } - }; - - let decoding_key = match self - .jwks_client - .get_key(&config.issuer, &config.jwks_url, kid_str) - .await - { - Some(key) => key, - None => { - error!("Failed to get decoding key from JWKS for kid: {}", kid_str); - return self.decode_with_fallback(token, validation, algorithm); - } - }; - let mut validation = Validation::new(algorithm); - validation.set_issuer(std::slice::from_ref(&config.issuer)); - validation.set_audience(std::slice::from_ref(&config.audience)); - - let mut result = jsonwebtoken::decode::(token, &decoding_key, &validation) - .map_err(|e| { - error!("Failed to decode JWT: {}", e); - IggyError::Unauthenticated - })?; - - result.claims.sub = config.user_id.to_string(); - - Ok(result) - } - - /// fallback to standard JWT validation if JWKS validation fails - fn decode_with_fallback( - &self, - token: &str, - validation: Option<&Validation>, - algorithm: Algorithm, - ) -> Result, IggyError> { - let validation = validation.ok_or_else(|| { - IggyError::InvalidJwtAlgorithm(Self::map_algorithm_to_string(algorithm)) - })?; - - jsonwebtoken::decode::(token, &self.validator.key, validation) - .map_err(|_| IggyError::Unauthenticated) - } - - fn map_algorithm_to_string(algorithm: Algorithm) -> String { - match algorithm { - Algorithm::HS256 => "HS256", - Algorithm::HS384 => "HS384", - Algorithm::HS512 => "HS512", - Algorithm::RS256 => "RS256", - Algorithm::RS384 => "RS384", - Algorithm::RS512 => "RS512", - _ => "Unknown", - } - .to_string() - } - - pub async fn revoke_token(&self, token_id: &str, expiry: u64) -> Result<(), IggyError> { - let mut revoked_tokens = self.revoked_tokens.write().await; - revoked_tokens.insert(token_id.to_string(), expiry); - self.tokens_storage - .save_revoked_access_token(&RevokedAccessToken { - id: token_id.to_string(), - expiry, - }) - .await - .error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - failed to save revoked access token: {token_id}" - ) - })?; - info!("Revoked access token with ID: {token_id}"); - Ok(()) - } - - pub async fn is_token_revoked(&self, token_id: &str) -> bool { - let revoked_tokens = self.revoked_tokens.read().await; - revoked_tokens.contains_key(token_id) - } -} - -/// Normalize issuer URL by lowercasing scheme and host, preserving path case -/// -/// Example: "HTTPS://Example.COM/PATH" -> "https://example.com/PATH" -fn normalize_issuer_url(url: &str) -> String { - match url.split_once("://") { - Some((scheme, rest)) => { - let scheme = scheme.to_lowercase(); - // Find end of host (first '/' or end of string) - let (host, path) = match rest.find('/') { - Some(idx) => rest.split_at(idx), - None => (rest, ""), - }; - format!("{}://{}{}", scheme, host.to_lowercase(), path) - } - None => url.trim_end_matches('/').to_lowercase(), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_normalize_issuer_url_basic() { - assert_eq!( - normalize_issuer_url("HTTPS://Example.COM/PATH"), - "https://example.com/PATH" - ); - } - - #[test] - fn test_normalize_issuer_url_no_path() { - assert_eq!( - normalize_issuer_url("HTTPS://Example.COM"), - "https://example.com" - ); - } - - #[test] - fn test_normalize_issuer_url_no_scheme() { - assert_eq!(normalize_issuer_url("Example.COM"), "example.com"); - } - - #[test] - fn test_normalize_issuer_url_trailing_slash() { - assert_eq!( - normalize_issuer_url("HTTPS://Example.COM/"), - "https://example.com/" - ); - } - - #[test] - fn test_normalize_issuer_url_preserves_path_case() { - assert_eq!( - normalize_issuer_url("https://EXAMPLE.com/MyPath/SubPath"), - "https://example.com/MyPath/SubPath" - ); - } - - #[test] - fn test_normalize_issuer_url_already_normalized() { - assert_eq!( - normalize_issuer_url("https://example.com/path"), - "https://example.com/path" - ); - } -} diff --git a/core/server/src/http/jwt/middleware.rs b/core/server/src/http/jwt/middleware.rs deleted file mode 100644 index 0ec31ccdad..0000000000 --- a/core/server/src/http/jwt/middleware.rs +++ /dev/null @@ -1,105 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::http::jwt::json_web_token::Identity; -use crate::http::shared::{AppState, RequestDetails}; -use axum::body::Body; -use axum::{ - extract::State, - http::{Request, StatusCode}, - middleware::Next, - response::Response, -}; -use err_trail::ErrContext; -use send_wrapper::SendWrapper; -use std::sync::Arc; - -const COMPONENT: &str = "JWT_MIDDLEWARE"; -const AUTHORIZATION: &str = "authorization"; -const BEARER: &str = "Bearer "; -const UNAUTHORIZED: StatusCode = StatusCode::UNAUTHORIZED; - -const PUBLIC_PATHS: &[&str] = &[ - "/", - "/metrics", - "/ping", - "/users/login", - "/users/refresh-token", - "/personal-access-tokens/login", - "/ui", -]; - -pub async fn jwt_auth( - State(state): State>, - mut request: Request, - next: Next, -) -> Result { - if PUBLIC_PATHS.contains(&request.uri().path()) { - return Ok(next.run(request).await); - } - - let bearer = request - .headers() - .get(AUTHORIZATION) - .ok_or(UNAUTHORIZED) - .error(|e: &StatusCode| { - format!("{COMPONENT} (error: {e}) - missing or inaccessible Authorization header") - })? - .to_str() - .error(|e: &axum::http::header::ToStrError| { - format!("{COMPONENT} (error: {e}) - invalid authorization header format") - }) - .map_err(|_| UNAUTHORIZED)?; - - if !bearer.starts_with(BEARER) { - return Err(StatusCode::UNAUTHORIZED); - } - - let jwt_token = &bearer[BEARER.len()..]; - let token_header = jsonwebtoken::decode_header(jwt_token) - .error(|e: &jsonwebtoken::errors::Error| { - format!("{COMPONENT} (error: {e}) - failed to decode JWT header") - }) - .map_err(|_| UNAUTHORIZED)?; - // `decode` may fetch JWKS via the cyper client, which is `!Send` since - // cyper 0.9 (`Rc`-backed). axum requires this middleware's future to be - // `Send`, so wrap the `!Send` sub-futures in `SendWrapper` -- the same - // pattern the rest of the HTTP layer uses for compio shard ops. - // Sound under compio's thread-per-core model: the future is never - // polled from another thread. - let jwt_claims = SendWrapper::new(state.jwt_manager.decode(jwt_token, token_header.alg)) - .await - .map_err(|_| UNAUTHORIZED)?; - if SendWrapper::new(state.jwt_manager.is_token_revoked(&jwt_claims.claims.jti)).await { - return Err(StatusCode::UNAUTHORIZED); - } - - let request_details = request.extensions().get::().unwrap(); - let user_id = jwt_claims - .claims - .sub - .parse::() - .map_err(|_| UNAUTHORIZED)?; - let identity = Identity { - token_id: jwt_claims.claims.jti, - token_expiry: jwt_claims.claims.exp, - user_id, - ip_address: request_details.ip_address, - }; - request.extensions_mut().insert(identity); - Ok(next.run(request).await) -} diff --git a/core/server/src/http/jwt/mod.rs b/core/server/src/http/jwt/mod.rs deleted file mode 100644 index e63d2165fd..0000000000 --- a/core/server/src/http/jwt/mod.rs +++ /dev/null @@ -1,24 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod json_web_token; -pub mod jwks; -pub mod jwt_manager; -pub mod middleware; -pub mod storage; - -pub const COMPONENT: &str = "HTTP_JWT"; diff --git a/core/server/src/http/jwt/storage.rs b/core/server/src/http/jwt/storage.rs deleted file mode 100644 index c56c7509a3..0000000000 --- a/core/server/src/http/jwt/storage.rs +++ /dev/null @@ -1,149 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::http::jwt::COMPONENT; -use crate::{ - http::jwt::json_web_token::RevokedAccessToken, streaming::persistence::persister::PersisterKind, -}; -use ahash::AHashMap; -use anyhow::Context; -use err_trail::ErrContext; -use iggy_common::IggyError; -use std::sync::Arc; -use tracing::{error, info}; - -#[derive(Debug)] -pub struct TokenStorage { - persister: Arc, - path: String, -} - -impl TokenStorage { - pub fn new(persister: Arc, path: &str) -> Self { - Self { - persister, - path: path.to_owned(), - } - } - - pub async fn load_all_revoked_access_tokens( - &self, - ) -> Result, IggyError> { - // Check if file exists by trying to get metadata (equivalent to original file open check) - let file_size = match compio::fs::metadata(&self.path).await { - Err(_) => { - info!("No revoked access tokens found to load."); - return Ok(vec![]); - } - Ok(metadata) => metadata.len() as usize, - }; - - info!("Loading revoked access tokens from: {}", self.path); - - let buffer = compio::fs::read(&self.path) - .await - .error(|e: &std::io::Error| { - format!( - "{COMPONENT} (error: {e}) - failed to read file into buffer, path: {}", - self.path - ) - }) - .map_err(|e| { - error!("Cannot open revoked access tokens file: {e}"); - IggyError::CannotReadFile - })?; - - if buffer.len() != file_size { - error!( - "File size mismatch: expected {file_size}, got {}", - buffer.len() - ); - return Err(IggyError::CannotReadFile); - } - - let tokens: AHashMap = rmp_serde::from_slice(&buffer) - .with_context(|| "Failed to deserialize revoked access tokens") - .map_err(|_| IggyError::CannotDeserializeResource)?; - - let tokens = tokens - .into_iter() - .map(|(id, expiry)| RevokedAccessToken { id, expiry }) - .collect::>(); - - info!("Loaded {} revoked access tokens", tokens.len()); - Ok(tokens) - } - - pub async fn save_revoked_access_token( - &self, - token: &RevokedAccessToken, - ) -> Result<(), IggyError> { - let tokens = self.load_all_revoked_access_tokens().await?; - let mut map = tokens - .into_iter() - .map(|token| (token.id, token.expiry)) - .collect::>(); - map.insert(token.id.to_owned(), token.expiry); - let bytes = rmp_serde::to_vec(&map) - .with_context(|| "Failed to serialize revoked access tokens") - .map_err(|_| IggyError::CannotSerializeResource)?; - self.persister - .overwrite(&self.path, bytes) - .await - .error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - failed to overwrite file, path: {}", - self.path - ) - })?; - Ok(()) - } - - pub async fn delete_revoked_access_tokens(&self, id: &[String]) -> Result<(), IggyError> { - let tokens = self - .load_all_revoked_access_tokens() - .await - .error(|e: &IggyError| { - format!("{COMPONENT} (error: {e}) - failed to load revoked access tokens") - })?; - if tokens.is_empty() { - return Ok(()); - } - - let mut map = tokens - .into_iter() - .map(|token| (token.id, token.expiry)) - .collect::>(); - for id in id { - map.remove(id); - } - - let bytes = rmp_serde::to_vec(&map) - .with_context(|| "Failed to serialize revoked access tokens") - .map_err(|_| IggyError::CannotSerializeResource)?; - self.persister - .overwrite(&self.path, bytes) - .await - .error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - failed to overwrite file, path: {}", - self.path - ) - })?; - Ok(()) - } -} diff --git a/core/server/src/http/mapper.rs b/core/server/src/http/mapper.rs deleted file mode 100644 index eaf031a6a8..0000000000 --- a/core/server/src/http/mapper.rs +++ /dev/null @@ -1,329 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::http::jwt::json_web_token::GeneratedToken; -use crate::metadata::{ConsumerGroupMeta, InnerMetadata, PartitionMeta, StreamMeta, TopicMeta}; -use crate::streaming::clients::client_manager::Client; -use crate::streaming::users::user::User; -use iggy_common::PersonalAccessToken; -use iggy_common::{ConsumerGroupDetails, ConsumerGroupInfo, ConsumerGroupMember, IggyByteSize}; -use iggy_common::{IdentityInfo, PersonalAccessTokenInfo, TokenInfo, TopicDetails}; -use iggy_common::{UserInfo, UserInfoDetails}; - -pub fn map_user(user: &User) -> UserInfoDetails { - UserInfoDetails { - id: user.id, - username: user.username.clone(), - created_at: user.created_at, - status: user.status, - permissions: user.permissions.clone(), - } -} - -pub fn map_users(users: &[&User]) -> Vec { - let mut users_data = Vec::with_capacity(users.len()); - for user in users { - let user = UserInfo { - id: user.id, - username: user.username.clone(), - created_at: user.created_at, - status: user.status, - }; - users_data.push(user); - } - users_data.sort_by_key(|u| u.id); - users_data -} - -pub fn map_personal_access_tokens( - personal_access_tokens: &[PersonalAccessToken], -) -> Vec { - let mut personal_access_tokens_data = Vec::with_capacity(personal_access_tokens.len()); - for personal_access_token in personal_access_tokens { - let personal_access_token = PersonalAccessTokenInfo { - name: (*personal_access_token.name).to_owned(), - expiry_at: personal_access_token.expiry_at, - }; - personal_access_tokens_data.push(personal_access_token); - } - personal_access_tokens_data.sort_by(|a, b| a.name.cmp(&b.name)); - personal_access_tokens_data -} - -pub fn map_client(client: &Client) -> iggy_common::ClientInfoDetails { - iggy_common::ClientInfoDetails { - client_id: client.session.client_id, - user_id: client.user_id, - transport: client.transport.to_string(), - address: client.session.ip_address.to_string(), - consumer_groups_count: client.consumer_groups.len() as u32, - consumer_groups: client - .consumer_groups - .iter() - .map(|consumer_group| ConsumerGroupInfo { - stream_id: consumer_group.stream_id, - topic_id: consumer_group.topic_id, - group_id: consumer_group.group_id, - }) - .collect(), - } -} - -pub fn map_clients(clients: &[Client]) -> Vec { - let mut all_clients = Vec::new(); - for client in clients { - let client = iggy_common::ClientInfo { - client_id: client.session.client_id, - user_id: client.user_id, - transport: client.transport.to_string(), - address: client.session.ip_address.to_string(), - consumer_groups_count: client.consumer_groups.len() as u32, - }; - all_clients.push(client); - } - - all_clients.sort_by_key(|c| c.client_id); - all_clients -} - -pub fn map_generated_access_token_to_identity_info(token: GeneratedToken) -> IdentityInfo { - IdentityInfo { - user_id: token.user_id, - access_token: Some(TokenInfo { - token: token.access_token, - expiry: token.access_token_expiry, - }), - } -} - -/// Map a stream from SharedMetadata to StreamDetails (with topics) -pub fn map_stream_details_from_metadata(stream_meta: &StreamMeta) -> iggy_common::StreamDetails { - // Get topic IDs sorted - let mut topic_ids: Vec<_> = stream_meta.topics.iter().map(|(k, _)| k).collect(); - topic_ids.sort_unstable(); - - // Map topics - let mut topics = Vec::with_capacity(topic_ids.len()); - for topic_id in topic_ids { - if let Some(topic_meta) = stream_meta.topics.get(topic_id) { - topics.push(map_topic_from_metadata(topic_meta)); - } - } - - // Aggregate stats - let (total_size, total_messages) = aggregate_stream_stats(stream_meta); - - iggy_common::StreamDetails { - id: stream_meta.id as u32, - created_at: stream_meta.created_at, - name: stream_meta.name.to_string(), - topics_count: topics.len() as u32, - size: IggyByteSize::from(total_size), - messages_count: total_messages, - topics, - } -} - -/// Map a stream from SharedMetadata to Stream (without topics) -pub fn map_stream_from_metadata(stream_meta: &StreamMeta) -> iggy_common::Stream { - let (total_size, total_messages) = aggregate_stream_stats(stream_meta); - - iggy_common::Stream { - id: stream_meta.id as u32, - created_at: stream_meta.created_at, - name: stream_meta.name.to_string(), - topics_count: stream_meta.topics.len() as u32, - size: IggyByteSize::from(total_size), - messages_count: total_messages, - } -} - -/// Map all streams from SharedMetadata -pub fn map_streams_from_metadata(metadata: &InnerMetadata) -> Vec { - let mut stream_ids: Vec<_> = metadata.streams.iter().map(|(k, _)| k).collect(); - stream_ids.sort_unstable(); - - let mut streams = Vec::with_capacity(stream_ids.len()); - for stream_id in stream_ids { - if let Some(stream_meta) = metadata.streams.get(stream_id) { - streams.push(map_stream_from_metadata(stream_meta)); - } - } - streams -} - -/// Map a topic from SharedMetadata to Topic (without partitions) -pub fn map_topic_from_metadata(topic_meta: &TopicMeta) -> iggy_common::Topic { - let (total_size, total_messages) = aggregate_topic_stats(topic_meta); - - iggy_common::Topic { - id: topic_meta.id as u32, - created_at: topic_meta.created_at, - name: topic_meta.name.to_string(), - size: IggyByteSize::from(total_size), - partitions_count: topic_meta.partitions.len() as u32, - messages_count: total_messages, - message_expiry: topic_meta.message_expiry, - compression_algorithm: topic_meta.compression_algorithm, - max_topic_size: topic_meta.max_topic_size, - replication_factor: topic_meta.replication_factor, - } -} - -/// Map all topics for a stream from SharedMetadata -pub fn map_topics_from_metadata(stream_meta: &StreamMeta) -> Vec { - let mut topic_ids: Vec<_> = stream_meta.topics.iter().map(|(k, _)| k).collect(); - topic_ids.sort_unstable(); - - let mut topics = Vec::with_capacity(topic_ids.len()); - for topic_id in topic_ids { - if let Some(topic_meta) = stream_meta.topics.get(topic_id) { - topics.push(map_topic_from_metadata(topic_meta)); - } - } - topics -} - -/// Map a topic from SharedMetadata to TopicDetails (with partitions) -pub fn map_topic_details_from_metadata(topic_meta: &TopicMeta) -> TopicDetails { - // Get partition IDs sorted - let mut partition_ids: Vec<_> = topic_meta - .partitions - .iter() - .enumerate() - .map(|(k, _)| k) - .collect(); - partition_ids.sort_unstable(); - - // Map partitions - let mut partitions = Vec::with_capacity(partition_ids.len()); - for partition_id in partition_ids { - if let Some(partition_meta) = topic_meta.partitions.get(partition_id) { - partitions.push(map_partition_from_metadata(partition_meta)); - } - } - - // Aggregate stats - let (total_size, total_messages) = aggregate_topic_stats(topic_meta); - - TopicDetails { - id: topic_meta.id as u32, - created_at: topic_meta.created_at, - name: topic_meta.name.to_string(), - size: IggyByteSize::from(total_size), - messages_count: total_messages, - partitions_count: partitions.len() as u32, - partitions, - message_expiry: topic_meta.message_expiry, - compression_algorithm: topic_meta.compression_algorithm, - max_topic_size: topic_meta.max_topic_size, - replication_factor: topic_meta.replication_factor, - } -} - -/// Map a partition from SharedMetadata -pub fn map_partition_from_metadata(partition_meta: &PartitionMeta) -> iggy_common::Partition { - let stats = &partition_meta.stats; - let segments_count = stats.segments_count_inconsistent(); - let size_bytes = stats.size_bytes_inconsistent(); - let messages_count = stats.messages_count_inconsistent(); - let current_offset = stats.current_offset(); - - iggy_common::Partition { - id: partition_meta.id as u32, - created_at: partition_meta.created_at, - segments_count, - current_offset, - size: IggyByteSize::from(size_bytes), - messages_count, - } -} - -/// Map a consumer group from SharedMetadata -pub fn map_consumer_group_from_metadata(cg_meta: &ConsumerGroupMeta) -> iggy_common::ConsumerGroup { - iggy_common::ConsumerGroup { - id: cg_meta.id as u32, - name: cg_meta.name.to_string(), - partitions_count: cg_meta.partitions.len() as u32, - members_count: cg_meta.members.len() as u32, - } -} - -/// Map a consumer group to ConsumerGroupDetails from SharedMetadata -pub fn map_consumer_group_details_from_metadata( - cg_meta: &ConsumerGroupMeta, -) -> ConsumerGroupDetails { - let members: Vec = cg_meta - .members - .iter() - .map(|(_, member)| ConsumerGroupMember { - id: member.id as u32, - partitions_count: member.partitions.len() as u32, - partitions: member.partitions.iter().map(|&p| p as u32).collect(), - }) - .collect(); - - ConsumerGroupDetails { - id: cg_meta.id as u32, - name: cg_meta.name.to_string(), - partitions_count: cg_meta.partitions.len() as u32, - members_count: members.len() as u32, - members, - } -} - -/// Map all consumer groups for a topic from SharedMetadata -pub fn map_consumer_groups_from_metadata( - topic_meta: &TopicMeta, -) -> Vec { - let mut group_ids: Vec<_> = topic_meta.consumer_groups.iter().map(|(k, _)| k).collect(); - group_ids.sort_unstable(); - - let mut groups = Vec::with_capacity(group_ids.len()); - for group_id in group_ids { - if let Some(cg_meta) = topic_meta.consumer_groups.get(group_id) { - groups.push(map_consumer_group_from_metadata(cg_meta)); - } - } - groups -} - -fn aggregate_stream_stats(stream_meta: &StreamMeta) -> (u64, u64) { - let mut total_size = 0u64; - let mut total_messages = 0u64; - - for (_, topic_meta) in stream_meta.topics.iter() { - for partition_meta in topic_meta.partitions.iter() { - total_size += partition_meta.stats.size_bytes_inconsistent(); - total_messages += partition_meta.stats.messages_count_inconsistent(); - } - } - - (total_size, total_messages) -} - -fn aggregate_topic_stats(topic_meta: &TopicMeta) -> (u64, u64) { - let mut total_size = 0u64; - let mut total_messages = 0u64; - - for partition_meta in topic_meta.partitions.iter() { - total_size += partition_meta.stats.size_bytes_inconsistent(); - total_messages += partition_meta.stats.messages_count_inconsistent(); - } - - (total_size, total_messages) -} diff --git a/core/server/src/http/messages.rs b/core/server/src/http/messages.rs deleted file mode 100644 index b51df4aef1..0000000000 --- a/core/server/src/http/messages.rs +++ /dev/null @@ -1,164 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::http::COMPONENT; -use crate::http::error::CustomError; -use crate::http::jwt::json_web_token::Identity; -use crate::http::shared::AppState; -use crate::shard::system::messages::PollingArgs; -use crate::shard::transmission::message::ResolvedPartition; -use crate::streaming::segments::IggyMessagesBatchMut; -use crate::streaming::session::Session; -use axum::extract::{Path, Query, State}; -use axum::http::StatusCode; -use axum::routing::get; -use axum::{Extension, Json, Router, debug_handler}; -use err_trail::ErrContext; -use iggy_common::Identifier; -use iggy_common::Validatable; -use iggy_common::{Consumer, PollMessages, SendMessages}; -use iggy_common::{IggyError, IggyMessagesBatch, PolledMessages}; -use send_wrapper::SendWrapper; -use server_common::{IggyIndexesMut, PooledBuffer}; -use std::sync::Arc; -use tracing::instrument; - -pub fn router(state: Arc) -> Router { - Router::new() - .route( - "/streams/{stream_id}/topics/{topic_id}/messages", - get(poll_messages).post(send_messages), - ) - .route( - "/streams/{stream_id}/topics/{topic_id}/messages/flush/{partition_id}/{fsync}", - get(flush_unsaved_buffer), - ) - .with_state(state) -} - -#[debug_handler] -async fn poll_messages( - State(state): State>, - Extension(identity): Extension, - Path((stream_id, topic_id)): Path<(String, String)>, - mut query: Query, -) -> Result, CustomError> { - query.stream_id = Identifier::from_str_value(&stream_id)?; - query.topic_id = Identifier::from_str_value(&topic_id)?; - query.validate()?; - - let consumer = Consumer::new(query.0.consumer.id); - - let session = Session::stateless(identity.user_id, identity.ip_address); - - let poll_future = SendWrapper::new(state.shard.poll_messages( - session.client_id, - session.get_user_id(), - query.0.stream_id, - query.0.topic_id, - consumer, - query.0.partition_id, - PollingArgs::new(query.0.strategy, query.0.count, query.0.auto_commit), - )); - - let (metadata, messages) = poll_future - .await - .error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - failed to poll messages, stream ID: {}, topic ID: {}, partition ID: {:?}", - stream_id, topic_id, query.0.partition_id - ) - })?; - let polled_messages = messages.into_polled_messages(metadata); - - Ok(Json(polled_messages)) -} - -#[debug_handler] -async fn send_messages( - State(state): State>, - Extension(identity): Extension, - Path((stream_id, topic_id)): Path<(String, String)>, - Json(mut command): Json, -) -> Result { - command.stream_id = Identifier::from_str_value(&stream_id)?; - command.topic_id = Identifier::from_str_value(&topic_id)?; - command.partitioning.length = command.partitioning.value.len() as u8; - command.validate()?; - - let batch = make_mutable(command.batch); - let command_stream_id = command.stream_id; - let command_topic_id = command.topic_id; - let partitioning = command.partitioning; - - let append_future = SendWrapper::new(state.shard.append_messages( - identity.user_id, - command_stream_id, - command_topic_id, - &partitioning, - batch, - )); - - append_future - .await - .error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - failed to append messages, stream ID: {stream_id}, topic ID: {topic_id}" - ) - })?; - - Ok(StatusCode::CREATED) -} - -#[debug_handler] -#[instrument(skip_all, name = "trace_flush_unsaved_buffer", fields(iggy_user_id = identity.user_id, iggy_stream_id = stream_id, iggy_topic_id = topic_id, iggy_partition_id = partition_id, iggy_fsync = fsync))] -async fn flush_unsaved_buffer( - State(state): State>, - Extension(identity): Extension, - Path((stream_id, topic_id, partition_id, fsync)): Path<(String, String, u32, bool)>, -) -> Result { - let stream_id_ident = Identifier::from_str_value(&stream_id)?; - let topic_id_ident = Identifier::from_str_value(&topic_id)?; - let partition_id = partition_id as usize; - - let shard = state.shard.shard(); - let topic = shard.resolve_topic(&stream_id_ident, &topic_id_ident)?; - let partition = ResolvedPartition { - stream_id: topic.stream_id, - topic_id: topic.topic_id, - partition_id, - }; - - let flush_future = - SendWrapper::new(shard.flush_unsaved_buffer(identity.user_id, partition, fsync)); - flush_future.await?; - Ok(StatusCode::OK) -} - -fn make_mutable(batch: IggyMessagesBatch) -> IggyMessagesBatchMut { - let (_, indexes, messages) = batch.decompose(); - let (base_position, indexes_buffer) = indexes.decompose(); - - let mut indexes_buffer_mut = PooledBuffer::with_capacity(indexes_buffer.len()); - indexes_buffer_mut.extend_from_slice(&indexes_buffer); - let indexes_mut = IggyIndexesMut::from_bytes(indexes_buffer_mut, base_position); - - let mut messages_buffer_mut = PooledBuffer::with_capacity(messages.len()); - messages_buffer_mut.extend_from_slice(&messages); - - IggyMessagesBatchMut::from_indexes_and_messages(indexes_mut, messages_buffer_mut) -} diff --git a/core/server/src/http/metrics.rs b/core/server/src/http/metrics.rs index 2866e5c09a..bb18a8a9b9 100644 --- a/core/server/src/http/metrics.rs +++ b/core/server/src/http/metrics.rs @@ -15,21 +15,283 @@ // specific language governing permissions and limitations // under the License. -use crate::http::shared::AppState; -use axum::body::Body; -use axum::{ - extract::State, - http::{Request, StatusCode}, - middleware::Next, - response::Response, -}; -use std::sync::Arc; - -pub async fn metrics( - State(state): State>, - request: Request, - next: Next, -) -> Result { - state.shard.shard().metrics.increment_http_requests(); - Ok(next.run(request).await) +//! The `[http.metrics]` scrape surface: the legacy-parity metric registry +//! (entity gauges plus the request counter), its public scrape handler, and +//! the config gate deciding whether the route is mounted. + +use axum::extract::State; +use configs::http::HttpMetricsConfig; +use consensus::MetadataHandle; +use iggy_common::IggyError; +use metadata::impls::metadata::StreamsFrontend; +use prometheus_client::encoding::text::encode; +use prometheus_client::metrics::counter::Counter; +use prometheus_client::metrics::gauge::Gauge; +use prometheus_client::registry::Registry; +use send_wrapper::SendWrapper; +use tracing::error; + +use crate::http::state::HttpState; + +/// The legacy server's metric set, registered under the same names and help +/// texts so existing dashboards and alerts keep working unchanged. +/// +/// Unlike the legacy server, the entity gauges are not counted at mutation +/// sites: [`get_metrics`] samples the live state on every scrape, so a gauge +/// can never drift from the state it describes. +pub(in crate::http) struct HttpMetrics { + registry: Registry, + http_requests: Counter, + streams: Gauge, + topics: Gauge, + partitions: Gauge, + segments: Gauge, + messages: Gauge, + users: Gauge, + clients: Gauge, +} + +impl HttpMetrics { + pub(in crate::http) fn init() -> Self { + let mut registry = Registry::default(); + let http_requests = Counter::default(); + let streams = Gauge::default(); + let topics = Gauge::default(); + let partitions = Gauge::default(); + let segments = Gauge::default(); + let messages = Gauge::default(); + let users = Gauge::default(); + let clients = Gauge::default(); + registry.register( + "http_requests", + "total count of http_requests", + http_requests.clone(), + ); + registry.register("streams", "total count of streams", streams.clone()); + registry.register("topics", "total count of topics", topics.clone()); + registry.register( + "partitions", + "total count of partitions", + partitions.clone(), + ); + registry.register("segments", "total count of segments", segments.clone()); + registry.register("messages", "total count of messages", messages.clone()); + registry.register("users", "total count of users", users.clone()); + registry.register("clients", "total count of clients", clients.clone()); + Self { + registry, + http_requests, + streams, + topics, + partitions, + segments, + messages, + users, + clients, + } + } + + /// Handle for the router's request-counting layer. The counter is + /// `Arc`-backed, so bumping the clone bumps the registered metric. + pub(in crate::http) fn request_counter(&self) -> Counter { + self.http_requests.clone() + } + + fn formatted_output(&self) -> String { + let mut buffer = String::new(); + if let Err(error) = encode(&mut buffer, &self.registry) { + error!(%error, "failed to encode metrics"); + } + buffer + } +} + +/// Resolve the configured scrape path: `None` when `[http.metrics]` is +/// disabled, so the route is never mounted and the endpoint answers 404. +/// +/// axum's `Router::route` panics on a path without a leading `/`, so an +/// enabled endpoint missing one is rejected as a configuration error before +/// the router is assembled. +/// +/// # Errors +/// +/// Returns [`IggyError::InvalidConfiguration`] when metrics are enabled and +/// the endpoint does not start with `/`. +pub(in crate::http) fn validated_endpoint( + config: &HttpMetricsConfig, +) -> Result, IggyError> { + if !config.enabled { + return Ok(None); + } + if !config.endpoint.starts_with('/') { + error!( + endpoint = %config.endpoint, + "invalid http.metrics.endpoint: the path must start with '/'" + ); + return Err(IggyError::InvalidConfiguration); + } + Ok(Some(config.endpoint.clone())) +} + +/// `GET `: the metric set in prometheus text +/// exposition. Public - reached without proving a credential, exactly like the +/// legacy endpoint. +/// +/// The entity gauges sample the same reads `/stats` serves: the metadata STM +/// stream and user maps plus the stats-registry rollups, whose partition-plane +/// increments are relaxed, so scraped values are approximate while writes are +/// in flight. The clients count scatter-gathers the per-shard session managers +/// exactly like `GET /clients` and turns partial when a shard misses the reply +/// deadline. +pub(in crate::http) async fn get_metrics(State(state): State) -> String { + let (streams_count, topics_count, partitions_count, segments_count, messages_count) = state + .shard + .plane + .metadata() + .mux_stm + .streams() + .read(|streams| { + let mut topics_count = 0u64; + let mut partitions_count = 0u64; + let mut segments_count = 0u64; + let mut messages_count = 0u64; + for (_, stream) in &streams.items { + topics_count = topics_count.saturating_add(stream.topics.len() as u64); + segments_count = segments_count + .saturating_add(u64::from(stream.stats.segments_count_inconsistent())); + messages_count = + messages_count.saturating_add(stream.stats.messages_count_inconsistent()); + for (_, topic) in &stream.topics { + partitions_count = + partitions_count.saturating_add(topic.partitions.len() as u64); + } + } + ( + streams.items.len() as u64, + topics_count, + partitions_count, + segments_count, + messages_count, + ) + }); + let users_count = state + .shard + .plane + .metadata() + .mux_stm + .users() + .read(|users| users.items.len() as u64); + let clients_count = SendWrapper::new(state.shard.list_all_clients()).await.len() as u64; + + let metrics = &state.metrics; + metrics.streams.set(gauge_value(streams_count)); + metrics.topics.set(gauge_value(topics_count)); + metrics.partitions.set(gauge_value(partitions_count)); + metrics.segments.set(gauge_value(segments_count)); + metrics.messages.set(gauge_value(messages_count)); + metrics.users.set(gauge_value(users_count)); + metrics.clients.set(gauge_value(clients_count)); + metrics.formatted_output() +} + +/// Clamp a count into the gauge's `i64` domain; only `messages` can pass +/// `i64::MAX` even in theory, the rest are bounded far below it. +fn gauge_value(count: u64) -> i64 { + i64::try_from(count).unwrap_or(i64::MAX) +} + +#[cfg(test)] +mod tests { + use super::*; + + const PARITY_METRIC_NAMES: [&str; 8] = [ + "http_requests", + "streams", + "topics", + "partitions", + "segments", + "messages", + "users", + "clients", + ]; + + fn metrics_config(enabled: bool, endpoint: &str) -> HttpMetricsConfig { + HttpMetricsConfig { + enabled, + endpoint: endpoint.to_owned(), + } + } + + #[test] + fn formatted_output_exposes_every_parity_metric() { + let metrics = HttpMetrics::init(); + let output = metrics.formatted_output(); + for name in PARITY_METRIC_NAMES { + assert!( + output.contains(&format!("# TYPE {name} ")), + "metric {name} missing from exposition:\n{output}" + ); + } + assert!( + output.ends_with("# EOF\n"), + "missing exposition trailer:\n{output}" + ); + } + + #[test] + fn scraped_values_land_in_the_exposition() { + let metrics = HttpMetrics::init(); + metrics.streams.set(1); + metrics.topics.set(2); + metrics.partitions.set(3); + metrics.segments.set(4); + metrics.messages.set(5); + metrics.users.set(6); + metrics.clients.set(7); + metrics.request_counter().inc(); + let output = metrics.formatted_output(); + for line in [ + "streams 1", + "topics 2", + "partitions 3", + "segments 4", + "messages 5", + "users 6", + "clients 7", + "http_requests_total 1", + ] { + assert!( + output.contains(&format!("\n{line}\n")), + "expected `{line}` in exposition:\n{output}" + ); + } + } + + #[test] + fn gauge_value_clamps_past_i64_range() { + assert_eq!(gauge_value(42), 42); + assert_eq!(gauge_value(u64::MAX), i64::MAX); + } + + #[test] + fn validated_endpoint_disabled_yields_none() { + assert!(matches!( + validated_endpoint(&metrics_config(false, "/metrics")), + Ok(None) + )); + } + + #[test] + fn validated_endpoint_returns_enabled_path() { + let endpoint = validated_endpoint(&metrics_config(true, "/metrics")).unwrap(); + assert_eq!(endpoint.as_deref(), Some("/metrics")); + } + + #[test] + fn validated_endpoint_rejects_missing_leading_slash() { + assert!(matches!( + validated_endpoint(&metrics_config(true, "metrics")), + Err(IggyError::InvalidConfiguration) + )); + } } diff --git a/core/server/src/http/mod.rs b/core/server/src/http/mod.rs deleted file mode 100644 index ad3186a412..0000000000 --- a/core/server/src/http/mod.rs +++ /dev/null @@ -1,40 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod diagnostics; -pub mod error; -pub mod http_server; -mod http_shard_wrapper; -pub mod jwt; -mod mapper; -pub mod metrics; -pub mod shared; - -pub mod consumer_groups; -pub mod consumer_offsets; -pub mod messages; -pub mod partitions; -pub mod personal_access_tokens; -pub mod segments; -pub mod streams; -pub mod system; -pub mod topics; -pub mod users; -#[cfg(feature = "iggy-web")] -pub mod web; - -pub const COMPONENT: &str = "HTTP"; diff --git a/core/server/src/http/partitions.rs b/core/server/src/http/partitions.rs deleted file mode 100644 index 46dd3853f8..0000000000 --- a/core/server/src/http/partitions.rs +++ /dev/null @@ -1,104 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::http::error::CustomError; -use crate::http::jwt::json_web_token::Identity; -use crate::http::shared::AppState; -use crate::shard::transmission::frame::ShardResponse; -use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; -use axum::extract::{Path, Query, State}; -use axum::http::StatusCode; -use axum::routing::post; -use axum::{Extension, Json, Router, debug_handler}; -use iggy_binary_protocol::requests::partitions::{ - CreatePartitionsRequest as WireCreatePartitions, - DeletePartitionsRequest as WireDeletePartitions, -}; -use iggy_common::Identifier; -use iggy_common::Validatable; -use iggy_common::create_partitions::CreatePartitions; -use iggy_common::delete_partitions::DeletePartitions; -use iggy_common::wire_conversions::identifier_to_wire; -use std::sync::Arc; -use tracing::instrument; - -pub fn router(state: Arc) -> Router { - Router::new() - .route( - "/streams/{stream_id}/topics/{topic_id}/partitions", - post(create_partitions).delete(delete_partitions), - ) - .with_state(state) -} - -#[debug_handler] -#[instrument(skip_all, name = "trace_create_partitions", fields(iggy_user_id = identity.user_id, iggy_stream_id = stream_id, iggy_topic_id = topic_id))] -async fn create_partitions( - State(state): State>, - Extension(identity): Extension, - Path((stream_id, topic_id)): Path<(String, String)>, - Json(mut command): Json, -) -> Result { - command.stream_id = Identifier::from_str_value(&stream_id)?; - command.topic_id = Identifier::from_str_value(&topic_id)?; - command.validate()?; - - let wire_command = WireCreatePartitions { - stream_id: identifier_to_wire(&command.stream_id)?, - topic_id: identifier_to_wire(&command.topic_id)?, - partitions_count: command.partitions_count, - }; - let request = ShardRequest::control_plane(ShardRequestPayload::CreatePartitionsRequest { - user_id: identity.user_id, - command: wire_command, - }); - - match state.shard.send_to_control_plane(request).await? { - ShardResponse::CreatePartitionsResponse => Ok(StatusCode::CREATED), - ShardResponse::ErrorResponse(err) => Err(err.into()), - _ => unreachable!("Expected CreatePartitionsResponse"), - } -} - -#[debug_handler] -#[instrument(skip_all, name = "trace_delete_partitions", fields(iggy_user_id = identity.user_id, iggy_stream_id = stream_id, iggy_topic_id = topic_id))] -async fn delete_partitions( - State(state): State>, - Extension(identity): Extension, - Path((stream_id, topic_id)): Path<(String, String)>, - mut query: Query, -) -> Result { - query.stream_id = Identifier::from_str_value(&stream_id)?; - query.topic_id = Identifier::from_str_value(&topic_id)?; - query.validate()?; - - let wire_command = WireDeletePartitions { - stream_id: identifier_to_wire(&query.stream_id)?, - topic_id: identifier_to_wire(&query.topic_id)?, - partitions_count: query.partitions_count, - }; - let request = ShardRequest::control_plane(ShardRequestPayload::DeletePartitionsRequest { - user_id: identity.user_id, - command: wire_command, - }); - - match state.shard.send_to_control_plane(request).await? { - ShardResponse::DeletePartitionsResponse => Ok(StatusCode::NO_CONTENT), - ShardResponse::ErrorResponse(err) => Err(err.into()), - _ => unreachable!("Expected DeletePartitionsResponse"), - } -} diff --git a/core/server/src/http/personal_access_tokens.rs b/core/server/src/http/personal_access_tokens.rs deleted file mode 100644 index bb8d2a8d41..0000000000 --- a/core/server/src/http/personal_access_tokens.rs +++ /dev/null @@ -1,148 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::http::COMPONENT; -use crate::http::error::CustomError; -use crate::http::jwt::json_web_token::Identity; -use crate::http::mapper; -use crate::http::mapper::map_generated_access_token_to_identity_info; -use crate::http::shared::AppState; -use crate::shard::transmission::frame::ShardResponse; -use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; -use axum::extract::{Path, State}; -use axum::http::StatusCode; -use axum::routing::{delete, get, post}; -use axum::{Extension, Json, Router, debug_handler}; -use err_trail::ErrContext; -use iggy_binary_protocol::WireName; -use iggy_binary_protocol::requests::personal_access_tokens::{ - CreatePersonalAccessTokenRequest as WireCreatePat, - DeletePersonalAccessTokenRequest as WireDeletePat, -}; -use iggy_common::IdentityInfo; -use iggy_common::PersonalAccessTokenInfo; -use iggy_common::Validatable; -use iggy_common::create_personal_access_token::CreatePersonalAccessToken; -use iggy_common::login_with_personal_access_token::LoginWithPersonalAccessToken; -use iggy_common::{IggyError, RawPersonalAccessToken}; -use secrecy::ExposeSecret; -use std::sync::Arc; -use tracing::instrument; - -pub fn router(state: Arc) -> Router { - Router::new() - .route( - "/personal-access-tokens", - get(get_personal_access_tokens).post(create_personal_access_token), - ) - .route( - "/personal-access-tokens/{name}", - delete(delete_personal_access_token), - ) - .route( - "/personal-access-tokens/login", - post(login_with_personal_access_token), - ) - .with_state(state) -} - -#[debug_handler] -async fn get_personal_access_tokens( - State(state): State>, - Extension(identity): Extension, -) -> Result>, CustomError> { - let personal_access_tokens = state - .shard - .shard() - .get_personal_access_tokens(identity.user_id) - .error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - failed to get personal access tokens, user ID: {}", - identity.user_id - ) - })?; - let personal_access_tokens = mapper::map_personal_access_tokens(&personal_access_tokens); - Ok(Json(personal_access_tokens)) -} - -#[debug_handler] -#[instrument(skip_all, name = "trace_create_personal_access_token", fields(iggy_user_id = identity.user_id))] -async fn create_personal_access_token( - State(state): State>, - Extension(identity): Extension, - Json(command): Json, -) -> Result, CustomError> { - command.validate()?; - - let wire_command = WireCreatePat { - name: WireName::new(&command.name) - .map_err(|_| IggyError::InvalidPersonalAccessTokenName)?, - expiry: command.expiry.into(), - }; - let request = - ShardRequest::control_plane(ShardRequestPayload::CreatePersonalAccessTokenRequest { - user_id: identity.user_id, - command: wire_command, - }); - - match state.shard.send_to_control_plane(request).await? { - ShardResponse::CreatePersonalAccessTokenResponse(_, token) => { - Ok(Json(RawPersonalAccessToken { token })) - } - ShardResponse::ErrorResponse(err) => Err(err.into()), - _ => unreachable!("Expected CreatePersonalAccessTokenResponse"), - } -} - -#[debug_handler] -#[instrument(skip_all, name = "trace_delete_personal_access_token", fields(iggy_user_id = identity.user_id))] -async fn delete_personal_access_token( - State(state): State>, - Extension(identity): Extension, - Path(name): Path, -) -> Result { - let wire_command = WireDeletePat { - name: WireName::new(&name).map_err(|_| IggyError::InvalidPersonalAccessTokenName)?, - }; - let request = - ShardRequest::control_plane(ShardRequestPayload::DeletePersonalAccessTokenRequest { - user_id: identity.user_id, - command: wire_command, - }); - - match state.shard.send_to_control_plane(request).await? { - ShardResponse::DeletePersonalAccessTokenResponse => Ok(StatusCode::NO_CONTENT), - ShardResponse::ErrorResponse(err) => Err(err.into()), - _ => unreachable!("Expected DeletePersonalAccessTokenResponse"), - } -} - -#[instrument(skip_all, name = "trace_login_with_personal_access_token")] -async fn login_with_personal_access_token( - State(state): State>, - Json(command): Json, -) -> Result, CustomError> { - let user = state - .shard - .shard() - .login_with_personal_access_token(command.token.expose_secret(), None) - .error(|e: &IggyError| { - format!("{COMPONENT} (error: {e}) - failed to login with personal access token") - })?; - let tokens = state.jwt_manager.generate(user.id)?; - Ok(Json(map_generated_access_token_to_identity_info(tokens))) -} diff --git a/core/server-ng/src/http/reads.rs b/core/server/src/http/reads.rs similarity index 95% rename from core/server-ng/src/http/reads.rs rename to core/server/src/http/reads.rs index 9246ad6109..9955242065 100644 --- a/core/server-ng/src/http/reads.rs +++ b/core/server/src/http/reads.rs @@ -19,14 +19,16 @@ //! metadata-STM read entry, and the wire/domain identifier resolvers the read //! and data-plane routes ground their scopes through. -use crate::bootstrap::ServerNgShard; +use crate::bootstrap::ServerShard; use bytes::Bytes; use consensus::MetadataHandle; use iggy_binary_protocol::WireIdentifier; +use iggy_binary_protocol::codes::GET_STATS_CODE; use iggy_common::wire_conversions::identifier_to_wire; use iggy_common::{Identifier, IggyError}; use metadata::impls::metadata::StreamsFrontend; use metadata::permissioner::Permissioner; +use send_wrapper::SendWrapper; use std::rc::Rc; use crate::http::error::{Consistency, ReadError}; @@ -78,8 +80,9 @@ pub(in crate::http) fn authorize_read( /// a TCP read of the same entity return byte-identical bodies. /// /// Reads never touch consensus or a VSR session: `build_non_replicated_response` -/// is a pure STM read. It is synchronous, so this helper is too - no submit -/// await, no gate, no `SendWrapper`. An absent entity surfaces as +/// is a pure STM read, with one exception - the stats read's cross-shard +/// connected-client gather, an async broadcast run here (under `SendWrapper`, +/// same as `/metrics`) before the sync builder. An absent entity surfaces as /// [`NonReplicatedResponse::Empty`], mapped to 404 here because every REST read /// whose entity can be missing shares that not-found shape. pub(in crate::http) async fn read_local( @@ -92,6 +95,12 @@ pub(in crate::http) async fn read_local( ) -> Result { await_recovery_barrier(&state.shard).await?; authorize_read(state, identity, consistency, rule)?; + let clients_count = if code == GET_STATS_CODE { + u32::try_from(SendWrapper::new(state.shard.list_all_clients()).await.len()) + .unwrap_or(u32::MAX) + } else { + 0 + }; match build_non_replicated_response( &state.shard, code, @@ -99,6 +108,7 @@ pub(in crate::http) async fn read_local( Some(identity.user_id), &state.roster, identity.client_ip, + clients_count, ) .map_err(ReadError::Rejected)? { @@ -149,7 +159,7 @@ const fn barrier_state(barrier: u64, commit_min: u64, expired: bool) -> BarrierW /// state a client already saw acked; the caller retries against a converged /// cluster. pub(in crate::http) async fn await_recovery_barrier( - shard: &Rc, + shard: &Rc, ) -> Result<(), ReadError> { const POLL: std::time::Duration = std::time::Duration::from_millis(10); diff --git a/core/server-ng/src/http/reply.rs b/core/server/src/http/reply.rs similarity index 99% rename from core/server-ng/src/http/reply.rs rename to core/server/src/http/reply.rs index d039c64c78..9c3a59f889 100644 --- a/core/server-ng/src/http/reply.rs +++ b/core/server/src/http/reply.rs @@ -105,7 +105,7 @@ pub(in crate::http) fn send_confirmations( Err(error) => { warn!( ?error, - "server-ng HTTP: undecodable send_messages commit confirmation" + "server HTTP: undecodable send_messages commit confirmation" ); None } @@ -315,7 +315,7 @@ mod tests { let commit = 9; let body = Bytes::from_static(b"body"); - // server-ng reply builders, all funnelled through build_reply_with_body. + // Reply builders, all funnelled through build_reply_with_body. for status in [ build_reply_with_body(header, 42, 7, commit, 0, |_| {}) .header() diff --git a/core/server/src/http/segments.rs b/core/server/src/http/segments.rs deleted file mode 100644 index e83be5de26..0000000000 --- a/core/server/src/http/segments.rs +++ /dev/null @@ -1,120 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::http::COMPONENT; -use crate::http::error::CustomError; -use crate::http::jwt::json_web_token::Identity; -use crate::http::shared::AppState; -use crate::shard::transmission::frame::ShardResponse; -use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; -use axum::extract::{Path, Query, State}; -use axum::http::StatusCode; -use axum::routing::delete; -use axum::{Extension, Router, debug_handler}; -use err_trail::ErrContext; -use iggy_binary_protocol::requests::segments::DeleteSegmentsRequest; -use iggy_common::Identifier; -use iggy_common::Validatable; -use iggy_common::delete_segments::DeleteSegments; -use iggy_common::wire_conversions::identifier_to_wire; -use send_wrapper::SendWrapper; -use server_common::sharding::IggyNamespace; -use std::sync::Arc; -use tracing::instrument; - -pub fn router(state: Arc) -> Router { - Router::new() - .route( - "/streams/{stream_id}/topics/{topic_id}/partitions/{partition_id}", - delete(delete_segments), - ) - .with_state(state) -} - -#[debug_handler] -#[instrument(skip_all, name = "trace_delete_segments", fields(iggy_user_id = identity.user_id, iggy_stream_id = stream_id, iggy_topic_id = topic_id))] -async fn delete_segments( - State(state): State>, - Extension(identity): Extension, - Path((stream_id, topic_id, partition_id)): Path<(String, String, u32)>, - mut query: Query, -) -> Result { - query.stream_id = Identifier::from_str_value(&stream_id)?; - query.topic_id = Identifier::from_str_value(&topic_id)?; - query.partition_id = partition_id; - query.validate()?; - let segments_count = query.segments_count; - - let partition = state.shard.shard().resolve_partition_for_delete_segments( - identity.user_id, - &query.stream_id, - &query.topic_id, - partition_id as usize, - )?; - - let namespace = IggyNamespace::new( - partition.stream_id, - partition.topic_id, - partition.partition_id, - ); - let request = ShardRequest::data_plane( - namespace, - ShardRequestPayload::DeleteSegments { segments_count }, - ); - - let delete_future = SendWrapper::new(state.shard.shard().send_to_data_plane(request)); - match delete_future.await? { - ShardResponse::DeleteSegments { - deleted_segments, - deleted_messages, - } => { - state - .shard - .shard() - .metrics - .decrement_segments(deleted_segments as u32); - state - .shard - .shard() - .metrics - .decrement_messages(deleted_messages); - } - ShardResponse::ErrorResponse(err) => return Err(err.into()), - _ => unreachable!("Expected DeleteSegments"), - } - - let wire_command = DeleteSegmentsRequest { - stream_id: identifier_to_wire(&query.stream_id)?, - topic_id: identifier_to_wire(&query.topic_id)?, - partition_id: query.partition_id, - segments_count, - }; - let entry_command = crate::state::command::EntryCommand::DeleteSegments(wire_command); - let state_future = SendWrapper::new( - state - .shard - .shard() - .state - .apply(identity.user_id, &entry_command), - ); - state_future.await.error(|e: &iggy_common::IggyError| { - format!( - "{COMPONENT} (error: {e}) - failed to apply delete segments, stream ID: {stream_id}, topic ID: {topic_id}" - ) - })?; - Ok(StatusCode::NO_CONTENT) -} diff --git a/core/server-ng/src/http/session.rs b/core/server/src/http/session.rs similarity index 98% rename from core/server-ng/src/http/session.rs rename to core/server/src/http/session.rs index f6fd3df06a..2e104268d4 100644 --- a/core/server-ng/src/http/session.rs +++ b/core/server/src/http/session.rs @@ -85,10 +85,10 @@ pub(in crate::http) struct HttpSession { pub(in crate::http) key: String, /// Shard-0 client id minted for this credential; its top 16 bits are 0, so /// it shares the shard-0 id space with TCP virtual clients without - /// colliding. Fills `RequestHeader.client` on every write. + /// colliding. Fills `RoutedRequestHeader.client` on every write. pub(in crate::http) client_id: u128, /// Cluster session number returned by the VSR `Register` commit. Fills - /// `RequestHeader.session` on every write. + /// `RoutedRequestHeader.session` on every write. pub(in crate::http) session: u64, /// User the credential authenticated as. Consumed by the write path for /// authorization. @@ -384,7 +384,7 @@ mod tests { // deployment untouched. #[test] fn runtime_cap_at_default_config_equals_pinned_default() { - let configured = configs::ng_metadata::MetadataConfig::default().clients_table_max; + let configured = configs::metadata::MetadataConfig::default().clients_table_max; assert_eq!(max_http_sessions(configured), DEFAULT_MAX_HTTP_SESSIONS); } diff --git a/core/server-ng/src/http/state.rs b/core/server/src/http/state.rs similarity index 97% rename from core/server-ng/src/http/state.rs rename to core/server/src/http/state.rs index 7efb8bcc4c..0c8a2cca29 100644 --- a/core/server-ng/src/http/state.rs +++ b/core/server/src/http/state.rs @@ -27,7 +27,7 @@ use std::sync::Arc; use axum::http::{HeaderName, HeaderValue}; use axum::response::Response; -use configs::server_ng::NgSystemConfig; +use configs::server::ServerSystemConfig; use consensus::{MetadataHandle, VsrConsensus}; use futures::channel::oneshot; use iggy_common::{ClusterMetadata, IggyTimestamp}; @@ -37,7 +37,7 @@ use send_wrapper::SendWrapper; use tokio::sync::Mutex; use tracing::warn; -use crate::bootstrap::ServerNgShard; +use crate::bootstrap::ServerShard; use crate::cluster_meta::ClusterRoster; use crate::dispatch::submit_register_on_owner; use crate::http::error::{AuthError, ReadError, primary_redirect_location}; @@ -69,12 +69,12 @@ pub(in crate::http) type HttpState = SendWrapper>; /// session table so every handler and the [`Authenticated`] extractor reach /// them through one axum `State`. pub(in crate::http) struct HttpInner { - pub(in crate::http) shard: Rc, + pub(in crate::http) shard: Rc, pub(in crate::http) jwt: JwtManager, /// Read-only server config for the snapshot collector (log directory + /// runtime config paths); the shard does not expose config on the read /// path. - pub(in crate::http) system_config: Arc, + pub(in crate::http) system_config: Arc, /// Per-credential VSR sessions keyed by JWT `jti` / PAT hash. `RefCell` is /// sound here - shard 0 is single-threaded and the `SendWrapper` state /// bridge tolerates the `!Sync` interior - but the guard must never be held @@ -256,7 +256,7 @@ impl HttpInner { { warn!( attempt, - "server-ng HTTP: minted client id was already registered; re-minting" + "server HTTP: minted client id was already registered; re-minting" ); } Err(error) => return Err(error), @@ -325,7 +325,7 @@ impl HttpInner { .await .map_err(|_| AuthError::SessionUnavailable)? .map_err(|error| { - warn!(?error, "server-ng HTTP: VSR Register submit failed"); + warn!(?error, "server HTTP: VSR Register submit failed"); match error { // The Register never entered the pipeline, so re-issuing // it anywhere is safe; the transient-not-accepted body @@ -361,7 +361,7 @@ impl HttpInner { client_id, user_id, watermark = bound.watermark, - "server-ng HTTP: minted client id already had a committed session for this user" + "server HTTP: minted client id already had a committed session for this user" ); return Err(AuthError::SessionIdTaken); } diff --git a/core/server/src/http/streams.rs b/core/server/src/http/streams.rs deleted file mode 100644 index 742575144c..0000000000 --- a/core/server/src/http/streams.rs +++ /dev/null @@ -1,200 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::http::error::CustomError; -use crate::http::jwt::json_web_token::Identity; -use crate::http::shared::AppState; -use crate::shard::transmission::frame::ShardResponse; -use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; -use axum::extract::{Path, State}; -use axum::http::StatusCode; -use axum::routing::{delete, get}; -use axum::{Extension, Json, Router, debug_handler}; -use iggy_binary_protocol::WireName; -use iggy_binary_protocol::requests::streams::{ - CreateStreamRequest as WireCreateStream, DeleteStreamRequest as WireDeleteStream, - PurgeStreamRequest as WirePurgeStream, UpdateStreamRequest as WireUpdateStream, -}; -use iggy_common::Identifier; -use iggy_common::Validatable; -use iggy_common::create_stream::CreateStream; -use iggy_common::update_stream::UpdateStream; -use iggy_common::wire_conversions::identifier_to_wire; -use iggy_common::{IggyError, Stream, StreamDetails}; -use std::sync::Arc; -use tracing::instrument; - -pub fn router(state: Arc) -> Router { - Router::new() - .route("/streams", get(get_streams).post(create_stream)) - .route( - "/streams/{stream_id}", - get(get_stream).put(update_stream).delete(delete_stream), - ) - .route("/streams/{stream_id}/purge", delete(purge_stream)) - .with_state(state) -} - -#[debug_handler] -async fn get_stream( - State(state): State>, - Extension(identity): Extension, - Path(stream_id): Path, -) -> Result, CustomError> { - let stream_id = Identifier::from_str_value(&stream_id)?; - - let shard = state.shard.shard(); - let numeric_stream_id = shard - .metadata - .get_stream_id(&stream_id) - .ok_or(CustomError::ResourceNotFound)?; - - shard - .metadata - .perm_get_stream(identity.user_id, numeric_stream_id)?; - - let stream_meta = shard - .metadata - .get_stream(numeric_stream_id) - .ok_or(CustomError::ResourceNotFound)?; - - let stream_details = crate::http::mapper::map_stream_details_from_metadata(&stream_meta); - - Ok(Json(stream_details)) -} - -#[debug_handler] -async fn get_streams( - State(state): State>, - Extension(identity): Extension, -) -> Result>, CustomError> { - let shard = state.shard.shard(); - - shard.metadata.perm_get_streams(identity.user_id)?; - - let streams = shard - .metadata - .with_metadata(crate::http::mapper::map_streams_from_metadata); - - Ok(Json(streams)) -} - -#[debug_handler] -#[instrument(skip_all, name = "trace_create_stream", fields(iggy_user_id = identity.user_id))] -async fn create_stream( - State(state): State>, - Extension(identity): Extension, - Json(command): Json, -) -> Result, CustomError> { - command.validate()?; - - let wire_command = WireCreateStream { - name: WireName::new(&command.name).map_err(|_| IggyError::InvalidStreamName)?, - }; - let request = ShardRequest::control_plane(ShardRequestPayload::CreateStreamRequest { - user_id: identity.user_id, - command: wire_command, - }); - - match state.shard.send_to_control_plane(request).await? { - ShardResponse::CreateStreamResponse(data) => { - let stream_meta = state - .shard - .shard() - .metadata - .get_stream(data.id as usize) - .expect("Stream must exist after creation"); - let response = crate::http::mapper::map_stream_details_from_metadata(&stream_meta); - Ok(Json(response)) - } - ShardResponse::ErrorResponse(err) => Err(err.into()), - _ => unreachable!("Expected CreateStreamResponse"), - } -} - -#[debug_handler] -#[instrument(skip_all, name = "trace_update_stream", fields(iggy_user_id = identity.user_id, iggy_stream_id = stream_id))] -async fn update_stream( - State(state): State>, - Extension(identity): Extension, - Path(stream_id): Path, - Json(mut command): Json, -) -> Result { - command.stream_id = Identifier::from_str_value(&stream_id)?; - command.validate()?; - - let wire_command = WireUpdateStream { - stream_id: identifier_to_wire(&command.stream_id)?, - name: WireName::new(&command.name).map_err(|_| IggyError::InvalidStreamName)?, - }; - let request = ShardRequest::control_plane(ShardRequestPayload::UpdateStreamRequest { - user_id: identity.user_id, - command: wire_command, - }); - - match state.shard.send_to_control_plane(request).await? { - ShardResponse::UpdateStreamResponse => Ok(StatusCode::NO_CONTENT), - ShardResponse::ErrorResponse(err) => Err(err.into()), - _ => unreachable!("Expected UpdateStreamResponse"), - } -} - -#[debug_handler] -#[instrument(skip_all, name = "trace_delete_stream", fields(iggy_user_id = identity.user_id, iggy_stream_id = stream_id))] -async fn delete_stream( - State(state): State>, - Extension(identity): Extension, - Path(stream_id): Path, -) -> Result { - let stream_id = Identifier::from_str_value(&stream_id)?; - - let request = ShardRequest::control_plane(ShardRequestPayload::DeleteStreamRequest { - user_id: identity.user_id, - command: WireDeleteStream { - stream_id: identifier_to_wire(&stream_id)?, - }, - }); - - match state.shard.send_to_control_plane(request).await? { - ShardResponse::DeleteStreamResponse => Ok(StatusCode::NO_CONTENT), - ShardResponse::ErrorResponse(err) => Err(err.into()), - _ => unreachable!("Expected DeleteStreamResponse"), - } -} - -#[debug_handler] -#[instrument(skip_all, name = "trace_purge_stream", fields(iggy_user_id = identity.user_id, iggy_stream_id = stream_id))] -async fn purge_stream( - State(state): State>, - Extension(identity): Extension, - Path(stream_id): Path, -) -> Result { - let stream_id = Identifier::from_str_value(&stream_id)?; - - let request = ShardRequest::control_plane(ShardRequestPayload::PurgeStreamRequest { - user_id: identity.user_id, - command: WirePurgeStream { - stream_id: identifier_to_wire(&stream_id)?, - }, - }); - - match state.shard.send_to_control_plane(request).await? { - ShardResponse::PurgeStreamResponse => Ok(StatusCode::NO_CONTENT), - ShardResponse::ErrorResponse(err) => Err(err.into()), - _ => unreachable!("Expected PurgeStreamResponse"), - } -} diff --git a/core/server-ng/src/http/submit.rs b/core/server/src/http/submit.rs similarity index 97% rename from core/server-ng/src/http/submit.rs rename to core/server/src/http/submit.rs index 07c45b1824..f9543cda8d 100644 --- a/core/server-ng/src/http/submit.rs +++ b/core/server/src/http/submit.rs @@ -26,14 +26,14 @@ use bytes::Bytes; use consensus::MetadataHandle; use futures::channel::oneshot; use iggy_binary_protocol::consensus::Command2; -use iggy_binary_protocol::{GenericHeader, Operation, ReplyHeader, RequestHeader}; +use iggy_binary_protocol::{GenericHeader, Operation, ReplyHeader, RoutedRequestHeader}; use iggy_common::IggyError; use message_bus::BusMessage; use metadata::impls::metadata::StreamsFrontend; use server_common::Message; use tracing::warn; -use crate::bootstrap::ServerNgShard; +use crate::bootstrap::ServerShard; use crate::dispatch::{ dispatch_partition_request, resolve_delete_segments_truncate, submit_client_request_on_owner, submit_logout_on_owner, @@ -98,7 +98,7 @@ pub(in crate::http) async fn submit_committed( session: &Rc, operation: Operation, body: &[u8], -) -> Result<(RequestHeader, Message, Option), WriteError> { +) -> Result<(RoutedRequestHeader, Message, Option), WriteError> { // Control writes are authorized in-apply on the metadata STM: a denial // comes back as `Unauthorized` in the committed result section, which // `committed_payload` maps to a 403. No pre-submit gate here, so the @@ -178,12 +178,12 @@ pub(in crate::http) async fn submit_committed( /// that the replicated apply grades to `InvalidCredentials` (see /// `verify_and_rewrite_change_password`). async fn submit_gated( - shard: &Rc, + shard: &Rc, session: &HttpSession, operation: Operation, max_tokens_per_user: u32, body: &[u8], -) -> Result<(RequestHeader, Message, Option), WriteError> { +) -> Result<(RoutedRequestHeader, Message, Option), WriteError> { let mut next_request_id = session.gate.lock().await; // Burn the id at stamp time: every exit below (rewrite rejection, // unresolved delete-segments, unanswered submit, exhausted transient @@ -343,10 +343,10 @@ pub(in crate::http) async fn logout_session(state: &HttpInner, session: &Rc {} Ok(Err(error)) => warn!( ?error, - "server-ng HTTP: VSR Logout submit failed; slot lingers until eviction" + "server HTTP: VSR Logout submit failed; slot lingers until eviction" ), Err(_canceled) => warn!( - "server-ng HTTP: VSR Logout task dropped before replying; slot lingers until eviction" + "server HTTP: VSR Logout task dropped before replying; slot lingers until eviction" ), } state.forget_session(session); @@ -396,7 +396,7 @@ pub(in crate::http) async fn partition_write_replicated( warn!( ?error, ?operation, - "server-ng HTTP: partition write reply slot install failed" + "server HTTP: partition write reply slot install failed" ); PartitionWriteError::Unavailable })?; @@ -464,7 +464,7 @@ pub(in crate::http) async fn produce_unacked( /// Install this session's in-process reply target on first data-plane use. /// /// The registry key is the session's shard-0 client id - the same id stamped -/// into `RequestHeader.client` - so a partition reply routed through +/// into `RoutedRequestHeader.client` - so a partition reply routed through /// `send_to_client` lands on this entry and resolves the request-keyed slot. /// `None` from the registry means the key is already occupied; treat it as /// installed but leave the token unset so this session never tears down an diff --git a/core/server/src/http/system.rs b/core/server/src/http/system.rs deleted file mode 100644 index 7d8e26b941..0000000000 --- a/core/server/src/http/system.rs +++ /dev/null @@ -1,168 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::configs::http::HttpMetricsConfig; -use crate::http::COMPONENT; -use crate::http::error::CustomError; -use crate::http::jwt::json_web_token::Identity; -use crate::http::mapper; -use crate::http::shared::AppState; -use axum::body::Body; -use axum::extract::{Path, State}; -use axum::http::{HeaderMap, header}; -use axum::response::IntoResponse; -use axum::routing::{get, post}; -use axum::{Extension, Json, Router, debug_handler}; -use bytes::Bytes; -use chrono::Local; -use err_trail::ErrContext; -use iggy_common::Stats; -use iggy_common::get_snapshot::GetSnapshot; -use iggy_common::{ClientInfo, ClientInfoDetails, ClusterMetadata, IggyError, SystemSnapshotType}; -use send_wrapper::SendWrapper; -use std::sync::Arc; -use tracing::error; - -const NAME: &str = "Iggy API"; -const PONG: &str = "pong"; - -pub fn router(state: Arc, metrics_config: &HttpMetricsConfig) -> Router { - let mut router = Router::new() - .route("/", get(|| async { NAME })) - .route("/ping", get(|| async { PONG })) - .route("/stats", get(get_stats)) - .route("/cluster/metadata", get(get_cluster_metadata)) - .route("/clients", get(get_clients)) - .route("/clients/{client_id}", get(get_client)) - .route("/snapshot", post(get_snapshot)); - if metrics_config.enabled { - router = router.route(&metrics_config.endpoint, get(get_metrics)); - } - - router.with_state(state) -} - -#[debug_handler] -async fn get_metrics(State(state): State>) -> Result { - let metrics_formatted_output = state.shard.shard().metrics.get_formatted_output(); - Ok(metrics_formatted_output) -} - -#[debug_handler] -async fn get_stats( - State(state): State>, - Extension(identity): Extension, -) -> Result, CustomError> { - state - .shard - .shard() - .metadata - .perm_get_stats(identity.user_id)?; - let stats_future = SendWrapper::new(state.shard.shard().get_stats()); - let stats = stats_future.await.error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - failed to get stats, user ID: {}", - identity.user_id - ) - })?; - Ok(Json(stats)) -} - -async fn get_cluster_metadata( - State(state): State>, - Extension(identity): Extension, -) -> Result, CustomError> { - let _ = identity; // authenticated via middleware - let cluster_metadata = state.shard.shard().get_cluster_metadata(); - Ok(Json(cluster_metadata)) -} - -async fn get_client( - State(state): State>, - Extension(identity): Extension, - Path(client_id): Path, -) -> Result, CustomError> { - state - .shard - .shard() - .metadata - .perm_get_client(identity.user_id)?; - let Some(client) = state.shard.shard().get_client(client_id) else { - return Err(CustomError::ResourceNotFound); - }; - - let client = mapper::map_client(&client); - Ok(Json(client)) -} - -#[debug_handler] -async fn get_clients( - State(state): State>, - Extension(identity): Extension, -) -> Result>, CustomError> { - state - .shard - .shard() - .metadata - .perm_get_clients(identity.user_id)?; - let clients = state.shard.shard().get_clients(); - let clients = mapper::map_clients(&clients); - Ok(Json(clients)) -} - -#[debug_handler] -async fn get_snapshot( - State(state): State>, - Extension(identity): Extension, - Json(command): Json, -) -> Result { - state - .shard - .shard() - .metadata - .perm_get_snapshot(identity.user_id)?; - if command.snapshot_types.contains(&SystemSnapshotType::All) && command.snapshot_types.len() > 1 - { - error!("When using 'All' snapshot type, no other types can be specified"); - return Err(IggyError::InvalidCommand.into()); - } - - let snapshot_future = SendWrapper::new( - state - .shard - .shard() - .get_snapshot(command.compression, &command.snapshot_types), - ); - - let snapshot = snapshot_future - .await - .error(|e: &IggyError| format!("{COMPONENT} (error: {e}) - failed to get snapshot"))?; - - let zip_data = Bytes::from(snapshot.0); - let filename = format!("iggy_snapshot_{}.zip", Local::now().format("%Y%m%d_%H%M%S")); - - let mut headers = HeaderMap::new(); - headers.insert( - header::CONTENT_TYPE, - header::HeaderValue::from_static("application/zip"), - ); - headers.insert( - header::CONTENT_DISPOSITION, - header::HeaderValue::from_str(&format!("attachment; filename=\"{filename}\"")).unwrap(), - ); - Ok((headers, Body::from(zip_data))) -} diff --git a/core/server-ng/src/http/tls.rs b/core/server/src/http/tls.rs similarity index 92% rename from core/server-ng/src/http/tls.rs rename to core/server/src/http/tls.rs index b4e0137a5a..a7b8510955 100644 --- a/core/server-ng/src/http/tls.rs +++ b/core/server/src/http/tls.rs @@ -17,7 +17,7 @@ //! HTTPS for the shard-0 REST listener. //! -//! server-ng is thread-per-core `compio/io_uring`, so it cannot reuse the +//! The server is thread-per-core `compio/io_uring`, so it cannot reuse the //! legacy `axum-server` TLS acceptor. The plain-HTTP path uses //! `cyper_axum::serve`; the TLS path cannot, because that serve loop wraps //! its IO in a `compio` `Split` (one `BiLock`): hyper parks a pending read @@ -59,7 +59,7 @@ use tower_http::add_extension::AddExtension; use tracing::{debug, error}; use crate::http::ClientAddr; -use crate::server_error::ServerNgError; +use crate::server_error::ServerError; /// hyper's auto-builder serves whichever protocol the client selects via /// ALPN; advertise the same pair `axum-server` negotiates by default on the @@ -82,14 +82,14 @@ type Handshaken = (TlsStream, SocketAddr); /// /// # Errors /// -/// [`ServerNgError::ListenerCredentials`] with `transport: "http.tls"` if +/// [`ServerError::ListenerCredentials`] with `transport: "http.tls"` if /// the PEM files cannot be read or the certificate / key pair is rejected. pub fn load_http_tls_server_config( tls: &HttpTlsConfig, -) -> Result, ServerNgError> { +) -> Result, ServerError> { let credentials = load_pem(Path::new(&tls.cert_file), Path::new(&tls.key_file)).map_err(|source| { - ServerNgError::ListenerCredentials { + ServerError::ListenerCredentials { transport: "http.tls", source, } @@ -171,7 +171,7 @@ async fn serve_connection( futures::select! { result = conn.as_mut().fuse() => { if let Err(error) = result { - debug!(%peer, %error, "server-ng HTTPS connection terminated with error"); + debug!(%peer, %error, "server HTTPS connection terminated with error"); } break; } @@ -199,12 +199,12 @@ impl + 'static> Executor for LocalExecutor { /// the only HTTP-specific step is the ALPN advertisement. fn build_server_config( credentials: TlsServerCredentials, -) -> Result, ServerNgError> { +) -> Result, ServerError> { install_default_crypto_provider(); let mut config = rustls::ServerConfig::builder() .with_no_client_auth() .with_single_cert(credentials.cert_chain, credentials.key_der) - .map_err(|error| ServerNgError::ListenerCredentials { + .map_err(|error| ServerError::ListenerCredentials { transport: "http.tls", source: std::io::Error::other(format!( "http TLS server config rejected credentials: {error}" @@ -227,14 +227,14 @@ async fn accept_pump( loop { futures::select! { () = shutdown.wait().fuse() => { - debug!("server-ng HTTPS accept pump shutting down"); + debug!("server HTTPS accept pump shutting down"); break; } result = listener.accept().fuse() => match result { Ok((stream, peer)) => { spawn_handshake(&acceptor, &connections, handshake_grace, stream, peer); } - Err(error) => error!(%error, "server-ng HTTPS accept failed"), + Err(error) => error!(%error, "server HTTPS accept failed"), }, } } @@ -260,9 +260,9 @@ fn spawn_handshake( // shutdown, when the serve loop is already tearing down. let _ = connections.send((tls, peer)).await; } - Ok(Err(error)) => debug!(%peer, %error, "server-ng HTTPS handshake failed"), + Ok(Err(error)) => debug!(%peer, %error, "server HTTPS handshake failed"), Err(_elapsed) => { - debug!(%peer, grace = ?handshake_grace, "server-ng HTTPS handshake timed out"); + debug!(%peer, grace = ?handshake_grace, "server HTTPS handshake timed out"); } } }) diff --git a/core/server/src/http/topics.rs b/core/server/src/http/topics.rs deleted file mode 100644 index c1b12d5a94..0000000000 --- a/core/server/src/http/topics.rs +++ /dev/null @@ -1,263 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::http::COMPONENT; -use crate::http::error::CustomError; -use crate::http::jwt::json_web_token::Identity; -use crate::http::shared::AppState; -use crate::shard::transmission::frame::ShardResponse; -use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; -use axum::extract::{Path, State}; -use axum::http::StatusCode; -use axum::routing::{delete, get}; -use axum::{Extension, Json, Router, debug_handler}; -use err_trail::ErrContext; -use iggy_binary_protocol::WireName; -use iggy_binary_protocol::requests::topics::{ - CreateTopicRequest as WireCreateTopic, DeleteTopicRequest as WireDeleteTopic, - PurgeTopicRequest as WirePurgeTopic, UpdateTopicRequest as WireUpdateTopic, -}; -use iggy_common::Identifier; -use iggy_common::Validatable; -use iggy_common::create_topic::CreateTopic; -use iggy_common::update_topic::UpdateTopic; -use iggy_common::wire_conversions::identifier_to_wire; -use iggy_common::{IggyError, Topic, TopicDetails}; -use std::sync::Arc; -use tracing::instrument; - -pub fn router(state: Arc) -> Router { - Router::new() - .route( - "/streams/{stream_id}/topics", - get(get_topics).post(create_topic), - ) - .route( - "/streams/{stream_id}/topics/{topic_id}", - get(get_topic).put(update_topic).delete(delete_topic), - ) - .route( - "/streams/{stream_id}/topics/{topic_id}/purge", - delete(purge_topic), - ) - .with_state(state) -} - -#[debug_handler] -async fn get_topic( - State(state): State>, - Extension(identity): Extension, - Path((stream_id, topic_id)): Path<(String, String)>, -) -> Result, CustomError> { - let identity_stream_id = Identifier::from_str_value(&stream_id)?; - let identity_topic_id = Identifier::from_str_value(&topic_id)?; - - let shard = state.shard.shard(); - - let numeric_stream_id = shard - .metadata - .get_stream_id(&identity_stream_id) - .ok_or(CustomError::ResourceNotFound)?; - - let numeric_topic_id = shard - .metadata - .get_topic_id(numeric_stream_id, &identity_topic_id) - .ok_or(CustomError::ResourceNotFound)?; - - shard - .metadata - .perm_get_topic(identity.user_id, numeric_stream_id, numeric_topic_id) - .error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - permission denied to get topic with ID: {topic_id} in stream with ID: {stream_id} for user with ID: {}", - identity.user_id, - ) - })?; - - let topic_meta = shard - .metadata - .get_topic(numeric_stream_id, numeric_topic_id) - .ok_or(CustomError::ResourceNotFound)?; - - let topic_details = crate::http::mapper::map_topic_details_from_metadata(&topic_meta); - - Ok(Json(topic_details)) -} - -#[debug_handler] -async fn get_topics( - State(state): State>, - Extension(identity): Extension, - Path(stream_id): Path, -) -> Result>, CustomError> { - let stream_id_ident = Identifier::from_str_value(&stream_id)?; - let shard = state.shard.shard(); - - let numeric_stream_id = shard - .metadata - .get_stream_id(&stream_id_ident) - .ok_or(CustomError::ResourceNotFound)?; - - shard - .metadata - .perm_get_topics(identity.user_id, numeric_stream_id) - .error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - permission denied to get topics in stream with ID: {stream_id} for user with ID: {}", - identity.user_id, - ) - })?; - - let stream_meta = shard - .metadata - .get_stream(numeric_stream_id) - .ok_or(CustomError::ResourceNotFound)?; - let topics = crate::http::mapper::map_topics_from_metadata(&stream_meta); - - Ok(Json(topics)) -} - -#[debug_handler] -#[instrument(skip_all, name = "trace_create_topic", fields(iggy_user_id = identity.user_id, iggy_stream_id = stream_id))] -async fn create_topic( - State(state): State>, - Extension(identity): Extension, - Path(stream_id): Path, - Json(mut command): Json, -) -> Result, CustomError> { - command.stream_id = Identifier::from_str_value(&stream_id)?; - command.validate()?; - - let numeric_stream_id = state - .shard - .shard() - .metadata - .get_stream_id(&command.stream_id) - .ok_or(CustomError::ResourceNotFound)?; - - let wire_command = WireCreateTopic { - stream_id: identifier_to_wire(&command.stream_id)?, - partitions_count: command.partitions_count, - compression_algorithm: command.compression_algorithm.as_code(), - message_expiry: command.message_expiry.into(), - max_topic_size: command.max_topic_size.into(), - replication_factor: command.replication_factor.unwrap_or(0), - name: WireName::new(&command.name).map_err(|_| IggyError::InvalidTopicName)?, - }; - let request = ShardRequest::control_plane(ShardRequestPayload::CreateTopicRequest { - user_id: identity.user_id, - command: wire_command, - }); - - match state.shard.send_to_control_plane(request).await? { - ShardResponse::CreateTopicResponse(data) => { - let topic_meta = state - .shard - .shard() - .metadata - .get_topic(numeric_stream_id, data.id as usize) - .expect("Topic must exist after creation"); - let response = crate::http::mapper::map_topic_details_from_metadata(&topic_meta); - Ok(Json(response)) - } - ShardResponse::ErrorResponse(err) => Err(err.into()), - _ => unreachable!("Expected CreateTopicResponse"), - } -} - -#[debug_handler] -#[instrument(skip_all, name = "trace_update_topic", fields(iggy_user_id = identity.user_id, iggy_stream_id = stream_id, iggy_topic_id = topic_id))] -async fn update_topic( - State(state): State>, - Extension(identity): Extension, - Path((stream_id, topic_id)): Path<(String, String)>, - Json(mut command): Json, -) -> Result { - command.stream_id = Identifier::from_str_value(&stream_id)?; - command.topic_id = Identifier::from_str_value(&topic_id)?; - command.validate()?; - - let wire_command = WireUpdateTopic { - stream_id: identifier_to_wire(&command.stream_id)?, - topic_id: identifier_to_wire(&command.topic_id)?, - compression_algorithm: command.compression_algorithm.as_code(), - message_expiry: command.message_expiry.into(), - max_topic_size: command.max_topic_size.into(), - replication_factor: command.replication_factor.unwrap_or(0), - name: WireName::new(&command.name).map_err(|_| IggyError::InvalidTopicName)?, - }; - let request = ShardRequest::control_plane(ShardRequestPayload::UpdateTopicRequest { - user_id: identity.user_id, - command: wire_command, - }); - - match state.shard.send_to_control_plane(request).await? { - ShardResponse::UpdateTopicResponse => Ok(StatusCode::NO_CONTENT), - ShardResponse::ErrorResponse(err) => Err(err.into()), - _ => unreachable!("Expected UpdateTopicResponse"), - } -} - -#[debug_handler] -#[instrument(skip_all, name = "trace_delete_topic", fields(iggy_user_id = identity.user_id, iggy_stream_id = stream_id, iggy_topic_id = topic_id))] -async fn delete_topic( - State(state): State>, - Extension(identity): Extension, - Path((stream_id, topic_id)): Path<(String, String)>, -) -> Result { - let stream_id = Identifier::from_str_value(&stream_id)?; - let topic_id = Identifier::from_str_value(&topic_id)?; - - let request = ShardRequest::control_plane(ShardRequestPayload::DeleteTopicRequest { - user_id: identity.user_id, - command: WireDeleteTopic { - stream_id: identifier_to_wire(&stream_id)?, - topic_id: identifier_to_wire(&topic_id)?, - }, - }); - - match state.shard.send_to_control_plane(request).await? { - ShardResponse::DeleteTopicResponse => Ok(StatusCode::NO_CONTENT), - ShardResponse::ErrorResponse(err) => Err(err.into()), - _ => unreachable!("Expected DeleteTopicResponse"), - } -} - -#[debug_handler] -#[instrument(skip_all, name = "trace_purge_topic", fields(iggy_user_id = identity.user_id, iggy_stream_id = stream_id, iggy_topic_id = topic_id))] -async fn purge_topic( - State(state): State>, - Extension(identity): Extension, - Path((stream_id, topic_id)): Path<(String, String)>, -) -> Result { - let stream_id = Identifier::from_str_value(&stream_id)?; - let topic_id = Identifier::from_str_value(&topic_id)?; - - let request = ShardRequest::control_plane(ShardRequestPayload::PurgeTopicRequest { - user_id: identity.user_id, - command: WirePurgeTopic { - stream_id: identifier_to_wire(&stream_id)?, - topic_id: identifier_to_wire(&topic_id)?, - }, - }); - - match state.shard.send_to_control_plane(request).await? { - ShardResponse::PurgeTopicResponse => Ok(StatusCode::NO_CONTENT), - ShardResponse::ErrorResponse(err) => Err(err.into()), - _ => unreachable!("Expected PurgeTopicResponse"), - } -} diff --git a/core/server/src/http/users.rs b/core/server/src/http/users.rs deleted file mode 100644 index 1fc6da7567..0000000000 --- a/core/server/src/http/users.rs +++ /dev/null @@ -1,331 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::http::COMPONENT; -use crate::http::error::CustomError; -use crate::http::jwt::json_web_token::Identity; -use crate::http::mapper; -use crate::http::mapper::map_generated_access_token_to_identity_info; -use crate::http::shared::AppState; -use crate::shard::transmission::frame::ShardResponse; -use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; -use crate::streaming::session::Session; -use crate::streaming::users::user::User; -use ::iggy_common::change_password::ChangePassword; -use ::iggy_common::create_user::CreateUser; -use ::iggy_common::update_permissions::UpdatePermissions; -use ::iggy_common::update_user::UpdateUser; -use axum::extract::{Path, State}; -use axum::http::StatusCode; -use axum::routing::{delete, get, post, put}; -use axum::{Extension, Json, Router, debug_handler}; -use err_trail::ErrContext; -use iggy_binary_protocol::WireName; -use iggy_binary_protocol::requests::users::{ - ChangePasswordRequest as WireChangePassword, CreateUserRequest as WireCreateUser, - DeleteUserRequest as WireDeleteUser, UpdatePermissionsRequest as WireUpdatePermissions, - UpdateUserRequest as WireUpdateUser, -}; -use iggy_common::Identifier; -use iggy_common::IdentityInfo; -use iggy_common::Validatable; -use iggy_common::login_user::LoginUser; -use iggy_common::wire_conversions::{identifier_to_wire, permissions_to_wire}; -use iggy_common::{IggyError, UserInfo, UserInfoDetails}; -use secrecy::ExposeSecret; -use send_wrapper::SendWrapper; -use serde::Deserialize; -use std::sync::Arc; -use tracing::instrument; - -pub fn router(state: Arc) -> Router { - Router::new() - .route("/users", get(get_users).post(create_user)) - .route( - "/users/{user_id}", - get(get_user).put(update_user).delete(delete_user), - ) - .route("/users/{user_id}/permissions", put(update_permissions)) - .route("/users/{user_id}/password", put(change_password)) - .route("/users/login", post(login_user)) - .route("/users/logout", delete(logout_user)) - .route("/users/refresh-token", post(refresh_token)) - .with_state(state) -} - -#[debug_handler] -async fn get_user( - State(state): State>, - Extension(identity): Extension, - Path(user_id): Path, -) -> Result, CustomError> { - let identifier_user_id = Identifier::from_str_value(&user_id)?; - let Ok(user) = state.shard.shard().find_user(&identifier_user_id) else { - return Err(CustomError::ResourceNotFound); - }; - let Some(user) = user else { - return Err(CustomError::ResourceNotFound); - }; - - if user.id != identity.user_id { - state - .shard - .shard() - .metadata - .perm_get_user(identity.user_id)?; - } - - let user = mapper::map_user(&user); - Ok(Json(user)) -} - -#[debug_handler] -async fn get_users( - State(state): State>, - Extension(identity): Extension, -) -> Result>, CustomError> { - state - .shard - .shard() - .metadata - .perm_get_users(identity.user_id)?; - - let users = state.shard.shard().get_users(); - let user_refs: Vec<&User> = users.iter().collect(); - let users = mapper::map_users(&user_refs); - Ok(Json(users)) -} - -#[debug_handler] -#[instrument(skip_all, name = "trace_create_user", fields(iggy_user_id = identity.user_id))] -async fn create_user( - State(state): State>, - Extension(identity): Extension, - Json(command): Json, -) -> Result, CustomError> { - command.validate()?; - - let wire_command = WireCreateUser { - username: WireName::new(&command.username).map_err(|_| IggyError::InvalidUsername)?, - password: command.password.expose_secret().to_string(), - status: command.status.as_code(), - permissions: command.permissions.as_ref().map(permissions_to_wire), - }; - let request = ShardRequest::control_plane(ShardRequestPayload::CreateUserRequest { - user_id: identity.user_id, - command: wire_command, - }); - - match state.shard.send_to_control_plane(request).await? { - ShardResponse::CreateUserResponse(user) => { - let response = mapper::map_user(&user); - Ok(Json(response)) - } - ShardResponse::ErrorResponse(err) => Err(err.into()), - _ => unreachable!("Expected CreateUserResponse"), - } -} - -#[debug_handler] -#[instrument(skip_all, name = "trace_update_user", fields(iggy_user_id = identity.user_id, iggy_updated_user_id = user_id))] -async fn update_user( - State(state): State>, - Extension(identity): Extension, - Path(user_id): Path, - Json(mut command): Json, -) -> Result { - command.user_id = Identifier::from_str_value(&user_id)?; - command.validate()?; - - let wire_command = WireUpdateUser { - user_id: identifier_to_wire(&command.user_id)?, - username: command - .username - .as_deref() - .map(WireName::new) - .transpose() - .map_err(|_| IggyError::InvalidUsername)?, - status: command.status.map(|s| s.as_code()), - }; - let request = ShardRequest::control_plane(ShardRequestPayload::UpdateUserRequest { - user_id: identity.user_id, - command: wire_command, - }); - - match state.shard.send_to_control_plane(request).await? { - ShardResponse::UpdateUserResponse(_) => Ok(StatusCode::NO_CONTENT), - ShardResponse::ErrorResponse(err) => Err(err.into()), - _ => unreachable!("Expected UpdateUserResponse"), - } -} - -#[debug_handler] -#[instrument(skip_all, name = "trace_update_permissions", fields(iggy_user_id = identity.user_id, iggy_updated_user_id = user_id))] -async fn update_permissions( - State(state): State>, - Extension(identity): Extension, - Path(user_id): Path, - Json(mut command): Json, -) -> Result { - command.user_id = Identifier::from_str_value(&user_id)?; - command.validate()?; - - let wire_command = WireUpdatePermissions { - user_id: identifier_to_wire(&command.user_id)?, - permissions: command.permissions.as_ref().map(permissions_to_wire), - }; - let request = ShardRequest::control_plane(ShardRequestPayload::UpdatePermissionsRequest { - user_id: identity.user_id, - command: wire_command, - }); - - match state.shard.send_to_control_plane(request).await? { - ShardResponse::UpdatePermissionsResponse => Ok(StatusCode::NO_CONTENT), - ShardResponse::ErrorResponse(err) => Err(err.into()), - _ => unreachable!("Expected UpdatePermissionsResponse"), - } -} - -#[debug_handler] -#[instrument(skip_all, name = "trace_change_password", fields(iggy_user_id = identity.user_id, iggy_updated_user_id = user_id))] -async fn change_password( - State(state): State>, - Extension(identity): Extension, - Path(user_id): Path, - Json(mut command): Json, -) -> Result { - command.user_id = Identifier::from_str_value(&user_id)?; - command.validate()?; - - let wire_command = WireChangePassword { - user_id: identifier_to_wire(&command.user_id)?, - current_password: command.current_password.expose_secret().to_string(), - new_password: command.new_password.expose_secret().to_string(), - }; - let request = ShardRequest::control_plane(ShardRequestPayload::ChangePasswordRequest { - user_id: identity.user_id, - command: wire_command, - }); - - match state.shard.send_to_control_plane(request).await? { - ShardResponse::ChangePasswordResponse => Ok(StatusCode::NO_CONTENT), - ShardResponse::ErrorResponse(err) => Err(err.into()), - _ => unreachable!("Expected ChangePasswordResponse"), - } -} - -#[debug_handler] -#[instrument(skip_all, name = "trace_delete_user", fields(iggy_user_id = identity.user_id, iggy_deleted_user_id = user_id))] -async fn delete_user( - State(state): State>, - Extension(identity): Extension, - Path(user_id): Path, -) -> Result { - let user_id = Identifier::from_str_value(&user_id)?; - - let wire_command = WireDeleteUser { - user_id: identifier_to_wire(&user_id)?, - }; - let request = ShardRequest::control_plane(ShardRequestPayload::DeleteUserRequest { - user_id: identity.user_id, - command: wire_command, - }); - - match state.shard.send_to_control_plane(request).await? { - ShardResponse::DeleteUserResponse(_) => Ok(StatusCode::NO_CONTENT), - ShardResponse::ErrorResponse(err) => Err(err.into()), - _ => unreachable!("Expected DeleteUserResponse"), - } -} - -#[debug_handler] -#[instrument(skip_all, name = "trace_login_user")] -async fn login_user( - State(state): State>, - Json(command): Json, -) -> Result, CustomError> { - let user = state - .shard - .shard() - .login_user(&command.username, command.password.expose_secret(), None) - .error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - failed to login, username: {}", - command.username - ) - })?; - let tokens = state.jwt_manager.generate(user.id)?; - Ok(Json(map_generated_access_token_to_identity_info(tokens))) -} - -#[debug_handler] -#[instrument(skip_all, name = "trace_logout_user", fields(iggy_user_id = identity.user_id))] -async fn logout_user( - State(state): State>, - Extension(identity): Extension, -) -> Result { - let session = Session::stateless(identity.user_id, identity.ip_address); - state - .shard - .shard() - .logout_user(&session) - .error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - failed to logout, user ID: {}", - identity.user_id - ) - })?; - - { - let revoke_token_future = SendWrapper::new( - state - .jwt_manager - .revoke_token(&identity.token_id, identity.token_expiry), - ); - - revoke_token_future.await.error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - failed to revoke token, user ID: {}", - identity.user_id - ) - })?; - } - - Ok(StatusCode::NO_CONTENT) -} - -#[debug_handler] -async fn refresh_token( - State(state): State>, - Json(command): Json, -) -> Result, CustomError> { - let token = { - let refresh_token_future = - SendWrapper::new(state.jwt_manager.refresh_token(&command.token)); - - refresh_token_future - .await - .error(|e: &IggyError| format!("{COMPONENT} (error: {e}) - failed to refresh token"))? - }; - - Ok(Json(map_generated_access_token_to_identity_info(token))) -} - -#[derive(Debug, Deserialize)] -struct RefreshToken { - token: String, -} diff --git a/core/server/src/http/web.rs b/core/server/src/http/web.rs deleted file mode 100644 index 76e740e701..0000000000 --- a/core/server/src/http/web.rs +++ /dev/null @@ -1,83 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use axum::Router; -use axum::body::Body; -use axum::extract::Path; -use axum::http::{Response, StatusCode, header}; -use axum::response::IntoResponse; -use axum::routing::get; -use rust_embed::{Embed, EmbeddedFile}; - -#[derive(Embed)] -#[folder = "../../web/build/static/"] -#[allow_missing = true] -struct WebAssets; - -impl WebAssets { - fn get_file(path: &str) -> Option { - ::get(path) - } -} - -pub fn router() -> Router { - Router::new() - .route("/ui/{*wildcard}", get(serve_web_asset)) - .route("/ui", get(serve_index)) - .route("/ui/", get(serve_index)) -} - -async fn serve_index() -> impl IntoResponse { - serve_file("index.html") -} - -async fn serve_web_asset(Path(wildcard): Path) -> impl IntoResponse { - if let Some(response) = try_serve_file(&wildcard) { - return response; - } - - if !wildcard.contains('.') { - return serve_file("index.html"); - } - - Response::builder() - .status(StatusCode::NOT_FOUND) - .body(Body::from("Not Found")) - .unwrap() -} - -fn try_serve_file(path: &str) -> Option> { - let asset = WebAssets::get_file(path)?; - let mime = mime_guess::from_path(path).first_or_octet_stream(); - - Some( - Response::builder() - .status(StatusCode::OK) - .header(header::CONTENT_TYPE, mime.as_ref()) - .body(Body::from(asset.data.into_owned())) - .unwrap(), - ) -} - -fn serve_file(path: &str) -> Response { - try_serve_file(path).unwrap_or_else(|| { - Response::builder() - .status(StatusCode::NOT_FOUND) - .body(Body::from("Not Found")) - .unwrap() - }) -} diff --git a/core/server-ng/src/http/wire.rs b/core/server/src/http/wire.rs similarity index 97% rename from core/server-ng/src/http/wire.rs rename to core/server/src/http/wire.rs index e96866f80a..c7026d09f5 100644 --- a/core/server-ng/src/http/wire.rs +++ b/core/server/src/http/wire.rs @@ -16,7 +16,7 @@ // under the License. //! HTTP -> wire request mappers: produce/poll/consumer-offset encoders and -//! the control-plane [`Message`] builder shared by the write path. +//! the control-plane [`Message`] builder shared by the write path. use bytes::{Bytes, BytesMut}; use iggy_binary_protocol::consensus::{Command2, HEADER_SIZE}; @@ -28,7 +28,7 @@ use iggy_binary_protocol::requests::consumer_offsets::{ use iggy_binary_protocol::requests::messages::{ PollMessagesRequest, RawMessage, SendMessagesEncoder, }; -use iggy_binary_protocol::{AckLevel, Operation, RequestHeader}; +use iggy_binary_protocol::{AckLevel, Operation, RoutedRequestHeader}; use iggy_common::get_consumer_offset::GetConsumerOffset; use iggy_common::poll_messages::DEFAULT_PARTITION_ID; use iggy_common::store_consumer_offset::StoreConsumerOffset; @@ -179,7 +179,7 @@ pub(in crate::http) const fn resync_required_polled_messages() -> PolledMessages } } -/// Build a `Message` for a control-plane write by filling a zeroed +/// Build a `Message` for a control-plane write by filling a zeroed /// `#[repr(C)]` header, mirroring `wire::rewrite_request_body` and the partition /// reconciler's prepare builder. `body` is the already-encoded wire request, /// copied in after the header. @@ -189,14 +189,14 @@ pub(in crate::http) fn build_request_message( session_id: u64, request_id: u64, body: &[u8], -) -> Message { +) -> Message { let total = HEADER_SIZE + body.len(); - let mut message = Message::::new(total); + let mut message = Message::::new(total); message.as_mut_slice()[HEADER_SIZE..].copy_from_slice(body); - let header = bytemuck::checked::try_from_bytes_mut::( + let header = bytemuck::checked::try_from_bytes_mut::( &mut message.as_mut_slice()[..HEADER_SIZE], ) - .expect("zeroed bytes form a valid RequestHeader"); + .expect("zeroed bytes form a valid RoutedRequestHeader"); header.command = Command2::Request; header.operation = operation; header.client = client_id; diff --git a/core/server/src/io/mod.rs b/core/server/src/io/mod.rs deleted file mode 100644 index 4efe7ebd13..0000000000 --- a/core/server/src/io/mod.rs +++ /dev/null @@ -1,20 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub use server_common::fs_utils; - -pub mod storage; diff --git a/core/server/src/io/storage.rs b/core/server/src/io/storage.rs deleted file mode 100644 index c065e50081..0000000000 --- a/core/server/src/io/storage.rs +++ /dev/null @@ -1,171 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::future::Future; - -use compio::{ - buf::{IoBuf, IoBufMut}, - io::{AsyncReadAtExt, AsyncWriteAtExt}, -}; - -pub trait Storage { - fn read_exact_at( - &self, - buf: B, - pos: u64, - ) -> impl Future>; - fn write_all_at( - &self, - buf: B, - pos: u64, - ) -> impl Future>; -} - -pub struct OpenOpts { - keep_fd: bool, - path: String, -} - -impl OpenOpts { - pub fn ephemeral(path: String) -> Self { - Self { - keep_fd: false, - path, - } - } - - pub fn permanent(path: String) -> Self { - Self { - keep_fd: true, - path, - } - } -} - -pub enum StorageImpl { - Block(BlockStorage), -} - -impl StorageImpl { - pub fn read_exact_at( - &self, - buf: B, - pos: u64, - ) -> impl Future> { - match self { - StorageImpl::Block(storage) => storage.read_exact_at(buf, pos), - } - } - - pub fn write_all_at( - &mut self, - buf: B, - pos: u64, - ) -> impl Future> { - match self { - StorageImpl::Block(storage) => storage.write_all_at(buf, pos), - } - } -} - -pub struct BlockStorage { - file: Option, - path: Option, -} - -impl BlockStorage { - pub async fn xd(&self) { - let file = self.file.as_ref().unwrap(); - let buf = Vec::new(); - (&*file).write_all_at(buf, 0).await.unwrap(); - } -} - -impl BlockStorage { - pub async fn new(opts: OpenOpts) -> Result { - let path = opts.path; - let keep_fd = opts.keep_fd; - let file = if keep_fd { - let file = compio::fs::OpenOptions::new() - .create(true) - .read(true) - .write(true) - .open(&path) - .await?; - Some(file) - } else { - None - }; - let path = if file.is_some() { None } else { Some(path) }; - Ok(Self { file, path }) - } -} - -impl Storage for BlockStorage { - async fn read_exact_at(&self, buf: B, pos: u64) -> Result { - let (result, buf) = match &self.file { - Some(file) => file.read_exact_at(buf, pos).await.into(), - None => { - let path = self.path.as_ref().unwrap(); - let file = compio::fs::File::open(path).await?; - file.read_exact_at(buf, pos).await.into() - } - }; - result?; - Ok(buf) - } - - async fn write_all_at(&self, buf: B, pos: u64) -> Result { - let (result, buf) = match self.file { - Some(ref file) => (&*file).write_all_at(buf, pos).await.into(), - None => { - let path = self.path.as_ref().unwrap(); - let mut file = compio::fs::OpenOptions::new() - .create(true) - .write(true) - .open(path) - .await?; - file.write_all_at(buf, pos).await.into() - } - }; - result?; - Ok(buf) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn open_opts_ephemeral_creates_correct_config() { - let path = "path/to/file".to_string(); - let opts = OpenOpts::ephemeral(path.clone()); - - assert!(!opts.keep_fd); - assert_eq!(opts.path, path); - } - - #[test] - fn open_opts_permanent_creates_correct_config() { - let path = "path/to/file".to_string(); - let opts = OpenOpts::permanent(path.clone()); - - assert!(opts.keep_fd); - assert_eq!(opts.path, path); - } -} diff --git a/core/server/src/lib.rs b/core/server/src/lib.rs index feddf428d6..5ad252b5d5 100644 --- a/core/server/src/lib.rs +++ b/core/server/src/lib.rs @@ -15,39 +15,35 @@ // specific language governing permissions and limitations // under the License. -#[cfg(not(feature = "disable-mimalloc"))] -use mimalloc::MiMalloc; +#![allow(clippy::future_not_send)] use iggy_common::SemanticVersion; -#[cfg(not(feature = "disable-mimalloc"))] -#[global_allocator] -static GLOBAL: MiMalloc = MiMalloc; - -#[cfg(windows)] -compile_error!("iggy-server doesn't support windows."); +pub const VERSION: &str = env!("CARGO_PKG_VERSION"); +pub const SEMANTIC_VERSION: SemanticVersion = SemanticVersion::parse_const(VERSION); -pub mod args; -pub mod binary; +pub mod auth; pub mod bootstrap; -pub(crate) mod compat; -pub mod configs; -pub mod diagnostics; -pub mod http; -pub mod io; -pub mod metadata; -pub mod quic; -pub mod sender; +pub(crate) mod cluster_meta; +pub mod config_writer; +pub mod consumer_group; +pub mod dispatch; +pub(crate) mod http; +pub mod login_register; +pub(crate) mod offset_recovery; +pub mod partition_helpers; +pub mod partition_reconciler; +pub mod pat; +pub(crate) mod personal_access_token_cleaner; +pub mod responses; +pub(crate) mod segment_cleaner; +pub(crate) mod segment_recovery; pub mod server_error; -pub mod shard; -pub mod state; -pub mod streaming; -pub mod tcp; -pub mod websocket; - -pub use server_common::log; - -pub const VERSION: &str = env!("CARGO_PKG_VERSION"); -pub const SEMANTIC_VERSION: SemanticVersion = SemanticVersion::parse_const(VERSION); -pub const IGGY_ROOT_USERNAME_ENV: &str = "IGGY_ROOT_USERNAME"; -pub const IGGY_ROOT_PASSWORD_ENV: &str = "IGGY_ROOT_PASSWORD"; +pub mod session_manager; +pub(crate) mod snapshot; +#[cfg(feature = "systemd")] +pub mod systemd; +pub mod users; +#[cfg(feature = "iggy-web")] +pub(crate) mod web; +pub mod wire; diff --git a/core/server-ng/src/login_register.rs b/core/server/src/login_register.rs similarity index 100% rename from core/server-ng/src/login_register.rs rename to core/server/src/login_register.rs diff --git a/core/server/src/main.rs b/core/server/src/main.rs index 1f0ac18d90..f6a8e63e36 100644 --- a/core/server/src/main.rs +++ b/core/server/src/main.rs @@ -15,549 +15,86 @@ // specific language governing permissions and limitations // under the License. -use anyhow::Result; +#![allow(clippy::future_not_send)] + +mod args; + +use args::Args; use clap::Parser; -use dashmap::DashMap; -use dotenvy::dotenv; -use err_trail::ErrContext; -use figlet_rs::FIGlet; -use iggy_common::SemanticVersion; -use iggy_common::{Aes256GcmEncryptor, EncryptorKind, IggyError}; -use server::SEMANTIC_VERSION; -use server::args::Args; +use configs::server::ServerConfig; use server::bootstrap::{ - create_directories, create_shard_connections, create_shard_executor, load_config, - load_metadata, resolve_persister, update_system_info, + apply_default_root_credentials, bootstrap, load_config, prepare_runtime_dirs, }; -use server::diagnostics::{ASYNCIFY_POOL_DISABLED_PANIC_MSG, print_incomplete_io_uring_ops_info}; -use server::io::fs_utils; -use server::log::logger::Logging; -use server::metadata::{Metadata, create_metadata_handles}; use server::server_error::ServerError; -use server::shard::system::info::SystemInfo; -use server::shard::{IggyShard, calculate_shard_assignment}; -use server::state::file::FileState; -use server::state::system::SystemState; -use server::streaming::clients::client_manager::{Client, ClientManager}; -use server::streaming::diagnostics::metrics::Metrics; -use server::streaming::storage::SystemStorage; -use server::streaming::utils::ptr::EternalPtr; -use server_common::MemoryPool; -use server_common::log::{LoggingSettings, TelemetrySettings}; -use server_common::sharding::{IggyNamespace, PartitionLocation, ShardId}; -use shard_allocator::ShardAllocator; -use std::panic::AssertUnwindSafe; -use std::rc::Rc; -use std::str::FromStr; -use std::sync::Arc; -use std::sync::atomic::{AtomicBool, AtomicU32, AtomicU64, Ordering}; -use std::sync::mpsc; -use std::thread::JoinHandle; +use server_common::log::Logging; use system_stats::capture_allowed_cpus; -use tracing::{error, info, instrument, warn}; - -const COMPONENT: &str = "MAIN"; -const SHARDS_TABLE_CAPACITY: usize = 16384; - -static SHUTDOWN_START_TIME: AtomicU64 = AtomicU64::new(0); -static SHUTDOWN_INITIATED: AtomicBool = AtomicBool::new(false); -// Separate latch from the ring-setup one inside -// `enrich_runtime_create_error`: a shard that fails ring setup (e.g. partial -// ENOMEM under a tight RLIMIT_MEMLOCK) must not consume the latch and -// suppress the unsupported-opcode diagnostic from a sibling shard that did -// start. Setup vs runtime io_uring failures can co-occur across shards. -static SHARD_RUNTIME_DIAGNOSTIC: std::sync::Once = std::sync::Once::new(); - -enum ShardExitStatus { - Success, - Error(String), - Panic(String), -} - -fn initiate_shutdown( - reason: &str, - shutdown_handles: &[(u16, server::shard::transmission::connector::StopSender)], -) { - if SHUTDOWN_INITIATED.swap(true, Ordering::SeqCst) { - return; - } - - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_millis() as u64) - .unwrap_or(0); - SHUTDOWN_START_TIME.store(now, Ordering::SeqCst); - - info!("{reason}, initiating graceful shutdown..."); +use tracing::{error, info}; - for (shard_id, stop_sender) in shutdown_handles { - if let Err(e) = stop_sender.try_send(()) { - error!("Failed to send shutdown signal to shard {shard_id}: {e}"); - } +fn main() -> Result<(), ServerError> { + // This prelude must stay ahead of the first thread the process ever + // spawns: `--with-default-root-credentials` writes to the environment, + // and `set_var` is only sound while single-threaded. `early_init` just + // installs the tracing registry (buffered until `late_init`) and starts + // no worker of its own, so it can run here and make the warnings below + // visible. `create_shard_executor` also reads its capacity knob from the + // environment, which is why the `.env` load has to precede it. + let args = Args::parse(); + // `logging` owns the tracing appender worker guards; it must outlive the + // shard threads or every log line after bootstrap is silently dropped. + let mut logging = Logging::new(server::VERSION); + logging.early_init(); + server_common::print_build_info!(server::VERSION); + if let Ok(env_path) = std::env::var("IGGY_ENV_PATH") { + let _ = dotenvy::from_path(&env_path); + } else { + let _ = dotenvy::dotenv(); } -} - -fn extract_panic_message(payload: Box) -> String { - payload - .downcast_ref::<&str>() - .map(|s| s.to_string()) - .or_else(|| payload.downcast_ref::().cloned()) - .unwrap_or_else(|| "unknown panic".to_string()) -} + // SAFETY: no thread has been spawned yet, see the comment above. + unsafe { apply_default_root_credentials(args.with_default_root_credentials) }; -fn print_ascii_art(text: &str) { - let standard_font = FIGlet::standard().unwrap(); - let figure = standard_font.convert(text); - println!("{}", figure.unwrap()); -} - -#[instrument(skip_all, name = "trace_start_server")] -fn main() -> Result<(), ServerError> { // Before shard threads pin themselves: a pinned capture sees one core. capture_allowed_cpus(); - let rt = match compio::runtime::Runtime::new() { + let bootstrap_runtime = match server_common::create_shard_executor() { Ok(rt) => rt, Err(e) => { let e = server_common::diagnostics::enrich_runtime_create_error(e); - panic!("Cannot create runtime: {e}"); + panic!("Cannot create server bootstrap executor: {e}"); } }; - rt.block_on(async move { - if let Ok(env_path) = std::env::var("IGGY_ENV_PATH") { - if dotenvy::from_path(&env_path).is_ok() { - println!("Loaded environment variables from path: {env_path}"); - } - } else if let Ok(path) = dotenv() { - println!( - "Loaded environment variables from .env file at path: {}", - path.display() - ); - } - let args = Args::parse(); - print_ascii_art("Iggy Server"); - - let is_follower = args.follower; - let replica_id = args.replica_id; - - // FIRST DISCRETE LOADING STEP. - // Initialize early logging before config parsing so we can log during bootstrap. - let mut logging = Logging::new(server::VERSION); - logging.early_init(); - server_common::print_build_info!(server::VERSION); - - // SECOND DISCRETE LOADING STEP. - // Load config and create directories. - // Remove `local_data` directory if run with `--fresh` flag. - let config = load_config().await.error(|e: &ServerError| { - format!("{COMPONENT} (error: {e}) - failed to load config during bootstrap") - })?; - if args.fresh { - let system_path = config.system.get_system_path(); - if compio::fs::metadata(&system_path).await.is_ok() { - warn!( - "Removing system path at: {} because `--fresh` flag was set", - system_path - ); - if let Err(e) = fs_utils::remove_dir_all(&system_path).await { - warn!("Failed to remove system path at {system_path}: {e}"); - } - } - } - - // THIRD DISCRETE LOADING STEP. - // Create directories. - create_directories(&config.system).await?; - - // FOURTH DISCRETE LOADING STEP. - // Complete logging setup with config (file output, telemetry). - // From this point on, logs are persisted to file and telemetry is active. - logging.late_init( - config.system.get_system_path(), - &LoggingSettings::from(&config.system.logging), - &TelemetrySettings::from(&config.telemetry), - )?; - - if is_follower { - info!("Server is running in FOLLOWER mode for testing leader redirection"); - } - - if args.with_default_root_credentials { - let username_set = std::env::var("IGGY_ROOT_USERNAME").is_ok(); - let password_set = std::env::var("IGGY_ROOT_PASSWORD").is_ok(); - - if !username_set || !password_set { - if !username_set { - unsafe { - std::env::set_var("IGGY_ROOT_USERNAME", "iggy"); - } - } - if !password_set { - unsafe { - std::env::set_var("IGGY_ROOT_PASSWORD", "iggy"); - } - } - info!( - "Using default root credentials (username: iggy, password: iggy) - FOR DEVELOPMENT ONLY! \ - If root user already exists, existing credentials will be reused." - ); - } else { - warn!( - "--with-default-root-credentials flag is ignored because root credentials are already set via environment variables" - ); - } - } - - // FIFTH DISCRETE LOADING STEP. - MemoryPool::init_pool(&config.system.memory_pool.into_other()); - - // SIXTH DISCRETE LOADING STEP. - let partition_persister = resolve_persister(config.system.partition.enforce_fsync); - let storage = SystemStorage::new(config.system.clone(), partition_persister); - - // SEVENTH DISCRETE LOADING STEP. - let current_version = SEMANTIC_VERSION; - info!("Current semantic version: {:?}", current_version); - - let mut system_info; - let load_system_info = storage.info.load().await; - match load_system_info { - Ok(info) => { - system_info = info; - } - Err(e) => { - if let IggyError::ResourceNotFound(_) = e { - info!("System info not found, creating..."); - system_info = SystemInfo::default(); - update_system_info(&storage, &mut system_info, ¤t_version).await?; - } else { - panic!("Failed to load system info from disk. {e}"); - } - } - } - info!("Loaded {system_info}."); - let loaded_version = SemanticVersion::from_str(&system_info.version.version)?; - if current_version.is_equal_to(&loaded_version) { - info!("System version {current_version} is up to date."); - } else if current_version.is_greater_than(&loaded_version) { - info!( - "System version {current_version} is greater than {loaded_version}, checking the available migrations..." - ); - update_system_info(&storage, &mut system_info, ¤t_version).await?; - } else { - info!( - "System version {current_version} is lower than {loaded_version}, possible downgrade." - ); - update_system_info(&storage, &mut system_info, ¤t_version).await?; - } - - // EIGHTH DISCRETE LOADING STEP. - info!( - "Server-side encryption is {}.", - match config.system.encryption.enabled { - true => "enabled", - false => "disabled", - } - ); - let encryptor: Option = match config.system.encryption.enabled { - true => Some(EncryptorKind::Aes256Gcm( - Aes256GcmEncryptor::from_base64_key(&config.system.encryption.key).unwrap(), - )), - false => None, - }; - - // TENTH DISCRETE LOADING STEP. - let state_persister = resolve_persister(config.system.state.enforce_fsync); - let state_current_index = Arc::new(AtomicU64::new(0)); - let state_entries_count = Arc::new(AtomicU64::new(0)); - let state_current_leader = Arc::new(AtomicU32::new(0)); - let state_term = Arc::new(AtomicU64::new(0)); - let state = FileState::new( - &config.system.get_state_messages_file_path(), - ¤t_version, - state_persister, - encryptor.clone(), - state_current_index.clone(), - state_entries_count.clone(), - state_current_leader.clone(), - state_term.clone(), - ); - let state = SystemState::load(state).await?; - let (streams_state, users_state) = state.decompose(); - - // Create left-right handles for metadata - let (mut metadata_writer, metadata_reader) = create_metadata_handles(); - // Create shared metadata reader - each shard will get a clone - let metadata = Metadata::new(metadata_reader); - - // Load initial metadata using the writer - load_metadata( - users_state.into_values(), - streams_state.into_values(), - &mut metadata_writer, - ); - - // ELEVENTH DISCRETE LOADING STEP. - let shard_allocator = ShardAllocator::new( - &config.system.sharding.cpu_allocation, - config.system.sharding.pin_cores, - )?; - let shard_assignment = shard_allocator.to_shard_assignments()?; - - #[cfg(feature = "disable-mimalloc")] - warn!("Using default system allocator because code was build with `disable-mimalloc` feature"); - #[cfg(not(feature = "disable-mimalloc"))] - info!("Using mimalloc allocator"); - - // DISCRETE STEP. - // Increment the metrics. - let metrics = Metrics::init(); - - // TWELFTH DISCRETE LOADING STEP. - info!( - "Enable TCP socket migration across shards: {}.", - config.tcp.socket_migration - ); - - info!("Starting {} shard(s)", shard_assignment.len()); - let (connections, shutdown_handles) = create_shard_connections(&shard_assignment); - let shards_count = shard_assignment.len(); - let mut handles: Vec> = Vec::with_capacity(shards_count); - - // Channel for shard completion notifications - let (shard_done_tx, shard_done_rx) = mpsc::channel::<(u16, ShardExitStatus)>(); - - // TODO: Persist the shards table and load it from the disk, so it does not have to be - // THIRTEENTH DISCRETE LOADING STEP. - // Shared resources bootstrap. - let shards_table = Box::new(DashMap::with_capacity(SHARDS_TABLE_CAPACITY)); - let shards_table = Box::leak(shards_table); - let shards_table: EternalPtr> = shards_table.into(); - - let client_manager = Box::new(DashMap::new()); - let client_manager = Box::leak(client_manager); - let client_manager: EternalPtr> = client_manager.into(); - let client_manager = ClientManager::new(client_manager); - - // Populate shards_table from SharedMetadata partitions (hierarchical traversal) - metadata.with_metadata(|metadata| { - for (stream_id, stream_meta) in metadata.streams.iter() { - for (topic_id, topic_meta) in stream_meta.topics.iter() { - for (partition_id, _partition_meta) in topic_meta.partitions.iter().enumerate() { - let ns = IggyNamespace::new(stream_id, topic_id, partition_id); - let shard_id = ShardId::new(calculate_shard_assignment( - &ns, - shard_assignment.len() as u32, - )); - // epoch is reconciler-only; unused by legacy server. - let location = PartitionLocation::new(shard_id, 0); - shards_table.insert(ns, location); - } - } - } - }); - - // Wrap metadata_writer in Option so we can take it for shard 0 - let mut metadata_writer_opt = Some(metadata_writer); - - for (id, assignment) in shard_assignment - .into_iter() - .enumerate() - .map(|(idx, assignment)| (idx as u16, assignment)) - { - let shards_table = shards_table.clone(); - let connections = connections.clone(); - let config = config.clone(); - let encryptor = encryptor.clone(); - let metrics = metrics.clone(); - let current_version = current_version.clone(); - let state_persister = resolve_persister(config.system.state.enforce_fsync); - let state = FileState::new( - &config.system.get_state_messages_file_path(), - ¤t_version, - state_persister, - encryptor.clone(), - state_current_index.clone(), - state_entries_count.clone(), - state_current_leader.clone(), - state_term.clone(), - ); - let client_manager = client_manager.clone(); - let shard_metadata = metadata.clone(); - - // Take metadata_writer for shard 0 only - let shard_metadata_writer = if id == 0 { - metadata_writer_opt.take() - } else { - None - }; - - let shard_done_tx = shard_done_tx.clone(); - let handle = std::thread::Builder::new() - .name(format!("shard-{id}")) - .spawn(move || { - let result = std::panic::catch_unwind(AssertUnwindSafe(|| { - if let Err(e) = assignment.bind_cpu() { - error!("Failed to bind cpu: {e:?}"); - } - - if let Err(e) = assignment.bind_memory() { - error!("Failed to bind memory: {e:?}"); - } - - let rt = match create_shard_executor() { - Ok(rt) => rt, - Err(e) => { - // Prints the verbose remediation once across - // all shard threads; the panic message itself - // carries the one-line fix. - let e = - server_common::diagnostics::enrich_runtime_create_error(e); - panic!("Cannot create shard-{id} executor: {e}"); - } - }; - rt.block_on(async move { - let mut builder = IggyShard::builder(); - builder = builder - .id(id) - .state(state) - .shards_table(shards_table) - .connections(connections) - .clients_manager(client_manager) - .config(config) - .encryptor(encryptor) - .version(current_version) - .metrics(metrics) - .is_follower(is_follower) - .current_replica_id(replica_id) - .metadata(shard_metadata); - - if let Some(writer) = shard_metadata_writer { - builder = builder.metadata_writer(writer); - } - - let shard = builder.build(); - - let shard = Rc::new(shard); - - if let Err(e) = shard.run().await { - error!("Failed to run shard-{id}: {e}"); - return Err(e.to_string()); - } - info!("Shard {id} run completed"); - - Ok(()) - }) - })); - - let status = match result { - Ok(Ok(())) => ShardExitStatus::Success, - Ok(Err(msg)) => ShardExitStatus::Error(msg), - Err(panic_payload) => { - ShardExitStatus::Panic(extract_panic_message(panic_payload)) - } - }; - - let _ = shard_done_tx.send((id, status)); - }) - .unwrap_or_else(|e| panic!("Failed to spawn thread for shard-{id}: {e}")); - handles.push(handle); - } - - drop(shard_done_tx); - - let shutdown_handles_for_signal = shutdown_handles.clone(); - ctrlc::set_handler(move || { - initiate_shutdown( - "Received shutdown signal (SIGTERM/SIGINT)", - &shutdown_handles_for_signal, - ); - }) - .expect("Error setting Ctrl-C handler"); - - info!("Iggy server is running. Press Ctrl+C or send SIGTERM to shutdown."); - - let mut completed_shards = 0usize; - let mut failure_message: Option = None; - - while completed_shards < shards_count { - match shard_done_rx.recv() { - Ok((shard_id, status)) => { - completed_shards += 1; - - match status { - ShardExitStatus::Success => { - info!("Shard {shard_id} exited successfully"); - } - ShardExitStatus::Error(msg) => { - error!("Shard {shard_id} exited with error: {msg}"); - if failure_message.is_none() { - failure_message = - Some(format!("Shard {shard_id} exited with error: {msg}")); - } - initiate_shutdown( - &format!("Shard {shard_id} exited with error"), - &shutdown_handles, - ); - } - ShardExitStatus::Panic(msg) => { - error!("Shard {shard_id} panicked: {msg}"); - if msg.contains(ASYNCIFY_POOL_DISABLED_PANIC_MSG) { - SHARD_RUNTIME_DIAGNOSTIC - .call_once(print_incomplete_io_uring_ops_info); - } - if failure_message.is_none() { - failure_message = - Some(format!("Shard {shard_id} panicked: {msg}")); - } - initiate_shutdown( - &format!("Shard {shard_id} panicked"), - &shutdown_handles, - ); - } - } - } - Err(_) => { - error!("Shard completion channel closed unexpectedly"); - break; - } - } - } - - for (idx, handle) in handles.into_iter().enumerate() { - if let Err(e) = handle.join() { - warn!("Shard {idx} thread join returned panic: {e:?}"); - } - } - - let shutdown_duration_msg = { - let start_time = SHUTDOWN_START_TIME.load(Ordering::SeqCst); - if start_time > 0 { - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_millis() as u64) - .unwrap_or(0); - let elapsed = now.saturating_sub(start_time); - format!(" (shutdown took {} ms)", elapsed) - } else { - String::new() - } - }; - - if let Some(msg) = failure_message { - error!( - "Server shutting down due to shard failure.{}", - shutdown_duration_msg - ); - return Err(ServerError::ShardFailure { message: msg }); - } - - info!( - "All shards have shut down. Iggy server is exiting.{}", - shutdown_duration_msg - ); + // Bootstrap on a temporary runtime: load config, prepare the data + // directory, init the memory pool. Then drop the runtime and spawn the + // per-shard runtimes - each shard thread builds its OWN + // `compio::runtime::Runtime` via `create_shard_executor`, pinned to + // its CPU. + let bootstrap_result: Result = bootstrap_runtime.block_on(async { + let config = load_config().await?; + prepare_runtime_dirs(&config, &mut logging, args.fresh).await?; + server_common::MemoryPool::init_pool(&config.system.memory_pool.into_other()); + + Ok(config) + }); + let config = bootstrap_result?; + drop(bootstrap_runtime); + + let shards = bootstrap(config, args.replica_id)?; + if let Err(error) = shards.install_ctrlc_handler() { + // Without a working SIGINT handler the server has no way to + // observe an operator Ctrl-C and the shutdown flag would never + // flip, leaving shard threads parked indefinitely. Fail fast + // rather than boot into an un-killable state. + error!(error = %error, "failed to install Ctrl-C handler; aborting boot"); + std::process::exit(1); + } - Ok(()) - }) + info!("server running; waiting on shard threads"); + let joined = shards.join_all(); + #[cfg(feature = "systemd")] + if let Err(error) = &joined { + server::systemd::notify_shutdown_failure(error); + } + joined?; + info!("server shutdown complete"); + Ok(()) } diff --git a/core/server/src/metadata/absorb.rs b/core/server/src/metadata/absorb.rs deleted file mode 100644 index a90734f2aa..0000000000 --- a/core/server/src/metadata/absorb.rs +++ /dev/null @@ -1,500 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::metadata::ConsumerGroupMemberMeta; -use crate::metadata::inner::InnerMetadata; -use crate::metadata::ops::MetadataOp; -use crate::metadata::{StreamId, UserId}; -use crate::streaming::polling_consumer::ConsumerGroupId; -use iggy_common::Permissions; -use left_right::Absorb; -use std::sync::atomic::Ordering; - -impl Absorb for InnerMetadata { - fn absorb_first(&mut self, op: &mut MetadataOp, other: &Self) { - apply_op(self, op, true, other); - } - - fn absorb_second(&mut self, op: MetadataOp, other: &Self) { - apply_op(self, &op, false, other); - } - - fn sync_with(&mut self, first: &Self) { - *self = first.clone(); - } -} - -fn apply_op( - metadata: &mut InnerMetadata, - op: &MetadataOp, - populate_ids: bool, - _reader_copy: &InnerMetadata, -) { - match op { - MetadataOp::Initialize(initial) => { - *metadata = (**initial).clone(); - rebuild_all_permission_indexes(metadata); - } - - MetadataOp::AddStream { meta, assigned_id } => { - let entry = metadata.streams.vacant_entry(); - let id = entry.key(); - if populate_ids { - assigned_id.store(id, Ordering::Release); - } - let mut meta = meta.clone(); - meta.id = id; - let name = meta.name.clone(); - entry.insert(meta); - metadata.stream_index.insert(name, id); - } - - MetadataOp::UpdateStream { id, new_name } => { - if let Some(stream) = metadata.streams.get_mut(*id) { - let old_name = stream.name.clone(); - stream.name = new_name.clone(); - metadata.stream_index.remove(&old_name); - metadata.stream_index.insert(new_name.clone(), *id); - } - } - - MetadataOp::DeleteStream { id } => { - if metadata.streams.contains(*id) { - let stream = metadata.streams.remove(*id); - metadata.stream_index.remove(&stream.name); - clear_stream_permission_indexes(metadata, *id); - } - } - - MetadataOp::AddTopic { - stream_id, - meta, - assigned_id, - } => { - if let Some(stream) = metadata.streams.get_mut(*stream_id) { - let entry = stream.topics.vacant_entry(); - let id = entry.key(); - if populate_ids { - assigned_id.store(id, Ordering::Release); - } - let mut meta = meta.clone(); - meta.id = id; - let name = meta.name.clone(); - entry.insert(meta); - stream.topic_index.insert(name, id); - } - } - - MetadataOp::UpdateTopic { - stream_id, - topic_id, - new_name, - message_expiry, - compression_algorithm, - max_topic_size, - replication_factor, - } => { - if let Some(stream) = metadata.streams.get_mut(*stream_id) - && let Some(topic) = stream.topics.get_mut(*topic_id) - { - let old_name = topic.name.clone(); - - topic.name = new_name.clone(); - topic.message_expiry = *message_expiry; - topic.compression_algorithm = *compression_algorithm; - topic.max_topic_size = *max_topic_size; - topic.replication_factor = *replication_factor; - - if old_name != *new_name { - stream.topic_index.remove(&old_name); - stream.topic_index.insert(new_name.clone(), *topic_id); - } - } - } - - MetadataOp::DeleteTopic { - stream_id, - topic_id, - } => { - if let Some(stream) = metadata.streams.get_mut(*stream_id) - && stream.topics.contains(*topic_id) - { - let topic = stream.topics.remove(*topic_id); - stream.topic_index.remove(&topic.name); - } - } - - MetadataOp::AddPartitions { - stream_id, - topic_id, - partitions, - revision_id, - } => { - if partitions.is_empty() { - return; - } - if let Some(stream) = metadata.streams.get_mut(*stream_id) - && let Some(topic) = stream.topics.get_mut(*topic_id) - { - for meta in partitions { - let mut meta = meta.clone(); - meta.id = topic.partitions.len(); - meta.revision_id = *revision_id; - topic.partitions.push(meta); - } - } - } - - MetadataOp::DeletePartitions { - stream_id, - topic_id, - count, - } => { - if *count == 0 { - return; - } - if let Some(stream) = metadata.streams.get_mut(*stream_id) - && let Some(topic) = stream.topics.get_mut(*topic_id) - { - let new_len = topic.partitions.len().saturating_sub(*count as usize); - topic.partitions.truncate(new_len); - } - } - - MetadataOp::AddUser { meta, assigned_id } => { - let entry = metadata.users.vacant_entry(); - let id = entry.key(); - if populate_ids { - assigned_id.store(id, Ordering::Release); - } - let mut meta = meta.clone(); - meta.id = id as u32; - let username = meta.username.clone(); - let permissions = meta.permissions.clone(); - entry.insert(meta); - metadata.user_index.insert(username, id as u32); - update_permission_indexes(metadata, id as u32, permissions.as_deref()); - } - - MetadataOp::UpdateUserMeta { id, meta } => { - let user_id = *id as usize; - if let Some(old_user) = metadata.users.get(user_id) - && old_user.username != meta.username - { - metadata.user_index.remove(&old_user.username); - metadata.user_index.insert(meta.username.clone(), *id); - } - if metadata.users.contains(user_id) { - let permissions = meta.permissions.clone(); - metadata.users[user_id] = meta.clone(); - update_permission_indexes(metadata, *id, permissions.as_deref()); - } - } - - MetadataOp::DeleteUser { id } => { - let user_id = *id as usize; - if metadata.users.contains(user_id) { - let user = metadata.users.remove(user_id); - metadata.user_index.remove(&user.username); - } - metadata.personal_access_tokens.remove(id); - clear_permission_indexes(metadata, *id); - } - - MetadataOp::AddPersonalAccessToken { user_id, pat } => { - metadata - .personal_access_tokens - .entry(*user_id) - .or_default() - .insert(pat.token.clone(), pat.clone()); - } - - MetadataOp::DeletePersonalAccessToken { - user_id, - token_hash, - } => { - if let Some(user_pats) = metadata.personal_access_tokens.get_mut(user_id) { - user_pats.remove(token_hash); - } - } - - MetadataOp::AddConsumerGroup { - stream_id, - topic_id, - meta, - assigned_id, - } => { - if let Some(stream) = metadata.streams.get_mut(*stream_id) - && let Some(topic) = stream.topics.get_mut(*topic_id) - { - let entry = topic.consumer_groups.vacant_entry(); - let id = entry.key(); - if populate_ids { - assigned_id.store(id, Ordering::Release); - } - let mut meta = meta.clone(); - meta.id = id; - let name = meta.name.clone(); - entry.insert(meta); - topic.consumer_group_index.insert(name, id); - } - } - - MetadataOp::DeleteConsumerGroup { - stream_id, - topic_id, - group_id, - } => { - if let Some(stream) = metadata.streams.get_mut(*stream_id) - && let Some(topic) = stream.topics.get_mut(*topic_id) - && topic.consumer_groups.contains(*group_id) - { - let group = topic.consumer_groups.remove(*group_id); - topic.consumer_group_index.remove(&group.name); - } - } - - MetadataOp::JoinConsumerGroup { - stream_id, - topic_id, - group_id, - client_id, - member_id, - valid_client_ids, - completable_revocations, - } => { - if let Some(stream) = metadata.streams.get_mut(*stream_id) - && let Some(topic) = stream.topics.get_mut(*topic_id) - && let Some(group) = topic.consumer_groups.get_mut(*group_id) - { - if let Some(valid_ids) = valid_client_ids { - let stale_members: Vec = group - .members - .iter() - .filter(|(_, m)| !valid_ids.contains(&m.client_id)) - .map(|(slot_id, _)| slot_id) - .collect(); - - for slot_id in stale_members { - group.members.remove(slot_id); - } - } - - let next_id = group - .members - .iter() - .map(|(_, m)| m.id) - .max() - .map(|m| m + 1) - .unwrap_or(0); - - if populate_ids { - member_id.store(next_id, Ordering::Release); - } - - if !group.members.iter().any(|(_, m)| m.client_id == *client_id) { - let new_member = ConsumerGroupMemberMeta::new(next_id, *client_id); - group.members.insert(new_member); - group.rebalance_cooperative(); - } - - if populate_ids { - let cg_id = ConsumerGroupId(*group_id); - let found = group.find_completable_revocations(&topic.partitions, cg_id); - if !found.is_empty() { - *completable_revocations.lock().unwrap() = found; - } - } - } - } - - MetadataOp::LeaveConsumerGroup { - stream_id, - topic_id, - group_id, - client_id, - removed_member_id, - } => { - if let Some(stream) = metadata.streams.get_mut(*stream_id) - && let Some(topic) = stream.topics.get_mut(*topic_id) - && let Some(group) = topic.consumer_groups.get_mut(*group_id) - { - let member_to_remove: Option = group - .members - .iter() - .find(|(_, m)| m.client_id == *client_id) - .map(|(id, _)| id); - - if let Some(member_id) = member_to_remove { - if populate_ids { - removed_member_id.store(member_id, Ordering::Release); - } - // Partitions owned by the leaving member - let leaving_partitions: Vec = group - .members - .get(member_id) - .map(|m| m.partitions.clone()) - .unwrap_or_default(); - - group.members.remove(member_id); - group.rebalance_members(); - - // Clear polled offsets only for the leaving member's partitions - let consumer_group_id = ConsumerGroupId(*group_id); - for partition_id in leaving_partitions { - if let Some(partition) = topic.partitions.get(partition_id) { - let guard = partition.last_polled_offsets.pin(); - guard.remove(&consumer_group_id); - } - } - } - } - } - - MetadataOp::RebalanceConsumerGroupsForTopic { - stream_id, - topic_id, - partitions_count, - } => { - if let Some(stream) = metadata.streams.get_mut(*stream_id) - && let Some(topic) = stream.topics.get_mut(*topic_id) - { - let partition_ids: Vec = (0..*partitions_count as usize).collect(); - let group_ids: Vec<_> = topic.consumer_groups.iter().map(|(id, _)| id).collect(); - - for group_id in group_ids { - if let Some(group) = topic.consumer_groups.get_mut(group_id) { - group.partitions = partition_ids.clone(); - group.rebalance_members(); - - let consumer_group_id = ConsumerGroupId(group_id); - for partition in topic.partitions.iter() { - let guard = partition.last_polled_offsets.pin(); - guard.remove(&consumer_group_id); - } - } - } - } - } - - MetadataOp::CompletePartitionRevocation { - stream_id, - topic_id, - group_id, - member_slab_id, - member_id, - partition_id, - timed_out: _, - } => { - if let Some(stream) = metadata.streams.get_mut(*stream_id) - && let Some(topic) = stream.topics.get_mut(*topic_id) - && let Some(group) = topic.consumer_groups.get_mut(*group_id) - { - // Pre-validated by maybe_complete_pending_revocation before dispatch. - group.complete_revocation(*member_slab_id, *member_id, *partition_id); - } - } - } -} - -fn clear_permission_indexes(metadata: &mut InnerMetadata, user_id: UserId) { - metadata.users_global_permissions.remove(&user_id); - metadata.users_can_poll_all_streams.remove(&user_id); - metadata.users_can_send_all_streams.remove(&user_id); - metadata - .users_stream_permissions - .retain(|(uid, _), _| *uid != user_id); - metadata - .users_can_poll_stream - .retain(|(uid, _)| *uid != user_id); - metadata - .users_can_send_stream - .retain(|(uid, _)| *uid != user_id); -} - -fn clear_stream_permission_indexes(metadata: &mut InnerMetadata, stream_id: StreamId) { - metadata - .users_stream_permissions - .retain(|(_, sid), _| *sid != stream_id); - metadata - .users_can_poll_stream - .retain(|(_, sid)| *sid != stream_id); - metadata - .users_can_send_stream - .retain(|(_, sid)| *sid != stream_id); -} - -fn update_permission_indexes( - metadata: &mut InnerMetadata, - user_id: UserId, - permissions: Option<&Permissions>, -) { - clear_permission_indexes(metadata, user_id); - - let Some(permissions) = permissions else { - return; - }; - - if permissions.global.poll_messages { - metadata.users_can_poll_all_streams.insert(user_id); - } - - if permissions.global.send_messages { - metadata.users_can_send_all_streams.insert(user_id); - } - - metadata - .users_global_permissions - .insert(user_id, permissions.global.clone()); - - let Some(streams) = &permissions.streams else { - return; - }; - - for (stream_id, stream_perm) in streams { - if stream_perm.poll_messages { - metadata.users_can_poll_stream.insert((user_id, *stream_id)); - } - - if stream_perm.send_messages { - metadata.users_can_send_stream.insert((user_id, *stream_id)); - } - - metadata - .users_stream_permissions - .insert((user_id, *stream_id), stream_perm.clone()); - } -} - -fn rebuild_all_permission_indexes(metadata: &mut InnerMetadata) { - metadata.users_global_permissions.clear(); - metadata.users_stream_permissions.clear(); - metadata.users_can_poll_all_streams.clear(); - metadata.users_can_send_all_streams.clear(); - metadata.users_can_poll_stream.clear(); - metadata.users_can_send_stream.clear(); - - let user_permissions: Vec<_> = metadata - .users - .iter() - .map(|(_, user)| (user.id, user.permissions.clone())) - .collect(); - - for (user_id, permissions) in user_permissions { - update_permission_indexes(metadata, user_id, permissions.as_deref()); - } -} diff --git a/core/server/src/metadata/consumer_group.rs b/core/server/src/metadata/consumer_group.rs deleted file mode 100644 index c619939c6f..0000000000 --- a/core/server/src/metadata/consumer_group.rs +++ /dev/null @@ -1,315 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::metadata::consumer_group_member::{ - CompletableRevocation, ConsumerGroupMemberMeta, PendingRevocation, -}; -use crate::metadata::partition::PartitionMeta; -use crate::metadata::{ConsumerGroupId, PartitionId}; -use crate::streaming::polling_consumer::ConsumerGroupId as CgId; -use iggy_common::IggyTimestamp; -use slab::Slab; -use std::collections::HashSet; -use std::sync::Arc; -use std::sync::atomic::Ordering; -use tracing::warn; - -#[derive(Clone, Debug)] -pub struct ConsumerGroupMeta { - pub id: ConsumerGroupId, - pub name: Arc, - pub partitions: Vec, - pub members: Slab, -} - -impl ConsumerGroupMeta { - /// Full rebalance: clear all assignments and redistribute round-robin. - /// Used when a member leaves or partition count changes. - pub fn rebalance_members(&mut self) { - let partition_count = self.partitions.len(); - let member_count = self.members.len(); - - if member_count == 0 || partition_count == 0 { - return; - } - - // Clear all member partitions and pending revocations - let member_ids: Vec = self.members.iter().map(|(id, _)| id).collect(); - for &member_id in &member_ids { - if let Some(member) = self.members.get_mut(member_id) { - member.partitions.clear(); - member.pending_revocations.clear(); - } - } - - // Rebuild assignments (round-robin) - for (i, &partition_id) in self.partitions.iter().enumerate() { - let member_idx = i % member_count; - if let Some(&member_id) = member_ids.get(member_idx) - && let Some(member) = self.members.get_mut(member_id) - { - member.partitions.push(partition_id); - } - } - } - - /// Cooperative rebalance: assign unassigned partitions to idle members and - /// mark excess partitions on over-assigned members as pending revocation. - pub fn rebalance_cooperative(&mut self) { - let member_count = self.members.len(); - if member_count == 0 || self.partitions.is_empty() { - return; - } - - // Find which partitions are already assigned - let mut assigned: HashSet = HashSet::new(); - for (_, member) in self.members.iter() { - for &pid in &member.partitions { - assigned.insert(pid); - } - } - - // Step 1: Assign unassigned partitions to idle members - let unassigned: Vec = self - .partitions - .iter() - .copied() - .filter(|pid| !assigned.contains(pid)) - .collect(); - - if !unassigned.is_empty() { - let idle_member_ids: Vec = self - .members - .iter() - .filter(|(_, m)| m.partitions.is_empty()) - .map(|(id, _)| id) - .collect(); - - if !idle_member_ids.is_empty() { - for (i, partition_id) in unassigned.into_iter().enumerate() { - let member_idx = i % idle_member_ids.len(); - if let Some(member) = self.members.get_mut(idle_member_ids[member_idx]) { - member.partitions.push(partition_id); - } - } - } - } - - // Step 2: Mark excess partitions as pending revocation - let partition_count = self.partitions.len(); - let fair_share = partition_count / member_count; - let remainder = partition_count % member_count; - - // Collect members already targeted by a pending revocation - let revocation_targets: HashSet = self - .members - .iter() - .flat_map(|(_, m)| { - m.pending_revocations - .iter() - .map(|revocation| revocation.target_slab_id) - }) - .collect(); - - // Collect idle members (no partitions and not already a revocation target) - let idle_slab_ids: Vec = self - .members - .iter() - .filter(|(id, m)| m.partitions.is_empty() && !revocation_targets.contains(id)) - .map(|(id, _)| id) - .collect(); - - if idle_slab_ids.is_empty() { - return; - } - - // Two-pass distribution: we first collect ALL excess partitions, then distribute - // them round-robin. This ensures even distribution when one member holds many - // partitions and multiple idle members join. Without this, the first idle member - // would receive all excess partitions while others starve. - // - // Example: 16 partitions held by 1 member, 15 idle members join - // Single-pass: member1 gets 15, members 2-15 get 0-1 each (unbalanced) - // Two-pass: each of 16 members gets exactly 1 partition - - // Pass 1: Collect excess partitions from over-assigned members - let member_ids: Vec = self.members.iter().map(|(id, _)| id).collect(); - let mut members_with_remainder = remainder; - let mut all_excess: Vec<(PartitionId, usize)> = Vec::new(); - - for &mid in &member_ids { - let Some(member) = self.members.get(mid) else { - continue; - }; - - let pending: HashSet = member - .pending_revocations - .iter() - .map(|revocation| revocation.partition_id) - .collect(); - let effective_count = member - .partitions - .iter() - .filter(|p| !pending.contains(p)) - .count(); - - let max_allowed = if members_with_remainder > 0 { - fair_share + 1 - } else { - fair_share - }; - - if effective_count <= max_allowed { - if effective_count > fair_share { - members_with_remainder = members_with_remainder.saturating_sub(1); - } - continue; - } - - let excess_count = effective_count - max_allowed; - if members_with_remainder > 0 && effective_count > fair_share { - members_with_remainder = members_with_remainder.saturating_sub(1); - } - - let revocable: Vec = member - .partitions - .iter() - .rev() - .filter(|p| !pending.contains(p)) - .copied() - .collect(); - - for partition_id in revocable.into_iter().take(excess_count) { - all_excess.push((partition_id, mid)); - } - } - - // Pass 2: Distribute collected partitions round-robin across idle members - let now = IggyTimestamp::now().as_micros(); - for (i, (partition_id, source_mid)) in all_excess.into_iter().enumerate() { - // Modulo ensures partitions cycle through idle members evenly - let idle_id = idle_slab_ids[i % idle_slab_ids.len()]; - let target_member_id = self - .members - .get(idle_id) - .map(|m| m.id) - .unwrap_or(usize::MAX); - if let Some(member) = self.members.get_mut(source_mid) { - member.pending_revocations.push(PendingRevocation { - partition_id, - target_slab_id: idle_id, - target_member_id, - created_at_micros: now, - }); - } - } - } - - /// Find revocations completable immediately (never polled or already committed). - pub fn find_completable_revocations( - &self, - partitions: &[PartitionMeta], - cg_id: CgId, - ) -> Vec { - let mut result = Vec::new(); - for (slab_id, member) in self.members.iter() { - for revocation in &member.pending_revocations { - let partition = match partitions.get(revocation.partition_id) { - Some(p) => p, - None => continue, - }; - let last_polled = { - let guard = partition.last_polled_offsets.pin(); - guard.get(&cg_id).map(|v| v.load(Ordering::Acquire)) - }; - let can_complete = match last_polled { - None => true, - Some(polled) => { - let offsets_guard = partition.consumer_group_offsets.pin(); - offsets_guard - .get(&cg_id) - .map(|offset| offset.offset.load(Ordering::Acquire)) - .is_some_and(|committed| committed >= polled) - } - }; - if can_complete { - result.push(CompletableRevocation { - slab_id, - member_id: member.id, - partition_id: revocation.partition_id, - }); - } - } - } - result - } - - /// Complete a pending revocation, moving the partition to the target member. - pub fn complete_revocation( - &mut self, - member_slab_id: usize, - member_id: usize, - partition_id: PartitionId, - ) -> bool { - let target_info = if let Some(member) = self.members.get_mut(member_slab_id) { - if member.id != member_id { - warn!( - "Revocation rejected: member ID mismatch (slab={member_slab_id}, expected={member_id}, actual={})", - member.id - ); - return false; - } - let pos = member - .pending_revocations - .iter() - .position(|revocation| revocation.partition_id == partition_id); - if let Some(pos) = pos { - let removed = member.pending_revocations.remove(pos); - member.partitions.retain(|&p| p != partition_id); - Some((removed.target_slab_id, removed.target_member_id)) - } else { - warn!( - "Revocation rejected: no pending revocation for partition={partition_id} on slab={member_slab_id}" - ); - None - } - } else { - warn!("Revocation rejected: source slab={member_slab_id} not found"); - None - }; - - if let Some((target_slab, expected_target_id)) = target_info { - if let Some(target_member) = self.members.get_mut(target_slab) { - if target_member.id != expected_target_id { - warn!( - "Revocation target slab={target_slab} reused (expected={expected_target_id}, actual={}), full rebalance", - target_member.id - ); - self.rebalance_members(); - return true; - } - target_member.partitions.push(partition_id); - return true; - } - warn!("Revocation target slab={target_slab} gone, full rebalance"); - self.rebalance_members(); - return true; - } - - false - } -} diff --git a/core/server/src/metadata/consumer_group_member.rs b/core/server/src/metadata/consumer_group_member.rs deleted file mode 100644 index 7605de0cc7..0000000000 --- a/core/server/src/metadata/consumer_group_member.rs +++ /dev/null @@ -1,58 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::metadata::{ClientId, ConsumerGroupMemberId, PartitionId}; -use std::sync::Arc; -use std::sync::atomic::AtomicUsize; - -/// A partition pending cooperative revocation from this member to a target member. -#[derive(Clone, Debug)] -pub struct PendingRevocation { - pub partition_id: PartitionId, - pub target_slab_id: usize, - pub target_member_id: usize, - pub created_at_micros: u64, -} - -/// A revocation that can be completed immediately (never polled or already committed). -#[derive(Clone, Debug)] -pub struct CompletableRevocation { - pub slab_id: usize, - pub member_id: usize, - pub partition_id: PartitionId, -} - -#[derive(Clone, Debug)] -pub struct ConsumerGroupMemberMeta { - pub id: ConsumerGroupMemberId, - pub client_id: ClientId, - pub partitions: Vec, - pub partition_index: Arc, - pub pending_revocations: Vec, -} - -impl ConsumerGroupMemberMeta { - pub fn new(id: ConsumerGroupMemberId, client_id: ClientId) -> Self { - Self { - id, - client_id, - partitions: Vec::new(), - partition_index: Arc::new(AtomicUsize::new(0)), - pending_revocations: Vec::new(), - } - } -} diff --git a/core/server/src/metadata/inner.rs b/core/server/src/metadata/inner.rs deleted file mode 100644 index 4bcc165f54..0000000000 --- a/core/server/src/metadata/inner.rs +++ /dev/null @@ -1,54 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::metadata::{StreamId, StreamMeta, UserId, UserMeta}; -use ahash::{AHashMap, AHashSet}; -use iggy_common::{GlobalPermissions, PersonalAccessToken, StreamPermissions}; -use slab::Slab; -use std::sync::Arc; - -#[derive(Clone, Default)] -pub struct InnerMetadata { - /// Streams indexed by StreamId (slab-assigned) - pub streams: Slab, - - /// Users indexed by UserId (slab-assigned) - pub users: Slab, - - /// Forward indexes (name → ID) - pub stream_index: AHashMap, StreamId>, - pub user_index: AHashMap, UserId>, - - /// user_id -> (token_hash -> PAT) - pub personal_access_tokens: AHashMap, PersonalAccessToken>>, - - // Permission indexes (auto-maintained by absorb) - pub users_global_permissions: AHashMap, - pub users_stream_permissions: AHashMap<(UserId, StreamId), StreamPermissions>, - - // Hot-path optimizations for message send/poll - pub users_can_poll_all_streams: AHashSet, - pub users_can_send_all_streams: AHashSet, - pub users_can_poll_stream: AHashSet<(UserId, StreamId)>, - pub users_can_send_stream: AHashSet<(UserId, StreamId)>, -} - -impl InnerMetadata { - pub fn new() -> Self { - Self::default() - } -} diff --git a/core/server/src/metadata/mod.rs b/core/server/src/metadata/mod.rs deleted file mode 100644 index 6e557c9ddc..0000000000 --- a/core/server/src/metadata/mod.rs +++ /dev/null @@ -1,71 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Shared metadata module providing a single source of truth for all shards. -//! -//! This module provides a `LeftRight`-based approach where all shards read from -//! a shared snapshot, and only shard 0 can write. -//! -//! # Architecture -//! -//! - `InnerMetadata` (inner.rs): Immutable snapshot with all metadata -//! - `Metadata` (reader.rs): Thread-safe read handle for querying metadata -//! - Entity types: `StreamMeta`, `TopicMeta`, `PartitionMeta`, `UserMeta`, `ConsumerGroupMeta` -//! - Consumer offsets are stored in `PartitionMeta` for cross-shard visibility - -mod absorb; -mod consumer_group; -mod consumer_group_member; -mod inner; -pub mod ops; -mod partition; -mod reader; -mod stream; -mod topic; -mod user; -mod writer; - -pub use consumer_group::ConsumerGroupMeta; -pub use consumer_group_member::ConsumerGroupMemberMeta; -pub use inner::InnerMetadata; -pub use ops::MetadataOp; -pub use partition::PartitionMeta; -pub use reader::{Metadata, PartitionInitInfo}; -pub(crate) use reader::{ - resolve_consumer_group_id_inner, resolve_stream_id_inner, resolve_topic_id_inner, -}; -pub use stream::StreamMeta; -pub use topic::TopicMeta; -pub use user::UserMeta; -pub use writer::MetadataWriter; - -pub type MetadataReadHandle = left_right::ReadHandle; -pub type StreamId = usize; -pub type TopicId = usize; -pub type PartitionId = usize; -pub type UserId = u32; -pub type ClientId = u32; -pub type ConsumerGroupId = usize; -pub type ConsumerGroupMemberId = usize; -pub type ConsumerGroupKey = (StreamId, TopicId, ConsumerGroupId); - -pub fn create_metadata_handles() -> (MetadataWriter, MetadataReadHandle) { - let (write_handle, read_handle) = left_right::new::(); - let mut writer = MetadataWriter::new(write_handle); - writer.publish(); - (writer, read_handle) -} diff --git a/core/server/src/metadata/ops.rs b/core/server/src/metadata/ops.rs deleted file mode 100644 index a6d80c2376..0000000000 --- a/core/server/src/metadata/ops.rs +++ /dev/null @@ -1,134 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::metadata::consumer_group_member::CompletableRevocation; -use crate::metadata::inner::InnerMetadata; -use crate::metadata::{ - ConsumerGroupId, ConsumerGroupMeta, PartitionId, PartitionMeta, StreamId, StreamMeta, TopicId, - TopicMeta, UserId, UserMeta, -}; -use iggy_common::{CompressionAlgorithm, IggyExpiry, MaxTopicSize, PersonalAccessToken}; -use std::sync::Arc; -use std::sync::Mutex; -use std::sync::atomic::AtomicUsize; - -#[derive(Clone)] -pub enum MetadataOp { - Initialize(Box), - - AddStream { - meta: StreamMeta, - assigned_id: Arc, - }, - UpdateStream { - id: StreamId, - new_name: Arc, - }, - DeleteStream { - id: StreamId, - }, - AddTopic { - stream_id: StreamId, - meta: TopicMeta, - assigned_id: Arc, - }, - UpdateTopic { - stream_id: StreamId, - topic_id: TopicId, - new_name: Arc, - message_expiry: IggyExpiry, - compression_algorithm: CompressionAlgorithm, - max_topic_size: MaxTopicSize, - replication_factor: u8, - }, - DeleteTopic { - stream_id: StreamId, - topic_id: TopicId, - }, - AddPartitions { - stream_id: StreamId, - topic_id: TopicId, - partitions: Vec, - revision_id: u64, - }, - DeletePartitions { - stream_id: StreamId, - topic_id: TopicId, - count: u32, - }, - AddUser { - meta: UserMeta, - assigned_id: Arc, - }, - UpdateUserMeta { - id: UserId, - meta: UserMeta, - }, - DeleteUser { - id: UserId, - }, - - AddPersonalAccessToken { - user_id: UserId, - pat: PersonalAccessToken, - }, - DeletePersonalAccessToken { - user_id: UserId, - token_hash: Arc, - }, - AddConsumerGroup { - stream_id: StreamId, - topic_id: TopicId, - meta: ConsumerGroupMeta, - assigned_id: Arc, - }, - DeleteConsumerGroup { - stream_id: StreamId, - topic_id: TopicId, - group_id: ConsumerGroupId, - }, - JoinConsumerGroup { - stream_id: StreamId, - topic_id: TopicId, - group_id: ConsumerGroupId, - client_id: u32, - member_id: Arc, - valid_client_ids: Option>, - completable_revocations: Arc>>, - }, - LeaveConsumerGroup { - stream_id: StreamId, - topic_id: TopicId, - group_id: ConsumerGroupId, - client_id: u32, - removed_member_id: Arc, - }, - RebalanceConsumerGroupsForTopic { - stream_id: StreamId, - topic_id: TopicId, - partitions_count: u32, - }, - CompletePartitionRevocation { - stream_id: StreamId, - topic_id: TopicId, - group_id: ConsumerGroupId, - member_slab_id: usize, - member_id: usize, - partition_id: PartitionId, - timed_out: bool, - }, -} diff --git a/core/server/src/metadata/partition.rs b/core/server/src/metadata/partition.rs deleted file mode 100644 index 97cb9c7934..0000000000 --- a/core/server/src/metadata/partition.rs +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::metadata::PartitionId; -use crate::streaming::partitions::consumer_group_offsets::ConsumerGroupOffsets; -use crate::streaming::partitions::consumer_offsets::ConsumerOffsets; -use crate::streaming::polling_consumer::ConsumerGroupId; -use crate::streaming::stats::PartitionStats; -use iggy_common::IggyTimestamp; -use std::sync::Arc; -use std::sync::atomic::AtomicU64; - -#[derive(Clone, Debug)] -pub struct PartitionMeta { - pub id: PartitionId, - pub created_at: IggyTimestamp, - /// Monotonically increasing version to detect stale local_partitions entries. - /// Set to the Metadata version when the partition was created. - pub revision_id: u64, - pub stats: Arc, - pub consumer_offsets: Arc, - pub consumer_group_offsets: Arc, - /// Last offset polled by each consumer group from this partition. - pub last_polled_offsets: Arc>>, -} diff --git a/core/server/src/metadata/reader.rs b/core/server/src/metadata/reader.rs deleted file mode 100644 index fa284777a1..0000000000 --- a/core/server/src/metadata/reader.rs +++ /dev/null @@ -1,1920 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::metadata::{ - ConsumerGroupId, ConsumerGroupMeta, InnerMetadata, MetadataReadHandle, PartitionId, - PartitionMeta, StreamId, StreamMeta, TopicId, TopicMeta, UserId, UserMeta, -}; -use crate::shard::transmission::message::{ResolvedPartition, ResolvedTopic}; -use crate::streaming::partitions::consumer_group_offsets::ConsumerGroupOffsets; -use crate::streaming::partitions::consumer_offsets::ConsumerOffsets; -use crate::streaming::polling_consumer::PollingConsumer; -use crate::streaming::stats::{PartitionStats, StreamStats, TopicStats}; -use iggy_common::{ - IdKind, Identifier, IggyError, IggyExpiry, IggyTimestamp, MaxTopicSize, PersonalAccessToken, -}; -use left_right::ReadGuard; -use server_common::sharding::IggyNamespace; -use std::sync::Arc; -use std::sync::atomic::Ordering; - -/// Thread-safe wrapper for GlobalMetadata using left-right for lock-free reads. -/// Uses hierarchical structure: streams contain topics, topics contain partitions and consumer groups. -/// All mutations go through MetadataWriter (shard 0 only). -/// -/// Each shard should own its own `Metadata` instance (cloned from a common source). -/// The underlying data is shared via left-right's internal mechanism. -#[derive(Clone)] -pub struct Metadata { - inner: MetadataReadHandle, -} - -impl Metadata { - pub fn new(reader: MetadataReadHandle) -> Self { - Self { inner: reader } - } - - #[inline] - pub(super) fn load(&self) -> ReadGuard<'_, InnerMetadata> { - self.inner - .enter() - .expect("metadata not initialized - writer must publish before reads") - } - - pub fn get_stream_id(&self, identifier: &Identifier) -> Option { - let metadata = self.load(); - match identifier.kind { - IdKind::Numeric => { - let stream_id = identifier.get_u32_value().ok()? as StreamId; - if metadata.streams.get(stream_id).is_some() { - Some(stream_id) - } else { - None - } - } - IdKind::String => { - let name = identifier.get_cow_str_value().ok()?; - metadata.stream_index.get(name.as_ref()).copied() - } - } - } - - pub fn stream_name_exists(&self, name: &str) -> bool { - self.load().stream_index.contains_key(name) - } - - pub fn get_topic_id(&self, stream_id: StreamId, identifier: &Identifier) -> Option { - let metadata = self.load(); - let stream = metadata.streams.get(stream_id)?; - - match identifier.kind { - IdKind::Numeric => { - let topic_id = identifier.get_u32_value().ok()? as TopicId; - if stream.topics.get(topic_id).is_some() { - Some(topic_id) - } else { - None - } - } - IdKind::String => { - let name = identifier.get_cow_str_value().ok()?; - stream.topic_index.get(&Arc::from(name.as_ref())).copied() - } - } - } - - pub fn get_user_id(&self, identifier: &Identifier) -> Option { - let metadata = self.load(); - match identifier.kind { - IdKind::Numeric => Some(identifier.get_u32_value().ok()? as UserId), - IdKind::String => { - let name = identifier.get_cow_str_value().ok()?; - metadata.user_index.get(name.as_ref()).copied() - } - } - } - - pub fn get_consumer_group_id( - &self, - stream_id: StreamId, - topic_id: TopicId, - identifier: &Identifier, - ) -> Option { - let metadata = self.load(); - let stream = metadata.streams.get(stream_id)?; - let topic = stream.topics.get(topic_id)?; - - match identifier.kind { - IdKind::Numeric => { - let group_id = identifier.get_u32_value().ok()? as ConsumerGroupId; - if topic.consumer_groups.get(group_id).is_some() { - Some(group_id) - } else { - None - } - } - IdKind::String => { - let name = identifier.get_cow_str_value().ok()?; - topic - .consumer_group_index - .get(&Arc::from(name.as_ref())) - .copied() - } - } - } - - pub fn stream_exists(&self, id: StreamId) -> bool { - self.load().streams.get(id).is_some() - } - - pub fn topic_exists(&self, stream_id: StreamId, topic_id: TopicId) -> bool { - self.load() - .streams - .get(stream_id) - .and_then(|s| s.topics.get(topic_id)) - .is_some() - } - - pub fn partition_exists( - &self, - stream_id: StreamId, - topic_id: TopicId, - partition_id: PartitionId, - ) -> bool { - self.load() - .streams - .get(stream_id) - .and_then(|s| s.topics.get(topic_id)) - .and_then(|t| t.partitions.get(partition_id)) - .is_some() - } - - pub fn user_exists(&self, id: UserId) -> bool { - self.load().users.get(id as usize).is_some() - } - - pub fn consumer_group_exists( - &self, - stream_id: StreamId, - topic_id: TopicId, - group_id: ConsumerGroupId, - ) -> bool { - self.load() - .streams - .get(stream_id) - .and_then(|s| s.topics.get(topic_id)) - .and_then(|t| t.consumer_groups.get(group_id)) - .is_some() - } - - pub fn consumer_group_exists_by_name( - &self, - stream_id: StreamId, - topic_id: TopicId, - name: &str, - ) -> bool { - self.load() - .streams - .get(stream_id) - .and_then(|s| s.topics.get(topic_id)) - .map(|t| t.consumer_group_index.contains_key(name)) - .unwrap_or(false) - } - - pub fn streams_count(&self) -> usize { - self.load().streams.len() - } - - pub fn next_stream_id(&self) -> usize { - self.load().streams.vacant_key() - } - - pub fn topics_count(&self, stream_id: StreamId) -> usize { - self.load() - .streams - .get(stream_id) - .map(|s| s.topics.len()) - .unwrap_or(0) - } - - pub fn next_topic_id(&self, stream_id: StreamId) -> Option { - self.load() - .streams - .get(stream_id) - .map(|s| s.topics.vacant_key()) - } - - pub fn partitions_count(&self, stream_id: StreamId, topic_id: TopicId) -> usize { - self.load() - .streams - .get(stream_id) - .and_then(|s| s.topics.get(topic_id)) - .map(|t| t.partitions.len()) - .unwrap_or(0) - } - - pub fn get_partitions_count(&self, stream_id: StreamId, topic_id: TopicId) -> Option { - self.load() - .streams - .get(stream_id) - .and_then(|s| s.topics.get(topic_id)) - .map(|t| t.partitions.len()) - } - - pub fn get_next_partition_id(&self, stream_id: StreamId, topic_id: TopicId) -> Option { - let metadata = self.load(); - let topic = metadata.streams.get(stream_id)?.topics.get(topic_id)?; - let partitions_count = topic.partitions.len(); - - if partitions_count == 0 { - return None; - } - - let counter = &topic.round_robin_counter; - let current = counter - .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |c| { - Some((c + 1) % partitions_count) - }) - .unwrap(); - Some(current % partitions_count) - } - - /// Resolve consumer group partition under a single metadata read guard. - pub fn resolve_consumer_group_partition( - &self, - stream_id: StreamId, - topic_id: TopicId, - group_identifier: &Identifier, - client_id: u32, - explicit_partition_id: Option, - calculate_partition_id: bool, - ) -> Result, IggyError> { - let metadata = self.load(); - - // Step 1: Resolve group ID - let group_id = { - let stream = metadata.streams.get(stream_id).ok_or_else(|| { - IggyError::ConsumerGroupIdNotFound( - group_identifier.clone(), - Identifier::numeric(topic_id as u32).unwrap(), - ) - })?; - let topic = stream.topics.get(topic_id).ok_or_else(|| { - IggyError::ConsumerGroupIdNotFound( - group_identifier.clone(), - Identifier::numeric(topic_id as u32).unwrap(), - ) - })?; - match group_identifier.kind { - IdKind::Numeric => { - let gid = group_identifier.get_u32_value().map_err(|_| { - IggyError::ConsumerGroupIdNotFound( - group_identifier.clone(), - Identifier::numeric(topic_id as u32).unwrap(), - ) - })? as ConsumerGroupId; - if topic.consumer_groups.get(gid).is_none() { - return Err(IggyError::ConsumerGroupIdNotFound( - group_identifier.clone(), - Identifier::numeric(topic_id as u32).unwrap(), - )); - } - gid - } - IdKind::String => { - let name = group_identifier.get_cow_str_value().map_err(|_| { - IggyError::ConsumerGroupIdNotFound( - group_identifier.clone(), - Identifier::numeric(topic_id as u32).unwrap(), - ) - })?; - *topic - .consumer_group_index - .get(name.as_ref()) - .ok_or_else(|| { - IggyError::ConsumerGroupIdNotFound( - group_identifier.clone(), - Identifier::numeric(topic_id as u32).unwrap(), - ) - })? - } - } - }; - - // Step 2: Find member by client_id (same read guard, same metadata snapshot) - let group = metadata - .streams - .get(stream_id) - .and_then(|s| s.topics.get(topic_id)) - .and_then(|t| t.consumer_groups.get(group_id)) - .ok_or_else(|| { - IggyError::ConsumerGroupIdNotFound( - group_identifier.clone(), - Identifier::numeric(topic_id as u32).unwrap(), - ) - })?; - - let (member_slab_id, member) = group - .members - .iter() - .find(|(_, m)| m.client_id == client_id) - .ok_or_else(|| { - IggyError::ConsumerGroupMemberNotFound( - client_id, - group_identifier.clone(), - Identifier::numeric(topic_id as u32).unwrap(), - ) - })?; - - // Step 3a: If explicit partition_id provided, validate member owns it - if let Some(pid) = explicit_partition_id { - let pid_usize = pid as usize; - if !member.partitions.contains(&pid_usize) { - // Member doesn't own this partition — check if it's pending revocation - let is_pending = member - .pending_revocations - .iter() - .any(|revocation| revocation.partition_id == pid_usize); - if !is_pending { - return Ok(None); - } - } - return Ok(Some(( - PollingConsumer::consumer_group(group_id, member_slab_id), - pid_usize, - ))); - } - - // Step 3b: Round-robin partition selection (same snapshot, no race) - if member.pending_revocations.is_empty() { - // Fast path - let partitions = &member.partitions; - let count = partitions.len(); - if count == 0 { - return Ok(None); - } - let counter = &member.partition_index; - let partition_id = if calculate_partition_id { - let current = counter - .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |c| { - Some((c + 1) % count) - }) - .unwrap(); - partitions[current % count] - } else { - let current = counter.load(Ordering::Relaxed); - partitions[current % count] - }; - return Ok(Some(( - PollingConsumer::consumer_group(group_id, member_slab_id), - partition_id, - ))); - } - - // Slow path: skip revoked partitions - let effective_count = member.partitions.len() - member.pending_revocations.len(); - if effective_count == 0 { - return Ok(None); - } - - let counter = &member.partition_index; - let idx = if calculate_partition_id { - counter - .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |c| { - Some((c + 1) % effective_count) - }) - .unwrap() - % effective_count - } else { - counter.load(Ordering::Relaxed) % effective_count - }; - - let mut seen = 0; - for &pid in &member.partitions { - let is_revoked = member - .pending_revocations - .iter() - .any(|revocation| revocation.partition_id == pid); - if is_revoked { - continue; - } - if seen == idx { - return Ok(Some(( - PollingConsumer::consumer_group(group_id, member_slab_id), - pid, - ))); - } - seen += 1; - } - - Ok(None) - } - - /// Record the last offset returned to a CG member during poll. - pub fn record_polled_offset( - &self, - stream_id: StreamId, - topic_id: TopicId, - group_id: ConsumerGroupId, - partition_id: PartitionId, - offset: u64, - ) { - use crate::streaming::polling_consumer::ConsumerGroupId as CgIdNewtype; - - let metadata = self.load(); - if let Some(partition) = metadata - .streams - .get(stream_id) - .and_then(|s| s.topics.get(topic_id)) - .and_then(|t| t.partitions.get(partition_id)) - { - let key = CgIdNewtype(group_id); - let guard = partition.last_polled_offsets.pin(); - match guard.get(&key) { - Some(existing) => { - existing.store(offset, Ordering::Release); - } - None => { - guard.insert(key, Arc::new(std::sync::atomic::AtomicU64::new(offset))); - } - } - } - } - - pub fn users_count(&self) -> usize { - self.load().users.len() - } - - pub fn username_exists(&self, username: &str) -> bool { - self.load().user_index.contains_key(username) - } - - pub fn consumer_groups_count(&self, stream_id: StreamId, topic_id: TopicId) -> usize { - self.load() - .streams - .get(stream_id) - .and_then(|s| s.topics.get(topic_id)) - .map(|t| t.consumer_groups.len()) - .unwrap_or(0) - } - - pub fn get_stream_stats(&self, id: StreamId) -> Option> { - self.load().streams.get(id).map(|s| s.stats.clone()) - } - - pub fn get_topic_stats( - &self, - stream_id: StreamId, - topic_id: TopicId, - ) -> Option> { - self.load() - .streams - .get(stream_id) - .and_then(|s| s.topics.get(topic_id)) - .map(|t| t.stats.clone()) - } - - pub fn get_partition_stats(&self, ns: &IggyNamespace) -> Option> { - self.load() - .streams - .get(ns.stream_id()) - .and_then(|s| s.topics.get(ns.topic_id())) - .and_then(|t| t.partitions.get(ns.partition_id())) - .map(|p| p.stats.clone()) - } - - pub fn get_partition_stats_by_ids( - &self, - stream_id: StreamId, - topic_id: TopicId, - partition_id: PartitionId, - ) -> Option> { - self.load() - .streams - .get(stream_id) - .and_then(|s| s.topics.get(topic_id)) - .and_then(|t| t.partitions.get(partition_id)) - .map(|p| p.stats.clone()) - } - - pub fn get_partition_consumer_offsets( - &self, - stream_id: StreamId, - topic_id: TopicId, - partition_id: PartitionId, - ) -> Option> { - self.load() - .streams - .get(stream_id) - .and_then(|s| s.topics.get(topic_id)) - .and_then(|t| t.partitions.get(partition_id)) - .map(|p| p.consumer_offsets.clone()) - } - - pub fn get_partition_consumer_group_offsets( - &self, - stream_id: StreamId, - topic_id: TopicId, - partition_id: PartitionId, - ) -> Option> { - self.load() - .streams - .get(stream_id) - .and_then(|s| s.topics.get(topic_id)) - .and_then(|t| t.partitions.get(partition_id)) - .map(|p| p.consumer_group_offsets.clone()) - } - - pub fn get_user(&self, id: UserId) -> Option { - self.load().users.get(id as usize).cloned() - } - - pub fn get_all_users(&self) -> Vec { - self.load().users.iter().map(|(_, u)| u.clone()).collect() - } - - pub fn get_stream(&self, id: StreamId) -> Option { - self.load().streams.get(id).cloned() - } - - pub fn get_topic(&self, stream_id: StreamId, topic_id: TopicId) -> Option { - self.load() - .streams - .get(stream_id) - .and_then(|s| s.topics.get(topic_id).cloned()) - } - - pub fn get_partition( - &self, - stream_id: StreamId, - topic_id: TopicId, - partition_id: PartitionId, - ) -> Option { - self.load() - .streams - .get(stream_id) - .and_then(|s| s.topics.get(topic_id)) - .and_then(|t| t.partitions.get(partition_id).cloned()) - } - - pub fn get_consumer_group( - &self, - stream_id: StreamId, - topic_id: TopicId, - group_id: ConsumerGroupId, - ) -> Option { - self.load() - .streams - .get(stream_id) - .and_then(|s| s.topics.get(topic_id)) - .and_then(|t| t.consumer_groups.get(group_id).cloned()) - } - - pub fn get_user_personal_access_tokens(&self, user_id: UserId) -> Vec { - self.load() - .personal_access_tokens - .get(&user_id) - .map(|pats| pats.values().cloned().collect()) - .unwrap_or_default() - } - - pub fn get_personal_access_token_by_hash( - &self, - token_hash: &str, - ) -> Option { - let token_hash_arc: Arc = Arc::from(token_hash); - let metadata = self.load(); - for user_pats in metadata.personal_access_tokens.values() { - if let Some(pat) = user_pats.get(&token_hash_arc) { - return Some(pat.clone()); - } - } - None - } - - pub fn user_pat_count(&self, user_id: UserId) -> usize { - self.load() - .personal_access_tokens - .get(&user_id) - .map(|pats| pats.len()) - .unwrap_or(0) - } - - pub fn user_has_pat_with_name(&self, user_id: UserId, name: &str) -> bool { - self.load() - .personal_access_tokens - .get(&user_id) - .map(|pats| pats.values().any(|pat| &*pat.name == name)) - .unwrap_or(false) - } - - pub fn find_pat_token_hash_by_name(&self, user_id: UserId, name: &str) -> Option> { - self.load() - .personal_access_tokens - .get(&user_id) - .and_then(|pats| { - pats.iter() - .find(|(_, pat)| &*pat.name == name) - .map(|(hash, _)| hash.clone()) - }) - } - - pub fn is_consumer_group_member( - &self, - stream_id: StreamId, - topic_id: TopicId, - group_id: ConsumerGroupId, - client_id: u32, - ) -> bool { - let metadata = self.load(); - metadata - .streams - .get(stream_id) - .and_then(|s| s.topics.get(topic_id)) - .and_then(|t| t.consumer_groups.get(group_id)) - .map(|g| g.members.iter().any(|(_, m)| m.client_id == client_id)) - .unwrap_or(false) - } - - /// Execute a closure with read access to the metadata snapshot. - /// This is the safe way to perform complex read operations that need - /// atomic access to multiple metadata fields. - /// - /// The closure receives an immutable reference to the metadata and must - /// return owned data (not references). This ensures the ReadGuard is - /// dropped before any async operations can occur. - #[inline] - pub fn with_metadata(&self, f: F) -> R - where - F: FnOnce(&InnerMetadata) -> R, - { - let guard = self.load(); - f(&guard) - } - - /// Get all partition IDs for a topic, sorted. - pub fn get_partition_ids(&self, stream_id: StreamId, topic_id: TopicId) -> Vec { - self.with_metadata(|m| { - m.streams - .get(stream_id) - .and_then(|s| s.topics.get(topic_id)) - .map(|t| { - let mut ids: Vec<_> = t.partitions.iter().enumerate().map(|(k, _)| k).collect(); - ids.sort_unstable(); - ids - }) - .unwrap_or_default() - }) - } - - /// Get all topic IDs for a stream, sorted. - pub fn get_topic_ids(&self, stream_id: StreamId) -> Vec { - self.with_metadata(|m| { - m.streams - .get(stream_id) - .map(|s| { - let mut ids: Vec<_> = s.topics.iter().map(|(k, _)| k).collect(); - ids.sort_unstable(); - ids - }) - .unwrap_or_default() - }) - } - - /// Get all stream IDs, sorted. - pub fn get_stream_ids(&self) -> Vec { - self.with_metadata(|m| { - let mut ids: Vec<_> = m.streams.iter().map(|(k, _)| k).collect(); - ids.sort_unstable(); - ids - }) - } - - /// Get all namespaces (stream/topic/partition combinations). - pub fn get_all_namespaces(&self) -> Vec { - self.with_metadata(|m| { - let mut namespaces = Vec::new(); - for (stream_id, stream) in m.streams.iter() { - for (topic_id, topic) in stream.topics.iter() { - for (partition_id, _) in topic.partitions.iter().enumerate() { - namespaces.push(IggyNamespace::new(stream_id, topic_id, partition_id)); - } - } - } - namespaces - }) - } - - /// Get topic configuration (message_expiry, max_topic_size). - pub fn get_topic_config( - &self, - stream_id: StreamId, - topic_id: TopicId, - ) -> Option<(IggyExpiry, MaxTopicSize)> { - self.with_metadata(|m| { - m.streams - .get(stream_id) - .and_then(|s| s.topics.get(topic_id)) - .map(|t| (t.message_expiry, t.max_topic_size)) - }) - } - - /// Get partition initialization info needed for LocalPartition setup. - pub fn get_partition_init_info( - &self, - stream_id: StreamId, - topic_id: TopicId, - partition_id: PartitionId, - ) -> Option { - self.with_metadata(|m| { - m.streams - .get(stream_id) - .and_then(|s| s.topics.get(topic_id)) - .and_then(|t| t.partitions.get(partition_id)) - .map(|p| PartitionInitInfo { - created_at: p.created_at, - revision_id: p.revision_id, - stats: p.stats.clone(), - consumer_offsets: p.consumer_offsets.clone(), - consumer_group_offsets: p.consumer_group_offsets.clone(), - }) - }) - } - - /// Get consumer group member ID for a client. - pub fn get_consumer_group_member_id( - &self, - stream_id: StreamId, - topic_id: TopicId, - group_id: ConsumerGroupId, - client_id: u32, - ) -> Option { - self.with_metadata(|m| { - m.streams - .get(stream_id) - .and_then(|s| s.topics.get(topic_id)) - .and_then(|t| t.consumer_groups.get(group_id)) - .and_then(|g| { - g.members - .iter() - .find(|(_, member)| member.client_id == client_id) - .map(|(id, _)| id) - }) - }) - } - - /// Get all consumer groups for a topic. - pub fn get_all_consumer_groups( - &self, - stream_id: StreamId, - topic_id: TopicId, - ) -> Vec { - self.with_metadata(|m| { - m.streams - .get(stream_id) - .and_then(|s| s.topics.get(topic_id)) - .map(|t| t.consumer_groups.iter().map(|(_, cg)| cg.clone()).collect()) - .unwrap_or_default() - }) - } - - /// Inheritance: manage_streams → read_streams → read_topics → poll_messages - pub fn perm_poll_messages( - &self, - user_id: u32, - stream_id: StreamId, - topic_id: TopicId, - ) -> Result<(), IggyError> { - let metadata = self.load(); - - if metadata.users_can_poll_all_streams.contains(&user_id) { - return Ok(()); - } - - if let Some(global) = metadata.users_global_permissions.get(&user_id) - && (global.read_topics - || global.manage_topics - || global.read_streams - || global.manage_streams) - { - return Ok(()); - } - - if metadata - .users_can_poll_stream - .contains(&(user_id, stream_id)) - { - return Ok(()); - } - - let Some(stream_permissions) = metadata.users_stream_permissions.get(&(user_id, stream_id)) - else { - return Err(IggyError::Unauthorized); - }; - - if stream_permissions.manage_stream || stream_permissions.read_stream { - return Ok(()); - } - - if stream_permissions.manage_topics || stream_permissions.read_topics { - return Ok(()); - } - - if stream_permissions.poll_messages { - return Ok(()); - } - - if let Some(topics) = &stream_permissions.topics - && let Some(topic_permissions) = topics.get(&topic_id) - && (topic_permissions.manage_topic - || topic_permissions.read_topic - || topic_permissions.poll_messages) - { - return Ok(()); - } - - Err(IggyError::Unauthorized) - } - - /// Inheritance: manage_streams → manage_topics → send_messages - pub fn perm_append_messages( - &self, - user_id: u32, - stream_id: StreamId, - topic_id: TopicId, - ) -> Result<(), IggyError> { - let metadata = self.load(); - - if metadata.users_can_send_all_streams.contains(&user_id) { - return Ok(()); - } - - if let Some(global) = metadata.users_global_permissions.get(&user_id) - && (global.manage_streams || global.manage_topics) - { - return Ok(()); - } - - if metadata - .users_can_send_stream - .contains(&(user_id, stream_id)) - { - return Ok(()); - } - - let Some(stream_permissions) = metadata.users_stream_permissions.get(&(user_id, stream_id)) - else { - return Err(IggyError::Unauthorized); - }; - - if stream_permissions.manage_stream || stream_permissions.manage_topics { - return Ok(()); - } - - if stream_permissions.send_messages { - return Ok(()); - } - - if let Some(topics) = &stream_permissions.topics - && let Some(topic_permissions) = topics.get(&topic_id) - && (topic_permissions.manage_topic || topic_permissions.send_messages) - { - return Ok(()); - } - - Err(IggyError::Unauthorized) - } - - pub fn perm_get_stream(&self, user_id: u32, stream_id: StreamId) -> Result<(), IggyError> { - let metadata = self.load(); - - if let Some(global_permissions) = metadata.users_global_permissions.get(&user_id) - && (global_permissions.manage_streams || global_permissions.read_streams) - { - return Ok(()); - } - - if let Some(stream_permissions) = - metadata.users_stream_permissions.get(&(user_id, stream_id)) - && (stream_permissions.manage_stream || stream_permissions.read_stream) - { - return Ok(()); - } - - Err(IggyError::Unauthorized) - } - - pub fn perm_get_streams(&self, user_id: u32) -> Result<(), IggyError> { - let metadata = self.load(); - - if let Some(global_permissions) = metadata.users_global_permissions.get(&user_id) - && (global_permissions.manage_streams || global_permissions.read_streams) - { - return Ok(()); - } - - Err(IggyError::Unauthorized) - } - - pub fn perm_create_stream(&self, user_id: u32) -> Result<(), IggyError> { - let metadata = self.load(); - - if let Some(global_permissions) = metadata.users_global_permissions.get(&user_id) - && global_permissions.manage_streams - { - return Ok(()); - } - - Err(IggyError::Unauthorized) - } - - pub fn perm_update_stream(&self, user_id: u32, stream_id: StreamId) -> Result<(), IggyError> { - self.perm_manage_stream(user_id, stream_id) - } - - pub fn perm_delete_stream(&self, user_id: u32, stream_id: StreamId) -> Result<(), IggyError> { - self.perm_manage_stream(user_id, stream_id) - } - - pub fn perm_purge_stream(&self, user_id: u32, stream_id: StreamId) -> Result<(), IggyError> { - self.perm_manage_stream(user_id, stream_id) - } - - fn perm_manage_stream(&self, user_id: u32, stream_id: StreamId) -> Result<(), IggyError> { - let metadata = self.load(); - - if let Some(global_permissions) = metadata.users_global_permissions.get(&user_id) - && global_permissions.manage_streams - { - return Ok(()); - } - - if let Some(stream_permissions) = - metadata.users_stream_permissions.get(&(user_id, stream_id)) - && stream_permissions.manage_stream - { - return Ok(()); - } - - Err(IggyError::Unauthorized) - } - - /// Inheritance: manage_streams → read_streams → read_topics - pub fn perm_get_topic( - &self, - user_id: u32, - stream_id: StreamId, - topic_id: TopicId, - ) -> Result<(), IggyError> { - let metadata = self.load(); - - if let Some(global) = metadata.users_global_permissions.get(&user_id) - && (global.read_streams - || global.manage_streams - || global.manage_topics - || global.read_topics) - { - return Ok(()); - } - - if let Some(stream_permissions) = - metadata.users_stream_permissions.get(&(user_id, stream_id)) - { - if stream_permissions.manage_stream - || stream_permissions.read_stream - || stream_permissions.manage_topics - || stream_permissions.read_topics - { - return Ok(()); - } - - if let Some(topics) = &stream_permissions.topics - && let Some(topic_permissions) = topics.get(&topic_id) - && (topic_permissions.manage_topic || topic_permissions.read_topic) - { - return Ok(()); - } - } - - Err(IggyError::Unauthorized) - } - - pub fn perm_get_topics(&self, user_id: u32, stream_id: StreamId) -> Result<(), IggyError> { - let metadata = self.load(); - - if let Some(global) = metadata.users_global_permissions.get(&user_id) - && (global.read_streams - || global.manage_streams - || global.manage_topics - || global.read_topics) - { - return Ok(()); - } - - if let Some(stream_permissions) = - metadata.users_stream_permissions.get(&(user_id, stream_id)) - && (stream_permissions.manage_stream - || stream_permissions.read_stream - || stream_permissions.manage_topics - || stream_permissions.read_topics) - { - return Ok(()); - } - - Err(IggyError::Unauthorized) - } - - /// Inheritance: manage_streams → manage_topics - pub fn perm_create_topic(&self, user_id: u32, stream_id: StreamId) -> Result<(), IggyError> { - let metadata = self.load(); - - if let Some(global) = metadata.users_global_permissions.get(&user_id) - && (global.manage_streams || global.manage_topics) - { - return Ok(()); - } - - if let Some(stream_permissions) = - metadata.users_stream_permissions.get(&(user_id, stream_id)) - && (stream_permissions.manage_stream || stream_permissions.manage_topics) - { - return Ok(()); - } - - Err(IggyError::Unauthorized) - } - - pub fn perm_update_topic( - &self, - user_id: u32, - stream_id: StreamId, - topic_id: TopicId, - ) -> Result<(), IggyError> { - self.perm_manage_topic(user_id, stream_id, topic_id) - } - - pub fn perm_delete_topic( - &self, - user_id: u32, - stream_id: StreamId, - topic_id: TopicId, - ) -> Result<(), IggyError> { - self.perm_manage_topic(user_id, stream_id, topic_id) - } - - pub fn perm_purge_topic( - &self, - user_id: u32, - stream_id: StreamId, - topic_id: TopicId, - ) -> Result<(), IggyError> { - self.perm_manage_topic(user_id, stream_id, topic_id) - } - - /// Inheritance: manage_streams → manage_topics - fn perm_manage_topic( - &self, - user_id: u32, - stream_id: StreamId, - topic_id: TopicId, - ) -> Result<(), IggyError> { - let metadata = self.load(); - - if let Some(global) = metadata.users_global_permissions.get(&user_id) - && (global.manage_streams || global.manage_topics) - { - return Ok(()); - } - - if let Some(stream_permissions) = - metadata.users_stream_permissions.get(&(user_id, stream_id)) - { - if stream_permissions.manage_stream || stream_permissions.manage_topics { - return Ok(()); - } - - if let Some(topics) = &stream_permissions.topics - && let Some(topic_permissions) = topics.get(&topic_id) - && topic_permissions.manage_topic - { - return Ok(()); - } - } - - Err(IggyError::Unauthorized) - } - - pub fn perm_create_partitions( - &self, - user_id: u32, - stream_id: StreamId, - topic_id: TopicId, - ) -> Result<(), IggyError> { - self.perm_update_topic(user_id, stream_id, topic_id) - } - - pub fn perm_delete_partitions( - &self, - user_id: u32, - stream_id: StreamId, - topic_id: TopicId, - ) -> Result<(), IggyError> { - self.perm_update_topic(user_id, stream_id, topic_id) - } - - pub fn perm_delete_segments( - &self, - user_id: u32, - stream_id: StreamId, - topic_id: TopicId, - ) -> Result<(), IggyError> { - self.perm_update_topic(user_id, stream_id, topic_id) - } - - pub fn perm_create_consumer_group( - &self, - user_id: u32, - stream_id: StreamId, - topic_id: TopicId, - ) -> Result<(), IggyError> { - self.perm_get_topic(user_id, stream_id, topic_id) - } - - pub fn perm_delete_consumer_group( - &self, - user_id: u32, - stream_id: StreamId, - topic_id: TopicId, - ) -> Result<(), IggyError> { - self.perm_get_topic(user_id, stream_id, topic_id) - } - - pub fn perm_get_consumer_group( - &self, - user_id: u32, - stream_id: StreamId, - topic_id: TopicId, - ) -> Result<(), IggyError> { - self.perm_get_topic(user_id, stream_id, topic_id) - } - - pub fn perm_get_consumer_groups( - &self, - user_id: u32, - stream_id: StreamId, - topic_id: TopicId, - ) -> Result<(), IggyError> { - self.perm_get_topic(user_id, stream_id, topic_id) - } - - pub fn perm_join_consumer_group( - &self, - user_id: u32, - stream_id: StreamId, - topic_id: TopicId, - ) -> Result<(), IggyError> { - self.perm_get_topic(user_id, stream_id, topic_id) - } - - pub fn perm_leave_consumer_group( - &self, - user_id: u32, - stream_id: StreamId, - topic_id: TopicId, - ) -> Result<(), IggyError> { - self.perm_get_topic(user_id, stream_id, topic_id) - } - - pub fn perm_get_consumer_offset( - &self, - user_id: u32, - stream_id: StreamId, - topic_id: TopicId, - ) -> Result<(), IggyError> { - self.perm_poll_messages(user_id, stream_id, topic_id) - } - - pub fn perm_store_consumer_offset( - &self, - user_id: u32, - stream_id: StreamId, - topic_id: TopicId, - ) -> Result<(), IggyError> { - self.perm_poll_messages(user_id, stream_id, topic_id) - } - - pub fn perm_delete_consumer_offset( - &self, - user_id: u32, - stream_id: StreamId, - topic_id: TopicId, - ) -> Result<(), IggyError> { - self.perm_poll_messages(user_id, stream_id, topic_id) - } - - pub fn perm_get_user(&self, user_id: u32) -> Result<(), IggyError> { - self.perm_read_users(user_id) - } - - pub fn perm_get_users(&self, user_id: u32) -> Result<(), IggyError> { - self.perm_read_users(user_id) - } - - pub fn perm_create_user(&self, user_id: u32) -> Result<(), IggyError> { - self.perm_manage_users(user_id) - } - - pub fn perm_delete_user(&self, user_id: u32) -> Result<(), IggyError> { - self.perm_manage_users(user_id) - } - - pub fn perm_update_user(&self, user_id: u32) -> Result<(), IggyError> { - self.perm_manage_users(user_id) - } - - pub fn perm_update_permissions(&self, user_id: u32) -> Result<(), IggyError> { - self.perm_manage_users(user_id) - } - - pub fn perm_change_password(&self, user_id: u32) -> Result<(), IggyError> { - self.perm_manage_users(user_id) - } - - fn perm_manage_users(&self, user_id: u32) -> Result<(), IggyError> { - let metadata = self.load(); - - if let Some(global_permissions) = metadata.users_global_permissions.get(&user_id) - && global_permissions.manage_users - { - return Ok(()); - } - - Err(IggyError::Unauthorized) - } - - fn perm_read_users(&self, user_id: u32) -> Result<(), IggyError> { - let metadata = self.load(); - - if let Some(global_permissions) = metadata.users_global_permissions.get(&user_id) - && (global_permissions.manage_users || global_permissions.read_users) - { - return Ok(()); - } - - Err(IggyError::Unauthorized) - } - - pub fn perm_get_stats(&self, user_id: u32) -> Result<(), IggyError> { - self.perm_get_server_info(user_id) - } - - pub fn perm_get_clients(&self, user_id: u32) -> Result<(), IggyError> { - self.perm_get_server_info(user_id) - } - - pub fn perm_get_client(&self, user_id: u32) -> Result<(), IggyError> { - self.perm_get_server_info(user_id) - } - - pub fn perm_get_snapshot(&self, user_id: u32) -> Result<(), IggyError> { - self.perm_get_server_info(user_id) - } - - fn perm_get_server_info(&self, user_id: u32) -> Result<(), IggyError> { - let metadata = self.load(); - - if let Some(global_permissions) = metadata.users_global_permissions.get(&user_id) - && (global_permissions.manage_servers || global_permissions.read_servers) - { - return Ok(()); - } - - Err(IggyError::Unauthorized) - } - - /// Atomically resolve, authorize, and return stream metadata. - pub fn query_stream( - &self, - user_id: u32, - stream_id: &Identifier, - ) -> Result, IggyError> { - self.with_metadata(|m| { - let sid = match resolve_stream_id_inner(m, stream_id) { - Some(s) => s, - None => return Ok(None), - }; - perm_get_stream_inner(m, user_id, sid)?; - Ok(m.streams.get(sid).cloned()) - }) - } - - /// Atomically authorize and return all streams. - pub fn query_streams(&self, user_id: u32) -> Result, IggyError> { - self.with_metadata(|m| { - perm_get_streams_inner(m, user_id)?; - Ok(m.streams.iter().map(|(_, s)| s.clone()).collect()) - }) - } - - /// Atomically resolve, authorize, and return topic metadata. - pub fn query_topic( - &self, - user_id: u32, - stream_id: &Identifier, - topic_id: &Identifier, - ) -> Result, IggyError> { - self.with_metadata(|m| { - let sid = match resolve_stream_id_inner(m, stream_id) { - Some(s) => s, - None => return Ok(None), - }; - let tid = match resolve_topic_id_inner(m, sid, topic_id) { - Some(id) => id, - None => return Ok(None), - }; - perm_get_topic_inner(m, user_id, sid, tid)?; - Ok(m.streams.get(sid).and_then(|s| s.topics.get(tid).cloned())) - }) - } - - /// Atomically resolve, authorize, and return all topics for a stream. - pub fn query_topics( - &self, - user_id: u32, - stream_id: &Identifier, - ) -> Result>, IggyError> { - self.with_metadata(|m| { - let sid = match resolve_stream_id_inner(m, stream_id) { - Some(s) => s, - None => return Ok(None), - }; - perm_get_topics_inner(m, user_id, sid)?; - Ok(m.streams - .get(sid) - .map(|s| s.topics.iter().map(|(_, t)| t.clone()).collect())) - }) - } - - /// Atomically resolve, authorize, and return consumer group metadata. - pub fn query_consumer_group( - &self, - user_id: u32, - stream_id: &Identifier, - topic_id: &Identifier, - group_id: &Identifier, - ) -> Result, IggyError> { - self.with_metadata(|m| { - let sid = match resolve_stream_id_inner(m, stream_id) { - Some(s) => s, - None => return Ok(None), - }; - let tid = match resolve_topic_id_inner(m, sid, topic_id) { - Some(id) => id, - None => return Ok(None), - }; - let gid = match resolve_consumer_group_id_inner(m, sid, tid, group_id) { - Some(id) => id, - None => return Ok(None), - }; - perm_get_consumer_group_inner(m, user_id, sid, tid)?; - Ok(m.streams - .get(sid) - .and_then(|s| s.topics.get(tid)) - .and_then(|t| t.consumer_groups.get(gid).cloned())) - }) - } - - /// Atomically resolve, authorize, and return all consumer groups for a topic. - pub fn query_consumer_groups( - &self, - user_id: u32, - stream_id: &Identifier, - topic_id: &Identifier, - ) -> Result>, IggyError> { - self.with_metadata(|m| { - let sid = match resolve_stream_id_inner(m, stream_id) { - Some(s) => s, - None => return Ok(None), - }; - let tid = match resolve_topic_id_inner(m, sid, topic_id) { - Some(id) => id, - None => return Ok(None), - }; - perm_get_consumer_group_inner(m, user_id, sid, tid)?; - Ok(m.streams - .get(sid) - .and_then(|s| s.topics.get(tid)) - .map(|t| t.consumer_groups.iter().map(|(_, cg)| cg.clone()).collect())) - }) - } - - /// Atomically resolve, authorize, and return user metadata. - /// Permission check skipped when requesting own data. - pub fn query_user( - &self, - requesting_user_id: u32, - target_user_id: &Identifier, - ) -> Result, IggyError> { - self.with_metadata(|m| { - let uid = match resolve_user_id_inner(m, target_user_id) { - Some(id) => id, - None => return Ok(None), - }; - if uid != requesting_user_id { - perm_get_user_inner(m, requesting_user_id)?; - } - Ok(m.users.get(uid as usize).cloned()) - }) - } - - /// Atomically authorize and return all users. - pub fn query_users(&self, user_id: u32) -> Result, IggyError> { - self.with_metadata(|m| { - perm_get_users_inner(m, user_id)?; - Ok(m.users.iter().map(|(_, u)| u.clone()).collect()) - }) - } - - /// Atomically resolve topic and check permission for consumer offset query. - /// Returns resolved topic for use with get_consumer_offset. - pub fn resolve_for_consumer_offset( - &self, - user_id: u32, - stream_id: &Identifier, - topic_id: &Identifier, - ) -> Result, IggyError> { - self.with_metadata(|m| { - let sid = match resolve_stream_id_inner(m, stream_id) { - Some(s) => s, - None => return Ok(None), - }; - let tid = match resolve_topic_id_inner(m, sid, topic_id) { - Some(id) => id, - None => return Ok(None), - }; - perm_get_consumer_offset_inner(m, user_id, sid, tid)?; - Ok(Some(ResolvedTopic { - stream_id: sid, - topic_id: tid, - })) - }) - } - - /// Atomically resolve topic and check append permission. - pub fn resolve_for_append( - &self, - user_id: u32, - stream_id: &Identifier, - topic_id: &Identifier, - ) -> Result { - self.with_metadata(|m| { - let sid = resolve_stream_id_inner(m, stream_id) - .ok_or_else(|| IggyError::StreamIdNotFound(stream_id.clone()))?; - let tid = resolve_topic_id_inner(m, sid, topic_id) - .ok_or_else(|| IggyError::TopicIdNotFound(stream_id.clone(), topic_id.clone()))?; - perm_append_messages_inner(m, user_id, sid, tid)?; - Ok(ResolvedTopic { - stream_id: sid, - topic_id: tid, - }) - }) - } - - /// Atomically resolve topic and check poll permission. - pub fn resolve_for_poll( - &self, - user_id: u32, - stream_id: &Identifier, - topic_id: &Identifier, - ) -> Result { - self.with_metadata(|m| { - let sid = resolve_stream_id_inner(m, stream_id) - .ok_or_else(|| IggyError::StreamIdNotFound(stream_id.clone()))?; - let tid = resolve_topic_id_inner(m, sid, topic_id) - .ok_or_else(|| IggyError::TopicIdNotFound(stream_id.clone(), topic_id.clone()))?; - perm_poll_messages_inner(m, user_id, sid, tid)?; - Ok(ResolvedTopic { - stream_id: sid, - topic_id: tid, - }) - }) - } - - /// Atomically resolve topic and check store consumer offset permission. - pub fn resolve_for_store_consumer_offset( - &self, - user_id: u32, - stream_id: &Identifier, - topic_id: &Identifier, - ) -> Result { - self.with_metadata(|m| { - let sid = resolve_stream_id_inner(m, stream_id) - .ok_or_else(|| IggyError::StreamIdNotFound(stream_id.clone()))?; - let tid = resolve_topic_id_inner(m, sid, topic_id) - .ok_or_else(|| IggyError::TopicIdNotFound(stream_id.clone(), topic_id.clone()))?; - perm_get_consumer_offset_inner(m, user_id, sid, tid)?; - Ok(ResolvedTopic { - stream_id: sid, - topic_id: tid, - }) - }) - } - - /// Atomically resolve topic and check delete consumer offset permission. - pub fn resolve_for_delete_consumer_offset( - &self, - user_id: u32, - stream_id: &Identifier, - topic_id: &Identifier, - ) -> Result { - self.with_metadata(|m| { - let sid = resolve_stream_id_inner(m, stream_id) - .ok_or_else(|| IggyError::StreamIdNotFound(stream_id.clone()))?; - let tid = resolve_topic_id_inner(m, sid, topic_id) - .ok_or_else(|| IggyError::TopicIdNotFound(stream_id.clone(), topic_id.clone()))?; - perm_get_consumer_offset_inner(m, user_id, sid, tid)?; - Ok(ResolvedTopic { - stream_id: sid, - topic_id: tid, - }) - }) - } - - /// Atomically resolve partition and check delete segments permission. - pub fn resolve_for_delete_segments( - &self, - user_id: u32, - stream_id: &Identifier, - topic_id: &Identifier, - partition_id: PartitionId, - ) -> Result { - self.with_metadata(|m| { - let sid = resolve_stream_id_inner(m, stream_id) - .ok_or_else(|| IggyError::StreamIdNotFound(stream_id.clone()))?; - let tid = resolve_topic_id_inner(m, sid, topic_id) - .ok_or_else(|| IggyError::TopicIdNotFound(stream_id.clone(), topic_id.clone()))?; - let exists = m - .streams - .get(sid) - .and_then(|s| s.topics.get(tid)) - .and_then(|t| t.partitions.get(partition_id)) - .is_some(); - if !exists { - return Err(IggyError::PartitionNotFound( - partition_id, - topic_id.clone(), - stream_id.clone(), - )); - } - perm_manage_topic_inner(m, user_id, sid, tid)?; - Ok(ResolvedPartition { - stream_id: sid, - topic_id: tid, - partition_id, - }) - }) - } -} - -/// Information needed to initialize a LocalPartition. -#[derive(Clone, Debug)] -pub struct PartitionInitInfo { - pub created_at: IggyTimestamp, - pub revision_id: u64, - pub stats: Arc, - pub consumer_offsets: Arc, - pub consumer_group_offsets: Arc, -} - -pub(crate) fn resolve_stream_id_inner( - m: &InnerMetadata, - stream_id: &Identifier, -) -> Option { - match stream_id.kind { - IdKind::Numeric => { - let sid = stream_id.get_u32_value().ok()? as StreamId; - if m.streams.get(sid).is_some() { - Some(sid) - } else { - None - } - } - IdKind::String => { - let name = stream_id.get_cow_str_value().ok()?; - m.stream_index.get(name.as_ref()).copied() - } - } -} - -pub(crate) fn resolve_topic_id_inner( - m: &InnerMetadata, - stream_id: StreamId, - topic_id: &Identifier, -) -> Option { - let stream = m.streams.get(stream_id)?; - match topic_id.kind { - IdKind::Numeric => { - let tid = topic_id.get_u32_value().ok()? as TopicId; - if stream.topics.get(tid).is_some() { - Some(tid) - } else { - None - } - } - IdKind::String => { - let name = topic_id.get_cow_str_value().ok()?; - stream.topic_index.get(&Arc::from(name.as_ref())).copied() - } - } -} - -pub(crate) fn resolve_consumer_group_id_inner( - m: &InnerMetadata, - stream_id: StreamId, - topic_id: TopicId, - group_id: &Identifier, -) -> Option { - let stream = m.streams.get(stream_id)?; - let topic = stream.topics.get(topic_id)?; - match group_id.kind { - IdKind::Numeric => { - let gid = group_id.get_u32_value().ok()? as ConsumerGroupId; - if topic.consumer_groups.get(gid).is_some() { - Some(gid) - } else { - None - } - } - IdKind::String => { - let name = group_id.get_cow_str_value().ok()?; - topic - .consumer_group_index - .get(&Arc::from(name.as_ref())) - .copied() - } - } -} - -fn resolve_user_id_inner(m: &InnerMetadata, user_id: &Identifier) -> Option { - match user_id.kind { - IdKind::Numeric => Some(user_id.get_u32_value().ok()?), - IdKind::String => { - let name = user_id.get_cow_str_value().ok()?; - m.user_index.get(name.as_ref()).copied() - } - } -} - -fn perm_get_stream_inner( - m: &InnerMetadata, - user_id: u32, - stream_id: StreamId, -) -> Result<(), IggyError> { - if let Some(global) = m.users_global_permissions.get(&user_id) - && (global.manage_streams || global.read_streams) - { - return Ok(()); - } - if let Some(stream_perm) = m.users_stream_permissions.get(&(user_id, stream_id)) - && (stream_perm.manage_stream || stream_perm.read_stream) - { - return Ok(()); - } - Err(IggyError::Unauthorized) -} - -fn perm_get_streams_inner(m: &InnerMetadata, user_id: u32) -> Result<(), IggyError> { - if let Some(global) = m.users_global_permissions.get(&user_id) - && (global.manage_streams || global.read_streams) - { - return Ok(()); - } - Err(IggyError::Unauthorized) -} - -/// Inheritance: manage_streams -> manage_topics -> manage_topic -fn perm_manage_topic_inner( - m: &InnerMetadata, - user_id: u32, - stream_id: StreamId, - topic_id: TopicId, -) -> Result<(), IggyError> { - if let Some(global) = m.users_global_permissions.get(&user_id) - && (global.manage_streams || global.manage_topics) - { - return Ok(()); - } - - if let Some(stream_permissions) = m.users_stream_permissions.get(&(user_id, stream_id)) { - if stream_permissions.manage_stream || stream_permissions.manage_topics { - return Ok(()); - } - - if let Some(topics) = &stream_permissions.topics - && let Some(topic_permissions) = topics.get(&topic_id) - && topic_permissions.manage_topic - { - return Ok(()); - } - } - - Err(IggyError::Unauthorized) -} - -fn perm_get_topic_inner( - m: &InnerMetadata, - user_id: u32, - stream_id: StreamId, - topic_id: TopicId, -) -> Result<(), IggyError> { - if let Some(global) = m.users_global_permissions.get(&user_id) - && (global.read_streams - || global.manage_streams - || global.manage_topics - || global.read_topics) - { - return Ok(()); - } - - if let Some(stream_permissions) = m.users_stream_permissions.get(&(user_id, stream_id)) { - if stream_permissions.manage_stream - || stream_permissions.read_stream - || stream_permissions.manage_topics - || stream_permissions.read_topics - { - return Ok(()); - } - - if let Some(topics) = &stream_permissions.topics - && let Some(topic_permissions) = topics.get(&topic_id) - && (topic_permissions.manage_topic || topic_permissions.read_topic) - { - return Ok(()); - } - } - - Err(IggyError::Unauthorized) -} - -fn perm_get_topics_inner( - m: &InnerMetadata, - user_id: u32, - stream_id: StreamId, -) -> Result<(), IggyError> { - if let Some(global) = m.users_global_permissions.get(&user_id) - && (global.read_streams - || global.manage_streams - || global.manage_topics - || global.read_topics) - { - return Ok(()); - } - - if let Some(stream_permissions) = m.users_stream_permissions.get(&(user_id, stream_id)) - && (stream_permissions.manage_stream - || stream_permissions.read_stream - || stream_permissions.manage_topics - || stream_permissions.read_topics) - { - return Ok(()); - } - - Err(IggyError::Unauthorized) -} - -fn perm_get_consumer_group_inner( - m: &InnerMetadata, - user_id: u32, - stream_id: StreamId, - topic_id: TopicId, -) -> Result<(), IggyError> { - perm_get_topic_inner(m, user_id, stream_id, topic_id) -} - -fn perm_get_user_inner(m: &InnerMetadata, user_id: u32) -> Result<(), IggyError> { - if let Some(global) = m.users_global_permissions.get(&user_id) - && (global.manage_users || global.read_users) - { - return Ok(()); - } - Err(IggyError::Unauthorized) -} - -fn perm_get_users_inner(m: &InnerMetadata, user_id: u32) -> Result<(), IggyError> { - perm_get_user_inner(m, user_id) -} - -fn perm_get_consumer_offset_inner( - m: &InnerMetadata, - user_id: u32, - stream_id: StreamId, - topic_id: TopicId, -) -> Result<(), IggyError> { - if m.users_can_poll_all_streams.contains(&user_id) { - return Ok(()); - } - - if let Some(global) = m.users_global_permissions.get(&user_id) - && (global.read_topics - || global.manage_topics - || global.read_streams - || global.manage_streams) - { - return Ok(()); - } - - if m.users_can_poll_stream.contains(&(user_id, stream_id)) { - return Ok(()); - } - - let Some(stream_permissions) = m.users_stream_permissions.get(&(user_id, stream_id)) else { - return Err(IggyError::Unauthorized); - }; - - if stream_permissions.manage_stream || stream_permissions.read_stream { - return Ok(()); - } - - if stream_permissions.manage_topics || stream_permissions.read_topics { - return Ok(()); - } - - if stream_permissions.poll_messages { - return Ok(()); - } - - if let Some(topics) = &stream_permissions.topics - && let Some(topic_permissions) = topics.get(&topic_id) - && (topic_permissions.manage_topic - || topic_permissions.read_topic - || topic_permissions.poll_messages) - { - return Ok(()); - } - - Err(IggyError::Unauthorized) -} - -fn perm_poll_messages_inner( - m: &InnerMetadata, - user_id: u32, - stream_id: StreamId, - topic_id: TopicId, -) -> Result<(), IggyError> { - perm_get_consumer_offset_inner(m, user_id, stream_id, topic_id) -} - -fn perm_append_messages_inner( - m: &InnerMetadata, - user_id: u32, - stream_id: StreamId, - topic_id: TopicId, -) -> Result<(), IggyError> { - if m.users_can_send_all_streams.contains(&user_id) { - return Ok(()); - } - - if let Some(global) = m.users_global_permissions.get(&user_id) - && (global.manage_streams || global.manage_topics) - { - return Ok(()); - } - - if m.users_can_send_stream.contains(&(user_id, stream_id)) { - return Ok(()); - } - - let Some(stream_permissions) = m.users_stream_permissions.get(&(user_id, stream_id)) else { - return Err(IggyError::Unauthorized); - }; - - if stream_permissions.manage_stream - || stream_permissions.manage_topics - || stream_permissions.send_messages - { - return Ok(()); - } - - if let Some(topics) = &stream_permissions.topics - && let Some(topic_permissions) = topics.get(&topic_id) - && (topic_permissions.manage_topic || topic_permissions.send_messages) - { - return Ok(()); - } - - Err(IggyError::Unauthorized) -} diff --git a/core/server/src/metadata/stream.rs b/core/server/src/metadata/stream.rs deleted file mode 100644 index 02aee459dd..0000000000 --- a/core/server/src/metadata/stream.rs +++ /dev/null @@ -1,64 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::metadata::topic::TopicMeta; -use crate::metadata::{StreamId, TopicId}; -use crate::streaming::stats::StreamStats; -use ahash::AHashMap; -use iggy_common::IggyTimestamp; -use slab::Slab; -use std::sync::Arc; - -/// Stream metadata stored in the shared snapshot. -#[derive(Clone, Debug)] -pub struct StreamMeta { - pub id: StreamId, - pub name: Arc, - pub created_at: IggyTimestamp, - pub stats: Arc, - pub topics: Slab, - pub topic_index: AHashMap, TopicId>, -} - -impl StreamMeta { - pub fn new(id: StreamId, name: Arc, created_at: IggyTimestamp) -> Self { - Self { - id, - name, - created_at, - stats: Arc::new(StreamStats::default()), - topics: Slab::new(), - topic_index: AHashMap::default(), - } - } - - pub fn with_stats( - id: StreamId, - name: Arc, - created_at: IggyTimestamp, - stats: Arc, - ) -> Self { - Self { - id, - name, - created_at, - stats, - topics: Slab::new(), - topic_index: AHashMap::default(), - } - } -} diff --git a/core/server/src/metadata/topic.rs b/core/server/src/metadata/topic.rs deleted file mode 100644 index ba0a88b3da..0000000000 --- a/core/server/src/metadata/topic.rs +++ /dev/null @@ -1,72 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::metadata::consumer_group::ConsumerGroupMeta; -use crate::metadata::partition::PartitionMeta; -use crate::metadata::{ConsumerGroupId, TopicId}; -use crate::streaming::stats::TopicStats; -use ahash::AHashMap; -use iggy_common::{CompressionAlgorithm, IggyExpiry, IggyTimestamp, MaxTopicSize}; -use slab::Slab; -use std::sync::Arc; -use std::sync::atomic::AtomicUsize; - -/// Topic metadata stored in the shared snapshot. -#[derive(Clone, Debug)] -pub struct TopicMeta { - pub id: TopicId, - pub name: Arc, - pub created_at: IggyTimestamp, - pub message_expiry: IggyExpiry, - pub compression_algorithm: CompressionAlgorithm, - pub max_topic_size: MaxTopicSize, - pub replication_factor: u8, - pub stats: Arc, - pub partitions: Vec, - pub consumer_groups: Slab, - pub consumer_group_index: AHashMap, ConsumerGroupId>, - pub round_robin_counter: Arc, -} - -impl TopicMeta { - #[allow(clippy::too_many_arguments)] - pub fn with_stats( - id: TopicId, - name: Arc, - created_at: IggyTimestamp, - message_expiry: IggyExpiry, - compression_algorithm: CompressionAlgorithm, - max_topic_size: MaxTopicSize, - replication_factor: u8, - stats: Arc, - ) -> Self { - Self { - id, - name, - created_at, - message_expiry, - compression_algorithm, - max_topic_size, - replication_factor, - stats, - partitions: Vec::new(), - consumer_groups: Slab::new(), - consumer_group_index: AHashMap::default(), - round_robin_counter: Arc::new(AtomicUsize::new(0)), - } - } -} diff --git a/core/server/src/metadata/writer.rs b/core/server/src/metadata/writer.rs deleted file mode 100644 index 4712679c82..0000000000 --- a/core/server/src/metadata/writer.rs +++ /dev/null @@ -1,617 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::metadata::consumer_group_member::CompletableRevocation; -use crate::metadata::inner::InnerMetadata; -use crate::metadata::ops::MetadataOp; -use crate::metadata::reader::Metadata; -use crate::metadata::{ - ConsumerGroupId, ConsumerGroupMeta, PartitionId, PartitionMeta, StreamId, StreamMeta, TopicId, - TopicMeta, UserId, UserMeta, -}; -use crate::streaming::partitions::consumer_group_offsets::ConsumerGroupOffsets; -use crate::streaming::partitions::consumer_offsets::ConsumerOffsets; -use crate::streaming::stats::{PartitionStats, StreamStats, TopicStats}; -use iggy_common::{ - CompressionAlgorithm, Identifier, IggyError, IggyExpiry, IggyTimestamp, MaxTopicSize, - Permissions, PersonalAccessToken, UserStatus, -}; -use left_right::WriteHandle; -use slab::Slab; -use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::{Arc, Mutex}; - -pub struct MetadataWriter { - inner: WriteHandle, - revision: u64, -} - -impl MetadataWriter { - pub fn new(handle: WriteHandle) -> Self { - Self { - inner: handle, - revision: 0, - } - } - - fn next_revision(&mut self) -> u64 { - self.revision += 1; - self.revision - } - - pub fn append(&mut self, op: MetadataOp) { - self.inner.append(op); - } - - pub fn publish(&mut self) { - self.inner.publish(); - } - - pub fn initialize(&mut self, initial: InnerMetadata) { - self.append(MetadataOp::Initialize(Box::new(initial))); - self.publish(); - } - - pub fn add_stream(&mut self, meta: StreamMeta) -> StreamId { - let assigned_id = Arc::new(AtomicUsize::new(usize::MAX)); - self.append(MetadataOp::AddStream { - meta, - assigned_id: assigned_id.clone(), - }); - self.publish(); - let id = assigned_id.load(Ordering::Acquire); - debug_assert_ne!(id, usize::MAX, "add_stream should always succeed"); - id - } - - pub fn update_stream(&mut self, id: StreamId, new_name: Arc) { - self.append(MetadataOp::UpdateStream { id, new_name }); - self.publish(); - } - - pub fn delete_stream(&mut self, id: StreamId) { - self.append(MetadataOp::DeleteStream { id }); - self.publish(); - } - - pub fn add_topic(&mut self, stream_id: StreamId, meta: TopicMeta) -> Option { - let assigned_id = Arc::new(AtomicUsize::new(usize::MAX)); - self.append(MetadataOp::AddTopic { - stream_id, - meta, - assigned_id: assigned_id.clone(), - }); - self.publish(); - let id = assigned_id.load(Ordering::Acquire); - if id == usize::MAX { None } else { Some(id) } - } - - #[allow(clippy::too_many_arguments)] - pub fn update_topic( - &mut self, - stream_id: StreamId, - topic_id: TopicId, - new_name: Arc, - message_expiry: IggyExpiry, - compression_algorithm: CompressionAlgorithm, - max_topic_size: MaxTopicSize, - replication_factor: u8, - ) { - self.append(MetadataOp::UpdateTopic { - stream_id, - topic_id, - new_name, - message_expiry, - compression_algorithm, - max_topic_size, - replication_factor, - }); - self.publish(); - } - - pub fn delete_topic(&mut self, stream_id: StreamId, topic_id: TopicId) { - self.append(MetadataOp::DeleteTopic { - stream_id, - topic_id, - }); - self.publish(); - } - - /// Add partitions to a topic. Returns the assigned partition IDs (sequential from current count). - pub fn add_partitions( - &mut self, - reader: &Metadata, - stream_id: StreamId, - topic_id: TopicId, - partitions: Vec, - ) -> Vec { - if partitions.is_empty() { - return Vec::new(); - } - - let count_before = reader - .get_partitions_count(stream_id, topic_id) - .expect("stream and topic must exist when adding partitions"); - let count = partitions.len(); - - let revision_id = self.next_revision(); - self.append(MetadataOp::AddPartitions { - stream_id, - topic_id, - partitions, - revision_id, - }); - self.publish(); - - (count_before..count_before + count).collect() - } - - /// Delete partitions from the end of a topic. - pub fn delete_partitions(&mut self, stream_id: StreamId, topic_id: TopicId, count: u32) { - if count == 0 { - return; - } - self.append(MetadataOp::DeletePartitions { - stream_id, - topic_id, - count, - }); - self.publish(); - } - - pub fn add_user(&mut self, meta: UserMeta) -> UserId { - let assigned_id = Arc::new(AtomicUsize::new(usize::MAX)); - self.append(MetadataOp::AddUser { - meta, - assigned_id: assigned_id.clone(), - }); - self.publish(); - let id = assigned_id.load(Ordering::Acquire); - debug_assert_ne!(id, usize::MAX, "add_user should always succeed"); - id as UserId - } - - pub fn update_user_meta(&mut self, id: UserId, meta: UserMeta) { - self.append(MetadataOp::UpdateUserMeta { id, meta }); - self.publish(); - } - - pub fn delete_user(&mut self, id: UserId) { - self.append(MetadataOp::DeleteUser { id }); - self.publish(); - } - - pub fn add_personal_access_token(&mut self, user_id: UserId, pat: PersonalAccessToken) { - self.append(MetadataOp::AddPersonalAccessToken { user_id, pat }); - self.publish(); - } - - pub fn delete_personal_access_token(&mut self, user_id: UserId, token_hash: Arc) { - self.append(MetadataOp::DeletePersonalAccessToken { - user_id, - token_hash, - }); - self.publish(); - } - - pub fn add_consumer_group( - &mut self, - stream_id: StreamId, - topic_id: TopicId, - meta: ConsumerGroupMeta, - ) -> Option { - let assigned_id = Arc::new(AtomicUsize::new(usize::MAX)); - self.append(MetadataOp::AddConsumerGroup { - stream_id, - topic_id, - meta, - assigned_id: assigned_id.clone(), - }); - self.publish(); - let id = assigned_id.load(Ordering::Acquire); - if id == usize::MAX { None } else { Some(id) } - } - - pub fn delete_consumer_group( - &mut self, - stream_id: StreamId, - topic_id: TopicId, - group_id: ConsumerGroupId, - ) { - self.append(MetadataOp::DeleteConsumerGroup { - stream_id, - topic_id, - group_id, - }); - self.publish(); - } - - pub fn join_consumer_group( - &mut self, - stream_id: StreamId, - topic_id: TopicId, - group_id: ConsumerGroupId, - client_id: u32, - valid_client_ids: Option>, - ) -> (Option, Vec) { - let member_id = Arc::new(AtomicUsize::new(usize::MAX)); - let completable = Arc::new(Mutex::new(Vec::new())); - self.append(MetadataOp::JoinConsumerGroup { - stream_id, - topic_id, - group_id, - client_id, - member_id: member_id.clone(), - valid_client_ids, - completable_revocations: completable.clone(), - }); - self.publish(); - let id = member_id.load(Ordering::Acquire); - let revocations = match Arc::try_unwrap(completable) { - Ok(mutex) => mutex.into_inner().unwrap(), - Err(arc) => std::mem::take(&mut *arc.lock().unwrap()), - }; - (if id == usize::MAX { None } else { Some(id) }, revocations) - } - - pub fn leave_consumer_group( - &mut self, - stream_id: StreamId, - topic_id: TopicId, - group_id: ConsumerGroupId, - client_id: u32, - ) -> Option { - let removed_member_id = Arc::new(AtomicUsize::new(usize::MAX)); - self.append(MetadataOp::LeaveConsumerGroup { - stream_id, - topic_id, - group_id, - client_id, - removed_member_id: removed_member_id.clone(), - }); - self.publish(); - let id = removed_member_id.load(Ordering::Acquire); - if id == usize::MAX { None } else { Some(id) } - } - - pub fn rebalance_consumer_groups_for_topic( - &mut self, - stream_id: StreamId, - topic_id: TopicId, - partitions_count: u32, - ) { - self.append(MetadataOp::RebalanceConsumerGroupsForTopic { - stream_id, - topic_id, - partitions_count, - }); - self.publish(); - } - - #[allow(clippy::too_many_arguments)] - pub fn complete_partition_revocation( - &mut self, - stream_id: StreamId, - topic_id: TopicId, - group_id: ConsumerGroupId, - member_slab_id: usize, - member_id: usize, - partition_id: PartitionId, - timed_out: bool, - ) { - self.append(MetadataOp::CompletePartitionRevocation { - stream_id, - topic_id, - group_id, - member_slab_id, - member_id, - partition_id, - timed_out, - }); - self.publish(); - } - - // High-level registration methods with validation - - pub fn create_stream( - &mut self, - reader: &Metadata, - name: Arc, - created_at: IggyTimestamp, - ) -> Result<(StreamId, Arc), IggyError> { - if reader.stream_name_exists(&name) { - return Err(IggyError::StreamNameAlreadyExists(name.to_string())); - } - - let stats = Arc::new(StreamStats::default()); - let meta = StreamMeta::with_stats(0, name, created_at, stats.clone()); - let id = self.add_stream(meta); - Ok((id, stats)) - } - - pub fn try_update_stream( - &mut self, - reader: &Metadata, - id: StreamId, - new_name: Arc, - ) -> Result<(), IggyError> { - let guard = reader.load(); - let Some(stream) = guard.streams.get(id) else { - return Err(IggyError::StreamIdNotFound( - Identifier::numeric(id as u32).unwrap(), - )); - }; - - if stream.name == new_name { - return Ok(()); - } - - if let Some(&existing_id) = guard.stream_index.get(&new_name) - && existing_id != id - { - return Err(IggyError::StreamNameAlreadyExists(new_name.to_string())); - } - drop(guard); - - self.update_stream(id, new_name); - Ok(()) - } - - #[allow(clippy::too_many_arguments)] - pub fn create_topic( - &mut self, - reader: &Metadata, - stream_id: StreamId, - name: Arc, - created_at: IggyTimestamp, - message_expiry: IggyExpiry, - compression_algorithm: CompressionAlgorithm, - max_topic_size: MaxTopicSize, - replication_factor: u8, - ) -> Result<(TopicId, Arc), IggyError> { - let parent_stats = reader.get_stream_stats(stream_id).ok_or_else(|| { - IggyError::StreamIdNotFound(Identifier::numeric(stream_id as u32).unwrap()) - })?; - - let guard = reader.load(); - let Some(stream) = guard.streams.get(stream_id) else { - return Err(IggyError::StreamIdNotFound( - Identifier::numeric(stream_id as u32).unwrap(), - )); - }; - - if stream.topic_index.contains_key(&name) { - return Err(IggyError::TopicNameAlreadyExists( - name.to_string(), - Identifier::numeric(stream_id as u32).unwrap(), - )); - } - drop(guard); - - let stats = Arc::new(TopicStats::new(parent_stats)); - let meta = TopicMeta { - id: 0, - name, - created_at, - message_expiry, - compression_algorithm, - max_topic_size, - replication_factor, - stats: stats.clone(), - partitions: Vec::new(), - consumer_groups: Slab::new(), - consumer_group_index: ahash::AHashMap::default(), - round_robin_counter: Arc::new(AtomicUsize::new(0)), - }; - - // change to create_topic - let id = self.add_topic(stream_id, meta).ok_or_else(|| { - IggyError::StreamIdNotFound(Identifier::numeric(stream_id as u32).unwrap()) - })?; - Ok((id, stats)) - } - - #[allow(clippy::too_many_arguments)] - pub fn try_update_topic( - &mut self, - reader: &Metadata, - stream_id: StreamId, - topic_id: TopicId, - new_name: Arc, - message_expiry: IggyExpiry, - compression_algorithm: CompressionAlgorithm, - max_topic_size: MaxTopicSize, - replication_factor: u8, - ) -> Result<(), IggyError> { - let guard = reader.load(); - let Some(stream) = guard.streams.get(stream_id) else { - return Err(IggyError::StreamIdNotFound( - Identifier::numeric(stream_id as u32).unwrap(), - )); - }; - - let Some(topic) = stream.topics.get(topic_id) else { - return Err(IggyError::TopicIdNotFound( - Identifier::numeric(topic_id as u32).unwrap(), - Identifier::numeric(stream_id as u32).unwrap(), - )); - }; - - if topic.name != new_name - && let Some(&existing_id) = stream.topic_index.get(&new_name) - && existing_id != topic_id - { - return Err(IggyError::TopicNameAlreadyExists( - new_name.to_string(), - Identifier::numeric(stream_id as u32).unwrap(), - )); - } - drop(guard); - - self.update_topic( - stream_id, - topic_id, - new_name, - message_expiry, - compression_algorithm, - max_topic_size, - replication_factor, - ); - Ok(()) - } - - pub fn register_partitions( - &mut self, - reader: &Metadata, - stream_id: StreamId, - topic_id: TopicId, - count: usize, - created_at: IggyTimestamp, - ) -> Vec<(PartitionId, Arc)> { - if count == 0 { - return Vec::new(); - } - - let parent_stats = reader - .get_topic_stats(stream_id, topic_id) - .expect("Parent topic stats must exist before registering partitions"); - - let mut metas = Vec::with_capacity(count); - let mut stats_list = Vec::with_capacity(count); - - for _ in 0..count { - let stats = Arc::new(PartitionStats::new(parent_stats.clone())); - metas.push(PartitionMeta { - id: 0, - created_at, - revision_id: 0, - stats: stats.clone(), - consumer_offsets: Arc::new(ConsumerOffsets::with_capacity(0)), - consumer_group_offsets: Arc::new(ConsumerGroupOffsets::with_capacity(0)), - last_polled_offsets: Arc::new(papaya::HashMap::new()), - }); - stats_list.push(stats); - } - - let ids = self.add_partitions(reader, stream_id, topic_id, metas); - ids.into_iter().zip(stats_list).collect() - } - - pub fn create_user( - &mut self, - reader: &Metadata, - username: Arc, - password_hash: Arc, - status: UserStatus, - permissions: Option>, - max_users: usize, - ) -> Result { - if reader.username_exists(&username) { - return Err(IggyError::UserAlreadyExists); - } - - if reader.users_count() >= max_users { - return Err(IggyError::UsersLimitReached); - } - - let meta = UserMeta { - id: 0, - username, - password_hash, - status, - permissions, - created_at: IggyTimestamp::now(), - }; - let id = self.add_user(meta); - Ok(id) - } - - pub fn update_user( - &mut self, - reader: &Metadata, - id: UserId, - username: Option>, - status: Option, - ) -> Result { - let Some(mut meta) = reader.get_user(id) else { - return Err(IggyError::ResourceNotFound(format!("user:{id}"))); - }; - - if let Some(new_username) = username { - if meta.username != new_username && reader.username_exists(&new_username) { - return Err(IggyError::UserAlreadyExists); - } - meta.username = new_username; - } - - if let Some(new_status) = status { - meta.status = new_status; - } - - let updated = meta.clone(); - self.update_user_meta(id, meta); - Ok(updated) - } - - pub fn create_consumer_group( - &mut self, - reader: &Metadata, - stream_id: StreamId, - topic_id: TopicId, - name: Arc, - partitions_count: u32, - ) -> Result { - let guard = reader.load(); - let Some(stream) = guard.streams.get(stream_id) else { - return Err(IggyError::StreamIdNotFound( - Identifier::numeric(stream_id as u32).unwrap(), - )); - }; - - let Some(topic) = stream.topics.get(topic_id) else { - return Err(IggyError::TopicIdNotFound( - Identifier::numeric(topic_id as u32).unwrap(), - Identifier::numeric(stream_id as u32).unwrap(), - )); - }; - - if topic.consumer_group_index.contains_key(&name) { - return Err(IggyError::ConsumerGroupNameAlreadyExists( - name.to_string(), - Identifier::numeric(topic_id as u32).unwrap(), - )); - } - drop(guard); - - let meta = ConsumerGroupMeta { - id: 0, - name, - partitions: (0..partitions_count as usize).collect(), - members: Slab::new(), - }; - - let id = self - .add_consumer_group(stream_id, topic_id, meta) - .ok_or_else(|| { - IggyError::TopicIdNotFound( - Identifier::numeric(topic_id as u32).unwrap(), - Identifier::numeric(stream_id as u32).unwrap(), - ) - })?; - Ok(id) - } -} diff --git a/core/server-ng/src/offset_recovery.rs b/core/server/src/offset_recovery.rs similarity index 73% rename from core/server-ng/src/offset_recovery.rs rename to core/server/src/offset_recovery.rs index 2e70291938..afb60c8209 100644 --- a/core/server-ng/src/offset_recovery.rs +++ b/core/server/src/offset_recovery.rs @@ -15,17 +15,19 @@ // specific language governing permissions and limitations // under the License. -//! server-ng-owned consumer offset recovery. +//! Server-owned consumer offset recovery. //! //! Forked from `server::streaming::partitions::storage` (the legacy -//! `load_consumer_offsets` / `load_consumer_group_offsets`) so server-ng +//! `load_consumer_offsets` / `load_consumer_group_offsets`) so server //! owns the loaders for the offset files its own persistence path writes, -//! without depending on the legacy `server` crate. The on-disk format is -//! shared with the legacy server today: one file per consumer (numeric -//! file name = consumer id) holding a single little-endian `u64` offset. +//! without depending on the legacy `server` crate. One file per consumer (numeric +//! file name = consumer id) holding a little-endian `u64` offset then a checksum over +//! it; see [`partitions::offset_storage`]. The legacy server stays compatible both +//! ways: it reads the first eight bytes and stops, and a file it wrote itself decodes +//! here as unchecksummed. use iggy_common::{ConsumerGroupId, ConsumerKind, ConsumerOffset, IggyError}; -use std::io::Read; +use partitions::offset_storage::{OffsetRecord, decode_offset_record}; use std::sync::atomic::AtomicU64; use tracing::{error, trace, warn}; @@ -169,23 +171,48 @@ pub fn load_consumer_group_offsets( } fn read_offset_file(path: &str, offset_kind: &'static str) -> Option { - let mut file = match std::fs::File::open(path) { - Ok(file) => file, + let bytes = match std::fs::read(path) { + Ok(bytes) => bytes, Err(e) => { warn!( - "{COMPONENT} (error: {e}) - failed to open offset file, \ + "{COMPONENT} (error: {e}) - failed to read offset file, \ path: {path}, skipping." ); return None; } }; - let mut offset = [0; 8]; - if let Err(e) = file.read_exact(&mut offset) { - warn!( - "{COMPONENT} (error: {e}) - failed to read {offset_kind} from file \ - (truncated or corrupt?), path: {path}, skipping." - ); - return None; + match decode_offset_record(&bytes) { + OffsetRecord::Value { offset, .. } => Some(AtomicU64::new(offset)), + OffsetRecord::Torn => { + warn!( + "{COMPONENT} - failed to read {offset_kind} from file (truncated), \ + path: {path}, skipping." + ); + None + } + // Skipped rather than loaded: resuming from a cursor provably not the one + // written reads as ordinary redelivery or a gap, never as corruption. + // + // And unlinked, not just skipped: the offset map starts cold every boot, so a + // file left behind is re-read by the first auto-commit and trips the commit + // path again. + OffsetRecord::Corrupt { + offset, + expected, + found, + } => { + error!( + "{COMPONENT} - {offset_kind} file failed its checksum \ + (offset: {offset}, expected: {expected}, found: {found}), \ + path: {path}, removing it and resuming this consumer from the start." + ); + if let Err(e) = std::fs::remove_file(path) { + error!( + "{COMPONENT} (error: {e}) - could not remove the corrupt \ + {offset_kind} file, path: {path}; remove it manually." + ); + } + None + } } - Some(AtomicU64::new(u64::from_le_bytes(offset))) } diff --git a/core/server-ng/src/partition_helpers.rs b/core/server/src/partition_helpers.rs similarity index 94% rename from core/server-ng/src/partition_helpers.rs rename to core/server/src/partition_helpers.rs index a7a291eb76..48ea83a791 100644 --- a/core/server-ng/src/partition_helpers.rs +++ b/core/server/src/partition_helpers.rs @@ -25,9 +25,9 @@ //! consumer-offset configuration, and initial-segment provisioning. use crate::offset_recovery::{load_consumer_group_offsets, load_consumer_offsets}; -use crate::server_error::ServerNgError; +use crate::server_error::ServerError; use compio::fs::create_dir_all; -use configs::server_ng::ServerNgConfig; +use configs::server::ServerConfig; use consensus::{LocalPipeline, VsrConsensus, VsrState}; use iggy_common::{ ConsumerGroupOffsets, ConsumerOffsets, IggyError, IggyTimestamp, PartitionStats, @@ -54,15 +54,15 @@ use tracing::{error, info, warn}; /// /// # Errors /// -/// Returns [`ServerNgError::RecoveredNamespaceOutOfBounds`] if any of +/// Returns [`ServerError::RecoveredNamespaceOutOfBounds`] if any of /// `stream_id`, `topic_id`, or `partition_id` exceed the configured /// maxima. pub const fn validate_namespace_bounds( - config: &ServerNgConfig, + config: &ServerConfig, stream_id: usize, topic_id: usize, partition_id: usize, -) -> Result<(), ServerNgError> { +) -> Result<(), ServerError> { let namespace = &config.extra.namespace; if stream_id < namespace.max_streams && topic_id < namespace.max_topics @@ -71,7 +71,7 @@ pub const fn validate_namespace_bounds( return Ok(()); } - Err(ServerNgError::RecoveredNamespaceOutOfBounds { + Err(ServerError::RecoveredNamespaceOutOfBounds { stream_id, topic_id, partition_id, @@ -96,7 +96,7 @@ pub async fn create_partition_file_hierarchy( stream_id: usize, topic_id: usize, partition_id: usize, - config: &ServerNgConfig, + config: &ServerConfig, ) -> Result<(), IggyError> { let partition_path = config .system @@ -174,15 +174,15 @@ pub async fn create_partition_file_hierarchy( /// /// # Errors /// -/// Returns [`ServerNgError::ConsumerOffsetsLoad`] when the on-disk files +/// Returns [`ServerError::ConsumerOffsetsLoad`] when the on-disk files /// exist but fail to decode. A stored offset ahead of `current_offset` is /// clamped (with a warning), not an error. pub fn configure_consumer_offsets( partition: &mut IggyPartition>, - config: &ServerNgConfig, + config: &ServerConfig, namespace: IggyNamespace, current_offset: u64, -) -> Result<(), ServerNgError> { +) -> Result<(), ServerError> { let stream_id = namespace.stream_id(); let topic_id = namespace.topic_id(); let partition_id = namespace.partition_id(); @@ -270,7 +270,7 @@ fn load_partition_consumer_offsets( stream_id: usize, topic_id: usize, partition_id: usize, -) -> Result, ServerNgError> { +) -> Result, ServerError> { if !Path::new(path).exists() { return Ok(Vec::new()); } @@ -281,7 +281,7 @@ fn load_partition_consumer_offsets( return Ok(Vec::new()); } - Err(ServerNgError::ConsumerOffsetsLoad { + Err(ServerError::ConsumerOffsetsLoad { consumer_kind, stream_id, topic_id, @@ -297,7 +297,7 @@ fn load_partition_consumer_group_offsets( stream_id: usize, topic_id: usize, partition_id: usize, -) -> Result, ServerNgError> { +) -> Result, ServerError> { if !Path::new(path).exists() { return Ok(Vec::new()); } @@ -308,7 +308,7 @@ fn load_partition_consumer_group_offsets( return Ok(Vec::new()); } - Err(ServerNgError::ConsumerOffsetsLoad { + Err(ServerError::ConsumerOffsetsLoad { consumer_kind: "consumer group", stream_id, topic_id, @@ -327,15 +327,15 @@ fn load_partition_consumer_group_offsets( /// /// # Errors /// -/// Returns [`ServerNgError`] on segment-storage creation failure or +/// Returns [`ServerError`] on segment-storage creation failure or /// writer initialisation failure. pub async fn ensure_initial_segment( partition: &mut IggyPartition>, - config: &ServerNgConfig, + config: &ServerConfig, stream_id: usize, topic_id: usize, partition_id: usize, -) -> Result<(), ServerNgError> { +) -> Result<(), ServerError> { if partition.log.has_segments() { return Ok(()); } @@ -401,6 +401,11 @@ pub async fn ensure_initial_segment( messages_size_counter, config.system.partition.enforce_fsync, false, + config + .system + .segment + .preallocate + .then_some(config.system.segment.size), ) .await .map_err(|source| { @@ -457,14 +462,14 @@ pub async fn ensure_initial_segment( /// /// # Errors /// -/// [`ServerNgError::PartitionSuperblockIo`] when the directory or a slot +/// [`ServerError::PartitionSuperblockIo`] when the directory or a slot /// cannot be read; the `VersionUnknown` / `Unverifiable` / `Undecodable` / /// `IdentityMismatch` variants when a record exists but cannot be trusted. pub(crate) async fn open_partition_superblock( partition_dir: &str, identity: ReplicaIdentity, -) -> Result<(Rc, Option), ServerNgError> { - let io_error = |source| ServerNgError::PartitionSuperblockIo { +) -> Result<(Rc, Option), ServerError> { + let io_error = |source| ServerError::PartitionSuperblockIo { dir: PathBuf::from(partition_dir), source, }; @@ -478,7 +483,7 @@ pub(crate) async fn open_partition_superblock( let recovered_state = match latest { SuperblockContents::Present(bytes) => { Some(VsrState::try_from(bytes.as_slice()).map_err(|source| { - ServerNgError::PartitionSuperblockUndecodable { + ServerError::PartitionSuperblockUndecodable { dir: PathBuf::from(partition_dir), source, } @@ -487,13 +492,13 @@ pub(crate) async fn open_partition_superblock( SuperblockContents::Unreadable { version: Some(version), } => { - return Err(ServerNgError::PartitionSuperblockVersionUnknown { + return Err(ServerError::PartitionSuperblockVersionUnknown { dir: PathBuf::from(partition_dir), version, }); } SuperblockContents::Unreadable { version: None } => { - return Err(ServerNgError::PartitionSuperblockUnverifiable { + return Err(ServerError::PartitionSuperblockUnverifiable { dir: PathBuf::from(partition_dir), }); } @@ -501,7 +506,7 @@ pub(crate) async fn open_partition_superblock( }; if let Some(state) = recovered_state.as_ref() { let mismatch = |field, expected: u128, found: u128| { - Err(ServerNgError::PartitionSuperblockIdentityMismatch { + Err(ServerError::PartitionSuperblockIdentityMismatch { dir: PathBuf::from(partition_dir), field, expected, @@ -542,7 +547,7 @@ pub(crate) fn restore_partition_view( // a replica that came back at view 0 is otherwise indistinguishable from one // that resumed correctly until it votes. info!( - namespace_raw = consensus.namespace(), + namespace_raw = consensus.group(), view = state.view, log_view = state.log_view, "restored partition view from its superblock" @@ -573,11 +578,11 @@ pub(crate) fn restore_partition_view( /// /// # Errors /// -/// Returns [`ServerNgError`] when bounds validation, directory creation, +/// Returns [`ServerError`] when bounds validation, directory creation, /// superblock recovery, or segment provisioning fails. #[allow(clippy::too_many_arguments)] pub async fn build_partition_fresh( - config: &ServerNgConfig, + config: &ServerConfig, namespace: IggyNamespace, stats: Arc, created_revision: u64, @@ -585,7 +590,7 @@ pub async fn build_partition_fresh( self_replica_id: u8, replica_count: u8, bus: Rc, -) -> Result>, ServerNgError> { +) -> Result>, ServerError> { let stream_id = namespace.stream_id(); let topic_id = namespace.topic_id(); let partition_id = namespace.partition_id(); @@ -694,7 +699,6 @@ pub async fn build_partition_fresh( partition.offset.store(0, Ordering::Release); partition.dirty_offset.store(0, Ordering::Relaxed); partition.should_increment_offset = false; - partition.stats.set_current_offset(0); debug_assert!( !partition.log.has_segments(), "fresh partition must not carry recovered segments" @@ -734,7 +738,7 @@ pub async fn delete_partitions_from_disk( stream_id: usize, topic_id: usize, partition_id: usize, - config: &ServerNgConfig, + config: &ServerConfig, ) -> Result<(), IggyError> { let partition_path = config .system @@ -875,7 +879,7 @@ mod tests { let refused = open_partition_superblock(&dir, test_identity()).await; match refused { - Err(ServerNgError::PartitionSuperblockIdentityMismatch { field, .. }) => { + Err(ServerError::PartitionSuperblockIdentityMismatch { field, .. }) => { assert_eq!(field, IdentityField::Cluster); } Err(other) => panic!("expected an identity mismatch, got {other}"), diff --git a/core/server-ng/src/partition_reconciler.rs b/core/server/src/partition_reconciler.rs similarity index 94% rename from core/server-ng/src/partition_reconciler.rs rename to core/server/src/partition_reconciler.rs index 63d6c2a5cb..b08cf52290 100644 --- a/core/server-ng/src/partition_reconciler.rs +++ b/core/server/src/partition_reconciler.rs @@ -169,10 +169,10 @@ //! discriminator, like `checkpoint_id` on every prepare //! -- `PrepareHeader.reserved` has room, but it is a `#[repr(C)]` wire change. -use crate::bootstrap::ServerNgShard; +use crate::bootstrap::ServerShard; use crate::partition_helpers::{build_partition_fresh, delete_partitions_from_disk}; use ahash::{AHashMap, AHashSet}; -use configs::server_ng::ServerNgConfig; +use configs::server::ServerConfig; use consensus::{MetadataHandle, PartitionsHandle}; use futures::FutureExt; use iggy_common::{ConsumerGroupId, IggyTimestamp}; @@ -219,9 +219,9 @@ enum FailureCause { } pub struct ReconcilerCtx { - pub shard: Rc, + pub shard: Rc, pub total_shards: u16, - pub config: Rc, + pub config: Rc, pub cluster_id: u128, pub self_replica_id: u8, pub replica_count: u8, @@ -238,9 +238,9 @@ pub struct ReconcilerCtx { impl ReconcilerCtx { #[must_use] pub fn new( - shard: Rc, + shard: Rc, total_shards: u16, - config: Rc, + config: Rc, cluster_id: u128, self_replica_id: u8, replica_count: u8, @@ -413,6 +413,10 @@ struct PassCounters { /// tombstone and re-wakes us without bumping `Streams::revision`, so an /// armed skip would swallow that wake and strand the rebuild forever. deferred: usize, + /// Namespaces an earlier pass already built, whose `InsertOwned` the pump + /// has not applied yet. Counted so the pass does not arm the fast-skip + /// while work is in flight; applying it bumps no revision. + already_staged: usize, } impl PassCounters { @@ -428,6 +432,7 @@ impl PassCounters { + self.purges_staged + self.deferred + self.parked_reclaimed + + self.already_staged } } @@ -476,9 +481,9 @@ async fn reconcile_once(ctx: &ReconcilerCtx) -> bool { let target_set: AHashSet = target.iter().map(|(ns, _)| *ns).collect(); let mut counters = PassCounters::default(); - let staged = reconcile_additions(ctx, target, &mut counters).await; + reconcile_additions(ctx, target, &mut counters).await; reconcile_removals(ctx, &target_set, &mut counters).await; - reconcile_parked_frames(ctx, &staged, &mut counters); + reconcile_parked_frames(ctx, &mut counters); reconcile_consumer_group_offsets(ctx, &mut counters).await; reconcile_segment_truncations(ctx, &mut counters); reconcile_partition_purges(ctx, &mut counters); @@ -506,6 +511,7 @@ async fn reconcile_once(ctx: &ReconcilerCtx) -> bool { backoff_skipped = counters.backoff_skipped, stale = counters.stale, deferred = counters.deferred, + already_staged = counters.already_staged, parked_reclaimed = counters.parked_reclaimed, purges_staged = counters.purges_staged, trims_pending = counters.trims_pending, @@ -521,19 +527,14 @@ async fn reconcile_once(ctx: &ReconcilerCtx) -> bool { true } -/// Returns the namespaces whose `ReconcileOp::InsertOwned` this pass staged. The -/// pump applies the op on its own task, so they are not in `IggyPartitions` yet -/// and [`reconcile_parked_frames`] would read them as un-materialised, aging -/// their frames on the pass that built them. async fn reconcile_additions( ctx: &ReconcilerCtx, target: Vec<(IggyNamespace, u64)>, counters: &mut PassCounters, -) -> AHashSet { +) { let shard_id = ctx.shard.id; let partitions = ctx.shard.plane.partitions(); let total_shards = u32::from(ctx.total_shards); - let mut staged = AHashSet::new(); for (ns, epoch) in target { if partitions.contains(&ns) { @@ -583,7 +584,7 @@ async fn reconcile_additions( // means the local partition is a prior incarnation carrying // stale segments/offsets/log. Tear it down; the // post-ConfirmRemove wake rebuilds it fresh next pass. - if ctx.shard.shards_table().epoch_for(ns) == Some(epoch) { + if shards_table_has_epoch(ctx, ns, epoch) { continue; } trace!( @@ -599,10 +600,15 @@ async fn reconcile_additions( let owning_shard = calculate_shard_assignment(&ns, total_shards); if owning_shard != shard_id { - // Compare the epoch, not just presence: a delete + recreate recycles - // the slab keys, so the row survives with the DEAD incarnation's - // `created_revision`. A presence-only gate never refreshes it, and - // nothing else writes a non-owner's row. + // Compare the epoch, not just presence: a delete + recreate + // recycles the slab keys, so the row survives with the DEAD + // incarnation's `created_revision`. A presence-only gate never + // refreshes it, and nothing else writes a non-owner's row. + // + // No mirror of the staged-`InsertOwned` guard below, deliberately: + // a lagging pump costs one duplicate `InsertRouted` per pass, and + // the apply is an idempotent row overwrite, while scanning the op + // queue per routed namespace would go quadratic. if !shards_table_has_epoch(ctx, ns, epoch) { ctx.shard.enqueue_reconcile_op(ReconcileOp::InsertRouted { namespace: ns, @@ -614,6 +620,17 @@ async fn reconcile_additions( continue; } + // An earlier pass already built this one and the pump has not applied it + // yet, so the `contains` test above reads false for finished work. + // Rebuilding is not a wasted-effort question: the second build shares + // the namespace's `PartitionStats` with the queued sibling and re-opens + // segment 0 with `file_exists = false`, truncating the file that + // sibling is about to serve. + if ctx.shard.has_staged_insert_owned(ns) { + counters.already_staged += 1; + continue; + } + let now = Instant::now(); if ctx.is_backed_off(ns, FailureCause::Add, now) { counters.backoff_skipped += 1; @@ -648,7 +665,6 @@ async fn reconcile_additions( }); ctx.record_success(ns, FailureCause::Add); counters.materialised += 1; - staged.insert(ns); } Err(err) => { ctx.record_failure(ns, FailureCause::Add, now); @@ -664,8 +680,6 @@ async fn reconcile_additions( } } } - - staged } /// Retire parked frames the shard cannot serve, age the ones it might. @@ -705,18 +719,15 @@ async fn reconcile_additions( /// timeout and no committed op dies on a local-convergence signal. Residency /// only; see `ParkedFrame::passes`. /// -/// `staged_this_pass` is exempt: its `InsertOwned` is queued but not applied, so -/// it reads as un-materialised here. Not a one-pass concession. -/// `reconcile_additions` has no cross-pass guard against a queued-but-unapplied -/// op (it tests `partitions.contains`, false the whole time it sits in the -/// queue), so it re-stages every pass until the pump drains. The exemption -/// therefore covers arbitrary pump lag; dropping it ages frames on every -/// commit-driven pass the pump falls behind. -fn reconcile_parked_frames( - ctx: &ReconcilerCtx, - staged_this_pass: &AHashSet, - counters: &mut PassCounters, -) { +/// A namespace with a staged, unapplied `InsertOwned` is exempt: its partition +/// is on the way but reads as un-materialised here. The queue is asked per +/// parked namespace ([`shard::IggyShard::has_staged_insert_owned`]) rather than +/// carrying a set over from the additions pass, so the answer cannot go stale +/// across `reconcile_removals`' awaits; `parked` is empty on the steady path, +/// so the scan costs nothing there. The exemption spans passes, not just the +/// one that built the namespace, covering arbitrary pump lag; dropping it ages +/// frames on every commit-driven pass the pump falls behind. +fn reconcile_parked_frames(ctx: &ReconcilerCtx, counters: &mut PassCounters) { let parked = ctx.shard.parked_namespaces(); if parked.is_empty() { return; @@ -724,7 +735,7 @@ fn reconcile_parked_frames( let partitions = ctx.shard.plane.partitions(); let total_shards = u32::from(ctx.total_shards); for ns in parked { - if staged_this_pass.contains(&ns) { + if ctx.shard.has_staged_insert_owned(ns) { continue; } // Tombstoned namespaces are still in the map, so `contains` below reads @@ -1134,7 +1145,7 @@ fn reconcile_partition_purges(ctx: &ReconcilerCtx, counters: &mut PassCounters) } } -pub fn install_tick_handler(shard: &Rc, wake_tx: WakeTx) { +pub fn install_tick_handler(shard: &Rc, wake_tx: WakeTx) { let shard_id = shard.id; let handler = Rc::new(move || { if let Err(err) = wake_tx.try_send(()) { @@ -1147,9 +1158,10 @@ pub fn install_tick_handler(shard: &Rc, wake_tx: WakeTx) { #[cfg(test)] mod tests { use super::{ - FailureCause, FailureRecord, ReconcilerCtx, delete_partitions_from_disk, reconcile_once, + FailureCause, FailureRecord, ReconcilerCtx, build_partition_fresh, + delete_partitions_from_disk, fetch_partition_stats, reconcile_once, }; - use configs::server_ng::{NgSystemConfig, ServerNgConfig}; + use configs::server::{ServerConfig, ServerSystemConfig}; use consensus::{MetadataHandle, PartitionsHandle}; use iggy_binary_protocol::codec::WireEncode; use iggy_binary_protocol::primitives::identifier::WireName; @@ -1163,7 +1175,7 @@ mod tests { PurgeTopicRequest, }; use iggy_binary_protocol::{ - Command2, GenericHeader, Operation, PrepareHeader, ReplyHeader, RequestHeader, + Command2, GenericHeader, Operation, PrepareHeader, ReplyHeader, RoutedRequestHeader, WireIdentifier, }; use message_bus::IggyMessageBus; @@ -1181,6 +1193,7 @@ mod tests { use std::mem::size_of; use std::rc::Rc; use std::sync::Arc; + use std::sync::atomic::Ordering; use std::time::Instant; use tempfile::TempDir; @@ -1249,7 +1262,7 @@ mod tests { header.command = Command2::Prepare; header.size = u32::try_from(header_size).expect("prepare size fits u32"); header.operation = Operation::SendMessages; - header.namespace = namespace.inner(); + header.group = namespace.inner(); header.op = op; msg.into_generic() } @@ -1281,17 +1294,17 @@ mod tests { namespace: IggyNamespace, body_len: usize, ) -> Message { - let header_size = size_of::(); + let header_size = size_of::(); let total_size = header_size + body_len; - let mut msg = Message::::new(total_size); - let header = bytemuck::checked::try_from_bytes_mut::( + let mut msg = Message::::new(total_size); + let header = bytemuck::checked::try_from_bytes_mut::( &mut msg.as_mut_slice()[..header_size], ) - .expect("zeroed bytes form a valid RequestHeader"); + .expect("zeroed bytes form a valid RoutedRequestHeader"); header.command = Command2::Request; header.size = u32::try_from(total_size).expect("request size fits u32"); header.operation = Operation::SendMessages; - header.namespace = namespace.inner(); + header.group = namespace.inner(); // Header validation rejects a zero session / request on a non-register // op, and the park path runs after that validation. header.session = 1; @@ -1417,24 +1430,24 @@ mod tests { .expect("JoinConsumerGroup apply succeeds"); } - fn test_config(tmp: &TempDir) -> ServerNgConfig { - let mut cfg = ServerNgConfig::default(); - // `NgSystemConfig` is not `Clone`, so `Arc::make_mut` is out; build a + fn test_config(tmp: &TempDir) -> ServerConfig { + let mut cfg = ServerConfig::default(); + // `ServerSystemConfig` is not `Clone`, so `Arc::make_mut` is out; build a // fresh value via struct-update syntax and swap the Arc wholesale. // Only `path` differs from the default; every other field uses the // runtime's defaults. - let system = NgSystemConfig { + let system = ServerSystemConfig { path: tmp.path().to_string_lossy().into_owned(), - ..NgSystemConfig::default() + ..ServerSystemConfig::default() }; cfg.system = Arc::new(system); cfg } - /// Assemble a fully functional `ServerNgShard` for reconciler tests. + /// Assemble a fully functional `ServerShard` for reconciler tests. /// Uses `IggyShard::without_inbox` so no inter-shard pump runs; the /// reconciler can be driven directly by `reconcile_once`. - fn build_test_shard(shard_id: u16, config: &ServerNgConfig, mux: TestMux) -> Rc { + fn build_test_shard(shard_id: u16, config: &ServerConfig, mux: TestMux) -> Rc { let bus = Rc::new(IggyMessageBus::with_config(shard_id, config)); let metadata: IggyMetadata< consensus::VsrConsensus>, @@ -1448,7 +1461,9 @@ mod tests { messages_required_to_save: 1, size_of_messages_required_to_save: iggy_common::IggyByteSize::from(1024_u64), enforce_fsync: false, + validate_checksum: true, segment_size: config.system.segment.size, + preallocate_segments: false, encryptor: None, }, ); @@ -1479,7 +1494,7 @@ mod tests { /// in this shard's inbox and reading as success. fn build_test_shard_with_inbox( shard_id: u16, - config: &ServerNgConfig, + config: &ServerConfig, mux: TestMux, capacity: usize, ) -> (Rc, shard::Receiver) { @@ -1527,7 +1542,7 @@ mod tests { fn make_ctx( shard: Rc, total_shards: u16, - config: Rc, + config: Rc, ) -> Rc { Rc::new(ReconcilerCtx::new( shard, @@ -1595,61 +1610,149 @@ mod tests { ); } - /// Regression (deferred-apply window): the reconciler stages - /// `ReconcileOp::InsertOwned` from a task separate from the pump that - /// applies it, so under a commit burst it can run a second pass before - /// the pump drains the first pass's staged ops. Both passes then - /// observe `!contains(ns)` and build the same namespace. The pump's - /// apply must be idempotent, else the second `insert` orphans the first - /// partition (leaked VSR group + writers) and inflates `len`. - /// `reconcile_pass` applies inline and cannot surface this, so here we - /// run two passes and only then drain once. + /// The cross-pass guard: a pass must not rebuild a namespace an earlier pass + /// already built and left queued. Rebuilding is not merely wasted work -- + /// the second build shares the namespace's `PartitionStats` with the queued + /// sibling and re-opens segment 0 truncating -- so a pass has to recognise + /// the staged op, not just `partitions.contains`. #[compio::test] - async fn deferred_apply_window_does_not_duplicate_owned_partition() { + async fn second_pass_does_not_rebuild_a_namespace_already_staged() { let tmp = TempDir::new().expect("tempdir for system path"); let config = test_config(&tmp); let mux = TestMux::default(); seed_stream(&mux, 1, "stream-a"); - seed_topic( - &mux, - 2, - 0, - "topic-a", - vec![assignment(0, 1), assignment(1, 2)], - ); + seed_topic(&mux, 2, 0, "topic-a", vec![assignment(0, 1)]); let shard = build_test_shard(0, &config, mux); let ctx = make_ctx(Rc::clone(&shard), 1, Rc::new(config)); + let ns = IggyNamespace::new(0, 0, 0); - // Two passes with no pump drain in between: models the reconciler, - // woken by a second commit tick, running pass N+1 before the pump - // applies pass N's `InsertOwned`. Both passes see the namespaces as - // unmaterialised and stage a build for each, so the queue holds two - // `InsertOwned` per namespace when the pump finally drains. reconcile_once(&ctx).await; + assert!( + ctx.shard.has_staged_insert_owned(ns), + "the first pass must leave an unapplied InsertOwned to guard against" + ); + + // Second pass while that op is still queued: `partitions.contains(ns)` + // is false, so only the staged-op guard can stop the rebuild. reconcile_once(&ctx).await; ctx.shard.apply_reconcile_ops(); + assert_eq!( + shard.plane.partitions().len(), + 1, + "the namespace must materialise exactly once" + ); - let partitions = shard.plane.partitions(); + // Counting builds, not partitions: the pump discards the redundant + // `InsertOwned` either way, so `len` cannot tell one build from two. + // `ensure_initial_segment` plants exactly one segment per build and + // folds it into the namespace's shared stats, so this counter is the + // observable that separates them. + let stats = fetch_partition_stats(&ctx, ns).expect("materialised namespace has stats"); assert_eq!( - partitions.len(), - 2, - "deferred-apply window must not duplicate partitions: \ - each namespace materialises exactly once" + stats.segments_count_inconsistent(), + 1, + "a second build ran: its initial segment was folded into the \ + namespace's shared stats on top of the live incarnation's" + ); + } + + /// The stats registry keys on the namespace, not the incarnation, so + /// `current_offset` moves on adoption only: the pump seeds it from the + /// incarnation it inserts, and a build that never becomes addressable + /// leaves it alone. Seeding from the build instead zeroed it under the live + /// incarnation, after which the partition plane's admission check read an + /// empty offset space and answered every `store_consumer_offset` above 0 + /// with `InvalidOffset` (error 4100) until the next send re-seeded it. + /// + /// Both incarnations are built and staged by hand: the adopted one so its + /// counter is non-zero BEFORE insertion (a reconciler build always adopts + /// at 0, where the publish is indistinguishable from a no-op), and the + /// redundant one because `has_staged_insert_owned` now stops a pass from + /// producing it. + #[compio::test] + async fn discarded_build_leaves_live_partition_offset_intact() { + const COMMITTED_OFFSET: u64 = 3; + const LIVE_EPOCH: u64 = 1; + + let tmp = TempDir::new().expect("tempdir for system path"); + let config = test_config(&tmp); + let mux = TestMux::default(); + seed_stream(&mux, 1, "stream-a"); + seed_topic(&mux, 2, 0, "topic-a", vec![assignment(0, 1)]); + + let shard = build_test_shard(0, &config, mux); + let ctx = make_ctx(Rc::clone(&shard), 1, Rc::new(config.clone())); + let ns = IggyNamespace::new(0, 0, 0); + + let stats = fetch_partition_stats(&ctx, ns).expect("committed namespace has stats"); + let live = build_partition_fresh( + &config, + ns, + Arc::clone(&stats), + LIVE_EPOCH, + CLUSTER_ID, + 0, + 1, + Rc::clone(&ctx.shard.bus), + ) + .await + .expect("live build succeeds"); + // What a recovery leaves behind: an incarnation whose own counter is + // ahead of the zeroed shared stats until adoption publishes it. + live.offset.store(COMMITTED_OFFSET, Ordering::Release); + ctx.shard.enqueue_reconcile_op(ReconcileOp::InsertOwned { + namespace: ns, + partition: Box::new(live), + epoch: LIVE_EPOCH, + }); + ctx.shard.apply_reconcile_ops(); + + assert_eq!( + stats.current_offset(), + COMMITTED_OFFSET, + "adoption must publish the incarnation's offset into the shared stats" + ); + + let redundant = build_partition_fresh( + &config, + ns, + Arc::clone(&stats), + LIVE_EPOCH + 1, + CLUSTER_ID, + 0, + 1, + Rc::clone(&ctx.shard.bus), + ) + .await + .expect("redundant build succeeds over the live incarnation's path"); + ctx.shard.enqueue_reconcile_op(ReconcileOp::InsertOwned { + namespace: ns, + partition: Box::new(redundant), + epoch: LIVE_EPOCH + 1, + }); + ctx.shard.apply_reconcile_ops(); + + assert_eq!( + shard.plane.partitions().len(), + 1, + "the redundant build must be discarded, not adopted: a second insert \ + overwrites the ns -> idx entry and orphans the first partition, \ + leaking its VSR group and segment writers" + ); + // The epoch, not `shard_for`: on a single shard an adopt would write + // `ShardId::new(0)` too, but it would stamp the redundant op's epoch. + assert_eq!( + shard.shards_table().epoch_for(ns), + Some(LIVE_EPOCH), + "the discarded op must not rewrite the routing row" + ); + assert_eq!( + stats.current_offset(), + COMMITTED_OFFSET, + "a discarded build must not reset the live incarnation's current_offset" ); - for partition_id in 0..2 { - let ns = IggyNamespace::new(0, 0, partition_id); - assert!( - partitions.contains(&ns), - "namespace {ns:?} must be addressable exactly once" - ); - assert_eq!( - shard.shards_table().shard_for(ns), - Some(0), - "shards_table must point at the owning shard" - ); - } } /// Multi-shard scenario: only the partition whose hash maps to diff --git a/core/server-ng/src/pat.rs b/core/server/src/pat.rs similarity index 93% rename from core/server-ng/src/pat.rs rename to core/server/src/pat.rs index dc22f20004..26c8ea6db4 100644 --- a/core/server-ng/src/pat.rs +++ b/core/server/src/pat.rs @@ -27,7 +27,7 @@ use iggy_binary_protocol::requests::personal_access_tokens::{ CreatePersonalAccessTokenRequest as WireCreatePersonalAccessTokenRequest, DeletePersonalAccessTokenRequest as WireDeletePersonalAccessTokenRequest, }; -use iggy_binary_protocol::{Operation, RequestHeader, WireDecode, WireEncode}; +use iggy_binary_protocol::{Operation, RoutedRequestHeader, WireDecode, WireEncode}; use iggy_common::IggyError; use metadata::stm::user::{ CreatePersonalAccessTokenRequest as ReplicatedCreatePersonalAccessTokenRequest, @@ -42,8 +42,8 @@ pub(crate) fn maybe_rewrite_pat_request( transport_client_id: u128, max_tokens_per_user: u32, pat_count_of: impl FnOnce(u32) -> usize, - request: Message, -) -> Result<(Message, Option), IggyError> { + request: Message, +) -> Result<(Message, Option), IggyError> { let user_id = match request.header().operation { Operation::CreatePersonalAccessToken | Operation::DeletePersonalAccessToken => sessions .borrow() @@ -69,8 +69,8 @@ pub(crate) fn rewrite_pat_request_for_user( user_id: u32, max_tokens_per_user: u32, pat_count_of: impl FnOnce(u32) -> usize, - request: Message, -) -> Result<(Message, Option), IggyError> { + request: Message, +) -> Result<(Message, Option), IggyError> { let body = request_body(&request); let mut raw_token = None; let rewritten = match request.header().operation { @@ -151,10 +151,10 @@ mod tests { const MAX_TOKENS: u32 = 3; const AT_LIMIT_TOKENS: [(&str, u8); 3] = [("one", b'a'), ("two", b'b'), ("three", b'c')]; - fn create_pat_request(name: &str) -> Message { - let header_len = std::mem::size_of::(); - let mut template = Message::::new(header_len); - let header = bytemuck::checked::try_from_bytes_mut::( + fn create_pat_request(name: &str) -> Message { + let header_len = std::mem::size_of::(); + let mut template = Message::::new(header_len); + let header = bytemuck::checked::try_from_bytes_mut::( &mut template.as_mut_slice()[..header_len], ) .expect("zeroed bytes are a valid request header"); @@ -266,9 +266,9 @@ mod tests { #[test] fn given_delete_op_when_at_limit_should_pass_the_gate() { - let header_len = std::mem::size_of::(); - let mut template = Message::::new(header_len); - let header = bytemuck::checked::try_from_bytes_mut::( + let header_len = std::mem::size_of::(); + let mut template = Message::::new(header_len); + let header = bytemuck::checked::try_from_bytes_mut::( &mut template.as_mut_slice()[..header_len], ) .expect("zeroed bytes are a valid request header"); diff --git a/core/server-ng/src/personal_access_token_cleaner.rs b/core/server/src/personal_access_token_cleaner.rs similarity index 96% rename from core/server-ng/src/personal_access_token_cleaner.rs rename to core/server/src/personal_access_token_cleaner.rs index 9b150129d5..a55093605a 100644 --- a/core/server-ng/src/personal_access_token_cleaner.rs +++ b/core/server/src/personal_access_token_cleaner.rs @@ -22,7 +22,7 @@ //! proposes it once and every replica applies the commit. Backups never //! propose, so cleanup cannot race across the cluster. -use crate::bootstrap::ServerNgShard; +use crate::bootstrap::ServerShard; use consensus::MetadataHandle; use iggy_binary_protocol::WireName; use iggy_common::IggyTimestamp; @@ -54,7 +54,7 @@ enum Pass { /// Run the cleaner until `stop` fires. Wakes every `interval`; expiry is /// wall-clock driven, so no metadata-commit wake is needed. -pub async fn run_pat_cleaner(shard: Rc, stop: Receiver<()>, interval: Duration) { +pub async fn run_pat_cleaner(shard: Rc, stop: Receiver<()>, interval: Duration) { trace!( shard = shard.id, interval_ms = interval.as_millis(), @@ -81,7 +81,7 @@ pub async fn run_pat_cleaner(shard: Rc, stop: Receiver<()>, inter } /// Run one cleanup pass, deleting at most [`MAX_DELETIONS_PER_PASS`] tokens. -async fn clean_expired_tokens(shard: &Rc, stop: &Receiver<()>) -> Pass { +async fn clean_expired_tokens(shard: &Rc, stop: &Receiver<()>) -> Pass { let metadata = shard.plane.metadata(); if !metadata.is_caught_up_primary() { return Pass::Drained; diff --git a/core/server/src/quic/listener.rs b/core/server/src/quic/listener.rs deleted file mode 100644 index 6cd3d50d65..0000000000 --- a/core/server/src/quic/listener.rs +++ /dev/null @@ -1,239 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::{self, HandlerResult, MAX_CONTROL_FRAME_PAYLOAD}; -use crate::sender::SenderKind; -use crate::server_error::ConnectionError; -use crate::shard::IggyShard; -use crate::shard::task_registry::ShutdownToken; -use crate::streaming::session::Session; -use anyhow::anyhow; -use compio::io::AsyncReadExt; -use compio::quic::{Connection, Endpoint, RecvStream, SendStream}; -use futures::FutureExt; -use iggy_binary_protocol::RequestFrame; -use iggy_binary_protocol::codes::{GET_CLUSTER_METADATA_CODE, SEND_MESSAGES_CODE, command_name}; -use iggy_common::{IggyError, TransportProtocol}; -use std::rc::Rc; -use tracing::{debug, error, info, trace, warn}; - -pub async fn start( - endpoint: Endpoint, - shard: Rc, - shutdown: ShutdownToken, -) -> Result<(), IggyError> { - loop { - let accept_future = endpoint.wait_incoming(); - - futures::select! { - _ = shutdown.wait().fuse() => { - debug!( "QUIC listener received shutdown signal, no longer accepting connections"); - break; - } - incoming_conn = accept_future.fuse() => { - match incoming_conn { - Some(incoming_conn) => { - let remote_addr = incoming_conn.remote_address(); - info!("Received incoming QUIC connection from {}", remote_addr); - - if shard.is_shutting_down() { - info!( "Rejecting new QUIC connection from {} during shutdown", remote_addr); - continue; - } - - trace!("Incoming connection from client: {}", remote_addr); - let shard_for_conn = shard.clone(); - - shard.task_registry.spawn_connection(async move { - trace!("Accepting connection from {}", remote_addr); - match incoming_conn.await { - Ok(connection) => { - trace!("Connection established from {}", remote_addr); - if let Err(error) = handle_connection(connection, shard_for_conn).await { - error!("QUIC connection from {} has failed: {error}", remote_addr); - } - } - Err(error) => { - error!( - "Error when accepting incoming connection from {}: {:?}", - remote_addr, error - ); - } - } - }); - } - None => { - info!("QUIC endpoint closed for shard {}", shard.id); - break; - } - } - } - } - } - Ok(()) -} - -async fn handle_connection( - connection: Connection, - shard: Rc, -) -> Result<(), ConnectionError> { - let address = connection.remote_address(); - info!("Client has connected: {address}"); - let session = Rc::new(shard.add_client(&address, TransportProtocol::Quic)); - - let client_id = session.client_id; - debug!( - "Added {} client with session: {} for IP address: {}", - TransportProtocol::Quic, - session, - address - ); - - let conn_stop_receiver = shard.task_registry.add_connection(client_id); - - loop { - let shard = shard.clone(); - futures::select! { - // Check for shutdown signal - _ = conn_stop_receiver.recv().fuse() => { - info!("QUIC connection {} shutting down gracefully", client_id); - break; - } - // Accept new connection - stream_result = accept_stream(&connection, shard.clone(), client_id).fuse() => { - match stream_result? { - Some(stream) => { - let shard_clone = shard.clone(); - let session_rc = session.clone(); - - shard.task_registry.spawn_connection(async move { - if let Err(err) = handle_stream(stream, shard_clone, &session_rc).await { - error!("Error when handling QUIC stream: {:?}", err) - } - }); - } - None => break, // Connection closed - } - } - } - } - - shard.delete_client(client_id).await; - shard.task_registry.remove_connection(&client_id); - info!("QUIC connection {} closed", client_id); - Ok(()) -} - -type BiStream = (SendStream, RecvStream); - -async fn accept_stream( - connection: &Connection, - _shard: Rc, - _client_id: u32, -) -> Result, ConnectionError> { - match connection.accept_bi().await { - Err(compio::quic::ConnectionError::ApplicationClosed { .. }) => { - info!("Connection closed"); - Ok(None) - } - Err(error) => { - error!("Error when accepting QUIC connection: {:?}", error); - Err(error.into()) - } - Ok(stream) => Ok(Some(stream)), - } -} - -async fn handle_stream( - stream: BiStream, - shard: Rc, - session: &Session, -) -> anyhow::Result<()> { - let (send_stream, mut recv_stream) = stream; - - let header_buf = [0u8; RequestFrame::HEADER_SIZE]; - let compio::BufResult(result, header_buf) = recv_stream.read_exact(header_buf).await; - result?; - - let length = u32::from_le_bytes(header_buf[0..4].try_into().unwrap()); - let code = u32::from_le_bytes(header_buf[4..8].try_into().unwrap()); - - let cmd_name = command_name(code).unwrap_or("unknown"); - trace!("Received a QUIC request, length: {length}, code: {code} ({cmd_name})"); - - let payload_length = RequestFrame::payload_length(length) - .map_err(|_| anyhow!("Invalid frame length: {length}"))?; - - let mut sender = SenderKind::get_quic_sender(send_stream, recv_stream); - - let result = if code == SEND_MESSAGES_CODE { - dispatch::dispatch_send_messages(&mut sender, payload_length, session, &shard).await - } else { - if payload_length > MAX_CONTROL_FRAME_PAYLOAD { - sender - .send_error_response(IggyError::InvalidCommand) - .await?; - return Ok(()); - } - let payload = dispatch::read_payload(&mut sender, payload_length).await?; - let frame = RequestFrame::from_parts(code, &payload); - dispatch::dispatch(frame, &mut sender, session, &shard).await - }; - - match result { - Ok(HandlerResult::Finished) => { - trace!( - "Command was handled successfully, session: {:?}. QUIC response was sent.", - session - ); - Ok(()) - } - Ok(HandlerResult::Migrated { to_shard }) => { - warn!("Unexpected migration on QUIC: to_shard {to_shard}, session: {session:?}"); - Ok(()) - } - Err(e) => { - // Special handling for GetClusterMetadata when clustering is disabled - if code == GET_CLUSTER_METADATA_CODE && matches!(e, IggyError::FeatureUnavailable) { - debug!( - "GetClusterMetadata command not available (clustering disabled), session: {:?}.", - session - ); - sender.send_error_response(e).await?; - trace!("QUIC error response was sent."); - Ok(()) - } else { - error!( - "Command was not handled successfully, session: {:?}, error: {e}.", - session - ); - // Only return a connection-terminating error for client not found or stale - if matches!(e, IggyError::ClientNotFound(_) | IggyError::StaleClient) { - sender.send_error_response(e.clone()).await?; - trace!("QUIC error response was sent."); - error!("Session will be deleted."); - Err(anyhow!("Client invalid: {e}")) - } else { - // For all other errors, send response and continue the connection - sender.send_error_response(e).await?; - trace!("QUIC error response was sent."); - Ok(()) - } - } - } - } -} diff --git a/core/server/src/quic/mod.rs b/core/server/src/quic/mod.rs deleted file mode 100644 index cc600f01c3..0000000000 --- a/core/server/src/quic/mod.rs +++ /dev/null @@ -1,22 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -mod listener; -pub mod quic_server; -pub mod quic_socket; - -pub const COMPONENT: &str = "QUIC"; diff --git a/core/server/src/quic/quic_server.rs b/core/server/src/quic/quic_server.rs deleted file mode 100644 index 3effc7cea6..0000000000 --- a/core/server/src/quic/quic_server.rs +++ /dev/null @@ -1,222 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::configs::quic::QuicConfig; -use crate::quic::{COMPONENT, listener, quic_socket}; -use crate::server_error::QuicError; -use crate::shard::IggyShard; -use crate::shard::task_registry::ShutdownToken; -use crate::shard::transmission::event::ShardEvent; -use anyhow::Result; -use compio::quic::{ - Endpoint, EndpointConfig, IdleTimeout, ServerBuilder, ServerConfig, TransportConfig, VarInt, -}; -use err_trail::ErrContext; -use rustls::crypto::ring::default_provider; -use rustls::pki_types::{CertificateDer, PrivateKeyDer}; -use std::fs::File; -use std::io::BufReader; -use std::net::SocketAddr; -use std::rc::Rc; -use std::sync::Arc; -use tracing::info; -use tracing::{error, trace, warn}; - -/// Starts the QUIC server. -/// Returns the address the server is listening on. -pub async fn spawn_quic_server( - shard: Rc, - shutdown: ShutdownToken, -) -> Result<(), iggy_common::IggyError> { - // Ensure rustls crypto provider is installed (thread-safe, idempotent) - if rustls::crypto::CryptoProvider::get_default().is_none() { - if let Err(e) = default_provider().install_default() { - warn!( - "Failed to install rustls crypto provider: {:?}. This may be normal if another thread installed it first.", - e - ); - } else { - trace!("Rustls crypto provider installed successfully"); - } - } else { - trace!("Rustls crypto provider already installed"); - } - - let config = shard.config.quic.clone(); - let mut addr: SocketAddr = config.address.parse().map_err(|e| { - error!("Failed to parse QUIC address '{}': {}", config.address, e); - iggy_common::IggyError::QuicError - })?; - - if shard.id != 0 && addr.port() == 0 { - info!("Waiting for QUIC address from shard 0..."); - loop { - if let Some(bound_addr) = shard.quic_bound_address.get() { - addr = bound_addr; - info!("Received QUIC address: {}", addr); - break; - } - compio::time::sleep(std::time::Duration::from_millis(50)).await; - } - } - - info!( - "Initializing Iggy QUIC server on shard {} for address {}", - shard.id, addr - ); - - let server_config = configure_quic(&config).map_err(|e| { - error!("Failed to configure QUIC server: {:?}", e); - iggy_common::IggyError::QuicError - })?; - trace!("Building UDP socket for QUIC endpoint on {}", addr); - - let socket = quic_socket::build(&addr, &config.socket); - socket.bind(&addr.into()).map_err(|e| { - error!("Failed to bind socket: {}", e); - iggy_common::IggyError::CannotBindToSocket(addr.to_string()) - })?; - socket.set_nonblocking(true).map_err(|e| { - error!("Failed to set nonblocking: {}", e); - iggy_common::IggyError::QuicError - })?; - - let std_socket: std::net::UdpSocket = socket.into(); - let socket = compio::net::UdpSocket::from_std(std_socket).map_err(|e| { - error!("Failed to convert std socket to compio socket: {:?}", e); - iggy_common::IggyError::QuicError - })?; - trace!("Creating QUIC endpoint with server config"); - - let endpoint = Endpoint::new(socket, EndpointConfig::default(), Some(server_config), None) - .map_err(|e| { - error!("Failed to create QUIC endpoint: {:?}", e); - iggy_common::IggyError::QuicError - })?; - - let actual_addr = endpoint.local_addr().map_err(|e| { - error!("Failed to get local address: {e}"); - iggy_common::IggyError::CannotBindToSocket(addr.to_string()) - })?; - - info!("Iggy QUIC server has started on: {:?}", actual_addr); - - if shard.id == 0 { - // Store bound address locally - shard.quic_bound_address.set(Some(actual_addr)); - - if addr.port() == 0 { - // Notify config writer on shard 0 - let _ = shard.config_writer_notify.try_send(()); - - // Broadcast to other shards for SO_REUSEPORT binding - let event = ShardEvent::AddressBound { - protocol: iggy_common::TransportProtocol::Quic, - address: actual_addr, - }; - shard.broadcast_event_to_all_shards(event).await?; - } - } else { - shard.quic_bound_address.set(Some(actual_addr)); - } - - listener::start(endpoint, shard, shutdown).await -} - -fn configure_quic(config: &QuicConfig) -> Result { - let (certificates, private_key) = match config.certificate.self_signed { - true => generate_self_signed_cert()?, - false => load_certificates(&config.certificate.cert_file, &config.certificate.key_file)?, - }; - - let builder = ServerBuilder::new_with_single_cert(certificates, private_key) - .error(|e: &rustls::Error| { - format!("{COMPONENT} (error: {e}) - failed to create QUIC server builder") - }) - .map_err(|_| QuicError::ConfigCreationError)?; - let mut transport = TransportConfig::default(); - transport.initial_mtu(config.initial_mtu.as_bytes_u64() as u16); - transport.send_window(config.send_window.as_bytes_u64()); - transport.receive_window( - VarInt::try_from(config.receive_window.as_bytes_u64()).map_err(|e| { - error!("{COMPONENT} (error: {e}) - invalid receive window"); - QuicError::TransportConfigError - })?, - ); - transport.datagram_send_buffer_size(config.datagram_send_buffer_size.as_bytes_u64() as usize); - transport.max_concurrent_bidi_streams( - VarInt::try_from(config.max_concurrent_bidi_streams).map_err(|e| { - error!("{COMPONENT} (error: {e}) - invalid bidi stream limit"); - QuicError::TransportConfigError - })?, - ); - - if !config.keep_alive_interval.is_zero() { - transport.keep_alive_interval(Some(config.keep_alive_interval.get_duration())); - } - if !config.max_idle_timeout.is_zero() { - let max_idle_timeout = IdleTimeout::try_from(config.max_idle_timeout.get_duration()) - .map_err(|e| { - error!("{COMPONENT} (error: {e}) - invalid idle timeout"); - QuicError::TransportConfigError - })?; - transport.max_idle_timeout(Some(max_idle_timeout)); - } - - let mut server_config = builder.build(); - server_config.transport_config(Arc::new(transport)); - Ok(server_config) -} - -fn generate_self_signed_cert<'a>() -> Result<(Vec>, PrivateKeyDer<'a>), QuicError> -{ - server_common::generate_self_signed_certificate("localhost").map_err(|e| { - error!("{COMPONENT} (error: {e}) - failed to generate self-signed certificate"); - QuicError::CertGenerationError - }) -} - -fn load_certificates( - cert_file: &str, - key_file: &str, -) -> Result<(Vec>, PrivateKeyDer<'static>), QuicError> { - let mut cert_chain_reader = BufReader::new( - File::open(cert_file) - .error(|e: &std::io::Error| { - format!("{COMPONENT} (error: {e}) - failed to open cert file: {cert_file}") - }) - .map_err(|_| QuicError::CertLoadError)?, - ); - let certs = rustls_pemfile::certs(&mut cert_chain_reader) - .map(|x| CertificateDer::from(x.unwrap().to_vec())) - .collect(); - let mut key_reader = BufReader::new( - File::open(key_file) - .error(|e: &std::io::Error| { - format!("{COMPONENT} (error: {e}) - failed to open key file: {key_file}") - }) - .map_err(|_| QuicError::CertLoadError)?, - ); - let mut keys = rustls_pemfile::rsa_private_keys(&mut key_reader) - .filter(|key| key.is_ok()) - .map(|key| PrivateKeyDer::try_from(key.unwrap().secret_pkcs1_der().to_vec())) - .collect::, _>>() - .error(|e: &&str| format!("{COMPONENT} (error: {e}) - failed to parse private key")) - .map_err(|_| QuicError::CertLoadError)?; - let key = keys.remove(0); - Ok((certs, key)) -} diff --git a/core/server/src/quic/quic_socket.rs b/core/server/src/quic/quic_socket.rs deleted file mode 100644 index 91acc54a1f..0000000000 --- a/core/server/src/quic/quic_socket.rs +++ /dev/null @@ -1,66 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use socket2::{Domain, Protocol, Socket, Type}; -use std::net::SocketAddr; -use std::num::TryFromIntError; - -use crate::configs::quic::QuicSocketConfig; - -/// Build a UDP socket for the given address and configure the options that are -/// required by the server -pub fn build(addr: &SocketAddr, config: &QuicSocketConfig) -> Socket { - // Choose the correct address family based on the target address - let socket = Socket::new(Domain::for_address(*addr), Type::DGRAM, Some(Protocol::UDP)) - .expect("Unable to create a UDP socket"); - - // Allow multiple sockets (shards) to bind to the same address - socket - .set_reuse_address(true) - .expect("Unable to set SO_REUSEADDR on socket"); - - // SO_REUSEPORT is only available on Unix-like systems - #[cfg(unix)] - socket - .set_reuse_port(true) - .expect("Unable to set SO_REUSEPORT on socket"); - - // Configure socket buffer sizes and keepalive if override is enabled - if config.override_defaults { - config - .recv_buffer_size - .as_bytes_u64() - .try_into() - .map_err(|e: TryFromIntError| std::io::Error::other(e.to_string())) - .and_then(|size| socket.set_recv_buffer_size(size)) - .expect("Unable to set SO_RCVBUF on socket"); - - config - .send_buffer_size - .as_bytes_u64() - .try_into() - .map_err(|e: TryFromIntError| std::io::Error::other(e.to_string())) - .and_then(|size| socket.set_send_buffer_size(size)) - .expect("Unable to set SO_SNDBUF on socket"); - - socket - .set_keepalive(config.keepalive) - .expect("Unable to set SO_KEEPALIVE on socket"); - } - - socket -} diff --git a/core/server-ng/src/responses.rs b/core/server/src/responses.rs similarity index 97% rename from core/server-ng/src/responses.rs rename to core/server/src/responses.rs index 8d79bf648c..7adb4a46da 100644 --- a/core/server-ng/src/responses.rs +++ b/core/server/src/responses.rs @@ -76,7 +76,7 @@ use iggy_binary_protocol::responses::users::get_users::GetUsersResponse; use iggy_binary_protocol::responses::users::user_response::UserResponse; use iggy_binary_protocol::{ Command2, GenericHeader, IGGY_PROTOCOL_VERSION, KIND_CONSUMER_GROUP, Operation, ReplyHeader, - RequestHeader, WireDecode, WireEncode, WireIdentifier, WireName, WirePartitioning, + RoutedRequestHeader, WireDecode, WireEncode, WireIdentifier, WireName, WirePartitioning, }; use iggy_common::{EncryptorKind, Identifier, IggyError, IggyTimestamp}; use journal::superblock::SuperblockStore; @@ -156,7 +156,7 @@ where // No session record (shouldn't happen on an auth-gated // read). Report the connection id with the "no user" // sentinel + TCP default rather than impersonating root - // (user id 0 is a real user; server-ng is 0-based). + // (user id 0 is a real user; server is 0-based). #[allow(clippy::cast_possible_truncation)] ClientResponse { client_id: transport_client_id as u32, @@ -466,18 +466,29 @@ where if let Some(namespace) = streams.namespace_from_partition(stream_id, topic_id, partition_id) { return Ok(namespace); } - // Tell a bad partition id apart from a bad stream/topic: the former is a - // typed not-found the client can act on, the latter keeps the generic - // rejection every caller already handles. + // Name the level that missed - partition, topic, or stream - with the + // legacy typed not-found, so a client can tell an addressing typo from an + // empty partition. Callers that shape their own reply (empty poll, group + // gather) treat every variant the same, so the split is reply-visible only + // where a caller denies typed. if streams.topic_partition_ids(stream_id, topic_id).is_some() { - Err(IggyError::PartitionNotFound( + return Err(IggyError::PartitionNotFound( partition_id as usize, wire_identifier_for_display(topic_id), wire_identifier_for_display(stream_id), - )) - } else { - Err(IggyError::InvalidIdentifier) + )); } + Err(streams.read(|inner| { + let Some(resolved_stream) = resolve_stream_id(inner, stream_id) else { + return stream_not_found(stream_id); + }; + if resolve_topic_id(inner, resolved_stream, topic_id).is_none() { + return topic_not_found(stream_id, topic_id); + } + // Unreachable while `topic_partition_ids` misses only on stream/topic; + // kept as the safe generic rejection should that invariant drift. + IggyError::InvalidIdentifier + })) } /// Best-effort conversion for error payloads only: the wire reply carries just @@ -496,7 +507,10 @@ fn wire_identifier_for_display(id: &WireIdentifier) -> Identifier { /// stays with the per-transport gates that run before this builder. `client_ip` /// is the caller's transport-level peer address, used only by the /// cluster-metadata read to pick each node's advertised address; `None` -/// degrades to the catch-all address. +/// degrades to the catch-all address. `clients_count` is the cross-shard +/// connected-client total, used only by the stats read: it comes from the async +/// `ListClients` scatter-gather, which this sync builder cannot run, so both +/// transport callers gather it up front (0 for every other opcode). pub(crate) fn build_non_replicated_response( shard: &Rc>, code: u32, @@ -504,6 +518,7 @@ pub(crate) fn build_non_replicated_response( user_id: Option, roster: &ClusterRoster, client_ip: Option, + clients_count: u32, ) -> Result where B: ShellBus, @@ -517,7 +532,7 @@ where build_cluster_metadata_response(roster, shard, client_ip).to_bytes(), )), GET_STATS_CODE => Ok(NonReplicatedResponse::Bytes( - build_stats_response(shard)?.to_bytes(), + build_stats_response(shard, clients_count)?.to_bytes(), )), GET_STREAM_CODE => { let request = @@ -607,7 +622,7 @@ where NonReplicatedResponse::Bytes(GetConsumerGroupsResponse { groups }.to_bytes()) })) } - // server-ng has no on-demand flush primitive, so it denies honestly. + // The server has no on-demand flush primitive, so it denies honestly. // The non-replicated catch-all's empty-ok would otherwise attest a // durability guarantee the server never gave. FLUSH_UNSAVED_BUFFER_CODE => Err(IggyError::FeatureUnavailable), @@ -680,6 +695,7 @@ where fn build_stats_response( shard: &Rc>, + clients_count: u32, ) -> Result where B: ShellBus, @@ -766,12 +782,7 @@ where partitions_count, segments_count, messages_count, - // Connected clients are per-shard `SessionManager` state, aggregated - // across shards only by the async `ListClients` broadcast (see - // `get_clients`). This sync single-shard read can't gather it, and one - // shard's local count is a fraction of the total, so report 0 rather - // than a misleading partial. - clients_count: 0, + clients_count, consumer_groups_count, hostname: system.hostname, os_name: system.os_name, @@ -1242,7 +1253,7 @@ pub(crate) enum NonReplicatedResponse { impl NonReplicatedResponse { pub(crate) fn into_reply( self, - request_header: &RequestHeader, + request_header: &RoutedRequestHeader, client_id: u128, session: u64, commit: u64, @@ -1257,7 +1268,7 @@ impl NonReplicatedResponse { } pub(crate) fn build_empty_reply( - request_header: &RequestHeader, + request_header: &RoutedRequestHeader, client_id: u128, session: u64, commit: u64, @@ -1273,7 +1284,7 @@ pub(crate) fn build_empty_reply( /// reply, and only the partition primary's pre-pipeline deny pins it to 0, /// stamped through `consensus::build_deny_reply_from_request`. pub(crate) fn build_deny_reply( - request_header: &RequestHeader, + request_header: &RoutedRequestHeader, client_id: u128, session: u64, commit: u64, @@ -1293,7 +1304,7 @@ pub(crate) fn build_deny_reply( const SERVER_VERSION: &str = env!("CARGO_PKG_VERSION"); pub(crate) fn build_login_register_reply( - request_header: &RequestHeader, + request_header: &RoutedRequestHeader, client_id: u128, session: u64, commit: u64, @@ -1324,7 +1335,7 @@ pub(crate) fn build_login_register_reply( } pub(crate) fn build_reply_from_bytes( - request_header: &RequestHeader, + request_header: &RoutedRequestHeader, client_id: u128, session: u64, commit: u64, @@ -1347,7 +1358,7 @@ pub(crate) fn build_reply_from_bytes( /// (no token, a committed business rejection, or an eviction frame) the /// committed reply passes through unchanged. pub(crate) fn build_raw_pat_reply( - request_header: &RequestHeader, + request_header: &RoutedRequestHeader, committed: Message, raw_token: Option, ) -> Result, IggyError> { @@ -1408,7 +1419,7 @@ pub(crate) fn build_raw_pat_reply( } pub(crate) fn build_reply_with_body( - request_header: &RequestHeader, + request_header: &RoutedRequestHeader, client_id: u128, session: u64, commit: u64, @@ -1437,7 +1448,6 @@ pub(crate) fn build_reply_with_body( timestamp: request_header.timestamp, request: request_header.request, operation: request_header.operation, - namespace: request_header.namespace, ..Default::default() }; write_body(&mut reply.as_mut_slice()[header_len..total_size]); @@ -1629,10 +1639,10 @@ pub(crate) fn build_consumer_offset_body( mod tests { use super::*; - fn pat_request_header() -> RequestHeader { - let zeroed = [0u8; std::mem::size_of::()]; - let mut header = *bytemuck::checked::try_from_bytes::(&zeroed) - .expect("zeroed bytes form a valid RequestHeader"); + fn pat_request_header() -> RoutedRequestHeader { + let zeroed = [0u8; std::mem::size_of::()]; + let mut header = *bytemuck::checked::try_from_bytes::(&zeroed) + .expect("zeroed bytes form a valid RoutedRequestHeader"); header.command = Command2::Request; header.operation = Operation::CreatePersonalAccessToken; header.client = 42; diff --git a/core/server-ng/src/segment_cleaner.rs b/core/server/src/segment_cleaner.rs similarity index 97% rename from core/server-ng/src/segment_cleaner.rs rename to core/server/src/segment_cleaner.rs index bffd016168..49bc338ff4 100644 --- a/core/server-ng/src/segment_cleaner.rs +++ b/core/server/src/segment_cleaner.rs @@ -26,7 +26,7 @@ //! with reads. This mirrors the legacy server's `MessagesCleaner` -> //! message-pump `CleanTopicMessages` path. -use crate::bootstrap::ServerNgShard; +use crate::bootstrap::ServerShard; use consensus::{MetadataHandle, PartitionsHandle}; use iggy_common::{IggyExpiry, IggyTimestamp, MaxTopicSize}; use metadata::impls::metadata::StreamsFrontend; @@ -38,7 +38,7 @@ use tracing::trace; /// Run the cleaner until `stop` fires. Wakes every `interval`; expiry and size /// are evaluated against wall-clock and resident bytes, so no metadata-commit /// wake is needed. -pub async fn run_segment_cleaner(shard: Rc, stop: Receiver<()>, interval: Duration) { +pub async fn run_segment_cleaner(shard: Rc, stop: Receiver<()>, interval: Duration) { trace!( shard = shard.id, interval_ms = interval.as_millis(), @@ -57,7 +57,7 @@ pub async fn run_segment_cleaner(shard: Rc, stop: Receiver<()>, i /// Stage a cleaner pass for every partition this shard owns whose topic has a /// retention policy. Reads config off-pump and hands the resolved decision to /// the pump; partitions with no policy are skipped without a frame. -fn stage_owned_partitions(shard: &Rc) { +fn stage_owned_partitions(shard: &Rc) { let now = IggyTimestamp::now(); let namespaces: Vec<_> = shard.plane.partitions().namespaces().copied().collect(); let streams = shard.plane.metadata().mux_stm.streams(); diff --git a/core/server-ng/src/segment_recovery.rs b/core/server/src/segment_recovery.rs similarity index 96% rename from core/server-ng/src/segment_recovery.rs rename to core/server/src/segment_recovery.rs index ef9ba54839..ecdeaa1709 100644 --- a/core/server-ng/src/segment_recovery.rs +++ b/core/server/src/segment_recovery.rs @@ -15,20 +15,20 @@ // specific language governing permissions and limitations // under the License. -//! server-ng-owned segment recovery. +//! Server-owned segment recovery. //! -//! Previously the bootstrap path borrowed `server::bootstrap::load_segments` -//! from the legacy `server` crate to hydrate persisted segments. That loader +//! Previously the bootstrap path borrowed `load_segments` from the legacy +//! server implementation to hydrate persisted segments. That loader //! reads the legacy 16-byte dense per-message index through -//! `server_common::IndexReader`, but server-ng persists a 24-byte sparse index +//! `server_common::IndexReader`, but the server persists a 24-byte sparse index //! (`partitions::IggyIndexWriter`: one entry per flush, absolute `offset`, //! `timestamp`, and batch-start `position`). Reading the 24-byte file with the //! 16-byte parser mis-strides it (the "Index data must be exactly 16 bytes" -//! recovery panic). This module is the server-ng-owned loader, reading the same +//! recovery panic). This module is the server-owned loader, reading the same //! 24-byte format its writer emits. -use crate::server_error::{PartitionChainRefusal, ServerNgError}; -use configs::server_ng::ServerNgConfig; +use crate::server_error::{PartitionChainRefusal, ServerError}; +use configs::server::ServerConfig; use iggy_common::{IggyByteSize, IggyError, PartitionStats}; use partitions::state_transfer::STAGING_SUFFIX; use partitions::{IggyIndexReader, Segment}; @@ -61,12 +61,12 @@ pub struct RecoveredSegment { /// read, or if a segment's index references a batch beyond the end of its /// messages file (torn write). pub async fn load_persisted_segments( - config: &ServerNgConfig, + config: &ServerConfig, stream_id: usize, topic_id: usize, partition_id: usize, stats: &PartitionStats, -) -> Result, ServerNgError> { +) -> Result, ServerError> { let partition_path = config .system .get_partition_path(stream_id, topic_id, partition_id); @@ -207,9 +207,9 @@ fn ensure_contiguous_chain( stream_id: usize, topic_id: usize, partition_id: usize, -) -> Result<(), ServerNgError> { +) -> Result<(), ServerError> { let refused = |reason| { - Err(ServerNgError::PartitionChainRefused { + Err(ServerError::PartitionChainRefused { dir: PathBuf::from(partition_path), stream_id, topic_id, @@ -264,9 +264,7 @@ fn ensure_contiguous_chain( /// Sweeps boot-time scratch (`.staging` spill, orphan `.index`) and returns the /// start offset parsed out of every remaining zero-padded `.log` file name. A /// missing directory means a never-persisted partition. -fn sweep_scratch_files_and_collect_offsets( - partition_path: &str, -) -> Result, ServerNgError> { +fn sweep_scratch_files_and_collect_offsets(partition_path: &str) -> Result, ServerError> { let entries = match fs::read_dir(partition_path) { Ok(entries) => entries, Err(source) if source.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), @@ -345,7 +343,7 @@ async fn recover_segment_bounds( stream_id: usize, topic_id: usize, partition_id: usize, -) -> Result, ServerNgError> { +) -> Result, ServerError> { let reader = IggyIndexReader::new(index_path).await.map_err(|source| { error!( stream_id, @@ -420,7 +418,7 @@ async fn recover_segment_bounds( position = extent; } if !walked_any { - return Err(ServerNgError::RecoveredSegmentSizeDivergence { + return Err(ServerError::RecoveredSegmentSizeDivergence { stream_id, topic_id, partition_id, diff --git a/core/server/src/sender/mod.rs b/core/server/src/sender/mod.rs deleted file mode 100644 index c6a0b4acaa..0000000000 --- a/core/server/src/sender/mod.rs +++ /dev/null @@ -1,257 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -mod quic_sender; -mod tcp_sender; -mod tcp_tls_sender; -mod websocket_sender; -mod websocket_tls_sender; - -pub use quic_sender::QuicSender; -pub use tcp_sender::TcpSender; -pub use tcp_tls_sender::TcpTlsSender; -pub use websocket_sender::WebSocketSender; -pub use websocket_tls_sender::WebSocketTlsSender; - -use compio::BufResult; -use compio::buf::IoBufMut; -use compio::io::{AsyncReadExt, AsyncWriteExt}; -use compio::net::TcpStream; -use compio::quic::{RecvStream, SendStream}; -use compio::tls::TlsStream; -use iggy_common::IggyError; -use server_common::PooledBuffer; -use std::future::Future; -#[cfg(unix)] -use std::os::fd::{AsFd, OwnedFd}; -use tracing::debug; -#[cfg(unix)] -use tracing::error; - -macro_rules! forward_async_methods { - ( - $( - async fn $method_name:ident - $(<$($generic:ident $(: $bound:path)?),+>)? - ( - &mut self $(, $arg:ident : $arg_ty:ty )* - ) -> $ret:ty ; - )* - ) => { - $( - pub async fn $method_name - $(<$($generic $(: $bound)?),+>)? - (&mut self, $( $arg: $arg_ty ),* ) -> $ret { - match self { - Self::Tcp(d) => d.$method_name$(::<$($generic),+>)?($( $arg ),*).await, - Self::TcpTls(s) => s.$method_name$(::<$($generic),+>)?($( $arg ),*).await, - Self::Quic(s) => s.$method_name$(::<$($generic),+>)?($( $arg ),*).await, - Self::WebSocket(s) => s.$method_name$(::<$($generic),+>)?($( $arg ),*).await, - Self::WebSocketTls(s) => s.$method_name$(::<$($generic),+>)?($( $arg ),*).await, - } - } - )* - } -} - -pub trait Sender { - fn read(&mut self, buffer: B) -> impl Future, B)>; - fn send_empty_ok_response(&mut self) -> impl Future>; - fn send_ok_response(&mut self, payload: &[u8]) -> impl Future>; - fn send_ok_response_vectored( - &mut self, - length: &[u8], - slices: Vec, - ) -> impl Future>; - fn send_error_response( - &mut self, - error: IggyError, - ) -> impl Future>; - fn shutdown(&mut self) -> impl Future>; -} - -#[allow(clippy::large_enum_variant)] -#[derive(Debug)] -pub enum SenderKind { - Tcp(TcpSender), - TcpTls(TcpTlsSender), - Quic(QuicSender), - WebSocket(WebSocketSender), - WebSocketTls(WebSocketTlsSender), -} - -impl SenderKind { - pub fn get_tcp_sender(stream: TcpStream) -> Self { - Self::Tcp(TcpSender { - stream: Some(stream), - }) - } - - pub fn get_tcp_tls_sender(stream: TlsStream) -> Self { - Self::TcpTls(TcpTlsSender { stream }) - } - - pub fn get_quic_sender(send_stream: SendStream, recv_stream: RecvStream) -> Self { - Self::Quic(QuicSender { - send: send_stream, - recv: recv_stream, - }) - } - - pub fn get_websocket_sender(stream: WebSocketSender) -> Self { - Self::WebSocket(stream) - } - - pub fn get_websocket_tls_sender(stream: WebSocketTlsSender) -> Self { - Self::WebSocketTls(stream) - } - - #[cfg(unix)] - pub fn take_and_migrate_tcp(&mut self) -> Option { - match self { - SenderKind::Tcp(tcp_sender) => { - let stream = tcp_sender.stream.take()?; - let poll_fd = stream.into_poll_fd().ok()?; - - let raw_fd = poll_fd.as_fd(); - let Ok(owned_fd) = nix::unistd::dup(raw_fd) else { - // TODO(tungtose): recover tcp stream? - error!("Failed to dup fd"); - return None; - }; - - Some(owned_fd) - } - // TODO(tungtose): support TCP TLS - _ => None, - } - } - - forward_async_methods! { - async fn read(&mut self, buffer: B) -> (Result<(), IggyError>, B); - async fn send_empty_ok_response(&mut self) -> Result<(), IggyError>; - async fn send_ok_response(&mut self, payload: &[u8]) -> Result<(), IggyError>; - async fn send_ok_response_vectored(&mut self, length: &[u8], slices: Vec) -> Result<(), IggyError>; - async fn send_error_response(&mut self, error: IggyError) -> Result<(), IggyError>; - async fn shutdown(&mut self) -> Result<(), IggyError>; - } -} - -const STATUS_OK: &[u8] = &[0; 4]; - -pub(crate) async fn read(stream: &mut T, buffer: B) -> (Result<(), IggyError>, B) -where - T: AsyncReadExt + AsyncWriteExt + Unpin, - B: IoBufMut, -{ - let BufResult(result, buffer) = stream.read_exact(buffer).await; - match (result, buffer) { - (Ok(_), buffer) => (Ok(()), buffer), - (Err(e), buffer) => { - if e.kind() == std::io::ErrorKind::UnexpectedEof { - (Err(IggyError::ConnectionClosed), buffer) - } else { - (Err(IggyError::TcpError), buffer) - } - } - } -} - -pub(crate) async fn send_empty_ok_response(stream: &mut T) -> Result<(), IggyError> -where - T: AsyncReadExt + AsyncWriteExt + Unpin, -{ - send_ok_response(stream, &[]).await -} - -pub(crate) async fn send_ok_response(stream: &mut T, payload: &[u8]) -> Result<(), IggyError> -where - T: AsyncReadExt + AsyncWriteExt + Unpin, -{ - send_response(stream, STATUS_OK, payload).await -} - -pub(crate) async fn send_ok_response_vectored( - stream: &mut T, - length: &[u8], - slices: Vec, -) -> Result<(), IggyError> -where - T: AsyncReadExt + AsyncWriteExt + Unpin, -{ - send_response_vectored(stream, STATUS_OK, length, slices).await -} - -pub(crate) async fn send_error_response( - stream: &mut T, - error: IggyError, -) -> Result<(), IggyError> -where - T: AsyncReadExt + AsyncWriteExt + Unpin, -{ - send_response(stream, &error.as_code().to_le_bytes(), &[]).await -} - -pub(crate) async fn send_response( - stream: &mut T, - status: &[u8], - payload: &[u8], -) -> Result<(), IggyError> -where - T: AsyncReadExt + AsyncWriteExt + Unpin, -{ - debug!( - "Sending response of len: {} with status: {:?}...", - payload.len(), - status - ); - let length = (payload.len() as u32).to_le_bytes(); - stream - .write_all([status, &length, payload].concat()) - .await - .0 - .map_err(|_| IggyError::TcpError)?; - debug!("Sent response with status: {:?}", status); - Ok(()) -} - -pub(crate) async fn send_response_vectored( - stream: &mut T, - status: &[u8], - length: &[u8], - mut slices: Vec, -) -> Result<(), IggyError> -where - T: AsyncReadExt + AsyncWriteExt + Unpin, -{ - let resp_status = u32::from_le_bytes(status.try_into().unwrap()); - debug!( - "Sending vectored response of len: {} with status: {:?}...", - slices.len(), - resp_status - ); - let status = PooledBuffer::from(status); - let length = PooledBuffer::from(length); - slices.splice(0..0, [status, length]); - stream - .write_vectored_all(slices) - .await - .0 - .map_err(|_| IggyError::TcpError)?; - debug!("Sent response with status: {:?}", resp_status); - Ok(()) -} diff --git a/core/server/src/sender/quic_sender.rs b/core/server/src/sender/quic_sender.rs deleted file mode 100644 index 2759e3b34d..0000000000 --- a/core/server/src/sender/quic_sender.rs +++ /dev/null @@ -1,140 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use super::Sender; -use compio::BufResult; -use compio::buf::IoBufMut; -use compio::io::{AsyncReadExt, AsyncWriteExt}; -use compio::quic::{ClosedStream, RecvStream, SendStream}; -use err_trail::ErrContext; -use iggy_common::IggyError; -use server_common::PooledBuffer; -use tracing::{debug, error}; - -const COMPONENT: &str = "QUIC"; -const STATUS_OK: &[u8] = &[0; 4]; - -#[derive(Debug)] -pub struct QuicSender { - pub(crate) send: SendStream, - pub(crate) recv: RecvStream, -} - -impl Sender for QuicSender { - /// Reads data from the QUIC stream directly into the buffer. - async fn read(&mut self, buffer: B) -> (Result<(), IggyError>, B) { - let BufResult(result, buffer) = - ::read_exact(&mut self.recv, buffer).await; - match (result, buffer) { - (Ok(_), buffer) => (Ok(()), buffer), - (Err(error), buffer) => { - error!("Failed to read from the stream: {:?}", error); - (Err(IggyError::QuicError), buffer) - } - } - } - - async fn send_empty_ok_response(&mut self) -> Result<(), IggyError> { - self.send_ok_response(&[]).await - } - - async fn send_ok_response(&mut self, payload: &[u8]) -> Result<(), IggyError> { - self.send_response(STATUS_OK, payload).await - } - - async fn send_error_response(&mut self, error: IggyError) -> Result<(), IggyError> { - self.send_response(&error.as_code().to_le_bytes(), &[]) - .await - } - - async fn shutdown(&mut self) -> Result<(), IggyError> { - Ok(()) - } - - async fn send_ok_response_vectored( - &mut self, - length: &[u8], - slices: Vec, - ) -> Result<(), IggyError> { - debug!("Sending vectored response with status: {:?}...", STATUS_OK); - - let headers = [STATUS_OK, length].concat(); - let BufResult(result, _) = self.send.write_all(headers).await; - result - .error(|e: &std::io::Error| { - format!("{COMPONENT} (error: {e}) - failed to write headers to stream") - }) - .map_err(|_| IggyError::QuicError)?; - - let mut total_bytes_written = 0; - - for slice in slices { - let slice_len = slice.len(); - if slice_len > 0 { - let BufResult(result, _) = self.send.write_all(slice).await; - result - .error(|e: &std::io::Error| { - format!("{COMPONENT} (error: {e}) - failed to write slice to stream") - }) - .map_err(|_| IggyError::QuicError)?; - - total_bytes_written += slice_len; - } - } - - debug!( - "Sent vectored response: {} bytes of payload", - total_bytes_written - ); - - self.send - .finish() - .error(|e: &ClosedStream| { - format!("{COMPONENT} (error: {e}) - failed to finish send stream") - }) - .map_err(|_| IggyError::QuicError)?; - - debug!("Sent vectored response with status: {:?}", STATUS_OK); - Ok(()) - } -} - -impl QuicSender { - async fn send_response(&mut self, status: &[u8], payload: &[u8]) -> Result<(), IggyError> { - debug!( - "Sending response of len: {} with status: {:?}...", - payload.len(), - status - ); - let length = (payload.len() as u32).to_le_bytes(); - let data = [status, &length, payload].concat(); - let BufResult(result, _) = self.send.write_all(data).await; - result - .error(|e: &std::io::Error| { - format!("{COMPONENT} (error: {e}) - failed to write buffer to the stream") - }) - .map_err(|_| IggyError::QuicError)?; - self.send - .finish() - .error(|e: &ClosedStream| { - format!("{COMPONENT} (error: {e}) - failed to finish send stream") - }) - .map_err(|_| IggyError::QuicError)?; - debug!("Sent response with status: {:?}", status); - Ok(()) - } -} diff --git a/core/server/src/sender/tcp_sender.rs b/core/server/src/sender/tcp_sender.rs deleted file mode 100644 index 15dba86da3..0000000000 --- a/core/server/src/sender/tcp_sender.rs +++ /dev/null @@ -1,90 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use super::Sender; -use compio::buf::IoBufMut; -use compio::io::AsyncWrite; -use compio::net::TcpStream; -use err_trail::ErrContext; -use iggy_common::IggyError; -use server_common::PooledBuffer; - -const COMPONENT: &str = "TCP"; - -#[derive(Debug)] -pub struct TcpSender { - pub(crate) stream: Option, -} - -impl Sender for TcpSender { - async fn read(&mut self, buffer: B) -> (Result<(), IggyError>, B) { - match self.stream.as_mut() { - Some(stream) => super::read(stream, buffer).await, - None => (Err(IggyError::ConnectionClosed), buffer), - } - } - - async fn send_empty_ok_response(&mut self) -> Result<(), IggyError> { - match self.stream.as_mut() { - Some(stream) => super::send_empty_ok_response(stream).await, - None => Err(IggyError::ConnectionClosed), - } - } - - async fn send_ok_response(&mut self, payload: &[u8]) -> Result<(), IggyError> { - match self.stream.as_mut() { - Some(stream) => super::send_ok_response(stream, payload).await, - None => Err(IggyError::ConnectionClosed), - } - } - - async fn send_error_response(&mut self, error: IggyError) -> Result<(), IggyError> { - match self.stream.as_mut() { - Some(stream) => super::send_error_response(stream, error).await, - None => Err(IggyError::ConnectionClosed), - } - } - - async fn shutdown(&mut self) -> Result<(), IggyError> { - match self.stream.as_mut() { - Some(stream) => stream - .shutdown() - .await - .error(|e: &std::io::Error| { - format!("{COMPONENT} (error: {e}) - failed to shutdown TCP stream") - }) - .map_err(|e| IggyError::IoError(e.to_string())), - - None => Err(IggyError::ConnectionClosed), - } - } - - async fn send_ok_response_vectored( - &mut self, - length: &[u8], - slices: Vec, - ) -> Result<(), IggyError> { - if self.stream.is_none() { - tracing::error!("Tried to send but stream is None!"); - } - match self.stream.as_mut() { - Some(stream) => super::send_ok_response_vectored(stream, length, slices).await, - - None => Err(IggyError::ConnectionClosed), - } - } -} diff --git a/core/server/src/sender/tcp_tls_sender.rs b/core/server/src/sender/tcp_tls_sender.rs deleted file mode 100644 index 346aa46d2a..0000000000 --- a/core/server/src/sender/tcp_tls_sender.rs +++ /dev/null @@ -1,96 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use super::Sender; -use compio::buf::IoBufMut; -use compio::io::AsyncWrite; -use compio::net::TcpStream; -use compio::tls::TlsStream; -use err_trail::ErrContext; -use iggy_common::IggyError; -use server_common::PooledBuffer; - -const COMPONENT: &str = "TCP"; - -#[derive(Debug)] -pub struct TcpTlsSender { - pub(crate) stream: TlsStream, -} - -impl Sender for TcpTlsSender { - async fn read(&mut self, buffer: B) -> (Result<(), IggyError>, B) { - super::read(&mut self.stream, buffer).await - } - - async fn send_empty_ok_response(&mut self) -> Result<(), IggyError> { - super::send_empty_ok_response(&mut self.stream).await?; - self.stream - .flush() - .await - .error(|e: &std::io::Error| { - format!("failed to flush TCP stream after sending response: {e}") - }) - .map_err(|_| IggyError::TcpError) - } - - async fn send_ok_response(&mut self, payload: &[u8]) -> Result<(), IggyError> { - super::send_ok_response(&mut self.stream, payload).await?; - self.stream - .flush() - .await - .error(|e: &std::io::Error| { - format!("failed to flush TCP stream after sending response: {e}") - }) - .map_err(|_| IggyError::TcpError) - } - - async fn send_error_response(&mut self, error: IggyError) -> Result<(), IggyError> { - super::send_error_response(&mut self.stream, error).await?; - self.stream - .flush() - .await - .error(|e: &std::io::Error| { - format!("failed to flush TCP stream after sending response: {e}") - }) - .map_err(|_| IggyError::TcpError) - } - - async fn shutdown(&mut self) -> Result<(), IggyError> { - self.stream - .shutdown() - .await - .error(|e: &std::io::Error| { - format!("{COMPONENT} (error: {e}) - failed to shutdown TCP TLS stream") - }) - .map_err(|e| IggyError::IoError(e.to_string())) - } - - async fn send_ok_response_vectored( - &mut self, - length: &[u8], - slices: Vec, - ) -> Result<(), IggyError> { - super::send_ok_response_vectored(&mut self.stream, length, slices).await?; - self.stream - .flush() - .await - .error(|e: &std::io::Error| { - format!("failed to flush TCP stream after sending response: {e}") - }) - .map_err(|_| IggyError::TcpError) - } -} diff --git a/core/server/src/sender/websocket_sender.rs b/core/server/src/sender/websocket_sender.rs deleted file mode 100644 index 9e92a72f05..0000000000 --- a/core/server/src/sender/websocket_sender.rs +++ /dev/null @@ -1,206 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use super::Sender; -use bytes::{BufMut, BytesMut}; -use compio::buf::IoBufMut; -use compio::net::TcpStream; -use compio::ws::WebSocketStream; -use compio::ws::tungstenite::{Error as TungsteniteError, Message}; -use iggy_common::IggyError; -use server_common::PooledBuffer; -use std::ptr; -use tracing::{debug, warn}; - -const READ_BUFFER_CAPACITY: usize = 8192; -const WRITE_BUFFER_CAPACITY: usize = 8192; -const STATUS_OK: &[u8] = &[0; 4]; - -pub struct WebSocketSender { - pub(crate) stream: WebSocketStream, - pub(crate) read_buffer: BytesMut, - pub(crate) write_buffer: BytesMut, -} - -impl WebSocketSender { - pub fn new(stream: WebSocketStream) -> Self { - Self { - stream, - read_buffer: BytesMut::with_capacity(READ_BUFFER_CAPACITY), - write_buffer: BytesMut::with_capacity(WRITE_BUFFER_CAPACITY), - } - } - - async fn flush_write_buffer(&mut self) -> Result<(), IggyError> { - if self.write_buffer.is_empty() { - return Ok(()); - } - let data = self.write_buffer.split().freeze(); - debug!("WebSocket sending data: {:?}", data.to_vec()); - - self.stream.send(Message::Binary(data)).await.map_err(|e| { - debug!("WebSocket send error: {:?}", e); - match e { - TungsteniteError::ConnectionClosed | TungsteniteError::AlreadyClosed => { - IggyError::ConnectionClosed - } - TungsteniteError::Io(ref io_err) - if io_err.kind() == std::io::ErrorKind::BrokenPipe => - { - warn!("Broken pipe detected (client closed connection)"); - IggyError::ConnectionClosed - } - _ => IggyError::TcpError, - } - }) - } -} - -impl std::fmt::Debug for WebSocketSender { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("WebSocketSender").finish() - } -} - -impl Sender for WebSocketSender { - async fn read(&mut self, mut buffer: B) -> (Result<(), IggyError>, B) { - let required_len = buffer.buf_capacity(); - if required_len == 0 { - return (Ok(()), buffer); - } - - while self.read_buffer.len() < required_len { - match self.stream.read().await { - Ok(Message::Binary(data)) => { - self.read_buffer.extend_from_slice(&data); - } - Ok(Message::Close(_)) => { - return (Err(IggyError::ConnectionClosed), buffer); - } - Ok(Message::Ping(data)) => { - if self.stream.send(Message::Pong(data)).await.is_err() { - return (Err(IggyError::ConnectionClosed), buffer); - } - } - Ok(_) => { /* Ignore other message types */ } - Err(_) => { - return (Err(IggyError::ConnectionClosed), buffer); - } - } - } - - let data_to_copy = self.read_buffer.split_to(required_len); - - unsafe { - ptr::copy_nonoverlapping( - data_to_copy.as_ptr(), - buffer.buf_mut_ptr().cast::(), - required_len, - ); - buffer.set_len(required_len); - } - - (Ok(()), buffer) - } - - async fn send_empty_ok_response(&mut self) -> Result<(), IggyError> { - self.send_ok_response(&[]).await - } - - async fn send_ok_response(&mut self, payload: &[u8]) -> Result<(), IggyError> { - debug!( - "Sending WebSocket response with status: OK, payload length: {}", - payload.len() - ); - - let length = (payload.len() as u32).to_le_bytes(); - let total_size = STATUS_OK.len() + length.len() + payload.len(); - - if self.write_buffer.len() + total_size > self.write_buffer.capacity() { - self.flush_write_buffer().await?; - } - - self.write_buffer.put_slice(STATUS_OK); - self.write_buffer.put_slice(&length); - self.write_buffer.put_slice(payload); - - self.flush_write_buffer().await - } - - async fn send_error_response(&mut self, error: IggyError) -> Result<(), IggyError> { - let status = &error.as_code().to_le_bytes(); - debug!("Sending WebSocket error response with status: {:?}", status); - let length = 0u32.to_le_bytes(); - let total_size = status.len() + length.len(); - - if self.write_buffer.len() + total_size > self.write_buffer.capacity() { - self.flush_write_buffer().await?; - } - self.write_buffer.put_slice(status); - self.write_buffer.put_slice(&length); - self.flush_write_buffer().await - } - - async fn shutdown(&mut self) -> Result<(), IggyError> { - self.flush_write_buffer().await?; - - match self.stream.close(None).await { - Ok(_) => Ok(()), - Err(e) => match e { - TungsteniteError::ConnectionClosed | TungsteniteError::AlreadyClosed => { - debug!("WebSocket connection already closed: {}", e); - Ok(()) - } - _ => Err(IggyError::CannotCloseWebSocketConnection(format!("{}", e))), - }, - } - } - - async fn send_ok_response_vectored( - &mut self, - length: &[u8], - slices: Vec, - ) -> Result<(), IggyError> { - self.flush_write_buffer().await?; - - let total_payload_size = slices.iter().map(|s| s.len()).sum::(); - let total_size = STATUS_OK.len() + length.len() + total_payload_size; - - let mut response_bytes = BytesMut::with_capacity(total_size); - response_bytes.put_slice(STATUS_OK); - response_bytes.put_slice(length); - for slice in slices { - response_bytes.put_slice(&slice); - } - - self.stream - .send(Message::Binary(response_bytes.freeze())) - .await - .map_err(|e| match e { - TungsteniteError::ConnectionClosed | TungsteniteError::AlreadyClosed => { - IggyError::ConnectionClosed - } - TungsteniteError::Io(ref io_err) - if io_err.kind() == std::io::ErrorKind::BrokenPipe => - { - warn!("Broken pipe in vectored send - client closed connection"); - IggyError::ConnectionClosed - } - _ => IggyError::TcpError, - }) - } -} diff --git a/core/server/src/sender/websocket_tls_sender.rs b/core/server/src/sender/websocket_tls_sender.rs deleted file mode 100644 index 8a25854274..0000000000 --- a/core/server/src/sender/websocket_tls_sender.rs +++ /dev/null @@ -1,185 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use super::Sender; -use bytes::{BufMut, BytesMut}; -use compio::buf::IoBufMut; -use compio::net::TcpStream; -use compio::ws::WebSocketStream; -use compio::ws::tungstenite::{Error as TungsteniteError, Message}; -use iggy_common::IggyError; -use server_common::PooledBuffer; -use std::ptr; -use tracing::debug; - -const READ_BUFFER_CAPACITY: usize = 8192; -const WRITE_BUFFER_CAPACITY: usize = 8192; -const STATUS_OK: &[u8] = &[0; 4]; - -pub struct WebSocketTlsSender { - pub(crate) stream: WebSocketStream, - pub(crate) read_buffer: BytesMut, - pub(crate) write_buffer: BytesMut, -} - -impl WebSocketTlsSender { - pub fn new(stream: WebSocketStream) -> Self { - Self { - stream, - read_buffer: BytesMut::with_capacity(READ_BUFFER_CAPACITY), - write_buffer: BytesMut::with_capacity(WRITE_BUFFER_CAPACITY), - } - } - - async fn flush_write_buffer(&mut self) -> Result<(), IggyError> { - if self.write_buffer.is_empty() { - return Ok(()); - } - let data = self.write_buffer.split().freeze(); - self.stream - .send(Message::Binary(data)) - .await - .map_err(|_| IggyError::TcpError) - } -} - -impl std::fmt::Debug for WebSocketTlsSender { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("WebSocketTlsSender").finish() - } -} - -impl Sender for WebSocketTlsSender { - async fn read(&mut self, mut buffer: B) -> (Result<(), IggyError>, B) { - let required_len = buffer.buf_capacity(); - if required_len == 0 { - return (Ok(()), buffer); - } - - while self.read_buffer.len() < required_len { - match self.stream.read().await { - Ok(Message::Binary(data)) => { - self.read_buffer.extend_from_slice(&data); - } - Ok(Message::Close(_)) => { - return (Err(IggyError::ConnectionClosed), buffer); - } - Ok(Message::Ping(data)) => { - if self.stream.send(Message::Pong(data)).await.is_err() { - return (Err(IggyError::ConnectionClosed), buffer); - } - } - Ok(_) => { /* Ignore other message types */ } - Err(_) => { - return (Err(IggyError::ConnectionClosed), buffer); - } - } - } - - let data_to_copy = self.read_buffer.split_to(required_len); - - unsafe { - ptr::copy_nonoverlapping( - data_to_copy.as_ptr(), - buffer.buf_mut_ptr().cast::(), - required_len, - ); - buffer.set_len(required_len); - } - - (Ok(()), buffer) - } - - async fn send_empty_ok_response(&mut self) -> Result<(), IggyError> { - self.send_ok_response(&[]).await - } - - async fn send_ok_response(&mut self, payload: &[u8]) -> Result<(), IggyError> { - debug!( - "Sending WebSocket TLS response with status: OK, payload length: {}", - payload.len() - ); - - let length = (payload.len() as u32).to_le_bytes(); - let total_size = STATUS_OK.len() + length.len() + payload.len(); - - if self.write_buffer.len() + total_size > self.write_buffer.capacity() { - self.flush_write_buffer().await?; - } - - self.write_buffer.put_slice(STATUS_OK); - self.write_buffer.put_slice(&length); - self.write_buffer.put_slice(payload); - - self.flush_write_buffer().await - } - - async fn send_error_response(&mut self, error: IggyError) -> Result<(), IggyError> { - let status = &error.as_code().to_le_bytes(); - debug!( - "Sending WebSocket TLS error response with status: {:?}", - status - ); - let length = 0u32.to_le_bytes(); - let total_size = status.len() + length.len(); - - if self.write_buffer.len() + total_size > self.write_buffer.capacity() { - self.flush_write_buffer().await?; - } - self.write_buffer.put_slice(status); - self.write_buffer.put_slice(&length); - self.flush_write_buffer().await - } - - async fn shutdown(&mut self) -> Result<(), IggyError> { - self.flush_write_buffer().await?; - - match self.stream.close(None).await { - Ok(_) => Ok(()), - Err(e) => match e { - TungsteniteError::ConnectionClosed | TungsteniteError::AlreadyClosed => { - debug!("WebSocket TLS connection already closed: {}", e); - Ok(()) - } - _ => Err(IggyError::CannotCloseWebSocketConnection(format!("{}", e))), - }, - } - } - - async fn send_ok_response_vectored( - &mut self, - length: &[u8], - slices: Vec, - ) -> Result<(), IggyError> { - self.flush_write_buffer().await?; - - let total_payload_size = slices.iter().map(|s| s.len()).sum::(); - let total_size = STATUS_OK.len() + length.len() + total_payload_size; - - let mut response_bytes = BytesMut::with_capacity(total_size); - response_bytes.put_slice(STATUS_OK); - response_bytes.put_slice(length); - for slice in slices { - response_bytes.put_slice(&slice); - } - - self.stream - .send(Message::Binary(response_bytes.freeze())) - .await - .map_err(|_| IggyError::TcpError) - } -} diff --git a/core/server/src/server_error.rs b/core/server/src/server_error.rs index 94d20d4c56..85281932c0 100644 --- a/core/server/src/server_error.rs +++ b/core/server/src/server_error.rs @@ -15,87 +15,398 @@ // specific language governing permissions and limitations // under the License. -use compio::quic::{ConnectionError as QuicConnectionError, ReadError, WriteError}; -use error_set::error_set; -use std::array::TryFromSliceError; -use std::io; +use consensus::VsrStateError; +use metadata::impls::recovery::RecoveryError; +use server_common::log::LogError; +use shard::ShardCtorError; +use shard_allocator::ShardingError; +use std::path::PathBuf; +use thiserror::Error; -error_set!( - ServerError := NumaError || ConfigurationError || ArchiverError || ConnectionError || LogError || CompatError || QuicError || ShardError +#[derive(Debug, Error)] +#[non_exhaustive] +pub enum ServerError { + #[error(transparent)] + Iggy(Box), + #[error("failed to load server config")] + Config(#[source] configs::ConfigurationError), + #[error("failed to allocate shards from sharding.cpu_allocation")] + ShardAllocator(#[source] ShardingError), + #[error("failed to bind shard {shard_id} to its CPU set")] + CpuAffinityFailed { + shard_id: u16, + #[source] + source: ShardingError, + }, + #[error("failed to bind shard {shard_id} memory to its NUMA node")] + MemoryAffinityFailed { + shard_id: u16, + #[source] + source: ShardingError, + }, + #[error("failed to spawn OS thread for shard {shard_id}")] + ShardSpawnFailed { + shard_id: u16, + #[source] + source: std::io::Error, + }, + // `{source}` is deliberately part of the Display text: the shard-join + // failure report and `%error` log fields print Display only, and the + // source carries the io_uring remediation folded in by + // `server_common::diagnostics::enrich_runtime_create_error`. + #[error("failed to create io_uring runtime for shard {shard_id}: {source}")] + ShardRuntimeCreateFailed { + shard_id: u16, + #[source] + source: std::io::Error, + }, + #[error( + "shard allocator produced zero shards; server must run at least one \ + shard (check [system.sharding] cpu_allocation)" + )] + ShardsCountZero, + #[error( + "computed shards_count = {count} exceeds the maximum of {} shards per \ + server; shard ids must fit in u16 and stay below the OWNER_NONE \ + sentinel", + message_bus::OWNER_NONE - 1 + )] + ShardsCountOverflow { count: usize }, + #[error("system.sharding.inbox_capacity must be in 1..={max}; got {value}")] + InvalidInboxCapacity { value: usize, max: usize }, + #[error("system.sharding.shutdown_drain_timeout must be in (0, {max:?}]; got {value:?}")] + InvalidShutdownDrainTimeout { + value: std::time::Duration, + max: std::time::Duration, + }, + #[error("system.sharding.shutdown_poll_interval must be in (0, {max:?}]; got {value:?}")] + InvalidShutdownPollInterval { + value: std::time::Duration, + max: std::time::Duration, + }, + #[error( + "system.sharding.shutdown_poll_interval ({poll:?}) must be <= \ + shutdown_drain_timeout ({drain:?})" + )] + ShutdownPollExceedsDrain { + poll: std::time::Duration, + drain: std::time::Duration, + }, + #[error("failed to serialize current server config")] + CurrentConfigSerialize(#[source] toml::ser::Error), + #[error("failed to write current server config at {path}")] + CurrentConfigWrite { + path: String, + #[source] + source: std::io::Error, + }, + #[error("failed to initialize server logging")] + Logging(#[source] LogError), + #[error("failed to recover metadata snapshot and journal")] + MetadataRecovery(#[source] RecoveryError), + #[error("failed to open partition superblock at {dir}")] + PartitionSuperblockIo { + dir: PathBuf, + #[source] + source: std::io::Error, + }, + // Quarantines the one partition rather than treating the group as fresh or + // reading through to a superseded view: mirrors the metadata plane's + // `RecoveryError::SuperblockUnreadable` policy, minus the boot refusal, + // because one unreadable partition directory must not strand every healthy + // group on the shard. + #[error( + "partition superblock at {dir} is present but its format version \ + {version} is unrecognized by this build (a downgrade, or a corrupt \ + version field)" + )] + PartitionSuperblockVersionUnknown { dir: PathBuf, version: u16 }, + #[error( + "partition superblock at {dir} is present but a copy holds bytes that \ + do not verify (bit-rot or a checksum failure), so its latest \ + generation cannot be established" + )] + PartitionSuperblockUnverifiable { dir: PathBuf }, + #[error( + "partition superblock at {dir} was checksum-clean but did not decode; \ + tombstoning this partition rather than inferring a stale view" + )] + PartitionSuperblockUndecodable { + dir: PathBuf, + #[source] + source: VsrStateError, + }, + #[error( + "partition superblock at {dir} belongs to a different {field}: expected \ + {expected}, found {found}; a copied or misplaced data directory, or the \ + cluster was resized without reconfiguration" + )] + PartitionSuperblockIdentityMismatch { + dir: PathBuf, + field: metadata::IdentityField, + expected: u128, + found: u128, + }, + // Per-partition, not fatal: the boot path fences this one group (quarantines + // its segment files and materialises it fresh) instead of taking the node + // down for one damaged local chain. The shapes it reports are exactly what a + // failed state-transfer quarantine leaves behind, and the rebuild recovers + // the data from a peer. + #[error( + "partition {stream_id}/{topic_id}/{partition_id} at {dir} recovered an \ + unusable segment chain: {reason}" + )] + PartitionChainRefused { + dir: PathBuf, + stream_id: usize, + topic_id: usize, + partition_id: usize, + reason: PartitionChainRefusal, + }, + #[error( + "shard {shard_id} aborted while waiting for shard-0 to broadcast the metadata \ + factory bundle; shard 0 dropped its sender (most likely it failed to recover)" + )] + MetadataHandoffAborted { shard_id: u16 }, + #[error( + "shard 0 aborted before binding listeners with {remaining} peer shard(s) still loading \ + their on-disk partitions; a peer most likely failed during bootstrap (shutdown flag set)" + )] + ShardBootstrapBarrierAborted { remaining: usize }, + #[error("failed to parse {context} socket address '{address}'")] + SocketAddressParse { + context: &'static str, + address: String, + #[source] + source: std::net::AddrParseError, + }, + #[error("cluster enabled but no node is configured for replica {replica_id}")] + ClusterNodeNotFound { replica_id: u8 }, + #[error("cluster node count {count} exceeds supported u8 replica count")] + ClusterReplicaCountTooLarge { count: usize }, + #[error("cluster mode requires --replica-id to identify the current node")] + MissingReplicaId, + #[error( + "--replica-id {supplied} was passed with cluster.enabled=false; the WAL would commit \ + under replica {default} which permanently fixes this node's identity. Either set \ + cluster.enabled=true with a matching nodes[] entry, or drop --replica-id" + )] + ReplicaIdRequiresCluster { supplied: u8, default: u8 }, + #[error( + "cluster node for replica {replica_id} is missing ports.{transport}; cluster mode \ + requires an explicit roster port for every enabled transport" + )] + ClusterPortMissing { + transport: &'static str, + replica_id: u8, + }, + #[error( + "cluster bootstrap with empty metadata requires both {username_env} and {password_env} to be set before server can create the root user deterministically" + )] + ClusterRootCredentialsRequired { + username_env: &'static str, + password_env: &'static str, + }, + #[error( + "{provided_env} is set but {missing_env} is not; the root user credentials must be \ + provided as a pair" + )] + RootCredentialsIncomplete { + provided_env: &'static str, + missing_env: &'static str, + }, + #[error("{env_name} must be {min}..={max} characters long; got {length}")] + RootCredentialLength { + env_name: &'static str, + length: usize, + min: usize, + max: usize, + }, + #[error("--fresh could not remove the system path at {path}")] + FreshWipeFailed { + path: PathBuf, + #[source] + source: std::io::Error, + }, + #[error( + "recovered segment for stream {stream_id}, topic {topic_id}, partition {partition_id} at start_offset {start_offset} has message/index divergence (messages_size={messages_size_bytes}, indexed_size={indexed_size_bytes}, end_offset={end_offset}); recovery aborted before opening listeners. Restore the partition from a healthy replica or snapshot, or move the segment aside for offline repair before restarting." + )] + RecoveredSegmentSizeDivergence { + stream_id: usize, + topic_id: usize, + partition_id: usize, + start_offset: u64, + end_offset: u64, + messages_size_bytes: u64, + indexed_size_bytes: u64, + }, + #[error( + "failed to load persisted {consumer_kind} offsets for stream {stream_id}, topic {topic_id}, partition {partition_id} from {path}" + )] + ConsumerOffsetsLoad { + consumer_kind: &'static str, + stream_id: usize, + topic_id: usize, + partition_id: usize, + path: String, + #[source] + source: Box, + }, + #[error( + "recovered namespace stream {stream_id}, topic {topic_id}, partition {partition_id} exceeds configured limits (max_streams={max_streams}, max_topics={max_topics}, max_partitions={max_partitions})" + )] + RecoveredNamespaceOutOfBounds { + stream_id: usize, + topic_id: usize, + partition_id: usize, + max_streams: usize, + max_topics: usize, + max_partitions: usize, + }, + #[error("failed to load {transport} listener credentials")] + ListenerCredentials { + transport: &'static str, + #[source] + source: std::io::Error, + }, + #[error("failed to build the HTTP forward client: {reason}")] + HttpForwardClient { reason: String }, + #[error("failed to construct IggyShard from bootstrap inputs")] + ShardConstruction(#[source] ShardCtorError), + #[error("{} shard thread(s) failed: {}", failures.len(), format_shard_failures(failures))] + ShardJoinFailures { failures: Vec }, +} - IoError := { - #[display("IO error")] - IoError(io::Error), +/// Why a recovered segment chain cannot be served. +/// +/// Both shapes mean the same thing operationally -- the local files do not form +/// a chain this replica can serve -- but they are distinguished because they +/// point at different causes: an empty non-tail segment is a failed rebuild's +/// orphan pairing, a hole is a stray or half-unlinked file. +#[derive(Debug)] +pub enum PartitionChainRefusal { + EmptyNonTailSegment { + empty_start: u64, + next_start: u64, + }, + Hole { + previous_start: u64, + previous_end: u64, + next_start: u64, + }, +} - #[display("Write error")] - WriteError(WriteError), - - #[display("Read error")] - ReadToEndError(ReadError) - } - - NumaError := { - #[display("{0}")] - Sharding(shard_allocator::ShardingError), +impl std::fmt::Display for PartitionChainRefusal { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::EmptyNonTailSegment { + empty_start, + next_start, + } => write!( + f, + "segment {empty_start} is empty yet {next_start} follows it, so the \ + chain cannot be served past it" + ), + Self::Hole { + previous_start, + previous_end, + next_start, + } => write!( + f, + "segment {previous_start} ends at offset {previous_end} but the next \ + starts at {next_start}, leaving a hole" + ), + } } +} - ConfigurationError := { - ConfigurationError(configs::ConfigurationError), - } - - ArchiverError := { - #[display("File to archive not found: {}", file_path)] - FileToArchiveNotFound { file_path: String }, - - #[display("Cannot initialize S3 archiver")] - CannotInitializeS3Archiver, +/// Per-shard outcome captured by [`crate::bootstrap::ShardHandles::join_all`] +/// when a shard either returned `Err` or panicked. +/// +/// Bundled into [`ServerError::ShardJoinFailures`] so the operator sees +/// every failing shard rather than only the first one, which previously +/// lived in the trace log alone. +#[derive(Debug)] +pub struct ShardJoinFailure { + pub shard_id: u16, + pub kind: ShardJoinFailureKind, +} - #[display("Invalid S3 credentials")] - InvalidS3Credentials, +#[derive(Debug)] +pub enum ShardJoinFailureKind { + Error(Box), + Panic { + message: String, + }, + /// The shard thread never finished inside `shutdown_join_timeout` + /// and was abandoned so process exit is not blocked forever. + Wedged { + waited: std::time::Duration, + }, +} - #[display("HTTP request error: {0}")] - CyperError(cyper::Error), - - #[display("Cannot archive file: {}", file_path)] - CannotArchiveFile { file_path: String }, - } || IoError - - ConnectionError := { - #[display("Connection error")] - QuicConnectionError(QuicConnectionError), - } || IoError || CommonError - - LogError := { - #[display("{0}")] - Logging(server_common::log::LogError), +fn format_shard_failures(failures: &[ShardJoinFailure]) -> String { + use std::fmt::Write as _; + let mut out = String::new(); + for (idx, failure) in failures.iter().enumerate() { + if idx > 0 { + out.push_str("; "); + } + match &failure.kind { + ShardJoinFailureKind::Error(err) => { + let _ = write!(out, "shard {} -> {err}", failure.shard_id); + } + ShardJoinFailureKind::Panic { message } => { + let _ = write!(out, "shard {} panicked: {message}", failure.shard_id); + } + ShardJoinFailureKind::Wedged { waited } => { + let _ = write!( + out, + "shard {} wedged: thread still running after {waited:?}, abandoned", + failure.shard_id + ); + } + } } + out +} - CompatError := { - #[display("Index migration error")] - IndexMigrationError, - } || IoError || CommonError - - CommonError := { - #[display("Try from slice error")] - TryFromSliceError(TryFromSliceError), - - #[display("SDK error")] - SdkError(iggy_common::IggyError), +impl From for ServerError { + fn from(source: iggy_common::IggyError) -> Self { + Self::Iggy(Box::new(source)) } +} - QuicError := { - #[display("Cert load error")] - CertLoadError, - #[display("Cert generation error")] - CertGenerationError, - #[display("Config creation error")] - ConfigCreationError, - #[display("Transport config error")] - TransportConfigError, - } +#[cfg(test)] +mod tests { + use super::*; - ShardError := { - #[display("Shard failed: {}", message)] - ShardFailure { message: String }, + #[test] + fn shard_join_failures_display_aggregates_all_entries() { + let failures = vec![ + ShardJoinFailure { + shard_id: 0, + kind: ShardJoinFailureKind::Error(Box::new(ServerError::MissingReplicaId)), + }, + ShardJoinFailure { + shard_id: 2, + kind: ShardJoinFailureKind::Panic { + message: "boom".to_string(), + }, + }, + ]; + let rendered = ServerError::ShardJoinFailures { failures }.to_string(); + assert!( + rendered.starts_with("2 shard thread(s) failed:"), + "expected count prefix, got {rendered}" + ); + assert!( + rendered.contains("shard 0 ->"), + "shard 0 entry missing: {rendered}" + ); + assert!( + rendered.contains("shard 2 panicked: boom"), + "shard 2 panic entry missing: {rendered}" + ); } -); +} diff --git a/core/server-ng/src/session_manager.rs b/core/server/src/session_manager.rs similarity index 99% rename from core/server-ng/src/session_manager.rs rename to core/server/src/session_manager.rs index 9a35f6a1bd..11fca2a5ba 100644 --- a/core/server-ng/src/session_manager.rs +++ b/core/server/src/session_manager.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -//! Transport-to-consensus session bridge for server-ng. +//! Transport-to-consensus session bridge for server. //! //! Maps ephemeral transport connections to durable consensus sessions. //! Each connection goes through: `connect → login → register → bound`. @@ -50,8 +50,8 @@ pub enum ConnectionState { Authenticated { user_id: u32 }, /// Register committed through consensus. Connection is bound to a /// `(client_id, session)` pair. Requests on this connection use - /// these values to populate `RequestHeader.client` and - /// `RequestHeader.session`. + /// these values to populate `RoutedRequestHeader.client` and + /// `RoutedRequestHeader.session`. Bound { user_id: u32, client_id: u128, @@ -84,7 +84,7 @@ pub struct Connection { /// Bridges transport connections to consensus sessions. /// /// NOT thread-safe: each shard owns one `SessionManager` on its -/// single-threaded compio runtime, the same way the rest of server-ng +/// single-threaded compio runtime, the same way the rest of server /// is structured. All mutators take `&mut self`; the type carries no /// internal locking. /// diff --git a/core/server/src/shard/builder.rs b/core/server/src/shard/builder.rs deleted file mode 100644 index 817276484e..0000000000 --- a/core/server/src/shard/builder.rs +++ /dev/null @@ -1,197 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use super::{ - IggyShard, TaskRegistry, transmission::connector::ShardConnector, - transmission::frame::ShardFrame, -}; -use crate::metadata::{Metadata, MetadataWriter}; -use crate::streaming::partitions::local_partitions::LocalPartitions; -use crate::{ - configs::server::ServerConfig, - state::file::FileState, - streaming::{ - clients::client_manager::ClientManager, diagnostics::metrics::Metrics, - utils::ptr::EternalPtr, - }, -}; -use ahash::AHashSet; -use dashmap::DashMap; -use iggy_common::EncryptorKind; -use iggy_common::SemanticVersion; -use server_common::sharding::{IggyNamespace, PartitionLocation}; -use std::{ - cell::{Cell, RefCell}, - rc::Rc, - sync::atomic::AtomicBool, -}; - -#[derive(Default)] -pub struct IggyShardBuilder { - id: Option, - shards_table: Option>>, - state: Option, - client_manager: Option, - connections: Option>>, - config: Option, - encryptor: Option, - version: Option, - metrics: Option, - is_follower: bool, - /// Runtime-supplied replica identity. Resolved from the `--replica-id` - /// CLI flag and matched against `config.cluster.nodes[*].replica_id` at - /// startup. `None` when cluster mode is disabled. - current_replica_id: Option, - metadata: Option, - metadata_writer: Option, -} - -impl IggyShardBuilder { - pub fn id(mut self, id: u16) -> Self { - self.id = Some(id); - self - } - - pub fn connections(mut self, connections: Vec>) -> Self { - self.connections = Some(connections); - self - } - - pub fn config(mut self, config: ServerConfig) -> Self { - self.config = Some(config); - self - } - - pub fn shards_table( - mut self, - shards_table: EternalPtr>, - ) -> Self { - self.shards_table = Some(shards_table); - self - } - - pub fn clients_manager(mut self, client_manager: ClientManager) -> Self { - self.client_manager = Some(client_manager); - self - } - - pub fn encryptor(mut self, encryptor: Option) -> Self { - self.encryptor = encryptor; - self - } - - pub fn version(mut self, version: SemanticVersion) -> Self { - self.version = Some(version); - self - } - - pub fn state(mut self, state: FileState) -> Self { - self.state = Some(state); - self - } - - pub fn metrics(mut self, metrics: Metrics) -> Self { - self.metrics = Some(metrics); - self - } - - pub fn is_follower(mut self, is_follower: bool) -> Self { - self.is_follower = is_follower; - self - } - - pub fn current_replica_id(mut self, current_replica_id: Option) -> Self { - self.current_replica_id = current_replica_id; - self - } - - pub fn metadata(mut self, metadata: Metadata) -> Self { - self.metadata = Some(metadata); - self - } - - pub fn metadata_writer(mut self, metadata_writer: MetadataWriter) -> Self { - self.metadata_writer = Some(metadata_writer); - self - } - - // TODO: Too much happens in there, some of those bootstrapping logic should be moved outside. - pub fn build(self) -> IggyShard { - let id = self.id.unwrap(); - let shards_table = self.shards_table.unwrap(); - let state = self.state.unwrap(); - let config = self.config.unwrap(); - let connections = self.connections.unwrap(); - let encryptor = self.encryptor; - let client_manager = self.client_manager.unwrap(); - let version = self.version.unwrap(); - let metadata = self.metadata.expect("metadata is required"); - let (stop_receiver, frame_receiver) = connections - .iter() - .filter(|c| c.id == id) - .map(|c| (c.stop_receiver.clone(), c.receiver.clone())) - .next() - .expect("Failed to find connection with the specified ID"); - - // Collect all stop_senders for broadcasting shutdown to all shards - let all_stop_senders: Vec<_> = connections.iter().map(|c| c.stop_sender.clone()).collect(); - let shards = connections; - - // Initialize metrics - let metrics = self.metrics.unwrap_or_else(Metrics::init); - - // Create TaskRegistry with all stop_senders for critical task failures - let task_registry = Rc::new(TaskRegistry::new(id, all_stop_senders)); - - // Create notification channel for config writer - let (config_writer_notify, config_writer_receiver) = async_channel::bounded(1); - - // Trigger initial check in case servers bind before task starts - let _ = config_writer_notify.try_send(()); - - // Create per-shard stores (wrapped in RefCell for interior mutability) - let local_partitions = RefCell::new(LocalPartitions::new()); - - IggyShard { - id, - shards, - shards_table, - metadata, - metadata_writer: self.metadata_writer.map(RefCell::new), - local_partitions, - pending_partition_inits: RefCell::new(AHashSet::new()), - encryptor, - config, - _version: version, - state, - stop_receiver, - messages_receiver: Cell::new(Some(frame_receiver)), - metrics, - is_follower: self.is_follower, - current_replica_id: self.current_replica_id, - is_shutting_down: AtomicBool::new(false), - tcp_bound_address: Cell::new(None), - quic_bound_address: Cell::new(None), - websocket_bound_address: Cell::new(None), - http_bound_address: Cell::new(None), - config_writer_notify, - config_writer_receiver, - task_registry, - client_manager, - } - } -} diff --git a/core/server/src/shard/communication.rs b/core/server/src/shard/communication.rs deleted file mode 100644 index a67014110a..0000000000 --- a/core/server/src/shard/communication.rs +++ /dev/null @@ -1,198 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::shard::{ - BROADCAST_TIMEOUT, COMPONENT, IggyShard, - transmission::{ - connector::ShardConnector, - event::ShardEvent, - frame::{ShardFrame, ShardResponse}, - message::{ShardMessage, ShardRequest}, - }, -}; -use futures::future::join_all; -use hash32::{Hasher, Murmur3Hasher}; -use iggy_common::{Identifier, IggyError}; -use server_common::sharding::{IggyNamespace, PartitionLocation}; -use std::hash::Hasher as _; -use tracing::{error, info, warn}; - -impl IggyShard { - /// Sends a control-plane request to shard 0's message pump. - pub async fn send_to_control_plane( - &self, - request: ShardRequest, - ) -> Result { - let shard0 = &self.shards[0]; - shard0 - .send_request(ShardMessage::Request(request)) - .await - .map_err(|err| { - error!( - "{COMPONENT} - failed to send control-plane request to shard 0, error: {err}" - ); - err - }) - } - - /// Sends a data-plane request to the shard owning the partition. - pub async fn send_to_data_plane( - &self, - request: ShardRequest, - ) -> Result { - let ns = request - .routing - .as_ref() - .expect("data-plane request requires namespace"); - let shard = self - .find_shard(ns) - .ok_or_else(|| self.namespace_not_found_error(ns))?; - shard - .send_request(ShardMessage::Request(request)) - .await - .map_err(|err| { - error!( - "{COMPONENT} - failed to send data-plane request to shard {}, error: {err}", - shard.id - ); - err - }) - } - - /// Converts a missing namespace in shards_table to the appropriate entity-not-found error. - fn namespace_not_found_error(&self, ns: &IggyNamespace) -> IggyError { - let stream_id = - Identifier::numeric(ns.stream_id() as u32).expect("numeric identifier is always valid"); - let topic_id = - Identifier::numeric(ns.topic_id() as u32).expect("numeric identifier is always valid"); - - if self.metadata.get_stream_id(&stream_id).is_none() { - return IggyError::StreamIdNotFound(stream_id); - } - - if self - .metadata - .get_topic_id(ns.stream_id(), &topic_id) - .is_none() - { - return IggyError::TopicIdNotFound(stream_id, topic_id); - } - - IggyError::PartitionNotFound(ns.partition_id(), topic_id, stream_id) - } - - pub async fn broadcast_event_to_all_shards(&self, event: ShardEvent) -> Result<(), IggyError> { - if self.is_shutting_down() { - info!("Skipping broadcast during shutdown for event: {}", event); - return Ok(()); - } - - let event_type = event.to_string(); - let futures = self - .shards - .iter() - .filter(|s| s.id != self.id) - .map(|shard| { - let event = event.clone(); - let conn = shard.clone(); - let shard_id = shard.id; - let event_type = event_type.clone(); - - async move { - let (sender, receiver) = async_channel::bounded(1); - conn.send(ShardFrame::new(ShardMessage::Event(event), Some(sender))); - - match compio::time::timeout(BROADCAST_TIMEOUT, receiver.recv()).await { - Ok(Ok(_)) => Ok(()), - Ok(Err(e)) => { - warn!( - "Broadcast to shard {} failed for event {}: channel error: {}", - shard_id, event_type, e - ); - Err(()) - } - Err(e) => { - warn!( - "Broadcast to shard {} failed for event {}: timeout waiting for response after {:?}, elapsed: {:?}", - shard_id, event_type, - BROADCAST_TIMEOUT, - e - ); - Err(()) - } - } - } - }) - .collect::>(); - - if futures.is_empty() { - return Ok(()); - } - - let results = join_all(futures).await; - let has_failures = results.iter().any(|r| r.is_err()); - - if has_failures { - Err(IggyError::ShardCommunicationError) - } else { - Ok(()) - } - } - - pub fn find_shard(&self, namespace: &IggyNamespace) -> Option<&ShardConnector> { - self.shards_table.get(namespace).map(|location| { - self.shards - .iter() - .find(|shard| shard.id == *location.shard_id) - .expect("Shard not found in the shards table.") - }) - } - - pub fn remove_shard_table_record(&self, namespace: &IggyNamespace) -> PartitionLocation { - self.shards_table - .remove(namespace) - .map(|(_, location)| location) - .expect("remove_shard_table_record: namespace not found") - } - - pub fn insert_shard_table_record(&self, ns: IggyNamespace, location: PartitionLocation) { - self.shards_table.insert(ns, location); - } - - pub fn get_current_shard_namespaces(&self) -> Vec { - self.shards_table - .iter() - .filter_map(|entry| { - let (ns, location) = entry.pair(); - if *location.shard_id == self.id { - Some(*ns) - } else { - None - } - }) - .collect() - } -} - -// Utility function for shard assignment calculation -pub fn calculate_shard_assignment(ns: &IggyNamespace, upperbound: u32) -> u16 { - let mut hasher = Murmur3Hasher::default(); - hasher.write_u64(ns.inner()); - let hash = hasher.finish32(); - // Murmur3 has problems with weak lower bits for small integer inputs, so we use bits from the middle. - ((hash >> 16) % upperbound) as u16 -} diff --git a/core/server/src/shard/execution.rs b/core/server/src/shard/execution.rs deleted file mode 100644 index 14664aa9b7..0000000000 --- a/core/server/src/shard/execution.rs +++ /dev/null @@ -1,732 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::wire_id_to_identifier; -use crate::streaming::users::user::User; -use crate::streaming::utils::crypto; -use crate::{ - shard::{ - IggyShard, - transmission::{ - event::ShardEvent, - frame::{ConsumerGroupResponseData, StreamResponseData, TopicResponseData}, - message::ResolvedTopic, - }, - }, - state::{ - command::EntryCommand, - models::{ - CreateConsumerGroupWithId, CreatePersonalAccessTokenWithHash, CreateStreamWithId, - CreateTopicWithId, CreateUserWithId, - }, - }, - streaming::polling_consumer::ConsumerGroupId, -}; -use iggy_binary_protocol::requests::{ - consumer_groups::*, partitions::*, personal_access_tokens::*, streams::*, topics::*, users::*, -}; -use iggy_common::wire_conversions::wire_permissions_to_permissions; -use iggy_common::{ - CompressionAlgorithm, Identifier, IggyError, IggyExpiry, MaxTopicSize, PersonalAccessToken, - UserStatus, -}; -use secrecy::{ExposeSecret, SecretString}; - -pub async fn execute_create_stream( - shard: &IggyShard, - user_id: u32, - wire: CreateStreamRequest, -) -> Result { - shard.metadata.perm_create_stream(user_id)?; - - let stream_id = shard.create_stream(wire.name.to_string()).await?; - - let response_data = shard.metadata.with_metadata(|m| { - let stream = m - .streams - .get(stream_id) - .expect("stream missing from metadata after creation"); - StreamResponseData { - id: stream_id as u32, - name: stream.name.clone(), - created_at: stream.created_at, - } - }); - - shard - .state - .apply( - user_id, - &EntryCommand::CreateStream(CreateStreamWithId { - stream_id: stream_id as u32, - command: wire, - }), - ) - .await?; - - Ok(response_data) -} - -pub async fn execute_update_stream( - shard: &IggyShard, - user_id: u32, - wire: UpdateStreamRequest, -) -> Result<(), IggyError> { - let stream_id = wire_id_to_identifier(&wire.stream_id)?; - let stream = shard.resolve_stream(&stream_id)?; - shard.metadata.perm_update_stream(user_id, stream.id())?; - - shard.update_stream(stream, wire.name.to_string())?; - - shard - .state - .apply(user_id, &EntryCommand::UpdateStream(wire)) - .await?; - - Ok(()) -} - -pub async fn execute_delete_stream( - shard: &IggyShard, - user_id: u32, - wire: DeleteStreamRequest, -) -> Result<(), IggyError> { - let stream_id = wire_id_to_identifier(&wire.stream_id)?; - let stream = shard.resolve_stream(&stream_id)?; - shard.metadata.perm_delete_stream(user_id, stream.id())?; - - // Capture all topic/partition info BEFORE deletion for broadcast - let topics_with_partitions: Vec<(usize, Vec)> = shard - .metadata - .get_topic_ids(stream.id()) - .into_iter() - .map(|topic_id| { - let partition_ids = shard.metadata.get_partition_ids(stream.id(), topic_id); - (topic_id, partition_ids) - }) - .collect(); - - shard.delete_stream(stream).await?; - - shard - .state - .apply(user_id, &EntryCommand::DeleteStream(wire)) - .await?; - - // Broadcast DeletedPartitions to all shards for each topic's partitions (best-effort) - for (topic_id, partition_ids) in topics_with_partitions { - if partition_ids.is_empty() { - continue; - } - let event = ShardEvent::DeletedPartitions { - stream_id: Identifier::numeric(stream.id() as u32) - .expect("numeric identifier is always valid"), - topic_id: Identifier::numeric(topic_id as u32) - .expect("numeric identifier is always valid"), - partitions_count: partition_ids.len() as u32, - partition_ids, - }; - if let Err(e) = shard.broadcast_event_to_all_shards(event).await { - tracing::warn!("Broadcast failed: {e}. Shards will sync on restart."); - } - } - - Ok(()) -} - -pub async fn execute_purge_stream( - shard: &IggyShard, - user_id: u32, - wire: PurgeStreamRequest, -) -> Result<(), IggyError> { - let stream_id = wire_id_to_identifier(&wire.stream_id)?; - let stream = shard.resolve_stream(&stream_id)?; - shard.metadata.perm_purge_stream(user_id, stream.id())?; - - shard.purge_stream(stream).await?; - shard.purge_stream_local(stream).await?; - - shard - .state - .apply(user_id, &EntryCommand::PurgeStream(wire)) - .await?; - - let event = ShardEvent::PurgedStream { - stream_id: Identifier::numeric(stream.id() as u32) - .expect("numeric identifier is always valid"), - }; - if let Err(e) = shard.broadcast_event_to_all_shards(event).await { - tracing::warn!("Broadcast failed: {e}. Shards will sync on restart."); - } - - Ok(()) -} - -pub async fn execute_create_topic( - shard: &IggyShard, - user_id: u32, - wire: CreateTopicRequest, -) -> Result { - let stream_id = wire_id_to_identifier(&wire.stream_id)?; - let compression = CompressionAlgorithm::from_code(wire.compression_algorithm)?; - let message_expiry = IggyExpiry::from(wire.message_expiry); - let max_topic_size = MaxTopicSize::from(wire.max_topic_size); - let replication_factor = if wire.replication_factor == 0 { - None - } else { - Some(wire.replication_factor) - }; - - let stream = shard.resolve_stream(&stream_id)?; - shard.metadata.perm_create_topic(user_id, stream.id())?; - - let topic_id = shard - .create_topic( - stream, - wire.name.to_string(), - message_expiry, - compression, - max_topic_size, - replication_factor, - ) - .await?; - - let resolved_topic = ResolvedTopic { - stream_id: stream.id(), - topic_id, - }; - let partition_infos = shard - .create_partitions(resolved_topic, wire.partitions_count) - .await?; - - let response_data = shard.metadata.with_metadata(|m| { - let topic = m - .streams - .get(stream.id()) - .and_then(|s| s.topics.get(topic_id)) - .expect("topic missing from metadata after creation"); - TopicResponseData { - id: topic_id as u32, - name: topic.name.clone(), - created_at: topic.created_at, - partitions: partition_infos.clone(), - message_expiry: topic.message_expiry, - compression_algorithm: topic.compression_algorithm, - max_topic_size: topic.max_topic_size, - replication_factor: topic.replication_factor, - } - }); - - shard - .state - .apply( - user_id, - &EntryCommand::CreateTopic(CreateTopicWithId { - topic_id: topic_id as u32, - command: wire, - }), - ) - .await?; - - let event = ShardEvent::CreatedPartitions { - stream_id: Identifier::numeric(stream.id() as u32) - .expect("numeric identifier is always valid"), - topic_id: Identifier::numeric(topic_id as u32).expect("numeric identifier is always valid"), - partitions: partition_infos, - }; - if let Err(e) = shard.broadcast_event_to_all_shards(event).await { - tracing::warn!("Broadcast failed: {e}. Shards will sync on restart."); - } - - Ok(response_data) -} - -pub async fn execute_update_topic( - shard: &IggyShard, - user_id: u32, - wire: UpdateTopicRequest, -) -> Result<(), IggyError> { - let stream_id = wire_id_to_identifier(&wire.stream_id)?; - let topic_id = wire_id_to_identifier(&wire.topic_id)?; - let compression = CompressionAlgorithm::from_code(wire.compression_algorithm)?; - let message_expiry = IggyExpiry::from(wire.message_expiry); - let max_topic_size = MaxTopicSize::from(wire.max_topic_size); - let replication_factor = if wire.replication_factor == 0 { - None - } else { - Some(wire.replication_factor) - }; - - let topic = shard.resolve_topic(&stream_id, &topic_id)?; - shard - .metadata - .perm_update_topic(user_id, topic.stream_id, topic.topic_id)?; - - shard.update_topic( - topic, - wire.name.to_string(), - message_expiry, - compression, - max_topic_size, - replication_factor, - )?; - - shard - .state - .apply(user_id, &EntryCommand::UpdateTopic(wire)) - .await?; - - Ok(()) -} - -pub async fn execute_delete_topic( - shard: &IggyShard, - user_id: u32, - wire: DeleteTopicRequest, -) -> Result<(), IggyError> { - let stream_id = wire_id_to_identifier(&wire.stream_id)?; - let topic_id = wire_id_to_identifier(&wire.topic_id)?; - let topic = shard.resolve_topic(&stream_id, &topic_id)?; - shard - .metadata - .perm_delete_topic(user_id, topic.stream_id, topic.topic_id)?; - - // Capture partition_ids BEFORE deletion for broadcast - let partition_ids = shard - .metadata - .get_partition_ids(topic.stream_id, topic.topic_id); - - shard.delete_topic(topic).await?; - - shard - .state - .apply(user_id, &EntryCommand::DeleteTopic(wire)) - .await?; - - // Broadcast to all shards to clean up their local_partitions entries (best-effort) - let event = ShardEvent::DeletedPartitions { - stream_id: Identifier::numeric(topic.stream_id as u32) - .expect("numeric identifier is always valid"), - topic_id: Identifier::numeric(topic.topic_id as u32) - .expect("numeric identifier is always valid"), - partitions_count: partition_ids.len() as u32, - partition_ids, - }; - if let Err(e) = shard.broadcast_event_to_all_shards(event).await { - tracing::warn!("Broadcast failed: {e}. Shards will sync on restart."); - } - - Ok(()) -} - -pub async fn execute_purge_topic( - shard: &IggyShard, - user_id: u32, - wire: PurgeTopicRequest, -) -> Result<(), IggyError> { - let stream_id = wire_id_to_identifier(&wire.stream_id)?; - let topic_id = wire_id_to_identifier(&wire.topic_id)?; - let topic = shard.resolve_topic(&stream_id, &topic_id)?; - shard - .metadata - .perm_purge_topic(user_id, topic.stream_id, topic.topic_id)?; - - shard.purge_topic(topic).await?; - shard.purge_topic_local(topic).await?; - - shard - .state - .apply(user_id, &EntryCommand::PurgeTopic(wire)) - .await?; - - let event = ShardEvent::PurgedTopic { - stream_id: Identifier::numeric(topic.stream_id as u32) - .expect("numeric identifier is always valid"), - topic_id: Identifier::numeric(topic.topic_id as u32) - .expect("numeric identifier is always valid"), - }; - if let Err(e) = shard.broadcast_event_to_all_shards(event).await { - tracing::warn!("Broadcast failed: {e}. Shards will sync on restart."); - } - - Ok(()) -} - -pub async fn execute_create_partitions( - shard: &IggyShard, - user_id: u32, - wire: CreatePartitionsRequest, -) -> Result<(), IggyError> { - let stream_id = wire_id_to_identifier(&wire.stream_id)?; - let topic_id = wire_id_to_identifier(&wire.topic_id)?; - let topic = shard.resolve_topic(&stream_id, &topic_id)?; - shard - .metadata - .perm_create_partitions(user_id, topic.stream_id, topic.topic_id)?; - - let partition_infos = shard - .create_partitions(topic, wire.partitions_count) - .await?; - let total_partition_count = shard - .metadata - .partitions_count(topic.stream_id, topic.topic_id) as u32; - shard.writer().rebalance_consumer_groups_for_topic( - topic.stream_id, - topic.topic_id, - total_partition_count, - ); - - shard - .state - .apply(user_id, &EntryCommand::CreatePartitions(wire)) - .await?; - - let event = ShardEvent::CreatedPartitions { - stream_id: Identifier::numeric(topic.stream_id as u32) - .expect("numeric identifier is always valid"), - topic_id: Identifier::numeric(topic.topic_id as u32) - .expect("numeric identifier is always valid"), - partitions: partition_infos, - }; - if let Err(e) = shard.broadcast_event_to_all_shards(event).await { - tracing::warn!("Broadcast failed: {e}. Shards will sync on restart."); - } - - Ok(()) -} - -pub async fn execute_delete_partitions( - shard: &IggyShard, - user_id: u32, - wire: DeletePartitionsRequest, -) -> Result<(), IggyError> { - let stream_id = wire_id_to_identifier(&wire.stream_id)?; - let topic_id = wire_id_to_identifier(&wire.topic_id)?; - let topic = shard.resolve_topic(&stream_id, &topic_id)?; - shard - .metadata - .perm_delete_partitions(user_id, topic.stream_id, topic.topic_id)?; - - let deleted_partition_ids = shard - .delete_partitions(topic, wire.partitions_count) - .await?; - - let remaining_partition_count = shard - .metadata - .partitions_count(topic.stream_id, topic.topic_id) - as u32; - shard.writer().rebalance_consumer_groups_for_topic( - topic.stream_id, - topic.topic_id, - remaining_partition_count, - ); - - shard - .state - .apply(user_id, &EntryCommand::DeletePartitions(wire)) - .await?; - - let event = ShardEvent::DeletedPartitions { - stream_id: Identifier::numeric(topic.stream_id as u32) - .expect("numeric identifier is always valid"), - topic_id: Identifier::numeric(topic.topic_id as u32) - .expect("numeric identifier is always valid"), - partitions_count: deleted_partition_ids.len() as u32, - partition_ids: deleted_partition_ids.clone(), - }; - if let Err(e) = shard.broadcast_event_to_all_shards(event).await { - tracing::warn!("Broadcast failed: {e}. Shards will sync on restart."); - } - - Ok(()) -} - -pub async fn execute_create_consumer_group( - shard: &IggyShard, - user_id: u32, - wire: CreateConsumerGroupRequest, -) -> Result { - let stream_id = wire_id_to_identifier(&wire.stream_id)?; - let topic_id = wire_id_to_identifier(&wire.topic_id)?; - let topic = shard.resolve_topic(&stream_id, &topic_id)?; - shard - .metadata - .perm_create_consumer_group(user_id, topic.stream_id, topic.topic_id)?; - - let group_id = shard.create_consumer_group(topic, wire.name.to_string())?; - - let response_data = shard - .metadata - .get_consumer_group(topic.stream_id, topic.topic_id, group_id) - .map(|cg| ConsumerGroupResponseData { - id: group_id as u32, - name: cg.name.clone(), - partitions_count: cg.partitions.len() as u32, - }) - .expect("consumer group missing from metadata after creation"); - - shard - .state - .apply( - user_id, - &EntryCommand::CreateConsumerGroup(CreateConsumerGroupWithId { - group_id: group_id as u32, - command: wire, - }), - ) - .await?; - - Ok(response_data) -} - -pub async fn execute_delete_consumer_group( - shard: &IggyShard, - user_id: u32, - wire: DeleteConsumerGroupRequest, -) -> Result<(), IggyError> { - let stream_id = wire_id_to_identifier(&wire.stream_id)?; - let topic_id = wire_id_to_identifier(&wire.topic_id)?; - let group_id = wire_id_to_identifier(&wire.group_id)?; - let group = shard.resolve_consumer_group(&stream_id, &topic_id, &group_id)?; - shard - .metadata - .perm_delete_consumer_group(user_id, group.stream_id, group.topic_id)?; - - let deleted = shard.delete_consumer_group(group)?; - - let cg_id = ConsumerGroupId(deleted.group_id); - shard - .delete_consumer_group_offsets( - cg_id, - group.stream_id, - group.topic_id, - &deleted.partition_ids, - ) - .await?; - - shard - .state - .apply(user_id, &EntryCommand::DeleteConsumerGroup(wire)) - .await?; - - Ok(()) -} - -pub fn execute_join_consumer_group( - shard: &IggyShard, - user_id: u32, - client_id: u32, - wire: JoinConsumerGroupRequest, -) -> Result<(), IggyError> { - let stream_id = wire_id_to_identifier(&wire.stream_id)?; - let topic_id = wire_id_to_identifier(&wire.topic_id)?; - let group_id = wire_id_to_identifier(&wire.group_id)?; - let group = shard.resolve_consumer_group(&stream_id, &topic_id, &group_id)?; - shard - .metadata - .perm_join_consumer_group(user_id, group.stream_id, group.topic_id)?; - - shard.join_consumer_group(client_id, group)?; - - Ok(()) -} - -pub fn execute_leave_consumer_group( - shard: &IggyShard, - user_id: u32, - client_id: u32, - wire: LeaveConsumerGroupRequest, -) -> Result<(), IggyError> { - let stream_id = wire_id_to_identifier(&wire.stream_id)?; - let topic_id = wire_id_to_identifier(&wire.topic_id)?; - let group_id = wire_id_to_identifier(&wire.group_id)?; - let group = shard.resolve_consumer_group(&stream_id, &topic_id, &group_id)?; - shard - .metadata - .perm_leave_consumer_group(user_id, group.stream_id, group.topic_id)?; - - shard.leave_consumer_group(client_id, group)?; - - Ok(()) -} - -pub async fn execute_create_user( - shard: &IggyShard, - user_id: u32, - wire: CreateUserRequest, -) -> Result { - shard.metadata.perm_create_user(user_id)?; - - let username = wire.username.to_string(); - let password = SecretString::from(wire.password.clone()); - let status = UserStatus::from_code(wire.status)?; - let permissions = wire - .permissions - .as_ref() - .map(wire_permissions_to_permissions); - - let user = shard.create_user(&username, password.expose_secret(), status, permissions)?; - - // Hash the password before persisting to WAL - let mut wal_wire = wire; - wal_wire.password = crypto::hash_password(password.expose_secret()); - - shard - .state - .apply( - user_id, - &EntryCommand::CreateUser(CreateUserWithId { - user_id: user.id, - command: wal_wire, - }), - ) - .await?; - - Ok(user) -} - -pub async fn execute_delete_user( - shard: &IggyShard, - user_id: u32, - wire: DeleteUserRequest, -) -> Result { - shard.metadata.perm_delete_user(user_id)?; - - let target_id = wire_id_to_identifier(&wire.user_id)?; - let user = shard.delete_user(&target_id)?; - - shard - .state - .apply(user_id, &EntryCommand::DeleteUser(wire)) - .await?; - - Ok(user) -} - -pub async fn execute_update_user( - shard: &IggyShard, - user_id: u32, - wire: UpdateUserRequest, -) -> Result { - shard.metadata.perm_update_user(user_id)?; - - let target_id = wire_id_to_identifier(&wire.user_id)?; - let username = wire.username.as_ref().map(|n| n.to_string()); - let status = wire.status.map(UserStatus::from_code).transpose()?; - let user = shard.update_user(&target_id, username, status)?; - - shard - .state - .apply(user_id, &EntryCommand::UpdateUser(wire)) - .await?; - - Ok(user) -} - -pub async fn execute_change_password( - shard: &IggyShard, - user_id: u32, - wire: ChangePasswordRequest, -) -> Result<(), IggyError> { - let target_id = wire_id_to_identifier(&wire.user_id)?; - let target_user = shard.get_user(&target_id)?; - if target_user.id != user_id { - shard.metadata.perm_change_password(user_id)?; - } - - shard.change_password(&target_id, &wire.current_password, &wire.new_password)?; - - // Clear current password and hash new password before persisting to WAL - let wal_wire = ChangePasswordRequest { - user_id: wire.user_id, - current_password: String::new(), - new_password: crypto::hash_password(&wire.new_password), - }; - - shard - .state - .apply(user_id, &EntryCommand::ChangePassword(wal_wire)) - .await?; - - Ok(()) -} - -pub async fn execute_update_permissions( - shard: &IggyShard, - user_id: u32, - wire: UpdatePermissionsRequest, -) -> Result<(), IggyError> { - shard.metadata.perm_update_permissions(user_id)?; - - let target_id = wire_id_to_identifier(&wire.user_id)?; - let target_user = shard.get_user(&target_id)?; - if target_user.is_root() { - return Err(IggyError::CannotChangePermissions(target_user.id)); - } - - let permissions = wire - .permissions - .as_ref() - .map(wire_permissions_to_permissions); - shard.update_permissions(&target_id, permissions)?; - - shard - .state - .apply(user_id, &EntryCommand::UpdatePermissions(wire)) - .await?; - - Ok(()) -} - -pub async fn execute_create_personal_access_token( - shard: &IggyShard, - user_id: u32, - wire: CreatePersonalAccessTokenRequest, -) -> Result<(PersonalAccessToken, String), IggyError> { - let name = wire.name.to_string(); - let expiry = IggyExpiry::from(wire.expiry); - let (personal_access_token, token) = - shard.create_personal_access_token(user_id, &name, expiry)?; - - shard - .state - .apply( - user_id, - &EntryCommand::CreatePersonalAccessToken(CreatePersonalAccessTokenWithHash { - hash: personal_access_token.token.to_string(), - command: wire, - }), - ) - .await?; - - Ok((personal_access_token, token)) -} - -pub async fn execute_delete_personal_access_token( - shard: &IggyShard, - user_id: u32, - wire: DeletePersonalAccessTokenRequest, -) -> Result<(), IggyError> { - shard.delete_personal_access_token(user_id, wire.name.as_str())?; - - shard - .state - .apply(user_id, &EntryCommand::DeletePersonalAccessToken(wire)) - .await?; - - Ok(()) -} diff --git a/core/server/src/shard/handlers.rs b/core/server/src/shard/handlers.rs deleted file mode 100644 index 44cc1c0cc2..0000000000 --- a/core/server/src/shard/handlers.rs +++ /dev/null @@ -1,614 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use super::*; -use crate::sender::SenderKind; -use crate::{ - shard::{ - IggyShard, execution, - transmission::{ - event::ShardEvent, - frame::ShardResponse, - message::{ShardMessage, ShardRequest, ShardRequestPayload}, - }, - }, - tcp::{ - connection_handler::{ConnectionAction, handle_connection, handle_error}, - tcp_listener::cleanup_connection, - }, -}; -use compio::net::TcpStream; -use iggy_common::{IggyError, TransportProtocol}; -use nix::sys::stat::SFlag; -use server_common::sharding::IggyNamespace; -use std::os::fd::{FromRawFd, IntoRawFd}; -use tracing::info; - -pub(super) async fn handle_shard_message( - shard: &Rc, - message: ShardMessage, -) -> Option { - match message { - ShardMessage::Request(request) => match handle_request(shard, request).await { - Ok(response) => Some(response), - Err(err) => Some(ShardResponse::ErrorResponse(err)), - }, - ShardMessage::Event(event) => match handle_event(shard, event).await { - Ok(_) => Some(ShardResponse::Event), - Err(err) => Some(ShardResponse::ErrorResponse(err)), - }, - } -} - -async fn handle_request( - shard: &Rc, - request: ShardRequest, -) -> Result { - // Data-plane operations extract namespace from routing - let namespace = request.routing; - match request.payload { - ShardRequestPayload::SendMessages { batch } => { - let batch = shard.maybe_encrypt_messages(batch)?; - let messages_count = batch.count(); - - let namespace = namespace.expect("SendMessages requires routing namespace"); - - shard.ensure_partition(&namespace).await?; - - shard - .append_messages_to_local_partition(&namespace, batch, &shard.config.system) - .await?; - - shard.metrics.increment_messages(messages_count as u64); - Ok(ShardResponse::SendMessages) - } - ShardRequestPayload::PollMessages { args, consumer } => { - let namespace = namespace.expect("PollMessages requires routing namespace"); - - if args.count == 0 { - let current_offset = shard - .local_partitions - .borrow() - .get(&namespace) - .map(|p| p.offset.load(std::sync::atomic::Ordering::Relaxed)) - .unwrap_or(0); - return Ok(ShardResponse::PollMessages(( - iggy_common::IggyPollMetadata::new( - namespace.partition_id() as u32, - current_offset, - ), - crate::streaming::segments::IggyMessagesBatchSet::empty(), - ))); - } - - let auto_commit = args.auto_commit; - - shard.ensure_partition(&namespace).await?; - - let (poll_metadata, batches) = shard - .poll_messages_from_local_partition(&namespace, consumer, args) - .await?; - - if auto_commit && !batches.is_empty() { - let offset = batches - .last_offset() - .expect("Batch set should have at least one batch"); - shard - .auto_commit_consumer_offset_from_local_partition(&namespace, consumer, offset) - .await?; - } - Ok(ShardResponse::PollMessages((poll_metadata, batches))) - } - ShardRequestPayload::FlushUnsavedBuffer { fsync } => { - let ns = namespace.expect("FlushUnsavedBuffer requires routing namespace"); - let flushed_count = shard - .flush_unsaved_buffer_from_local_partitions(&ns, fsync) - .await?; - Ok(ShardResponse::FlushUnsavedBuffer { flushed_count }) - } - ShardRequestPayload::DeleteSegments { segments_count } => { - let ns = namespace.expect("DeleteSegments requires routing namespace"); - let (deleted_segments, deleted_messages) = shard - .delete_oldest_segments( - ns.stream_id(), - ns.topic_id(), - ns.partition_id(), - segments_count, - ) - .await?; - Ok(ShardResponse::DeleteSegments { - deleted_segments, - deleted_messages, - }) - } - ShardRequestPayload::CleanTopicMessages { - stream_id, - topic_id, - partition_ids, - } => { - let (deleted_segments, deleted_messages) = shard - .clean_topic_messages(stream_id, topic_id, &partition_ids) - .await?; - Ok(ShardResponse::CleanTopicMessages { - deleted_segments, - deleted_messages, - }) - } - ShardRequestPayload::CreatePartitionsRequest { user_id, command } => { - assert_eq!( - shard.id, 0, - "CreatePartitionsRequest should only be handled by shard0" - ); - - execution::execute_create_partitions(shard, user_id, command).await?; - Ok(ShardResponse::CreatePartitionsResponse) - } - ShardRequestPayload::DeletePartitionsRequest { user_id, command } => { - assert_eq!( - shard.id, 0, - "DeletePartitionsRequest should only be handled by shard0" - ); - - execution::execute_delete_partitions(shard, user_id, command).await?; - Ok(ShardResponse::DeletePartitionsResponse) - } - ShardRequestPayload::CreateStreamRequest { user_id, command } => { - assert_eq!( - shard.id, 0, - "CreateStreamRequest should only be handled by shard0" - ); - - let result = execution::execute_create_stream(shard, user_id, command).await?; - Ok(ShardResponse::CreateStreamResponse(result)) - } - ShardRequestPayload::CreateTopicRequest { user_id, command } => { - assert_eq!( - shard.id, 0, - "CreateTopicRequest should only be handled by shard0" - ); - - let result = execution::execute_create_topic(shard, user_id, command).await?; - Ok(ShardResponse::CreateTopicResponse(result)) - } - ShardRequestPayload::UpdateTopicRequest { user_id, command } => { - assert_eq!( - shard.id, 0, - "UpdateTopicRequest should only be handled by shard0" - ); - - execution::execute_update_topic(shard, user_id, command).await?; - Ok(ShardResponse::UpdateTopicResponse) - } - ShardRequestPayload::DeleteTopicRequest { user_id, command } => { - assert_eq!( - shard.id, 0, - "DeleteTopicRequest should only be handled by shard0" - ); - - execution::execute_delete_topic(shard, user_id, command).await?; - Ok(ShardResponse::DeleteTopicResponse) - } - ShardRequestPayload::CreateUserRequest { user_id, command } => { - assert_eq!( - shard.id, 0, - "CreateUserRequest should only be handled by shard0" - ); - let user = execution::execute_create_user(shard, user_id, command).await?; - Ok(ShardResponse::CreateUserResponse(user)) - } - ShardRequestPayload::GetStats { .. } => { - assert_eq!(shard.id, 0, "GetStats should only be handled by shard0"); - let stats = shard.get_stats().await?; - Ok(ShardResponse::GetStatsResponse(stats)) - } - ShardRequestPayload::DeleteUserRequest { user_id, command } => { - assert_eq!( - shard.id, 0, - "DeleteUserRequest should only be handled by shard0" - ); - let user = execution::execute_delete_user(shard, user_id, command).await?; - Ok(ShardResponse::DeleteUserResponse(user)) - } - ShardRequestPayload::UpdateStreamRequest { user_id, command } => { - assert_eq!( - shard.id, 0, - "UpdateStreamRequest should only be handled by shard0" - ); - - execution::execute_update_stream(shard, user_id, command).await?; - Ok(ShardResponse::UpdateStreamResponse) - } - ShardRequestPayload::DeleteStreamRequest { user_id, command } => { - assert_eq!( - shard.id, 0, - "DeleteStreamRequest should only be handled by shard0" - ); - - execution::execute_delete_stream(shard, user_id, command).await?; - Ok(ShardResponse::DeleteStreamResponse) - } - ShardRequestPayload::UpdatePermissionsRequest { user_id, command } => { - assert_eq!( - shard.id, 0, - "UpdatePermissionsRequest should only be handled by shard0" - ); - execution::execute_update_permissions(shard, user_id, command).await?; - Ok(ShardResponse::UpdatePermissionsResponse) - } - ShardRequestPayload::ChangePasswordRequest { user_id, command } => { - assert_eq!( - shard.id, 0, - "ChangePasswordRequest should only be handled by shard0" - ); - execution::execute_change_password(shard, user_id, command).await?; - Ok(ShardResponse::ChangePasswordResponse) - } - ShardRequestPayload::UpdateUserRequest { user_id, command } => { - assert_eq!( - shard.id, 0, - "UpdateUserRequest should only be handled by shard0" - ); - let user = execution::execute_update_user(shard, user_id, command).await?; - Ok(ShardResponse::UpdateUserResponse(user)) - } - ShardRequestPayload::CreateConsumerGroupRequest { user_id, command } => { - assert_eq!( - shard.id, 0, - "CreateConsumerGroupRequest should only be handled by shard0" - ); - - let result = execution::execute_create_consumer_group(shard, user_id, command).await?; - Ok(ShardResponse::CreateConsumerGroupResponse(result)) - } - ShardRequestPayload::JoinConsumerGroupRequest { - user_id, - client_id, - command, - } => { - assert_eq!( - shard.id, 0, - "JoinConsumerGroupRequest should only be handled by shard0" - ); - - execution::execute_join_consumer_group(shard, user_id, client_id, command)?; - Ok(ShardResponse::JoinConsumerGroupResponse) - } - ShardRequestPayload::LeaveConsumerGroupRequest { - user_id, - client_id, - command, - } => { - assert_eq!( - shard.id, 0, - "LeaveConsumerGroupRequest should only be handled by shard0" - ); - - execution::execute_leave_consumer_group(shard, user_id, client_id, command)?; - Ok(ShardResponse::LeaveConsumerGroupResponse) - } - ShardRequestPayload::DeleteConsumerGroupRequest { user_id, command } => { - assert_eq!( - shard.id, 0, - "DeleteConsumerGroupRequest should only be handled by shard0" - ); - - execution::execute_delete_consumer_group(shard, user_id, command).await?; - Ok(ShardResponse::DeleteConsumerGroupResponse) - } - ShardRequestPayload::CreatePersonalAccessTokenRequest { user_id, command } => { - assert_eq!( - shard.id, 0, - "CreatePersonalAccessTokenRequest should only be handled by shard0" - ); - - let (personal_access_token, token) = - execution::execute_create_personal_access_token(shard, user_id, command).await?; - - Ok(ShardResponse::CreatePersonalAccessTokenResponse( - personal_access_token, - token, - )) - } - ShardRequestPayload::DeletePersonalAccessTokenRequest { user_id, command } => { - assert_eq!( - shard.id, 0, - "DeletePersonalAccessTokenRequest should only be handled by shard0" - ); - - execution::execute_delete_personal_access_token(shard, user_id, command).await?; - - Ok(ShardResponse::DeletePersonalAccessTokenResponse) - } - ShardRequestPayload::LeaveConsumerGroupMetadataOnly { - stream_id, - topic_id, - group_id, - client_id, - } => { - assert_eq!( - shard.id, 0, - "LeaveConsumerGroupMetadataOnly should only be handled by shard0" - ); - - shard - .writer() - .leave_consumer_group(stream_id, topic_id, group_id, client_id); - - Ok(ShardResponse::LeaveConsumerGroupMetadataOnlyResponse) - } - ShardRequestPayload::CompletePartitionRevocation { - stream_id, - topic_id, - group_id, - member_slab_id, - member_id, - partition_id, - timed_out, - } => { - assert_eq!( - shard.id, 0, - "CompletePartitionRevocation should only be handled by shard0" - ); - - shard.writer().complete_partition_revocation( - stream_id, - topic_id, - group_id, - member_slab_id, - member_id, - partition_id, - timed_out, - ); - - Ok(ShardResponse::CompletePartitionRevocationResponse) - } - ShardRequestPayload::SocketTransfer { - fd, - from_shard, - client_id, - user_id, - address, - initial_data, - } => { - info!( - "Received socket transfer msg, fd: {fd:?}, from_shard: {from_shard}, address: {address}" - ); - - // Safety: The fd already != 1. - let stat = nix::sys::stat::fstat(&fd) - .map_err(|e| IggyError::IoError(format!("Invalid fd: {}", e)))?; - - if !SFlag::from_bits_truncate(stat.st_mode).contains(SFlag::S_IFSOCK) { - return Err(IggyError::IoError(format!("fd {:?} is not a socket", fd))); - } - - // restore TcpStream from fd - let tcp_stream = unsafe { TcpStream::from_raw_fd(fd.into_raw_fd()) }; - let session = shard.add_client(&address, TransportProtocol::Tcp); - session.set_user_id(user_id); - session.set_migrated(); - - let mut sender = SenderKind::get_tcp_sender(tcp_stream); - let conn_stop_receiver = shard.task_registry.add_connection(session.client_id); - let shard_for_conn = shard.clone(); - let registry = shard.task_registry.clone(); - let registry_clone = registry.clone(); - - let batch = shard.maybe_encrypt_messages(initial_data)?; - let messages_count = batch.count(); - - let ns = namespace.expect("SocketTransfer requires routing namespace"); - shard.ensure_partition(&ns).await?; - - shard - .append_messages_to_local_partition(&ns, batch, &shard.config.system) - .await?; - - shard.metrics.increment_messages(messages_count as u64); - - sender.send_empty_ok_response().await?; - - registry.spawn_connection(async move { - match handle_connection(&session, &mut sender, &shard_for_conn, conn_stop_receiver) - .await - { - Ok(ConnectionAction::Migrated { to_shard }) => { - info!("Migrated to shard {to_shard}, ignore cleanup connection"); - } - Ok(ConnectionAction::Finished) => { - cleanup_connection( - &mut sender, - client_id, - address, - ®istry_clone, - &shard_for_conn, - ) - .await; - } - Err(err) => { - handle_error(err); - cleanup_connection( - &mut sender, - client_id, - address, - ®istry_clone, - &shard_for_conn, - ) - .await; - } - } - }); - - Ok(ShardResponse::SocketTransferResponse) - } - ShardRequestPayload::PurgeStreamRequest { user_id, command } => { - assert_eq!( - shard.id, 0, - "PurgeStreamRequest should only be handled by shard0" - ); - - execution::execute_purge_stream(shard, user_id, command).await?; - Ok(ShardResponse::PurgeStreamResponse) - } - ShardRequestPayload::PurgeTopicRequest { user_id, command } => { - assert_eq!( - shard.id, 0, - "PurgeTopicRequest should only be handled by shard0" - ); - - execution::execute_purge_topic(shard, user_id, command).await?; - Ok(ShardResponse::PurgeTopicResponse) - } - } -} - -pub async fn handle_event(shard: &Rc, event: ShardEvent) -> Result<(), IggyError> { - match event { - ShardEvent::DeletedPartitions { - stream_id, - topic_id, - partitions_count: _, - partition_ids, - } => { - // SharedMetadata was already updated by the request handler before broadcasting. - // Here we only need to clean up local local_partitions entries on all shards. - // - // For DeleteTopic, the topic is already removed from metadata, so we extract - // numeric IDs directly from the Identifier (which must be numeric in that case). - // For DeletePartitions, the topic still exists, so metadata lookup works. - let numeric_stream_id = stream_id - .get_u32_value() - .map(|v| v as usize) - .unwrap_or_else(|_| shard.metadata.get_stream_id(&stream_id).unwrap_or_default()); - let numeric_topic_id = - topic_id - .get_u32_value() - .map(|v| v as usize) - .unwrap_or_else(|_| { - shard - .metadata - .get_topic_id(numeric_stream_id, &topic_id) - .unwrap_or_default() - }); - let mut partitions = shard.local_partitions.borrow_mut(); - for partition_id in partition_ids { - let ns = IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_id); - partitions.remove(&ns); - } - Ok(()) - } - ShardEvent::PurgedStream { stream_id } => { - let stream = shard.resolve_stream(&stream_id)?; - shard.purge_stream_local(stream).await?; - Ok(()) - } - ShardEvent::PurgedTopic { - stream_id, - topic_id, - } => { - let topic = shard.resolve_topic(&stream_id, &topic_id)?; - shard.purge_topic_local(topic).await?; - Ok(()) - } - ShardEvent::AddressBound { protocol, address } => { - info!( - "Received AddressBound event for {:?} with address: {}", - protocol, address - ); - match protocol { - TransportProtocol::Tcp => { - shard.tcp_bound_address.set(Some(address)); - let _ = shard.config_writer_notify.try_send(()); - } - TransportProtocol::Quic => { - shard.quic_bound_address.set(Some(address)); - let _ = shard.config_writer_notify.try_send(()); - } - TransportProtocol::Http => { - shard.http_bound_address.set(Some(address)); - let _ = shard.config_writer_notify.try_send(()); - } - TransportProtocol::WebSocket => { - shard.websocket_bound_address.set(Some(address)); - let _ = shard.config_writer_notify.try_send(()); - } - } - Ok(()) - } - ShardEvent::CreatedPartitions { - stream_id, - topic_id, - partitions, - } => { - let numeric_stream_id = match shard.metadata.get_stream_id(&stream_id) { - Some(id) => id, - None => { - tracing::warn!( - "CreatedPartitions: stream {:?} not found in SharedMetadata", - stream_id - ); - return Ok(()); - } - }; - let numeric_topic_id = match shard.metadata.get_topic_id(numeric_stream_id, &topic_id) { - Some(id) => id, - None => { - tracing::warn!( - "CreatedPartitions: topic {:?} not found in SharedMetadata for stream {}", - topic_id, - numeric_stream_id - ); - return Ok(()); - } - }; - - let shards_count = shard.get_available_shards_count(); - for partition_info in partitions { - let ns = IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_info.id); - let owner_shard_id = crate::shard::calculate_shard_assignment(&ns, shards_count); - - if shard.id == owner_shard_id as u16 { - shard.ensure_partition(&ns).await?; - } - } - Ok(()) - } - ShardEvent::FlushUnsavedBuffer { - stream_id, - topic_id, - partition_id, - fsync, - } => { - let numeric_stream_id = match shard.metadata.get_stream_id(&stream_id) { - Some(id) => id, - None => return Ok(()), - }; - let numeric_topic_id = match shard.metadata.get_topic_id(numeric_stream_id, &topic_id) { - Some(id) => id, - None => return Ok(()), - }; - - let ns = IggyNamespace::new(numeric_stream_id, numeric_topic_id, partition_id); - if shard.local_partitions.borrow().get(&ns).is_some() { - shard - .flush_unsaved_buffer_from_local_partitions(&ns, fsync) - .await?; - } - Ok(()) - } - } -} diff --git a/core/server/src/shard/mod.rs b/core/server/src/shard/mod.rs deleted file mode 100644 index 03f0bad1b1..0000000000 --- a/core/server/src/shard/mod.rs +++ /dev/null @@ -1,482 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use self::tasks::{continuous, periodic}; -use crate::{ - bootstrap::load_segments, - configs::server::ServerConfig, - metadata::{Metadata, MetadataWriter}, - shard::{task_registry::TaskRegistry, transmission::frame::ShardFrame}, - state::file::FileState, - streaming::{ - clients::client_manager::ClientManager, - diagnostics::metrics::Metrics, - partitions::{local_partition::LocalPartition, local_partitions::LocalPartitions}, - session::Session, - utils::ptr::EternalPtr, - }, -}; -use ahash::AHashSet; -use builder::IggyShardBuilder; -use dashmap::DashMap; -use iggy_common::SemanticVersion; -use iggy_common::{EncryptorKind, IggyByteSize, IggyError}; -use server_common::sharding::{IggyNamespace, PartitionLocation}; -use std::{ - cell::{Cell, RefCell}, - net::SocketAddr, - rc::Rc, - sync::{ - Arc, - atomic::{AtomicBool, AtomicU64, Ordering}, - }, - time::{Duration, Instant}, -}; -use tracing::{debug, error, info, instrument, warn}; -use transmission::connector::{Receiver, ShardConnector, StopReceiver}; - -pub mod builder; -pub mod execution; -pub mod handlers; -pub mod system; -pub mod task_registry; -pub mod tasks; -pub mod transmission; - -#[cfg(feature = "systemd")] -pub mod systemd; - -mod communication; - -pub use communication::calculate_shard_assignment; - -pub const COMPONENT: &str = "SHARD"; -pub const SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(10); -pub const BROADCAST_TIMEOUT: Duration = Duration::from_secs(20); - -pub struct IggyShard { - pub id: u16, - shards: Vec>, - _version: SemanticVersion, - - pub(crate) metadata: Metadata, - pub(crate) metadata_writer: Option>, - pub(crate) local_partitions: RefCell, - pub(crate) pending_partition_inits: RefCell>, - - pub(crate) shards_table: EternalPtr>, - pub(crate) state: FileState, - - pub(crate) encryptor: Option, - pub(crate) config: ServerConfig, - pub(crate) client_manager: ClientManager, - pub(crate) metrics: Metrics, - pub(crate) is_follower: bool, - /// Index into `config.cluster.nodes` that describes this running node. - /// `Some` only when cluster mode is enabled; validated at bootstrap to - /// match exactly one entry in the nodes list. - pub(crate) current_replica_id: Option, - pub messages_receiver: Cell>>, - pub(crate) stop_receiver: StopReceiver, - pub(crate) is_shutting_down: AtomicBool, - pub(crate) tcp_bound_address: Cell>, - pub(crate) quic_bound_address: Cell>, - pub(crate) websocket_bound_address: Cell>, - pub(crate) http_bound_address: Cell>, - pub(crate) config_writer_notify: async_channel::Sender<()>, - config_writer_receiver: async_channel::Receiver<()>, - pub(crate) task_registry: Rc, -} - -impl IggyShard { - pub fn builder() -> IggyShardBuilder { - Default::default() - } - - pub fn writer(&self) -> std::cell::RefMut<'_, crate::metadata::MetadataWriter> { - self.metadata_writer - .as_ref() - .expect("MetadataWriter only available on shard 0") - .borrow_mut() - } - - pub async fn init(&self) -> Result<(), IggyError> { - self.load_segments().await?; - let _ = self.load_users().await; - Ok(()) - } - - fn init_tasks(self: &Rc) { - continuous::spawn_message_pump(self.clone()); - - // Spawn config writer task on shard 0 if we need to wait for bound addresses - if self.id == 0 - && (self.config.tcp.enabled - || self.config.quic.enabled - || self.config.http.enabled - || self.config.websocket.enabled) - { - tasks::oneshot::spawn_config_writer_task(self); - } - - if self.config.tcp.enabled { - continuous::spawn_tcp_server(self.clone()); - } - - if self.config.http.enabled && self.id == 0 { - continuous::spawn_http_server(self.clone()); - } - - // JWT token cleaner task is spawned inside HTTP server because it needs `AppState`. - - // TODO(hubcio): QUIC doesn't properly work on all shards, especially tests `concurrent` and `system_scenario`. - // it's probably related to Endpoint not Cloned between shards, but all shards are creating its own instance. - // This way packet CID is invalid. (crypto-related stuff) - if self.config.quic.enabled && self.id == 0 { - continuous::spawn_quic_server(self.clone()); - } - if self.config.websocket.enabled { - continuous::spawn_websocket_server(self.clone()); - } - - if self.config.message_saver.enabled { - periodic::spawn_message_saver(self.clone()); - } - - if self.config.data_maintenance.messages.cleaner_enabled { - periodic::spawn_message_cleaner(self.clone()); - } - - if self.config.heartbeat.enabled { - periodic::spawn_heartbeat_verifier(self.clone()); - } - - if self.config.personal_access_token.cleaner.enabled { - periodic::spawn_personal_access_token_cleaner(self.clone()); - } - - if self.id == 0 { - periodic::spawn_revocation_timeout_checker(self.clone()); - } - - if !self.config.system.logging.sysinfo_print_interval.is_zero() && self.id == 0 { - periodic::spawn_sysinfo_printer(self.clone()); - } - - #[cfg(feature = "systemd")] - if self.id == 0 { - periodic::spawn_systemd_watchdog(self.clone()); - } - } - - pub async fn run(self: &Rc) -> Result<(), IggyError> { - let now: Instant = Instant::now(); - - info!("Starting..."); - self.init().await?; - - // TODO: Fixme - //self.assert_init(); - - self.init_tasks(); - let (shutdown_complete_tx, shutdown_complete_rx) = async_channel::bounded(1); - let stop_receiver = self.get_stop_receiver(); - let shard_for_shutdown = self.clone(); - - // Spawn shutdown handler - compio::runtime::spawn(async move { - let _ = stop_receiver.recv().await; - #[cfg(feature = "systemd")] - if shard_for_shutdown.id == 0 { - systemd::notify_stopping(); - } - let drained = shard_for_shutdown.trigger_shutdown().await; - #[cfg(feature = "systemd")] - if shard_for_shutdown.id == 0 && !drained { - warn!("Graceful shutdown timed out; some tasks did not drain in time"); - systemd::notify_status("graceful shutdown timed out"); - } - #[cfg(not(feature = "systemd"))] - let _ = drained; - let _ = shutdown_complete_tx.send(()).await; - }) - .detach(); - - let elapsed = now.elapsed(); - info!("Initialized in {} ms.", elapsed.as_millis()); - - shutdown_complete_rx.recv().await.ok(); - Ok(()) - } - - async fn load_segments(&self) -> Result<(), IggyError> { - for shard_entry in self.shards_table.iter() { - let (namespace, location) = shard_entry.pair(); - - if *location.shard_id == self.id { - let stream_id = namespace.stream_id(); - let topic_id: usize = namespace.topic_id(); - let partition_id = namespace.partition_id(); - - info!( - "Loading segments for stream: {}, topic: {}, partition: {}", - stream_id, topic_id, partition_id - ); - - let partition_path = - self.config - .system - .get_partition_path(stream_id, topic_id, partition_id); - - let init_info = self - .metadata - .get_partition_init_info(stream_id, topic_id, partition_id) - .expect("Partition must exist in SharedMetadata"); - let created_at = init_info.created_at; - let stats = init_info.stats; - - use crate::streaming::partitions::helpers::create_message_deduplicator; - use crate::streaming::partitions::storage::{ - load_consumer_group_offsets, load_consumer_offsets, - }; - - let consumer_offset_path = - self.config - .system - .get_consumer_offsets_path(stream_id, topic_id, partition_id); - let consumer_group_offsets_path = self - .config - .system - .get_consumer_group_offsets_path(stream_id, topic_id, partition_id); - - // Reuse metadata's Arcs so both metadata and local_partitions - // reference the same allocation — writes via store_consumer_offset - // (metadata path) are visible to delete_oldest_segments (local path). - let consumer_offsets = init_info.consumer_offsets; - let consumer_group_offsets = init_info.consumer_group_offsets; - - { - let guard = consumer_offsets.pin(); - for co in load_consumer_offsets(&consumer_offset_path).unwrap_or_default() { - guard.insert(co.consumer_id as usize, co); - } - } - - { - let guard = consumer_group_offsets.pin(); - for (cg_id, co) in load_consumer_group_offsets(&consumer_group_offsets_path) - .unwrap_or_default() - { - guard.insert(cg_id, co); - } - } - - let message_deduplicator = - create_message_deduplicator(&self.config.system).map(Arc::new); - - match load_segments( - &self.config.system, - stream_id, - topic_id, - partition_id, - partition_path, - stats.clone(), - ) - .await - { - Ok(mut loaded_log) => { - if !loaded_log.has_segments() { - info!( - "No segments found on disk for partition ID: {} for topic ID: {} for stream ID: {}, creating initial segment", - partition_id, topic_id, stream_id - ); - let segment = crate::streaming::segments::Segment::new( - 0, - self.config.system.segment.size, - ); - let storage = - crate::streaming::segments::storage::create_segment_storage( - &self.config.system, - stream_id, - topic_id, - partition_id, - 0, - 0, - 0, - ) - .await?; - loaded_log.add_persisted_segment(segment, storage); - stats.increment_segments_count(1); - } - - // Use the max end_offset across segments that have data, - // not just the active segment. Handles the edge case where - // the active segment is empty (rotated right before shutdown). - let current_offset = loaded_log - .segments() - .iter() - .filter(|s| s.size > IggyByteSize::default()) - .map(|s| s.end_offset) - .max() - .unwrap_or(0); - stats.set_current_offset(current_offset); - - // Check if ANY segment has data. Cannot use current_offset > 0 - // because a single message at offset 0 yields current_offset = 0 - // yet must still increment on the next append. - let should_increment_offset = loaded_log - .segments() - .iter() - .any(|s| s.size > IggyByteSize::default()); - - // After a crash (OOM, SIGKILL), auto_commit may have persisted - // a consumer offset beyond what was flushed to disk. Clamp to - // the partition's actual offset to prevent permanent empty polls. - { - let guard = consumer_offsets.pin(); - for entry in guard.iter() { - let stored = entry.1.offset.load(Ordering::Relaxed); - if stored > current_offset { - warn!( - "Consumer {} offset {} ahead of partition offset {} \ - for stream {}, topic {}, partition {} - clamping \ - (crash recovery)", - entry.0, - stored, - current_offset, - stream_id, - topic_id, - partition_id - ); - entry.1.offset.store(current_offset, Ordering::Relaxed); - } - } - } - { - let guard = consumer_group_offsets.pin(); - for entry in guard.iter() { - let stored = entry.1.offset.load(Ordering::Relaxed); - if stored > current_offset { - warn!( - "Consumer group {:?} offset {} ahead of partition \ - offset {} for stream {}, topic {}, partition {} - \ - clamping (crash recovery)", - entry.0, - stored, - current_offset, - stream_id, - topic_id, - partition_id - ); - entry.1.offset.store(current_offset, Ordering::Relaxed); - } - } - } - - // Initialize journal base_offset so the three-tier - // routing in ops.rs computes correct in_memory_floor. - // Without this, journal defaults to base_offset=0 which - // causes disk reads to be skipped after restart. - if should_increment_offset { - use crate::streaming::partitions::journal::{Inner, Journal}; - loaded_log.journal_mut().init(Inner { - base_offset: current_offset + 1, - ..Default::default() - }); - } - - let revision_id = init_info.revision_id; - - let partition = LocalPartition::with_log( - loaded_log, - stats, - Arc::new(AtomicU64::new(current_offset)), - consumer_offsets, - consumer_group_offsets, - message_deduplicator, - created_at, - revision_id, - should_increment_offset, - ); - - self.local_partitions - .borrow_mut() - .insert(*namespace, partition); - - info!( - "Successfully loaded segments for stream: {}, topic: {}, partition: {}", - stream_id, topic_id, partition_id - ); - } - Err(e) => { - error!( - "Failed to load segments for stream: {}, topic: {}, partition: {}: {}", - stream_id, topic_id, partition_id, e - ); - return Err(e); - } - } - } - } - - Ok(()) - } - - async fn load_users(&self) -> Result<(), IggyError> { - let users_count = self.metadata.users_count(); - self.metrics.increment_users(users_count as u32); - info!("Initialized {} user(s).", users_count); - Ok(()) - } - - pub fn assert_init(&self) -> Result<(), IggyError> { - Ok(()) - } - - pub fn is_shutting_down(&self) -> bool { - self.is_shutting_down.load(Ordering::Relaxed) - } - - pub fn get_stop_receiver(&self) -> StopReceiver { - self.stop_receiver.clone() - } - - #[instrument(skip_all, name = "trace_shutdown")] - pub async fn trigger_shutdown(&self) -> bool { - self.is_shutting_down.store(true, Ordering::SeqCst); - debug!("Shard {} shutdown state set", self.id); - self.task_registry.graceful_shutdown(SHUTDOWN_TIMEOUT).await - } - - pub fn get_available_shards_count(&self) -> u32 { - self.shards.len() as u32 - } - - pub fn ensure_authenticated(&self, session: &Session) -> Result<(), IggyError> { - if !session.is_active() { - error!("{COMPONENT} - session is inactive, session: {session}"); - return Err(IggyError::StaleClient); - } - - if session.is_authenticated() { - Ok(()) - } else { - error!("{COMPONENT} - unauthenticated access attempt, session: {session}"); - Err(IggyError::Unauthenticated) - } - } -} diff --git a/core/server/src/shard/system/clients.rs b/core/server/src/shard/system/clients.rs deleted file mode 100644 index 91cfccfadd..0000000000 --- a/core/server/src/shard/system/clients.rs +++ /dev/null @@ -1,92 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::shard::IggyShard; -use crate::shard::transmission::frame::ShardResponse; -use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; -use crate::streaming::clients::client_manager::Client; -use crate::streaming::session::Session; -use iggy_common::TransportProtocol; -use std::net::SocketAddr; -use tracing::{error, info, warn}; - -impl IggyShard { - pub fn add_client(&self, address: &SocketAddr, transport: TransportProtocol) -> Session { - let session = self.client_manager.add_client(address, transport); - self.metrics.increment_clients(1); - session - } - - pub async fn delete_client(&self, client_id: u32) { - let consumer_groups: Vec<(u32, u32, u32)>; - - { - let client = self.client_manager.try_get_client(client_id); - if client.is_none() { - error!("Client with ID: {client_id} was not found in the client manager.",); - return; - } - - self.metrics.decrement_clients(1); - let client = client.unwrap(); - consumer_groups = client - .consumer_groups - .iter() - .map(|c| (c.stream_id, c.topic_id, c.group_id)) - .collect(); - - info!( - "Deleted {} client with ID: {} for IP address: {}", - client.transport, client.session.client_id, client.session.ip_address - ); - } - - for (stream_id, topic_id, consumer_group_id) in consumer_groups.into_iter() { - let request = - ShardRequest::control_plane(ShardRequestPayload::LeaveConsumerGroupMetadataOnly { - stream_id: stream_id as usize, - topic_id: topic_id as usize, - group_id: consumer_group_id as usize, - client_id, - }); - - match self.send_to_control_plane(request).await { - Ok(ShardResponse::LeaveConsumerGroupMetadataOnlyResponse) => {} - Ok(ShardResponse::ErrorResponse(err)) => { - warn!( - "Failed to leave consumer group {consumer_group_id} for client {client_id} during cleanup: {err}" - ); - } - Ok(_) => {} - Err(err) => { - warn!( - "Failed to send leave consumer group request for client {client_id} during cleanup: {err}" - ); - } - } - } - self.client_manager.delete_client(client_id); - } - - pub fn get_client(&self, client_id: u32) -> Option { - self.client_manager.try_get_client(client_id) - } - - pub fn get_clients(&self) -> Vec { - self.client_manager.get_clients() - } -} diff --git a/core/server/src/shard/system/cluster.rs b/core/server/src/shard/system/cluster.rs deleted file mode 100644 index e4e2688795..0000000000 --- a/core/server/src/shard/system/cluster.rs +++ /dev/null @@ -1,196 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::shard::IggyShard; -use crate::streaming::utils::address::{extract_ip, extract_port}; -use iggy_common::{ - ClusterMetadata, ClusterNode, ClusterNodeRole, ClusterNodeStatus, TransportEndpoints, -}; -use tracing::trace; - -impl IggyShard { - pub fn get_cluster_metadata(&self) -> ClusterMetadata { - let mut nodes = Vec::new(); - - if !self.config.cluster.enabled { - // Single-node: report ourselves as the sole leader; identity - // fields fall back to transport addresses since no cluster list - // is configured. - let current_endpoints = self.get_actual_bound_ports().unwrap_or_else(|| { - TransportEndpoints::new( - extract_port(&self.config.tcp.address), - extract_port(&self.config.quic.address), - extract_port(&self.config.http.address), - extract_port(&self.config.websocket.address), - ) - }); - - nodes.push(ClusterNode { - name: "iggy-node".to_string(), - ip: extract_ip(&self.config.tcp.address), - endpoints: current_endpoints, - role: ClusterNodeRole::Leader, - status: ClusterNodeStatus::Healthy, - }); - - return ClusterMetadata { - name: "single-node".to_string(), - nodes, - }; - } - - // Cluster mode: resolve the current node from the roster by the - // runtime-supplied replica_id. Bootstrap already validated that - // exactly one entry matches; missing `current_replica_id` here - // would indicate a bootstrap bug, so fall back to a safe default - // rather than panicking on the hot metadata path. - let current_id = self.current_replica_id; - let current_node = current_id - .and_then(|id| { - self.config - .cluster - .nodes - .iter() - .find(|node| node.replica_id == id) - }) - .or_else(|| self.config.cluster.nodes.first()); - - let Some(current_node) = current_node else { - // Nodes list is empty even though cluster.enabled=true. Validator - // refuses this state at startup; treat defensively if reached. - return ClusterMetadata { - name: self.config.cluster.name.clone(), - nodes, - }; - }; - - // Use the actual bound ports for the current node so tests that bind - // to port 0 still report the OS-assigned port over the wire. - let current_endpoints = self.get_actual_bound_ports().unwrap_or_else(|| { - TransportEndpoints::new( - current_node - .ports - .tcp - .unwrap_or_else(|| extract_port(&self.config.tcp.address)), - current_node - .ports - .quic - .unwrap_or_else(|| extract_port(&self.config.quic.address)), - current_node - .ports - .http - .unwrap_or_else(|| extract_port(&self.config.http.address)), - current_node - .ports - .websocket - .unwrap_or_else(|| extract_port(&self.config.websocket.address)), - ) - }); - - nodes.push(ClusterNode { - name: current_node.name.clone(), - ip: current_node.ip.clone(), - endpoints: current_endpoints, - role: if self.is_follower { - ClusterNodeRole::Follower - } else { - ClusterNodeRole::Leader - }, - status: ClusterNodeStatus::Healthy, - }); - - for peer in self - .config - .cluster - .nodes - .iter() - .filter(|node| node.replica_id != current_node.replica_id) - { - let endpoints = TransportEndpoints::new( - peer.ports - .tcp - .unwrap_or_else(|| extract_port(&self.config.tcp.address)), - peer.ports - .quic - .unwrap_or_else(|| extract_port(&self.config.quic.address)), - peer.ports - .http - .unwrap_or_else(|| extract_port(&self.config.http.address)), - peer.ports - .websocket - .unwrap_or_else(|| extract_port(&self.config.websocket.address)), - ); - - nodes.push(ClusterNode { - name: peer.name.clone(), - ip: peer.ip.clone(), - endpoints, - role: if self.is_follower { - ClusterNodeRole::Leader - } else { - ClusterNodeRole::Follower - }, - status: ClusterNodeStatus::Healthy, - }); - } - - ClusterMetadata { - name: self.config.cluster.name.clone(), - nodes, - } - } - - /// Get actual bound ports from the shard's bound addresses - /// This is needed when server binds to port 0 (OS-assigned port) - fn get_actual_bound_ports(&self) -> Option { - let tcp_port = self - .tcp_bound_address - .get() - .map(|addr| addr.port()) - .unwrap_or_else(|| extract_port(&self.config.tcp.address)); - - let quic_port = self - .quic_bound_address - .get() - .map(|addr| addr.port()) - .unwrap_or_else(|| extract_port(&self.config.quic.address)); - - let http_port = self - .http_bound_address - .get() - .map(|addr| addr.port()) - .unwrap_or_else(|| extract_port(&self.config.http.address)); - - let websocket_port = self - .websocket_bound_address - .get() - .map(|addr| addr.port()) - .unwrap_or_else(|| extract_port(&self.config.websocket.address)); - - trace!( - "Using actual bound ports - TCP: {}, QUIC: {}, HTTP: {}, WebSocket: {}", - tcp_port, quic_port, http_port, websocket_port - ); - - Some(TransportEndpoints::new( - tcp_port, - quic_port, - http_port, - websocket_port, - )) - } -} diff --git a/core/server/src/shard/system/consumer_groups.rs b/core/server/src/shard/system/consumer_groups.rs deleted file mode 100644 index 7c3b446c19..0000000000 --- a/core/server/src/shard/system/consumer_groups.rs +++ /dev/null @@ -1,205 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use super::COMPONENT; -use crate::shard::IggyShard; -use crate::shard::transmission::message::{ResolvedConsumerGroup, ResolvedTopic}; -use err_trail::ErrContext; -use iggy_common::Identifier; -use iggy_common::IggyError; -use std::sync::Arc; - -pub struct DeletedConsumerGroup { - pub group_id: usize, - pub partition_ids: Vec, -} - -impl IggyShard { - pub fn create_consumer_group( - &self, - topic: ResolvedTopic, - name: String, - ) -> Result { - let stream = topic.stream_id; - let topic_id = topic.topic_id; - - let partitions_count = self.metadata.partitions_count(stream, topic_id) as u32; - - let id = self - .writer() - .create_consumer_group( - &self.metadata, - stream, - topic_id, - Arc::from(name.as_str()), - partitions_count, - ) - .map_err(|e| { - if let IggyError::ConsumerGroupNameAlreadyExists(_, _) = &e { - IggyError::ConsumerGroupNameAlreadyExists( - name.clone(), - Identifier::numeric(topic_id as u32).unwrap(), - ) - } else { - e - } - })?; - - Ok(id) - } - - pub fn delete_consumer_group( - &self, - group: ResolvedConsumerGroup, - ) -> Result { - let stream = group.stream_id; - let topic = group.topic_id; - let group_id = group.group_id; - - let partition_ids = self - .metadata - .get_consumer_group(stream, topic, group_id) - .map(|cg| cg.partitions.clone()) - .unwrap_or_default(); - - self.client_manager - .delete_consumer_group(stream, topic, group_id); - - self.writer().delete_consumer_group(stream, topic, group_id); - - Ok(DeletedConsumerGroup { - group_id, - partition_ids, - }) - } - - /// Join runs on shard 0 (control plane), single-threaded — no concurrent joins. - pub fn join_consumer_group( - &self, - client_id: u32, - group: ResolvedConsumerGroup, - ) -> Result<(), IggyError> { - let valid_client_ids: Vec = self - .client_manager - .get_clients() - .iter() - .map(|c| c.session.client_id) - .collect(); - - let (_, completable) = self.writer().join_consumer_group( - group.stream_id, - group.topic_id, - group.group_id, - client_id, - Some(valid_client_ids), - ); - - for revocation in completable { - self.writer().complete_partition_revocation( - group.stream_id, - group.topic_id, - group.group_id, - revocation.slab_id, - revocation.member_id, - revocation.partition_id, - false, - ); - } - - if let Some(cg) = - self.metadata - .get_consumer_group(group.stream_id, group.topic_id, group.group_id) - && let Some((_, member)) = cg.members.iter().find(|(_, m)| m.client_id == client_id) - && member.partitions.is_empty() - && !cg.partitions.is_empty() - { - let current_valid_ids: Vec = self - .client_manager - .get_clients() - .iter() - .map(|c| c.session.client_id) - .collect(); - - let potentially_stale: Vec = cg - .members - .iter() - .filter(|(_, m)| { - !m.partitions.is_empty() && !current_valid_ids.contains(&m.client_id) - }) - .map(|(_, m)| m.client_id) - .collect(); - - if !potentially_stale.is_empty() { - tracing::info!( - "join_consumer_group: new member {client_id} has no partitions, found stale members: {potentially_stale:?}, forcing leave" - ); - - for stale_client_id in potentially_stale { - let _ = self.writer().leave_consumer_group( - group.stream_id, - group.topic_id, - group.group_id, - stale_client_id, - ); - } - } - } - - self.client_manager - .join_consumer_group(client_id, group.stream_id, group.topic_id, group.group_id) - .error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - failed to make client join consumer group for client ID: {}", - client_id - ) - })?; - - Ok(()) - } - - pub fn leave_consumer_group( - &self, - client_id: u32, - group: ResolvedConsumerGroup, - ) -> Result<(), IggyError> { - let member_id = self.writer().leave_consumer_group( - group.stream_id, - group.topic_id, - group.group_id, - client_id, - ); - - if member_id.is_none() { - return Err(IggyError::ConsumerGroupMemberNotFound( - client_id, - Identifier::numeric(group.group_id as u32).unwrap(), - Identifier::numeric(group.topic_id as u32).unwrap(), - )); - } - - self.client_manager - .leave_consumer_group(client_id, group.stream_id, group.topic_id, group.group_id) - .error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - failed to make client leave consumer group for client ID: {}", - client_id - ) - })?; - - Ok(()) - } -} diff --git a/core/server/src/shard/system/consumer_offsets.rs b/core/server/src/shard/system/consumer_offsets.rs deleted file mode 100644 index 76234ab070..0000000000 --- a/core/server/src/shard/system/consumer_offsets.rs +++ /dev/null @@ -1,489 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use super::COMPONENT; -use crate::{ - shard::IggyShard, - shard::transmission::message::{ResolvedTopic, ShardRequest, ShardRequestPayload}, - streaming::{ - partitions::consumer_offset::ConsumerOffset, - polling_consumer::{ConsumerGroupId, PollingConsumer}, - }, -}; -use err_trail::ErrContext; -use iggy_common::{Consumer, ConsumerKind, ConsumerOffsetInfo, Identifier, IggyError}; -use server_common::sharding::IggyNamespace; -use std::sync::atomic::Ordering; - -impl IggyShard { - pub async fn store_consumer_offset( - &self, - client_id: u32, - consumer: Consumer, - topic: ResolvedTopic, - partition_id: Option, - offset: u64, - ) -> Result<(PollingConsumer, usize), IggyError> { - let Some((polling_consumer, partition_id)) = self.resolve_consumer_with_partition_id( - topic, - &consumer, - client_id, - partition_id, - false, - )? - else { - return Err(IggyError::NotResolvedConsumer(consumer.id)); - }; - - self.validate_partition_offset(topic.stream_id, topic.topic_id, partition_id, offset)?; - - self.store_consumer_offset_base( - topic.stream_id, - topic.topic_id, - &polling_consumer, - partition_id, - offset, - ); - self.persist_consumer_offset_to_disk( - topic.stream_id, - topic.topic_id, - &polling_consumer, - partition_id, - ) - .await?; - - self.maybe_complete_pending_revocation( - &polling_consumer, - topic.stream_id, - topic.topic_id, - partition_id, - ) - .await; - - Ok((polling_consumer, partition_id)) - } - - pub async fn get_consumer_offset( - &self, - client_id: u32, - consumer: Consumer, - topic: ResolvedTopic, - partition_id: Option, - ) -> Result, IggyError> { - let (polling_consumer, partition_id) = match consumer.kind { - ConsumerKind::Consumer => { - let Some((polling_consumer, partition_id)) = self - .resolve_consumer_with_partition_id( - topic, - &consumer, - client_id, - partition_id, - false, - )? - else { - return Err(IggyError::NotResolvedConsumer(consumer.id.clone())); - }; - (polling_consumer, partition_id) - } - ConsumerKind::ConsumerGroup => { - // Reading offsets doesn't require group membership — offsets are stored - // per consumer group (not per member), so any client can query the - // group's progress. Only store_consumer_offset enforces membership. - let cg_id = self - .metadata - .get_consumer_group_id(topic.stream_id, topic.topic_id, &consumer.id) - .ok_or_else(|| { - IggyError::ConsumerGroupIdNotFound( - consumer.id.clone(), - Identifier::numeric(topic.topic_id as u32).unwrap(), - ) - })?; - let partition_id = partition_id.unwrap_or(0) as usize; - (PollingConsumer::consumer_group(cg_id, 0), partition_id) - } - }; - - if !self - .metadata - .partition_exists(topic.stream_id, topic.topic_id, partition_id) - { - return Err(IggyError::PartitionNotFound( - partition_id, - Identifier::numeric(topic.topic_id as u32).expect("valid topic id"), - Identifier::numeric(topic.stream_id as u32).expect("valid stream id"), - )); - } - - let ns = IggyNamespace::new(topic.stream_id, topic.topic_id, partition_id); - let partition_current_offset = self - .metadata - .get_partition_stats(&ns) - .map(|s| s.current_offset()) - .unwrap_or(0); - - let offset = match polling_consumer { - PollingConsumer::Consumer(id, _) => { - let offsets = self.metadata.get_partition_consumer_offsets( - topic.stream_id, - topic.topic_id, - partition_id, - ); - offsets.and_then(|co| { - let guard = co.pin(); - guard.get(&id).map(|item| ConsumerOffsetInfo { - partition_id: partition_id as u32, - current_offset: partition_current_offset, - stored_offset: item.offset.load(Ordering::Relaxed), - }) - }) - } - PollingConsumer::ConsumerGroup(consumer_group_id, _) => { - let offsets = self.metadata.get_partition_consumer_group_offsets( - topic.stream_id, - topic.topic_id, - partition_id, - ); - offsets.and_then(|co| { - let guard = co.pin(); - guard - .get(&consumer_group_id) - .map(|item| ConsumerOffsetInfo { - partition_id: partition_id as u32, - current_offset: partition_current_offset, - stored_offset: item.offset.load(Ordering::Relaxed), - }) - }) - } - }; - Ok(offset) - } - - pub async fn delete_consumer_offset( - &self, - client_id: u32, - consumer: Consumer, - topic: ResolvedTopic, - partition_id: Option, - ) -> Result<(PollingConsumer, usize), IggyError> { - let Some((polling_consumer, partition_id)) = self.resolve_consumer_with_partition_id( - topic, - &consumer, - client_id, - partition_id, - false, - )? - else { - return Err(IggyError::NotResolvedConsumer(consumer.id)); - }; - - if !self - .metadata - .partition_exists(topic.stream_id, topic.topic_id, partition_id) - { - return Err(IggyError::PartitionNotFound( - partition_id, - Identifier::numeric(topic.topic_id as u32).expect("valid topic id"), - Identifier::numeric(topic.stream_id as u32).expect("valid stream id"), - )); - } - - let path = self.delete_consumer_offset_base( - topic.stream_id, - topic.topic_id, - &polling_consumer, - partition_id, - )?; - self.delete_consumer_offset_from_disk(&path).await?; - Ok((polling_consumer, partition_id)) - } - - pub async fn delete_consumer_group_offsets( - &self, - cg_id: ConsumerGroupId, - stream_id: usize, - topic_id: usize, - partition_ids: &[usize], - ) -> Result<(), IggyError> { - for &partition_id in partition_ids { - if !self - .metadata - .partition_exists(stream_id, topic_id, partition_id) - { - tracing::trace!( - "{COMPONENT} - partition {partition_id} not found in stream {stream_id}/topic {topic_id}, skipping offset cleanup for consumer group {}", - cg_id.0 - ); - continue; - } - - let offsets = self.metadata.get_partition_consumer_group_offsets( - stream_id, - topic_id, - partition_id, - ); - - let Some(offsets) = offsets else { - continue; - }; - - let path = offsets.pin().remove(&cg_id).map(|item| item.path.clone()); - - if let Some(path) = path { - self.delete_consumer_offset_from_disk(&path) - .await - .error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - failed to delete consumer group offset file for group with ID: {} in partition {} of topic with ID: {} and stream with ID: {}", - cg_id, partition_id, topic_id, stream_id - ) - })?; - } - } - - Ok(()) - } - - fn store_consumer_offset_base( - &self, - stream_id: usize, - topic_id: usize, - polling_consumer: &PollingConsumer, - partition_id: usize, - offset: u64, - ) { - match polling_consumer { - PollingConsumer::Consumer(id, _) => { - let Some(offsets) = - self.metadata - .get_partition_consumer_offsets(stream_id, topic_id, partition_id) - else { - return; - }; - - let guard = offsets.pin(); - let entry = guard.get_or_insert_with(*id, || { - let dir_path = self.config.system.get_consumer_offsets_path( - stream_id, - topic_id, - partition_id, - ); - let path = format!("{}/{}", dir_path, id); - ConsumerOffset::new(ConsumerKind::Consumer, *id as u32, offset, path) - }); - entry.offset.store(offset, Ordering::Release); - } - PollingConsumer::ConsumerGroup(cg_id, _) => { - let Some(offsets) = self.metadata.get_partition_consumer_group_offsets( - stream_id, - topic_id, - partition_id, - ) else { - return; - }; - - let guard = offsets.pin(); - let entry = guard.get_or_insert_with(*cg_id, || { - let dir_path = self.config.system.get_consumer_group_offsets_path( - stream_id, - topic_id, - partition_id, - ); - let path = format!("{}/{}", dir_path, cg_id.0); - ConsumerOffset::new(ConsumerKind::ConsumerGroup, cg_id.0 as u32, offset, path) - }); - entry.offset.store(offset, Ordering::Release); - } - } - } - - fn delete_consumer_offset_base( - &self, - stream_id: usize, - topic_id: usize, - polling_consumer: &PollingConsumer, - partition_id: usize, - ) -> Result { - match polling_consumer { - PollingConsumer::Consumer(id, _) => { - let offsets = self - .metadata - .get_partition_consumer_offsets(stream_id, topic_id, partition_id) - .ok_or_else(|| IggyError::ConsumerOffsetNotFound(*id))?; - - let guard = offsets.pin(); - let offset = guard - .remove(id) - .ok_or_else(|| IggyError::ConsumerOffsetNotFound(*id))?; - Ok(offset.path.clone()) - } - PollingConsumer::ConsumerGroup(cg_id, _) => { - let offsets = self - .metadata - .get_partition_consumer_group_offsets(stream_id, topic_id, partition_id) - .ok_or_else(|| IggyError::ConsumerOffsetNotFound(cg_id.0))?; - - let guard = offsets.pin(); - let offset = guard - .remove(cg_id) - .ok_or_else(|| IggyError::ConsumerOffsetNotFound(cg_id.0))?; - Ok(offset.path.clone()) - } - } - } - - async fn persist_consumer_offset_to_disk( - &self, - stream_id: usize, - topic_id: usize, - polling_consumer: &PollingConsumer, - partition_id: usize, - ) -> Result<(), IggyError> { - use crate::streaming::partitions::storage::persist_offset; - - let (offset_value, path) = match polling_consumer { - PollingConsumer::Consumer(id, _) => { - let offsets = self - .metadata - .get_partition_consumer_offsets(stream_id, topic_id, partition_id) - .ok_or_else(|| IggyError::ConsumerOffsetNotFound(*id))?; - - let guard = offsets.pin(); - let item = guard - .get(id) - .ok_or_else(|| IggyError::ConsumerOffsetNotFound(*id))?; - (item.offset.load(Ordering::Relaxed), item.path.clone()) - } - PollingConsumer::ConsumerGroup(cg_id, _) => { - let offsets = self - .metadata - .get_partition_consumer_group_offsets(stream_id, topic_id, partition_id) - .ok_or_else(|| IggyError::ConsumerOffsetNotFound(cg_id.0))?; - - let guard = offsets.pin(); - let item = guard - .get(cg_id) - .ok_or_else(|| IggyError::ConsumerOffsetNotFound(cg_id.0))?; - (item.offset.load(Ordering::Relaxed), item.path.clone()) - } - }; - persist_offset(&path, offset_value).await - } - - pub async fn delete_consumer_offset_from_disk(&self, path: &str) -> Result<(), IggyError> { - crate::streaming::partitions::storage::delete_persisted_offset(path).await - } - - /// Enumerates and deletes all consumer/group offset files for a partition from disk. - /// Uses filesystem paths from config rather than in-memory state (which may already be cleared). - pub async fn delete_all_consumer_offset_files( - &self, - stream_id: usize, - topic_id: usize, - partition_id: usize, - ) -> Result<(), IggyError> { - let consumers_path = - self.config - .system - .get_consumer_offsets_path(stream_id, topic_id, partition_id); - let groups_path = - self.config - .system - .get_consumer_group_offsets_path(stream_id, topic_id, partition_id); - - Self::delete_all_files_in_dir(&consumers_path).await?; - Self::delete_all_files_in_dir(&groups_path).await?; - Ok(()) - } - - /// Complete a pending partition revocation if this offset commit satisfies it. - pub(crate) async fn maybe_complete_pending_revocation( - &self, - polling_consumer: &PollingConsumer, - stream_id: usize, - topic_id: usize, - partition_id: usize, - ) { - let PollingConsumer::ConsumerGroup(group_id, member_id) = polling_consumer else { - return; - }; - - let completion_info = self.metadata.with_metadata(|m| { - let topic = m.streams.get(stream_id)?.topics.get(topic_id)?; - let group = topic.consumer_groups.get(group_id.0)?; - let member = group.members.get(member_id.0)?; - if !member - .pending_revocations - .iter() - .any(|revocation| revocation.partition_id == partition_id) - { - return None; - } - let partition = topic.partitions.get(partition_id)?; - let last_polled = { - let guard = partition.last_polled_offsets.pin(); - guard.get(group_id).map(|v| v.load(Ordering::Acquire)) - }; - let can_complete = match last_polled { - None => true, - Some(polled) => { - let guard = partition.consumer_group_offsets.pin(); - guard - .get(group_id) - .map(|co| co.offset.load(Ordering::Acquire)) - .is_some_and(|c| c >= polled) - } - }; - if can_complete { Some(member.id) } else { None } - }); - - if let Some(logical_member_id) = completion_info { - let request = - ShardRequest::control_plane(ShardRequestPayload::CompletePartitionRevocation { - stream_id, - topic_id, - group_id: group_id.0, - member_slab_id: member_id.0, - member_id: logical_member_id, - partition_id, - timed_out: false, - }); - let _ = self.send_to_control_plane(request).await; - } - } - - async fn delete_all_files_in_dir(dir: &str) -> Result<(), IggyError> { - let entries = match std::fs::read_dir(dir) { - Ok(entries) => entries, - Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()), - Err(e) => { - return Err(IggyError::IoError(format!( - "Failed to read directory {dir}: {e}" - ))); - } - }; - for entry in entries.flatten() { - let path = entry.path(); - if path.is_file() { - crate::streaming::partitions::storage::delete_persisted_offset( - &path.to_string_lossy(), - ) - .await?; - } - } - Ok(()) - } -} diff --git a/core/server/src/shard/system/info.rs b/core/server/src/shard/system/info.rs deleted file mode 100644 index 4d7ba7e7bc..0000000000 --- a/core/server/src/shard/system/info.rs +++ /dev/null @@ -1,78 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use iggy_common::SemanticVersion; -use serde::{Deserialize, Serialize}; -use std::collections::hash_map::DefaultHasher; -use std::fmt::Display; -use std::hash::{Hash, Hasher}; - -#[derive(Debug, Serialize, Deserialize, Default)] -pub struct SystemInfo { - pub version: Version, - pub migrations: Vec, -} - -#[derive(Debug, Serialize, Deserialize, Default)] -pub struct Version { - pub version: String, - pub hash: String, -} - -#[derive(Debug, Serialize, Deserialize, Default)] -pub struct Migration { - pub id: u32, - pub name: String, - pub hash: String, - pub applied_at: u64, -} - -impl SystemInfo { - pub fn update_version(&mut self, version: &SemanticVersion) { - self.version.version = version.to_string(); - let mut hasher = DefaultHasher::new(); - self.version.hash.hash(&mut hasher); - self.version.hash = hasher.finish().to_string(); - } -} - -impl Hash for SystemInfo { - fn hash(&self, state: &mut H) { - self.version.version.hash(state); - for migration in &self.migrations { - migration.hash(state); - } - } -} - -impl Hash for Migration { - fn hash(&self, state: &mut H) { - self.id.hash(state); - } -} - -impl Display for Version { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "version: {}", self.version) - } -} - -impl Display for SystemInfo { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "system info, {}", self.version) - } -} diff --git a/core/server/src/shard/system/messages.rs b/core/server/src/shard/system/messages.rs deleted file mode 100644 index 8770ac905c..0000000000 --- a/core/server/src/shard/system/messages.rs +++ /dev/null @@ -1,695 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use super::COMPONENT; -use crate::shard::IggyShard; -use crate::shard::transmission::frame::ShardResponse; -use crate::shard::transmission::message::{ - ResolvedPartition, ResolvedTopic, ShardRequest, ShardRequestPayload, -}; -use crate::streaming::partitions::journal::Journal; -use crate::streaming::polling_consumer::PollingConsumer; -use crate::streaming::segments::{IggyIndexesMut, IggyMessagesBatchMut, IggyMessagesBatchSet}; -use err_trail::ErrContext; -use iggy_common::IggyPollMetadata; -use iggy_common::{ - Consumer, EncryptorKind, IGGY_MESSAGE_HEADER_SIZE, Identifier, IggyError, PollingStrategy, -}; -use server_common::PooledBuffer; -use server_common::sharding::IggyNamespace; -use std::sync::atomic::Ordering; -use tracing::error; - -impl IggyShard { - /// Appends messages to partition. Permission must be checked by caller via - /// `resolve_topic_for_append()` before calling this method. - pub async fn append_messages( - &self, - partition: ResolvedPartition, - batch: IggyMessagesBatchMut, - ) -> Result<(), IggyError> { - if batch.count() == 0 { - return Ok(()); - } - - let namespace = IggyNamespace::new( - partition.stream_id, - partition.topic_id, - partition.partition_id, - ); - - let payload = ShardRequestPayload::SendMessages { batch }; - let request = ShardRequest::data_plane(namespace, payload); - - match self.send_to_data_plane(request).await? { - ShardResponse::SendMessages => Ok(()), - ShardResponse::ErrorResponse(err) => Err(err), - _ => unreachable!("Expected SendMessages response"), - } - } - - /// Polls messages from partition. Permission must be checked by caller via - /// `resolve_topic_for_poll()` before calling this method. - pub async fn poll_messages( - &self, - client_id: u32, - topic: ResolvedTopic, - consumer: Consumer, - maybe_partition_id: Option, - args: PollingArgs, - ) -> Result<(IggyPollMetadata, IggyMessagesBatchSet), IggyError> { - let Some((consumer, partition_id)) = self.resolve_consumer_with_partition_id( - topic, - &consumer, - client_id, - maybe_partition_id, - true, - )? - else { - return Ok((IggyPollMetadata::new(0, 0), IggyMessagesBatchSet::empty())); - }; - - let namespace = IggyNamespace::new(topic.stream_id, topic.topic_id, partition_id); - - let payload = ShardRequestPayload::PollMessages { consumer, args }; - let request = ShardRequest::data_plane(namespace, payload); - - let (metadata, batch) = match self.send_to_data_plane(request).await? { - ShardResponse::PollMessages(result) => result, - ShardResponse::ErrorResponse(err) => return Err(err), - _ => unreachable!("Expected PollMessages response"), - }; - - let batch = if let Some(encryptor) = &self.encryptor { - self.decrypt_messages(batch, encryptor).await? - } else { - batch - }; - - // Track last offset sent to CG member for cooperative rebalance. - if let PollingConsumer::ConsumerGroup(group_id, _) = &consumer - && let Some(last_offset) = batch.last_offset() - { - self.metadata.record_polled_offset( - topic.stream_id, - topic.topic_id, - group_id.0, - partition_id, - last_offset, - ); - } - - Ok((metadata, batch)) - } - - pub async fn flush_unsaved_buffer( - &self, - user_id: u32, - partition: ResolvedPartition, - fsync: bool, - ) -> Result<(), IggyError> { - self.metadata - .perm_append_messages(user_id, partition.stream_id, partition.topic_id) - .error(|e: &IggyError| { - format!("{COMPONENT} (error: {e}) - permission denied to flush unsaved buffer for user {} on stream ID: {}, topic ID: {}", user_id, partition.stream_id as u32, partition.topic_id as u32) - })?; - - let namespace = IggyNamespace::new( - partition.stream_id, - partition.topic_id, - partition.partition_id, - ); - let payload = ShardRequestPayload::FlushUnsavedBuffer { fsync }; - let request = ShardRequest::data_plane(namespace, payload); - - match self.send_to_data_plane(request).await? { - ShardResponse::FlushUnsavedBuffer { .. } => Ok(()), - ShardResponse::ErrorResponse(err) => Err(err), - _ => unreachable!("Expected FlushUnsavedBuffer response"), - } - } - - /// Flushes unsaved messages from the partition store to disk. - /// Returns the number of messages saved. - pub(crate) async fn flush_unsaved_buffer_from_local_partitions( - &self, - namespace: &IggyNamespace, - fsync: bool, - ) -> Result { - let frozen_batches = { - let mut partitions = self.local_partitions.borrow_mut(); - let Some(partition) = partitions.get_mut(namespace) else { - return Ok(0); - }; - if !partition.log.has_segments() || partition.log.journal().is_empty() { - return Ok(0); - } - let batches = partition.log.journal_mut().commit(); - partition.log.ensure_indexes(); - batches.append_indexes_to(partition.log.active_indexes_mut().unwrap()); - - let frozen: Vec<_> = batches - .into_inner() - .into_iter() - .map(|mut b| b.freeze()) - .collect(); - partition.log.set_in_flight(frozen.clone()); - frozen - }; - - let saved_count = self - .persist_frozen_batches_to_disk(namespace, frozen_batches) - .await?; - - if fsync { - self.fsync_all_messages_from_local_partitions(namespace) - .await?; - } - - Ok(saved_count) - } - - pub(crate) async fn fsync_all_messages_from_local_partitions( - &self, - namespace: &IggyNamespace, - ) -> Result<(), IggyError> { - let storage = { - let partitions = self.local_partitions.borrow(); - let Some(partition) = partitions.get(namespace) else { - return Ok(()); - }; - if !partition.log.has_segments() { - return Ok(()); - } - partition.log.active_storage().clone() - }; - - if storage.messages_writer.is_none() || storage.index_writer.is_none() { - return Ok(()); - } - - if let Some(ref messages_writer) = storage.messages_writer - && let Err(e) = messages_writer.fsync().await - { - tracing::error!( - "Failed to fsync messages writer for partition {:?}: {}", - namespace, - e - ); - return Err(e); - } - - if let Some(ref index_writer) = storage.index_writer - && let Err(e) = index_writer.fsync().await - { - tracing::error!( - "Failed to fsync index writer for partition {:?}: {}", - namespace, - e - ); - return Err(e); - } - - Ok(()) - } - - pub(crate) async fn auto_commit_consumer_offset_from_local_partition( - &self, - namespace: &IggyNamespace, - consumer: PollingConsumer, - offset: u64, - ) -> Result<(), IggyError> { - let (offset_value, path) = { - let partitions = self.local_partitions.borrow(); - let partition = partitions.get(namespace).ok_or_else(|| { - IggyError::PartitionNotFound( - namespace.partition_id(), - Identifier::numeric(namespace.topic_id() as u32).unwrap(), - Identifier::numeric(namespace.stream_id() as u32).unwrap(), - ) - })?; - - match consumer { - PollingConsumer::Consumer(consumer_id, _) => { - tracing::trace!( - "Auto-committing offset {} for consumer {} on partition {:?}", - offset, - consumer_id, - namespace - ); - let hdl = partition.consumer_offsets.pin(); - let item = hdl.get_or_insert( - consumer_id, - crate::streaming::partitions::consumer_offset::ConsumerOffset::default_for_consumer( - consumer_id as u32, - &self.config.system.get_consumer_offsets_path( - namespace.stream_id(), - namespace.topic_id(), - namespace.partition_id(), - ), - ), - ); - item.offset.store(offset, Ordering::Release); - (item.offset.load(Ordering::Relaxed), item.path.clone()) - } - PollingConsumer::ConsumerGroup(consumer_group_id, _) => { - tracing::trace!( - "Auto-committing offset {} for consumer group {} on partition {:?}", - offset, - consumer_group_id.0, - namespace - ); - let hdl = partition.consumer_group_offsets.pin(); - let item = hdl.get_or_insert( - consumer_group_id, - crate::streaming::partitions::consumer_offset::ConsumerOffset::default_for_consumer_group( - consumer_group_id, - &self.config.system.get_consumer_group_offsets_path( - namespace.stream_id(), - namespace.topic_id(), - namespace.partition_id(), - ), - ), - ); - item.offset.store(offset, Ordering::Release); - (item.offset.load(Ordering::Relaxed), item.path.clone()) - } - } - }; - - crate::streaming::partitions::storage::persist_offset(&path, offset_value).await?; - - self.maybe_complete_pending_revocation( - &consumer, - namespace.stream_id(), - namespace.topic_id(), - namespace.partition_id(), - ) - .await; - - Ok(()) - } - - /// Appends a batch to the active segment, flushing to disk and rotating if needed. - /// - /// Safety: called exclusively from the message pump — segment indices captured before - /// internal `.await` points (prepare_for_persistence, persist, rotate) remain valid - /// because no other handler can modify the segment vec while this frame is in progress. - pub(crate) async fn append_messages_to_local_partition( - &self, - namespace: &IggyNamespace, - mut batch: IggyMessagesBatchMut, - config: &crate::configs::system::SystemConfig, - ) -> Result<(), IggyError> { - let ( - current_offset, - current_position, - segment_start_offset, - segment_index, - message_deduplicator, - ) = { - let partitions = self.local_partitions.borrow(); - let partition = partitions - .get(namespace) - .expect("local_partitions: partition must exist"); - - let current_offset = if partition.should_increment_offset { - partition.offset.load(Ordering::Relaxed) + 1 - } else { - 0 - }; - - let segment = partition.log.active_segment(); - let segment_index = partition.log.segments().len() - 1; - - ( - current_offset, - segment.current_position, - segment.start_offset, - segment_index, - partition.message_deduplicator.clone(), - ) - }; - - batch - .prepare_for_persistence( - segment_start_offset, - current_offset, - current_position, - message_deduplicator.as_ref(), - ) - .await; - - let (journal_messages_count, journal_size, is_full) = { - let mut partitions = self.local_partitions.borrow_mut(); - let partition = partitions - .get_mut(namespace) - .expect("local_partitions: partition must exist"); - - let segment = &mut partition.log.segments_mut()[segment_index]; - - if segment.start_timestamp == 0 { - segment.start_timestamp = batch.first_timestamp().unwrap(); - } - - let batch_messages_size = batch.size(); - let batch_messages_count = batch.count(); - - partition - .stats - .increment_size_bytes(batch_messages_size as u64); - partition - .stats - .increment_messages_count(batch_messages_count as u64); - - segment.end_timestamp = batch.last_timestamp().unwrap(); - segment.end_offset = batch.last_offset().unwrap(); - - let (journal_messages_count, journal_size) = - partition.log.journal_mut().append(batch)?; - - let last_offset = if batch_messages_count == 0 { - current_offset - } else { - current_offset + batch_messages_count as u64 - 1 - }; - - if partition.should_increment_offset { - partition.offset.store(last_offset, Ordering::Relaxed); - } else { - partition.should_increment_offset = true; - partition.offset.store(last_offset, Ordering::Relaxed); - } - partition.stats.set_current_offset(last_offset); - partition.log.segments_mut()[segment_index].current_position += batch_messages_size; - - let is_full = partition.log.segments()[segment_index].is_full(); - - (journal_messages_count, journal_size, is_full) - }; - - let unsaved_messages_count_exceeded = - journal_messages_count >= config.partition.messages_required_to_save; - let unsaved_messages_size_exceeded = journal_size - >= config - .partition - .size_of_messages_required_to_save - .as_bytes_u64() as u32; - - if is_full || unsaved_messages_count_exceeded || unsaved_messages_size_exceeded { - let frozen_batches = { - let mut partitions = self.local_partitions.borrow_mut(); - let partition = partitions - .get_mut(namespace) - .expect("local_partitions: partition must exist"); - let batches = partition.log.journal_mut().commit(); - partition.log.ensure_indexes(); - batches.append_indexes_to(partition.log.active_indexes_mut().unwrap()); - - let frozen: Vec<_> = batches - .into_inner() - .into_iter() - .map(|mut b| b.freeze()) - .collect(); - partition.log.set_in_flight(frozen.clone()); - frozen - }; - - self.persist_frozen_batches_to_disk(namespace, frozen_batches) - .await?; - - if is_full { - self.rotate_segment_in_local_partitions(namespace).await?; - } - } - - Ok(()) - } - - /// Persists already-frozen batches to disk. Caller must have set in_flight buffer. - async fn persist_frozen_batches_to_disk( - &self, - namespace: &IggyNamespace, - frozen_batches: Vec, - ) -> Result { - let batch_count: u32 = frozen_batches.iter().map(|b| b.count()).sum(); - - if batch_count == 0 { - return Ok(0); - } - - let (messages_writer, index_writer) = { - let partitions = self.local_partitions.borrow(); - let partition = partitions - .get(namespace) - .expect("local_partitions: partition must exist"); - - if !partition.log.has_segments() { - return Ok(0); - } - - let messages_writer = partition - .log - .active_storage() - .messages_writer - .as_ref() - .expect("Messages writer not initialized") - .clone(); - let index_writer = partition - .log - .active_storage() - .index_writer - .as_ref() - .expect("Index writer not initialized") - .clone(); - (messages_writer, index_writer) - }; - - let saved = messages_writer - .as_ref() - .save_frozen_batches(&frozen_batches) - .await?; - - let unsaved_indexes_slice = { - let partitions = self.local_partitions.borrow(); - let partition = partitions - .get(namespace) - .expect("local_partitions: partition must exist"); - let segment_index = partition.log.segments().len() - 1; - partition.log.indexes()[segment_index] - .as_ref() - .expect("indexes must exist for segment being persisted") - .unsaved_slice() - }; - - index_writer - .as_ref() - .save_indexes(unsaved_indexes_slice) - .await?; - - tracing::trace!( - "Persisted {} messages on disk for partition: {:?}, total bytes written: {}.", - batch_count, - namespace, - saved - ); - - { - let mut partitions = self.local_partitions.borrow_mut(); - let partition = partitions - .get_mut(namespace) - .expect("local_partitions: partition must exist"); - - let segment_index = partition.log.segments().len() - 1; - let indexes = partition.log.indexes_mut()[segment_index] - .as_mut() - .expect("indexes must exist for segment being persisted"); - indexes.mark_saved(); - - let segment = &mut partition.log.segments_mut()[segment_index]; - segment.size = - iggy_common::IggyByteSize::from(segment.size.as_bytes_u64() + saved.as_bytes_u64()); - - partition.log.clear_in_flight(); - } - - Ok(batch_count) - } - - pub(crate) async fn poll_messages_from_local_partition( - &self, - namespace: &IggyNamespace, - consumer: crate::streaming::polling_consumer::PollingConsumer, - args: PollingArgs, - ) -> Result<(IggyPollMetadata, IggyMessagesBatchSet), IggyError> { - crate::streaming::partitions::ops::poll_messages( - &self.local_partitions, - namespace, - consumer, - args, - ) - .await - } - - async fn decrypt_messages( - &self, - batches: IggyMessagesBatchSet, - encryptor: &EncryptorKind, - ) -> Result { - let mut decrypted_batches = Vec::with_capacity(batches.containers_count()); - for batch in batches.iter() { - let mut indexes = IggyIndexesMut::with_capacity(batch.count() as usize, 0); - let mut decrypted_messages = PooledBuffer::with_capacity(batch.size() as usize); - let mut position = 0; - - for message in batch.iter() { - let mut header = message.header().to_header(); - let offset = header.offset; - let payload = encryptor.decrypt(message.payload()); - match payload { - Ok(payload) => { - // Update the header with the decrypted payload length - header.payload_length = payload.len() as u32; - - // Decrypt user headers if present - let decrypted_user_headers = if let Some(user_headers) = - message.user_headers() - { - match encryptor.decrypt(user_headers) { - Ok(decrypted) => { - header.user_headers_length = decrypted.len() as u32; - Some(decrypted) - } - Err(error) => { - error!( - "Cannot decrypt the message user headers at offset: {offset}. Error: {error}" - ); - continue; - } - } - } else { - None - }; - - decrypted_messages.extend_from_slice(&header.to_bytes()); - decrypted_messages.extend_from_slice(&payload); - if let Some(ref user_headers) = decrypted_user_headers { - decrypted_messages.extend_from_slice(user_headers); - } - position += IGGY_MESSAGE_HEADER_SIZE - + payload.len() - + header.user_headers_length as usize; - indexes.insert(0, position as u32, 0); - } - Err(error) => { - error!("Cannot decrypt the message at offset: {offset}. Error: {error}",); - continue; - } - } - } - let decrypted_batch = - IggyMessagesBatchMut::from_indexes_and_messages(indexes, decrypted_messages); - decrypted_batches.push(decrypted_batch); - } - - Ok(IggyMessagesBatchSet::from_vec(decrypted_batches)) - } - - pub fn maybe_encrypt_messages( - &self, - batch: IggyMessagesBatchMut, - ) -> Result { - let encryptor = match self.encryptor.as_ref() { - Some(encryptor) => encryptor, - None => return Ok(batch), - }; - let mut encrypted_messages = PooledBuffer::with_capacity(batch.size() as usize * 2); - let mut indexes = IggyIndexesMut::with_capacity(batch.count() as usize, 0); - let mut position = 0; - - for message in batch.iter() { - let header = message.header().to_header(); - let offset = header.offset; - let payload_bytes = message.payload(); - let user_headers_bytes = message.user_headers(); - - let encrypted_payload = encryptor.encrypt(payload_bytes); - match encrypted_payload { - Ok(encrypted_payload) => { - let mut updated_header = header; - updated_header.payload_length = encrypted_payload.len() as u32; - - // Encrypt user headers if present - let encrypted_user_headers = if let Some(user_headers_bytes) = - user_headers_bytes - { - match encryptor.encrypt(user_headers_bytes) { - Ok(encrypted) => { - updated_header.user_headers_length = encrypted.len() as u32; - Some(encrypted) - } - Err(error) => { - error!( - "Cannot encrypt the message user headers at offset: {offset}. Error: {error}" - ); - continue; - } - } - } else { - None - }; - - encrypted_messages.extend_from_slice(&updated_header.to_bytes()); - encrypted_messages.extend_from_slice(&encrypted_payload); - if let Some(ref encrypted_user_headers) = encrypted_user_headers { - encrypted_messages.extend_from_slice(encrypted_user_headers); - } - position += IGGY_MESSAGE_HEADER_SIZE - + encrypted_payload.len() - + updated_header.user_headers_length as usize; - indexes.insert(0, position as u32, 0); - } - Err(error) => { - error!("Cannot encrypt the message at offset: {offset}. Error: {error}",); - continue; - } - } - } - - Ok(IggyMessagesBatchMut::from_indexes_and_messages( - indexes, - encrypted_messages, - )) - } -} - -#[derive(Debug)] -pub struct PollingArgs { - pub strategy: PollingStrategy, - pub count: u32, - pub auto_commit: bool, -} - -impl PollingArgs { - pub fn new(strategy: PollingStrategy, count: u32, auto_commit: bool) -> Self { - Self { - strategy, - count, - auto_commit, - } - } -} diff --git a/core/server/src/shard/system/mod.rs b/core/server/src/shard/system/mod.rs deleted file mode 100644 index f326ea08e4..0000000000 --- a/core/server/src/shard/system/mod.rs +++ /dev/null @@ -1,35 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod clients; -pub mod cluster; -pub mod consumer_groups; -pub mod consumer_offsets; -pub mod info; -pub mod messages; -pub mod partitions; -pub mod personal_access_tokens; -pub mod segments; -pub mod snapshot; -pub mod stats; -pub mod storage; -pub mod streams; -pub mod topics; -pub mod users; -pub mod utils; - -pub const COMPONENT: &str = "SHARD_SYSTEM"; diff --git a/core/server/src/shard/system/partitions.rs b/core/server/src/shard/system/partitions.rs deleted file mode 100644 index 519601bc1e..0000000000 --- a/core/server/src/shard/system/partitions.rs +++ /dev/null @@ -1,443 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::metadata::PartitionMeta; -use crate::shard::IggyShard; -use crate::shard::calculate_shard_assignment; -use crate::shard::transmission::event::PartitionInfo; -use crate::shard::transmission::message::ResolvedTopic; -use crate::streaming::partitions::consumer_group_offsets::ConsumerGroupOffsets; -use crate::streaming::partitions::consumer_offsets::ConsumerOffsets; -use crate::streaming::partitions::local_partition::LocalPartition; -use crate::streaming::partitions::storage::create_partition_file_hierarchy; -use crate::streaming::partitions::storage::delete_partitions_from_disk; -use crate::streaming::segments::Segment; -use crate::streaming::segments::storage::create_segment_storage; -use crate::streaming::stats::PartitionStats; -use iggy_common::Identifier; -use iggy_common::IggyError; -use iggy_common::IggyTimestamp; -use server_common::sharding::IggyNamespace; -use server_common::sharding::{PartitionLocation, ShardId}; -use std::sync::Arc; -use std::time::Duration; -use tracing::{info, warn}; - -const PARTITION_INIT_BASE_INTERVAL: Duration = Duration::from_micros(100); -const PARTITION_INIT_MAX_INTERVAL: Duration = Duration::from_millis(50); -const PARTITION_INIT_TIMEOUT: Duration = Duration::from_secs(5); - -impl IggyShard { - pub async fn create_partitions( - &self, - topic: ResolvedTopic, - partitions_count: u32, - ) -> Result, IggyError> { - let stream = topic.stream_id; - let topic_id = topic.topic_id; - - let created_at = IggyTimestamp::now(); - let shards_count = self.get_available_shards_count(); - - let parent_stats = self - .metadata - .get_topic_stats(stream, topic_id) - .expect("Parent topic stats must exist"); - - let count_before = self - .metadata - .get_partitions_count(stream, topic_id) - .unwrap_or(0); - let partition_ids: Vec = - (count_before..count_before + partitions_count as usize).collect(); - let partition_infos: Vec = partition_ids - .iter() - .map(|&id| PartitionInfo { id, created_at }) - .collect(); - - for info in &partition_infos { - create_partition_file_hierarchy(stream, topic_id, info.id, &self.config.system).await?; - } - - let metas: Vec = (0..partitions_count) - .map(|_| PartitionMeta { - id: 0, - created_at, - revision_id: 0, - stats: Arc::new(PartitionStats::new(parent_stats.clone())), - consumer_offsets: Arc::new(ConsumerOffsets::with_capacity(0)), - consumer_group_offsets: Arc::new(ConsumerGroupOffsets::with_capacity(0)), - last_polled_offsets: Arc::new(papaya::HashMap::new()), - }) - .collect(); - - let assigned_ids = self - .writer() - .add_partitions(&self.metadata, stream, topic_id, metas); - debug_assert_eq!( - assigned_ids, partition_ids, - "Partition IDs mismatch: expected {:?}, got {:?}", - partition_ids, assigned_ids - ); - - self.metrics.increment_partitions(partitions_count); - self.metrics.increment_segments(partitions_count); - - for info in &partition_infos { - let partition_id = info.id; - let ns = IggyNamespace::new(stream, topic_id, partition_id); - let shard_id = ShardId::new(calculate_shard_assignment(&ns, shards_count)); - let is_current_shard = self.id == *shard_id; - // epoch is reconciler-only; unused by legacy server. - let location = PartitionLocation::new(shard_id, 0); - self.insert_shard_table_record(ns, location); - - if is_current_shard { - self.ensure_partition(&ns).await?; - } - } - Ok(partition_infos) - } - - /// Ensures partition is initialized in local_partitions. Idempotent. - /// Returns error if partition doesn't exist in metadata. - /// If another task is already initializing the partition, waits for it to complete. - pub async fn ensure_partition(&self, ns: &IggyNamespace) -> Result<(), IggyError> { - use std::time::Instant; - - let deadline = Instant::now() + PARTITION_INIT_TIMEOUT; - let mut backoff = PARTITION_INIT_BASE_INTERVAL; - - loop { - // partition_needs_init handles both fresh and stale entries: - // - Returns None if fresh entry exists (same revision_id) - // - Removes stale entry and returns Some if revision_id differs - // - Returns Some if no entry exists - let Some(created_at) = self.partition_needs_init(ns)? else { - return Ok(()); - }; - - if Instant::now() >= deadline { - warn!( - "Partition initialization timed out after {:?} for stream: {}, topic: {}, partition: {}", - PARTITION_INIT_TIMEOUT, - ns.stream_id(), - ns.topic_id(), - ns.partition_id() - ); - return Err(IggyError::TaskTimeout); - } - - let is_pending = self.pending_partition_inits.borrow().contains(ns); - if is_pending { - compio::time::sleep(backoff).await; - backoff = (backoff * 2).min(PARTITION_INIT_MAX_INTERVAL); - continue; - } - - // Double-check after potential yield - another task may have claimed it - { - let mut pending = self.pending_partition_inits.borrow_mut(); - if pending.contains(ns) { - continue; - } - pending.insert(*ns); - } - - let result = self.init_partition_inner(ns, created_at).await; - self.pending_partition_inits.borrow_mut().remove(ns); - return result; - } - } - - /// Returns `Ok(Some(timestamp))` if partition needs initialization, - /// `Ok(None)` if already initialized, or `Err` if partition doesn't exist in metadata. - fn partition_needs_init(&self, ns: &IggyNamespace) -> Result, IggyError> { - let init_info = - self.metadata - .get_partition_init_info(ns.stream_id(), ns.topic_id(), ns.partition_id()); - - let revision_id = init_info.as_ref().map(|m| m.revision_id); - - let needs_init = { - let partitions = self.local_partitions.borrow(); - match (partitions.get(ns), revision_id) { - (Some(data), Some(rev)) if data.revision_id == rev => false, - (Some(_), _) => { - drop(partitions); - self.local_partitions.borrow_mut().remove(ns); - true - } - (None, _) => true, - } - }; - - if needs_init { - let created_at = init_info.map(|m| m.created_at).ok_or_else(|| { - IggyError::PartitionNotFound( - ns.partition_id(), - Identifier::numeric(ns.topic_id() as u32).unwrap(), - Identifier::numeric(ns.stream_id() as u32).unwrap(), - ) - })?; - Ok(Some(created_at)) - } else { - Ok(None) - } - } - - async fn init_partition_inner( - &self, - ns: &IggyNamespace, - created_at: IggyTimestamp, - ) -> Result<(), IggyError> { - let stream_id = ns.stream_id(); - let topic_id = ns.topic_id(); - let partition_id = ns.partition_id(); - - info!( - "Initializing partition in local_partitions: partition ID: {} for topic ID: {} for stream ID: {}", - partition_id, topic_id, stream_id - ); - - let stats = self - .metadata - .get_partition_stats_by_ids(stream_id, topic_id, partition_id) - .expect("Partition stats must exist in SharedMetadata"); - - let partition_path = - self.config - .system - .get_partition_path(stream_id, topic_id, partition_id); - - let mut loaded_log = crate::bootstrap::load_segments( - &self.config.system, - stream_id, - topic_id, - partition_id, - partition_path, - stats.clone(), - ) - .await?; - - if !loaded_log.has_segments() { - info!( - "No segments found on disk for partition ID: {} for topic ID: {} for stream ID: {}, creating initial segment", - partition_id, topic_id, stream_id - ); - - let start_offset = 0; - let segment = Segment::new(start_offset, self.config.system.segment.size); - - let storage = create_segment_storage( - &self.config.system, - stream_id, - topic_id, - partition_id, - 0, - 0, - start_offset, - ) - .await?; - - loaded_log.add_persisted_segment(segment, storage); - stats.increment_segments_count(1); - } - - // Use the max end_offset across segments that have data, not just - // the active segment. Mirrors the fix in shard/mod.rs bootstrap. - let current_offset = loaded_log - .segments() - .iter() - .filter(|s| s.size > iggy_common::IggyByteSize::default()) - .map(|s| s.end_offset) - .max() - .unwrap_or(0); - - let should_increment_offset = loaded_log - .segments() - .iter() - .any(|s| s.size > iggy_common::IggyByteSize::default()); - - // Initialize journal base_offset so three-tier routing works correctly. - if should_increment_offset { - use crate::streaming::partitions::journal::{Inner, Journal}; - loaded_log.journal_mut().init(Inner { - base_offset: current_offset + 1, - ..Default::default() - }); - } - - let (revision_id, consumer_offsets, consumer_group_offsets) = self - .metadata - .get_partition_init_info(stream_id, topic_id, partition_id) - .map(|info| { - ( - info.revision_id, - info.consumer_offsets, - info.consumer_group_offsets, - ) - }) - .unwrap_or_else(|| { - ( - 0, - Arc::new(ConsumerOffsets::with_capacity(0)), - Arc::new(ConsumerGroupOffsets::with_capacity(0)), - ) - }); - - // Clamp consumer offsets that are ahead of partition offset (crash recovery). - { - let guard = consumer_offsets.pin(); - for entry in guard.iter() { - let stored = entry.1.offset.load(std::sync::atomic::Ordering::Relaxed); - if stored > current_offset { - tracing::warn!( - "Consumer {} offset {} ahead of partition offset {} \ - for stream {}, topic {}, partition {} - clamping \ - (lazy init recovery)", - entry.0, - stored, - current_offset, - stream_id, - topic_id, - partition_id - ); - entry - .1 - .offset - .store(current_offset, std::sync::atomic::Ordering::Relaxed); - } - } - } - { - let guard = consumer_group_offsets.pin(); - for entry in guard.iter() { - let stored = entry.1.offset.load(std::sync::atomic::Ordering::Relaxed); - if stored > current_offset { - tracing::warn!( - "Consumer group {:?} offset {} ahead of partition \ - offset {} for stream {}, topic {}, partition {} - \ - clamping (lazy init recovery)", - entry.0, - stored, - current_offset, - stream_id, - topic_id, - partition_id - ); - entry - .1 - .offset - .store(current_offset, std::sync::atomic::Ordering::Relaxed); - } - } - } - - let partition = LocalPartition::with_log( - loaded_log, - stats, - std::sync::Arc::new(std::sync::atomic::AtomicU64::new(current_offset)), - consumer_offsets, - consumer_group_offsets, - None, - created_at, - revision_id, - should_increment_offset, - ); - - self.local_partitions.borrow_mut().insert(*ns, partition); - - info!( - "Initialized partition in local_partitions: partition ID: {} for topic ID: {} for stream ID: {} with offset: {}", - partition_id, topic_id, stream_id, current_offset - ); - - Ok(()) - } - - pub async fn delete_partitions( - &self, - topic: ResolvedTopic, - partitions_count: u32, - ) -> Result, IggyError> { - let stream = topic.stream_id; - let topic_id = topic.topic_id; - - self.validate_partitions_count(topic, partitions_count)?; - - let all_partition_ids = self.metadata.get_partition_ids(stream, topic_id); - - let partitions_to_delete: Vec = all_partition_ids - .into_iter() - .rev() - .take(partitions_count as usize) - .collect(); - - let topic_stats = self.metadata.get_topic_stats(stream, topic_id); - - let mut total_messages_count: u64 = 0; - let mut total_segments_count: u32 = 0; - let mut total_size_bytes: u64 = 0; - - for partition_id in &partitions_to_delete { - if let Some(stats) = - self.metadata - .get_partition_stats_by_ids(stream, topic_id, *partition_id) - { - total_segments_count += stats.segments_count_inconsistent(); - total_messages_count += stats.messages_count_inconsistent(); - total_size_bytes += stats.size_bytes_inconsistent(); - } - } - - self.writer() - .delete_partitions(stream, topic_id, partitions_to_delete.len() as u32); - - for partition_id in &partitions_to_delete { - let ns = IggyNamespace::new(stream, topic_id, *partition_id); - self.remove_shard_table_record(&ns); - self.local_partitions.borrow_mut().remove(&ns); - } - - for partition_id in &partitions_to_delete { - self.delete_partition_dir(stream, topic_id, *partition_id) - .await?; - } - - self.metrics - .decrement_partitions(partitions_to_delete.len() as u32); - self.metrics.decrement_segments(total_segments_count); - - if let Some(parent) = topic_stats { - parent.decrement_messages_count(total_messages_count); - parent.decrement_size_bytes(total_size_bytes); - parent.decrement_segments_count(total_segments_count); - } - - Ok(partitions_to_delete) - } - - async fn delete_partition_dir( - &self, - stream_id: usize, - topic_id: usize, - partition_id: usize, - ) -> Result<(), IggyError> { - delete_partitions_from_disk(stream_id, topic_id, partition_id, &self.config.system).await - } -} diff --git a/core/server/src/shard/system/personal_access_tokens.rs b/core/server/src/shard/system/personal_access_tokens.rs deleted file mode 100644 index 8c69a8ad3b..0000000000 --- a/core/server/src/shard/system/personal_access_tokens.rs +++ /dev/null @@ -1,149 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use super::COMPONENT; -use crate::shard::IggyShard; -use crate::streaming::session::Session; -use crate::streaming::users::user::User; -use err_trail::ErrContext; -use iggy_common::IggyError; -use iggy_common::IggyExpiry; -use iggy_common::IggyTimestamp; -use iggy_common::PersonalAccessToken; -use tracing::{error, info}; - -impl IggyShard { - pub fn get_personal_access_tokens( - &self, - user_id: u32, - ) -> Result, IggyError> { - let _ = self.get_user(&user_id.try_into()?).error(|e: &IggyError| { - format!("{COMPONENT} (error: {e}) - failed to get user with id: {user_id}") - })?; - - info!("Loading personal access tokens for user with ID: {user_id}...",); - - let personal_access_tokens = self.metadata.get_user_personal_access_tokens(user_id); - - info!( - "Loaded {} personal access tokens for user with ID: {user_id}.", - personal_access_tokens.len(), - ); - Ok(personal_access_tokens) - } - - pub fn create_personal_access_token( - &self, - user_id: u32, - name: &str, - expiry: IggyExpiry, - ) -> Result<(PersonalAccessToken, String), IggyError> { - let _ = self.get_user(&user_id.try_into()?).error(|e: &IggyError| { - format!("{COMPONENT} (error: {e}) - failed to get user with id: {user_id}") - })?; - - let max_token_per_user = self.config.personal_access_token.max_tokens_per_user; - let current_count = self.metadata.user_pat_count(user_id); - if current_count as u32 >= max_token_per_user { - error!( - "User with ID: {user_id} has reached the maximum number of personal access tokens: {max_token_per_user}.", - ); - return Err(IggyError::PersonalAccessTokensLimitReached( - user_id, - max_token_per_user, - )); - } - - let (personal_access_token, token) = - PersonalAccessToken::new(user_id, name, IggyTimestamp::now(), expiry); - - let pat_name = personal_access_token.name.clone(); - if self.metadata.user_has_pat_with_name(user_id, &pat_name) { - error!("Personal access token: {pat_name} for user with ID: {user_id} already exists."); - return Err(IggyError::PersonalAccessTokenAlreadyExists( - pat_name.to_string(), - user_id, - )); - } - - self.writer() - .add_personal_access_token(user_id, personal_access_token.clone()); - info!("Created personal access token: {pat_name} for user with ID: {user_id}."); - - Ok((personal_access_token, token)) - } - - pub fn delete_personal_access_token(&self, user_id: u32, name: &str) -> Result<(), IggyError> { - let token_hash = - self.metadata - .find_pat_token_hash_by_name(user_id, name) - .ok_or_else(|| { - error!( - "Personal access token: {name} for user with ID: {user_id} does not exist.", - ); - IggyError::ResourceNotFound(name.to_owned()) - })?; - - info!("Deleting personal access token: {name} for user with ID: {user_id}..."); - self.writer() - .delete_personal_access_token(user_id, token_hash); - info!("Deleted personal access token: {name} for user with ID: {user_id}."); - Ok(()) - } - - pub fn login_with_personal_access_token( - &self, - token: &str, - session: Option<&Session>, - ) -> Result { - let token_hash = PersonalAccessToken::hash_token(token); - - let personal_access_token = self - .metadata - .get_personal_access_token_by_hash(&token_hash) - .ok_or_else(|| { - let redacted_token = if token.len() > 4 { - format!("{}****", &token[..4]) - } else { - "****".to_string() - }; - error!("Personal access token: {redacted_token} does not exist."); - IggyError::ResourceNotFound(token.to_owned()) - })?; - - if personal_access_token.is_expired(IggyTimestamp::now()) { - error!( - "Personal access token: {} for user with ID: {} has expired.", - personal_access_token.name, personal_access_token.user_id - ); - return Err(IggyError::PersonalAccessTokenExpired( - (*personal_access_token.name).to_owned(), - personal_access_token.user_id, - )); - } - - let user = self - .get_user(&personal_access_token.user_id.try_into()?) - .error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - failed to get user with id: {}", - personal_access_token.user_id - ) - })?; - self.login_user_with_credentials(&user.username, None, session) - } -} diff --git a/core/server/src/shard/system/segments.rs b/core/server/src/shard/system/segments.rs deleted file mode 100644 index fb48846a76..0000000000 --- a/core/server/src/shard/system/segments.rs +++ /dev/null @@ -1,586 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::configs::cache_indexes::CacheIndexesConfig; -use crate::shard::IggyShard; -use crate::streaming::segments::Segment; -use iggy_common::{ConsumerKind, IggyError, IggyExpiry, IggyTimestamp, MaxTopicSize}; -use server_common::sharding::IggyNamespace; - -impl IggyShard { - /// Performs all cleanup for a topic's partitions: time-based expiry then size-based trimming. - /// - /// Runs entirely inside the message pump's serialized loop — reads partition state and - /// deletes segments atomically with no TOCTOU window. - pub(crate) async fn clean_topic_messages( - &self, - stream_id: usize, - topic_id: usize, - partition_ids: &[usize], - ) -> Result<(u64, u64), IggyError> { - let (expiry, max_topic_size) = self - .metadata - .get_topic_config(stream_id, topic_id) - .unwrap_or(( - self.config.system.topic.message_expiry, - MaxTopicSize::Unlimited, - )); - - let mut total_segments = 0u64; - let mut total_messages = 0u64; - - // Phase 1: time-based expiry - if !matches!(expiry, IggyExpiry::NeverExpire) { - let now = IggyTimestamp::now(); - for &partition_id in partition_ids { - let (s, m) = self - .delete_expired_segments_for_partition( - stream_id, - topic_id, - partition_id, - now, - expiry, - ) - .await?; - total_segments += s; - total_messages += m; - } - } - - // Phase 2: size-based trimming - if !matches!(max_topic_size, MaxTopicSize::Unlimited) { - let max_bytes = max_topic_size.as_bytes_u64(); - let threshold = max_bytes * 9 / 10; - - loop { - let current_size = self - .metadata - .with_metadata(|m| { - m.streams - .get(stream_id) - .and_then(|s| s.topics.get(topic_id)) - .map(|t| t.stats.size_bytes_inconsistent()) - }) - .unwrap_or(0); - - if current_size < threshold { - break; - } - - let Some((target_partition_id, target_offset)) = - self.find_oldest_sealed_segment(stream_id, topic_id, partition_ids) - else { - break; - }; - - let (s, m) = self - .remove_segment_by_offset( - stream_id, - topic_id, - target_partition_id, - target_offset, - ) - .await?; - if s == 0 { - break; - } - total_segments += s; - total_messages += m; - } - } - - Ok((total_segments, total_messages)) - } - - /// Deletes all expired sealed segments from a single partition. - async fn delete_expired_segments_for_partition( - &self, - stream_id: usize, - topic_id: usize, - partition_id: usize, - now: IggyTimestamp, - expiry: IggyExpiry, - ) -> Result<(u64, u64), IggyError> { - let ns = IggyNamespace::new(stream_id, topic_id, partition_id); - - let expired_offsets: Vec = { - let partitions = self.local_partitions.borrow(); - let Some(partition) = partitions.get(&ns) else { - return Ok((0, 0)); - }; - - let min_committed = Self::min_committed_offset( - &partition.consumer_offsets, - &partition.consumer_group_offsets, - ); - - let segments = partition.log.segments(); - let last_idx = segments.len().saturating_sub(1); - let mut offsets = Vec::new(); - - for (idx, seg) in segments.iter().enumerate() { - if idx == last_idx || !seg.is_expired(now, expiry) { - continue; - } - if let Some((barrier, kind, id)) = &min_committed - && seg.end_offset > *barrier - { - tracing::warn!( - "Segment [{}..{}] blocked from expiry-based deletion \ - by {kind} (ID: {id}) at offset {barrier} \ - in partition {partition_id} (stream: {stream_id}, topic: {topic_id})", - seg.start_offset, - seg.end_offset, - ); - continue; - } - offsets.push(seg.start_offset); - } - - offsets - }; - - let mut total_segments = 0u64; - let mut total_messages = 0u64; - for offset in expired_offsets { - let (s, m) = self - .remove_segment_by_offset(stream_id, topic_id, partition_id, offset) - .await?; - total_segments += s; - total_messages += m; - } - Ok((total_segments, total_messages)) - } - - /// Finds the oldest sealed segment across the given partitions, comparing by timestamp. - /// Returns `(partition_id, start_offset)` or `None` if no deletable segments exist. - fn find_oldest_sealed_segment( - &self, - stream_id: usize, - topic_id: usize, - partition_ids: &[usize], - ) -> Option<(usize, u64)> { - let partitions = self.local_partitions.borrow(); - let mut oldest: Option<(usize, u64, u64)> = None; - - for &partition_id in partition_ids { - let ns = IggyNamespace::new(stream_id, topic_id, partition_id); - let Some(partition) = partitions.get(&ns) else { - continue; - }; - - let segments = partition.log.segments(); - if segments.len() <= 1 { - continue; - } - - let first = &segments[0]; - if !first.sealed { - continue; - } - - let min_committed = Self::min_committed_offset( - &partition.consumer_offsets, - &partition.consumer_group_offsets, - ); - if let Some((barrier, kind, id)) = &min_committed - && first.end_offset > *barrier - { - tracing::warn!( - "Segment [{}..{}] blocked from size-based deletion \ - by {kind} (ID: {id}) at offset {barrier} \ - in partition {partition_id} (stream: {stream_id}, topic: {topic_id})", - first.start_offset, - first.end_offset, - ); - continue; - } - - match &oldest { - None => oldest = Some((partition_id, first.start_offset, first.start_timestamp)), - Some((_, _, ts)) if first.start_timestamp < *ts => { - oldest = Some((partition_id, first.start_offset, first.start_timestamp)); - } - _ => {} - } - } - - oldest.map(|(pid, offset, _)| (pid, offset)) - } - - /// Removes a single segment identified by its start_offset from the given partition. - /// Skips if the segment no longer exists or is the active (last) segment. - async fn remove_segment_by_offset( - &self, - stream_id: usize, - topic_id: usize, - partition_id: usize, - start_offset: u64, - ) -> Result<(u64, u64), IggyError> { - let ns = IggyNamespace::new(stream_id, topic_id, partition_id); - - let removed = { - let mut partitions = self.local_partitions.borrow_mut(); - let Some(partition) = partitions.get_mut(&ns) else { - return Ok((0, 0)); - }; - - let log = &mut partition.log; - let last_idx = log.segments().len().saturating_sub(1); - - let Some(idx) = log - .segments() - .iter() - .position(|s| s.start_offset == start_offset) - else { - return Ok((0, 0)); - }; - - if idx == last_idx { - tracing::warn!( - "Refusing to delete active segment (start_offset: {start_offset}) \ - for partition ID: {partition_id}" - ); - return Ok((0, 0)); - } - - let segment = log.segments_mut().remove(idx); - let storage = log.storages_mut().remove(idx); - log.indexes_mut().remove(idx); - - Some((segment, storage, partition.stats.clone())) - }; - - let Some((segment, mut storage, stats)) = removed else { - return Ok((0, 0)); - }; - - let segment_size = segment.size.as_bytes_u64(); - let end_offset = segment.end_offset; - let messages_in_segment = if start_offset == end_offset { - 0 - } else { - (end_offset - start_offset) + 1 - }; - - let _ = storage.shutdown(); - let (messages_path, index_path) = storage.segment_and_index_paths(); - - if let Some(path) = messages_path - && let Err(e) = compio::fs::remove_file(&path).await - { - tracing::error!("Failed to delete messages file {}: {}", path, e); - } - - if let Some(path) = index_path - && let Err(e) = compio::fs::remove_file(&path).await - { - tracing::error!("Failed to delete index file {}: {}", path, e); - } - - stats.decrement_size_bytes(segment_size); - stats.decrement_segments_count(1); - stats.decrement_messages_count(messages_in_segment); - - tracing::info!( - "Deleted segment (start: {}, end: {}, size: {}, messages: {}) from partition {}", - start_offset, - end_offset, - segment_size, - messages_in_segment, - partition_id - ); - - Ok((1, messages_in_segment)) - } - - /// Deletes the N oldest **sealed** segments from a partition, preserving the active segment - /// and partition offset. Reuses `remove_segment_by_offset` - same logic as the message cleaner. - /// - /// Segments containing unconsumed messages are protected by a barrier: deletion is skipped - /// when `end_offset > min_committed_offset` and a warning is logged identifying the - /// blocking consumer. If no consumers exist, there is no barrier. - pub(crate) async fn delete_oldest_segments( - &self, - stream_id: usize, - topic_id: usize, - partition_id: usize, - segments_count: u32, - ) -> Result<(u64, u64), IggyError> { - let ns = IggyNamespace::new(stream_id, topic_id, partition_id); - - let sealed_offsets: Vec = { - let partitions = self.local_partitions.borrow(); - let Some(partition) = partitions.get(&ns) else { - return Ok((0, 0)); - }; - - let min_committed = Self::min_committed_offset( - &partition.consumer_offsets, - &partition.consumer_group_offsets, - ); - - let segments = partition.log.segments(); - let last_idx = segments.len().saturating_sub(1); - let mut offsets = Vec::new(); - let mut collected = 0u32; - - for (idx, seg) in segments.iter().enumerate() { - if collected >= segments_count { - break; - } - if idx == last_idx || !seg.sealed { - continue; - } - if let Some((barrier, kind, id)) = &min_committed - && seg.end_offset > *barrier - { - tracing::warn!( - "Segment [{}..{}] blocked from size-based deletion \ - by {kind} (ID: {id}) at offset {barrier} \ - in partition {partition_id} (stream: {stream_id}, topic: {topic_id})", - seg.start_offset, - seg.end_offset, - ); - continue; - } - offsets.push(seg.start_offset); - collected += 1; - } - - offsets - }; - - let mut total_segments = 0u64; - let mut total_messages = 0u64; - for offset in sealed_offsets { - let (s, m) = self - .remove_segment_by_offset(stream_id, topic_id, partition_id, offset) - .await?; - total_segments += s; - total_messages += m; - } - Ok((total_segments, total_messages)) - } - - /// Returns the minimum committed offset across all consumers and consumer groups, - /// along with the identity of the consumer holding it. Returns `None` if no - /// consumers exist (no barrier). - fn min_committed_offset( - consumer_offsets: &crate::streaming::partitions::consumer_offsets::ConsumerOffsets, - consumer_group_offsets: &crate::streaming::partitions::consumer_group_offsets::ConsumerGroupOffsets, - ) -> Option<(u64, ConsumerKind, u32)> { - let co_guard = consumer_offsets.pin(); - let cg_guard = consumer_group_offsets.pin(); - let consumers = co_guard.iter().map(|(_, co)| { - ( - co.offset.load(std::sync::atomic::Ordering::Relaxed), - co.kind, - co.consumer_id, - ) - }); - let groups = cg_guard.iter().map(|(_, co)| { - ( - co.offset.load(std::sync::atomic::Ordering::Relaxed), - co.kind, - co.consumer_id, - ) - }); - consumers.chain(groups).min_by_key(|(offset, _, _)| *offset) - } - - /// Drains all segments, deletes their files, and re-initializes the partition log at offset 0. - /// Used exclusively by `purge_topic_inner` — a destructive full reset. - pub(crate) async fn purge_all_segments( - &self, - stream_id: usize, - topic_id: usize, - partition_id: usize, - ) -> Result<(), IggyError> { - let namespace = IggyNamespace::new(stream_id, topic_id, partition_id); - - // Drain segments from local_partitions - let (segments, storages, stats) = { - let mut partitions = self.local_partitions.borrow_mut(); - let partition = partitions - .get_mut(&namespace) - .expect("purge_all_segments: partition must exist in local_partitions"); - - let upperbound = partition.log.segments().len(); - let segments = partition - .log - .segments_mut() - .drain(..upperbound) - .collect::>(); - let storages = partition - .log - .storages_mut() - .drain(..upperbound) - .collect::>(); - let _ = partition - .log - .indexes_mut() - .drain(..upperbound) - .collect::>(); - (segments, storages, partition.stats.clone()) - }; - - for (mut storage, segment) in storages.into_iter().zip(segments) { - let (msg_writer, index_writer) = storage.shutdown(); - let start_offset = segment.start_offset; - - let log_path = if let Some(msg_writer) = msg_writer { - let path = msg_writer.path(); - drop(msg_writer); - path - } else { - self.config.system.get_messages_file_path( - stream_id, - topic_id, - partition_id, - start_offset, - ) - }; - drop(index_writer); - - let index_path = - self.config - .system - .get_index_path(stream_id, topic_id, partition_id, start_offset); - - for path in [&log_path, &index_path] { - match compio::fs::remove_file(path).await { - Ok(()) => {} - Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - tracing::debug!("File already gone at path: {path}"); - } - Err(e) => { - tracing::error!("Failed to delete file at path: {path}, err: {e}"); - return Err(IggyError::CannotDeleteFile); - } - } - } - } - - self.init_log_in_local_partitions(&namespace).await?; - stats.increment_segments_count(1); - Ok(()) - } - - /// Creates a fresh segment at offset 0 after all segments have been drained. - /// - /// The log is momentarily empty between `delete_segments`' drain and this call, which is - /// safe because the message pump serializes all handlers — no concurrent operation can - /// observe the empty state. - async fn init_log_in_local_partitions( - &self, - namespace: &IggyNamespace, - ) -> Result<(), IggyError> { - use crate::streaming::segments::storage::create_segment_storage; - - let start_offset = 0; - let segment = Segment::new(start_offset, self.config.system.segment.size); - - let storage = create_segment_storage( - &self.config.system, - namespace.stream_id(), - namespace.topic_id(), - namespace.partition_id(), - 0, // messages_size - 0, // indexes_size - start_offset, - ) - .await?; - - let mut partitions = self.local_partitions.borrow_mut(); - if let Some(partition) = partitions.get_mut(namespace) { - partition.log.add_persisted_segment(segment, storage); - // Reset offset when starting fresh with a new segment at offset 0 - partition - .offset - .store(start_offset, std::sync::atomic::Ordering::SeqCst); - partition.should_increment_offset = false; - } - Ok(()) - } - - /// Rotate to a new segment when the current segment is full. - /// The new segment starts at the next offset after the current segment's end. - /// Seals the old segment so it becomes eligible for expiry-based cleanup. - /// - /// Safety: called exclusively from the message pump (via append handler) — the captured - /// `old_segment_index` remains valid across the `create_segment_storage` await because - /// no other handler can modify the segment vec while this frame is in progress. - pub(crate) async fn rotate_segment_in_local_partitions( - &self, - namespace: &IggyNamespace, - ) -> Result<(), IggyError> { - use crate::streaming::segments::storage::create_segment_storage; - - let (start_offset, old_segment_index) = { - let mut partitions = self.local_partitions.borrow_mut(); - let partition = partitions - .get_mut(namespace) - .expect("rotate_segment: partition must exist"); - let old_segment_index = partition.log.segments().len() - 1; - let active_segment = partition.log.active_segment_mut(); - active_segment.sealed = true; - (active_segment.end_offset + 1, old_segment_index) - }; - - let segment = Segment::new(start_offset, self.config.system.segment.size); - - let storage = create_segment_storage( - &self.config.system, - namespace.stream_id(), - namespace.topic_id(), - namespace.partition_id(), - 0, // messages_size - 0, // indexes_size - start_offset, - ) - .await?; - - let mut partitions = self.local_partitions.borrow_mut(); - if let Some(partition) = partitions.get_mut(namespace) { - // Clear old segment's indexes if cache_indexes is not set to All. - // This prevents memory accumulation from keeping index buffers for sealed segments. - if !matches!( - self.config.system.segment.cache_indexes, - CacheIndexesConfig::All - ) { - partition.log.indexes_mut()[old_segment_index] = None; - } - - // Close writers for the sealed segment - they're never needed after sealing. - // This releases file handles and associated kernel/io_uring resources. - let old_storage = &mut partition.log.storages_mut()[old_segment_index]; - let _ = old_storage.shutdown(); - - partition.log.add_persisted_segment(segment, storage); - partition.stats.increment_segments_count(1); - tracing::info!( - "Rotated to new segment at offset {} for partition {} (stream {}, topic {})", - start_offset, - namespace.partition_id(), - namespace.stream_id(), - namespace.topic_id() - ); - } - Ok(()) - } -} diff --git a/core/server/src/shard/system/snapshot/mod.rs b/core/server/src/shard/system/snapshot/mod.rs deleted file mode 100644 index e3c802f939..0000000000 --- a/core/server/src/shard/system/snapshot/mod.rs +++ /dev/null @@ -1,257 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -mod procdump; - -use crate::configs::system::SystemConfig; -use crate::shard::IggyShard; -use async_zip::base::write::ZipFileWriter; -use async_zip::{Compression, ZipEntryBuilder}; -use compio::fs::OpenOptions; -use compio::io::AsyncWriteAtExt; -use iggy_common::{IggyDuration, IggyError, Snapshot, SnapshotCompression, SystemSnapshotType}; -use std::path::PathBuf; -use std::time::Instant; -use tempfile::NamedTempFile; -use tracing::{error, info}; - -// NOTE(hubcio): compio has a `process` module, but it currently blocks the executor when the runtime -// has thread_pool_limit(0) configured (which we do on non-macOS platforms in bootstrap.rs). -// To use compio::process::Command, we need to either: -// 1. Enable thread pool by removing/increasing thread_pool_limit(0) -// 2. Use std::process::Command with compio::runtime::spawn_blocking (requires thread pool) -// 3. Find alternative approach that doesn't rely on thread pool -// See: https://compio.rs/docs/compio/process and bootstrap::create_shard_executor -use std::process::Command; - -impl IggyShard { - pub async fn get_snapshot( - &self, - compression: SnapshotCompression, - snapshot_types: &Vec, - ) -> Result { - let snapshot_types = if snapshot_types.contains(&SystemSnapshotType::All) { - if snapshot_types.len() > 1 { - error!("When using 'All' snapshot type, no other types can be specified"); - return Err(IggyError::InvalidCommand); - } - &SystemSnapshotType::all_snapshot_types() - } else { - snapshot_types - }; - - let mut zip_writer = ZipFileWriter::new(Vec::new()); - let compression = match compression { - SnapshotCompression::Stored => Compression::Stored, - SnapshotCompression::Deflated => Compression::Deflate, - SnapshotCompression::Bzip2 => Compression::Bz, - SnapshotCompression::Lzma => Compression::Lzma, - SnapshotCompression::Xz => Compression::Xz, - SnapshotCompression::Zstd => Compression::Zstd, - }; - - info!("Executing snapshot commands: {:?}", snapshot_types); - let now = Instant::now(); - - for snapshot_type in snapshot_types { - info!("Processing snapshot type: {:?}", snapshot_type); - match get_command_result(snapshot_type, &self.config.system).await { - Ok(temp_file) => { - info!( - "Got temp file for {:?}: {}", - snapshot_type, - temp_file.path().display() - ); - let filename = format!("{snapshot_type}.txt"); - let entry = ZipEntryBuilder::new(filename.clone().into(), compression); - - // Read file using compio fs - let content = match compio::fs::read(temp_file.path()).await { - Ok(data) => data, - Err(e) => { - error!("Failed to read temporary file: {}", e); - continue; - } - }; - - info!( - "Read {} bytes from temp file for {}", - content.len(), - filename - ); - - if let Err(e) = zip_writer.write_entry_whole(entry, &content).await { - error!("Failed to write to snapshot file: {}", e); - continue; - } - info!("Wrote entry {} to zip file", filename); - } - Err(e) => { - error!( - "Failed to execute command for snapshot type {:?}: {}", - snapshot_type, e - ); - continue; - } - } - } - - info!( - "Snapshot commands {:?} finished in {}", - snapshot_types, - IggyDuration::new(now.elapsed()) - ); - - let zip_data = zip_writer - .close() - .await - .map_err(|_| IggyError::SnapshotFileCompletionFailed)?; - - info!("Final zip size: {} bytes", zip_data.len()); - Ok(Snapshot::new(zip_data)) - } -} - -async fn write_command_output_to_temp_file( - command: &mut Command, -) -> Result { - let output = command.output()?; - - info!( - "Command output: {} bytes, stderr: {}", - output.stdout.len(), - String::from_utf8_lossy(&output.stderr) - ); - - let temp_file = NamedTempFile::new()?; - - // Use compio to write the file - create/truncate to ensure clean write - let mut file = OpenOptions::new() - .create(true) - .write(true) - .truncate(true) - .open(temp_file.path()) - .await?; - - // Write the command output - compio takes ownership of the buffer - let stdout = output.stdout; - let (result, _buf) = file.write_all_at(stdout, 0).await.into(); - result?; - - file.sync_all().await?; - - info!( - "Wrote {} bytes to temp file: {}", - _buf.len(), - temp_file.path().display() - ); - - Ok(temp_file) -} - -async fn get_filesystem_overview() -> Result { - write_command_output_to_temp_file(Command::new("ls").args(["-la", "/tmp", "/proc"])).await -} - -async fn get_process_info() -> Result { - let temp_file = NamedTempFile::new()?; - let mut file = OpenOptions::new() - .create(true) - .write(true) - .open(temp_file.path()) - .await?; - - let mut position = 0; - let ps_output = Command::new("ps").arg("aux").output()?; - let (result, written) = file - .write_all_at(b"=== Process List (ps aux) ===\n", 0) - .await - .into(); - result?; - position += written.len() as u64; - - let (result, written) = file.write_all_at(ps_output.stdout, position).await.into(); - result?; - position += written.len() as u64; - - let (result, written) = file.write_all_at(b"\n\n", position).await.into(); - result?; - position += written.len() as u64; - - let (result, written) = file - .write_all_at(b"=== Detailed Process Information ===\n", position) - .await - .into(); - result?; - position += written.len() as u64; - - let proc_info = procdump::get_proc_info().await?; - let bytes = proc_info.as_bytes().to_owned(); - let (result, _) = file.write_all_at(bytes, position).await.into(); - result?; - file.sync_all().await?; - - Ok(temp_file) -} - -async fn get_resource_usage() -> Result { - write_command_output_to_temp_file(Command::new("top").args(["-H", "-b", "-n", "1"])).await -} - -async fn get_test_snapshot() -> Result { - write_command_output_to_temp_file(Command::new("echo").arg("test")).await -} - -async fn get_server_logs(config: &SystemConfig) -> Result { - let base_directory = PathBuf::from(config.get_system_path()); - let logs_subdirectory = PathBuf::from(&config.logging.path); - let logs_path = base_directory.join(logs_subdirectory); - - let list_and_cat = format!( - r#"ls -tr "{logs}" | xargs -I {{}} cat "{logs}/{{}}" "#, - logs = logs_path.display() - ); - - write_command_output_to_temp_file(Command::new("sh").args(["-c", &list_and_cat])).await -} - -async fn get_server_config(config: &SystemConfig) -> Result { - let base_directory = PathBuf::from(config.get_system_path()); - let config_path = base_directory.join("runtime").join("current_config.toml"); - - write_command_output_to_temp_file(Command::new("cat").arg(config_path)).await -} - -async fn get_command_result( - snapshot_type: &SystemSnapshotType, - config: &SystemConfig, -) -> Result { - match snapshot_type { - SystemSnapshotType::FilesystemOverview => get_filesystem_overview().await, - SystemSnapshotType::ProcessList => get_process_info().await, - SystemSnapshotType::ResourceUsage => get_resource_usage().await, - SystemSnapshotType::Test => get_test_snapshot().await, - SystemSnapshotType::ServerLogs => get_server_logs(config).await, - SystemSnapshotType::ServerConfig => get_server_config(config).await, - SystemSnapshotType::All => { - // This should not be reached (we filter out `All`` at the call site) - unreachable!( - "SystemSnapshotType::All should be handled before calling get_command_result()." - ) - } - } -} diff --git a/core/server/src/shard/system/snapshot/procdump.rs b/core/server/src/shard/system/snapshot/procdump.rs deleted file mode 100644 index 7c6f5b6183..0000000000 --- a/core/server/src/shard/system/snapshot/procdump.rs +++ /dev/null @@ -1,213 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -/// Parse the contents of a /proc/[pid]/task/[tid]/stat file into a human-readable format -fn parse_stat(contents: &str) -> String { - let fields: Vec<&str> = contents.split_whitespace().collect(); - if fields.len() < 52 { - return format!("Invalid stat format: {contents}"); - } - - let cmd_start = contents.find('(').unwrap_or(0); - let cmd_end = contents.rfind(')').unwrap_or(contents.len()); - let comm = &contents[cmd_start + 1..cmd_end]; - - let mut result = String::new(); - result.push_str(&format!("PID: {}\n", fields[0])); - result.push_str(&format!("Command: {comm}\n")); - result.push_str(&format!( - "State: {} ({})\n", - fields[2], - match fields[2] { - "R" => "Running", - "S" => "Sleeping (interruptible)", - "D" => "Waiting in uninterruptible disk sleep", - "Z" => "Zombie", - "T" => "Stopped", - "t" => "Tracing stop", - "W" => "Paging", - "X" | "x" => "Dead", - "K" => "Wakekill", - "P" => "Parked", - _ => "Unknown", - } - )); - result.push_str(&format!("Parent PID: {}\n", fields[3])); - result.push_str(&format!("Process Group: {}\n", fields[4])); - result.push_str(&format!("Session ID: {}\n", fields[5])); - result.push_str(&format!("TTY: {}\n", fields[6])); - result.push_str(&format!("Foreground Process Group: {}\n", fields[7])); - result.push_str(&format!("Kernel Flags: {}\n", fields[8])); - result.push_str(&format!("Minor Faults: {}\n", fields[9])); - result.push_str(&format!("Children Minor Faults: {}\n", fields[10])); - result.push_str(&format!("Major Faults: {}\n", fields[11])); - result.push_str(&format!("Children Major Faults: {}\n", fields[12])); - result.push_str(&format!("User Mode Time: {} ticks\n", fields[13])); - result.push_str(&format!("System Mode Time: {} ticks\n", fields[14])); - result.push_str(&format!("Children User Mode Time: {} ticks\n", fields[15])); - result.push_str(&format!( - "Children System Mode Time: {} ticks\n", - fields[16] - )); - result.push_str(&format!("Priority: {}\n", fields[17])); - result.push_str(&format!("Nice Value: {}\n", fields[18])); - result.push_str(&format!("Number of Threads: {}\n", fields[19])); - result.push_str(&format!("Real-time Priority: {}\n", fields[39])); - result.push_str(&format!( - "Policy: {} ({})\n", - fields[40], - match fields[40] { - "0" => "SCHED_NORMAL/OTHER", - "1" => "SCHED_FIFO", - "2" => "SCHED_RR", - "3" => "SCHED_BATCH", - "5" => "SCHED_IDLE", - "6" => "SCHED_DEADLINE", - _ => "Unknown", - } - )); - result.push_str(&format!( - "Aggregated Block I/O Delays: {} ticks\n", - fields[41] - )); - result.push_str(&format!("Guest Time: {} ticks\n", fields[42])); - result.push_str(&format!("Children Guest Time: {} ticks\n", fields[43])); - - result -} - -/// Get detailed information about the system's processes and related /proc data -pub async fn get_proc_info() -> Result { - let static_proc_files = vec![ - "/proc/uptime", - "/proc/cpuinfo", - "/proc/stat", - "/proc/meminfo", - "/proc/interrupts", - "/proc/softirqs", - "/proc/latency", - "/proc/buddyinfo", - "/proc/slabinfo", - "/proc/vmstat", - "/proc/loadavg", - "/proc/cmdline", - "/proc/version", - "/proc/net/sockstat", - "/proc/net/snmp", - "/proc/net/netlink", - "/proc/net/netstat", - "/proc/net/dev", - "/proc/net/packet", - "/proc/net/tcp", - "/proc/net/tcp6", - "/proc/net/udp", - "/proc/net/udp6", - "/proc/net/raw", - "/proc/net/raw6", - "/proc/net/icmp", - "/proc/net/icmp6", - "/proc/net/udplite", - "/proc/net/udplite6", - "/proc/net/unix", - "/proc/net/softnet_stat", - "/proc/tty/drivers", - "/proc/sys/kernel/pid_max", - "/proc/sys/kernel/random/boot_id", - "/proc/mounts", - "/proc/modules", - ]; - - let mut result = String::new(); - - async fn dump_file(result: &mut String, path: &str) -> Result<(), std::io::Error> { - match std::fs::read_to_string(path) { - Ok(contents) => { - result.push_str(&format!("=== {path} ===\n")); - - if path.ends_with("/stat") && path.contains("/task/") { - result.push_str(&parse_stat(&contents)); - } else { - result.push_str(&contents); - } - - result.push_str("\n\n"); - } - Err(e) => { - if let Ok(metadata) = std::fs::metadata(path) { - if metadata.is_dir() && path.ends_with("/fd") { - result.push_str(&format!("=== {path} (directory) ===\n")); - if let Ok(mut rd) = std::fs::read_dir(path) { - while let Some(Ok(entry)) = rd.next() { - let fd_path = entry.path(); - match std::fs::read_link(&fd_path) { - Ok(link) => { - result.push_str(&format!( - "{} -> {}\n", - fd_path.display(), - link.display() - )); - } - Err(_) => { - result.push_str(&format!( - "{} (unreadable symlink)\n", - fd_path.display() - )); - } - } - } - } - result.push('\n'); - } else { - result.push_str(&format!("=== {path} ERROR: {e} ===\n\n")); - } - } else { - result.push_str(&format!("=== {path} ERROR: {e} ===\n\n")); - } - } - } - Ok(()) - } - - for path in &static_proc_files { - dump_file(&mut result, path).await?; - } - - let mut proc_dir = std::fs::read_dir("/proc")?; - while let Some(Ok(entry)) = proc_dir.next() { - let file_type = entry.file_type()?; - if file_type.is_dir() - && let Ok(pid) = entry.file_name().to_string_lossy().parse::() - { - let pid_paths = vec![ - format!("/proc/{}/cmdline", pid), - format!("/proc/{}/statm", pid), - format!("/proc/{}/cgroup", pid), - format!("/proc/{}/task/{}/stat", pid, pid), - format!("/proc/{}/task/{}/status", pid, pid), - format!("/proc/{}/task/{}/wchan", pid, pid), - format!("/proc/{}/task/{}/syscall", pid, pid), - format!("/proc/{}/task/{}/fd", pid, pid), - ]; - - for p in pid_paths { - dump_file(&mut result, &p).await?; - } - } - } - - Ok(result) -} diff --git a/core/server/src/shard/system/stats.rs b/core/server/src/shard/system/stats.rs deleted file mode 100644 index a1d0114cf0..0000000000 --- a/core/server/src/shard/system/stats.rs +++ /dev/null @@ -1,120 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::shard::IggyShard; -use crate::{SEMANTIC_VERSION, VERSION}; -use iggy_common::{IggyDuration, IggyError, Stats}; -use std::cell::RefCell; -use sysinfo::System as SysinfoSystem; -use system_stats::SystemProbe; - -thread_local! { - static SYSINFO: RefCell> = const { RefCell::new(None) }; -} - -impl IggyShard { - pub async fn get_stats(&self) -> Result { - assert_eq!(self.id, 0, "GetStats should only be called on shard0"); - - let probe = SYSINFO.with_borrow_mut(|slot| { - let sys = slot.get_or_insert_with(SysinfoSystem::new); - SystemProbe::capture(sys) - }); - - let clients_count = self.client_manager.get_clients().len() as u32; - let hostname = sysinfo::System::host_name().unwrap_or("unknown_hostname".to_string()); - let os_name = sysinfo::System::name().unwrap_or("unknown_os_name".to_string()); - let os_version = - sysinfo::System::long_os_version().unwrap_or("unknown_os_version".to_string()); - let kernel_version = - sysinfo::System::kernel_version().unwrap_or("unknown_kernel_version".to_string()); - - let mut stats = Stats { - process_id: probe.process_id, - cpu_usage: probe.cpu_usage, - total_cpu_usage: probe.total_cpu_usage, - memory_usage: probe.memory_usage.into(), - total_memory: probe.total_memory.into(), - available_memory: probe.available_memory.into(), - run_time: IggyDuration::new_from_secs(probe.run_time_secs), - start_time: IggyDuration::new_from_secs(probe.start_time_secs) - .as_micros() - .into(), - read_bytes: probe.read_bytes.into(), - written_bytes: probe.written_bytes.into(), - threads_count: probe.threads_count, - clients_count, - hostname, - os_name, - os_version, - kernel_version, - iggy_server_version: VERSION.to_owned(), - iggy_server_semver: SEMANTIC_VERSION.get_numeric_version().ok(), - ..Default::default() - }; - - let (streams_count, topics_count, partitions_count, consumer_groups_count, stream_ids) = - self.metadata.with_metadata(|m| { - let mut topics = 0u32; - let mut partitions = 0u32; - let mut cg = 0u32; - let ids: Vec<_> = m.streams.iter().map(|(k, _)| k).collect(); - for (_, stream) in m.streams.iter() { - topics += stream.topics.len() as u32; - for (_, topic) in stream.topics.iter() { - partitions += topic.partitions.len() as u32; - cg += topic.consumer_groups.len() as u32; - } - } - (m.streams.len() as u32, topics, partitions, cg, ids) - }); - - stats.streams_count = streams_count; - stats.topics_count = topics_count; - stats.partitions_count = partitions_count; - stats.consumer_groups_count = consumer_groups_count; - - for stream_id in stream_ids { - if let Some(stream_stat) = self.metadata.get_stream_stats(stream_id) { - stats.messages_count += stream_stat.messages_count_inconsistent(); - stats.segments_count += stream_stat.segments_count_inconsistent(); - stats.messages_size_bytes += stream_stat.size_bytes_inconsistent().into(); - } - } - - match fs2::available_space(&self.config.system.path) { - Ok(space) => stats.free_disk_space = space.into(), - Err(err) => { - tracing::warn!( - "Failed to get available disk space for '{}': {err}", - self.config.system.path - ); - } - } - match fs2::total_space(&self.config.system.path) { - Ok(space) => stats.total_disk_space = space.into(), - Err(err) => { - tracing::warn!( - "Failed to get total disk space for '{}': {err}", - self.config.system.path - ); - } - } - - Ok(stats) - } -} diff --git a/core/server/src/shard/system/storage.rs b/core/server/src/shard/system/storage.rs deleted file mode 100644 index eb736a83cd..0000000000 --- a/core/server/src/shard/system/storage.rs +++ /dev/null @@ -1,99 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use super::COMPONENT; -use crate::shard::system::info::SystemInfo; -use crate::streaming::persistence::persister::PersisterKind; -use crate::streaming::utils::file; -use anyhow::Context; -use compio::buf::IoBuf; -use compio::io::AsyncReadAtExt; -use err_trail::ErrContext; -use iggy_common::IggyError; -use server_common::PooledBuffer; -use std::sync::Arc; -use tracing::info; - -#[derive(Debug)] -pub struct FileSystemInfoStorage { - persister: Arc, - path: String, -} - -impl FileSystemInfoStorage { - pub fn new(path: String, persister: Arc) -> Self { - Self { path, persister } - } - - pub async fn load(&self) -> Result { - let file = file::open(&self.path).await; - if file.is_err() { - return Err(IggyError::ResourceNotFound(self.path.to_owned())); - } - - let file = file.unwrap(); - let file_size = file - .metadata() - .await - .error(|e: &std::io::Error| { - format!( - "{COMPONENT} (error: {e}) - failed to retrieve metadata for file at path: {}", - self.path - ) - }) - .map_err(|_| IggyError::CannotReadFileMetadata)? - .len() as usize; - - let file = file::open(&self.path) - .await - .map_err(|_| IggyError::CannotReadFile)?; - let buffer = PooledBuffer::with_capacity(file_size); - let (result, buffer) = file - .read_exact_at(buffer.slice(0..file_size), 0) - .await - .into(); - result - .error(|e: &std::io::Error| { - format!( - "{COMPONENT} Failed to read system info from file at path: {} (error: {e})", - self.path - ) - }) - .map_err(|_| IggyError::CannotReadFile)?; - let system_info = rmp_serde::from_slice(&buffer) - .with_context(|| "Failed to deserialize system info") - .map_err(|_| IggyError::CannotDeserializeResource)?; - Ok(system_info) - } - - pub async fn save(&self, system_info: &SystemInfo) -> Result<(), IggyError> { - let data = rmp_serde::to_vec(system_info) - .with_context(|| "Failed to serialize system info") - .map_err(|_| IggyError::CannotSerializeResource)?; - self.persister - .overwrite(&self.path, data) - .await - .error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - failed to overwrite file at path: {}", - self.path - ) - })?; - info!("Saved system info, {system_info}"); - Ok(()) - } -} diff --git a/core/server/src/shard/system/streams.rs b/core/server/src/shard/system/streams.rs deleted file mode 100644 index ef80c7a1d7..0000000000 --- a/core/server/src/shard/system/streams.rs +++ /dev/null @@ -1,179 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::metadata::StreamMeta; -use crate::shard::IggyShard; -use crate::shard::transmission::message::{ResolvedStream, ResolvedTopic}; -use crate::streaming::streams::storage::{create_stream_file_hierarchy, delete_stream_directory}; -use iggy_common::{IggyError, IggyTimestamp}; -use server_common::sharding::IggyNamespace; -use std::sync::Arc; - -/// Info returned when a stream is deleted - contains what callers need for logging/events. -pub struct DeletedStreamInfo { - pub id: usize, - pub name: String, -} - -impl IggyShard { - pub async fn create_stream(&self, name: String) -> Result { - let name_arc = Arc::from(name.as_str()); - if self.metadata.stream_name_exists(&name_arc) { - return Err(IggyError::StreamNameAlreadyExists(name)); - } - - let stream_id = self.metadata.next_stream_id(); - create_stream_file_hierarchy(stream_id, &self.config.system).await?; - - let created_at = IggyTimestamp::now(); - let stats = Arc::new(crate::streaming::stats::StreamStats::default()); - let meta = StreamMeta::with_stats(0, name_arc, created_at, stats); - let assigned_id = self.writer().add_stream(meta); - debug_assert_eq!( - assigned_id, stream_id, - "Stream ID mismatch: expected {stream_id}, got {assigned_id}" - ); - - self.metrics.increment_streams(1); - Ok(stream_id) - } - - pub fn update_stream(&self, stream: ResolvedStream, name: String) -> Result<(), IggyError> { - self.writer() - .try_update_stream(&self.metadata, stream.id(), Arc::from(name.as_str())) - } - - pub async fn delete_stream( - &self, - stream: ResolvedStream, - ) -> Result { - let stream_id = stream.id(); - - let (topics_with_partitions, stream_name, stats, topics_count, partitions_count) = - self.metadata.with_metadata(|m| { - let stream_meta = m - .streams - .get(stream_id) - .expect("Stream metadata must exist"); - let twp: Vec<_> = stream_meta - .topics - .iter() - .map(|(topic_id, topic)| { - let partition_ids: Vec = (0..topic.partitions.len()).collect(); - (topic_id, partition_ids) - }) - .collect(); - let partitions_count: usize = stream_meta - .topics - .iter() - .map(|(_, t)| t.partitions.len()) - .sum(); - ( - twp, - stream_meta.name.to_string(), - stream_meta.stats.clone(), - stream_meta.topics.len(), - partitions_count, - ) - }); - - { - let namespaces: Vec<_> = topics_with_partitions - .iter() - .flat_map(|(topic_id, partition_ids)| { - partition_ids - .iter() - .map(|&partition_id| IggyNamespace::new(stream_id, *topic_id, partition_id)) - }) - .collect(); - let mut partitions = self.local_partitions.borrow_mut(); - for ns in namespaces { - partitions.remove(&ns); - } - } - - self.metrics.decrement_streams(1); - self.metrics.decrement_topics(topics_count as u32); - self.metrics.decrement_partitions(partitions_count as u32); - self.metrics - .decrement_messages(stats.messages_count_inconsistent()); - self.metrics - .decrement_segments(stats.segments_count_inconsistent()); - - self.writer().delete_stream(stream_id); - - let stream_info = DeletedStreamInfo { - id: stream_id, - name: stream_name, - }; - - self.client_manager - .delete_consumer_groups_for_stream(stream_id); - - let namespaces_to_remove: Vec<_> = self - .shards_table - .iter() - .filter_map(|entry| { - let (ns, _) = entry.pair(); - if ns.stream_id() == stream_id { - Some(*ns) - } else { - None - } - }) - .collect(); - - for ns in namespaces_to_remove { - self.remove_shard_table_record(&ns); - } - - delete_stream_directory(stream_id, &topics_with_partitions, &self.config.system).await?; - Ok(stream_info) - } - - /// Clears in-memory state for all topics in a stream. - pub async fn purge_stream(&self, stream: ResolvedStream) -> Result<(), IggyError> { - let stream_id = stream.id(); - let topic_ids = self.metadata.get_topic_ids(stream_id); - - for topic_id in topic_ids { - let topic = ResolvedTopic { - stream_id, - topic_id, - }; - self.purge_topic(topic).await?; - } - - Ok(()) - } - - /// Disk cleanup for local partitions across all topics in a stream. - pub(crate) async fn purge_stream_local(&self, stream: ResolvedStream) -> Result<(), IggyError> { - let stream_id = stream.id(); - let topic_ids = self.metadata.get_topic_ids(stream_id); - - for topic_id in topic_ids { - let topic = ResolvedTopic { - stream_id, - topic_id, - }; - self.purge_topic_local(topic).await?; - } - - Ok(()) - } -} diff --git a/core/server/src/shard/system/topics.rs b/core/server/src/shard/system/topics.rs deleted file mode 100644 index a8d99ed0e4..0000000000 --- a/core/server/src/shard/system/topics.rs +++ /dev/null @@ -1,250 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::metadata::TopicMeta; -use crate::shard::IggyShard; -use crate::shard::transmission::message::{ResolvedStream, ResolvedTopic}; -use crate::streaming::topics::storage::{create_topic_file_hierarchy, delete_topic_directory}; -use iggy_common::{ - CompressionAlgorithm, Identifier, IggyError, IggyExpiry, IggyTimestamp, MaxTopicSize, -}; -use server_common::sharding::IggyNamespace; -use std::sync::Arc; - -/// Info returned when a topic is deleted - contains what callers need for logging/events. -pub struct DeletedTopicInfo { - pub id: usize, - pub name: String, - pub stream_id: usize, -} - -impl IggyShard { - #[allow(clippy::too_many_arguments)] - pub async fn create_topic( - &self, - stream: ResolvedStream, - name: String, - message_expiry: IggyExpiry, - compression: CompressionAlgorithm, - max_topic_size: MaxTopicSize, - replication_factor: Option, - ) -> Result { - let stream_id = stream.0; - - let config = &self.config.system; - let message_expiry = config.resolve_message_expiry(message_expiry); - let max_topic_size = config.resolve_max_topic_size(max_topic_size)?; - - let name_arc = Arc::from(name.as_str()); - let parent_stats = self.metadata.get_stream_stats(stream_id).ok_or_else(|| { - IggyError::StreamIdNotFound(Identifier::numeric(stream_id as u32).unwrap()) - })?; - - let name_exists = self.metadata.with_metadata(|m| { - m.streams - .get(stream_id) - .map(|s| s.topic_index.contains_key(&name_arc)) - .unwrap_or(false) - }); - if name_exists { - return Err(IggyError::TopicNameAlreadyExists( - name, - Identifier::numeric(stream_id as u32).unwrap(), - )); - } - - let topic_id = self.metadata.next_topic_id(stream_id).ok_or_else(|| { - IggyError::StreamIdNotFound(Identifier::numeric(stream_id as u32).unwrap()) - })?; - create_topic_file_hierarchy(stream_id, topic_id, &self.config.system).await?; - - let created_at = IggyTimestamp::now(); - let stats = Arc::new(crate::streaming::stats::TopicStats::new(parent_stats)); - let topic_meta = TopicMeta { - id: 0, - name: name_arc, - created_at, - message_expiry, - compression_algorithm: compression, - max_topic_size, - replication_factor: replication_factor.unwrap_or(1), - stats, - partitions: Vec::new(), - consumer_groups: slab::Slab::new(), - consumer_group_index: ahash::AHashMap::default(), - round_robin_counter: Arc::new(std::sync::atomic::AtomicUsize::new(0)), - }; - let assigned_id = self - .writer() - .add_topic(stream_id, topic_meta) - .ok_or_else(|| { - IggyError::StreamIdNotFound(Identifier::numeric(stream_id as u32).unwrap()) - })?; - debug_assert_eq!( - assigned_id, topic_id, - "Topic ID mismatch: expected {topic_id}, got {assigned_id}" - ); - - self.metrics.increment_topics(1); - Ok(topic_id) - } - - #[allow(clippy::too_many_arguments)] - pub fn update_topic( - &self, - topic: ResolvedTopic, - name: String, - message_expiry: IggyExpiry, - compression_algorithm: CompressionAlgorithm, - max_topic_size: MaxTopicSize, - replication_factor: Option, - ) -> Result<(), IggyError> { - self.writer().try_update_topic( - &self.metadata, - topic.stream_id, - topic.topic_id, - Arc::from(name.as_str()), - message_expiry, - compression_algorithm, - max_topic_size, - replication_factor.unwrap_or(1), - ) - } - - pub async fn delete_topic(&self, topic: ResolvedTopic) -> Result { - let stream = topic.stream_id; - let topic_id = topic.topic_id; - - let (partition_ids, topic_name, messages_count, size_bytes, segments_count, parent_stats) = - self.metadata.with_metadata(|m| { - let stream_meta = m.streams.get(stream).expect("Stream metadata must exist"); - let topic_meta = stream_meta - .topics - .get(topic_id) - .expect("Topic metadata must exist"); - let pids: Vec = (0..topic_meta.partitions.len()).collect(); - ( - pids, - topic_meta.name.to_string(), - topic_meta.stats.messages_count_inconsistent(), - topic_meta.stats.size_bytes_inconsistent(), - topic_meta.stats.segments_count_inconsistent(), - topic_meta.stats.parent().clone(), - ) - }); - - { - let mut partitions = self.local_partitions.borrow_mut(); - for &partition_id in &partition_ids { - let ns = IggyNamespace::new(stream, topic_id, partition_id); - partitions.remove(&ns); - } - } - - self.writer().delete_topic(stream, topic_id); - - let topic_info = DeletedTopicInfo { - id: topic_id, - name: topic_name, - stream_id: stream, - }; - - self.client_manager - .delete_consumer_groups_for_topic(stream, topic_id); - - let namespaces_to_remove: Vec<_> = self - .shards_table - .iter() - .filter_map(|entry| { - let (ns, _) = entry.pair(); - if ns.stream_id() == stream && ns.topic_id() == topic_id { - Some(*ns) - } else { - None - } - }) - .collect(); - - for ns in namespaces_to_remove { - self.remove_shard_table_record(&ns); - } - - delete_topic_directory(stream, topic_id, &partition_ids, &self.config.system).await?; - - parent_stats.decrement_messages_count(messages_count); - parent_stats.decrement_size_bytes(size_bytes); - parent_stats.decrement_segments_count(segments_count); - self.metrics.decrement_topics(1); - Ok(topic_info) - } - - /// Clears in-memory state for a topic: consumer offsets and stats. - /// Called on the control plane before broadcasting to other shards. - pub async fn purge_topic(&self, topic: ResolvedTopic) -> Result<(), IggyError> { - let stream = topic.stream_id; - let topic_id = topic.topic_id; - let partition_ids = self.metadata.get_partition_ids(stream, topic_id); - - for &partition_id in &partition_ids { - if let Some(offsets) = - self.metadata - .get_partition_consumer_offsets(stream, topic_id, partition_id) - { - offsets.pin().clear(); - } - if let Some(offsets) = - self.metadata - .get_partition_consumer_group_offsets(stream, topic_id, partition_id) - { - offsets.pin().clear(); - } - } - - // Zero partition stats — propagation handles topic and stream counters. - // Topic stats must NOT be zeroed separately to avoid double-decrementing the stream. - for &partition_id in &partition_ids { - let ns = IggyNamespace::new(stream, topic_id, partition_id); - if let Some(partition_stats) = self.metadata.get_partition_stats(&ns) { - partition_stats.zero_out_all(); - } - } - - Ok(()) - } - - /// Disk cleanup for local partitions: deletes consumer offset files and purges segments. - /// Called on each shard (including shard 0) after in-memory state is cleared. - pub(crate) async fn purge_topic_local(&self, topic: ResolvedTopic) -> Result<(), IggyError> { - let stream = topic.stream_id; - let topic_id = topic.topic_id; - let partition_ids = self.metadata.get_partition_ids(stream, topic_id); - - for &partition_id in &partition_ids { - let ns = IggyNamespace::new(stream, topic_id, partition_id); - if !self.local_partitions.borrow().contains(&ns) { - continue; - } - - self.delete_all_consumer_offset_files(stream, topic_id, partition_id) - .await?; - self.purge_all_segments(stream, topic_id, partition_id) - .await?; - } - - Ok(()) - } -} diff --git a/core/server/src/shard/system/users.rs b/core/server/src/shard/system/users.rs deleted file mode 100644 index 69a9d66e85..0000000000 --- a/core/server/src/shard/system/users.rs +++ /dev/null @@ -1,301 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use super::COMPONENT; -use crate::metadata::UserMeta; -use crate::shard::IggyShard; -use crate::streaming::session::Session; -use crate::streaming::users::user::User; -use crate::streaming::utils::crypto; -use dashmap::DashMap; -use err_trail::ErrContext; -use iggy_common::Identifier; -use iggy_common::IggyError; -use iggy_common::Permissions; -use iggy_common::UserStatus; -use std::sync::Arc; -use tracing::{error, warn}; - -const MAX_USERS: usize = u32::MAX as usize; - -impl IggyShard { - fn user_from_meta(&self, meta: &UserMeta) -> User { - let pats = self.metadata.get_user_personal_access_tokens(meta.id); - let pat_map = DashMap::new(); - for pat in pats { - pat_map.insert(pat.token.clone(), pat); - } - User { - id: meta.id, - status: meta.status, - username: meta.username.to_string(), - password: meta.password_hash.to_string(), - created_at: meta.created_at, - permissions: meta.permissions.as_ref().map(|p| (**p).clone()), - personal_access_tokens: pat_map, - } - } - - fn get_user_from_metadata(&self, identifier: &Identifier) -> Result, IggyError> { - let user_id = match self.metadata.get_user_id(identifier) { - Some(id) => id, - None => return Ok(None), - }; - - Ok(self - .metadata - .get_user(user_id) - .map(|meta| self.user_from_meta(&meta))) - } - - pub fn find_user(&self, user_id: &Identifier) -> Result, IggyError> { - self.try_get_user(user_id) - } - - pub fn get_user(&self, user_id: &Identifier) -> Result { - self.try_get_user(user_id)? - .ok_or(IggyError::ResourceNotFound(user_id.to_string())) - } - - pub fn try_get_user(&self, user_id: &Identifier) -> Result, IggyError> { - self.get_user_from_metadata(user_id) - } - - pub fn get_users(&self) -> Vec { - self.metadata - .get_all_users() - .iter() - .map(|meta| self.user_from_meta(meta)) - .collect() - } - - pub fn create_user( - &self, - username: &str, - password: &str, - status: UserStatus, - permissions: Option, - ) -> Result { - let password_hash = crypto::hash_password(password); - - let user_id = self - .writer() - .create_user( - &self.metadata, - Arc::from(username), - Arc::from(password_hash.as_str()), - status, - permissions.map(Arc::new), - MAX_USERS, - ) - .inspect_err(|e| match e { - IggyError::UserAlreadyExists => error!("User: {username} already exists."), - IggyError::UsersLimitReached => error!("Available users limit reached."), - _ => {} - })?; - - self.metrics.increment_users(1); - - self.get_user(&user_id.try_into()?).error(|e: &IggyError| { - format!("{COMPONENT} (error: {e}) - failed to get user with id: {user_id}") - }) - } - - pub fn delete_user(&self, user_id: &Identifier) -> Result { - let user = self.get_user(user_id).error(|e: &IggyError| { - format!("{COMPONENT} (error: {e}) - failed to get user with id: {user_id}") - })?; - - if user.is_root() { - error!("Cannot delete the root user."); - return Err(IggyError::CannotDeleteUser(user.id)); - } - - let user_u32_id = user.id; - - self.client_manager - .delete_clients_for_user(user_u32_id) - .error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - failed to delete clients for user with ID: {user_u32_id}" - ) - })?; - self.metrics.decrement_users(1); - - self.writer().delete_user(user_u32_id); - - Ok(user) - } - - pub fn update_user( - &self, - user_id: &Identifier, - username: Option, - status: Option, - ) -> Result { - let user = self.get_user(user_id)?; - let numeric_user_id = user.id; - - let updated_meta = self.writer().update_user( - &self.metadata, - numeric_user_id, - username.map(|u| Arc::from(u.as_str())), - status, - )?; - - Ok(self.user_from_meta(&updated_meta)) - } - - pub fn update_permissions( - &self, - user_id: &Identifier, - permissions: Option, - ) -> Result<(), IggyError> { - let user: User = self.get_user(user_id).error(|e: &IggyError| { - format!("{COMPONENT} (error: {e}) - failed to get user with id: {user_id}") - })?; - - let current_meta = self - .metadata - .get_user(user.id) - .ok_or_else(|| IggyError::ResourceNotFound(user_id.to_string()))?; - - let updated_meta = UserMeta { - id: current_meta.id, - username: current_meta.username, - password_hash: current_meta.password_hash, - status: current_meta.status, - permissions: permissions.map(Arc::new), - created_at: current_meta.created_at, - }; - - self.writer().update_user_meta(user.id, updated_meta); - - Ok(()) - } - - pub fn change_password( - &self, - user_id: &Identifier, - current_password: &str, - new_password: &str, - ) -> Result<(), IggyError> { - let user = self.get_user(user_id).error(|e: &IggyError| { - format!( - "{COMPONENT} change password (error: {e}) - failed to get user with id: {user_id}" - ) - })?; - - if !crypto::verify_password(current_password, &user.password) { - error!( - "Invalid current password for user: {} with ID: {user_id}.", - user.username - ); - return Err(IggyError::InvalidCredentials); - } - - let current_meta = self - .metadata - .get_user(user.id) - .ok_or_else(|| IggyError::ResourceNotFound(user_id.to_string()))?; - - let new_password_hash = crypto::hash_password(new_password); - let updated_meta = UserMeta { - id: current_meta.id, - username: current_meta.username, - password_hash: Arc::from(new_password_hash.as_str()), - status: current_meta.status, - permissions: current_meta.permissions, - created_at: current_meta.created_at, - }; - - self.writer().update_user_meta(user.id, updated_meta); - - Ok(()) - } - - pub fn login_user( - &self, - username: &str, - password: &str, - session: Option<&Session>, - ) -> Result { - self.login_user_with_credentials(username, Some(password), session) - } - - pub fn login_user_with_credentials( - &self, - username: &str, - password: Option<&str>, - session: Option<&Session>, - ) -> Result { - let user = match self.get_user(&username.try_into()?) { - Ok(user) => user, - Err(_) => { - error!("Cannot login user: {username} (not found)."); - return Err(IggyError::InvalidCredentials); - } - }; - - if !user.is_active() { - warn!("User: {username} with ID: {} is inactive.", user.id); - return Err(IggyError::UserInactive); - } - - if let Some(password) = password - && !crypto::verify_password(password, &user.password) - { - warn!( - "Invalid password for user: {username} with ID: {}.", - user.id - ); - return Err(IggyError::InvalidCredentials); - } - - if session.is_none() { - return Ok(user); - } - - let session = session.unwrap(); - if session.is_authenticated() { - warn!( - "User: {} with ID: {} was already authenticated, removing the previous session...", - user.username, - session.get_user_id() - ); - self.logout_user(session)?; - } - session.set_user_id(user.id); - self.client_manager - .set_user_id(session.client_id, user.id) - .error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - failed to set user_id to client, client ID: {}, user ID: {}", - session.client_id, user.id - ) - })?; - Ok(user) - } - - pub fn logout_user(&self, session: &Session) -> Result<(), IggyError> { - let client_id = session.client_id; - if client_id > 0 { - self.client_manager.clear_user_id(client_id)?; - } - Ok(()) - } -} diff --git a/core/server/src/shard/system/utils.rs b/core/server/src/shard/system/utils.rs deleted file mode 100644 index fc5f37a5f7..0000000000 --- a/core/server/src/shard/system/utils.rs +++ /dev/null @@ -1,255 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::{ - metadata::{resolve_consumer_group_id_inner, resolve_stream_id_inner, resolve_topic_id_inner}, - shard::{ - IggyShard, - transmission::message::{ - ResolvedConsumerGroup, ResolvedPartition, ResolvedStream, ResolvedTopic, - }, - }, - streaming::polling_consumer::PollingConsumer, -}; -use iggy_common::{Consumer, ConsumerKind, Identifier, IggyError}; - -impl IggyShard { - /// Resolves stream identifier to typed `ResolvedStream`. - pub fn resolve_stream(&self, stream_id: &Identifier) -> Result { - self.metadata - .with_metadata(|m| resolve_stream_inner(m, stream_id)) - } - - /// Resolves topic from identifiers. Returns StreamIdNotFound if stream doesn't exist, - /// TopicIdNotFound if topic doesn't exist. - pub fn resolve_topic( - &self, - stream_id: &Identifier, - topic_id: &Identifier, - ) -> Result { - self.metadata.with_metadata(|m| { - let stream = resolve_stream_inner(m, stream_id)?; - let id = resolve_topic_id_inner(m, stream.0, topic_id) - .ok_or_else(|| IggyError::TopicIdNotFound(stream_id.clone(), topic_id.clone()))?; - Ok(ResolvedTopic { - stream_id: stream.0, - topic_id: id, - }) - }) - } - - /// Resolves partition from identifiers. Returns appropriate error at each level. - pub fn resolve_partition( - &self, - stream_id: &Identifier, - topic_id: &Identifier, - partition_id: usize, - ) -> Result { - self.metadata.with_metadata(|m| { - let stream = resolve_stream_inner(m, stream_id)?; - let tid = resolve_topic_id_inner(m, stream.0, topic_id) - .ok_or_else(|| IggyError::TopicIdNotFound(stream_id.clone(), topic_id.clone()))?; - - let exists = m - .streams - .get(stream.0) - .and_then(|s| s.topics.get(tid)) - .and_then(|t| t.partitions.get(partition_id)) - .is_some(); - - if !exists { - return Err(IggyError::PartitionNotFound( - partition_id, - topic_id.clone(), - stream_id.clone(), - )); - } - - Ok(ResolvedPartition { - stream_id: stream.0, - topic_id: tid, - partition_id, - }) - }) - } - - /// Resolves consumer group from identifiers. Returns appropriate error at each level. - pub fn resolve_consumer_group( - &self, - stream_id: &Identifier, - topic_id: &Identifier, - group_id: &Identifier, - ) -> Result { - self.metadata.with_metadata(|m| { - let stream = resolve_stream_inner(m, stream_id)?; - let tid = resolve_topic_id_inner(m, stream.0, topic_id) - .ok_or_else(|| IggyError::TopicIdNotFound(stream_id.clone(), topic_id.clone()))?; - - let gid = - resolve_consumer_group_id_inner(m, stream.0, tid, group_id).ok_or_else(|| { - IggyError::ConsumerGroupIdNotFound(group_id.clone(), topic_id.clone()) - })?; - - Ok(ResolvedConsumerGroup { - stream_id: stream.0, - topic_id: tid, - group_id: gid, - }) - }) - } - - /// Validates that partitions_count does not exceed actual partition count. - pub fn validate_partitions_count( - &self, - topic: ResolvedTopic, - partitions_count: u32, - ) -> Result<(), IggyError> { - let actual = self - .metadata - .partitions_count(topic.stream_id, topic.topic_id); - if partitions_count > actual as u32 { - return Err(IggyError::InvalidPartitionsCount); - } - Ok(()) - } - - /// Validates that consumer_offset does not exceed actual partition offset. - pub fn validate_partition_offset( - &self, - stream_id: usize, - topic_id: usize, - partition_id: usize, - consumer_offset: u64, - ) -> Result<(), IggyError> { - let partition_stats = self - .metadata - .get_partition_stats_by_ids(stream_id, topic_id, partition_id) - .ok_or(IggyError::PartitionNotFound( - partition_id, - Identifier::numeric(topic_id as u32).expect("numeric identifier is always valid"), - Identifier::numeric(stream_id as u32).expect("numeric identifier is always valid"), - ))?; - - // Also rejects storing any offset if the partition is completely empty (i.e., has never contained any messages). - if (partition_stats.messages_count_inconsistent() == 0 - && partition_stats.current_offset() == 0) - || consumer_offset > partition_stats.current_offset() - { - return Err(IggyError::InvalidOffset(consumer_offset)); - } - Ok(()) - } - - /// Resolves consumer with partition ID for polling/offset operations. - /// For consumer groups, all lookups happen under a single metadata read guard. - pub fn resolve_consumer_with_partition_id( - &self, - topic: ResolvedTopic, - consumer: &Consumer, - client_id: u32, - partition_id: Option, - calculate_partition_id: bool, - ) -> Result, IggyError> { - match consumer.kind { - ConsumerKind::Consumer => { - let partition_id = partition_id.unwrap_or(0); - Ok(Some(( - PollingConsumer::consumer(&consumer.id, partition_id as usize), - partition_id as usize, - ))) - } - ConsumerKind::ConsumerGroup => { - if self.client_manager.try_get_client(client_id).is_none() { - return Err(IggyError::StaleClient); - } - - self.metadata.resolve_consumer_group_partition( - topic.stream_id, - topic.topic_id, - &consumer.id, - client_id, - partition_id, - calculate_partition_id, - ) - } - } - } - - /// Resolves topic and verifies user has append permission atomically. - pub fn resolve_topic_for_append( - &self, - user_id: u32, - stream_id: &Identifier, - topic_id: &Identifier, - ) -> Result { - self.metadata - .resolve_for_append(user_id, stream_id, topic_id) - } - - /// Resolves topic and verifies user has poll permission atomically. - pub fn resolve_topic_for_poll( - &self, - user_id: u32, - stream_id: &Identifier, - topic_id: &Identifier, - ) -> Result { - self.metadata.resolve_for_poll(user_id, stream_id, topic_id) - } - - /// Resolves topic and verifies user has permission to store consumer offset atomically. - pub fn resolve_topic_for_store_consumer_offset( - &self, - user_id: u32, - stream_id: &Identifier, - topic_id: &Identifier, - ) -> Result { - self.metadata - .resolve_for_store_consumer_offset(user_id, stream_id, topic_id) - } - - /// Resolves topic and verifies user has permission to delete consumer offset atomically. - pub fn resolve_topic_for_delete_consumer_offset( - &self, - user_id: u32, - stream_id: &Identifier, - topic_id: &Identifier, - ) -> Result { - self.metadata - .resolve_for_delete_consumer_offset(user_id, stream_id, topic_id) - } - - /// Resolves partition and verifies user has permission to delete segments atomically. - pub fn resolve_partition_for_delete_segments( - &self, - user_id: u32, - stream_id: &Identifier, - topic_id: &Identifier, - partition_id: usize, - ) -> Result { - self.metadata - .resolve_for_delete_segments(user_id, stream_id, topic_id, partition_id) - } -} - -fn resolve_stream_inner( - m: &crate::metadata::InnerMetadata, - stream_id: &Identifier, -) -> Result { - resolve_stream_id_inner(m, stream_id) - .map(ResolvedStream) - .ok_or_else(|| IggyError::StreamIdNotFound(stream_id.clone())) -} diff --git a/core/server/src/shard/systemd.rs b/core/server/src/shard/systemd.rs deleted file mode 100644 index a10c2fbba5..0000000000 --- a/core/server/src/shard/systemd.rs +++ /dev/null @@ -1,45 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Thin wrappers around `sd_notify` so every systemd interaction on the server -//! side lives in one place (mirrors `core/ai/mcp/src/systemd.rs`). - -use tracing::warn; - -/// Tell systemd the service has finished start-up (`READY=1`). -pub fn notify_ready() { - if let Err(e) = sd_notify::notify(&[sd_notify::NotifyState::Ready]) { - warn!("Failed to send systemd READY=1 notification: {e}"); - } -} - -/// Tell systemd the service has begun shutting down (`STOPPING=1`). -pub fn notify_stopping() { - let _ = sd_notify::notify(&[sd_notify::NotifyState::Stopping]); -} - -/// Surface a non-fatal shutdown problem in `systemctl status` / journald. -pub fn notify_status(status: &str) { - let _ = sd_notify::notify(&[sd_notify::NotifyState::Status(status)]); -} - -/// Send a single watchdog keep-alive ping (`WATCHDOG=1`). -pub fn ping_watchdog() { - if let Err(e) = sd_notify::notify(&[sd_notify::NotifyState::Watchdog]) { - warn!("Failed to send systemd watchdog ping: {e}"); - } -} diff --git a/core/server/src/shard/task_registry/builders.rs b/core/server/src/shard/task_registry/builders.rs deleted file mode 100644 index ac61d27ff6..0000000000 --- a/core/server/src/shard/task_registry/builders.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod continuous; -pub mod oneshot; -pub mod periodic; - -use super::registry::TaskRegistry; - -// Marker type for when no shutdown callback is provided -pub struct NoShutdown; - -impl TaskRegistry { - pub fn periodic(&self, name: &'static str) -> periodic::PeriodicBuilder<'_, (), NoShutdown> { - periodic::PeriodicBuilder::new(self, name) - } - - pub fn continuous( - &self, - name: &'static str, - ) -> continuous::ContinuousBuilder<'_, (), NoShutdown> { - continuous::ContinuousBuilder::new(self, name) - } - - pub fn oneshot(&self, name: &'static str) -> oneshot::OneShotBuilder<'_, (), NoShutdown> { - oneshot::OneShotBuilder::new(self, name) - } -} diff --git a/core/server/src/shard/task_registry/builders/continuous.rs b/core/server/src/shard/task_registry/builders/continuous.rs deleted file mode 100644 index 4f4aad0a2e..0000000000 --- a/core/server/src/shard/task_registry/builders/continuous.rs +++ /dev/null @@ -1,109 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use super::NoShutdown; -use crate::shard::task_registry::ShutdownToken; -use crate::shard::task_registry::registry::TaskRegistry; -use iggy_common::IggyError; -use std::ops::AsyncFnOnce; - -pub struct ContinuousBuilder<'a, Task, OnShutdown = NoShutdown> { - reg: &'a TaskRegistry, - name: &'static str, - critical: bool, - run_fn: Option, - on_shutdown: Option, -} - -impl<'a> ContinuousBuilder<'a, (), NoShutdown> { - pub fn new(reg: &'a TaskRegistry, name: &'static str) -> Self { - Self { - reg, - name, - critical: false, - run_fn: None, - on_shutdown: None, - } - } -} - -impl<'a, Task, OnShutdown> ContinuousBuilder<'a, Task, OnShutdown> { - pub fn critical(mut self, c: bool) -> Self { - self.critical = c; - self - } - - pub fn on_shutdown( - self, - f: NewShutdown, - ) -> ContinuousBuilder<'a, Task, NewShutdown> - where - NewShutdown: AsyncFnOnce(Result<(), IggyError>) + 'static, - { - ContinuousBuilder { - reg: self.reg, - name: self.name, - critical: self.critical, - run_fn: self.run_fn, - on_shutdown: Some(f), - } - } -} - -impl<'a, OnShutdown> ContinuousBuilder<'a, (), OnShutdown> { - pub fn run(self, f: NewTask) -> ContinuousBuilder<'a, NewTask, OnShutdown> - where - NewTask: AsyncFnOnce(ShutdownToken) -> Result<(), IggyError> + 'static, - { - ContinuousBuilder { - reg: self.reg, - name: self.name, - critical: self.critical, - run_fn: Some(f), - on_shutdown: self.on_shutdown, - } - } -} - -impl<'a, Task> ContinuousBuilder<'a, Task, NoShutdown> -where - Task: AsyncFnOnce(ShutdownToken) -> Result<(), IggyError> + 'static, -{ - pub fn spawn(self) { - if let Some(f) = self.run_fn { - self.reg - .spawn_continuous_closure(self.name, self.critical, f, Some(|_| async {})); - } else { - panic!("run() must be called before spawn()"); - } - } -} - -impl<'a, Task, OnShutdown> ContinuousBuilder<'a, Task, OnShutdown> -where - Task: AsyncFnOnce(ShutdownToken) -> Result<(), IggyError> + 'static, - OnShutdown: AsyncFnOnce(Result<(), IggyError>) + 'static, -{ - pub fn spawn(self) { - if let Some(f) = self.run_fn { - self.reg - .spawn_continuous_closure(self.name, self.critical, f, self.on_shutdown); - } else { - panic!("run() must be called before spawn()"); - } - } -} diff --git a/core/server/src/shard/task_registry/builders/oneshot.rs b/core/server/src/shard/task_registry/builders/oneshot.rs deleted file mode 100644 index abf0b727c5..0000000000 --- a/core/server/src/shard/task_registry/builders/oneshot.rs +++ /dev/null @@ -1,120 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use super::NoShutdown; -use crate::shard::task_registry::ShutdownToken; -use crate::shard::task_registry::registry::TaskRegistry; -use iggy_common::IggyError; -use std::ops::AsyncFnOnce; -use std::time::Duration; - -pub struct OneShotBuilder<'a, Task, OnShutdown = NoShutdown> { - reg: &'a TaskRegistry, - name: &'static str, - critical: bool, - timeout: Option, - run_fn: Option, - on_shutdown: Option, -} - -impl<'a> OneShotBuilder<'a, (), NoShutdown> { - pub fn new(reg: &'a TaskRegistry, name: &'static str) -> Self { - Self { - reg, - name, - critical: false, - timeout: None, - run_fn: None, - on_shutdown: None, - } - } -} - -impl<'a, Task, OnShutdown> OneShotBuilder<'a, Task, OnShutdown> { - pub fn critical(mut self, c: bool) -> Self { - self.critical = c; - self - } - - pub fn timeout(mut self, d: Duration) -> Self { - self.timeout = Some(d); - self - } - - pub fn on_shutdown(self, f: NewShutdown) -> OneShotBuilder<'a, Task, NewShutdown> - where - NewShutdown: AsyncFnOnce(Result<(), IggyError>) + 'static, - { - OneShotBuilder { - reg: self.reg, - name: self.name, - critical: self.critical, - timeout: self.timeout, - run_fn: self.run_fn, - on_shutdown: Some(f), - } - } -} - -impl<'a, OnShutdown> OneShotBuilder<'a, (), OnShutdown> { - pub fn run(self, f: NewTask) -> OneShotBuilder<'a, NewTask, OnShutdown> - where - NewTask: AsyncFnOnce(ShutdownToken) -> Result<(), IggyError> + 'static, - { - OneShotBuilder { - reg: self.reg, - name: self.name, - critical: self.critical, - timeout: self.timeout, - run_fn: Some(f), - on_shutdown: self.on_shutdown, - } - } -} - -impl<'a, Task> OneShotBuilder<'a, Task, NoShutdown> -where - Task: AsyncFnOnce(ShutdownToken) -> Result<(), IggyError> + 'static, -{ - pub fn spawn(self) { - let run_fn = self.run_fn.expect("run() must be called before spawn()"); - self.reg.spawn_oneshot_closure( - self.name, - self.critical, - self.timeout, - run_fn, - Some(|_| async {}), - ); - } -} - -impl<'a, Task, OnShutdown> OneShotBuilder<'a, Task, OnShutdown> -where - Task: AsyncFnOnce(ShutdownToken) -> Result<(), IggyError> + 'static, - OnShutdown: AsyncFnOnce(Result<(), IggyError>) + 'static, -{ - pub fn spawn(self) { - let run_fn = self.run_fn.expect("run() must be called before spawn()"); - self.reg.spawn_oneshot_closure( - self.name, - self.critical, - self.timeout, - run_fn, - self.on_shutdown, - ); - } -} diff --git a/core/server/src/shard/task_registry/builders/periodic.rs b/core/server/src/shard/task_registry/builders/periodic.rs deleted file mode 100644 index e91c4b8107..0000000000 --- a/core/server/src/shard/task_registry/builders/periodic.rs +++ /dev/null @@ -1,135 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use super::NoShutdown; -use crate::shard::task_registry::ShutdownToken; -use crate::shard::task_registry::registry::TaskRegistry; -use iggy_common::IggyError; -use std::ops::{AsyncFn, AsyncFnOnce}; -use std::time::Duration; - -pub struct PeriodicBuilder<'a, Tick, OnShutdown = NoShutdown> { - reg: &'a TaskRegistry, - name: &'static str, - critical: bool, - period: Option, - last_on_shutdown: bool, - tick_fn: Option, - on_shutdown: Option, -} - -impl<'a> PeriodicBuilder<'a, (), NoShutdown> { - pub fn new(reg: &'a TaskRegistry, name: &'static str) -> Self { - Self { - reg, - name, - critical: false, - period: None, - last_on_shutdown: false, - tick_fn: None, - on_shutdown: None, - } - } -} - -impl<'a, Tick, OnShutdown> PeriodicBuilder<'a, Tick, OnShutdown> { - pub fn every(mut self, d: Duration) -> Self { - self.period = Some(d); - self - } - - pub fn critical(mut self, c: bool) -> Self { - self.critical = c; - self - } - - pub fn last_tick_on_shutdown(mut self, v: bool) -> Self { - self.last_on_shutdown = v; - self - } - - pub fn on_shutdown(self, f: NewShutdown) -> PeriodicBuilder<'a, Tick, NewShutdown> - where - NewShutdown: AsyncFnOnce(Result<(), IggyError>) + 'static, - { - PeriodicBuilder { - reg: self.reg, - name: self.name, - critical: self.critical, - period: self.period, - last_on_shutdown: self.last_on_shutdown, - tick_fn: self.tick_fn, - on_shutdown: Some(f), - } - } -} - -impl<'a> PeriodicBuilder<'a, ()> { - pub fn tick(self, f: NewTick) -> PeriodicBuilder<'a, NewTick> - where - NewTick: AsyncFn(ShutdownToken) -> Result<(), IggyError> + 'static, - { - PeriodicBuilder { - reg: self.reg, - name: self.name, - critical: self.critical, - period: self.period, - last_on_shutdown: self.last_on_shutdown, - tick_fn: Some(f), - on_shutdown: self.on_shutdown, - } - } -} - -impl<'a, Tick> PeriodicBuilder<'a, Tick, NoShutdown> -where - Tick: AsyncFn(ShutdownToken) -> Result<(), IggyError> + 'static, -{ - pub fn spawn(self) { - let period = self.period.expect("period required - use .every()"); - let tick_fn = self.tick_fn.expect("tick function required - use .tick()"); - - self.reg.spawn_periodic_closure( - self.name, - period, - self.critical, - self.last_on_shutdown, - tick_fn, - Some(|_| async {}), - ); - } -} - -impl<'a, Tick, OnShutdown> PeriodicBuilder<'a, Tick, OnShutdown> -where - Tick: AsyncFn(ShutdownToken) -> Result<(), IggyError> + 'static, - OnShutdown: AsyncFnOnce(Result<(), IggyError>) + 'static, -{ - pub fn spawn(self) { - let period = self.period.expect("period required - use .every()"); - let tick_fn = self.tick_fn.expect("tick function required - use .tick()"); - - self.reg.spawn_periodic_closure( - self.name, - period, - self.critical, - self.last_on_shutdown, - tick_fn, - self.on_shutdown, - ); - } -} diff --git a/core/server/src/shard/task_registry/mod.rs b/core/server/src/shard/task_registry/mod.rs deleted file mode 100644 index 8020c34af1..0000000000 --- a/core/server/src/shard/task_registry/mod.rs +++ /dev/null @@ -1,23 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod builders; -pub mod registry; -pub mod shutdown; - -pub use registry::TaskRegistry; -pub use shutdown::{Shutdown, ShutdownToken}; diff --git a/core/server/src/shard/task_registry/registry.rs b/core/server/src/shard/task_registry/registry.rs deleted file mode 100644 index 531cdcbdcb..0000000000 --- a/core/server/src/shard/task_registry/registry.rs +++ /dev/null @@ -1,732 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use super::shutdown::{Shutdown, ShutdownToken}; -use crate::shard::transmission::connector::StopSender; -use compio::runtime::JoinHandle; -use futures::FutureExt; -use futures::future::join_all; -use iggy_common::IggyError; -use std::cell::RefCell; -use std::collections::HashMap; -use std::future::Future; -use std::ops::{AsyncFn, AsyncFnOnce}; -use std::panic::AssertUnwindSafe; -use std::time::{Duration, Instant}; -use tracing::{debug, error, trace, warn}; - -#[derive(Debug)] -enum Kind { - Continuous, - Periodic, - OneShot, -} - -#[derive(Debug)] -struct TaskHandle { - name: String, - kind: Kind, - handle: JoinHandle>, - critical: bool, -} - -pub struct TaskRegistry { - shard_id: u16, - shutdown: Shutdown, - shutdown_token: ShutdownToken, - all_stop_senders: Vec, - long_running: RefCell>, - oneshots: RefCell>, - connections: RefCell>>, - shutting_down: RefCell, -} - -impl TaskRegistry { - pub fn new(shard_id: u16, all_stop_senders: Vec) -> Self { - let (s, t) = Shutdown::new(); - Self { - shard_id, - shutdown: s, - shutdown_token: t, - all_stop_senders, - long_running: RefCell::new(vec![]), - oneshots: RefCell::new(vec![]), - connections: RefCell::new(HashMap::new()), - shutting_down: RefCell::new(false), - } - } - - pub fn shutdown_token(&self) -> ShutdownToken { - self.shutdown_token.clone() - } - - pub(crate) fn spawn_continuous_closure( - &self, - name: &'static str, - critical: bool, - f: Task, - on_shutdown: Option, - ) where - Task: AsyncFnOnce(ShutdownToken) -> Result<(), IggyError> + 'static, - OnShutdown: AsyncFnOnce(Result<(), IggyError>) + 'static, - { - if *self.shutting_down.borrow() { - warn!( - "Attempted to spawn continuous task '{}' during shutdown", - name - ); - return; - } - - let shutdown = self.shutdown_token.clone(); - let shard_id = self.shard_id; - let all_stop_senders = self.all_stop_senders.clone(); - - let handle = compio::runtime::spawn(async move { - trace!("continuous '{}' starting on shard {}", name, shard_id); - - let fut = AssertUnwindSafe(f(shutdown)).catch_unwind(); - let result = fut.await; - - let (r, should_trigger_shutdown) = match result { - Ok(r) => { - match &r { - Ok(()) => debug!("continuous '{}' completed on shard {}", name, shard_id), - Err(e) => { - error!("continuous '{}' failed on shard {}: {}", name, shard_id, e); - } - } - // Trigger shutdown for critical task errors - let trigger = critical && r.is_err(); - (r, trigger) - } - Err(panic_payload) => { - let panic_msg = panic_payload - .downcast_ref::<&str>() - .map(|s| s.to_string()) - .or_else(|| panic_payload.downcast_ref::().cloned()) - .unwrap_or_else(|| "unknown panic".to_string()); - error!( - "continuous '{}' panicked on shard {}: {}", - name, shard_id, panic_msg - ); - // Trigger shutdown for critical task panics - (Err(IggyError::Error), critical) - } - }; - - // Execute on_shutdown callback if provided - if let Some(shutdown_fn) = on_shutdown { - trace!("continuous '{}' executing on_shutdown callback", name); - shutdown_fn(r.clone()).await; - } - - // Trigger shutdown for ALL shards when critical task fails - if should_trigger_shutdown { - error!( - "Critical task '{}' failed on shard {}, triggering shutdown for all shards", - name, shard_id - ); - for stop_sender in &all_stop_senders { - let _ = stop_sender.try_send(()); - } - } - - r - }); - - self.long_running.borrow_mut().push(TaskHandle { - name: name.into(), - kind: Kind::Continuous, - handle, - critical, - }); - } - - pub(crate) fn spawn_periodic_closure( - &self, - name: &'static str, - period: Duration, - critical: bool, - last_on_shutdown: bool, - tick_fn: Tick, - on_shutdown: Option, - ) where - Tick: AsyncFn(ShutdownToken) -> Result<(), IggyError> + 'static, - OnShutdown: AsyncFnOnce(Result<(), IggyError>) + 'static, - { - if *self.shutting_down.borrow() { - warn!( - "Attempted to spawn periodic task '{}' during shutdown", - name - ); - return; - } - - let shutdown = self.shutdown_token.clone(); - let shutdown_for_task = self.shutdown_token.clone(); - let shard_id = self.shard_id; - let all_stop_senders = self.all_stop_senders.clone(); - - let handle = compio::runtime::spawn(async move { - trace!( - "periodic '{}' every {:?} on shard {}", - name, period, shard_id - ); - - loop { - if !shutdown.sleep_or_shutdown(period).await { - break; - } - - let fut = AssertUnwindSafe(tick_fn(shutdown_for_task.clone())).catch_unwind(); - match fut.await { - Ok(Ok(())) => {} - Ok(Err(e)) => { - error!( - "periodic '{}' tick failed on shard {}: {}", - name, shard_id, e - ); - } - Err(panic_payload) => { - let panic_msg = panic_payload - .downcast_ref::<&str>() - .map(|s| s.to_string()) - .or_else(|| panic_payload.downcast_ref::().cloned()) - .unwrap_or_else(|| "unknown panic".to_string()); - error!( - "periodic '{}' tick panicked on shard {}: {}", - name, shard_id, panic_msg - ); - if critical { - error!( - "Critical periodic task '{}' panicked on shard {}, triggering shutdown", - name, shard_id - ); - for stop_sender in &all_stop_senders { - let _ = stop_sender.try_send(()); - } - return Err(IggyError::Error); - } - } - } - } - - if last_on_shutdown { - const FINAL_TICK_TIMEOUT: Duration = Duration::from_secs(5); - trace!( - "periodic '{}' executing final tick on shutdown (timeout: {:?})", - name, FINAL_TICK_TIMEOUT - ); - - let fut = tick_fn(shutdown_for_task); - match compio::time::timeout(FINAL_TICK_TIMEOUT, fut).await { - Ok(Ok(())) => trace!("periodic '{}' final tick completed", name), - Ok(Err(e)) => error!("periodic '{}' final tick failed: {}", name, e), - Err(_) => error!( - "periodic '{}' final tick timed out after {:?}", - name, FINAL_TICK_TIMEOUT - ), - } - } - - let result = Ok(()); - - if let Some(on_shutdown) = on_shutdown { - on_shutdown(result.clone()).await; - } - - result - }); - - self.long_running.borrow_mut().push(TaskHandle { - name: name.into(), - kind: Kind::Periodic, - handle, - critical, - }); - } - - pub(crate) fn spawn_oneshot_closure( - &self, - name: &'static str, - critical: bool, - timeout: Option, - f: Task, - on_shutdown: Option, - ) where - Task: AsyncFnOnce(ShutdownToken) -> Result<(), IggyError> + 'static, - OnShutdown: AsyncFnOnce(Result<(), IggyError>) + 'static, - { - if *self.shutting_down.borrow() { - warn!("Attempted to spawn oneshot task '{}' during shutdown", name); - return; - } - - let shutdown = self.shutdown_token.clone(); - let shard_id = self.shard_id; - let all_stop_senders = self.all_stop_senders.clone(); - - let handle = compio::runtime::spawn(async move { - trace!("oneshot '{}' starting on shard {}", name, shard_id); - - let fut = if let Some(d) = timeout { - let inner_fut = AssertUnwindSafe(f(shutdown)).catch_unwind(); - match compio::time::timeout(d, inner_fut).await { - Ok(Ok(r)) => Ok(r), - Ok(Err(panic_payload)) => Err(panic_payload), - Err(_) => Ok(Err(IggyError::TaskTimeout)), - } - } else { - AssertUnwindSafe(f(shutdown)).catch_unwind().await - }; - - let r = match fut { - Ok(r) => { - match &r { - Ok(()) => trace!("oneshot '{}' completed on shard {}", name, shard_id), - Err(e) => { - error!("oneshot '{}' failed on shard {}: {}", name, shard_id, e); - if critical { - error!( - "Critical oneshot task '{}' failed on shard {}, triggering shutdown", - name, shard_id - ); - for stop_sender in &all_stop_senders { - let _ = stop_sender.try_send(()); - } - } - } - } - r - } - Err(panic_payload) => { - let panic_msg = panic_payload - .downcast_ref::<&str>() - .map(|s| s.to_string()) - .or_else(|| panic_payload.downcast_ref::().cloned()) - .unwrap_or_else(|| "unknown panic".to_string()); - error!( - "oneshot '{}' panicked on shard {}: {}", - name, shard_id, panic_msg - ); - if critical { - error!( - "Critical oneshot task '{}' panicked on shard {}, triggering shutdown", - name, shard_id - ); - for stop_sender in &all_stop_senders { - let _ = stop_sender.try_send(()); - } - } - Err(IggyError::Error) - } - }; - - if let Some(on_shutdown) = on_shutdown { - on_shutdown(r.clone()).await; - } - - r - }); - - self.oneshots.borrow_mut().push(TaskHandle { - name: name.into(), - kind: Kind::OneShot, - handle, - critical, - }); - } - - pub async fn graceful_shutdown(&self, timeout: Duration) -> bool { - let start = Instant::now(); - *self.shutting_down.borrow_mut() = true; - self.shutdown_connections(); - self.shutdown.trigger(); - - // First shutdown long-running tasks (continuous and periodic) - let long = self.long_running.take(); - let long_ok = if !long.is_empty() { - debug!( - "Shutting down {} long-running task(s) on shard {}", - long.len(), - self.shard_id - ); - self.await_with_timeout(long, timeout).await - } else { - true - }; - - // Calculate remaining time for oneshots - let elapsed = start.elapsed(); - let remaining = timeout.saturating_sub(elapsed); - - // Then shutdown oneshot tasks with remaining time - let ones = self.oneshots.take(); - let ones_ok = if !ones.is_empty() { - if remaining.is_zero() { - warn!( - "No time remaining for {} oneshot task(s) on shard {}, they will be cancelled", - ones.len(), - self.shard_id - ); - false - } else { - debug!( - "Shutting down {} oneshot task(s) on shard {} with {:?} remaining", - ones.len(), - self.shard_id, - remaining - ); - self.await_with_timeout(ones, remaining).await - } - } else { - true - }; - - let total_elapsed = start.elapsed(); - if long_ok && ones_ok { - debug!( - "Graceful shutdown completed successfully on shard {} in {:?}", - self.shard_id, total_elapsed - ); - } else { - warn!( - "Graceful shutdown completed with failures on shard {} in {:?}", - self.shard_id, total_elapsed - ); - } - - long_ok && ones_ok - } - - async fn await_with_timeout(&self, tasks: Vec, timeout: Duration) -> bool { - if tasks.is_empty() { - return true; - } - let results = join_all(tasks.into_iter().map(|t| async move { - match compio::time::timeout(timeout, t.handle).await { - Ok(Ok(Ok(()))) => true, - Ok(Ok(Err(e))) => { - error!("task '{}' of kind {:?} failed: {}", t.name, t.kind, e); - !t.critical - } - Ok(Err(_)) => { - error!("task '{}' of kind {:?} panicked", t.name, t.kind); - !t.critical - } - Err(_) => { - error!( - "task '{}' of kind {:?} timed out after {:?}", - t.name, t.kind, timeout - ); - !t.critical - } - } - })) - .await; - - results.into_iter().all(|x| x) - } - - #[cfg(test)] - async fn await_all(&self, tasks: Vec) -> bool { - if tasks.is_empty() { - return true; - } - let results = join_all(tasks.into_iter().map(|t| async move { - match t.handle.await { - Ok(Ok(())) => true, - Ok(Err(e)) => { - error!("task '{}' failed: {}", t.name, e); - !t.critical - } - Err(_) => { - error!("task '{}' panicked", t.name); - !t.critical - } - } - })) - .await; - results.into_iter().all(|x| x) - } - - pub fn add_connection(&self, client_id: u32) -> async_channel::Receiver<()> { - let (tx, rx) = async_channel::bounded(1); - self.connections.borrow_mut().insert(client_id, tx); - rx - } - - pub fn remove_connection(&self, client_id: &u32) { - self.connections.borrow_mut().remove(client_id); - } - - fn shutdown_connections(&self) { - // Close all connection channels to signal shutdown - // We use close() instead of send_blocking() to avoid potential blocking - for tx in self.connections.borrow().values() { - tx.close(); - } - } - - /// Spawn a connection handler that doesn't need to be tracked for shutdown. - /// These handlers have their own shutdown mechanism via connection channels. - /// If the handler panics, shutdown is triggered for all shards. - pub fn spawn_connection(&self, future: F) - where - F: Future + 'static, - { - let shard_id = self.shard_id; - let all_stop_senders = self.all_stop_senders.clone(); - - compio::runtime::spawn(async move { - let fut = AssertUnwindSafe(future).catch_unwind(); - if let Err(panic_payload) = fut.await { - let panic_msg = panic_payload - .downcast_ref::<&str>() - .map(|s| s.to_string()) - .or_else(|| panic_payload.downcast_ref::().cloned()) - .unwrap_or_else(|| "unknown panic".to_string()); - - error!( - "Connection handler panicked on shard {}: {}, triggering shutdown", - shard_id, panic_msg - ); - - for stop_sender in &all_stop_senders { - let _ = stop_sender.try_send(()); - } - } - }) - .detach(); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn create_test_registry(shard_id: u16) -> TaskRegistry { - let (stop_sender, _stop_receiver) = async_channel::bounded(1); - TaskRegistry::new(shard_id, vec![stop_sender]) - } - - #[compio::test] - async fn test_oneshot_completion_detection() { - let registry = create_test_registry(1); - - // Spawn a failing non-critical task - registry - .oneshot("failing_non_critical") - .run(|_shutdown| async { Err(IggyError::Error) }) - .spawn(); - - // Spawn a successful task - registry - .oneshot("successful") - .run(|_shutdown| async { Ok(()) }) - .spawn(); - - // Wait for all tasks - let all_ok = registry.await_all(registry.oneshots.take()).await; - - // Should return true because the failing task is not critical - assert!(all_ok); - } - - #[compio::test] - async fn test_oneshot_critical_failure() { - let registry = create_test_registry(1); - - // Spawn a failing critical task - registry - .oneshot("failing_critical") - .critical(true) - .run(|_shutdown| async { Err(IggyError::Error) }) - .spawn(); - - // Wait for all tasks - let all_ok = registry.await_all(registry.oneshots.take()).await; - - // Should return false because the failing task is critical - assert!(!all_ok); - } - - #[compio::test] - async fn test_shutdown_prevents_spawning() { - let registry = create_test_registry(1); - - // Trigger shutdown - *registry.shutting_down.borrow_mut() = true; - - let initial_count = registry.oneshots.borrow().len(); - - // Try to spawn after shutdown - registry - .oneshot("should_not_spawn") - .run(|_shutdown| async { Ok(()) }) - .spawn(); - - // Task should not be added - assert_eq!(registry.oneshots.borrow().len(), initial_count); - } - - #[compio::test] - async fn test_timeout_error() { - let registry = create_test_registry(1); - - // Create a task that will timeout - let handle = compio::runtime::spawn(async move { - compio::time::sleep(Duration::from_secs(10)).await; - Ok(()) - }); - - let task_handle = TaskHandle { - name: "timeout_test".to_string(), - kind: Kind::OneShot, - handle, - critical: false, - }; - - let tasks = vec![task_handle]; - let all_ok = registry - .await_with_timeout(tasks, Duration::from_millis(50)) - .await; - - // Should return true because the task is not critical - assert!(all_ok); - } - - #[compio::test] - async fn test_composite_timeout() { - let registry = create_test_registry(1); - - // Create a long-running task that takes 100ms - let long_handle = compio::runtime::spawn(async move { - compio::time::sleep(Duration::from_millis(100)).await; - Ok(()) - }); - - registry.long_running.borrow_mut().push(TaskHandle { - name: "long_task".to_string(), - kind: Kind::Continuous, - handle: long_handle, - critical: false, - }); - - // Create a oneshot that would succeed quickly - let oneshot_handle = compio::runtime::spawn(async move { - compio::time::sleep(Duration::from_millis(10)).await; - Ok(()) - }); - - registry.oneshots.borrow_mut().push(TaskHandle { - name: "quick_oneshot".to_string(), - kind: Kind::OneShot, - handle: oneshot_handle, - critical: false, - }); - - // Give total timeout of 150ms - // Long-running should complete in ~100ms - // Oneshot should have ~50ms remaining, which is enough - let all_ok = registry.graceful_shutdown(Duration::from_millis(150)).await; - assert!(all_ok); - } - - #[compio::test] - async fn test_composite_timeout_insufficient() { - let registry = create_test_registry(1); - - // Create a long-running task that takes 50ms - let long_handle = compio::runtime::spawn(async move { - compio::time::sleep(Duration::from_millis(50)).await; - Ok(()) - }); - - registry.long_running.borrow_mut().push(TaskHandle { - name: "long_task".to_string(), - kind: Kind::Continuous, - handle: long_handle, - critical: false, - }); - - // Create a oneshot that would take 100ms (much longer) - let oneshot_handle = compio::runtime::spawn(async move { - compio::time::sleep(Duration::from_millis(100)).await; - Ok(()) - }); - - registry.oneshots.borrow_mut().push(TaskHandle { - name: "slow_oneshot".to_string(), - kind: Kind::OneShot, - handle: oneshot_handle, - critical: true, // Make it critical so failure is detected - }); - - // Give total timeout of 60ms - // Long-running should complete in ~50ms - // Oneshot would need 100ms but only has ~10ms, so it should definitely fail - let all_ok = registry.graceful_shutdown(Duration::from_millis(60)).await; - assert!(!all_ok); // Should fail because critical oneshot times out - } - - #[compio::test] - async fn test_periodic_last_tick_timeout() { - // This test verifies that periodic tasks with last_tick_on_shutdown - // don't hang shutdown if the final tick takes too long - let registry = create_test_registry(1); - - // Create a handle that simulates a periodic task whose final tick will hang - let handle = compio::runtime::spawn(async move { - // Simulate the periodic task loop that already exited - // Now simulate the last_tick_on_shutdown logic with a hanging tick - const FINAL_TICK_TIMEOUT: Duration = Duration::from_millis(100); - let fut = async { - // This would hang for 500ms without timeout - compio::time::sleep(Duration::from_millis(500)).await; - Ok::<(), IggyError>(()) - }; - - match compio::time::timeout(FINAL_TICK_TIMEOUT, fut).await { - Ok(Ok(())) => {} - Ok(Err(_)) => {} - Err(_) => { - // Timeout occurred as expected - } - } - Ok(()) - }); - - registry.long_running.borrow_mut().push(TaskHandle { - name: "periodic_with_slow_final".to_string(), - kind: Kind::Periodic, - handle, - critical: false, - }); - - // Shutdown should complete in ~100ms (the FINAL_TICK_TIMEOUT), not 500ms - let start = std::time::Instant::now(); - let all_ok = registry.graceful_shutdown(Duration::from_secs(1)).await; - let elapsed = start.elapsed(); - - // Should complete in about 100ms due to the timeout, not hang for 500ms - assert!(elapsed >= Duration::from_millis(80)); // At least 80ms - assert!(elapsed < Duration::from_millis(200)); // But less than 200ms (not the full 500ms) - assert!(all_ok); - } -} diff --git a/core/server/src/shard/task_registry/shutdown.rs b/core/server/src/shard/task_registry/shutdown.rs deleted file mode 100644 index 09eab0a3ab..0000000000 --- a/core/server/src/shard/task_registry/shutdown.rs +++ /dev/null @@ -1,232 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use async_channel::{Receiver, Sender, bounded}; -use futures::FutureExt; -use std::sync::Arc; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::time::Duration; -use tracing::trace; - -/// Coordinates graceful shutdown across multiple tasks -#[derive(Clone)] -pub struct Shutdown { - sender: Sender<()>, - is_triggered: Arc, -} - -impl Shutdown { - pub fn new() -> (Self, ShutdownToken) { - let (sender, receiver) = bounded(1); - let is_triggered = Arc::new(AtomicBool::new(false)); - - let shutdown = Self { - sender, - is_triggered: is_triggered.clone(), - }; - - let token = ShutdownToken { - receiver, - is_triggered, - }; - - (shutdown, token) - } - - pub fn trigger(&self) { - if self.is_triggered.swap(true, Ordering::SeqCst) { - return; - } - - trace!("Triggering shutdown signal"); - let _ = self.sender.close(); - } - - pub fn is_triggered(&self) -> bool { - self.is_triggered.load(Ordering::Relaxed) - } -} - -/// Token held by tasks to receive shutdown signals -#[derive(Clone)] -pub struct ShutdownToken { - receiver: Receiver<()>, - is_triggered: Arc, -} - -impl ShutdownToken { - /// Wait for shutdown signal - pub async fn wait(&self) { - let _ = self.receiver.recv().await; - } - - /// Check if shutdown has been triggered (non-blocking) - pub fn is_triggered(&self) -> bool { - self.is_triggered.load(Ordering::Relaxed) - } - - /// Sleep for the specified duration or until shutdown is triggered - /// Returns true if the full duration elapsed, false if shutdown was triggered - pub async fn sleep_or_shutdown(&self, duration: Duration) -> bool { - futures::select! { - _ = self.wait().fuse() => false, - _ = compio::time::sleep(duration).fuse() => !self.is_triggered(), - } - } - - /// Creates a scoped shutdown pair (child `Shutdown`, combined `ShutdownToken`). - /// - /// This is a bit complicated, but it needs to be this way to avoid deadlocks. - /// - /// The returned token fires when EITHER the parent or the child is triggered, - /// while a child trigger does NOT propagate back to the parent. - /// Internally spawns a tiny forwarder to merge both signals into one channel, - /// so callers can await a single `wait()` and use fast `is_triggered()` checks - /// without writing `select!` at every call site. - /// Use when a subtree needs cancelation that respects parent cancelation, - /// but can also be canceled locally. - pub fn child(&self) -> (Shutdown, ShutdownToken) { - let (child_shutdown, child_token) = Shutdown::new(); - let parent_receiver = self.receiver.clone(); - let child_receiver = child_token.receiver.clone(); - - let (combined_sender, combined_receiver) = bounded(1); - let combined_is_triggered = Arc::new(AtomicBool::new(false)); - - let parent_triggered = self.is_triggered.clone(); - let child_triggered = child_token.is_triggered.clone(); - let combined_flag_for_task = combined_is_triggered.clone(); - - compio::runtime::spawn(async move { - futures::select! { - _ = parent_receiver.recv().fuse() => { - trace!("Child token triggered by parent shutdown"); - }, - _ = child_receiver.recv().fuse() => { - trace!("Child token triggered by child shutdown"); - }, - } - - if parent_triggered.load(Ordering::Relaxed) || child_triggered.load(Ordering::Relaxed) { - combined_flag_for_task.store(true, Ordering::SeqCst); - } - - let _ = combined_sender.close(); - }) - .detach(); - - let combined_token = ShutdownToken { - receiver: combined_receiver, - is_triggered: combined_is_triggered, - }; - - (child_shutdown, combined_token) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[compio::test] - async fn test_shutdown_trigger() { - let (shutdown, token) = Shutdown::new(); - - assert!(!token.is_triggered()); - - shutdown.trigger(); - - assert!(token.is_triggered()); - - token.wait().await; - } - - #[compio::test] - async fn test_sleep_or_shutdown_completes() { - let (_shutdown, token) = Shutdown::new(); - - let completed = token.sleep_or_shutdown(Duration::from_millis(10)).await; - assert!(completed); - } - - #[compio::test] - async fn test_sleep_or_shutdown_interrupted() { - let (shutdown, token) = Shutdown::new(); - - // Trigger shutdown after a short delay - let shutdown_clone = shutdown.clone(); - compio::runtime::spawn(async move { - compio::time::sleep(Duration::from_millis(10)).await; - shutdown_clone.trigger(); - }) - .detach(); - - // Should be interrupted - let completed = token.sleep_or_shutdown(Duration::from_secs(10)).await; - assert!(!completed); - } - - #[compio::test] - async fn test_child_token_parent_trigger() { - let (parent_shutdown, parent_token) = Shutdown::new(); - let (_child_shutdown, combined_token) = parent_token.child(); - - assert!(!combined_token.is_triggered()); - - // Trigger parent shutdown - parent_shutdown.trigger(); - - // Combined token should be triggered - combined_token.wait().await; - assert!(combined_token.is_triggered()); - } - - #[compio::test] - async fn test_child_token_child_trigger() { - let (_parent_shutdown, parent_token) = Shutdown::new(); - let (child_shutdown, combined_token) = parent_token.child(); - - assert!(!combined_token.is_triggered()); - - // Trigger child shutdown - child_shutdown.trigger(); - - // Combined token should be triggered - combined_token.wait().await; - assert!(combined_token.is_triggered()); - } - - #[compio::test] - async fn test_child_token_no_polling_overhead() { - let (_parent_shutdown, parent_token) = Shutdown::new(); - let (_child_shutdown, combined_token) = parent_token.child(); - - // Test that we can create many child tokens without performance issues - let start = std::time::Instant::now(); - for _ in 0..100 { - let _ = combined_token.child(); - } - let elapsed = start.elapsed(); - - // Should complete very quickly since there's no polling - assert!( - elapsed.as_millis() < 100, - "Creating child tokens took too long: {:?}", - elapsed - ); - } -} diff --git a/core/server/src/shard/tasks/continuous/http_server.rs b/core/server/src/shard/tasks/continuous/http_server.rs deleted file mode 100644 index 2b88639905..0000000000 --- a/core/server/src/shard/tasks/continuous/http_server.rs +++ /dev/null @@ -1,40 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::bootstrap::resolve_persister; -use crate::http::http_server::start_http_server; -use crate::shard::IggyShard; -use crate::shard::task_registry::ShutdownToken; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::info; - -pub fn spawn_http_server(shard: Rc) { - let shard_clone = shard.clone(); - shard - .task_registry - .continuous("http_server") - .critical(true) - .run(move |shutdown| http_server(shard_clone, shutdown)) - .spawn(); -} - -async fn http_server(shard: Rc, shutdown: ShutdownToken) -> Result<(), IggyError> { - info!("Starting HTTP server on shard: {}", shard.id); - let persister = resolve_persister(shard.config.system.partition.enforce_fsync); - start_http_server(shard.config.http.clone(), persister, shard, shutdown).await -} diff --git a/core/server/src/shard/tasks/continuous/message_pump.rs b/core/server/src/shard/tasks/continuous/message_pump.rs deleted file mode 100644 index 2837ef626d..0000000000 --- a/core/server/src/shard/tasks/continuous/message_pump.rs +++ /dev/null @@ -1,135 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::shard::task_registry::ShutdownToken; -use crate::shard::transmission::frame::ShardFrame; -use crate::shard::{IggyShard, handlers::handle_shard_message}; -use futures::FutureExt; -use std::rc::Rc; -use tracing::{debug, error, info}; - -pub fn spawn_message_pump(shard: Rc) { - let shard_clone = shard.clone(); - shard - .task_registry - .continuous("message_pump") - .critical(true) - .run(move |shutdown| message_pump(shard_clone, shutdown)) - .spawn(); -} - -/// Single serialization point for all partition mutations on this shard. -/// -/// Every operation that mutates `local_partitions` — appends, segment rotation, flush, -/// segment deletion — is dispatched exclusively through this pump. The loop awaits each -/// `process_frame` to completion before dequeuing the next message, so handlers never -/// interleave even across internal `.await` points (disk I/O, fsync). -/// -/// Periodic tasks (message_saver, message_cleaner) run as separate futures on the same -/// compio thread but **cannot** mutate partitions directly. They read partition metadata -/// via `borrow()` and enqueue mutation requests back into this pump's channel. Those -/// requests block on a response that is only sent after the current frame completes, -/// guaranteeing strict ordering. -/// -/// This invariant replaces per-partition write locks and eliminates TOCTOU races between -/// concurrent handlers. All `pub(crate)` mutation methods on `IggyShard` (e.g. -/// `append_messages_to_local_partition`, `delete_expired_segments`, -/// `rotate_segment_in_local_partitions`) assume they are called from within this pump. -async fn message_pump( - shard: Rc, - shutdown: ShutdownToken, -) -> Result<(), iggy_common::IggyError> { - let Some(messages_receiver) = shard.messages_receiver.take() else { - info!("Message receiver already taken; pump not started"); - return Ok(()); - }; - - info!("Starting message passing task"); - - let receiver = messages_receiver.inner; - - loop { - futures::select! { - _ = shutdown.wait().fuse() => { - debug!("Message pump shutting down"); - break; - } - frame = receiver.recv_async().fuse() => { - match frame { - Ok(frame) => process_frame(&shard, frame).await, - Err(_) => { - debug!("Message receiver closed; exiting pump"); - break; - } - } - } - } - } - - // Drain remaining frames before flushing — any in-flight appends must - // complete so their data lands in the journal before we flush to disk. - while let Ok(frame) = receiver.try_recv() { - process_frame(&shard, frame).await; - } - - flush_and_fsync_all_partitions(&shard).await; - - Ok(()) -} - -async fn process_frame(shard: &Rc, frame: ShardFrame) { - let ShardFrame { - message, - response_sender, - } = frame; - if let (Some(response), Some(tx)) = - (handle_shard_message(shard, message).await, response_sender) - { - let _ = tx.send(response).await; - } -} - -/// Final flush + fsync of all local partitions. Runs inside the pump after -/// the main loop exits, so no other pump frame can interleave. -async fn flush_and_fsync_all_partitions(shard: &Rc) { - let namespaces = shard.get_current_shard_namespaces(); - if namespaces.is_empty() { - return; - } - - let mut flushed = 0u32; - for ns in &namespaces { - match shard - .flush_unsaved_buffer_from_local_partitions(ns, false) - .await - { - Ok(saved) if saved > 0 => flushed += 1, - Ok(_) => {} - Err(e) => error!("Shutdown flush failed for partition {:?}: {}", ns, e), - } - } - if flushed > 0 { - info!("Shutdown: flushed {flushed} partitions."); - } - - for ns in &namespaces { - if let Err(e) = shard.fsync_all_messages_from_local_partitions(ns).await { - error!("Shutdown fsync failed for partition {:?}: {}", ns, e); - } - } - info!("Shutdown: fsync complete for all partitions."); -} diff --git a/core/server/src/shard/tasks/continuous/mod.rs b/core/server/src/shard/tasks/continuous/mod.rs deleted file mode 100644 index 421d77be7e..0000000000 --- a/core/server/src/shard/tasks/continuous/mod.rs +++ /dev/null @@ -1,28 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -mod http_server; -mod message_pump; -mod quic_server; -mod tcp_server; -mod websocket_server; - -pub use http_server::spawn_http_server; -pub use message_pump::spawn_message_pump; -pub use quic_server::spawn_quic_server; -pub use tcp_server::spawn_tcp_server; -pub use websocket_server::spawn_websocket_server; diff --git a/core/server/src/shard/tasks/continuous/quic_server.rs b/core/server/src/shard/tasks/continuous/quic_server.rs deleted file mode 100644 index 93af50455e..0000000000 --- a/core/server/src/shard/tasks/continuous/quic_server.rs +++ /dev/null @@ -1,36 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::quic::quic_server; -use crate::shard::IggyShard; -use crate::shard::task_registry::ShutdownToken; -use iggy_common::IggyError; -use std::rc::Rc; - -pub fn spawn_quic_server(shard: Rc) { - let shard_clone = shard.clone(); - shard - .task_registry - .continuous("quic_server") - .critical(true) - .run(move |shutdown| quic_server_task(shard_clone, shutdown)) - .spawn(); -} - -async fn quic_server_task(shard: Rc, shutdown: ShutdownToken) -> Result<(), IggyError> { - quic_server::spawn_quic_server(shard, shutdown).await -} diff --git a/core/server/src/shard/tasks/continuous/tcp_server.rs b/core/server/src/shard/tasks/continuous/tcp_server.rs deleted file mode 100644 index 7a4348f94b..0000000000 --- a/core/server/src/shard/tasks/continuous/tcp_server.rs +++ /dev/null @@ -1,36 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::shard::IggyShard; -use crate::shard::task_registry::ShutdownToken; -use crate::tcp::tcp_server; -use iggy_common::IggyError; -use std::rc::Rc; - -pub fn spawn_tcp_server(shard: Rc) { - let shard_clone = shard.clone(); - shard - .task_registry - .continuous("tcp_server") - .critical(true) - .run(move |shutdown| tcp_server_task(shard_clone, shutdown)) - .spawn(); -} - -async fn tcp_server_task(shard: Rc, shutdown: ShutdownToken) -> Result<(), IggyError> { - tcp_server::spawn_tcp_server(shard, shutdown).await -} diff --git a/core/server/src/shard/tasks/continuous/websocket_server.rs b/core/server/src/shard/tasks/continuous/websocket_server.rs deleted file mode 100644 index 2219534861..0000000000 --- a/core/server/src/shard/tasks/continuous/websocket_server.rs +++ /dev/null @@ -1,38 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::shard::IggyShard; -use crate::shard::task_registry::ShutdownToken; -use iggy_common::IggyError; -use std::rc::Rc; - -pub fn spawn_websocket_server(shard: Rc) { - let shard_clone = shard.clone(); - shard - .task_registry - .continuous("websocket_server") - .critical(true) - .run(move |shutdown| websocket_server_task(shard_clone, shutdown)) - .spawn(); -} - -async fn websocket_server_task( - shard: Rc, - shutdown: ShutdownToken, -) -> Result<(), IggyError> { - crate::websocket::websocket_server::spawn_websocket_server(shard, shutdown).await -} diff --git a/core/server/src/shard/tasks/mod.rs b/core/server/src/shard/tasks/mod.rs deleted file mode 100644 index 7c5f23ba8e..0000000000 --- a/core/server/src/shard/tasks/mod.rs +++ /dev/null @@ -1,20 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod continuous; -pub mod oneshot; -pub mod periodic; diff --git a/core/server/src/shard/tasks/oneshot/config_writer.rs b/core/server/src/shard/tasks/oneshot/config_writer.rs deleted file mode 100644 index c48a7c8ef2..0000000000 --- a/core/server/src/shard/tasks/oneshot/config_writer.rs +++ /dev/null @@ -1,142 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::shard::IggyShard; -use crate::shard::task_registry::ShutdownToken; -use compio::io::AsyncWriteAtExt; -use err_trail::ErrContext; -use futures::FutureExt; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::{info, warn}; - -pub fn spawn_config_writer_task(shard: &Rc) { - let shard_clone = shard.clone(); - shard - .task_registry - .oneshot("config_writer") - .critical(false) - .run(move |shutdown_token| async move { write_config(shard_clone, shutdown_token).await }) - .spawn(); -} - -async fn write_config( - shard: Rc, - shutdown_token: ShutdownToken, -) -> Result<(), IggyError> { - let shard_clone = shard.clone(); - let tcp_enabled = shard.config.tcp.enabled; - let quic_enabled = shard.config.quic.enabled; - let http_enabled = shard.config.http.enabled; - let websocket_enabled = shard.config.websocket.enabled; - - let notify_receiver = shard_clone.config_writer_receiver.clone(); - - // Wait for notifications until all servers have bound, or shutdown is triggered - loop { - futures::select! { - _ = shutdown_token.wait().fuse() => { - warn!("config_writer: shutdown triggered before all servers bound, skipping config write"); - return Ok(()); - } - result = notify_receiver.recv().fuse() => { - if result.is_err() { - return Err(IggyError::CannotWriteToFile).error( - |_: &IggyError| "config_writer: notification channel closed before all servers bound", - ); - } - } - } - - let tcp_ready = !tcp_enabled || shard_clone.tcp_bound_address.get().is_some(); - let quic_ready = !quic_enabled || shard_clone.quic_bound_address.get().is_some(); - let http_ready = !http_enabled || shard_clone.http_bound_address.get().is_some(); - let websocket_ready = - !websocket_enabled || shard_clone.websocket_bound_address.get().is_some(); - - if tcp_ready && quic_ready && http_ready && websocket_ready { - break; - } - } - - #[cfg(feature = "systemd")] - crate::shard::systemd::notify_ready(); - - let mut current_config = shard_clone.config.clone(); - - let tcp_addr = shard_clone.tcp_bound_address.get(); - let quic_addr = shard_clone.quic_bound_address.get(); - let http_addr = shard_clone.http_bound_address.get(); - let websocket_addr = shard_clone.websocket_bound_address.get(); - - info!( - "Config writer: TCP addr = {:?}, QUIC addr = {:?}, HTTP addr = {:?}, WebSocket addr = {:?}", - tcp_addr, quic_addr, http_addr, websocket_addr - ); - - if let Some(tcp_addr) = tcp_addr { - current_config.tcp.address = tcp_addr.to_string(); - } - - if let Some(quic_addr) = quic_addr { - current_config.quic.address = quic_addr.to_string(); - } - - if let Some(http_addr) = http_addr { - current_config.http.address = http_addr.to_string(); - } - - if let Some(websocket_addr) = websocket_addr { - current_config.websocket.address = websocket_addr.to_string(); - } - - let runtime_path = current_config.system.get_runtime_path(); - let config_path = format!("{runtime_path}/current_config.toml"); - let content = toml::to_string(¤t_config) - .map_err(|_| IggyError::CannotWriteToFile) - .error(|_: &IggyError| "config_writer: cannot serialize current_config")?; - - let mut file = compio::fs::OpenOptions::new() - .write(true) - .create(true) - .truncate(true) - .open(&config_path) - .await - .map_err(|_| IggyError::CannotWriteToFile) - .error(|_: &IggyError| { - format!("config_writer: failed to open current config at {config_path}") - })?; - - file.write_all_at(content.into_bytes(), 0) - .await - .0 - .map_err(|_| IggyError::CannotWriteToFile) - .error(|_: &IggyError| { - format!("config_writer: failed to write current config to {config_path}") - })?; - - file.sync_all() - .await - .map_err(|_| IggyError::CannotWriteToFile) - .error(|_: &IggyError| { - format!("config_writer: failed to fsync current config to {config_path}") - })?; - - info!("Current config written and synced to: {config_path} with all bound addresses",); - - Ok(()) -} diff --git a/core/server/src/shard/tasks/oneshot/mod.rs b/core/server/src/shard/tasks/oneshot/mod.rs deleted file mode 100644 index 3bc0d71138..0000000000 --- a/core/server/src/shard/tasks/oneshot/mod.rs +++ /dev/null @@ -1,20 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -mod config_writer; - -pub use config_writer::spawn_config_writer_task; diff --git a/core/server/src/shard/tasks/periodic/heartbeat_verifier.rs b/core/server/src/shard/tasks/periodic/heartbeat_verifier.rs deleted file mode 100644 index f2c1c16848..0000000000 --- a/core/server/src/shard/tasks/periodic/heartbeat_verifier.rs +++ /dev/null @@ -1,86 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::shard::IggyShard; -use iggy_common::{IggyDuration, IggyError, IggyTimestamp}; -use std::rc::Rc; -use tracing::{debug, info, trace, warn}; - -const MAX_THRESHOLD: f64 = 1.2; - -pub fn spawn_heartbeat_verifier(shard: Rc) { - let period = shard.config.heartbeat.interval.get_duration(); - let interval = iggy_common::IggyDuration::from(period); - let max_interval = - iggy_common::IggyDuration::from((MAX_THRESHOLD * interval.as_micros() as f64) as u64); - info!( - "Heartbeats will be verified every: {}. Max allowed interval: {}.", - interval, max_interval - ); - let shard_clone = shard.clone(); - shard - .task_registry - .periodic("verify_heartbeats") - .every(period) - .tick(move |_shutdown| verify_heartbeats(shard_clone.clone())) - .spawn(); -} - -async fn verify_heartbeats(shard: Rc) -> Result<(), IggyError> { - trace!("Verifying heartbeats..."); - - // Get the period from config to compute max_interval - let period = shard.config.heartbeat.interval.get_duration(); - let interval = IggyDuration::from(period); - let max_interval = IggyDuration::from((MAX_THRESHOLD * interval.as_micros() as f64) as u64); - - let clients = shard.client_manager.get_clients(); - - let now = IggyTimestamp::now(); - let heartbeat_to = IggyTimestamp::from(now.as_micros() - max_interval.as_micros()); - debug!("Verifying heartbeats at: {now}, max allowed timestamp: {heartbeat_to}"); - - let mut stale_clients = Vec::new(); - for client in clients { - if client.last_heartbeat.as_micros() < heartbeat_to.as_micros() { - warn!( - "Stale client session: {}, last heartbeat at: {}, max allowed timestamp: {heartbeat_to}", - client.session, client.last_heartbeat, - ); - client.session.set_stale(); - stale_clients.push(client.session.client_id); - } else { - debug!( - "Valid heartbeat at: {} for client session: {}, max allowed timestamp: {heartbeat_to}", - client.last_heartbeat, client.session, - ); - } - } - - if stale_clients.is_empty() { - return Ok(()); - } - - let count = stale_clients.len(); - - for client_id in stale_clients { - shard.delete_client(client_id).await; - } - info!("Removed {count} stale clients."); - - Ok(()) -} diff --git a/core/server/src/shard/tasks/periodic/jwt_token_cleaner.rs b/core/server/src/shard/tasks/periodic/jwt_token_cleaner.rs deleted file mode 100644 index bd9943612d..0000000000 --- a/core/server/src/shard/tasks/periodic/jwt_token_cleaner.rs +++ /dev/null @@ -1,60 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::http::shared::AppState; -use crate::shard::IggyShard; -use iggy_common::{IggyError, IggyTimestamp}; -use std::rc::Rc; -use std::sync::Arc; -use std::time::Duration; -use tracing::{error, info, trace}; - -const JWT_TOKENS_CLEANER_PERIOD: Duration = Duration::from_secs(300); - -pub fn spawn_jwt_token_cleaner(shard: Rc, app_state: Arc) { - info!( - "JWT token cleaner is enabled, expired revoked tokens will be deleted every: {} seconds.", - JWT_TOKENS_CLEANER_PERIOD.as_secs() - ); - shard - .task_registry - .periodic("clear_jwt_tokens") - .every(JWT_TOKENS_CLEANER_PERIOD) - .tick(move |_shutdown| clear_jwt_tokens(app_state.clone())) - .spawn(); -} - -async fn clear_jwt_tokens(app_state: Arc) -> Result<(), IggyError> { - trace!("Checking for expired revoked JWT tokens..."); - - let now = IggyTimestamp::now().to_secs(); - - match app_state - .jwt_manager - .delete_expired_revoked_tokens(now) - .await - { - Ok(()) => { - trace!("Successfully cleaned up expired revoked JWT tokens"); - } - Err(err) => { - error!("Failed to delete expired revoked JWT tokens: {}", err); - } - } - - Ok(()) -} diff --git a/core/server/src/shard/tasks/periodic/message_cleaner.rs b/core/server/src/shard/tasks/periodic/message_cleaner.rs deleted file mode 100644 index a4bac188be..0000000000 --- a/core/server/src/shard/tasks/periodic/message_cleaner.rs +++ /dev/null @@ -1,120 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::shard::IggyShard; -use crate::shard::transmission::frame::ShardResponse; -use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; -use iggy_common::IggyError; -use server_common::sharding::IggyNamespace; -use std::rc::Rc; -use tracing::{error, info, trace}; - -pub fn spawn_message_cleaner(shard: Rc) { - if !shard.config.data_maintenance.messages.cleaner_enabled { - info!("Message cleaner is disabled."); - return; - } - - let period = shard - .config - .data_maintenance - .messages - .interval - .get_duration(); - info!( - "Message cleaner is enabled, expired segments will be automatically deleted every: {:?}", - period - ); - let shard_clone = shard.clone(); - shard - .task_registry - .periodic("clean_messages") - .every(period) - .tick(move |_shutdown| clean_messages(shard_clone.clone())) - .spawn(); -} - -/// Groups namespaces by topic and sends a single `CleanTopicMessages` per topic to the pump. -/// All segment inspection and deletion happens inside the pump handler — no TOCTOU. -async fn clean_messages(shard: Rc) -> Result<(), IggyError> { - trace!("Cleaning expired messages..."); - - let namespaces = shard.get_current_shard_namespaces(); - - let mut topics: std::collections::HashMap<(usize, usize), Vec> = - std::collections::HashMap::new(); - - for ns in namespaces { - topics - .entry((ns.stream_id(), ns.topic_id())) - .or_default() - .push(ns.partition_id()); - } - - let mut total_deleted_segments = 0u64; - let mut total_deleted_messages = 0u64; - - for ((stream_id, topic_id), partition_ids) in topics { - let ns = IggyNamespace::new(stream_id, topic_id, partition_ids[0]); - let payload = ShardRequestPayload::CleanTopicMessages { - stream_id, - topic_id, - partition_ids, - }; - let request = ShardRequest::data_plane(ns, payload); - - match shard.send_to_data_plane(request).await { - Ok(ShardResponse::CleanTopicMessages { - deleted_segments, - deleted_messages, - }) => { - if deleted_segments > 0 { - info!( - "Deleted {} segments and {} messages for stream {}, topic {}", - deleted_segments, deleted_messages, stream_id, topic_id - ); - shard.metrics.decrement_segments(deleted_segments as u32); - shard.metrics.decrement_messages(deleted_messages); - total_deleted_segments += deleted_segments; - total_deleted_messages += deleted_messages; - } - } - Ok(ShardResponse::ErrorResponse(err)) => { - error!( - "Failed to clean messages for stream {}, topic {}: {}", - stream_id, topic_id, err - ); - } - Ok(_) => unreachable!("Expected CleanTopicMessages response"), - Err(err) => { - error!( - "Failed to send CleanTopicMessages for stream {}, topic {}: {}", - stream_id, topic_id, err - ); - } - } - } - - if total_deleted_segments > 0 { - info!( - "Total cleaned: {} segments and {} messages", - total_deleted_segments, total_deleted_messages - ); - } - - Ok(()) -} diff --git a/core/server/src/shard/tasks/periodic/message_saver.rs b/core/server/src/shard/tasks/periodic/message_saver.rs deleted file mode 100644 index f1e973378c..0000000000 --- a/core/server/src/shard/tasks/periodic/message_saver.rs +++ /dev/null @@ -1,71 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::shard::IggyShard; -use crate::shard::transmission::frame::ShardResponse; -use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::{error, info, trace}; - -pub fn spawn_message_saver(shard: Rc) { - let period = shard.config.message_saver.interval.get_duration(); - let enforce_fsync = shard.config.message_saver.enforce_fsync; - info!( - "Message saver is enabled, buffered messages will be automatically saved every: {:?}, enforce fsync: {enforce_fsync}.", - period - ); - let shard_clone = shard.clone(); - shard - .task_registry - .periodic("save_messages") - .every(period) - // No last_tick_on_shutdown — the pump handles final flush + fsync - // during its own shutdown (see message_pump.rs). - .tick(move |_shutdown| save_messages(shard_clone.clone())) - .spawn(); -} - -async fn save_messages(shard: Rc) -> Result<(), IggyError> { - trace!("Saving buffered messages..."); - - let namespaces = shard.get_current_shard_namespaces(); - let mut partitions_flushed = 0u32; - - for ns in namespaces { - let payload = ShardRequestPayload::FlushUnsavedBuffer { fsync: false }; - let request = ShardRequest::data_plane(ns, payload); - match shard.send_to_data_plane(request).await { - Ok(ShardResponse::FlushUnsavedBuffer { flushed_count }) if flushed_count > 0 => { - partitions_flushed += 1; - } - Ok(ShardResponse::FlushUnsavedBuffer { .. }) => {} - Ok(ShardResponse::ErrorResponse(err)) => { - error!("Failed to save messages for partition {:?}: {}", ns, err); - } - Err(err) => { - error!("Failed to save messages for partition {:?}: {}", ns, err); - } - _ => {} - } - } - - if partitions_flushed > 0 { - info!("Flushed {partitions_flushed} partitions."); - } - Ok(()) -} diff --git a/core/server/src/shard/tasks/periodic/mod.rs b/core/server/src/shard/tasks/periodic/mod.rs deleted file mode 100644 index 55799697a3..0000000000 --- a/core/server/src/shard/tasks/periodic/mod.rs +++ /dev/null @@ -1,36 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -mod heartbeat_verifier; -mod jwt_token_cleaner; -mod message_cleaner; -mod message_saver; -mod personal_access_token_cleaner; -mod revocation_timeout; -mod sysinfo_printer; -#[cfg(feature = "systemd")] -mod systemd_watchdog; - -pub use heartbeat_verifier::spawn_heartbeat_verifier; -pub use jwt_token_cleaner::spawn_jwt_token_cleaner; -pub use message_cleaner::spawn_message_cleaner; -pub use message_saver::spawn_message_saver; -pub use personal_access_token_cleaner::spawn_personal_access_token_cleaner; -pub use revocation_timeout::spawn_revocation_timeout_checker; -pub use sysinfo_printer::spawn_sysinfo_printer; -#[cfg(feature = "systemd")] -pub use systemd_watchdog::spawn_systemd_watchdog; diff --git a/core/server/src/shard/tasks/periodic/personal_access_token_cleaner.rs b/core/server/src/shard/tasks/periodic/personal_access_token_cleaner.rs deleted file mode 100644 index e258929e0e..0000000000 --- a/core/server/src/shard/tasks/periodic/personal_access_token_cleaner.rs +++ /dev/null @@ -1,85 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::shard::IggyShard; -use iggy_common::{IggyError, IggyTimestamp}; -use std::rc::Rc; -use tracing::{info, trace}; - -pub fn spawn_personal_access_token_cleaner(shard: Rc) { - if shard.id != 0 { - return; - } - let period = shard - .config - .personal_access_token - .cleaner - .interval - .get_duration(); - info!( - "Personal access token cleaner is enabled, expired tokens will be deleted every: {:?}.", - period - ); - let shard_clone = shard.clone(); - shard - .task_registry - .periodic("clear_personal_access_tokens") - .every(period) - .tick(move |_shutdown| clear_personal_access_tokens(shard_clone.clone())) - .spawn(); -} - -async fn clear_personal_access_tokens(shard: Rc) -> Result<(), IggyError> { - trace!("Checking for expired personal access tokens..."); - - let now = IggyTimestamp::now(); - let mut total_removed = 0; - - let user_ids: Vec = shard - .metadata - .get_all_users() - .iter() - .map(|u| u.id) - .collect(); - - for user_id in user_ids { - let pats = shard.metadata.get_user_personal_access_tokens(user_id); - - let expired_tokens: Vec<_> = pats - .iter() - .filter(|pat| pat.is_expired(now)) - .map(|pat| (pat.name.clone(), pat.token.clone())) - .collect(); - - for (name, token_hash) in expired_tokens { - shard - .writer() - .delete_personal_access_token(user_id, token_hash); - info!( - "Removed expired personal access token '{}' for user ID {}", - name, user_id - ); - total_removed += 1; - } - } - - if total_removed > 0 { - info!("Removed {total_removed} expired personal access tokens"); - } - - Ok(()) -} diff --git a/core/server/src/shard/tasks/periodic/revocation_timeout.rs b/core/server/src/shard/tasks/periodic/revocation_timeout.rs deleted file mode 100644 index 69fa251b1e..0000000000 --- a/core/server/src/shard/tasks/periodic/revocation_timeout.rs +++ /dev/null @@ -1,105 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::shard::IggyShard; -use crate::shard::transmission::message::{ShardRequest, ShardRequestPayload}; -use iggy_common::{IggyError, IggyTimestamp}; -use std::rc::Rc; -use tracing::{info, trace, warn}; - -pub fn spawn_revocation_timeout_checker(shard: Rc) { - let interval = shard.config.consumer_group.rebalancing_check_interval; - let timeout = shard.config.consumer_group.rebalancing_timeout; - info!( - "Pending partition revocations will be checked every: {}. Timeout: {}.", - interval, timeout - ); - let shard_clone = shard.clone(); - shard - .task_registry - .periodic("check_revocation_timeouts") - .every(interval.get_duration()) - .tick(move |_shutdown| check_revocation_timeouts(shard_clone.clone())) - .spawn(); -} - -async fn check_revocation_timeouts(shard: Rc) -> Result<(), IggyError> { - trace!("Checking pending revocation timeouts..."); - - let now = IggyTimestamp::now().as_micros(); - let timeout_micros = shard.config.consumer_group.rebalancing_timeout.as_micros(); - - let timed_out = shard.metadata.with_metadata(|metadata| { - metadata - .streams - .iter() - .flat_map(|(stream_id, stream)| { - stream.topics.iter().flat_map(move |(topic_id, topic)| { - topic.consumer_groups.iter().flat_map(move |(_, group)| { - let group_id = group.id; - group.members.iter().flat_map(move |(slab_id, member)| { - let member_id = member.id; - member - .pending_revocations - .iter() - .filter(move |revocation| { - now.saturating_sub(revocation.created_at_micros) - >= timeout_micros - }) - .map(move |revocation| { - ( - stream_id, - topic_id, - group_id, - slab_id, - member_id, - revocation.partition_id, - ) - }) - }) - }) - }) - }) - .collect::>() - }); - - if timed_out.is_empty() { - return Ok(()); - } - - let count = timed_out.len(); - for (stream_id, topic_id, group_id, member_slab_id, member_id, partition_id) in timed_out { - warn!( - "Force-completing timed out revocation: stream={stream_id}, topic={topic_id}, group={group_id}, \ - member_slab={member_slab_id}, partition={partition_id}", - ); - let request = - ShardRequest::control_plane(ShardRequestPayload::CompletePartitionRevocation { - stream_id, - topic_id, - group_id, - member_slab_id, - member_id, - partition_id, - timed_out: true, - }); - let _ = shard.send_to_control_plane(request).await; - } - info!("Force-completed {count} timed out partition revocations."); - - Ok(()) -} diff --git a/core/server/src/shard/tasks/periodic/sysinfo_printer.rs b/core/server/src/shard/tasks/periodic/sysinfo_printer.rs deleted file mode 100644 index c70f78e694..0000000000 --- a/core/server/src/shard/tasks/periodic/sysinfo_printer.rs +++ /dev/null @@ -1,103 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::shard::IggyShard; -use human_repr::HumanCount; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::{error, info, trace}; - -pub fn spawn_sysinfo_printer(shard: Rc) { - let period = shard - .config - .system - .logging - .sysinfo_print_interval - .get_duration(); - info!( - "System info logger is enabled, OS info will be printed every: {:?}", - period - ); - let shard_clone = shard.clone(); - shard - .task_registry - .periodic("print_sysinfo") - .every(period) - .tick(move |_shutdown| print_sysinfo(shard_clone.clone())) - .spawn(); -} - -fn get_open_file_descriptors() -> Option { - #[cfg(target_os = "linux")] - { - let pid = std::process::id(); - let fd_path = format!("/proc/{}/fd", pid); - if let Ok(entries) = std::fs::read_dir(&fd_path) { - return Some(entries.count()); - } - } - None -} - -async fn print_sysinfo(shard: Rc) -> Result<(), IggyError> { - trace!("Printing system information..."); - - let stats = match shard.get_stats().await { - Ok(stats) => stats, - Err(e) => { - error!("Failed to get system information. Error: {e}"); - return Ok(()); - } - }; - - let free_memory_percent = (stats.available_memory.as_bytes_u64() as f64 - / stats.total_memory.as_bytes_u64() as f64) - * 100f64; - - let threads_info = if stats.threads_count > 0 { - format!(", Threads: {}", stats.threads_count) - } else { - String::new() - }; - - let open_files_info = if let Some(open_files) = get_open_file_descriptors() { - format!(", OpenFDs: {}", open_files) - } else { - String::new() - }; - - info!( - "CPU: {:.2}%/{:.2}% (IggyUsage/Total), Mem: {:.2}%/{}/{}/{} (Free/IggyUsage/TotalUsed/Total), Disk: {}/{} (Free/Total), IggyUsage: {}, Clients: {}, Messages: {}, Read: {}, Written: {}{}{}", - stats.cpu_usage, - stats.total_cpu_usage, - free_memory_percent, - stats.memory_usage, - stats.total_memory - stats.available_memory, - stats.total_memory, - stats.free_disk_space, - stats.total_disk_space, - stats.messages_size_bytes, - stats.clients_count.human_count_bare().to_string(), - stats.messages_count.human_count_bare().to_string(), - stats.read_bytes, - stats.written_bytes, - threads_info, - open_files_info, - ); - - Ok(()) -} diff --git a/core/server/src/shard/transmission/connector.rs b/core/server/src/shard/transmission/connector.rs deleted file mode 100644 index 309d9fe0bd..0000000000 --- a/core/server/src/shard/transmission/connector.rs +++ /dev/null @@ -1,106 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use super::{ - frame::{ShardFrame, ShardResponse}, - message::ShardMessage, -}; -use iggy_common::IggyError; -use tracing::error; - -pub type StopSender = async_channel::Sender<()>; -pub type StopReceiver = async_channel::Receiver<()>; - -/// Inter-shard communication channel -pub struct ShardConnector { - pub id: u16, - pub sender: flume::Sender, - pub receiver: Receiver, - pub stop_receiver: StopReceiver, - pub stop_sender: StopSender, -} - -impl Clone for ShardConnector { - fn clone(&self) -> Self { - Self { - id: self.id, - sender: self.sender.clone(), - receiver: self.receiver.clone(), - stop_receiver: self.stop_receiver.clone(), - stop_sender: self.stop_sender.clone(), - } - } -} - -impl ShardConnector { - /// Creates a new shard connector with unbounded capacity. - pub fn new(id: u16) -> Self { - let (sender, receiver) = flume::unbounded(); - let (stop_sender, stop_receiver) = async_channel::bounded(1); - Self { - id, - sender, - receiver: Receiver::new(receiver), - stop_receiver, - stop_sender, - } - } - - /// Sends a message to this shard. - /// - /// For unbounded channels, this operation is infallible and never blocks. - pub fn send(&self, data: T) { - let _ = self.sender.send(data); - } -} - -impl ShardConnector { - /// Sends a request and waits for a response. - /// This implements the request-response pattern for inter-shard communication. - pub async fn send_request(&self, message: ShardMessage) -> Result { - let (sender, receiver) = async_channel::bounded(1); - // Note: sender needs to be passed to ShardFrame to keep the channel open - self.send(ShardFrame::new(message, Some(sender))); - - receiver.recv().await.map_err(|err| { - error!("Failed to receive response from shard {}: {err}", self.id); - IggyError::ShardCommunicationError - }) - } -} - -/// Wrapper around flume's Receiver that provides Clone capability. -/// -/// This wraps the flume receiver to allow cloning while still providing -/// access to the underlying receiver for direct use. -pub struct Receiver { - pub inner: flume::Receiver, -} - -impl Receiver { - fn new(receiver: flume::Receiver) -> Self { - Self { inner: receiver } - } -} - -impl Clone for Receiver { - fn clone(&self) -> Self { - Self { - inner: self.inner.clone(), - } - } -} diff --git a/core/server/src/shard/transmission/event.rs b/core/server/src/shard/transmission/event.rs deleted file mode 100644 index b243e26dd2..0000000000 --- a/core/server/src/shard/transmission/event.rs +++ /dev/null @@ -1,70 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use iggy_common::{Identifier, IggyTimestamp, TransportProtocol}; -use std::net::SocketAddr; -use strum::Display; - -/// Minimal partition info for event broadcasting (no slab dependency) -#[derive(Debug, Clone)] -pub struct PartitionInfo { - pub id: usize, - pub created_at: IggyTimestamp, -} - -/// Events that require broadcasting between shards. -/// -/// Note: Metadata events (CreatedStream, DeletedStream, CreatedTopic, DeletedTopic, -/// UpdatedStream, UpdatedTopic, CreatedConsumerGroup, DeletedConsumerGroup) are NOT -/// broadcast because SharedMetadata is already visible to all shards via LeftRight. -/// Only events that require per-shard local actions are broadcast. -#[derive(Debug, Clone, Display)] -#[strum(serialize_all = "PascalCase")] -pub enum ShardEvent { - /// Flush unsaved buffer to disk for a specific partition - FlushUnsavedBuffer { - stream_id: Identifier, - topic_id: Identifier, - partition_id: usize, - fsync: bool, - }, - /// Purge all messages, consumer groups and consumer group offsets from a topic - PurgedTopic { - stream_id: Identifier, - topic_id: Identifier, - }, - /// Purges all topics in a stream - PurgedStream { stream_id: Identifier }, - /// New partitions created (requires per-shard log initialization) - CreatedPartitions { - stream_id: Identifier, - topic_id: Identifier, - partitions: Vec, - }, - /// Partitions deleted (requires per-shard log cleanup) - DeletedPartitions { - stream_id: Identifier, - topic_id: Identifier, - partitions_count: u32, - partition_ids: Vec, - }, - /// Transport address bound (for config file writing) - AddressBound { - protocol: TransportProtocol, - address: SocketAddr, - }, -} diff --git a/core/server/src/shard/transmission/frame.rs b/core/server/src/shard/transmission/frame.rs deleted file mode 100644 index fb77565deb..0000000000 --- a/core/server/src/shard/transmission/frame.rs +++ /dev/null @@ -1,116 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::{ - shard::transmission::message::ShardMessage, - streaming::{segments::IggyMessagesBatchSet, users::user::User}, -}; -use async_channel::Sender; -use iggy_common::{ - CompressionAlgorithm, IggyError, IggyExpiry, IggyPollMetadata, IggyTimestamp, MaxTopicSize, - PersonalAccessToken, Stats, -}; -use std::sync::Arc; - -/// Data needed to construct a stream creation response. -#[derive(Debug)] -pub struct StreamResponseData { - pub id: u32, - pub name: Arc, - pub created_at: IggyTimestamp, -} - -/// Data needed to construct a topic creation response. -#[derive(Debug)] -pub struct TopicResponseData { - pub id: u32, - pub name: Arc, - pub created_at: IggyTimestamp, - pub partitions: Vec, - pub message_expiry: IggyExpiry, - pub compression_algorithm: CompressionAlgorithm, - pub max_topic_size: MaxTopicSize, - pub replication_factor: u8, -} - -/// Data needed to construct a consumer group creation response. -#[derive(Debug)] -pub struct ConsumerGroupResponseData { - pub id: u32, - pub name: Arc, - pub partitions_count: u32, -} - -// TODO: make nice types in common module so that each command has respective *Response struct, i.e. CreateStream -> CreateStreamResponse -#[derive(Debug)] -pub enum ShardResponse { - PollMessages((IggyPollMetadata, IggyMessagesBatchSet)), - SendMessages, - FlushUnsavedBuffer { - flushed_count: u32, - }, - DeleteSegments { - deleted_segments: u64, - deleted_messages: u64, - }, - CleanTopicMessages { - deleted_segments: u64, - deleted_messages: u64, - }, - Event, - CreateStreamResponse(StreamResponseData), - DeleteStreamResponse, - CreateTopicResponse(TopicResponseData), - UpdateTopicResponse, - DeleteTopicResponse, - CreateUserResponse(User), - DeleteUserResponse(User), - GetStatsResponse(Stats), - CreatePartitionsResponse, - DeletePartitionsResponse, - UpdateStreamResponse, - SocketTransferResponse, - UpdatePermissionsResponse, - ChangePasswordResponse, - UpdateUserResponse(User), - CreateConsumerGroupResponse(ConsumerGroupResponseData), - JoinConsumerGroupResponse, - LeaveConsumerGroupResponse, - DeleteConsumerGroupResponse, - CreatePersonalAccessTokenResponse(PersonalAccessToken, String), - DeletePersonalAccessTokenResponse, - LeaveConsumerGroupMetadataOnlyResponse, - CompletePartitionRevocationResponse, - PurgeStreamResponse, - PurgeTopicResponse, - ErrorResponse(IggyError), -} - -#[derive(Debug)] -pub struct ShardFrame { - pub message: ShardMessage, - pub response_sender: Option>, -} - -impl ShardFrame { - pub fn new(message: ShardMessage, response_sender: Option>) -> Self { - Self { - message, - response_sender, - } - } -} diff --git a/core/server/src/shard/transmission/message.rs b/core/server/src/shard/transmission/message.rs deleted file mode 100644 index f1acf9777f..0000000000 --- a/core/server/src/shard/transmission/message.rs +++ /dev/null @@ -1,254 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::{ - shard::{system::messages::PollingArgs, transmission::event::ShardEvent}, - streaming::{polling_consumer::PollingConsumer, segments::IggyMessagesBatchMut}, -}; -use iggy_binary_protocol::requests::{ - consumer_groups::*, partitions::*, personal_access_tokens::*, streams::*, topics::*, users::*, -}; -use server_common::sharding::IggyNamespace; - -use std::{net::SocketAddr, os::fd::OwnedFd}; - -/// Resolved stream ID. Contains only the numeric ID - `Identifier` stays at handler boundary. -#[derive(Debug, Clone, Copy)] -pub struct ResolvedStream(pub usize); - -impl ResolvedStream { - pub fn id(self) -> usize { - self.0 - } -} - -/// Resolved topic with parent stream context. -#[derive(Debug, Clone, Copy)] -pub struct ResolvedTopic { - pub stream_id: usize, - pub topic_id: usize, -} - -/// Resolved partition with full context. -#[derive(Debug, Clone, Copy)] -pub struct ResolvedPartition { - pub stream_id: usize, - pub topic_id: usize, - pub partition_id: usize, -} - -/// Resolved consumer group with full context. -#[derive(Debug, Clone, Copy)] -pub struct ResolvedConsumerGroup { - pub stream_id: usize, - pub topic_id: usize, - pub group_id: usize, -} - -#[allow(clippy::large_enum_variant)] -#[derive(Debug)] -pub enum ShardMessage { - Request(ShardRequest), - Event(ShardEvent), -} - -/// Routing envelope determining which shard handles the request. -#[derive(Debug)] -pub struct ShardRequest { - /// None = shard 0 (control-plane), Some = partition owner (data-plane) - pub routing: Option, - pub payload: ShardRequestPayload, -} - -impl ShardRequest { - /// Control-plane operations always route to shard 0 - pub fn control_plane(payload: ShardRequestPayload) -> Self { - Self { - routing: None, - payload, - } - } - - /// Data-plane operations route by partition namespace - pub fn data_plane(namespace: IggyNamespace, payload: ShardRequestPayload) -> Self { - Self { - routing: Some(namespace), - payload, - } - } -} - -#[derive(Debug)] -pub enum ShardRequestPayload { - // Data-plane operations: namespace provided via ShardRequest - SendMessages { - batch: IggyMessagesBatchMut, - }, - PollMessages { - consumer: PollingConsumer, - args: PollingArgs, - }, - FlushUnsavedBuffer { - fsync: bool, - }, - DeleteSegments { - segments_count: u32, - }, - CleanTopicMessages { - stream_id: usize, - topic_id: usize, - partition_ids: Vec, - }, - SocketTransfer { - fd: OwnedFd, - from_shard: u16, - client_id: u32, - user_id: u32, - address: SocketAddr, - initial_data: IggyMessagesBatchMut, - }, - - // Control-plane: stream operations - CreateStreamRequest { - user_id: u32, - command: CreateStreamRequest, - }, - UpdateStreamRequest { - user_id: u32, - command: UpdateStreamRequest, - }, - DeleteStreamRequest { - user_id: u32, - command: DeleteStreamRequest, - }, - PurgeStreamRequest { - user_id: u32, - command: PurgeStreamRequest, - }, - - // Control-plane: topic operations - CreateTopicRequest { - user_id: u32, - command: CreateTopicRequest, - }, - UpdateTopicRequest { - user_id: u32, - command: UpdateTopicRequest, - }, - DeleteTopicRequest { - user_id: u32, - command: DeleteTopicRequest, - }, - PurgeTopicRequest { - user_id: u32, - command: PurgeTopicRequest, - }, - - // Control-plane: partition operations - CreatePartitionsRequest { - user_id: u32, - command: CreatePartitionsRequest, - }, - DeletePartitionsRequest { - user_id: u32, - command: DeletePartitionsRequest, - }, - - // Control-plane: user operations - CreateUserRequest { - user_id: u32, - command: CreateUserRequest, - }, - UpdateUserRequest { - user_id: u32, - command: UpdateUserRequest, - }, - DeleteUserRequest { - user_id: u32, - command: DeleteUserRequest, - }, - UpdatePermissionsRequest { - user_id: u32, - command: UpdatePermissionsRequest, - }, - ChangePasswordRequest { - user_id: u32, - command: ChangePasswordRequest, - }, - - // Control-plane: consumer group operations - CreateConsumerGroupRequest { - user_id: u32, - command: CreateConsumerGroupRequest, - }, - DeleteConsumerGroupRequest { - user_id: u32, - command: DeleteConsumerGroupRequest, - }, - JoinConsumerGroupRequest { - user_id: u32, - client_id: u32, - command: JoinConsumerGroupRequest, - }, - LeaveConsumerGroupRequest { - user_id: u32, - client_id: u32, - command: LeaveConsumerGroupRequest, - }, - LeaveConsumerGroupMetadataOnly { - stream_id: usize, - topic_id: usize, - group_id: usize, - client_id: u32, - }, - CompletePartitionRevocation { - stream_id: usize, - topic_id: usize, - group_id: usize, - member_slab_id: usize, - member_id: usize, - partition_id: usize, - timed_out: bool, - }, - - // Control-plane: PAT operations - CreatePersonalAccessTokenRequest { - user_id: u32, - command: CreatePersonalAccessTokenRequest, - }, - DeletePersonalAccessTokenRequest { - user_id: u32, - command: DeletePersonalAccessTokenRequest, - }, - - // Control-plane: stats - GetStats { - user_id: u32, - }, -} - -impl From for ShardMessage { - fn from(request: ShardRequest) -> Self { - ShardMessage::Request(request) - } -} - -impl From for ShardMessage { - fn from(event: ShardEvent) -> Self { - ShardMessage::Event(event) - } -} diff --git a/core/server/src/shard/transmission/mod.rs b/core/server/src/shard/transmission/mod.rs deleted file mode 100644 index cc326cb27d..0000000000 --- a/core/server/src/shard/transmission/mod.rs +++ /dev/null @@ -1,21 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod connector; -pub mod event; -pub mod frame; -pub mod message; diff --git a/core/server-ng/src/snapshot.rs b/core/server/src/snapshot.rs similarity index 96% rename from core/server-ng/src/snapshot.rs rename to core/server/src/snapshot.rs index 8761265ae8..fca3b6e529 100644 --- a/core/server-ng/src/snapshot.rs +++ b/core/server/src/snapshot.rs @@ -29,7 +29,7 @@ use std::time::Instant; use async_zip::base::write::ZipFileWriter; use async_zip::{Compression, ZipEntryBuilder}; -use configs::server_ng::NgSystemConfig; +use configs::server::ServerSystemConfig; use futures::channel::oneshot; use iggy_common::{IggyDuration, IggyError, SnapshotCompression, SystemSnapshotType}; use tracing::{error, info, warn}; @@ -55,7 +55,7 @@ static SNAPSHOT_IN_PROGRESS: AtomicBool = AtomicBool::new(false); /// [`SNAPSHOT_IN_PROGRESS`]); a concurrent request busy-rejects with /// [`IggyError::SnapshotFileCompletionFailed`]. pub async fn collect( - system_config: Arc, + system_config: Arc, compression: SnapshotCompression, snapshot_types: Vec, ) -> Result, IggyError> { @@ -128,7 +128,7 @@ impl Drop for SnapshotInProgressGuard { } fn collect_blocking( - system_config: &NgSystemConfig, + system_config: &ServerSystemConfig, compression: SnapshotCompression, snapshot_types: &[SystemSnapshotType], ) -> Result, IggyError> { @@ -156,7 +156,7 @@ fn collect_blocking( fn capture( snapshot_type: &SystemSnapshotType, - system_config: &NgSystemConfig, + system_config: &ServerSystemConfig, ) -> io::Result> { match snapshot_type { SystemSnapshotType::FilesystemOverview => { @@ -195,7 +195,7 @@ fn process_list() -> io::Result> { Ok(content) } -fn server_logs(system_config: &NgSystemConfig) -> io::Result> { +fn server_logs(system_config: &ServerSystemConfig) -> io::Result> { // Mirror the logger's path derivation (server_common `Logging::late_init`): // it canonicalizes the configured subdirectory before joining the system // path, so a relative `logging.path` that already exists resolves against the @@ -224,7 +224,7 @@ fn server_logs(system_config: &NgSystemConfig) -> io::Result> { Ok(content) } -fn server_config(system_config: &NgSystemConfig) -> io::Result> { +fn server_config(system_config: &ServerSystemConfig) -> io::Result> { let config_path = PathBuf::from(system_config.get_runtime_path()).join("current_config.toml"); std::fs::read(config_path) } @@ -277,7 +277,7 @@ mod tests { // second collector thread) rather than piling up threads. let held = SnapshotInProgressGuard::acquire().expect("flag starts free"); let result = futures::executor::block_on(collect( - Arc::new(NgSystemConfig::default()), + Arc::new(ServerSystemConfig::default()), SnapshotCompression::Stored, vec![SystemSnapshotType::Test], )); diff --git a/core/server-ng/src/snapshot/procdump.rs b/core/server/src/snapshot/procdump.rs similarity index 100% rename from core/server-ng/src/snapshot/procdump.rs rename to core/server/src/snapshot/procdump.rs diff --git a/core/server/src/state/command.rs b/core/server/src/state/command.rs deleted file mode 100644 index 9138df06d7..0000000000 --- a/core/server/src/state/command.rs +++ /dev/null @@ -1,268 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::state::models::{ - CreateConsumerGroupWithId, CreatePersonalAccessTokenWithHash, CreateStreamWithId, - CreateTopicWithId, CreateUserWithId, -}; -use bytes::{BufMut, BytesMut}; -use iggy_binary_protocol::codes::{ - CHANGE_PASSWORD_CODE, CREATE_CONSUMER_GROUP_CODE, CREATE_PARTITIONS_CODE, - CREATE_PERSONAL_ACCESS_TOKEN_CODE, CREATE_STREAM_CODE, CREATE_TOPIC_CODE, CREATE_USER_CODE, - DELETE_CONSUMER_GROUP_CODE, DELETE_PARTITIONS_CODE, DELETE_PERSONAL_ACCESS_TOKEN_CODE, - DELETE_SEGMENTS_CODE, DELETE_STREAM_CODE, DELETE_TOPIC_CODE, DELETE_USER_CODE, - PURGE_STREAM_CODE, PURGE_TOPIC_CODE, UPDATE_PERMISSIONS_CODE, UPDATE_STREAM_CODE, - UPDATE_TOPIC_CODE, UPDATE_USER_CODE, -}; -use iggy_binary_protocol::requests::{ - consumer_groups::DeleteConsumerGroupRequest, - partitions::{CreatePartitionsRequest, DeletePartitionsRequest}, - personal_access_tokens::DeletePersonalAccessTokenRequest, - segments::DeleteSegmentsRequest, - streams::{DeleteStreamRequest, PurgeStreamRequest, UpdateStreamRequest}, - topics::{DeleteTopicRequest, PurgeTopicRequest, UpdateTopicRequest}, - users::{ - ChangePasswordRequest, DeleteUserRequest, UpdatePermissionsRequest, UpdateUserRequest, - }, -}; -use iggy_binary_protocol::{WireDecode, WireEncode, WireError}; -use std::fmt::{Display, Formatter}; - -#[derive(Debug)] -pub enum EntryCommand { - CreateStream(CreateStreamWithId), - UpdateStream(UpdateStreamRequest), - DeleteStream(DeleteStreamRequest), - PurgeStream(PurgeStreamRequest), - CreateTopic(CreateTopicWithId), - UpdateTopic(UpdateTopicRequest), - DeleteTopic(DeleteTopicRequest), - PurgeTopic(PurgeTopicRequest), - CreatePartitions(CreatePartitionsRequest), - DeletePartitions(DeletePartitionsRequest), - DeleteSegments(DeleteSegmentsRequest), - CreateConsumerGroup(CreateConsumerGroupWithId), - DeleteConsumerGroup(DeleteConsumerGroupRequest), - CreateUser(CreateUserWithId), - UpdateUser(UpdateUserRequest), - DeleteUser(DeleteUserRequest), - ChangePassword(ChangePasswordRequest), - UpdatePermissions(UpdatePermissionsRequest), - CreatePersonalAccessToken(CreatePersonalAccessTokenWithHash), - DeletePersonalAccessToken(DeletePersonalAccessTokenRequest), -} - -impl WireEncode for EntryCommand { - fn encoded_size(&self) -> usize { - let inner_size = match self { - EntryCommand::CreateStream(cmd) => cmd.encoded_size(), - EntryCommand::UpdateStream(cmd) => cmd.encoded_size(), - EntryCommand::DeleteStream(cmd) => cmd.encoded_size(), - EntryCommand::PurgeStream(cmd) => cmd.encoded_size(), - EntryCommand::CreateTopic(cmd) => cmd.encoded_size(), - EntryCommand::UpdateTopic(cmd) => cmd.encoded_size(), - EntryCommand::DeleteTopic(cmd) => cmd.encoded_size(), - EntryCommand::PurgeTopic(cmd) => cmd.encoded_size(), - EntryCommand::CreatePartitions(cmd) => cmd.encoded_size(), - EntryCommand::DeletePartitions(cmd) => cmd.encoded_size(), - EntryCommand::DeleteSegments(cmd) => cmd.encoded_size(), - EntryCommand::CreateConsumerGroup(cmd) => cmd.encoded_size(), - EntryCommand::DeleteConsumerGroup(cmd) => cmd.encoded_size(), - EntryCommand::CreateUser(cmd) => cmd.encoded_size(), - EntryCommand::UpdateUser(cmd) => cmd.encoded_size(), - EntryCommand::DeleteUser(cmd) => cmd.encoded_size(), - EntryCommand::ChangePassword(cmd) => cmd.encoded_size(), - EntryCommand::UpdatePermissions(cmd) => cmd.encoded_size(), - EntryCommand::CreatePersonalAccessToken(cmd) => cmd.encoded_size(), - EntryCommand::DeletePersonalAccessToken(cmd) => cmd.encoded_size(), - }; - 4 + 4 + inner_size - } - - fn encode(&self, buf: &mut BytesMut) { - let (code, inner_size) = match self { - EntryCommand::CreateStream(cmd) => (CREATE_STREAM_CODE, cmd.encoded_size()), - EntryCommand::UpdateStream(cmd) => (UPDATE_STREAM_CODE, cmd.encoded_size()), - EntryCommand::DeleteStream(cmd) => (DELETE_STREAM_CODE, cmd.encoded_size()), - EntryCommand::PurgeStream(cmd) => (PURGE_STREAM_CODE, cmd.encoded_size()), - EntryCommand::CreateTopic(cmd) => (CREATE_TOPIC_CODE, cmd.encoded_size()), - EntryCommand::UpdateTopic(cmd) => (UPDATE_TOPIC_CODE, cmd.encoded_size()), - EntryCommand::DeleteTopic(cmd) => (DELETE_TOPIC_CODE, cmd.encoded_size()), - EntryCommand::PurgeTopic(cmd) => (PURGE_TOPIC_CODE, cmd.encoded_size()), - EntryCommand::CreatePartitions(cmd) => (CREATE_PARTITIONS_CODE, cmd.encoded_size()), - EntryCommand::DeletePartitions(cmd) => (DELETE_PARTITIONS_CODE, cmd.encoded_size()), - EntryCommand::DeleteSegments(cmd) => (DELETE_SEGMENTS_CODE, cmd.encoded_size()), - EntryCommand::CreateConsumerGroup(cmd) => { - (CREATE_CONSUMER_GROUP_CODE, cmd.encoded_size()) - } - EntryCommand::DeleteConsumerGroup(cmd) => { - (DELETE_CONSUMER_GROUP_CODE, cmd.encoded_size()) - } - EntryCommand::CreateUser(cmd) => (CREATE_USER_CODE, cmd.encoded_size()), - EntryCommand::UpdateUser(cmd) => (UPDATE_USER_CODE, cmd.encoded_size()), - EntryCommand::DeleteUser(cmd) => (DELETE_USER_CODE, cmd.encoded_size()), - EntryCommand::ChangePassword(cmd) => (CHANGE_PASSWORD_CODE, cmd.encoded_size()), - EntryCommand::UpdatePermissions(cmd) => (UPDATE_PERMISSIONS_CODE, cmd.encoded_size()), - EntryCommand::CreatePersonalAccessToken(cmd) => { - (CREATE_PERSONAL_ACCESS_TOKEN_CODE, cmd.encoded_size()) - } - EntryCommand::DeletePersonalAccessToken(cmd) => { - (DELETE_PERSONAL_ACCESS_TOKEN_CODE, cmd.encoded_size()) - } - }; - buf.put_u32_le(code); - buf.put_u32_le(inner_size as u32); - match self { - EntryCommand::CreateStream(cmd) => cmd.encode(buf), - EntryCommand::UpdateStream(cmd) => cmd.encode(buf), - EntryCommand::DeleteStream(cmd) => cmd.encode(buf), - EntryCommand::PurgeStream(cmd) => cmd.encode(buf), - EntryCommand::CreateTopic(cmd) => cmd.encode(buf), - EntryCommand::UpdateTopic(cmd) => cmd.encode(buf), - EntryCommand::DeleteTopic(cmd) => cmd.encode(buf), - EntryCommand::PurgeTopic(cmd) => cmd.encode(buf), - EntryCommand::CreatePartitions(cmd) => cmd.encode(buf), - EntryCommand::DeletePartitions(cmd) => cmd.encode(buf), - EntryCommand::DeleteSegments(cmd) => cmd.encode(buf), - EntryCommand::CreateConsumerGroup(cmd) => cmd.encode(buf), - EntryCommand::DeleteConsumerGroup(cmd) => cmd.encode(buf), - EntryCommand::CreateUser(cmd) => cmd.encode(buf), - EntryCommand::UpdateUser(cmd) => cmd.encode(buf), - EntryCommand::DeleteUser(cmd) => cmd.encode(buf), - EntryCommand::ChangePassword(cmd) => cmd.encode(buf), - EntryCommand::UpdatePermissions(cmd) => cmd.encode(buf), - EntryCommand::CreatePersonalAccessToken(cmd) => cmd.encode(buf), - EntryCommand::DeletePersonalAccessToken(cmd) => cmd.encode(buf), - } - } -} - -impl WireDecode for EntryCommand { - fn decode(buf: &[u8]) -> Result<(Self, usize), WireError> { - if buf.len() < 8 { - return Err(WireError::UnexpectedEof { - offset: 0, - need: 8, - have: buf.len(), - }); - } - let code = u32::from_le_bytes(buf[0..4].try_into().unwrap()); - let length = u32::from_le_bytes(buf[4..8].try_into().unwrap()) as usize; - if buf.len() < 8 + length { - return Err(WireError::UnexpectedEof { - offset: 8, - need: length, - have: buf.len() - 8, - }); - } - let payload = &buf[8..8 + length]; - let consumed = 8 + length; - let cmd = match code { - CREATE_STREAM_CODE => { - EntryCommand::CreateStream(CreateStreamWithId::decode_from(payload)?) - } - UPDATE_STREAM_CODE => { - EntryCommand::UpdateStream(UpdateStreamRequest::decode_from(payload)?) - } - DELETE_STREAM_CODE => { - EntryCommand::DeleteStream(DeleteStreamRequest::decode_from(payload)?) - } - PURGE_STREAM_CODE => { - EntryCommand::PurgeStream(PurgeStreamRequest::decode_from(payload)?) - } - CREATE_TOPIC_CODE => { - EntryCommand::CreateTopic(CreateTopicWithId::decode_from(payload)?) - } - UPDATE_TOPIC_CODE => { - EntryCommand::UpdateTopic(UpdateTopicRequest::decode_from(payload)?) - } - DELETE_TOPIC_CODE => { - EntryCommand::DeleteTopic(DeleteTopicRequest::decode_from(payload)?) - } - PURGE_TOPIC_CODE => EntryCommand::PurgeTopic(PurgeTopicRequest::decode_from(payload)?), - CREATE_PARTITIONS_CODE => { - EntryCommand::CreatePartitions(CreatePartitionsRequest::decode_from(payload)?) - } - DELETE_PARTITIONS_CODE => { - EntryCommand::DeletePartitions(DeletePartitionsRequest::decode_from(payload)?) - } - DELETE_SEGMENTS_CODE => { - EntryCommand::DeleteSegments(DeleteSegmentsRequest::decode_from(payload)?) - } - CREATE_CONSUMER_GROUP_CODE => { - EntryCommand::CreateConsumerGroup(CreateConsumerGroupWithId::decode_from(payload)?) - } - DELETE_CONSUMER_GROUP_CODE => { - EntryCommand::DeleteConsumerGroup(DeleteConsumerGroupRequest::decode_from(payload)?) - } - CREATE_USER_CODE => EntryCommand::CreateUser(CreateUserWithId::decode_from(payload)?), - UPDATE_USER_CODE => EntryCommand::UpdateUser(UpdateUserRequest::decode_from(payload)?), - DELETE_USER_CODE => EntryCommand::DeleteUser(DeleteUserRequest::decode_from(payload)?), - CHANGE_PASSWORD_CODE => { - EntryCommand::ChangePassword(ChangePasswordRequest::decode_from(payload)?) - } - UPDATE_PERMISSIONS_CODE => { - EntryCommand::UpdatePermissions(UpdatePermissionsRequest::decode_from(payload)?) - } - CREATE_PERSONAL_ACCESS_TOKEN_CODE => EntryCommand::CreatePersonalAccessToken( - CreatePersonalAccessTokenWithHash::decode_from(payload)?, - ), - DELETE_PERSONAL_ACCESS_TOKEN_CODE => EntryCommand::DeletePersonalAccessToken( - DeletePersonalAccessTokenRequest::decode_from(payload)?, - ), - _ => return Err(WireError::UnknownCommand(code)), - }; - Ok((cmd, consumed)) - } -} - -impl Display for EntryCommand { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - match self { - EntryCommand::CreateStream(command) => write!(f, "CreateStream({command})"), - EntryCommand::UpdateStream(command) => write!(f, "UpdateStream({command:?})"), - EntryCommand::DeleteStream(command) => write!(f, "DeleteStream({command:?})"), - EntryCommand::PurgeStream(command) => write!(f, "PurgeStream({command:?})"), - EntryCommand::CreateTopic(command) => write!(f, "CreateTopic({command})"), - EntryCommand::UpdateTopic(command) => write!(f, "UpdateTopic({command:?})"), - EntryCommand::DeleteTopic(command) => write!(f, "DeleteTopic({command:?})"), - EntryCommand::PurgeTopic(command) => write!(f, "PurgeTopic({command:?})"), - EntryCommand::CreatePartitions(command) => write!(f, "CreatePartitions({command:?})"), - EntryCommand::DeletePartitions(command) => write!(f, "DeletePartitions({command:?})"), - EntryCommand::DeleteSegments(command) => write!(f, "DeleteSegments({command:?})"), - EntryCommand::CreateConsumerGroup(command) => { - write!(f, "CreateConsumerGroup({command})") - } - EntryCommand::DeleteConsumerGroup(command) => { - write!(f, "DeleteConsumerGroup({command:?})") - } - EntryCommand::CreateUser(command) => write!(f, "CreateUser({command})"), - EntryCommand::UpdateUser(command) => write!(f, "UpdateUser({command:?})"), - EntryCommand::DeleteUser(command) => write!(f, "DeleteUser({command:?})"), - EntryCommand::ChangePassword(command) => write!(f, "ChangePassword({command:?})"), - EntryCommand::UpdatePermissions(command) => { - write!(f, "UpdatePermissions({command:?})") - } - EntryCommand::CreatePersonalAccessToken(command) => { - write!(f, "CreatePersonalAccessToken({command})") - } - EntryCommand::DeletePersonalAccessToken(command) => { - write!(f, "DeletePersonalAccessToken({command:?})") - } - } - } -} diff --git a/core/server/src/state/entry.rs b/core/server/src/state/entry.rs deleted file mode 100644 index c4905a6dbe..0000000000 --- a/core/server/src/state/entry.rs +++ /dev/null @@ -1,150 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::state::command::EntryCommand; -use bytes::{BufMut, Bytes, BytesMut}; -use iggy_binary_protocol::{WireDecode, WireEncode}; -use iggy_common::IggyError; -use iggy_common::IggyTimestamp; -use iggy_common::calculate_checksum; -use std::fmt::{Display, Formatter}; - -/// State entry in the log -/// - `index` - Index (operation number) of the entry in the log -/// - `term` - Election term (view number) for replication -/// - `leader_id` - Leader ID for replication -/// - `version` - Server version based on semver as number e.g. 1.234.567 -> 1234567 -/// - `flags` - Reserved for future use -/// - `timestamp` - Timestamp when the command was issued -/// - `user_id` - User ID of the user who issued the command -/// - `checksum` - Checksum of the entry -/// - `code` - Command code -/// - `command` - Payload of the command -/// - `context` - Optional context e.g. used to enrich the payload with additional data -#[derive(Debug)] -pub struct StateEntry { - pub index: u64, - pub term: u64, - pub leader_id: u32, - pub version: u32, - pub flags: u64, - pub timestamp: IggyTimestamp, - pub user_id: u32, - pub checksum: u64, - pub context: Bytes, - pub command: Bytes, -} - -impl StateEntry { - #[allow(clippy::too_many_arguments)] - pub fn new( - index: u64, - term: u64, - leader_id: u32, - version: u32, - flags: u64, - timestamp: IggyTimestamp, - user_id: u32, - checksum: u64, - context: Bytes, - command: Bytes, - ) -> Self { - Self { - index, - term, - leader_id, - version, - flags, - timestamp, - user_id, - checksum, - context, - command, - } - } - - pub fn command(&self) -> Result { - EntryCommand::decode_from(&self.command).map_err(|e| { - tracing::warn!("wire decode error during WAL replay: {e}"); - IggyError::InvalidCommand - }) - } - - #[allow(clippy::too_many_arguments)] - pub fn calculate_checksum( - index: u64, - term: u64, - leader_id: u32, - version: u32, - flags: u64, - timestamp: IggyTimestamp, - user_id: u32, - context: &Bytes, - command: &Bytes, - ) -> u64 { - let mut bytes = - BytesMut::with_capacity(8 + 8 + 4 + 4 + 8 + 8 + 4 + 4 + context.len() + command.len()); - bytes.put_u64_le(index); - bytes.put_u64_le(term); - bytes.put_u32_le(leader_id); - bytes.put_u32_le(version); - bytes.put_u64_le(flags); - bytes.put_u64_le(timestamp.into()); - bytes.put_u32_le(user_id); - bytes.put_u32_le(context.len() as u32); - bytes.put_slice(context); - bytes.extend(command); - calculate_checksum(&bytes.freeze()) - } -} - -impl Display for StateEntry { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "StateEntry {{ index: {}, term: {}, leader ID: {}, version: {}, flags: {}, timestamp: {}, user ID: {}, checksum: {} }}", - self.index, - self.term, - self.leader_id, - self.version, - self.flags, - self.timestamp, - self.user_id, - self.checksum, - ) - } -} - -impl WireEncode for StateEntry { - fn encoded_size(&self) -> usize { - 8 + 8 + 4 + 4 + 8 + 8 + 4 + 8 + 4 + self.context.len() + self.command.len() - } - - fn encode(&self, buf: &mut BytesMut) { - buf.put_u64_le(self.index); - buf.put_u64_le(self.term); - buf.put_u32_le(self.leader_id); - buf.put_u32_le(self.version); - buf.put_u64_le(self.flags); - buf.put_u64_le(self.timestamp.into()); - buf.put_u32_le(self.user_id); - buf.put_u64_le(self.checksum); - buf.put_u32_le(self.context.len() as u32); - buf.put_slice(&self.context); - buf.extend_from_slice(&self.command); - } -} diff --git a/core/server/src/state/file.rs b/core/server/src/state/file.rs deleted file mode 100644 index ff710f415d..0000000000 --- a/core/server/src/state/file.rs +++ /dev/null @@ -1,378 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::state::command::EntryCommand; -use crate::state::{COMPONENT, StateEntry}; -use crate::streaming::persistence::persister::PersisterKind; -use crate::streaming::utils::file; -use bytes::{Buf, BufMut, Bytes, BytesMut}; -use compio::io::AsyncReadExt; -use err_trail::ErrContext; -use iggy_binary_protocol::{WireDecode, WireEncode}; -use iggy_common::EncryptorKind; -use iggy_common::IggyByteSize; -use iggy_common::IggyError; -use iggy_common::IggyTimestamp; -use iggy_common::SemanticVersion; -use std::fmt::Debug; -use std::path::Path; -use std::sync::Arc; -use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; -use tracing::{debug, error, info}; - -pub const BUF_CURSOR_CAPACITY_BYTES: usize = 512 * 1000; -const FILE_STATE_PARSE_ERROR: &str = "STATE - failed to parse file state"; - -#[derive(Debug)] -pub struct FileState { - current_index: Arc, - entries_count: Arc, - current_leader: Arc, - term: Arc, - version: u32, - path: String, - persister: Arc, - encryptor: Option, -} - -impl FileState { - #[allow(clippy::too_many_arguments)] - pub fn new( - path: &str, - version: &SemanticVersion, - persister: Arc, - encryptor: Option, - current_index: Arc, - entries_count: Arc, - current_leader: Arc, - term: Arc, - ) -> Self { - Self { - current_index, - entries_count, - current_leader, - term, - path: path.into(), - persister, - encryptor, - version: version.get_numeric_version().expect("Invalid version"), - } - } - - pub fn current_index(&self) -> u64 { - self.current_index.load(Ordering::SeqCst) - } - - pub fn entries_count(&self) -> u64 { - self.entries_count.load(Ordering::SeqCst) - } - - pub fn term(&self) -> u64 { - self.term.load(Ordering::SeqCst) - } - - pub async fn init(&self) -> Result, IggyError> { - assert!(Path::new(&self.path).exists()); - - let entries = self - .load_entries() - .await - .error(|e: &IggyError| format!("{COMPONENT} (error: {e}) - failed to load entries"))?; - let entries_count = entries.len() as u64; - self.entries_count.store(entries_count, Ordering::SeqCst); - if entries_count == 0 { - self.current_index.store(0, Ordering::SeqCst); - } else { - let last_index = entries[entries_count as usize - 1].index; - self.current_index.store(last_index, Ordering::SeqCst); - } - - Ok(entries) - } - - pub async fn load_entries(&self) -> Result, IggyError> { - if !Path::new(&self.path).exists() { - return Err(IggyError::StateFileNotFound); - } - - let file = file::open(&self.path) - .await - .error(|e: &std::io::Error| { - format!( - "{COMPONENT} (error: {e}) - failed to open state file, path: {}", - self.path - ) - }) - .map_err(|_| IggyError::CannotReadFile)?; - let file_size = file - .metadata() - .await - .error(|e: &std::io::Error| { - format!( - "{COMPONENT} (error: {e}) - failed to load state file metadata, path: {}", - self.path - ) - }) - .map_err(|_| IggyError::CannotReadFileMetadata)? - .len(); - - if file_size == 0 { - info!("State file is empty"); - return Ok(Vec::new()); - } - - info!( - "Loading state, file size: {}", - IggyByteSize::from(file_size).as_human_string() - ); - let mut entries = Vec::new(); - let mut total_size: u64 = 0; - let mut cursor = std::io::Cursor::new(file); - let mut current_index = 0; - let mut entries_count = 0; - loop { - let index = cursor - .read_u64_le() - .await - .error(|e: &std::io::Error| format!("{FILE_STATE_PARSE_ERROR} index. {e}")) - .map_err(|_| IggyError::InvalidNumberEncoding)?; - total_size += 8; - // Greater than one, because one of the entries after a fresh reboot is the default root user. - if entries_count > 1 && index != current_index + 1 { - error!( - "State file is corrupted, expected index: {}, got: {}", - current_index + 1, - index - ); - return Err(IggyError::StateFileCorrupted); - } - - current_index = index; - entries_count += 1; - let term = cursor - .read_u64_le() - .await - .error(|e: &std::io::Error| format!("{FILE_STATE_PARSE_ERROR} term. {e}")) - .map_err(|_| IggyError::InvalidNumberEncoding)?; - total_size += 8; - let leader_id = cursor - .read_u32_le() - .await - .error(|e: &std::io::Error| format!("{FILE_STATE_PARSE_ERROR} leader_id. {e}")) - .map_err(|_| IggyError::InvalidNumberEncoding)?; - total_size += 4; - let version = cursor - .read_u32_le() - .await - .error(|e: &std::io::Error| format!("{FILE_STATE_PARSE_ERROR} version. {e}")) - .map_err(|_| IggyError::InvalidNumberEncoding)?; - total_size += 4; - let flags = cursor - .read_u64_le() - .await - .error(|e: &std::io::Error| format!("{FILE_STATE_PARSE_ERROR} flags. {e}")) - .map_err(|_| IggyError::InvalidNumberEncoding)?; - total_size += 8; - let timestamp = IggyTimestamp::from( - cursor - .read_u64_le() - .await - .error(|e: &std::io::Error| format!("{FILE_STATE_PARSE_ERROR} timestamp. {e}")) - .map_err(|_| IggyError::InvalidNumberEncoding)?, - ); - total_size += 8; - let user_id = cursor - .read_u32_le() - .await - .error(|e: &std::io::Error| format!("{FILE_STATE_PARSE_ERROR} user_id. {e}")) - .map_err(|_| IggyError::InvalidNumberEncoding)?; - total_size += 4; - let checksum = cursor - .read_u64_le() - .await - .error(|e: &std::io::Error| format!("{FILE_STATE_PARSE_ERROR} checksum. {e}")) - .map_err(|_| IggyError::InvalidNumberEncoding)?; - total_size += 8; - let context_length = cursor - .read_u32_le() - .await - .error(|e: &std::io::Error| { - format!("{FILE_STATE_PARSE_ERROR} context context_length. {e}") - }) - .map_err(|_| IggyError::InvalidNumberEncoding)? - as usize; - total_size += 4; - let mut context = BytesMut::with_capacity(context_length); - context.put_bytes(0, context_length); - let (result, context) = cursor.read_exact(context).await.into(); - - result - .error(|e: &std::io::Error| format!("{FILE_STATE_PARSE_ERROR} code. {e}")) - .map_err(|_| IggyError::CannotReadFile)?; - let context = context.freeze(); - total_size += context_length as u64; - let code = cursor - .read_u32_le() - .await - .error(|e: &std::io::Error| format!("{FILE_STATE_PARSE_ERROR} code. {e}")) - .map_err(|_| IggyError::InvalidNumberEncoding)?; - total_size += 4; - let mut command_length = cursor - .read_u32_le() - .await - .error(|e: &std::io::Error| format!("{FILE_STATE_PARSE_ERROR} command_length. {e}")) - .map_err(|_| IggyError::InvalidNumberEncoding)? - as usize; - total_size += 4; - let mut command = BytesMut::with_capacity(command_length); - command.put_bytes(0, command_length); - let (result, command) = cursor.read_exact(command).await.into(); - result - .error(|e: &std::io::Error| format!("{FILE_STATE_PARSE_ERROR} command. {e}")) - .map_err(|_| IggyError::CannotReadFile)?; - total_size += command_length as u64; - let command_payload; - if let Some(encryptor) = &self.encryptor { - debug!("Decrypting state entry with index: {index}"); - command_payload = Bytes::from(encryptor.decrypt(&command.freeze())?); - command_length = command_payload.len(); - } else { - command_payload = command.freeze(); - } - - let mut entry_command = BytesMut::with_capacity(4 + 4 + command_length); - entry_command.put_u32_le(code); - entry_command.put_u32_le(command_length as u32); - entry_command.extend(command_payload); - let command = entry_command.freeze(); - EntryCommand::decode_from(&command) - .map_err(|e| { - tracing::warn!("wire decode error during WAL replay: {e}"); - IggyError::InvalidCommand - }) - .error(|e: &IggyError| { - format!("{COMPONENT} (error: {e}) - failed to parse entry command from bytes") - })?; - let calculated_checksum = StateEntry::calculate_checksum( - index, term, leader_id, version, flags, timestamp, user_id, &context, &command, - ); - let entry = StateEntry::new( - index, - term, - leader_id, - version, - flags, - timestamp, - user_id, - calculated_checksum, - context, - command, - ); - debug!("Read state entry: {entry}"); - if entry.checksum != checksum { - return Err(IggyError::InvalidStateEntryChecksum( - entry.checksum, - checksum, - entry.index, - )); - } - - entries.push(entry); - if total_size == file_size { - break; - } - } - - info!("Loaded {entries_count} state entries, current index: {current_index}"); - Ok(entries) - } - - pub async fn apply(&self, user_id: u32, command: &EntryCommand) -> Result<(), IggyError> { - debug!("Applying state entry with command: {command}, user ID: {user_id}"); - let timestamp = IggyTimestamp::now(); - let index = if self.entries_count.load(Ordering::SeqCst) == 0 { - 0 - } else { - self.current_index.fetch_add(1, Ordering::SeqCst) + 1 - }; - let term = self.term.load(Ordering::SeqCst); - let current_leader = self.current_leader.load(Ordering::SeqCst); - let version = self.version; - let flags = 0; - let context = Bytes::new(); - let mut command = command.to_bytes(); - let checksum = StateEntry::calculate_checksum( - index, - term, - current_leader, - version, - flags, - timestamp, - user_id, - &context, - &command, - ); - - if let Some(encryptor) = &self.encryptor { - debug!("Encrypting state entry command with index: {index}"); - let command_code = command.slice(0..4).get_u32_le(); - let mut command_length = command.slice(4..8).get_u32_le() as usize; - let command_payload = command.slice(8..8 + command_length); - let encrypted_command_payload = encryptor - .encrypt(&command_payload) - .error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - failed to encrypt state entry command, index: {index}" - ) - })?; - command_length = encrypted_command_payload.len(); - let mut command_bytes = BytesMut::with_capacity(4 + 4 + command_length); - command_bytes.put_u32_le(command_code); - command_bytes.put_u32_le(command_length as u32); - command_bytes.extend(encrypted_command_payload); - command = command_bytes.freeze(); - } - - let entry = StateEntry::new( - index, - term, - current_leader, - version, - flags, - timestamp, - user_id, - checksum, - context, - command, - ); - let bytes = entry.to_bytes(); - let len = bytes.len(); - self.entries_count.fetch_add(1, Ordering::SeqCst); - self.persister - .append(&self.path, bytes) - .await - .error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - failed to append state entry data to file, path: {}, data size: {}", - self.path, - len - ) - })?; - debug!("Applied state entry: {entry}"); - Ok(()) - } -} diff --git a/core/server/src/state/models.rs b/core/server/src/state/models.rs deleted file mode 100644 index 15cd357789..0000000000 --- a/core/server/src/state/models.rs +++ /dev/null @@ -1,334 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use bytes::{BufMut, BytesMut}; -use iggy_binary_protocol::requests::{ - consumer_groups::CreateConsumerGroupRequest, - personal_access_tokens::CreatePersonalAccessTokenRequest, streams::CreateStreamRequest, - topics::CreateTopicRequest, users::CreateUserRequest, -}; -use iggy_binary_protocol::{WireDecode, WireEncode}; -use std::fmt; -use std::fmt::{Display, Formatter}; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct CreateStreamWithId { - pub stream_id: u32, - pub command: CreateStreamRequest, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct CreateTopicWithId { - pub topic_id: u32, - pub command: CreateTopicRequest, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct CreateConsumerGroupWithId { - pub group_id: u32, - pub command: CreateConsumerGroupRequest, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct CreateUserWithId { - pub user_id: u32, - pub command: CreateUserRequest, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct CreatePersonalAccessTokenWithHash { - pub hash: String, - pub command: CreatePersonalAccessTokenRequest, -} - -impl Display for CreateStreamWithId { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - write!( - f, - "CreateStreamWithId {{ name: {}, stream_id: {} }}", - self.command.name, self.stream_id - ) - } -} - -impl Display for CreateTopicWithId { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - write!( - f, - "CreateTopicWithId {{ name: {}, topic_id: {} }}", - self.command.name, self.topic_id - ) - } -} - -impl Display for CreateConsumerGroupWithId { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - write!( - f, - "CreateConsumerGroupWithId {{ name: {}, group_id: {} }}", - self.command.name, self.group_id - ) - } -} - -impl Display for CreateUserWithId { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - write!( - f, - "CreateUserWithId {{ username: {}, user_id: {} }}", - self.command.username, self.user_id - ) - } -} - -impl Display for CreatePersonalAccessTokenWithHash { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - write!( - f, - "CreatePersonalAccessTokenWithHash {{ name: {}, hash: [REDACTED] }}", - self.command.name, - ) - } -} - -// Wire format for WithId wrappers: id:u32_le | inner_length:u32_le | inner_bytes - -impl WireEncode for CreateStreamWithId { - fn encoded_size(&self) -> usize { - 4 + 4 + self.command.encoded_size() - } - - fn encode(&self, buf: &mut BytesMut) { - buf.put_u32_le(self.stream_id); - buf.put_u32_le(self.command.encoded_size() as u32); - self.command.encode(buf); - } -} - -impl WireDecode for CreateStreamWithId { - fn decode(buf: &[u8]) -> Result<(Self, usize), iggy_binary_protocol::WireError> { - if buf.len() < 8 { - return Err(iggy_binary_protocol::WireError::UnexpectedEof { - offset: 0, - need: 8, - have: buf.len(), - }); - } - let stream_id = u32::from_le_bytes(buf[0..4].try_into().unwrap()); - let command_length = u32::from_le_bytes(buf[4..8].try_into().unwrap()) as usize; - let total = 8usize.checked_add(command_length).ok_or( - iggy_binary_protocol::WireError::UnexpectedEof { - offset: 4, - need: command_length, - have: buf.len() - 8, - }, - )?; - if buf.len() < total { - return Err(iggy_binary_protocol::WireError::UnexpectedEof { - offset: 8, - need: command_length, - have: buf.len() - 8, - }); - } - let (command, _) = CreateStreamRequest::decode(&buf[8..total])?; - Ok((Self { stream_id, command }, total)) - } -} - -impl WireEncode for CreateTopicWithId { - fn encoded_size(&self) -> usize { - 4 + 4 + self.command.encoded_size() - } - - fn encode(&self, buf: &mut BytesMut) { - buf.put_u32_le(self.topic_id); - buf.put_u32_le(self.command.encoded_size() as u32); - self.command.encode(buf); - } -} - -impl WireDecode for CreateTopicWithId { - fn decode(buf: &[u8]) -> Result<(Self, usize), iggy_binary_protocol::WireError> { - if buf.len() < 8 { - return Err(iggy_binary_protocol::WireError::UnexpectedEof { - offset: 0, - need: 8, - have: buf.len(), - }); - } - let topic_id = u32::from_le_bytes(buf[0..4].try_into().unwrap()); - let command_length = u32::from_le_bytes(buf[4..8].try_into().unwrap()) as usize; - let total = 8usize.checked_add(command_length).ok_or( - iggy_binary_protocol::WireError::UnexpectedEof { - offset: 4, - need: command_length, - have: buf.len() - 8, - }, - )?; - if buf.len() < total { - return Err(iggy_binary_protocol::WireError::UnexpectedEof { - offset: 8, - need: command_length, - have: buf.len() - 8, - }); - } - let (command, _) = CreateTopicRequest::decode(&buf[8..total])?; - Ok((Self { topic_id, command }, total)) - } -} - -impl WireEncode for CreateConsumerGroupWithId { - fn encoded_size(&self) -> usize { - 4 + 4 + self.command.encoded_size() - } - - fn encode(&self, buf: &mut BytesMut) { - buf.put_u32_le(self.group_id); - buf.put_u32_le(self.command.encoded_size() as u32); - self.command.encode(buf); - } -} - -impl WireDecode for CreateConsumerGroupWithId { - fn decode(buf: &[u8]) -> Result<(Self, usize), iggy_binary_protocol::WireError> { - if buf.len() < 8 { - return Err(iggy_binary_protocol::WireError::UnexpectedEof { - offset: 0, - need: 8, - have: buf.len(), - }); - } - let group_id = u32::from_le_bytes(buf[0..4].try_into().unwrap()); - let command_length = u32::from_le_bytes(buf[4..8].try_into().unwrap()) as usize; - let total = 8usize.checked_add(command_length).ok_or( - iggy_binary_protocol::WireError::UnexpectedEof { - offset: 4, - need: command_length, - have: buf.len() - 8, - }, - )?; - if buf.len() < total { - return Err(iggy_binary_protocol::WireError::UnexpectedEof { - offset: 8, - need: command_length, - have: buf.len() - 8, - }); - } - let (command, _) = CreateConsumerGroupRequest::decode(&buf[8..total])?; - Ok((Self { group_id, command }, total)) - } -} - -impl WireEncode for CreateUserWithId { - fn encoded_size(&self) -> usize { - 4 + 4 + self.command.encoded_size() - } - - fn encode(&self, buf: &mut BytesMut) { - buf.put_u32_le(self.user_id); - buf.put_u32_le(self.command.encoded_size() as u32); - self.command.encode(buf); - } -} - -impl WireDecode for CreateUserWithId { - fn decode(buf: &[u8]) -> Result<(Self, usize), iggy_binary_protocol::WireError> { - if buf.len() < 8 { - return Err(iggy_binary_protocol::WireError::UnexpectedEof { - offset: 0, - need: 8, - have: buf.len(), - }); - } - let user_id = u32::from_le_bytes(buf[0..4].try_into().unwrap()); - let command_length = u32::from_le_bytes(buf[4..8].try_into().unwrap()) as usize; - let total = 8usize.checked_add(command_length).ok_or( - iggy_binary_protocol::WireError::UnexpectedEof { - offset: 4, - need: command_length, - have: buf.len() - 8, - }, - )?; - if buf.len() < total { - return Err(iggy_binary_protocol::WireError::UnexpectedEof { - offset: 8, - need: command_length, - have: buf.len() - 8, - }); - } - let (command, _) = CreateUserRequest::decode(&buf[8..total])?; - Ok((Self { user_id, command }, total)) - } -} - -impl WireEncode for CreatePersonalAccessTokenWithHash { - fn encoded_size(&self) -> usize { - 4 + self.hash.len() + 4 + self.command.encoded_size() - } - - fn encode(&self, buf: &mut BytesMut) { - buf.put_u32_le(self.hash.len() as u32); - buf.put_slice(self.hash.as_bytes()); - buf.put_u32_le(self.command.encoded_size() as u32); - self.command.encode(buf); - } -} - -impl WireDecode for CreatePersonalAccessTokenWithHash { - fn decode(buf: &[u8]) -> Result<(Self, usize), iggy_binary_protocol::WireError> { - if buf.len() < 4 { - return Err(iggy_binary_protocol::WireError::UnexpectedEof { - offset: 0, - need: 4, - have: buf.len(), - }); - } - let hash_length = u32::from_le_bytes(buf[0..4].try_into().unwrap()) as usize; - let mut pos = 4; - if buf.len() < pos + hash_length { - return Err(iggy_binary_protocol::WireError::UnexpectedEof { - offset: pos, - need: hash_length, - have: buf.len() - pos, - }); - } - let hash = std::str::from_utf8(&buf[pos..pos + hash_length]) - .map_err(|_| iggy_binary_protocol::WireError::InvalidUtf8 { offset: pos })? - .to_string(); - pos += hash_length; - if buf.len() < pos + 4 { - return Err(iggy_binary_protocol::WireError::UnexpectedEof { - offset: pos, - need: 4, - have: buf.len() - pos, - }); - } - let command_length = u32::from_le_bytes(buf[pos..pos + 4].try_into().unwrap()) as usize; - pos += 4; - if buf.len() < pos + command_length { - return Err(iggy_binary_protocol::WireError::UnexpectedEof { - offset: pos, - need: command_length, - have: buf.len() - pos, - }); - } - let (command, _) = - CreatePersonalAccessTokenRequest::decode(&buf[pos..pos + command_length])?; - pos += command_length; - Ok((Self { hash, command }, pos)) - } -} diff --git a/core/server/src/state/system.rs b/core/server/src/state/system.rs deleted file mode 100644 index 76a7b86174..0000000000 --- a/core/server/src/state/system.rs +++ /dev/null @@ -1,633 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::bootstrap::create_root_user; -use crate::state::file::FileState; -use crate::state::models::CreateUserWithId; -use crate::state::{COMPONENT, EntryCommand, StateEntry}; -use ahash::AHashMap; -use err_trail::ErrContext; -use iggy_binary_protocol::requests::users::CreateUserRequest; -use iggy_binary_protocol::{WireIdentifier, WireName}; -use iggy_common::CompressionAlgorithm; -use iggy_common::IggyError; -use iggy_common::IggyExpiry; -use iggy_common::IggyTimestamp; -use iggy_common::MaxTopicSize; -use iggy_common::PersonalAccessToken; -use iggy_common::defaults::DEFAULT_ROOT_USER_ID; -use iggy_common::wire_conversions::{permissions_to_wire, wire_permissions_to_permissions}; -use iggy_common::{Permissions, UserStatus}; -use std::collections::BTreeMap; -use std::fmt::Display; -use tracing::{debug, error, info}; - -#[derive(Debug, Clone)] -pub struct SystemState { - pub streams: BTreeMap, - pub users: AHashMap, -} - -impl SystemState { - pub fn decompose(self) -> (BTreeMap, AHashMap) { - (self.streams, self.users) - } -} - -#[derive(Debug, Clone)] -pub struct StreamState { - pub id: u32, - pub name: String, - pub created_at: IggyTimestamp, - pub topics: BTreeMap, -} - -#[derive(Debug, Clone)] -pub struct TopicState { - pub id: u32, - pub name: String, - pub partitions: BTreeMap, - pub consumer_groups: BTreeMap, - pub compression_algorithm: CompressionAlgorithm, - pub message_expiry: IggyExpiry, - pub max_topic_size: MaxTopicSize, - pub replication_factor: Option, - pub created_at: IggyTimestamp, -} - -#[derive(Debug, Clone)] -pub struct PartitionState { - pub id: u32, - pub created_at: IggyTimestamp, -} - -// TODO: consider converting token_hash to SecretString (requires updating the full hash flow across crates) -#[derive(Clone)] -pub struct PersonalAccessTokenState { - pub name: String, - pub token_hash: String, - pub expiry_at: Option, -} - -impl std::fmt::Debug for PersonalAccessTokenState { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("PersonalAccessTokenState") - .field("name", &self.name) - .field("token_hash", &"[REDACTED]") - .field("expiry_at", &self.expiry_at) - .finish() - } -} - -// TODO: consider converting password_hash to SecretString (requires updating the full hash flow across crates) -#[derive(Clone)] -pub struct UserState { - pub id: u32, - pub username: String, - pub password_hash: String, - pub status: UserStatus, - pub created_at: IggyTimestamp, - pub permissions: Option, - pub personal_access_tokens: AHashMap, -} - -impl std::fmt::Debug for UserState { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("UserState") - .field("id", &self.id) - .field("username", &self.username) - .field("password_hash", &"[REDACTED]") - .field("status", &self.status) - .field("created_at", &self.created_at) - .field("permissions", &self.permissions) - .field("personal_access_tokens", &self.personal_access_tokens) - .finish() - } -} - -#[derive(Debug, Clone)] -pub struct ConsumerGroupState { - pub id: u32, - pub name: String, -} - -impl SystemState { - pub async fn load(state: FileState) -> Result { - let mut state_entries = state.init().await.error(|e: &IggyError| { - format!("{COMPONENT} (error: {e}) - failed to initialize state entries") - })?; - - // Create root user if does not exist. - let root_exists = state_entries - .iter() - .any(|entry| { - entry - .command() - .map(|command| matches!(command, EntryCommand::CreateUser(payload) if payload.user_id == DEFAULT_ROOT_USER_ID)) - .unwrap_or_else(|err| { - error!("Failed to check if root user exists: {err}"); - false - }) - }); - - if !root_exists { - info!("No users found, creating the root user..."); - let root = create_root_user(); - let command = CreateUserRequest { - username: WireName::new(root.username.clone()) - .expect("root username must be valid"), - password: root.password.clone(), - status: root.status.as_code(), - permissions: root.permissions.as_ref().map(permissions_to_wire), - }; - state - .apply(0, &EntryCommand::CreateUser(CreateUserWithId { - user_id: root.id, - command - })) - .await - .error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - failed to apply create user command, username: {}", - root.username - ) - })?; - state_entries = state.init().await.error(|e: &IggyError| { - format!("{COMPONENT} (error: {e}) - failed to initialize state entries") - })?; - } - - let system_state = Self::init(state_entries).await.error(|e: &IggyError| { - format!("{COMPONENT} (error: {e}) - failed to initialize system state") - })?; - Ok(system_state) - } - - pub async fn init(entries: Vec) -> Result { - let mut streams = BTreeMap::new(); - let mut users = AHashMap::new(); - for entry in entries { - debug!("Processing state entry: {entry}",); - match entry.command().error(|e: &IggyError| { - format!( - "{COMPONENT} (error: {e}) - failed to retrieve state entry command: {entry}" - ) - })? { - EntryCommand::CreateStream(command) => { - info!("Creating stream: {command:?}"); - let stream_id = command.stream_id; - let stream = StreamState { - id: stream_id, - name: command.command.name.to_string(), - topics: BTreeMap::new(), - created_at: entry.timestamp, - }; - streams.insert(stream.id, stream); - } - EntryCommand::UpdateStream(command) => { - let stream_id = find_stream_id(&streams, &command.stream_id); - let stream = streams - .get_mut(&stream_id) - .unwrap_or_else(|| panic!("{}", format!("Stream: {stream_id} not found"))); - stream.name = command.name.to_string(); - } - EntryCommand::DeleteStream(command) => { - let stream_id = find_stream_id(&streams, &command.stream_id); - streams.remove(&stream_id); - } - EntryCommand::PurgeStream(command) => { - let stream_id = find_stream_id(&streams, &command.stream_id); - streams - .get(&stream_id) - .unwrap_or_else(|| panic!("{}", format!("Stream: {stream_id} not found"))); - } - EntryCommand::CreateTopic(command) => { - let stream_id = find_stream_id(&streams, &command.command.stream_id); - let stream = streams - .get_mut(&stream_id) - .unwrap_or_else(|| panic!("{}", format!("Stream: {stream_id} not found"))); - let topic_id = command.topic_id; - let wire = command.command; - let topic = TopicState { - id: topic_id, - name: wire.name.to_string(), - consumer_groups: BTreeMap::new(), - compression_algorithm: CompressionAlgorithm::from_code( - wire.compression_algorithm, - )?, - message_expiry: IggyExpiry::from(wire.message_expiry), - max_topic_size: MaxTopicSize::from(wire.max_topic_size), - replication_factor: if wire.replication_factor == 0 { - None - } else { - Some(wire.replication_factor) - }, - created_at: entry.timestamp, - partitions: if wire.partitions_count > 0 { - let mut partitions = BTreeMap::new(); - for i in 0..wire.partitions_count { - partitions.insert( - i, - PartitionState { - id: i, - created_at: entry.timestamp, - }, - ); - } - partitions - } else { - BTreeMap::new() - }, - }; - stream.topics.insert(topic.id, topic); - } - EntryCommand::UpdateTopic(command) => { - let stream_id = find_stream_id(&streams, &command.stream_id); - let stream = streams - .get_mut(&stream_id) - .unwrap_or_else(|| panic!("{}", format!("Stream: {stream_id} not found"))); - let topic_id = find_topic_id(&stream.topics, &command.topic_id); - let topic = stream - .topics - .get_mut(&topic_id) - .unwrap_or_else(|| panic!("{}", format!("Topic: {topic_id} not found"))); - topic.name = command.name.to_string(); - topic.compression_algorithm = - CompressionAlgorithm::from_code(command.compression_algorithm)?; - topic.message_expiry = IggyExpiry::from(command.message_expiry); - topic.max_topic_size = MaxTopicSize::from(command.max_topic_size); - topic.replication_factor = if command.replication_factor == 0 { - None - } else { - Some(command.replication_factor) - }; - } - EntryCommand::DeleteTopic(command) => { - let stream_id = find_stream_id(&streams, &command.stream_id); - let stream = streams - .get_mut(&stream_id) - .unwrap_or_else(|| panic!("{}", format!("Stream: {stream_id} not found"))); - let topic_id = find_topic_id(&stream.topics, &command.topic_id); - stream.topics.remove(&topic_id); - } - EntryCommand::PurgeTopic(command) => { - let stream_id = find_stream_id(&streams, &command.stream_id); - let stream = streams - .get(&stream_id) - .unwrap_or_else(|| panic!("{}", format!("Stream: {stream_id} not found"))); - let topic_id = find_topic_id(&stream.topics, &command.topic_id); - stream - .topics - .get(&topic_id) - .unwrap_or_else(|| panic!("{}", format!("Topic: {topic_id} not found"))); - } - EntryCommand::CreatePartitions(command) => { - let stream_id = find_stream_id(&streams, &command.stream_id); - let stream = streams - .get_mut(&stream_id) - .unwrap_or_else(|| panic!("{}", format!("Stream: {stream_id} not found"))); - let topic_id = find_topic_id(&stream.topics, &command.topic_id); - let topic = stream - .topics - .get_mut(&topic_id) - .unwrap_or_else(|| panic!("{}", format!("Topic: {topic_id} not found"))); - let last_partition_id = if topic.partitions.is_empty() { - 0 - } else { - topic - .partitions - .values() - .map(|p| p.id) - .max() - .unwrap_or_else(|| panic!("No partition found")) - }; - for i in 1..=command.partitions_count { - topic.partitions.insert( - last_partition_id + i, - PartitionState { - id: last_partition_id + i, - created_at: entry.timestamp, - }, - ); - } - } - EntryCommand::DeletePartitions(command) => { - let stream_id = find_stream_id(&streams, &command.stream_id); - let stream = streams - .get_mut(&stream_id) - .unwrap_or_else(|| panic!("{}", format!("Stream: {stream_id} not found"))); - let topic_id = find_topic_id(&stream.topics, &command.topic_id); - let topic = stream - .topics - .get_mut(&topic_id) - .unwrap_or_else(|| panic!("{}", format!("Topic: {topic_id} not found"))); - if topic.partitions.is_empty() { - continue; - } - - let last_partition_id = topic - .partitions - .values() - .map(|p| p.id) - .max() - .unwrap_or_else(|| panic!("No partition found")); - for i in 0..command.partitions_count { - topic.partitions.remove(&(last_partition_id - i)); - } - } - EntryCommand::DeleteSegments(command) => { - let stream_id = find_stream_id(&streams, &command.stream_id); - let stream = streams - .get_mut(&stream_id) - .unwrap_or_else(|| panic!("{}", format!("Stream: {stream_id} not found"))); - let topic_id = find_topic_id(&stream.topics, &command.topic_id); - let topic = stream - .topics - .get_mut(&topic_id) - .unwrap_or_else(|| panic!("{}", format!("Topic: {topic_id} not found"))); - if topic.partitions.is_empty() { - continue; - } - - let partition_id = command.partition_id; - - let _partition = - topic - .partitions - .get(&command.partition_id) - .unwrap_or_else(|| { - panic!("{}", format!("Partition {partition_id} not found.")) - }); - } - EntryCommand::CreateConsumerGroup(command) => { - let consumer_group_id = command.group_id; - let wire = command.command; - let stream_id = find_stream_id(&streams, &wire.stream_id); - let stream = streams - .get_mut(&stream_id) - .unwrap_or_else(|| panic!("{}", format!("Stream: {stream_id} not found"))); - let topic_id = find_topic_id(&stream.topics, &wire.topic_id); - let topic = stream - .topics - .get_mut(&topic_id) - .unwrap_or_else(|| panic!("{}", format!("Topic: {topic_id} not found"))); - let consumer_group = ConsumerGroupState { - id: consumer_group_id, - name: wire.name.to_string(), - }; - topic - .consumer_groups - .insert(consumer_group.id, consumer_group); - } - EntryCommand::DeleteConsumerGroup(command) => { - let stream_id = find_stream_id(&streams, &command.stream_id); - let stream = streams - .get_mut(&stream_id) - .unwrap_or_else(|| panic!("{}", format!("Stream: {stream_id} not found"))); - let topic_id = find_topic_id(&stream.topics, &command.topic_id); - let topic = stream - .topics - .get_mut(&topic_id) - .unwrap_or_else(|| panic!("{}", format!("Topic: {topic_id} not found"))); - let consumer_group_id = - find_consumer_group_id(&topic.consumer_groups, &command.group_id); - topic.consumer_groups.remove(&consumer_group_id); - } - EntryCommand::CreateUser(command) => { - let user_id = command.user_id; - let wire = command.command; - let user = UserState { - id: user_id, - username: wire.username.to_string(), - password_hash: wire.password, // already hashed at write time - status: UserStatus::from_code(wire.status)?, - created_at: entry.timestamp, - permissions: wire - .permissions - .as_ref() - .map(wire_permissions_to_permissions), - personal_access_tokens: AHashMap::new(), - }; - users.insert(user.id, user); - } - EntryCommand::UpdateUser(command) => { - let user_id = find_user_id(&users, &command.user_id); - let user = users - .get_mut(&user_id) - .unwrap_or_else(|| panic!("{}", format!("User: {user_id} not found"))); - if let Some(username) = &command.username { - user.username = username.to_string(); - } - if let Some(status) = command.status { - user.status = UserStatus::from_code(status)?; - } - } - EntryCommand::DeleteUser(command) => { - let user_id = find_user_id(&users, &command.user_id); - users.remove(&user_id); - } - EntryCommand::ChangePassword(command) => { - let user_id = find_user_id(&users, &command.user_id); - let user = users - .get_mut(&user_id) - .unwrap_or_else(|| panic!("{}", format!("User: {user_id} not found"))); - user.password_hash = command.new_password; // already hashed at write time - } - EntryCommand::UpdatePermissions(command) => { - let user_id = find_user_id(&users, &command.user_id); - let user = users - .get_mut(&user_id) - .unwrap_or_else(|| panic!("{}", format!("User: {user_id} not found"))); - user.permissions = command - .permissions - .as_ref() - .map(wire_permissions_to_permissions); - } - EntryCommand::CreatePersonalAccessToken(command) => { - let token_hash = command.hash; - let user_id = find_user_id(&users, &WireIdentifier::numeric(entry.user_id)); - let user = users - .get_mut(&user_id) - .unwrap_or_else(|| panic!("{}", format!("User: {user_id} not found"))); - let expiry_at = PersonalAccessToken::calculate_expiry_at( - entry.timestamp, - IggyExpiry::from(command.command.expiry), - ); - if let Some(expiry_at) = expiry_at - && expiry_at.as_micros() <= IggyTimestamp::now().as_micros() - { - debug!("Personal access token: {token_hash} has already expired."); - continue; - } - - let name = command.command.name.to_string(); - user.personal_access_tokens.insert( - name.clone(), - PersonalAccessTokenState { - name, - token_hash, - expiry_at, - }, - ); - } - EntryCommand::DeletePersonalAccessToken(command) => { - let user_id = find_user_id(&users, &WireIdentifier::numeric(entry.user_id)); - let user = users - .get_mut(&user_id) - .unwrap_or_else(|| panic!("{}", format!("User: {user_id} not found"))); - user.personal_access_tokens.remove(command.name.as_str()); - } - } - } - - let state = SystemState { streams, users }; - debug!("+++ State +++"); - debug!("{state}"); - debug!("+++ State +++"); - Ok(state) - } -} - -fn find_stream_id(streams: &BTreeMap, stream_id: &WireIdentifier) -> u32 { - match stream_id { - WireIdentifier::Numeric(id) => *id, - WireIdentifier::String(name) => { - let name = name.as_str(); - let stream = streams - .values() - .find(|s| s.name == name) - .unwrap_or_else(|| panic!("Stream: {name} not found")); - stream.id - } - } -} - -fn find_topic_id(topics: &BTreeMap, topic_id: &WireIdentifier) -> u32 { - match topic_id { - WireIdentifier::Numeric(id) => *id, - WireIdentifier::String(name) => { - let name = name.as_str(); - let topic = topics - .values() - .find(|s| s.name == name) - .unwrap_or_else(|| panic!("Topic: {name} not found")); - topic.id - } - } -} - -fn find_consumer_group_id( - groups: &BTreeMap, - group_id: &WireIdentifier, -) -> u32 { - match group_id { - WireIdentifier::Numeric(id) => *id, - WireIdentifier::String(name) => { - let name = name.as_str(); - let group = groups - .values() - .find(|s| s.name == name) - .unwrap_or_else(|| panic!("Consumer group: {name} not found")); - group.id - } - } -} - -fn find_user_id(users: &AHashMap, user_id: &WireIdentifier) -> u32 { - match user_id { - WireIdentifier::Numeric(id) => *id, - WireIdentifier::String(name) => { - let name = name.as_str(); - let user = users - .values() - .find(|s| s.username == name) - .unwrap_or_else(|| panic!("User: {name} not found")); - user.id - } - } -} - -impl Display for SystemState { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "Streams:")?; - for stream in self.streams.iter() { - write!(f, "\n================\n")?; - write!(f, "{}", stream.1)?; - } - write!(f, "Users:")?; - for user in self.users.iter() { - write!(f, "\n================\n")?; - write!(f, "{}", user.1)?; - } - Ok(()) - } -} - -impl Display for ConsumerGroupState { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "ConsumerGroup -> ID: {}, Name: {}", self.id, self.name) - } -} - -impl Display for UserState { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let permissions = if let Some(permissions) = &self.permissions { - permissions.to_string() - } else { - "no_permissions".to_string() - }; - write!( - f, - "User -> ID: {}, Username: {}, Status: {}, Permissions: {}", - self.id, self.username, self.status, permissions - ) - } -} - -impl Display for StreamState { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "Stream -> ID: {}, Name: {}", self.id, self.name,)?; - for topic in self.topics.iter() { - write!(f, "\n {}", topic.1)?; - } - Ok(()) - } -} - -impl Display for TopicState { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "Topic -> ID: {}, Name: {}", self.id, self.name,)?; - for partition in self.partitions.iter() { - write!(f, "\n {}", partition.1)?; - } - write!(f, "\nConsumer Groups:")?; - for consumer_group in self.consumer_groups.iter() { - write!(f, "\n {}", consumer_group.1)?; - } - Ok(()) - } -} - -impl Display for PartitionState { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "Partition -> ID: {}, Created At: {}", - self.id, self.created_at - ) - } -} diff --git a/core/server/src/streaming/clients/client_manager.rs b/core/server/src/streaming/clients/client_manager.rs deleted file mode 100644 index f8ba859378..0000000000 --- a/core/server/src/streaming/clients/client_manager.rs +++ /dev/null @@ -1,233 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::streaming::session::Session; -use crate::streaming::utils::ptr::EternalPtr; -use dashmap::DashMap; -use iggy_common::IggyTimestamp; -use iggy_common::TransportProtocol; -use iggy_common::UserId; -use iggy_common::{IggyError, calculate_32}; -use std::net::SocketAddr; - -pub struct ClientManager { - clients: EternalPtr>, -} - -impl ClientManager { - pub fn new(clients: EternalPtr>) -> Self { - Self { clients } - } -} - -impl Clone for ClientManager { - fn clone(&self) -> Self { - Self { - clients: self.clients.clone(), - } - } -} - -#[derive(Debug, Clone)] -pub struct Client { - pub user_id: Option, - pub session: Session, - pub transport: TransportProtocol, - pub consumer_groups: Vec, - pub last_heartbeat: IggyTimestamp, -} - -#[derive(Debug, Clone)] -pub struct ConsumerGroup { - pub stream_id: u32, - pub topic_id: u32, - pub group_id: u32, -} - -impl ClientManager { - pub fn add_client(&self, address: &SocketAddr, transport: TransportProtocol) -> Session { - let client_id = calculate_32(address.to_string().as_bytes()); - let session = Session::from_client_id(client_id, *address); - let client = Client { - user_id: None, - session: session.clone(), - transport, - consumer_groups: Vec::new(), - last_heartbeat: IggyTimestamp::now(), - }; - self.clients.insert(client_id, client); - session - } - - pub fn set_user_id(&self, client_id: u32, user_id: UserId) -> Result<(), IggyError> { - self.clients - .get_mut(&client_id) - .ok_or(IggyError::ClientNotFound(client_id))? - .user_id = Some(user_id); - Ok(()) - } - - pub fn clear_user_id(&self, client_id: u32) -> Result<(), IggyError> { - self.clients - .get_mut(&client_id) - .ok_or(IggyError::ClientNotFound(client_id))? - .user_id = None; - Ok(()) - } - - pub fn try_get_client(&self, client_id: u32) -> Option { - self.clients.get(&client_id).map(|c| c.clone()) - } - - pub fn try_get_client_mut( - &'_ self, - client_id: u32, - ) -> Option> { - self.clients.get_mut(&client_id) - } - - pub fn get_clients(&self) -> Vec { - self.clients - .iter() - .map(|entry| entry.value().clone()) - .collect() - } - - pub fn delete_clients_for_user(&self, user_id: UserId) -> Result<(), IggyError> { - let clients_to_remove: Vec = self - .clients - .iter() - .filter(|entry| entry.value().user_id == Some(user_id)) - .map(|entry| *entry.key()) - .collect(); - - for client_id in clients_to_remove { - self.clients.remove(&client_id); - } - Ok(()) - } - - pub fn delete_client(&self, client_id: u32) -> Option { - self.clients.remove(&client_id).map(|(_, client)| client) - } - - pub fn get_client_count(&self) -> usize { - self.clients.len() - } - - pub fn heartbeat(&mut self, client_id: u32) -> Result<(), IggyError> { - let mut client = self - .clients - .get_mut(&client_id) - .ok_or(IggyError::StaleClient)?; - client.last_heartbeat = IggyTimestamp::now(); - Ok(()) - } - - pub fn join_consumer_group( - &self, - client_id: u32, - stream_id: usize, - topic_id: usize, - group_id: usize, - ) -> Result<(), IggyError> { - let stream_id = stream_id as u32; - let topic_id = topic_id as u32; - let group_id = group_id as u32; - - let mut client = self - .clients - .get_mut(&client_id) - .ok_or(IggyError::StaleClient)?; - - if client.consumer_groups.iter().any(|consumer_group| { - consumer_group.group_id == group_id - && consumer_group.topic_id == topic_id - && consumer_group.stream_id == stream_id - }) { - return Ok(()); - } - - client.consumer_groups.push(ConsumerGroup { - stream_id, - topic_id, - group_id, - }); - Ok(()) - } - - pub fn leave_consumer_group( - &self, - client_id: u32, - stream_id: usize, - topic_id: usize, - consumer_group_id: usize, - ) -> Result<(), IggyError> { - let stream_id = stream_id as u32; - let topic_id = topic_id as u32; - let consumer_group_id = consumer_group_id as u32; - - let mut client = self - .clients - .get_mut(&client_id) - .ok_or(IggyError::StaleClient)?; - - if let Some(index) = client.consumer_groups.iter().position(|consumer_group| { - consumer_group.stream_id == stream_id - && consumer_group.topic_id == topic_id - && consumer_group.group_id == consumer_group_id - }) { - client.consumer_groups.remove(index); - } - Ok(()) - } - - pub fn delete_consumer_group(&self, stream_id: usize, topic_id: usize, group_id: usize) { - let stream_id = stream_id as u32; - let topic_id = topic_id as u32; - let group_id = group_id as u32; - - for mut client in self.clients.iter_mut() { - client.consumer_groups.retain(|consumer_group| { - !(consumer_group.stream_id == stream_id - && consumer_group.topic_id == topic_id - && consumer_group.group_id == group_id) - }); - } - } - - pub fn delete_consumer_groups_for_stream(&self, stream_id: usize) { - let stream_id = stream_id as u32; - - for mut client in self.clients.iter_mut() { - client - .consumer_groups - .retain(|consumer_group| consumer_group.stream_id != stream_id); - } - } - - pub fn delete_consumer_groups_for_topic(&self, stream_id: usize, topic_id: usize) { - let stream_id = stream_id as u32; - let topic_id = topic_id as u32; - - for mut client in self.clients.iter_mut() { - client.consumer_groups.retain(|consumer_group| { - !(consumer_group.stream_id == stream_id && consumer_group.topic_id == topic_id) - }); - } - } -} diff --git a/core/server/src/streaming/clients/mod.rs b/core/server/src/streaming/clients/mod.rs deleted file mode 100644 index 3048ad3603..0000000000 --- a/core/server/src/streaming/clients/mod.rs +++ /dev/null @@ -1,18 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod client_manager; diff --git a/core/server/src/streaming/deduplication/mod.rs b/core/server/src/streaming/deduplication/mod.rs deleted file mode 100644 index bd18c67f31..0000000000 --- a/core/server/src/streaming/deduplication/mod.rs +++ /dev/null @@ -1,18 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub use server_common::MessageDeduplicator; diff --git a/core/server/src/streaming/diagnostics/metrics.rs b/core/server/src/streaming/diagnostics/metrics.rs deleted file mode 100644 index a0a1dca062..0000000000 --- a/core/server/src/streaming/diagnostics/metrics.rs +++ /dev/null @@ -1,149 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use prometheus_client::encoding::text::encode; -use prometheus_client::metrics::counter::Counter; -use prometheus_client::metrics::gauge::Gauge; -use prometheus_client::registry::Registry; -use std::sync::Arc; -use tracing::error; - -#[derive(Debug, Clone)] -pub struct Metrics { - registry: Arc, - http_requests: Counter, - streams: Gauge, - topics: Gauge, - partitions: Gauge, - segments: Gauge, - messages: Gauge, - users: Gauge, - clients: Gauge, -} - -impl Metrics { - pub fn init() -> Self { - let mut registry = Registry::default(); - - let http_requests = Counter::default(); - let streams = Gauge::default(); - let topics = Gauge::default(); - let partitions = Gauge::default(); - let segments = Gauge::default(); - let messages = Gauge::default(); - let users = Gauge::default(); - let clients = Gauge::default(); - - registry.register( - "http_requests", - "total count of http_requests", - http_requests.clone(), - ); - registry.register("streams", "total count of streams", streams.clone()); - registry.register("topics", "total count of topics", topics.clone()); - registry.register( - "partitions", - "total count of partitions", - partitions.clone(), - ); - registry.register("segments", "total count of segments", segments.clone()); - registry.register("messages", "total count of messages", messages.clone()); - registry.register("users", "total count of users", users.clone()); - registry.register("clients", "total count of clients", clients.clone()); - - let registry = registry.into(); - Self { - registry, - http_requests, - streams, - topics, - partitions, - segments, - messages, - users, - clients, - } - } - - pub fn get_formatted_output(&self) -> String { - let mut buffer = String::new(); - if let Err(err) = encode(&mut buffer, &self.registry) { - error!("Failed to encode metrics: {}", err); - } - buffer - } - - pub fn increment_http_requests(&self) { - self.http_requests.inc(); - } - - pub fn increment_streams(&self, count: u32) { - self.streams.inc_by(count as i64); - } - - pub fn decrement_streams(&self, count: u32) { - self.streams.dec_by(count as i64); - } - - pub fn increment_topics(&self, count: u32) { - self.topics.inc_by(count as i64); - } - - pub fn decrement_topics(&self, count: u32) { - self.topics.dec_by(count as i64); - } - - pub fn increment_partitions(&self, count: u32) { - self.partitions.inc_by(count as i64); - } - - pub fn decrement_partitions(&self, count: u32) { - self.partitions.dec_by(count as i64); - } - - pub fn increment_segments(&self, count: u32) { - self.segments.inc_by(count as i64); - } - - pub fn decrement_segments(&self, count: u32) { - self.segments.dec_by(count as i64); - } - - pub fn increment_messages(&self, count: u64) { - self.messages.inc_by(count as i64); - } - - pub fn decrement_messages(&self, count: u64) { - self.messages.dec_by(count as i64); - } - - pub fn increment_users(&self, count: u32) { - self.users.inc_by(count as i64); - } - - pub fn decrement_users(&self, count: u32) { - self.users.dec_by(count as i64); - } - - pub fn increment_clients(&self, count: u32) { - self.clients.inc_by(count as i64); - } - - pub fn decrement_clients(&self, count: u32) { - self.clients.dec_by(count as i64); - } -} diff --git a/core/server/src/streaming/diagnostics/mod.rs b/core/server/src/streaming/diagnostics/mod.rs deleted file mode 100644 index 054bb9088b..0000000000 --- a/core/server/src/streaming/diagnostics/mod.rs +++ /dev/null @@ -1,18 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod metrics; diff --git a/core/server/src/streaming/mod.rs b/core/server/src/streaming/mod.rs deleted file mode 100644 index d370e9c2b8..0000000000 --- a/core/server/src/streaming/mod.rs +++ /dev/null @@ -1,31 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod clients; -pub mod deduplication; -pub mod diagnostics; -pub mod partitions; -pub mod persistence; -pub mod polling_consumer; -pub mod segments; -pub mod session; -pub mod stats; -pub mod storage; -pub mod streams; -pub mod topics; -pub mod users; -pub mod utils; diff --git a/core/server/src/streaming/partitions/consumer_group_offsets.rs b/core/server/src/streaming/partitions/consumer_group_offsets.rs deleted file mode 100644 index 623d397a5c..0000000000 --- a/core/server/src/streaming/partitions/consumer_group_offsets.rs +++ /dev/null @@ -1,18 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub use iggy_common::ConsumerGroupOffsets; diff --git a/core/server/src/streaming/partitions/consumer_offset.rs b/core/server/src/streaming/partitions/consumer_offset.rs deleted file mode 100644 index 9cff9043b9..0000000000 --- a/core/server/src/streaming/partitions/consumer_offset.rs +++ /dev/null @@ -1,18 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub use iggy_common::ConsumerOffset; diff --git a/core/server/src/streaming/partitions/consumer_offsets.rs b/core/server/src/streaming/partitions/consumer_offsets.rs deleted file mode 100644 index 4dba7dd28e..0000000000 --- a/core/server/src/streaming/partitions/consumer_offsets.rs +++ /dev/null @@ -1,18 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub use iggy_common::ConsumerOffsets; diff --git a/core/server/src/streaming/partitions/helpers.rs b/core/server/src/streaming/partitions/helpers.rs deleted file mode 100644 index 89f4e73c10..0000000000 --- a/core/server/src/streaming/partitions/helpers.rs +++ /dev/null @@ -1,36 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::{configs::system::SystemConfig, streaming::deduplication::MessageDeduplicator}; - -pub fn create_message_deduplicator(config: &SystemConfig) -> Option { - if !config.message_deduplication.enabled { - return None; - } - let max_entries = if config.message_deduplication.max_entries > 0 { - Some(config.message_deduplication.max_entries) - } else { - None - }; - let expiry = if !config.message_deduplication.expiry.is_zero() { - Some(config.message_deduplication.expiry) - } else { - None - }; - - Some(MessageDeduplicator::new(max_entries, expiry)) -} diff --git a/core/server/src/streaming/partitions/in_flight.rs b/core/server/src/streaming/partitions/in_flight.rs deleted file mode 100644 index ca726df8f3..0000000000 --- a/core/server/src/streaming/partitions/in_flight.rs +++ /dev/null @@ -1,18 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub use server_common::IggyMessagesBatchSetInFlight; diff --git a/core/server/src/streaming/partitions/journal.rs b/core/server/src/streaming/partitions/journal.rs deleted file mode 100644 index 87bd9cfcb5..0000000000 --- a/core/server/src/streaming/partitions/journal.rs +++ /dev/null @@ -1,212 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::streaming::segments::{IggyMessagesBatchMut, IggyMessagesBatchSet}; -use iggy_common::{IggyByteSize, IggyError}; -use std::fmt::Debug; - -#[derive(Default, Debug)] -pub struct Inner { - /// Base offset for the next journal epoch. After commit(), set to - /// current_offset + 1. Used in `append()`: `current_offset = base_offset + - /// messages_count - 1`. - pub base_offset: u64, - pub current_offset: u64, - pub first_timestamp: u64, - pub end_timestamp: u64, - pub messages_count: u32, - pub size: IggyByteSize, -} - -#[derive(Debug)] -pub struct MemoryMessageJournal { - batches: IggyMessagesBatchSet, - inner: Inner, -} - -impl MemoryMessageJournal { - /// Create an empty journal for a fresh partition (no existing data). - pub fn empty() -> Self { - Self { - batches: IggyMessagesBatchSet::default(), - inner: Inner::default(), - } - } - - /// Create an empty journal positioned at the given offset. Used after - /// bootstrap when the partition already has data on disk up to some offset. - pub fn at_offset(base_offset: u64) -> Self { - Self { - batches: IggyMessagesBatchSet::default(), - inner: Inner { - base_offset, - ..Default::default() - }, - } - } -} - -impl Journal for MemoryMessageJournal { - type Container = IggyMessagesBatchSet; - type Entry = IggyMessagesBatchMut; - type Inner = Inner; - type AppendResult = Result<(u32, u32), IggyError>; - - fn append(&mut self, entry: Self::Entry) -> Self::AppendResult { - let batch_messages_count = entry.count(); - tracing::trace!( - "Coalescing batch with base_offset: {}, current_offset: {}, self.messages_count: {}, batch.count: {}", - self.inner.base_offset, - self.inner.current_offset, - self.inner.messages_count, - batch_messages_count - ); - - // Defense-in-depth: on first append after empty/default state, correct - // base_offset from the batch's actual first offset. Mirrors the existing - // first_timestamp initialization pattern below. Catches code paths that - // create a journal without calling init(). - if self.inner.messages_count == 0 - && let Some(first_offset) = entry.first_offset() - { - // Allow disagreement when either side is 0 (fresh partition or - // reset after purge). Only flag when both are non-zero and differ. - debug_assert!( - self.inner.base_offset == 0 - || first_offset == 0 - || self.inner.base_offset == first_offset, - "journal base_offset ({}) disagrees with batch first_offset ({})", - self.inner.base_offset, - first_offset - ); - self.inner.base_offset = first_offset; - } - - let batch_size = entry.size(); - let first_timestamp = entry.first_timestamp().unwrap(); - let last_timestamp = entry.last_timestamp().unwrap(); - self.batches.add_batch(entry); - - if self.inner.first_timestamp == 0 { - self.inner.first_timestamp = first_timestamp; - } - self.inner.end_timestamp = last_timestamp; - self.inner.messages_count += batch_messages_count; - self.inner.current_offset = self.inner.base_offset + self.inner.messages_count as u64 - 1; - self.inner.size = IggyByteSize::from(self.inner.size.as_bytes_u64() + batch_size as u64); - - Ok((self.inner.messages_count, self.inner.size.as_bytes_u32())) - } - - async fn flush(&self) -> Result<(), IggyError> { - Ok(()) - } - - fn init(&mut self, inner: Self::Inner) { - self.inner = inner - } - - fn get(&self, filter: impl FnOnce(&Self::Container) -> U) -> U { - filter(&self.batches) - } - - fn commit(&mut self) -> Self::Container { - self.inner.base_offset = self.inner.current_offset + 1; - self.inner.first_timestamp = 0; - self.inner.end_timestamp = 0; - self.inner.size = IggyByteSize::default(); - self.inner.messages_count = 0; - std::mem::take(&mut self.batches) - } - - fn is_empty(&self) -> bool { - self.batches.is_empty() - } - - fn inner(&self) -> &Self::Inner { - &self.inner - } - - fn first_offset(&self) -> Option { - if self.is_empty() { - None - } else { - Some(self.inner.base_offset) - } - } - - fn last_offset(&self) -> Option { - if self.is_empty() { - None - } else { - Some(self.inner.current_offset) - } - } - - fn first_timestamp(&self) -> Option { - if self.is_empty() || self.inner.first_timestamp == 0 { - None - } else { - Some(self.inner.first_timestamp) - } - } - - fn last_timestamp(&self) -> Option { - if self.is_empty() || self.inner.end_timestamp == 0 { - None - } else { - Some(self.inner.end_timestamp) - } - } -} - -pub trait Journal { - type Container; - type Entry; - type Inner; - type AppendResult; - - fn init(&mut self, inner: Self::Inner); - - fn append(&mut self, entry: Self::Entry) -> Self::AppendResult; - - fn get(&self, filter: impl FnOnce(&Self::Container) -> U) -> U; - - fn commit(&mut self) -> Self::Container; - - fn is_empty(&self) -> bool; - - fn inner(&self) -> &Self::Inner; - - /// First offset of data in the journal, or None if empty. - fn first_offset(&self) -> Option; - - /// Last offset of data in the journal, or None if empty. - fn last_offset(&self) -> Option; - - /// Timestamp of first message in journal, or None if empty. - fn first_timestamp(&self) -> Option; - - /// Timestamp of last message in journal, or None if empty. - fn last_timestamp(&self) -> Option; - - // `flush` is only useful in case of an journal that has disk backed WAL. - // This could be merged together with `append`, but not doing this for two reasons. - // 1. In case of the `Journal` being used as part of structure that utilizes interior mutability, async with borrow_mut is not possible. - // 2. Having it as separate function allows for more optimal usage patterns, e.g. batching multiple appends before flushing. - fn flush(&self) -> impl Future>; -} diff --git a/core/server/src/streaming/partitions/local_partition.rs b/core/server/src/streaming/partitions/local_partition.rs deleted file mode 100644 index de49720efa..0000000000 --- a/core/server/src/streaming/partitions/local_partition.rs +++ /dev/null @@ -1,98 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Per-shard partition data. -//! -//! Each shard runs on a single-threaded compio runtime, so per-shard data -//! needs NO synchronization. - -use super::{ - consumer_group_offsets::ConsumerGroupOffsets, consumer_offsets::ConsumerOffsets, - journal::MemoryMessageJournal, log::SegmentedLog, -}; -use crate::streaming::{deduplication::MessageDeduplicator, stats::PartitionStats}; -use iggy_common::IggyTimestamp; -use std::sync::{Arc, atomic::AtomicU64}; - -/// Per-shard partition data - mutable, single-threaded access. -#[derive(Debug)] -pub struct LocalPartition { - pub log: SegmentedLog, - pub offset: Arc, - pub consumer_offsets: Arc, - pub consumer_group_offsets: Arc, - pub message_deduplicator: Option>, - pub stats: Arc, - pub created_at: IggyTimestamp, - pub revision_id: u64, - pub should_increment_offset: bool, -} - -impl LocalPartition { - /// Create new partition data with default log. - #[allow(clippy::too_many_arguments)] - pub fn new( - stats: Arc, - offset: Arc, - consumer_offsets: Arc, - consumer_group_offsets: Arc, - message_deduplicator: Option>, - created_at: IggyTimestamp, - revision_id: u64, - should_increment_offset: bool, - ) -> Self { - Self { - log: SegmentedLog::new( - crate::streaming::partitions::journal::MemoryMessageJournal::empty(), - ), - offset, - consumer_offsets, - consumer_group_offsets, - message_deduplicator, - stats, - created_at, - revision_id, - should_increment_offset, - } - } - - /// Create partition data with existing log (e.g., loaded from disk). - #[allow(clippy::too_many_arguments)] - pub fn with_log( - log: SegmentedLog, - stats: Arc, - offset: Arc, - consumer_offsets: Arc, - consumer_group_offsets: Arc, - message_deduplicator: Option>, - created_at: IggyTimestamp, - revision_id: u64, - should_increment_offset: bool, - ) -> Self { - Self { - log, - offset, - consumer_offsets, - consumer_group_offsets, - message_deduplicator, - stats, - created_at, - revision_id, - should_increment_offset, - } - } -} diff --git a/core/server/src/streaming/partitions/local_partitions.rs b/core/server/src/streaming/partitions/local_partitions.rs deleted file mode 100644 index 1b3073fcbe..0000000000 --- a/core/server/src/streaming/partitions/local_partitions.rs +++ /dev/null @@ -1,213 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Per-shard partition storage. -//! -//! Single-threaded (compio runtime) - NO synchronization needed! - -use super::local_partition::LocalPartition; -use server_common::sharding::IggyNamespace; -use std::collections::HashMap; - -/// Per-shard partition storage. -/// Single-threaded (compio runtime) - NO synchronization needed! -#[derive(Debug, Default)] -pub struct LocalPartitions { - partitions: HashMap, -} - -impl LocalPartitions { - pub fn new() -> Self { - Self { - partitions: HashMap::new(), - } - } - - pub fn with_capacity(capacity: usize) -> Self { - Self { - partitions: HashMap::with_capacity(capacity), - } - } - - #[inline] - pub fn get(&self, ns: &IggyNamespace) -> Option<&LocalPartition> { - self.partitions.get(ns) - } - - #[inline] - pub fn get_mut(&mut self, ns: &IggyNamespace) -> Option<&mut LocalPartition> { - self.partitions.get_mut(ns) - } - - #[inline] - pub fn insert(&mut self, ns: IggyNamespace, data: LocalPartition) { - self.partitions.insert(ns, data); - } - - #[inline] - pub fn remove(&mut self, ns: &IggyNamespace) -> Option { - self.partitions.remove(ns) - } - - #[inline] - pub fn contains(&self, ns: &IggyNamespace) -> bool { - self.partitions.contains_key(ns) - } - - #[inline] - pub fn len(&self) -> usize { - self.partitions.len() - } - - #[inline] - pub fn is_empty(&self) -> bool { - self.partitions.is_empty() - } - - /// Iterate over all namespaces owned by this shard. - pub fn namespaces(&self) -> impl Iterator { - self.partitions.keys() - } - - /// Iterate over all partition data. - pub fn iter(&self) -> impl Iterator { - self.partitions.iter() - } - - /// Iterate over all partition data mutably. - pub fn iter_mut(&mut self) -> impl Iterator { - self.partitions.iter_mut() - } - - /// Remove multiple partitions at once. - pub fn remove_many(&mut self, namespaces: &[IggyNamespace]) -> Vec { - namespaces - .iter() - .filter_map(|ns| self.partitions.remove(ns)) - .collect() - } - - /// Get partition data, initializing if not present. - pub fn get_or_init(&mut self, ns: IggyNamespace, init: F) -> &mut LocalPartition - where - F: FnOnce() -> LocalPartition, - { - self.partitions.entry(ns).or_insert_with(init) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::streaming::{ - partitions::{ - consumer_group_offsets::ConsumerGroupOffsets, consumer_offsets::ConsumerOffsets, - }, - stats::{PartitionStats, StreamStats, TopicStats}, - }; - use iggy_common::IggyTimestamp; - use std::sync::{Arc, atomic::AtomicU64}; - - fn create_test_partition() -> LocalPartition { - let stream_stats = Arc::new(StreamStats::default()); - let topic_stats = Arc::new(TopicStats::new(stream_stats)); - let partition_stats = Arc::new(PartitionStats::new(topic_stats)); - - LocalPartition::new( - partition_stats, - Arc::new(AtomicU64::new(0)), - Arc::new(ConsumerOffsets::with_capacity(10)), - Arc::new(ConsumerGroupOffsets::with_capacity(10)), - None, - IggyTimestamp::now(), - 1, - true, - ) - } - - #[test] - fn test_basic_operations() { - let mut partitions = LocalPartitions::new(); - let ns = IggyNamespace::new(1, 1, 0); - - assert!(!partitions.contains(&ns)); - assert!(partitions.is_empty()); - - partitions.insert(ns, create_test_partition()); - - assert!(partitions.contains(&ns)); - assert_eq!(partitions.len(), 1); - assert!(partitions.get(&ns).is_some()); - assert!(partitions.get_mut(&ns).is_some()); - - let removed = partitions.remove(&ns); - assert!(removed.is_some()); - assert!(!partitions.contains(&ns)); - assert!(partitions.is_empty()); - } - - #[test] - fn test_iteration() { - let mut partitions = LocalPartitions::new(); - let ns1 = IggyNamespace::new(1, 1, 0); - let ns2 = IggyNamespace::new(1, 1, 1); - let ns3 = IggyNamespace::new(1, 2, 0); - - partitions.insert(ns1, create_test_partition()); - partitions.insert(ns2, create_test_partition()); - partitions.insert(ns3, create_test_partition()); - - let namespaces: Vec<_> = partitions.namespaces().collect(); - assert_eq!(namespaces.len(), 3); - - let pairs: Vec<_> = partitions.iter().collect(); - assert_eq!(pairs.len(), 3); - } - - #[test] - fn test_remove_many() { - let mut partitions = LocalPartitions::new(); - let ns1 = IggyNamespace::new(1, 1, 0); - let ns2 = IggyNamespace::new(1, 1, 1); - let ns3 = IggyNamespace::new(1, 2, 0); - - partitions.insert(ns1, create_test_partition()); - partitions.insert(ns2, create_test_partition()); - partitions.insert(ns3, create_test_partition()); - - let removed = partitions.remove_many(&[ns1, ns2]); - assert_eq!(removed.len(), 2); - assert!(!partitions.contains(&ns1)); - assert!(!partitions.contains(&ns2)); - assert!(partitions.contains(&ns3)); - } - - #[test] - fn test_get_or_init() { - let mut partitions = LocalPartitions::new(); - let ns = IggyNamespace::new(1, 1, 0); - - assert!(!partitions.contains(&ns)); - - let _ = partitions.get_or_init(ns, create_test_partition); - assert!(partitions.contains(&ns)); - - // Second call should not reinitialize - let data = partitions.get_or_init(ns, || panic!("Should not be called")); - assert!(data.should_increment_offset); - } -} diff --git a/core/server/src/streaming/partitions/log.rs b/core/server/src/streaming/partitions/log.rs deleted file mode 100644 index 16318553e5..0000000000 --- a/core/server/src/streaming/partitions/log.rs +++ /dev/null @@ -1,207 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::streaming::{ - partitions::{in_flight::IggyMessagesBatchSetInFlight, journal::Journal}, - segments::{IggyIndexesMut, Segment, storage::Storage}, -}; -use iggy_common::{INDEX_SIZE, IggyMessagesBatch}; -use ringbuffer::AllocRingBuffer; -use std::fmt::Debug; - -const SEGMENTS_CAPACITY: usize = 1024; -const ACCESS_MAP_CAPACITY: usize = 8; -const SIZE_16MB: usize = 16 * 1024 * 1024; - -#[derive(Debug)] -pub struct SegmentedLog -where - J: Journal + Debug, -{ - journal: J, - // Ring buffer tracking recently accessed segment indices for cleanup optimization. - // A background task uses this to identify and close file descriptors for unused segments. - _access_map: AllocRingBuffer, - _cache: (), - segments: Vec, - indexes: Vec>, - storage: Vec, - in_flight: IggyMessagesBatchSetInFlight, -} - -impl Default for SegmentedLog -where - J: Journal + Debug + Default, -{ - fn default() -> Self { - Self { - journal: J::default(), - _access_map: AllocRingBuffer::with_capacity_power_of_2(ACCESS_MAP_CAPACITY), - _cache: (), - segments: Vec::with_capacity(SEGMENTS_CAPACITY), - storage: Vec::with_capacity(SEGMENTS_CAPACITY), - indexes: Vec::with_capacity(SEGMENTS_CAPACITY), - in_flight: IggyMessagesBatchSetInFlight::default(), - } - } -} - -impl SegmentedLog -where - J: Journal + Debug, -{ - pub fn new(journal: J) -> Self { - Self { - journal, - _access_map: AllocRingBuffer::with_capacity_power_of_2(ACCESS_MAP_CAPACITY), - _cache: (), - segments: Vec::with_capacity(SEGMENTS_CAPACITY), - storage: Vec::with_capacity(SEGMENTS_CAPACITY), - indexes: Vec::with_capacity(SEGMENTS_CAPACITY), - in_flight: IggyMessagesBatchSetInFlight::default(), - } - } - - pub fn has_segments(&self) -> bool { - !self.segments.is_empty() - } - - pub fn segments(&self) -> &Vec { - &self.segments - } - - pub fn segments_mut(&mut self) -> &mut Vec { - &mut self.segments - } - - pub fn storages_mut(&mut self) -> &mut Vec { - &mut self.storage - } - - pub fn storages(&self) -> &Vec { - &self.storage - } - - pub fn active_segment(&self) -> &Segment { - self.segments - .last() - .expect("active segment called on empty log") - } - - pub fn active_segment_mut(&mut self) -> &mut Segment { - self.segments - .last_mut() - .expect("active segment called on empty log") - } - - pub fn active_storage(&self) -> &Storage { - self.storage - .last() - .expect("active storage called on empty log") - } - - pub fn active_storage_mut(&mut self) -> &mut Storage { - self.storage - .last_mut() - .expect("active storage called on empty log") - } - - pub fn indexes(&self) -> &Vec> { - &self.indexes - } - - pub fn indexes_mut(&mut self) -> &mut Vec> { - &mut self.indexes - } - - pub fn active_indexes(&self) -> Option<&IggyIndexesMut> { - self.indexes - .last() - .expect("active indexes called on empty log") - .as_ref() - } - - pub fn active_indexes_mut(&mut self) -> Option<&mut IggyIndexesMut> { - self.indexes - .last_mut() - .expect("active indexes called on empty log") - .as_mut() - } - - pub fn clear_active_indexes(&mut self) { - let indexes = self - .indexes - .last_mut() - .expect("active indexes called on empty log"); - *indexes = None; - } - - pub fn ensure_indexes(&mut self) { - let indexes = self - .indexes - .last_mut() - .expect("active indexes called on empty log"); - if indexes.is_none() { - let capacity = SIZE_16MB / INDEX_SIZE; - *indexes = Some(IggyIndexesMut::with_capacity(capacity, 0)); - } - } - - pub fn add_persisted_segment(&mut self, segment: Segment, storage: Storage) { - self.segments.push(segment); - self.storage.push(storage); - self.indexes.push(None); - } - - pub fn set_segment_indexes(&mut self, segment_index: usize, indexes: IggyIndexesMut) { - if let Some(segment_indexes) = self.indexes.get_mut(segment_index) { - *segment_indexes = Some(indexes); - } - } - - pub fn in_flight(&self) -> &IggyMessagesBatchSetInFlight { - &self.in_flight - } - - pub fn in_flight_mut(&mut self) -> &mut IggyMessagesBatchSetInFlight { - &mut self.in_flight - } - - pub fn set_in_flight(&mut self, batches: Vec) { - self.in_flight.set(batches); - } - - pub fn clear_in_flight(&mut self) { - self.in_flight.clear(); - } -} - -impl SegmentedLog -where - J: Journal + Debug, -{ - pub fn journal_mut(&mut self) -> &mut J { - &mut self.journal - } - - pub fn journal(&self) -> &J { - &self.journal - } -} - -impl Log for SegmentedLog where J: Journal + Debug {} -pub trait Log {} diff --git a/core/server/src/streaming/partitions/mod.rs b/core/server/src/streaming/partitions/mod.rs deleted file mode 100644 index 110edaa1a9..0000000000 --- a/core/server/src/streaming/partitions/mod.rs +++ /dev/null @@ -1,33 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod consumer_group_offsets; -pub mod consumer_offset; -pub mod consumer_offsets; -pub mod helpers; -pub mod in_flight; -pub mod journal; -pub mod local_partition; -pub mod local_partitions; -pub mod log; -pub mod ops; -#[cfg(test)] -mod ops_tests; -pub mod segments; -pub mod storage; - -pub const COMPONENT: &str = "STREAMING_PARTITIONS"; diff --git a/core/server/src/streaming/partitions/ops.rs b/core/server/src/streaming/partitions/ops.rs deleted file mode 100644 index e2d206b413..0000000000 --- a/core/server/src/streaming/partitions/ops.rs +++ /dev/null @@ -1,730 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Shared partition operations that can be used by both production code and tests. -//! -//! This module provides the core logic for polling and loading messages from partitions, -//! avoiding code duplication between `IggyShard` and test harnesses. -//! -//! # Safety invariants -//! -//! The snapshot-then-read pattern in [`get_messages_by_offset`] and -//! [`poll_messages_by_timestamp`] is safe only under **single-threaded shard -//! execution** (compio runtime). Between the metadata snapshot and the actual -//! reads, no other shard request can mutate the partition state because the -//! message pump processes one request at a time. -//! -//! The poll + auto_commit sequence in the handler (`handlers.rs`) is likewise -//! non-atomic but safe for the same reason. -//! -//! If the architecture ever moves to multi-threaded shard processing or adds -//! compaction/message deletion, these invariants must be re-evaluated. - -use super::journal::Journal; -use super::local_partitions::LocalPartitions; -use crate::shard::system::messages::PollingArgs; -use crate::streaming::polling_consumer::PollingConsumer; -use crate::streaming::segments::IggyMessagesBatchSet; -use iggy_common::IggyPollMetadata; -use iggy_common::{IggyError, PollingKind}; -use server_common::sharding::IggyNamespace; -use std::cell::RefCell; -use std::sync::atomic::Ordering; - -/// Poll messages from a partition partitions. -/// -/// This is the core polling logic shared between production code and tests. -pub async fn poll_messages( - local_partitions: &RefCell, - namespace: &IggyNamespace, - consumer: PollingConsumer, - args: PollingArgs, -) -> Result<(IggyPollMetadata, IggyMessagesBatchSet), IggyError> { - let partition_id = namespace.partition_id(); - let count = args.count; - let strategy = args.strategy; - let value = strategy.value; - - // Handle timestamp polling separately - it has different logic - if strategy.kind == PollingKind::Timestamp { - return poll_messages_by_timestamp(local_partitions, namespace, value, count).await; - } - - // Phase 1: Extract metadata and determine start offset - let (metadata, start_offset) = { - let store = local_partitions.borrow(); - let partition = store - .get(namespace) - .expect("local_partitions: partition must exist for poll"); - - let current_offset = partition.offset.load(Ordering::Relaxed); - let metadata = IggyPollMetadata::new(partition_id as u32, current_offset); - - let start_offset = match strategy.kind { - PollingKind::Offset => { - let offset = value; - if offset > current_offset { - return Ok((metadata, IggyMessagesBatchSet::empty())); - } - offset - } - PollingKind::First => partition - .log - .segments() - .first() - .map(|segment| segment.start_offset) - .unwrap_or(0), - PollingKind::Last => { - let mut requested_count = count as u64; - if requested_count > current_offset + 1 { - requested_count = current_offset + 1; - } - 1 + current_offset - requested_count - } - PollingKind::Next => { - let stored_offset = match consumer { - PollingConsumer::Consumer(id, _) => partition - .consumer_offsets - .pin() - .get(&id) - .map(|item| item.offset.load(Ordering::Relaxed)), - PollingConsumer::ConsumerGroup(cg_id, _) => partition - .consumer_group_offsets - .pin() - .get(&cg_id) - .map(|item| item.offset.load(Ordering::Relaxed)), - }; - match stored_offset { - Some(offset) => offset + 1, - None => partition - .log - .segments() - .first() - .map(|segment| segment.start_offset) - .unwrap_or(0), - } - } - PollingKind::Timestamp => unreachable!("Timestamp handled above"), - }; - - if start_offset > current_offset || count == 0 { - return Ok((metadata, IggyMessagesBatchSet::empty())); - } - - (metadata, start_offset) - }; - - // Phase 2: Get messages using hybrid disk+journal logic - let batches = get_messages_by_offset(local_partitions, namespace, start_offset, count).await?; - Ok((metadata, batches)) -} - -/// Get messages by offset, handling the hybrid disk+journal case. -pub async fn get_messages_by_offset( - local_partitions: &RefCell, - namespace: &IggyNamespace, - start_offset: u64, - count: u32, -) -> Result { - if count == 0 { - return Ok(IggyMessagesBatchSet::empty()); - } - - // Snapshot journal and in-flight metadata for routing decisions. - let (journal_first_offset, in_flight_empty, in_flight_first, in_flight_last) = { - let store = local_partitions.borrow(); - let partition = store - .get(namespace) - .expect("local_partitions: partition must exist for poll"); - - let journal = partition.log.journal(); - let in_flight = partition.log.in_flight(); - ( - journal.first_offset(), - in_flight.is_empty(), - in_flight.first_offset(), - in_flight.last_offset(), - ) - }; - - // Lookup ordered by ascending offset: disk -> in-flight -> journal. - // - // Offsets are sequential: disk < in-flight < journal. A request may span - // multiple tiers, so we advance `current` through each sequentially. - // See issue #2715. - - let mut combined = IggyMessagesBatchSet::empty(); - let mut remaining = count; - let mut current = start_offset; - - // Lowest in-memory tier boundary (if any). Disk handles offsets below this. - let in_memory_floor = if !in_flight_empty { - in_flight_first - } else { - journal_first_offset.unwrap_or(u64::MAX) - }; - - // Disk (pre-tier): offsets below the lowest in-memory tier. - if remaining > 0 && current < in_memory_floor { - let disk_count = - ((in_memory_floor.min(current + remaining as u64) - current) as u32).min(remaining); - let disk_messages = - load_messages_from_disk(local_partitions, namespace, current, disk_count).await?; - let loaded = disk_messages.count(); - if loaded > 0 { - current += loaded as u64; - remaining = remaining.saturating_sub(loaded); - combined.add_batch_set(disk_messages); - } - } - - // In-flight: committed data being persisted to disk. - if remaining > 0 && !in_flight_empty && current >= in_flight_first && current <= in_flight_last - { - let in_flight_count = ((in_flight_last - current + 1) as u32).min(remaining); - let in_flight_batches = { - let store = local_partitions.borrow(); - let partition = store - .get(namespace) - .expect("local_partitions: partition must exist for poll"); - partition - .log - .in_flight() - .get_by_offset(current, in_flight_count) - .to_vec() - }; - if !in_flight_batches.is_empty() { - let mut result = IggyMessagesBatchSet::empty(); - result.add_immutable_batches(&in_flight_batches); - let sliced = result.get_by_offset(current, in_flight_count); - let loaded = sliced.count(); - if loaded > 0 { - current += loaded as u64; - remaining = remaining.saturating_sub(loaded); - combined.add_batch_set(sliced); - } - } - } - - // Journal: may hold data from recent appends. - if remaining > 0 - && let Some(jfo) = journal_first_offset - && current >= jfo - { - let journal_messages = { - let store = local_partitions.borrow(); - let partition = store - .get(namespace) - .expect("local_partitions: partition must exist for poll"); - partition - .log - .journal() - .get(|batches| batches.get_by_offset(current, remaining)) - }; - if !journal_messages.is_empty() { - combined.add_batch_set(journal_messages); - } - } - - Ok(combined) -} - -/// Poll messages by timestamp. -async fn poll_messages_by_timestamp( - local_partitions: &RefCell, - namespace: &IggyNamespace, - timestamp: u64, - count: u32, -) -> Result<(IggyPollMetadata, IggyMessagesBatchSet), IggyError> { - let partition_id = namespace.partition_id(); - - // Snapshot metadata from journal and in-flight for routing decisions. - let ( - metadata, - journal_first_ts, - journal_last_ts, - in_flight_empty, - in_flight_first_ts, - in_flight_last_ts, - ) = { - let store = local_partitions.borrow(); - let partition = store - .get(namespace) - .expect("local_partitions: partition must exist for poll"); - - let current_offset = partition.offset.load(Ordering::Relaxed); - let metadata = IggyPollMetadata::new(partition_id as u32, current_offset); - - let journal = partition.log.journal(); - - let in_flight = partition.log.in_flight(); - let (ife, ifts, ilts) = if in_flight.is_empty() { - (true, 0u64, 0u64) - } else { - let first_ts = in_flight - .batches() - .first() - .and_then(|b| b.first_timestamp()) - .unwrap_or(0); - let last_ts = in_flight - .batches() - .last() - .and_then(|b| b.last_timestamp()) - .unwrap_or(0); - (false, first_ts, last_ts) - }; - - ( - metadata, - journal.first_timestamp(), - journal.last_timestamp(), - ife, - ifts, - ilts, - ) - }; - - if count == 0 { - return Ok((metadata, IggyMessagesBatchSet::empty())); - } - - // Three-tier timestamp lookup: disk -> in-flight -> journal. - // Same structure as offset-based polling (see issue #2715). - - let mut combined = IggyMessagesBatchSet::empty(); - let mut remaining = count; - - // Phase 1: Disk - timestamps before in-flight range. - let disk_upper_ts = if !in_flight_empty { - in_flight_first_ts - } else { - journal_first_ts.unwrap_or(u64::MAX) - }; - - if timestamp < disk_upper_ts && remaining > 0 { - let disk_messages = - load_messages_from_disk_by_timestamp(local_partitions, namespace, timestamp, remaining) - .await?; - let loaded = disk_messages.count(); - if loaded > 0 { - remaining = remaining.saturating_sub(loaded); - combined.add_batch_set(disk_messages); - } - } - - // Phase 2: In-flight - committed data being persisted. - if remaining > 0 && !in_flight_empty && timestamp <= in_flight_last_ts { - let in_flight_batches = { - let store = local_partitions.borrow(); - let partition = store - .get(namespace) - .expect("local_partitions: partition must exist for poll"); - partition.log.in_flight().batches().to_vec() - }; - if !in_flight_batches.is_empty() { - let mut batch_set = IggyMessagesBatchSet::empty(); - batch_set.add_immutable_batches(&in_flight_batches); - let filtered = batch_set.get_by_timestamp(timestamp, remaining); - let loaded = filtered.count(); - if loaded > 0 { - remaining = remaining.saturating_sub(loaded); - combined.add_batch_set(filtered); - } - } - } - - // Phase 3: Journal - newest appends (post-commit). - if remaining > 0 - && let Some(jlts) = journal_last_ts - && timestamp <= jlts - { - let journal_messages = { - let store = local_partitions.borrow(); - let partition = store - .get(namespace) - .expect("local_partitions: partition must exist for poll"); - partition - .log - .journal() - .get(|batches| batches.get_by_timestamp(timestamp, remaining)) - }; - if !journal_messages.is_empty() { - combined.add_batch_set(journal_messages); - } - } - - Ok((metadata, combined)) -} - -/// Load messages from disk by offset. -pub async fn load_messages_from_disk( - local_partitions: &RefCell, - namespace: &IggyNamespace, - start_offset: u64, - count: u32, -) -> Result { - if count == 0 { - return Ok(IggyMessagesBatchSet::empty()); - } - - // Get segment range containing the requested offset - let segment_range = { - let store = local_partitions.borrow(); - let partition = store - .get(namespace) - .expect("local_partitions: partition must exist"); - - let segments = partition.log.segments(); - if segments.is_empty() { - return Ok(IggyMessagesBatchSet::empty()); - } - - let start = segments - .iter() - .rposition(|segment| segment.start_offset <= start_offset) - .unwrap_or(0); - let end = segments.len(); - start..end - }; - - let mut remaining_count = count; - let mut batches = IggyMessagesBatchSet::empty(); - let mut current_offset = start_offset; - - for idx in segment_range { - if remaining_count == 0 { - break; - } - - let (segment_start_offset, segment_end_offset) = { - let store = local_partitions.borrow(); - let partition = store - .get(namespace) - .expect("local_partitions: partition must exist"); - - let segment = &partition.log.segments()[idx]; - (segment.start_offset, segment.end_offset) - }; - - let offset = if current_offset < segment_start_offset { - segment_start_offset - } else { - current_offset - }; - - let mut end_offset = offset + (remaining_count - 1) as u64; - if end_offset > segment_end_offset { - end_offset = segment_end_offset; - } - - let messages = load_segment_messages( - local_partitions, - namespace, - idx, - offset, - end_offset, - remaining_count, - segment_start_offset, - ) - .await?; - - let loaded_count = messages.count(); - if loaded_count > 0 { - batches.add_batch_set(messages); - remaining_count = remaining_count.saturating_sub(loaded_count); - current_offset = end_offset + 1; - } else { - break; - } - } - - Ok(batches) -} - -/// Load messages from a specific segment. -async fn load_segment_messages( - local_partitions: &RefCell, - namespace: &IggyNamespace, - idx: usize, - start_offset: u64, - end_offset: u64, - count: u32, - segment_start_offset: u64, -) -> Result { - let relative_start_offset = (start_offset - segment_start_offset) as u32; - - // Check journal for this segment's data (handles callers outside get_messages_by_offset). - let journal_data = { - let store = local_partitions.borrow(); - let partition = store - .get(namespace) - .expect("local_partitions: partition must exist"); - - let journal = partition.log.journal(); - - if let (Some(jfo), Some(jlo)) = (journal.first_offset(), journal.last_offset()) - && start_offset >= jfo - && end_offset <= jlo - { - Some(journal.get(|batches| batches.get_by_offset(start_offset, count))) - } else { - None - } - }; - - if let Some(batches) = journal_data { - return Ok(batches); - } - - // Load from disk - let (index_reader, messages_reader, indexes) = { - let store = local_partitions.borrow(); - let partition = store - .get(namespace) - .expect("local_partitions: partition must exist"); - - let storages = partition.log.storages(); - if idx >= storages.len() { - return Ok(IggyMessagesBatchSet::empty()); - } - - let index_reader = storages[idx] - .index_reader - .as_ref() - .expect("Index reader not initialized") - .clone(); - let messages_reader = storages[idx] - .messages_reader - .as_ref() - .expect("Messages reader not initialized") - .clone(); - let indexes_vec = partition.log.indexes(); - let indexes = indexes_vec - .get(idx) - .and_then(|opt| opt.as_ref()) - .map(|indexes| { - indexes - .slice_by_offset(relative_start_offset, count) - .unwrap_or_default() - }); - (index_reader, messages_reader, indexes) - }; - - let indexes_to_read = if let Some(indexes) = indexes { - if !indexes.is_empty() { - Some(indexes) - } else { - index_reader - .as_ref() - .load_from_disk_by_offset(relative_start_offset, count) - .await? - } - } else { - index_reader - .as_ref() - .load_from_disk_by_offset(relative_start_offset, count) - .await? - }; - - if indexes_to_read.is_none() { - return Ok(IggyMessagesBatchSet::empty()); - } - - let indexes_to_read = indexes_to_read.unwrap(); - let batch = messages_reader - .as_ref() - .load_messages_from_disk(indexes_to_read) - .await?; - - batch.validate_checksums_and_offsets(start_offset)?; - - Ok(IggyMessagesBatchSet::from(batch)) -} - -/// Load messages from disk by timestamp. -async fn load_messages_from_disk_by_timestamp( - local_partitions: &RefCell, - namespace: &IggyNamespace, - timestamp: u64, - count: u32, -) -> Result { - if count == 0 { - return Ok(IggyMessagesBatchSet::empty()); - } - - // Find segment range that might contain messages >= timestamp - let segment_range = { - let store = local_partitions.borrow(); - let partition = store - .get(namespace) - .expect("local_partitions: partition must exist"); - - let segments = partition.log.segments(); - if segments.is_empty() { - return Ok(IggyMessagesBatchSet::empty()); - } - - let start = segments - .iter() - .position(|segment| segment.end_timestamp >= timestamp) - .unwrap_or(segments.len()); - - if start >= segments.len() { - return Ok(IggyMessagesBatchSet::empty()); - } - - start..segments.len() - }; - - let mut remaining_count = count; - let mut batches = IggyMessagesBatchSet::empty(); - - for idx in segment_range { - if remaining_count == 0 { - break; - } - - let segment_end_timestamp = { - let store = local_partitions.borrow(); - let partition = store - .get(namespace) - .expect("local_partitions: partition must exist"); - partition.log.segments()[idx].end_timestamp - }; - - if segment_end_timestamp < timestamp { - continue; - } - - let messages = load_segment_messages_by_timestamp( - local_partitions, - namespace, - idx, - timestamp, - remaining_count, - ) - .await?; - - let messages_count = messages.count(); - if messages_count == 0 { - continue; - } - - remaining_count = remaining_count.saturating_sub(messages_count); - batches.add_batch_set(messages); - } - - Ok(batches) -} - -/// Load messages from a specific segment by timestamp. -async fn load_segment_messages_by_timestamp( - local_partitions: &RefCell, - namespace: &IggyNamespace, - idx: usize, - timestamp: u64, - count: u32, -) -> Result { - if count == 0 { - return Ok(IggyMessagesBatchSet::empty()); - } - - // Check journal first - let journal_data = { - let store = local_partitions.borrow(); - let partition = store - .get(namespace) - .expect("local_partitions: partition must exist"); - - let journal = partition.log.journal(); - - if let (Some(jfts), Some(jlts)) = (journal.first_timestamp(), journal.last_timestamp()) - && timestamp >= jfts - && timestamp <= jlts - { - Some(journal.get(|batches| batches.get_by_timestamp(timestamp, count))) - } else { - None - } - }; - - if let Some(batches) = journal_data { - return Ok(batches); - } - - // Load from disk - let (index_reader, messages_reader, indexes) = { - let store = local_partitions.borrow(); - let partition = store - .get(namespace) - .expect("local_partitions: partition must exist"); - - let storages = partition.log.storages(); - if idx >= storages.len() { - return Ok(IggyMessagesBatchSet::empty()); - } - - let index_reader = storages[idx] - .index_reader - .as_ref() - .expect("Index reader not initialized") - .clone(); - let messages_reader = storages[idx] - .messages_reader - .as_ref() - .expect("Messages reader not initialized") - .clone(); - let indexes_vec = partition.log.indexes(); - let indexes = indexes_vec - .get(idx) - .and_then(|opt| opt.as_ref()) - .map(|indexes| { - indexes - .slice_by_timestamp(timestamp, count) - .unwrap_or_default() - }); - (index_reader, messages_reader, indexes) - }; - - let indexes_to_read = if let Some(indexes) = indexes { - if !indexes.is_empty() { - Some(indexes) - } else { - index_reader - .as_ref() - .load_from_disk_by_timestamp(timestamp, count) - .await? - } - } else { - index_reader - .as_ref() - .load_from_disk_by_timestamp(timestamp, count) - .await? - }; - - if indexes_to_read.is_none() { - return Ok(IggyMessagesBatchSet::empty()); - } - - let indexes_to_read = indexes_to_read.unwrap(); - let batch = messages_reader - .as_ref() - .load_messages_from_disk(indexes_to_read) - .await?; - - Ok(IggyMessagesBatchSet::from(batch)) -} diff --git a/core/server/src/streaming/partitions/ops_tests.rs b/core/server/src/streaming/partitions/ops_tests.rs deleted file mode 100644 index 7769666f4f..0000000000 --- a/core/server/src/streaming/partitions/ops_tests.rs +++ /dev/null @@ -1,358 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Tests for the in-flight buffer visibility gap fix (issue #2715). -//! -//! These tests set up "State C" directly in memory: -//! - in-flight holds offsets [0..N-1] (committed journal, not yet on disk) -//! - journal holds offsets [N..N+M-1] (new appends after commit) -//! - no actual disk data -//! -//! Before the fix, Cases 1-3 in get_messages_by_offset never checked -//! in-flight, causing the consumer to miss committed data and either -//! get empty results or skip directly to journal offsets. - -#[cfg(test)] -mod tests { - use crate::streaming::partitions::consumer_group_offsets::ConsumerGroupOffsets; - use crate::streaming::partitions::consumer_offsets::ConsumerOffsets; - use crate::streaming::partitions::journal::{Inner, Journal}; - use crate::streaming::partitions::local_partition::LocalPartition; - use crate::streaming::partitions::local_partitions::LocalPartitions; - use crate::streaming::partitions::ops; - use crate::streaming::polling_consumer::PollingConsumer; - use crate::streaming::stats::{PartitionStats, StreamStats, TopicStats}; - use iggy_common::{IggyByteSize, IggyMessage, PollingStrategy, Sizeable}; - use server_common::sharding::IggyNamespace; - use server_common::{MemoryPool, MemoryPoolConfigOther}; - use std::cell::RefCell; - use std::sync::Arc; - use std::sync::atomic::AtomicU64; - - fn init_memory_pool() { - static INIT: std::sync::Once = std::sync::Once::new(); - INIT.call_once(|| { - let config = MemoryPoolConfigOther { - enabled: false, - size: IggyByteSize::from(64 * 1024 * 1024u64), - bucket_capacity: 256, - }; - MemoryPool::init_pool(&config); - }); - } - - fn create_test_partition(current_offset: u64) -> LocalPartition { - let stream_stats = Arc::new(StreamStats::default()); - let topic_stats = Arc::new(TopicStats::new(stream_stats)); - let partition_stats = Arc::new(PartitionStats::new(topic_stats)); - - LocalPartition::new( - partition_stats, - Arc::new(AtomicU64::new(current_offset)), - Arc::new(ConsumerOffsets::with_capacity(10)), - Arc::new(ConsumerGroupOffsets::with_capacity(10)), - None, - iggy_common::IggyTimestamp::now(), - 1, - true, - ) - } - - fn create_batch(count: u32) -> server_common::IggyMessagesBatchMut { - let messages: Vec = (0..count) - .map(|_| { - IggyMessage::builder() - .payload(bytes::Bytes::from("test-payload")) - .build() - .unwrap() - }) - .collect(); - - let messages_size: u32 = messages - .iter() - .map(|m| m.get_size_bytes().as_bytes_u32()) - .sum(); - server_common::IggyMessagesBatchMut::from_messages(&messages, messages_size) - } - - /// Sets up "State C": in-flight holds committed data, journal holds new - /// appends that arrived after commit but before persist completes. - /// - /// Layout: - /// segment metadata: [0..journal_end] (no actual disk data) - /// in-flight: [0..in_flight_count-1] - /// journal: [in_flight_count..in_flight_count+journal_count-1] - /// partition.offset: in_flight_count + journal_count - 1 - async fn setup_state_c( - in_flight_count: u32, - journal_count: u32, - ) -> (RefCell, IggyNamespace) { - init_memory_pool(); - let ns = IggyNamespace::new(1, 1, 0); - - let in_flight_end = in_flight_count as u64 - 1; - let journal_base = in_flight_end + 1; - let journal_end = journal_base + journal_count as u64 - 1; - - let mut partition = create_test_partition(journal_end); - - let segment = iggy_common::Segment::new(0, IggyByteSize::from(1_073_741_824u64)); - let storage = server_common::SegmentStorage::default(); - partition.log.add_persisted_segment(segment, storage); - - let seg = &mut partition.log.segments_mut()[0]; - seg.end_offset = journal_end; - seg.start_timestamp = 1; - seg.end_timestamp = 2; - - let mut in_flight_batch = create_batch(in_flight_count); - in_flight_batch.prepare_for_persistence(0, 0, 0, None).await; - let in_flight_size = in_flight_batch.size(); - partition.log.set_in_flight(vec![in_flight_batch.freeze()]); - - let journal_inner = Inner { - base_offset: journal_base, - current_offset: 0, - first_timestamp: 0, - end_timestamp: 0, - messages_count: 0, - size: IggyByteSize::default(), - }; - partition.log.journal_mut().init(journal_inner); - - let mut journal_batch = create_batch(journal_count); - journal_batch - .prepare_for_persistence(0, journal_base, in_flight_size, None) - .await; - partition.log.journal_mut().append(journal_batch).unwrap(); - - let mut store = LocalPartitions::new(); - store.insert(ns, partition); - (RefCell::new(store), ns) - } - - // ----------------------------------------------------------------------- - // Issue #2715: In-flight buffer must be reachable when journal is non-empty - // ----------------------------------------------------------------------- - - #[compio::test] - async fn in_flight_reachable_when_journal_non_empty() { - let (store, ns) = setup_state_c(10, 5).await; - let batches = ops::get_messages_by_offset(&store, &ns, 0, 5) - .await - .unwrap(); - assert_eq!(batches.count(), 5); - assert_eq!(batches.first_offset(), Some(0)); - } - - #[compio::test] - async fn spanning_in_flight_and_journal_returns_all_in_order() { - let (store, ns) = setup_state_c(10, 5).await; - let batches = ops::get_messages_by_offset(&store, &ns, 0, 15) - .await - .unwrap(); - assert_eq!(batches.count(), 15); - assert_eq!(batches.first_offset(), Some(0)); - } - - #[compio::test] - async fn polling_next_starts_from_in_flight_not_journal() { - let (store, ns) = setup_state_c(10, 5).await; - let consumer = PollingConsumer::Consumer(1, 0); - let args = - crate::shard::system::messages::PollingArgs::new(PollingStrategy::next(), 15, false); - let (metadata, batches) = ops::poll_messages(&store, &ns, consumer, args) - .await - .unwrap(); - assert_eq!(batches.first_offset(), Some(0)); - assert!(metadata.current_offset >= 14); - } - - #[compio::test] - async fn single_message_at_in_flight_journal_boundary() { - let (store, ns) = setup_state_c(10, 5).await; - let batches = ops::get_messages_by_offset(&store, &ns, 9, 1) - .await - .unwrap(); - assert_eq!(batches.count(), 1); - assert_eq!(batches.first_offset(), Some(9)); - } - - #[compio::test] - async fn single_message_from_in_flight_at_offset_zero() { - let (store, ns) = setup_state_c(10, 5).await; - let batches = ops::get_messages_by_offset(&store, &ns, 0, 1) - .await - .unwrap(); - assert_eq!(batches.count(), 1); - assert_eq!(batches.first_offset(), Some(0)); - } - - // ----------------------------------------------------------------------- - // Existing correct behavior must still work - // ----------------------------------------------------------------------- - - #[compio::test] - async fn in_flight_reachable_when_journal_empty() { - init_memory_pool(); - let ns = IggyNamespace::new(1, 1, 0); - let mut partition = create_test_partition(9); - - let segment = iggy_common::Segment::new(0, IggyByteSize::from(1_073_741_824u64)); - partition - .log - .add_persisted_segment(segment, server_common::SegmentStorage::default()); - let seg = &mut partition.log.segments_mut()[0]; - seg.end_offset = 9; - seg.start_timestamp = 1; - seg.end_timestamp = 2; - - let mut batch = create_batch(10); - batch.prepare_for_persistence(0, 0, 0, None).await; - partition.log.set_in_flight(vec![batch.freeze()]); - - let mut store = LocalPartitions::new(); - store.insert(ns, partition); - let store = RefCell::new(store); - - let batches = ops::get_messages_by_offset(&store, &ns, 0, 10) - .await - .unwrap(); - assert_eq!(batches.count(), 10); - } - - #[compio::test] - async fn journal_reachable_when_in_flight_empty() { - init_memory_pool(); - let ns = IggyNamespace::new(1, 1, 0); - let mut partition = create_test_partition(9); - - let segment = iggy_common::Segment::new(0, IggyByteSize::from(1_073_741_824u64)); - partition - .log - .add_persisted_segment(segment, server_common::SegmentStorage::default()); - let seg = &mut partition.log.segments_mut()[0]; - seg.end_offset = 9; - seg.start_timestamp = 1; - seg.end_timestamp = 2; - - partition.log.journal_mut().init(Inner { - base_offset: 0, - current_offset: 0, - first_timestamp: 0, - end_timestamp: 0, - messages_count: 0, - size: IggyByteSize::default(), - }); - - let mut batch = create_batch(10); - batch.prepare_for_persistence(0, 0, 0, None).await; - partition.log.journal_mut().append(batch).unwrap(); - - let mut store = LocalPartitions::new(); - store.insert(ns, partition); - let store = RefCell::new(store); - - let batches = ops::get_messages_by_offset(&store, &ns, 0, 10) - .await - .unwrap(); - assert_eq!(batches.count(), 10); - } - - #[compio::test] - async fn journal_single_message_at_specific_offset() { - let (store, ns) = setup_state_c(10, 5).await; - let batches = ops::get_messages_by_offset(&store, &ns, 12, 1) - .await - .unwrap(); - assert_eq!(batches.count(), 1); - assert_eq!(batches.first_offset(), Some(12)); - } - - // ----------------------------------------------------------------------- - // Bug reproduction: journal base_offset=0 after restart causes offset skip - // ----------------------------------------------------------------------- - - /// Verifies that journal self-heals base_offset on first append. - /// Without self-healing, a journal created via Default would have - /// base_offset=0, causing incorrect offset calculations. - #[compio::test] - async fn journal_self_heals_base_offset_on_first_append() { - init_memory_pool(); - - let mut journal = crate::streaming::partitions::journal::MemoryMessageJournal::empty(); - assert_eq!(journal.inner().base_offset, 0); - - let mut batch = create_batch(5); - batch.prepare_for_persistence(0, 100, 0, None).await; - journal.append(batch).unwrap(); - - assert_eq!( - journal.inner().base_offset, - 100, - "Journal should self-heal base_offset from batch's first offset" - ); - assert_eq!( - journal.inner().current_offset, - 104, - "current_offset should be base_offset + messages_count - 1" - ); - } - - /// Verifies that slice_by_offset returns None when start_offset is below - /// the batch's range, instead of clamping to index 0 (the old bug). - #[compio::test] - async fn slice_by_offset_rejects_offset_below_range() { - init_memory_pool(); - - let mut batch = create_batch(10); - batch.prepare_for_persistence(0, 100, 0, None).await; - - let result = batch.slice_by_offset(95, 10); - - assert!( - result.is_none(), - "slice_by_offset should return None when start_offset(95) < first_offset(100), \ - got {} messages at offset {:?}", - result.as_ref().map(|r| r.count()).unwrap_or(0), - result.as_ref().and_then(|r| r.first_offset()) - ); - } - - /// After proper journal initialization, polling across the in-flight/journal - /// boundary returns contiguous messages with no gaps. - #[compio::test] - async fn post_restart_poll_with_correct_journal_init_no_skip() { - let (store, ns) = setup_state_c(100, 10).await; - - let batches = ops::get_messages_by_offset(&store, &ns, 95, 10) - .await - .unwrap(); - - assert_eq!(batches.count(), 10, "should return exactly 10 messages"); - assert_eq!( - batches.first_offset(), - Some(95), - "first message should be at requested offset 95" - ); - assert_eq!( - batches.last_offset(), - Some(104), - "last message should be at offset 104 (contiguous)" - ); - } -} diff --git a/core/server/src/streaming/partitions/segments.rs b/core/server/src/streaming/partitions/segments.rs deleted file mode 100644 index 8e40174abe..0000000000 --- a/core/server/src/streaming/partitions/segments.rs +++ /dev/null @@ -1,123 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub struct DeletedSegment { - pub end_offset: u64, - pub messages_count: u32, -} - -/* -impl Partition { - pub fn get_segments_count(&self) -> u32 { - self.segments.len() as u32 - } - - pub fn get_segments(&self) -> &Vec { - &self.segments - } - - pub fn get_segments_mut(&mut self) -> &mut Vec { - &mut self.segments - } - - pub fn get_segment(&self, start_offset: u64) -> Option<&Segment> { - self.segments - .iter() - .find(|s| s.start_offset() == start_offset) - } - - pub fn get_segment_mut(&mut self, start_offset: u64) -> Option<&mut Segment> { - self.segments - .iter_mut() - .find(|s| s.start_offset() == start_offset) - } - - pub async fn get_expired_segments_start_offsets(&self, now: IggyTimestamp) -> Vec { - let mut expired_segments = Vec::new(); - for segment in &self.segments { - if segment.is_expired(now).await { - expired_segments.push(segment.start_offset()); - } - } - - expired_segments.sort(); - expired_segments - } - - pub async fn add_persisted_segment(&mut self, start_offset: u64) -> Result<(), IggyError> { - info!( - "Creating the new segment for partition with ID: {}, stream with ID: {}, topic with ID: {}...", - self.partition_id, self.stream_id, self.topic_id - ); - let mut new_segment = Segment::create( - self.stream_id, - self.topic_id, - self.partition_id, - start_offset, - self.config.clone(), - self.message_expiry, - self.size_of_parent_stream.clone(), - self.size_of_parent_topic.clone(), - self.size_bytes.clone(), - self.messages_count_of_parent_stream.clone(), - self.messages_count_of_parent_topic.clone(), - self.messages_count.clone(), - true, - ); - new_segment.open().await.error(|e: &IggyError| { - format!("{COMPONENT} (error: {e}) - failed to persist new segment: {new_segment}",) - })?; - self.segments.push(new_segment); - self.segments_count_of_parent_stream - .fetch_add(1, Ordering::SeqCst); - self.segments.sort_by_key(|a| a.start_offset()); - Ok(()) - } - - pub async fn delete_segment(&mut self, start_offset: u64) -> Result { - let deleted_segment; - { - let segment = self.get_segment_mut(start_offset); - if segment.is_none() { - return Err(IggyError::SegmentNotFound); - } - - let segment = segment.unwrap(); - segment.delete().await.error(|e: &IggyError| { - format!("{COMPONENT} (error: {e}) - failed to delete segment: {segment}",) - })?; - - deleted_segment = DeletedSegment { - end_offset: segment.end_offset(), - messages_count: segment.get_messages_count(), - }; - } - - self.segments_count_of_parent_stream - .fetch_sub(1, Ordering::SeqCst); - - self.segments.retain(|s| s.start_offset() != start_offset); - self.segments.sort_by_key(|a| a.start_offset()); - info!( - "Segment with start offset: {} has been deleted from partition with ID: {}, stream with ID: {}, topic with ID: {}", - start_offset, self.partition_id, self.stream_id, self.topic_id - ); - Ok(deleted_segment) - } -} - -*/ diff --git a/core/server/src/streaming/partitions/storage.rs b/core/server/src/streaming/partitions/storage.rs deleted file mode 100644 index e641b3ec31..0000000000 --- a/core/server/src/streaming/partitions/storage.rs +++ /dev/null @@ -1,346 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use super::COMPONENT; -use crate::{ - configs::system::SystemConfig, io::fs_utils::remove_dir_all, - streaming::partitions::consumer_offset::ConsumerOffset, - streaming::polling_consumer::ConsumerGroupId, -}; -use compio::{ - fs::{self, OpenOptions, create_dir_all}, - io::AsyncWriteAtExt, -}; -use iggy_common::{ConsumerKind, IggyError}; -use std::{io::Read, path::Path, sync::atomic::AtomicU64}; -use tracing::{error, trace, warn}; - -pub async fn create_partition_file_hierarchy( - stream_id: usize, - topic_id: usize, - partition_id: usize, - config: &SystemConfig, -) -> Result<(), IggyError> { - let partition_path = config.get_partition_path(stream_id, topic_id, partition_id); - tracing::info!( - "Saving partition with ID: {} for stream with ID: {} and topic with ID: {}...", - partition_id, - stream_id, - topic_id - ); - if !Path::new(&partition_path).exists() && create_dir_all(&partition_path).await.is_err() { - return Err(IggyError::CannotCreatePartitionDirectory( - partition_id, - stream_id, - topic_id, - )); - } - - let offset_path = config.get_offsets_path(stream_id, topic_id, partition_id); - if !Path::new(&offset_path).exists() && create_dir_all(&offset_path).await.is_err() { - tracing::error!( - "Failed to create offsets directory for partition with ID: {} for stream with ID: {} and topic with ID: {}.", - partition_id, - stream_id, - topic_id - ); - return Err(IggyError::CannotCreatePartition( - partition_id, - stream_id, - topic_id, - )); - } - - let consumer_offset_path = config.get_consumer_offsets_path(stream_id, topic_id, partition_id); - if !Path::new(&consumer_offset_path).exists() - && create_dir_all(&consumer_offset_path).await.is_err() - { - tracing::error!( - "Failed to create consumer offsets directory for partition with ID: {} for stream with ID: {} and topic with ID: {}.", - partition_id, - stream_id, - topic_id - ); - return Err(IggyError::CannotCreatePartition( - partition_id, - stream_id, - topic_id, - )); - } - - let consumer_group_offsets_path = - config.get_consumer_group_offsets_path(stream_id, topic_id, partition_id); - if !Path::new(&consumer_group_offsets_path).exists() - && create_dir_all(&consumer_group_offsets_path).await.is_err() - { - tracing::error!( - "Failed to create consumer group offsets directory for partition with ID: {} for stream with ID: {} and topic with ID: {}.", - partition_id, - stream_id, - topic_id - ); - return Err(IggyError::CannotCreatePartition( - partition_id, - stream_id, - topic_id, - )); - } - - tracing::info!( - "Saved partition with start ID: {} for stream with ID: {} and topic with ID: {}, path: {}.", - partition_id, - stream_id, - topic_id, - partition_path - ); - - Ok(()) -} - -pub async fn delete_partitions_from_disk( - stream_id: usize, - topic_id: usize, - partition_id: usize, - config: &SystemConfig, -) -> Result<(), IggyError> { - let partition_path = config.get_partition_path(stream_id, topic_id, partition_id); - remove_dir_all(&partition_path).await.map_err(|_| { - IggyError::CannotDeletePartitionDirectory(stream_id, topic_id, partition_id) - })?; - tracing::info!( - "Deleted partition files for partition with ID: {} stream with ID: {} and topic with ID: {}.", - partition_id, - stream_id, - topic_id - ); - Ok(()) -} - -pub async fn delete_persisted_offset(path: &str) -> Result<(), IggyError> { - if !Path::new(path).exists() { - tracing::trace!("Consumer offset file does not exist: {path}."); - return Ok(()); - } - - if fs::remove_file(path).await.is_err() { - tracing::error!("Cannot delete consumer offset file: {path}."); - return Err(IggyError::CannotDeleteConsumerOffsetFile(path.to_owned())); - } - Ok(()) -} - -pub async fn persist_offset(path: &str, offset: u64) -> Result<(), IggyError> { - let mut file = OpenOptions::new() - .write(true) - .create(true) - .open(path) - .await - .map_err(|_| IggyError::CannotOpenConsumerOffsetsFile(path.to_owned()))?; - let buf = offset.to_le_bytes(); - file.write_all_at(buf, 0) - .await - .0 - .map_err(|_| IggyError::CannotWriteToFile)?; - tracing::trace!("Stored consumer offset value: {}, path: {}", offset, path); - Ok(()) -} - -pub fn load_consumer_offsets(path: &str) -> Result, IggyError> { - trace!("Loading consumer offsets from path: {path}..."); - let dir_entries = std::fs::read_dir(path); - if dir_entries.is_err() { - return Err(IggyError::CannotReadConsumerOffsets(path.to_owned())); - } - - let mut consumer_offsets = Vec::new(); - let dir_entries = dir_entries.unwrap(); - for dir_entry in dir_entries { - let dir_entry = match dir_entry { - Ok(entry) => entry, - Err(e) => { - warn!( - "Failed to read directory entry in consumer offsets path: {path}, \ - error: {e}, skipping." - ); - continue; - } - }; - - let metadata = match dir_entry.metadata() { - Ok(m) => m, - Err(e) => { - warn!( - "Failed to read metadata for entry in consumer offsets path: {path}, \ - error: {e}, skipping." - ); - continue; - } - }; - - if metadata.is_dir() { - continue; - } - - let name = dir_entry.file_name().to_string_lossy().to_string(); - let consumer_id = match name.parse::() { - Ok(id) => id, - Err(_) => { - warn!( - "Unexpected non-numeric consumer offset file: '{}', skipping.", - name - ); - continue; - } - }; - - let path = dir_entry.path(); - let path = path.to_str(); - if path.is_none() { - error!("Invalid consumer ID path for file with name: '{}'.", name); - continue; - } - - let path = path.unwrap().to_string(); - let file = match std::fs::File::open(&path) { - Ok(f) => f, - Err(e) => { - warn!( - "{COMPONENT} (error: {e}) - failed to open offset file, \ - path: {path}, skipping." - ); - continue; - } - }; - let mut cursor = std::io::Cursor::new(file); - let mut offset = [0; 8]; - if let Err(e) = cursor.get_mut().read_exact(&mut offset) { - warn!( - "{COMPONENT} (error: {e}) - failed to read consumer offset from file \ - (truncated or corrupt?), path: {path}, skipping." - ); - continue; - } - let offset = AtomicU64::new(u64::from_le_bytes(offset)); - - consumer_offsets.push(ConsumerOffset { - kind: ConsumerKind::Consumer, - consumer_id, - offset, - path, - }); - } - - consumer_offsets.sort_by_key(|o| o.consumer_id); - Ok(consumer_offsets) -} - -pub fn load_consumer_group_offsets( - path: &str, -) -> Result, IggyError> { - trace!("Loading consumer group offsets from path: {path}..."); - let dir_entries = std::fs::read_dir(path); - if dir_entries.is_err() { - return Err(IggyError::CannotReadConsumerOffsets(path.to_owned())); - } - - let mut consumer_group_offsets = Vec::new(); - let dir_entries = dir_entries.unwrap(); - for dir_entry in dir_entries { - let dir_entry = match dir_entry { - Ok(entry) => entry, - Err(e) => { - warn!( - "Failed to read directory entry in consumer group offsets path: {path}, \ - error: {e}, skipping." - ); - continue; - } - }; - - let metadata = match dir_entry.metadata() { - Ok(m) => m, - Err(e) => { - warn!( - "Failed to read metadata for entry in consumer group offsets path: {path}, \ - error: {e}, skipping." - ); - continue; - } - }; - - if metadata.is_dir() { - continue; - } - - let name = dir_entry.file_name().to_string_lossy().to_string(); - - let consumer_group_id = match name.parse::() { - Ok(id) => id, - Err(_) => { - warn!( - "Unexpected non-numeric consumer group offset file: '{}', skipping.", - name - ); - continue; - } - }; - let consumer_group_id = ConsumerGroupId(consumer_group_id as usize); - - let path = dir_entry.path(); - let path = path.to_str(); - if path.is_none() { - error!( - "Invalid consumer group offset path for file with name: '{}'.", - name - ); - continue; - } - - let path = path.unwrap().to_string(); - let file = match std::fs::File::open(&path) { - Ok(f) => f, - Err(e) => { - warn!( - "{COMPONENT} (error: {e}) - failed to open offset file, \ - path: {path}, skipping." - ); - continue; - } - }; - let mut cursor = std::io::Cursor::new(file); - let mut offset = [0; 8]; - if let Err(e) = cursor.get_mut().read_exact(&mut offset) { - warn!( - "{COMPONENT} (error: {e}) - failed to read consumer group offset from file \ - (truncated or corrupt?), path: {path}, skipping." - ); - continue; - } - let offset = AtomicU64::new(u64::from_le_bytes(offset)); - - let consumer_offset = ConsumerOffset { - kind: ConsumerKind::ConsumerGroup, - consumer_id: consumer_group_id.0 as u32, - offset, - path, - }; - - consumer_group_offsets.push((consumer_group_id, consumer_offset)); - } - - Ok(consumer_group_offsets) -} diff --git a/core/server/src/streaming/persistence/mod.rs b/core/server/src/streaming/persistence/mod.rs deleted file mode 100644 index 23f1d21799..0000000000 --- a/core/server/src/streaming/persistence/mod.rs +++ /dev/null @@ -1,20 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod persister; - -pub const COMPONENT: &str = "STREAMING_PERSISTENCE"; diff --git a/core/server/src/streaming/persistence/persister.rs b/core/server/src/streaming/persistence/persister.rs deleted file mode 100644 index 466b68c944..0000000000 --- a/core/server/src/streaming/persistence/persister.rs +++ /dev/null @@ -1,166 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::streaming::persistence::COMPONENT; -use crate::streaming::utils::file; -use compio::buf::IoBuf; -use compio::fs::remove_file; -use compio::io::AsyncWriteAtExt; -use err_trail::ErrContext; -use iggy_common::IggyError; -use std::fmt::Debug; - -#[derive(Debug)] -pub enum PersisterKind { - File(FilePersister), - FileWithSync(FileWithSyncPersister), -} - -impl PersisterKind { - pub async fn append(&self, path: &str, bytes: B) -> Result<(), IggyError> { - match self { - PersisterKind::File(p) => p.append(path, bytes).await, - PersisterKind::FileWithSync(p) => p.append(path, bytes).await, - } - } - - pub async fn overwrite(&self, path: &str, bytes: B) -> Result<(), IggyError> { - match self { - PersisterKind::File(p) => p.overwrite(path, bytes).await, - PersisterKind::FileWithSync(p) => p.overwrite(path, bytes).await, - } - } - - pub async fn delete(&self, path: &str) -> Result<(), IggyError> { - match self { - PersisterKind::File(p) => p.delete(path).await, - PersisterKind::FileWithSync(p) => p.delete(path).await, - } - } -} - -#[derive(Debug)] -pub struct FilePersister; - -impl FilePersister { - pub async fn append(&self, path: &str, bytes: B) -> Result<(), IggyError> { - let (mut file, position) = file::append(path) - .await - .error(|e: &std::io::Error| { - format!("{COMPONENT} (error: {e}) - failed to append to file: {path}") - }) - .map_err(|_| IggyError::CannotAppendToFile)?; - file.write_all_at(bytes, position) - .await - .0 - .error(|e: &std::io::Error| { - format!("{COMPONENT} (error: {e}) - failed to write data to file: {path}") - }) - .map_err(|_| IggyError::CannotWriteToFile)?; - Ok(()) - } - - pub async fn overwrite(&self, path: &str, bytes: B) -> Result<(), IggyError> { - let mut file = file::overwrite(path) - .await - .error(|e: &std::io::Error| { - format!("{COMPONENT} (error: {e}) - failed to overwrite file: {path}") - }) - .map_err(|_| IggyError::CannotOverwriteFile)?; - let position = 0; - file.write_all_at(bytes, position) - .await - .0 - .error(|e: &std::io::Error| { - format!("{COMPONENT} (error: {e}) - failed to write data to file: {path}") - }) - .map_err(|_| IggyError::CannotWriteToFile)?; - Ok(()) - } - - pub async fn delete(&self, path: &str) -> Result<(), IggyError> { - remove_file(path) - .await - .error(|e: &std::io::Error| { - format!("{COMPONENT} (error: {e}) - failed to delete file: {path}") - }) - .map_err(|_| IggyError::CannotDeleteFile)?; - Ok(()) - } -} - -#[derive(Debug)] -pub struct FileWithSyncPersister; - -impl FileWithSyncPersister { - pub async fn append(&self, path: &str, bytes: B) -> Result<(), IggyError> { - let (mut file, position) = file::append(path) - .await - .error(|e: &std::io::Error| { - format!("{COMPONENT} (error: {e}) - failed to append to file: {path}") - }) - .map_err(|_| IggyError::CannotAppendToFile)?; - file.write_all_at(bytes, position) - .await - .0 - .error(|e: &std::io::Error| { - format!("{COMPONENT} (error: {e}) - failed to write data to file: {path}") - }) - .map_err(|_| IggyError::CannotWriteToFile)?; - file.sync_all() - .await - .error(|e: &std::io::Error| { - format!("{COMPONENT} (error: {e}) - failed to sync file after appending: {path}") - }) - .map_err(|_| IggyError::CannotSyncFile)?; - Ok(()) - } - - pub async fn overwrite(&self, path: &str, bytes: B) -> Result<(), IggyError> { - let mut file = file::overwrite(path) - .await - .error(|e: &std::io::Error| { - format!("{COMPONENT} (error: {e}) - failed to overwrite file: {path}") - }) - .map_err(|_| IggyError::CannotOverwriteFile)?; - let position = 0; - file.write_all_at(bytes, position) - .await - .0 - .error(|e: &std::io::Error| { - format!("{COMPONENT} (error: {e}) - failed to write data to file: {path}") - }) - .map_err(|_| IggyError::CannotWriteToFile)?; - file.sync_all() - .await - .error(|e: &std::io::Error| { - format!("{COMPONENT} (error: {e}) - failed to sync file after overwriting: {path}") - }) - .map_err(|_| IggyError::CannotSyncFile)?; - Ok(()) - } - - pub async fn delete(&self, path: &str) -> Result<(), IggyError> { - remove_file(path) - .await - .error(|e: &std::io::Error| { - format!("{COMPONENT} (error: {e}) - failed to delete file: {path}") - }) - .map_err(|_| IggyError::CannotDeleteFile)?; - Ok(()) - } -} diff --git a/core/server/src/streaming/polling_consumer.rs b/core/server/src/streaming/polling_consumer.rs deleted file mode 100644 index 02f657f288..0000000000 --- a/core/server/src/streaming/polling_consumer.rs +++ /dev/null @@ -1,129 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub use iggy_common::ConsumerGroupId; -use iggy_common::{IdKind, Identifier, calculate_32}; -use std::fmt::{Display, Formatter}; - -#[derive(Debug, PartialEq, Eq, Hash, Copy, Clone)] -pub struct MemberId(pub usize); - -impl Display for MemberId { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.0) - } -} - -#[derive(Debug, PartialEq, Copy, Clone)] -pub enum PollingConsumer { - Consumer(usize, usize), // Consumer ID + Partition ID - ConsumerGroup(ConsumerGroupId, MemberId), // Consumer Group ID + Member ID -} - -impl PollingConsumer { - pub fn consumer(consumer_id: &Identifier, partition_id: usize) -> Self { - PollingConsumer::Consumer(Self::resolve_consumer_id(consumer_id), partition_id) - } - - pub fn consumer_group(consumer_group_id: usize, member_id: usize) -> Self { - PollingConsumer::ConsumerGroup(ConsumerGroupId(consumer_group_id), MemberId(member_id)) - } - - pub fn resolve_consumer_id(identifier: &Identifier) -> usize { - match identifier.kind { - IdKind::Numeric => identifier.get_u32_value().unwrap() as usize, - IdKind::String => calculate_32(&identifier.value) as usize, - } - } -} - -impl Display for PollingConsumer { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - match self { - PollingConsumer::Consumer(consumer_id, partition_id) => write!( - f, - "consumer ID: {consumer_id}, partition ID: {partition_id}" - ), - PollingConsumer::ConsumerGroup(consumer_group_id, member_id) => { - write!( - f, - "consumer group ID: {consumer_group_id}, member ID: {member_id}" - ) - } - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use iggy_common::Consumer; - - #[test] - fn given_consumer_with_numeric_id_polling_consumer_should_be_created() { - let consumer_id_value = 1; - let partition_id = 3; - let consumer_id = Identifier::numeric(consumer_id_value).unwrap(); - let consumer = Consumer::new(consumer_id); - let polling_consumer = PollingConsumer::consumer(&consumer.id, partition_id); - - assert_eq!( - polling_consumer, - PollingConsumer::Consumer(consumer_id_value as usize, partition_id) - ); - } - - #[test] - fn given_consumer_with_named_id_polling_consumer_should_be_created() { - let consumer_name = "consumer"; - let partition_id = 3; - let consumer_id = Identifier::named(consumer_name).unwrap(); - let consumer = Consumer::new(consumer_id); - - let resolved_consumer_id = PollingConsumer::resolve_consumer_id(&consumer.id); - let polling_consumer = PollingConsumer::consumer(&consumer.id, partition_id); - - assert_eq!( - polling_consumer, - PollingConsumer::Consumer(resolved_consumer_id, partition_id) - ); - } - - #[test] - fn given_consumer_group_with_numeric_id_polling_consumer_group_should_be_created() { - let group_id = 1; - let client_id = 2; - let polling_consumer = PollingConsumer::consumer_group(group_id, client_id); - - match polling_consumer { - PollingConsumer::ConsumerGroup(consumer_group_id, member_id) => { - assert_eq!(consumer_group_id, ConsumerGroupId(group_id)); - assert_eq!(member_id, MemberId(client_id)); - } - _ => panic!("Expected ConsumerGroup"), - } - } - - #[test] - fn given_distinct_named_ids_unique_polling_consumer_ids_should_be_created() { - let name1 = Identifier::named("consumer1").unwrap(); - let name2 = Identifier::named("consumer2").unwrap(); - let id1 = PollingConsumer::resolve_consumer_id(&name1); - let id2 = PollingConsumer::resolve_consumer_id(&name2); - assert_ne!(id1, id2); - } -} diff --git a/core/server/src/streaming/segments/indexes/index_reader.rs b/core/server/src/streaming/segments/indexes/index_reader.rs deleted file mode 100644 index 3decdb46e7..0000000000 --- a/core/server/src/streaming/segments/indexes/index_reader.rs +++ /dev/null @@ -1,19 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#[allow(unused_imports)] -pub use server_common::IndexReader; diff --git a/core/server/src/streaming/segments/indexes/index_writer.rs b/core/server/src/streaming/segments/indexes/index_writer.rs deleted file mode 100644 index c77bada32e..0000000000 --- a/core/server/src/streaming/segments/indexes/index_writer.rs +++ /dev/null @@ -1,18 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub use server_common::IndexWriter; diff --git a/core/server/src/streaming/segments/indexes/mod.rs b/core/server/src/streaming/segments/indexes/mod.rs deleted file mode 100644 index d6bfd8ffaf..0000000000 --- a/core/server/src/streaming/segments/indexes/mod.rs +++ /dev/null @@ -1,22 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -mod index_reader; -mod index_writer; - -pub use index_writer::IndexWriter; -pub use server_common::IggyIndexesMut; diff --git a/core/server/src/streaming/segments/memory_journal.rs b/core/server/src/streaming/segments/memory_journal.rs deleted file mode 100644 index 5cd17fb5a6..0000000000 --- a/core/server/src/streaming/segments/memory_journal.rs +++ /dev/null @@ -1,17 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - diff --git a/core/server/src/streaming/segments/messages/messages_reader.rs b/core/server/src/streaming/segments/messages/messages_reader.rs deleted file mode 100644 index 3398351b8d..0000000000 --- a/core/server/src/streaming/segments/messages/messages_reader.rs +++ /dev/null @@ -1,19 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#[allow(unused_imports)] -pub use server_common::MessagesReader; diff --git a/core/server/src/streaming/segments/messages/messages_writer.rs b/core/server/src/streaming/segments/messages/messages_writer.rs deleted file mode 100644 index b9911b201c..0000000000 --- a/core/server/src/streaming/segments/messages/messages_writer.rs +++ /dev/null @@ -1,18 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub use server_common::MessagesWriter; diff --git a/core/server/src/streaming/segments/messages/mod.rs b/core/server/src/streaming/segments/messages/mod.rs deleted file mode 100644 index fbd279ec7c..0000000000 --- a/core/server/src/streaming/segments/messages/mod.rs +++ /dev/null @@ -1,21 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -mod messages_reader; -mod messages_writer; - -pub use messages_writer::MessagesWriter; diff --git a/core/server/src/streaming/segments/mod.rs b/core/server/src/streaming/segments/mod.rs deleted file mode 100644 index df4bd3c53e..0000000000 --- a/core/server/src/streaming/segments/mod.rs +++ /dev/null @@ -1,34 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -mod indexes; -mod messages; -mod segment; -mod types; - -pub mod storage; - -pub use indexes::IggyIndexesMut; -pub use indexes::IndexWriter; -pub use messages::MessagesWriter; -pub use segment::Segment; -pub use types::IggyMessageHeaderViewMut; -pub use types::IggyMessageViewMut; -pub use types::IggyMessagesBatchMut; -pub use types::IggyMessagesBatchSet; - -pub use crate::configs::validators::SEGMENT_MAX_SIZE_BYTES; diff --git a/core/server/src/streaming/segments/segment.rs b/core/server/src/streaming/segments/segment.rs deleted file mode 100644 index f2938d3541..0000000000 --- a/core/server/src/streaming/segments/segment.rs +++ /dev/null @@ -1,18 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub use iggy_common::Segment; diff --git a/core/server/src/streaming/segments/storage.rs b/core/server/src/streaming/segments/storage.rs deleted file mode 100644 index 8d0212ce39..0000000000 --- a/core/server/src/streaming/segments/storage.rs +++ /dev/null @@ -1,50 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub use server_common::SegmentStorage as Storage; - -use crate::configs::system::SystemConfig; -use iggy_common::IggyError; - -/// Creates a new storage for the specified partition with the given start offset -pub async fn create_segment_storage( - config: &SystemConfig, - stream_id: usize, - topic_id: usize, - partition_id: usize, - messages_size: u64, - indexes_size: u64, - start_offset: u64, -) -> Result { - let messages_path = - config.get_messages_file_path(stream_id, topic_id, partition_id, start_offset); - let index_path = config.get_index_path(stream_id, topic_id, partition_id, start_offset); - let log_fsync = config.partition.enforce_fsync; - let index_fsync = config.partition.enforce_fsync; - let file_exists = false; - - Storage::new( - &messages_path, - &index_path, - messages_size, - indexes_size, - log_fsync, - index_fsync, - file_exists, - ) - .await -} diff --git a/core/server/src/streaming/segments/types/mod.rs b/core/server/src/streaming/segments/types/mod.rs deleted file mode 100644 index a822fd3fc5..0000000000 --- a/core/server/src/streaming/segments/types/mod.rs +++ /dev/null @@ -1,19 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub use iggy_common::{IggyMessageHeaderViewMut, IggyMessageViewMut}; -pub use server_common::{IggyMessagesBatchMut, IggyMessagesBatchSet}; diff --git a/core/server/src/streaming/session.rs b/core/server/src/streaming/session.rs deleted file mode 100644 index 57984663fe..0000000000 --- a/core/server/src/streaming/session.rs +++ /dev/null @@ -1,105 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use iggy_common::UserId; -use std::cell::Cell; -use std::fmt::Display; -use std::net::SocketAddr; - -// This might be extended with more fields in the future e.g. custom name, permissions etc. -#[derive(Debug, Clone)] -pub struct Session { - pub client_id: u32, - user_id: Cell, - active: Cell, - pub ip_address: SocketAddr, - pub migrated: Cell, -} - -impl Session { - pub fn new(client_id: u32, user_id: UserId, ip_address: SocketAddr) -> Self { - Self { - client_id, - user_id: Cell::new(user_id), - active: Cell::new(true), - migrated: Cell::new(false), - ip_address, - } - } - - pub fn stateless(user_id: UserId, ip_address: SocketAddr) -> Self { - Self::new(0, user_id, ip_address) - } - - pub fn from_client_id(client_id: u32, ip_address: SocketAddr) -> Self { - Self::new(client_id, u32::MAX, ip_address) - } - - pub fn get_user_id(&self) -> UserId { - self.user_id.get() - } - - pub fn set_user_id(&self, user_id: UserId) { - self.user_id.set(user_id); - } - - pub fn set_stale(&self) { - self.active.set(false); - } - - /// Returns true if this session has been migrated to another shard. - /// - /// Prevents socket ping-ponging between shards. Subsequent wrong-shard requests use message forwarding instead - pub fn is_migrated(&self) -> bool { - self.migrated.get() - } - - pub fn set_migrated(&self) { - self.migrated.set(true) - } - - pub fn clear_user_id(&self) { - self.set_user_id(u32::MAX); - } - - pub fn is_active(&self) -> bool { - self.active.get() - } - - pub fn is_authenticated(&self) -> bool { - self.get_user_id() != u32::MAX - } -} - -impl Display for Session { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let user_id = self.get_user_id(); - if user_id != u32::MAX { - write!( - f, - "client ID: {}, user ID: {}, IP address: {}", - self.client_id, user_id, self.ip_address - ) - } else { - write!( - f, - "client ID: {}, IP address: {}", - self.client_id, self.ip_address - ) - } - } -} diff --git a/core/server/src/streaming/stats/mod.rs b/core/server/src/streaming/stats/mod.rs deleted file mode 100644 index a745b9bb3a..0000000000 --- a/core/server/src/streaming/stats/mod.rs +++ /dev/null @@ -1,18 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub use iggy_common::{PartitionStats, StreamStats, TopicStats}; diff --git a/core/server/src/streaming/storage.rs b/core/server/src/streaming/storage.rs deleted file mode 100644 index 9c91b52dbc..0000000000 --- a/core/server/src/streaming/storage.rs +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use super::persistence::persister::PersisterKind; -use crate::configs::system::SystemConfig; -use crate::shard::system::storage::FileSystemInfoStorage; -use std::sync::Arc; - -#[derive(Debug, Clone)] -pub struct SystemStorage { - pub info: Arc, - pub persister: Arc, -} - -impl SystemStorage { - pub fn new(config: Arc, persister: Arc) -> Self { - Self { - info: Arc::new(FileSystemInfoStorage::new( - config.get_state_info_path(), - persister.clone(), - )), - persister, - } - } -} diff --git a/core/server/src/streaming/streams/mod.rs b/core/server/src/streaming/streams/mod.rs deleted file mode 100644 index 976caf7a52..0000000000 --- a/core/server/src/streaming/streams/mod.rs +++ /dev/null @@ -1,20 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod storage; - -pub const COMPONENT: &str = "STREAMING_STREAMS"; diff --git a/core/server/src/streaming/streams/storage.rs b/core/server/src/streaming/streams/storage.rs deleted file mode 100644 index 5068c1d1e2..0000000000 --- a/core/server/src/streaming/streams/storage.rs +++ /dev/null @@ -1,65 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::{configs::system::SystemConfig, io::fs_utils::remove_dir_all}; -use compio::fs::create_dir_all; -use iggy_common::IggyError; -use std::path::Path; - -pub async fn create_stream_file_hierarchy( - id: usize, - config: &SystemConfig, -) -> Result<(), IggyError> { - let path = config.get_stream_path(id); - - if !Path::new(&path).exists() && create_dir_all(&path).await.is_err() { - return Err(IggyError::CannotCreateStreamDirectory( - id as u32, - path.clone(), - )); - } - - tracing::info!("Saved stream with ID: {}.", id); - Ok(()) -} - -/// Delete stream directory using only IDs. -/// Does not require slab access - works with SharedMetadata. -/// topics_with_partitions: Vec<(topic_id, Vec)> -pub async fn delete_stream_directory( - stream_id: usize, - topics_with_partitions: &[(usize, Vec)], - config: &SystemConfig, -) -> Result<(), IggyError> { - use crate::streaming::topics::storage::delete_topic_directory; - - let stream_path = config.get_stream_path(stream_id); - if !Path::new(&stream_path).exists() { - return Err(IggyError::StreamDirectoryNotFound(stream_path)); - } - - // Delete all topics - for (topic_id, partition_ids) in topics_with_partitions { - delete_topic_directory(stream_id, *topic_id, partition_ids, config).await?; - } - - remove_dir_all(&stream_path) - .await - .map_err(|_| IggyError::CannotDeleteStreamDirectory(stream_id as u32))?; - tracing::info!("Deleted stream files for stream with ID: {}.", stream_id); - Ok(()) -} diff --git a/core/server/src/streaming/topics/helpers.rs b/core/server/src/streaming/topics/helpers.rs deleted file mode 100644 index 119222b427..0000000000 --- a/core/server/src/streaming/topics/helpers.rs +++ /dev/null @@ -1,33 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use iggy_common::calculate_32; - -pub fn calculate_partition_id_by_messages_key_hash( - upperbound: usize, - messages_key: &[u8], -) -> usize { - let messages_key_hash = calculate_32(messages_key) as usize; - let partition_id = messages_key_hash % upperbound; - tracing::trace!( - "Calculated partition ID: {} for messages key: {:?}, hash: {}", - partition_id, - messages_key, - messages_key_hash - ); - partition_id -} diff --git a/core/server/src/streaming/topics/mod.rs b/core/server/src/streaming/topics/mod.rs deleted file mode 100644 index da72843cce..0000000000 --- a/core/server/src/streaming/topics/mod.rs +++ /dev/null @@ -1,21 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod helpers; -pub mod storage; - -pub const COMPONENT: &str = "STREAMING_TOPICS"; diff --git a/core/server/src/streaming/topics/storage.rs b/core/server/src/streaming/topics/storage.rs deleted file mode 100644 index 10774d9a5e..0000000000 --- a/core/server/src/streaming/topics/storage.rs +++ /dev/null @@ -1,82 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use compio::fs::create_dir_all; -use iggy_common::IggyError; -use std::path::Path; - -use crate::{ - configs::system::SystemConfig, io::fs_utils::remove_dir_all, - streaming::partitions::storage::delete_partitions_from_disk, -}; - -pub async fn create_topic_file_hierarchy( - stream_id: usize, - topic_id: usize, - config: &SystemConfig, -) -> Result<(), IggyError> { - let topic_path = config.get_topic_path(stream_id, topic_id); - let partitions_path = config.get_partitions_path(stream_id, topic_id); - if !Path::new(&topic_path).exists() && create_dir_all(&topic_path).await.is_err() { - return Err(IggyError::CannotCreateTopicDirectory( - topic_id, stream_id, topic_path, - )); - } - tracing::info!( - "Saved topic with ID: {}. for stream with ID: {}", - topic_id, - stream_id - ); - - if !Path::new(&partitions_path).exists() && create_dir_all(&partitions_path).await.is_err() { - return Err(IggyError::CannotCreatePartitionsDirectory( - stream_id, topic_id, - )); - } - Ok(()) -} - -/// Delete topic directory and all partition subdirectories using only IDs. -/// Does not require slab access - works with SharedMetadata. -pub async fn delete_topic_directory( - stream_id: usize, - topic_id: usize, - partition_ids: &[usize], - config: &SystemConfig, -) -> Result<(), IggyError> { - let topic_path = config.get_topic_path(stream_id, topic_id); - if !Path::new(&topic_path).exists() { - return Err(IggyError::TopicDirectoryNotFound(topic_path)); - } - - // Delete partition directories - for &partition_id in partition_ids { - delete_partitions_from_disk(stream_id, topic_id, partition_id, config).await?; - } - - // Delete the topic directory itself - remove_dir_all(&topic_path).await.map_err(|_| { - IggyError::CannotDeleteTopicDirectory(topic_id as u32, stream_id as u32, topic_path) - })?; - - tracing::info!( - "Deleted topic files for topic with ID: {} in stream with ID: {}.", - topic_id, - stream_id - ); - Ok(()) -} diff --git a/core/server/src/streaming/users/mod.rs b/core/server/src/streaming/users/mod.rs deleted file mode 100644 index 36e32eca12..0000000000 --- a/core/server/src/streaming/users/mod.rs +++ /dev/null @@ -1,18 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod user; diff --git a/core/server/src/streaming/users/user.rs b/core/server/src/streaming/users/user.rs deleted file mode 100644 index 334370dd40..0000000000 --- a/core/server/src/streaming/users/user.rs +++ /dev/null @@ -1,137 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::streaming::utils::crypto; -use dashmap::DashMap; -use iggy_common::IggyTimestamp; -use iggy_common::PersonalAccessToken; -use iggy_common::UserStatus; -use iggy_common::defaults::*; -use iggy_common::{Permissions, UserId}; -use std::sync::Arc; - -#[derive(Debug, Clone)] -pub struct User { - pub id: UserId, - pub status: UserStatus, - pub username: String, - pub password: String, - pub created_at: IggyTimestamp, - pub permissions: Option, - pub personal_access_tokens: DashMap, PersonalAccessToken>, -} - -impl Default for User { - fn default() -> Self { - Self { - id: 0, - status: UserStatus::Active, - username: "user".to_string(), - password: "secret".to_string(), - created_at: IggyTimestamp::now(), - permissions: None, - personal_access_tokens: DashMap::new(), - } - } -} - -impl User { - pub fn empty(id: UserId) -> Self { - Self { - id, - ..Default::default() - } - } - - pub fn new( - id: u32, - username: &str, - password: &str, - status: UserStatus, - permissions: Option, - ) -> Self { - Self::with_password( - id, - username, - crypto::hash_password(password), - status, - permissions, - ) - } - - pub fn with_password( - id: u32, - username: &str, - password: String, - status: UserStatus, - permissions: Option, - ) -> Self { - Self { - id, - username: username.into(), - password, - created_at: IggyTimestamp::now(), - status, - permissions, - personal_access_tokens: DashMap::new(), - } - } - - pub fn root(username: &str, password: &str) -> Self { - Self::new( - DEFAULT_ROOT_USER_ID, - username, - password, - UserStatus::Active, - Some(Permissions::root()), - ) - } - - pub fn is_root(&self) -> bool { - self.id == DEFAULT_ROOT_USER_ID - } - - pub fn is_active(&self) -> bool { - self.status == UserStatus::Active - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn given_root_user_data_and_credentials_should_be_valid() { - let user = User::root(DEFAULT_ROOT_USERNAME, DEFAULT_ROOT_PASSWORD); - assert_eq!(user.id, DEFAULT_ROOT_USER_ID); - assert_eq!(user.username, DEFAULT_ROOT_USERNAME); - assert_ne!(user.password, DEFAULT_ROOT_PASSWORD); - assert!(crypto::verify_password( - DEFAULT_ROOT_PASSWORD, - &user.password - )); - assert_eq!(user.status, UserStatus::Active); - assert!(user.created_at.as_micros() > 0); - } - - #[test] - fn should_be_created_given_specific_status() { - let status = UserStatus::Inactive; - let user = User::new(1, "test", "test", status, None); - assert_eq!(user.status, status); - } -} diff --git a/core/server/src/streaming/utils/address.rs b/core/server/src/streaming/utils/address.rs deleted file mode 100644 index 32bf83f3ac..0000000000 --- a/core/server/src/streaming/utils/address.rs +++ /dev/null @@ -1,75 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -/// Extracts IP from an address string like "127.0.0.1:8090" or "[::1]:8090" -pub fn extract_ip(address: &str) -> String { - if let Some(colon_pos) = address.rfind(':') { - // Handle IPv6 addresses like [::1]:8090 - if address.starts_with('[') - && let Some(bracket_pos) = address.rfind(']') - { - return address[1..bracket_pos].to_string(); - } - // Handle IPv4 addresses like 127.0.0.1:8090 - return address[..colon_pos].to_string(); - } - address.to_string() -} - -/// Extracts port from an address string like "127.0.0.1:8090" -pub fn extract_port(address: &str) -> u16 { - if let Some(colon_pos) = address.rfind(':') - && let Ok(port) = address[colon_pos + 1..].parse::() - { - return port; - } - 0 -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_extract_ip_ipv4() { - assert_eq!(extract_ip("127.0.0.1:8090"), "127.0.0.1"); - assert_eq!(extract_ip("192.168.1.100:3000"), "192.168.1.100"); - } - - #[test] - fn test_extract_ip_ipv6() { - assert_eq!(extract_ip("[::1]:8090"), "::1"); - assert_eq!(extract_ip("[2001:db8::1]:443"), "2001:db8::1"); - } - - #[test] - fn test_extract_ip_no_port() { - assert_eq!(extract_ip("127.0.0.1"), "127.0.0.1"); - } - - #[test] - fn test_extract_port() { - assert_eq!(extract_port("127.0.0.1:8090"), 8090); - assert_eq!(extract_port("192.168.1.100:3000"), 3000); - assert_eq!(extract_port("[::1]:8090"), 8090); - } - - #[test] - fn test_extract_port_no_port() { - assert_eq!(extract_port("127.0.0.1"), 0); - } -} diff --git a/core/server/src/streaming/utils/file.rs b/core/server/src/streaming/utils/file.rs deleted file mode 100644 index 75d833d39a..0000000000 --- a/core/server/src/streaming/utils/file.rs +++ /dev/null @@ -1,54 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use compio::fs::{File, OpenOptions, remove_file}; -use std::path::Path; - -pub async fn open(path: &str) -> Result { - OpenOptions::new().read(true).open(path).await -} - -pub async fn append(path: &str) -> Result<(File, u64), std::io::Error> { - let file = OpenOptions::new() - .create(true) - .write(true) - .open(path) - .await?; - let position = file.metadata().await?.len(); - Ok((file, position)) -} - -pub async fn overwrite(path: &str) -> Result { - OpenOptions::new() - .create(true) - .write(true) - .truncate(false) - .open(path) - .await -} - -pub async fn remove(path: &str) -> Result<(), std::io::Error> { - remove_file(path).await -} - -pub async fn rename(old_path: &str, new_path: &str) -> Result<(), std::io::Error> { - compio::fs::rename(Path::new(old_path), Path::new(new_path)).await -} - -pub async fn exists(path: &str) -> Result { - std::fs::exists(path) -} diff --git a/core/server/src/streaming/utils/mod.rs b/core/server/src/streaming/utils/mod.rs deleted file mode 100644 index b03232126f..0000000000 --- a/core/server/src/streaming/utils/mod.rs +++ /dev/null @@ -1,22 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod address; -pub mod file; -pub mod ptr; -pub use iggy_common::random_id; -pub use server_common::crypto; diff --git a/core/server/src/streaming/utils/ptr.rs b/core/server/src/streaming/utils/ptr.rs deleted file mode 100644 index a98977049b..0000000000 --- a/core/server/src/streaming/utils/ptr.rs +++ /dev/null @@ -1,70 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::{ops::Deref, ptr::NonNull}; - -// Wrapper around an imutable pointer to a 'static value, that can be cloned and sent across threads. -pub struct EternalPtr { - ptr: NonNull, - _marker: std::marker::PhantomData<&'static T>, -} - -impl From> for EternalPtr { - fn from(value: NonNull) -> Self { - Self { - ptr: value, - _marker: std::marker::PhantomData, - } - } -} - -impl From<&'static T> for EternalPtr { - fn from(value: &'static T) -> Self { - Self { - ptr: value.into(), - _marker: std::marker::PhantomData, - } - } -} - -impl From<&'static mut T> for EternalPtr { - fn from(value: &'static mut T) -> Self { - Self { - ptr: value.into(), - _marker: std::marker::PhantomData, - } - } -} - -impl Clone for EternalPtr { - fn clone(&self) -> Self { - Self { - ptr: self.ptr, - _marker: std::marker::PhantomData, - } - } -} - -impl Deref for EternalPtr { - type Target = T; - - fn deref(&self) -> &Self::Target { - unsafe { self.ptr.as_ref() } - } -} - -unsafe impl Send for EternalPtr {} diff --git a/core/server/src/systemd.rs b/core/server/src/systemd.rs new file mode 100644 index 0000000000..5ef2ba80be --- /dev/null +++ b/core/server/src/systemd.rs @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Thin wrappers around `sd_notify` so every systemd interaction on the +//! server side lives in one place (mirrors `core/ai/mcp/src/systemd.rs`). + +use crate::server_error::{ServerError, ShardJoinFailureKind}; +use message_bus::{IggyMessageBus, ShutdownToken}; +use std::time::Duration; +use tracing::{info, warn}; + +/// Tell systemd the service has finished start-up (`READY=1`). +pub fn notify_ready() { + if let Err(error) = sd_notify::notify(&[sd_notify::NotifyState::Ready]) { + warn!("Failed to send systemd READY=1 notification: {error}"); + } +} + +/// Tell systemd the service has begun shutting down (`STOPPING=1`), which +/// also stops the watchdog timer from counting against a long drain. +pub fn notify_stopping() { + let _ = sd_notify::notify(&[sd_notify::NotifyState::Stopping]); +} + +/// Surface a dirty shutdown in `systemctl status` / journald. +pub fn notify_shutdown_failure(error: &ServerError) { + let wedged = matches!( + error, + ServerError::ShardJoinFailures { failures } + if failures + .iter() + .any(|failure| matches!(failure.kind, ShardJoinFailureKind::Wedged { .. })) + ); + let status = if wedged { + "graceful shutdown timed out" + } else { + "shard threads failed during shutdown" + }; + let _ = sd_notify::notify(&[sd_notify::NotifyState::Status(status)]); +} + +/// Start the `WATCHDOG=1` keep-alive. Does nothing unless the unit set +/// `WatchdogSec=`. Tracked on the bus so `bus.shutdown()` reaps the task. +pub fn spawn_watchdog(bus: &IggyMessageBus) { + let Some(timeout) = sd_notify::watchdog_enabled() else { + return; + }; + + let interval = timeout / 2; + info!( + "Systemd watchdog enabled, pinging every {}s (timeout: {}s).", + interval.as_secs(), + timeout.as_secs() + ); + + let handle = compio::runtime::spawn(run_watchdog(bus.token(), interval)); + bus.track_background(handle); +} + +async fn run_watchdog(token: ShutdownToken, interval: Duration) { + while token.sleep_or_shutdown(interval).await { + if let Err(error) = sd_notify::notify(&[sd_notify::NotifyState::Watchdog]) { + warn!("Failed to send systemd watchdog ping: {error}"); + } + } +} diff --git a/core/server/src/tcp/connection_handler.rs b/core/server/src/tcp/connection_handler.rs deleted file mode 100644 index d48012258d..0000000000 --- a/core/server/src/tcp/connection_handler.rs +++ /dev/null @@ -1,184 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::{self, HandlerResult, MAX_CONTROL_FRAME_PAYLOAD}; -use crate::sender::SenderKind; -use crate::server_error::ConnectionError; -use crate::shard::IggyShard; -use crate::streaming::session::Session; -use async_channel::Receiver; -use bytes::BytesMut; -use futures::FutureExt; -use iggy_binary_protocol::RequestFrame; -use iggy_binary_protocol::codes::{GET_CLUSTER_METADATA_CODE, SEND_MESSAGES_CODE, command_name}; -use iggy_common::IggyError; -use std::io::ErrorKind; -use std::rc::Rc; -use tracing::{debug, error, info}; - -/// Connection lifecycle action after command handling. -pub enum ConnectionAction { - /// Continue handling connection on current shard. - Finished, - - /// Connection migrated to another shard, exit without cleanup. - Migrated { to_shard: u16 }, -} - -pub(crate) async fn handle_connection( - session: &Session, - sender: &mut SenderKind, - shard: &Rc, - stop_receiver: Receiver<()>, -) -> Result { - let mut header_buffer = BytesMut::with_capacity(RequestFrame::HEADER_SIZE); - loop { - let read_future = sender.read(header_buffer); - // TODO(hubcio): this futures::select! call is translated to epoll_wait syscall for every - // message, which adds around 100 us median latency. We could instead just call sender.shutdown() - // if some atomic bool is set, since this is all happenng within single thread. - let (_, mut header_buf) = futures::select! { - _ = stop_receiver.recv().fuse() => { - info!("Connection stop signal received for session: {}", session); - let _ = sender.send_error_response(IggyError::Disconnected).await; - return Ok(ConnectionAction::Finished); - } - result = read_future.fuse() => { - match result { - (Ok(_), buf) => (Ok::<(), IggyError>(()), buf), - (Err(error), buf) => { - header_buffer = buf; - if error.as_code() == IggyError::ConnectionClosed.as_code() { - return Err(ConnectionError::from(error)); - } else { - error!("got error: {:?}", error); - sender.send_error_response(error).await?; - continue; - } - } - } - } - }; - - let length = u32::from_le_bytes(header_buf[0..4].try_into().unwrap()); - let code = u32::from_le_bytes(header_buf[4..8].try_into().unwrap()); - header_buf.clear(); - header_buffer = header_buf; - - let cmd_name = command_name(code).unwrap_or("unknown"); - debug!("Received a TCP request, length: {length}, code: {code} ({cmd_name})"); - - let payload_length = match RequestFrame::payload_length(length) { - Ok(len) => len, - Err(_) => { - sender - .send_error_response(IggyError::InvalidCommand) - .await?; - continue; - } - }; - - let result = if code == SEND_MESSAGES_CODE { - dispatch::dispatch_send_messages(sender, payload_length, session, shard).await - } else { - if payload_length > MAX_CONTROL_FRAME_PAYLOAD { - sender - .send_error_response(IggyError::InvalidCommand) - .await?; - continue; - } - let payload = dispatch::read_payload(sender, payload_length).await?; - let frame = RequestFrame::from_parts(code, &payload); - dispatch::dispatch(frame, sender, session, shard).await - }; - - match result { - Ok(handler_result) => match handler_result { - HandlerResult::Finished => { - debug!( - "Command {code} ({cmd_name}) was handled successfully, session: {session}. TCP response was sent." - ); - } - HandlerResult::Migrated { to_shard } => { - info!( - "Command {code} ({cmd_name}) was transferred to shard {to_shard}, session: {session}." - ); - - return Ok(ConnectionAction::Migrated { to_shard }); - } - }, - Err(error) => { - if code == GET_CLUSTER_METADATA_CODE - && matches!(error, IggyError::FeatureUnavailable) - { - debug!( - "GetClusterMetadata command not available (clustering disabled), session: {session}." - ); - sender.send_error_response(error).await?; - debug!("TCP error response was sent to: {session}."); - } else { - error!( - "Command with code {code} ({cmd_name}) was not handled successfully, session: {session}, error: {error}." - ); - - if matches!(error, IggyError::ClientNotFound(_) | IggyError::StaleClient) { - sender.send_error_response(error.clone()).await?; - debug!("TCP error response was sent to: {session}."); - error!("Session: {session} will be deleted."); - return Err(ConnectionError::from(error)); - } else { - sender.send_error_response(error).await?; - debug!("TCP error response was sent to: {session}."); - } - } - } - } - } -} - -pub(crate) fn handle_error(error: ConnectionError) { - match error { - ConnectionError::IoError(e) => match e.kind() { - ErrorKind::UnexpectedEof => { - info!("Connection has been closed."); - } - ErrorKind::ConnectionAborted => { - info!("Connection has been aborted."); - } - ErrorKind::ConnectionRefused => { - info!("Connection has been refused."); - } - ErrorKind::ConnectionReset => { - info!("Connection has been reset."); - } - _ => { - error!("Connection has failed: {e}"); - } - }, - ConnectionError::SdkError(sdk_error) => match sdk_error { - IggyError::ConnectionClosed => { - debug!("Client closed connection."); - } - _ => { - error!("Failure in internal SDK call: {sdk_error}"); - } - }, - _ => { - error!("Connection has failed: {error}"); - } - } -} diff --git a/core/server/src/tcp/mod.rs b/core/server/src/tcp/mod.rs deleted file mode 100644 index e5f9d42cba..0000000000 --- a/core/server/src/tcp/mod.rs +++ /dev/null @@ -1,73 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod connection_handler; -pub mod tcp_listener; -pub mod tcp_server; -pub mod tcp_socket; -pub mod tcp_tls_listener; - -pub const COMPONENT: &str = "TCP"; - -/// Bind a `TcpListener` via the compio 0.19 `TcpSocket` builder. -/// -/// compio 0.19 removed `TcpListener::bind_with_options(addr, SocketOpts)`; -/// this is the shared replacement for every legacy listener bind. -/// `SO_REUSEPORT` is always set (thread-per-core: many sockets bind the -/// same addr+port); `reuseaddr` is opt-in. When `tuning` is `Some` and -/// `override_defaults` is set, the socket buffers / keepalive / nodelay -/// are applied (a zero linger maps to `set_zero_linger`; a non-zero -/// linger has no compio-0.19 successor and is dropped). -pub(crate) async fn bind_reuseport_listener( - addr: std::net::SocketAddr, - reuseaddr: bool, - tuning: Option<&crate::configs::tcp::TcpSocketConfig>, -) -> std::io::Result { - use compio::net::TcpSocket; - - let socket = match addr { - std::net::SocketAddr::V4(_) => TcpSocket::new_v4().await?, - std::net::SocketAddr::V6(_) => TcpSocket::new_v6().await?, - }; - socket.set_reuseport(true)?; - if reuseaddr { - socket.set_reuseaddr(true)?; - } - if let Some(config) = tuning - && config.override_defaults - { - let recv_buffer_size = config - .recv_buffer_size - .as_bytes_u64() - .try_into() - .expect("Failed to parse recv_buffer_size for TCP socket"); - let send_buffer_size = config - .send_buffer_size - .as_bytes_u64() - .try_into() - .expect("Failed to parse send_buffer_size for TCP socket"); - socket.set_recv_buffer_size(recv_buffer_size)?; - socket.set_send_buffer_size(send_buffer_size)?; - socket.set_keepalive(config.keepalive)?; - if config.linger.get_duration().is_zero() { - socket.set_zero_linger()?; - } - socket.set_nodelay(config.nodelay)?; - } - socket.bind(addr).await?; - socket.listen(1024).await -} diff --git a/core/server/src/tcp/tcp_listener.rs b/core/server/src/tcp/tcp_listener.rs deleted file mode 100644 index 3f53e4e640..0000000000 --- a/core/server/src/tcp/tcp_listener.rs +++ /dev/null @@ -1,171 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::configs::tcp::TcpSocketConfig; - -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::shard::task_registry::{ShutdownToken, TaskRegistry}; -use crate::shard::transmission::event::ShardEvent; -use crate::tcp::connection_handler::{ConnectionAction, handle_connection, handle_error}; -use compio::net::TcpListener; -use err_trail::ErrContext; -use futures::FutureExt; -use iggy_common::{IggyError, TransportProtocol}; -use std::net::SocketAddr; -use std::rc::Rc; -use std::time::Duration; -use tracing::{debug, error, info}; - -async fn create_listener( - addr: SocketAddr, - config: &TcpSocketConfig, -) -> Result { - crate::tcp::bind_reuseport_listener(addr, false, Some(config)).await -} - -pub async fn start( - server_name: &'static str, - mut addr: SocketAddr, - config: &TcpSocketConfig, - shard: Rc, - shutdown: ShutdownToken, -) -> Result<(), IggyError> { - if shard.id != 0 && addr.port() == 0 { - info!("Waiting for TCP address from shard 0..."); - loop { - if let Some(bound_addr) = shard.tcp_bound_address.get() { - addr = bound_addr; - info!("Received TCP address: {}", addr); - break; - } - compio::time::sleep(Duration::from_millis(50)).await; - } - } - - let listener = create_listener(addr, config) - .await - .map_err(|_| IggyError::CannotBindToSocket(addr.to_string())) - .error(|err: &IggyError| { - format!("Failed to bind {server_name} server to address: {addr}, {err}") - })?; - let actual_addr = listener.local_addr().map_err(|e| { - error!("Failed to get local address: {}", e); - IggyError::CannotBindToSocket(addr.to_string()) - })?; - info!("{} server has started on: {:?}", server_name, actual_addr); - - if shard.id == 0 { - // Store bound address locally - shard.tcp_bound_address.set(Some(actual_addr)); - - if addr.port() == 0 { - // Notify config writer on shard 0 - let _ = shard.config_writer_notify.try_send(()); - - // Broadcast to other shards for SO_REUSEPORT binding - let event = ShardEvent::AddressBound { - protocol: TransportProtocol::Tcp, - address: actual_addr, - }; - shard.broadcast_event_to_all_shards(event).await?; - } - } - - accept_loop(server_name, listener, shard, shutdown).await -} - -async fn accept_loop( - server_name: &'static str, - listener: TcpListener, - shard: Rc, - shutdown: ShutdownToken, -) -> Result<(), IggyError> { - loop { - let shard = shard.clone(); - let accept_future = listener.accept(); - futures::select! { - _ = shutdown.wait().fuse() => { - debug!("{} received shutdown signal, no longer accepting connections", server_name); - break; - } - result = accept_future.fuse() => { - match result { - Ok((stream, address)) => { - if shard.is_shutting_down() { - info!("Rejecting new connection from {} during shutdown", address); - continue; - } - let shard_clone = shard.clone(); - info!("Accepted new TCP connection: {}", address); - let transport = TransportProtocol::Tcp; - let session = shard_clone.add_client(&address, transport); - info!("Added {} client with session: {} for IP address: {}", transport, session, address); - - let client_id = session.client_id; - info!("Created new session: {}", session); - let mut sender = SenderKind::get_tcp_sender(stream); - - let conn_stop_receiver = shard.task_registry.add_connection(client_id); - - let shard_for_conn = shard_clone.clone(); - let registry = shard.task_registry.clone(); - let registry_clone = registry.clone(); - registry.spawn_connection(async move { - match handle_connection(&session, &mut sender, &shard_for_conn, conn_stop_receiver).await { - Ok(ConnectionAction::Migrated { to_shard }) => { - info!("Migrated to shard {to_shard}, ignore cleanup connection"); - } - Ok(ConnectionAction::Finished) => { - cleanup_connection(&mut sender, client_id, address, ®istry_clone, &shard_for_conn).await; - } - Err(err) => { - handle_error(err); - cleanup_connection(&mut sender, client_id, address, ®istry_clone, &shard_for_conn).await; - }, - } - }); - } - Err(error) => error!("Unable to accept TCP socket. {}", error), - } - } - } - } - Ok(()) -} - -pub async fn cleanup_connection( - sender: &mut SenderKind, - client_id: u32, - address: SocketAddr, - registry: &Rc, - shard: &IggyShard, -) { - registry.remove_connection(&client_id); - shard.delete_client(client_id).await; - if let Err(error) = sender.shutdown().await { - error!( - "Failed to shutdown for client {}, address {}: {}", - client_id, address, error - ); - } else { - info!( - "Successfully closed for client {}, address {}", - client_id, address - ); - } -} diff --git a/core/server/src/tcp/tcp_server.rs b/core/server/src/tcp/tcp_server.rs deleted file mode 100644 index 3f9bcdb2ef..0000000000 --- a/core/server/src/tcp/tcp_server.rs +++ /dev/null @@ -1,56 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::shard::IggyShard; -use crate::shard::task_registry::ShutdownToken; -use crate::tcp::{tcp_listener, tcp_tls_listener}; -use iggy_common::IggyError; -use std::net::SocketAddr; -use std::rc::Rc; -use tracing::info; - -/// Starts the TCP server. -pub async fn spawn_tcp_server( - shard: Rc, - shutdown: ShutdownToken, -) -> Result<(), IggyError> { - let server_name = if shard.config.tcp.tls.enabled { - "Iggy TCP TLS" - } else { - "Iggy TCP" - }; - let socket_config = &shard.config.tcp.socket; - let addr: SocketAddr = shard - .config - .tcp - .address - .parse() - .expect("Failed to parse TCP address"); - info!("Initializing {} server...", server_name); - - match shard.config.tcp.tls.enabled { - true => { - tcp_tls_listener::start(server_name, addr, socket_config, shard.clone(), shutdown) - .await? - } - false => { - tcp_listener::start(server_name, addr, socket_config, shard.clone(), shutdown).await? - } - }; - - Ok(()) -} diff --git a/core/server/src/tcp/tcp_socket.rs b/core/server/src/tcp/tcp_socket.rs deleted file mode 100644 index 6946e0c86f..0000000000 --- a/core/server/src/tcp/tcp_socket.rs +++ /dev/null @@ -1,97 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use socket2::{Domain, Protocol, Socket, Type}; -use std::num::TryFromIntError; - -use crate::configs::tcp::TcpSocketConfig; - -pub fn build(ipv6: bool, config: &TcpSocketConfig) -> Socket { - let socket = if ipv6 { - Socket::new(Domain::IPV6, Type::STREAM, Some(Protocol::TCP)) - .expect("Unable to create an ipv6 socket") - } else { - Socket::new(Domain::IPV4, Type::STREAM, Some(Protocol::TCP)) - .expect("Unable to create an ipv4 socket") - }; - - // Required by the thread-per-core model... - // We create bunch of sockets on different threads, that bind to exactly the same address and port. - socket - .set_reuse_address(true) - .expect("Unable to set SO_REUSEADDR on socket"); - socket - .set_reuse_port(true) - .expect("Unable to set SO_REUSEPORT on socket"); - - if config.override_defaults { - config - .recv_buffer_size - .as_bytes_u64() - .try_into() - .map_err(|e: TryFromIntError| std::io::Error::other(e.to_string())) - .and_then(|size| socket.set_recv_buffer_size(size)) - .expect("Unable to set SO_RCVBUF on socket"); - config - .send_buffer_size - .as_bytes_u64() - .try_into() - .map_err(|e: TryFromIntError| std::io::Error::other(e.to_string())) - .and_then(|size| socket.set_send_buffer_size(size)) - .expect("Unable to set SO_SNDBUF on socket"); - socket - .set_keepalive(config.keepalive) - .expect("Unable to set SO_KEEPALIVE on socket"); - socket - .set_tcp_nodelay(config.nodelay) - .expect("Unable to set TCP_NODELAY on socket"); - socket - .set_linger(Some(config.linger.get_duration())) - .expect("Unable to set SO_LINGER on socket"); - } - - socket -} - -#[cfg(test)] -mod tests { - use std::time::Duration; - - use iggy_common::{IggyByteSize, IggyDuration}; - - use super::*; - - #[test] - fn given_override_defaults_socket_should_be_configured() { - let buffer_size = 425984; - let linger_dur = Duration::new(1, 0); - let config = TcpSocketConfig { - override_defaults: true, - recv_buffer_size: IggyByteSize::from(buffer_size), - send_buffer_size: IggyByteSize::from(buffer_size), - keepalive: true, - nodelay: true, - linger: IggyDuration::new(linger_dur), - }; - let socket = build(false, &config); - assert!(socket.recv_buffer_size().unwrap() >= buffer_size as usize); - assert!(socket.send_buffer_size().unwrap() >= buffer_size as usize); - assert!(socket.keepalive().unwrap()); - assert!(socket.tcp_nodelay().unwrap()); - assert_eq!(socket.linger().unwrap(), Some(linger_dur)); - } -} diff --git a/core/server/src/tcp/tcp_tls_listener.rs b/core/server/src/tcp/tcp_tls_listener.rs deleted file mode 100644 index c4c200e059..0000000000 --- a/core/server/src/tcp/tcp_tls_listener.rs +++ /dev/null @@ -1,224 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::configs::tcp::TcpSocketConfig; -use crate::sender::SenderKind; -use crate::shard::IggyShard; -use crate::shard::task_registry::ShutdownToken; -use crate::shard::transmission::event::ShardEvent; -use crate::tcp::connection_handler::{handle_connection, handle_error}; -use compio::net::TcpListener; -use compio::tls::TlsAcceptor; -use err_trail::ErrContext; -use futures::FutureExt; -use iggy_common::{IggyError, TransportProtocol}; -use rustls::ServerConfig; -use rustls::pki_types::{CertificateDer, PrivateKeyDer}; -use rustls_pemfile::{certs, private_key}; -use std::io::BufReader; -use std::net::SocketAddr; -use std::rc::Rc; -use std::sync::Arc; -use std::time::Duration; -use tracing::{error, info, trace, warn}; - -pub(crate) async fn start( - server_name: &'static str, - mut addr: SocketAddr, - config: &TcpSocketConfig, - shard: Rc, - shutdown: ShutdownToken, -) -> Result<(), IggyError> { - if shard.id != 0 && addr.port() == 0 { - info!("Waiting for TCP address from shard 0..."); - loop { - if let Some(bound_addr) = shard.tcp_bound_address.get() { - addr = bound_addr; - info!("Received TCP address: {}", addr); - break; - } - compio::time::sleep(Duration::from_millis(50)).await; - } - } - - let listener = create_listener(addr, config) - .await - .map_err(|_| IggyError::CannotBindToSocket(addr.to_string())) - .error(|err: &IggyError| { - format!("Failed to bind {server_name} server to address: {addr}, {err}") - })?; - - let actual_addr = listener.local_addr().map_err(|_e| { - error!("Failed to get local address: {_e}"); - IggyError::CannotBindToSocket(addr.to_string()) - })?; - - if shard.id == 0 { - shard.tcp_bound_address.set(Some(actual_addr)); - if addr.port() == 0 { - // Notify config writer on shard 0 - let _ = shard.config_writer_notify.try_send(()); - - let event = ShardEvent::AddressBound { - protocol: TransportProtocol::Tcp, - address: actual_addr, - }; - shard.broadcast_event_to_all_shards(event).await?; - } - } - - // Ensure rustls crypto provider is installed - if rustls::crypto::CryptoProvider::get_default().is_none() - && let Err(e) = rustls::crypto::ring::default_provider().install_default() - { - warn!( - "Failed to install rustls crypto provider: {:?}. This may be normal if another thread installed it first.", - e - ); - } else { - trace!("Rustls crypto provider installed or already present"); - } - - // Load or generate TLS certificates - let tls_config = &shard.config.tcp.tls; - let (certs, key) = - if tls_config.self_signed && !std::path::Path::new(&tls_config.cert_file).exists() { - info!("Generating self-signed certificate for TCP TLS server"); - generate_self_signed_cert() - .unwrap_or_else(|e| panic!("Failed to generate self-signed certificate: {e}")) - } else { - info!( - "Loading certificates from cert_file: {}, key_file: {}", - tls_config.cert_file, tls_config.key_file - ); - load_certificates(&tls_config.cert_file, &tls_config.key_file) - .unwrap_or_else(|e| panic!("Failed to load certificates: {e}")) - }; - - let server_config = ServerConfig::builder() - .with_no_client_auth() - .with_single_cert(certs, key) - .unwrap_or_else(|e| panic!("Unable to create TLS server config: {e}")); - - let acceptor = TlsAcceptor::from(Arc::new(server_config)); - - info!("{} server has started on: {:?}", server_name, actual_addr); - - accept_loop(server_name, listener, acceptor, shard, shutdown).await -} - -async fn create_listener( - addr: SocketAddr, - config: &TcpSocketConfig, -) -> Result { - crate::tcp::bind_reuseport_listener(addr, false, Some(config)).await -} - -async fn accept_loop( - server_name: &'static str, - listener: TcpListener, - acceptor: TlsAcceptor, - shard: Rc, - shutdown: ShutdownToken, -) -> Result<(), IggyError> { - loop { - let shard = shard.clone(); - let accept_future = listener.accept(); - futures::select! { - _ = shutdown.wait().fuse() => { - info!("{} received shutdown signal, no longer accepting connections", server_name); - break; - } - result = accept_future.fuse() => { - match result { - Ok((stream, address)) => { - if shard.is_shutting_down() { - info!("Rejecting new TLS connection from {} during shutdown", address); - continue; - } - info!("Accepted new TCP connection for TLS handshake: {}", address); - let shard_clone = shard.clone(); - let acceptor = acceptor.clone(); - - // Perform TLS handshake in a separate task to avoid blocking the accept loop - let registry = shard.task_registry.clone(); - let registry_clone = registry.clone(); - registry.spawn_connection(async move { - match acceptor.accept(stream).await { - Ok(tls_stream) => { - // TLS handshake successful, now create session - info!("TLS handshake successful, adding TCP client: {}", address); - let transport = TransportProtocol::Tcp; - let session = shard_clone.add_client(&address, transport); - info!("Added {} client with session: {} for IP address: {}", transport, session, address); - - let client_id = session.client_id; - info!("Created new session: {}", session); - - let conn_stop_receiver = registry_clone.add_connection(client_id); - let shard_for_conn = shard_clone.clone(); - let mut sender = SenderKind::get_tcp_tls_sender(tls_stream); - if let Err(error) = handle_connection(&session, &mut sender, &shard_for_conn, conn_stop_receiver).await { - handle_error(error); - } - shard_for_conn.delete_client(session.client_id).await; - registry_clone.remove_connection(&client_id); - - if let Err(error) = sender.shutdown().await { - error!("Failed to shutdown TCP TLS stream for client: {}, address: {}. {}", client_id, address, error); - } else { - info!("Successfully closed TCP TLS stream for client: {}, address: {}.", client_id, address); - } - } - Err(e) => { - error!("Failed to accept TLS connection from '{}': {}", address, e); - // No session was created, so no cleanup needed - } - } - }); - } - Err(error) => error!("Unable to accept TCP TLS socket. {}", error), - } - } - } - } - Ok(()) -} - -fn generate_self_signed_cert() --> Result<(Vec>, PrivateKeyDer<'static>), Box> { - server_common::generate_self_signed_certificate("localhost") -} - -fn load_certificates( - cert_file: &str, - key_file: &str, -) -> Result<(Vec>, PrivateKeyDer<'static>), Box> { - let cert_file = std::fs::File::open(cert_file)?; - let mut cert_reader = BufReader::new(cert_file); - let certs: Vec<_> = certs(&mut cert_reader).collect::, _>>()?; - - if certs.is_empty() { - return Err("No certificates found in certificate file".into()); - } - - let key_file = std::fs::File::open(key_file)?; - let mut key_reader = BufReader::new(key_file); - let key = private_key(&mut key_reader)?.ok_or("No private key found in key file")?; - - Ok((certs, key)) -} diff --git a/core/server-ng/src/users.rs b/core/server/src/users.rs similarity index 98% rename from core/server-ng/src/users.rs rename to core/server/src/users.rs index 94a5cb270c..802d092e34 100644 --- a/core/server-ng/src/users.rs +++ b/core/server/src/users.rs @@ -41,7 +41,7 @@ use bytes::Bytes; use consensus::MetadataHandle; use iggy_binary_protocol::codec::{WireDecode, WireEncode}; use iggy_binary_protocol::requests::users::{ChangePasswordRequest, CreateUserRequest}; -use iggy_binary_protocol::{Operation, PrepareHeader, RequestHeader}; +use iggy_binary_protocol::{Operation, PrepareHeader, RoutedRequestHeader}; use iggy_common::IggyError; use journal::superblock::SuperblockStore; use journal::{Journal, JournalHandle}; @@ -60,8 +60,8 @@ use std::rc::Rc; /// undecodable password body. pub(crate) fn maybe_rewrite_user_password_request( shard: &Rc>, - request: Message, -) -> Result, IggyError> + request: Message, +) -> Result, IggyError> where B: ShellBus, MJ: JournalHandle + 'static, diff --git a/core/server-ng/src/web.rs b/core/server/src/web.rs similarity index 100% rename from core/server-ng/src/web.rs rename to core/server/src/web.rs diff --git a/core/server/src/websocket/connection_handler.rs b/core/server/src/websocket/connection_handler.rs deleted file mode 100644 index 77ed24d7fc..0000000000 --- a/core/server/src/websocket/connection_handler.rs +++ /dev/null @@ -1,175 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::binary::dispatch::{self, HandlerResult, MAX_CONTROL_FRAME_PAYLOAD}; -use crate::sender::SenderKind; -use crate::server_error::ConnectionError; -use crate::shard::IggyShard; -use crate::streaming::session::Session; -use async_channel::Receiver; -use bytes::BytesMut; -use futures::FutureExt; -use iggy_binary_protocol::RequestFrame; -use iggy_binary_protocol::codes::{SEND_MESSAGES_CODE, command_name}; -use iggy_common::IggyError; -use std::io::ErrorKind; -use std::rc::Rc; -use tracing::{debug, error, info, warn}; - -pub(crate) async fn handle_connection( - session: &Session, - sender: &mut SenderKind, - shard: &Rc, - stop_receiver: Receiver<()>, -) -> Result<(), ConnectionError> { - let mut header_buffer = BytesMut::with_capacity(RequestFrame::HEADER_SIZE); - - loop { - let read_future = sender.read(header_buffer); - let (_, mut header_buf) = futures::select! { - _ = stop_receiver.recv().fuse() => { - info!("Connection stop signal received for session: {}", session); - let _ = sender.send_error_response(IggyError::Disconnected).await; - return Ok(()); - } - result = read_future.fuse() => { - match result { - (Ok(_), buf) => (Ok::<(), IggyError>(()), buf), - (Err(error), buf) => { - header_buffer = buf; - if error.as_code() == IggyError::ConnectionClosed.as_code() { - return Err(ConnectionError::from(error)); - } else { - error!("got error: {:?}", error); - sender.send_error_response(error).await?; - continue; - } - } - } - } - }; - - let length = u32::from_le_bytes(header_buf[0..4].try_into().unwrap()); - let code = u32::from_le_bytes(header_buf[4..8].try_into().unwrap()); - header_buf.clear(); - header_buffer = header_buf; - - let cmd_name = command_name(code).unwrap_or("unknown"); - debug!("Received a WebSocket request, length: {length}, code: {code} ({cmd_name})"); - - let payload_length = match RequestFrame::payload_length(length) { - Ok(len) => len, - Err(_) => { - sender - .send_error_response(IggyError::InvalidCommand) - .await?; - continue; - } - }; - - let result = if code == SEND_MESSAGES_CODE { - dispatch::dispatch_send_messages(sender, payload_length, session, shard).await - } else { - if payload_length > MAX_CONTROL_FRAME_PAYLOAD { - sender - .send_error_response(IggyError::InvalidCommand) - .await?; - continue; - } - let payload = dispatch::read_payload(sender, payload_length).await?; - let frame = RequestFrame::from_parts(code, &payload); - dispatch::dispatch(frame, sender, session, shard).await - }; - - match result { - Ok(HandlerResult::Finished) => { - debug!( - "Command {code} ({cmd_name}) was handled successfully, session: {session}. WebSocket response was sent." - ); - } - Ok(HandlerResult::Migrated { to_shard }) => { - warn!("Unexpected migration on WebSocket: to_shard {to_shard}, session: {session}"); - } - Err(error) => match error { - IggyError::TcpError | IggyError::ConnectionClosed | IggyError::Disconnected => { - warn!( - "Client {} closed connection during request processing", - session.client_id - ); - return Err(ConnectionError::from(IggyError::ConnectionClosed)); - } - IggyError::ClientNotFound(_) | IggyError::StaleClient => { - error!("Command failed for session: {session}, error: {error}."); - sender.send_error_response(error.clone()).await?; - return Err(ConnectionError::from(error)); - } - _ => { - error!("Command failed for session: {session}, error: {error}."); - match sender.send_error_response(error).await { - Ok(_) => { - debug!("WebSocket error response was sent to: {session}."); - } - Err(IggyError::ConnectionClosed) => { - warn!( - "Could not send error response to {} - client already disconnected", - session.client_id - ); - return Err(ConnectionError::from(IggyError::ConnectionClosed)); - } - Err(send_err) => { - error!("Failed to send error response: {send_err}"); - return Err(ConnectionError::from(send_err)); - } - } - } - }, - } - } -} - -pub(crate) fn handle_error(error: ConnectionError) { - match error { - ConnectionError::IoError(e) => match e.kind() { - ErrorKind::UnexpectedEof => { - info!("WebSocket connection has been closed."); - } - ErrorKind::ConnectionAborted => { - info!("WebSocket connection has been aborted."); - } - ErrorKind::ConnectionRefused => { - info!("WebSocket connection has been refused."); - } - ErrorKind::ConnectionReset => { - info!("WebSocket connection has been reset."); - } - _ => { - error!("WebSocket connection has failed: {e}"); - } - }, - ConnectionError::SdkError(sdk_error) => match sdk_error { - IggyError::ConnectionClosed => { - debug!("Client closed WebSocket connection."); - } - _ => { - error!("Failure in internal SDK call: {sdk_error}"); - } - }, - _ => { - error!("WebSocket connection has failed: {error}"); - } - } -} diff --git a/core/server/src/websocket/mod.rs b/core/server/src/websocket/mod.rs deleted file mode 100644 index 435648fe81..0000000000 --- a/core/server/src/websocket/mod.rs +++ /dev/null @@ -1,64 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -pub mod connection_handler; -pub mod websocket_listener; -pub mod websocket_server; -pub mod websocket_tls_listener; - -pub const COMPONENT: &str = "WEBSOCKET"; - -use crate::configs::websocket::WebSocketConfig; -use compio::ws::tungstenite::protocol::WebSocketConfig as CompioWsConfig; -use iggy_common::IggyByteSize; - -/// Build a `compio-ws`-compatible `WebSocketConfig` from the server config. -/// -/// The standalone `tungstenite` crate may be a different major version than -/// the one re-exported by `compio-ws`, so we construct the config through -/// `compio::ws::tungstenite` to guarantee type compatibility. -pub fn build_compio_ws_config(config: &WebSocketConfig) -> CompioWsConfig { - let mut ws = CompioWsConfig::default(); - - if let Some(ref s) = config.read_buffer_size - && let Ok(b) = s.parse::() - { - ws = ws.read_buffer_size(b.as_bytes_u64() as usize); - } - if let Some(ref s) = config.write_buffer_size - && let Ok(b) = s.parse::() - { - ws = ws.write_buffer_size(b.as_bytes_u64() as usize); - } - if let Some(ref s) = config.max_write_buffer_size - && let Ok(b) = s.parse::() - { - ws = ws.max_write_buffer_size(b.as_bytes_u64() as usize); - } - if let Some(ref s) = config.max_message_size - && let Ok(b) = s.parse::() - { - ws = ws.max_message_size(Some(b.as_bytes_u64() as usize)); - } - if let Some(ref s) = config.max_frame_size - && let Ok(b) = s.parse::() - { - ws = ws.max_frame_size(Some(b.as_bytes_u64() as usize)); - } - ws = ws.accept_unmasked_frames(config.accept_unmasked_frames); - ws -} diff --git a/core/server/src/websocket/websocket_listener.rs b/core/server/src/websocket/websocket_listener.rs deleted file mode 100644 index 0f8512b154..0000000000 --- a/core/server/src/websocket/websocket_listener.rs +++ /dev/null @@ -1,182 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::configs::websocket::WebSocketConfig; -use crate::sender::{SenderKind, WebSocketSender}; -use crate::shard::IggyShard; -use crate::shard::task_registry::ShutdownToken; -use crate::shard::transmission::event::ShardEvent; -use crate::websocket::connection_handler::{handle_connection, handle_error}; -use compio::net::TcpListener; -use compio::ws::accept_async_with_config; -use err_trail::ErrContext; -use futures::FutureExt; -use iggy_common::IggyError; -use iggy_common::TransportProtocol; -use std::net::SocketAddr; -use std::rc::Rc; -use tracing::{debug, error, info}; - -async fn create_listener(addr: SocketAddr) -> Result { - // Thread-per-core: many sockets bind the same addr+port (SO_REUSEPORT). - crate::tcp::bind_reuseport_listener(addr, true, None).await -} - -pub async fn start( - config: WebSocketConfig, - shard: Rc, - shutdown: ShutdownToken, -) -> Result<(), IggyError> { - let mut addr: SocketAddr = config - .address - .parse() - .error(|e: &std::net::AddrParseError| { - format!( - "WebSocket (error: {e}) - failed to parse address: {}", - config.address - ) - }) - .map_err(|_| IggyError::InvalidConfiguration)?; - - if shard.id != 0 && addr.port() == 0 { - info!("Waiting for WebSocket address from shard 0..."); - loop { - if let Some(bound_addr) = shard.websocket_bound_address.get() { - addr = bound_addr; - info!("Received WebSocket address from shard 0: {}", addr); - break; - } - // Small delay to prevent busy waiting - compio::time::sleep(std::time::Duration::from_millis(10)).await; - } - } - - let listener = create_listener(addr) - .await - .error(|e: &std::io::Error| { - format!("WebSocket (error: {e}) - failed to bind to address: {addr}") - }) - .map_err(|_| IggyError::CannotBindToSocket(addr.to_string()))?; - - let local_addr = listener.local_addr().unwrap(); - info!("{} has started on: ws://{}", "WebSocket Server", local_addr); - - // Notify shard about the bound address - let event = ShardEvent::AddressBound { - protocol: TransportProtocol::WebSocket, - address: local_addr, - }; - - if shard.id == 0 { - // Store bound address locally first - shard.websocket_bound_address.set(Some(local_addr)); - - if addr.port() == 0 { - // Broadcast to other shards for SO_REUSEPORT binding - shard.broadcast_event_to_all_shards(event).await?; - } - } else { - // Non-shard0 just handles the event locally - crate::shard::handlers::handle_event(&shard, event) - .await - .ok(); - } - - let ws_config = super::build_compio_ws_config(&config); - info!( - "WebSocket config: max_message_size: {:?}, max_frame_size: {:?}, accept_unmasked_frames: {}", - config.max_message_size, config.max_frame_size, config.accept_unmasked_frames - ); - - accept_loop(listener, Some(ws_config), shard, shutdown).await -} - -async fn accept_loop( - listener: TcpListener, - ws_config: Option, - shard: Rc, - shutdown: ShutdownToken, -) -> Result<(), IggyError> { - loop { - let shard = shard.clone(); - let accept_future = listener.accept(); - - futures::select! { - _ = shutdown.wait().fuse() => { - debug!("WebSocket Server received shutdown signal, no longer accepting connections"); - break; - } - result = accept_future.fuse() => { - match result { - Ok((tcp_stream, remote_addr)) => { - if shard.is_shutting_down() { - info!("Rejecting new WebSocket connection from {} during shutdown", remote_addr); - continue; - } - info!("Accepted new WebSocket connection from: {}", remote_addr); - - let shard_clone = shard.clone(); - let ws_config_clone = ws_config; - let registry = shard.task_registry.clone(); - let registry_clone = registry.clone(); - - registry.spawn_connection(async move { - match accept_async_with_config(tcp_stream, ws_config_clone).await { - Ok(websocket) => { - info!("WebSocket handshake successful from: {}", remote_addr); - - let session = shard_clone.add_client(&remote_addr, TransportProtocol::WebSocket); - let client_id = session.client_id; - - let sender = WebSocketSender::new(websocket); - let mut sender_kind = SenderKind::get_websocket_sender(sender); - let client_stop_receiver = registry_clone.add_connection(client_id); - - if let Err(error) = handle_connection(&session, &mut sender_kind, &shard_clone, client_stop_receiver).await { - handle_error(error); - } - shard_clone.delete_client(session.client_id).await; - registry_clone.remove_connection(&client_id); - - match sender_kind.shutdown().await { - Ok(_) => { - info!("Successfully closed WebSocket stream for client: {}, address: {}.", client_id, remote_addr); - } - Err(_) => { - // shutdown failures during client disconnect are expected and normal - // real errors would have been caught earlier in handle_connection - debug!("WebSocket shutdown completed with error for client: {} (likely client already disconnected)", client_id); - } - } - } - Err(error) => { - error!("WebSocket handshake failed from {}: {:?}", remote_addr, error); - } - } - }); - } - Err(error) => { - error!("Failed to accept WebSocket connection: {}", error); - } - } - } - } - } - - info!("WebSocket Server listener has stopped"); - Ok(()) -} diff --git a/core/server/src/websocket/websocket_server.rs b/core/server/src/websocket/websocket_server.rs deleted file mode 100644 index 9ce40cc3a4..0000000000 --- a/core/server/src/websocket/websocket_server.rs +++ /dev/null @@ -1,59 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::shard::IggyShard; -use crate::shard::task_registry::ShutdownToken; -use crate::websocket::websocket_listener; -use crate::websocket::websocket_tls_listener; -use iggy_common::IggyError; -use std::rc::Rc; -use tracing::{error, info}; - -pub async fn spawn_websocket_server( - shard: Rc, - shutdown: ShutdownToken, -) -> Result<(), IggyError> { - let config = shard.config.websocket.clone(); - - if !config.enabled { - info!("WebSocket server is disabled."); - return Ok(()); - } - - let server_name = if config.tls.enabled { - "WebSocket TLS" - } else { - "WebSocket" - }; - - info!( - "Starting {} server on: {} for shard: {}...", - server_name, config.address, shard.id - ); - - let result = match config.tls.enabled { - true => websocket_tls_listener::start(config, shard.clone(), shutdown).await, - false => websocket_listener::start(config, shard.clone(), shutdown).await, - }; - - if let Err(error) = result { - error!("{} server has failed to start, error: {error}", server_name); - return Err(error); - } - - Ok(()) -} diff --git a/core/server/src/websocket/websocket_tls_listener.rs b/core/server/src/websocket/websocket_tls_listener.rs deleted file mode 100644 index ea8787d9ed..0000000000 --- a/core/server/src/websocket/websocket_tls_listener.rs +++ /dev/null @@ -1,272 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::configs::websocket::WebSocketConfig; -use crate::sender::{SenderKind, WebSocketTlsSender}; -use crate::shard::IggyShard; -use crate::shard::task_registry::ShutdownToken; -use crate::shard::transmission::event::ShardEvent; -use crate::websocket::connection_handler::{handle_connection, handle_error}; -use compio::net::TcpListener; -use compio::tls::{MaybeTlsStream, TlsAcceptor}; -use compio::ws::accept_async_with_config; -use err_trail::ErrContext; -use futures::FutureExt; -use iggy_common::{IggyError, TransportProtocol}; -use rustls::ServerConfig; -use rustls::pki_types::{CertificateDer, PrivateKeyDer}; -use rustls_pemfile::{certs, private_key}; -use std::io::BufReader; -use std::net::SocketAddr; -use std::rc::Rc; -use std::sync::Arc; -use tracing::{debug, error, info, trace, warn}; - -async fn create_listener(addr: SocketAddr) -> Result { - // Thread-per-core: many sockets bind the same addr+port (SO_REUSEPORT). - crate::tcp::bind_reuseport_listener(addr, true, None).await -} - -pub async fn start( - config: WebSocketConfig, - shard: Rc, - shutdown: ShutdownToken, -) -> Result<(), IggyError> { - let mut addr: SocketAddr = config - .address - .parse() - .error(|e: &std::net::AddrParseError| { - format!( - "WebSocket TLS (error: {e}) - failed to parse address: {}", - config.address - ) - }) - .map_err(|_| IggyError::InvalidConfiguration)?; - - if shard.id != 0 && addr.port() == 0 { - info!("Waiting for WebSocket TLS address from shard 0..."); - loop { - if let Some(bound_addr) = shard.websocket_bound_address.get() { - addr = bound_addr; - info!("Received WebSocket TLS address from shard 0: {}", addr); - break; - } - compio::time::sleep(std::time::Duration::from_millis(10)).await; - } - } - - let listener = create_listener(addr) - .await - .error(|e: &std::io::Error| { - format!("WebSocket TLS (error: {e}) - failed to bind to address: {addr}") - }) - .map_err(|_| IggyError::CannotBindToSocket(addr.to_string()))?; - - let local_addr = listener.local_addr().unwrap(); - - // Notify shard about the bound address - let event = ShardEvent::AddressBound { - protocol: TransportProtocol::WebSocket, - address: local_addr, - }; - - if shard.id == 0 { - // Store bound address locally first - shard.websocket_bound_address.set(Some(local_addr)); - - if addr.port() == 0 { - // Broadcast to other shards for SO_REUSEPORT binding - shard.broadcast_event_to_all_shards(event).await?; - } - } else { - // Non-shard0 just handles the event locally - crate::shard::handlers::handle_event(&shard, event) - .await - .ok(); - } - - // Ensure rustls crypto provider is installed - if rustls::crypto::CryptoProvider::get_default().is_none() - && let Err(e) = rustls::crypto::ring::default_provider().install_default() - { - warn!( - "Failed to install rustls crypto provider: {:?}. This may be normal if another thread installed it first.", - e - ); - } else { - trace!("Rustls crypto provider installed or already present"); - } - - // Load or generate TLS certificates - let tls_config = &shard.config.websocket.tls; - let (certs, key) = - if tls_config.self_signed && !std::path::Path::new(&tls_config.cert_file).exists() { - info!("Generating self-signed certificate for WebSocket TLS server"); - generate_self_signed_cert() - .unwrap_or_else(|e| panic!("Failed to generate self-signed certificate: {e}")) - } else { - info!( - "Loading certificates from cert_file: {}, key_file: {}", - tls_config.cert_file, tls_config.key_file - ); - load_certificates(&tls_config.cert_file, &tls_config.key_file) - .unwrap_or_else(|e| panic!("Failed to load certificates: {e}")) - }; - - let server_config = ServerConfig::builder() - .with_no_client_auth() - .with_single_cert(certs, key) - .unwrap_or_else(|e| panic!("Unable to create TLS server config: {e}")); - - let acceptor = TlsAcceptor::from(Arc::new(server_config)); - - info!( - "{} has started on: wss://{}", - "WebSocket TLS Server", local_addr - ); - let ws_config = super::build_compio_ws_config(&config); - info!( - "WebSocket TLS config: max_message_size: {:?}, max_frame_size: {:?}, accept_unmasked_frames: {}", - config.max_message_size, config.max_frame_size, config.accept_unmasked_frames - ); - - let result = accept_loop(listener, acceptor, ws_config, shard.clone(), shutdown).await; - - info!( - "WebSocket TLS listener task exiting with result: {:?}", - result - ); - - result -} - -async fn accept_loop( - listener: TcpListener, - acceptor: TlsAcceptor, - ws_config: compio::ws::tungstenite::protocol::WebSocketConfig, - shard: Rc, - shutdown: ShutdownToken, -) -> Result<(), IggyError> { - info!("WebSocket TLS accept loop started, waiting for connections..."); - - loop { - let shard = shard.clone(); - let acceptor = acceptor.clone(); - let accept_future = listener.accept(); - - futures::select! { - _ = shutdown.wait().fuse() => { - debug!("WebSocket TLS Server received shutdown signal, no longer accepting connections"); - break; - } - result = accept_future.fuse() => { - match result { - Ok((tcp_stream, remote_addr)) => { - if shard.is_shutting_down() { - info!("Rejecting new WebSocket TLS connection from {} during shutdown", remote_addr); - continue; - } - info!("Accepted new TCP connection for WebSocket TLS handshake from: {}", remote_addr); - - let shard_clone = shard.clone(); - let ws_config_clone = ws_config; - let registry = shard.task_registry.clone(); - let registry_clone = registry.clone(); - - registry.spawn_connection(async move { - match acceptor.accept(tcp_stream).await { - Ok(tls_stream) => { - info!("TLS handshake successful for {}, performing WebSocket upgrade...", remote_addr); - - // compio-ws 0.4 drives TLS via `MaybeTlsStream` - // (`TlsStream` is not `Splittable`); wrap the handshaked - // stream so the WS layer owns it. - let tls_stream = MaybeTlsStream::new_tls(tls_stream); - match accept_async_with_config(tls_stream, Some(ws_config_clone)).await { - Ok(websocket) => { - info!("WebSocket TLS handshake successful from: {}", remote_addr); - - let session = shard_clone.add_client(&remote_addr, TransportProtocol::WebSocket); - let client_id = session.client_id; - - let sender = WebSocketTlsSender::new(websocket); - let mut sender_kind = SenderKind::WebSocketTls(sender); - let client_stop_receiver = registry_clone.add_connection(client_id); - - if let Err(error) = handle_connection(&session, &mut sender_kind, &shard_clone, client_stop_receiver).await { - handle_error(error); - } - shard_clone.delete_client(session.client_id).await; - registry_clone.remove_connection(&client_id); - - match sender_kind.shutdown().await { - Ok(_) => { - info!("Successfully closed WebSocket TLS stream for client: {}, address: {}.", client_id, remote_addr); - } - Err(_) => { - // shutdown failures during client disconnect are expected and normal - // real errors would have been caught earlier in handle_connection - debug!("WebSocket TLS shutdown completed with error for client: {} (likely client already disconnected)", client_id); - } - } - } - Err(error) => { - error!("WebSocket handshake failed on TLS connection from {}: {:?}", remote_addr, error); - } - } - } - Err(error) => { - error!("TLS handshake failed from {}: {:?}", remote_addr, error); - } - } - }); - } - Err(error) => { - error!("Failed to accept WebSocket TLS connection: {}", error); - } - } - } - } - } - - info!("WebSocket TLS Server listener has stopped"); - Ok(()) -} - -fn generate_self_signed_cert() --> Result<(Vec>, PrivateKeyDer<'static>), Box> { - server_common::generate_self_signed_certificate("localhost") -} - -fn load_certificates( - cert_file: &str, - key_file: &str, -) -> Result<(Vec>, PrivateKeyDer<'static>), Box> { - let cert_file = std::fs::File::open(cert_file)?; - let mut cert_reader = BufReader::new(cert_file); - let certs: Vec<_> = certs(&mut cert_reader).collect::, _>>()?; - - if certs.is_empty() { - return Err("No certificates found in certificate file".into()); - } - - let key_file = std::fs::File::open(key_file)?; - let mut key_reader = BufReader::new(key_file); - let key = private_key(&mut key_reader)?.ok_or("No private key found in key file")?; - - Ok((certs, key)) -} diff --git a/core/server/src/wire.rs b/core/server/src/wire.rs new file mode 100644 index 0000000000..02e21a5a3a --- /dev/null +++ b/core/server/src/wire.rs @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Leaf wire helpers shared by the request-handling modules. +//! +//! Request-body slicing, the `usize -> u32` wire conversion, and the +//! transport-kind discriminant mapping. + +use bytes::Bytes; +use iggy_binary_protocol::RoutedRequestHeader; +use iggy_common::IggyError; +use message_bus::installer::conn_info::ClientTransportKind; +use server_common::Message; + +pub(crate) fn request_body(request: &Message) -> &[u8] { + &request.as_slice()[std::mem::size_of::()..request.header().size as usize] +} + +/// Check a client's `request_checksum` against the body it stamps. +/// +/// Must run BEFORE any body rewrite: PAT / password / consumer-group paths +/// substitute server-chosen bytes. Zero is "unstamped" and skips the check, so an +/// SDK predating the stamp still works. +/// +/// # Errors +/// [`IggyError::InvalidFormat`] when the stamp disagrees with the body. +pub(crate) fn verify_request_checksum( + request: &Message, +) -> Result<(), IggyError> { + let stamped = request.header().request_checksum; + if stamped == 0 || u128::from(iggy_common::calculate_checksum(request_body(request))) == stamped + { + return Ok(()); + } + Err(IggyError::InvalidFormat) +} + +/// Map the transport kind to the legacy wire discriminant +/// (`1=TCP, 2=QUIC, 4=WebSocket`); TLS variants report their base +/// transport. `ClientTransportKind` is `#[non_exhaustive]`, so any other +/// (TCP, TCP-TLS, or a future) variant falls back to TCP. +pub(crate) const fn transport_kind_to_wire(kind: ClientTransportKind) -> u8 { + match kind { + ClientTransportKind::Quic => 2, + ClientTransportKind::Ws | ClientTransportKind::Wss => 4, + _ => 1, + } +} + +pub(crate) fn usize_to_u32(value: usize) -> Result { + u32::try_from(value).map_err(|_| IggyError::InvalidIdentifier) +} + +/// Rebuild a request message with `body` replacing the original payload, +/// preserving the header (and fixing `size`). Used by the primary-side +/// request rewrites that swap a secret-bearing wire body for the +/// hash-carrying replicated body before consensus. +pub(crate) fn rewrite_request_body( + request: &Message, + body: &Bytes, +) -> Result, IggyError> { + let total_size = std::mem::size_of::() + .checked_add(body.len()) + .ok_or(IggyError::InvalidConfiguration)?; + let size = u32::try_from(total_size).map_err(|_| IggyError::InvalidConfiguration)?; + let mut rewritten = Message::::new(total_size); + let header = bytemuck::checked::try_from_bytes_mut::( + &mut rewritten.as_mut_slice()[..std::mem::size_of::()], + ) + .expect("zeroed bytes are a valid request header"); + *header = *request.header(); + header.size = size; + // Both describe the body just replaced, and nothing recomputes them for a + // `RoutedRequestHeader` -- the prepare projection derives its own `checksum_body` + // downstream. Clear rather than recompute; carrying them forward is a stale claim. + header.checksum = 0; + header.checksum_body = 0; + // `request_checksum` is deliberately NOT touched: it stamps what the CLIENT sent, + // already validated at admission. Re-stamping it over the substituted body would + // make the client-table reuse check compare a value no client ever produced. + rewritten.as_mut_slice()[std::mem::size_of::()..].copy_from_slice(body); + Ok(rewritten) +} + +#[cfg(test)] +mod tests { + use super::{request_body, rewrite_request_body}; + use bytes::Bytes; + use iggy_binary_protocol::{Command2, Operation, RoutedRequestHeader}; + use server_common::Message; + use std::mem::size_of; + + fn request(body: &[u8], request_checksum: u128) -> Message { + let total_size = size_of::() + body.len(); + let mut message = Message::::new(total_size).transmute_header( + |_, header: &mut RoutedRequestHeader| { + header.command = Command2::Request; + header.operation = Operation::CreateStream; + header.client = 1; + header.session = 1; + header.request = 9; + header.size = u32::try_from(total_size).expect("fits u32"); + header.request_checksum = request_checksum; + header.checksum = 0xdead; + header.checksum_body = 0xbeef; + }, + ); + message.as_mut_slice()[size_of::()..].copy_from_slice(body); + message + } + + #[test] + fn given_a_body_rewrite_should_keep_the_client_stamp_and_clear_the_stale_seals() { + // The secret-bearing wire body is swapped for the hash-carrying replicated + // one. `request_checksum` describes what the client sent and admission has + // already checked it, so it must survive; the other two describe the body + // that just went away. + let original = request(b"plaintext-secret", 0x1234); + let rewritten = rewrite_request_body(&original, &Bytes::from_static(b"argon2-hash")) + .expect("the rewritten body fits a request message"); + + assert_eq!( + rewritten.header().request_checksum, + 0x1234, + "the client's stamp must not be re-signed over server-substituted bytes" + ); + assert_eq!(rewritten.header().checksum, 0); + assert_eq!(rewritten.header().checksum_body, 0); + assert_eq!(request_body(&rewritten), b"argon2-hash"); + assert_eq!( + rewritten.header().size as usize, + size_of::() + b"argon2-hash".len(), + "`size` follows the new body, so `request_body` bounds it correctly" + ); + } + + #[test] + fn given_an_unstamped_request_when_rewriting_should_stay_unstamped() { + // Zero means "unstamped" all the way through the client table, so a rewrite + // must not manufacture a stamp for a client that sent none. + let original = request(b"plaintext-secret", 0); + let rewritten = rewrite_request_body(&original, &Bytes::from_static(b"argon2-hash")) + .expect("the rewritten body fits a request message"); + + assert_eq!(rewritten.header().request_checksum, 0); + } +} diff --git a/core/server-ng/tests/sdk_e2e.rs b/core/server/tests/sdk_e2e.rs similarity index 91% rename from core/server-ng/tests/sdk_e2e.rs rename to core/server/tests/sdk_e2e.rs index 2cbd5d64cb..0e52e310e8 100644 --- a/core/server-ng/tests/sdk_e2e.rs +++ b/core/server/tests/sdk_e2e.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -//! Process-level boot smoke test for the `iggy-server-ng` binary. +//! Process-level boot smoke test for the `iggy-server` binary. //! //! Spawns the production binary against an isolated tempdir with every //! transport except TCP disabled and `IGGY_TCP_ADDRESS` bound to port 0, @@ -33,7 +33,7 @@ use std::time::{Duration, Instant}; use tempfile::TempDir; use toml::Value; -/// `iggy-server-ng` bootstrap can take ~5s on cold caches before the TCP +/// `iggy-server` bootstrap can take ~5s on cold caches before the TCP /// listener binds and `current_config.toml` is written; 30s is generous /// enough for slow CI runners without hanging the suite indefinitely. const STARTUP_TIMEOUT: Duration = Duration::from_secs(30); @@ -54,8 +54,8 @@ struct TestServer { impl TestServer { fn start() -> Self { let data_dir = TempDir::new().expect("tempdir for system.path"); - let mut cmd = Command::cargo_bin("iggy-server-ng") - .expect("iggy-server-ng binary must be built by the test runner"); + let mut cmd = Command::cargo_bin("iggy-server") + .expect("iggy-server binary must be built by the test runner"); cmd.env("IGGY_SYSTEM_PATH", data_dir.path()) // Ephemeral port; the actual bound port is read back from // `runtime/current_config.toml` after the listener binds. @@ -71,7 +71,7 @@ impl TestServer { let mut child = cmd .spawn() - .expect("iggy-server-ng must spawn under cargo test"); + .expect("iggy-server must spawn under cargo test"); let runtime_path = data_dir.path().join("runtime").join("current_config.toml"); let tcp_addr = match wait_for_bound_tcp(&runtime_path, STARTUP_TIMEOUT) { @@ -82,7 +82,7 @@ impl TestServer { // when `data_dir` goes out of scope at the panic. let _ = child.kill(); let _ = child.wait(); - panic!("server-ng startup failed: {err}"); + panic!("server startup failed: {err}"); } }; @@ -158,7 +158,7 @@ fn read_bound_tcp(config_path: &Path) -> Result { Ok(addr) } -/// Spawn `iggy-server-ng` and verify the production binary bootstraps +/// Spawn `iggy-server` and verify the production binary bootstraps /// cleanly. /// /// Specifically asserts: @@ -168,7 +168,7 @@ fn read_bound_tcp(config_path: &Path) -> Result { /// `current_config.toml`). /// * The bound port is `connect()`-able from this test process. #[tokio::test(flavor = "current_thread")] -async fn server_ng_bootstraps_and_binds_ephemeral_tcp_port() { +async fn server_bootstraps_and_binds_ephemeral_tcp_port() { let server = TestServer::start(); assert_ne!( server.tcp_addr.port(), @@ -180,5 +180,5 @@ async fn server_ng_bootstraps_and_binds_ephemeral_tcp_port() { // `connect()`-able once; doing it again here is a sanity check that // the listener stays bound for the lifetime of the test harness. let _ = StdTcpStream::connect_timeout(&server.tcp_addr, Duration::from_secs(1)) - .expect("server-ng TCP listener must accept connections post-bootstrap"); + .expect("server TCP listener must accept connections post-bootstrap"); } diff --git a/core/server_common/src/consensus_message.rs b/core/server_common/src/consensus_message.rs index 4fe65638be..66f973a999 100644 --- a/core/server_common/src/consensus_message.rs +++ b/core/server_common/src/consensus_message.rs @@ -20,11 +20,14 @@ use iggy_binary_protocol::{ Command2, CommitHeader, ConsensusError, ConsensusHeader, DoViewChangeHeader, GenericHeader, Operation, PrepareHeader, PrepareOkHeader, RepairPrepareHeader, RepairRangeReplyHeader, RequestHeader, RequestPreparesHeader, RequestStartViewHeader, RequestStateChunkHeader, - RequestStateTransferHeader, StartViewChangeHeader, StartViewHeader, StateChunkHeader, - StateTransferTargetHeader, + RequestStateTransferHeader, RoutedRequestHeader, StartViewChangeHeader, StartViewHeader, + StateChunkHeader, StateTransferTargetHeader, }; use smallvec::SmallVec; -use std::{marker::PhantomData, mem::size_of}; +use std::{ + marker::PhantomData, + mem::{offset_of, size_of}, +}; pub const MESSAGE_ALIGN: usize = 4096; @@ -237,6 +240,9 @@ where let bytes = >::header_storage(&self.backing); let typed = bytemuck::checked::try_from_bytes::(&bytes[..size_of::()]) .map_err(|_| ConsensusError::InvalidBitPattern)?; + // Before `validate`: a header that did not survive the link intact cannot + // have any of its fields believed, and `validate` reads them. + typed.verify_frame()?; typed.validate()?; Ok(Message { @@ -273,6 +279,9 @@ where let bytes = >::header_storage(&self.backing); let typed = bytemuck::checked::try_from_bytes::(&bytes[..size_of::()]) .map_err(|_| ConsensusError::InvalidBitPattern)?; + // Before `validate`: a header that did not survive the link intact cannot + // have any of its fields believed, and `validate` reads them. + typed.verify_frame()?; typed.validate()?; let typed_message = unsafe { &*std::ptr::from_ref(self).cast::>() }; @@ -372,6 +381,31 @@ where } } +impl Message { + /// Retype the client-wire request into the server-internal + /// [`RoutedRequestHeader`] shape in place, with `group` starting unset. + /// + /// The two layouts share every field offset (const-asserted where they + /// are declared) and `group` claims the client header's reserved tail, + /// so the promotion zeroes those eight bytes instead of rebuilding the + /// whole 256-byte header. This is the only sanctioned crossing between + /// the two layouts: transmute-based reads across them would alias + /// `group` with reserved bytes a client may have sent nonzero. + /// + /// # Panics + /// + /// Panics if the retyped header fails [`RoutedRequestHeader`] validation; + /// unreachable when `self` already passed [`RequestHeader`] validation, + /// which enforces the same field rules. + #[must_use] + pub fn into_routed(self) -> Message { + let group_offset = offset_of!(RoutedRequestHeader, group); + let mut owned = self.into_owned(); + owned.as_mut_slice()[group_offset..group_offset + size_of::()].fill(0); + Message::try_from(owned).expect("retyped request message must stay valid") + } +} + impl Message where H: ConsensusHeader, @@ -495,7 +529,7 @@ where #[derive(Debug)] pub enum MessageBag { - Request(Message), + Request(Message), Prepare(Message), PrepareOk(Message), StartViewChange(Message), @@ -600,7 +634,9 @@ where match command { Command2::Prepare => Ok(Self::Prepare(value.try_into_typed::()?)), - Command2::Request => Ok(Self::Request(value.try_into_typed::()?)), + Command2::Request => Ok(Self::Request( + value.try_into_typed::()?, + )), Command2::PrepareOk => Ok(Self::PrepareOk(value.try_into_typed::()?)), Command2::StartViewChange => Ok(Self::StartViewChange( value.try_into_typed::()?, @@ -654,17 +690,19 @@ where #[cfg(test)] mod tests { use super::*; - use iggy_binary_protocol::{Operation, ReplyHeader}; + use iggy_binary_protocol::{ + HEADER_SIZE, Operation, ReplyHeader, RequestHeader, frame_checksum_bytes, + }; use smallvec::smallvec; // Field offsets via `offset_of!`: a field reorder fails to compile here // rather than silently corrupting test bytes. - const SIZE_OFF: usize = std::mem::offset_of!(RequestHeader, size); - const COMMAND_OFF: usize = std::mem::offset_of!(RequestHeader, command); - const REQUEST_CLIENT_OFF: usize = std::mem::offset_of!(RequestHeader, client); - const REQUEST_OPERATION_OFF: usize = std::mem::offset_of!(RequestHeader, operation); - const REQUEST_SESSION_OFF: usize = std::mem::offset_of!(RequestHeader, session); - const REQUEST_REQUEST_OFF: usize = std::mem::offset_of!(RequestHeader, request); + const SIZE_OFF: usize = std::mem::offset_of!(RoutedRequestHeader, size); + const COMMAND_OFF: usize = std::mem::offset_of!(RoutedRequestHeader, command); + const REQUEST_CLIENT_OFF: usize = std::mem::offset_of!(RoutedRequestHeader, client); + const REQUEST_OPERATION_OFF: usize = std::mem::offset_of!(RoutedRequestHeader, operation); + const REQUEST_SESSION_OFF: usize = std::mem::offset_of!(RoutedRequestHeader, session); + const REQUEST_REQUEST_OFF: usize = std::mem::offset_of!(RoutedRequestHeader, request); fn header_bytes(command: Command2, size: u32) -> Owned { header_bytes_sized(command, size, 256) @@ -684,10 +722,68 @@ mod tests { // `Register` needs session 0 and request 0, which zeroed bytes // already satisfy. buf[REQUEST_OPERATION_OFF] = Operation::Register as u8; + seal_header_bytes(buf); } o } + /// Seal a hand-built frame the way a real sender does. + /// + /// Control headers are rejected on the typed parse unless `checksum` covers the + /// rest of the header, so a fixture that skips this tests the rejection path. + fn seal_header_bytes(buf: &mut [u8]) { + let header: &[u8; HEADER_SIZE] = buf[..HEADER_SIZE].try_into().expect("frame is a header"); + let checksum = frame_checksum_bytes(header); + buf[..size_of::()].copy_from_slice(&checksum.to_le_bytes()); + } + + /// A `DoViewChange` frame carrying a one-entry suffix, sealed. + /// + /// One entry rather than none because a bitset bit is only legal within the + /// suffix, so an empty frame cannot express the attack this seals against. + fn sealed_do_view_change() -> Owned { + const DVC_SIZE: usize = HEADER_SIZE * 2; + let mut owned = Owned::::zeroed(DVC_SIZE); + { + let buf = owned.as_mut_slice(); + buf[SIZE_OFF..SIZE_OFF + 4].copy_from_slice(&(DVC_SIZE as u32).to_le_bytes()); + buf[COMMAND_OFF] = Command2::DoViewChange as u8; + seal_header_bytes(buf); + } + owned + } + + #[test] + fn given_a_sealed_do_view_change_when_dispatching_should_accept() { + let generic = Message::::try_from(sealed_do_view_change()) + .expect("a sealed DoViewChange frames correctly"); + assert!(matches!( + MessageBag::try_from(generic), + Ok(MessageBag::DoViewChange(_)) + )); + } + + #[test] + fn given_a_flipped_nack_bit_when_dispatching_should_reject_the_frame() { + // Why the header seal exists. `validate` accepts this frame: the bit sits + // inside the one-entry suffix, where a legitimate nack lives. Downstream the + // bitset goes to the merge unchanged and authorises truncating a committed op. + const NACK_OFF: usize = std::mem::offset_of!(DoViewChangeHeader, nack_bitset); + + let mut owned = sealed_do_view_change(); + owned.as_mut_slice()[NACK_OFF] ^= 0x01; + + let generic = + Message::::try_from(owned).expect("framing does not inspect the bitset"); + assert!( + matches!( + MessageBag::try_from(generic), + Err(ConsensusError::FrameChecksumMismatch { .. }) + ), + "a manufactured nack must not reach the merge" + ); + } + // MessageBag round-trip for the probe + repair command family. Locks // RangeEvicted delivery in particular: RepairDone and RangeEvicted share // one header layout and BOTH must survive the typed parse -- a strict @@ -712,6 +808,8 @@ mod tests { let buf = owned.as_mut_slice(); buf[FROM_OP_OFF..FROM_OP_OFF + 8].copy_from_slice(&1u64.to_le_bytes()); buf[TO_OP_OFF..TO_OP_OFF + 8].copy_from_slice(&1u64.to_le_bytes()); + // Re-seal: the range was written after `header_bytes` sealed. + seal_header_bytes(buf); } let generic = Message::::try_from(owned) .unwrap_or_else(|e| panic!("{command:?} failed generic framing: {e}")); @@ -737,7 +835,7 @@ mod tests { #[test] #[should_panic(expected = "size must be at least header size")] fn message_new_smaller_than_header_panics() { - let _ = Message::::new(100); + let _ = Message::::new(100); } // try_from(Owned): validation gates the unsafe construction @@ -745,7 +843,7 @@ mod tests { #[test] fn try_from_owned_too_short_returns_err() { let owned = Owned::::zeroed(100); - let result = Message::::try_from(owned); + let result = Message::::try_from(owned); assert!(matches!(result, Err(ConsensusError::InvalidCommand { .. }))); } @@ -753,13 +851,13 @@ mod tests { fn try_from_owned_invalid_bit_pattern_returns_err() { let mut owned = Owned::::zeroed(256); owned.as_mut_slice()[COMMAND_OFF] = 99; // outside Command2's discriminant range - let result = Message::::try_from(owned); + let result = Message::::try_from(owned); assert!(matches!(result, Err(ConsensusError::InvalidBitPattern))); } #[test] fn try_from_owned_buffer_shorter_than_claimed_size_returns_err() { - // Header parses cleanly (RequestHeader::validate doesn't gate on + // Header parses cleanly (RoutedRequestHeader::validate doesn't gate on // size), but the encoded `size` field claims more bytes than the // backing buffer holds. The buffer-bounds check at the bottom of // `Message::try_from` must reject. (Both this case and the @@ -769,7 +867,7 @@ mod tests { let owned = header_bytes(Command2::Request, 999); // header_bytes already produces a 256-byte buffer; size=999 > 256, // so try_from rejects via `bytes.len() < header.size()`. - let result = Message::::try_from(owned); + let result = Message::::try_from(owned); assert!(matches!(result, Err(ConsensusError::InvalidCommand { .. }))); } @@ -779,8 +877,11 @@ mod tests { // so only the construction-time `size` floor rejects it (the // buffer-length check passes). Guards the `[size_of::()..size]` // underflow at every downstream call site. - let owned = header_bytes(Command2::Request, size_of::() as u32 - 1); - let result = Message::::try_from(owned); + let owned = header_bytes( + Command2::Request, + size_of::() as u32 - 1, + ); + let result = Message::::try_from(owned); assert!(matches!(result, Err(ConsensusError::InvalidCommand { .. }))); } @@ -789,7 +890,7 @@ mod tests { #[test] fn as_generic_view_reads_command_byte() { let owned = header_bytes(Command2::Request, 256); - let typed = Message::::try_from(owned).expect("valid"); + let typed = Message::::try_from(owned).expect("valid"); let generic = typed.as_generic(); assert_eq!(generic.header().command, Command2::Request); assert_eq!(generic.total_len(), 256); @@ -799,11 +900,11 @@ mod tests { #[test] fn try_as_typed_command_mismatch_returns_err_without_unsafe_cast() { - // bytes are a valid Prepare; asking for RequestHeader must fail + // bytes are a valid Prepare; asking for RoutedRequestHeader must fail // *before* the unsafe ptr-cast inside try_as_typed. let owned = header_bytes(Command2::Prepare, 256); let generic = Message::::try_from(owned).expect("valid"); - let result = generic.try_as_typed::(); + let result = generic.try_as_typed::(); assert!(matches!( result, Err(ConsensusError::InvalidCommand { @@ -815,7 +916,8 @@ mod tests { #[test] fn try_as_typed_invalid_validation_returns_err() { - // RequestHeader::validate rejects operation=Register with non-zero session. + // `RequestHeader::validate` rejects operation=Register with non-zero + // session; the routed shape shares the same field rules. let mut owned = header_bytes(Command2::Request, 256); { let buf = owned.as_mut_slice(); @@ -833,7 +935,7 @@ mod tests { fn try_into_typed_command_mismatch_returns_err() { let owned = header_bytes(Command2::Prepare, 256); let generic = Message::::try_from(owned).expect("valid"); - let result = generic.try_into_typed::(); + let result = generic.try_into_typed::(); assert!(matches!( result, Err(ConsensusError::InvalidCommand { @@ -897,7 +999,7 @@ mod tests { } #[test] - fn messagebag_dispatch_request_with_invalid_register_session_returns_err() { + fn client_wire_decode_of_request_with_invalid_register_session_returns_err() { // `RequestHeader::validate` rejects Register with non-zero session. let mut owned = header_bytes(Command2::Request, 256); { @@ -906,16 +1008,16 @@ mod tests { buf[REQUEST_SESSION_OFF..REQUEST_SESSION_OFF + 8].copy_from_slice(&5u64.to_le_bytes()); } let generic = Message::::try_from(owned).expect("valid generic"); - let result = MessageBag::try_from(generic); + let result = generic.try_into_typed::(); assert!(matches!(result, Err(ConsensusError::InvalidField(_)))); } // Ingress validation runs on every client frame at the network boundary, - // reached through `MessageBag::try_from` -> `try_into_typed` -> - // `RequestHeader::validate`. Several dedup and authz conclusions rest on - // it running, so pin the field rules rather than the plumbing: whatever - // `request_preflight` and the operation gate see downstream has already - // passed these. + // reached through `try_into_typed` -> `RequestHeader::validate` before + // dispatch promotes the frame to `RoutedRequestHeader`. Several dedup and + // authz conclusions rest on it running, so pin the field rules rather than + // the plumbing: whatever `request_preflight` and the operation gate see + // downstream has already passed these. #[test] fn ingress_validation_enforces_the_request_header_field_rules() { // (operation, session, request, must_pass) @@ -946,7 +1048,7 @@ mod tests { .copy_from_slice(&request.to_le_bytes()); } let generic = Message::::try_from(owned).expect("valid generic"); - let accepted = MessageBag::try_from(generic).is_ok(); + let accepted = generic.try_into_typed::().is_ok(); assert_eq!( accepted, must_pass, "{operation:?} with session={session} request={request}" @@ -968,7 +1070,7 @@ mod tests { } let generic = Message::::try_from(owned).expect("valid generic"); assert!(matches!( - MessageBag::try_from(generic), + generic.try_into_typed::(), Err(ConsensusError::InvalidField(_)) )); } @@ -978,7 +1080,7 @@ mod tests { #[test] fn request_message_deep_copy_independent() { let owned = header_bytes(Command2::Request, 256); - let mut msg = Message::::try_from(owned).expect("valid"); + let mut msg = Message::::try_from(owned).expect("valid"); let copy = msg.deep_copy(); // Mutate the original's bytes; the deep copy must be untouched. msg.as_mut_slice()[200] = 0xab; @@ -991,7 +1093,7 @@ mod tests { #[test] fn transmute_header_request_to_prepare() { let owned = header_bytes(Command2::Request, 256); - let msg = Message::::try_from(owned).expect("valid"); + let msg = Message::::try_from(owned).expect("valid"); let prepared: Message = msg.transmute_header::(|_old, new| { new.command = Command2::Prepare; @@ -1000,6 +1102,71 @@ mod tests { assert_eq!(prepared.header().command, Command2::Prepare); } + // into_routed: in-place client-wire -> routed retype + + // Promotion must carry the data-bearing reserved prefix verbatim (the + // non-replicated op code lives in `reserved[0..4]`) and unset only the + // `group` tail, whatever junk the client sent in those eight bytes. + #[test] + fn into_routed_keeps_reserved_prefix_and_unsets_group() { + const RESERVED_OFF: usize = std::mem::offset_of!(RequestHeader, reserved); + + let mut owned = header_bytes(Command2::Request, 256); + { + let buf = owned.as_mut_slice(); + for (index, byte) in buf[RESERVED_OFF..RESERVED_OFF + 60].iter_mut().enumerate() { + *byte = u8::try_from(index).expect("60 fits u8") + 1; + } + } + let request = Message::::try_from(owned).expect("valid client frame"); + let client_header = *request.header(); + + let routed = request.into_routed(); + let header = routed.header(); + assert_eq!( + header.reserved[..], + client_header.reserved[..52], + "the reserved prefix carries data and must survive promotion" + ); + assert_eq!( + header.group, 0, + "the client-sent reserved tail must not leak into `group`" + ); + assert_eq!(header.client, client_header.client); + assert_eq!(header.operation, client_header.operation); + assert_eq!(header.session, client_header.session); + assert_eq!(header.request, client_header.request); + assert_eq!(header.user_id, client_header.user_id); + } + + // A peer-wire `Command2::Request` decodes as `RoutedRequestHeader`, so its + // validate must enforce the client-boundary field rules: a forged + // `client = 0` frame would otherwise reach the client table's hard assert + // and abort the metadata primary, and a `Reserved` operation would replay + // that client's cached register reply. + #[test] + fn messagebag_dispatch_rejects_request_with_zero_client() { + let mut owned = header_bytes(Command2::Request, 256); + owned.as_mut_slice()[REQUEST_CLIENT_OFF..REQUEST_CLIENT_OFF + 16] + .copy_from_slice(&0u128.to_le_bytes()); + let generic = Message::::try_from(owned).expect("valid generic"); + assert!(matches!( + MessageBag::try_from(generic), + Err(ConsensusError::InvalidField(_)) + )); + } + + #[test] + fn messagebag_dispatch_rejects_request_with_reserved_operation() { + let mut owned = header_bytes(Command2::Request, 256); + owned.as_mut_slice()[REQUEST_OPERATION_OFF] = Operation::Reserved as u8; + let generic = Message::::try_from(owned).expect("valid generic"); + assert!(matches!( + MessageBag::try_from(generic), + Err(ConsensusError::InvalidField(_)) + )); + } + // ResponseBacking via SmallVec #[test] diff --git a/core/server_common/src/indexes_mut.rs b/core/server_common/src/indexes_mut.rs index b31c59089f..394b234cb5 100644 --- a/core/server_common/src/indexes_mut.rs +++ b/core/server_common/src/indexes_mut.rs @@ -171,7 +171,7 @@ impl IggyIndexesMut { // Bound to exactly one entry rather than to the buffer end: a file // whose length is not a whole multiple of INDEX_SIZE (e.g. a 24-byte - // server-ng sparse index read back through this 16-byte reader on a + // server sparse index read back through this 16-byte reader on a // mixed-format recovery) would otherwise hand an oversized slice to the // view and trip its length assertion. let start = (self.count() - 1) as usize * INDEX_SIZE; diff --git a/core/server_common/src/send_messages2.rs b/core/server_common/src/send_messages2.rs index f0d9a7f88d..4dacba2e2b 100644 --- a/core/server_common/src/send_messages2.rs +++ b/core/server_common/src/send_messages2.rs @@ -19,7 +19,7 @@ use crate::consensus_message::{MESSAGE_ALIGN, Message}; use crate::iobuf::Owned; use crate::sharding::IggyNamespace; use bytes::{Bytes, BytesMut}; -use iggy_binary_protocol::{PrepareHeader, RequestHeader}; +use iggy_binary_protocol::{PrepareHeader, RoutedRequestHeader}; use iggy_common::{EncryptorKind, INDEX_SIZE, IggyError, random_id}; use std::hash::Hasher; use twox_hash::XxHash3_64; @@ -196,20 +196,20 @@ impl SendMessages2Owned { pub fn encode_request( self, - mut request_header: RequestHeader, - ) -> Result, IggyError> { - let total_size = std::mem::size_of::() + self.header.total_size(); + mut request_header: RoutedRequestHeader, + ) -> Result, IggyError> { + let total_size = std::mem::size_of::() + self.header.total_size(); // The converted body differs in size from the legacy wire body the // header described; a stale `size` truncates the rebuilt blob for // every downstream slice (stamping, journal reads). request_header.size = u32::try_from(total_size).map_err(|_| IggyError::InvalidCommand)?; let mut buffer = Owned::::zeroed(total_size); let bytes = buffer.as_mut_slice(); - bytes[0..std::mem::size_of::()] + bytes[0..std::mem::size_of::()] .copy_from_slice(bytemuck::bytes_of(&request_header)); self.header.encode_into( - &mut bytes[std::mem::size_of::() - ..std::mem::size_of::() + COMMAND_HEADER_SIZE], + &mut bytes[std::mem::size_of::() + ..std::mem::size_of::() + COMMAND_HEADER_SIZE], ); bytes[PREPARE_SPLIT_POINT..PREPARE_SPLIT_POINT + self.blob.len()] .copy_from_slice(&self.blob); @@ -473,12 +473,12 @@ pub(crate) type FrozenBatchHeader = crate::iobuf::Frozen; /// [`IggyError::InvalidCommand`] on an undecodable batch; encryption errors /// propagate from the encryptor. pub fn encrypt_batch_request( - message: Message, + message: Message, encryptor: &EncryptorKind, -) -> Result, IggyError> { +) -> Result, IggyError> { let request_header = *message.header(); let total_size = request_header.size as usize; - let body = &message.as_slice()[std::mem::size_of::()..total_size]; + let body = &message.as_slice()[std::mem::size_of::()..total_size]; let batch = decode_batch_slice(body)?; let mut blob = BytesMut::with_capacity(batch.blob().len() * 2); @@ -537,26 +537,27 @@ pub enum ChecksumMode { pub fn convert_request_message( namespace: IggyNamespace, - message: Message, + message: Message, checksum: ChecksumMode, -) -> Result, IggyError> { +) -> Result, IggyError> { let request_header = *message.header(); let total_size = request_header.size as usize; - let body = &message.as_slice()[std::mem::size_of::()..total_size]; + let body = &message.as_slice()[std::mem::size_of::()..total_size]; // A canonical body enters the pipeline verbatim, so it must end exactly at // `batch_length`: `size` and `batch_length` are independent client-supplied // fields and `decode_batch_slice` only lower-bounds the frame. A suffix past // `batch_length` is covered by no checksum, still rides the buffer to disk, // and desyncs the segment walk that advances by `batch_length`. - match decode_batch_slice(body).map(|batch| batch.header.total_size()) { - Ok(batch_length) if body.len() == batch_length => Ok(message), + match decode_batch_slice(body) { + Ok(batch) if batch.message_count() == 0 => Err(IggyError::InvalidCommand), + Ok(batch) if body.len() == batch.header.total_size() => Ok(message), Ok(_) => Err(IggyError::InvalidCommand), Err(_) => transcode_legacy_request(namespace, body, request_header, checksum), } } /// Transcode a legacy `SendMessages` request body directly into the canonical -/// `[RequestHeader][256B SendMessages2Header][blob]` form, writing each message +/// `[RoutedRequestHeader][256B SendMessages2Header][blob]` form, writing each message /// record straight into the final aligned buffer. /// /// Fused replacement for the `from_legacy_request(..).encode_request(..)` @@ -571,10 +572,13 @@ pub fn convert_request_message( fn transcode_legacy_request( namespace: IggyNamespace, body: &[u8], - mut request_header: RequestHeader, + mut request_header: RoutedRequestHeader, checksum: ChecksumMode, -) -> Result, IggyError> { +) -> Result, IggyError> { let (message_count, messages) = legacy_messages_slice(body)?; + if message_count == 0 { + return Err(IggyError::InvalidCommand); + } let mut parsed = Vec::with_capacity(message_count as usize); let mut origin_timestamp = u64::MAX; let mut cursor = 0usize; @@ -598,7 +602,7 @@ fn transcode_legacy_request( origin_timestamp = 0; } - let header_size = std::mem::size_of::(); + let header_size = std::mem::size_of::(); let batch_length = COMMAND_HEADER_SIZE .checked_add(blob_len) .ok_or(IggyError::InvalidCommand)?; @@ -678,6 +682,37 @@ fn transcode_legacy_request( /// chunk and steps by `batch_length`. Callers whose buffer is meant to BE the /// batch must reject the surplus themselves - see [`convert_request_message`]. pub fn decode_batch_slice(body: &[u8]) -> Result, IggyError> { + decode_batch_slice_with(body, BatchIntegrity::Verify) +} + +/// How much of a batch record [`decode_batch_slice_with`] proves before returning it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BatchIntegrity { + /// Re-hash the batch and reject it unless it matches its own `batch_checksum`. + Verify, + /// Check the framing only, and hand back whatever it describes. The caller is + /// accepting bytes that may not be the ones written. + LayoutOnly, +} + +/// [`decode_batch_slice`] with the integrity level chosen by the caller. +/// +/// An enum, not a bool: the one caller that passes anything but +/// [`BatchIntegrity::Verify`] is the disk poll under its operator knob, and +/// `..., false)` there reads like a detail rather than opting a read out of +/// corruption detection. +/// +/// Layout checks are not optional either way: a short or self-inconsistent record is +/// rejected regardless, because the caller would otherwise index past it. +/// +/// # Errors +/// [`IggyError::InvalidCommand`] for a short or inconsistent record, and +/// [`IggyError::InvalidBatchChecksum`] under [`BatchIntegrity::Verify`] when the batch +/// does not match. Callers that must tell corruption from a partial tail need both. +pub fn decode_batch_slice_with( + body: &[u8], + integrity: BatchIntegrity, +) -> Result, IggyError> { if body.len() < COMMAND_HEADER_SIZE { return Err(IggyError::InvalidCommand); } @@ -690,13 +725,18 @@ pub fn decode_batch_slice(body: &[u8]) -> Result, IggyError let blob = &body[COMMAND_HEADER_SIZE..COMMAND_HEADER_SIZE + blob_len]; let batch = SendMessages2Ref { header, blob }; - let expected_checksum = verify_and_recompute_batch_checksum(&batch)?; - if header.batch_checksum != expected_checksum { - return Err(IggyError::InvalidBatchChecksum( - header.batch_checksum, - expected_checksum, - header.base_offset, - )); + match integrity { + BatchIntegrity::Verify => { + let expected_checksum = verify_and_recompute_batch_checksum(&batch)?; + if header.batch_checksum != expected_checksum { + return Err(IggyError::InvalidBatchChecksum( + header.batch_checksum, + expected_checksum, + header.base_offset, + )); + } + } + BatchIntegrity::LayoutOnly => validate_batch_layout(&batch)?, } Ok(batch) @@ -726,15 +766,13 @@ pub fn decode_prepare_slice(bytes: &[u8]) -> Result, IggyEr /// /// INVARIANT: `bytes` MUST be node-local self-stamped - /// [`stamp_prepare_for_persistence`] recomputed the batch checksum over the -/// exact blob on THIS node - or already integrity-checked at their network +/// exact blob on the local node - or already integrity-checked at network /// ingress. There is no consensus-layer blob validation: the `PrepareHeader` -/// integrity fields are inert zeros. A replicated `SendMessages` prepare is -/// gated per-message on receipt by [`verify_received_send_messages`], and a -/// repaired prepare is validated via [`decode_prepare_slice`]; both run BEFORE -/// the bytes reach any trusted decode. NEVER call this on unvalidated network -/// bytes - it would let a corrupted blob pass undetected. The full-body -/// per-message checksum pass dominates produce-path CPU, so trusted call sites -/// that only read header meta skip it. +/// integrity fields are inert zeros. Replicated and repaired prepares are +/// validated via [`decode_prepare_slice`] before the bytes reach any trusted +/// decode. Calling this on unvalidated network bytes would let a corrupted blob +/// pass undetected. The full-body per-message checksum pass dominates +/// produce-path CPU, so trusted call sites that only read header meta skip it. /// /// # Errors /// @@ -824,35 +862,6 @@ pub fn stamp_prepare_for_persistence( Ok((message, command, command.message_count)) } -/// Verify every per-message checksum in a received `SendMessages` prepare. -/// -/// The FIRST blob-integrity check on the replicated path: the `PrepareHeader` -/// integrity fields are inert zeros and the batch checksum is recomputed -/// locally at stamp, so transit corruption of a message body would otherwise -/// reach apply undetected. Backups call this before journaling a replicated -/// prepare; on a mismatch the caller fails closed (drop, no `PrepareOk`) and the -/// primary retransmits on prepare-timeout. -/// -/// The stored `batch_checksum` is not consulted (a received prepare is -/// pre-stamp, `base_offset` / `base_timestamp` zero): integrity rests on the -/// per-message checksums, recomputed over the stamp-invariant cover -/// (`header[8..48] || payload || user_headers`), which excludes the 256B command -/// header, so it holds whether or not this node has stamped yet. Shares the -/// frame walk with the validating decoders via -/// [`verify_and_recompute_batch_checksum`], discarding its recomputed batch -/// value. -/// -/// # Errors -/// -/// [`IggyError::InvalidCommand`] if the records do not tile `message_count` -/// exactly (a length-field corruption desyncs the walk); -/// [`IggyError::InvalidMessageChecksum`] on the first per-message mismatch. -pub fn verify_received_send_messages(bytes: &[u8]) -> Result<(), IggyError> { - let batch = decode_prepare_slice_trusted(bytes)?; - verify_and_recompute_batch_checksum(&batch)?; - Ok(()) -} - fn legacy_messages_slice(body: &[u8]) -> Result<(u32, &[u8]), IggyError> { if body.len() < 4 { return Err(IggyError::InvalidCommand); @@ -1000,6 +1009,24 @@ fn verify_and_recompute_batch_checksum(batch: &SendMessages2Ref<'_>) -> Result) -> Result<(), IggyError> { + let mut framed = 0u32; + let mut covered = 0usize; + for message in batch.iter_with_offsets() { + framed += 1; + covered = message.end; + } + if framed != batch.message_count() || covered != batch.blob().len() { + return Err(IggyError::InvalidCommand); + } + Ok(()) +} + fn read_u32(bytes: &[u8], offset: usize) -> Result { bytes .get(offset..offset + 4) @@ -1179,8 +1206,8 @@ mod tests { } /// `[PrepareHeader][256B batch header][blob]` carrying real per-message - /// records + checksums from the production encoder, left pre-stamp - /// (`base_offset` / `base_timestamp` zero) as a follower receives it. + /// records and checksums from the production encoder, with the initial zero + /// base offset and timestamp. fn prepare_with_messages(messages: &IggyMessages2) -> Owned { let namespace = IggyNamespace::new(1, 1, 7); let owned = @@ -1188,42 +1215,6 @@ mod tests { prepare_from_owned(&owned) } - #[test] - fn verify_received_send_messages_accepts_clean_batch() { - let owned = prepare_with_messages(&sample_messages()); - verify_received_send_messages(owned.as_slice()) - .expect("a clean batch passes the receive gate"); - } - - #[test] - fn verify_received_send_messages_rejects_flipped_payload_byte() { - let mut owned = prepare_with_messages(&sample_messages()); - // First payload begins right after the first message's 48B header. - let payload_index = PREPARE_SPLIT_POINT + MESSAGE_HEADER_SIZE; - owned.as_mut_slice()[payload_index] ^= 0xFF; - assert!( - matches!( - verify_received_send_messages(owned.as_slice()), - Err(IggyError::InvalidMessageChecksum(..)) - ), - "a flipped payload byte must fail the per-message checksum", - ); - } - - #[test] - fn verify_received_send_messages_rejects_flipped_stored_checksum() { - let mut owned = prepare_with_messages(&sample_messages()); - // The first message's stored checksum is the first 8 bytes of the blob. - owned.as_mut_slice()[PREPARE_SPLIT_POINT] ^= 0xFF; - assert!( - matches!( - verify_received_send_messages(owned.as_slice()), - Err(IggyError::InvalidMessageChecksum(..)) - ), - "a flipped stored checksum must fail the per-message check", - ); - } - #[test] fn checksum_oneshot_matches_streaming_reference() { // Formula pin: the per-message checksum is XxHash3-64 (default seed) @@ -1381,14 +1372,14 @@ mod tests { body } - fn legacy_request_message(body: &[u8]) -> Message { - let header_size = std::mem::size_of::(); + fn legacy_request_message(body: &[u8]) -> Message { + let header_size = std::mem::size_of::(); let total = header_size + body.len(); let mut buffer = Owned::::zeroed(total); { - let header: &mut RequestHeader = + let header: &mut RoutedRequestHeader = bytemuck::checked::try_from_bytes_mut(&mut buffer.as_mut_slice()[..header_size]) - .expect("zeroed bytes form a valid RequestHeader"); + .expect("zeroed bytes form a valid RoutedRequestHeader"); header.command = Command2::Request; header.operation = Operation::SendMessages; header.client = 1; @@ -1400,6 +1391,29 @@ mod tests { Message::try_from(buffer).expect("legacy request message is valid") } + #[test] + fn convert_request_message_rejects_empty_canonical_and_legacy_batches() { + let namespace = IggyNamespace::new(1, 1, 3); + let messages = IggyMessages2::with_capacity(0); + let canonical = SendMessages2Owned::from_messages(namespace, &messages) + .expect("build empty canonical batch"); + let mut canonical_body = vec![0; canonical.header.total_size()]; + canonical + .header + .encode_into(&mut canonical_body[..COMMAND_HEADER_SIZE]); + let legacy_body = legacy_send_messages_body(&messages); + + for mode in [ChecksumMode::Compute, ChecksumMode::Skip] { + let canonical_result = + convert_request_message(namespace, legacy_request_message(&canonical_body), mode); + assert!(matches!(canonical_result, Err(IggyError::InvalidCommand))); + + let legacy_result = + convert_request_message(namespace, legacy_request_message(&legacy_body), mode); + assert!(matches!(legacy_result, Err(IggyError::InvalidCommand))); + } + } + #[test] fn convert_request_message_transcodes_legacy_to_canonical_bytes() { // Golden: the fused legacy transcode must emit the exact canonical batch @@ -1420,7 +1434,7 @@ mod tests { let legacy = legacy_request_message(&legacy_send_messages_body(&messages)); let converted = convert_request_message(namespace, legacy, ChecksumMode::Compute) .expect("legacy body transcodes"); - let header_size = std::mem::size_of::(); + let header_size = std::mem::size_of::(); let actual_body = &converted.as_slice()[header_size..converted.header().size as usize]; assert_eq!( @@ -1447,7 +1461,7 @@ mod tests { let namespace = IggyNamespace::new(1, 1, 3); let messages = sample_messages(); let body = legacy_send_messages_body(&messages); - let header_size = std::mem::size_of::(); + let header_size = std::mem::size_of::(); let computed = convert_request_message( namespace, @@ -1490,7 +1504,7 @@ mod tests { // batch and returns it unchanged. Every decode must succeed. let namespace = IggyNamespace::new(1, 1, 3); let messages = sample_messages(); - let header_size = std::mem::size_of::(); + let header_size = std::mem::size_of::(); let legacy = legacy_request_message(&legacy_send_messages_body(&messages)); let canonical = convert_request_message(namespace, legacy, ChecksumMode::Compute) @@ -1523,19 +1537,19 @@ mod tests { const TRAILING_JUNK_CASES: [&[u8]; 2] = [&[0xAA], &[0xFF; 64]]; /// Canonical `SendMessages` request carrying `junk` past `batch_length`, with - /// `RequestHeader.size` inflated to cover it. `size` and `batch_length` are + /// `RoutedRequestHeader.size` inflated to cover it. `size` and `batch_length` are /// independent wire fields, so a non-conforming client can emit this. - fn canonical_request_with_trailing_bytes(junk: &[u8]) -> Message { + fn canonical_request_with_trailing_bytes(junk: &[u8]) -> Message { let namespace = IggyNamespace::new(1, 1, 3); let owned = SendMessages2Owned::from_messages(namespace, &sample_messages()).expect("build batch"); - let header_size = std::mem::size_of::(); + let header_size = std::mem::size_of::(); let total = header_size + owned.header.total_size() + junk.len(); let mut buffer = Owned::::zeroed(total); { - let header: &mut RequestHeader = + let header: &mut RoutedRequestHeader = bytemuck::checked::try_from_bytes_mut(&mut buffer.as_mut_slice()[..header_size]) - .expect("zeroed bytes form a valid RequestHeader"); + .expect("zeroed bytes form a valid RoutedRequestHeader"); header.command = Command2::Request; header.operation = Operation::SendMessages; header.client = 1; @@ -1553,8 +1567,7 @@ mod tests { Message::try_from(buffer).expect("request message is valid") } - /// The replicated counterpart: a pre-stamp `Prepare` whose `size` covers - /// `junk` past `batch_length`. + /// A `Prepare` whose `size` covers `junk` past `batch_length`. fn prepare_with_trailing_bytes(junk: &[u8]) -> Owned { let namespace = IggyNamespace::new(1, 1, 7); let owned = @@ -1612,18 +1625,11 @@ mod tests { } #[test] - fn verify_received_send_messages_rejects_trailing_bytes_past_batch_length() { - // Replica ingest boundary. The gate clamps the blob to `batch_length` - // before verifying, so without an exact-frame check a primary could plant - // bytes that no per-message checksum covers on every backup. + fn decode_prepare_slice_rejects_trailing_bytes_past_batch_length() { + // Replica ingest must reject bytes beyond `batch_length` because no + // per-message checksum covers them. for junk in TRAILING_JUNK_CASES { let owned = prepare_with_trailing_bytes(junk); - let result = verify_received_send_messages(owned.as_slice()); - assert!( - matches!(result, Err(IggyError::InvalidCommand)), - "{} trailing bytes must fail the receive gate, got {result:?}", - junk.len(), - ); assert!( matches!( decode_prepare_slice(owned.as_slice()), diff --git a/core/server_common/src/sharding/mod.rs b/core/server_common/src/sharding/mod.rs index 761e41d045..b73169f526 100644 --- a/core/server_common/src/sharding/mod.rs +++ b/core/server_common/src/sharding/mod.rs @@ -22,10 +22,9 @@ mod shard_id; pub use local_idx::LocalIdx; pub use namespace::{ - IggyNamespace, MAX_PARTITIONS, MAX_STREAMS, MAX_TOPICS, METADATA_CONSENSUS_NAMESPACE, - NamespaceCapacityError, PACKED_NAMESPACE_BITS, PACKED_NAMESPACE_MAX, PARTITION_BITS, - PARTITION_MASK, PARTITION_SHIFT, STREAM_BITS, STREAM_MASK, STREAM_SHIFT, TOPIC_BITS, - TOPIC_MASK, TOPIC_SHIFT, + IggyNamespace, MAX_PARTITIONS, MAX_STREAMS, MAX_TOPICS, METADATA_GROUP, NamespaceCapacityError, + PACKED_NAMESPACE_BITS, PACKED_NAMESPACE_MAX, PARTITION_BITS, PARTITION_MASK, PARTITION_SHIFT, + STREAM_BITS, STREAM_MASK, STREAM_SHIFT, TOPIC_BITS, TOPIC_MASK, TOPIC_SHIFT, }; pub use partition_location::PartitionLocation; pub use shard_id::ShardId; diff --git a/core/server_common/src/sharding/namespace.rs b/core/server_common/src/sharding/namespace.rs index 839edc0a24..435b076c41 100644 --- a/core/server_common/src/sharding/namespace.rs +++ b/core/server_common/src/sharding/namespace.rs @@ -27,7 +27,7 @@ // shard. Re-exported here for ergonomics of existing call sites. pub use iggy_binary_protocol::namespace::{ - MAX_PARTITIONS, MAX_STREAMS, MAX_TOPICS, METADATA_CONSENSUS_NAMESPACE, PACKED_NAMESPACE_BITS, + MAX_PARTITIONS, MAX_STREAMS, MAX_TOPICS, METADATA_GROUP, PACKED_NAMESPACE_BITS, PACKED_NAMESPACE_MAX, PARTITION_BITS, PARTITION_MASK, PARTITION_SHIFT, STREAM_BITS, STREAM_MASK, STREAM_SHIFT, TOPIC_BITS, TOPIC_MASK, TOPIC_SHIFT, bits_required, }; @@ -149,7 +149,7 @@ impl IggyNamespace { #[cfg(test)] mod tests { use super::{ - IggyNamespace, MAX_PARTITIONS, MAX_STREAMS, MAX_TOPICS, METADATA_CONSENSUS_NAMESPACE, + IggyNamespace, MAX_PARTITIONS, MAX_STREAMS, MAX_TOPICS, METADATA_GROUP, NamespaceCapacityError, PACKED_NAMESPACE_BITS, PACKED_NAMESPACE_MAX, }; @@ -158,26 +158,26 @@ mod tests { // STREAM_BITS / TOPIC_BITS / PARTITION_BITS that closes the gap between // the packed range and the sentinel will fail to build. const _: () = { - assert!(METADATA_CONSENSUS_NAMESPACE > PACKED_NAMESPACE_MAX); + assert!(METADATA_GROUP > PACKED_NAMESPACE_MAX); assert!(PACKED_NAMESPACE_BITS == 12 + 12 + 20); assert!(PACKED_NAMESPACE_MAX == (1u64 << PACKED_NAMESPACE_BITS) - 1); }; #[test] fn metadata_sentinel_cannot_collide_with_any_packable_namespace() { - assert!(!IggyNamespace::is_packable(METADATA_CONSENSUS_NAMESPACE)); + assert!(!IggyNamespace::is_packable(METADATA_GROUP)); // The (0, 0, 0) corner is intentionally a legal partition, which is // precisely why `0` is unsuitable as the metadata sentinel. let zero = IggyNamespace::new(0, 0, 0); assert_eq!(zero.inner(), 0); assert!(IggyNamespace::is_packable(zero.inner())); - assert_ne!(zero.inner(), METADATA_CONSENSUS_NAMESPACE); + assert_ne!(zero.inner(), METADATA_GROUP); // Maximum packable triple stays inside the packed range. let max = IggyNamespace::new(MAX_STREAMS - 1, MAX_TOPICS - 1, MAX_PARTITIONS - 1); assert!(IggyNamespace::is_packable(max.inner())); - assert_ne!(max.inner(), METADATA_CONSENSUS_NAMESPACE); + assert_ne!(max.inner(), METADATA_GROUP); } #[test] diff --git a/core/shard/Cargo.toml b/core/shard/Cargo.toml index 439221c447..874fb53532 100644 --- a/core/shard/Cargo.toml +++ b/core/shard/Cargo.toml @@ -25,7 +25,7 @@ publish = false [features] # Simulator-only test hook (`IggyShard::init_partition`): bypasses the # reconciler's `ReconcileOp::InsertOwned` funnel, mutating `IggyPartitions` -# off the pump task. A `-p iggy-server-ng` build excludes it; `cargo build +# off the pump task. A `-p iggy-server` build excludes it; `cargo build # --workspace` unifies features and compiles it into the shared `shard` # unit (simulator requests it). Benign: no production caller. simulator = [] diff --git a/core/shard/src/lib.rs b/core/shard/src/lib.rs index 28537d1c5f..d898e6a909 100644 --- a/core/shard/src/lib.rs +++ b/core/shard/src/lib.rs @@ -28,21 +28,22 @@ pub use router::CONSENSUS_TICK_INTERVAL; #[cfg(any(test, feature = "simulator"))] use consensus::LocalPipeline; use consensus::{ - ChunkProgress, CommitOutcome, Consensus, ConsensusClock, MetadataHandle, MuxPlane, - PartitionsHandle, Pipeline, Plane, PlaneKind, STATE_TRANSFER_MAX_DECODE_RETRIES, - STATE_TRANSFER_MAX_STALL_RETRIES, Sequencer, VsrAction, VsrConsensus, - build_deny_reply_from_request_header, + ChunkProgress, CommitOutcome, Consensus, ConsensusClock, DVC_HEADERS_MAX, DvcHeaderKind, + DvcSuffix, MergedLog, MetadataHandle, MuxPlane, PartitionsHandle, Pipeline, Plane, PlaneKind, + STATE_TRANSFER_MAX_DECODE_RETRIES, STATE_TRANSFER_MAX_STALL_RETRIES, Sequencer, Status, + VsrAction, VsrConsensus, build_deny_reply_from_request_header, dvc_blank, dvc_header_kind, + encode_prepare_headers, restamp_prepare_view, verify_prepare_integrity, }; #[cfg(any(test, feature = "simulator"))] use crossfire::AsyncRxTrait; use crossfire::TrySendError; use futures::FutureExt; use iggy_binary_protocol::{ - Command2, CommitHeader, DoViewChangeHeader, GenericHeader, Operation, PrepareHeader, - PrepareOkHeader, RepairPrepareHeader, RepairRangeReplyHeader, RequestHeader, + CHECKSUM_UNSEALED, Command2, CommitHeader, ConsensusHeader, DoViewChangeHeader, GenericHeader, + Operation, PrepareHeader, PrepareOkHeader, RepairPrepareHeader, RepairRangeReplyHeader, RequestPreparesHeader, RequestStartViewHeader, RequestStateChunkHeader, - RequestStateTransferHeader, StartViewChangeHeader, StartViewHeader, StateChunkHeader, - StateTransferTargetHeader, + RequestStateTransferHeader, RoutedRequestHeader, StartViewChangeHeader, StartViewHeader, + StateChunkHeader, StateTransferTargetHeader, }; #[cfg(any(test, feature = "simulator"))] use iggy_common::PartitionStats; @@ -216,7 +217,7 @@ pub enum MetadataSubmit { /// Handler shard 0 runs for an inbound [`MetadataSubmit`]. /// -/// server-ng wires it to `submit_register_in_process` / +/// The server wires it to `submit_register_in_process` / /// `submit_logout_in_process` / `submit_request_in_process` and sends the /// result back over the frame's `reply` sender. A peer shard (no consensus) /// must never receive this frame. @@ -249,7 +250,7 @@ pub struct ConnectedClientInfo { } /// Handler each shard runs for an inbound [`LifecycleFrame::ListClients`]. -/// server-ng wires it to read the shard's `SessionManager` and push its +/// The server wires it to read the shard's `SessionManager` and push its /// connected clients back over the carried reply sender. pub type ListClientsHandler = Rc>)>; @@ -332,7 +333,7 @@ pub enum PartitionReadReply { } /// Handler the owning shard runs for an inbound -/// [`LifecycleFrame::PartitionRead`]. server-ng wires it to its partitions +/// [`LifecycleFrame::PartitionRead`]. The server wires it to its partitions /// plane; the handler pushes the result back over the carried reply sender. pub type PartitionReadHandler = Rc)>; @@ -728,7 +729,7 @@ impl ShardFrame { /// the receiver pulls the window chunk by chunk instead (each walked /// `RepairDone` immediately requests the next chunk while progress holds). /// -/// Runtime default; server-ng overrides the live ceiling per shard from +/// Runtime default; the server overrides the live ceiling per shard from /// `[cluster] repair_chunk_max` at bootstrap. pub const REPAIR_CHUNK_MAX: u64 = 128; @@ -865,7 +866,7 @@ const SEGMENT_SIZE_CEILING_BYTES: u64 = 1 << 30; /// The most one segment can overshoot its size cap: rotation checks the cap /// AFTER appending, so a segment closes at most one maximum-size batch past it. /// -/// Derived from the BUS frame cap, not `MAX_PAYLOAD_SIZE`: server-ng never +/// Derived from the BUS frame cap, not `MAX_PAYLOAD_SIZE`: the server never /// enforces the latter (its only enforcement sites are the legacy server and the /// SDK batch types), so the largest appendable batch is whatever the message bus /// will frame. This tracks the shipped `message_bus.max_message_size` default; an @@ -878,7 +879,7 @@ const SEGMENT_SIZE_OVERSHOOT_BYTES: u64 = 64 * 1024 * 1024; /// /// Mirrors `[partition] transfer_artifact_bytes_max`. Free const so the config /// crate's copy can be pinned to it by a `const _: () = assert!(..)` at the -/// server-ng build edge, the way every other runtime default is. +/// server build edge, the way every other runtime default is. pub const PARTITION_ARTIFACT_LEN_DEFAULT: u64 = SEGMENT_SIZE_CEILING_BYTES + SEGMENT_SIZE_OVERSHOOT_BYTES; @@ -1186,13 +1187,13 @@ where on_metadata_submit: MetadataSubmitHandler, /// Handler for inbound [`LifecycleFrame::ListClients`] broadcast - /// queries. Every shard receives these (not just shard 0); server-ng + /// queries. Every shard receives these (not just shard 0); the server /// wires it to its per-shard `SessionManager`. Defaults to a no-op for /// the simulator stub ctor. on_list_clients: ListClientsHandler, /// Handler for inbound [`LifecycleFrame::PartitionRead`] queries. - /// server-ng wires it to this shard's partitions plane. Defaults to a + /// The server wires it to this shard's partitions plane. Defaults to a /// no-op for the simulator stub ctor. on_partition_read: PartitionReadHandler, @@ -1288,30 +1289,30 @@ where shard_park_shedding: Cell, /// Live ceiling on prepares served per `RequestPrepares` round. Defaults - /// to [`REPAIR_CHUNK_MAX`]; server-ng overrides it from + /// to [`REPAIR_CHUNK_MAX`]; the server overrides it from /// `[cluster] repair_chunk_max` at bootstrap. repair_chunk_max: Cell, /// Live stalled-repair retry threshold in consensus ticks. Defaults to - /// [`partitions::REPAIR_RETRY_TICKS`]; server-ng overrides it from + /// [`partitions::REPAIR_RETRY_TICKS`]; the server overrides it from /// `[cluster] repair_retry_interval` at bootstrap. repair_retry_ticks: Cell, /// Live `[partition] transfer_served_cache_bytes_max`: the byte budget for /// segment payloads this shard keeps resident to serve chunk requests. - /// Defaults to [`SERVED_SEGMENT_CACHE_BYTES_DEFAULT`]; server-ng + /// Defaults to [`SERVED_SEGMENT_CACHE_BYTES_DEFAULT`]; the server /// overrides it at bootstrap. served_segment_cache_bytes_max: Cell, /// Live `[partition] transfer_artifact_bytes_max`: the alloc ceiling for one /// RECEIVED artifact. Defaults to [`PARTITION_ARTIFACT_LEN_DEFAULT`]; - /// server-ng overrides it at bootstrap. + /// the server overrides it at bootstrap. partition_artifact_len_max: Cell, /// Live `[message_bus] max_message_size`. Bounds a served state chunk: a /// frame above this is rejected by the RECEIVING transport, which tears /// down the whole replica connection. Defaults to a value that leaves - /// [`STATE_CHUNK_LEN`] usable; server-ng overrides it at bootstrap. + /// [`STATE_CHUNK_LEN`] usable; the server overrides it at bootstrap. bus_max_message_size: Cell, /// Consecutive metadata state-transfer rounds that made no progress. @@ -1775,6 +1776,34 @@ where let _ = sender.try_send(ShardFrame::lifecycle(LifecycleFrame::ReconcileApply)); } + /// `true` when an `InsertOwned` for `namespace` is built and queued but not + /// yet applied. + /// + /// The reconciler's own "already handled" test is `IggyPartitions::contains`, + /// which only turns true once the pump applies, so without this a pass run + /// during that lag rebuilds a namespace an earlier pass already built. The + /// queue IS the record of that in-flight work, so asking it cannot drift + /// from reality the way a parallel set would: every op leaves the queue + /// through `apply_reconcile_ops`, which either inserts or discards. + /// + /// Deliberately blind to `epoch`. Matching it would let a delete + recreate + /// landing inside the lag build a second incarnation over the queued one's + /// on-disk path, which is the case this exists to prevent; the recreate is + /// not lost, it costs one pass. The queued (dead-epoch) op applies, and the + /// next pass reads the epoch mismatch off the routing row and takes the + /// stale-incarnation teardown into a clean rebuild. + pub fn has_staged_insert_owned(&self, namespace: IggyNamespace) -> bool { + self.reconcile_queue.borrow().iter().any(|op| { + matches!( + op, + ReconcileOp::InsertOwned { + namespace: staged_namespace, + .. + } if *staged_namespace == namespace + ) + }) + } + /// Stage a segment-cleaner pass for `namespace` on this shard's pump. The /// timer task resolves retention config off-pump and stamps `now`; the pump /// is the single writer of partition state, so the deletion runs there, @@ -1882,18 +1911,30 @@ where epoch, } => { // Idempotent apply, mirroring `ConfirmRemove` (idempotent - // via `remove`'s `None` early-return). The reconciler - // stages this from a task separate from the pump, so under - // a commit burst two passes can each observe - // `!contains(ns)` and build the same namespace before - // either drains here. A second unconditional `insert` - // would push a duplicate partition and overwrite the - // `ns -> idx` entry, orphaning the first (its VSR group + - // segment writers leak and `len` inflates). The discarded - // build is a fresh empty incarnation over the same on-disk - // path the kept one owns, so dropping it just closes a few - // fds. + // via `remove`'s `None` early-return). An unconditional + // `insert` over a live namespace would push a duplicate + // partition and overwrite the `ns -> idx` entry, orphaning + // the first: its VSR group + segment writers leak and `len` + // inflates. + // + // A backstop, not the mechanism. `reconcile_additions` + // skips a namespace whose `InsertOwned` is already staged + // ([`Self::has_staged_insert_owned`]), so a second op for a + // live namespace should not be built at all. Dropping one + // here is damage control rather than a free no-op: the + // build already planted its initial segment over the live + // incarnation's path and folded that into the namespace's + // shared stats. if partitions.contains(&namespace) { + tracing::error!( + shard = self_shard_id, + ns_raw = namespace.inner(), + epoch, + "discarding duplicate InsertOwned for a live namespace: the \ + staged-op guard was bypassed and the build re-planted segment 0 \ + over the live incarnation's path" + ); + self.metrics.record_duplicate_partition_build_discarded(); drop(partition); continue; } @@ -2247,7 +2288,7 @@ where { match MessageBag::try_from(message) { Ok(MessageBag::Request(request)) => { - let routing = (request.header().operation, request.header().namespace); + let routing = (request.header().operation, request.header().group); match self.park_if_unmaterialised(request, routing.0, routing.1) { // The incarnation fence runs only here, on client traffic. // A backup denying what the primary admitted would diverge @@ -2272,7 +2313,7 @@ where } } Ok(MessageBag::Prepare(prepare)) => { - let routing = (prepare.header().operation, prepare.header().namespace); + let routing = (prepare.header().operation, prepare.header().group); // A tombstoned prepare still flows to the plane: replicated // traffic has no client awaiting a reply on this node, and // the plane's own tombstone guard drops it. @@ -2329,7 +2370,7 @@ where Ok(MessageBag::RequestStateChunk(ref msg)) => self.on_request_state_chunk(msg).await, Ok(MessageBag::StateChunk(ref msg)) => self.on_state_chunk(msg).await, Err(e) => { - tracing::warn!(shard = self.id, error = %e, "dropping message with invalid command"); + tracing::warn!(shard = self.id, error = %e, "dropping unparsable consensus frame"); } } } @@ -2708,7 +2749,7 @@ where /// `frame_drops_total{variant=partition,reason=park_dropped}`. fn deny_parked_client_request(&self, frame: ParkedFrame) -> bool { if frame.message.header().command == Command2::Request - && let Ok(request) = frame.message.try_into_typed::() + && let Ok(request) = frame.message.try_into_typed::() { return self.stage_transient_deny(request.header()); } @@ -2913,7 +2954,7 @@ where /// budget. Sent directly over the bus; delivery failure is terminal for /// this reply (the client recovers via its own read-timeout). #[allow(clippy::future_not_send)] - async fn deny_partition_request_transient(&self, request_header: &RequestHeader) { + async fn deny_partition_request_transient(&self, request_header: &RoutedRequestHeader) { let reply = build_deny_reply_from_request_header( request_header, IggyError::TransientNotAccepted.as_code(), @@ -2950,7 +2991,7 @@ where /// /// Returns whether the pump took it. A shard with no sender stages nothing, /// so assuming success logs an answer for a request destroyed unanswered. - fn stage_transient_deny(&self, request_header: &RequestHeader) -> bool { + fn stage_transient_deny(&self, request_header: &RoutedRequestHeader) -> bool { let reply = build_deny_reply_from_request_header( request_header, IggyError::TransientNotAccepted.as_code(), @@ -2983,7 +3024,7 @@ where } #[allow(clippy::future_not_send)] - pub async fn on_request(&self, request: Message) + pub async fn on_request(&self, request: Message) where B: MessageBus, MJ: JournalHandle, @@ -3134,7 +3175,7 @@ where /// production runtime path; bootstrap recovery uses `load_partition`), /// so it must never run in production. VSR replica id comes from /// `PartitionConsensusConfig`, not `self.id` (the local shard index). A - /// `-p iggy-server-ng` build excludes the `simulator` feature and this + /// `-p iggy-server` build excludes the `simulator` feature and this /// method; `cargo build --workspace` compiles it in but with no /// production caller. /// `superblock` is this group's durable `(view, log_view)` store. Passing @@ -3201,7 +3242,7 @@ where } /// Resolve the single partition a VSR control frame addresses, keyed by - /// `header.namespace`. Warns and returns `None` when the namespace matches + /// `header.group`. Warns and returns `None` when the namespace matches /// neither metadata nor a live partition consensus. Returns `&mut` because /// `on_do_view_change` / `on_commit` need it for `commit_journal`; the read- /// only callers reborrow `&`. Pump-only (sole mutator), so the `&mut` formed @@ -3230,7 +3271,7 @@ where return None; }; debug_assert_eq!( - partition.consensus().namespace(), + partition.consensus().group(), namespace, "keyed partition lookup must match the frame namespace" ); @@ -3255,8 +3296,9 @@ where let planes = self.plane.inner(); if let Some(ref consensus) = planes.0.consensus - && consensus.namespace() == header.namespace + && consensus.group() == header.group { + refresh_metadata_dvc_suffix(consensus, planes.0.journal.as_ref()); let actions = consensus.handle_start_view_change(PlaneKind::Metadata, &header); let (local_actions, wire_actions) = split_local_actions(actions); dispatch_vsr_actions(consensus, planes.0.journal.as_ref(), &local_actions).await; @@ -3268,13 +3310,14 @@ where let Some(partition) = self.resolve_partition_target( &planes.1.0, - header.namespace, + header.group, header.view, header.replica, "StartViewChange", ) else { return; }; + refresh_partition_dvc_suffix(partition); let consensus = partition.consensus(); let actions = consensus.handle_start_view_change(PlaneKind::Partitions, &header); let (local_actions, wire_actions) = split_local_actions(actions); @@ -3304,9 +3347,20 @@ where let planes = self.plane.inner(); if let Some(ref consensus) = planes.0.consensus - && consensus.namespace() == header.namespace + && consensus.group() == header.group { - let actions = consensus.handle_do_view_change(PlaneKind::Metadata, &header); + refresh_metadata_dvc_suffix(consensus, planes.0.journal.as_ref()); + let Some(suffix_body) = control_suffix_body_verified(&msg, header.checksum_body) else { + tracing::warn!( + shard = self.id, + from_replica = header.replica, + view = header.view, + "dropping do_view_change whose body failed its checksum" + ); + return; + }; + let actions = + consensus.handle_do_view_change(PlaneKind::Metadata, &header, suffix_body); let (local_actions, wire_actions) = split_local_actions(actions); dispatch_vsr_actions(consensus, planes.0.journal.as_ref(), &local_actions).await; if planes.0.persist_superblock_if_needed(consensus).await { @@ -3327,15 +3381,25 @@ where let config = planes.1.0.config(); let Some(partition) = self.resolve_partition_target( &planes.1.0, - header.namespace, + header.group, header.view, header.replica, "DoViewChange", ) else { return; }; + refresh_partition_dvc_suffix(partition); let consensus = partition.consensus(); - let actions = consensus.handle_do_view_change(PlaneKind::Partitions, &header); + let Some(suffix_body) = control_suffix_body_verified(&msg, header.checksum_body) else { + tracing::warn!( + shard = self.id, + from_replica = header.replica, + view = header.view, + "dropping do_view_change whose body failed its checksum" + ); + return; + }; + let actions = consensus.handle_do_view_change(PlaneKind::Partitions, &header, suffix_body); let (local_actions, wire_actions) = split_local_actions(actions); // Locals go to the partition dispatcher ONLY: `RebuildPipeline` // executes there (`dispatch_vsr_actions` bails on `journal: None`) @@ -3357,8 +3421,7 @@ where } } - #[allow(clippy::future_not_send)] - #[allow(clippy::too_many_lines)] + #[allow(clippy::future_not_send, clippy::too_many_lines)] async fn on_start_view(&self, msg: Message) where B: MessageBus, @@ -3374,15 +3437,31 @@ where let planes = self.plane.inner(); if let Some(ref consensus) = planes.0.consensus - && consensus.namespace() == header.namespace + && consensus.group() == header.group { - let actions = consensus.handle_start_view(PlaneKind::Metadata, &header); + let Some(suffix_body) = control_suffix_body_verified(&msg, header.checksum_body) else { + tracing::warn!( + shard = self.id, + from_replica = header.replica, + view = header.view, + "dropping start_view whose body failed its checksum" + ); + return; + }; + let actions = consensus.handle_start_view(PlaneKind::Metadata, &header, suffix_body); // Every rejection path (wrong primary, old view, stale incarnation, // below the commit floor, self-sent) returns no actions, and an // adopted StartView always emits at least `CommitJournal`. That // makes emptiness the adoption signal -- and the arms below must // not fire on a StartView this replica did not adopt. let adopted = !actions.is_empty(); + if adopted { + // First chance to spot a local entry disagreeing with the view's log. + // Ahead of the local dispatch below: it truncates the journal that + // `RebuildPipeline` reads back, so a rebuild before it would seed the + // pipeline from the entries this is about to drop. + self.reconcile_metadata_view_divergence().await; + } let (local_actions, wire_actions) = split_local_actions(actions); dispatch_vsr_actions(consensus, planes.0.journal.as_ref(), &local_actions).await; if planes.0.persist_superblock_if_needed(consensus).await { @@ -3460,24 +3539,40 @@ where // entirely, but `IggyPartition::transfer` is `pub` and cleared inside the // partitions crate, so an externally maintained count would drift; that // refactor is a prerequisite, not a detail.) - let transfers_inflight = if Self::may_arm_partition_transfer(&planes.1.0, header.namespace) - { + let transfers_inflight = if Self::may_arm_partition_transfer(&planes.1.0, header.group) { self.partition_transfers_inflight() } else { 0 }; let Some(partition) = self.resolve_partition_target( &planes.1.0, - header.namespace, + header.group, header.view, header.replica, "StartView", ) else { return; }; - let consensus = partition.consensus(); - let actions = consensus.handle_start_view(PlaneKind::Partitions, &header); + let Some(suffix_body) = control_suffix_body_verified(&msg, header.checksum_body) else { + tracing::warn!( + shard = self.id, + from_replica = header.replica, + view = header.view, + "dropping start_view whose body failed its checksum" + ); + return; + }; + let actions = + partition + .consensus() + .handle_start_view(PlaneKind::Partitions, &header, suffix_body); let adopted = !actions.is_empty(); + if adopted && let Some(pending) = partition.consensus().pending_view_log() { + // Ahead of the local dispatch, which rebuilds the pipeline out of the + // journal this rewrites. Same position as the metadata arm's twin. + reconcile_partition_view_divergence(self.id, partition, &pending).await; + } + let consensus = partition.consensus(); let (local_actions, wire_actions) = split_local_actions(actions); // Locals go to the partition dispatcher ONLY: `RebuildPipeline` // executes there (`dispatch_vsr_actions` bails on `journal: None`) @@ -3496,7 +3591,7 @@ where { tracing::info!( shard = self.id, - namespace_raw = header.namespace, + namespace_raw = header.group, peer = header.replica, "adopted a live view while awaiting transfer; requesting partition state transfer" ); @@ -3550,7 +3645,7 @@ where let planes = self.plane.inner(); if let Some(ref consensus) = planes.0.consensus - && consensus.namespace() == header.namespace + && consensus.group() == header.group { match consensus.handle_commit(&header) { CommitOutcome::Advanced => { @@ -3598,7 +3693,7 @@ where let config = planes.1.0.config(); let Some(partition) = self.resolve_partition_target( &planes.1.0, - header.namespace, + header.group, header.view, header.replica, "Commit", @@ -3652,7 +3747,7 @@ where let header = *msg.header(); let planes = self.plane.inner(); if let Some(ref consensus) = planes.0.consensus - && consensus.namespace() == header.namespace + && consensus.group() == header.group { let actions = consensus.handle_request_start_view(PlaneKind::Metadata, &header); let (local_actions, wire_actions) = split_local_actions(actions); @@ -3665,7 +3760,7 @@ where let Some(partition) = planes .1 .0 - .get_mut_by_ns(&IggyNamespace::from_raw(header.namespace)) + .get_mut_by_ns(&IggyNamespace::from_raw(header.group)) else { return; }; @@ -3709,9 +3804,16 @@ where let repair_chunk_max = self.repair_chunk_max.get(); let planes = self.plane.inner(); if let Some(ref consensus) = planes.0.consensus - && consensus.namespace() == header.namespace + && consensus.group() == header.group { - if !consensus.is_normal() { + // Served in `ViewChange` too: the replicas holding a missing body are + // exactly those in `ViewChange`, so refusing would deadlock the repair + // the new primary waits on. Read-only; the requester decides. + if consensus.is_transferring() { + // A transfer rewrites local state wholesale; journal not stable yet. + return; + } + if !matches!(consensus.status(), Status::Normal | Status::ViewChange) { return; } let Some(journal) = planes.0.journal.as_ref() else { @@ -3720,11 +3822,21 @@ where let journal = journal.handle(); let cluster = consensus.cluster(); let self_id = consensus.replica(); - let to_op = header.to_op.min(consensus.commit_max()); + let to_op = repair_serve_ceiling( + header.to_op, + consensus.commit_max(), + consensus.sequencer().current_sequence(), + ); // Skip the compacted prefix (below the snapshot floor) in one // RangeEvicted notice, then serve contiguously until the range // ends or the WAL runs out. - let mut from_op = header.from_op; + // + // Floored, not just walked up to: `validate` asks only for + // `1 <= from_op <= to_op`, and the walk steps op by op with no `.await`, + // so a peer sending `from_op = 1` against a large compacted frontier pins + // the pump against a 10 ms tick. Nothing at or below the watermark is + // servable anyway. The partition arm jumps to `retained_from` likewise. + let mut from_op = header.from_op.max(journal.snapshot_op() + 1); #[allow(clippy::cast_possible_truncation)] while from_op <= to_op && journal.header(from_op as usize).is_none() { from_op += 1; @@ -3741,7 +3853,7 @@ where Command2::RangeEvicted, header.nonce, from_op, - header.namespace, + header.group, ) .await; self.send_repair_range_reply( @@ -3751,7 +3863,7 @@ where Command2::RepairDone, header.nonce, header.from_op.saturating_sub(1), - header.namespace, + header.group, ) .await; return; @@ -3764,7 +3876,7 @@ where Command2::RangeEvicted, header.nonce, from_op, - header.namespace, + header.group, ) .await; } @@ -3793,16 +3905,13 @@ where Command2::RepairDone, header.nonce, served_through, - header.namespace, + header.group, ) .await; return; } - let Some(partition) = planes - .1 - .0 - .get_mut_by_ns(&IggyNamespace::from_raw(header.namespace)) - else { + let namespace = IggyNamespace::from_raw(header.group); + let Some(partition) = planes.1.0.get_mut_by_ns(&namespace) else { return; }; if !partition.consensus().is_normal() { @@ -3817,7 +3926,6 @@ where // Defer instead: no RepairDone is sent, the rejoiner's stall retry // re-asks, and the local purge (one reconciler wake away) installs // the floor the fence below serves behind. - let namespace = IggyNamespace::from_raw(header.namespace); let committed_purge = self .plane .metadata() @@ -3832,7 +3940,7 @@ where self.metrics.record_partition_repair_serve_deferred(); tracing::debug!( shard = self.id, - namespace_raw = header.namespace, + namespace_raw = header.group, committed_purge, applied_purge = partition.applied_purge_generation(), "deferring repair serve until the committed purge applies locally" @@ -3876,7 +3984,7 @@ where Command2::RangeEvicted, header.nonce, retained_from, - header.namespace, + header.group, ) .await; from_op = retained_from; @@ -3899,12 +4007,12 @@ where Command2::RepairDone, header.nonce, served_through, - header.namespace, + header.group, ) .await; tracing::info!( shard = self.id, - namespace_raw = header.namespace, + namespace_raw = header.group, target, from_op = header.from_op, to_op, @@ -3916,7 +4024,7 @@ where /// Ingest one repaired prepare. Metadata journals it into the WAL (the /// commit walk at `RepairDone` applies it); partitions journal + stage it /// through the same apply path as live replication, minus fence and ack. - #[allow(clippy::future_not_send)] + #[allow(clippy::future_not_send, clippy::too_many_lines)] async fn on_repair_prepare(&self, msg: Message) where B: MessageBus, @@ -3930,7 +4038,7 @@ where tracing::debug!( shard = self.id, op = msg.header().0.op, - namespace_raw = msg.header().0.namespace, + namespace_raw = msg.header().0.group, "repair prepare received" ); // Convert to a live-prepare frame exactly once, here at the apply @@ -3946,7 +4054,7 @@ where let header = *msg.header(); let planes = self.plane.inner(); // Legacy acceptance: pre-upgrade metadata WAL entries were journaled - // before prepares stamped `consensus.namespace()`, and repair ships + // before prepares stamped `consensus.group()`, and repair ships // stored bytes verbatim, so without it a mixed-version metadata repair // re-ships the same 0-stamped entries forever. // @@ -3960,15 +4068,58 @@ where // being metadata mutations, so `is_metadata` alone is too narrow), which // is why both sites share it rather than re-deriving the set. let metadata_plane_op = header.operation.is_metadata_plane(); - let legacy_metadata_claim = header.namespace == 0 && metadata_plane_op; + let legacy_metadata_claim = header.group == 0 && metadata_plane_op; if let Some(ref consensus) = planes.0.consensus - && (consensus.namespace() == header.namespace || legacy_metadata_claim) + && (consensus.group() == header.group || legacy_metadata_claim) { let session = *self.metadata_repair.borrow(); let Some(session) = session else { return; }; - if header.op > session.to_op || header.op <= consensus.commit_min() { + if header.op > session.to_op { + return; + } + let is_primary = consensus.is_primary_for_view(consensus.view()); + let commit_min = consensus.commit_min(); + // In place: runs once per repaired prepare, and the clone is two Vecs. + let in_scope = consensus + .with_pending_view_log(|pending| { + repair_op_in_scope(Some(pending), is_primary, commit_min, header.op) + }) + .unwrap_or_else(|| repair_op_in_scope(None, is_primary, commit_min, header.op)); + if !in_scope { + return; + } + // Applies to both planes, and is why a backup parks a log at all. The + // view already decided which prepare belongs at this op; a different + // one forks the log. An op the parked log omits is unconstrained. + let disagrees = consensus + .with_pending_view_log(|pending| { + pending + .headers + .iter() + .chain(pending.committed_elsewhere.iter()) + .find(|expected| expected.op == header.op) + .is_some_and(|expected| expected.checksum != header.checksum) + }) + .unwrap_or(false); + if disagrees { + tracing::warn!( + shard = self.id, + op = header.op, + "discarding repaired prepare that disagrees with the merged log" + ); + return; + } + // Recompute both integrity fields before durable storage: everything + // above treats `header.checksum` as an opaque token, so a corrupted + // frame passes whenever its flipped value satisfies the comparisons. + if let Err(reason) = verify_prepare_integrity(&header, msg.as_slice()) { + tracing::warn!( + shard = self.id, + op = header.op, + "discarding repaired prepare: {reason}" + ); return; } let Some(journal) = planes.0.journal.as_ref() else { @@ -4017,7 +4168,7 @@ where shard = self.id, op = header.op, operation = ?header.operation, - namespace_raw = header.namespace, + namespace_raw = header.group, "dropping a metadata-plane repair prepare this shard cannot journal" ); return; @@ -4025,10 +4176,22 @@ where let Some(partition) = planes .1 .0 - .get_mut_by_ns(&IggyNamespace::from_raw(header.namespace)) + .get_mut_by_ns(&IggyNamespace::from_raw(header.group)) else { return; }; + // The partition arm reaches the WAL via `apply_repaired_prepare` with no + // view fence and no ack, so this is its only integrity gate. Without it a + // repaired partition prepare is journaled on the serving peer's word alone. + if let Err(reason) = verify_prepare_integrity(&header, msg.as_slice()) { + tracing::warn!( + shard = self.id, + op = header.op, + namespace_raw = header.group, + "discarding repaired partition prepare: {reason}" + ); + return; + } partition.apply_repaired_prepare(msg).await; } @@ -4050,7 +4213,7 @@ where let header = *msg.header(); let planes = self.plane.inner(); if let Some(ref consensus) = planes.0.consensus - && consensus.namespace() == header.namespace + && consensus.group() == header.group { let session = *self.metadata_repair.borrow(); let Some(session) = session else { @@ -4090,7 +4253,7 @@ where session.nonce, commit_min + 1, session.to_op, - header.namespace, + header.group, ) .await; } @@ -4143,14 +4306,13 @@ where // Counted BEFORE the `&mut partition` below exists, and only when an arm // is possible at all: see the StartView site for why the scan is gated // rather than replaced with a counter. - let transfers_inflight = if Self::may_arm_partition_transfer(&planes.1.0, header.namespace) - { + let transfers_inflight = if Self::may_arm_partition_transfer(&planes.1.0, header.group) { self.partition_transfers_inflight() } else { 0 }; let config = planes.1.0.config().clone(); - let namespace = IggyNamespace::from_raw(header.namespace); + let namespace = IggyNamespace::from_raw(header.group); let Some(partition) = planes.1.0.get_mut_by_ns(&namespace) else { return; }; @@ -4188,7 +4350,7 @@ where self.metrics.record_partition_repair_serve_deferred(); tracing::debug!( shard = self.id, - namespace_raw = header.namespace, + namespace_raw = header.group, committed_purge, applied_purge = partition.applied_purge_generation(), command = ?header.command, @@ -4227,7 +4389,7 @@ where // arming here would defeat its backoff. tracing::info!( shard = self.id, - namespace_raw = header.namespace, + namespace_raw = header.group, floor, to_op, peer = header.replica, @@ -4253,7 +4415,7 @@ where // the scheduled re-arm) owns recovery from here. tracing::info!( shard = self.id, - namespace_raw = header.namespace, + namespace_raw = header.group, floor, to_op, "partition repair floor refused; transfer in flight or scheduled" @@ -4264,7 +4426,7 @@ where if partition.repair.is_none() { tracing::info!( shard = self.id, - namespace_raw = header.namespace, + namespace_raw = header.group, through_op = header.op, "partition journal repair complete" ); @@ -4283,7 +4445,7 @@ where nonce, commit_min + 1, to_op, - header.namespace, + header.group, ) .await; } @@ -4319,8 +4481,9 @@ where h.nonce = nonce; h.from_op = from_op; h.to_op = to_op; - h.namespace = namespace; + h.group = namespace; h.size = size_of::() as u32; + h.seal(); }); if self .bus @@ -4393,8 +4556,9 @@ where h.replica = self_id; h.nonce = nonce; h.op = op; - h.namespace = namespace; + h.group = namespace; h.size = size_of::() as u32; + h.seal(); }); let _ = self .bus @@ -4402,120 +4566,532 @@ where .await; } - /// Start metadata tail journal-repair from `peer` when the commit walk - /// gap-stopped below the known frontier. Shared by `StartView` adoption - /// and the post-install step of a state transfer. + /// Partition-plane twin of [`Self::advance_pending_metadata_view`]. + /// + /// No `RequestPrepares` stream to arm: the partition journal is not durable + /// yet, so coverage either holds or a peer must retransmit. Same invariant + /// either way: the view does not start until this replica can serve its log. #[allow(clippy::future_not_send)] - async fn maybe_request_metadata_repair

(&self, consensus: &VsrConsensus, peer: u8) + async fn advance_pending_partition_view(&self, namespace: IggyNamespace) where B: MessageBus, - P: Pipeline, + MJ: JournalHandle, + ::Target: Journal< + ::Storage, + Entry = Message, + Header = PrepareHeader, + >, { - if consensus.is_normal() - && !consensus.is_transferring() - && consensus.commit_min() < consensus.commit_max() - && self.metadata_repair.borrow().is_none() - { - let nonce = iggy_common::random_id::get_uuid(); - let to_op = consensus.commit_max(); - let from_op = consensus.commit_min() + 1; - *self.metadata_repair.borrow_mut() = Some(MetadataRepairSession { - nonce, - to_op, - peer, - idle_ticks: 0, - }); - tracing::info!( - shard = self.id, - from_op, - to_op, - "metadata behind the group frontier; requesting repair" - ); - self.send_request_prepares( - consensus.cluster(), - consensus.replica(), - peer, - nonce, - from_op, - to_op, - consensus.namespace(), - ) - .await; + let partitions = self.plane.partitions(); + let started = { + let Some(partition) = partitions.get_mut_by_ns(&namespace) else { + return; + }; + if !partition + .consensus() + .is_primary_for_view(partition.consensus().view()) + { + return; + } + let Some(pending) = partition.consensus().pending_view_log() else { + return; + }; + // Before the scan can mean anything: the repair ingest skips an op it + // already holds a header for, so the scan would report a gap nothing + // fills. Backups reach this on StartView adoption; a primary-elect has no + // adoption to hang it off. + reconcile_partition_view_divergence(self.id, partition, &pending).await; + let consensus = partition.consensus(); + // Identity, not presence: see the metadata twin. The floor is the local + // commit point, the partition twin of the metadata snapshot floor: + // `evict_prefix` clears the header vec for the flushed (committed) + // prefix, so a survivor that flushes on every commit holds NO resident + // header for the op the merged window opens on. Demanding one parks the + // primary-elect in `ViewChange` forever, and the rotation lands + // primaryship on whichever replica still has its window resident -- a + // fresh rejoiner with nothing but repair-ingested entries, which then + // cannot serve the state transfer it itself needs. A committed op + // cannot diverge from the merged log, and its bytes stay serveable + // from the evicted ring or the flushed segments. + let missing = { + let journal = partition.log.journal(); + first_op_not_covered(&pending, consensus.commit_min(), |op| { + journal.inner.header_by_op(op) + }) + }; + if let Some(missing_op) = missing { + tracing::debug!( + shard = self.id, + namespace_raw = namespace.inner(), + missing_op, + op_head = pending.op_head, + "partition view change waiting on op {missing_op} before starting the view" + ); + return; + } + + let actions = consensus.start_pending_view(PlaneKind::Partitions); + let (local_actions, wire_actions) = split_local_actions(actions); + // Locals go to the partition dispatcher ONLY: `RebuildPipeline` + // executes there (`dispatch_vsr_actions` bails on `journal: None`) + // and `CommitJournal` is a no-op in both. + dispatch_partition_journal_actions(consensus, partition, &local_actions).await; + // `start_pending_view` flips this replica into `Normal` for the new + // view, so the `StartView` it emits advertises a view the superblock + // must already record. Same gate as the `on_do_view_change` and + // `on_start_view` partition arms. + if partition.persist_superblock_if_needed().await { + dispatch_vsr_actions::(consensus, None, &wire_actions).await; + dispatch_partition_journal_actions(consensus, partition, &wire_actions).await; + } + local_actions + .iter() + .any(|action| matches!(action, VsrAction::CommitJournal)) + }; + if started { + let config = partitions.config(); + if let Some(partition) = partitions.get_mut_by_ns(&namespace) { + partition.commit_journal(config).await; + } } } - #[allow(clippy::future_not_send, clippy::cast_possible_truncation)] - async fn send_request_state_transfer

( - &self, - consensus: &VsrConsensus, - target: u8, - nonce: u128, - ) where + /// Re-request the remaining repair window when the stream has gone quiet. + /// + /// Repair frames are fire-and-forget, so a lost one leaves the session armed + /// forever with the commit walk pinned below the frontier. + #[allow(clippy::future_not_send)] + async fn retry_stalled_metadata_repair

(&self, consensus: &VsrConsensus) + where B: MessageBus, P: Pipeline, { - let msg = - Message::::new(size_of::()) - .transmute_header(|_, h: &mut RequestStateTransferHeader| { - h.command = Command2::RequestStateTransfer; - h.cluster = consensus.cluster(); - h.replica = consensus.replica(); - h.nonce = nonce; - h.namespace = consensus.namespace(); - h.size = size_of::() as u32; - }); - let _ = self - .bus - .send_to_replica(target, msg.into_generic().into_frozen()) - .await; + // Stall retry (mirrors `tick_partitions`): a lost frame must not wedge it. + let repair_retry_ticks = self.repair_retry_ticks.get(); + let stalled = { + // `ViewChange` too: a parked view change repairs toward its merged log + // and cannot start until the window fills. Gating on `Normal` alone + // defers a dropped frame to the 500-tick escalation. + let repairing_view = + consensus.view_log_is_pending() && consensus.is_primary_for_view(consensus.view()); + let mut session = self.metadata_repair.borrow_mut(); + session.as_mut().and_then(|session| { + if !consensus.is_normal() && !repairing_view { + return None; + } + session.idle_ticks += 1; + if session.idle_ticks < repair_retry_ticks { + return None; + } + session.idle_ticks = 0; + Some((session.peer, session.nonce, session.to_op)) + }) + }; + if let Some((peer, nonce, to_op)) = stalled { + // Primary-elect only. Its window starts at the merged log's commit + // point, which can sit below local `commit_min` (the headers inherited + // from senders behind the canonical log_view live there), so + // `commit_min + 1` would skip them. A backup's parked `StartView` + // suffix is only a verification reference; resuming from its commit + // point would restart at the view's opening head, not at the gap. + let from_op = consensus + .is_primary_for_view(consensus.view()) + .then(|| consensus.with_pending_view_log(|pending| pending.commit_max.max(1))) + .flatten() + .unwrap_or_else(|| consensus.commit_min() + 1); + if from_op <= to_op { + tracing::info!( + shard = self.id, + from_op, + to_op, + peer, + "metadata repair stalled; re-requesting remaining window" + ); + self.send_request_prepares( + consensus.cluster(), + consensus.replica(), + peer, + nonce, + from_op, + to_op, + consensus.group(), + ) + .await; + } + } } - /// Answer a `RequestStateTransfer`: `offer = None` sends a header-only - /// `available = 0` (the requester falls back to journal repair or - /// retries elsewhere); an offer ships its encoded state manifest as the - /// frame body. - #[allow( - clippy::future_not_send, - clippy::cast_possible_truncation, - clippy::too_many_arguments - )] - async fn send_state_transfer_target( - &self, - cluster: u128, - self_id: u8, - target: u8, - nonce: u128, - namespace: u64, - descriptor: TransferDescriptor<'_>, - ) where + /// Compare this replica's log against the headers the view decided, and drop or + /// report where they disagree. + /// + /// Without this, divergence is silent and permanent: the replica acks with its + /// own checksum, the primary rejects the ack, and journal repair skips an op + /// it already has a header for. + /// + /// Both roles. A backup runs it against the `StartView` suffix it adopted, the + /// primary-elect against the merged log before the coverage scan in + /// [`Self::advance_pending_metadata_view`]. The merge does NOT reconcile the + /// primary's log for it: nothing installs the merged headers into the journal, + /// `RebuildPipeline` reads the pipeline back out of it, and `CommitJournal` + /// applies whatever sits at each op up to the merged commit point. + /// + /// The split at the announced commit point is what matters. Above it a + /// disagreement is ordinary, so the entry is dropped and the primary's + /// retransmission refills the range. At or below it, this replica applied + /// something the view says was different, which only state transfer fixes, so + /// it is reported and left alone. + /// + /// Truncation uses `Journal::truncate_from`, not `drain`: `drain` advances + /// `snapshot_op` past what it removed, marking ops that must stay refillable + /// as evictable. + #[allow(clippy::future_not_send)] + async fn reconcile_metadata_view_divergence(&self) + where B: MessageBus, + MJ: JournalHandle, + ::Target: Journal< + ::Storage, + Entry = Message, + Header = PrepareHeader, + >, + M: MetadataStm, { - let manifest = descriptor - .offer - .map(|(entries, _)| consensus::encode_state_manifest(entries)); - let total_size = - size_of::() + manifest.as_ref().map_or(0, Vec::len); - let mut msg = Message::::new(total_size); - if let Some(manifest) = &manifest { - msg.as_mut_slice()[size_of::()..].copy_from_slice(manifest); - } - let msg = msg.transmute_header(|_, h: &mut StateTransferTargetHeader| { - h.command = Command2::StateTransferTarget; - h.cluster = cluster; - h.replica = self_id; - h.nonce = nonce; - h.namespace = namespace; - h.size = total_size as u32; - // The serving replica's own progress travels with every descriptor, - // available or not: it is what lets a receiver refuse an offer from - // a replica that knows less than it does. - h.view = descriptor.view; - h.commit_max = descriptor.commit_max; - h.unavailable_transient = u8::from(descriptor.transient); - if let Some((_, commit_op)) = descriptor.offer { - h.available = 1; + let metadata = self.plane.metadata(); + let Some(ref consensus) = metadata.consensus else { + return; + }; + let Some(pending) = consensus.pending_view_log() else { + return; + }; + let Some(journal) = metadata.journal.as_ref() else { + return; + }; + + // Truncation is safe only above what this replica has *applied*, which is + // not the view's commit point: `pending.commit_max` is the new primary's + // number and a backup can sit above it. Splitting on the view's number + // would drop already-executed ops with no rollback, and silently. + let applied_floor = pending.commit_max.max(consensus.commit_min()); + + let mut repairable_from: Option = None; + for canonical in &pending.headers { + let Some(local) = usize::try_from(canonical.op) + .ok() + .and_then(|slot| journal.handle().header(slot)) + else { + continue; + }; + if header_is_view_entry(&local, canonical) { + continue; + } + if canonical.op <= applied_floor { + tracing::error!( + shard = self.id, + op = canonical.op, + view = consensus.view(), + commit_max = pending.commit_max, + commit_min = consensus.commit_min(), + local_checksum = local.checksum, + canonical_checksum = canonical.checksum, + "committed op {} disagrees with the view that just started; this replica \ + applied a different op as committed and cannot be reconciled by log repair", + canonical.op + ); + continue; + } + repairable_from = Some(repairable_from.map_or(canonical.op, |op| op.min(canonical.op))); + } + + // A suffix ABOVE the announced head is named by nobody, so the loop cannot + // see it, and it is exactly the log that AGREES in-window (restart and + // re-adopt), where `repairable_from` never arms. Adoption drops the head + // under it, and the next prepare at `op_head + 1` then collides in `append`, + // which refuses the slot even when the ops match. Floored at the applied + // point too: an executed op is not rollback-able whatever the head says. + let above_head = pending.op_head.max(applied_floor) + 1; + if journal + .handle() + .last_op() + .is_some_and(|last_op| last_op >= above_head) + { + repairable_from = Some(repairable_from.map_or(above_head, |op| op.min(above_head))); + } + + let Some(from_op) = repairable_from else { + return; + }; + // SERIALIZATION: the drain guard excludes `truncate_from` against `drain`, + // NOT against an append; that is metadata's private `journal_gate`. It holds + // by call-site placement, not construction: the pump is single-threaded, and + // both callers run with a view change parked, so no submit is admitted and no + // repair prepare is in flight for these ops. Routing shard-side journal + // mutations through gate-taking metadata methods would make it structural. + match journal.handle().truncate_from(from_op).await { + Ok(removed) => { + // The snapshot's `(op, commit)` tag does not move when entries are + // removed under it, so the next `DoViewChange` would advertise the + // dropped headers and offer bodies this replica cannot serve. + consensus.invalidate_local_dvc_suffix(); + tracing::warn!( + shard = self.id, + from_op, + removed, + op_head = pending.op_head, + view = consensus.view(), + "dropped {removed} uncommitted entries from op {from_op} that disagreed with \ + the view's log; the primary's retransmission refills the range" + ); + } + Err(error) => { + tracing::error!( + shard = self.id, + from_op, + %error, + "could not drop the diverging uncommitted entries from op {from_op}; journal \ + repair skips ops it already holds a header for, so this replica will not \ + converge at those ops until it is restarted" + ); + } + } + } + + /// Drive a parked view change to completion. + /// + /// A DVC quorum decides the log before this replica necessarily holds it, so + /// the merged log parks in consensus and this replica stays in `ViewChange`, + /// announcing and preparing nothing: `StartView` promises it can serve every + /// op it names, and a backup adopting that head asks for the bodies at once. + /// + /// Check coverage, then start the view or pull missing bodies from a peer that + /// offered them in its DVC. Only those peers: a cleared present bit means the + /// body was never held or cannot be read back. + #[allow(clippy::future_not_send)] + async fn advance_pending_metadata_view(&self) + where + B: MessageBus, + MJ: JournalHandle, + ::Target: Journal< + ::Storage, + Entry = Message, + Header = PrepareHeader, + >, + M: MetadataStm, + { + let metadata = self.plane.metadata(); + let Some(ref consensus) = metadata.consensus else { + return; + }; + // Primary-elect only. A backup's parked `StartView` suffix is only what + // its ingest verifies bodies against; driving repair from it would put a + // rejoining node on the tail-repair path when its gap sits below every + // peer's retention floor, racing the view probe that picks state transfer. + if !consensus.is_primary_for_view(consensus.view()) { + return; + } + // Before the coverage scan, and a primary-elect's only shot at it: the repair + // ingest skips an op it already holds a header for, so a diverging entry would + // never be replaced. No-op when nothing diverges. + self.reconcile_metadata_view_divergence().await; + let Some(pending) = consensus.pending_view_log() else { + return; + }; + let Some(journal) = metadata.journal.as_ref() else { + return; + }; + + // Floor on what this replica can be asked to hold before starting the view. + // Entries at or below the snapshot watermark are compacted, so no repair puts + // one back: demanding one parks the view change forever on an op already + // applied and durable in the snapshot. + let repair_floor = journal.handle().snapshot_op(); + let missing = first_op_not_covered(&pending, repair_floor, |op| { + usize::try_from(op) + .ok() + .and_then(|slot| journal.handle().header(slot)) + .map(|header| *header) + }); + + let Some(missing_op) = missing else { + let actions = consensus.start_pending_view(PlaneKind::Metadata); + tracing::info!( + shard = self.id, + view = consensus.view(), + op_head = pending.op_head, + commit_max = pending.commit_max, + "merged log is locally serveable; starting the view" + ); + if metadata.persist_superblock_if_needed(consensus).await { + dispatch_vsr_actions(consensus, metadata.journal.as_ref(), &actions).await; + } + if actions + .iter() + .any(|action| matches!(action, VsrAction::CommitJournal)) + && !consensus.is_transferring() + { + metadata.commit_journal().await; + } + return; + }; + + if self.metadata_repair.borrow().is_some() { + // Stream already running; the stall retry covers it drying up. + return; + } + let sources = consensus.pending_view_body_sources(missing_op); + let Some(peer) = sources.first().copied() else { + // The merge only returns a startable log when some replica offered each + // body, so an empty source list means that offer was withdrawn (peer + // restarted, or moved on). Let the view-change timeout escalate. + tracing::warn!( + shard = self.id, + missing_op, + "no replica offers op {missing_op} for the merged log; view change is stalled" + ); + return; + }; + + let nonce = iggy_common::random_id::get_uuid(); + *self.metadata_repair.borrow_mut() = Some(MetadataRepairSession { + nonce, + to_op: pending.op_head, + peer, + idle_ticks: 0, + }); + tracing::info!( + shard = self.id, + missing_op, + peer, + to_op = pending.op_head, + "repairing toward the merged log before starting the view" + ); + self.send_request_prepares( + consensus.cluster(), + consensus.replica(), + peer, + nonce, + missing_op, + pending.op_head, + consensus.group(), + ) + .await; + } + + /// Start metadata tail journal-repair from `peer` when the commit walk + /// gap-stopped below the known frontier. Shared by `StartView` adoption + /// and the post-install step of a state transfer. + #[allow(clippy::future_not_send)] + async fn maybe_request_metadata_repair

(&self, consensus: &VsrConsensus, peer: u8) + where + B: MessageBus, + P: Pipeline, + { + if consensus.is_normal() + && !consensus.is_transferring() + && consensus.commit_min() < consensus.commit_max() + && self.metadata_repair.borrow().is_none() + { + let nonce = iggy_common::random_id::get_uuid(); + let to_op = consensus.commit_max(); + let from_op = consensus.commit_min() + 1; + *self.metadata_repair.borrow_mut() = Some(MetadataRepairSession { + nonce, + to_op, + peer, + idle_ticks: 0, + }); + tracing::info!( + shard = self.id, + from_op, + to_op, + "metadata behind the group frontier; requesting repair" + ); + self.send_request_prepares( + consensus.cluster(), + consensus.replica(), + peer, + nonce, + from_op, + to_op, + consensus.group(), + ) + .await; + } + } + + #[allow(clippy::future_not_send, clippy::cast_possible_truncation)] + async fn send_request_state_transfer

( + &self, + consensus: &VsrConsensus, + target: u8, + nonce: u128, + ) where + B: MessageBus, + P: Pipeline, + { + let msg = + Message::::new(size_of::()) + .transmute_header(|_, h: &mut RequestStateTransferHeader| { + h.command = Command2::RequestStateTransfer; + h.cluster = consensus.cluster(); + h.replica = consensus.replica(); + h.nonce = nonce; + h.group = consensus.group(); + h.size = size_of::() as u32; + h.seal(); + }); + let _ = self + .bus + .send_to_replica(target, msg.into_generic().into_frozen()) + .await; + } + + /// Answer a `RequestStateTransfer`: `offer = None` sends a header-only + /// `available = 0` (the requester falls back to journal repair or + /// retries elsewhere); an offer ships its encoded state manifest as the + /// frame body. + #[allow( + clippy::future_not_send, + clippy::cast_possible_truncation, + clippy::too_many_arguments + )] + async fn send_state_transfer_target( + &self, + cluster: u128, + self_id: u8, + target: u8, + nonce: u128, + namespace: u64, + descriptor: TransferDescriptor<'_>, + ) where + B: MessageBus, + { + let manifest = descriptor + .offer + .map(|(entries, _)| consensus::encode_state_manifest(entries)); + let total_size = + size_of::() + manifest.as_ref().map_or(0, Vec::len); + let mut msg = Message::::new(total_size); + if let Some(manifest) = &manifest { + msg.as_mut_slice()[size_of::()..].copy_from_slice(manifest); + } + let msg = msg.transmute_header(|_, h: &mut StateTransferTargetHeader| { + h.command = Command2::StateTransferTarget; + h.cluster = cluster; + h.replica = self_id; + h.nonce = nonce; + h.group = namespace; + h.size = total_size as u32; + // The serving replica's own progress travels with every descriptor, + // available or not: it is what lets a receiver refuse an offer from + // a replica that knows less than it does. + h.view = descriptor.view; + h.commit_max = descriptor.commit_max; + h.unavailable_transient = u8::from(descriptor.transient); + if let Some((_, commit_op)) = descriptor.offer { + h.available = 1; h.commit_op = commit_op; } + h.seal(); }); let _ = self .bus @@ -4547,11 +5123,12 @@ where h.cluster = cluster; h.replica = self_id; h.nonce = nonce; - h.namespace = namespace; + h.group = namespace; h.artifact = artifact; h.offset = offset; h.len = len; h.size = size_of::() as u32; + h.seal(); }); let _ = self .bus @@ -4579,7 +5156,7 @@ where .0 .consensus .as_ref() - .is_some_and(|consensus| consensus.namespace() == header.namespace); + .is_some_and(|consensus| consensus.group() == header.group); if !metadata_frame { return self.on_partition_request_state_transfer(msg).await; } @@ -4599,7 +5176,7 @@ where let cached = self .state_transfer_offers .borrow_mut() - .get_mut(&(header.namespace, header.replica)) + .get_mut(&(header.group, header.replica)) .filter(|served| served.nonce == header.nonce) .and_then(|served| { let ServedOffer::Metadata(offer) = &served.offer else { @@ -4623,7 +5200,7 @@ where self_id, header.replica, header.nonce, - header.namespace, + header.group, TransferDescriptor::available( &offer.manifest(), offer.commit_op, @@ -4651,7 +5228,7 @@ where self_id, header.replica, header.nonce, - header.namespace, + header.group, TransferDescriptor::available( &offer.manifest(), offer.commit_op, @@ -4661,7 +5238,7 @@ where ) .await; self.state_transfer_offers.borrow_mut().insert( - (header.namespace, header.replica), + (header.group, header.replica), ServedStateTransfer { nonce: header.nonce, offer: ServedOffer::Metadata(offer), @@ -4686,7 +5263,7 @@ where self_id, header.replica, header.nonce, - header.namespace, + header.group, TransferDescriptor::unavailable( false, consensus.view(), @@ -4731,7 +5308,7 @@ where .0 .consensus .as_ref() - .is_some_and(|consensus| consensus.namespace() == header.namespace); + .is_some_and(|consensus| consensus.group() == header.group); if !metadata_frame { return self.on_partition_state_transfer_target(msg).await; } @@ -4909,7 +5486,7 @@ where consensus.replica(), peer, nonce, - consensus.namespace(), + consensus.group(), artifact, offset, len, @@ -5022,7 +5599,7 @@ where .0 .consensus .as_ref() - .is_some_and(|consensus| consensus.namespace() == header.namespace); + .is_some_and(|consensus| consensus.group() == header.group); if !metadata_frame { return self.on_partition_request_state_chunk(msg).await; } @@ -5044,7 +5621,7 @@ where let reply = { let mut offers = self.state_transfer_offers.borrow_mut(); let served = offers - .get_mut(&(header.namespace, header.replica)) + .get_mut(&(header.group, header.replica)) .filter(|served| served.nonce == header.nonce); served.map_or( Some(ChunkReply::Unavailable { transient: true }), @@ -5094,10 +5671,11 @@ where h.cluster = cluster; h.replica = self_id; h.nonce = header.nonce; - h.namespace = header.namespace; + h.group = header.group; h.artifact = header.artifact; h.offset = header.offset; h.size = total_size as u32; + h.seal(); }, ))) }, @@ -5122,7 +5700,7 @@ where self_id, header.replica, header.nonce, - header.namespace, + header.group, TransferDescriptor::unavailable( transient, consensus.view(), @@ -5164,7 +5742,7 @@ where .0 .consensus .as_ref() - .is_some_and(|consensus| consensus.namespace() == header.namespace); + .is_some_and(|consensus| consensus.group() == header.group); if !metadata_frame { return self.on_partition_state_chunk(msg).await; } @@ -5504,6 +6082,16 @@ where }; let consensus = partition.consensus(); + // Only while a view change is live. A `Normal` tick has no consumer: + // `start_election` records no DoViewChange, and every path that does + // either refreshes at its own call site (the SVC and DVC handlers, + // still `Normal` at that point) or runs in `ViewChange`. + // + // Ungated, this rebuilt a 128-entry window every 10 ms per advancing + // partition: a linear `header_by_op` scan per entry plus 32 KiB. + if consensus.status() != Status::Normal { + refresh_partition_dvc_suffix(partition); + } let actions = consensus.tick(PlaneKind::Partitions); // The tick emits view-scoped sends (heartbeats, view-change // retransmits), so it persists first like every dispatch site; @@ -5517,6 +6105,9 @@ where dispatch_partition_journal_actions(consensus, partition, &wire_actions).await; } + // Finish a view change whose quorum decided ahead of the local log. + self.advance_pending_partition_view(namespace).await; + // Stall retry: repair frames are fire-and-forget, so a lost // frame (or a peer that went silent mid-stream) would leave the // session armed forever with commit_min pinned below commit_max. @@ -5787,7 +6378,7 @@ where let Some(partition) = planes .1 .0 - .get_mut_by_ns(&IggyNamespace::from_raw(header.namespace)) + .get_mut_by_ns(&IggyNamespace::from_raw(header.group)) else { return; }; @@ -5801,7 +6392,7 @@ where let cached = self .state_transfer_offers .borrow_mut() - .get_mut(&(header.namespace, header.replica)) + .get_mut(&(header.group, header.replica)) .filter(|served| served.nonce == header.nonce) .and_then(|served| { let ServedOffer::Partition(offer) = &served.offer else { @@ -5818,14 +6409,14 @@ where Some(offer) => { tracing::debug!( shard = self.id, - namespace_raw = header.namespace, + namespace_raw = header.group, requester = header.replica, "re-answering a partition state transfer request from the offer \ already served" ); Some(offer) } - None if !self.may_serve_another_partition_transfer(header.namespace) => { + None if !self.may_serve_another_partition_transfer(header.group) => { // Admission control, because the served-payload budget is a // BYTE budget and the pulls that overrun it do not degrade // gracefully. Each concurrent pull holds a different segment @@ -5838,7 +6429,7 @@ where // Refusing the surplus is what makes the admitted ones finish. tracing::info!( shard = self.id, - namespace_raw = header.namespace, + namespace_raw = header.group, requester = header.replica, "already serving as many partition transfers as the served-payload \ budget holds; refusing until one completes" @@ -5849,7 +6440,7 @@ where self_id, header.replica, header.nonce, - header.namespace, + header.group, TransferDescriptor::unavailable(true, view, commit_max), ) .await; @@ -5861,15 +6452,15 @@ where // and holds nothing in the offers map meanwhile. self.partition_offer_builds .borrow_mut() - .insert(header.namespace, 0); + .insert(header.group, 0); match partition.state_transfer_offer(&config).await { Ok(offer) => { self.partition_offer_builds .borrow_mut() - .remove(&header.namespace); + .remove(&header.group); tracing::info!( shard = self.id, - namespace_raw = header.namespace, + namespace_raw = header.group, requester = header.replica, commit_op = offer.commit_op, artifacts = offer.artifact_count(), @@ -5877,7 +6468,7 @@ where "serving partition state transfer" ); self.state_transfer_offers.borrow_mut().insert( - (header.namespace, header.replica), + (header.group, header.replica), ServedStateTransfer { nonce: header.nonce, offer: ServedOffer::Partition(Rc::clone(&offer)), @@ -5905,12 +6496,12 @@ where if !building { self.partition_offer_builds .borrow_mut() - .remove(&header.namespace); + .remove(&header.group); } let transient = reason.transient(); tracing::info!( shard = self.id, - namespace_raw = header.namespace, + namespace_raw = header.group, requester = header.replica, transient, %reason, @@ -5922,7 +6513,7 @@ where self_id, header.replica, header.nonce, - header.namespace, + header.group, TransferDescriptor::unavailable(transient, view, commit_max), ) .await; @@ -5946,7 +6537,7 @@ where self_id, header.replica, header.nonce, - header.namespace, + header.group, TransferDescriptor::available(&offer.manifest(), offer.commit_op, view, commit_max), ) .await; @@ -5985,11 +6576,7 @@ where return; } let planes = self.plane.inner(); - let Some(partition) = planes - .1 - .0 - .get_by_ns(&IggyNamespace::from_raw(header.namespace)) - else { + let Some(partition) = planes.1.0.get_by_ns(&IggyNamespace::from_raw(header.group)) else { return; }; let cluster = partition.consensus().cluster(); @@ -6001,7 +6588,7 @@ where let attempt = 'attempt: { let mut offers = self.state_transfer_offers.borrow_mut(); let served = offers - .get_mut(&(header.namespace, header.replica)) + .get_mut(&(header.group, header.replica)) .filter(|served| served.nonce == header.nonce); let Some(served) = served else { break 'attempt ChunkAttempt::Reply(Some(ChunkReply::Unavailable { @@ -6025,7 +6612,7 @@ where match self .served_segment_cache .borrow_mut() - .get(header.namespace, source.entry.checksum) + .get(header.group, source.entry.checksum) { Some(payload) => { segment_payload = payload; @@ -6062,7 +6649,7 @@ where // the serving side's proof that the pull ran to completion. tracing::info!( shard = self.id, - namespace_raw = header.namespace, + namespace_raw = header.group, requester = header.replica, "partition state transfer fully served" ); @@ -6077,10 +6664,14 @@ where h.cluster = cluster; h.replica = self_id; h.nonce = header.nonce; - h.namespace = header.namespace; + h.group = header.group; h.artifact = header.artifact; h.offset = header.offset; h.size = total_size as u32; + // `StateChunk` is `FRAME_SEALED`: the receiver's router + // drops an unsealed frame before any handler sees it, so + // a missing seal starves the pull silently. + h.seal(); }, )))) }; @@ -6100,7 +6691,7 @@ where let reason = match loaded { Ok(bytes) => { self.served_segment_cache.borrow_mut().insert( - header.namespace, + header.group, entry.checksum, Rc::new(bytes), self.served_segment_cache_bytes_max.get(), @@ -6116,7 +6707,7 @@ where let transient = reason.transient(); tracing::warn!( shard = self.id, - namespace_raw = header.namespace, + namespace_raw = header.group, artifact = header.artifact, path = %log_path, transient, @@ -6125,7 +6716,7 @@ where ); self.state_transfer_offers .borrow_mut() - .remove(&(header.namespace, header.replica)); + .remove(&(header.group, header.replica)); // The builder cache too: it is keyed by commit_op alone, // and GC unlinks files WITHOUT a commit, so the restarted // requester would otherwise be handed the same offer with @@ -6145,7 +6736,7 @@ where Some(ChunkReply::Unavailable { transient }) => { tracing::info!( shard = self.id, - namespace_raw = header.namespace, + namespace_raw = header.group, requester = header.replica, transient, "partition chunk request for an unknown offer; telling requester to restart" @@ -6155,7 +6746,7 @@ where self_id, header.replica, header.nonce, - header.namespace, + header.group, // Usually TRANSIENT -- retention GC'd a served segment, or // the offer aged out between two chunks, and the restarted // session converges -- but a load that failed on a local @@ -6168,7 +6759,7 @@ where None => { tracing::warn!( shard = self.id, - namespace_raw = header.namespace, + namespace_raw = header.group, requester = header.replica, artifact = header.artifact, offset = header.offset, @@ -6414,7 +7005,7 @@ where { tracing::info!( shard = self.id, - namespace_raw = partition.consensus().namespace(), + namespace_raw = partition.consensus().group(), cap = Self::PARTITION_TRANSFERS_INFLIGHT_MAX, "partition transfer slots exhausted; deferring this arm" ); @@ -6467,7 +7058,7 @@ where let to_op = consensus.commit_max(); let cluster = consensus.cluster(); let self_id = consensus.replica(); - let namespace = consensus.namespace(); + let namespace = consensus.group(); partition.repair = Some(partitions::RepairSession { nonce, to_op, @@ -6507,7 +7098,7 @@ where let Some(partition) = planes .1 .0 - .get_mut_by_ns(&IggyNamespace::from_raw(header.namespace)) + .get_mut_by_ns(&IggyNamespace::from_raw(header.group)) else { return; }; @@ -6530,7 +7121,7 @@ where let transient = header.unavailable_transient == 1; tracing::info!( shard = self.id, - namespace_raw = header.namespace, + namespace_raw = header.group, peer = header.replica, transient, "partition transfer peer cannot serve; backing off before re-arming" @@ -6576,7 +7167,7 @@ where if header.commit_op > header.commit_max { tracing::warn!( shard = self.id, - namespace_raw = header.namespace, + namespace_raw = header.group, peer = header.replica, serving_commit_op = header.commit_op, serving_commit_max = header.commit_max, @@ -6590,7 +7181,7 @@ where if header.view < local_view || header.commit_max < local_commit_max { tracing::warn!( shard = self.id, - namespace_raw = header.namespace, + namespace_raw = header.group, peer = header.replica, serving_view = header.view, serving_commit_max = header.commit_max, @@ -6611,7 +7202,7 @@ where Err(error) => { tracing::warn!( shard = self.id, - namespace_raw = header.namespace, + namespace_raw = header.group, %error, "partition transfer manifest rejected" ); @@ -6640,7 +7231,7 @@ where if !kind_capped || total_len > Self::PARTITION_TRANSFER_TOTAL_LEN_MAX { tracing::warn!( shard = self.id, - namespace_raw = header.namespace, + namespace_raw = header.group, total_len, "partition transfer manifest exceeds artifact caps; refusing descriptor" ); @@ -6660,7 +7251,7 @@ where if !reused.is_empty() { tracing::info!( shard = self.id, - namespace_raw = header.namespace, + namespace_raw = header.group, peer = header.replica, adopted = reused.len(), artifacts = entries.len(), @@ -6707,7 +7298,7 @@ where if consensus.state_transfer_stage() == consensus::StateTransferStage::AwaitingTarget { consensus.set_state_transfer_stage(consensus::StateTransferStage::Fetching); } - self.on_partition_transfer_progress(header.namespace).await; + self.on_partition_transfer_progress(header.group).await; } /// Receive one partition chunk; spill a completed segment artifact, and @@ -6724,7 +7315,7 @@ where let Some(partition) = planes .1 .0 - .get_mut_by_ns(&IggyNamespace::from_raw(header.namespace)) + .get_mut_by_ns(&IggyNamespace::from_raw(header.group)) else { return; }; @@ -6758,7 +7349,7 @@ where session.idle_ticks = 0; } partition.note_transfer_progress(); - self.on_partition_transfer_progress(header.namespace).await; + self.on_partition_transfer_progress(header.group).await; } /// Drive an in-flight partition transfer: spill newly completed segment @@ -7121,7 +7712,7 @@ where // cannot tell the two apart; the serving node's own logs can. tracing::warn!( shard = self.id, - namespace_raw = partition.consensus().namespace(), + namespace_raw = partition.consensus().group(), peer, refusals, "partition state transfer has been refused {refusals} times in a row; the peer \ @@ -7169,7 +7760,7 @@ where }; tracing::info!( shard = self.id, - namespace_raw = partition.consensus().namespace(), + namespace_raw = partition.consensus().group(), failures, next_peer, after_ticks, @@ -7315,7 +7906,7 @@ where let metadata_served = metadata.consensus.as_ref().is_some_and(|consensus| { offers .keys() - .any(|(namespace, _)| *namespace == consensus.namespace()) + .any(|(namespace, _)| *namespace == consensus.group()) }); if !metadata_served { metadata.clear_state_transfer_offer_cache(); @@ -7360,6 +7951,10 @@ where return; }; + // See the partition tick: no snapshot consumer on a `Normal` tick. + if consensus.status() != Status::Normal { + refresh_metadata_dvc_suffix(consensus, metadata.journal.as_ref()); + } let actions = consensus.tick(PlaneKind::Metadata); let (local_actions, wire_actions) = split_local_actions(actions); @@ -7383,6 +7978,9 @@ where // nothing is stranded. metadata.resume_stranded_commits().await; + self.advance_pending_metadata_view().await; + self.expire_idle_state_transfer_offers(); + // Stall retry for an in-flight state transfer: descriptor or chunk // frames are fire-and-forget, so a lost one must not wedge the // session (and the boot flow behind it) forever. @@ -7438,45 +8036,7 @@ where } } - // Stall retry, mirroring `tick_partitions`: a lost repair frame must - // not wedge the session forever. - let repair_retry_ticks = self.repair_retry_ticks.get(); - let stalled = { - let mut session = self.metadata_repair.borrow_mut(); - session.as_mut().and_then(|session| { - if !consensus.is_normal() { - return None; - } - session.idle_ticks += 1; - if session.idle_ticks < repair_retry_ticks { - return None; - } - session.idle_ticks = 0; - Some((session.peer, session.nonce, session.to_op)) - }) - }; - if let Some((peer, nonce, to_op)) = stalled { - let from_op = consensus.commit_min() + 1; - if from_op <= to_op { - tracing::info!( - shard = self.id, - from_op, - to_op, - peer, - "metadata repair stalled; re-requesting remaining window" - ); - self.send_request_prepares( - consensus.cluster(), - consensus.replica(), - peer, - nonce, - from_op, - to_op, - consensus.namespace(), - ) - .await; - } - } + self.retry_stalled_metadata_repair(consensus).await; } } @@ -7498,7 +8058,7 @@ where view = consensus.view(), op = consensus.sequencer().current_sequence(), commit = consensus.commit_max(), - namespace = consensus.namespace(), + namespace = consensus.group(), "answering stale-view heartbeat with StartView" ); // Unsolicited, answering a stale-view heartbeat rather than a probe, so there is @@ -7511,28 +8071,573 @@ where commit: consensus.commit_max(), incarnation: 0, target: None, - namespace: consensus.namespace(), + group: consensus.group(), + // Correcting a peer on a stale view, not concluding a view change: this + // publishes the settled frontier, which the peer reaches by repair. + suffix: Vec::new(), }; dispatch_vsr_actions::(consensus, None, &[action]).await; } -/// Re-stamp a stored prepare with the current view before retransmission. -/// After a view change the primary re-sends its uncommitted suffix as its -/// own prepares (VSR), but the journal keeps the original view stamp and -/// `replicate_preflight` fences `header.view < view` as deposed-primary -/// traffic -- a verbatim replay of the stored bytes would be ignored -/// forever, wedging the commit walk on every peer. The stored buffer is -/// shared with the journal, so the patch runs on an owned copy. -fn restamp_prepare_view(stored: &[u8], view: u32) -> Option> { - const VIEW_OFFSET: usize = std::mem::offset_of!(PrepareHeader, view); - let mut owned = server_common::iobuf::Owned::::copy_from_slice(stored); - owned.as_mut_slice()[VIEW_OFFSET..VIEW_OFFSET + std::mem::size_of::()] - .copy_from_slice(&view.to_ne_bytes()); - Message::::try_from(owned) - .ok() - .map(Message::into_frozen) -} - +/// Rebuild the new primary's pipeline over `from_op..=to_op` from local journal +/// headers. +/// +/// A gap means the caller started the view before its journal could serve the +/// merged log: a bug in the transition, not a data condition. Nothing is +/// truncated, because truncating to the last findable op discards ops committed +/// on a quorum and already acknowledged. The pipeline is left short, the commit +/// walk stalls at the gap, and repair fills it in. +fn rebuild_pipeline_entries( + consensus: &VsrConsensus, + self_id: u8, + from_op: u64, + to_op: u64, + header_at: impl Fn(u64) -> Option, +) where + B: MessageBus, + P: Pipeline, +{ + let mut gap_at = None; + let entries: Vec<_> = (from_op..=to_op) + .map_while(|op| { + let header = header_at(op).or_else(|| { + gap_at = Some(op); + None + })?; + // Lift the monotonic timestamp floor to the rebuilt log so + // post-view-change prepares cannot stamp below committed ones. + consensus.observe_prepare_timestamp(header.timestamp); + let mut entry = consensus::PipelineEntry::new(header); + entry.add_ack(self_id); + Some(entry) + }) + .collect(); + + if let Some(missing_op) = gap_at { + tracing::error!( + replica = self_id, + missing_op, + range_start = from_op, + range_end = to_op, + rebuilt = entries.len(), + "RebuildPipeline: journal gap at op {missing_op} while starting a view; leaving the \ + sequencer at {to_op} and stalling the commit walk. Truncating here would discard ops \ + the view change proved recoverable." + ); + } + + let mut pipeline = consensus.pipeline().borrow_mut(); + for entry in entries { + pipeline.push(entry); + } +} + +/// Snapshot this replica's uncommitted suffix into consensus, if the journal has +/// moved since the last snapshot. +/// +/// Called before every handler that could start or join a view change: consensus +/// records its own `DoViewChange` there and has no journal to read. A stale +/// snapshot is never reused; consensus tags it with its `(op, commit)` and falls +/// back to an empty suffix, stalling the view change rather than nacking an op +/// since acquired. +fn refresh_metadata_dvc_suffix(consensus: &VsrConsensus, journal: Option<&MJ>) +where + B: MessageBus, + P: Pipeline, + MJ: JournalHandle, + ::Target: Journal< + ::Storage, + Entry = Message, + Header = PrepareHeader, + >, +{ + if !consensus.local_dvc_suffix_stale() { + return; + } + let op = consensus.sequencer().current_sequence(); + let commit = consensus.commit_max().min(op); + let pending = adopted_view_headers(consensus); + consensus.set_local_dvc_suffix(build_metadata_dvc_suffix( + journal, + commit, + op, + pending.as_ref().map(|pending| pending.headers.as_slice()), + )); +} + +/// The adopted view's headers, when they describe a log this replica has NOT itself +/// decided. +/// +/// `None` for the primary-elect holding the log its own merge produced: that log is +/// a proposal it is still repairing toward and may contain ops a later view +/// truncated, so stitching it into its own `DoViewChange` would re-assert them. +/// +/// A backup's parked log is the opposite: headers the view already decided and +/// announced, which this replica acknowledged and is repairing to hold. +fn adopted_view_headers(consensus: &VsrConsensus) -> Option +where + B: MessageBus, + P: Pipeline, +{ + if consensus.is_primary_for_view(consensus.view()) { + return None; + } + consensus.pending_view_log() +} + +/// Snapshot a partition's uncommitted suffix into its consensus. +/// +/// Same contract as [`Self::refresh_metadata_dvc_suffix`]. The partition journal +/// is in-memory only, so after a restart it reads empty and this replica votes +/// all-nack: correct, since the ops really are lost and the merge needs a peer +/// that still holds them. +/// +/// Read through `repair_header`, not the resident headers: the committed prefix +/// leaves those as soon as its bytes reach a segment, which on a caught-up +/// replica includes the commit point itself. +fn refresh_partition_dvc_suffix(partition: &partitions::IggyPartition) +where + B: MessageBus, + SB: SuperblockStore, +{ + let consensus = partition.consensus(); + if !consensus.local_dvc_suffix_stale() { + return; + } + let op = consensus.sequencer().current_sequence(); + let commit = consensus.commit_max().min(op); + let journal = partition.log.journal(); + let pending = adopted_view_headers(consensus); + // The window materialized once: probing `repair_header` per op is two linear + // scans each, up to `DVC_HEADERS_MAX` of them, on every SVC/DVC arrival and + // non-Normal tick, on the pump. The internal clamp only narrows this range. + let head = op.max( + pending + .as_ref() + .and_then(|pending| pending.headers.first()) + .map_or(0, |header| header.op), + ); + let window = journal.inner.repair_headers_in(commit.max(1)..=head); + let suffix = build_dvc_suffix( + commit, + op, + |entry_op| window.get(&entry_op).copied(), + pending.as_ref().map(|pending| pending.headers.as_slice()), + ); + consensus.set_local_dvc_suffix(suffix); +} + +/// The suffix headers a `DoViewChange` or `StartView` carries, as raw bytes. +/// +/// `size` is attacker-controlled, so it is clamped to what arrived; a short read +/// decodes as a malformed suffix and the DVC is dropped. +fn control_suffix_body(msg: &Message) -> &[u8] +where + H: iggy_binary_protocol::ConsensusHeader, +{ + let slice = msg.as_slice(); + let start = size_of::(); + let end = (msg.header().size() as usize).min(slice.len()); + if end <= start { + return &[]; + } + &slice[start..end] +} + +/// Seal a control-message body. Zero for an empty body, which is the unsealed +/// sentinel every other integrity field in this protocol uses. +fn control_body_checksum(body: &[u8]) -> u128 { + if body.is_empty() { + return 0; + } + u128::from(iggy_common::calculate_checksum(body)) +} + +/// The body of a control frame, once it matches the checksum its header carries. +/// +/// `None` means corruption in transit and the frame must be dropped whole: the +/// header numbers describe a body that did not arrive intact, so neither half is +/// trustworthy. This is what covers a body-carrying control message end to end. +/// +/// Keyed on whether a body is present, NOT on whether `checksum_body` looks +/// sealed: skipping the check when that field reads zero makes the layer +/// bypassable by clearing the one field that decides whether anything is checked. +/// A frame legitimately carries no body (a sender with nothing uncommitted, a +/// probe-answer `StartView`), so emptiness is the only exemption. A non-empty body +/// always came from a sender that seals it, and a zero checksum there is corruption. +fn control_suffix_body_verified(msg: &Message, checksum_body: u128) -> Option<&[u8]> +where + H: iggy_binary_protocol::ConsensusHeader, +{ + let body = control_suffix_body(msg); + if body.is_empty() { + // Nothing to verify. `checksum_body` is irrelevant either way. + return Some(body); + } + if control_body_checksum(body) == checksum_body { + Some(body) + } else { + None + } +} + +/// Whether a repaired prepare at `op` falls inside the range this replica is +/// currently repairing. +/// +/// A parked log means two things depending on who parked it, and only one is a +/// repair window. The primary-elect parked the log its merge decided and repairs +/// toward exactly that range, so the range IS its scope, including ops at or +/// below `commit_min`: those are the headers inherited from senders behind the +/// canonical `log_view`, which the ordinary rule would reject and header repair +/// cannot walk back to. A backup's parked `StartView` suffix is only what its +/// ingest verifies bodies against, and its repair runs for the whole view, so +/// reading that range as a scope would discard every later op. +fn repair_op_in_scope( + pending: Option<&MergedLog>, + is_primary_elect: bool, + commit_min: u64, + op: u64, +) -> bool { + pending + .filter(|_| is_primary_elect) + .map_or(op > commit_min, |pending| { + (op >= pending.commit_max.max(1) && op <= pending.op_head) + || pending + .committed_elsewhere + .iter() + .any(|expected| expected.op == op) + }) +} + +/// Ceiling on the op range a repair request may ask this replica to walk. +/// +/// Not `commit_max` alone: a new primary repairing toward a merged log needs the +/// uncommitted suffix the view change kept, which sits above every commit point. +/// +/// Bounded by the local frontier all the same. `RequestPreparesHeader::validate` +/// accepts any `from_op <= to_op`, so `u64::MAX` is legal, and the metadata serve +/// path then walks op by op with no `.await` -- on a single-threaded shard pump +/// that ends the shard rather than merely serving slowly. Nothing above the +/// frontier is servable, so the clamp costs nothing. +fn repair_serve_ceiling(requested_to_op: u64, commit_max: u64, head: u64) -> u64 { + requested_to_op.min(commit_max.max(head)) +} + +/// Read this replica's uncommitted suffix out of the metadata journal, for the +/// window `commit..=op`. +/// +/// The nack bit is load-bearing, and is set only where absence *proves* this +/// replica never prepared the op: +/// * Above the commit point, a missing header is proof: the WAL refuses to boot +/// on interior corruption, so a hole in a journal that opened never arrived. +/// * At or below it, a checkpoint may have compacted the header away. Those slots +/// go out blank and un-nacked, read as "no information" rather than licence to +/// truncate an op this replica considers committed. +/// +/// Deriving the suffix on demand is also why it needs no durable record: the +/// merged log is in memory and bodies are fetched whole, so the WAL is the only +/// thing that ever backs a nack and recomputing after a restart gives the same +/// answer. A torn tail is the one exception, and it changes the answer correctly: +/// recovery truncates the incomplete append, which fsyncs before the ack, so no +/// replication quorum could have counted it. +fn build_metadata_dvc_suffix( + journal: Option<&J>, + commit: u64, + op: u64, + view_headers: Option<&[PrepareHeader]>, +) -> DvcSuffix +where + J: JournalHandle, + ::Target: Journal< + ::Storage, + Entry = Message, + Header = PrepareHeader, + >, +{ + let Some(journal) = journal else { + return DvcSuffix::empty(); + }; + let handle = journal.handle(); + build_dvc_suffix( + commit, + op, + |entry_op| { + usize::try_from(entry_op) + .ok() + .and_then(|slot| handle.header(slot)) + .map(|header| *header) + }, + view_headers, + ) +} + +/// Plane-independent core of the suffix read. `header_at` answers "do I hold +/// this op, and what is its header". +fn build_dvc_suffix( + commit: u64, + op: u64, + header_at: impl Fn(u64) -> Option, + view_headers: Option<&[PrepareHeader]>, +) -> DvcSuffix { + // Stitch the adopted view's headers over the journal, high-to-low. + // + // Reading the journal alone is only correct for a replica whose journal IS its + // log. A backup that adopted a `StartView` is header-poor by design: the suffix + // went to `pending_view_log` and the bodies are still being repaired, so the + // journal holds nothing at those ops and would report them blank AND nacked, + // since a hole above the commit point is normally proof the op never arrived. + // Here it proves only unfinished repair, and enough such senders reach a nack + // quorum against ops the view just decided to keep. + // + // The head rises to the view's head too, so a later view change cannot let the + // op backtrack below what this replica already acknowledged. + let view_head = view_headers + .and_then(<[PrepareHeader]>::first) + .map_or(0, |header| header.op); + let op = op.max(view_head); + if op == 0 { + return DvcSuffix::empty(); + } + // Window runs from the commit point up, floored at 1 because ops are 1-based. + // That floor is a scan bound only: the lines below can raise it above the + // commit point, so no reader may read it back as one. See `merge_commit_max`. + let mut low = commit.max(1); + if low > op { + return DvcSuffix::empty(); + } + if op - low + 1 > DVC_HEADERS_MAX as u64 { + // Defensive: every plane's `prepare_queue_depth` is capped below + // `DVC_HEADERS_MAX` so `op - commit` cannot reach this. If it does, the + // clamped-away ops go out described by nobody and the merge stalls rather + // than deciding wrongly. Keep the highest entries, whose fate the view + // change decides, and log it rather than shipping a different window. + let clamped = op - DVC_HEADERS_MAX as u64 + 1; + tracing::warn!( + commit, + op, + window_from = clamped, + "uncommitted suffix wider than {DVC_HEADERS_MAX} entries; truncating the DVC window \ + from below. Ops {}..={} are now undecidable and will stall the view change", + commit + 1, + clamped - 1 + ); + low = clamped; + } + + let len = usize::try_from(op - low + 1).unwrap_or(DVC_HEADERS_MAX); + let mut headers = Vec::with_capacity(len); + let mut nack_bitset = 0u128; + let mut present_bitset = 0u128; + for (index, entry_op) in (low..=op).rev().enumerate() { + if let Some(header) = header_at(entry_op) { + headers.push(header); + // A header in the index means the entry is in the WAL at a known + // offset, the same condition `on_request_prepares` serves from. + present_bitset |= 1u128 << index; + } else if let Some(header) = + view_headers.and_then(|headers| view_header_at(headers, entry_op)) + { + // Held from the adopted view rather than from the journal, so the + // header is reported and the op is NOT nacked: this replica knows + // the op exists and simply cannot serve its body yet. No present + // bit for the same reason. + headers.push(*header); + } else { + headers.push(dvc_blank(entry_op)); + if entry_op > commit { + nack_bitset |= 1u128 << index; + } else { + // The commit point, the one slot that goes out blank AND + // un-nacked. The merge scans it and may not discard it, so a + // sender is asking the new primary to take the header from + // someone else; if every sender in the quorum does that, the + // op is undecidable and the view never starts. + // + // Every compaction path is supposed to leave this header behind + // (the metadata checkpoint drain stops one op short, a + // partition serves it from the evicted ring), so reaching here + // means a replica whose log genuinely starts above its own + // commit point: a state-transfer receiver that jumped its + // commit floor to a snapshot whose prepares it never held. + tracing::warn!( + op = entry_op, + commit, + "no header at this replica's commit point; the DVC reports it blank and \ + cannot nack it, so the view change stalls unless a peer supplies it" + ); + } + } + } + DvcSuffix::new(headers, nack_bitset, present_bitset) +} + +/// Partition-plane twin of `Shard::reconcile_metadata_view_divergence`: same split +/// at the announced commit point, dropping above it and reporting at or below. +/// +/// Worse to skip here than on the metadata plane, which is why this exists. +/// Partition `append` has no slot-collision check, so a re-prepared op pushes a +/// duplicate header and rewrites `op_to_storage_offset`, and `committed_prefix` walks +/// positionally, so the stale entry is what `evict_prefix` flushes to the segment: +/// durable divergent bytes, no error anywhere. +#[allow(clippy::future_not_send)] +async fn reconcile_partition_view_divergence( + shard: u16, + partition: &mut IggyPartition, + pending: &MergedLog, +) where + B: MessageBus, + SB: journal::superblock::SuperblockStore, +{ + // Truncation is safe only above what this replica has *applied*, which is not + // the view's commit point: a backup can sit above it. + let applied_floor = pending.commit_max.max(partition.consensus().commit_min()); + + let mut repairable_from: Option = None; + for canonical in &pending.headers { + let Some(local) = partition.log.journal().inner.header_by_op(canonical.op) else { + continue; + }; + if header_is_view_entry(&local, canonical) { + continue; + } + if canonical.op <= applied_floor { + tracing::error!( + shard, + namespace_raw = partition.consensus().group(), + op = canonical.op, + view = partition.consensus().view(), + commit_max = pending.commit_max, + commit_min = partition.consensus().commit_min(), + local_checksum = local.checksum, + canonical_checksum = canonical.checksum, + "committed partition op {} disagrees with the view that just started; this \ + replica applied a different op and log repair cannot reconcile it", + canonical.op + ); + continue; + } + repairable_from = Some(repairable_from.map_or(canonical.op, |op| op.min(canonical.op))); + } + + // The suffix above the announced head, which no canonical header names. As on + // the metadata twin, except here `append` pushes a duplicate rather than erroring. + let above_head = pending.op_head.max(applied_floor) + 1; + if partition + .log + .journal() + .inner + .last_op() + .is_some_and(|last_op| last_op >= above_head) + { + repairable_from = Some(repairable_from.map_or(above_head, |op| op.min(above_head))); + } + + let Some(from_op) = repairable_from else { + return; + }; + match partition.truncate_uncommitted_from(from_op).await { + Ok(removed) => { + tracing::warn!( + shard, + namespace_raw = partition.consensus().group(), + from_op, + removed, + op_head = pending.op_head, + view = partition.consensus().view(), + "dropped {removed} uncommitted partition entries from op {from_op} that \ + disagreed with the view's log; the primary's retransmission refills the range" + ); + } + Err(error) => { + tracing::error!( + shard, + namespace_raw = partition.consensus().group(), + from_op, + %error, + "could not drop the diverging uncommitted partition entries from op \ + {from_op}; repair skips ops it already holds, so this replica will not \ + converge there until restarted" + ); + } + } +} + +/// Whether a locally journaled header IS the entry the view's log names at that op. +/// +/// Identity, not presence: otherwise a stale prepare at the right op reads as +/// coverage everywhere: the repair ingest skips it as already held, +/// `RebuildPipeline` seeds the pipeline from it and self-acks, `CommitJournal` +/// applies it. `identity_checksum` excludes `view`, so a restamp still compares equal. +/// +/// An unsealed checksum on either side is not evidence (pre-seal WAL, partition-plane +/// prepare), so it counts as agreement, as in `dvc_suffix_decode`. +const fn header_is_view_entry(local: &PrepareHeader, canonical: &PrepareHeader) -> bool { + local.checksum == CHECKSUM_UNSEALED + || canonical.checksum == CHECKSUM_UNSEALED + || local.checksum == canonical.checksum +} + +/// The lowest op in the merged log this replica cannot serve, or `None` when the +/// view can start. +/// +/// Coverage is identity, not presence (see [`header_is_view_entry`]): starting a view +/// over a differing entry commits this replica's own operation where the view says +/// another belongs. +/// +/// Covers every op the merged log names, including headers inherited from senders +/// behind the canonical `log_view`, which sit below the canonical window where header +/// repair cannot walk back to them. `repair_floor` drops the ops whose journal entry +/// is legitimately gone AND whose identity is already settled: on the metadata plane +/// ops compacted under a snapshot, on the partition plane ops at or below the local +/// commit point, whose flushed entries `evict_prefix` moves out of the header vec. +/// Neither can diverge from the merged log (a committed or compacted op is the +/// quorum's op), and no repair puts the journal entry back, so demanding one parks +/// the view change forever. +fn first_op_not_covered( + pending: &MergedLog, + repair_floor: u64, + header_at: impl Fn(u64) -> Option, +) -> Option { + let held = |op: u64| { + let Some(local) = header_at(op) else { + return false; + }; + pending + .headers + .iter() + .chain(pending.committed_elsewhere.iter()) + .find(|header| header.op == op) + .is_none_or(|canonical| header_is_view_entry(&local, canonical)) + }; + (pending.commit_max.max(1).max(repair_floor + 1)..=pending.op_head) + .find(|op| !held(*op)) + .or_else(|| { + pending + .committed_elsewhere + .iter() + .map(|header| header.op) + .filter(|op| *op > repair_floor) + .find(|op| !held(*op)) + }) +} + +/// The adopted view's header at `op`, or `None` when the view says nothing about +/// it. +/// +/// Headers run high-to-low from the view's head, so the slot is arithmetic. The +/// op is re-checked rather than assumed: a mismatch means the range is not the +/// contiguous run this indexing needs, and inventing a header for the wrong op +/// is worse than reporting none. +fn view_header_at(view_headers: &[PrepareHeader], op: u64) -> Option<&PrepareHeader> { + let head = view_headers.first()?.op; + let index = usize::try_from(head.checked_sub(op)?).ok()?; + let header = view_headers.get(index)?; + if header.op != op || matches!(dvc_header_kind(header), DvcHeaderKind::Blank) { + return None; + } + Some(header) +} + /// Dispatch a list of `VsrAction`s by constructing the appropriate /// protocol messages and sending them via the consensus message bus. #[allow( @@ -7606,22 +8711,23 @@ async fn dispatch_vsr_actions( !advertises_view || !consensus.needs_superblock_persist(), "durable-before-send violated: dispatching a view-scoped action for \ namespace {} while the superblock is behind the in-memory view {}", - consensus.namespace(), + consensus.group(), consensus.view(), ); } for action in actions { match action { - VsrAction::SendStartViewChange { view, namespace } => { + VsrAction::SendStartViewChange { view, group } => { let msg = Message::::new(size_of::()) .transmute_header(|_, h: &mut StartViewChangeHeader| { h.command = Command2::StartViewChange; h.cluster = cluster; h.replica = self_id; h.view = *view; - h.namespace = *namespace; + h.group = *group; h.size = size_of::() as u32; + h.seal(); }); broadcast(msg.into_generic().into_frozen()).await; } @@ -7631,23 +8737,41 @@ async fn dispatch_vsr_actions( log_view, op, commit, - namespace, + group, + suffix, } => { - let msg = Message::::new(size_of::()) - .transmute_header(|_, h: &mut DoViewChangeHeader| { - h.command = Command2::DoViewChange; - h.cluster = cluster; - h.replica = self_id; - h.view = *view; - h.log_view = *log_view; - h.op = *op; - h.commit = *commit; - h.namespace = *namespace; - h.size = size_of::() as u32; - }); - send(*target, msg.into_generic().into_frozen()).await; + let header_size = size_of::(); + let total_size = header_size + suffix.encoded_len(); + let mut msg = Message::::new(total_size); + // Body first: `transmute_header` zeroes only the header region, so + // anything past it survives. Same order as the manifest build. + suffix.encode_into(&mut msg.as_mut_slice()[header_size..total_size]); + let body_checksum = control_body_checksum(&msg.as_slice()[header_size..total_size]); + let nack_bitset = suffix.nack_bitset(); + let present_bitset = suffix.present_bitset(); + let msg = msg.transmute_header(|_, h: &mut DoViewChangeHeader| { + h.command = Command2::DoViewChange; + h.cluster = cluster; + h.replica = self_id; + h.view = *view; + h.log_view = *log_view; + h.op = *op; + h.commit = *commit; + h.group = *group; + h.nack_bitset = nack_bitset; + h.present_bitset = present_bitset; + h.checksum_body = body_checksum; + h.size = total_size as u32; + // Last: covers the bitsets a new primary truncates on. + h.seal(); + }); + // Broadcast, not unicast to `target`: a backup seeing a DVC for a + // newer view adopts it instead of waiting out its heartbeat + // timeout, which converges the view change in one round. + let _ = target; + broadcast(msg.into_generic().into_frozen()).await; } - VsrAction::SendRequestStartView { view, namespace } => { + VsrAction::SendRequestStartView { view, group } => { // Stamp this replica's incarnation so the answering StartView can // echo it, proving to us the reply post-dates our restart. let incarnation = consensus.incarnation(); @@ -7659,8 +8783,9 @@ async fn dispatch_vsr_actions( h.replica = self_id; h.view = *view; h.incarnation = incarnation; - h.namespace = *namespace; + h.group = *group; h.size = size_of::() as u32; + h.seal(); }); broadcast(msg.into_generic().into_frozen()).await; } @@ -7670,20 +8795,28 @@ async fn dispatch_vsr_actions( commit, incarnation, target, - namespace, + group, + suffix, } => { - let msg = Message::::new(size_of::()) - .transmute_header(|_, h: &mut StartViewHeader| { - h.command = Command2::StartView; - h.cluster = cluster; - h.replica = self_id; - h.view = *view; - h.op = *op; - h.commit = *commit; - h.incarnation = *incarnation; - h.namespace = *namespace; - h.size = size_of::() as u32; - }); + let header_size = size_of::(); + let total_size = header_size + suffix.len() * size_of::(); + let mut msg = Message::::new(total_size); + // Body first: `transmute_header` zeroes only the header region. + encode_prepare_headers(suffix, &mut msg.as_mut_slice()[header_size..total_size]); + let body_checksum = control_body_checksum(&msg.as_slice()[header_size..total_size]); + let msg = msg.transmute_header(|_, h: &mut StartViewHeader| { + h.checksum_body = body_checksum; + h.command = Command2::StartView; + h.cluster = cluster; + h.replica = self_id; + h.view = *view; + h.op = *op; + h.commit = *commit; + h.incarnation = *incarnation; + h.group = *group; + h.size = total_size as u32; + h.seal(); + }); let frozen = msg.into_generic().into_frozen(); // A probe echo is addressed to its requester: the incarnation it // carries is that replica's freshness proof, and a peer recovering @@ -7699,7 +8832,7 @@ async fn dispatch_vsr_actions( from_op, to_op, target, - namespace, + group, } => { let Some(journal) = journal else { continue; @@ -7722,8 +8855,9 @@ async fn dispatch_vsr_actions( h.prepare_checksum = prepare_header.checksum; h.request = prepare_header.request; h.operation = prepare_header.operation; - h.namespace = *namespace; + h.group = *group; h.size = size_of::() as u32; + h.seal(); }); send(*target, msg.into_generic().into_frozen()).await; } @@ -7738,15 +8872,10 @@ async fn dispatch_vsr_actions( continue; }; // Freeze the retransmit payload once; clone per target. - let frozen = if prepare.header().view == current_view { - prepare.into_generic().into_frozen() - } else { - let Some(restamped) = - restamp_prepare_view(prepare.as_slice(), current_view) - else { - continue; - }; - restamped + let Some(frozen) = + restamp_prepare_view(prepare.into_generic().into_frozen(), current_view) + else { + continue; }; for replica in replicas { send(*replica, frozen.clone()).await; @@ -7757,49 +8886,12 @@ async fn dispatch_vsr_actions( let Some(journal) = journal else { continue; }; - // Collect headers before borrowing the pipeline to avoid - // holding borrow_mut() across journal reads. - let mut gap_at = None; - let entries: Vec<_> = (*from_op..=*to_op) - .map_while(|op| { - let Some(header) = journal.handle().header(op as usize) else { - gap_at = Some(op); - return None; - }; - // New-primary path: lift the monotonic timestamp - // floor to the rebuilt log so post-view-change - // prepares cannot stamp below committed ones. - consensus.observe_prepare_timestamp(header.timestamp); - let mut entry = consensus::PipelineEntry::new(*header); - entry.add_ack(self_id); - Some(entry) - }) - .collect(); - if let Some(missing_op) = gap_at { - // A primary's own uncommitted suffix has no repair - // source: peers ack'd nothing above the gap or the DVC - // merge would have carried it, so the range is decided - // lost. Truncate the sequencer to the last op we could - // rebuild so the next client prepare chains correctly. - let rebuilt_up_to = missing_op.saturating_sub(1); - tracing::warn!( - replica = self_id, - missing_op, - range_start = from_op, - range_end = to_op, - rebuilt = entries.len(), - "RebuildPipeline: journal gap at op {missing_op}, \ - truncating sequencer from {to_op} to {rebuilt_up_to} \ - ({}/{} ops rebuilt)", - entries.len(), - to_op - from_op + 1, - ); - consensus.sequencer().set_sequence(rebuilt_up_to); - } - let mut pipeline = consensus.pipeline().borrow_mut(); - for entry in entries { - pipeline.push(entry); - } + rebuild_pipeline_entries(consensus, self_id, *from_op, *to_op, |op| { + usize::try_from(op) + .ok() + .and_then(|slot| journal.handle().header(slot)) + .map(|header| *header) + }); } // Handled by the caller (shard view change handlers) since it // requires access to the plane's commit_journal method. @@ -7807,7 +8899,7 @@ async fn dispatch_vsr_actions( VsrAction::SendCommit { view, commit, - namespace, + group, timestamp_monotonic, } => { let msg = Message::::new(size_of::()).transmute_header( @@ -7817,9 +8909,10 @@ async fn dispatch_vsr_actions( h.replica = self_id; h.view = *view; h.commit = *commit; - h.namespace = *namespace; + h.group = *group; h.timestamp_monotonic = *timestamp_monotonic; h.size = size_of::() as u32; + h.seal(); }, ); broadcast(msg.into_generic().into_frozen()).await; @@ -7865,7 +8958,7 @@ async fn dispatch_partition_journal_actions( || !consensus.needs_superblock_persist(), "durable-before-send violated: dispatching a view-scoped action for \ namespace {} while the superblock is behind the in-memory view {}", - consensus.namespace(), + consensus.group(), consensus.view(), ); } @@ -7877,7 +8970,7 @@ async fn dispatch_partition_journal_actions( from_op, to_op, target, - namespace, + group, } => { for op in *from_op..=*to_op { let Some(prepare_header) = journal.header_by_op(op) else { @@ -7896,8 +8989,9 @@ async fn dispatch_partition_journal_actions( h.prepare_checksum = prepare_header.checksum; h.request = prepare_header.request; h.operation = prepare_header.operation; - h.namespace = *namespace; + h.group = *group; h.size = size_of::() as u32; + h.seal(); }); send(*target, msg.into_generic().into_frozen()).await; } @@ -7925,15 +9019,8 @@ async fn dispatch_partition_journal_actions( // above and avoids both the per-target 4 KiB memcpy // and the prior `.expect` that would panic the shard // on a corrupted journal entry. - let prepare = if header.view == current_view { - prepare - } else { - let Some(restamped) = - restamp_prepare_view(prepare.as_slice(), current_view) - else { - continue; - }; - restamped + let Some(prepare) = restamp_prepare_view(prepare, current_view) else { + continue; }; for replica in replicas { send(*replica, prepare.clone()).await; @@ -7941,42 +9028,9 @@ async fn dispatch_partition_journal_actions( } } VsrAction::RebuildPipeline { from_op, to_op } => { - let mut gap_at = None; - let entries: Vec<_> = (*from_op..=*to_op) - .map_while(|op| { - let Some(header) = journal.header_by_op(op) else { - gap_at = Some(op); - return None; - }; - // New-primary path: lift the monotonic timestamp - // floor to the rebuilt log so post-view-change - // prepares cannot stamp below committed ones. - consensus.observe_prepare_timestamp(header.timestamp); - let mut entry = consensus::PipelineEntry::new(header); - entry.add_ack(self_id); - Some(entry) - }) - .collect(); - if let Some(missing_op) = gap_at { - let rebuilt_up_to = missing_op.saturating_sub(1); - tracing::warn!( - replica = self_id, - missing_op, - range_start = from_op, - range_end = to_op, - rebuilt = entries.len(), - "RebuildPipeline: journal gap at op {missing_op}, \ - truncating sequencer from {to_op} to {rebuilt_up_to} \ - ({}/{} ops rebuilt)", - entries.len(), - to_op - from_op + 1, - ); - consensus.sequencer().set_sequence(rebuilt_up_to); - } - let mut pipeline = consensus.pipeline().borrow_mut(); - for entry in entries { - pipeline.push(entry); - } + rebuild_pipeline_entries(consensus, self_id, *from_op, *to_op, |op| { + journal.header_by_op(op) + }); } _ => {} } @@ -8008,7 +9062,8 @@ mod persist_gate_tests { commit: 3, incarnation: 0, target: None, - namespace: 7, + group: 7, + suffix: Vec::new(), }, VsrAction::CommitJournal, rebuild(), @@ -8028,12 +9083,422 @@ mod persist_gate_tests { #[test] fn given_send_only_actions_when_split_should_leave_locals_empty() { - let actions = vec![VsrAction::SendStartViewChange { - view: 2, - namespace: 7, - }]; + let actions = vec![VsrAction::SendStartViewChange { view: 2, group: 7 }]; let (local, wire) = split_local_actions(actions); assert!(local.is_empty()); assert_eq!(wire.len(), 1); } } + +#[cfg(test)] +mod repair_scope_tests { + //! Who parked the log decides what it means. + + use super::{MergedLog, repair_op_in_scope, repair_serve_ceiling}; + use iggy_binary_protocol::{Command2, PrepareHeader}; + + fn header(op: u64) -> PrepareHeader { + PrepareHeader { + command: Command2::Prepare, + op, + ..Default::default() + } + } + + /// A view that started at op 100 with commit 98. + fn parked() -> MergedLog { + MergedLog { + op_head: 100, + commit_max: 98, + headers: (98..=100).rev().map(header).collect(), + committed_elsewhere: Vec::new(), + } + } + + #[test] + fn given_a_backup_with_a_parked_log_when_repairing_above_the_view_head_should_accept() { + // A backup keeps its parked `StartView` suffix for the whole view, so at + // op 200 the parked head is 100 ops stale. Reading it as a repair scope + // silently discards the served op: the retry loops, the commit walk + // freezes, checkpointing stops, and the backup stops acking. + assert!( + repair_op_in_scope(Some(&parked()), false, 149, 150), + "a backup repairs for the whole view, not just the view-start range" + ); + } + + #[test] + fn given_a_backup_with_a_parked_log_when_repairing_below_commit_min_should_reject() { + // A backup's parked log grants no licence to re-ingest committed ops. + assert!(!repair_op_in_scope(Some(&parked()), false, 149, 149)); + } + + #[test] + fn given_a_primary_elect_when_repairing_toward_its_merged_log_should_use_it_as_the_scope() { + let pending = parked(); + // Inside the merged range, including inherited headers below `commit_min`. + assert!(repair_op_in_scope(Some(&pending), true, 99, 98)); + assert!(repair_op_in_scope(Some(&pending), true, 99, 100)); + // Outside it: the primary-elect is not repairing toward these. + assert!(!repair_op_in_scope(Some(&pending), true, 99, 101)); + assert!(!repair_op_in_scope(Some(&pending), true, 99, 97)); + // With nothing parked, the ordinary commit-point rule applies. + assert!(!repair_op_in_scope(None, false, 149, 149)); + assert!(repair_op_in_scope(None, false, 149, 150)); + } + + #[test] + fn given_a_primary_elect_when_an_op_is_committed_elsewhere_should_accept_it() { + let mut pending = parked(); + pending.committed_elsewhere.push(header(42)); + assert!(repair_op_in_scope(Some(&pending), true, 99, 42)); + } + + #[test] + fn given_a_repair_request_when_serving_should_clamp_to_the_frontier_but_not_below_it() { + // `validate` accepts any `to_op >= from_op` and the serve path walks op by + // op with no `.await`, so an unclamped ceiling hangs the whole shard. + assert_eq!(repair_serve_ceiling(u64::MAX, 40, 90), 90); + assert_eq!(repair_serve_ceiling(50, 40, 90), 50); + // The suffix a new primary repairs toward sits above every commit point, + // so clamping to `commit_max` alone deadlocks the view change. + assert_eq!(repair_serve_ceiling(90, 40, 90), 90); + // `commit_max` above the local head still counts: heartbeats outrun prepares. + assert_eq!(repair_serve_ceiling(u64::MAX, 120, 90), 120); + } +} + +#[cfg(test)] +mod view_coverage_tests { + //! Holding an op is not holding the view's op. + + use super::{MergedLog, first_op_not_covered}; + use iggy_binary_protocol::{Command2, Operation, PrepareHeader}; + + fn sealed(op: u64, request: u64) -> PrepareHeader { + let mut header = PrepareHeader { + command: Command2::Prepare, + operation: Operation::CreateStream, + op, + request, + ..Default::default() + }; + header.checksum = header.identity_checksum(); + header + } + + #[test] + fn given_a_diverging_entry_when_scanning_should_report_it_like_a_hole() { + // Op 99 is present and is not the view's op 99. Reading presence as coverage + // starts the view over an operation the view says is something else, which + // `CommitJournal` then applies at or below the commit point unchecked. + let pending = MergedLog { + op_head: 100, + commit_max: 98, + headers: (98..=100).rev().map(|op| sealed(op, 1)).collect(), + committed_elsewhere: Vec::new(), + }; + let held = [sealed(100, 1), sealed(99, 7), sealed(98, 1)]; + let missing = first_op_not_covered(&pending, 0, |op| { + held.iter().find(|header| header.op == op).copied() + }); + assert_eq!(missing, Some(99)); + } + + #[test] + fn given_an_evicted_committed_window_when_floored_should_start_the_view() { + // The wedge behind the partition_state_transfer regressions: a survivor + // that flushes on every commit holds NO resident journal header (the + // flush evicts them), so a merged window opening on its own committed op + // reads as a hole nothing can fill -- no repair re-journals a committed + // op. The floor (local commit point) must count it as covered, or the + // primary-elect parks in `ViewChange` forever and the rotation hands + // primaryship to an empty rejoiner that then cannot be served the state + // transfer it needs. + let pending = MergedLog { + op_head: 256, + commit_max: 256, + headers: vec![sealed(256, 1)], + committed_elsewhere: Vec::new(), + }; + let nothing_resident = |_: u64| None; + assert_eq!( + first_op_not_covered(&pending, 0, nothing_resident), + Some(256), + "unfloored, the evicted committed op reads as an unfillable hole" + ); + assert_eq!( + first_op_not_covered(&pending, 256, nothing_resident), + None, + "floored at the local commit point, the view starts" + ); + } +} + +#[cfg(test)] +mod dvc_suffix_window_tests { + //! The suffix window's floor is a scan bound, not a commit point. + //! + //! Reading the lowest suffix op back as a proven commit point assumes suffix + //! generation stops at the sender's commit. These pin the two paths that break + //! that premise, so it cannot be quietly reintroduced. + + use super::{DVC_HEADERS_MAX, build_dvc_suffix}; + use iggy_binary_protocol::{Command2, Operation, PrepareHeader}; + + /// A real prepare at `op`. The operation must not be `Reserved`: that is + /// exactly `dvc_blank`, and `dvc_header_kind` classifies by equality with it. + fn held(op: u64) -> PrepareHeader { + PrepareHeader { + command: Command2::Prepare, + operation: Operation::CreateStream, + op, + ..Default::default() + } + } + + /// The lowest op the built window describes. + fn floor(suffix: &consensus::DvcSuffix) -> Option { + suffix.headers().last().map(|header| header.op) + } + + /// A view's headers for `low..=high`, high-to-low as the suffix carries them. + fn view_headers(low: u64, high: u64) -> Vec { + (low..=high).rev().map(held).collect() + } + + #[test] + fn given_an_adopted_view_when_the_journal_is_empty_should_report_its_headers_unnacked() { + // A backup that adopted a `StartView` put the suffix in `pending_view_log` + // and is still repairing bodies, so its journal holds nothing at those ops. + // Reading the journal alone reports them blank AND nacked, which reaches a + // nack quorum against ops the view had just decided to keep. + let view = view_headers(3, 5); + let suffix = build_dvc_suffix(2, 0, |_| None, Some(&view)); + + assert_eq!( + suffix.len(), + 4, + "the window rises to the view's head even with an empty journal" + ); + assert_eq!( + floor(&suffix), + Some(2), + "the floor is still the commit point" + ); + assert_eq!( + suffix.nack_bitset(), + 0, + "a header held from the adopted view is not a nack" + ); + assert_eq!( + suffix.present_bitset(), + 0, + "and its body is not servable, so no present bit either" + ); + } + + #[test] + fn given_no_adopted_view_when_the_journal_is_empty_should_nack() { + // The contrast: without an adopted view the same holes really are proof. + let suffix = build_dvc_suffix(2, 5, |_| None, None); + assert_eq!( + suffix.nack_bitset(), + 0b0111, + "ops 5, 4 and 3 nack; op 2 is the commit point" + ); + } + + #[test] + fn given_an_adopted_view_when_the_journal_covers_part_should_prefer_the_journal() { + // Journal first, so an op whose body this replica can serve keeps its + // present bit; the view fills only what the journal is missing. + let view = view_headers(3, 5); + let suffix = build_dvc_suffix(2, 5, |op| (op == 5).then(|| held(op)), Some(&view)); + + assert_eq!(suffix.len(), 4); + assert_eq!(suffix.present_bitset(), 0b0001, "only op 5 is servable"); + assert_eq!(suffix.nack_bitset(), 0, "the view covers ops 4 and 3"); + + // The head is the max of the two, never the view's alone. + let short_view = view_headers(3, 4); + let deeper = build_dvc_suffix(2, 6, |op| Some(held(op)), Some(&short_view)); + assert_eq!(deeper.headers().first().map(|header| header.op), Some(6)); + assert_eq!(deeper.present_bitset(), 0b1_1111, "ops 6 down to 2"); + } + + #[test] + fn given_a_blank_view_entry_should_not_report_it_as_held() { + // A blank is the view saying "no header here", not one this replica holds. + let mut view = view_headers(3, 5); + view[1] = consensus::dvc_blank(4); + let suffix = build_dvc_suffix(2, 0, |_| None, Some(&view)); + + assert_eq!(suffix.nack_bitset(), 0b010, "only the blank op nacks"); + } + + #[test] + fn given_no_header_at_the_commit_point_should_report_it_blank_and_undecidable() { + // The window's floor is the commit point, and a blank there is the one + // entry that goes out with neither a header nor a nack. The merge scans + // that op and may not discard it, so a quorum of these deadlocks the view + // change. Pinned here because both compaction paths are meant to keep the + // header alive precisely so this shape never leaves a healthy replica. + let suffix = build_dvc_suffix(5, 5, |_| None, None); + + assert_eq!(suffix.len(), 1); + assert_eq!(floor(&suffix), Some(5)); + assert_eq!( + suffix.nack_bitset(), + 0, + "the commit point is never nacked, whatever the journal says" + ); + assert_eq!(suffix.present_bitset(), 0); + } + + #[test] + fn given_a_window_at_the_depth_ceiling_when_building_should_floor_at_the_commit() { + // At the deepest legal prepare-queue depth the window still starts exactly + // at the commit point, so nothing is clamped and no op goes undescribed. + // Config ceilings and `LocalPipeline::with_capacities` enforce the depth. + let depth = DVC_HEADERS_MAX as u64 - 1; + let commit = 500; + let op = commit + depth; + let suffix = build_dvc_suffix(commit, op, |op| Some(held(op)), None); + + assert_eq!(suffix.len(), DVC_HEADERS_MAX, "the widest window that fits"); + assert_eq!( + floor(&suffix), + Some(commit), + "at the ceiling the floor is still the commit point" + ); + } + + #[test] + fn given_a_window_past_the_depth_ceiling_when_building_should_clamp_above_the_commit() { + // One op deeper and the window clamps: the floor sits 501 ops above the + // sender's commit, with no marker on the frame saying so. + let commit = 500; + let op = commit + DVC_HEADERS_MAX as u64; + let suffix = build_dvc_suffix(commit, op, |op| Some(held(op)), None); + + assert_eq!(suffix.len(), DVC_HEADERS_MAX); + assert_eq!( + floor(&suffix), + Some(op - DVC_HEADERS_MAX as u64 + 1), + "the clamped floor sits above the commit point" + ); + assert!(floor(&suffix) > Some(commit)); + + // Second path, at any depth: ops are 1-based, so commit 0 floors at op 1. + let from_zero = build_dvc_suffix(0, 3, |op| Some(held(op)), None); + assert_eq!(floor(&from_zero), Some(1)); + } + + #[test] + fn given_a_compacted_log_when_building_should_still_describe_the_commit_point() { + // The commit point goes out blank AND un-nacked, so the merge can neither + // adopt nor discard it: a quorum that all compacted to the same op deadlocks + // and no further message fixes it. Both planes must keep that header + // reachable (metadata's drain stops one op short, a partition serves it from + // the evicted ring); nothing in `build_dvc_suffix` enforces it. + let commit = 500; + let compacted = |op: u64| (op >= commit).then(|| held(op)); + let suffix = build_dvc_suffix(commit, commit + 3, compacted, None); + + let commit_index = suffix.index_of(commit + 3, commit).expect("in window"); + assert!( + suffix.valid_header_at(commit_index).is_some(), + "a blank at the commit point is undecidable for the merge" + ); + assert!( + suffix.offers_body(commit_index), + "the commit point must be servable, or the merge stalls waiting for a peer" + ); + assert!( + !suffix.nacks(commit_index), + "the commit point can never be nacked" + ); + } +} + +#[cfg(test)] +mod control_frame_tests { + //! A control frame's body must be verified on a rule corruption cannot switch + //! off. Keying on `checksum_body` looking sealed is bypassable by zeroing it. + + use super::{control_body_checksum, control_suffix_body_verified}; + use iggy_binary_protocol::{Command2, DoViewChangeHeader, PrepareHeader}; + use server_common::Message; + use std::mem::size_of; + + /// A `DoViewChange` frame carrying `entries` blank suffix headers. + fn frame(entries: usize, checksum_body: u128) -> Message { + let header_size = size_of::(); + let total = header_size + entries * size_of::(); + let mut msg = Message::::new(total); + for (index, byte) in msg.as_mut_slice()[header_size..total] + .iter_mut() + .enumerate() + { + *byte = u8::try_from(index % 251).expect("modulus fits u8"); + } + msg.transmute_header(|_, header: &mut DoViewChangeHeader| { + header.command = Command2::DoViewChange; + header.checksum_body = checksum_body; + header.size = u32::try_from(total).expect("frame fits u32"); + }) + } + + #[test] + fn given_a_sealed_body_when_verifying_should_accept() { + let header_size = size_of::(); + let unsealed = frame(2, 0); + let sealed_value = control_body_checksum( + &unsealed.as_slice()[header_size..unsealed.header().size as usize], + ); + let msg = frame(2, sealed_value); + + assert!( + control_suffix_body_verified(&msg, msg.header().checksum_body).is_some(), + "a correctly sealed body must be accepted" + ); + } + + #[test] + fn given_a_body_with_a_zeroed_checksum_when_verifying_should_reject() { + // A non-empty body always came from a sender that seals it, so a zero here is + // corruption. Treating it as "unsealed, skip" disables the layer by clearing + // the one field that decides whether anything is checked. + let msg = frame(2, 0); + assert!( + control_suffix_body_verified(&msg, msg.header().checksum_body).is_none(), + "a non-empty body with a zeroed checksum must be rejected, not waved through" + ); + } + + #[test] + fn given_a_corrupted_body_when_verifying_should_reject() { + let header_size = size_of::(); + let unsealed = frame(2, 0); + let sealed_value = control_body_checksum( + &unsealed.as_slice()[header_size..unsealed.header().size as usize], + ); + let mut msg = frame(2, sealed_value); + msg.as_mut_slice()[header_size] ^= 0xFF; + + assert!( + control_suffix_body_verified(&msg, msg.header().checksum_body).is_none(), + "a body that does not match its checksum must be rejected" + ); + } + + #[test] + fn given_a_header_only_frame_when_verifying_should_accept() { + // A sender with nothing uncommitted contributes numbers only, no body. + let msg = frame(0, 0); + let body = control_suffix_body_verified(&msg, msg.header().checksum_body) + .expect("a header-only frame has nothing to verify"); + assert!(body.is_empty()); + } +} diff --git a/core/shard/src/metrics.rs b/core/shard/src/metrics.rs index 659550217e..2df15c357e 100644 --- a/core/shard/src/metrics.rs +++ b/core/shard/src/metrics.rs @@ -186,6 +186,7 @@ pub struct ShardMetrics { partitions_materialised_total: Counter, partitions_removed_total: Counter, partitions_reconcile_failures_total: Counter, + partitions_duplicate_builds_discarded_total: Counter, partition_transfer_refusals_total: Counter, partition_frames_rejected_stale_total: Counter, partition_frames_rejected_ahead_total: Counter, @@ -220,6 +221,7 @@ impl ShardMetrics { partitions_materialised_total: Counter::default(), partitions_removed_total: Counter::default(), partitions_reconcile_failures_total: Counter::default(), + partitions_duplicate_builds_discarded_total: Counter::default(), partition_transfer_refusals_total: Counter::default(), partition_frames_rejected_stale_total: Counter::default(), partition_frames_rejected_ahead_total: Counter::default(), @@ -259,6 +261,16 @@ impl ShardMetrics { self.partitions_removed_total.inc(); } + /// Bumped when the pump discards a duplicate `InsertOwned` for a namespace + /// that is already live. The reconciler's staged-op guard should make this + /// unreachable, so a non-zero value is a caught correctness anomaly, not + /// routine churn: the discarded build re-planted segment 0 over the live + /// incarnation's path and folded its initial segment into the shared stats + /// before the pump caught it. + pub fn record_duplicate_partition_build_discarded(&self) { + self.partitions_duplicate_builds_discarded_total.inc(); + } + /// Bumped each time `build_partition_fresh` or /// `delete_partitions_from_disk` returns `Err`. The reconciler retries /// next tick, but a sustained climb surfaces a stuck partition (disk diff --git a/core/shard/src/router.rs b/core/shard/src/router.rs index 0dbf0c1f61..b683af2bc0 100644 --- a/core/shard/src/router.rs +++ b/core/shard/src/router.rs @@ -27,7 +27,7 @@ use iggy_binary_protocol::{ConsensusHeader, GenericHeader, Operation, PrepareHea use journal::superblock::SuperblockStore; use journal::{Journal, JournalHandle}; use message_bus::{ConnectionInstaller, MessageBus, ReplicaHandshakeDoneFn}; -use server_common::sharding::{IggyNamespace, METADATA_CONSENSUS_NAMESPACE}; +use server_common::sharding::{IggyNamespace, METADATA_GROUP}; use server_common::{Message, MessageBag}; /// How often the shard pump drives `VsrConsensus::tick`. @@ -46,63 +46,63 @@ fn extract_routing(bag: MessageBag) -> (Operation, u64, Message) match bag { MessageBag::Request(r) => { let h = *r.header(); - (h.operation, h.namespace, r.into_generic()) + (h.operation, h.group, r.into_generic()) } MessageBag::Prepare(p) => { let h = *p.header(); - (h.operation, h.namespace, p.into_generic()) + (h.operation, h.group, p.into_generic()) } MessageBag::PrepareOk(p) => { let h = *p.header(); - (h.operation, h.namespace, p.into_generic()) + (h.operation, h.group, p.into_generic()) } MessageBag::StartViewChange(m) => { let h = *m.header(); - (h.operation(), h.namespace, m.into_generic()) + (h.operation(), h.group, m.into_generic()) } MessageBag::DoViewChange(m) => { let h = *m.header(); - (h.operation(), h.namespace, m.into_generic()) + (h.operation(), h.group, m.into_generic()) } MessageBag::StartView(m) => { let h = *m.header(); - (h.operation(), h.namespace, m.into_generic()) + (h.operation(), h.group, m.into_generic()) } MessageBag::Commit(m) => { let h = *m.header(); - (h.operation(), h.namespace, m.into_generic()) + (h.operation(), h.group, m.into_generic()) } MessageBag::RequestStartView(m) => { let h = *m.header(); - (h.operation(), h.namespace, m.into_generic()) + (h.operation(), h.group, m.into_generic()) } MessageBag::RequestPrepares(m) => { let h = *m.header(); - (h.operation(), h.namespace, m.into_generic()) + (h.operation(), h.group, m.into_generic()) } MessageBag::RepairPrepare(m) => { let h = *m.header(); - (h.0.operation, h.0.namespace, m.into_generic()) + (h.0.operation, h.0.group, m.into_generic()) } MessageBag::RepairRangeReply(m) => { let h = *m.header(); - (h.operation(), h.namespace, m.into_generic()) + (h.operation(), h.group, m.into_generic()) } MessageBag::RequestStateTransfer(m) => { let h = *m.header(); - (h.operation(), h.namespace, m.into_generic()) + (h.operation(), h.group, m.into_generic()) } MessageBag::StateTransferTarget(m) => { let h = *m.header(); - (h.operation(), h.namespace, m.into_generic()) + (h.operation(), h.group, m.into_generic()) } MessageBag::RequestStateChunk(m) => { let h = *m.header(); - (h.operation(), h.namespace, m.into_generic()) + (h.operation(), h.group, m.into_generic()) } MessageBag::StateChunk(m) => { let h = *m.header(); - (h.operation(), h.namespace, m.into_generic()) + (h.operation(), h.group, m.into_generic()) } } } @@ -150,7 +150,7 @@ where // consensus-op additions need a version fence (release_min / // release_max bounds on the replica plane) before this arm is // safe to hit. - tracing::warn!(shard = self.id, error = %e, "dropping message with invalid command"); + tracing::warn!(shard = self.id, error = %e, "dropping unparsable consensus frame"); return; } }; @@ -163,7 +163,7 @@ where /// The simulator's dispatch shell uses this to drive `SimClient` requests /// through the real `on_client_request` path (auth, session binding, /// consensus submit, reply) instead of the raw `dispatch` routing the - /// shell-off fast path takes. No production caller: a `-p iggy-server-ng` + /// shell-off fast path takes. No production caller: a `-p iggy-server` /// build excludes the `simulator` feature and this method. #[cfg(any(test, feature = "simulator"))] pub fn deliver_client_request(&self, client_id: u128, message: Message) { @@ -195,7 +195,7 @@ where /// frame (`StartViewChange`, `DoViewChange`, `StartView`, `Commit`) /// or a client `Register` request. The owning consensus group is /// identified by `namespace_u64`: - /// - `METADATA_CONSENSUS_NAMESPACE` -> shard 0. + /// - `METADATA_GROUP` -> shard 0. /// - packable `IggyNamespace::inner()` -> the shard owning that /// partition's consensus group. fn route_typed( @@ -242,7 +242,7 @@ where "route_typed: operation {operation:?} fell through unclassified; \ expected is_metadata / is_partition / is_vsr_reserved" ); - if namespace_u64 == METADATA_CONSENSUS_NAMESPACE { + if namespace_u64 == METADATA_GROUP { self.try_send_to_target(0, generic, operation); return; } @@ -572,7 +572,7 @@ where // Only shard 0 owns the metadata consensus group, and // `forward_metadata_submit` always addresses shard 0, so a // non-zero shard here is a routing bug. The handler (wired - // by server-ng) replies `None` on the carried sender if it + // by the server) replies `None` on the carried sender if it // cannot submit, so the awaiting peer never blocks forever. debug_assert_eq!( self.id, 0, @@ -583,7 +583,7 @@ where LifecycleFrame::ListClients { reply } => { // Every shard handles this (not shard-0-only): each replies // with the clients whose connections it homes. The handler - // (wired by server-ng) reads this shard's `SessionManager` + // (wired by the server) reads this shard's `SessionManager` // and pushes the list over `reply`. (self.on_list_clients)(reply); } @@ -594,7 +594,7 @@ where } => { // Addressed to the shard owning `namespace` (the sender // resolved it via the shards table). The handler (wired by - // server-ng) runs the read against this shard's partitions + // the server) runs the read against this shard's partitions // plane and pushes the result over `reply`; a dropped // sender means the read is skipped and the gather side // times out. diff --git a/core/shard_allocator/src/lib.rs b/core/shard_allocator/src/lib.rs index c1cf2395ae..f0aadae26b 100644 --- a/core/shard_allocator/src/lib.rs +++ b/core/shard_allocator/src/lib.rs @@ -28,6 +28,7 @@ use cpu_allocation::{CpuAllocation, NumaConfig, allowed_cpus}; use hwlocality::Topology; use hwlocality::bitmap::SpecializedBitmapRef; use hwlocality::cpu::cpuset::CpuSet; +#[cfg(target_os = "linux")] use hwlocality::memory::binding::{MemoryBindingFlags, MemoryBindingPolicy}; use hwlocality::object::types::ObjectType::{self, NUMANode}; #[cfg(target_os = "linux")] @@ -236,36 +237,45 @@ impl ShardInfo { /// Pin the calling thread's memory to this shard's NUMA node so /// allocations stay local and fast. Does nothing if no node is set. + /// On non-Linux this does nothing (no-op), mirroring [`Self::bind_cpu`]. pub fn bind_memory(&self) -> Result<(), ShardingError> { - if let Some(node_id) = self.numa_node { - let topology = Topology::new().map_err(|err| ShardingError::TopologyDetection { - msg: err.to_string(), - })?; - - let node = topology - .objects_with_type(ObjectType::NUMANode) - .nth(node_id) - .ok_or(ShardingError::InvalidNode { - requested: node_id, - available: topology.objects_with_type(ObjectType::NUMANode).count(), + #[cfg(target_os = "linux")] + { + if let Some(node_id) = self.numa_node { + let topology = Topology::new().map_err(|err| ShardingError::TopologyDetection { + msg: err.to_string(), })?; - if let Some(nodeset) = node.nodeset() { - topology - .bind_memory( - nodeset, - MemoryBindingPolicy::Bind, - MemoryBindingFlags::THREAD | MemoryBindingFlags::STRICT, - ) - .map_err(|err| { - tracing::error!("Failed to bind memory {:?}", err); - ShardingError::BindingFailed + let node = topology + .objects_with_type(ObjectType::NUMANode) + .nth(node_id) + .ok_or(ShardingError::InvalidNode { + requested: node_id, + available: topology.objects_with_type(ObjectType::NUMANode).count(), })?; - info!("Memory bound to NUMA node {node_id}"); + if let Some(nodeset) = node.nodeset() { + topology + .bind_memory( + nodeset, + MemoryBindingPolicy::Bind, + MemoryBindingFlags::THREAD | MemoryBindingFlags::STRICT, + ) + .map_err(|err| { + tracing::error!("Failed to bind memory {:?}", err); + ShardingError::BindingFailed + })?; + + info!("Memory bound to NUMA node {node_id}"); + } } } + #[cfg(not(target_os = "linux"))] + { + tracing::debug!("NUMA memory binding skipped on non-Linux platform"); + } + Ok(()) } } diff --git a/core/simulator/Cargo.toml b/core/simulator/Cargo.toml index 32a3dae471..37df9a52e6 100644 --- a/core/simulator/Cargo.toml +++ b/core/simulator/Cargo.toml @@ -43,7 +43,7 @@ rand_xoshiro = { workspace = true } secrecy = { workspace = true } # `default-features = false` drops the mimalloc global allocator and the # web-embed feature; the sim only needs the dispatch/bootstrap library. -server-ng = { path = "../server-ng", default-features = false } +server = { path = "../server", default-features = false } server_common = { path = "../server_common", features = ["simulator"] } shard = { path = "../shard", features = ["simulator"] } strum = { workspace = true } diff --git a/core/simulator/src/client.rs b/core/simulator/src/client.rs index 028adb38cf..02ea373231 100644 --- a/core/simulator/src/client.rs +++ b/core/simulator/src/client.rs @@ -43,12 +43,12 @@ use iggy_binary_protocol::requests::users::{ UpdatePermissionsRequest, UpdateUserRequest, }; use iggy_binary_protocol::{ - AckLevel, ClientVersionInfo, IGGY_PROTOCOL_VERSION, Operation, RequestHeader, WireEncode, + AckLevel, ClientVersionInfo, IGGY_PROTOCOL_VERSION, Operation, RoutedRequestHeader, WireEncode, WireIdentifier, WireName, WirePartitioning, WirePollingStrategy, }; use metadata::stm::user::{CreatePersonalAccessTokenRequest, DeletePersonalAccessTokenRequest}; use secrecy::SecretString; -use server_common::sharding::{IggyNamespace, METADATA_CONSENSUS_NAMESPACE}; +use server_common::sharding::{IggyNamespace, METADATA_GROUP}; use server_common::{Message, iobuf::Owned}; use std::cell::Cell; @@ -130,7 +130,7 @@ impl SimClient { /// offset into a disjoint range ([`PARTITION_ID_BASE`]). A partition id can /// therefore never equal a metadata id, so a delayed or duplicated partition /// reply is never misattributed to a metadata entry in the auditor's - /// `(client, request)` map (which would trip the namespace guard and drop a + /// `(client, request)` map (which would trip the group guard and drop a /// live metadata op). This holds regardless of reply duplication, not only /// while clients are one-in-flight. fn request_id_for(&self, operation: Operation) -> u64 { @@ -162,9 +162,9 @@ impl SimClient { /// # Panics /// Panics if the register request buffer is invalid. #[allow(clippy::cast_possible_truncation)] - pub fn register(&self) -> Message { - let header_size = std::mem::size_of::(); - let header = RequestHeader { + pub fn register(&self) -> Message { + let header_size = std::mem::size_of::(); + let header = RoutedRequestHeader { command: iggy_binary_protocol::Command2::Request, operation: Operation::Register, size: header_size as u32, @@ -173,8 +173,8 @@ impl SimClient { request: 0, // Register is a vsr-reserved op: the shard router picks its // target by comparing this against the metadata consensus - // namespace, not by op class. - namespace: METADATA_CONSENSUS_NAMESPACE, + // group, not by op class. + group: METADATA_GROUP, ..Default::default() }; @@ -198,7 +198,7 @@ impl SimClient { /// Panics if a credential exceeds the wire name/secret bounds or the /// request buffer is invalid. #[allow(clippy::cast_possible_truncation)] - pub fn login(&self, username: &str, password: &str) -> Message { + pub fn login(&self, username: &str, password: &str) -> Message { let body = LoginRegisterRequest { version_info: ClientVersionInfo { protocol_version: IGGY_PROTOCOL_VERSION, @@ -211,16 +211,16 @@ impl SimClient { } .to_bytes(); - let header_size = std::mem::size_of::(); + let header_size = std::mem::size_of::(); let total_size = header_size + body.len(); - let header = RequestHeader { + let header = RoutedRequestHeader { command: iggy_binary_protocol::Command2::Request, operation: Operation::Register, size: total_size as u32, client: self.client_id, session: 0, request: 0, - namespace: METADATA_CONSENSUS_NAMESPACE, + group: METADATA_GROUP, ..Default::default() }; @@ -233,7 +233,7 @@ impl SimClient { /// # Panics /// Panics if the stream name is not a valid wire name. - pub fn create_stream(&self, name: &str) -> Message { + pub fn create_stream(&self, name: &str) -> Message { let wire = CreateStreamRequest { name: WireName::new(name).expect("stream name must be valid"), }; @@ -244,7 +244,7 @@ impl SimClient { /// # Panics /// Panics if the stream name cannot be converted to a `WireIdentifier`. - pub fn delete_stream(&self, name: &str) -> Message { + pub fn delete_stream(&self, name: &str) -> Message { let wire = DeleteStreamRequest { stream_id: WireIdentifier::named(name).expect("stream name must be valid"), }; @@ -256,7 +256,7 @@ impl SimClient { /// # Panics /// Panics if the new name or the existing stream name is not a valid /// `WireName`. - pub fn update_stream(&self, stream: &str, new_name: &str) -> Message { + pub fn update_stream(&self, stream: &str, new_name: &str) -> Message { let wire = UpdateStreamRequest { stream_id: WireIdentifier::named(stream).expect("stream name must be valid"), name: WireName::new(new_name).expect("stream name must be valid"), @@ -266,7 +266,7 @@ impl SimClient { /// # Panics /// Panics if `stream` is not a valid `WireName`. - pub fn purge_stream(&self, stream: &str) -> Message { + pub fn purge_stream(&self, stream: &str) -> Message { let wire = PurgeStreamRequest { stream_id: WireIdentifier::named(stream).expect("stream name must be valid"), }; @@ -280,7 +280,7 @@ impl SimClient { stream: &str, name: &str, partitions_count: u32, - ) -> Message { + ) -> Message { let wire = CreateTopicRequest { stream_id: WireIdentifier::named(stream).expect("stream name must be valid"), partitions_count, @@ -300,7 +300,7 @@ impl SimClient { stream: &str, topic: &str, new_name: &str, - ) -> Message { + ) -> Message { let wire = UpdateTopicRequest { stream_id: WireIdentifier::named(stream).expect("stream name must be valid"), topic_id: WireIdentifier::named(topic).expect("topic name must be valid"), @@ -315,7 +315,7 @@ impl SimClient { /// # Panics /// Panics if `stream` or `topic` is not a valid `WireName`. - pub fn delete_topic(&self, stream: &str, topic: &str) -> Message { + pub fn delete_topic(&self, stream: &str, topic: &str) -> Message { let wire = DeleteTopicRequest { stream_id: WireIdentifier::named(stream).expect("stream name must be valid"), topic_id: WireIdentifier::named(topic).expect("topic name must be valid"), @@ -325,7 +325,7 @@ impl SimClient { /// # Panics /// Panics if `stream` or `topic` is not a valid `WireName`. - pub fn purge_topic(&self, stream: &str, topic: &str) -> Message { + pub fn purge_topic(&self, stream: &str, topic: &str) -> Message { let wire = PurgeTopicRequest { stream_id: WireIdentifier::named(stream).expect("stream name must be valid"), topic_id: WireIdentifier::named(topic).expect("topic name must be valid"), @@ -340,7 +340,7 @@ impl SimClient { stream: &str, topic: &str, partitions_count: u32, - ) -> Message { + ) -> Message { let wire = CreatePartitionsRequest { stream_id: WireIdentifier::named(stream).expect("stream name must be valid"), topic_id: WireIdentifier::named(topic).expect("topic name must be valid"), @@ -356,7 +356,7 @@ impl SimClient { stream: &str, topic: &str, partitions_count: u32, - ) -> Message { + ) -> Message { let wire = DeletePartitionsRequest { stream_id: WireIdentifier::named(stream).expect("stream name must be valid"), topic_id: WireIdentifier::named(topic).expect("topic name must be valid"), @@ -373,7 +373,7 @@ impl SimClient { topic: &str, partition_id: u32, segments_count: u32, - ) -> Message { + ) -> Message { let wire = DeleteSegmentsRequest { stream_id: WireIdentifier::named(stream).expect("stream name must be valid"), topic_id: WireIdentifier::named(topic).expect("topic name must be valid"), @@ -390,7 +390,7 @@ impl SimClient { stream: &str, topic: &str, name: &str, - ) -> Message { + ) -> Message { let wire = CreateConsumerGroupRequest { stream_id: WireIdentifier::named(stream).expect("stream name must be valid"), topic_id: WireIdentifier::named(topic).expect("topic name must be valid"), @@ -406,7 +406,7 @@ impl SimClient { stream: &str, topic: &str, group: &str, - ) -> Message { + ) -> Message { let wire = DeleteConsumerGroupRequest { stream_id: WireIdentifier::named(stream).expect("stream name must be valid"), topic_id: WireIdentifier::named(topic).expect("topic name must be valid"), @@ -422,7 +422,7 @@ impl SimClient { username: &str, password: &str, status: u8, - ) -> Message { + ) -> Message { let wire = CreateUserRequest { username: WireName::new(username).expect("username must be valid"), password: password.to_string(), @@ -440,7 +440,7 @@ impl SimClient { user: &str, new_username: Option<&str>, status: Option, - ) -> Message { + ) -> Message { let wire = UpdateUserRequest { user_id: WireIdentifier::named(user).expect("username must be valid"), username: new_username.map(|n| WireName::new(n).expect("username must be valid")), @@ -451,7 +451,7 @@ impl SimClient { /// # Panics /// Panics if `user` is not a valid `WireName`. - pub fn delete_user(&self, user: &str) -> Message { + pub fn delete_user(&self, user: &str) -> Message { let wire = DeleteUserRequest { user_id: WireIdentifier::named(user).expect("username must be valid"), }; @@ -465,7 +465,7 @@ impl SimClient { user: &str, current_password: &str, new_password: &str, - ) -> Message { + ) -> Message { let wire = ChangePasswordRequest { user_id: WireIdentifier::named(user).expect("username must be valid"), current_password: current_password.to_string(), @@ -476,7 +476,7 @@ impl SimClient { /// # Panics /// Panics if `user` is not a valid `WireName`. - pub fn update_permissions(&self, user: &str) -> Message { + pub fn update_permissions(&self, user: &str) -> Message { let wire = UpdatePermissionsRequest { user_id: WireIdentifier::named(user).expect("username must be valid"), permissions: None, @@ -486,7 +486,11 @@ impl SimClient { /// # Panics /// Panics if `name` is not a valid `WireName`. - pub fn create_personal_access_token(&self, name: &str, expiry: u64) -> Message { + pub fn create_personal_access_token( + &self, + name: &str, + expiry: u64, + ) -> Message { let wire = CreatePersonalAccessTokenRequest { user_id: 0, name: WireName::new(name).expect("PAT name must be valid"), @@ -502,7 +506,7 @@ impl SimClient { /// # Panics /// Panics if `name` is not a valid `WireName`. - pub fn delete_personal_access_token(&self, name: &str) -> Message { + pub fn delete_personal_access_token(&self, name: &str) -> Message { let wire = DeletePersonalAccessTokenRequest { user_id: 0, name: WireName::new(name).expect("PAT name must be valid"), @@ -519,16 +523,16 @@ impl SimClient { /// path converts it to `SendMessages2` via `transcode_legacy_request`. /// /// # Panics - /// Panics if a namespace id exceeds `u32` or the request buffer is invalid. + /// Panics if a group id exceeds `u32` or the request buffer is invalid. pub fn send_messages( &self, - namespace: IggyNamespace, + group: IggyNamespace, messages: &[Bytes], - ) -> Message { - let to_u32 = |v: usize| u32::try_from(v).expect("namespace id fits u32"); - let stream_id = WireIdentifier::Numeric(to_u32(namespace.stream_id())); - let topic_id = WireIdentifier::Numeric(to_u32(namespace.topic_id())); - let partitioning = WirePartitioning::PartitionId(to_u32(namespace.partition_id())); + ) -> Message { + let to_u32 = |v: usize| u32::try_from(v).expect("group id fits u32"); + let stream_id = WireIdentifier::Numeric(to_u32(group.stream_id())); + let topic_id = WireIdentifier::Numeric(to_u32(group.topic_id())); + let partitioning = WirePartitioning::PartitionId(to_u32(group.partition_id())); // Stamp a deterministic, non-zero id per message. `id: 0` (the real // SDK's server-assigned path) would make the server mint an unseeded @@ -549,11 +553,11 @@ impl SimClient { let mut buf = BytesMut::with_capacity(size); SendMessagesEncoder::encode(&mut buf, &stream_id, &topic_id, &partitioning, &raw); - self.build_request_with_namespace(Operation::SendMessages, &buf, namespace) + self.build_request_with_namespace(Operation::SendMessages, &buf, group) } /// Build a `POLL_MESSAGES` request for an individual consumer, reading - /// `count` messages from offset 0 of `namespace`'s partition. + /// `count` messages from offset 0 of `group`'s partition. /// /// A `NonReplicated` read: the command code sits in the header's /// `reserved` prefix, and the request id ECHOES the current metadata @@ -564,8 +568,8 @@ impl SimClient { /// # Panics /// Panics if the session is unbound or the request buffer is invalid. #[allow(clippy::cast_possible_truncation)] - pub fn poll_messages(&self, namespace: IggyNamespace, count: u32) -> Message { - let (stream_id, topic_id, partition_id) = namespace_ids(namespace); + pub fn poll_messages(&self, group: IggyNamespace, count: u32) -> Message { + let (stream_id, topic_id, partition_id) = namespace_ids(group); let body = PollMessagesRequest { consumer: WireConsumer::consumer(WireIdentifier::Numeric(self.client_id as u32)), stream_id, @@ -577,11 +581,11 @@ impl SimClient { } .to_bytes(); - let header_size = std::mem::size_of::(); + let header_size = std::mem::size_of::(); let total_size = header_size + body.len(); let mut reserved = [0u8; 52]; reserved[..4].copy_from_slice(&POLL_MESSAGES_CODE.to_le_bytes()); - let header = RequestHeader { + let header = RoutedRequestHeader { command: iggy_binary_protocol::Command2::Request, operation: Operation::NonReplicated, size: total_size as u32, @@ -589,7 +593,7 @@ impl SimClient { session: self.session_id(), request: self.request_counter.get(), reserved, - namespace: namespace.inner(), + group: group.inner(), ..Default::default() }; @@ -602,12 +606,12 @@ impl SimClient { pub fn store_consumer_offset( &self, - namespace: IggyNamespace, + group: IggyNamespace, consumer_kind: u8, consumer_id: u32, offset: u64, - ) -> Message { - let (stream_id, topic_id, partition_id) = namespace_ids(namespace); + ) -> Message { + let (stream_id, topic_id, partition_id) = namespace_ids(group); let request = StoreConsumerOffsetRequest { consumer: namespace_consumer(consumer_kind, consumer_id), stream_id, @@ -618,17 +622,17 @@ impl SimClient { self.build_request_with_namespace( Operation::StoreConsumerOffset, &request.to_bytes(), - namespace, + group, ) } pub fn delete_consumer_offset( &self, - namespace: IggyNamespace, + group: IggyNamespace, consumer_kind: u8, consumer_id: u32, - ) -> Message { - let (stream_id, topic_id, partition_id) = namespace_ids(namespace); + ) -> Message { + let (stream_id, topic_id, partition_id) = namespace_ids(group); let request = DeleteConsumerOffsetRequest { consumer: namespace_consumer(consumer_kind, consumer_id), stream_id, @@ -638,7 +642,7 @@ impl SimClient { self.build_request_with_namespace( Operation::DeleteConsumerOffset, &request.to_bytes(), - namespace, + group, ) } @@ -647,16 +651,16 @@ impl SimClient { /// /// # Panics /// Panics on payload too large for `Owned::<4096>` or invalid - /// `Message` parse; both are simulator misconfig. + /// `Message` parse; both are simulator misconfig. pub fn store_consumer_offset_2( &self, - namespace: IggyNamespace, + group: IggyNamespace, consumer_kind: u8, consumer_id: u32, offset: u64, ack: AckLevel, - ) -> Message { - let (stream_id, topic_id, partition_id) = namespace_ids(namespace); + ) -> Message { + let (stream_id, topic_id, partition_id) = namespace_ids(group); let request = StoreConsumerOffset2Request { consumer: namespace_consumer(consumer_kind, consumer_id), stream_id, @@ -668,7 +672,7 @@ impl SimClient { self.build_request_with_namespace( Operation::StoreConsumerOffset2, &request.to_bytes(), - namespace, + group, ) } @@ -676,15 +680,15 @@ impl SimClient { /// /// # Panics /// Panics on payload too large for `Owned::<4096>` or invalid - /// `Message` parse; both are simulator misconfig. + /// `Message` parse; both are simulator misconfig. pub fn delete_consumer_offset_2( &self, - namespace: IggyNamespace, + group: IggyNamespace, consumer_kind: u8, consumer_id: u32, ack: AckLevel, - ) -> Message { - let (stream_id, topic_id, partition_id) = namespace_ids(namespace); + ) -> Message { + let (stream_id, topic_id, partition_id) = namespace_ids(group); let request = DeleteConsumerOffset2Request { consumer: namespace_consumer(consumer_kind, consumer_id), stream_id, @@ -695,7 +699,7 @@ impl SimClient { self.build_request_with_namespace( Operation::DeleteConsumerOffset2, &request.to_bytes(), - namespace, + group, ) } @@ -703,12 +707,12 @@ impl SimClient { &self, operation: Operation, payload: &[u8], - namespace: IggyNamespace, - ) -> Message { - let header_size = std::mem::size_of::(); + group: IggyNamespace, + ) -> Message { + let header_size = std::mem::size_of::(); let total_size = header_size + payload.len(); - let header = self.header(operation, namespace.inner(), total_size); + let header = self.header(operation, group.inner(), total_size); let header_bytes = bytemuck::bytes_of(&header); let mut buffer = Vec::with_capacity(total_size); @@ -719,14 +723,14 @@ impl SimClient { .expect("request buffer must contain a valid request message") } - fn build_request(&self, operation: Operation, payload: &[u8]) -> Message { - let header_size = std::mem::size_of::(); + fn build_request(&self, operation: Operation, payload: &[u8]) -> Message { + let header_size = std::mem::size_of::(); let total_size = header_size + payload.len(); // Every `build_request` caller is a metadata-plane op (partition // ops go through `build_request_with_namespace`), and metadata - // requests carry the metadata consensus namespace on the wire. - let header = self.header(operation, METADATA_CONSENSUS_NAMESPACE, total_size); + // requests carry the metadata consensus group on the wire. + let header = self.header(operation, METADATA_GROUP, total_size); let header_bytes = bytemuck::bytes_of(&header); let mut buffer = Vec::with_capacity(total_size); @@ -738,8 +742,8 @@ impl SimClient { } #[allow(clippy::cast_possible_truncation)] - fn header(&self, operation: Operation, namespace: u64, total_size: usize) -> RequestHeader { - RequestHeader { + fn header(&self, operation: Operation, group: u64, total_size: usize) -> RoutedRequestHeader { + RoutedRequestHeader { command: iggy_binary_protocol::Command2::Request, operation, size: total_size as u32, @@ -755,7 +759,7 @@ impl SimClient { timestamp: 0, // TODO: Use actual timestamp session: self.session_id(), request: self.request_id_for(operation), - namespace, + group, ..Default::default() } } @@ -772,11 +776,11 @@ const fn namespace_consumer(kind: u8, consumer_id: u32) -> WireConsumer { } } -/// Decompose a namespace into the `(stream_id, topic_id, partition_id)` wire +/// Decompose a group into the `(stream_id, topic_id, partition_id)` wire /// identifiers the consumer-offset requests carry. Namespace ids are small /// test values that always fit `u32`. fn namespace_ids(ns: IggyNamespace) -> (WireIdentifier, WireIdentifier, Option) { - let to_u32 = |v: usize| u32::try_from(v).expect("namespace id fits u32"); + let to_u32 = |v: usize| u32::try_from(v).expect("group id fits u32"); ( WireIdentifier::Numeric(to_u32(ns.stream_id())), WireIdentifier::Numeric(to_u32(ns.topic_id())), diff --git a/core/simulator/src/deps.rs b/core/simulator/src/deps.rs index de7c2240ed..2502b8e9b5 100644 --- a/core/simulator/src/deps.rs +++ b/core/simulator/src/deps.rs @@ -176,6 +176,37 @@ impl>> Journal for SimJournal { where Self: 'a; + fn last_op(&self) -> Option { + self.last_op.get() + } + + /// Drop the suffix, so a simulated backup whose entries disagree with a started + /// view reconciles the way a real one does. Mirrors + /// `PrepareJournal::truncate_from`, whose watermark stays put; here it never moves. + async fn truncate_from(&self, from_op: u64) -> std::io::Result { + if from_op == 0 { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "truncate_from: ops are 1-based, so 0 would discard the whole journal", + )); + } + #[cfg(debug_assertions)] + let _guard = JournalAccessGuard::new(&self.accessing); + let headers = unsafe { &mut *self.headers.get() }; + let offsets = unsafe { &mut *self.offsets.get() }; + let doomed: Vec = headers + .keys() + .copied() + .filter(|op| *op >= from_op) + .collect(); + for op in &doomed { + headers.remove(op); + offsets.remove(op); + } + self.last_op.set(headers.keys().copied().max()); + Ok(doomed.len()) + } + /// The simulated journal retains everything for the run, so nothing is /// ever superseded by a snapshot. Answered explicitly (the trait has no /// default) so a simulated state transfer has to opt into a watermark @@ -268,6 +299,24 @@ impl SimJournal { self.last_op.get() } + /// Forget one op, leaving a hole exactly where a lost prepare would. + /// + /// Tests only. The alternative is choreographing `Prepare`, `Commit` and + /// `RepairPrepare` drops on a directed link until a replica falls behind, which + /// is fragile to tune; the scenarios are about what a replica does with a hole, + /// not how it got one. + /// + /// `last_op` is deliberately left alone: a hole below the head must not look like + /// a shorter log, since that is the state a view change has to survive. + pub fn forget_op(&self, op: u64) -> bool { + #[cfg(debug_assertions)] + let _guard = JournalAccessGuard::new(&self.accessing); + let headers = unsafe { &mut *self.headers.get() }; + let offsets = unsafe { &mut *self.offsets.get() }; + offsets.remove(&op); + headers.remove(&op).is_some() + } + /// The committed watermark to restore after a restart, mirroring /// `metadata::recover`. On a solo cluster every appended op commits the instant /// it is durable, so the head IS the commit point; otherwise the highest diff --git a/core/simulator/src/lib.rs b/core/simulator/src/lib.rs index 64aecc440a..d4c2ba0c79 100644 --- a/core/simulator/src/lib.rs +++ b/core/simulator/src/lib.rs @@ -164,7 +164,7 @@ impl Simulator { } /// [`Simulator::new`] with `shards_per_replica` shards on every replica, - /// meshed exactly like server-ng bootstrap: metadata plane on shard 0, + /// meshed exactly like the server bootstrap: metadata plane on shard 0, /// partitions hash-assigned, one pump task per shard. /// /// # Panics @@ -186,7 +186,7 @@ impl Simulator { } /// [`Simulator::with_shards`] with the deterministic dispatch shell on: - /// every shard wires server-ng's real dispatch handlers, so a client + /// every shard wires the server's real dispatch handlers, so a client /// request runs as a task the seeded executor interleaves with the /// pump. Off (the default) keeps the raw-`on_message` fast path. /// @@ -276,7 +276,7 @@ impl Simulator { let mut shards = Vec::with_capacity(usize::from(shards_per_replica)); let mut stop_txs = Vec::with_capacity(usize::from(shards_per_replica)); let mut pump_tasks = Vec::with_capacity(usize::from(shards_per_replica)); - // Single-writer metadata (mirrors server-ng bootstrap): shard 0 + // Single-writer metadata (mirrors the server bootstrap): shard 0 // builds the writable STM and mints a factory bundle; every peer // shard rebuilds a reader-mode mirror from it and sees committed // metadata through the shared read handle. Shards are built in index @@ -317,7 +317,7 @@ impl Simulator { ); } - // Same wiring as server-ng bootstrap: one pump task per + // Same wiring as the server bootstrap: one pump task per // shard, stopped only by the (held) stop channel or a // crash abort. let (stop_tx, stop_rx) = shard::channel::<()>(1); @@ -743,7 +743,7 @@ impl Simulator { let metadata_incarnation = self.replicas[idx].metadata_incarnation + 1; // Partition superblocks carry forward too: a group re-materialised after // the restart must recover its recorded view from the same store, exactly - // as a rebooted server-ng partition reads the record in its directory. + // as a rebooted server partition reads the record in its directory. let partition_superblocks = std::mem::take(&mut *self.replicas[idx].partition_superblocks.borrow_mut()); @@ -814,7 +814,7 @@ impl Simulator { }; // Re-materialise every group this replica had before the crash, as a - // rebooted server-ng re-opens every partition directory it owns. This + // rebooted the server re-opens every partition directory it owns. This // is what makes the carried-forward superblock load-bearing: the group // recovers the `(view, log_view)` it recorded instead of re-entering // view 0. @@ -945,7 +945,7 @@ impl Simulator { /// the routing row on every shard of that replica. /// /// Shared by [`SimCluster::init_partition`] and the restart path: a rebooted -/// server-ng re-opens every partition directory it owns, so the sim has to +/// server re-opens every partition directory it owns, so the sim has to /// re-materialise too, otherwise the superblock a restart carries forward is /// never read back and the recovered-view branch is dead code. fn materialise_partition(replica: &SimReplica, namespace: IggyNamespace) { @@ -1667,13 +1667,16 @@ mod tests { // shifts the trace. Re-lock on intentional changes; expect re-locks until // error discriminants and reply bodies stabilize the wire format. // - // Re-locked when the sim adopted METADATA_CONSENSUS_NAMESPACE (1<<63) + // Re-locked when the sim adopted METADATA_GROUP (1<<63) // for metadata requests and the metadata consensus group, replacing // the sim-only 0: reply headers and the per-group timeout-jitter seed // (replica_id ^ namespace) both changed. The old 0 only ever routed // correctly because `hash % 1 == 0` at one shard per replica. + // Re-locked again when replies stopped echoing a group id (the + // client wire lost its namespace field): the reply-hash tuple + // dropped that component. assert_eq!( - h1, 0x530D_499C_5DBE_A2BE, + h1, 0xCF1F_BC79_B44A_65F7, "workload reply hash drifted from locked baseline" ); } @@ -2319,15 +2322,7 @@ mod tests { } for reply in sim.step() { let h = reply.header(); - ( - h.client, - h.request, - h.op, - h.commit, - h.namespace, - h.operation as u8, - ) - .hash(&mut hasher); + (h.client, h.request, h.op, h.commit, h.operation as u8).hash(&mut hasher); let cmds = wl.on_reply(&reply); apply_sim_commands(&mut sim, &cmds); replies_seen += 1; @@ -2618,7 +2613,7 @@ mod tests { sim.schedule_hash() } - /// Turning the dispatch shell on wires server-ng's real deferred + /// Turning the dispatch shell on wires the server's real deferred /// handlers on every shard. With no client traffic none of them is /// reached, so the consensus plane both replays deterministically and /// matches the shell-off schedule: the toggle is genuinely off the @@ -2951,7 +2946,7 @@ mod tests { let header: &PrepareOkHeader = bytemuck::checked::from_bytes( &packet.message.as_slice()[..std::mem::size_of::()], ); - header.namespace == BLOCKED_NS.load(Ordering::Relaxed) + header.group == BLOCKED_NS.load(Ordering::Relaxed) } server_common::MemoryPool::init_pool(&server_common::MemoryPoolConfigOther { @@ -3007,13 +3002,16 @@ mod tests { // ns_b shares the client, the replicas, and the shard, but has its // own consensus group: it must commit while ns_a stays wedged. let msg = client.send_messages(ns_b, &[Bytes::from_static(b"independent")]); + // Replies no longer carry a group id; correlate by the request id + // this send was stamped with. + let ns_b_request = msg.header().request; sim.submit_request(client_id, 0, msg.into_generic()); let mut independent_replies = 0usize; for _ in 0..100 { for reply in sim.step() { assert_eq!( - reply.header().namespace, - ns_b.inner(), + reply.header().request, + ns_b_request, "only ns_b may commit while ns_a's acks are blocked" ); independent_replies += 1; @@ -3032,7 +3030,9 @@ mod tests { let mut drained_replies = 0usize; for _ in 0..800 { for reply in sim.step() { - if reply.header().namespace == ns_a.inner() { + // Only ns_a replies are outstanding once ns_b committed + // above, so every reply counts toward the drain. + if reply.header().request != ns_b_request { drained_replies += 1; } } @@ -3097,3 +3097,162 @@ mod tests { assert_no_frame_drops(&sim); } } + +#[cfg(test)] +mod view_change_data_loss_tests { + //! A committed, client-acknowledged op must survive a view change even when + //! the replica that becomes primary is the one missing it. + //! + //! Without the sender's log suffix on the `DoViewChange`, the new primary adopts + //! the winner's op NUMBER, rebuilds its pipeline from its OWN journal, hits the + //! hole, and truncates the range as "decided lost" -- discarding an op journaled + //! on a quorum and already replied to. The next client op then reuses the number + //! and collides with the stale entry on the up-to-date backup. + //! + //! The hole here is punched at the commit point, so the assertion that catches a + //! regression is "the op came back", not "the head did not regress": with nothing + //! uncommitted there is no pipeline rebuild to truncate. The `dvc_merge` unit + //! tests cover the sequencer-truncation path directly. + + use super::*; + use consensus::{Sequencer, Status}; + use journal::Journal; + + /// Whether a replica's shard-0 metadata consensus is a settled primary in a + /// view past the one that crashed. + fn is_new_metadata_primary(sim: &Simulator, replica: u8) -> bool { + sim.replicas[replica as usize].shards[0] + .plane + .metadata() + .consensus + .as_ref() + .is_some_and(|consensus| { + consensus.view() > 0 + && consensus.status() == Status::Normal + && consensus.is_primary() + }) + } + + /// `(head op, commit_max)` of a replica's shard-0 metadata consensus. + fn metadata_progress(sim: &Simulator, replica: u8) -> (u64, u64) { + let consensus = sim.replicas[replica as usize].shards[0] + .plane + .metadata() + .consensus + .as_ref() + .expect("shard 0 owns metadata consensus"); + ( + consensus.sequencer().current_sequence(), + consensus.commit_max(), + ) + } + + /// Whether a replica's metadata journal holds `op`. + fn metadata_holds(sim: &Simulator, replica: u8, op: u64) -> bool { + let journal = sim.replicas[replica as usize].shards[0] + .plane + .metadata() + .journal + .as_ref() + .expect("shard 0 owns the metadata journal"); + let slot = usize::try_from(op).expect("op fits usize"); + Journal::header(journal.as_ref(), slot).is_some() + } + + /// Drop `op` from a replica's metadata journal, leaving a hole. + fn metadata_forget(sim: &Simulator, replica: u8, op: u64) -> bool { + sim.replicas[replica as usize].shards[0] + .plane + .metadata() + .journal + .as_ref() + .expect("shard 0 owns the metadata journal") + .forget_op(op) + } + + #[test] + fn given_committed_op_missing_on_next_primary_when_primary_crashes_should_survive_view_change() + { + server_common::MemoryPool::init_pool(&server_common::MemoryPoolConfigOther { + enabled: false, + size: iggy_common::IggyByteSize::from(0u64), + bucket_capacity: 1, + }); + + let replica_count: u8 = 3; + let client_id: u128 = 1; + let network_opts = packet::PacketSimulatorOptions { + node_count: replica_count, + client_count: 1, + ..packet::PacketSimulatorOptions::default() + }; + let mut sim = Simulator::new( + replica_count as usize, + std::iter::once(client_id), + network_opts, + ); + let client = SimClient::new(client_id); + + // Commit some metadata ops so there is a log to lose. Registering binds + // a session, and seeding a stream/topic/partition commits several more. + sim.register_client_with_primary(&client); + sim.seed_stream_topic_partition(IggyNamespace::new(1, 1, 0)); + for _ in 0..200 { + sim.step(); + } + + // Replica 0 is primary for view 0, so replica 1 is primary-elect for view 1 + // (view % replica_count): the replica whose hole decides the outcome. + let next_primary: u8 = 1; + let (_, committed) = metadata_progress(&sim, next_primary); + assert!( + committed > 0, + "the test needs committed metadata ops to be able to lose one" + ); + + // Every replica must hold the op: the point is that it IS recoverable, and + // only the incoming primary lacks it. + for replica in 0..replica_count { + assert!( + metadata_holds(&sim, replica, committed), + "replica {replica} must hold op {committed} before the hole is punched" + ); + } + + // Punch the hole: the incoming primary forgets an op its peers still hold. + assert!( + metadata_forget(&sim, next_primary, committed), + "op {committed} must have been present to forget" + ); + + sim.replica_crash(0); + for _ in 0..1500 { + sim.step(); + } + + // A primary must emerge among the survivors. + let primary = (1..replica_count) + .find(|&replica| is_new_metadata_primary(&sim, replica)) + .expect("a metadata primary must be elected after the old one crashes"); + + let (head, commit_max) = metadata_progress(&sim, primary); + + // The committed op must not have been discarded. + assert!( + head >= committed, + "the new primary's head ({head}) regressed below the committed op ({committed}); \ + a committed, acknowledged op was discarded by the view change" + ); + assert!( + commit_max >= committed, + "commit_max ({commit_max}) regressed below the committed op ({committed})" + ); + + // And back in the new primary's journal: the view change repaired the hole + // from a peer that offered the body, rather than declaring the op lost. + assert!( + metadata_holds(&sim, primary, committed), + "op {committed} must be repaired back into the new primary's journal" + ); + } +} diff --git a/core/simulator/src/replica.rs b/core/simulator/src/replica.rs index 9abf73c325..2764b96da6 100644 --- a/core/simulator/src/replica.rs +++ b/core/simulator/src/replica.rs @@ -19,7 +19,7 @@ use crate::bus::{SharedSimOutbox, SimOutbox}; use crate::deps::SimSuperblock; use crate::deps::{MemStorage, SimJournal, SimMuxStateMachine, SimSnapshot}; use configs::server::PersonalAccessTokenConfig; -use configs::server_ng::NgSystemConfig; +use configs::server::ServerSystemConfig; use consensus::{ConsensusClock, LocalPipeline, Sequencer, VsrConsensus, VsrState}; use iggy_common::IggyByteSize; use iggy_common::variadic; @@ -28,9 +28,9 @@ use metadata::stm::stream::{Streams, StreamsInner}; use metadata::stm::user::{Users, UsersInner}; use metadata::{IggyMetadata, apply_committed_prepare}; use partitions::{IggyPartitions, PartitionsConfig}; +use server::bootstrap::{ShellHandlers, ShellShardHandle, wire_shell_handlers}; use server_common::crypto; -use server_common::sharding::{METADATA_CONSENSUS_NAMESPACE, ShardId}; -use server_ng::bootstrap::{ShellHandlers, ShellShardHandle, wire_shell_handlers}; +use server_common::sharding::{METADATA_GROUP, ShardId}; use shard::shards_table::PapayaShardsTable; use std::cell::RefCell; use std::rc::Rc; @@ -52,7 +52,7 @@ pub const SHELL_ROOT_PASSWORD: &str = "iggy"; // // `PapayaShardsTable` (the production namespace -> shard routing table) // instead of the always-`None` `()` impl: each shard owns its own table -// instance, exactly as server-ng wires it. Until rows are seeded the +// instance, exactly as the server wires it. Until rows are seeded the // router falls back to the deterministic hash assignment, which at one // shard per replica always resolves to shard 0. pub type Replica = shard::IggyShard< @@ -70,7 +70,7 @@ pub type Replica = shard::IggyShard< /// /// Shard 0 (the sole writer) mints one via `factory_bundle`; every peer shard /// rebuilds a reader-mode mirror from it with `from_factory_bundle`, exactly as -/// server-ng bootstrap does with its `ServerNgMetadataBundle`. +/// the server bootstrap does with its `ServerMetadataBundle`. /// `Clone + Send + Sync`. pub type SimMetadataBundle = ::Bundle; @@ -90,11 +90,11 @@ pub const SIM_INBOX_CAPACITY: usize = 8192; /// /// `shell` selects the dispatch handlers. Off is the fast path: inert /// no-ops, so the simulator drives raw client frames straight into -/// `IggyShard::on_message`. On wires server-ng's real deferred dispatch +/// `IggyShard::on_message`. On wires the server's real deferred dispatch /// handlers (via [`wire_shell_handlers`]), exactly as production does, so /// a client request runs as a task concurrent with the pump. /// -/// Mirrors server-ng bootstrap's single-writer metadata: the consensus +/// Mirrors the server bootstrap's single-writer metadata: the consensus /// group, journal, snapshot, and the only writable metadata STM live on /// shard 0. Shard 0 mints a [`SimMetadataBundle`] (returned as the second /// tuple element); every peer shard passes it back in as `reader_bundle` @@ -126,7 +126,7 @@ pub fn new_shard( recovered_state: Option, incarnation: u128, ) -> (Rc, Option) { - // Metadata is single-writer, mirroring server-ng bootstrap. Shard 0 owns + // Metadata is single-writer, mirroring the server bootstrap. Shard 0 owns // the only writable STM; every peer shard rebuilds a reader-mode mirror from // shard 0's factory bundle and sees committed metadata through the shared // left-right read handle (each apply `publish`es, bounding reader staleness @@ -182,7 +182,7 @@ pub fn new_shard( CLUSTER_ID, replica_id, replica_count, - METADATA_CONSENSUS_NAMESPACE, + METADATA_GROUP, SharedSimOutbox(Rc::clone(bus)), LocalPipeline::new(), clock.clone(), @@ -193,7 +193,7 @@ pub fn new_shard( consensus.set_incarnation(incarnation); // View/log_view come from the durable superblock; op, commit, and the // last-prepare markers come from the retained WAL. Independent inputs, - // mirroring server-ng's restore_metadata_consensus. + // mirroring the server's restore_metadata_consensus. let last_header = metadata_journal .as_ref() .and_then(|journal| journal.last_header()); @@ -283,7 +283,9 @@ pub fn new_shard( messages_required_to_save: 1000, size_of_messages_required_to_save: IggyByteSize::from(4 * 1024 * 1024), enforce_fsync: false, //Disable fsync for simulation + validate_checksum: true, segment_size: IggyByteSize::from(1024 * 1024 * 1024), + preallocate_segments: false, encryptor: None, }; @@ -314,7 +316,7 @@ pub fn new_shard( wire_shell_handlers( &SharedSimOutbox(Rc::clone(bus)), &shard_handle, - Arc::new(NgSystemConfig::default()), + Arc::new(ServerSystemConfig::default()), // Default-config PAT cap, like the system config above, so sim // ingress admits exactly what a default-configured server does. PersonalAccessTokenConfig::default().max_tokens_per_user, diff --git a/core/simulator/src/workload/auditor.rs b/core/simulator/src/workload/auditor.rs index 2cfe2c28db..d9616ec4ec 100644 --- a/core/simulator/src/workload/auditor.rs +++ b/core/simulator/src/workload/auditor.rs @@ -122,7 +122,7 @@ impl ServerAuditor { /// classifies, applies effects, decrements the counter. /// - [`OnReply::NsMismatch`]: entry consumed but reply namespace /// diverged from the request namespace. Caller decrements but - /// skips effects + `note_committed`. Unreachable today (server-ng + /// skips effects + `note_committed`. Unreachable today (the server /// echoes the request namespace); guards future routing/dedup /// bugs from wedging a client at `CLIENT_REQUEST_QUEUE_MAX = 1`. /// - [`OnReply::Unknown`]: no matching entry (duplicate cached @@ -148,16 +148,11 @@ impl ServerAuditor { return OnReply::Unknown; }; - // Reply's namespace must match the namespace the request was - // submitted to. A mismatch means the reply landed in the wrong - // VSR group's bookkeeping; refuse to apply effects against the - // wrong shadow bucket. Entry already consumed. - if entry.request_namespace != header.namespace { - self.stats.replies_unknown += 1; - return OnReply::NsMismatch; - } - - let ns_key = (header.client, header.namespace); + // Replies no longer echo a group id (the client wire has no + // namespace field at all), so correlation rests entirely on the + // (client, request) key that fetched `entry`; the group the request + // was submitted to comes from the sim's own bookkeeping. + let ns_key = (header.client, entry.request_namespace); let last_commit = self .last_commit_watermark_per_client_ns .entry(ns_key) diff --git a/core/simulator/src/workload/effect.rs b/core/simulator/src/workload/effect.rs index f9adb7edd1..a9d8c868c2 100644 --- a/core/simulator/src/workload/effect.rs +++ b/core/simulator/src/workload/effect.rs @@ -42,6 +42,16 @@ pub enum Effect { stream: String, name: String, }, + AddPartitions { + stream: String, + topic: String, + count: u32, + }, + RemovePartitions { + stream: String, + topic: String, + count: u32, + }, AddUser { name: String, }, diff --git a/core/simulator/src/workload/mod.rs b/core/simulator/src/workload/mod.rs index 045e946fa3..2e641066cb 100644 --- a/core/simulator/src/workload/mod.rs +++ b/core/simulator/src/workload/mod.rs @@ -40,7 +40,7 @@ use crate::workload::ops::InFlight; use actions::Action; use auditor::{OnReply, ServerAuditor}; use effect::SimCommand; -use iggy_binary_protocol::{ReplyHeader, RequestHeader, result_code}; +use iggy_binary_protocol::{ReplyHeader, RoutedRequestHeader, result_code}; use invariants::Invariants; use metadata::stm::result::result_code_recognized; use options::WorkloadOptions; @@ -138,7 +138,10 @@ impl Workload { /// PRNG before `sample` runs, so they advance the trace even when `sample` /// returns `None` (a targeted outcome whose precondition is unmet, e.g. a /// duplicate-name target with an empty shadow). `samples_none` counts these. - pub fn build_request(&mut self, client: &SimClient) -> Option<(u8, Message)> { + pub fn build_request( + &mut self, + client: &SimClient, + ) -> Option<(u8, Message)> { if !self.client_idle(client.client_id()) { return None; } @@ -167,7 +170,7 @@ impl Workload { action, input, outcome, - request_namespace: header.namespace, + request_namespace: header.group, }, ); *self diff --git a/core/simulator/src/workload/ops/change_password.rs b/core/simulator/src/workload/ops/change_password.rs index edd2e0cf90..59668c92c9 100644 --- a/core/simulator/src/workload/ops/change_password.rs +++ b/core/simulator/src/workload/ops/change_password.rs @@ -18,7 +18,7 @@ //! `ChangePassword` op. Targets `Ok` (rotate live user's password) or //! `UserNotFound` (fabricated user). -use iggy_binary_protocol::RequestHeader; +use iggy_binary_protocol::RoutedRequestHeader; use rand::RngExt; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -68,7 +68,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.change_password(&input.user, &input.current_password, &input.new_password) } diff --git a/core/simulator/src/workload/ops/create_consumer_group.rs b/core/simulator/src/workload/ops/create_consumer_group.rs index 66113e65b9..df1bc5c639 100644 --- a/core/simulator/src/workload/ops/create_consumer_group.rs +++ b/core/simulator/src/workload/ops/create_consumer_group.rs @@ -21,7 +21,7 @@ //! (a fabricated parent stream), `TopicNotFound` (a live stream with a //! fabricated topic), or `NameAlreadyExists` (an existing group name). -use iggy_binary_protocol::RequestHeader; +use iggy_binary_protocol::RoutedRequestHeader; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -87,7 +87,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.create_consumer_group(&input.stream, &input.topic, &input.name) } diff --git a/core/simulator/src/workload/ops/create_partitions.rs b/core/simulator/src/workload/ops/create_partitions.rs index 4ee8eb3004..2fdc6a2c4f 100644 --- a/core/simulator/src/workload/ops/create_partitions.rs +++ b/core/simulator/src/workload/ops/create_partitions.rs @@ -19,10 +19,11 @@ //! //! Targets `Ok` (live topic), `StreamNotFound` (fabricated parent stream), or //! `TopicNotFound` (live stream, fabricated topic). `InvalidPartitionsCount` -//! not targeted. Shadow tracks no partition counts, so every outcome predicts -//! `Effect::None`. +//! not targeted (only reachable through partition-id overflow). A committed +//! `Ok` grows the shadow's per-topic partition count, which +//! `delete_partitions` samples against. -use iggy_binary_protocol::RequestHeader; +use iggy_binary_protocol::RoutedRequestHeader; use rand::RngExt; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -86,7 +87,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.create_partitions(&input.stream, &input.topic, input.partitions_count) } @@ -100,7 +101,13 @@ pub const fn classify_reply(code: u32) -> Outcome { } #[must_use] -pub const fn predicted_effect(_input: &Input, _outcome: Outcome) -> Effect { - // Shadow tracks no per-topic partition counts. - Effect::None +pub fn predicted_effect(input: &Input, outcome: Outcome) -> Effect { + match outcome { + Outcome::Ok => Effect::AddPartitions { + stream: input.stream.clone(), + topic: input.topic.clone(), + count: input.partitions_count, + }, + _ => Effect::None, + } } diff --git a/core/simulator/src/workload/ops/create_personal_access_token.rs b/core/simulator/src/workload/ops/create_personal_access_token.rs index da01cbc48c..561b515343 100644 --- a/core/simulator/src/workload/ops/create_personal_access_token.rs +++ b/core/simulator/src/workload/ops/create_personal_access_token.rs @@ -19,7 +19,7 @@ //! (a live token name). `InvalidExpiry` is not targeted; tokens never expire //! (expiry = 0). -use iggy_binary_protocol::RequestHeader; +use iggy_binary_protocol::RoutedRequestHeader; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -56,7 +56,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.create_personal_access_token(&input.name, input.expiry) } diff --git a/core/simulator/src/workload/ops/create_stream.rs b/core/simulator/src/workload/ops/create_stream.rs index c0b3047efb..cf5e68ace8 100644 --- a/core/simulator/src/workload/ops/create_stream.rs +++ b/core/simulator/src/workload/ops/create_stream.rs @@ -18,7 +18,7 @@ //! `CreateStream` op. Targets `Ok` with a fresh name, or `NameAlreadyExists` //! by reusing a live stream name from the shadow. -use iggy_binary_protocol::RequestHeader; +use iggy_binary_protocol::RoutedRequestHeader; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -53,7 +53,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.create_stream(&input.name) } diff --git a/core/simulator/src/workload/ops/create_topic.rs b/core/simulator/src/workload/ops/create_topic.rs index 7d7b2f7612..07fd654a1d 100644 --- a/core/simulator/src/workload/ops/create_topic.rs +++ b/core/simulator/src/workload/ops/create_topic.rs @@ -19,7 +19,7 @@ //! `StreamNotFound` (a fabricated parent stream), or `NameAlreadyExists` (an //! existing topic name under its live stream). -use iggy_binary_protocol::RequestHeader; +use iggy_binary_protocol::RoutedRequestHeader; use rand::RngExt; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -84,7 +84,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.create_topic(&input.stream, &input.name, input.partitions_count) } diff --git a/core/simulator/src/workload/ops/create_user.rs b/core/simulator/src/workload/ops/create_user.rs index 55fb8993ef..c5bce99db5 100644 --- a/core/simulator/src/workload/ops/create_user.rs +++ b/core/simulator/src/workload/ops/create_user.rs @@ -18,7 +18,7 @@ //! `CreateUser` op. Targets `Ok` (fresh username) or `UserAlreadyExists` (a //! live username from the shadow). Status fixed at 1 (Active). -use iggy_binary_protocol::RequestHeader; +use iggy_binary_protocol::RoutedRequestHeader; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -59,7 +59,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.create_user(&input.username, &input.password, input.status) } diff --git a/core/simulator/src/workload/ops/delete_consumer_group.rs b/core/simulator/src/workload/ops/delete_consumer_group.rs index cfbe89961e..996ea1d3c1 100644 --- a/core/simulator/src/workload/ops/delete_consumer_group.rs +++ b/core/simulator/src/workload/ops/delete_consumer_group.rs @@ -22,7 +22,7 @@ //! `ConsumerGroupNotFound` (a live stream/topic with a fabricated group //! name), mirroring the legacy resolution ladder. -use iggy_binary_protocol::RequestHeader; +use iggy_binary_protocol::RoutedRequestHeader; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -86,7 +86,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.delete_consumer_group(&input.stream, &input.topic, &input.group) } diff --git a/core/simulator/src/workload/ops/delete_consumer_offset.rs b/core/simulator/src/workload/ops/delete_consumer_offset.rs index 0c71b6031e..c05c7960ac 100644 --- a/core/simulator/src/workload/ops/delete_consumer_offset.rs +++ b/core/simulator/src/workload/ops/delete_consumer_offset.rs @@ -17,7 +17,7 @@ //! `DeleteConsumerOffset` op. Live namespace via shadow. -use iggy_binary_protocol::RequestHeader; +use iggy_binary_protocol::RoutedRequestHeader; use rand::RngExt; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -63,7 +63,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.delete_consumer_offset(input.ns, input.consumer_kind, input.consumer_id) } diff --git a/core/simulator/src/workload/ops/delete_consumer_offset_2.rs b/core/simulator/src/workload/ops/delete_consumer_offset_2.rs index 5913b018d1..ad2a6201a5 100644 --- a/core/simulator/src/workload/ops/delete_consumer_offset_2.rs +++ b/core/simulator/src/workload/ops/delete_consumer_offset_2.rs @@ -17,7 +17,7 @@ //! `DeleteConsumerOffset2` op. Namespace-routed with `AckLevel`. -use iggy_binary_protocol::{AckLevel, RequestHeader}; +use iggy_binary_protocol::{AckLevel, RoutedRequestHeader}; use rand::RngExt; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -71,7 +71,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.delete_consumer_offset_2(input.ns, input.consumer_kind, input.consumer_id, input.ack) } diff --git a/core/simulator/src/workload/ops/delete_partitions.rs b/core/simulator/src/workload/ops/delete_partitions.rs index 46ee23e73f..3711f89ddf 100644 --- a/core/simulator/src/workload/ops/delete_partitions.rs +++ b/core/simulator/src/workload/ops/delete_partitions.rs @@ -17,11 +17,14 @@ //! `DeletePartitions` op. //! -//! Targets `Ok` (a live topic), `StreamNotFound` (a fabricated parent stream), -//! or `TopicNotFound` (a live stream with a fabricated topic). Partition counts -//! are not tracked in the shadow, so every outcome predicts `Effect::None`. +//! Targets `Ok` (a live topic, count within its shadow-tracked partition +//! count), `StreamNotFound` (a fabricated parent stream), `TopicNotFound` (a +//! live stream with a fabricated topic), or `InvalidPartitionsCount` (a live +//! topic, count one past its partition count - the server commits the typed +//! rejection instead of acking a silent no-op). A committed `Ok` shrinks the +//! shadow's per-topic partition count. -use iggy_binary_protocol::RequestHeader; +use iggy_binary_protocol::{MAX_PARTITIONS_PER_REQUEST, RoutedRequestHeader}; use rand::RngExt; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -40,7 +43,12 @@ pub struct Input { pub partitions_count: u32, } -pub const OUTCOMES: &[Outcome] = &[Outcome::Ok, Outcome::StreamNotFound, Outcome::TopicNotFound]; +pub const OUTCOMES: &[Outcome] = &[ + Outcome::Ok, + Outcome::StreamNotFound, + Outcome::TopicNotFound, + Outcome::InvalidPartitionsCount, +]; pub fn sample( shadow: &mut Shadow, @@ -51,7 +59,15 @@ pub fn sample( match outcome { Outcome::Ok => { let (stream, topic) = shadow.pick_topic_pair(prng)?; - let partitions_count = 1 + prng.random_range(0..4u32); + let live = *shadow + .topic_partitions + .get(&(stream.clone(), topic.clone()))?; + if live == 0 { + // Any nonzero count would over-delete; that is the + // `InvalidPartitionsCount` target, not `Ok`. + return None; + } + let partitions_count = 1 + prng.random_range(0..live.min(4)); Some(Input { stream, topic, @@ -78,11 +94,29 @@ pub fn sample( partitions_count, }) } + Outcome::InvalidPartitionsCount => { + let (stream, topic) = shadow.pick_topic_pair(prng)?; + let live = *shadow + .topic_partitions + .get(&(stream.clone(), topic.clone()))?; + // One past the live count is the smallest guaranteed over-delete. + // Past the per-request cap the pre-consensus gate would answer + // `TooManyPartitions` instead, so the target is unrealizable. + let partitions_count = live.checked_add(1)?; + if partitions_count > MAX_PARTITIONS_PER_REQUEST { + return None; + } + Some(Input { + stream, + topic, + partitions_count, + }) + } } } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.delete_partitions(&input.stream, &input.topic, input.partitions_count) } @@ -96,7 +130,13 @@ pub const fn classify_reply(code: u32) -> Outcome { } #[must_use] -pub const fn predicted_effect(_input: &Input, _outcome: Outcome) -> Effect { - // Partition counts are not tracked per topic in the shadow. - Effect::None +pub fn predicted_effect(input: &Input, outcome: Outcome) -> Effect { + match outcome { + Outcome::Ok => Effect::RemovePartitions { + stream: input.stream.clone(), + topic: input.topic.clone(), + count: input.partitions_count, + }, + _ => Effect::None, + } } diff --git a/core/simulator/src/workload/ops/delete_personal_access_token.rs b/core/simulator/src/workload/ops/delete_personal_access_token.rs index 2bd54304eb..a829f5fa73 100644 --- a/core/simulator/src/workload/ops/delete_personal_access_token.rs +++ b/core/simulator/src/workload/ops/delete_personal_access_token.rs @@ -18,7 +18,7 @@ //! `DeletePersonalAccessToken` op. Targets `Ok` (a live token) or `NotFound` //! (a fabricated name). -use iggy_binary_protocol::RequestHeader; +use iggy_binary_protocol::RoutedRequestHeader; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -51,7 +51,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.delete_personal_access_token(&input.name) } diff --git a/core/simulator/src/workload/ops/delete_segments.rs b/core/simulator/src/workload/ops/delete_segments.rs index 399016f77e..abbb173ff5 100644 --- a/core/simulator/src/workload/ops/delete_segments.rs +++ b/core/simulator/src/workload/ops/delete_segments.rs @@ -26,7 +26,7 @@ //! in `Action` and the dispatch table so the surface compiles and the v2.4 //! outcome expansion lands cleanly post-upgrade. -use iggy_binary_protocol::RequestHeader; +use iggy_binary_protocol::RoutedRequestHeader; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -61,7 +61,7 @@ pub const fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.delete_segments( &input.stream, &input.topic, diff --git a/core/simulator/src/workload/ops/delete_stream.rs b/core/simulator/src/workload/ops/delete_stream.rs index 008b9f04a0..97aa7f050c 100644 --- a/core/simulator/src/workload/ops/delete_stream.rs +++ b/core/simulator/src/workload/ops/delete_stream.rs @@ -18,7 +18,7 @@ //! `DeleteStream` op. Targets `Ok` with a live stream name, or `StreamNotFound` //! with a fabricated name that was never created. -use iggy_binary_protocol::RequestHeader; +use iggy_binary_protocol::RoutedRequestHeader; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -51,7 +51,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.delete_stream(&input.name) } diff --git a/core/simulator/src/workload/ops/delete_topic.rs b/core/simulator/src/workload/ops/delete_topic.rs index 32e640ba7e..4dfd8241fb 100644 --- a/core/simulator/src/workload/ops/delete_topic.rs +++ b/core/simulator/src/workload/ops/delete_topic.rs @@ -19,7 +19,7 @@ //! fabricated parent stream), or `TopicNotFound` (a live stream with a //! fabricated topic). -use iggy_binary_protocol::RequestHeader; +use iggy_binary_protocol::RoutedRequestHeader; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -63,7 +63,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.delete_topic(&input.stream, &input.topic) } diff --git a/core/simulator/src/workload/ops/delete_user.rs b/core/simulator/src/workload/ops/delete_user.rs index 964b8eee87..55c039eb72 100644 --- a/core/simulator/src/workload/ops/delete_user.rs +++ b/core/simulator/src/workload/ops/delete_user.rs @@ -18,7 +18,7 @@ //! `DeleteUser` op. Targets `Ok` (a live user) or `UserNotFound` (a fabricated //! username). -use iggy_binary_protocol::RequestHeader; +use iggy_binary_protocol::RoutedRequestHeader; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -53,7 +53,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.delete_user(&input.user) } diff --git a/core/simulator/src/workload/ops/mod.rs b/core/simulator/src/workload/ops/mod.rs index be965e9108..df6ffed485 100644 --- a/core/simulator/src/workload/ops/mod.rs +++ b/core/simulator/src/workload/ops/mod.rs @@ -55,7 +55,7 @@ pub mod update_stream; pub mod update_topic; pub mod update_user; -use iggy_binary_protocol::RequestHeader; +use iggy_binary_protocol::RoutedRequestHeader; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -84,7 +84,7 @@ macro_rules! op_dispatch { /// In-flight entry recorded on submit, removed on reply. /// - /// `request_namespace` is the `header.namespace` the request was + /// `request_namespace` is the `header.group` the request was /// submitted with; the auditor cross-checks it against the /// reply's namespace so a misrouted reply cannot update the /// wrong VSR group's bookkeeping. @@ -130,7 +130,7 @@ macro_rules! op_dispatch { } #[must_use] - pub fn build_message(client: &SimClient, input: &InFlightInput) -> Message { + pub fn build_message(client: &SimClient, input: &InFlightInput) -> Message { match input { $( InFlightInput::$variant(i) => $module::build_message(client, i), )* } diff --git a/core/simulator/src/workload/ops/purge_stream.rs b/core/simulator/src/workload/ops/purge_stream.rs index 50033df79d..a7f89c66d8 100644 --- a/core/simulator/src/workload/ops/purge_stream.rs +++ b/core/simulator/src/workload/ops/purge_stream.rs @@ -18,7 +18,7 @@ //! `PurgeStream` op. Targets `Ok` (live stream) or `StreamNotFound` //! (fabricated, never-created name). -use iggy_binary_protocol::RequestHeader; +use iggy_binary_protocol::RoutedRequestHeader; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -51,7 +51,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.purge_stream(&input.stream) } diff --git a/core/simulator/src/workload/ops/purge_topic.rs b/core/simulator/src/workload/ops/purge_topic.rs index 00ae96e30e..875c625442 100644 --- a/core/simulator/src/workload/ops/purge_topic.rs +++ b/core/simulator/src/workload/ops/purge_topic.rs @@ -18,7 +18,7 @@ //! `PurgeTopic` op. Targets `Ok` (live topic), `StreamNotFound` (fabricated //! parent stream), or `TopicNotFound` (live stream, fabricated topic). -use iggy_binary_protocol::RequestHeader; +use iggy_binary_protocol::RoutedRequestHeader; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -62,7 +62,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.purge_topic(&input.stream, &input.topic) } diff --git a/core/simulator/src/workload/ops/send_messages.rs b/core/simulator/src/workload/ops/send_messages.rs index 7e58a27f99..fda0c06565 100644 --- a/core/simulator/src/workload/ops/send_messages.rs +++ b/core/simulator/src/workload/ops/send_messages.rs @@ -22,7 +22,7 @@ //! 3. one `prng.random()` per payload to disambiguate body bytes use bytes::Bytes; -use iggy_binary_protocol::RequestHeader; +use iggy_binary_protocol::RoutedRequestHeader; use rand::RngExt; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -74,7 +74,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.send_messages(input.ns, &input.payloads) } diff --git a/core/simulator/src/workload/ops/store_consumer_offset.rs b/core/simulator/src/workload/ops/store_consumer_offset.rs index feda1b47cf..edefbfc722 100644 --- a/core/simulator/src/workload/ops/store_consumer_offset.rs +++ b/core/simulator/src/workload/ops/store_consumer_offset.rs @@ -18,7 +18,7 @@ //! `StoreConsumerOffset` op. Pre-`AckLevel` manual encoding. Live //! namespace via shadow, fabricated consumer kind/id. Samples Success. -use iggy_binary_protocol::RequestHeader; +use iggy_binary_protocol::RoutedRequestHeader; use rand::RngExt; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -58,7 +58,7 @@ pub fn sample( // Draw against the configured ceiling, then clamp to committed // reality so the offset is reachable. Clamping post-draw keeps // the PRNG draw order (and determinism hash baseline) intact - // while staying valid once server-ng validates offsets. + // while staying valid once the server validates offsets. let raw: u64 = prng.random_range(0..options.max_offset.max(1)); let high = shadow.sends_committed(ns).max(1); let offset = raw % high; @@ -73,7 +73,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.store_consumer_offset( input.ns, input.consumer_kind, diff --git a/core/simulator/src/workload/ops/store_consumer_offset_2.rs b/core/simulator/src/workload/ops/store_consumer_offset_2.rs index c4b2785a8b..baac692d07 100644 --- a/core/simulator/src/workload/ops/store_consumer_offset_2.rs +++ b/core/simulator/src/workload/ops/store_consumer_offset_2.rs @@ -23,7 +23,7 @@ //! 4. `offset` range draw //! 5. `ack` ratio draw -use iggy_binary_protocol::{AckLevel, RequestHeader}; +use iggy_binary_protocol::{AckLevel, RoutedRequestHeader}; use rand::RngExt; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -64,7 +64,7 @@ pub fn sample( // Draw against the configured ceiling, then clamp to committed // reality so the offset is reachable. Clamping post-draw keeps // the PRNG draw order (and determinism hash baseline) intact - // while staying valid once server-ng validates offsets. + // while staying valid once the server validates offsets. let raw: u64 = prng.random_range(0..options.max_offset.max(1)); let high = shadow.sends_committed(ns).max(1); let offset = raw % high; @@ -86,7 +86,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.store_consumer_offset_2( input.ns, input.consumer_kind, diff --git a/core/simulator/src/workload/ops/update_permissions.rs b/core/simulator/src/workload/ops/update_permissions.rs index 75169a3822..b3aad70e8e 100644 --- a/core/simulator/src/workload/ops/update_permissions.rs +++ b/core/simulator/src/workload/ops/update_permissions.rs @@ -18,7 +18,7 @@ //! `UpdatePermissions` op. Targets `Ok` (live user) or `UserNotFound` //! (fabricated user). No permissions payload, so every outcome is `Effect::None`. -use iggy_binary_protocol::RequestHeader; +use iggy_binary_protocol::RoutedRequestHeader; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -53,7 +53,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.update_permissions(&input.user) } diff --git a/core/simulator/src/workload/ops/update_stream.rs b/core/simulator/src/workload/ops/update_stream.rs index ef70236d8a..f6f2285096 100644 --- a/core/simulator/src/workload/ops/update_stream.rs +++ b/core/simulator/src/workload/ops/update_stream.rs @@ -21,7 +21,7 @@ //! (fabricated stream). `NameAlreadyExists` (rename onto a live name) not //! targeted, but the server still classifies it on a race. -use iggy_binary_protocol::RequestHeader; +use iggy_binary_protocol::RoutedRequestHeader; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -63,7 +63,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.update_stream(&input.stream, &input.new_name) } diff --git a/core/simulator/src/workload/ops/update_topic.rs b/core/simulator/src/workload/ops/update_topic.rs index a06f2a61dc..a4e36367e5 100644 --- a/core/simulator/src/workload/ops/update_topic.rs +++ b/core/simulator/src/workload/ops/update_topic.rs @@ -21,7 +21,7 @@ //! parent stream), or `TopicNotFound` (live stream, fabricated topic). //! `NameAlreadyExists` not targeted. -use iggy_binary_protocol::RequestHeader; +use iggy_binary_protocol::RoutedRequestHeader; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -77,7 +77,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.update_topic(&input.stream, &input.topic, &input.new_name) } diff --git a/core/simulator/src/workload/ops/update_user.rs b/core/simulator/src/workload/ops/update_user.rs index 045c377b11..c0c15aade4 100644 --- a/core/simulator/src/workload/ops/update_user.rs +++ b/core/simulator/src/workload/ops/update_user.rs @@ -18,7 +18,7 @@ //! `UpdateUser` op. Targets `Ok` (rename live user to fresh name) or //! `UserNotFound` (fabricated user). `UsernameAlreadyExists` not targeted. -use iggy_binary_protocol::RequestHeader; +use iggy_binary_protocol::RoutedRequestHeader; use rand_xoshiro::Xoshiro256Plus; use server_common::Message; @@ -74,7 +74,7 @@ pub fn sample( } #[must_use] -pub fn build_message(client: &SimClient, input: &Input) -> Message { +pub fn build_message(client: &SimClient, input: &Input) -> Message { client.update_user(&input.user, input.new_username.as_deref(), input.status) } diff --git a/core/simulator/src/workload/shadow.rs b/core/simulator/src/workload/shadow.rs index 3d02b693e1..69f64b2e47 100644 --- a/core/simulator/src/workload/shadow.rs +++ b/core/simulator/src/workload/shadow.rs @@ -46,6 +46,11 @@ pub struct Shadow { pub stream_names: IndexSet, /// Live topics by `(stream, topic)`. Only added if parent stream lives. pub topic_names: IndexSet<(String, String)>, + /// Partition count per live topic, keyed like `topic_names`. Lets + /// `create/delete_partitions` sample in-bounds vs over-count deliberately + /// (the server rejects an over-count delete with a committed + /// `InvalidPartitionsCount`, so the count must be known at sample time). + pub topic_partitions: HashMap<(String, String), u32>, pub user_names: IndexSet, pub pat_names: IndexSet, pub consumer_group_names: IndexSet<(String, String, String)>, @@ -80,6 +85,7 @@ impl Shadow { namespaces_live, stream_names: IndexSet::new(), topic_names: IndexSet::new(), + topic_partitions: HashMap::new(), user_names: IndexSet::new(), pat_names: IndexSet::new(), consumer_group_names: IndexSet::new(), @@ -191,31 +197,23 @@ impl Shadow { let applied = match e { Effect::None => true, Effect::AddStream { name } => self.stream_names.insert(name), - Effect::RemoveStream { name } => { - let removed = self.stream_names.shift_remove(&name); - self.topic_names.retain(|(s, _)| s != &name); - self.consumer_group_names.retain(|(s, _, _)| s != &name); - removed - } + Effect::RemoveStream { name } => self.remove_stream(&name), Effect::AddTopic { stream, name, - partitions: _, - } => { - if self.stream_names.contains(&stream) { - self.topic_names.insert((stream, name)) - } else { - false - } - } - Effect::RemoveTopic { stream, name } => { - let removed = self - .topic_names - .shift_remove(&(stream.clone(), name.clone())); - self.consumer_group_names - .retain(|(s, t, _)| !(s == &stream && t == &name)); - removed - } + partitions, + } => self.add_topic(stream, name, partitions), + Effect::RemoveTopic { stream, name } => self.remove_topic(&stream, &name), + Effect::AddPartitions { + stream, + topic, + count, + } => self.add_partitions(stream, topic, count), + Effect::RemovePartitions { + stream, + topic, + count, + } => self.remove_partitions(stream, topic, count), Effect::AddUser { name } => { // Matches `create_user::sample`'s pw-{name} baseline. let password = format!("pw-{name}"); @@ -286,6 +284,60 @@ impl Shadow { } } + fn remove_stream(&mut self, name: &str) -> bool { + let removed = self.stream_names.shift_remove(name); + self.topic_names.retain(|(s, _)| s != name); + self.topic_partitions.retain(|(s, _), _| s != name); + self.consumer_group_names.retain(|(s, _, _)| s != name); + removed + } + + fn add_topic(&mut self, stream: String, name: String, partitions: u32) -> bool { + if self.stream_names.contains(&stream) { + self.topic_partitions + .insert((stream.clone(), name.clone()), partitions); + self.topic_names.insert((stream, name)) + } else { + false + } + } + + fn remove_topic(&mut self, stream: &str, name: &str) -> bool { + let removed = self + .topic_names + .shift_remove(&(stream.to_string(), name.to_string())); + self.topic_partitions + .remove(&(stream.to_string(), name.to_string())); + self.consumer_group_names + .retain(|(s, t, _)| !(s == stream && t == name)); + removed + } + + fn add_partitions(&mut self, stream: String, topic: String, count: u32) -> bool { + self.topic_partitions + .get_mut(&(stream, topic)) + .is_some_and(|partitions| { + *partitions = partitions.saturating_add(count); + true + }) + } + + /// The committed delete was in bounds on the server; a shadow count below + /// it means a concurrent commit defeated the sample-time precondition, + /// same as the other `applied = false` paths. + fn remove_partitions(&mut self, stream: String, topic: String, count: u32) -> bool { + self.topic_partitions + .get_mut(&(stream, topic)) + .is_some_and(|partitions| { + if *partitions >= count { + *partitions -= count; + true + } else { + false + } + }) + } + fn rename_stream(&mut self, old: &str, new: &str) -> bool { if !self.stream_names.shift_remove(old) { return false; @@ -304,6 +356,16 @@ impl Shadow { } }) .collect(); + self.topic_partitions = std::mem::take(&mut self.topic_partitions) + .into_iter() + .map(|((s, t), partitions)| { + if s == old { + ((new_owned.clone(), t), partitions) + } else { + ((s, t), partitions) + } + }) + .collect(); self.consumer_group_names = std::mem::take(&mut self.consumer_group_names) .into_iter() .map(|(s, t, g)| { @@ -326,6 +388,13 @@ impl Shadow { } self.topic_names .insert((stream.to_string(), new.to_string())); + if let Some(partitions) = self + .topic_partitions + .remove(&(stream.to_string(), old.to_string())) + { + self.topic_partitions + .insert((stream.to_string(), new.to_string()), partitions); + } let new_owned = new.to_string(); self.consumer_group_names = std::mem::take(&mut self.consumer_group_names) .into_iter() diff --git a/examples/go/README.md b/examples/go/README.md index 0a57967e7c..d463dee8bc 100644 --- a/examples/go/README.md +++ b/examples/go/README.md @@ -9,16 +9,14 @@ To run any example, first start a VSR server and then run the desired example. For server configuration options and help: ```bash -# TODO: change to iggy-server once legacy server is removed (core/server has VSR support) -cargo run --bin iggy-server-ng --features vsr -- --help +cargo run --bin iggy-server -- --help ``` You can also customize the server using environment variables: ```bash ## Example: Enable HTTP transport and set custom address -# TODO: change to iggy-server once legacy server is removed (core/server has VSR support) -IGGY_HTTP_ENABLED=true IGGY_TCP_ADDRESS=0.0.0.0:8090 cargo run --bin iggy-server-ng --features vsr +IGGY_HTTP_ENABLED=true IGGY_TCP_ADDRESS=0.0.0.0:8090 cargo run --bin iggy-server ``` You can run multiple producers and consumers simultaneously to observe how messages are distributed across clients. @@ -51,8 +49,7 @@ All examples can be executed directly from the repository. Follow these steps: 1. **Start the Iggy server**: the Go SDK speaks the VSR wire protocol, so the examples need a VSR server. - - `cargo run --bin iggy-server-ng --features vsr` + `cargo run --bin iggy-server` 2. **Run desired example**: `go run ./xxx/xxx/main.go` 3. **Check source code**: Examples include detailed comments explaining concepts and usage patterns diff --git a/examples/java/README.md b/examples/java/README.md index aca1b8be65..4507d90095 100644 --- a/examples/java/README.md +++ b/examples/java/README.md @@ -6,29 +6,24 @@ Java 17 and Gradle 9.2.1 are recommended for running the examples. ## Running Examples -Iggy requires valid credentials to authenticate client requests. The examples assume that the server is using the default root credentials, which can be enabled in one of two ways: +The Java SDK speaks the VSR (Viewstamped Replication) wire protocol, so the examples run against the VSR server. -1. Start the server with default credentials: +Iggy requires valid credentials to authenticate client requests. The examples assume that the server is using the default root credentials, set through environment variables before starting the server: - ```bash - cargo run --bin iggy-server -- --with-default-root-credentials - ``` +macOS/Linux: -2. Set the appropriate environment variables before starting the server with `cargo run --bin iggy-server`: - - macOS/Linux: - - ```bash - export IGGY_ROOT_USERNAME=iggy - export IGGY_ROOT_PASSWORD=iggy - ``` +```bash +export IGGY_ROOT_USERNAME=iggy +export IGGY_ROOT_PASSWORD=iggy +cargo run --bin iggy-server +``` - Windows(Powershell): +Windows(Powershell): - ```bash - $env:IGGY_ROOT_USERNAME = "iggy" - $env:IGGY_ROOT_PASSWORD = "iggy" - ``` +```bash +$env:IGGY_ROOT_USERNAME = "iggy" +$env:IGGY_ROOT_PASSWORD = "iggy" +``` > **Note**
> This setup is intended only for development and testing, not production use. @@ -36,32 +31,15 @@ Iggy requires valid credentials to authenticate client requests. The examples as By default, all server data is stored in the `local_data` directory (this can be changed via `system.path` in `config.toml`). Root credentials are applied **only on the very first startup**, when no data directory exists yet. -Once the server has created and populated the data directory, the existing stored credentials will always be used, and supplying the `--with-default-root-credentials` flag or setting the environment variables will no longer override them. - -If the server has already been started once and your example returns `Error: InvalidCredentials`, then this means the stored credentials differ from the defaults. +Once the server has created and populated the data directory, the existing stored credentials will always be used, and setting the environment variables will no longer override them. -You can reset the credentials in one of two ways: - -1. Delete the existing data directory, then start the server again with the default-credential flag or environment variables. -2. Use the `--fresh` flag to force a reset: - - ```bash - cargo run --bin iggy-server -- --with-default-root-credentials --fresh - ``` - - This will ignore any existing data directory and re-initialize it with the default credentials. - -For server configuration options and help: - -```bash -cargo run --bin iggy-server -- --help -``` +If the server has already been started once and your example returns `Error: InvalidCredentials`, then this means the stored credentials differ from the defaults. Delete the existing data directory, then start the server again with the environment variables set. You can also customize the server using environment variables: ```bash -## Example: Enable HTTP transport and set custom address -IGGY_HTTP_ENABLED=true IGGY_TCP_ADDRESS=0.0.0.0:8090 cargo run --bin iggy-server +## Example: set a custom TCP address +IGGY_TCP_ADDRESS=0.0.0.0:8090 cargo run --bin iggy-server ``` ## Basic Examples @@ -136,7 +114,7 @@ Shows how to use the stream builder API to create and configure streams with cus ### Async Producer -High-throughput async production with pipelining: +Non-blocking batch production with concurrent request submission: ```bash ./gradlew runAsyncProducer @@ -145,7 +123,7 @@ High-throughput async production with pipelining: Shows: - CompletableFuture chaining patterns -- Pipelining multiple sends without blocking +- Submitting multiple sends without blocking - Performance comparison with blocking client ### Async Consumer @@ -210,7 +188,7 @@ The Iggy Java SDK provides two client types: **blocking (synchronous)** and **as - Need high throughput - Application is already async/reactive (Spring WebFlux, Vert.x) -- Want to pipeline multiple requests over a single connection +- Want to compose non-blocking requests with `CompletableFuture` - Building services that handle many concurrent streams ## Key Async Patterns @@ -228,16 +206,20 @@ client.connect() }); ``` -### Pipelining for Throughput +### Submitting Multiple Sends ```java -List> sends = new ArrayList<>(); +List> sends = new ArrayList<>(); for (int i = 0; i < 10; i++) { sends.add(client.messages().sendMessages(...)); } CompletableFuture.allOf(sends.toArray(new CompletableFuture[0])).join(); ``` +The client accepts these calls without blocking, but its single VSR-pinned TCP +connection processes them in order. Batch more messages into each send to improve +throughput. + ### Thread Pool Offloading ```java diff --git a/examples/node/src/tcp-tls/consumer.ts b/examples/node/src/tcp-tls/consumer.ts index 9a7ec5229a..6817193123 100644 --- a/examples/node/src/tcp-tls/consumer.ts +++ b/examples/node/src/tcp-tls/consumer.ts @@ -23,6 +23,7 @@ // // Prerequisites: // Start the Iggy server with TLS enabled: +// IGGY_ROOT_USERNAME=iggy IGGY_ROOT_PASSWORD=iggy \ // IGGY_TCP_TLS_ENABLED=true \ // IGGY_TCP_TLS_CERT_FILE=core/certs/iggy_cert.pem \ // IGGY_TCP_TLS_KEY_FILE=core/certs/iggy_key.pem \ diff --git a/examples/node/src/tcp-tls/producer.ts b/examples/node/src/tcp-tls/producer.ts index 7b51cc541e..98f7dc99c2 100644 --- a/examples/node/src/tcp-tls/producer.ts +++ b/examples/node/src/tcp-tls/producer.ts @@ -23,6 +23,7 @@ // // Prerequisites: // Start the Iggy server with TLS enabled: +// IGGY_ROOT_USERNAME=iggy IGGY_ROOT_PASSWORD=iggy \ // IGGY_TCP_TLS_ENABLED=true \ // IGGY_TCP_TLS_CERT_FILE=core/certs/iggy_cert.pem \ // IGGY_TCP_TLS_KEY_FILE=core/certs/iggy_key.pem \ @@ -33,7 +34,15 @@ import { readFileSync } from 'node:fs'; import { Client, Partitioning } from 'apache-iggy'; -import { BATCHES_LIMIT, cleanup, initSystem, log, MESSAGES_PER_BATCH, sleep } from '../utils'; +import { + BATCHES_LIMIT, + cleanup, + initSystem, + log, + MESSAGES_PER_BATCH, + PARTITION_ID, + sleep +} from '../utils'; async function produceMessages( client: Client, @@ -62,11 +71,14 @@ async function produceMessages( }); try { + // The VSR client routes each send to an explicit partition. + // TODO(hubcio): Balanced partitioning to be implemented; not decided + // yet whether it'll be on server side or client side. await client.message.send({ streamId, topicId, messages, - partition: Partitioning.Balanced, + partition: Partitioning.PartitionId(PARTITION_ID), }); } catch (error) { log('Error sending messages: %o', error); diff --git a/examples/python/getting-started/consumer.py b/examples/python/getting-started/consumer.py index bb10e91382..db0a6edb1f 100755 --- a/examples/python/getting-started/consumer.py +++ b/examples/python/getting-started/consumer.py @@ -19,8 +19,16 @@ import asyncio import typing import urllib.parse - -from apache_iggy import IggyClient, PollingStrategy, ReceiveMessage +from datetime import timedelta + +from apache_iggy import ( + AutoLogin, + IggyClient, + PollingStrategy, + ReceiveMessage, + TcpConfig, + TcpReconnectionConfig, +) from loguru import logger STREAM_NAME = "sample-stream" @@ -91,34 +99,34 @@ def parse_args() -> ArgNamespace: return ArgNamespace(**vars(args)) -def build_connection_string(args) -> str: - """Build a connection string with TLS support.""" - - conn_str = f"iggy://{args.username}:{args.password}@{args.tcp_server_address}" - - if args.tls: - # Extract domain from server address (host:port -> host) - host = args.tcp_server_address.split(":")[0] - query_params = ["tls=true", f"tls_domain={host}"] - - # Add CA file if provided - if args.tls_ca_file: - query_params.append(f"tls_ca_file={args.tls_ca_file}") - conn_str += "?" + "&".join(query_params) +def build_config(args: ArgNamespace) -> TcpConfig: + """Build a TCP client configuration with auto-login and reconnection.""" - return conn_str + return TcpConfig( + server_address=args.tcp_server_address, + auto_login=AutoLogin.username_password(args.username, args.password), + reconnection=TcpReconnectionConfig( + enabled=True, + interval=timedelta(seconds=1), + ), + tls_enabled=args.tls, + tls_ca_file=args.tls_ca_file or None, + ) async def main(): args: ArgNamespace = parse_args() + try: + config = build_config(args) + except ValueError as error: + logger.error(f"Invalid client configuration: {error}") + return + logger.info(f"Connecting to {args.tcp_server_address} (TLS: {args.tls})") - # Build connection string with TLS support - connection_string = build_connection_string(args) - logger.info(f"Connection string: {connection_string}") - - client = IggyClient.from_connection_string(connection_string) + client = IggyClient(config) try: logger.info("Connecting to IggyClient...") + # No login_user() call: auto_login replays the credentials on every connect. await client.connect() logger.info("Connected.") await consume_messages(client) diff --git a/examples/python/getting-started/producer.py b/examples/python/getting-started/producer.py index 642399edb0..23f964fa81 100755 --- a/examples/python/getting-started/producer.py +++ b/examples/python/getting-started/producer.py @@ -19,8 +19,16 @@ import asyncio import typing import urllib.parse - -from apache_iggy import IggyClient, StreamDetails, TopicDetails +from datetime import timedelta + +from apache_iggy import ( + AutoLogin, + IggyClient, + StreamDetails, + TcpConfig, + TcpReconnectionConfig, + TopicDetails, +) from apache_iggy import SendMessage as Message from loguru import logger @@ -92,33 +100,33 @@ def parse_args() -> ArgNamespace: return ArgNamespace(**vars(args)) -def build_connection_string(args) -> str: - """Build a connection string with TLS support.""" - - conn_str = f"iggy://{args.username}:{args.password}@{args.tcp_server_address}" - - if args.tls: - # Extract domain from server address (host:port -> host) - host = args.tcp_server_address.split(":")[0] - query_params = ["tls=true", f"tls_domain={host}"] +def build_config(args: ArgNamespace) -> TcpConfig: + """Build a TCP client configuration with auto-login and reconnection.""" - # Add CA file if provided - if args.tls_ca_file: - query_params.append(f"tls_ca_file={args.tls_ca_file}") - conn_str += "?" + "&".join(query_params) - - return conn_str + return TcpConfig( + server_address=args.tcp_server_address, + auto_login=AutoLogin.username_password(args.username, args.password), + reconnection=TcpReconnectionConfig( + enabled=True, + interval=timedelta(seconds=1), + ), + tls_enabled=args.tls, + tls_ca_file=args.tls_ca_file or None, + ) async def main(): args: ArgNamespace = parse_args() - # Build connection string with TLS support - connection_string = build_connection_string(args) - logger.info(f"Connection string: {connection_string}") + try: + config = build_config(args) + except ValueError as error: + logger.error(f"Invalid client configuration: {error}") + return logger.info(f"Connecting to {args.tcp_server_address} (TLS: {args.tls})") - client = IggyClient.from_connection_string(connection_string) + client = IggyClient(config) logger.info("Connecting to IggyClient") + # No login_user() call: auto_login replays the credentials on every connect. await client.connect() logger.info("Connected.") await init_system(client) diff --git a/foreign/cpp/tests/e2e/client.cpp b/foreign/cpp/tests/e2e/client.cpp index 64405fe92b..53c1a96a51 100644 --- a/foreign/cpp/tests/e2e/client.cpp +++ b/foreign/cpp/tests/e2e/client.cpp @@ -448,40 +448,32 @@ TEST_F(LowLevelE2E_Client, GetClientsReflectsSessionRemovalAfterDisconnect) { } TEST_F(LowLevelE2E_Client, GetClientsReflectsLoggedOutSessionAsUnauthenticated) { - RecordProperty("description", - "Keeps a logged out session visible in get_clients and get_client, but marks it unauthenticated."); + RecordProperty("description", "Drops a logged out session from get_clients and reports it missing in get_client."); iggy::ffi::Client *first_client = GetLoggedInClient(); iggy::ffi::Client *second_client = GetLoggedInClient(); iggy::ffi::ClientInfoDetails first_me{}; - iggy::ffi::ClientInfoDetails logged_out_client{}; - rust::Vec clients_after_logout; ASSERT_NO_THROW({ first_me = first_client->get_me(); }); + // The VSR server drops the client-table entry on logout (an unauthenticated + // session is not tracked), unlike the legacy server which kept it visible + // without a user id. ASSERT_NO_THROW(first_client->logout_user()); - ASSERT_NO_THROW({ - clients_after_logout = second_client->get_clients(); - logged_out_client = second_client->get_client(first_me.client_id); - }); - - bool found_first = false; - for (const auto &client : clients_after_logout) { - if (client.client_id != first_me.client_id) { - continue; + constexpr auto removal_timeout = std::chrono::seconds(5); + constexpr auto removal_poll_interval = std::chrono::milliseconds(10); + const auto deadline = std::chrono::steady_clock::now() + removal_timeout; + bool removed = false; + do { + const auto clients = second_client->get_clients(); + removed = std::none_of(clients.begin(), clients.end(), + [&first_me](const auto &client) { return client.client_id == first_me.client_id; }); + if (removed) { + break; } - - found_first = true; - EXPECT_FALSE(client.has_user_id); - EXPECT_EQ(static_cast(client.address), static_cast(first_me.address)); - EXPECT_EQ(static_cast(client.transport), static_cast(first_me.transport)); - break; - } - - EXPECT_TRUE(found_first); - EXPECT_EQ(logged_out_client.client_id, first_me.client_id); - EXPECT_FALSE(logged_out_client.has_user_id); - EXPECT_EQ(static_cast(logged_out_client.address), static_cast(first_me.address)); - EXPECT_EQ(static_cast(logged_out_client.transport), static_cast(first_me.transport)); + std::this_thread::sleep_for(removal_poll_interval); + } while (std::chrono::steady_clock::now() < deadline); + ASSERT_TRUE(removed); + ASSERT_THROW(second_client->get_client(first_me.client_id), std::exception); } TEST_F(LowLevelE2E_Client, LoginWithoutConnect) { @@ -567,9 +559,12 @@ TEST_F(LowLevelE2E_Client, GetStatsBeforeLoginThrows) { ASSERT_THROW(client->get_stats(), std::exception); } -TEST_F(LowLevelE2E_Client, FlushUnsavedBufferSucceedsForExistingPartition) { +// The VSR server has no unsaved-buffer primitive (writes are journaled at +// commit); FLUSH_UNSAVED_BUFFER denies typed with FeatureUnavailable even for +// resolvable targets. +TEST_F(LowLevelE2E_Client, FlushUnsavedBufferThrowsForExistingPartition) { RecordProperty("description", - "Creates a stream and topic, sends one message, and flushes the partition buffer successfully."); + "Rejects flush_unsaved_buffer with the feature-unavailable error for an existing partition."); const std::string stream_name = GetRandomName(); const std::string topic_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); @@ -585,13 +580,14 @@ TEST_F(LowLevelE2E_Client, FlushUnsavedBufferSucceedsForExistingPartition) { ASSERT_NO_THROW(client->send_messages(make_numeric_identifier(stream.id), make_numeric_identifier(0), "partition_id", partition_id_bytes(0), std::move(messages))); - ASSERT_NO_THROW( - client->flush_unsaved_buffer(make_numeric_identifier(stream.id), make_numeric_identifier(0), 0, true)); + ASSERT_THROW(client->flush_unsaved_buffer(make_numeric_identifier(stream.id), make_numeric_identifier(0), 0, true), + std::exception); } -TEST_F(LowLevelE2E_Client, FlushUnsavedBufferSucceedsForExistingEmptyPartition) { - RecordProperty("description", - "Succeeds when flush_unsaved_buffer is called for an existing partition with no unsaved messages."); +TEST_F(LowLevelE2E_Client, FlushUnsavedBufferThrowsForExistingEmptyPartition) { + RecordProperty( + "description", + "Rejects flush_unsaved_buffer with the feature-unavailable error for a partition with no unsaved messages."); const std::string stream_name = GetRandomName(); const std::string topic_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); @@ -602,8 +598,8 @@ TEST_F(LowLevelE2E_Client, FlushUnsavedBufferSucceedsForExistingEmptyPartition) ASSERT_NO_THROW(client->create_topic(make_numeric_identifier(stream.id), topic_name, 1, "none", 0, "never_expire", 0, "server_default")); - ASSERT_NO_THROW( - client->flush_unsaved_buffer(make_numeric_identifier(stream.id), make_numeric_identifier(0), 0, true)); + ASSERT_THROW(client->flush_unsaved_buffer(make_numeric_identifier(stream.id), make_numeric_identifier(0), 0, true), + std::exception); } TEST_F(LowLevelE2E_Client, FlushUnsavedBufferBeforeLoginThrows) { @@ -694,8 +690,9 @@ TEST_F(LowLevelE2E_Client, FlushUnsavedBufferAfterTopicDeletedThrows) { std::exception); } -TEST_F(LowLevelE2E_Client, FlushUnsavedBufferTwiceSucceeds) { - RecordProperty("description", "Allows flush_unsaved_buffer to be called twice in a row for the same partition."); +TEST_F(LowLevelE2E_Client, FlushUnsavedBufferTwiceThrows) { + RecordProperty("description", + "Rejects flush_unsaved_buffer with the feature-unavailable error consistently across repeat calls."); const std::string stream_name = GetRandomName(); const std::string topic_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); @@ -711,10 +708,10 @@ TEST_F(LowLevelE2E_Client, FlushUnsavedBufferTwiceSucceeds) { ASSERT_NO_THROW(client->send_messages(make_numeric_identifier(stream.id), make_numeric_identifier(0), "partition_id", partition_id_bytes(0), std::move(messages))); - ASSERT_NO_THROW( - client->flush_unsaved_buffer(make_numeric_identifier(stream.id), make_numeric_identifier(0), 0, true)); - ASSERT_NO_THROW( - client->flush_unsaved_buffer(make_numeric_identifier(stream.id), make_numeric_identifier(0), 0, true)); + ASSERT_THROW(client->flush_unsaved_buffer(make_numeric_identifier(stream.id), make_numeric_identifier(0), 0, true), + std::exception); + ASSERT_THROW(client->flush_unsaved_buffer(make_numeric_identifier(stream.id), make_numeric_identifier(0), 0, true), + std::exception); } TEST_F(LowLevelE2E_Client, FlushUnsavedBufferWithInvalidPartitionIdsThrows) { @@ -1566,15 +1563,19 @@ TEST_F(LowLevelE2E_Client, GetClientsReflectsAdditionalSession) { EXPECT_TRUE(found_after); } -TEST_F(LowLevelE2E_Client, GetClusterMetadataBeforeLoginThrows) { +TEST_F(LowLevelE2E_Client, GetClusterMetadataBeforeLoginSucceeds) { RecordProperty( "description", - "Rejects get_cluster_metadata before connect, after connect but before login, and after disconnect."); + "Serves get_cluster_metadata to a connected but unauthenticated client, and rejects it without a connection."); iggy::ffi::Client *client = GetLoggedOutClient(); ASSERT_THROW(client->get_cluster_metadata(), std::exception); ASSERT_NO_THROW(client->connect()); - ASSERT_THROW(client->get_cluster_metadata(), std::exception); + // By design pre-login on the VSR server: an SDK must read the roster to + // find the primary before it can authenticate (redirect bootstrap). + iggy::ffi::ClusterMetadata metadata{}; + ASSERT_NO_THROW({ metadata = client->get_cluster_metadata(); }); + ASSERT_EQ(metadata.nodes.size(), 1u); ASSERT_NO_THROW(client->login_user("iggy", "iggy")); ASSERT_NO_THROW(client->disconnect()); ASSERT_THROW(client->get_cluster_metadata(), std::exception); @@ -1631,6 +1632,8 @@ TEST_F(LowLevelE2E_Client, PingSucceedsForNewConnection) { RecordProperty("description", "Successfully pings the server from a fresh unauthenticated client session."); iggy::ffi::Client *client = GetLoggedOutClient(); + // The VSR client has no lazy connect; ping still needs no authentication. + ASSERT_NO_THROW(client->connect()); ASSERT_NO_THROW(client->ping()); } diff --git a/foreign/cpp/tests/e2e/consumer_group.cpp b/foreign/cpp/tests/e2e/consumer_group.cpp index 3ca41d3e17..7863b66300 100644 --- a/foreign/cpp/tests/e2e/consumer_group.cpp +++ b/foreign/cpp/tests/e2e/consumer_group.cpp @@ -722,19 +722,21 @@ TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupsReflectsJoinedGroupMembersCou EXPECT_NE(groups[0].members_count, groups[1].members_count); } -TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupsOnNonExistentStreamReturnsEmpty) { - RecordProperty("description", "Returns an empty list when the stream does not exist."); +// The VSR server rejects consumer-group reads whose parent stream or topic is +// absent with the legacy typed not-found; the legacy server answered them with +// an empty list. +TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupsOnNonExistentStreamThrows) { + RecordProperty("description", "Throws when the stream does not exist."); const std::string stream_name = GetRandomName(); const std::string topic_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); - const auto groups = - client->get_consumer_groups(make_string_identifier(stream_name), make_string_identifier(topic_name)); - EXPECT_TRUE(groups.empty()); + ASSERT_THROW(client->get_consumer_groups(make_string_identifier(stream_name), make_string_identifier(topic_name)), + std::exception); } -TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupsOnNonExistentTopicReturnsEmpty) { - RecordProperty("description", "Returns an empty list when the topic does not exist."); +TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupsOnNonExistentTopicThrows) { + RecordProperty("description", "Throws when the topic does not exist."); const std::string stream_name = GetRandomName(); const std::string topic_name = GetRandomName(); iggy::ffi::Client *client = GetLoggedInClient(); @@ -742,9 +744,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupsOnNonExistentTopicReturnsEmpt ASSERT_NO_THROW(client->create_stream(stream_name)); TrackStream(stream_name); - const auto groups = - client->get_consumer_groups(make_string_identifier(stream_name), make_string_identifier(topic_name)); - EXPECT_TRUE(groups.empty()); + ASSERT_THROW(client->get_consumer_groups(make_string_identifier(stream_name), make_string_identifier(topic_name)), + std::exception); } TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupsIsStableAcrossBackToBackCalls) { @@ -836,8 +837,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupsReturnsCorrectNumberOfGroups) EXPECT_TRUE(groups_after_delete.empty()); } -TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupsAfterStreamDeletionReturnsEmpty) { - RecordProperty("description", "Returns an empty list after deleting the stream that owned the groups."); +TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupsAfterStreamDeletionThrows) { + RecordProperty("description", "Throws after deleting the stream that owned the groups."); const std::string stream_name = GetRandomName(); const std::string topic_name = GetRandomName(); const std::string first_group_name = GetRandomName(); @@ -859,13 +860,12 @@ TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupsAfterStreamDeletionReturnsEmp ForgetTrackedConsumerGroup(stream_name, topic_name, second_group_name); ForgetTrackedStream(stream_name); - const auto groups = - client->get_consumer_groups(make_string_identifier(stream_name), make_string_identifier(topic_name)); - EXPECT_TRUE(groups.empty()); + ASSERT_THROW(client->get_consumer_groups(make_string_identifier(stream_name), make_string_identifier(topic_name)), + std::exception); } -TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupsAfterTopicDeletionReturnsEmpty) { - RecordProperty("description", "Returns an empty list after deleting the topic that owned the groups."); +TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupsAfterTopicDeletionThrows) { + RecordProperty("description", "Throws after deleting the topic that owned the groups."); const std::string stream_name = GetRandomName(); const std::string topic_name = GetRandomName(); const std::string first_group_name = GetRandomName(); @@ -887,9 +887,8 @@ TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupsAfterTopicDeletionReturnsEmpt ForgetTrackedConsumerGroup(stream_name, topic_name, first_group_name); ForgetTrackedConsumerGroup(stream_name, topic_name, second_group_name); - const auto groups = - client->get_consumer_groups(make_string_identifier(stream_name), make_string_identifier(topic_name)); - EXPECT_TRUE(groups.empty()); + ASSERT_THROW(client->get_consumer_groups(make_string_identifier(stream_name), make_string_identifier(topic_name)), + std::exception); } TEST_F(LowLevelE2E_ConsumerGroup, GetConsumerGroupBeforeLoginThrows) { @@ -1134,7 +1133,10 @@ TEST_F(LowLevelE2E_ConsumerGroup, DeleteConsumerGroupAndRecreateWithSameNameSucc const auto recreated_group = client->create_consumer_group(make_string_identifier(stream_name), make_string_identifier(topic_name), group_name); TrackConsumerGroup(stream_name, topic_name, group_name); - ASSERT_EQ(recreated_group.id, 0u); + // The VSR server mints group ids monotonically; a recreate gets a + // fresh id (the deleted group held 0), unlike the legacy server which + // reused the freed slot. + ASSERT_GT(recreated_group.id, 0u); ASSERT_EQ(recreated_group.name, group_name); ASSERT_EQ(recreated_group.members_count, 0u); ASSERT_TRUE(recreated_group.members.empty()); diff --git a/foreign/cpp/tests/e2e/message.cpp b/foreign/cpp/tests/e2e/message.cpp index e20ef6008f..f805127a34 100644 --- a/foreign/cpp/tests/e2e/message.cpp +++ b/foreign/cpp/tests/e2e/message.cpp @@ -51,9 +51,10 @@ TEST_F(LowLevelE2E_Message, SendAndPollMessagesRoundTrip) { ASSERT_NO_THROW(sent = client->send_messages(make_numeric_identifier(stream.id), make_numeric_identifier(0), "partition_id", partition_id_bytes(0), std::move(messages))); - ASSERT_TRUE(sent.confirmations.empty()) - << "The legacy server reports no offsets, so the confirmation list must stay empty, got " - << sent.confirmations.size(); + ASSERT_EQ(sent.confirmations.size(), 1u) + << "The VSR server reports the written partition's offsets, so a single-partition send " + << "must carry exactly one confirmation"; + EXPECT_EQ(sent.confirmations.front().partition_id, 0u); auto polled = client->poll_messages(make_numeric_identifier(stream.id), make_numeric_identifier(0), 0, "consumer", make_numeric_identifier(1), "offset", 0, 100, false); diff --git a/foreign/csharp/Directory.Packages.props b/foreign/csharp/Directory.Packages.props index 14e554e308..c0f382e4fa 100644 --- a/foreign/csharp/Directory.Packages.props +++ b/foreign/csharp/Directory.Packages.props @@ -30,13 +30,14 @@ - - + + + - + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/ClusterRedirectionTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/ClusterRedirectionTests.cs index 1eafa8ea10..006946f2aa 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/ClusterRedirectionTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/ClusterRedirectionTests.cs @@ -26,15 +26,15 @@ namespace Apache.Iggy.Tests.Integrations; public class ClusterRedirectionTests { - [ClassDataSource(Shared = SharedType.PerAssembly)] - public required IggyClusterFixture Fixture { get; init; } + [ClassDataSource(Shared = SharedType.PerAssembly)] + public required RedirectionClusterFixture Fixture { get; init; } [Test] - public async Task ConnectToFollower_Should_ReturnClusterMetadataWithTwoNodes() + public async Task ConnectToFollower_Should_ReturnClusterMetadataWithAllNodes() { using var client = IggyClientFactory.CreateClient(new IggyClientConfigurator { - BaseAddress = Fixture.GetFollowerAddress(), + BaseAddress = await Fixture.GetFollowerTcpAddressAsync(), Protocol = Protocol.Tcp, ReconnectionSettings = new ReconnectionSettings { Enabled = false }, AutoLoginSettings = new AutoLoginSettings { Enabled = false } @@ -45,8 +45,8 @@ public async Task ConnectToFollower_Should_ReturnClusterMetadataWithTwoNodes() var metadata = await client.GetClusterMetadataAsync(); metadata.ShouldNotBeNull(); - metadata.Name.ShouldBe("test-cluster"); - metadata.Nodes.Length.ShouldBe(2); + metadata.Name.ShouldBe("test-vsr-cluster"); + metadata.Nodes.Length.ShouldBe(3); metadata.Nodes.ShouldContain(n => n.Role == ClusterNodeRole.Leader); metadata.Nodes.ShouldContain(n => n.Role == ClusterNodeRole.Follower); } @@ -56,7 +56,7 @@ public async Task ConnectToFollowerWithAutoLogin_Should_RedirectToLeader() { using var client = IggyClientFactory.CreateClient(new IggyClientConfigurator { - BaseAddress = Fixture.GetFollowerAddress(), + BaseAddress = await Fixture.GetFollowerTcpAddressAsync(), Protocol = Protocol.Tcp, ReconnectionSettings = new ReconnectionSettings { Enabled = true }, AutoLoginSettings = new AutoLoginSettings @@ -70,7 +70,7 @@ public async Task ConnectToFollowerWithAutoLogin_Should_RedirectToLeader() var address = client.GetCurrentAddress(); address.ShouldNotBeNullOrEmpty(); - address.ShouldBe(Fixture.GetLeaderAddress()); + address.ShouldBe(await Fixture.GetIggyAddressAsync(Protocol.Tcp)); } [Test] @@ -78,7 +78,7 @@ public async Task ConnectToFollowerWithManualLogin_Should_RedirectToLeader() { using var client = IggyClientFactory.CreateClient(new IggyClientConfigurator { - BaseAddress = Fixture.GetFollowerAddress(), + BaseAddress = await Fixture.GetFollowerTcpAddressAsync(), Protocol = Protocol.Tcp, ReconnectionSettings = new ReconnectionSettings { Enabled = true }, AutoLoginSettings = new AutoLoginSettings { Enabled = false } @@ -88,16 +88,15 @@ public async Task ConnectToFollowerWithManualLogin_Should_RedirectToLeader() var address = client.GetCurrentAddress(); address.ShouldNotBeNullOrEmpty(); - address.ShouldBe(Fixture.GetLeaderAddress()); + address.ShouldBe(await Fixture.GetIggyAddressAsync(Protocol.Tcp)); } [Test] - [Skip("Currently personal access token exist only on leader. Unskip when it will be available on follower.")] public async Task ConnectToFollowerWithPersonalAccessToken_Should_RedirectToLeader() { using var leaderClient = IggyClientFactory.CreateClient(new IggyClientConfigurator { - BaseAddress = Fixture.GetLeaderAddress(), + BaseAddress = await Fixture.GetIggyAddressAsync(Protocol.Tcp), Protocol = Protocol.Tcp, ReconnectionSettings = new ReconnectionSettings { Enabled = false }, AutoLoginSettings = new AutoLoginSettings { Enabled = false } @@ -111,7 +110,7 @@ public async Task ConnectToFollowerWithPersonalAccessToken_Should_RedirectToLead using var client = IggyClientFactory.CreateClient(new IggyClientConfigurator { - BaseAddress = Fixture.GetFollowerAddress(), + BaseAddress = await Fixture.GetFollowerTcpAddressAsync(), Protocol = Protocol.Tcp, ReconnectionSettings = new ReconnectionSettings { Enabled = true }, AutoLoginSettings = new AutoLoginSettings { Enabled = false } @@ -124,6 +123,6 @@ public async Task ConnectToFollowerWithPersonalAccessToken_Should_RedirectToLead var address = client.GetCurrentAddress(); address.ShouldNotBeNullOrEmpty(); - address.ShouldBe(Fixture.GetLeaderAddress()); + address.ShouldBe(await Fixture.GetIggyAddressAsync(Protocol.Tcp)); } } diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/ConsumerGroupTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/ConsumerGroupTests.cs index ee13df7d74..c6ed9ed56a 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/ConsumerGroupTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/ConsumerGroupTests.cs @@ -81,8 +81,8 @@ public async Task GetConsumerGroupById_Should_Return_ValidResponse(Protocol prot var cg = await client.CreateConsumerGroupAsync(Identifier.String(streamName), Identifier.String(TopicName), GroupName); - var response = await client.GetConsumerGroupByIdAsync( - Identifier.String(streamName), Identifier.String(TopicName), + var response = await client.GetConsumerGroupByIdAsync(Identifier.String(streamName), + Identifier.String(TopicName), Identifier.Numeric(cg!.Id)); response.ShouldNotBeNull(); @@ -194,15 +194,14 @@ public async Task GetConsumerGroupById_WithMembers_Should_Return_ValidResponse(P var clients = new List(); for (var i = 0; i < 2; i++) { - var memberClient = await Fixture.CreateClient(Protocol.Tcp); + var memberClient = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); clients.Add(memberClient); - await memberClient.LoginUserAsync("iggy", "iggy"); await memberClient.JoinConsumerGroupAsync(Identifier.String(streamName), Identifier.String(TopicName), Identifier.Numeric(cg!.Id)); } - var response = await client.GetConsumerGroupByIdAsync( - Identifier.String(streamName), Identifier.String(TopicName), + var response = await client.GetConsumerGroupByIdAsync(Identifier.String(streamName), + Identifier.String(TopicName), Identifier.Numeric(cg!.Id)); response.ShouldNotBeNull(); diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/FetchMessagesTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/FetchMessagesTests.cs index d75d95cee1..f9ae474a23 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/FetchMessagesTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/FetchMessagesTests.cs @@ -23,6 +23,7 @@ using Apache.Iggy.IggyClient; using Apache.Iggy.Kinds; using Apache.Iggy.Messages; +using Apache.Iggy.Tests.Integrations.Attributes; using Apache.Iggy.Tests.Integrations.Fixtures; using Shouldly; using Partitioning = Apache.Iggy.Kinds.Partitioning; @@ -90,23 +91,23 @@ public async Task PollMessages_WithNoHeaders_Should_PollMessages_Successfully(Pr [Test] [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] - public async Task PollMessages_InvalidTopic_Should_Throw_InvalidResponse(Protocol protocol) + public async Task PollMessages_InvalidTopic_Should_Throw_NotFound(Protocol protocol) { var (client, streamName) = await CreateStreamWithMessages(protocol); - var invalidFetchRequest = new MessageFetchRequest - { - Count = 10, - AutoCommit = true, - Consumer = Consumer.New(1), - PartitionId = 0, - PollingStrategy = PollingStrategy.Next(), - StreamId = Identifier.String(streamName), - TopicId = Identifier.Numeric(2137) - }; - + // A missing topic is an addressing error, not an empty partition: + // HTTP answers 404, TCP the typed topic-not-found rejection. await Should.ThrowAsync(() => - client.PollMessagesAsync(invalidFetchRequest)); + client.PollMessagesAsync(new MessageFetchRequest + { + Count = 10, + AutoCommit = true, + Consumer = Consumer.New(1), + PartitionId = 0, + PollingStrategy = PollingStrategy.Next(), + StreamId = Identifier.String(streamName), + TopicId = Identifier.Numeric(2137) + })); } [Test] diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyClusterFixture.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyClusterFixture.cs deleted file mode 100644 index 73c33edeac..0000000000 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyClusterFixture.cs +++ /dev/null @@ -1,250 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -using System.Net; -using System.Net.Sockets; -using DotNet.Testcontainers.Builders; -using DotNet.Testcontainers.Containers; -using DotNet.Testcontainers.Networks; -using TUnit.Core.Interfaces; - -namespace Apache.Iggy.Tests.Integrations.Fixtures; - -public class IggyClusterFixture : IAsyncInitializer, IAsyncDisposable -{ - private const string LeaderAlias = "iggy-leader"; - private const string FollowerAlias = "iggy-follower"; - - // Split the host-port pool by .NET major version so parallel `dotnet test` - // processes (net8.0 + net10.0) can never pick the same port and race docker's - // allocator. The ranges sit below Linux's default ephemeral range (32768+), so - // the kernel's auto-allocation won't steal from us either. - // net8.0 - 30800..30899 - // net10.0 - 31000..31099 - // Future TFMs slot in without overlap (e.g. net12.0 - 31200..31299). - private const ushort PortRangeSize = 100; - private static readonly ushort BasePort = (ushort)(30000 + Environment.Version.Major * 100); - private static readonly ushort EndPort = (ushort)(BasePort + PortRangeSize); - - // Listeners only need to outlive the eight ReservePort() calls in the - // constructor so we don't pick the same port twice within one fixture. - // Partitioned ranges already guarantee sibling processes can't race us, so - // we can release them as soon as picking is done. - private readonly List _portReservations = []; - private readonly IContainer _followerContainer; - private readonly ushort _followerHttpPort; - private readonly ushort _followerQuicPort; - - private readonly ushort _followerTcpPort; - private readonly ushort _followerWsPort; - private readonly IContainer _leaderContainer; - private readonly ushort _leaderHttpPort; - private readonly ushort _leaderQuicPort; - - private readonly ushort _leaderTcpPort; - private readonly ushort _leaderWsPort; - - private readonly INetwork _network; - - private string DockerImage => - Environment.GetEnvironmentVariable("IGGY_SERVER_DOCKER_IMAGE") ?? "apache/iggy:edge"; - - private static string? LogDirectory => - Environment.GetEnvironmentVariable("IGGY_TEST_LOGS_DIR"); - - public IggyClusterFixture() - { - try - { - _leaderTcpPort = ReservePort(); - _leaderHttpPort = ReservePort(); - _leaderQuicPort = ReservePort(); - _leaderWsPort = ReservePort(); - _followerTcpPort = ReservePort(); - _followerHttpPort = ReservePort(); - _followerQuicPort = ReservePort(); - _followerWsPort = ReservePort(); - } - finally - { - ReleaseReservedPorts(); - } - - _network = new NetworkBuilder() - .WithName($"iggy-cluster-{Guid.NewGuid():N}") - .Build(); - - // Cluster.nodes roster env vars are byte-identical on both - // containers; only the bind addresses and the --replica-id CLI arg - // differ per node. - var clusterRosterEnv = new Dictionary - { - ["IGGY_CLUSTER_ENABLED"] = "true", - ["IGGY_CLUSTER_NAME"] = "test-cluster", - ["IGGY_CLUSTER_NODES_0_NAME"] = "leader-node", - ["IGGY_CLUSTER_NODES_0_IP"] = "127.0.0.1", - ["IGGY_CLUSTER_NODES_0_REPLICA_ID"] = "0", - ["IGGY_CLUSTER_NODES_0_PORTS_TCP"] = _leaderTcpPort.ToString(), - ["IGGY_CLUSTER_NODES_0_PORTS_QUIC"] = _leaderQuicPort.ToString(), - ["IGGY_CLUSTER_NODES_0_PORTS_HTTP"] = _leaderHttpPort.ToString(), - ["IGGY_CLUSTER_NODES_0_PORTS_WEBSOCKET"] = _leaderWsPort.ToString(), - ["IGGY_CLUSTER_NODES_1_NAME"] = "follower-node", - ["IGGY_CLUSTER_NODES_1_IP"] = "127.0.0.1", - ["IGGY_CLUSTER_NODES_1_REPLICA_ID"] = "1", - ["IGGY_CLUSTER_NODES_1_PORTS_TCP"] = _followerTcpPort.ToString(), - ["IGGY_CLUSTER_NODES_1_PORTS_QUIC"] = _followerQuicPort.ToString(), - ["IGGY_CLUSTER_NODES_1_PORTS_HTTP"] = _followerHttpPort.ToString(), - ["IGGY_CLUSTER_NODES_1_PORTS_WEBSOCKET"] = _followerWsPort.ToString(), - }; - - _leaderContainer = new ContainerBuilder(DockerImage) - .WithName($"iggy-leader-{Guid.NewGuid():N}") - .WithCommand("--replica-id", "0") - .WithNetwork(_network) - .WithNetworkAliases(LeaderAlias) - .WithPortBinding(_leaderTcpPort.ToString(), _leaderTcpPort.ToString()) - .WithPortBinding(_leaderHttpPort.ToString(), _leaderHttpPort.ToString()) - .WithEnvironment("RUST_LOG", "trace") - .WithEnvironment("IGGY_SYSTEM_LOGGING_LEVEL", "trace") - .WithEnvironment("IGGY_ROOT_USERNAME", "iggy") - .WithEnvironment("IGGY_ROOT_PASSWORD", "iggy") - .WithEnvironment("IGGY_SYSTEM_PATH", "local_data_leader") - .WithEnvironment("IGGY_TCP_ADDRESS", $"0.0.0.0:{_leaderTcpPort}") - .WithEnvironment("IGGY_HTTP_ADDRESS", $"0.0.0.0:{_leaderHttpPort}") - .WithEnvironment("IGGY_QUIC_ADDRESS", $"0.0.0.0:{_leaderQuicPort}") - .WithEnvironment("IGGY_WEBSOCKET_ADDRESS", $"0.0.0.0:{_leaderWsPort}") - .WithEnvironment(clusterRosterEnv) - .WithPrivileged(true) - .WithCleanUp(true) - .WithWaitStrategy(Wait.ForUnixContainer().UntilInternalTcpPortIsAvailable(_leaderTcpPort)) - .Build(); - - _followerContainer = new ContainerBuilder(DockerImage) - .WithName($"iggy-follower-{Guid.NewGuid():N}") - .WithCommand("--follower", "--replica-id", "1") - .WithNetwork(_network) - .WithNetworkAliases(FollowerAlias) - .WithPortBinding(_followerTcpPort.ToString(), _followerTcpPort.ToString()) - .WithPortBinding(_followerHttpPort.ToString(), _followerHttpPort.ToString()) - .WithEnvironment("RUST_LOG", "trace") - .WithEnvironment("IGGY_SYSTEM_LOGGING_LEVEL", "trace") - .WithEnvironment("IGGY_ROOT_USERNAME", "iggy") - .WithEnvironment("IGGY_ROOT_PASSWORD", "iggy") - .WithEnvironment("IGGY_SYSTEM_PATH", "local_data_follower") - .WithEnvironment("IGGY_TCP_ADDRESS", $"0.0.0.0:{_followerTcpPort}") - .WithEnvironment("IGGY_HTTP_ADDRESS", $"0.0.0.0:{_followerHttpPort}") - .WithEnvironment("IGGY_QUIC_ADDRESS", $"0.0.0.0:{_followerQuicPort}") - .WithEnvironment("IGGY_WEBSOCKET_ADDRESS", $"0.0.0.0:{_followerWsPort}") - .WithEnvironment(clusterRosterEnv) - .WithPrivileged(true) - .WithCleanUp(true) - .WithWaitStrategy(Wait.ForUnixContainer().UntilInternalTcpPortIsAvailable(_followerTcpPort)) - .Build(); - } - - public async ValueTask DisposeAsync() - { - await SaveContainerLogsAsync(_leaderContainer, "leader"); - await SaveContainerLogsAsync(_followerContainer, "follower"); - await _followerContainer.StopAsync(); - await _leaderContainer.StopAsync(); - await _network.DeleteAsync(); - } - - public async Task InitializeAsync() - { - await _network.CreateAsync(); - await Task.WhenAll(_leaderContainer.StartAsync(), _followerContainer.StartAsync()); - } - - public string GetLeaderAddress() - { - return $"127.0.0.1:{_leaderTcpPort}"; - } - - public string GetFollowerAddress() - { - return $"127.0.0.1:{_followerTcpPort}"; - } - - private ushort ReservePort() - { - for (ushort candidate = BasePort; candidate < EndPort; candidate++) - { - try - { - var listener = new TcpListener(IPAddress.Loopback, candidate); - listener.Start(); - _portReservations.Add(listener); - return candidate; - } - catch (SocketException) - { - // Port is held by a previous ReservePort() in this fixture - // (the common case) or by something else on the host; keep - // walking the range. - } - } - - throw new InvalidOperationException( - $"No free ports available in [{BasePort}, {EndPort}) for .NET {Environment.Version.Major}.x."); - } - - private void ReleaseReservedPorts() - { - foreach (var listener in _portReservations) - { - listener.Stop(); - } - - _portReservations.Clear(); - } - - private static async Task SaveContainerLogsAsync(IContainer container, string role) - { - if (string.IsNullOrEmpty(LogDirectory)) - { - return; - } - - try - { - Directory.CreateDirectory(LogDirectory); - var dotnetVersion = $"net{Environment.Version.Major}.{Environment.Version.Minor}"; - var logFilePath = Path.Combine(LogDirectory, $"iggy-{role}-{dotnetVersion}-{container.Name}.log"); - - var (stdout, stderr) = await container.GetLogsAsync(); - - await using var writer = new StreamWriter(logFilePath); - if (!string.IsNullOrEmpty(stdout)) - { - await writer.WriteLineAsync("=== STDOUT ==="); - await writer.WriteLineAsync(stdout); - } - - if (!string.IsNullOrEmpty(stderr)) - { - await writer.WriteLineAsync("=== STDERR ==="); - await writer.WriteLineAsync(stderr); - } - } - catch (Exception ex) - { - Console.WriteLine($"Failed to save {role} container logs: {ex.Message}"); - } - } -} diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyServerFixture.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyServerFixture.cs index 8c6718b607..1670cbcdf4 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyServerFixture.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyServerFixture.cs @@ -21,35 +21,35 @@ using Apache.Iggy.Factory; using Apache.Iggy.IggyClient; using Apache.Iggy.Tests.Integrations.Helpers; -using DotNet.Testcontainers.Builders; -using DotNet.Testcontainers.Containers; using TUnit.Core.Interfaces; namespace Apache.Iggy.Tests.Integrations.Fixtures; +///

+/// Runs the suite against an iggy-server : a standalone node by default, +/// or a replicated cluster when IGGY_TEST_CLUSTER_NODES asks for one, so every test commits through +/// consensus. The SDK frames TCP with the VSR wire protocol. +/// public class IggyServerFixture : IAsyncInitializer, IAsyncDisposable { private readonly string _containerId = Guid.NewGuid().ToString(); - protected IContainer? IggyContainer; + private readonly SemaphoreSlim _startGate = new(1, 1); + + private VsrCluster? _cluster; /// /// Docker image to use. Can be overridden via IGGY_SERVER_DOCKER_IMAGE environment variable - /// or by subclasses. Defaults to apache/iggy:edge if not specified. + /// or by subclasses. Defaults to the locally built iggy-server:test; build it with + /// docker build -f core/server/Dockerfile -t iggy-server:test . from the repository root. /// - private string DockerImage => - Environment.GetEnvironmentVariable("IGGY_SERVER_DOCKER_IMAGE") ?? "apache/iggy:edge"; + protected virtual string DockerImage => + Environment.GetEnvironmentVariable("IGGY_SERVER_DOCKER_IMAGE") ?? "iggy-server:test"; /// - /// Environment variables for the container. Override in subclasses to customize. + /// Names the containers and network of this fixture's cluster, so a `docker ps` during a run + /// shows which fixture owns which node. /// - protected virtual Dictionary EnvironmentVariables => new() - { - { "IGGY_ROOT_USERNAME", "iggy" }, - { "IGGY_ROOT_PASSWORD", "iggy" }, - { "IGGY_TCP_ADDRESS", "0.0.0.0:8090" }, - { "IGGY_HTTP_ADDRESS", "0.0.0.0:3000" }, - { "IGGY_SYSTEM_TOPIC_MESSAGE_EXPIRY", "10m" } - }; + protected virtual string ClusterName => "general"; /// /// Enables iggy server trace logs. @@ -57,159 +57,85 @@ public class IggyServerFixture : IAsyncInitializer, IAsyncDisposable protected bool EnabledServerTraceLogs => true; /// - /// Resource mappings (volumes, etc.) for the container. Override in subclasses to add custom mappings. + /// Extra environment variables for every node, layered over the cluster's base configuration. + /// Override in subclasses to customize. /// - protected virtual ResourceMapping[] ResourceMappings => []; + protected virtual Dictionary EnvironmentVariables => []; /// - /// Directory for container log files. Set via IGGY_TEST_LOGS_DIR environment variable. - /// If not set, container logs will not be saved to file. + /// Resource mappings (certificates, etc.) mounted into every node. Override in subclasses. /// - private static string? LogDirectory => - Environment.GetEnvironmentVariable("IGGY_TEST_LOGS_DIR"); - - public IggyServerFixture() - { - var builder = new ContainerBuilder(DockerImage) - .WithPortBinding(3000, true) - .WithPortBinding(8090, true) - .WithWaitStrategy(Wait.ForUnixContainer() - .UntilInternalTcpPortIsAvailable(8090) - .UntilHttpRequestIsSucceeded(request => request - .ForPort(3000) - .ForPath("/ping"))) - .WithName(_containerId) - .WithPrivileged(true) - .WithCleanUp(true); - - foreach (var (key, value) in EnvironmentVariables) - { - builder = builder.WithEnvironment(key, value); - } - - if (EnabledServerTraceLogs) - { - builder = builder - .WithEnvironment("IGGY_SYSTEM_LOGGING_LEVEL", "trace") - .WithEnvironment("RUST_LOG", "trace"); - } - - foreach (var mapping in ResourceMappings) - { - builder = builder.WithResourceMapping(mapping.Source, mapping.Destination); - } + protected virtual ResourceMapping[] ResourceMappings => []; - IggyContainer = builder.Build(); - } + /// + /// Cluster size, mirroring the Rust integration harness knob of the same name: the + /// IGGY_TEST_CLUSTER_NODES environment variable decides (default 1, a standalone node with + /// clustering disabled; 2 or more, a replicated cluster). A subclass can pin its own size + /// instead, the way the redirection cluster stays three nodes whatever the knob says. + /// + protected virtual int NodeCount => + int.TryParse(Environment.GetEnvironmentVariable("IGGY_TEST_CLUSTER_NODES"), out var count) && count >= 1 + ? count + : 1; public async ValueTask DisposeAsync() { - if (IggyContainer == null) + if (_cluster != null) { - return; + await _cluster.DisposeAsync(); } - - await SaveContainerLogsAsync(); - await IggyContainer.StopAsync(); } - public virtual async Task InitializeAsync() - { - await IggyContainer!.StartAsync(); - - await CreateTcpClient(); - await CreateHttpClient(); - } - - private async Task SaveContainerLogsAsync() - { - if (string.IsNullOrEmpty(LogDirectory)) - { - return; - } - - try - { - Directory.CreateDirectory(LogDirectory); - var dotnetVersion = $"net{Environment.Version.Major}.{Environment.Version.Minor}"; - var logFilePath = Path.Combine(LogDirectory, $"iggy-server-{dotnetVersion}-{_containerId}.log"); - - var (stdout, stderr) = await IggyContainer!.GetLogsAsync(); - - await using var writer = new StreamWriter(logFilePath); - if (!string.IsNullOrEmpty(stdout)) - { - await writer.WriteLineAsync("=== STDOUT ==="); - await writer.WriteLineAsync(stdout); - } - - if (!string.IsNullOrEmpty(stderr)) - { - await writer.WriteLineAsync("=== STDERR ==="); - await writer.WriteLineAsync(stderr); - } - } - catch (Exception ex) - { - Console.WriteLine($"Failed to save container logs: {ex.Message}"); - } - } - - public async Task> CreateClients() + /// + /// The cluster starts on first use, so a run that never dials the server does not pay for it. + /// + public Task InitializeAsync() { - var dictionary = new Dictionary(); - dictionary[Protocol.Tcp] = await CreateTcpClient(); - dictionary[Protocol.Http] = await CreateHttpClient(); - - return dictionary; + return Task.CompletedTask; } + /// + /// Under VSR the register handshake is the login, so auto-login on top of the explicit one would register + /// twice on the same connection: the server answers the second one by replaying the binding it already + /// holds, and the client then carries a client id the server never bound, which consumer-group + /// membership is keyed by. + /// public async Task CreateAuthenticatedClient(Protocol protocol, string userName = "iggy", - string password = "iggy") + string password = "iggy", IMessageEncryptor? encryptor = null) { - return protocol == Protocol.Tcp - ? await CreateTcpClient(userName, password) - : await CreateHttpClient(userName, password); - } - - public async Task CreateTcpClient(string userName = "iggy", string password = "iggy", - bool connect = true, IMessageEncryptor? encryptor = null) - { - var client = await CreateClient(Protocol.Tcp, connect: connect, encryptor: encryptor); - - if (connect) - { - await client.LoginUserAsync(userName, password); - } + var client = await CreateClient(protocol, protocol == Protocol.Http, + encryptor: encryptor, userName: userName, password: password); + await client.LoginUserAsync(userName, password); return client; } - public async Task CreateHttpClient(string userName = "iggy", string password = "iggy", - IMessageEncryptor? encryptor = null) + /// + /// A connected client that has not logged in, so the caller owns the handshake. + /// + public async Task CreateUnauthenticatedClient(Protocol protocol) { - var client = await CreateClient(Protocol.Http, encryptor: encryptor); - - await client.LoginUserAsync(userName, password); - - return client; + return await CreateClient(protocol); } - public async Task CreateClient(Protocol protocol, Protocol? targetContainer = null, - bool connect = true, IMessageEncryptor? encryptor = null) + /// + /// overrides the cluster address, so a test can dial the server through a + /// proxy while keeping the rest of the configuration identical. + /// + public async Task CreateClient(Protocol protocol, bool autoLogin = false, bool connect = true, + IMessageEncryptor? encryptor = null, string? address = null, string userName = "iggy", + string password = "iggy") { - var address = GetIggyAddress(protocol); - var client = IggyClientFactory.CreateClient(new IggyClientConfigurator { - BaseAddress = address, + BaseAddress = address ?? await GetIggyAddressAsync(protocol), Protocol = protocol, ReconnectionSettings = new ReconnectionSettings { Enabled = true }, AutoLoginSettings = new AutoLoginSettings { - Enabled = true, - Username = "iggy", - Password = "iggy" + Enabled = autoLogin, + Username = userName, + Password = password }, MessageEncryptor = encryptor }); @@ -222,20 +148,58 @@ public async Task CreateClient(Protocol protocol, Protocol? targetC return client; } - public virtual string GetIggyAddress(Protocol protocol) + public async Task GetIggyAddressAsync(Protocol protocol) + { + var cluster = await EnsureClusterStartedAsync(); + + return protocol == Protocol.Tcp ? cluster.LeaderTcpAddress : cluster.LeaderHttpAddress; + } + + /// + /// A backup node of the initial view, for tests that dial a follower and expect the client to be + /// redirected to the primary. + /// + public async Task GetFollowerTcpAddressAsync() { - var port = protocol == Protocol.Tcp - ? IggyContainer!.GetMappedPublicPort(8090) - : IggyContainer!.GetMappedPublicPort(3000); + var cluster = await EnsureClusterStartedAsync(); - return protocol == Protocol.Tcp - ? $"127.0.0.1:{port}" - : $"http://127.0.0.1:{port}"; + return cluster.FollowerTcpAddress; } public static IEnumerable> ProtocolData() { - yield return () => Protocol.Http; - yield return () => Protocol.Tcp; + return [() => Protocol.Http, () => Protocol.Tcp]; + } + + private async Task EnsureClusterStartedAsync() + { + await _startGate.WaitAsync(); + try + { + if (_cluster == null) + { + var cluster = new VsrCluster(DockerImage, ClusterName, _containerId, + EnabledServerTraceLogs, NodeCount, EnvironmentVariables, ResourceMappings); + try + { + await cluster.StartAsync(); + } + catch + { + // A later retry rebuilds the cluster under the same name, so a half-started one + // must not leave its network or containers behind. + await cluster.DisposeAsync(); + throw; + } + + _cluster = cluster; + } + + return _cluster; + } + finally + { + _startGate.Release(); + } } } diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyTlsServerFixture.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyTlsServerFixture.cs index 0d7ff109df..2b412503b2 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyTlsServerFixture.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/IggyTlsServerFixture.cs @@ -25,6 +25,8 @@ namespace Apache.Iggy.Tests.Integrations.Fixtures; /// public class IggyTlsServerFixture : IggyServerFixture { + protected override string ClusterName => "tls"; + /// /// Environment variables with TLS configuration enabled. /// @@ -43,8 +45,5 @@ public class IggyTlsServerFixture : IggyServerFixture new("Certs", "/app/certs/") ]; - public override async Task InitializeAsync() - { - await IggyContainer!.StartAsync(); - } + protected override int NodeCount => 1; } diff --git a/core/server/src/http/shared.rs b/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/RedirectionClusterFixture.cs similarity index 65% rename from core/server/src/http/shared.rs rename to foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/RedirectionClusterFixture.cs index faab1b8cb6..ddcc2dcf6a 100644 --- a/core/server/src/http/shared.rs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/RedirectionClusterFixture.cs @@ -15,19 +15,15 @@ // specific language governing permissions and limitations // under the License. -use super::http_shard_wrapper::HttpSafeShard; -use crate::http::jwt::jwt_manager::JwtManager; -use std::net::SocketAddr; -use ulid::Ulid; +namespace Apache.Iggy.Tests.Integrations.Fixtures; -pub struct AppState { - pub jwt_manager: JwtManager, - pub shard: HttpSafeShard, -} +/// +/// A dedicated three-node cluster for the redirection tests, pinned regardless of the +/// IGGY_TEST_CLUSTER_NODES knob: dialing a follower needs real replicas to redirect between. +/// +public class RedirectionClusterFixture : IggyServerFixture +{ + protected override string ClusterName => "redirection"; -#[derive(Debug, Copy, Clone)] -pub struct RequestDetails { - #[allow(dead_code)] - pub request_id: Ulid, - pub ip_address: SocketAddr, + protected override int NodeCount => 3; } diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/VsrCluster.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/VsrCluster.cs new file mode 100644 index 0000000000..512b86a0e2 --- /dev/null +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/Fixtures/VsrCluster.cs @@ -0,0 +1,438 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Net; +using System.Net.Sockets; +using Apache.Iggy.Tests.Integrations.Helpers; +using Docker.DotNet; +using Docker.DotNet.Models; +using DotNet.Testcontainers.Builders; +using DotNet.Testcontainers.Containers; +using DotNet.Testcontainers.Networks; + +namespace Apache.Iggy.Tests.Integrations.Fixtures; + +/// +/// The iggy-server deployment a run scoped to that server uses. A single node runs with +/// clustering disabled, exercising the wire protocol without replication; two or more nodes form +/// a real roster that puts every test through consensus. +/// +internal sealed class VsrCluster : IAsyncDisposable +{ + private const string ClusterName = "test-vsr-cluster"; + + // TCP and HTTP get real host ports on every node: a view change can make any replica the + // primary clients are redirected to, so every advertised 127.0.0.1:port must be dialable from + // the host. QUIC/WebSocket/replica listeners are reachable only on the cluster network and get + // base + node. The offset is not for the network (each node has its own IP there) but because + // the server rejects a roster where two advertised endpoints collide. + private const ushort InternalQuicPortBase = 8200; + private const ushort InternalWebSocketPortBase = 8300; + private const ushort InternalReplicaPortBase = 8400; + + // Host-port pool, partitioned per .NET major so the parallel `dotnet test` processes + // (net8.0 -> 29800..29949, net10.0 -> 30000..30149) never race each other for a port. + private static readonly ushort BasePort = (ushort)(29000 + Environment.Version.Major * 100); + private static readonly ushort EndPort = (ushort)(BasePort + 150); + + // A reserved port is released before its container binds it, so another cluster in this process + // could scan onto it in that window. Ports stay claimed for the process lifetime to close it. + private static readonly HashSet ClaimedPorts = []; + + private readonly IContainer?[] _containers; + private readonly IReadOnlyDictionary? _extraEnvironment; + private readonly string _idSuffix; + private readonly string _image; + private readonly string _name; + private INetwork? _network; + private readonly int _nodeCount; + private readonly NodePorts[] _ports; + private readonly IReadOnlyList? _resourceMappings; + private readonly bool _traceLogs; + + /// Replica 0 - primary of the initial view, and the node the tests talk to. + public string LeaderTcpAddress => $"127.0.0.1:{_ports[0].Tcp}"; + + /// The same node's REST surface, which serves classic framing rather than VSR. + public string LeaderHttpAddress => $"http://127.0.0.1:{_ports[0].Http}"; + + /// A backup of the initial view, so a client dialing it gets redirected to the primary. + public string FollowerTcpAddress => ClusterEnabled + ? $"127.0.0.1:{_ports[1].Tcp}" + : throw new InvalidOperationException( + "A follower address requires a cluster; set IGGY_TEST_CLUSTER_NODES to 2 or more."); + + /// The same backup node's REST surface. + public string FollowerHttpAddress => ClusterEnabled + ? $"http://127.0.0.1:{_ports[1].Http}" + : throw new InvalidOperationException( + "A follower address requires a cluster; set IGGY_TEST_CLUSTER_NODES to 2 or more."); + + private bool ClusterEnabled => _nodeCount > 1; + + private static string? LogDirectory => + Environment.GetEnvironmentVariable("IGGY_TEST_LOGS_DIR"); + + /// + /// of 1 starts a standalone node with clustering disabled, higher + /// values a cluster with that many replicas. labels the containers and + /// network after the owning fixture. wins over the + /// base configuration, and are mounted into every node. + /// + public VsrCluster(string image, string name, string idSuffix, bool traceLogs, int nodeCount, + IReadOnlyDictionary? extraEnvironment = null, + IReadOnlyList? resourceMappings = null) + { + _image = image; + _name = name; + _idSuffix = idSuffix; + _traceLogs = traceLogs; + _extraEnvironment = extraEnvironment; + _resourceMappings = resourceMappings; + _nodeCount = nodeCount; + _ports = ReservePorts(_nodeCount); + _containers = new IContainer[_nodeCount]; + } + + public async ValueTask DisposeAsync() + { + for (var node = 0; node < _nodeCount; node++) + { + if (_containers[node] == null) + { + continue; + } + + try + { + await SaveContainerLogsAsync(_containers[node]!, $"iggy-server-{node}"); + } + catch (Exception e) + { + Console.WriteLine($"Failed to save the logs of iggy-server-{node}: {e}"); + } + } + + foreach (var container in _containers) + { + if (container == null) + { + continue; + } + + try + { + await container.DisposeAsync(); + } + catch (Exception e) + { + Console.WriteLine($"Failed to dispose an iggy-server container: {e}"); + } + } + + if (_network != null) + { + try + { + await _network.DeleteAsync(); + } + catch (Exception e) + { + Console.WriteLine($"Failed to delete the iggy-server network: {e}"); + } + } + } + + /// + /// The containers are built here rather than in the constructor because a roster needs every + /// node's IP before any node starts, and those IPs come from the subnet the network is + /// created with. + /// + public async Task StartAsync() + { + var nodeAddresses = Array.Empty(); + if (ClusterEnabled) + { + var subnetPrefix = await CreateNetworkAsync(); + // The gateway takes .1, so node addresses start at .10. + nodeAddresses = Enumerable.Range(0, _nodeCount) + .Select(node => $"{subnetPrefix}.{10 + node}") + .ToArray(); + } + + Dictionary clusterEnvironment = BuildClusterEnvironment(nodeAddresses); + for (var node = 0; node < _nodeCount; node++) + { + _containers[node] = BuildNodeContainer(node, nodeAddresses, clusterEnvironment); + } + + // A multi-node roster needs a quorum of replicas, so nothing commits until enough nodes are up. + await Task.WhenAll(_containers.Select(container => container!.StartAsync())); + } + + /// + /// Roster peers dial each other by literal IP - the ip field is parsed, not resolved - so every + /// node gets a static address, and Docker only honors static addresses on networks created with + /// an explicit subnet. The subnet is picked from a private /16 here; a candidate overlapping + /// another network on the host fails the create and the next one is tried. + /// + private async Task CreateNetworkAsync() + { + for (var candidate = 0; candidate < 256; candidate++) + { + var subnetPrefix = $"10.213.{candidate}"; + var network = new NetworkBuilder() + .WithName($"iggy-vsr-{_name}-{_idSuffix}") + .WithCreateParameterModifier(parameters => parameters.IPAM = new IPAM + { + Config = [new IPAMConfig { Subnet = $"{subnetPrefix}.0/24" }] + }) + .Build(); + + try + { + await network.CreateAsync(); + _network = network; + return subnetPrefix; + } + catch (DockerApiException) + { + // The subnet overlaps a network already on the host. + } + } + + throw new InvalidOperationException("No free /24 subnet in 10.213.0.0/16 for the cluster network."); + } + + private IContainer BuildNodeContainer(int node, IReadOnlyList nodeAddresses, + IReadOnlyDictionary clusterEnvironment) + { + var ports = _ports[node]; + var builder = new ContainerBuilder(_image) + .WithName($"iggy-vsr-{_name}-{node}-{_idSuffix}") + .WithEnvironment("IGGY_ROOT_USERNAME", "iggy") + .WithEnvironment("IGGY_ROOT_PASSWORD", "iggy") + .WithEnvironment("IGGY_SYSTEM_TOPIC_MESSAGE_EXPIRY", "10m") + .WithEnvironment("IGGY_SYSTEM_PATH", $"local_data_vsr_{node}") + .WithEnvironment("IGGY_TCP_ADDRESS", $"0.0.0.0:{ports.Tcp}") + .WithEnvironment("IGGY_HTTP_ADDRESS", $"0.0.0.0:{ports.Http}") + .WithEnvironment("IGGY_QUIC_ADDRESS", $"0.0.0.0:{ports.Quic}") + .WithEnvironment("IGGY_WEBSOCKET_ADDRESS", $"0.0.0.0:{ports.WebSocket}") + .WithEnvironment(clusterEnvironment) + .WithPrivileged(true) + .WithCleanUp(true) + .WithWaitStrategy(Wait.ForUnixContainer() + .UntilInternalTcpPortIsAvailable(ports.Tcp) + .UntilInternalTcpPortIsAvailable(ports.Http)); + + // Host bindings mirror the container port so the loopback address advertised in cluster + // metadata resolves to the node that advertised it. Every node needs one: view changes + // can make any replica the primary clients get redirected to. + builder = builder + .WithPortBinding(ports.Tcp.ToString(), ports.Tcp.ToString()) + .WithPortBinding(ports.Http.ToString(), ports.Http.ToString()); + + if (ClusterEnabled) + { + var address = nodeAddresses[node]; + builder = builder + .WithCommand("--replica-id", node.ToString()) + .WithNetwork(_network) + .WithNetworkAliases($"vsr-node-{node}") + .WithCreateParameterModifier(parameters => + AssignStaticAddress(parameters, _network!.Name, address)); + } + + if (_traceLogs) + { + builder = builder + .WithEnvironment("IGGY_SYSTEM_LOGGING_LEVEL", "trace") + .WithEnvironment("RUST_LOG", "trace"); + } + + if (_extraEnvironment != null) + { + foreach (var (key, value) in _extraEnvironment) + { + builder = builder.WithEnvironment(key, value); + } + } + + if (_resourceMappings != null) + { + foreach (var mapping in _resourceMappings) + { + builder = builder.WithResourceMapping(mapping.Source, mapping.Destination); + } + } + + return builder.Build(); + } + + private Dictionary BuildClusterEnvironment(IReadOnlyList nodeAddresses) + { + var environment = new Dictionary + { + ["IGGY_CLUSTER_ENABLED"] = ClusterEnabled ? "true" : "false" + }; + + if (!ClusterEnabled) + { + return environment; + } + + environment["IGGY_CLUSTER_NAME"] = ClusterName; + environment["IGGY_MESSAGE_BUS_RECONNECT_PERIOD"] = "100ms"; + + for (var node = 0; node < _nodeCount; node++) + { + var ports = _ports[node]; + environment[$"IGGY_CLUSTER_NODES_{node}_NAME"] = $"vsr-node-{node}"; + environment[$"IGGY_CLUSTER_NODES_{node}_IP"] = nodeAddresses[node]; + environment[$"IGGY_CLUSTER_NODES_{node}_ADVERTISED_ADDRESS"] = "127.0.0.1"; + environment[$"IGGY_CLUSTER_NODES_{node}_REPLICA_ID"] = node.ToString(); + environment[$"IGGY_CLUSTER_NODES_{node}_PORTS_TCP"] = ports.Tcp.ToString(); + environment[$"IGGY_CLUSTER_NODES_{node}_PORTS_HTTP"] = ports.Http.ToString(); + environment[$"IGGY_CLUSTER_NODES_{node}_PORTS_QUIC"] = ports.Quic.ToString(); + environment[$"IGGY_CLUSTER_NODES_{node}_PORTS_WEBSOCKET"] = ports.WebSocket.ToString(); + environment[$"IGGY_CLUSTER_NODES_{node}_PORTS_TCP_REPLICA"] = ports.Replica.ToString(); + } + + return environment; + } + + /// + /// Pins the container's address on the cluster network. Testcontainers has no first-class knob for it, + /// and the roster needs the address before any container starts. + /// + private static void AssignStaticAddress(CreateContainerParameters parameters, string networkName, + string address) + { + parameters.NetworkingConfig ??= new NetworkingConfig(); + parameters.NetworkingConfig.EndpointsConfig ??= new Dictionary(); + + if (!parameters.NetworkingConfig.EndpointsConfig.TryGetValue(networkName, out var endpoint)) + { + endpoint = new EndpointSettings(); + parameters.NetworkingConfig.EndpointsConfig[networkName] = endpoint; + } + + endpoint.IPAMConfig = new EndpointIPAMConfig { IPv4Address = address }; + } + + /// + /// Finds free host ports for the host-dialed endpoints by briefly binding them, so concurrent + /// clusters cannot pick the same port. The listeners are released before the containers start. + /// + private static NodePorts[] ReservePorts(int nodeCount) + { + var reservations = new List(); + try + { + var ports = new NodePorts[nodeCount]; + for (var node = 0; node < nodeCount; node++) + { + ports[node] = new NodePorts(ReservePort(reservations), + ReservePort(reservations), + (ushort)(InternalQuicPortBase + node), + (ushort)(InternalWebSocketPortBase + node), + (ushort)(InternalReplicaPortBase + node)); + } + + return ports; + } + finally + { + foreach (var listener in reservations) + { + listener.Stop(); + } + } + } + + private static ushort ReservePort(List reservations) + { + lock (ClaimedPorts) + { + for (var candidate = BasePort; candidate < EndPort; candidate++) + { + if (ClaimedPorts.Contains(candidate)) + { + continue; + } + + try + { + var listener = new TcpListener(IPAddress.Loopback, candidate); + listener.Start(); + reservations.Add(listener); + ClaimedPorts.Add(candidate); + return candidate; + } + catch (SocketException) + { + // Held by something else on the host. + } + } + } + + throw new InvalidOperationException( + $"No free ports available in [{BasePort}, {EndPort}) for .NET {Environment.Version.Major}.x."); + } + + private static async Task SaveContainerLogsAsync(IContainer container, string role) + { + if (string.IsNullOrEmpty(LogDirectory)) + { + return; + } + + try + { + Directory.CreateDirectory(LogDirectory); + var dotnetVersion = $"net{Environment.Version.Major}.{Environment.Version.Minor}"; + // Docker hands back names with a leading slash, which Path.Combine would read as a directory. + var containerName = container.Name.TrimStart('/'); + var logFilePath = Path.Combine(LogDirectory, $"{role}-{dotnetVersion}-{containerName}.log"); + + var (stdout, stderr) = await container.GetLogsAsync(); + + await using var writer = new StreamWriter(logFilePath); + if (!string.IsNullOrEmpty(stdout)) + { + await writer.WriteLineAsync("=== STDOUT ==="); + await writer.WriteLineAsync(stdout); + } + + if (!string.IsNullOrEmpty(stderr)) + { + await writer.WriteLineAsync("=== STDERR ==="); + await writer.WriteLineAsync(stderr); + } + } + catch (Exception ex) + { + Console.WriteLine($"Failed to save {role} container logs: {ex.Message}"); + } + } + + /// + /// The ports one node listens on. The leader's and first follower's TCP/HTTP are host ports + /// mirrored into the container; the rest are the fixed cluster-network ports. + /// + private readonly record struct NodePorts(ushort Tcp, ushort Http, ushort Quic, ushort WebSocket, ushort Replica); +} diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/FlushMessagesTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/FlushMessagesTests.cs index 778b707bac..ea258700b8 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/FlushMessagesTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/FlushMessagesTests.cs @@ -17,11 +17,8 @@ using Apache.Iggy.Enums; using Apache.Iggy.Exceptions; -using Apache.Iggy.IggyClient; -using Apache.Iggy.Messages; using Apache.Iggy.Tests.Integrations.Fixtures; using Shouldly; -using Partitioning = Apache.Iggy.Kinds.Partitioning; namespace Apache.Iggy.Tests.Integrations; @@ -30,62 +27,14 @@ public class FlushMessagesTests [ClassDataSource(Shared = SharedType.PerAssembly)] public required IggyServerFixture Fixture { get; init; } - private async Task<(IIggyClient client, string streamName, string topicName)> CreateStreamWithMessages( - Protocol protocol) - { - var client = await Fixture.CreateAuthenticatedClient(protocol); - - var streamName = $"flush-{Guid.NewGuid():N}"; - var topicName = "test-topic"; - - await client.CreateStreamAsync(streamName); - await client.CreateTopicAsync(Identifier.String(streamName), topicName, 1); - - await client.SendMessagesAsync(Identifier.String(streamName), - Identifier.String(topicName), Partitioning.None(), - [ - new Message(Guid.NewGuid(), "Test message 1"u8.ToArray()), - new Message(Guid.NewGuid(), "Test message 2"u8.ToArray()), - new Message(Guid.NewGuid(), "Test message 3"u8.ToArray()), - new Message(Guid.NewGuid(), "Test message 4"u8.ToArray()) - ]); - - return (client, streamName, topicName); - } - [Test] [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] - public async Task FlushUnsavedBuffer_WithFsync_Should_Flush_Successfully(Protocol protocol) + public async Task FlushUnsavedBuffer_Should_Throw_FeatureUnavailable(Protocol protocol) { - var (client, streamName, topicName) = await CreateStreamWithMessages(protocol); - - await Should.NotThrowAsync(() => - client.FlushUnsavedBufferAsync( - Identifier.String(streamName), - Identifier.String(topicName), 0, true)); - } - - [Test] - [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] - public async Task FlushUnsavedBuffer_WithOutFsync_Should_Flush_Successfully(Protocol protocol) - { - var (client, streamName, topicName) = await CreateStreamWithMessages(protocol); - - await Should.NotThrowAsync(() => - client.FlushUnsavedBufferAsync( - Identifier.String(streamName), - Identifier.String(topicName), 0, false)); - } - - [Test] - [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] - public async Task FlushUnsavedBuffer_Should_Throw_WhenPartition_DoesNotExist(Protocol protocol) - { - var (client, streamName, topicName) = await CreateStreamWithMessages(protocol); + var client = await Fixture.CreateAuthenticatedClient(protocol); - await Should.ThrowAsync(() => - client.FlushUnsavedBufferAsync( - Identifier.String(streamName), - Identifier.String(topicName), 55, false)); + await Should.ThrowAsync(() => + client.FlushUnsavedBufferAsync(Identifier.String("any-stream"), + Identifier.String("any-topic"), 1, false)); } } diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/HeaderEncryptionIntegrationTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/HeaderEncryptionIntegrationTests.cs index 4004a5929a..9de96652d1 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/HeaderEncryptionIntegrationTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/HeaderEncryptionIntegrationTests.cs @@ -45,12 +45,8 @@ public async Task SendMessages_WithEncryptedHeaders_Should_NotBeReadableWithoutD // Publisher on an encrypting client; a plain client raw-polls the same topic to prove the wire bytes // stay encrypted. - var encryptingClient = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient(encryptor: encryptor) - : await Fixture.CreateHttpClient(encryptor: encryptor); - var plainClient = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var encryptingClient = await Fixture.CreateAuthenticatedClient(protocol, encryptor: encryptor); + var plainClient = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStream(plainClient, protocol); var streamId = Identifier.String(testStream.StreamId); @@ -93,7 +89,11 @@ public async Task SendMessages_WithEncryptedHeaders_Should_NotBeReadableWithoutD Dictionary decryptedHeaders = BinaryMapper.MapHeaders(decryptedHeaderBytesResult); decryptedHeaders.Count.ShouldBe(3); - var typeHeader = decryptedHeaders[new HeaderKey { Kind = HeaderKind.String, Value = "type"u8.ToArray() }]; + var typeHeader = decryptedHeaders[new HeaderKey + { + Kind = HeaderKind.String, + Value = "type"u8.ToArray() + }]; Encoding.UTF8.GetString(typeHeader.Value).ShouldBe("test-message"); } @@ -104,9 +104,7 @@ public async Task ReceiveAsync_WithEncryptingClient_Should_DecryptHeadersCorrect var encryptor = CreateEncryptor(); // One encrypting client serves both publisher and consumer: it encrypts on send and decrypts on poll. - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient(encryptor: encryptor) - : await Fixture.CreateHttpClient(encryptor: encryptor); + var client = await Fixture.CreateAuthenticatedClient(protocol, encryptor: encryptor); var testStream = await CreateTestStream(client, protocol); var streamId = Identifier.String(testStream.StreamId); @@ -162,13 +160,25 @@ public async Task ReceiveAsync_WithEncryptingClient_Should_DecryptHeadersCorrect received.Message.UserHeaders.ShouldNotBeNull(); received.Message.UserHeaders!.Count.ShouldBe(3); - var batchHeader = received.Message.UserHeaders[new HeaderKey { Kind = HeaderKind.String, Value = "batch"u8.ToArray() }]; + var batchHeader = received.Message.UserHeaders[new HeaderKey + { + Kind = HeaderKind.String, + Value = "batch"u8.ToArray() + }]; BitConverter.ToUInt64(batchHeader.Value).ShouldBe(1UL); - var typeHeader = received.Message.UserHeaders[new HeaderKey { Kind = HeaderKind.String, Value = "type"u8.ToArray() }]; + var typeHeader = received.Message.UserHeaders[new HeaderKey + { + Kind = HeaderKind.String, + Value = "type"u8.ToArray() + }]; Encoding.UTF8.GetString(typeHeader.Value).ShouldBe("test-message"); - var encHeader = received.Message.UserHeaders[new HeaderKey { Kind = HeaderKind.String, Value = "encrypted"u8.ToArray() }]; + var encHeader = received.Message.UserHeaders[new HeaderKey + { + Kind = HeaderKind.String, + Value = "encrypted"u8.ToArray() + }]; encHeader.Value[0].ShouldBe((byte)1); } @@ -182,17 +192,41 @@ private static Dictionary CreateTestHeaders() return new Dictionary { { - new HeaderKey { Kind = HeaderKind.String, Value = "batch"u8.ToArray() }, - new HeaderValue { Kind = HeaderKind.Uint64, Value = BitConverter.GetBytes(1UL) } + new HeaderKey + { + Kind = HeaderKind.String, + Value = "batch"u8.ToArray() + }, + new HeaderValue + { + Kind = HeaderKind.Uint64, + Value = BitConverter.GetBytes(1UL) + } }, { - new HeaderKey { Kind = HeaderKind.String, Value = "type"u8.ToArray() }, - new HeaderValue { Kind = HeaderKind.String, Value = "test-message"u8.ToArray() } + new HeaderKey + { + Kind = HeaderKind.String, + Value = "type"u8.ToArray() + }, + new HeaderValue + { + Kind = HeaderKind.String, + Value = "test-message"u8.ToArray() + } }, { - new HeaderKey { Kind = HeaderKind.String, Value = "encrypted"u8.ToArray() }, - new HeaderValue { Kind = HeaderKind.Bool, Value = [1] } - }, + new HeaderKey + { + Kind = HeaderKind.String, + Value = "encrypted"u8.ToArray() + }, + new HeaderValue + { + Kind = HeaderKind.Bool, + Value = [1] + } + } }; } diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/Helpers/Eventually.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/Helpers/Eventually.cs new file mode 100644 index 0000000000..17cef93eee --- /dev/null +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/Helpers/Eventually.cs @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.Iggy.Tests.Integrations.Helpers; + +public static class Eventually +{ + private static readonly TimeSpan PollInterval = TimeSpan.FromMilliseconds(100); + + /// + /// Polls until the read satisfies the condition. Some server work commits ahead of the state it changes - + /// the server applies a purge by advancing a generation that its reconciler acts on a tick later - so the + /// first read after an acknowledged command can still show the old value. + /// + public static async Task ReadAsync(Func> read, Func condition, TimeSpan timeout) + { + var deadline = DateTime.UtcNow + timeout; + while (true) + { + var value = await read(); + if (condition(value) || DateTime.UtcNow >= deadline) + { + return value; + } + + await Task.Delay(PollInterval); + } + } +} diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/IggyConsumerTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/IggyConsumerTests.cs index 72f7987b8d..ef7c062f05 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/IggyConsumerTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/IggyConsumerTests.cs @@ -38,9 +38,7 @@ public class IggyConsumerTests [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task InitAsync_WithSingleConsumer_Should_Initialize_Successfully(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -63,9 +61,7 @@ public async Task InitAsync_WithSingleConsumer_Should_Initialize_Successfully(Pr [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task InitAsync_WithConsumerGroup_Should_Initialize_Successfully(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -88,13 +84,11 @@ public async Task InitAsync_WithConsumerGroup_Should_Initialize_Successfully(Pro [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task InitAsync_NewClient_Should_Initialize_Successfully(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStreamWithMessages(client, protocol); - var clientAddress = Fixture.GetIggyAddress(protocol); ; + var clientAddress = await Fixture.GetIggyAddressAsync(protocol); var consumer = IggyConsumerBuilder .Create(Identifier.String(testStream.StreamId), @@ -114,9 +108,7 @@ public async Task InitAsync_NewClient_Should_Initialize_Successfully(Protocol pr [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task InitAsync_CalledTwice_Should_NotThrow(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -139,9 +131,7 @@ public async Task InitAsync_CalledTwice_Should_NotThrow(Protocol protocol) [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task ReceiveAsync_WithoutInit_Should_Throw_ConsumerNotInitializedException(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -169,9 +159,7 @@ await Should.ThrowAsync(async () => [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task InitAsync_WithConsumerGroup_Should_CreateGroup_WhenNotExists(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -204,9 +192,7 @@ public async Task InitAsync_WithConsumerGroup_Should_CreateGroup_WhenNotExists(P public async Task InitAsync_WithConsumerGroup_Should_Throw_WhenGroupNotExists_AndAutoCreateDisabled( Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -230,9 +216,7 @@ public async Task InitAsync_WithConsumerGroup_Should_Throw_WhenGroupNotExists_An [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task InitAsync_WithConsumerGroup_Should_JoinGroup_Successfully(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -261,9 +245,7 @@ await client.CreateConsumerGroupAsync(Identifier.String(testStream.StreamId), [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task DisposeAsync_Should_LeaveConsumerGroup(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -294,9 +276,7 @@ public async Task DisposeAsync_Should_LeaveConsumerGroup(Protocol protocol) [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task ReceiveAsync_WithSingleConsumer_Should_ReceiveMessages_Successfully(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -339,9 +319,7 @@ public async Task ReceiveAsync_WithSingleConsumer_Should_ReceiveMessages_Success [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task ReceiveAsync_WithBatchSize_Should_RespectBatchSize(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -380,9 +358,7 @@ public async Task ReceiveAsync_WithBatchSize_Should_RespectBatchSize(Protocol pr [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task ReceiveAsync_WithPollingInterval_Should_RespectInterval(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -422,9 +398,7 @@ public async Task ReceiveAsync_WithPollingInterval_Should_RespectInterval(Protoc [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task ReceiveAsync_WithAutoCommitAfterReceive_Should_StoreOffset(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -472,9 +446,7 @@ public async Task ReceiveAsync_WithAutoCommitAfterReceive_Should_StoreOffset(Pro [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task ReceiveAsync_WithAutoCommitAfterPoll_Should_StoreOffset(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -521,9 +493,7 @@ public async Task ReceiveAsync_WithAutoCommitAfterPoll_Should_StoreOffset(Protoc [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task StoreOffsetAsync_Should_StoreOffset_Successfully(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -560,9 +530,7 @@ public async Task StoreOffsetAsync_Should_StoreOffset_Successfully(Protocol prot [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task DeleteOffsetAsync_Should_DeleteOffset_Successfully(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -605,9 +573,7 @@ public async Task DeleteOffsetAsync_Should_DeleteOffset_Successfully(Protocol pr [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task DisposeAsync_Should_NotThrow(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -629,9 +595,7 @@ public async Task DisposeAsync_Should_NotThrow(Protocol protocol) [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task DisposeAsync_CalledTwice_Should_NotThrow(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -654,9 +618,7 @@ public async Task DisposeAsync_CalledTwice_Should_NotThrow(Protocol protocol) [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task DisposeAsync_WithoutInit_Should_NotThrow(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -677,9 +639,7 @@ public async Task DisposeAsync_WithoutInit_Should_NotThrow(Protocol protocol) [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task OnPollingError_Should_Fire_WhenPollingFails(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -728,9 +688,7 @@ public async Task OnPollingError_Should_Fire_WhenPollingFails(Protocol protocol) [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task ReceiveAsync_WithOffsetStrategy_Should_StartFromOffset(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -767,9 +725,7 @@ public async Task ReceiveAsync_WithOffsetStrategy_Should_StartFromOffset(Protoco [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task ReceiveAsync_WithFirstStrategy_Should_StartFromBeginning(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -801,9 +757,7 @@ public async Task ReceiveAsync_WithFirstStrategy_Should_StartFromBeginning(Proto [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task ReceiveAsync_WithLastStrategy_Should_StartFromEnd(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStreamWithMessages(client, protocol); diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/IggyPublisherTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/IggyPublisherTests.cs index c1028ec0da..c50c1ba603 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/IggyPublisherTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/IggyPublisherTests.cs @@ -35,7 +35,8 @@ public class IggyPublisherTests [ClassDataSource(Shared = SharedType.PerAssembly)] public required IggyServerFixture Fixture { get; init; } - private async Task CreateTestStream(IIggyClient client, Protocol protocol, uint partitionsCount = 5) + private async Task CreateTestStream(IIggyClient client, Protocol protocol, + uint partitionsCount = 5) { var streamId = $"stream_{Guid.NewGuid()}_{protocol.ToString().ToLowerInvariant()}"; var topicId = "test_topic"; @@ -50,9 +51,7 @@ private async Task CreateTestStream(IIggyClient client, Protocol [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task InitAsync_Should_Initialize_Successfully(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStream(client, protocol); @@ -70,7 +69,7 @@ public async Task InitAsync_Should_Initialize_Successfully(Protocol protocol) [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task InitAsync_NewClient_Should_Initialize_Successfully(Protocol protocol) { - var client = Fixture.GetIggyAddress(protocol); + var client = await Fixture.GetIggyAddressAsync(protocol); var stream = Guid.NewGuid().ToString(); var topic = Guid.NewGuid().ToString(); @@ -91,9 +90,7 @@ public async Task InitAsync_NewClient_Should_Initialize_Successfully(Protocol pr [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task InitAsync_CalledTwice_Should_NotThrow(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStream(client, protocol); @@ -111,9 +108,7 @@ public async Task InitAsync_CalledTwice_Should_NotThrow(Protocol protocol) [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task SendMessages_WithoutInit_Should_Throw_PublisherNotInitializedException(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStream(client, protocol); @@ -132,9 +127,7 @@ public async Task SendMessages_WithoutInit_Should_Throw_PublisherNotInitializedE [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task SendMessages_Should_SendMessages_Successfully(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStream(client, protocol); @@ -170,9 +163,7 @@ public async Task SendMessages_Should_SendMessages_Successfully(Protocol protoco [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task SendMessages_WithEmptyList_Should_NotThrow(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStream(client, protocol); @@ -192,9 +183,7 @@ public async Task SendMessages_WithEmptyList_Should_NotThrow(Protocol protocol) [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task InitAsync_WithStreamAutoCreate_Should_CreateStream(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var streamId = $"auto_stream_{Guid.NewGuid()}_{protocol.ToString().ToLowerInvariant()}"; var topicId = "auto_topic"; @@ -221,9 +210,7 @@ public async Task InitAsync_WithStreamAutoCreate_Should_CreateStream(Protocol pr [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task InitAsync_WithTopicAutoCreate_Should_CreateTopic(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var streamId = $"stream_{Guid.NewGuid()}_{protocol.ToString().ToLowerInvariant()}"; var topicId = "auto_topic"; @@ -252,9 +239,7 @@ public async Task InitAsync_WithTopicAutoCreate_Should_CreateTopic(Protocol prot [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task InitAsync_WithoutAutoCreate_Should_Throw_WhenStreamNotExists(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var streamId = $"nonexistent_stream_{Guid.NewGuid()}"; var topicId = "test_topic"; @@ -272,9 +257,7 @@ public async Task InitAsync_WithoutAutoCreate_Should_Throw_WhenStreamNotExists(P [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task InitAsync_WithoutAutoCreate_Should_Throw_WhenTopicNotExists(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var streamId = $"stream_{Guid.NewGuid()}_{protocol.ToString().ToLowerInvariant()}"; var topicId = "nonexistent_topic"; @@ -295,9 +278,7 @@ public async Task InitAsync_WithoutAutoCreate_Should_Throw_WhenTopicNotExists(Pr [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task SendMessages_WithBackgroundSending_Should_SendMessages_Successfully(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStream(client, protocol); @@ -335,9 +316,7 @@ public async Task SendMessages_WithBackgroundSending_Should_SendMessages_Success [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task WaitUntilAllSends_Should_WaitForPendingMessages(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStream(client, protocol); @@ -372,9 +351,7 @@ public async Task WaitUntilAllSends_Should_WaitForPendingMessages(Protocol proto [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task WaitUntilAllSends_WithoutBackgroundSending_Should_ReturnImmediately(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStream(client, protocol); @@ -395,9 +372,7 @@ public async Task WaitUntilAllSends_WithoutBackgroundSending_Should_ReturnImmedi [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task SendMessages_ToMultiplePartitions_Should_DistributeMessages(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStream(client, protocol); @@ -442,9 +417,7 @@ public async Task SendMessages_ToMultiplePartitions_Should_DistributeMessages(Pr [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task SendMessages_WithBalancedPartitioning_Should_DistributeAcrossPartitions(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStream(client, protocol, 3); @@ -486,9 +459,7 @@ public async Task SendMessages_WithBalancedPartitioning_Should_DistributeAcrossP [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task DisposeAsync_Should_NotThrow(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStream(client, protocol); @@ -505,9 +476,7 @@ public async Task DisposeAsync_Should_NotThrow(Protocol protocol) [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task DisposeAsync_CalledTwice_Should_NotThrow(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStream(client, protocol); @@ -525,9 +494,7 @@ public async Task DisposeAsync_CalledTwice_Should_NotThrow(Protocol protocol) [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task DisposeAsync_WithoutInit_Should_NotThrow(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStream(client, protocol); @@ -544,9 +511,7 @@ public async Task DisposeAsync_WithoutInit_Should_NotThrow(Protocol protocol) [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task StreamId_Should_ReturnConfiguredStreamId(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStream(client, protocol); @@ -563,9 +528,7 @@ public async Task StreamId_Should_ReturnConfiguredStreamId(Protocol protocol) [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task TopicId_Should_ReturnConfiguredTopicId(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStream(client, protocol); @@ -582,9 +545,7 @@ public async Task TopicId_Should_ReturnConfiguredTopicId(Protocol protocol) [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task SendMessages_LargeMessageCount_Should_HandleCorrectly(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStream(client, protocol); @@ -622,9 +583,7 @@ public async Task SendMessages_LargeMessageCount_Should_HandleCorrectly(Protocol [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task SendAsync_RentedBatch_WithBackgroundSending_Should_RoundTrip(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStream(client, protocol); @@ -675,9 +634,7 @@ public async Task SendAsync_RentedBatch_WithBackgroundSending_Should_RoundTrip(P [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task SendAsync_SingleMessageRentedBatch_WithBackgroundSending_Should_RoundTrip(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStream(client, protocol); diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTlsConnectionTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTlsConnectionTests.cs index 4b4200a746..2255bbd4f5 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTlsConnectionTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTlsConnectionTests.cs @@ -34,7 +34,7 @@ public async Task Connect_WithTls_Should_Connect_Successfully() { using var client = IggyClientFactory.CreateClient(new IggyClientConfigurator { - BaseAddress = Fixture.GetIggyAddress(Protocol.Tcp), + BaseAddress = await Fixture.GetIggyAddressAsync(Protocol.Tcp), Protocol = Protocol.Tcp, ReconnectionSettings = new ReconnectionSettings { Enabled = false }, AutoLoginSettings = new AutoLoginSettings @@ -62,13 +62,15 @@ public async Task Connect_WithoutTls_Should_Throw_WhenTlsIsRequired() { using var client = IggyClientFactory.CreateClient(new IggyClientConfigurator { - BaseAddress = Fixture.GetIggyAddress(Protocol.Tcp), + BaseAddress = await Fixture.GetIggyAddressAsync(Protocol.Tcp), Protocol = Protocol.Tcp, ReconnectionSettings = new ReconnectionSettings { Enabled = false } }); + // The VSR register handshake runs inside ConnectAsync and dies against the TLS listener, so the + // client never reaches the connected state. await client.ConnectAsync(); - await Should.ThrowAsync(client.LoginUserAsync("iggy", "iggy")); + await Should.ThrowAsync(client.LoginUserAsync("iggy", "iggy")); } [Test] @@ -76,7 +78,7 @@ public async Task Connect_WithTls_CA_Should_Connect_Successfully() { using var client = IggyClientFactory.CreateClient(new IggyClientConfigurator { - BaseAddress = Fixture.GetIggyAddress(Protocol.Tcp), + BaseAddress = await Fixture.GetIggyAddressAsync(Protocol.Tcp), Protocol = Protocol.Tcp, ReconnectionSettings = new ReconnectionSettings { Enabled = false }, AutoLoginSettings = new AutoLoginSettings diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedConsumerTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedConsumerTests.cs index 3861ae812f..460b1c8cef 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedConsumerTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedConsumerTests.cs @@ -38,9 +38,7 @@ public class IggyTypedConsumerTests [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task ReceiveDeserializedAsync_Should_YieldMessages_WithCorrectData(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -77,9 +75,7 @@ public async Task ReceiveDeserializedAsync_Should_YieldMessages_WithCorrectData( public async Task ReceiveDeserializedAsync_WithoutInit_Should_Throw_ConsumerNotInitializedException( Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -101,9 +97,7 @@ await Should.ThrowAsync(async () => [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task ReceiveDeserializedAsync_WithAutoCommitAfterReceive_Should_StoreOffset(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -142,9 +136,7 @@ public async Task ReceiveDeserializedAsync_WithAutoCommitAfterReceive_Should_Sto public async Task ReceiveDeserializedAsync_WithFailingDeserializer_Should_YieldDeserializationFailed( Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStreamWithMessages(client, protocol); @@ -172,9 +164,7 @@ public async Task ReceiveDeserializedAsync_WithFailingDeserializer_Should_YieldD [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task ReceiveDeserializedAsync_Should_StopCleanly_OnCancellation(Protocol protocol) { - var client = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var testStream = await CreateTestStreamWithMessages(client, protocol); diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedPublisherTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedPublisherTests.cs index 64d32e40b4..84c5d50816 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedPublisherTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/IggyTypedPublisherTests.cs @@ -249,9 +249,7 @@ public async Task SendAsync_WithEncryptor_Should_RoundTrip_Decrypted(Protocol pr // Encryption is configured on the client. The publisher uses an encrypting client; a plain client polls // to prove the wire bytes are ciphertext, then decrypts manually. - var encryptingClient = protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient(encryptor: encryptor) - : await Fixture.CreateHttpClient(encryptor: encryptor); + var encryptingClient = await Fixture.CreateAuthenticatedClient(protocol, encryptor: encryptor); var plainClient = await Client(protocol); var stream = await CreateTestStream(plainClient, protocol); @@ -292,9 +290,7 @@ IggyPublisher publisher private async Task Client(Protocol protocol) { - return protocol == Protocol.Tcp - ? await Fixture.CreateTcpClient() - : await Fixture.CreateHttpClient(); + return await Fixture.CreateAuthenticatedClient(protocol); } // Base fluent methods return the non-generic builder, so apply them as statements to keep the typed Build(). diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/MessageEncryptionIntegrationTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/MessageEncryptionIntegrationTests.cs index 8e29ec1ba1..96426ee55f 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/MessageEncryptionIntegrationTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/MessageEncryptionIntegrationTests.cs @@ -195,9 +195,7 @@ private async Task SendBatch(IIggyClient client, Identifier streamId, Identifier private Task CreateClient(Protocol protocol, IMessageEncryptor encryptor) { - return protocol == Protocol.Tcp - ? Fixture.CreateTcpClient(encryptor: encryptor) - : Fixture.CreateHttpClient(encryptor: encryptor); + return Fixture.CreateAuthenticatedClient(protocol, encryptor: encryptor); } private static AesMessageEncryptor CreateEncryptor() diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/OffsetTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/OffsetTests.cs index 82cc193d7b..f96ced773f 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/OffsetTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/OffsetTests.cs @@ -98,15 +98,13 @@ await client.CreateConsumerGroupAsync(Identifier.String(streamName), // For HTTP, a separate TCP client joins (HTTP is stateless and doesn't track membership). if (protocol == Protocol.Tcp) { - await client.JoinConsumerGroupAsync( - Identifier.String(streamName), + await client.JoinConsumerGroupAsync(Identifier.String(streamName), Identifier.String(topicName), Identifier.String("test_consumer_group")); } else { - var tcpClient = await Fixture.CreateTcpClient(); - await tcpClient.JoinConsumerGroupAsync( - Identifier.String(streamName), + var tcpClient = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); + await tcpClient.JoinConsumerGroupAsync(Identifier.String(streamName), Identifier.String(topicName), Identifier.String("test_consumer_group")); } @@ -126,15 +124,13 @@ await client.CreateConsumerGroupAsync(Identifier.String(streamName), if (protocol == Protocol.Tcp) { - await client.JoinConsumerGroupAsync( - Identifier.String(streamName), + await client.JoinConsumerGroupAsync(Identifier.String(streamName), Identifier.String(topicName), Identifier.String("test_consumer_group")); } else { - var tcpClient = await Fixture.CreateTcpClient(); - await tcpClient.JoinConsumerGroupAsync( - Identifier.String(streamName), + var tcpClient = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); + await tcpClient.JoinConsumerGroupAsync(Identifier.String(streamName), Identifier.String(topicName), Identifier.String("test_consumer_group")); } @@ -161,15 +157,13 @@ await client.CreateConsumerGroupAsync(Identifier.String(streamName), if (protocol == Protocol.Tcp) { - await client.JoinConsumerGroupAsync( - Identifier.String(streamName), + await client.JoinConsumerGroupAsync(Identifier.String(streamName), Identifier.String(topicName), Identifier.String("test_consumer_group")); } else { - var tcpClient = await Fixture.CreateTcpClient(); - await tcpClient.JoinConsumerGroupAsync( - Identifier.String(streamName), + var tcpClient = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); + await tcpClient.JoinConsumerGroupAsync(Identifier.String(streamName), Identifier.String(topicName), Identifier.String("test_consumer_group")); } @@ -195,8 +189,7 @@ public async Task DeleteOffset_ConsumerGroup_Should_DeleteOffset_Successfully(Pr await client.CreateConsumerGroupAsync(Identifier.String(streamName), Identifier.String(topicName), "test_consumer_group"); - await client.JoinConsumerGroupAsync( - Identifier.String(streamName), + await client.JoinConsumerGroupAsync(Identifier.String(streamName), Identifier.String(topicName), Identifier.String("test_consumer_group")); await client.StoreOffsetAsync(Consumer.Group("test_consumer_group"), Identifier.String(streamName), diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/PartitionsTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/PartitionsTests.cs index 406b1d3ade..64e281ac48 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/PartitionsTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/PartitionsTests.cs @@ -43,8 +43,7 @@ await Should.NotThrowAsync(() => client.CreatePartitionsAsync(Identifier.String(streamName), Identifier.String(topicName), 3)); - var response = await client.GetTopicByIdAsync( - Identifier.String(streamName), Identifier.String(topicName)); + var response = await client.GetTopicByIdAsync(Identifier.String(streamName), Identifier.String(topicName)); response.ShouldNotBeNull(); response.PartitionsCount.ShouldBe(4u); } @@ -65,8 +64,7 @@ await Should.NotThrowAsync(() => client.DeletePartitionsAsync(Identifier.String(streamName), Identifier.String(topicName), 1)); - var response = await client.GetTopicByIdAsync( - Identifier.String(streamName), Identifier.String(topicName)); + var response = await client.GetTopicByIdAsync(Identifier.String(streamName), Identifier.String(topicName)); response.ShouldNotBeNull(); response.PartitionsCount.ShouldBe(3u); } diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/PersonalAccessTokenTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/PersonalAccessTokenTests.cs index 2e14ff637a..b8ffc39b91 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/PersonalAccessTokenTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/PersonalAccessTokenTests.cs @@ -87,7 +87,7 @@ public async Task LoginWithPersonalAccessToken_Should_Be_Successfully(Protocol p var name = $"lgn-{Guid.NewGuid():N}"[..20]; var response = await client.CreatePersonalAccessTokenAsync(name, Expiry); - var loginClient = await Fixture.CreateClient(protocol); + var loginClient = await Fixture.CreateClient(protocol, true); var authResponse = await loginClient.LoginWithPersonalAccessTokenAsync(response!.Token); authResponse.ShouldNotBeNull(); diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/RawCommandTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/RawCommandTests.cs index 49d156fe42..1cbfcd059c 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/RawCommandTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/RawCommandTests.cs @@ -51,8 +51,9 @@ public async Task SendBinaryRequest_Tcp_ShouldRejectSessionControlCodes(Protocol foreach (var code in new uint[] { 38, 39, 40, 44, 45 }) { - var exception = await Should.ThrowAsync( - () => client.SendBinaryRequestAsync(code, [])); + var exception + = await Should.ThrowAsync(() => + client.SendBinaryRequestAsync(code, [])); exception.StatusCode.ShouldBe(3); } } @@ -64,8 +65,8 @@ public async Task SendBinaryRequest_Tcp_ShouldPropagateServerError(Protocol prot { var client = await Fixture.CreateAuthenticatedClient(protocol); - var exception = await Should.ThrowAsync( - () => client.SendBinaryRequestAsync(60_000, [])); + var exception + = await Should.ThrowAsync(() => client.SendBinaryRequestAsync(60_000, [])); exception.StatusCode.ShouldBe(3); } @@ -77,7 +78,6 @@ public async Task SendBinaryRequest_Http_ShouldThrowFeatureUnavailable(Protocol { var client = await Fixture.CreateAuthenticatedClient(protocol); - await Should.ThrowAsync( - () => client.SendBinaryRequestAsync(1, [])); + await Should.ThrowAsync(() => client.SendBinaryRequestAsync(1, [])); } } diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/SegmentsTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/SegmentsTests.cs index eed1d01b0a..64a80a3905 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/SegmentsTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/SegmentsTests.cs @@ -42,8 +42,7 @@ public async Task DeleteSegments_WithZeroCount_Should_Succeed(Protocol protocol) // Deleting 0 segments should succeed without error (no-op) await Should.NotThrowAsync(() => - client.DeleteSegmentsAsync( - Identifier.String(streamName), + client.DeleteSegmentsAsync(Identifier.String(streamName), Identifier.String(topicName), 0, // partition_id (0-indexed) 0)); // segments_count = 0 @@ -62,8 +61,7 @@ public async Task DeleteSegments_Http_Should_Throw_FeatureUnavailable(Protocol p await client.CreateTopicAsync(Identifier.String(streamName), topicName, 1); await Should.ThrowAsync(() => - client.DeleteSegmentsAsync( - Identifier.String(streamName), + client.DeleteSegmentsAsync(Identifier.String(streamName), Identifier.String(topicName), 0, 0)); @@ -80,8 +78,7 @@ public async Task DeleteSegments_Should_Throw_WhenTopic_DoesNotExist(Protocol pr await client.CreateStreamAsync(streamName); await Should.ThrowAsync(() => - client.DeleteSegmentsAsync( - Identifier.String(streamName), + client.DeleteSegmentsAsync(Identifier.String(streamName), Identifier.String("non-existent-topic"), 0, // partition_id (0-indexed) 1)); // segments_count @@ -95,8 +92,7 @@ public async Task DeleteSegments_Should_Throw_WhenStream_DoesNotExist(Protocol p var client = await Fixture.CreateAuthenticatedClient(protocol); await Should.ThrowAsync(() => - client.DeleteSegmentsAsync( - Identifier.String($"nonexistent-stream-{Guid.NewGuid():N}"), + client.DeleteSegmentsAsync(Identifier.String($"nonexistent-stream-{Guid.NewGuid():N}"), Identifier.String("any-topic"), 0, // partition_id (0-indexed) 1)); // segments_count diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/SendMessagesTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/SendMessagesTests.cs index db109bfbdb..085f4d9b04 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/SendMessagesTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/SendMessagesTests.cs @@ -16,10 +16,12 @@ // under the License. using System.Text; +using Apache.Iggy.Contracts; using Apache.Iggy.Enums; using Apache.Iggy.Exceptions; using Apache.Iggy.Headers; using Apache.Iggy.IggyClient; +using Apache.Iggy.Kinds; using Apache.Iggy.Messages; using Apache.Iggy.Tests.Integrations.Fixtures; using Shouldly; @@ -41,20 +43,6 @@ public class SendMessagesTests [ClassDataSource(Shared = SharedType.PerAssembly)] public required IggyServerFixture Fixture { get; init; } - private async Task<(IIggyClient client, string streamName, string topicName)> CreateStreamAndTopic( - Protocol protocol) - { - var client = await Fixture.CreateAuthenticatedClient(protocol); - - var streamName = $"send-msg-{Guid.NewGuid():N}"; - var topicName = "test-topic"; - - await client.CreateStreamAsync(streamName); - await client.CreateTopicAsync(Identifier.String(streamName), topicName, 1); - - return (client, streamName, topicName); - } - [Test] [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task SendMessages_NoHeaders_Should_SendMessages_Successfully(Protocol protocol) @@ -142,4 +130,131 @@ await Should.ThrowAsync(() => client.SendMessagesAsync(Identifier.String(streamName), Identifier.Numeric(69), Partitioning.None(), messages)); } + + [Test] + [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] + public async Task SendMessages_Should_ReturnConfirmation_WithStreamTopicPartitionAndBaseOffset(Protocol protocol) + { + var context = await CreateStreamAndTopicWithIds(protocol, 4); + + var response = await SendBatchAsync(context.Client, context.StreamName, context.TopicName, + Partitioning.PartitionId(2), 2); + + var confirmation = response.Confirmations.ShouldHaveSingleItem(); + confirmation.StreamId.ShouldBe(context.StreamId); + confirmation.TopicId.ShouldBe(context.TopicId); + confirmation.PartitionId.ShouldBe(2u); + confirmation.BaseOffset.ShouldBe(0u); + } + + [Test] + [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] + public async Task SendMessages_Consecutive_Should_AdvanceBaseOffset_ByBatchSize(Protocol protocol) + { + var (client, streamName, topicName) = await CreateStreamAndTopic(protocol); + + var first = await SendBatchAsync(client, streamName, topicName, Partitioning.PartitionId(0), 3); + var second = await SendBatchAsync(client, streamName, topicName, Partitioning.PartitionId(0), 2); + var third = await SendBatchAsync(client, streamName, topicName, Partitioning.PartitionId(0), 1); + + first.Confirmations.ShouldHaveSingleItem().BaseOffset.ShouldBe(0u); + second.Confirmations.ShouldHaveSingleItem().BaseOffset.ShouldBe(3u); + third.Confirmations.ShouldHaveSingleItem().BaseOffset.ShouldBe(5u); + } + + [Test] + [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] + public async Task SendMessages_Confirmation_Should_MatchPolledMessageOffsets(Protocol protocol) + { + var (client, streamName, topicName) = await CreateStreamAndTopic(protocol); + + await SendBatchAsync(client, streamName, topicName, Partitioning.PartitionId(0), 4); + var response = await SendBatchAsync(client, streamName, topicName, Partitioning.PartitionId(0), 3); + var confirmation = response.Confirmations.ShouldHaveSingleItem(); + + var polled = await PollAsync(client, streamName, topicName, 0, + PollingStrategy.Offset(confirmation.BaseOffset)); + + polled.Messages.Count.ShouldBe(3); + polled.Messages[0].Header.Offset.ShouldBe(confirmation.BaseOffset); + } + + /// + /// Balanced partitioning is resolved client-side, so the confirmation is the only place the caller + /// learns where the batch landed. It must name the partition the messages actually poll back from. + /// + [Test] + [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] + public async Task SendMessages_Balanced_Confirmation_Should_NameThePartition_MessagesLandedIn(Protocol protocol) + { + const uint partitionsCount = 4; + var context = await CreateStreamAndTopicWithIds(protocol, partitionsCount); + + var response = await SendBatchAsync(context.Client, context.StreamName, context.TopicName, + Partitioning.None(), 2); + + var confirmation = response.Confirmations.ShouldHaveSingleItem(); + confirmation.PartitionId.ShouldBeLessThan(partitionsCount); + + var polled = await PollAsync(context.Client, context.StreamName, context.TopicName, confirmation.PartitionId, + PollingStrategy.Offset(0)); + polled.Messages.Count.ShouldBe(2); + polled.Messages[0].Header.Offset.ShouldBe(confirmation.BaseOffset); + } + + private static Task SendBatchAsync(IIggyClient client, string streamName, string topicName, + Partitioning partitioning, int count) + { + Message[] messages = Enumerable.Range(0, count) + .Select(index => new Message(Guid.NewGuid(), Encoding.UTF8.GetBytes($"confirm-payload-{index}"))) + .ToArray(); + + return client.SendMessagesAsync(Identifier.String(streamName), Identifier.String(topicName), partitioning, + messages); + } + + private static Task PollAsync(IIggyClient client, string streamName, string topicName, + uint partitionId, PollingStrategy strategy) + { + return client.PollMessagesAsync(new MessageFetchRequest + { + Count = 100, + AutoCommit = false, + Consumer = Consumer.New(1), + PartitionId = partitionId, + PollingStrategy = strategy, + StreamId = Identifier.String(streamName), + TopicId = Identifier.String(topicName) + }); + } + + private async Task<(IIggyClient client, string streamName, string topicName)> CreateStreamAndTopic( + Protocol protocol) + { + var context = await CreateStreamAndTopicWithIds(protocol); + + return (context.Client, context.StreamName, context.TopicName); + } + + private async Task CreateStreamAndTopicWithIds(Protocol protocol, uint partitionsCount = 1) + { + var client = await Fixture.CreateAuthenticatedClient(protocol); + + var streamName = $"send-msg-{Guid.NewGuid():N}"; + var topicName = "test-topic"; + + var stream = await client.CreateStreamAsync(streamName); + stream.ShouldNotBeNull(); + var topic = await client.CreateTopicAsync(Identifier.String(streamName), topicName, partitionsCount); + topic.ShouldNotBeNull(); + + return new StreamTopicContext(client, streamName, topicName, stream.Id, topic.Id); + } + + private sealed record StreamTopicContext( + IIggyClient Client, + string StreamName, + string TopicName, + uint StreamId, + uint TopicId); } diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/StreamsTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/StreamsTests.cs index aa47d00c8b..41d762bbf6 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/StreamsTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/StreamsTests.cs @@ -20,6 +20,7 @@ using Apache.Iggy.Exceptions; using Apache.Iggy.Messages; using Apache.Iggy.Tests.Integrations.Fixtures; +using Apache.Iggy.Tests.Integrations.Helpers; using Shouldly; using Partitioning = Apache.Iggy.Kinds.Partitioning; @@ -216,7 +217,9 @@ await client.SendMessagesAsync(Identifier.String(streamName), await Should.NotThrowAsync(() => client.PurgeStreamAsync(Identifier.String(streamName))); - stream = await client.GetStreamByIdAsync(Identifier.String(streamName)); + // The server commits the purge by advancing a generation its reconciler acts on a tick later. + stream = await Eventually.ReadAsync(() => client.GetStreamByIdAsync(Identifier.String(streamName)), + purged => purged?.MessagesCount == 0, TimeSpan.FromSeconds(10)); stream.ShouldNotBeNull(); stream.MessagesCount.ShouldBe(0u); stream.TopicsCount.ShouldBe(1); diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/SystemTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/SystemTests.cs index 3d803b4cc8..dd2b26ab94 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/SystemTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/SystemTests.cs @@ -55,8 +55,7 @@ public async Task GetClient_Should_Return_CorrectClient(Protocol protocol) { var client = await Fixture.CreateAuthenticatedClient(protocol); - var tcpClient = await Fixture.CreateClient(Protocol.Tcp); - await tcpClient.LoginUserAsync("iggy", "iggy"); + var tcpClient = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); var clientInfo = await tcpClient.GetMeAsync(); clientInfo.ShouldNotBeNull(); @@ -76,7 +75,7 @@ public async Task GetClient_Should_Return_CorrectClient(Protocol protocol) [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task GetMe_Tcp_Should_Return_MyClient(Protocol protocol) { - var client = await Fixture.CreateTcpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); var me = await client.GetMeAsync(); me.ShouldNotBeNull(); @@ -91,7 +90,7 @@ public async Task GetMe_Tcp_Should_Return_MyClient(Protocol protocol) [MethodDataSource(nameof(IggyServerFixture.ProtocolData))] public async Task GetMe_HTTP_Should_Throw_FeatureUnavailableException(Protocol protocol) { - var client = await Fixture.CreateHttpClient(); + var client = await Fixture.CreateAuthenticatedClient(protocol); await Should.ThrowAsync(() => client.GetMeAsync()); } @@ -103,8 +102,7 @@ public async Task GetClient_WithConsumerGroup_Should_Return_CorrectClient(Protoc var client = await Fixture.CreateAuthenticatedClient(protocol); var streamName = $"sys-cg-{Guid.NewGuid():N}"; - var tcpClient = await Fixture.CreateClient(Protocol.Tcp); - await tcpClient.LoginUserAsync("iggy", "iggy"); + var tcpClient = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); var stream = await tcpClient.CreateStreamAsync(streamName); await tcpClient.CreateTopicAsync(Identifier.String(streamName), "first_topic", 2); @@ -159,7 +157,7 @@ await client.SendMessagesAsync(Identifier.String(streamName), response.PartitionsCount.ShouldBeGreaterThanOrEqualTo(1); response.SegmentsCount.ShouldBeGreaterThanOrEqualTo(1); response.MessagesCount.ShouldBeGreaterThanOrEqualTo(1u); - response.ClientsCount.ShouldBeGreaterThanOrEqualTo(1); + // iggy-server leaves the connected-client tally out of its stats reply, so ClientsCount goes unchecked. response.Hostname.ShouldNotBeNullOrEmpty(); response.OsName.ShouldNotBeNullOrEmpty(); response.OsVersion.ShouldNotBeNullOrEmpty(); diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/TopicsTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/TopicsTests.cs index fa6a57796b..88d2c23f36 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/TopicsTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/TopicsTests.cs @@ -21,6 +21,7 @@ using Apache.Iggy.Exceptions; using Apache.Iggy.Messages; using Apache.Iggy.Tests.Integrations.Fixtures; +using Apache.Iggy.Tests.Integrations.Helpers; using Shouldly; using Partitioning = Apache.Iggy.Kinds.Partitioning; @@ -40,8 +41,8 @@ public async Task Create_NewTopic_Should_Return_Successfully(Protocol protocol) var streamName = $"topic-create-{Guid.NewGuid():N}"; await client.CreateStreamAsync(streamName); - var response = await client.CreateTopicAsync( - Identifier.String(streamName), "Test Topic", 2, CompressionAlgorithm.Gzip, + var response = await client.CreateTopicAsync(Identifier.String(streamName), "Test Topic", 2, + CompressionAlgorithm.Gzip, 1, TimeSpan.FromMinutes(10), 2_000_000_000); response.ShouldNotBeNull(); @@ -68,8 +69,8 @@ public async Task Create_DuplicateTopic_Should_Throw_InvalidResponse(Protocol pr await client.CreateStreamAsync(streamName); await client.CreateTopicAsync(Identifier.String(streamName), "Dup Topic", 1); - await Should.ThrowAsync( - client.CreateTopicAsync(Identifier.String(streamName), "Dup Topic", 1)); + await Should.ThrowAsync(client.CreateTopicAsync(Identifier.String(streamName), + "Dup Topic", 1)); } [Test] @@ -197,13 +198,11 @@ public async Task Update_ExistingTopic_Should_UpdateTopic_Successfully(Protocol var topicToUpdate = await client.CreateTopicAsync(Identifier.String(streamName), "topic-to-update", 1); topicToUpdate.ShouldNotBeNull(); - await Should.NotThrowAsync(client.UpdateTopicAsync( - Identifier.String(streamName), + await Should.NotThrowAsync(client.UpdateTopicAsync(Identifier.String(streamName), Identifier.Numeric(topicToUpdate.Id), "Updated Topic", CompressionAlgorithm.Gzip, 3_000_000_000, TimeSpan.FromMinutes(10), 3)); - var result = await client.GetTopicByIdAsync( - Identifier.String(streamName), + var result = await client.GetTopicByIdAsync(Identifier.String(streamName), Identifier.Numeric(topicToUpdate.Id)); result.ShouldNotBeNull(); result!.Name.ShouldBe("Updated Topic"); @@ -232,11 +231,13 @@ await client.SendMessagesAsync(Identifier.String(streamName), beforePurge.MessagesCount.ShouldBe(5u); beforePurge.Size.ShouldBeGreaterThan(0u); - await Should.NotThrowAsync(client.PurgeTopicAsync( - Identifier.String(streamName), Identifier.String("Purge Topic"))); + await Should.NotThrowAsync(client.PurgeTopicAsync(Identifier.String(streamName), + Identifier.String("Purge Topic"))); - var afterPurge = await client.GetTopicByIdAsync(Identifier.String(streamName), - Identifier.String("Purge Topic")); + // The server commits the purge by advancing a generation its reconciler acts on a tick later. + var afterPurge = await Eventually.ReadAsync( + () => client.GetTopicByIdAsync(Identifier.String(streamName), Identifier.String("Purge Topic")), + topic => topic?.MessagesCount == 0, TimeSpan.FromSeconds(10)); afterPurge.ShouldNotBeNull(); afterPurge!.MessagesCount.ShouldBe(0u); afterPurge.Size.ShouldBe(0u); @@ -253,8 +254,8 @@ public async Task Delete_ExistingTopic_Should_DeleteTopic_Successfully(Protocol var topicToDelete = await client.CreateTopicAsync(Identifier.String(streamName), "topic-to-delete", 1); topicToDelete.ShouldNotBeNull(); - await Should.NotThrowAsync(client.DeleteTopicAsync( - Identifier.String(streamName), Identifier.Numeric(topicToDelete.Id))); + await Should.NotThrowAsync(client.DeleteTopicAsync(Identifier.String(streamName), + Identifier.Numeric(topicToDelete.Id))); } [Test] @@ -266,8 +267,8 @@ public async Task Delete_NonExistingTopic_Should_Throw_InvalidResponse(Protocol var streamName = $"topic-delnone-{Guid.NewGuid():N}"; await client.CreateStreamAsync(streamName); - await Should.ThrowAsync(client.DeleteTopicAsync( - Identifier.String(streamName), Identifier.String("nonexistent-topic"))); + await Should.ThrowAsync(client.DeleteTopicAsync(Identifier.String(streamName), + Identifier.String("nonexistent-topic"))); } [Test] @@ -279,8 +280,8 @@ public async Task Get_NonExistingTopic_Should_Throw_InvalidResponse(Protocol pro var streamName = $"topic-getnone-{Guid.NewGuid():N}"; await client.CreateStreamAsync(streamName); - var topic = await client.GetTopicByIdAsync( - Identifier.String(streamName), Identifier.String("nonexistent-topic")); + var topic = await client.GetTopicByIdAsync(Identifier.String(streamName), + Identifier.String("nonexistent-topic")); topic.ShouldBeNull(); } diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/UsersTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/UsersTests.cs index c3820a3079..3c86131b29 100644 --- a/foreign/csharp/Iggy_SDK.Tests.Integration/UsersTests.cs +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/UsersTests.cs @@ -17,7 +17,6 @@ using Apache.Iggy.Contracts; using Apache.Iggy.Contracts.Auth; -using Apache.Iggy.Contracts.Http.Auth; using Apache.Iggy.Enums; using Apache.Iggy.Exceptions; using Apache.Iggy.Tests.Integrations.Fixtures; @@ -54,8 +53,8 @@ public async Task CreateUser_Duplicate_Should_Throw_InvalidResponse(Protocol pro var username = $"dup-{Guid.NewGuid():N}"[..20]; await client.CreateUserAsync(username, "test1", UserStatus.Active); - await Should.ThrowAsync( - client.CreateUserAsync(username, "test1", UserStatus.Active)); + await Should.ThrowAsync(client.CreateUserAsync(username, "test1", + UserStatus.Active)); } [Test] @@ -169,10 +168,11 @@ public async Task ChangePassword_Should_ChangePassword_Successfully(Protocol pro var username = $"chpw-{Guid.NewGuid():N}"[..20]; await client.CreateUserAsync(username, "old_password", UserStatus.Active); - await Should.NotThrowAsync(client.ChangePasswordAsync(Identifier.String(username), "old_password", "new_password")); + await Should.NotThrowAsync(client.ChangePasswordAsync(Identifier.String(username), "old_password", + "new_password")); // Verify password was actually changed by logging in with the new credentials - var loginClient = await Fixture.CreateClient(protocol); + var loginClient = await Fixture.CreateClient(protocol, true); var loginResponse = await loginClient.LoginUserAsync(username, "new_password"); loginResponse.ShouldNotBeNull(); loginResponse.UserId.ShouldBeGreaterThan(0); @@ -187,8 +187,8 @@ public async Task ChangePassword_WrongCurrentPassword_Should_Throw_InvalidRespon var username = $"chpwf-{Guid.NewGuid():N}"[..20]; await client.CreateUserAsync(username, "correct_password", UserStatus.Active); - await Should.ThrowAsync( - client.ChangePasswordAsync(Identifier.String(username), "wrong_password", "new_password")); + await Should.ThrowAsync(client.ChangePasswordAsync(Identifier.String(username), + "wrong_password", "new_password")); } [Test] @@ -200,7 +200,7 @@ public async Task LoginUser_Should_LoginUser_Successfully(Protocol protocol) var username = $"login-{Guid.NewGuid():N}"[..20]; await client.CreateUserAsync(username, "login_password", UserStatus.Active); - var loginClient = await Fixture.CreateClient(protocol); + var loginClient = await Fixture.CreateClient(protocol, true); var response = await loginClient.LoginUserAsync(username, "login_password"); response.ShouldNotBeNull(); diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/Vsr/VsrConsumerGroupTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/Vsr/VsrConsumerGroupTests.cs new file mode 100644 index 0000000000..5731c84abc --- /dev/null +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/Vsr/VsrConsumerGroupTests.cs @@ -0,0 +1,228 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Text; +using Apache.Iggy.Contracts; +using Apache.Iggy.Enums; +using Apache.Iggy.Exceptions; +using Apache.Iggy.IggyClient; +using Apache.Iggy.Kinds; +using Apache.Iggy.Messages; +using Apache.Iggy.Tests.Integrations.Fixtures; +using Shouldly; +using Partitioning = Apache.Iggy.Kinds.Partitioning; + +namespace Apache.Iggy.Tests.Integrations.Vsr; + +/// +/// Group polls under VSR: the server hands out an assignment, the client caches it and round-robins the +/// assigned partitions itself, so a poll without an explicit partition id never reaches the broker as one. +/// +public class VsrConsumerGroupTests +{ + private const uint PartitionsCount = 3; + private const string TopicName = "vsr-group-topic"; + + [ClassDataSource(Shared = SharedType.PerAssembly)] + public required IggyServerFixture Fixture { get; init; } + + [Test] + public async Task GroupPoll_Should_Drain_EveryAssignedPartition() + { + var (client, streamName, groupName) = await CreateGroup(); + await client.JoinConsumerGroupAsync(Identifier.String(streamName), Identifier.String(TopicName), + Identifier.String(groupName)); + + for (uint partitionId = 0; partitionId < PartitionsCount; partitionId++) + { + await SendAsync(client, streamName, partitionId); + } + + // One poll per assigned partition drains it; the sole member owns them all, so the round-robin has to + // hand out every partition exactly once before it wraps. + var polled = await DrainGroupAsync(client, streamName, groupName, PartitionsCount); + + polled.ShouldBe((int)PartitionsCount); + } + + [Test] + public async Task GroupPoll_Without_Joining_Should_Throw_MemberNotFound() + { + var (client, streamName, groupName) = await CreateGroup(); + + var exception = await Should.ThrowAsync( + PollGroupAsync(client, streamName, groupName)); + + exception.StatusCode.ShouldBe(5006); + } + + [Test] + public async Task GroupPoll_After_Leaving_Should_Throw_MemberNotFound() + { + var (client, streamName, groupName) = await CreateGroup(); + await client.JoinConsumerGroupAsync(Identifier.String(streamName), Identifier.String(TopicName), + Identifier.String(groupName)); + await PollGroupAsync(client, streamName, groupName); + + await client.LeaveConsumerGroupAsync(Identifier.String(streamName), Identifier.String(TopicName), + Identifier.String(groupName)); + + var exception = await Should.ThrowAsync( + PollGroupAsync(client, streamName, groupName)); + + exception.StatusCode.ShouldBe(5006); + } + + /// + /// A partition count change widens the assignment, and the ping is where the client re-syncs it. The + /// cached generation is asserted first: without it the test would pass on a client that re-synced on + /// every poll and never needed the heartbeat. + /// + [Test] + public async Task Ping_Should_Refresh_TheGroupAssignment_After_PartitionsAreAdded() + { + var (client, streamName, groupName) = await CreateGroup(); + await client.JoinConsumerGroupAsync(Identifier.String(streamName), Identifier.String(TopicName), + Identifier.String(groupName)); + await PollGroupAsync(client, streamName, groupName); + + await client.CreatePartitionsAsync(Identifier.String(streamName), Identifier.String(TopicName), 1); + await SendAsync(client, streamName, PartitionsCount); + + (await DrainGroupAsync(client, streamName, groupName, PartitionsCount + 1)).ShouldBe(0); + + await client.PingAsync(); + + (await DrainGroupAsync(client, streamName, groupName, PartitionsCount + 1)).ShouldBe(1); + } + + /// + /// A member holding no partitions is still a member, so its poll has to come back empty instead of + /// surfacing the not-a-member error the unassigned cursor otherwise looks like. One partition and two + /// members guarantees exactly one of them is in that state. + /// + [Test] + public async Task GroupPoll_By_AMemberWithoutPartitions_Should_ReturnEmpty() + { + var (first, streamName, groupName) = await CreateGroup(); + var topicName = $"vsr-single-partition-{Guid.NewGuid():N}"; + await first.CreateTopicAsync(Identifier.String(streamName), topicName, 1); + await first.CreateConsumerGroupAsync(Identifier.String(streamName), Identifier.String(topicName), groupName); + + var second = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); + await JoinAsync(first, streamName, topicName, groupName); + await JoinAsync(second, streamName, topicName, groupName); + + await first.SendMessagesAsync(Identifier.String(streamName), Identifier.String(topicName), + Partitioning.PartitionId(0), + [new Message(Guid.NewGuid(), Encoding.UTF8.GetBytes("vsr-single-partition-payload"))]); + + var firstPoll = await PollGroupAsync(first, streamName, topicName, groupName); + var secondPoll = await PollGroupAsync(second, streamName, topicName, groupName); + + // Whichever member drew the partition drains it, and the other one has nothing to poll. + (firstPoll.Messages.Count + secondPoll.Messages.Count).ShouldBe(1); + Math.Min(firstPoll.Messages.Count, secondPoll.Messages.Count).ShouldBe(0); + } + + /// + /// A second member rebalances the group, which leaves the first one round-robining partitions it no + /// longer owns. The fence has to re-sync the assignment underneath the poll: a client that surfaced the + /// ownership error instead would break every group app that did not special-case it. + /// + [Test] + public async Task GroupPoll_After_ASecondMemberJoins_Should_ResyncTheStaleAssignment() + { + var (first, streamName, groupName) = await CreateGroup(); + await JoinAsync(first, streamName, TopicName, groupName); + await DrainGroupAsync(first, streamName, groupName, PartitionsCount); + + var second = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); + await JoinAsync(second, streamName, TopicName, groupName); + + for (uint partitionId = 0; partitionId < PartitionsCount; partitionId++) + { + await SendAsync(first, streamName, partitionId); + } + + // The first member still holds the pre-rebalance assignment, so its polls fence until they re-sync. + // Between them the two members have to see every partition; a lost one means a fence was swallowed. + var drained = await DrainGroupAsync(first, streamName, groupName, PartitionsCount); + drained += await DrainGroupAsync(second, streamName, groupName, PartitionsCount); + + drained.ShouldBe((int)PartitionsCount); + } + + private static Task JoinAsync(IIggyClient client, string streamName, string topicName, string groupName) + { + return client.JoinConsumerGroupAsync(Identifier.String(streamName), Identifier.String(topicName), + Identifier.String(groupName)); + } + + private async Task<(IIggyClient Client, string StreamName, string GroupName)> CreateGroup() + { + var client = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); + var streamName = $"vsr-group-{Guid.NewGuid():N}"; + var groupName = $"vsr-group-name-{Guid.NewGuid():N}"; + + await client.CreateStreamAsync(streamName); + await client.CreateTopicAsync(Identifier.String(streamName), TopicName, PartitionsCount); + await client.CreateConsumerGroupAsync(Identifier.String(streamName), Identifier.String(TopicName), groupName); + + return (client, streamName, groupName); + } + + private static Task SendAsync(IIggyClient client, string streamName, uint partitionId) + { + return client.SendMessagesAsync(Identifier.String(streamName), Identifier.String(TopicName), + Partitioning.PartitionId((int)partitionId), + [new Message(Guid.NewGuid(), Encoding.UTF8.GetBytes($"vsr-group-payload-{partitionId}"))]); + } + + /// Polls once per assigned partition and returns how many messages came back in total. + private static async Task DrainGroupAsync(IIggyClient client, string streamName, string groupName, + uint polls) + { + var drained = 0; + for (var poll = 0; poll < polls; poll++) + { + drained += (await PollGroupAsync(client, streamName, groupName)).Messages.Count; + } + + return drained; + } + + private static Task PollGroupAsync(IIggyClient client, string streamName, string groupName) + { + return PollGroupAsync(client, streamName, TopicName, groupName); + } + + private static Task PollGroupAsync(IIggyClient client, string streamName, string topicName, + string groupName) + { + return client.PollMessagesAsync(new MessageFetchRequest + { + Count = 10, + AutoCommit = true, + Consumer = Consumer.Group(groupName), + PartitionId = null, + PollingStrategy = PollingStrategy.Next(), + StreamId = Identifier.String(streamName), + TopicId = Identifier.String(topicName) + }); + } +} diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/Vsr/VsrHandshakeTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/Vsr/VsrHandshakeTests.cs new file mode 100644 index 0000000000..6f77ae26c5 --- /dev/null +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/Vsr/VsrHandshakeTests.cs @@ -0,0 +1,164 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Apache.Iggy.Contracts; +using Apache.Iggy.Enums; +using Apache.Iggy.Exceptions; +using Apache.Iggy.Tests.Integrations.Fixtures; +using Shouldly; + +namespace Apache.Iggy.Tests.Integrations.Vsr; + +/// +/// The register handshake and the session it binds. Every other VSR suite depends on this one passing: +/// without a bound session the server fences every replicated request. +/// +public class VsrHandshakeTests +{ + [ClassDataSource(Shared = SharedType.PerAssembly)] + public required IggyServerFixture Fixture { get; init; } + + [Test] + public async Task Login_Should_BindSession_And_ServeReplicatedRequests() + { + var client = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); + + var name = $"vsr-handshake-{Guid.NewGuid():N}"; + var stream = await client.CreateStreamAsync(name); + + stream.ShouldNotBeNull(); + stream.Name.ShouldBe(name); + } + + /// + /// A re-login first logs out the bound session, then registers a fresh client identity. The metadata write + /// after re-login proves the request counter belongs to that new binding instead of replaying a cached + /// response from the old client table entry. + /// + [Test] + public async Task ReLogin_Should_AllowReplicatedRequests() + { + var client = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); + var name = $"vsr-relogin-{Guid.NewGuid():N}"; + await client.CreateStreamAsync(name); + + var response = await client.LoginUserAsync("iggy", "iggy"); + response.ShouldNotBeNull(); + + var afterRelogin = $"vsr-relogin-after-{Guid.NewGuid():N}"; + var stream = await client.CreateStreamAsync(afterRelogin); + stream.ShouldNotBeNull(); + stream.Name.ShouldBe(afterRelogin); + } + + [Test] + public async Task Logout_Should_UnbindTheSession_Until_TheClientRegistersAgain() + { + var client = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); + await client.LogoutUserAsync(); + + await Should.ThrowAsync(client.CreateStreamAsync($"vsr-logout-{Guid.NewGuid():N}")); + + await client.LoginUserAsync("iggy", "iggy"); + + var name = $"vsr-logout-relogin-{Guid.NewGuid():N}"; + (await client.CreateStreamAsync(name)).ShouldNotBeNull(); + } + + /// + /// A rejected register is a consumed one on the server, so the failure has to reset the session and + /// unwind the connection state. The retry with valid credentials is the assertion that matters: a client + /// left in Authenticating with the failed register's session still bound would fence it. + /// Wrong credentials come back as an InvalidCredentials eviction, not as the empty register reply, + /// and an eviction is terminal for the connection: the retry has to reconnect first. + /// + [Test] + public async Task Login_WithInvalidCredentials_Should_ResetTheSession_And_AllowARetry() + { + var client = await Fixture.CreateUnauthenticatedClient(Protocol.Tcp); + + var exception = await Should.ThrowAsync( + client.LoginUserAsync("iggy", "not-the-password")); + + exception.StatusCode.ShouldBe(42); + exception.Message.ShouldContain("Invalid credentials"); + + await client.ConnectAsync(); + (await client.LoginUserAsync("iggy", "iggy")).ShouldNotBeNull(); + + var name = $"vsr-failed-login-{Guid.NewGuid():N}"; + (await client.CreateStreamAsync(name)).ShouldNotBeNull(); + } + + [Test] + public async Task LoginWithPersonalAccessToken_Should_BindTheSession() + { + var client = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); + var token = await client.CreatePersonalAccessTokenAsync($"vsr-pat-{Guid.NewGuid():N}"); + token.ShouldNotBeNull(); + + var patClient = await Fixture.CreateUnauthenticatedClient(Protocol.Tcp); + var response = await patClient.LoginWithPersonalAccessTokenAsync(token.Token); + + response.ShouldNotBeNull(); + + var name = $"vsr-pat-stream-{Guid.NewGuid():N}"; + (await patClient.CreateStreamAsync(name)).ShouldNotBeNull(); + } + + [Test] + public async Task Ping_Should_Succeed_Before_TheSessionIsBound() + { + var client = await Fixture.CreateUnauthenticatedClient(Protocol.Tcp); + + // Non-replicated ops are sessionless, so an unbound client still pings. + await client.PingAsync(); + + await Should.ThrowAsync(client.CreateStreamAsync($"vsr-unbound-{Guid.NewGuid():N}")); + } + + [Test] + public async Task Ping_Should_Ride_NonReplicated_Without_GappingTheRequestCounter() + { + var client = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); + + // A ping that consumed a request id would gap the next metadata request, and the primary silently + // drops a gapped one - the create below would hang instead of failing loudly. + await client.PingAsync(); + var first = await client.CreateStreamAsync($"vsr-ping-first-{Guid.NewGuid():N}"); + await client.PingAsync(); + var second = await client.CreateStreamAsync($"vsr-ping-second-{Guid.NewGuid():N}"); + + first.ShouldNotBeNull(); + second.ShouldNotBeNull(); + } + + [Test] + public async Task Reads_Should_Ride_NonReplicated_Between_MetadataWrites() + { + var client = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); + + var name = $"vsr-reads-{Guid.NewGuid():N}"; + await client.CreateStreamAsync(name); + + IReadOnlyList streams = await client.GetStreamsAsync(); + streams.ShouldContain(stream => stream.Name == name); + + var second = $"vsr-reads-second-{Guid.NewGuid():N}"; + (await client.CreateStreamAsync(second)).ShouldNotBeNull(); + } +} diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/Vsr/VsrMessagingTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/Vsr/VsrMessagingTests.cs new file mode 100644 index 0000000000..d1fdbd3e47 --- /dev/null +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/Vsr/VsrMessagingTests.cs @@ -0,0 +1,165 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Text; +using Apache.Iggy.Contracts; +using Apache.Iggy.Enums; +using Apache.Iggy.IggyClient; +using Apache.Iggy.Kinds; +using Apache.Iggy.Messages; +using Apache.Iggy.Tests.Integrations.Fixtures; +using Shouldly; +using Partitioning = Apache.Iggy.Kinds.Partitioning; + +namespace Apache.Iggy.Tests.Integrations.Vsr; + +/// +/// The partition plane. Under VSR the broker never picks a partition, so the client resolves every +/// partitioning kind to an explicit id before the request leaves - these tests assert the resolution +/// lands where the Rust SDK's does. +/// +public class VsrMessagingTests +{ + private const uint PartitionsCount = 4; + private const string TopicName = "vsr-messages"; + + [ClassDataSource(Shared = SharedType.PerAssembly)] + public required IggyServerFixture Fixture { get; init; } + + [Test] + public async Task SendMessages_ToAnExplicitPartition_Should_PollBack_FromThatPartition() + { + var (client, streamName) = await CreateStreamAndTopic(); + + await SendAsync(client, streamName, Partitioning.PartitionId(2), 5); + + var polled = await PollAsync(client, streamName, 2); + polled.Messages.Count.ShouldBe(5); + polled.PartitionId.ShouldBe(2); + + (await PollAsync(client, streamName, 1)).Messages.ShouldBeEmpty(); + } + + /// + /// Balanced partitioning is resolved client-side by round-robin over the topic's partition count, so + /// a batch per partition ends up one message on each. + /// + [Test] + public async Task SendMessages_Balanced_Should_RoundRobin_AcrossEveryPartition() + { + var (client, streamName) = await CreateStreamAndTopic(); + + for (var i = 0; i < PartitionsCount; i++) + { + await SendAsync(client, streamName, Partitioning.None(), 1); + } + + List counts = await PollEveryPartitionAsync(client, streamName); + + counts.Sum().ShouldBe((int)PartitionsCount); + counts.ShouldAllBe(count => count == 1); + } + + /// + /// The message key hashes to one partition, so every message under the same key lands together and + /// two different keys are free to differ. Only the first is asserted: the hash is pinned by the unit + /// tests against the Rust vectors, and asserting two keys differ would be a coin flip. + /// + [Test] + public async Task SendMessages_ByMessageKey_Should_LandOn_ASinglePartition() + { + var (client, streamName) = await CreateStreamAndTopic(); + var key = Partitioning.EntityIdString($"key-{Guid.NewGuid():N}"); + + for (var i = 0; i < 6; i++) + { + await SendAsync(client, streamName, key, 1); + } + + List counts = await PollEveryPartitionAsync(client, streamName); + + counts.Sum().ShouldBe(6); + counts.Count(count => count > 0).ShouldBe(1); + } + + [Test] + public async Task ConsumerOffsets_Should_RoundTrip_ThroughTheResultSection() + { + var (client, streamName) = await CreateStreamAndTopic(); + await SendAsync(client, streamName, Partitioning.PartitionId(0), 3); + + var consumer = Consumer.New($"vsr-offset-{Guid.NewGuid():N}"); + await client.StoreOffsetAsync(consumer, Identifier.String(streamName), Identifier.String(TopicName), 1, 0); + + var stored = await client.GetOffsetAsync(consumer, Identifier.String(streamName), + Identifier.String(TopicName), 0); + stored.ShouldNotBeNull(); + stored.StoredOffset.ShouldBe(1u); + + await client.DeleteOffsetAsync(consumer, Identifier.String(streamName), Identifier.String(TopicName), 0); + + var cleared = await client.GetOffsetAsync(consumer, Identifier.String(streamName), + Identifier.String(TopicName), 0); + cleared.ShouldSatisfyAllConditions(() => (cleared is null || cleared.StoredOffset == 0).ShouldBeTrue()); + } + + private async Task<(IIggyClient Client, string StreamName)> CreateStreamAndTopic() + { + var client = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); + var streamName = $"vsr-msg-{Guid.NewGuid():N}"; + + await client.CreateStreamAsync(streamName); + await client.CreateTopicAsync(Identifier.String(streamName), TopicName, PartitionsCount); + + return (client, streamName); + } + + private static Task SendAsync(IIggyClient client, string streamName, Partitioning partitioning, int count) + { + Message[] messages = Enumerable.Range(0, count) + .Select(index => new Message(Guid.NewGuid(), Encoding.UTF8.GetBytes($"vsr-payload-{index}"))) + .ToArray(); + + return client.SendMessagesAsync(Identifier.String(streamName), Identifier.String(TopicName), partitioning, + messages); + } + + private static Task PollAsync(IIggyClient client, string streamName, uint partitionId) + { + return client.PollMessagesAsync(new MessageFetchRequest + { + Count = 100, + AutoCommit = false, + Consumer = Consumer.New(1), + PartitionId = partitionId, + PollingStrategy = PollingStrategy.Offset(0), + StreamId = Identifier.String(streamName), + TopicId = Identifier.String(TopicName) + }); + } + + private static async Task> PollEveryPartitionAsync(IIggyClient client, string streamName) + { + var counts = new List(); + for (uint partitionId = 0; partitionId < PartitionsCount; partitionId++) + { + counts.Add((await PollAsync(client, streamName, partitionId)).Messages.Count); + } + + return counts; + } +} diff --git a/foreign/csharp/Iggy_SDK.Tests.Integration/Vsr/VsrMetadataTests.cs b/foreign/csharp/Iggy_SDK.Tests.Integration/Vsr/VsrMetadataTests.cs new file mode 100644 index 0000000000..8f898d8e24 --- /dev/null +++ b/foreign/csharp/Iggy_SDK.Tests.Integration/Vsr/VsrMetadataTests.cs @@ -0,0 +1,132 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Apache.Iggy.Enums; +using Apache.Iggy.Exceptions; +using Apache.Iggy.Kinds; +using Apache.Iggy.Messages; +using Apache.Iggy.Tests.Integrations.Fixtures; +using Shouldly; +using Partitioning = Apache.Iggy.Kinds.Partitioning; + +namespace Apache.Iggy.Tests.Integrations.Vsr; + +/// +/// Control-plane operations through the consensus path: every one of these consumes a request id and +/// comes back with a committed result section the decoder has to strip before the typed mapper runs. +/// +public class VsrMetadataTests +{ + [ClassDataSource(Shared = SharedType.PerAssembly)] + public required IggyServerFixture Fixture { get; init; } + + [Test] + public async Task StreamLifecycle_Should_CommitThroughConsensus() + { + var client = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); + + var name = $"vsr-meta-stream-{Guid.NewGuid():N}"; + var created = await client.CreateStreamAsync(name); + created.ShouldNotBeNull(); + + var fetched = await client.GetStreamByIdAsync(Identifier.Numeric(created.Id)); + fetched.ShouldNotBeNull(); + fetched.Name.ShouldBe(name); + + await client.DeleteStreamAsync(Identifier.Numeric(created.Id)); + (await client.GetStreamsAsync()).ShouldNotContain(stream => stream.Name == name); + } + + [Test] + public async Task TopicLifecycle_Should_CommitThroughConsensus() + { + var client = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); + + var streamName = $"vsr-meta-topic-{Guid.NewGuid():N}"; + await client.CreateStreamAsync(streamName); + + var topic = await client.CreateTopicAsync(Identifier.String(streamName), "vsr-topic", 3); + topic.ShouldNotBeNull(); + topic.PartitionsCount.ShouldBe(3u); + + var fetched = await client.GetTopicByIdAsync(Identifier.String(streamName), Identifier.Numeric(topic.Id)); + fetched.ShouldNotBeNull(); + fetched.PartitionsCount.ShouldBe(3u); + + await client.DeleteTopicAsync(Identifier.String(streamName), Identifier.Numeric(topic.Id)); + (await client.GetTopicsAsync(Identifier.String(streamName))).ShouldBeEmpty(); + } + + /// + /// A committed rejection rides the result section with status 0 in the header, so the decoder has to + /// read the first result entry to see it. A silent success here would mean the section was skipped. + /// + [Test] + public async Task DuplicateStream_Should_Surface_TheCommittedRejection() + { + var client = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); + + var name = $"vsr-meta-dup-{Guid.NewGuid():N}"; + await client.CreateStreamAsync(name); + + await Should.ThrowAsync(client.CreateStreamAsync(name)); + } + + [Test] + public async Task UserLifecycle_Should_CommitThroughConsensus() + { + var client = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); + + var name = $"vsr-user-{Guid.NewGuid():N}"; + var user = await client.CreateUserAsync(name, "secret-password", UserStatus.Active); + user.ShouldNotBeNull(); + + var fetched = await client.GetUserAsync(Identifier.Numeric(user.Id)); + fetched.ShouldNotBeNull(); + fetched.Username.ShouldBe(name); + + await client.DeleteUserAsync(Identifier.Numeric(user.Id)); + (await client.GetUserAsync(Identifier.Numeric(user.Id))).ShouldBeNull(); + } + + /// + /// Partition ops read the request counter without advancing it. Interleaving them with metadata + /// writes catches the asymmetry: a partition op that consumed an id would gap the next metadata one + /// and the primary would silently drop it. + /// + [Test] + public async Task MetadataWrites_Should_KeepCommitting_Around_PartitionOps() + { + var client = await Fixture.CreateAuthenticatedClient(Protocol.Tcp); + + var streamName = $"vsr-meta-mixed-{Guid.NewGuid():N}"; + await client.CreateStreamAsync(streamName); + await client.CreateTopicAsync(Identifier.String(streamName), "vsr-mixed-topic", 1); + + await client.SendMessagesAsync(Identifier.String(streamName), Identifier.String("vsr-mixed-topic"), + Partitioning.PartitionId(0), + [new Message(Guid.NewGuid(), "vsr-mixed"u8.ToArray())]); + await client.StoreOffsetAsync(Consumer.New(1), Identifier.String(streamName), + Identifier.String("vsr-mixed-topic"), 0, 0); + + var second = $"vsr-meta-mixed-second-{Guid.NewGuid():N}"; + var created = await client.CreateStreamAsync(second); + + created.ShouldNotBeNull(); + created.Name.ShouldBe(second); + } +} diff --git a/foreign/csharp/Iggy_SDK/Configuration/AutoLoginSettings.cs b/foreign/csharp/Iggy_SDK/Configuration/AutoLoginSettings.cs index b1b434f563..3c88446eca 100644 --- a/foreign/csharp/Iggy_SDK/Configuration/AutoLoginSettings.cs +++ b/foreign/csharp/Iggy_SDK/Configuration/AutoLoginSettings.cs @@ -36,4 +36,19 @@ public class AutoLoginSettings /// Specifies the password for auto-login authentication /// public string Password { get; set; } = string.Empty; + + /// + /// Settings for a builder-owned client that signs in with the given credentials. The credentials must + /// reach the client and not only the explicit login the wrapper performs at startup: a reconnect or a + /// leader redirect drops the session, and without them the client comes back unauthenticated. + /// + internal static AutoLoginSettings For(string username, string password) + { + return new AutoLoginSettings + { + Enabled = !string.IsNullOrEmpty(username), + Username = username, + Password = password + }; + } } diff --git a/foreign/csharp/Iggy_SDK/Configuration/IggyClientConfigurator.cs b/foreign/csharp/Iggy_SDK/Configuration/IggyClientConfigurator.cs index b8c555a812..fad8604a6f 100644 --- a/foreign/csharp/Iggy_SDK/Configuration/IggyClientConfigurator.cs +++ b/foreign/csharp/Iggy_SDK/Configuration/IggyClientConfigurator.cs @@ -37,6 +37,12 @@ public sealed class IggyClientConfigurator /// public required Protocol Protocol { get; set; } + /// + /// The largest response frame accepted over , in bytes. + /// Default is 64 MiB, minimum is the 256-byte header. + /// + public int MaxResponseFrameSize { get; set; } = 64 * 1024 * 1024; + /// /// The size of the receive buffer in bytes. Default is 4096. /// diff --git a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Rented.cs b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Rented.cs index e75b60ae31..9b4bbfaeb8 100644 --- a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Rented.cs +++ b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.Rented.cs @@ -198,13 +198,7 @@ protected async Task PollRentedMessagesAsync(CancellationToken ct) LogFailedToDecryptMessage(ex, ex.Offset, ex.PartitionId); throw; } - catch (MalformedResponseException) - { - // Non-transient poison: rethrow so the generic catch below does not swallow it and re-poll forever. - // Base InvalidResponseException (server error status, possibly transient) falls through to retry. - throw; - } - catch (Exception ex) + catch (Exception ex) when (ex is not (MalformedResponseException or VsrRequestOutcomeUnknownException)) { LogFailedToPollMessages(ex); _consumerErrorEvents.Publish(new ConsumerErrorEventArgs(ex, "Failed to poll messages")); diff --git a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs index c1201d0274..b5317cfdef 100644 --- a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs +++ b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumer.cs @@ -154,7 +154,7 @@ public async Task InitAsync(CancellationToken ct = default) await _client.ConnectAsync(ct); - if (_config.CreateIggyClient) + if (!string.IsNullOrEmpty(_config.Login) && !_config.CreateIggyClient) { await _client.LoginUserAsync(_config.Login, _config.Password, ct); } @@ -441,13 +441,7 @@ private async Task PollMessagesAsync(CancellationToken ct) LogFailedToDecryptMessage(ex, ex.Offset, ex.PartitionId); throw; } - catch (MalformedResponseException) - { - // Non-transient poison: rethrow so the generic catch below does not swallow it and re-poll forever. - // Base InvalidResponseException (server error status, possibly transient) falls through to retry. - throw; - } - catch (Exception ex) + catch (Exception ex) when (ex is not (MalformedResponseException or VsrRequestOutcomeUnknownException)) { LogFailedToPollMessages(ex); _consumerErrorEvents.Publish(new ConsumerErrorEventArgs(ex, "Failed to poll messages")); diff --git a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerBuilder.cs b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerBuilder.cs index 937ca10c5a..fc768662bf 100644 --- a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerBuilder.cs +++ b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerBuilder.cs @@ -31,7 +31,7 @@ namespace Apache.Iggy.Consumers; /// public class IggyConsumerBuilder { - private IMessageEncryptor? _encryptor; + private protected IMessageEncryptor? _encryptor; internal Func? OnPollingError { get; set; } internal IggyConsumerConfig Config { get; set; } = new(); @@ -39,7 +39,7 @@ public class IggyConsumerBuilder /// /// Creates a new consumer builder that will create its own Iggy client. - /// You must configure connection settings using . + /// You must configure connection settings using WithConnection. /// /// The stream identifier to consume from /// The topic identifier to consume from @@ -245,6 +245,7 @@ public IggyConsumer Build() ReceiveBufferSize = Config.ReceiveBufferSize, SendBufferSize = Config.SendBufferSize, ReconnectionSettings = Config.ReconnectionSettings ?? new ReconnectionSettings(), + AutoLoginSettings = AutoLoginSettings.For(Config.Login, Config.Password), LoggerFactory = Config.LoggerFactory ?? NullLoggerFactory.Instance, MessageEncryptor = _encryptor }); diff --git a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerBuilderOfT.cs b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerBuilderOfT.cs index 0ccf68cc18..ad640ebeff 100644 --- a/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerBuilderOfT.cs +++ b/foreign/csharp/Iggy_SDK/Consumers/IggyConsumerBuilderOfT.cs @@ -19,7 +19,6 @@ using Apache.Iggy.Factory; using Apache.Iggy.IggyClient; using Apache.Iggy.Kinds; -using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; namespace Apache.Iggy.Consumers; @@ -95,7 +94,11 @@ public static IggyConsumerBuilder Create(IIggyClient iggyClient, Identifier s Protocol = Config.Protocol, BaseAddress = Config.Address, ReceiveBufferSize = Config.ReceiveBufferSize, - SendBufferSize = Config.SendBufferSize + SendBufferSize = Config.SendBufferSize, + ReconnectionSettings = Config.ReconnectionSettings ?? new ReconnectionSettings(), + AutoLoginSettings = AutoLoginSettings.For(Config.Login, Config.Password), + LoggerFactory = Config.LoggerFactory ?? NullLoggerFactory.Instance, + MessageEncryptor = _encryptor }); } @@ -133,8 +136,7 @@ protected override void Validate() } else { - throw new InvalidOperationException( - $"Config must be of type IggyConsumerConfig<{typeof(T).Name}>."); + throw new InvalidOperationException($"Config must be of type IggyConsumerConfig<{typeof(T).Name}>."); } } } diff --git a/foreign/csharp/Iggy_SDK/Contracts/SendMessagesResponse.cs b/foreign/csharp/Iggy_SDK/Contracts/SendMessagesResponse.cs new file mode 100644 index 0000000000..199bdc4c52 --- /dev/null +++ b/foreign/csharp/Iggy_SDK/Contracts/SendMessagesResponse.cs @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.Iggy.Contracts; + +/// +/// Commit confirmation for one partition written by a send messages request. +/// +/// +/// is the offset assigned to the first message of the batch in that +/// partition, bounded by two properties of the send path: +/// +/// +/// Delivery is at-least-once. An earlier retry of the same batch may already have committed +/// at a lower offset, so the value never implies uniqueness. +/// +/// +/// A batch is confirmed once it is committed in memory, not once it is fsynced. A +/// crash-restart can stamp a later batch with an offset a client has already recorded. +/// +/// +/// +public sealed class SendMessagesConfirmation +{ + /// + /// Stream identifier the batch was written to. + /// + public required uint StreamId { get; init; } + + /// + /// Topic identifier the batch was written to. + /// + public required uint TopicId { get; init; } + + /// + /// Partition the batch landed in. + /// + public required uint PartitionId { get; init; } + + /// + /// Offset assigned to the first message of the batch in that partition. + /// + public required ulong BaseOffset { get; init; } +} + +/// +/// Result of a send messages request. +/// +/// +/// An empty list means the batch committed with no offsets to +/// report (e.g. a background send merged into a larger batch). The server currently reports a +/// single partition per request; the list shape lets a later multi-partition send grow the +/// count without an API change. +/// +public sealed class SendMessagesResponse +{ + /// + /// One confirmation per partition the batch was committed to. + /// + public required IReadOnlyList Confirmations { get; init; } + + internal static SendMessagesResponse Empty { get; } = new() { Confirmations = [] }; +} diff --git a/foreign/csharp/Iggy_SDK/Exceptions/IggyInvalidStatusCodeException.cs b/foreign/csharp/Iggy_SDK/Exceptions/IggyInvalidStatusCodeException.cs index 6ba6e14d3e..9d73640d5c 100644 --- a/foreign/csharp/Iggy_SDK/Exceptions/IggyInvalidStatusCodeException.cs +++ b/foreign/csharp/Iggy_SDK/Exceptions/IggyInvalidStatusCodeException.cs @@ -25,10 +25,18 @@ public sealed class IggyInvalidStatusCodeException : Exception /// /// Status code returned by the server. /// - public int StatusCode { get; init; } + public int StatusCode { get; } - internal IggyInvalidStatusCodeException(int statusCode, string message) : base(message) + /// + /// Whether the status code was reported by the server rather than raised by the client. The two share one + /// code space, and only a server verdict may drive retry or failover: a locally raised code says nothing + /// about what the cluster did with the request. + /// + public bool FromServer { get; } + + internal IggyInvalidStatusCodeException(int statusCode, string message, bool fromServer = false) : base(message) { StatusCode = statusCode; + FromServer = fromServer; } } diff --git a/foreign/csharp/Iggy_SDK/Exceptions/VsrRequestOutcomeUnknownException.cs b/foreign/csharp/Iggy_SDK/Exceptions/VsrRequestOutcomeUnknownException.cs new file mode 100644 index 0000000000..f79334b282 --- /dev/null +++ b/foreign/csharp/Iggy_SDK/Exceptions/VsrRequestOutcomeUnknownException.cs @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.Iggy.Exceptions; + +/// +/// The request produced no server verdict after transmission began, so the server may already have committed +/// it. Replaying it on a fresh consensus session would bypass server-side deduplication, so the SDK refuses to +/// retry and surfaces this instead: the caller decides whether re-issuing the operation is safe. +/// +/// +/// Deliberately derived from rather than or +/// , whichever ended the request. Both of those are routinely caught +/// and either retried or swallowed, which are the two responses this type exists to prevent. The triggering +/// exception is preserved as . +/// +public sealed class VsrRequestOutcomeUnknownException(Exception innerException) + : Exception("The VSR request outcome is unknown because no server verdict arrived after transmission began.", + innerException); diff --git a/foreign/csharp/Iggy_SDK/Exceptions/VsrSessionEvictedException.cs b/foreign/csharp/Iggy_SDK/Exceptions/VsrSessionEvictedException.cs new file mode 100644 index 0000000000..824356618a --- /dev/null +++ b/foreign/csharp/Iggy_SDK/Exceptions/VsrSessionEvictedException.cs @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.Iggy.Exceptions; + +/// +/// An eviction frame arrived where a reply was expected. Internal: the transport decides whether the caller +/// sees or an unknown outcome, depending on whether the outstanding request could +/// still have committed. +/// +/// +/// The server emits evictions off its own heartbeat timer rather than as an answer, so the frame carries no +/// correlation with the request it interrupts and that request's real reply is never read. +/// +internal sealed class VsrSessionEvictedException(Exception verdict) + : Exception("The consensus session was evicted by the server.", verdict) +{ + /// The error the eviction reason maps to, for the requests it can safely be reported as. + internal Exception Verdict { get; } = verdict; +} diff --git a/foreign/csharp/Iggy_SDK/Factory/IggyClientFactory.cs b/foreign/csharp/Iggy_SDK/Factory/IggyClientFactory.cs index 4300464900..ef965f34bf 100644 --- a/foreign/csharp/Iggy_SDK/Factory/IggyClientFactory.cs +++ b/foreign/csharp/Iggy_SDK/Factory/IggyClientFactory.cs @@ -20,6 +20,7 @@ using Apache.Iggy.Enums; using Apache.Iggy.IggyClient; using Apache.Iggy.IggyClient.Implementations; +using Apache.Iggy.Vsr; namespace Apache.Iggy.Factory; @@ -44,8 +45,13 @@ public static class IggyClientFactory /// Thrown when the specified protocol in is not /// supported. /// + /// + /// Thrown when is below the 256-byte header. + /// public static IIggyClient CreateClient(IggyClientConfigurator options) { + Validate(options); + return options.Protocol switch { Protocol.Http => CreateIggyHttpClient(options), @@ -54,6 +60,15 @@ public static IIggyClient CreateClient(IggyClientConfigurator options) }; } + private static void Validate(IggyClientConfigurator options) + { + if (options.Protocol == Protocol.Tcp && options.MaxResponseFrameSize < VsrHeader.HEADER_SIZE) + { + throw new ArgumentOutOfRangeException(nameof(options), options.MaxResponseFrameSize, + $"MaxResponseFrameSize must be at least {VsrHeader.HEADER_SIZE} bytes."); + } + } + private static IIggyClient CreateIggyTcpClient(IggyClientConfigurator options) { return new TcpMessageStream(options, options.LoggerFactory); @@ -67,7 +82,7 @@ private static IIggyClient CreateIggyHttpClient(IggyClientConfigurator options) private static HttpClient CreateHttpClient(IggyClientConfigurator options) { - var client = new HttpClient(); + var client = new HttpClient(new TransientHttpRetryHandler(new HttpClientHandler())); client.BaseAddress = new Uri(options.BaseAddress); return client; } diff --git a/foreign/csharp/Iggy_SDK/IggyClient/IIggyPublisher.cs b/foreign/csharp/Iggy_SDK/IggyClient/IIggyPublisher.cs index 483814a630..7263c220cd 100644 --- a/foreign/csharp/Iggy_SDK/IggyClient/IIggyPublisher.cs +++ b/foreign/csharp/Iggy_SDK/IggyClient/IIggyPublisher.cs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +using Apache.Iggy.Contracts; using Apache.Iggy.Kinds; using Apache.Iggy.Messages; @@ -46,9 +47,11 @@ public interface IIggyPublisher /// The partitioning strategy that determines which partition receives the messages. /// The collection of messages to be sent. /// The cancellation token to cancel the operation. - /// A task representing the asynchronous operation. - Task SendMessagesAsync(Identifier streamId, Identifier topicId, Partitioning partitioning, IList messages, - CancellationToken token = default); + /// + /// Commit confirmations carrying the partition each batch landed in and its base offset. + /// + Task SendMessagesAsync(Identifier streamId, Identifier topicId, Partitioning partitioning, + IList messages, CancellationToken token = default); /// /// Sends a single message to the specified stream and topic. See @@ -60,19 +63,18 @@ Task SendMessagesAsync(Identifier streamId, Identifier topicId, Partitioning par /// payload memory (e.g. a pooled buffer) may be released /// once it completes. /// - Task SendMessagesAsync(Identifier streamId, Identifier topicId, Partitioning partitioning, Message message, - CancellationToken token = default) - => SendMessagesAsync(streamId, topicId, partitioning, new[] { message }, token); + Task SendMessagesAsync(Identifier streamId, Identifier topicId, Partitioning partitioning, + Message message, CancellationToken token = default) + { + return SendMessagesAsync(streamId, topicId, partitioning, new[] { message }, token); + } /// /// Forces a flush of the unsaved buffer to disk for a specific partition. /// /// - /// This method ensures that all pending messages in the in-memory buffer for the specified partition are written to - /// disk. - /// If is true, the data is both flushed to disk and synchronized (fsync), ensuring - /// durability. - /// If false, the data is only flushed to disk without synchronization. + /// This feature is not supported by the server. Durability is handled by replication and the journal, + /// so there is no client-flushable in-memory buffer. /// /// The stream identifier (numeric ID or name). /// The topic identifier (numeric ID or name). @@ -80,6 +82,7 @@ Task SendMessagesAsync(Identifier streamId, Identifier topicId, Partitioning par /// If true, the data is flushed and synchronized to disk (fsync). If false, only flushed. /// The cancellation token to cancel the operation. /// A task representing the asynchronous operation. + /// Always thrown; the server does not support this command. Task FlushUnsavedBufferAsync(Identifier streamId, Identifier topicId, uint partitionId, bool fsync, CancellationToken token = default); } diff --git a/foreign/csharp/Iggy_SDK/IggyClient/IIggySystem.cs b/foreign/csharp/Iggy_SDK/IggyClient/IIggySystem.cs index 32acfcec89..aab60f1c6c 100644 --- a/foreign/csharp/Iggy_SDK/IggyClient/IIggySystem.cs +++ b/foreign/csharp/Iggy_SDK/IggyClient/IIggySystem.cs @@ -86,7 +86,10 @@ public interface IIggySystem /// Sends a ping request to the server to verify connectivity. /// /// - /// This is a simple health check operation that can be used to verify the connection is active. + /// This is a simple health check operation that can be used to verify the connection is active. On the + /// VSR wire protocol it also re-syncs the assignment of every consumer group this client has joined, so + /// it costs one extra round trip per joined group. The SDK never calls it on its own: an application + /// that wants assignments refreshed has to ping on its own cadence. /// /// The cancellation token to cancel the operation. /// A task representing the asynchronous operation. diff --git a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs index 4aba83255e..b8f51c7a5e 100644 --- a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs +++ b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/HttpMessageStream.cs @@ -16,6 +16,7 @@ // under the License. using System.Buffers; +using System.IO.Hashing; using System.Net; using System.Net.Http.Headers; using System.Net.Http.Json; @@ -35,6 +36,7 @@ using Apache.Iggy.Messages; using Apache.Iggy.StringHandlers; using Apache.Iggy.Utils; +using Apache.Iggy.Vsr; using Partitioning = Apache.Iggy.Kinds.Partitioning; namespace Apache.Iggy.IggyClient.Implementations; @@ -47,6 +49,7 @@ public class HttpMessageStream : IIggyClient private const string Context = "csharp-sdk"; private readonly bool _allowAutoCommitWithEncryptor; + private readonly ConsumerGroupClientState _groupState = new(); private readonly HttpClient _httpClient; //TODO - create mechanism for refreshing jwt token @@ -204,6 +207,8 @@ public async Task DeleteTopicAsync(Identifier streamId, Identifier topicId, Canc { await HandleResponseAsync(response); } + + _groupState.InvalidatePartitionCount(new TopicKey(streamId, topicId)); } /// @@ -251,8 +256,8 @@ public async Task> GetTopicsAsync(Identifier stream } /// - public async Task SendMessagesAsync(Identifier streamId, Identifier topicId, Partitioning partitioning, - IList messages, + public async Task SendMessagesAsync(Identifier streamId, Identifier topicId, + Partitioning partitioning, IList messages, CancellationToken token = default) { if (MessageEncryptor is not null) @@ -266,6 +271,11 @@ public async Task SendMessagesAsync(Identifier streamId, Identifier topicId, Par messages = encrypted; } + if (partitioning.Kind != Enums.Partitioning.PartitionId) + { + partitioning = await ResolvePartitioningAsync(streamId, topicId, partitioning, token); + } + var request = new MessageSendRequest { StreamId = streamId, @@ -284,20 +294,19 @@ public async Task SendMessagesAsync(Identifier streamId, Identifier topicId, Par { await HandleResponseAsync(response); } + + return await response.Content.ReadFromJsonAsync(_jsonSerializerOptions, token) + ?? throw new InvalidResponseException("Send messages reply carried no confirmation body."); } - /// - public async Task FlushUnsavedBufferAsync(Identifier streamId, Identifier topicId, uint partitionId, bool fsync, + /// + /// This feature is not supported by the server. + /// + /// + public Task FlushUnsavedBufferAsync(Identifier streamId, Identifier topicId, uint partitionId, bool fsync, CancellationToken token = default) { - var url = CreateUrl($"/streams/{streamId}/topics/{topicId}/messages/flush/{partitionId}/{fsync}"); - - var response = await _httpClient.GetAsync(url, token); - - if (!response.IsSuccessStatusCode) - { - await HandleResponseAsync(response, true); - } + throw new FeatureUnavailableException(); } /// @@ -615,6 +624,8 @@ var response { await HandleResponseAsync(response); } + + _groupState.InvalidatePartitionCount(new TopicKey(streamId, topicId)); } /// @@ -646,6 +657,8 @@ public async Task CreatePartitionsAsync(Identifier streamId, Identifier topicId, { await HandleResponseAsync(response); } + + _groupState.InvalidatePartitionCount(new TopicKey(streamId, topicId)); } /// @@ -885,6 +898,41 @@ public string GetCurrentAddress() return _httpClient.BaseAddress?.ToString() ?? string.Empty; } + /// + /// Resolves balanced and message-key partitioning to an explicit partition id, mirroring the TCP client. + /// Server-side balanced resolution races partition-count changes (a send right after CreatePartitions can + /// land on a stale round-robin cycle), so the client picks the partition and sends it explicitly. + /// + private async ValueTask ResolvePartitioningAsync(Identifier streamId, Identifier topicId, + Partitioning partitioning, CancellationToken token) + { + var key = new TopicKey(streamId, topicId); + var partitionCount = _groupState.PartitionCount(key); + if (partitionCount is null) + { + var topic = await GetTopicByIdAsync(streamId, topicId, token) + ?? throw new IggyInvalidStatusCodeException((int)HttpStatusCode.NotFound, + $"Topic {topicId} was not found in stream {streamId}.", true); + _groupState.SetPartitionCount(key, topic.PartitionsCount); + partitionCount = topic.PartitionsCount; + } + + if (partitionCount == 0) + { + throw new IggyInvalidStatusCodeException((int)HttpStatusCode.NotFound, + $"Topic {topicId} in stream {streamId} has no partitions to resolve the message to.", true); + } + + var partition = partitioning.Kind switch + { + Enums.Partitioning.Balanced => _groupState.NextBalancedPartition(key, partitionCount.Value), + Enums.Partitioning.MessageKey => XxHash32.HashToUInt32(partitioning.Value) % partitionCount.Value, + _ => throw new FeatureUnavailableException() + }; + + return Partitioning.PartitionId((int)partition); + } + private void DecryptMessages(IReadOnlyList messages, uint partitionId) { foreach (var message in messages) @@ -981,20 +1029,30 @@ private static byte[] Decrypt(IMessageEncryptor encryptor, ReadOnlySpan da private static async Task HandleResponseAsync(HttpResponseMessage response, bool shouldThrowOnGetNotFound = false) { - if ((int)response.StatusCode > 300 - && (int)response.StatusCode < 500 - && !(response.RequestMessage!.Method == HttpMethod.Get && response.StatusCode == HttpStatusCode.NotFound && - !shouldThrowOnGetNotFound)) + if (response.IsSuccessStatusCode) { - var err = await response.Content.ReadAsStringAsync(); - var errorModel = JsonSerializer.Deserialize(err); - throw new IggyInvalidStatusCodeException(errorModel?.Id ?? -1, err); + return; } - if (response.StatusCode == HttpStatusCode.InternalServerError) + if (response.RequestMessage!.Method == HttpMethod.Get && response.StatusCode == HttpStatusCode.NotFound && + !shouldThrowOnGetNotFound) { - throw new Exception("Internal server error"); + return; } + + var err = await response.Content.ReadAsStringAsync(); + ErrorResponse? errorModel = null; + try + { + errorModel = JsonSerializer.Deserialize(err); + } + catch (JsonException) + { + // A gateway or proxy error body is not the server's JSON schema; the raw text still travels in + // the exception message. + } + + throw new IggyInvalidStatusCodeException(errorModel?.Id ?? -1, err, true); } private static string CreateUrl(ref MessageRequestInterpolationHandler message) diff --git a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.Vsr.cs b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.Vsr.cs new file mode 100644 index 0000000000..81898c0994 --- /dev/null +++ b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.Vsr.cs @@ -0,0 +1,978 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Buffers; +using System.Buffers.Binary; +using System.IO.Hashing; +using System.Runtime.ExceptionServices; +using Apache.Iggy.ConnectionStream; +using Apache.Iggy.Contracts; +using Apache.Iggy.Contracts.Auth; +using Apache.Iggy.Contracts.Tcp; +using Apache.Iggy.Enums; +using Apache.Iggy.Exceptions; +using Apache.Iggy.Kinds; +using Apache.Iggy.Messages; +using Apache.Iggy.Utils; +using Apache.Iggy.Vsr; +using Microsoft.Extensions.Logging; +using Partitioning = Apache.Iggy.Kinds.Partitioning; + +namespace Apache.Iggy.IggyClient.Implementations; + +/// +/// The consensus (VSR) half of the TCP client: the framed request path, the leader redirection, the +/// register handshake, and the client-side partitioning and consumer-group assignment the broker does not +/// resolve server-side. The command surface lives in . +/// +public sealed partial class TcpMessageStream +{ + /// + /// Upper bound for a whole VSR request: the transient replays and the leader failovers share it, and so + /// do the reply header and body reads. The connection is lockstep, so an unanswered read would hold the + /// sending semaphore forever and wedge every later request. + /// + private const int VsrRequestTimeoutMs = 30_000; + + /// Backoff between replays of a transiently refused request. + private const int VsrTransientRetryIntervalMs = 50; + + /// + /// Largest body still sent as one contiguous frame with its header. Beyond this the copy outweighs the + /// syscall and the extra segment it saves, so header and body go out as two writes. + /// + private const int VsrContiguousFrameLimit = 4 * 1024; + + /// + /// How long a request replays on the same connection + /// before the leader roster is re-checked. A node that stopped being primary refuses forever, so + /// replaying alone never recovers. + /// + private const int VsrTransientFailoverCheckMs = 2_000; + + /// + /// How long a transiently leaderless roster is polled before the connection proceeds on the current + /// node anyway. A restarted node cedes the primaryship its stale view assigns it, and the peers need + /// about one heartbeat timeout to elect. + /// + private const int VsrLeaderlessWaitMs = 5_000; + + private const int VsrLeaderlessPollMs = 250; + + /// + /// Cap on consecutive leader redirects, so a flapping roster cannot spin the connect loop or the + /// transient failover path. The budget is client-wide and resets on a roster check that finds the + /// current node is the leader, and on every request that completes, so a client that outlives more + /// leader changes than the cap does not latch onto a follower for good. + /// + private const int VsrMaxLeaderRedirects = 3; + + /// + /// Attempts a consumer-group poll gets before it gives up and reports an empty poll: one re-sync after + /// the coordinator fences a stale assignment, then one retry. + /// + private const int VsrGroupPollMaxAttempts = 2; + + /// + /// Partition id a fenced group poll echoes instead of a typed error, matching + /// RESYNC_REQUIRED_PARTITION_SENTINEL (u32::MAX). The reply header carries no status for an + /// empty poll, so the sentinel is the only channel the coordinator has to ask for a re-sync. + /// + private const int VsrResyncRequiredPartitionSentinel = -1; + + /// + /// Shared empty poll result. An idle consumer loop returns one on every iteration, and the instance owns + /// no rented buffer - disposes to nothing - so it is safe to hand out + /// repeatedly even after a caller disposes it. + /// + private static readonly PolledMessagesRental EmptyPolledMessages = new(EmptyMemoryOwner.Instance) + { + PartitionId = 0, + CurrentOffset = 0, + Messages = [] + }; + + private readonly ConsensusSession _consensusSession = new(); + private readonly ConsumerGroupClientState _groupState = new(); + private readonly byte[] _vsrReplyHeaderBuffer = new byte[VsrHeader.HEADER_SIZE]; + + // The redirect budget is refunded by a completed request, and the roster check a redirect runs is itself a + // request. Without this the refund lands between the check and the increment that reads the budget, and the + // counter never leaves zero. Nonzero for the duration of a roster read, so that refund is skipped. + private int _leaderProbeDepth; + + /// + /// Runs the consensus register handshake and binds the session it commits. Everything before the bind + /// is a consumed register on the server, so any failure resets the session: the next attempt must + /// re-register under a fresh client id rather than send requests the primary would fence. + /// + /// + /// A bound connection must commit logout before it can register again. The server treats a register on + /// an already-bound transport as an idempotent replay of the existing binding, so re-arming only the + /// local session would pair a fresh client id and request counter with the old server session. + /// + private async Task LoginRegisterAsync(int code, byte[] message, CancellationToken token) + { + for (var redirects = 0; ; redirects++) + { + if (_consensusSession.IsBound) + { + await LogoutUserAsync(token); + } + else if (_state == ConnectionState.Authenticated) + { + SetConnectionStateAsync(ConnectionState.Connected); + } + + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, code); + + SetConnectionStateAsync(ConnectionState.Authenticating); + + LoginRegisterResponse response; + try + { + Interlocked.Exchange(ref _skipAutoLoginOnce, 1); + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + response = LoginRegister.Deserialize(responseBuffer.Memory.Span); + _consensusSession.Bind(response.Session); + } + catch + { + await ResetConsensusSessionAsync(); + if (_state == ConnectionState.Authenticating) + { + SetConnectionStateAsync(ConnectionState.Connected); + } + + throw; + } + finally + { + Interlocked.Exchange(ref _skipAutoLoginOnce, 0); + } + + _logger.LogInformation( + "Authenticated against the server, version {ServerVersion}, protocol version {ServerProtocolVersion}", + response.ServerVersion, response.ServerProtocolVersion); + SetConnectionStateAsync(ConnectionState.Authenticated); + + var authResponse = new AuthResponse((int)response.UserId, null); + if (IsConnecting) + { + return authResponse; + } + + if (redirects >= VsrMaxLeaderRedirects) + { + _logger.LogWarning("Maximum leader redirections reached while registering, staying on {Address}", + _currentAddress); + + return authResponse; + } + + if (!await RedirectAsync(token)) + { + return authResponse; + } + + await ConnectAsync(false, token); + } + } + + /// + /// Whether the partitioning has to be resolved to an explicit partition id before the request is framed. + /// The broker never picks a partition, so balanced and message-key kinds resolve client-side. + /// + private static bool NeedsClientSidePartitioning(Partitioning partitioning) + { + return partitioning.Kind != Enums.Partitioning.PartitionId; + } + + private async Task SendMessagesResolvedAsync(Identifier streamId, Identifier topicId, + Partitioning partitioning, IList messages, CancellationToken token) + { + var resolved = await ResolvePartitioningAsync(streamId, topicId, partitioning, token); + + return await SendMessagesCoreAsync(streamId, topicId, resolved, AsSpan(messages), token); + } + + /// + /// Resolves balanced and message-key partitioning to an explicit partition id, mirroring + /// core/common/src/traits/binary_impls/messages.rs. The VSR broker never picks a partition, so + /// sending either kind on the wire would fail to route. + /// + private async ValueTask ResolvePartitioningAsync(Identifier streamId, Identifier topicId, + Partitioning partitioning, CancellationToken token) + { + var partitionCount = await TopicPartitionCountAsync(streamId, topicId, token); + if (partitionCount == 0) + { + throw VsrError.Exception(VsrError.TOPIC_ID_NOT_FOUND, + $"Topic {topicId} in stream {streamId} has no partitions to resolve the message to."); + } + + var partition = partitioning.Kind switch + { + Enums.Partitioning.Balanced => _groupState.NextBalancedPartition(new TopicKey(streamId, topicId), + partitionCount), + Enums.Partitioning.MessageKey => XxHash32.HashToUInt32(partitioning.Value) % partitionCount, + _ => throw VsrError.Exception(VsrError.FEATURE_UNAVAILABLE, + $"Partitioning kind {partitioning.Kind} cannot be resolved to a partition id.") + }; + + return Partitioning.PartitionId((int)partition); + } + + private async ValueTask TopicPartitionCountAsync(Identifier streamId, Identifier topicId, + CancellationToken token) + { + var key = new TopicKey(streamId, topicId); + if (_groupState.PartitionCount(key) is { } cached) + { + return cached; + } + + var topic = await GetTopicByIdAsync(streamId, topicId, token); + if (topic is null) + { + throw VsrError.Exception(VsrError.TOPIC_ID_NOT_FOUND, + $"Topic {topicId} was not found in stream {streamId}."); + } + + _groupState.SetPartitionCount(key, topic.PartitionsCount); + + return topic.PartitionsCount; + } + + /// + /// Polls one of the group member's assigned partitions, round-robin. A fence rejection - either the typed + /// error or the sentinel partition id an empty poll carries - re-syncs the assignment and retries once. + /// + private async Task PollGroupMessagesRentedAsync(Identifier streamId, Identifier topicId, + Consumer consumer, PollingStrategy pollingStrategy, uint count, bool autoCommit, CancellationToken token) + { + var key = new GroupKey(streamId, topicId, consumer.ConsumerId); + if (!_groupState.HasAssignment(key)) + { + await SyncGroupAssignmentAsync(streamId, topicId, consumer.ConsumerId, token); + } + + for (var attempt = 0; attempt < VsrGroupPollMaxAttempts; attempt++) + { + if (_groupState.NextGroupPartition(key) is not { } partitionId) + { + if (!_groupState.IsRegistered(key)) + { + throw VsrError.Exception(VsrError.CONSUMER_GROUP_MEMBER_NOT_FOUND, + $"Client is not a member of consumer group {consumer.ConsumerId} on topic {topicId}."); + } + + return EmptyPolledMessages; + } + + PolledMessagesRental? rental = null; + try + { + rental = await PollPartitionMessagesRentedAsync(streamId, topicId, partitionId, consumer, + pollingStrategy, count, autoCommit, token); + } + catch (IggyInvalidStatusCodeException e) when (e is + { + StatusCode: VsrError.CONSUMER_GROUP_PARTITION_NOT_OWNED, + FromServer: true + }) + { + // Both fence shapes - the typed error and the sentinel an empty poll carries - land on the same + // re-sync below. + } + + if (rental is not null) + { + if (rental.Messages.Count != 0 || rental.PartitionId != VsrResyncRequiredPartitionSentinel) + { + return rental; + } + + rental.Dispose(); + } + + _groupState.InvalidateAssignment(key); + await SyncGroupAssignmentAsync(streamId, topicId, consumer.ConsumerId, token); + } + + return EmptyPolledMessages; + } + + /// + /// Pulls the requesting member's assignment from the coordinator into the cache. An empty reply means the + /// client is not a member: the coordinator answers with an assignment header for any member, including + /// one holding zero partitions. + /// + private async Task SyncGroupAssignmentAsync(Identifier streamId, Identifier topicId, Identifier groupId, + CancellationToken token) + { + var message = TcpContracts.GetGroup(streamId, topicId, groupId); + var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; + TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.SYNC_CONSUMER_GROUP_CODE); + + using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); + + var key = new GroupKey(streamId, topicId, groupId); + if (responseBuffer.Memory.Length == 0) + { + // Deregistering is the only thing that observes a server-side removal (group deleted, member + // evicted); without it membership latches true and every later poll returns empty. + _groupState.DeregisterGroup(key); + + return; + } + + var assignment = SyncConsumerGroupAssignment.Decode(responseBuffer.Memory.Span); + _groupState.RegisterGroup(key, streamId, topicId, groupId); + _groupState.SetAssignment(key, assignment.Generation, assignment.Partitions); + } + + /// + /// Re-syncs every joined group so a widened assignment (a partition-count change, say) is picked up + /// without first hitting an ownership fence. One failing group is logged and skipped so it cannot stall + /// the rest. + /// + private async Task RefreshGroupAssignmentsAsync(CancellationToken token) + { + foreach (var group in _groupState.RegisteredGroups()) + { + try + { + await SyncGroupAssignmentAsync(group.StreamId, group.TopicId, group.GroupId, token); + } + catch (Exception e) when (e is not OperationCanceledException) + { + _logger.LogWarning(e, + "Failed to refresh the consumer group assignment for {StreamId}|{TopicId}|{GroupId}", + group.StreamId, group.TopicId, group.GroupId); + } + } + } + + + /// + /// Points the client at the current leader when it is not the node this connection is on, leaving the + /// stream closed for the caller to reconnect. The redirect budget is client-wide: the connect loop and + /// the transient failover path spend the same counter, and it is refunded as soon as a roster check + /// lands on the leader. + /// + private async Task RedirectAsync(CancellationToken token) + { + // The probe issues a request of its own, which takes the sending semaphore, so it cannot run under that + // lock. Only the commit below does. + var currentLeaderNode = await GetCurrentLeaderNodeAsync(token); + if (currentLeaderNode == null) + { + Interlocked.Exchange(ref _leaderRedirectCount, 0); + return false; + } + + var leaderAddress = ServerAddress.HostPort(currentLeaderNode.Ip, currentLeaderNode.Endpoints.Tcp); + if (ServerAddress.IsSame(leaderAddress, _currentAddress)) + { + Interlocked.Exchange(ref _leaderRedirectCount, 0); + return false; + } + + if (Interlocked.Increment(ref _leaderRedirectCount) > VsrMaxLeaderRedirects) + { + _logger.LogWarning("Maximum leader redirections reached, continuing on {Address}", _currentAddress); + return false; + } + + _logger.LogInformation("Leader address changed. Trying to reconnect to {Address}", + leaderAddress); + + // The address move and the drop are one step: a reader that saw the new address but the old stream would + // find every later redirect short-circuited by the address check above, with no path back to the leader. + await _sendingSemaphore.WaitAsync(token); + try + { + _currentAddress = leaderAddress; + DropVsrConnectionLocked(_stream); + } + finally + { + _sendingSemaphore.Release(); + } + + return true; + } + + private async Task GetCurrentLeaderNodeAsync(CancellationToken token) + { + var leaderlessDeadline = Environment.TickCount64 + VsrLeaderlessWaitMs; + Interlocked.Increment(ref _leaderProbeDepth); + try + { + while (true) + { + var clusterMetadata = await GetClusterMetadataAsync(token); + if (clusterMetadata == null) + { + return null; + } + + if (clusterMetadata.Nodes.Count() == 1) + { + return null; + } + + var leaderNode = clusterMetadata.Nodes.FirstOrDefault(x => + x.Role == ClusterNodeRole.Leader && x.Status == ClusterNodeStatus.Healthy); + if (leaderNode != null) + { + return leaderNode; + } + + if (Environment.TickCount64 >= leaderlessDeadline) + { + _logger.LogWarning("No leader in the cluster metadata after {WaitMs} ms, continuing on {Address}", + VsrLeaderlessWaitMs, _currentAddress); + + return null; + } + + await Task.Delay(VsrLeaderlessPollMs, token); + } + } + // todo: change after error refactoring, error code 5 is for feature not supported + catch (IggyInvalidStatusCodeException e) when (e is + { + StatusCode: VsrError.FEATURE_UNAVAILABLE, FromServer: true + }) + { + return null; + } + catch (Exception e) when (e is not OperationCanceledException) + { + _logger.LogWarning(e, "Failed to read the cluster metadata, continuing on {Address}", _currentAddress); + + return null; + } + finally + { + Interlocked.Decrement(ref _leaderProbeDepth); + } + } + + /// + /// Sends a consensus-framed request. The call sites still build the classic + /// [size u32][code u32][body] buffer, so the code is read back from it here and the body is written + /// right after the 256-byte consensus header - two writes, no concatenation. + /// + /// + /// One deadline bounds the whole request across transient replays AND leader failovers. Login and + /// register replay on this connection for the whole budget instead: the connect flow owns leader + /// redirection for the handshake, and reconnecting from underneath it would recurse. + /// + private async Task> SendRawVsrAsync(ReadOnlyMemory payload, CancellationToken token) + { + var code = (int)BinaryPrimitives.ReadUInt32LittleEndian(payload.Span.Slice(4, 4)); + ReadOnlyMemory body = payload[8..]; + var isLoginRegister = code is CommandCodes.LOGIN_REGISTER_CODE or CommandCodes.LOGIN_REGISTER_WITH_PAT_CODE; + var overallDeadline = Environment.TickCount64 + VsrRequestTimeoutMs; + var headerBuffer = ArrayPool.Shared.Rent(VsrHeader.HEADER_SIZE); + Memory header = headerBuffer.AsMemory(0, VsrHeader.HEADER_SIZE); + var requestEncoded = false; + TcpConnectionStream? lastStream = null; + + try + { + while (true) + { + var transientDeadline = isLoginRegister + ? overallDeadline + : Math.Min(overallDeadline, Environment.TickCount64 + VsrTransientFailoverCheckMs); + + var attempt = await SendVsrAttemptAsync(code, body, header, transientDeadline, overallDeadline, + token); + requestEncoded |= attempt.Encoded; + lastStream = attempt.Stream; + + if (attempt.Error is null) + { + // A roster read taken by RedirectAsync must not refund the budget it is about to be charged + // against, or the cap can never be reached. + if (Volatile.Read(ref _leaderRedirectCount) != 0 && Volatile.Read(ref _leaderProbeDepth) == 0) + { + Interlocked.Exchange(ref _leaderRedirectCount, 0); + } + + return attempt.Response!; + } + + if (attempt.Error is IggyInvalidStatusCodeException + { + StatusCode: VsrError.TRANSIENT_NOT_ACCEPTED, FromServer: true + } + && !isLoginRegister + && Environment.TickCount64 < overallDeadline) + { + if (await RedirectAsync(token)) + { + await ConnectAsync(token); + } + + continue; + } + + if (attempt.Error is VsrSessionEvictedException evicted) + { + if (attempt.RequestStarted && !VsrOperations.IsReplaySafeRead(code, isLoginRegister, body.Span)) + { + throw new VsrRequestOutcomeUnknownException(evicted); + } + + ExceptionDispatchInfo.Throw(evicted.Verdict); + } + + if (attempt.RequestStarted + && !VsrOperations.IsReplaySafeRead(code, isLoginRegister, body.Span) + && !IsDefinitiveVerdict(attempt.Error)) + { + throw new VsrRequestOutcomeUnknownException(attempt.Error); + } + + ExceptionDispatchInfo.Throw(attempt.Error); + } + } + catch (OperationCanceledException) + { + if (requestEncoded) + { + await DropVsrConnectionAsync(lastStream); + } + + throw; + } + finally + { + ArrayPool.Shared.Return(headerBuffer); + } + } + + /// + /// Whether the failure carries the server's verdict on this request. A lost connection, a reply frame + /// the client refused or discarded, and a NOT_COMMITTED that outlived its replay deadline all leave the + /// outcome of a request the server may still commit unknowable. + /// + private static bool IsDefinitiveVerdict(Exception error) + { + return error is IggyInvalidStatusCodeException + { + FromServer: true, + StatusCode: not VsrError.TRANSIENT_NOT_COMMITTED + }; + } + + /// + /// One attempt on the current connection: encode the header into , write the + /// frame, and replay it - same session, same request id - while the server answers transiently. + /// + private async ValueTask SendVsrAttemptAsync(int code, ReadOnlyMemory body, Memory header, + long transientDeadline, long readDeadline, CancellationToken token) + { + await _sendingSemaphore.WaitAsync(token); + + var encoded = false; + var requestStarted = false; + byte[]? frameBuffer = null; + + // Read the stream once for the whole attempt. Nothing may swap the field without the sending lock this + // call holds, so the frame and its reply cannot be split across two sockets, and a teardown after the + // lock is gone can tell this connection from a replacement a reconnect installed since. + var stream = _stream; + try + { + // A small request goes out as one write. Two writes cost two syscalls and, with Nagle disabled, two + // TCP segments (two TLS records when encrypted) for what the reference encoder sends as a single + // contiguous frame. Above the threshold the copy costs more than the extra write saves. Renting + // inside the try keeps the semaphore paired with its release even if the pool throws. + if (body.Length <= VsrContiguousFrameLimit) + { + frameBuffer = ArrayPool.Shared.Rent(VsrHeader.HEADER_SIZE + body.Length); + } + + VsrHeader.EncodeRequestHeader(header.Span, _consensusSession, code, body.Span); + + encoded = true; + + var frame = Memory.Empty; + if (frameBuffer is not null) + { + frame = frameBuffer.AsMemory(0, VsrHeader.HEADER_SIZE + body.Length); + header.CopyTo(frame); + body.CopyTo(frame[VsrHeader.HEADER_SIZE..]); + } + + while (true) + { + try + { + // Everything that fails without reaching the socket has to fail before this point: past it a + // failure is reported as an outcome the server alone knows, which for a replicated write + // tells the caller its request may have committed twice. + token.ThrowIfCancellationRequested(); + requestStarted = true; + + if (frameBuffer is not null) + { + await stream.SendAsync(frame, token); + } + else + { + await stream.SendAsync(header, token); + await stream.SendAsync(body, token); + } + + await stream.FlushAsync(token); + + IMemoryOwner response = await ReadVsrReplyAsync(stream, readDeadline, token); + + return VsrAttempt.Ok(response, stream); + } + catch (IggyInvalidStatusCodeException e) when (IsReplayableTransient(e, transientDeadline, + readDeadline)) + { + var governingDeadline = e.StatusCode == VsrError.TRANSIENT_NOT_COMMITTED + ? readDeadline + : transientDeadline; + var remaining = governingDeadline - Environment.TickCount64; + await Task.Delay((int)Math.Clamp(remaining, 0, VsrTransientRetryIntervalMs), token); + } + catch (Exception e) when (IsConnectionException(e)) + { + DropVsrConnectionLocked(stream); + + return VsrAttempt.Failed(encoded, e, requestStarted, stream); + } + catch (OperationCanceledException e) + { + DropVsrConnectionLocked(stream); + + return VsrAttempt.Failed(encoded, e, requestStarted, stream); + } + catch (Exception e) + { + return VsrAttempt.Failed(encoded, e, requestStarted, stream); + } + } + } + catch (OperationCanceledException e) + { + if (encoded) + { + DropVsrConnectionLocked(stream); + } + + return VsrAttempt.Failed(encoded, e, requestStarted, stream); + } + catch (Exception e) + { + return VsrAttempt.Failed(encoded, e, requestStarted, stream); + } + finally + { + if (frameBuffer is not null) + { + ArrayPool.Shared.Return(frameBuffer); + } + + _sendingSemaphore.Release(); + } + } + + private async Task> ReadVsrReplyAsync(TcpConnectionStream stream, long readDeadline, + CancellationToken token) + { + var remaining = readDeadline - Environment.TickCount64; + if (remaining <= 0) + { + throw new IOException($"Timed out after {VsrRequestTimeoutMs} ms waiting for a consensus reply."); + } + + // One timer for the whole reply: the deadline covers the frame, not each partial read, so a per-read + // source would both re-arm the budget and allocate a timer per socket read. + using var readCancellation = CancellationTokenSource.CreateLinkedTokenSource(token); + readCancellation.CancelAfter((int)Math.Min(remaining, VsrRequestTimeoutMs)); + + await ReadExactVsrAsync(stream, _vsrReplyHeaderBuffer, readCancellation.Token, token); + + var command = VsrHeader.PeekCommand(_vsrReplyHeaderBuffer); + if (command == Command2.Eviction) + { + var eviction = VsrHeader.ReadEviction(_vsrReplyHeaderBuffer); + _logger.LogWarning("Consensus session evicted by the server: {Reason}", eviction.Reason); + DropVsrConnectionLocked(stream); + + throw new VsrSessionEvictedException(VsrReplyDecoder.ToException(eviction)); + } + + if (command != Command2.Reply) + { + // Neither a reply nor an eviction: this frame was never an answer to the outstanding request, so + // whatever the peer does send for it would be read as the next request's reply and handed to the + // wrong caller. The size field of a frame the client cannot model is no basis for resynchronising. + DropVsrConnectionLocked(stream); + + throw VsrError.Exception(VsrError.INVALID_COMMAND, + $"Unexpected consensus frame {command} on a client connection."); + } + + int bodySize; + try + { + bodySize = VsrReplyDecoder.ReadBodySize(_vsrReplyHeaderBuffer); + if (VsrHeader.HEADER_SIZE + (long)bodySize > _configuration.MaxResponseFrameSize) + { + throw VsrError.Exception(VsrError.INVALID_COMMAND, + $"Reply frame of {VsrHeader.HEADER_SIZE + bodySize} bytes exceeds the configured maximum of " + + $"{_configuration.MaxResponseFrameSize} bytes."); + } + } + catch + { + // An announced size the client refuses to read - undersized, oversized - leaves the body on the + // wire, so the stream no longer sits on a frame boundary and the next reply would decode body bytes + // as a header. + DropVsrConnectionLocked(stream); + + throw; + } + + if (bodySize == 0) + { + VsrReplyDecoder.Decode(_vsrReplyHeaderBuffer, ReadOnlyMemory.Empty); + + return EmptyMemoryOwner.Instance; + } + + var buffer = ArrayPool.Shared.Rent(bodySize); + try + { + await ReadExactVsrAsync(stream, buffer.AsMemory(0, bodySize), readCancellation.Token, token); + ReadOnlyMemory decoded = VsrReplyDecoder.Decode(_vsrReplyHeaderBuffer, buffer.AsMemory(0, bodySize)); + if (decoded.IsEmpty) + { + ArrayPool.Shared.Return(buffer); + + return EmptyMemoryOwner.Instance; + } + + // The decoded payload is always a suffix of the body - the funnel only strips the leading + // committed result section. + return new PooledMemoryOwner(buffer, bodySize - decoded.Length, decoded.Length); + } + catch + { + ArrayPool.Shared.Return(buffer); + throw; + } + } + + private async ValueTask ReadExactVsrAsync(TcpConnectionStream stream, Memory buffer, + CancellationToken readToken, + CancellationToken token) + { + var totalRead = 0; + while (totalRead < buffer.Length) + { + int readBytes; + try + { + readBytes = await stream.ReadAsync(buffer[totalRead..], readToken); + } + catch (OperationCanceledException) when (!token.IsCancellationRequested) + { + throw new IOException($"Timed out after {VsrRequestTimeoutMs} ms waiting for a consensus reply."); + } + + if (readBytes == 0) + { + throw new IggyZeroBytesException(); + } + + totalRead += readBytes; + } + } + + /// + /// Drops the consensus session and the group state scoped to it. Consumer-group assignments are fenced by + /// a generation the coordinator tracks per session, so carrying them into a new session would fence every + /// poll until the first re-sync. The balanced cursors and the partition counts survive: neither is bound + /// to a session, and dropping them costs a metadata round trip per topic on the next produce. + /// + private void ResetConsensusSession() + { + _consensusSession.Reset(); + _groupState.ClearSessionScoped(); + } + + /// + /// Resets the session on behalf of a caller that does not hold the sending lock, so no request can be + /// encoding against the identity while it is re-armed. + /// + private async ValueTask ResetConsensusSessionAsync() + { + // Taking a disposed semaphore would replace the failure the caller is about to rethrow with an + // ObjectDisposedException, and a disposed client has nothing left to fence. Dispose can still land + // between the check and the wait, so the wait itself has to tolerate it. + if (_disposed || !await TryEnterSendingSemaphoreAsync()) + { + return; + } + + try + { + ResetConsensusSession(); + } + finally + { + _sendingSemaphore.Release(); + } + } + + /// + /// Takes the sending lock for a teardown that must not fail. Returns false once the client is disposed: + /// the caller is unwinding an earlier failure and has nothing left to fence. + /// + private async ValueTask TryEnterSendingSemaphoreAsync() + { + try + { + await _sendingSemaphore.WaitAsync(CancellationToken.None); + return true; + } + catch (ObjectDisposedException) + { + return false; + } + } + + /// + /// Drops the connection along with the session. A late or half-read reply would desync the framing of the + /// next request, so the stream cannot be reused. The caller must hold , + /// which owns every write to . + /// + /// + /// The connection the caller was using. A reconnect that completed in the meantime already closed it and + /// re-armed the session, so dropping anything but the live one would tear down a healthy replacement. + /// + private void DropVsrConnectionLocked(TcpConnectionStream? stream) + { + if (!ReferenceEquals(_stream, stream)) + { + return; + } + + ResetConsensusSession(); + _stream?.Close(); + SetConnectionStateAsync(ConnectionState.Disconnected); + } + + /// Drops the connection on behalf of a caller that no longer holds the sending lock. + private async ValueTask DropVsrConnectionAsync(TcpConnectionStream? stream) + { + // Dispose already closed the stream, and taking a disposed semaphore here would replace the + // cancellation the caller is about to rethrow with an ObjectDisposedException. Dispose can still land + // between the check and the wait, so the wait itself has to tolerate it. + // The request this drop belongs to was cancelled; the drop itself still has to run to completion. + if (_disposed || !await TryEnterSendingSemaphoreAsync()) + { + return; + } + + try + { + DropVsrConnectionLocked(stream); + } + finally + { + _sendingSemaphore.Release(); + } + } + + private static bool IsReplayableTransient(IggyInvalidStatusCodeException error, long transientDeadline, + long readDeadline) + { + if (!error.FromServer) + { + return false; + } + + return error.StatusCode switch + { + VsrError.TRANSIENT_NOT_COMMITTED => Environment.TickCount64 < readDeadline, + VsrError.TRANSIENT_NOT_ACCEPTED => Environment.TickCount64 < transientDeadline, + _ => false + }; + } + + /// Outcome of one call on the current connection. + /// Whether the header was encoded, i.e. whether a request id may have been consumed. + /// The decoded reply payload, non-null exactly when is null. + /// The failure that ended the attempt, or null on success. + /// + /// Whether any byte of the frame was written, which makes the server-side outcome unknowable on failure. + /// + /// + /// The connection the attempt ran on, so a caller that drops it after releasing the sending lock can tell + /// its own connection from a replacement a reconnect installed since. + /// + private readonly record struct VsrAttempt( + bool Encoded, + IMemoryOwner? Response, + Exception? Error, + bool RequestStarted, + TcpConnectionStream? Stream) + { + public static VsrAttempt Ok(IMemoryOwner response, TcpConnectionStream stream) + { + return new VsrAttempt(true, response, null, true, stream); + } + + public static VsrAttempt Failed(bool encoded, Exception error, bool requestStarted, + TcpConnectionStream? stream) + { + return new VsrAttempt(encoded, null, error, requestStarted, stream); + } + } + + /// Owns a pooled buffer while exposing only the decoded payload slice inside it. + internal sealed class PooledMemoryOwner(byte[] buffer, int start, int length) : IMemoryOwner + { + private int _disposed; + + public Memory Memory => buffer.AsMemory(start, length); + + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) == 0) + { + ArrayPool.Shared.Return(buffer); + } + } + } +} diff --git a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs index a2071c3513..146c93ee13 100644 --- a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs +++ b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TcpMessageStream.cs @@ -22,7 +22,6 @@ using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Security.Cryptography.X509Certificates; -using System.Text; using Apache.Iggy.Configuration; using Apache.Iggy.ConnectionStream; using Apache.Iggy.Contracts; @@ -35,15 +34,17 @@ using Apache.Iggy.Mappers; using Apache.Iggy.Messages; using Apache.Iggy.Utils; +using Apache.Iggy.Vsr; using Microsoft.Extensions.Logging; using Partitioning = Apache.Iggy.Kinds.Partitioning; namespace Apache.Iggy.IggyClient.Implementations; /// -/// A TCP client for interacting with the Iggy server. +/// A TCP client for interacting with the Iggy server over the consensus (VSR) framing. The framed request +/// path, leader redirection and register handshake live in TcpMessageStream.Vsr.cs. /// -public sealed class TcpMessageStream : IIggyClient +public sealed partial class TcpMessageStream : IIggyClient { private const int InvalidCommandStatus = 3; @@ -57,18 +58,28 @@ public sealed class TcpMessageStream : IIggyClient ]; private readonly IggyClientConfigurator _configuration; + private readonly SemaphoreSlim _connectGate = new(1, 1); private readonly EventAggregator _connectionEvents; private readonly SemaphoreSlim _connectionSemaphore; private readonly ILogger _logger; - private readonly byte[] _responseHeaderBuffer = new byte[BufferSizes.EXPECTED_RESPONSE_SIZE]; private readonly SemaphoreSlim _sendingSemaphore; private string _currentAddress = string.Empty; private X509Certificate2Collection _customCaStore = []; - private bool _isConnecting; + private volatile bool _disposed; + private int _isConnecting; private DateTimeOffset _lastConnectionTime; - private ConnectionState _state = ConnectionState.Disconnected; + private int _leaderRedirectCount; + + // Both are written by the connect and redirect paths, which do not hold the sending semaphore the request + // paths read them under, so they are accessed through Interlocked rather than as plain fields. Losing an + // update to the skip flag leaves a connection reporting Connected that never authenticated; losing one to + // the redirect counter over- or under-spends the redirect budget. + private int _skipAutoLoginOnce; + private volatile ConnectionState _state = ConnectionState.Disconnected; private TcpConnectionStream _stream = null!; + private bool IsConnecting => Volatile.Read(ref _isConnecting) != 0; + internal TcpMessageStream(IggyClientConfigurator configuration, ILoggerFactory loggerFactory) { _configuration = configuration; @@ -84,10 +95,14 @@ internal TcpMessageStream(IggyClientConfigurator configuration, ILoggerFactory l /// public void Dispose() { + _disposed = true; _stream?.Close(); _stream?.Dispose(); + + SetConnectionStateAsync(ConnectionState.Disconnected); _sendingSemaphore.Dispose(); _connectionSemaphore.Dispose(); + _connectGate.Dispose(); _connectionEvents.Clear(); } @@ -273,6 +288,7 @@ public async Task DeleteTopicAsync(Identifier streamId, Identifier topicId, Canc TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_TOPIC_CODE); await SendAckAsync(payload, token); + _groupState.InvalidatePartitionCount(new TopicKey(streamId, topicId)); } /// @@ -287,30 +303,38 @@ public async Task PurgeTopicAsync(Identifier streamId, Identifier topicId, Cance /// - public Task SendMessagesAsync(Identifier streamId, Identifier topicId, Partitioning partitioning, - IList messages, CancellationToken token = default) + public Task SendMessagesAsync(Identifier streamId, Identifier topicId, + Partitioning partitioning, IList messages, CancellationToken token = default) { + if (NeedsClientSidePartitioning(partitioning)) + { + return SendMessagesResolvedAsync(streamId, topicId, partitioning, messages, token); + } + return SendMessagesCoreAsync(streamId, topicId, partitioning, AsSpan(messages), token); } /// - public Task SendMessagesAsync(Identifier streamId, Identifier topicId, Partitioning partitioning, - Message message, CancellationToken token = default) + public Task SendMessagesAsync(Identifier streamId, Identifier topicId, + Partitioning partitioning, Message message, CancellationToken token = default) { + if (NeedsClientSidePartitioning(partitioning)) + { + return SendMessagesResolvedAsync(streamId, topicId, partitioning, [message], token); + } + ReadOnlySpan span = [message]; return SendMessagesCoreAsync(streamId, topicId, partitioning, span, token); } - /// - public async Task FlushUnsavedBufferAsync(Identifier streamId, Identifier topicId, uint partitionId, bool fsync, + /// + /// This feature is not supported by the server. + /// + /// + public Task FlushUnsavedBufferAsync(Identifier streamId, Identifier topicId, uint partitionId, bool fsync, CancellationToken token = default) { - var message = TcpContracts.FlushUnsavedBuffer(streamId, topicId, partitionId, fsync); - - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.FLUSH_UNSAVED_BUFFER_CODE); - - await SendAckAsync(payload, token); + throw new FeatureUnavailableException(); } /// @@ -324,38 +348,23 @@ public async Task PollMessagesAsync(Identifier streamId, Identif } /// - public async Task PollMessagesRentedAsync(Identifier streamId, Identifier topicId, + public Task PollMessagesRentedAsync(Identifier streamId, Identifier topicId, uint? partitionId, Consumer consumer, PollingStrategy pollingStrategy, uint count, bool autoCommit, CancellationToken token = default) { ThrowIfAutoCommitWithEncryptor(autoCommit); - var messageBufferSize = CalculateMessageBufferSize(streamId, topicId, consumer); - var payloadBufferSize = CalculatePayloadBufferSize(messageBufferSize); - var payload = ArrayPool.Shared.Rent(payloadBufferSize); - IMemoryOwner? responseBuffer = null; - - try - { - TcpContracts.GetMessages(payload.AsSpan().Slice(8, messageBufferSize), consumer, streamId, - topicId, pollingStrategy, count, autoCommit, partitionId); - BinaryPrimitives.WriteInt32LittleEndian(payload.AsSpan()[..4], messageBufferSize + 4); - BinaryPrimitives.WriteInt32LittleEndian(payload.AsSpan()[4..8], CommandCodes.POLL_MESSAGES_CODE); - - responseBuffer = await SendWithResponseAsync(payload.AsMemory(0, payloadBufferSize), token); - return BinaryMapper.MapRentedMessages(responseBuffer.Memory, responseBuffer, - _configuration.MessageEncryptor); - } - catch - { - responseBuffer?.Dispose(); - throw; - } - finally + // The broker routes explicit partitions only, so a group poll picks one of the member's assigned + // partitions client-side. + if (consumer.Type == ConsumerType.ConsumerGroup && partitionId is null) { - ArrayPool.Shared.Return(payload); + return PollGroupMessagesRentedAsync(streamId, topicId, consumer, pollingStrategy, count, autoCommit, + token); } + + return PollPartitionMessagesRentedAsync(streamId, topicId, partitionId, consumer, pollingStrategy, count, + autoCommit, token); } /// @@ -462,6 +471,7 @@ public async Task DeleteConsumerGroupAsync(Identifier streamId, Identifier topic TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_CONSUMER_GROUP_CODE); await SendAckAsync(payload, token); + _groupState.DeregisterGroup(new GroupKey(streamId, topicId, groupId)); } /// @@ -473,6 +483,10 @@ public async Task JoinConsumerGroupAsync(Identifier streamId, Identifier topicId TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.JOIN_CONSUMER_GROUP_CODE); await SendAckAsync(payload, token); + + // A join rebalances the group, so whatever this client holds for it is a generation behind and every + // poll under it would be fenced until the first re-sync. + _groupState.InvalidateAssignment(new GroupKey(streamId, topicId, groupId)); } /// @@ -484,6 +498,7 @@ public async Task LeaveConsumerGroupAsync(Identifier streamId, Identifier topicI TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LEAVE_CONSUMER_GROUP_CODE); await SendAckAsync(payload, token); + _groupState.DeregisterGroup(new GroupKey(streamId, topicId, groupId)); } /// @@ -495,6 +510,7 @@ public async Task DeletePartitionsAsync(Identifier streamId, Identifier topicId, TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.DELETE_PARTITIONS_CODE); await SendAckAsync(payload, token); + _groupState.InvalidatePartitionCount(new TopicKey(streamId, topicId)); } /// @@ -506,6 +522,7 @@ public async Task CreatePartitionsAsync(Identifier streamId, Identifier topicId, TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.CREATE_PARTITIONS_CODE); await SendAckAsync(payload, token); + _groupState.InvalidatePartitionCount(new TopicKey(streamId, topicId)); } /// @@ -578,6 +595,8 @@ public async Task PingAsync(CancellationToken token = default) TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.PING_CODE); await SendAckAsync(payload, token); + + await RefreshGroupAssignmentsAsync(token); } /// @@ -611,31 +630,9 @@ public async Task SendBinaryRequestAsync(uint code, byte[] payload, Canc } /// - public async Task ConnectAsync(CancellationToken token = default) + public Task ConnectAsync(CancellationToken token = default) { - if (_state is ConnectionState.Connected - or ConnectionState.Authenticating - or ConnectionState.Authenticated) - { - _logger.LogWarning("Connection is already connected"); - return; - } - - if (_lastConnectionTime != DateTimeOffset.MinValue) - { - await Task.Delay(_configuration.ReconnectionSettings.InitialDelay, token); - } - - SetConnectionStateAsync(ConnectionState.Connecting); - _isConnecting = true; - try - { - await TryEstablishConnectionAsync(token); - } - finally - { - _isConnecting = false; - } + return ConnectAsync(true, token); } /// @@ -775,30 +772,8 @@ public async Task ChangePasswordAsync(Identifier userId, string currentPassword, throw new NotConnectedException(); } - // TODO: Add binary protocol version - var message = TcpContracts.LoginUser(userName, password, SdkVersion.Value, "csharp-sdk"); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LOGIN_USER_CODE); - - SetConnectionStateAsync(ConnectionState.Authenticating); - using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, token); - - if (responseBuffer.Memory.Length == 0) - { - return null; - } - - var userId = BinaryPrimitives.ReadInt32LittleEndian(responseBuffer.Memory.Span[..responseBuffer.Memory.Length]); - SetConnectionStateAsync(ConnectionState.Authenticated); - - if (await RedirectAsync(token)) - { - await ConnectAsync(token); - return await LoginUserAsync(userName, password, token); - } - - var authResponse = new AuthResponse(userId, null); - return authResponse; + return await LoginRegisterAsync(CommandCodes.LOGIN_REGISTER_CODE, + LoginRegister.Serialize(userName, password), token); } /// @@ -808,7 +783,19 @@ public async Task LogoutUserAsync(CancellationToken token = default) var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LOGOUT_USER_CODE); - await SendAckAsync(payload, token); + try + { + await SendAckAsync(payload, token); + } + finally + { + await ResetConsensusSessionAsync(); + + if (_state == ConnectionState.Authenticated) + { + SetConnectionStateAsync(ConnectionState.Connected); + } + } } /// @@ -860,29 +847,86 @@ public async Task DeletePersonalAccessTokenAsync(string name, CancellationToken /// public async Task LoginWithPersonalAccessTokenAsync(string token, CancellationToken ct = default) { - var message = TcpContracts.LoginWithPersonalAccessToken(token); - var payload = new byte[4 + BufferSizes.INITIAL_BYTES_LENGTH + message.Length]; - TcpMessageStreamHelpers.CreatePayload(payload, message, CommandCodes.LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE); - - SetConnectionStateAsync(ConnectionState.Authenticating); - using IMemoryOwner responseBuffer = await SendWithResponseAsync(payload, ct); + return await LoginRegisterAsync(CommandCodes.LOGIN_REGISTER_WITH_PAT_CODE, + LoginRegister.SerializeWithPersonalAccessToken(token), ct); + } - if (responseBuffer.Memory.Length == 0) + /// + /// Connects, optionally without the configured auto login. A caller that authenticates itself right + /// after the connect passes false, so the connect does not spend a round trip on credentials the + /// caller is about to replace. + /// + private async Task ConnectAsync(bool autoLogin, CancellationToken token) + { + if (_state is ConnectionState.Connected + or ConnectionState.Authenticating + or ConnectionState.Authenticated) { - return null; + _logger.LogWarning("Connection is already connected"); + return; } - var userId = BinaryPrimitives.ReadInt32LittleEndian(responseBuffer.Memory.Span[..4]); + await _connectGate.WaitAsync(token); + Interlocked.Exchange(ref _isConnecting, 1); + try + { + if (_state is ConnectionState.Connected + or ConnectionState.Authenticating + or ConnectionState.Authenticated) + { + return; + } - SetConnectionStateAsync(ConnectionState.Authenticated); + if (_lastConnectionTime != DateTimeOffset.MinValue) + { + await Task.Delay(_configuration.ReconnectionSettings.InitialDelay, token); + } - if (await RedirectAsync(ct)) + SetConnectionStateAsync(ConnectionState.Connecting); + await TryEstablishConnectionAsync(autoLogin, token); + } + finally { - await ConnectAsync(ct); - return await LoginWithPersonalAccessTokenAsync(token, ct); + Interlocked.Exchange(ref _isConnecting, 0); + _connectGate.Release(); } + } - return new AuthResponse(userId, null); + private async Task PollPartitionMessagesRentedAsync(Identifier streamId, Identifier topicId, + uint? partitionId, Consumer consumer, PollingStrategy pollingStrategy, uint count, bool autoCommit, + CancellationToken token) + { + var messageBufferSize = CalculateMessageBufferSize(streamId, topicId, consumer); + var payloadBufferSize = CalculatePayloadBufferSize(messageBufferSize); + var payload = ArrayPool.Shared.Rent(payloadBufferSize); + IMemoryOwner? responseBuffer = null; + + try + { + TcpContracts.GetMessages(payload.AsSpan().Slice(8, messageBufferSize), consumer, streamId, + topicId, pollingStrategy, count, autoCommit, partitionId); + BinaryPrimitives.WriteInt32LittleEndian(payload.AsSpan()[..4], messageBufferSize + 4); + BinaryPrimitives.WriteInt32LittleEndian(payload.AsSpan()[4..8], CommandCodes.POLL_MESSAGES_CODE); + + responseBuffer = await SendWithResponseAsync(payload.AsMemory(0, payloadBufferSize), token); + if (responseBuffer.Memory.Length == 0) + { + responseBuffer.Dispose(); + return EmptyPolledMessages; + } + + return BinaryMapper.MapRentedMessages(responseBuffer.Memory, responseBuffer, + _configuration.MessageEncryptor); + } + catch + { + responseBuffer?.Dispose(); + throw; + } + finally + { + ArrayPool.Shared.Return(payload); + } } // Server-side autoCommit commits the batch offset before the client decrypts, so a decryption failure @@ -897,8 +941,8 @@ private void ThrowIfAutoCommitWithEncryptor(bool autoCommit) } } - private Task SendMessagesCoreAsync(Identifier streamId, Identifier topicId, Partitioning partitioning, - ReadOnlySpan messages, CancellationToken token) + private Task SendMessagesCoreAsync(Identifier streamId, Identifier topicId, + Partitioning partitioning, ReadOnlySpan messages, CancellationToken token) { var encryptor = _configuration.MessageEncryptor; @@ -924,15 +968,17 @@ private Task SendMessagesCoreAsync(Identifier streamId, Identifier topicId, Part throw; } - return SendAckAndDisposeAsync(payloadBuffer, payloadBufferSize, token); + return SendConfirmedAndDisposeAsync(payloadBuffer, payloadBufferSize, token); } - private async Task SendAckAndDisposeAsync(IMemoryOwner payloadBuffer, int payloadBufferSize, - CancellationToken token) + private async Task SendConfirmedAndDisposeAsync(IMemoryOwner payloadBuffer, + int payloadBufferSize, CancellationToken token) { try { - await SendAckAsync(payloadBuffer.Memory[..payloadBufferSize], token); + using IMemoryOwner responseBuffer = + await SendWithResponseAsync(payloadBuffer.Memory[..payloadBufferSize], token); + return BinaryMapper.MapSendMessages(responseBuffer.Memory.Span); } finally { @@ -962,59 +1008,101 @@ private static int FillSendMessagesPayload(Span buffer, int maxMessageBuff return messageBufferSize; } - private async Task TryEstablishConnectionAsync(CancellationToken token) + private async Task TryEstablishConnectionAsync(bool autoLogin, CancellationToken token) { var retryCount = 0; + var redirects = 0; var delay = _configuration.ReconnectionSettings.InitialDelay; do { - // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract - _stream?.Close(); - _stream?.Dispose(); + // The sending semaphore owns every write to _stream, so an in-flight request never observes the + // field changing between its write and its reply. + await _sendingSemaphore.WaitAsync(token); + try + { + _stream?.Dispose(); + + ResetConsensusSession(); + } + finally + { + _sendingSemaphore.Release(); + } if (string.IsNullOrEmpty(_currentAddress)) { _currentAddress = _configuration.BaseAddress; } - var urlPortSplitter = _currentAddress.Split(":"); - if (urlPortSplitter.Length > 2) + if (!ServerAddress.TryParse(_currentAddress, out var host, out var port)) { throw new InvalidBaseAddressException(); } + Socket? socket = null; try { - var socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp); + socket = new Socket(ServerAddress.AddressFamilyOf(host), SocketType.Stream, ProtocolType.Tcp); socket.SendBufferSize = _configuration.SendBufferSize; socket.ReceiveBufferSize = _configuration.ReceiveBufferSize; - await socket.ConnectAsync(urlPortSplitter[0], int.Parse(urlPortSplitter[1]), token); + // The protocol is request/reply, so a write is always the last one before + // the client blocks on the answer and Nagle has nothing to coalesce it with - it only delays the + // trailing segment of a large request until the previous one is acked. + socket.NoDelay = true; + + await socket.ConnectAsync(host, port, token); socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.KeepAlive, true); socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.TcpKeepAliveTime, 5); - SetConnectionStateAsync(ConnectionState.Connected); - _lastConnectionTime = DateTimeOffset.UtcNow; - - _stream = _configuration.TlsSettings.Enabled switch + var connectionStream = _configuration.TlsSettings.Enabled switch { true => await CreateSslStreamAndAuthenticate(socket, _configuration.TlsSettings), false => new TcpConnectionStream(new NetworkStream(socket, true)) }; - if (_configuration.AutoLoginSettings.Enabled) + await _sendingSemaphore.WaitAsync(token); + try + { + _stream = connectionStream; + } + finally + { + _sendingSemaphore.Release(); + } + + SetConnectionStateAsync(ConnectionState.Connected); + _lastConnectionTime = DateTimeOffset.UtcNow; + + socket = null; + + if (await RedirectAsync(token)) + { + await BackoffOrThrowAsync(); + continue; + } + + if (autoLogin && _configuration.AutoLoginSettings.Enabled && !ConsumeSkipAutoLogin()) { _logger.LogInformation("Auto login enabled. Trying to login with credentials: {Username}", _configuration.AutoLoginSettings.Username); await LoginUserAsync(_configuration.AutoLoginSettings.Username, _configuration.AutoLoginSettings.Password, token); + + if (await RedirectAsync(token)) + { + await BackoffOrThrowAsync(); + continue; + } } break; } catch (Exception e) { + socket?.Dispose(); + _logger.LogError(e, "Failed to connect"); if (!_configuration.ReconnectionSettings.Enabled || @@ -1045,37 +1133,38 @@ await LoginUserAsync(_configuration.AutoLoginSettings.Username, await Task.Delay(delay, token); } } while (true); - } - private async Task GetCurrentLeaderNodeAsync(CancellationToken token) - { - try + // A redirect restarts the loop without passing through the catch, so it spends no retry and waits for + // nothing. Its own budget rather than the reconnection one: following the roster to the leader is how a + // VSR connect succeeds, and it has to work with reconnection turned off. + async Task BackoffOrThrowAsync() { - var clusterMetadata = await GetClusterMetadataAsync(token); - if (clusterMetadata == null) - { - return null; - } - - // Single-node cluster (clustering disabled) - no redirection needed - if (clusterMetadata.Nodes.Count() == 1) - { - return null; - } - - var leaderNode = clusterMetadata.Nodes.FirstOrDefault(x => x.Role == ClusterNodeRole.Leader); - if (leaderNode == null) + if (++redirects > VsrMaxLeaderRedirects) { + SetConnectionStateAsync(ConnectionState.Disconnected); throw new MissingLeaderException(); } - return leaderNode; + _logger.LogInformation("Following leader redirect {Redirect} to {Address}", redirects, _currentAddress); + + await Task.Delay(delay, token); } - // todo: change after error refactoring, error code 5 is for feature not supported - catch (IggyInvalidStatusCodeException e) when (e.StatusCode == 5) + } + + /// + /// Whether this connect was triggered by a login or register request that will re-authenticate itself, + /// so the auto-login must sit this one out. Consumes the flag. + /// + private bool ConsumeSkipAutoLogin() + { + if (Interlocked.Exchange(ref _skipAutoLoginOnce, 0) == 0) { - return null; + return false; } + + _logger.LogInformation("Skipping auto login for a replayed register request"); + + return true; } private async Task CreateSslStreamAndAuthenticate(Socket socket, TlsSettings tlsSettings) @@ -1104,7 +1193,7 @@ private async Task> SendWithResponseAsync(ReadOnlyMemory> HandleReconnectionAsync(ReadOnlyMemory> SendRawAsync(ReadOnlyMemory payload, CancellationToken token) + private Task> SendRawAsync(ReadOnlyMemory payload, CancellationToken token) { + ObjectDisposedException.ThrowIf(_disposed, this); + if (_state is ConnectionState.Disconnected or ConnectionState.Connecting) { throw new NotConnectedException(); } - await _sendingSemaphore.WaitAsync(token); - - try - { - await _stream.SendAsync(payload, token); - await _stream.FlushAsync(token); - - // Read the 8-byte header (4 bytes status + 4 bytes length) - var totalRead = 0; - while (totalRead < BufferSizes.EXPECTED_RESPONSE_SIZE) - { - var readBytes - = await _stream.ReadAsync( - _responseHeaderBuffer.AsMemory(totalRead, BufferSizes.EXPECTED_RESPONSE_SIZE - totalRead), - token); - if (readBytes == 0) - { - throw new IggyZeroBytesException(); - } - - totalRead += readBytes; - } - - var response = TcpMessageStreamHelpers.GetResponseLengthAndStatus(_responseHeaderBuffer); - - if (response.Status != 0) - { - if (response.Length == 0) - { - throw new IggyInvalidStatusCodeException(response.Status, - $"Invalid response status code: {response.Status}"); - } - - - using var errorBuffer = ArrayPoolHelper.Rent(response.Length); - totalRead = 0; - while (totalRead < response.Length) - { - var readBytes - = await _stream.ReadAsync(errorBuffer.Memory.Slice(totalRead, response.Length - totalRead), - token); - if (readBytes == 0) - { - throw new IggyZeroBytesException(); - } - - totalRead += readBytes; - } - - throw new InvalidResponseException(Encoding.UTF8.GetString(errorBuffer.Memory.Span)); - } - - if (response.Length == 0) - { - return EmptyMemoryOwner.Instance; - } - - var responseBuffer = ArrayPoolHelper.Rent(response.Length); - try - { - totalRead = 0; - while (totalRead < response.Length) - { - var readBytes - = await _stream.ReadAsync(responseBuffer.Memory.Slice(totalRead, response.Length - totalRead), - token); - - if (readBytes == 0) - { - throw new IggyZeroBytesException(); - } - - totalRead += readBytes; - } - } - catch - { - responseBuffer.Dispose(); - throw; - } - - return responseBuffer; - } - finally - { - _sendingSemaphore.Release(); - } + return SendRawVsrAsync(payload, token); } private static bool IsConnectionException(Exception ex) @@ -1357,30 +1362,6 @@ private bool RemoteCertificateValidationCallback(object sender, X509Certificate? return false; } - private async Task RedirectAsync(CancellationToken token) - { - var currentLeaderNode = await GetCurrentLeaderNodeAsync(token); - if (currentLeaderNode == null) - { - return false; - } - - var leaderAddress = $"{currentLeaderNode.Ip}:{currentLeaderNode.Endpoints.Tcp}"; - if (leaderAddress == _currentAddress) - { - return false; - } - - _currentAddress = leaderAddress; - - _logger.LogInformation("Leader address changed. Trying to reconnect to {Address}", - leaderAddress); - - _stream.Close(); - SetConnectionStateAsync(ConnectionState.Disconnected); - return true; - } - internal sealed class EmptyMemoryOwner : IMemoryOwner { public static readonly EmptyMemoryOwner Instance = new(); diff --git a/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TransientHttpRetryHandler.cs b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TransientHttpRetryHandler.cs new file mode 100644 index 0000000000..014ee53f3b --- /dev/null +++ b/foreign/csharp/Iggy_SDK/IggyClient/Implementations/TransientHttpRetryHandler.cs @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Net; + +namespace Apache.Iggy.IggyClient.Implementations; + +/// +/// Replays requests the server answered with a retryable status: 503 (no caught-up primary, full +/// pipeline, view-change cancel, shard write budget) and 429 (session write cap). The server signals +/// both as transient with a Retry-After; the binary transports absorb the equivalent frames in their +/// in-client replay loop, so HTTP replays here for transport parity. A write 504 stays terminal: the +/// request may still commit, so its outcome is unknown and must surface to the caller. +/// +internal sealed class TransientHttpRetryHandler : DelegatingHandler +{ + private static readonly TimeSpan RetryDeadline = TimeSpan.FromSeconds(30); + private static readonly TimeSpan RetryInterval = TimeSpan.FromMilliseconds(50); + + public TransientHttpRetryHandler(HttpMessageHandler innerHandler) : base(innerHandler) + { + } + + protected override async Task SendAsync(HttpRequestMessage request, + CancellationToken cancellationToken) + { + var deadline = Environment.TickCount64 + (long)RetryDeadline.TotalMilliseconds; + byte[]? bufferedContent = null; + if (request.Content is not null) + { + bufferedContent = await request.Content.ReadAsByteArrayAsync(cancellationToken); + } + + while (true) + { + HttpResponseMessage response = await base.SendAsync(CloneRequest(request, bufferedContent), + cancellationToken); + if (!IsRetryable(request.Method, response.StatusCode) || Environment.TickCount64 >= deadline) + { + return response; + } + + var delay = response.Headers.RetryAfter?.Delta ?? RetryInterval; + response.Dispose(); + await Task.Delay(delay, cancellationToken); + } + } + + private static bool IsRetryable(HttpMethod method, HttpStatusCode statusCode) + { + if (statusCode is HttpStatusCode.ServiceUnavailable or HttpStatusCode.TooManyRequests) + { + return true; + } + + // A read that timed out in the partition plane (e.g. a poll racing the reconciler that has not yet + // materialised a fresh topic's partition group) has no side effects, so it replays. A write 504 is an + // unknown outcome and stays terminal. + return statusCode == HttpStatusCode.GatewayTimeout && method == HttpMethod.Get; + } + + /// + /// An is single-use, so every attempt sends a copy built from the + /// buffered body. + /// + private static HttpRequestMessage CloneRequest(HttpRequestMessage request, byte[]? bufferedContent) + { + var clone = new HttpRequestMessage(request.Method, request.RequestUri) + { + Version = request.Version, + VersionPolicy = request.VersionPolicy + }; + + foreach (var header in request.Headers) + { + clone.Headers.TryAddWithoutValidation(header.Key, header.Value); + } + + if (bufferedContent is not null) + { + clone.Content = new ByteArrayContent(bufferedContent); + foreach (var header in request.Content!.Headers) + { + clone.Content.Headers.TryAddWithoutValidation(header.Key, header.Value); + } + } + + foreach (var option in request.Options) + { + clone.Options.Set(new HttpRequestOptionsKey(option.Key), option.Value); + } + + return clone; + } +} diff --git a/foreign/csharp/Iggy_SDK/Iggy_SDK.csproj b/foreign/csharp/Iggy_SDK/Iggy_SDK.csproj index 776883aacd..79a6390366 100644 --- a/foreign/csharp/Iggy_SDK/Iggy_SDK.csproj +++ b/foreign/csharp/Iggy_SDK/Iggy_SDK.csproj @@ -27,7 +27,7 @@ net8.0;net10.0 Apache.Iggy Apache.Iggy - 0.8.1-edge.4 + 0.9.0-edge.1 true @@ -69,6 +69,7 @@ + diff --git a/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs b/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs index e65a4ef909..f5be657f20 100644 --- a/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs +++ b/foreign/csharp/Iggy_SDK/Mappers/BinaryMapper.cs @@ -604,8 +604,16 @@ internal static Dictionary MapHeaders(ReadOnlySpan ReadOnlySpan value = payload[position..(position + valueLength)]; position += valueLength; - headers[new HeaderKey { Kind = keyKind, Value = keyValue }] = - new HeaderValue { Kind = valueKind, Value = value.ToArray() }; + headers[new HeaderKey + { + Kind = keyKind, + Value = keyValue + }] = + new HeaderValue + { + Kind = valueKind, + Value = value.ToArray() + }; } return headers; @@ -682,8 +690,16 @@ internal static Dictionary MapHeaders(ReadOnlySpan ReadOnlySpan value = payload[position..(position + valueLength)]; position += valueLength; - headers[new HeaderKey { Kind = keyKind, Value = keyValue }] = - new HeaderValue { Kind = valueKind, Value = value.ToArray() }; + headers[new HeaderKey + { + Kind = keyKind, + Value = keyValue + }] = + new HeaderValue + { + Kind = valueKind, + Value = value.ToArray() + }; } return headers; @@ -740,6 +756,47 @@ internal static IReadOnlyList MapStreams(ReadOnlySpan payl return streams.AsReadOnly(); } + /// + /// Maps a send messages reply: [count:4][stream_id:4][topic_id:4][partition_id:4][base_offset:8]*. + /// + /// + /// Strict: the payload is the whole reply body, so any length that does not match the count + /// is a shape this build cannot read, not a prefix of a larger value. Entries are read at a + /// fixed 20-byte stride, so tolerating a tail would return garbage as a successful decode. + /// + internal static SendMessagesResponse MapSendMessages(ReadOnlySpan payload) + { + const int confirmationSize = 4 + 4 + 4 + 8; + + if (payload.Length < 4) + { + throw new InvalidResponseException("Send messages reply is shorter than the confirmation count prefix."); + } + + var count = BinaryPrimitives.ReadUInt32LittleEndian(payload[..4]); + if (payload.Length - 4 != count * (long)confirmationSize) + { + throw new InvalidResponseException( + $"Send messages reply length {payload.Length} does not match {count} confirmations."); + } + + var confirmations = new SendMessagesConfirmation[count]; + var position = 4; + for (var i = 0; i < confirmations.Length; i++) + { + confirmations[i] = new SendMessagesConfirmation + { + StreamId = BinaryPrimitives.ReadUInt32LittleEndian(payload[position..(position + 4)]), + TopicId = BinaryPrimitives.ReadUInt32LittleEndian(payload[(position + 4)..(position + 8)]), + PartitionId = BinaryPrimitives.ReadUInt32LittleEndian(payload[(position + 8)..(position + 12)]), + BaseOffset = BinaryPrimitives.ReadUInt64LittleEndian(payload[(position + 12)..(position + 20)]) + }; + position += confirmationSize; + } + + return new SendMessagesResponse { Confirmations = confirmations }; + } + internal static StreamResponse MapStream(ReadOnlySpan payload) { var (stream, position) = MapToStream(payload, 0); diff --git a/foreign/csharp/Iggy_SDK/Publishers/BackgroundMessageProcessor.cs b/foreign/csharp/Iggy_SDK/Publishers/BackgroundMessageProcessor.cs index 969118e408..cae62bfb5d 100644 --- a/foreign/csharp/Iggy_SDK/Publishers/BackgroundMessageProcessor.cs +++ b/foreign/csharp/Iggy_SDK/Publishers/BackgroundMessageProcessor.cs @@ -17,6 +17,7 @@ using System.Threading.Channels; using Apache.Iggy.Enums; +using Apache.Iggy.Exceptions; using Apache.Iggy.IggyClient; using Apache.Iggy.Messages; using Apache.Iggy.Utils; @@ -55,6 +56,11 @@ internal sealed partial class BackgroundMessageProcessor : IAsyncDisposable private int _inFlight; private PooledBufferWriter _payloadBuffer; + // Set by DisposeAsync so the loop finishes what it has instead of being cancelled mid-send. A send cancelled + // after its first byte is reported as an outcome only the server knows, which would make every ordinary + // shutdown publish a batch that may have committed twice. + private int _stopping; + public BackgroundMessageProcessor(IIggyClient client, IggyPublisherConfig config, ILoggerFactory loggerFactory) { _client = client; @@ -97,7 +103,7 @@ public async ValueTask DisposeAsync() _client.UnsubscribeConnectionEvents(ClientOnOnConnectionStateChanged); - await _cancellationTokenSource.CancelAsync(); + Volatile.Write(ref _stopping, 1); _writer.TryComplete(); var backgroundTaskTimedOut = false; @@ -118,9 +124,14 @@ public async ValueTask DisposeAsync() { LogBackgroundProcessorError(e); } + finally + { + await _cancellationTokenSource.CancelAsync(); + } } else { + await _cancellationTokenSource.CancelAsync(); DrainAndDispose(); } @@ -244,9 +255,11 @@ private async Task RunBackgroundProcessor(CancellationToken ct) { while (!ct.IsCancellationRequested) { + var stopping = Volatile.Read(ref _stopping) != 0; + if (!_canSend) { - if (!await timer.WaitForNextTickAsync(ct)) + if (stopping || !await timer.WaitForNextTickAsync(ct)) { break; } @@ -256,7 +269,7 @@ private async Task RunBackgroundProcessor(CancellationToken ct) if (!AccumulateBatch()) { - if (!await timer.WaitForNextTickAsync(ct)) + if (stopping || !await timer.WaitForNextTickAsync(ct)) { break; } @@ -477,6 +490,16 @@ private async Task SendWithRetry(List wire, CancellationToken ct) // Disposal cancellation, not a send failure; let the loop's cancellation handling take over. throw; } + catch (VsrRequestOutcomeUnknownException ex) + { + // May already have committed - report it as its own type rather than folding it into the + // generic failure path, so a subscriber can tell "not sent" from "possibly sent twice". + LogFailedToSendBatch(ex, wire.Count); + if (_messageBatchErrorAggregator.HasSubscribers) + { + _messageBatchErrorAggregator.Publish(new MessageBatchFailedEventArgs(ex, SnapshotForFailure(wire))); + } + } catch (Exception ex) { LogFailedToSendBatch(ex, wire.Count); @@ -507,6 +530,20 @@ private async Task SendWithRetry(List wire, CancellationToken ct) // remaining attempts instantly and publish a misleading "failed after N attempts" event. throw; } + catch (VsrRequestOutcomeUnknownException ex) + { + // The send may already have committed. The partition plane is sessionless - it keeps no + // client-table entry to deduplicate an append against - so a retry cannot be matched to the + // original under any client id and would simply append the batch twice. Report it instead. + LogFailedToSendBatch(ex, wire.Count); + if (_messageBatchErrorAggregator.HasSubscribers) + { + _messageBatchErrorAggregator.Publish(new MessageBatchFailedEventArgs(ex, SnapshotForFailure(wire), + attempt + 1)); + } + + return; + } catch (Exception ex) { lastException = ex; diff --git a/foreign/csharp/Iggy_SDK/Publishers/IggyPublisher.cs b/foreign/csharp/Iggy_SDK/Publishers/IggyPublisher.cs index c90bedfa60..7ec63bebf7 100644 --- a/foreign/csharp/Iggy_SDK/Publishers/IggyPublisher.cs +++ b/foreign/csharp/Iggy_SDK/Publishers/IggyPublisher.cs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +using Apache.Iggy.Contracts; using Apache.Iggy.Enums; using Apache.Iggy.Exceptions; using Apache.Iggy.IggyClient; @@ -173,7 +174,7 @@ public async Task InitAsync(CancellationToken ct = default) await Client.ConnectAsync(ct); LogInitializingPublisher(Config.StreamId, Config.TopicId); - if (Config.CreateIggyClient) + if (!string.IsNullOrEmpty(Config.Login) && !Config.CreateIggyClient) { await Client.LoginUserAsync(Config.Login, Config.Password, ct); LogUserLoggedIn(Config.Login); @@ -281,8 +282,12 @@ await Client.CreateTopicAsync(Config.StreamId, Config.TopicName, Config.TopicPar /// /// The messages to send. /// Cancellation token to cancel the send operation. + /// + /// Commit confirmations for a direct send. Empty when background sending is enabled: the + /// processor merges queued batches, so no 1:1 mapping to this call exists. + /// /// Thrown when attempting to send before initialization. - public async Task SendMessagesAsync(IList messages, CancellationToken ct = default) + public async Task SendMessagesAsync(IList messages, CancellationToken ct = default) { if (!IsInitialized) { @@ -292,20 +297,19 @@ public async Task SendMessagesAsync(IList messages, CancellationToken c if (messages.Count == 0) { - return; + return SendMessagesResponse.Empty; } if (Config.EnableBackgroundSending && BackgroundProcessor != null) { LogQueuingMessages(messages.Count); // Snapshot so a caller mutating the list after enqueue cannot change the batch read at flush time. - await SendReadyAsync(messages.ToArray(), null, ct); - } - else - { - await SendReadyAsync(messages, null, ct); - LogSuccessfullySentMessages(messages.Count); + return await SendReadyAsync(messages.ToArray(), null, ct); } + + var response = await SendReadyAsync(messages, null, ct); + LogSuccessfullySentMessages(messages.Count); + return response; } /// @@ -315,8 +319,11 @@ public async Task SendMessagesAsync(IList messages, CancellationToken c /// /// The rented batch to send. /// Cancellation token to cancel the send operation. + /// + /// Commit confirmations for a direct send; empty when background sending is enabled. + /// /// Thrown when attempting to send before initialization. - public async Task SendAsync(RentedMessageBatch batch, CancellationToken ct = default) + public async Task SendAsync(RentedMessageBatch batch, CancellationToken ct = default) { ArgumentNullException.ThrowIfNull(batch); @@ -332,29 +339,29 @@ public async Task SendAsync(RentedMessageBatch batch, CancellationToken ct = def if (messages.Count == 0) { batch.Dispose(); - return; + return SendMessagesResponse.Empty; } - await SendReadyAsync(messages, batch, ct); + return await SendReadyAsync(messages, batch, ct); } // Background: queued as one unit, owner disposed after its flush. Direct: sent now, owner disposed after. - private async Task SendReadyAsync(IList messages, IDisposable? owner, CancellationToken ct) + private async Task SendReadyAsync(IList messages, IDisposable? owner, + CancellationToken ct) { if (Config.EnableBackgroundSending && BackgroundProcessor != null) { await BackgroundProcessor.EnqueueAsync(new ReadyUnit(messages, owner), ct); + return SendMessagesResponse.Empty; } - else + + try { - try - { - await Client.SendMessagesAsync(Config.StreamId, Config.TopicId, Config.Partitioning, messages, ct); - } - finally - { - owner?.Dispose(); - } + return await Client.SendMessagesAsync(Config.StreamId, Config.TopicId, Config.Partitioning, messages, ct); + } + finally + { + owner?.Dispose(); } } diff --git a/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherBuilder.cs b/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherBuilder.cs index 47fa6f09df..b69b7e99db 100644 --- a/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherBuilder.cs +++ b/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherBuilder.cs @@ -33,7 +33,7 @@ namespace Apache.Iggy.Publishers; /// public class IggyPublisherBuilder { - private IMessageEncryptor? _encryptor; + private protected IMessageEncryptor? _encryptor; internal Func? OnBackgroundError { get; set; } internal Func? OnMessageBatchFailed { get; set; } @@ -305,6 +305,7 @@ public IggyPublisher Build() ReceiveBufferSize = Config.ReceiveBufferSize, SendBufferSize = Config.SendBufferSize, ReconnectionSettings = Config.ReconnectionSettings ?? new ReconnectionSettings(), + AutoLoginSettings = AutoLoginSettings.For(Config.Login, Config.Password), LoggerFactory = Config.LoggerFactory ?? NullLoggerFactory.Instance, MessageEncryptor = _encryptor }); diff --git a/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherBuilderOfT.cs b/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherBuilderOfT.cs index 962b401af6..6198c76433 100644 --- a/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherBuilderOfT.cs +++ b/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherBuilderOfT.cs @@ -90,7 +90,11 @@ public static IggyPublisherBuilder Create(IIggyClient iggyClient, Identifier Protocol = Config.Protocol, BaseAddress = Config.Address, ReceiveBufferSize = Config.ReceiveBufferSize, - SendBufferSize = Config.SendBufferSize + SendBufferSize = Config.SendBufferSize, + ReconnectionSettings = Config.ReconnectionSettings ?? new ReconnectionSettings(), + AutoLoginSettings = AutoLoginSettings.For(Config.Login, Config.Password), + LoggerFactory = Config.LoggerFactory ?? NullLoggerFactory.Instance, + MessageEncryptor = _encryptor }); } @@ -134,8 +138,7 @@ protected override void Validate() } else { - throw new InvalidOperationException( - $"Config must be of type IggyPublisherConfig<{typeof(T).Name}>."); + throw new InvalidOperationException($"Config must be of type IggyPublisherConfig<{typeof(T).Name}>."); } } } diff --git a/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherOfT.cs b/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherOfT.cs index ac8c790c6a..1c9fdf91fb 100644 --- a/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherOfT.cs +++ b/foreign/csharp/Iggy_SDK/Publishers/IggyPublisherOfT.cs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +using Apache.Iggy.Contracts; using Apache.Iggy.Exceptions; using Apache.Iggy.Extensions; using Apache.Iggy.Headers; @@ -60,7 +61,10 @@ public IggyPublisher(IIggyClient client, IggyPublisherConfig config, ILogger< /// Optional message ID. If null, the message is sent with ID 0 and the server assigns one /// Optional user headers to attach to the message /// Cancellation token - public async Task SendAsync(T data, Guid? messageId = null, + /// + /// Commit confirmations for a direct send; empty when background sending is enabled. + /// + public async Task SendAsync(T data, Guid? messageId = null, Dictionary? userHeaders = null, CancellationToken ct = default) { EnsureInitialized(); @@ -70,7 +74,7 @@ public async Task SendAsync(T data, Guid? messageId = null, if (BackgroundProcessor != null) { await BackgroundProcessor.EnqueueAsync(TypedUnit.Single(data, id, userHeaders, _serializer), ct); - return; + return SendMessagesResponse.Empty; } var writer = new PooledBufferWriter(); @@ -78,7 +82,7 @@ public async Task SendAsync(T data, Guid? messageId = null, { _serializer.Serialize(data, writer); var message = new Message(id, writer.Written, userHeaders); - await Client.SendMessagesAsync(Config.StreamId, Config.TopicId, Config.Partitioning, message, ct); + return await Client.SendMessagesAsync(Config.StreamId, Config.TopicId, Config.Partitioning, message, ct); } finally { @@ -91,7 +95,10 @@ public async Task SendAsync(T data, Guid? messageId = null, /// /// The collection of objects to serialize and send /// Cancellation token - public async Task SendAsync(IEnumerable data, CancellationToken ct = default) + /// + /// Commit confirmations for a direct send; empty when background sending is enabled. + /// + public async Task SendAsync(IEnumerable data, CancellationToken ct = default) { EnsureInitialized(); @@ -103,7 +110,7 @@ public async Task SendAsync(IEnumerable data, CancellationToken ct = default) await BackgroundProcessor.EnqueueAsync(unit, ct); } - return; + return SendMessagesResponse.Empty; } using var builder = new RentedMessageBatchBuilder(); @@ -113,7 +120,7 @@ public async Task SendAsync(IEnumerable data, CancellationToken ct = default) static (state, writer) => state.Serializer.Serialize(state.Data, writer)); } - await SendDirectBatchAsync(builder, ct); + return await SendDirectBatchAsync(builder, ct); } /// @@ -121,7 +128,10 @@ public async Task SendAsync(IEnumerable data, CancellationToken ct = default) /// /// The collection of items to send, each with optional message ID and headers /// Cancellation token - public async Task SendAsync( + /// + /// Commit confirmations for a direct send; empty when background sending is enabled. + /// + public async Task SendAsync( IEnumerable<(T data, Guid? messageId, Dictionary? userHeaders)> items, CancellationToken ct = default) { @@ -135,7 +145,7 @@ public async Task SendAsync( await BackgroundProcessor.EnqueueAsync(unit, ct); } - return; + return SendMessagesResponse.Empty; } using var builder = new RentedMessageBatchBuilder(); @@ -146,10 +156,11 @@ public async Task SendAsync( item.userHeaders); } - await SendDirectBatchAsync(builder, ct); + return await SendDirectBatchAsync(builder, ct); } - private async Task SendDirectBatchAsync(RentedMessageBatchBuilder builder, CancellationToken ct) + private async Task SendDirectBatchAsync(RentedMessageBatchBuilder builder, + CancellationToken ct) { var batch = builder.Build(); try @@ -157,10 +168,10 @@ private async Task SendDirectBatchAsync(RentedMessageBatchBuilder builder, Cance IList messages = batch.Messages; if (messages.Count == 0) { - return; + return SendMessagesResponse.Empty; } - await Client.SendMessagesAsync(Config.StreamId, Config.TopicId, Config.Partitioning, messages, ct); + return await Client.SendMessagesAsync(Config.StreamId, Config.TopicId, Config.Partitioning, messages, ct); } finally { diff --git a/foreign/csharp/Iggy_SDK/Utils/BufferSizes.cs b/foreign/csharp/Iggy_SDK/Utils/BufferSizes.cs index 90bc6a1955..cd5a9e871f 100644 --- a/foreign/csharp/Iggy_SDK/Utils/BufferSizes.cs +++ b/foreign/csharp/Iggy_SDK/Utils/BufferSizes.cs @@ -20,5 +20,4 @@ namespace Apache.Iggy.Utils; internal static class BufferSizes { internal const int INITIAL_BYTES_LENGTH = 4; - internal const int EXPECTED_RESPONSE_SIZE = 8; } diff --git a/foreign/csharp/Iggy_SDK/Utils/CommandCodes.cs b/foreign/csharp/Iggy_SDK/Utils/CommandCodes.cs index 8922d37d61..c7a3051e59 100644 --- a/foreign/csharp/Iggy_SDK/Utils/CommandCodes.cs +++ b/foreign/csharp/Iggy_SDK/Utils/CommandCodes.cs @@ -47,6 +47,8 @@ internal static class CommandCodes internal const int GET_CONSUMER_OFFSET_CODE = 120; internal const int STORE_CONSUMER_OFFSET_CODE = 121; internal const int DELETE_CONSUMER_OFFSET_CODE = 122; + internal const int STORE_CONSUMER_OFFSET_2_CODE = 123; + internal const int DELETE_CONSUMER_OFFSET_2_CODE = 124; internal const int GET_STREAM_CODE = 200; internal const int GET_STREAMS_CODE = 201; internal const int CREATE_STREAM_CODE = 202; @@ -68,4 +70,5 @@ internal static class CommandCodes internal const int DELETE_CONSUMER_GROUP_CODE = 603; internal const int JOIN_CONSUMER_GROUP_CODE = 604; internal const int LEAVE_CONSUMER_GROUP_CODE = 605; + internal const int SYNC_CONSUMER_GROUP_CODE = 606; } diff --git a/foreign/csharp/Iggy_SDK/Utils/ServerAddress.cs b/foreign/csharp/Iggy_SDK/Utils/ServerAddress.cs new file mode 100644 index 0000000000..2f85784610 --- /dev/null +++ b/foreign/csharp/Iggy_SDK/Utils/ServerAddress.cs @@ -0,0 +1,139 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Net; +using System.Net.Sockets; + +namespace Apache.Iggy.Utils; + +/// +/// Endpoint comparison for leader redirection. The configured address is whatever the caller wrote +/// (localhost:8090), while the cluster roster reports IPs, so the two are compared as parsed +/// endpoints rather than as strings. +/// +internal static class ServerAddress +{ + internal static bool IsSame(string first, string second) + { + return Normalize(first) == Normalize(second); + } + + /// + /// Renders host:port, bracketing a host that carries colons of its own (a bare IPv6 address). + /// Without the brackets the rendering can never round-trip through , so + /// it would compare unequal to every normalized address. + /// + internal static string HostPort(string host, ushort port) + { + return host.Contains(':') && !host.StartsWith('[') ? $"[{host}]:{port}" : $"{host}:{port}"; + } + + internal static bool TryParse(string address, out string host, out int port) + { + var parsed = TrySplitHostPort(address, out host, out var hostPort); + port = hostPort; + + return parsed; + } + + /// + /// The socket family a host has to be dialled on. A name resolves to whatever the resolver returns, and + /// the connect call handles that itself, so only a literal decides the family here. + /// + internal static AddressFamily AddressFamilyOf(string host) + { + return IPAddress.TryParse(host, out var ip) ? ip.AddressFamily : AddressFamily.InterNetwork; + } + + /// + /// Canonical host:port rendering: the host is lowercased, the loopback and unspecified aliases + /// collapse onto the loopback address, and an IP is re-rendered from its parsed form. The host is + /// replaced as a whole, never as a substring, so a node named my-localhost-1 keeps its name. + /// An address that is not host:port is only lowercased, which leaves it comparable but distinct. + /// + internal static string Normalize(string address) + { + if (!TrySplitHostPort(address, out var host, out var port)) + { + return address.ToLowerInvariant(); + } + + if (!IPAddress.TryParse(NormalizeHostAlias(host), out var ip)) + { + return $"{host.ToLowerInvariant()}:{port}"; + } + + return ip.AddressFamily == AddressFamily.InterNetworkV6 ? $"[{ip}]:{port}" : $"{ip}:{port}"; + } + + /// + /// A server bound to the unspecified address is reachable on the loopback one, and the roster may + /// report either, so both render the same way. + /// + private static string NormalizeHostAlias(string host) + { + if (host.Equals("localhost", StringComparison.OrdinalIgnoreCase)) + { + return "127.0.0.1"; + } + + if (!IPAddress.TryParse(host, out var ip)) + { + return host; + } + + if (ip.Equals(IPAddress.Any)) + { + return IPAddress.Loopback.ToString(); + } + + return ip.Equals(IPAddress.IPv6Any) ? IPAddress.IPv6Loopback.ToString() : host; + } + + private static bool TrySplitHostPort(string address, out string host, out ushort port) + { + host = string.Empty; + port = 0; + string portText; + + // A bracketed host is the only form that may carry colons of its own. + if (address.StartsWith('[')) + { + var closing = address.IndexOf(']'); + if (closing < 0 || closing + 1 >= address.Length || address[closing + 1] != ':') + { + return false; + } + + host = address[1..closing]; + portText = address[(closing + 2)..]; + } + else + { + var separator = address.IndexOf(':'); + if (separator <= 0 || address.IndexOf(':', separator + 1) >= 0) + { + return false; + } + + host = address[..separator]; + portText = address[(separator + 1)..]; + } + + return host.Length > 0 && ushort.TryParse(portText, out port); + } +} diff --git a/core/server/src/metadata/user.rs b/foreign/csharp/Iggy_SDK/Vsr/Command2.cs similarity index 69% rename from core/server/src/metadata/user.rs rename to foreign/csharp/Iggy_SDK/Vsr/Command2.cs index c82d49a441..c8f7a765d7 100644 --- a/core/server/src/metadata/user.rs +++ b/foreign/csharp/Iggy_SDK/Vsr/Command2.cs @@ -15,16 +15,16 @@ // specific language governing permissions and limitations // under the License. -use crate::metadata::UserId; -use iggy_common::{IggyTimestamp, Permissions, UserStatus}; -use std::sync::Arc; +namespace Apache.Iggy.Vsr; -#[derive(Clone, Debug)] -pub struct UserMeta { - pub id: UserId, - pub username: Arc, - pub password_hash: Arc, - pub status: UserStatus, - pub permissions: Option>, - pub created_at: IggyTimestamp, +/// +/// VSR frame discriminant, byte 60 of every consensus header. Only the frames a client emits or +/// receives are named; every other discriminant decodes as . +/// +internal enum Command2 : byte +{ + Reserved = 0, + Request = 5, + Reply = 8, + Eviction = 13 } diff --git a/foreign/csharp/Iggy_SDK/Vsr/ConsensusSession.cs b/foreign/csharp/Iggy_SDK/Vsr/ConsensusSession.cs new file mode 100644 index 0000000000..8cdbdcc16c --- /dev/null +++ b/foreign/csharp/Iggy_SDK/Vsr/ConsensusSession.cs @@ -0,0 +1,223 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Buffers.Binary; +using System.Security.Cryptography; +using Apache.Iggy.Exceptions; + +namespace Apache.Iggy.Vsr; + +/// +/// Consensus-level session state: the ephemeral client id, the session number the server hands back +/// when a register commits, and the monotonic request counter. +/// +/// +/// Every mutation and every read of the identity runs under one lock. The transport serialises the +/// re-arms against the requests with its sending lock, so a request always encodes from the identity +/// that is live for the whole time it is on the wire. +/// +internal sealed class ConsensusSession +{ +#if NET10_0_OR_GREATER + private readonly Lock _gate = new(); +#else + private readonly object _gate = new(); +#endif + private UInt128 _clientId; + private bool _registerPending; + private ulong _requestCounter; + private ulong? _session; + + /// Ephemeral client identifier, never persisted, non-zero. + internal UInt128 ClientId + { + get + { + lock (_gate) + { + return _clientId; + } + } + } + + /// Session fence epoch assigned by the server, null until a register commits. + internal ulong? Session + { + get + { + lock (_gate) + { + return _session; + } + } + } + + /// The id the next request id resolution will return. + internal ulong RequestCounter + { + get + { + lock (_gate) + { + return _requestCounter; + } + } + } + + internal bool IsBound + { + get + { + lock (_gate) + { + return _session.HasValue; + } + } + } + + internal ConsensusSession() : this(GenerateClientId()) + { + } + + internal ConsensusSession(UInt128 clientId) + { + _clientId = clientId; + _requestCounter = 1; + } + + /// + /// Resolves the identity one request header is encoded from, in a single atomic step so no reset can + /// interleave between the client id, the request id and the session. + /// + internal SessionFrame Resolve(VsrOperation operation) + { + lock (_gate) + { + return operation switch + { + VsrOperation.Register => RegisterFrameLocked(), + VsrOperation.NonReplicated => new SessionFrame(_clientId, _requestCounter, _session ?? 0), + _ => ReplicatedFrameLocked(operation) + }; + } + } + + /// Bind the session from a committed register reply. + /// The session number the register reply carried. + /// + /// No register is awaiting a binding, so the identity was re-armed while this one was in flight. Binding + /// regardless would pair the session with a client id the server never registered, and it would fence + /// every later request. + /// + /// + /// The register reply carried no session. The value comes off the wire, so a malformed reply has to + /// surface as a protocol error rather than as argument validation. + /// + internal void Bind(ulong session) + { + if (session == 0) + { + throw VsrError.Exception(VsrError.INVALID_FORMAT, "Register reply carried no session."); + } + + lock (_gate) + { + if (!_registerPending) + { + throw new NotConnectedException(); + } + + _registerPending = false; + _session = session; + } + } + + /// Forget the binding and the client id, e.g. after an eviction or a torn connection. + internal void Reset() + { + lock (_gate) + { + ReArmLocked(); + } + } + + /// + /// Begin a registration, re-arming the session if it was already used. A re-login mints a fresh + /// client id and clears the binding, so the register encodes cleanly and the server never has to + /// disambiguate a repeat register for the same client. Always returns 0. + /// + private SessionFrame RegisterFrameLocked() + { + // A second register while one is still unbound would re-arm the identity out from under the first, and + // the winner's Bind would then attach its session to the re-armed client id - the server answers every + // later request with NoSession and evicts. Refuse instead: the caller retries against a clean session. + if (_registerPending) + { + throw VsrError.Exception(VsrError.UNAUTHENTICATED, "A consensus register is already in flight."); + } + + if (_session.HasValue) + { + ReArmLocked(); + } + + _registerPending = true; + + return new SessionFrame(_clientId, 0, 0); + } + + private SessionFrame ReplicatedFrameLocked(VsrOperation operation) + { + var sessionId = _session ?? throw VsrError.Exception(VsrError.UNAUTHENTICATED, + "A replicated request requires a bound consensus session."); + + // Partition ops replicate in their own per-partition group with no client-table dedup, so they too + // must leave the metadata counter untouched. Only metadata operations and logout consume an id: the + // server tracks request ids for those alone, and it accepts any id above the client's watermark. + if (operation.IsPartition()) + { + return new SessionFrame(_clientId, _requestCounter, sessionId); + } + + var requestId = _requestCounter; + _requestCounter = checked(_requestCounter + 1); + + return new SessionFrame(_clientId, requestId, sessionId); + } + + private void ReArmLocked() + { + _clientId = GenerateClientId(); + _session = null; + _requestCounter = 1; + _registerPending = false; + } + + private static UInt128 GenerateClientId() + { + Span bytes = stackalloc byte[16]; + RandomNumberGenerator.Fill(bytes); + var lower = BinaryPrimitives.ReadUInt64LittleEndian(bytes[..8]); + var upper = BinaryPrimitives.ReadUInt64LittleEndian(bytes[8..]); + var clientId = new UInt128(upper, lower); + + return clientId == UInt128.Zero ? UInt128.One : clientId; + } +} + +/// The session identity one request header is encoded from, resolved as a single atomic snapshot. +internal readonly record struct SessionFrame(UInt128 ClientId, ulong RequestId, ulong SessionId); diff --git a/foreign/csharp/Iggy_SDK/Vsr/ConsumerGroupClientState.cs b/foreign/csharp/Iggy_SDK/Vsr/ConsumerGroupClientState.cs new file mode 100644 index 0000000000..e594443ae9 --- /dev/null +++ b/foreign/csharp/Iggy_SDK/Vsr/ConsumerGroupClientState.cs @@ -0,0 +1,301 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +namespace Apache.Iggy.Vsr; + +/// +/// Per-connection cache of consumer-group assignments and topic partition counts, mirroring +/// core/common/src/consumer_group_client_state.rs. Under VSR the broker never picks a partition, so +/// the client resolves group polls and balanced / message-key produce locally. The cursors have to survive +/// across calls, which is why this lives on the long-lived transport rather than on a request. +/// +internal sealed class ConsumerGroupClientState +{ + private readonly Dictionary _assignments = []; + private readonly Dictionary _balancedCursors = []; +#if NET10_0_OR_GREATER + private readonly Lock _gate = new(); +#else + private readonly object _gate = new(); +#endif + private readonly Dictionary _joinedGroups = []; + private readonly Dictionary _partitionCounts = []; + + /// Nothing tells this client when another one resizes a topic, so the count expires on its own. + private const long PartitionCountTtlMs = 30_000; + + /// True when a non-empty assignment is cached for the group. + internal bool HasAssignment(GroupKey key) + { + lock (_gate) + { + return _assignments.TryGetValue(key, out var assignment) && assignment.Partitions.Count > 0; + } + } + + /// + /// Replaces a group's cached assignment. A generation change is a rebalance, so the round-robin cursor + /// restarts rather than carrying an index that meant something else. + /// + internal void SetAssignment(GroupKey key, ulong generation, IReadOnlyList partitions) + { + lock (_gate) + { + if (!_assignments.TryGetValue(key, out var assignment)) + { + assignment = new GroupAssignment(); + _assignments[key] = assignment; + } + + if (assignment.Generation != generation) + { + assignment.Cursor = 0; + } + + assignment.Generation = generation; + assignment.Partitions = partitions; + } + } + + internal void InvalidateAssignment(GroupKey key) + { + lock (_gate) + { + _assignments.Remove(key); + } + } + + /// + /// The next assigned partition for a group poll, advancing the cursor. null when nothing is cached + /// or the member holds no partitions. + /// + internal uint? NextGroupPartition(GroupKey key) + { + lock (_gate) + { + if (!_assignments.TryGetValue(key, out var assignment) || assignment.Partitions.Count == 0) + { + return null; + } + + var index = assignment.Cursor % assignment.Partitions.Count; + assignment.Cursor = assignment.Cursor == int.MaxValue ? 0 : assignment.Cursor + 1; + + return assignment.Partitions[index]; + } + } + + /// The next balanced produce partition for a topic, advancing the cursor. + internal uint NextBalancedPartition(TopicKey key, uint partitionCount) + { + if (partitionCount == 0) + { + return 0; + } + + lock (_gate) + { + _balancedCursors.TryGetValue(key, out var cursor); + var partition = (uint)(cursor % partitionCount); + _balancedCursors[key] = cursor == int.MaxValue ? 0 : cursor + 1; + + return partition; + } + } + + internal uint? PartitionCount(TopicKey key) + { + lock (_gate) + { + if (!_partitionCounts.TryGetValue(key, out var cached)) + { + return null; + } + + if (Environment.TickCount64 >= cached.ExpiresAt) + { + _partitionCounts.Remove(key); + + return null; + } + + return cached.Count; + } + } + + internal void SetPartitionCount(TopicKey key, uint partitionCount) + { + lock (_gate) + { + _partitionCounts[key] = new CachedPartitionCount(partitionCount, + Environment.TickCount64 + PartitionCountTtlMs); + } + } + + /// + /// Forgets a topic's cached partition count immediately, for the changes this client makes itself. + /// Changes made by anyone else are covered by . + /// + internal void InvalidatePartitionCount(TopicKey key) + { + lock (_gate) + { + _partitionCounts.Remove(key); + } + } + + /// Records a joined group's identifiers so a later refresh can rebuild its sync request. + internal void RegisterGroup(GroupKey key, Identifier streamId, Identifier topicId, Identifier groupId) + { + lock (_gate) + { + _joinedGroups[key] = new GroupIdentifiers(streamId, topicId, groupId); + } + } + + internal void DeregisterGroup(GroupKey key) + { + lock (_gate) + { + _joinedGroups.Remove(key); + _assignments.Remove(key); + } + } + + /// + /// True when the last assignment sync saw this client as a member. A member mid-rebalance, or one holding + /// zero partitions, is still registered, so this asks a different question than + /// . + /// + internal bool IsRegistered(GroupKey key) + { + lock (_gate) + { + return _joinedGroups.ContainsKey(key); + } + } + + internal IReadOnlyList RegisteredGroups() + { + lock (_gate) + { + return _joinedGroups.Count == 0 ? [] : [.. _joinedGroups.Values]; + } + } + + /// + /// Drops what a consensus session owns. The assignments are fenced by a generation the coordinator tracks + /// per session, so carrying them across a reset would fence every poll, and membership has to be re-synced + /// before it can be trusted again. The balanced cursors and the cached partition counts stay: they belong + /// to a topic, not to a session, and clearing them restarts the produce round-robin at partition 0 and + /// costs a metadata round trip per topic on every reconnect. + /// + internal void ClearSessionScoped() + { + lock (_gate) + { + _assignments.Clear(); + _joinedGroups.Clear(); + } + } + + internal readonly record struct GroupIdentifiers(Identifier StreamId, Identifier TopicId, Identifier GroupId); + + private readonly record struct CachedPartitionCount(uint Count, long ExpiresAt); + + private sealed class GroupAssignment + { + internal IReadOnlyList Partitions { get; set; } = []; + internal ulong Generation { get; set; } + internal int Cursor { get; set; } + } +} + +/// +/// Cache key for a topic. The identifier kind is part of the key because a stream named "1" and the stream +/// with id 1 are different streams that share a rendering. Identifiers are compared by their wire bytes: +/// compares its value array by reference, and every call site builds a fresh one. +/// +internal readonly struct TopicKey(Identifier streamId, Identifier topicId) : IEquatable +{ + private Identifier StreamId { get; } = streamId; + private Identifier TopicId { get; } = topicId; + + public bool Equals(TopicKey other) + { + return IdentifierKey.Equal(StreamId, other.StreamId) && IdentifierKey.Equal(TopicId, other.TopicId); + } + + public override bool Equals(object? obj) + { + return obj is TopicKey other && Equals(other); + } + + public override int GetHashCode() + { + return HashCode.Combine(IdentifierKey.Hash(StreamId), IdentifierKey.Hash(TopicId)); + } + + public override string ToString() + { + return $"{StreamId}|{TopicId}"; + } +} + +/// Cache key for a consumer group on a topic. +internal readonly struct GroupKey(Identifier streamId, Identifier topicId, Identifier groupId) : IEquatable +{ + private TopicKey Topic { get; } = new(streamId, topicId); + private Identifier GroupId { get; } = groupId; + + public bool Equals(GroupKey other) + { + return Topic.Equals(other.Topic) && IdentifierKey.Equal(GroupId, other.GroupId); + } + + public override bool Equals(object? obj) + { + return obj is GroupKey other && Equals(other); + } + + public override int GetHashCode() + { + return HashCode.Combine(Topic.GetHashCode(), IdentifierKey.Hash(GroupId)); + } + + public override string ToString() + { + return $"{Topic}|{GroupId}"; + } +} + +internal static class IdentifierKey +{ + internal static bool Equal(Identifier first, Identifier second) + { + return first.Kind == second.Kind && first.Value.AsSpan().SequenceEqual(second.Value); + } + + internal static int Hash(Identifier identifier) + { + var hash = new HashCode(); + hash.Add((byte)identifier.Kind); + hash.AddBytes(identifier.Value); + + return hash.ToHashCode(); + } +} diff --git a/foreign/csharp/Iggy_SDK/Vsr/CredentialBounds.cs b/foreign/csharp/Iggy_SDK/Vsr/CredentialBounds.cs new file mode 100644 index 0000000000..11fde11871 --- /dev/null +++ b/foreign/csharp/Iggy_SDK/Vsr/CredentialBounds.cs @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Text; + +namespace Apache.Iggy.Vsr; + +/// +/// The credential bounds every server enforces, checked before encoding so an oversized credential is +/// reported as the typed status code instead of desyncing the u8 length prefix on the wire. Mirrors +/// core/common/src/http/users/defaults.rs. +/// +internal static class CredentialBounds +{ + internal const int MIN_USERNAME_LENGTH = 3; + internal const int MAX_USERNAME_LENGTH = 50; + internal const int MIN_PASSWORD_LENGTH = 3; + internal const int MAX_PASSWORD_LENGTH = 100; + + internal const int MIN_TOKEN_LENGTH = 1; + internal const int MAX_TOKEN_LENGTH = 255; + + internal static void ValidateUsername(string username) + { + var length = Encoding.UTF8.GetByteCount(username); + if (length is < MIN_USERNAME_LENGTH or > MAX_USERNAME_LENGTH) + { + throw VsrError.Exception(VsrError.INVALID_USERNAME, + $"Username must be {MIN_USERNAME_LENGTH}-{MAX_USERNAME_LENGTH} bytes, got {length}."); + } + } + + internal static void ValidatePassword(string password) + { + var length = Encoding.UTF8.GetByteCount(password); + if (length is < MIN_PASSWORD_LENGTH or > MAX_PASSWORD_LENGTH) + { + throw VsrError.Exception(VsrError.INVALID_PASSWORD, + $"Password must be {MIN_PASSWORD_LENGTH}-{MAX_PASSWORD_LENGTH} bytes, got {length}."); + } + } + + internal static void ValidateToken(string token) + { + var length = Encoding.UTF8.GetByteCount(token); + if (length is < MIN_TOKEN_LENGTH or > MAX_TOKEN_LENGTH) + { + throw VsrError.Exception(VsrError.INVALID_PERSONAL_ACCESS_TOKEN, + $"Personal access token must be {MIN_TOKEN_LENGTH}-{MAX_TOKEN_LENGTH} bytes, got {length}."); + } + } +} diff --git a/core/server-ng/src/args.rs b/foreign/csharp/Iggy_SDK/Vsr/EvictionReason.cs similarity index 53% rename from core/server-ng/src/args.rs rename to foreign/csharp/Iggy_SDK/Vsr/EvictionReason.cs index 8bff7717af..e51deeea0f 100644 --- a/core/server-ng/src/args.rs +++ b/foreign/csharp/Iggy_SDK/Vsr/EvictionReason.cs @@ -15,20 +15,28 @@ // specific language governing permissions and limitations // under the License. -use clap::Parser; +namespace Apache.Iggy.Vsr; -#[derive(Parser, Debug)] -#[command( - author = "Apache Iggy (Incubating)", - version, - about = "Apache Iggy server-ng", - long_about = "Apache Iggy server-ng\n\nUse --replica-id together with a shared cluster config to run one binary per cluster node." -)] -pub struct Args { - /// Identifies this node within `cluster.nodes` by its replica ID. - /// - /// Required when `cluster.enabled = true`. The value must match exactly - /// one `cluster.nodes[*].replica_id` entry in the loaded configuration. - #[arg(long, verbatim_doc_comment)] - pub replica_id: Option, +/// +/// Reason carried at byte 255 of an eviction frame. Session-terminal, never transient. +/// Discriminants are wire-pinned; a value outside this set decodes as . +/// +internal enum EvictionReason : byte +{ + Reserved = 0, + NoSession = 1, + ClientReleaseTooLow = 2, + ClientReleaseTooHigh = 3, + InvalidRequestOperation = 4, + InvalidRequestBody = 5, + InvalidRequestBodySize = 6, + SessionTooLow = 7, + SessionReleaseMismatch = 8, + InvalidCredentials = 9, + InvalidToken = 10, + UserInactive = 11, + SessionError = 12, + StaleClient = 13, + IncompatibleProtocol = 14, + MalformedLogin = 15 } diff --git a/foreign/csharp/Iggy_SDK/Vsr/LoginRegister.cs b/foreign/csharp/Iggy_SDK/Vsr/LoginRegister.cs new file mode 100644 index 0000000000..6d2958e139 --- /dev/null +++ b/foreign/csharp/Iggy_SDK/Vsr/LoginRegister.cs @@ -0,0 +1,168 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Buffers.Binary; +using System.Text; +using Apache.Iggy.Utils; + +namespace Apache.Iggy.Vsr; + +/// +/// The register handshake bodies. VSR replaces the legacy login codes with +/// LOGIN_REGISTER / LOGIN_REGISTER_WITH_PAT, whose bodies lead with the client version +/// info the server gates on before it touches credentials. +/// +internal static class LoginRegister +{ + internal const string SDK_NAME = "csharp-sdk"; + + /// Semver of the iggy_binary_protocol crate this SDK is built against. + internal const int PROTOCOL_VERSION_MAJOR = 0; + + internal const int PROTOCOL_VERSION_MINOR = 11; + internal const int PROTOCOL_VERSION_PATCH = 0; + + /// Packed protocol version: major << 20 | minor << 10 | patch, 10 bits each. + internal const uint PROTOCOL_VERSION = + ((uint)PROTOCOL_VERSION_MAJOR << 20) | ((uint)PROTOCOL_VERSION_MINOR << 10) | PROTOCOL_VERSION_PATCH; + + private const int MaxWireNameLength = 255; + + private static int VersionInfoLength => 4 + NameLength(SDK_NAME) + NameLength(SdkVersion.Value); + + internal static byte[] Serialize(string username, string password, string? clientContext = null) + { + CredentialBounds.ValidateUsername(username); + CredentialBounds.ValidatePassword(password); + + var writer = new BodyWriter(VersionInfoLength + NameLength(username) + NameLength(password) + 4 + + ContextLength(clientContext)); + writer.WriteVersionInfo(); + writer.WriteName(username, nameof(username)); + writer.WriteName(password, nameof(password)); + writer.WriteContext(clientContext); + + return writer.Buffer; + } + + internal static byte[] SerializeWithPersonalAccessToken(string token, string? clientContext = null) + { + CredentialBounds.ValidateToken(token); + + var writer = new BodyWriter(VersionInfoLength + NameLength(token) + 4 + ContextLength(clientContext)); + writer.WriteVersionInfo(); + writer.WriteName(token, nameof(token)); + writer.WriteContext(clientContext); + + return writer.Buffer; + } + + /// + /// [user_id u32][session u64][server_protocol_version u32][server_version len u8 + bytes]. + /// + internal static LoginRegisterResponse Deserialize(ReadOnlySpan body) + { + if (body.IsEmpty) + { + // The server fast-fails a terminal register failure (invalid credentials, invalid token, + // inactive user) with an empty reply instead of a typed error frame, and the reason is not + // recoverable from the wire. Same INVALID_FORMAT surface as the Rust SDK. + throw VsrError.Exception(VsrError.INVALID_FORMAT, + "Server rejected the login. The register reply is empty, which the server sends for invalid " + + "credentials, an invalid personal access token, or an inactive user."); + } + + if (body.Length < 17) + { + throw VsrError.Exception(VsrError.INVALID_FORMAT, "Register reply is truncated."); + } + + var serverVersionLength = body[16]; + if (serverVersionLength == 0 || body.Length < 17 + serverVersionLength) + { + throw VsrError.Exception(VsrError.INVALID_FORMAT, "Register reply carries a malformed server version."); + } + + return new LoginRegisterResponse(BinaryPrimitives.ReadUInt32LittleEndian(body[..4]), + BinaryPrimitives.ReadUInt64LittleEndian(body[4..12]), + BinaryPrimitives.ReadUInt32LittleEndian(body[12..16]), + Encoding.UTF8.GetString(body.Slice(17, serverVersionLength))); + } + + private static int NameLength(string value) + { + return 1 + Encoding.UTF8.GetByteCount(value); + } + + private static int ContextLength(string? clientContext) + { + return clientContext is null ? 0 : Encoding.UTF8.GetByteCount(clientContext); + } + + private struct BodyWriter + { + private int _position; + + internal BodyWriter(int length) + { + Buffer = new byte[length]; + _position = 0; + } + + internal byte[] Buffer { get; } + + internal void WriteVersionInfo() + { + BinaryPrimitives.WriteUInt32LittleEndian(Buffer.AsSpan(_position, 4), PROTOCOL_VERSION); + _position += 4; + WriteName(SDK_NAME, nameof(SDK_NAME)); + WriteName(SdkVersion.Value, nameof(SdkVersion)); + } + + internal void WriteName(string value, string name) + { + var length = Encoding.UTF8.GetByteCount(value); + if (length is 0 or > MaxWireNameLength) + { + throw new ArgumentException($"{name} must be 1-{MaxWireNameLength} UTF-8 bytes, got {length}.", name); + } + + Buffer[_position] = (byte)length; + _position += 1; + Encoding.UTF8.GetBytes(value, Buffer.AsSpan(_position, length)); + _position += length; + } + + internal void WriteContext(string? clientContext) + { + var length = ContextLength(clientContext); + BinaryPrimitives.WriteUInt32LittleEndian(Buffer.AsSpan(_position, 4), (uint)length); + _position += 4; + if (clientContext is not null && length > 0) + { + Encoding.UTF8.GetBytes(clientContext, Buffer.AsSpan(_position, length)); + _position += length; + } + } + } +} + +internal readonly record struct LoginRegisterResponse( + uint UserId, + ulong Session, + uint ServerProtocolVersion, + string ServerVersion); diff --git a/foreign/csharp/Iggy_SDK/Vsr/SyncConsumerGroup.cs b/foreign/csharp/Iggy_SDK/Vsr/SyncConsumerGroup.cs new file mode 100644 index 0000000000..5bc8d55fb8 --- /dev/null +++ b/foreign/csharp/Iggy_SDK/Vsr/SyncConsumerGroup.cs @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Buffers.Binary; + +namespace Apache.Iggy.Vsr; + +/// +/// Reply body of a SyncConsumerGroup request: the requesting member's current partition assignment +/// and the generation it belongs to. Mirrors +/// core/binary_protocol/src/responses/consumer_groups/sync_consumer_group.rs. +/// +/// +/// Wire format: [generation u64][partitions_count u32][partition_id u32]*. The request body is the +/// plain [stream][topic][group] identifier triple every other group request already builds. +/// +internal readonly record struct SyncConsumerGroupAssignment(ulong Generation, IReadOnlyList Partitions) +{ + internal static SyncConsumerGroupAssignment Decode(ReadOnlySpan body) + { + if (body.Length < 12) + { + throw Malformed(); + } + + var generation = BinaryPrimitives.ReadUInt64LittleEndian(body[..8]); + var count = BinaryPrimitives.ReadUInt32LittleEndian(body.Slice(8, 4)); + if (body.Length < 12 + (long)count * 4) + { + throw Malformed(); + } + + var partitions = new uint[count]; + for (var i = 0; i < partitions.Length; i++) + { + partitions[i] = BinaryPrimitives.ReadUInt32LittleEndian(body.Slice(12 + i * 4, 4)); + } + + return new SyncConsumerGroupAssignment(generation, partitions); + } + + private static Exception Malformed() + { + return VsrError.Exception(VsrError.INVALID_COMMAND, + "Consumer group assignment reply is too short for the partition count it declares."); + } +} diff --git a/foreign/csharp/Iggy_SDK/Vsr/VsrError.cs b/foreign/csharp/Iggy_SDK/Vsr/VsrError.cs new file mode 100644 index 0000000000..07e5306a83 --- /dev/null +++ b/foreign/csharp/Iggy_SDK/Vsr/VsrError.cs @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Apache.Iggy.Exceptions; + +namespace Apache.Iggy.Vsr; + +/// +/// Server error codes the VSR paths raise locally or surface from the wire. Values match +/// core/common/src/error/iggy_error.rs; the server shares this code space across the reply +/// status word, the committed result section and the eviction mapping. +/// +internal static class VsrError +{ + internal const int INVALID_COMMAND = 3; + internal const int INVALID_FORMAT = 4; + internal const int FEATURE_UNAVAILABLE = 5; + internal const int INVALID_IDENTIFIER = 6; + internal const int STALE_CLIENT = 30; + internal const int UNAUTHENTICATED = 40; + internal const int INVALID_CREDENTIALS = 42; + internal const int INVALID_USERNAME = 43; + internal const int INVALID_PASSWORD = 44; + internal const int INVALID_PERSONAL_ACCESS_TOKEN = 53; + internal const int TRANSIENT_NOT_COMMITTED = 57; + internal const int TRANSIENT_NOT_ACCEPTED = 58; + internal const int EMPTY_RESPONSE = 304; + internal const int TOPIC_ID_NOT_FOUND = 2010; + internal const int CONSUMER_GROUP_MEMBER_NOT_FOUND = 5006; + internal const int CONSUMER_GROUP_PARTITION_NOT_OWNED = 5009; + internal const int INCOMPATIBLE_PROTOCOL_VERSION = 14003; + + /// A failure the client raised itself, before or instead of a server verdict. + internal static IggyInvalidStatusCodeException Exception(int code, string message) + { + return new IggyInvalidStatusCodeException(code, message); + } + + /// + /// A verdict the server reported, on the reply status word, in the committed result section or in an + /// eviction frame. Only these may drive retry and failover. + /// + internal static IggyInvalidStatusCodeException FromServer(int code, string message) + { + return new IggyInvalidStatusCodeException(code, message, true); + } +} diff --git a/foreign/csharp/Iggy_SDK/Vsr/VsrHeader.cs b/foreign/csharp/Iggy_SDK/Vsr/VsrHeader.cs new file mode 100644 index 0000000000..ad58e0c56e --- /dev/null +++ b/foreign/csharp/Iggy_SDK/Vsr/VsrHeader.cs @@ -0,0 +1,156 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Buffers.Binary; + +namespace Apache.Iggy.Vsr; + +/// +/// The 256-byte consensus header, read and written by wire offset. Offsets mirror +/// core/binary_protocol/src/consensus/header.rs. Checksums stay zero: the server does not verify +/// them for client frames. +/// +internal static class VsrHeader +{ + internal const int HEADER_SIZE = 256; + + internal const int SIZE_OFFSET = 48; + internal const int COMMAND_OFFSET = 60; + + // The client wire carries no routing group: the server derives it (plane from + // the operation, partition target from the payload) and stamps it into its own + // internal header. Every field past the operation therefore sits eight bytes + // earlier than it did while the client header still carried one. + internal const int REQUEST_CLIENT_OFFSET = 128; + internal const int REQUEST_TIMESTAMP_OFFSET = 160; + internal const int REQUEST_ID_OFFSET = 168; + internal const int REQUEST_OPERATION_OFFSET = 176; + internal const int REQUEST_SESSION_OFFSET = 184; + internal const int REQUEST_RESERVED_OFFSET = 196; + + internal const int REPLY_OPERATION_OFFSET = 208; + internal const int REPLY_STATUS_OFFSET = 216; + + internal const int EVICTION_CLIENT_OFFSET = 128; + internal const int EVICTION_PROTOCOL_VERSION_OFFSET = 144; + internal const int EVICTION_PROTOCOL_VERSION_MIN_OFFSET = 148; + internal const int EVICTION_REASON_OFFSET = 255; + + /// + /// Encodes the request header for a classic command code and its body. Returns the total frame size + /// (header plus body). + /// + /// + /// Everything that can fail runs before a request id is consumed. The primary accepts any id above the + /// watermark, so a gap costs nothing, but a consumed id can never be handed back: re-encoding it for a + /// different request would let the client table answer that request from the first one's cached reply. + /// + internal static int EncodeRequestHeader(Span header, ConsensusSession session, int code, + ReadOnlySpan payload) + { + if (header.Length < HEADER_SIZE) + { + throw new ArgumentException($"Header buffer must be at least {HEADER_SIZE} bytes.", nameof(header)); + } + + header = header[..HEADER_SIZE]; + header.Clear(); + + var operation = VsrOperations.ForCode(code); + + if (payload.Length > int.MaxValue - HEADER_SIZE) + { + throw VsrError.Exception(VsrError.INVALID_COMMAND, "Request body exceeds the maximum frame size."); + } + + var frame = session.Resolve(operation); + var totalSize = HEADER_SIZE + payload.Length; + + BinaryPrimitives.WriteUInt32LittleEndian(header[SIZE_OFFSET..], (uint)totalSize); + header[COMMAND_OFFSET] = (byte)Command2.Request; + WriteUInt128(header[REQUEST_CLIENT_OFFSET..], frame.ClientId); + BinaryPrimitives.WriteUInt64LittleEndian(header[REQUEST_TIMESTAMP_OFFSET..], 0); + BinaryPrimitives.WriteUInt64LittleEndian(header[REQUEST_ID_OFFSET..], frame.RequestId); + header[REQUEST_OPERATION_OFFSET] = (byte)operation; + BinaryPrimitives.WriteUInt64LittleEndian(header[REQUEST_SESSION_OFFSET..], frame.SessionId); + + if (operation == VsrOperation.NonReplicated) + { + BinaryPrimitives.WriteUInt32LittleEndian(header[REQUEST_RESERVED_OFFSET..], (uint)code); + } + + return totalSize; + } + + internal static Command2 PeekCommand(ReadOnlySpan header) + { + return header[COMMAND_OFFSET] switch + { + (byte)Command2.Reply => Command2.Reply, + (byte)Command2.Eviction => Command2.Eviction, + _ => Command2.Reserved + }; + } + + /// Total frame size (header plus body) the peer announced. + internal static uint ReadSize(ReadOnlySpan header) + { + return BinaryPrimitives.ReadUInt32LittleEndian(header[SIZE_OFFSET..]); + } + + /// + /// Pre-commit deny channel. Nonzero means refused before commit with an empty body; a committed + /// rejection stamps 0 here and rides the result section instead. + /// + internal static uint ReadStatus(ReadOnlySpan header) + { + return BinaryPrimitives.ReadUInt32LittleEndian(header[REPLY_STATUS_OFFSET..]); + } + + internal static VsrOperation ReadReplyOperation(ReadOnlySpan header) + { + var operation = header[REPLY_OPERATION_OFFSET]; + if (!VsrOperations.IsKnown(operation)) + { + throw VsrError.Exception(VsrError.INVALID_COMMAND, $"Reply carries an unknown operation ({operation})."); + } + + return (VsrOperation)operation; + } + + internal static EvictionFrame ReadEviction(ReadOnlySpan header) + { + var reason = header[EVICTION_REASON_OFFSET]; + + return new EvictionFrame( + reason is > 0 and <= (byte)EvictionReason.MalformedLogin ? (EvictionReason)reason : null, + BinaryPrimitives.ReadUInt32LittleEndian(header[EVICTION_PROTOCOL_VERSION_OFFSET..]), + BinaryPrimitives.ReadUInt32LittleEndian(header[EVICTION_PROTOCOL_VERSION_MIN_OFFSET..])); + } + + private static void WriteUInt128(Span destination, UInt128 value) + { + BinaryPrimitives.WriteUInt64LittleEndian(destination, (ulong)value); + BinaryPrimitives.WriteUInt64LittleEndian(destination[8..], (ulong)(value >> 64)); + } +} + +/// Session-terminal eviction frame. Version fields are zero unless the reason is a protocol mismatch. +internal readonly record struct EvictionFrame( + EvictionReason? Reason, + uint ServerProtocolVersion, + uint ServerProtocolVersionMin); diff --git a/foreign/csharp/Iggy_SDK/Vsr/VsrOperation.cs b/foreign/csharp/Iggy_SDK/Vsr/VsrOperation.cs new file mode 100644 index 0000000000..5653a23c06 --- /dev/null +++ b/foreign/csharp/Iggy_SDK/Vsr/VsrOperation.cs @@ -0,0 +1,275 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Apache.Iggy.Utils; + +namespace Apache.Iggy.Vsr; + +/// +/// Replicated operation discriminant, byte 176 of a request header and byte 208 of a reply header. +/// Mirrors core/binary_protocol/src/consensus/operation.rs; discriminants are wire-pinned. +/// +internal enum VsrOperation : byte +{ + Reserved = 0, + Register = 1, + NonReplicated = 2, + Logout = 3, + + CreateTopicWithAssignments = 64, + CreatePartitionsWithAssignments = 65, + RemoveConsumerGroupMember = 66, + CompleteConsumerGroupRevocation = 67, + TruncatePartition = 68, + + CreateStream = 128, + UpdateStream = 129, + DeleteStream = 130, + PurgeStream = 131, + CreateTopic = 132, + UpdateTopic = 133, + DeleteTopic = 134, + PurgeTopic = 135, + CreatePartitions = 136, + DeletePartitions = 137, + DeleteSegments = 138, + CreateConsumerGroup = 139, + DeleteConsumerGroup = 140, + CreateUser = 141, + UpdateUser = 142, + DeleteUser = 143, + ChangePassword = 144, + UpdatePermissions = 145, + CreatePersonalAccessToken = 146, + DeletePersonalAccessToken = 147, + JoinConsumerGroup = 148, + LeaveConsumerGroup = 149, + + SendMessages = 160, + StoreConsumerOffset = 161, + DeleteConsumerOffset = 162, + StoreConsumerOffset2 = 164, + DeleteConsumerOffset2 = 165 +} + +internal static class VsrOperations +{ + private const byte InternalStart = (byte)VsrOperation.CreateTopicWithAssignments; + private const byte MetadataStart = (byte)VsrOperation.CreateStream; + private const byte PartitionStart = (byte)VsrOperation.SendMessages; + + /// + /// Non-replicated codes this build knows to leave no server-side state behind, so re-sending one after a + /// lost connection is indistinguishable from sending it once. Flushing an unsaved buffer is included: it + /// is idempotent by construction, a second flush writes nothing new. + /// + private static readonly HashSet NonReplicatedReadCodes = + [ + CommandCodes.PING_CODE, + CommandCodes.GET_STATS_CODE, + CommandCodes.GET_SNAPSHOT_CODE, + CommandCodes.GET_CLUSTER_METADATA_CODE, + CommandCodes.GET_ME_CODE, + CommandCodes.GET_CLIENT_CODE, + CommandCodes.GET_CLIENTS_CODE, + CommandCodes.GET_USER_CODE, + CommandCodes.GET_USERS_CODE, + CommandCodes.GET_PERSONAL_ACCESS_TOKENS_CODE, + CommandCodes.FLUSH_UNSAVED_BUFFER_CODE, + CommandCodes.GET_CONSUMER_OFFSET_CODE, + CommandCodes.GET_STREAM_CODE, + CommandCodes.GET_STREAMS_CODE, + CommandCodes.GET_TOPIC_CODE, + CommandCodes.GET_TOPICS_CODE, + CommandCodes.GET_CONSUMER_GROUP_CODE, + CommandCodes.GET_CONSUMER_GROUPS_CODE, + CommandCodes.SYNC_CONSUMER_GROUP_CODE + ]; + + /// + /// Maps a legacy command code to the operation its request header carries. An unmapped code rides + /// : the command table is a protocol registry, not a + /// per-server capability list, so the server is the authority on codes this SDK build does not know. + /// + internal static VsrOperation ForCode(int code) + { + return code switch + { + CommandCodes.LOGIN_REGISTER_CODE or CommandCodes.LOGIN_REGISTER_WITH_PAT_CODE => VsrOperation.Register, + + // VSR replaces the legacy login codes with the register handshake. A legacy login code arriving here + // means a caller bypassed the typed path, and sending it non-replicated would look like a working + // login while no session is ever bound. + CommandCodes.LOGIN_USER_CODE or CommandCodes.LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE => + throw VsrError.Exception(VsrError.INVALID_COMMAND, + $"Command {code} cannot be sent as a consensus request."), + CommandCodes.LOGOUT_USER_CODE => VsrOperation.Logout, + CommandCodes.CREATE_USER_CODE => VsrOperation.CreateUser, + CommandCodes.DELETE_USER_CODE => VsrOperation.DeleteUser, + CommandCodes.UPDATE_USER_CODE => VsrOperation.UpdateUser, + CommandCodes.UPDATE_PERMISSIONS_CODE => VsrOperation.UpdatePermissions, + CommandCodes.CHANGE_PASSWORD_CODE => VsrOperation.ChangePassword, + CommandCodes.CREATE_PERSONAL_ACCESS_TOKEN_CODE => VsrOperation.CreatePersonalAccessToken, + CommandCodes.DELETE_PERSONAL_ACCESS_TOKEN_CODE => VsrOperation.DeletePersonalAccessToken, + CommandCodes.SEND_MESSAGES_CODE => VsrOperation.SendMessages, + CommandCodes.STORE_CONSUMER_OFFSET_CODE => VsrOperation.StoreConsumerOffset, + CommandCodes.DELETE_CONSUMER_OFFSET_CODE => VsrOperation.DeleteConsumerOffset, + CommandCodes.STORE_CONSUMER_OFFSET_2_CODE => VsrOperation.StoreConsumerOffset2, + CommandCodes.DELETE_CONSUMER_OFFSET_2_CODE => VsrOperation.DeleteConsumerOffset2, + CommandCodes.CREATE_STREAM_CODE => VsrOperation.CreateStream, + CommandCodes.DELETE_STREAM_CODE => VsrOperation.DeleteStream, + CommandCodes.UPDATE_STREAM_CODE => VsrOperation.UpdateStream, + CommandCodes.PURGE_STREAM_CODE => VsrOperation.PurgeStream, + CommandCodes.CREATE_TOPIC_CODE => VsrOperation.CreateTopic, + CommandCodes.DELETE_TOPIC_CODE => VsrOperation.DeleteTopic, + CommandCodes.UPDATE_TOPIC_CODE => VsrOperation.UpdateTopic, + CommandCodes.PURGE_TOPIC_CODE => VsrOperation.PurgeTopic, + CommandCodes.CREATE_PARTITIONS_CODE => VsrOperation.CreatePartitions, + CommandCodes.DELETE_PARTITIONS_CODE => VsrOperation.DeletePartitions, + CommandCodes.DELETE_SEGMENTS_CODE => VsrOperation.DeleteSegments, + CommandCodes.CREATE_CONSUMER_GROUP_CODE => VsrOperation.CreateConsumerGroup, + CommandCodes.DELETE_CONSUMER_GROUP_CODE => VsrOperation.DeleteConsumerGroup, + CommandCodes.JOIN_CONSUMER_GROUP_CODE => VsrOperation.JoinConsumerGroup, + CommandCodes.LEAVE_CONSUMER_GROUP_CODE => VsrOperation.LeaveConsumerGroup, + _ => VsrOperation.NonReplicated + }; + } + + /// + /// Whether losing the connection mid-request leaves nothing for the server to deduplicate, so the request + /// can be re-issued on a fresh session by the reconnect path instead of failing the caller. + /// + internal static bool IsReplaySafeRead(int code, bool isLoginRegister, ReadOnlySpan body) + { + // A register lost mid-flight is replayable: the retry re-arms the session under a fresh client id, so + // it cannot be mistaken for the first one. At worst the server keeps an entry nobody binds, which it + // ages out. Reporting an unknown outcome here instead would deny the caller the retry that is in fact + // the only correct response. + if (isLoginRegister) + { + return true; + } + + var operation = ForCode(code); + + // A consumer offset write carries an absolute offset and the server applies it as an unconditional + // overwrite, on a plane that keeps no client table to dedup against, so a replay lands on the same + // value. Denying the retry here reports an unknown outcome for a blip on an offset commit, which + // takes down the consume loop over a write that was safe to repeat. + if (operation is VsrOperation.StoreConsumerOffset or VsrOperation.StoreConsumerOffset2 + or VsrOperation.DeleteConsumerOffset or VsrOperation.DeleteConsumerOffset2) + { + return true; + } + + if (operation != VsrOperation.NonReplicated) + { + return false; + } + + // A poll that auto-commits moves the consumer offset server-side, so a reply lost after the commit + // would make the replay start past a batch the caller never saw. auto_commit is the last body byte. + if (code == CommandCodes.POLL_MESSAGES_CODE) + { + return body.Length > 0 && body[^1] == 0; + } + + // Everything else non-replicated is replay-safe only if this build knows it to be a read. An unmapped + // code also lands on NonReplicated, and re-sending one the server implements as a mutation would apply + // it twice with nothing to deduplicate against. + return NonReplicatedReadCodes.Contains(code); + } + + /// + /// Whether the byte is a declared discriminant. Replies carry a server-controlled operation byte, so + /// an undeclared value is rejected rather than classified by range. + /// + internal static bool IsKnown(byte value) + { + return (VsrOperation)value switch + { + VsrOperation.Reserved or VsrOperation.Register or VsrOperation.NonReplicated or VsrOperation.Logout => + true, + >= VsrOperation.CreateTopicWithAssignments and <= VsrOperation.TruncatePartition => true, + >= VsrOperation.CreateStream and <= VsrOperation.LeaveConsumerGroup => true, + VsrOperation.SendMessages or VsrOperation.StoreConsumerOffset or VsrOperation.DeleteConsumerOffset + or VsrOperation.StoreConsumerOffset2 or VsrOperation.DeleteConsumerOffset2 => true, + _ => false + }; + } + + /// Replica / journal only; never emitted by a client. + internal static bool IsInternal(this VsrOperation operation) + { + return (byte)operation >= InternalStart && (byte)operation < MetadataStart; + } + + /// + /// Control-plane operations handled by shard 0. Enumerated member by member, mirroring + /// Operation::is_metadata, rather than tested as a numeric range: the metadata block runs to 159 + /// but the last member declared today is 149, so a range would silently exclude the next operation added + /// upstream. That operation would then also read as not result-framed, and a committed rejection in its + /// reply would decode as a successful payload. + /// + internal static bool IsMetadata(this VsrOperation operation) + { + return operation.IsInternal() || operation is VsrOperation.CreateStream + or VsrOperation.UpdateStream + or VsrOperation.DeleteStream + or VsrOperation.PurgeStream + or VsrOperation.CreateTopic + or VsrOperation.UpdateTopic + or VsrOperation.DeleteTopic + or VsrOperation.PurgeTopic + or VsrOperation.CreatePartitions + or VsrOperation.DeletePartitions + or VsrOperation.CreateConsumerGroup + or VsrOperation.DeleteConsumerGroup + or VsrOperation.CreateUser + or VsrOperation.UpdateUser + or VsrOperation.DeleteUser + or VsrOperation.ChangePassword + or VsrOperation.UpdatePermissions + or VsrOperation.CreatePersonalAccessToken + or VsrOperation.DeletePersonalAccessToken + or VsrOperation.JoinConsumerGroup + or VsrOperation.LeaveConsumerGroup; + } + + /// + /// Data-plane operations routed by namespace to the shard owning the partition. + /// is deliberately neither metadata nor partition: the + /// server resolves it to an internal TruncatePartition, yet it still carries a packed namespace. + /// + internal static bool IsPartition(this VsrOperation operation) + { + return (byte)operation >= PartitionStart; + } + + /// + /// Whether a reply for this operation leads its body with the committed result section. Metadata ops + /// always do; on the partition plane only the consumer-offset ops do. Register is result-framed only + /// when its body is non-empty, which is why that case stays in . + /// + internal static bool IsResultFramed(this VsrOperation operation) + { + return operation.IsMetadata() || operation is VsrOperation.StoreConsumerOffset + or VsrOperation.StoreConsumerOffset2 + or VsrOperation.DeleteConsumerOffset + or VsrOperation.DeleteConsumerOffset2; + } +} diff --git a/foreign/csharp/Iggy_SDK/Vsr/VsrReplyDecoder.cs b/foreign/csharp/Iggy_SDK/Vsr/VsrReplyDecoder.cs new file mode 100644 index 0000000000..efb2130157 --- /dev/null +++ b/foreign/csharp/Iggy_SDK/Vsr/VsrReplyDecoder.cs @@ -0,0 +1,206 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Buffers.Binary; + +namespace Apache.Iggy.Vsr; + +/// +/// Decodes a reply frame into the typed payload the classic response readers expect. +/// +internal static class VsrReplyDecoder +{ + internal const int RESULT_COUNT_LENGTH = 4; + internal const int RESULT_ENTRY_LENGTH = 8; + + /// + /// Runs the decode funnel in the order the wire contract requires: eviction frames first, then the + /// announced size, then the pre-commit status word, and only then the body and its committed result + /// section. Reading the status before the body is what lets an empty deny reply fail with the right + /// error instead of a truncation. + /// + internal static ReadOnlyMemory Decode(ReadOnlySpan header, ReadOnlyMemory body) + { + if (header.Length < VsrHeader.HEADER_SIZE) + { + throw VsrError.Exception(VsrError.EMPTY_RESPONSE, "Reply header is shorter than the consensus header."); + } + + switch (VsrHeader.PeekCommand(header)) + { + case Command2.Eviction: + throw ToException(VsrHeader.ReadEviction(header)); + case Command2.Reply: + break; + default: + throw VsrError.Exception(VsrError.INVALID_COMMAND, + $"Unexpected consensus frame ({header[VsrHeader.COMMAND_OFFSET]})."); + } + + var expectedBody = ReadBodySize(header); + if (body.Length < expectedBody) + { + throw VsrError.Exception(VsrError.INVALID_COMMAND, "Reply body is shorter than the announced frame size."); + } + + var status = VsrHeader.ReadStatus(header); + if (status != 0) + { + throw VsrError.FromServer((int)status, $"Server rejected the request with status {status}."); + } + + return SplitResultSection(VsrHeader.ReadReplyOperation(header), body[..expectedBody]); + } + + /// Body length announced by the frame, i.e. the bytes to read after the header. + internal static int ReadBodySize(ReadOnlySpan header) + { + var size = VsrHeader.ReadSize(header); + if (size < VsrHeader.HEADER_SIZE || size > int.MaxValue) + { + throw VsrError.Exception(VsrError.INVALID_COMMAND, $"Reply announced an invalid frame size ({size})."); + } + + return (int)size - VsrHeader.HEADER_SIZE; + } + + internal static Exception ToException(EvictionFrame eviction) + { + var (code, message) = eviction.Reason switch + { + // The five reasons below fall into the catch-all of the shared grader, so they carry INVALID_COMMAND + // even where a narrower status would read better. The message keeps the detail; the code is the part + // six SDKs agree on. + EvictionReason.ClientReleaseTooLow => (VsrError.INVALID_COMMAND, + "Client release is below the cluster minimum."), + EvictionReason.ClientReleaseTooHigh => (VsrError.INVALID_COMMAND, + "Client release is above the cluster maximum."), + EvictionReason.InvalidRequestOperation => (VsrError.INVALID_COMMAND, + "Server rejected the request operation."), + EvictionReason.InvalidRequestBody => (VsrError.INVALID_COMMAND, "Server rejected the request body."), + EvictionReason.InvalidRequestBodySize => (VsrError.INVALID_COMMAND, + "Server rejected the request body size."), + EvictionReason.InvalidCredentials => (VsrError.INVALID_CREDENTIALS, "Invalid credentials."), + EvictionReason.InvalidToken => (VsrError.INVALID_PERSONAL_ACCESS_TOKEN, "Invalid personal access token."), + EvictionReason.UserInactive => (VsrError.UNAUTHENTICATED, "User is inactive."), + EvictionReason.SessionError => (VsrError.UNAUTHENTICATED, "Session error."), + EvictionReason.NoSession => (VsrError.UNAUTHENTICATED, "No session for this client."), + EvictionReason.SessionTooLow => (VsrError.UNAUTHENTICATED, "Session is below the cluster minimum."), + EvictionReason.SessionReleaseMismatch => (VsrError.UNAUTHENTICATED, "Session release mismatch."), + EvictionReason.StaleClient => (VsrError.STALE_CLIENT, "Client missed too many heartbeats."), + EvictionReason.IncompatibleProtocol => IncompatibleProtocol(eviction), + EvictionReason.MalformedLogin => (VsrError.INVALID_FORMAT, "Malformed login body."), + + // Reserved and any reason this build cannot decode share the grader's catch-all. + null => (VsrError.INVALID_COMMAND, "Session evicted for an unrecognized reason."), + _ => (VsrError.INVALID_COMMAND, $"Session evicted ({eviction.Reason}).") + }; + + return VsrError.FromServer(code, message); + } + + /// + /// Leading result code of a committed reply body: 0 for success, otherwise the first entry's result. + /// null when the body cannot hold what the count claims - corruption, never a silent success. + /// + internal static uint? ReadResultCode(ReadOnlySpan body) + { + if (!TryReadUInt32(body, 0, out var count)) + { + return null; + } + + if (count == 0) + { + return 0; + } + + return TryReadUInt32(body, RESULT_COUNT_LENGTH + 4, out var result) ? result : null; + } + + /// Byte length of the leading result section, i.e. where the typed payload starts. + internal static int? ReadResultSectionLength(ReadOnlySpan body) + { + if (!TryReadUInt32(body, 0, out var count)) + { + return null; + } + + var length = RESULT_COUNT_LENGTH + (long)count * RESULT_ENTRY_LENGTH; + + return body.Length >= length ? (int)length : null; + } + + private static (int Code, string Message) IncompatibleProtocol(EvictionFrame eviction) + { + if (eviction.ServerProtocolVersionMin == 0 || + eviction.ServerProtocolVersion < eviction.ServerProtocolVersionMin) + { + return (VsrError.UNAUTHENTICATED, "Server rejected the client protocol version."); + } + + return (VsrError.INCOMPATIBLE_PROTOCOL_VERSION, + $"Client protocol version {LoginRegister.PROTOCOL_VERSION} is outside the range accepted by the server " + + $"({eviction.ServerProtocolVersionMin}..{eviction.ServerProtocolVersion})."); + } + + /// + /// Strips the committed result section from a result-framed reply and maps a committed rejection to + /// its error. Register replies are result-framed only when non-empty: a terminal register failure + /// ships an empty body and is passed through to fail the typed response decode. + /// + private static ReadOnlyMemory SplitResultSection(VsrOperation operation, ReadOnlyMemory body) + { + var resultFramed = operation.IsResultFramed() || + (operation == VsrOperation.Register && !body.IsEmpty); + if (!resultFramed) + { + return body; + } + + var code = ReadResultCode(body.Span); + if (code is null) + { + throw VsrError.Exception(VsrError.INVALID_COMMAND, "Reply carries a malformed committed result section."); + } + + if (code != 0) + { + throw VsrError.FromServer((int)code.Value, $"Server rejected the request with status {code.Value}."); + } + + var payloadStart = ReadResultSectionLength(body.Span) + ?? throw VsrError.Exception(VsrError.INVALID_COMMAND, + "Reply carries a truncated committed result section."); + + return body[payloadStart..]; + } + + private static bool TryReadUInt32(ReadOnlySpan body, int offset, out uint value) + { + if (body.Length < offset + 4) + { + value = 0; + + return false; + } + + value = BinaryPrimitives.ReadUInt32LittleEndian(body.Slice(offset, 4)); + + return true; + } +} diff --git a/foreign/csharp/Iggy_SDK_Tests/ClientTests/IggyClientFactoryTests.cs b/foreign/csharp/Iggy_SDK_Tests/ClientTests/IggyClientFactoryTests.cs new file mode 100644 index 0000000000..de665603d1 --- /dev/null +++ b/foreign/csharp/Iggy_SDK_Tests/ClientTests/IggyClientFactoryTests.cs @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Apache.Iggy.Configuration; +using Apache.Iggy.Enums; +using Apache.Iggy.Factory; + +namespace Apache.Iggy.Tests.ClientTests; + +public sealed class IggyClientFactoryTests +{ + [Fact] + public void CreateClient_CreatesTcpClient() + { + var options = new IggyClientConfigurator + { + BaseAddress = "127.0.0.1:8090", + Protocol = Protocol.Tcp + }; + + Assert.Equal(64 * 1024 * 1024, options.MaxResponseFrameSize); + + using var client = IggyClientFactory.CreateClient(options) as IDisposable; + Assert.NotNull(client); + } + + [Fact] + public void CreateClient_RejectsMaxResponseFrameSizeBelowHeader() + { + var options = new IggyClientConfigurator + { + BaseAddress = "127.0.0.1:8090", + Protocol = Protocol.Tcp, + MaxResponseFrameSize = 255 + }; + + Assert.Throws(() => IggyClientFactory.CreateClient(options)); + } + + [Fact] + public void CreateClient_AcceptsMaxResponseFrameSizeUnderHttp() + { + var options = new IggyClientConfigurator + { + BaseAddress = "http://127.0.0.1:3000", + Protocol = Protocol.Http, + MaxResponseFrameSize = 1 + }; + + using var client = IggyClientFactory.CreateClient(options) as IDisposable; + Assert.NotNull(client); + } +} diff --git a/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/IggyConsumerBuilderTests.cs b/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/IggyConsumerBuilderTests.cs index db0af7c3fc..8178484708 100644 --- a/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/IggyConsumerBuilderTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/ConsumerTests/IggyConsumerBuilderTests.cs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +using System.Text; using Apache.Iggy.Consumers; using Apache.Iggy.Encryption; using Apache.Iggy.Enums; @@ -69,4 +70,22 @@ public void Build_WithEncryptorAndAfterReceiveCommit_DoesNotThrow() Assert.NotNull(consumer); } + + [Fact] + public void TypedBuild_OverTcp_CreatesTheClient() + { + IggyConsumerBuilder builder = IggyConsumerBuilder + .Create(StreamId, TopicId, Consumer.New(1), new StringDeserializer()); + builder.WithConnection(Protocol.Tcp, "127.0.0.1:8090", "user", "pass"); + + Assert.NotNull(builder.Build()); + } + + private sealed class StringDeserializer : IDeserializer + { + public string Deserialize(ReadOnlyMemory data) + { + return Encoding.UTF8.GetString(data.Span); + } + } } diff --git a/foreign/csharp/Iggy_SDK_Tests/MapperTests/BinaryMapper.cs b/foreign/csharp/Iggy_SDK_Tests/MapperTests/BinaryMapper.cs index c86fb463db..e37aa35750 100644 --- a/foreign/csharp/Iggy_SDK_Tests/MapperTests/BinaryMapper.cs +++ b/foreign/csharp/Iggy_SDK_Tests/MapperTests/BinaryMapper.cs @@ -432,6 +432,82 @@ public void MapRentedMessages_WithEncryptor_TamperedCiphertext_ThrowsMessageDecr Assert.IsAssignableFrom(ex.InnerException); } + [Fact] + public void MapSendMessages_ReturnsConfirmations() + { + // Wire layout mirrors core/binary_protocol responses/messages/send_messages.rs: + // [count:4][stream_id:4][topic_id:4][partition_id:4][base_offset:8]* + var payload = new byte[4 + 20]; + BinaryPrimitives.WriteUInt32LittleEndian(payload.AsSpan(0, 4), 1); + BinaryPrimitives.WriteUInt32LittleEndian(payload.AsSpan(4, 4), 1); + BinaryPrimitives.WriteUInt32LittleEndian(payload.AsSpan(8, 4), 2); + BinaryPrimitives.WriteUInt32LittleEndian(payload.AsSpan(12, 4), 3); + BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(16, 8), 42); + + var response = Mappers.BinaryMapper.MapSendMessages(payload); + + var confirmation = Assert.Single(response.Confirmations); + Assert.Equal(1u, confirmation.StreamId); + Assert.Equal(2u, confirmation.TopicId); + Assert.Equal(3u, confirmation.PartitionId); + Assert.Equal(42ul, confirmation.BaseOffset); + } + + [Fact] + public void MapSendMessages_MultipleConfirmations_ReturnsAll() + { + var payload = new byte[4 + 3 * 20]; + BinaryPrimitives.WriteUInt32LittleEndian(payload.AsSpan(0, 4), 3); + for (var i = 0; i < 3; i++) + { + var position = 4 + i * 20; + BinaryPrimitives.WriteUInt32LittleEndian(payload.AsSpan(position, 4), 1); + BinaryPrimitives.WriteUInt32LittleEndian(payload.AsSpan(position + 4, 4), 2); + BinaryPrimitives.WriteUInt32LittleEndian(payload.AsSpan(position + 8, 4), (uint)i); + BinaryPrimitives.WriteUInt64LittleEndian(payload.AsSpan(position + 12, 8), (ulong)(100 + i)); + } + + var response = Mappers.BinaryMapper.MapSendMessages(payload); + + Assert.Equal(3, response.Confirmations.Count); + Assert.Equal(2u, response.Confirmations[2].PartitionId); + Assert.Equal(102ul, response.Confirmations[2].BaseOffset); + } + + [Fact] + public void MapSendMessages_EmptyBody_Throws() + { + Assert.Throws(() => Mappers.BinaryMapper.MapSendMessages([])); + } + + [Fact] + public void MapSendMessages_ZeroCount_ReturnsNoConfirmations() + { + var response = Mappers.BinaryMapper.MapSendMessages(new byte[4]); + + Assert.Empty(response.Confirmations); + } + + [Theory] + [InlineData(4 + 19)] // truncated entry + [InlineData(4 + 21)] // trailing byte + public void MapSendMessages_ShapeMismatch_Throws(int payloadLength) + { + var payload = new byte[payloadLength]; + BinaryPrimitives.WriteUInt32LittleEndian(payload.AsSpan(0, 4), 1); + + Assert.Throws(() => Mappers.BinaryMapper.MapSendMessages(payload)); + } + + [Fact] + public void MapSendMessages_BogusCount_DoesNotOverflow() + { + var payload = new byte[4]; + BinaryPrimitives.WriteUInt32LittleEndian(payload, uint.MaxValue); + + Assert.Throws(() => Mappers.BinaryMapper.MapSendMessages(payload)); + } + private static byte[] BuildEncryptedFrame(AesMessageEncryptor encryptor, ulong offset, ReadOnlySpan plainPayload, ReadOnlySpan plainHeaders) { diff --git a/foreign/csharp/Iggy_SDK_Tests/PublisherTests/IggyPublisherBuilderTests.cs b/foreign/csharp/Iggy_SDK_Tests/PublisherTests/IggyPublisherBuilderTests.cs new file mode 100644 index 0000000000..1172a752dd --- /dev/null +++ b/foreign/csharp/Iggy_SDK_Tests/PublisherTests/IggyPublisherBuilderTests.cs @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Buffers; +using System.Text; +using Apache.Iggy.Enums; +using Apache.Iggy.Publishers; + +namespace Apache.Iggy.Tests.PublisherTests; + +public class IggyPublisherBuilderTests +{ + private static readonly Identifier StreamId = Identifier.Numeric(1); + private static readonly Identifier TopicId = Identifier.Numeric(1); + + [Fact] + public void TypedBuild_OverTcp_CreatesTheClient() + { + IggyPublisherBuilder builder + = IggyPublisherBuilder.Create(StreamId, TopicId, new StringSerializer()); + builder.WithConnection(Protocol.Tcp, "127.0.0.1:8090", "user", "pass"); + + Assert.NotNull(builder.Build()); + } + + private sealed class StringSerializer : ISerializer + { + public void Serialize(string data, IBufferWriter writer) + { + writer.Write(Encoding.UTF8.GetBytes(data)); + } + } +} diff --git a/foreign/csharp/Iggy_SDK_Tests/PublisherTests/IggyTypedPublisherTests.cs b/foreign/csharp/Iggy_SDK_Tests/PublisherTests/IggyTypedPublisherTests.cs index 61776aec4c..2537b2f3bb 100644 --- a/foreign/csharp/Iggy_SDK_Tests/PublisherTests/IggyTypedPublisherTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/PublisherTests/IggyTypedPublisherTests.cs @@ -308,7 +308,7 @@ private static IIggyClient BuildClient(SendRecorder recorder) CancellationToken _) => { recorder.Record(stream, topic, partitioning, messages); - return Task.CompletedTask; + return Task.FromResult(new SendMessagesResponse { Confirmations = [] }); }); mock.Setup(c => c.SendMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) @@ -316,7 +316,7 @@ private static IIggyClient BuildClient(SendRecorder recorder) CancellationToken _) => { recorder.Record(stream, topic, partitioning, new[] { message }); - return Task.CompletedTask; + return Task.FromResult(new SendMessagesResponse { Confirmations = [] }); }); return mock.Object; diff --git a/foreign/csharp/Iggy_SDK_Tests/UtilityTests/SendUnitTests.cs b/foreign/csharp/Iggy_SDK_Tests/UtilityTests/SendUnitTests.cs index c7095f312a..ec5736bcd6 100644 --- a/foreign/csharp/Iggy_SDK_Tests/UtilityTests/SendUnitTests.cs +++ b/foreign/csharp/Iggy_SDK_Tests/UtilityTests/SendUnitTests.cs @@ -17,6 +17,7 @@ using System.Buffers; using System.Diagnostics; +using Apache.Iggy.Contracts; using Apache.Iggy.Extensions; using Apache.Iggy.IggyClient; using Apache.Iggy.Messages; @@ -211,7 +212,7 @@ public async Task Processor_Dispose_DisposesQueuedOwners() public async Task Processor_DrainStaysPending_UntilSendCompletes() { // Gate the send so the unit is in flight; the drain must not complete mid-flight. - var gate = new TaskCompletionSource(); + var gate = new TaskCompletionSource(); var client = new Mock(); client.Setup(c => c.SendMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>(), It.IsAny())) @@ -229,7 +230,7 @@ public async Task Processor_DrainStaysPending_UntilSendCompletes() await Task.Delay(100, TestContext.Current.CancellationToken); Assert.False(drain.IsCompleted); - gate.SetResult(); + gate.SetResult(new SendMessagesResponse { Confirmations = [] }); await drain.WaitAsync(TimeSpan.FromSeconds(5), TestContext.Current.CancellationToken); } @@ -238,7 +239,7 @@ private static Mock MockClient(List sentCounts) var client = new Mock(); client.Setup(c => c.SendMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>(), It.IsAny())) - .Returns(Task.CompletedTask) + .ReturnsAsync(new SendMessagesResponse { Confirmations = [] }) .Callback((Identifier _, Identifier _, Partitioning _, IList messages, CancellationToken _) => sentCounts.Add(messages.Count)); return client; diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/ConsensusSessionTests.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/ConsensusSessionTests.cs new file mode 100644 index 0000000000..51e4d7f0a1 --- /dev/null +++ b/foreign/csharp/Iggy_SDK_Tests/VsrTests/ConsensusSessionTests.cs @@ -0,0 +1,218 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Apache.Iggy.Exceptions; +using Apache.Iggy.Vsr; + +namespace Apache.Iggy.Tests.VsrTests; + +public sealed class ConsensusSessionTests +{ + [Fact] + public void NewSession_IsUnboundWithNonZeroClientId() + { + var session = new ConsensusSession(); + + Assert.False(session.IsBound); + Assert.Null(session.Session); + Assert.NotEqual(UInt128.Zero, session.ClientId); + } + + [Fact] + public void NewSession_MintsUniqueClientIds() + { + Assert.NotEqual(new ConsensusSession().ClientId, new ConsensusSession().ClientId); + } + + [Fact] + public void BeginRegister_OnFreshSessionKeepsClientId() + { + var session = new ConsensusSession(7); + + Assert.Equal(0UL, session.Resolve(VsrOperation.Register).RequestId); + Assert.Equal((UInt128)7, session.ClientId); + Assert.False(session.IsBound); + } + + /// + /// The re-arm mints the client id the register is encoded with, so the frame must carry the new one, not + /// the one the previous session used. + /// + [Fact] + public void Resolve_RegisterAfterBindReportsTheReArmedClientId() + { + var session = new ConsensusSession(7); + session.Resolve(VsrOperation.Register); + session.Bind(42); + + var frame = session.Resolve(VsrOperation.Register); + + Assert.Equal(session.ClientId, frame.ClientId); + Assert.NotEqual((UInt128)7, frame.ClientId); + } + + [Fact] + public void BeginRegister_AfterBindReArmsWithFreshClientId() + { + var session = new ConsensusSession(7); + session.Resolve(VsrOperation.Register); + session.Bind(42); + session.Resolve(VsrOperation.CreateStream); + + Assert.Equal(0UL, session.Resolve(VsrOperation.Register).RequestId); + Assert.NotEqual((UInt128)7, session.ClientId); + Assert.False(session.IsBound); + Assert.Equal(1UL, session.RequestCounter); + } + + /// + /// A register that never bound is cleared by the reset its failure path runs, and the retry re-arms onto a + /// fresh client id rather than reusing the one the server may have already seen. + /// + [Fact] + public void BeginRegister_AfterConsumedRegisterAndResetReArms() + { + var session = new ConsensusSession(7); + session.Resolve(VsrOperation.Register); + + session.Reset(); + + Assert.Equal(0UL, session.Resolve(VsrOperation.Register).RequestId); + Assert.False(session.IsBound); + Assert.NotEqual((UInt128)7, session.ClientId); + } + + [Fact] + public void NextRequestId_IsMonotonicAfterBind() + { + var session = new ConsensusSession(1); + session.Resolve(VsrOperation.Register); + session.Bind(10); + + Assert.Equal(1UL, session.Resolve(VsrOperation.CreateStream).RequestId); + Assert.Equal(2UL, session.Resolve(VsrOperation.CreateStream).RequestId); + Assert.Equal(3UL, session.RequestCounter); + } + + [Fact] + public void NextRequestId_BeforeBindThrows() + { + var session = new ConsensusSession(1); + + var error = Assert.Throws(() => session.Resolve(VsrOperation.CreateStream)); + + Assert.Equal(VsrError.UNAUTHENTICATED, error.StatusCode); + } + + [Fact] + public void Resolve_DoesNotConsumeAnIdForNonReplicatedOrPartitionOps() + { + var session = new ConsensusSession(1); + session.Resolve(VsrOperation.Register); + session.Bind(10); + + Assert.Equal(1UL, session.Resolve(VsrOperation.NonReplicated).RequestId); + Assert.Equal(1UL, session.Resolve(VsrOperation.SendMessages).RequestId); + Assert.Equal(1UL, session.RequestCounter); + } + + [Fact] + public void Bind_TwiceThrows() + { + var session = new ConsensusSession(1); + session.Resolve(VsrOperation.Register); + session.Bind(10); + + Assert.Throws(() => session.Bind(20)); + Assert.Equal(10UL, session.Session); + } + + [Fact] + public void Bind_ZeroThrows() + { + var session = new ConsensusSession(1); + + var exception = Assert.Throws(() => session.Bind(0)); + Assert.Equal(VsrError.INVALID_FORMAT, exception.StatusCode); + } + + [Fact] + public void Bind_WithoutAnInFlightRegisterThrows() + { + var session = new ConsensusSession(1); + + Assert.Throws(() => session.Bind(10)); + } + + /// + /// Binding runs after the sending lock is released, so a drop can have re-armed the identity since the + /// register committed. The reset clears the pending register, and binding regardless would pair the + /// session the server issued to the old client id with the one the re-arm minted. + /// + [Fact] + public void Bind_AfterTheIdentityReArmedThrows() + { + var session = new ConsensusSession(1); + session.Resolve(VsrOperation.Register); + + session.Reset(); + + Assert.Throws(() => session.Bind(10)); + Assert.False(session.IsBound); + } + + /// + /// Two concurrent registers would otherwise re-arm the identity under the first one, so its bind would + /// pair a committed session with a client id the server never saw. + /// + [Fact] + public void Resolve_SecondRegisterWhileOneIsInFlightThrows() + { + var session = new ConsensusSession(1); + session.Resolve(VsrOperation.Register); + + var error = Assert.Throws(() => session.Resolve(VsrOperation.Register)); + + Assert.Equal(VsrError.UNAUTHENTICATED, error.StatusCode); + } + + [Fact] + public void Resolve_RegisterIsAllowedAgainAfterReset() + { + var session = new ConsensusSession(1); + session.Resolve(VsrOperation.Register); + + session.Reset(); + + Assert.Equal(0UL, session.Resolve(VsrOperation.Register).RequestId); + } + + [Fact] + public void Reset_ClearsBindingAndCounter() + { + var session = new ConsensusSession(1); + session.Resolve(VsrOperation.Register); + session.Bind(10); + session.Resolve(VsrOperation.CreateStream); + + session.Reset(); + + Assert.False(session.IsBound); + Assert.Equal(1UL, session.RequestCounter); + Assert.NotEqual((UInt128)1, session.ClientId); + } +} diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/ConsumerGroupClientStateTests.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/ConsumerGroupClientStateTests.cs new file mode 100644 index 0000000000..c1c4f00ada --- /dev/null +++ b/foreign/csharp/Iggy_SDK_Tests/VsrTests/ConsumerGroupClientStateTests.cs @@ -0,0 +1,176 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.IO.Hashing; +using System.Text; +using Apache.Iggy.Vsr; + +namespace Apache.Iggy.Tests.VsrTests; + +public sealed class ConsumerGroupClientStateTests +{ + private static readonly GroupKey Key = + new(Identifier.Numeric(1), Identifier.Numeric(2), Identifier.Numeric(3)); + + private static readonly TopicKey Topic = new(Identifier.Numeric(1), Identifier.Numeric(2)); + + [Fact] + public void NextGroupPartition_RoundRobinsThenWraps() + { + var state = new ConsumerGroupClientState(); + state.SetAssignment(Key, 1, [0, 1, 2]); + + Assert.Equal([0, 1, 2, 0], [ + state.NextGroupPartition(Key), state.NextGroupPartition(Key), + state.NextGroupPartition(Key), state.NextGroupPartition(Key) + ]); + } + + [Fact] + public void SetAssignment_OnNewGenerationResetsCursor() + { + var state = new ConsumerGroupClientState(); + state.SetAssignment(Key, 1, [0, 1, 2]); + state.NextGroupPartition(Key); + state.NextGroupPartition(Key); + + state.SetAssignment(Key, 2, [5]); + + Assert.Equal(5u, state.NextGroupPartition(Key)); + } + + [Fact] + public void SetAssignment_OnSameGenerationKeepsCursor() + { + var state = new ConsumerGroupClientState(); + state.SetAssignment(Key, 1, [0, 1, 2]); + state.NextGroupPartition(Key); + + state.SetAssignment(Key, 1, [0, 1, 2]); + + Assert.Equal(1u, state.NextGroupPartition(Key)); + } + + [Fact] + public void NextGroupPartition_WithoutAssignmentIsNull() + { + var state = new ConsumerGroupClientState(); + + Assert.False(state.HasAssignment(Key)); + Assert.Null(state.NextGroupPartition(Key)); + } + + [Fact] + public void InvalidateAssignment_KeepsMembership() + { + var state = new ConsumerGroupClientState(); + state.RegisterGroup(Key, Identifier.Numeric(1), Identifier.Numeric(2), Identifier.Numeric(3)); + state.SetAssignment(Key, 1, [0]); + + state.InvalidateAssignment(Key); + + Assert.False(state.HasAssignment(Key)); + Assert.True(state.IsRegistered(Key)); + } + + [Fact] + public void MemberHoldingNoPartitions_StaysRegistered() + { + var state = new ConsumerGroupClientState(); + state.RegisterGroup(Key, Identifier.Numeric(1), Identifier.Numeric(2), Identifier.Numeric(3)); + state.SetAssignment(Key, 1, []); + + Assert.False(state.HasAssignment(Key)); + Assert.True(state.IsRegistered(Key)); + + state.DeregisterGroup(Key); + + Assert.False(state.IsRegistered(Key)); + } + + [Fact] + public void RegisteredGroups_ReturnsJoinedIdentifiers() + { + var state = new ConsumerGroupClientState(); + state.RegisterGroup(Key, Identifier.Numeric(1), Identifier.Numeric(2), Identifier.String("group")); + + IReadOnlyList groups = state.RegisteredGroups(); + + var group = Assert.Single(groups); + + Assert.Equal("1", group.StreamId.ToString()); + Assert.Equal("2", group.TopicId.ToString()); + Assert.Equal("group", group.GroupId.ToString()); + } + + [Fact] + public void NextBalancedPartition_RoundRobinsThenWraps() + { + var state = new ConsumerGroupClientState(); + + Assert.Equal([0, 1, 2, 0], [ + state.NextBalancedPartition(Topic, 3), state.NextBalancedPartition(Topic, 3), + state.NextBalancedPartition(Topic, 3), state.NextBalancedPartition(Topic, 3) + ]); + } + + [Fact] + public void NextBalancedPartition_WithNoPartitionsIsZero() + { + Assert.Equal(0u, new ConsumerGroupClientState().NextBalancedPartition(Topic, 0)); + } + + [Fact] + public void ClearSessionScoped_DropsMembershipAndAssignmentsButKeepsTopicState() + { + var state = new ConsumerGroupClientState(); + state.RegisterGroup(Key, Identifier.Numeric(1), Identifier.Numeric(2), Identifier.Numeric(3)); + state.SetAssignment(Key, 1, [0]); + state.SetPartitionCount(Topic, 4); + state.NextBalancedPartition(Topic, 4); + + state.ClearSessionScoped(); + + Assert.False(state.IsRegistered(Key)); + Assert.False(state.HasAssignment(Key)); + Assert.Equal(4u, state.PartitionCount(Topic)); + Assert.Equal(1u, state.NextBalancedPartition(Topic, 4)); + } + + [Fact] + public void TopicKey_SeparatesNumericFromNamedIdentifiers() + { + Assert.NotEqual(new TopicKey(Identifier.Numeric(1), Identifier.Numeric(1)), + new TopicKey(Identifier.String("1"), Identifier.String("1"))); + } + + /// + /// Message-key partitioning has to agree with the Rust client byte for byte, or the two SDKs put the same + /// key on different partitions. Vectors come from calculate_32 (XxHash32::oneshot(0, data)). + /// + [Theory] + [InlineData("", 0x02cc5d05u)] + [InlineData("a", 0x550d7456u)] + [InlineData("abc", 0x32d153ffu)] + [InlineData("hello world", 0xcebb6622u)] + [InlineData("iggy-message-key", 0xf54b51c9u)] + [InlineData("1234567890123456789012345", 0xb10c970eu)] + public void XxHash32_MatchesRustVectors(string value, uint expected) + { + Assert.Equal(expected, XxHash32.HashToUInt32(Encoding.UTF8.GetBytes(value))); + } +} diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/CredentialBoundsTests.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/CredentialBoundsTests.cs new file mode 100644 index 0000000000..4fc2e6b306 --- /dev/null +++ b/foreign/csharp/Iggy_SDK_Tests/VsrTests/CredentialBoundsTests.cs @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Apache.Iggy.Exceptions; +using Apache.Iggy.Vsr; + +namespace Apache.Iggy.Tests.VsrTests; + +/// +/// Mirrors validate_username / validate_password in +/// core/common/src/traits/binary_impls/mod.rs. +/// +public sealed class CredentialBoundsTests +{ + [Fact] + public void ValidateUsername_AcceptsTheServerBounds() + { + CredentialBounds.ValidateUsername(new string('a', CredentialBounds.MIN_USERNAME_LENGTH)); + CredentialBounds.ValidateUsername(new string('a', CredentialBounds.MAX_USERNAME_LENGTH)); + } + + [Fact] + public void ValidateUsername_RejectsOutOfBounds() + { + AssertRejects(VsrError.INVALID_USERNAME, + () => CredentialBounds.ValidateUsername(new string('a', CredentialBounds.MIN_USERNAME_LENGTH - 1))); + AssertRejects(VsrError.INVALID_USERNAME, + () => CredentialBounds.ValidateUsername(new string('a', CredentialBounds.MAX_USERNAME_LENGTH + 1))); + } + + [Fact] + public void ValidatePassword_AcceptsTheServerBounds() + { + CredentialBounds.ValidatePassword(new string('a', CredentialBounds.MIN_PASSWORD_LENGTH)); + CredentialBounds.ValidatePassword(new string('a', CredentialBounds.MAX_PASSWORD_LENGTH)); + } + + [Fact] + public void ValidatePassword_RejectsOutOfBounds() + { + AssertRejects(VsrError.INVALID_PASSWORD, + () => CredentialBounds.ValidatePassword(new string('a', CredentialBounds.MIN_PASSWORD_LENGTH - 1))); + AssertRejects(VsrError.INVALID_PASSWORD, + () => CredentialBounds.ValidatePassword(new string('a', CredentialBounds.MAX_PASSWORD_LENGTH + 1))); + } + + [Fact] + public void ValidatePassword_CountsUtf8BytesNotChars() + { + // 51 two-byte code points encode to 102 bytes, over the server's limit despite fitting in chars. + AssertRejects(VsrError.INVALID_PASSWORD, () => CredentialBounds.ValidatePassword(new string('ż', 51))); + } + + private static void AssertRejects(int statusCode, Action action) + { + Assert.Equal(statusCode, Assert.Throws(action).StatusCode); + } +} diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/LoginRegisterTests.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/LoginRegisterTests.cs new file mode 100644 index 0000000000..ea752241b2 --- /dev/null +++ b/foreign/csharp/Iggy_SDK_Tests/VsrTests/LoginRegisterTests.cs @@ -0,0 +1,165 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Buffers.Binary; +using System.Text; +using Apache.Iggy.Exceptions; +using Apache.Iggy.Utils; +using Apache.Iggy.Vsr; + +namespace Apache.Iggy.Tests.VsrTests; + +public sealed class LoginRegisterTests +{ + private static int AssertVersionInfo(byte[] body) + { + Assert.Equal(LoginRegister.PROTOCOL_VERSION, BinaryPrimitives.ReadUInt32LittleEndian(body)); + var position = 4; + position += AssertName(body, position, LoginRegister.SDK_NAME); + position += AssertName(body, position, SdkVersion.Value); + + return position; + } + + private static int AssertName(byte[] body, int position, string expected) + { + var length = body[position]; + Assert.Equal(Encoding.UTF8.GetByteCount(expected), length); + Assert.Equal(expected, Encoding.UTF8.GetString(body, position + 1, length)); + + return 1 + length; + } + + [Fact] + public void Serialize_WritesVersionInfoThenCredentialsThenContext() + { + var body = LoginRegister.Serialize("admin", "secret"); + + var position = AssertVersionInfo(body); + position += AssertName(body, position, "admin"); + position += AssertName(body, position, "secret"); + + Assert.Equal(0u, BinaryPrimitives.ReadUInt32LittleEndian(body.AsSpan(position))); + Assert.Equal(position + 4, body.Length); + } + + [Fact] + public void Serialize_AppendsTheClientContextWithAUInt32Length() + { + var body = LoginRegister.Serialize("admin", "secret", "ctx"); + + var position = AssertVersionInfo(body); + position += AssertName(body, position, "admin"); + position += AssertName(body, position, "secret"); + + Assert.Equal(3u, BinaryPrimitives.ReadUInt32LittleEndian(body.AsSpan(position))); + Assert.Equal("ctx", Encoding.UTF8.GetString(body, position + 4, 3)); + Assert.Equal(position + 7, body.Length); + } + + [Fact] + public void SerializeWithPersonalAccessToken_PutsTheTokenInTheCredentialSlot() + { + var body = LoginRegister.SerializeWithPersonalAccessToken("token"); + + var position = AssertVersionInfo(body); + position += AssertName(body, position, "token"); + + Assert.Equal(0u, BinaryPrimitives.ReadUInt32LittleEndian(body.AsSpan(position))); + Assert.Equal(position + 4, body.Length); + } + + [Fact] + public void Serialize_RejectsAnEmptyCredential() + { + var error = Assert.Throws(() => + LoginRegister.Serialize("admin", string.Empty)); + + Assert.Equal(VsrError.INVALID_PASSWORD, error.StatusCode); + } + + [Fact] + public void Serialize_RejectsACredentialAboveTheLengthPrefix() + { + var error = Assert.Throws(() => + LoginRegister.Serialize("admin", new string('x', 256))); + + Assert.Equal(VsrError.INVALID_PASSWORD, error.StatusCode); + } + + /// + /// The u8 length prefix is also guarded inside the writer, for the fields no credential check covers. + /// + [Fact] + public void SerializeWithPersonalAccessToken_RejectsATokenAboveTheLengthPrefix() + { + var error = Assert.Throws(() => + LoginRegister.SerializeWithPersonalAccessToken(new string('x', 256))); + + Assert.Equal(VsrError.INVALID_PERSONAL_ACCESS_TOKEN, error.StatusCode); + } + + [Fact] + public void SerializeWithPersonalAccessToken_RejectsAnEmptyToken() + { + var error = Assert.Throws(() => + LoginRegister.SerializeWithPersonalAccessToken(string.Empty)); + + Assert.Equal(VsrError.INVALID_PERSONAL_ACCESS_TOKEN, error.StatusCode); + } + + [Fact] + public void Deserialize_ReadsTheRegisterReply() + { + var body = VsrTestPayloads.Concat(VsrTestPayloads.UInt32(42), new byte[8], VsrTestPayloads.UInt32(10243), + [5], "0.8.0"u8.ToArray()); + BinaryPrimitives.WriteUInt64LittleEndian(body.AsSpan(4), 100); + + var response = LoginRegister.Deserialize(body); + + Assert.Equal(42u, response.UserId); + Assert.Equal(100UL, response.Session); + Assert.Equal(10243u, response.ServerProtocolVersion); + Assert.Equal("0.8.0", response.ServerVersion); + } + + [Fact] + public void Deserialize_TruncatedReplyIsInvalidFormat() + { + var body = VsrTestPayloads.Concat(VsrTestPayloads.UInt32(42), new byte[8], VsrTestPayloads.UInt32(10243), + [5], "0.8.0"u8.ToArray()); + + for (var length = 0; length < body.Length; length++) + { + var truncated = body[..length]; + var exception = Assert.Throws(() => + LoginRegister.Deserialize(truncated)); + + Assert.Equal(VsrError.INVALID_FORMAT, exception.StatusCode); + } + } + + [Fact] + public void Deserialize_EmptyReplyIsTheTerminalRegisterRejection() + { + var exception = Assert.Throws(() => + LoginRegister.Deserialize(ReadOnlySpan.Empty)); + + Assert.Equal(VsrError.INVALID_FORMAT, exception.StatusCode); + Assert.Contains("rejected the login", exception.Message); + } +} diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/ServerAddressTests.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/ServerAddressTests.cs new file mode 100644 index 0000000000..7288b45170 --- /dev/null +++ b/foreign/csharp/Iggy_SDK_Tests/VsrTests/ServerAddressTests.cs @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Apache.Iggy.Utils; + +namespace Apache.Iggy.Tests.VsrTests; + +/// +/// Mirrors core/sdk/src/leader_aware.rs test_is_same_address and +/// test_normalize_address: the leader redirect check has to agree with the Rust SDK, or the two +/// disagree on whether a roster entry names the node this connection is already on. +/// +public sealed class ServerAddressTests +{ + [Theory] + [InlineData("127.0.0.1:8090", "127.0.0.1:8090")] + [InlineData("localhost:8090", "127.0.0.1:8090")] + [InlineData("LOCALHOST:8090", "127.0.0.1:8090")] + [InlineData("[::1]:8090", "[::1]:8090")] + public void IsSame_MatchesEquivalentEndpoints(string first, string second) + { + Assert.True(ServerAddress.IsSame(first, second)); + Assert.True(ServerAddress.IsSame(second, first)); + } + + [Theory] + [InlineData("127.0.0.1:8090", "127.0.0.1:8091")] + [InlineData("192.168.1.1:8090", "127.0.0.1:8090")] + [InlineData("localhost:8090", "127.0.0.1:8091")] + [InlineData("iggy-1:8090", "iggy-2:8090")] + [InlineData("127.0.0.1:8090", "")] + public void IsSame_SeparatesDistinctEndpoints(string first, string second) + { + Assert.False(ServerAddress.IsSame(first, second)); + Assert.False(ServerAddress.IsSame(second, first)); + } + + [Theory] + [InlineData("localhost:8090", "127.0.0.1:8090")] + [InlineData("LOCALHOST:8090", "127.0.0.1:8090")] + [InlineData("[::]:8090", "[::1]:8090")] + [InlineData("0.0.0.0:8090", "127.0.0.1:8090")] + [InlineData("my-localhost-1:8090", "my-localhost-1:8090")] + public void Normalize_ResolvesHostAliases(string address, string expected) + { + Assert.Equal(expected, ServerAddress.Normalize(address)); + } + + /// + /// A host name that merely contains an alias is a different node, and a server bound to the unspecified + /// address answers on the loopback one. + /// + [Theory] + [InlineData("my-localhost-1:8090", "127.0.0.1:8090", false)] + [InlineData("localhost.example.com:8090", "127.0.0.1:8090", false)] + [InlineData("0.0.0.0:8090", "127.0.0.1:8090", true)] + [InlineData("[::]:8090", "[::1]:8090", true)] + [InlineData("0.0.0.0:8090", "[::1]:8090", false)] + public void IsSame_ResolvesHostAliasesWithoutSubstringMatching(string first, string second, bool same) + { + Assert.Equal(same, ServerAddress.IsSame(first, second)); + Assert.Equal(same, ServerAddress.IsSame(second, first)); + } + + [Fact] + public void IsSame_FallsBackToNormalizedStringsForUnparsableAddresses() + { + Assert.True(ServerAddress.IsSame("Iggy-Node:8090", "iggy-node:8090")); + Assert.False(ServerAddress.IsSame("iggy-node:8090", "iggy-node:8091")); + } +} diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/SyncConsumerGroupTests.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/SyncConsumerGroupTests.cs new file mode 100644 index 0000000000..1d185668b5 --- /dev/null +++ b/foreign/csharp/Iggy_SDK_Tests/VsrTests/SyncConsumerGroupTests.cs @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Buffers.Binary; +using Apache.Iggy.Exceptions; +using Apache.Iggy.Vsr; + +namespace Apache.Iggy.Tests.VsrTests; + +public sealed class SyncConsumerGroupTests +{ + [Fact] + public void Decode_ReadsGenerationAndPartitions() + { + var assignment = SyncConsumerGroupAssignment.Decode(Encode(7, [0, 2, 4])); + + Assert.Equal(7ul, assignment.Generation); + Assert.Equal([0, 2, 4], [.. assignment.Partitions]); + } + + [Fact] + public void Decode_ReadsEmptyAssignment() + { + var assignment = SyncConsumerGroupAssignment.Decode(Encode(1, [])); + + Assert.Equal(1ul, assignment.Generation); + Assert.Empty(assignment.Partitions); + } + + [Fact] + public void Decode_IgnoresTrailingBytes() + { + var body = Encode(1, [3]).Concat(new byte[8]).ToArray(); + + Assert.Equal([3], [.. SyncConsumerGroupAssignment.Decode(body).Partitions]); + } + + [Fact] + public void Decode_TruncatedBodyThrows() + { + var body = Encode(7, [1, 2]); + for (var length = 0; length < body.Length; length++) + { + var truncated = body[..length]; + var error = Assert.Throws(() => + SyncConsumerGroupAssignment.Decode(truncated)); + + Assert.Equal(VsrError.INVALID_COMMAND, error.StatusCode); + } + } + + /// + /// A count the body cannot back must fail rather than allocate for it: the value is attacker-reachable + /// through a corrupted frame. + /// + [Fact] + public void Decode_ImplausiblePartitionCountThrows() + { + var body = new byte[12]; + BinaryPrimitives.WriteUInt32LittleEndian(body.AsSpan(8, 4), uint.MaxValue); + + var error = Assert.Throws(() => SyncConsumerGroupAssignment.Decode(body)); + + Assert.Equal(VsrError.INVALID_COMMAND, error.StatusCode); + } + + private static byte[] Encode(ulong generation, uint[] partitions) + { + var body = new byte[12 + partitions.Length * 4]; + BinaryPrimitives.WriteUInt64LittleEndian(body.AsSpan(0, 8), generation); + BinaryPrimitives.WriteUInt32LittleEndian(body.AsSpan(8, 4), (uint)partitions.Length); + for (var i = 0; i < partitions.Length; i++) + { + BinaryPrimitives.WriteUInt32LittleEndian(body.AsSpan(12 + i * 4, 4), partitions[i]); + } + + return body; + } +} diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrHeaderTests.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrHeaderTests.cs new file mode 100644 index 0000000000..73eb1fbb4d --- /dev/null +++ b/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrHeaderTests.cs @@ -0,0 +1,283 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Buffers.Binary; +using Apache.Iggy.Exceptions; +using Apache.Iggy.Utils; +using Apache.Iggy.Vsr; + +namespace Apache.Iggy.Tests.VsrTests; + +public sealed class VsrHeaderTests +{ + private static ConsensusSession BoundSession(ulong session = 5) + { + var consensusSession = new ConsensusSession(0x0102_0304_0506_0708); + consensusSession.Resolve(VsrOperation.Register); + consensusSession.Bind(session); + + return consensusSession; + } + + private static byte[] Encode(ConsensusSession session, int code, byte[] payload, out int totalSize) + { + var header = new byte[VsrHeader.HEADER_SIZE]; + totalSize = VsrHeader.EncodeRequestHeader(header, session, code, payload); + + return header; + } + + private static ulong ReadUInt64(byte[] header, int offset) + { + return BinaryPrimitives.ReadUInt64LittleEndian(header.AsSpan(offset)); + } + + private static uint ReadUInt32(byte[] header, int offset) + { + return BinaryPrimitives.ReadUInt32LittleEndian(header.AsSpan(offset)); + } + + /// + /// The client no longer inspects a payload to route, so a partition-plane body it cannot interpret is + /// passed through for the server to resolve instead of failing at encode time. + /// + [Fact] + public void Encode_UnroutablePartitionPayloadIsPassedThroughToTheServer() + { + var session = BoundSession(); + var payload = VsrTestPayloads.ConsumerOffset(VsrTestPayloads.NumericIdentifier(4), + VsrTestPayloads.NumericIdentifier(5), null); + var header = new byte[VsrHeader.HEADER_SIZE]; + + var totalSize = VsrHeader.EncodeRequestHeader(header, session, + CommandCodes.STORE_CONSUMER_OFFSET_CODE, payload); + + Assert.Equal(VsrHeader.HEADER_SIZE + payload.Length, totalSize); + Assert.Equal((byte)VsrOperation.StoreConsumerOffset, header[VsrHeader.REQUEST_OPERATION_OFFSET]); + Assert.True(session.IsBound); + } + + [Fact] + public void Encode_RegisterUsesZeroRequestAndSession() + { + var session = new ConsensusSession(7); + var payload = LoginRegister.Serialize("admin", "secret"); + + var header = Encode(session, CommandCodes.LOGIN_REGISTER_CODE, payload, out var totalSize); + + Assert.Equal(VsrHeader.HEADER_SIZE + payload.Length, totalSize); + Assert.Equal((uint)totalSize, ReadUInt32(header, VsrHeader.SIZE_OFFSET)); + Assert.Equal((byte)Command2.Request, header[VsrHeader.COMMAND_OFFSET]); + Assert.Equal((byte)VsrOperation.Register, header[VsrHeader.REQUEST_OPERATION_OFFSET]); + Assert.Equal(0UL, ReadUInt64(header, VsrHeader.REQUEST_ID_OFFSET)); + Assert.Equal(0UL, ReadUInt64(header, VsrHeader.REQUEST_SESSION_OFFSET)); + Assert.Equal(0UL, ReadUInt64(header, VsrHeader.REQUEST_TIMESTAMP_OFFSET)); + Assert.Equal(7UL, ReadUInt64(header, VsrHeader.REQUEST_CLIENT_OFFSET)); + Assert.Equal(0UL, ReadUInt64(header, VsrHeader.REQUEST_CLIENT_OFFSET + 8)); + } + + [Fact] + public void Encode_WritesClientIdAsTwoLittleEndianHalvesLowFirst() + { + var session = new ConsensusSession(new UInt128(0xAABB_CCDD_EEFF_0011, 0x1122_3344_5566_7788)); + + var header = Encode(session, CommandCodes.PING_CODE, [], out _); + + Assert.Equal(0x1122_3344_5566_7788UL, ReadUInt64(header, VsrHeader.REQUEST_CLIENT_OFFSET)); + Assert.Equal(0xAABB_CCDD_EEFF_0011UL, ReadUInt64(header, VsrHeader.REQUEST_CLIENT_OFFSET + 8)); + } + + [Fact] + public void Encode_NonReplicatedDoesNotAdvanceCounterAndCarriesCodeInReserved() + { + var session = BoundSession(); + + var header = Encode(session, CommandCodes.PING_CODE, [], out _); + + Assert.Equal((byte)VsrOperation.NonReplicated, header[VsrHeader.REQUEST_OPERATION_OFFSET]); + Assert.Equal(1UL, ReadUInt64(header, VsrHeader.REQUEST_ID_OFFSET)); + Assert.Equal(1UL, session.RequestCounter); + Assert.Equal(5UL, ReadUInt64(header, VsrHeader.REQUEST_SESSION_OFFSET)); + Assert.Equal((uint)CommandCodes.PING_CODE, ReadUInt32(header, VsrHeader.REQUEST_RESERVED_OFFSET)); + } + + [Fact] + public void Encode_UnknownCodeRidesNonReplicated() + { + var session = BoundSession(); + + var header = Encode(session, 9999, [], out _); + + Assert.Equal((byte)VsrOperation.NonReplicated, header[VsrHeader.REQUEST_OPERATION_OFFSET]); + Assert.Equal(9999u, ReadUInt32(header, VsrHeader.REQUEST_RESERVED_OFFSET)); + } + + [Fact] + public void Encode_NonReplicatedWithoutSessionSendsSessionZero() + { + var session = new ConsensusSession(1); + + var header = Encode(session, CommandCodes.PING_CODE, [], out _); + + Assert.Equal(0UL, ReadUInt64(header, VsrHeader.REQUEST_SESSION_OFFSET)); + } + + [Fact] + public void Encode_MetadataAdvancesTheCounter() + { + var session = BoundSession(); + + var header = Encode(session, CommandCodes.CREATE_STREAM_CODE, [1, 2, 3], out _); + + Assert.Equal((byte)VsrOperation.CreateStream, header[VsrHeader.REQUEST_OPERATION_OFFSET]); + Assert.Equal(1UL, ReadUInt64(header, VsrHeader.REQUEST_ID_OFFSET)); + Assert.Equal(2UL, session.RequestCounter); + Assert.Equal(0u, ReadUInt32(header, VsrHeader.REQUEST_RESERVED_OFFSET)); + } + + [Fact] + public void Encode_LogoutAdvancesTheCounter() + { + var session = BoundSession(); + + var header = Encode(session, CommandCodes.LOGOUT_USER_CODE, [], out _); + + Assert.Equal((byte)VsrOperation.Logout, header[VsrHeader.REQUEST_OPERATION_OFFSET]); + Assert.Equal(1UL, ReadUInt64(header, VsrHeader.REQUEST_ID_OFFSET)); + Assert.Equal(2UL, session.RequestCounter); + } + + [Fact] + public void Encode_PartitionOpDoesNotAdvanceTheCounter() + { + var session = BoundSession(); + var payload = VsrTestPayloads.SendMessagesToPartition(2, 3, 4); + + var header = Encode(session, CommandCodes.SEND_MESSAGES_CODE, payload, out _); + + Assert.Equal((byte)VsrOperation.SendMessages, header[VsrHeader.REQUEST_OPERATION_OFFSET]); + Assert.Equal(1UL, ReadUInt64(header, VsrHeader.REQUEST_ID_OFFSET)); + Assert.Equal(1UL, session.RequestCounter); + } + + [Fact] + public void Encode_ReplicatedOpWithoutSessionIsUnauthenticated() + { + var session = new ConsensusSession(1); + + var exception = Assert.Throws(() => + Encode(session, CommandCodes.CREATE_STREAM_CODE, [1], out _)); + + Assert.Equal(VsrError.UNAUTHENTICATED, exception.StatusCode); + Assert.Equal(1UL, session.RequestCounter); + } + + [Fact] + public void Encode_ClearsStaleBytesFromAReusedBuffer() + { + var header = new byte[VsrHeader.HEADER_SIZE]; + Array.Fill(header, (byte)0xFF); + var session = BoundSession(); + + VsrHeader.EncodeRequestHeader(header, session, CommandCodes.CREATE_STREAM_CODE, [1]); + + Assert.Equal(0u, ReadUInt32(header, VsrHeader.REQUEST_RESERVED_OFFSET)); + Assert.All(header[..VsrHeader.SIZE_OFFSET], stale => Assert.Equal(0, stale)); + } + + [Fact] + public void Encode_RejectsAShortBuffer() + { + var session = BoundSession(); + + Assert.Throws(() => + VsrHeader.EncodeRequestHeader(new byte[VsrHeader.HEADER_SIZE - 1], session, CommandCodes.PING_CODE, [])); + } + + [Fact] + public void PeekCommand_MapsOnlyTheClientVisibleFrames() + { + var header = new byte[VsrHeader.HEADER_SIZE]; + + header[VsrHeader.COMMAND_OFFSET] = (byte)Command2.Reply; + Assert.Equal(Command2.Reply, VsrHeader.PeekCommand(header)); + + header[VsrHeader.COMMAND_OFFSET] = (byte)Command2.Eviction; + Assert.Equal(Command2.Eviction, VsrHeader.PeekCommand(header)); + + header[VsrHeader.COMMAND_OFFSET] = 6; + Assert.Equal(Command2.Reserved, VsrHeader.PeekCommand(header)); + } + + [Fact] + public void ReadReplyOperation_RejectsAnUnknownDiscriminant() + { + var header = new byte[VsrHeader.HEADER_SIZE]; + header[VsrHeader.REPLY_OPERATION_OFFSET] = 200; + + var exception = Assert.Throws(() => VsrHeader.ReadReplyOperation(header)); + + Assert.Equal(VsrError.INVALID_COMMAND, exception.StatusCode); + } + + [Fact] + public void ReadEviction_ReadsReasonAndProtocolWindow() + { + var header = new byte[VsrHeader.HEADER_SIZE]; + header[VsrHeader.EVICTION_REASON_OFFSET] = (byte)EvictionReason.IncompatibleProtocol; + BinaryPrimitives.WriteUInt32LittleEndian(header.AsSpan(VsrHeader.EVICTION_PROTOCOL_VERSION_OFFSET), 10243); + BinaryPrimitives.WriteUInt32LittleEndian(header.AsSpan(VsrHeader.EVICTION_PROTOCOL_VERSION_MIN_OFFSET), 10240); + + var eviction = VsrHeader.ReadEviction(header); + + Assert.Equal(EvictionReason.IncompatibleProtocol, eviction.Reason); + Assert.Equal(10243u, eviction.ServerProtocolVersion); + Assert.Equal(10240u, eviction.ServerProtocolVersionMin); + } + + /// + /// An undecodable reason stays null rather than collapsing onto a named one, and grades through the + /// shared grader's catch-all like every reason the Rust mapping does not name. + /// + [Fact] + public void ReadEviction_UnknownReasonDecodesAsUndecodable() + { + var header = new byte[VsrHeader.HEADER_SIZE]; + header[VsrHeader.EVICTION_REASON_OFFSET] = 200; + + Assert.Null(VsrHeader.ReadEviction(header).Reason); + Assert.Equal(VsrError.INVALID_COMMAND, + Assert.IsType(VsrReplyDecoder.ToException(VsrHeader.ReadEviction(header))) + .StatusCode); + } + + /// + /// Reason 0 is the Reserved sentinel the server rejects on the wire, so it decodes as an unrecognized + /// reason rather than as one. Rust grades it through the same catch-all. + /// + [Fact] + public void ReadEviction_ReservedReasonDecodesAsUnknown() + { + var header = new byte[VsrHeader.HEADER_SIZE]; + header[VsrHeader.EVICTION_REASON_OFFSET] = (byte)EvictionReason.Reserved; + + Assert.Null(VsrHeader.ReadEviction(header).Reason); + Assert.Equal(VsrError.INVALID_COMMAND, + Assert.IsType(VsrReplyDecoder.ToException(VsrHeader.ReadEviction(header))) + .StatusCode); + } +} diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrOperationTests.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrOperationTests.cs new file mode 100644 index 0000000000..bd5686e031 --- /dev/null +++ b/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrOperationTests.cs @@ -0,0 +1,167 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using Apache.Iggy.Utils; +using Apache.Iggy.Exceptions; +using Apache.Iggy.Vsr; + +namespace Apache.Iggy.Tests.VsrTests; + +public sealed class VsrOperationTests +{ + [Theory] + [InlineData(CommandCodes.LOGOUT_USER_CODE, (byte)VsrOperation.Logout)] + [InlineData(CommandCodes.CREATE_USER_CODE, (byte)VsrOperation.CreateUser)] + [InlineData(CommandCodes.DELETE_USER_CODE, (byte)VsrOperation.DeleteUser)] + [InlineData(CommandCodes.UPDATE_USER_CODE, (byte)VsrOperation.UpdateUser)] + [InlineData(CommandCodes.UPDATE_PERMISSIONS_CODE, (byte)VsrOperation.UpdatePermissions)] + [InlineData(CommandCodes.CHANGE_PASSWORD_CODE, (byte)VsrOperation.ChangePassword)] + [InlineData(CommandCodes.CREATE_PERSONAL_ACCESS_TOKEN_CODE, (byte)VsrOperation.CreatePersonalAccessToken)] + [InlineData(CommandCodes.DELETE_PERSONAL_ACCESS_TOKEN_CODE, (byte)VsrOperation.DeletePersonalAccessToken)] + [InlineData(CommandCodes.SEND_MESSAGES_CODE, (byte)VsrOperation.SendMessages)] + [InlineData(CommandCodes.STORE_CONSUMER_OFFSET_CODE, (byte)VsrOperation.StoreConsumerOffset)] + [InlineData(CommandCodes.DELETE_CONSUMER_OFFSET_CODE, (byte)VsrOperation.DeleteConsumerOffset)] + [InlineData(CommandCodes.STORE_CONSUMER_OFFSET_2_CODE, (byte)VsrOperation.StoreConsumerOffset2)] + [InlineData(CommandCodes.DELETE_CONSUMER_OFFSET_2_CODE, (byte)VsrOperation.DeleteConsumerOffset2)] + [InlineData(CommandCodes.CREATE_STREAM_CODE, (byte)VsrOperation.CreateStream)] + [InlineData(CommandCodes.DELETE_STREAM_CODE, (byte)VsrOperation.DeleteStream)] + [InlineData(CommandCodes.UPDATE_STREAM_CODE, (byte)VsrOperation.UpdateStream)] + [InlineData(CommandCodes.PURGE_STREAM_CODE, (byte)VsrOperation.PurgeStream)] + [InlineData(CommandCodes.CREATE_TOPIC_CODE, (byte)VsrOperation.CreateTopic)] + [InlineData(CommandCodes.DELETE_TOPIC_CODE, (byte)VsrOperation.DeleteTopic)] + [InlineData(CommandCodes.UPDATE_TOPIC_CODE, (byte)VsrOperation.UpdateTopic)] + [InlineData(CommandCodes.PURGE_TOPIC_CODE, (byte)VsrOperation.PurgeTopic)] + [InlineData(CommandCodes.CREATE_PARTITIONS_CODE, (byte)VsrOperation.CreatePartitions)] + [InlineData(CommandCodes.DELETE_PARTITIONS_CODE, (byte)VsrOperation.DeletePartitions)] + [InlineData(CommandCodes.DELETE_SEGMENTS_CODE, (byte)VsrOperation.DeleteSegments)] + [InlineData(CommandCodes.CREATE_CONSUMER_GROUP_CODE, (byte)VsrOperation.CreateConsumerGroup)] + [InlineData(CommandCodes.DELETE_CONSUMER_GROUP_CODE, (byte)VsrOperation.DeleteConsumerGroup)] + [InlineData(CommandCodes.JOIN_CONSUMER_GROUP_CODE, (byte)VsrOperation.JoinConsumerGroup)] + [InlineData(CommandCodes.LEAVE_CONSUMER_GROUP_CODE, (byte)VsrOperation.LeaveConsumerGroup)] + public void ForCode_MapsReplicatedCommands(int code, byte expected) + { + Assert.Equal((VsrOperation)expected, VsrOperations.ForCode(code)); + } + + [Theory] + [InlineData(CommandCodes.PING_CODE)] + [InlineData(CommandCodes.POLL_MESSAGES_CODE)] + [InlineData(CommandCodes.GET_STREAM_CODE)] + [InlineData(CommandCodes.GET_CONSUMER_OFFSET_CODE)] + [InlineData(9999)] + public void ForCode_ReadsAndUnknownCodesRideNonReplicated(int code) + { + Assert.Equal(VsrOperation.NonReplicated, VsrOperations.ForCode(code)); + } + + [Theory] + [InlineData(CommandCodes.LOGIN_REGISTER_CODE)] + [InlineData(CommandCodes.LOGIN_REGISTER_WITH_PAT_CODE)] + public void ForCode_MapsTheRegisterHandshake(int code) + { + Assert.Equal(VsrOperation.Register, VsrOperations.ForCode(code)); + } + + /// + /// A classic login sent as a consensus request would ride NonReplicated and look like it worked while no + /// session is ever bound, so the classification rejects it instead. + /// + [Theory] + [InlineData(CommandCodes.LOGIN_USER_CODE)] + [InlineData(CommandCodes.LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE)] + public void ForCode_RejectsLoginCodes(int code) + { + var error = Assert.Throws(() => VsrOperations.ForCode(code)); + + Assert.Equal(VsrError.INVALID_COMMAND, error.StatusCode); + } + + [Fact] + public void Classification_MatchesTheServerSideRanges() + { + Assert.True(VsrOperation.CreateTopicWithAssignments.IsInternal()); + Assert.True(VsrOperation.CreateTopicWithAssignments.IsMetadata()); + Assert.True(VsrOperation.CreateStream.IsMetadata()); + Assert.False(VsrOperation.CreateStream.IsPartition()); + Assert.True(VsrOperation.SendMessages.IsPartition()); + Assert.False(VsrOperation.SendMessages.IsMetadata()); + + // Resolved server-side to an internal truncate, so it is neither plane despite carrying a namespace. + Assert.False(VsrOperation.DeleteSegments.IsPartition()); + Assert.False(VsrOperation.DeleteSegments.IsMetadata()); + } + + [Fact] + public void IsResultFramed_CoversMetadataAndConsumerOffsetsOnly() + { + Assert.True(VsrOperation.CreateStream.IsResultFramed()); + Assert.True(VsrOperation.StoreConsumerOffset.IsResultFramed()); + Assert.True(VsrOperation.DeleteConsumerOffset2.IsResultFramed()); + Assert.False(VsrOperation.SendMessages.IsResultFramed()); + Assert.False(VsrOperation.NonReplicated.IsResultFramed()); + Assert.False(VsrOperation.Register.IsResultFramed()); + Assert.False(VsrOperation.Logout.IsResultFramed()); + } + + [Fact] + public void IsKnown_RejectsUndefinedDiscriminants() + { + Assert.True(VsrOperations.IsKnown((byte)VsrOperation.SendMessages)); + Assert.False(VsrOperations.IsKnown(163)); + Assert.False(VsrOperations.IsKnown(200)); + } + + /// Every arm of the control-plane table: shard 0 owns these, so a miss would route to a data shard. + [Theory] + [InlineData((byte)VsrOperation.CreateStream)] + [InlineData((byte)VsrOperation.UpdateStream)] + [InlineData((byte)VsrOperation.DeleteStream)] + [InlineData((byte)VsrOperation.PurgeStream)] + [InlineData((byte)VsrOperation.CreateTopic)] + [InlineData((byte)VsrOperation.UpdateTopic)] + [InlineData((byte)VsrOperation.DeleteTopic)] + [InlineData((byte)VsrOperation.PurgeTopic)] + [InlineData((byte)VsrOperation.CreatePartitions)] + [InlineData((byte)VsrOperation.DeletePartitions)] + [InlineData((byte)VsrOperation.CreateConsumerGroup)] + [InlineData((byte)VsrOperation.DeleteConsumerGroup)] + [InlineData((byte)VsrOperation.CreateUser)] + [InlineData((byte)VsrOperation.UpdateUser)] + [InlineData((byte)VsrOperation.DeleteUser)] + [InlineData((byte)VsrOperation.ChangePassword)] + [InlineData((byte)VsrOperation.UpdatePermissions)] + [InlineData((byte)VsrOperation.CreatePersonalAccessToken)] + [InlineData((byte)VsrOperation.DeletePersonalAccessToken)] + [InlineData((byte)VsrOperation.JoinConsumerGroup)] + [InlineData((byte)VsrOperation.LeaveConsumerGroup)] + public void IsMetadata_CoversTheWholeControlPlane(byte operation) + { + Assert.True(((VsrOperation)operation).IsMetadata()); + } + + [Theory] + [InlineData((byte)VsrOperation.SendMessages)] + [InlineData((byte)VsrOperation.StoreConsumerOffset)] + [InlineData((byte)VsrOperation.DeleteConsumerOffset2)] + [InlineData((byte)VsrOperation.DeleteSegments)] + [InlineData((byte)VsrOperation.NonReplicated)] + [InlineData((byte)VsrOperation.Logout)] + public void IsMetadata_LeavesTheDataPlaneOut(byte operation) + { + Assert.False(((VsrOperation)operation).IsMetadata()); + } +} diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrProtocolDriftTests.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrProtocolDriftTests.cs new file mode 100644 index 0000000000..afc9a6b7f5 --- /dev/null +++ b/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrProtocolDriftTests.cs @@ -0,0 +1,810 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Buffers.Binary; +using System.Globalization; +using System.Reflection; +using System.Text.RegularExpressions; +using Apache.Iggy.Exceptions; +using Apache.Iggy.Utils; +using Apache.Iggy.Vsr; + +namespace Apache.Iggy.Tests.VsrTests; + +/// +/// Asserts the .NET VSR constants still match the Rust wire definitions they were ported from. The unit +/// tests above pin the .NET side against itself; this one pins it against +/// core/binary_protocol/, so a Rust-side change fails here instead of in production framing. +/// Skipped when the Rust sources are not on disk (packaged SDK builds). +/// +public sealed class VsrProtocolDriftTests +{ + private const string HeaderPath = "core/binary_protocol/src/consensus/header.rs"; + private const string CommandPath = "core/binary_protocol/src/consensus/command.rs"; + private const string OperationPath = "core/binary_protocol/src/consensus/operation.rs"; + private const string CodesPath = "core/binary_protocol/src/codes.rs"; + private const string DispatchPath = "core/binary_protocol/src/dispatch.rs"; + private const string ErrorPath = "core/common/src/error/iggy_error.rs"; + private const string ManifestPath = "core/binary_protocol/Cargo.toml"; + private const string EvictionGraderPath = "core/common/src/error/eviction.rs"; + private const string ReplyResultPath = "core/binary_protocol/src/consensus/reply_result.rs"; + private const string CredentialDefaultsPath = "core/common/src/http/users/defaults.rs"; + + private const string LoginRegisterResponsePath = + "core/binary_protocol/src/responses/users/login_register.rs"; + + private const string SyncConsumerGroupResponsePath = + "core/binary_protocol/src/responses/consumer_groups/sync_consumer_group.rs"; + + /// + /// Codes the SDK deliberately does not resolve the way the Rust dispatch table declares them, with the + /// reason each one is exempt. Anything not listed here must agree with the table. + /// + private static readonly Dictionary CodeMappingExceptions = new() + { + // Non-replicated in the table because the legacy transport routes them; under VSR they are the + // consensus handshake itself and carry their own operations. + ["LOGIN_REGISTER_CODE"] = VsrOperation.Register, + ["LOGIN_REGISTER_WITH_PAT_CODE"] = VsrOperation.Register, + ["LOGOUT_USER_CODE"] = VsrOperation.Logout, + + // VSR has no legacy login. ForCode throws rather than encoding a request that would look like a + // working login while binding no session; null means "asserted to throw". + ["LOGIN_USER_CODE"] = null, + ["LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE"] = null + }; + + [Fact] + public void RequestHeaderOffsets_MatchTheRustLayout() + { + IReadOnlyDictionary offsets = RustStruct.Offsets(ReadRustSource(HeaderPath), "RequestHeader"); + + Assert.Equal(VsrHeader.SIZE_OFFSET, offsets["size"]); + Assert.Equal(VsrHeader.COMMAND_OFFSET, offsets["command"]); + Assert.Equal(VsrHeader.REQUEST_CLIENT_OFFSET, offsets["client"]); + Assert.Equal(VsrHeader.REQUEST_TIMESTAMP_OFFSET, offsets["timestamp"]); + Assert.Equal(VsrHeader.REQUEST_ID_OFFSET, offsets["request"]); + Assert.Equal(VsrHeader.REQUEST_OPERATION_OFFSET, offsets["operation"]); + Assert.Equal(VsrHeader.REQUEST_SESSION_OFFSET, offsets["session"]); + Assert.Equal(VsrHeader.REQUEST_RESERVED_OFFSET, offsets["reserved"]); + + // A routing group reintroduced on the client header would shift every field + // after it and silently reinstate a derivation this SDK no longer performs. + Assert.DoesNotContain("namespace", offsets.Keys); + Assert.DoesNotContain("group", offsets.Keys); + } + + [Fact] + public void ReplyHeaderOffsets_MatchTheRustLayout() + { + IReadOnlyDictionary offsets = RustStruct.Offsets(ReadRustSource(HeaderPath), "ReplyHeader"); + + Assert.Equal(VsrHeader.SIZE_OFFSET, offsets["size"]); + Assert.Equal(VsrHeader.COMMAND_OFFSET, offsets["command"]); + Assert.Equal(VsrHeader.REPLY_OPERATION_OFFSET, offsets["operation"]); + Assert.Equal(VsrHeader.REPLY_STATUS_OFFSET, offsets["status"]); + + Assert.DoesNotContain("namespace", offsets.Keys); + Assert.DoesNotContain("group", offsets.Keys); + } + + [Fact] + public void EvictionHeaderOffsets_MatchTheRustLayout() + { + IReadOnlyDictionary offsets = RustStruct.Offsets(ReadRustSource(HeaderPath), "EvictionHeader"); + + Assert.Equal(VsrHeader.COMMAND_OFFSET, offsets["command"]); + Assert.Equal(VsrHeader.EVICTION_CLIENT_OFFSET, offsets["client"]); + Assert.Equal(VsrHeader.EVICTION_PROTOCOL_VERSION_OFFSET, offsets["server_protocol_version"]); + Assert.Equal(VsrHeader.EVICTION_PROTOCOL_VERSION_MIN_OFFSET, offsets["server_protocol_version_min"]); + Assert.Equal(VsrHeader.EVICTION_REASON_OFFSET, offsets["reason"]); + } + + [Fact] + public void HeaderSize_MatchesTheRustConstant() + { + var source = ReadRustSource(HeaderPath); + var declared = Regex.Match(source, @"pub const HEADER_SIZE: usize = (\d+);"); + + Assert.True(declared.Success, "HEADER_SIZE is no longer declared in header.rs."); + Assert.Equal(VsrHeader.HEADER_SIZE, int.Parse(declared.Groups[1].Value, CultureInfo.InvariantCulture)); + Assert.Equal(VsrHeader.HEADER_SIZE, RustStruct.Size(source, "RequestHeader")); + Assert.Equal(VsrHeader.HEADER_SIZE, RustStruct.Size(source, "ReplyHeader")); + Assert.Equal(VsrHeader.HEADER_SIZE, RustStruct.Size(source, "EvictionHeader")); + } + + [Fact] + public void Command2Discriminants_MatchTheRustEnum() + { + IReadOnlyDictionary rust = RustEnum.Discriminants(ReadRustSource(CommandPath), "Command2"); + + AssertSubsetMatches(rust); + } + + [Fact] + public void EvictionReasons_MatchTheRustEnumExactly() + { + IReadOnlyDictionary rust = RustEnum.Discriminants(ReadRustSource(HeaderPath), "EvictionReason"); + + AssertExactMatch(rust); + } + + [Fact] + public void Operations_MatchTheRustEnumExactly() + { + IReadOnlyDictionary rust = RustEnum.Discriminants(ReadRustSource(OperationPath), "Operation"); + + AssertExactMatch(rust); + } + + [Fact] + public void ProtocolVersion_MatchesTheBinaryProtocolCrateVersion() + { + var manifest = ReadRustSource(ManifestPath); + var declared = Regex.Match(manifest, @"^version\s*=\s*""(\d+)\.(\d+)\.(\d+)", RegexOptions.Multiline); + + Assert.True(declared.Success, "iggy_binary_protocol no longer declares a semver version."); + Assert.Equal(LoginRegister.PROTOCOL_VERSION_MAJOR, Group(declared, 1)); + Assert.Equal(LoginRegister.PROTOCOL_VERSION_MINOR, Group(declared, 2)); + Assert.Equal(LoginRegister.PROTOCOL_VERSION_PATCH, Group(declared, 3)); + } + + /// + /// Pins against codes.rs as a set of values, the way the Node mirror + /// does. A name-keyed lookup alone lets a command added upstream pass unnoticed, because the constant it + /// would have to match does not exist here yet. Names are still compared where both sides declare them, + /// which catches a renumber that keeps the set size intact. + /// + [Fact] + public void CommandCodes_MatchTheRustCodes() + { + var rust = Regex.Matches(ReadRustSource(CodesPath), @"pub const (\w+_CODE): u32 = (\d+);") + .ToDictionary(code => code.Groups[1].Value, code => Group(code, 2)); + Assert.NotEmpty(rust); + + var dotnet = typeof(CommandCodes) + .GetFields(BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Static) + .Where(field => field is { IsLiteral: true, FieldType.Name: nameof(Int32) }) + .ToDictionary(field => field.Name, field => (int)field.GetRawConstantValue()!); + + var mismatches = rust + .Where(code => !dotnet.ContainsValue(code.Value)) + .Select(code => $"{code.Key} = {code.Value} is declared in Rust and missing from CommandCodes") + .Concat(dotnet + .Where(code => !rust.ContainsValue(code.Value)) + .Select(code => $"{code.Key} = {code.Value} is declared in CommandCodes and missing from Rust")) + .Concat(rust + .Where(code => dotnet.TryGetValue(code.Key, out var actual) && actual != code.Value) + .Select(code => $"{code.Key}: rust {code.Value}, .NET {dotnet[code.Key]}")) + .ToList(); + + Assert.Empty(mismatches); + } + + /// + /// Pins against the Rust dispatch table. The .NET mapping is a hand + /// written switch whose default arm is , so a command that gains + /// replication upstream would otherwise keep encoding as non-replicated here: it would apply on the node + /// that received it and never reach the others, diverging the replicas. This is the sweep + /// no_replicated_command_ever_resolves_to_non_replicated performs on the Rust side. + /// + [Fact] + public void CommandOperationTable_MatchesTheRustDispatchTable() + { + var source = ReadRustSource(DispatchPath); + + // Table entries wrap across lines once the arguments are long enough, so the separators have to match + // arbitrary whitespace rather than a single line. + var replicated = Regex.Matches(source, + @"CommandMeta::replicated\(\s*(\w+_CODE)\s*,\s*""[^""]*""\s*,\s*Operation::(\w+)\s*,?\s*\)"); + var nonReplicated = Regex.Matches(source, @"CommandMeta::non_replicated\(\s*(\w+_CODE)\s*,"); + + Assert.NotEmpty(replicated); + Assert.NotEmpty(nonReplicated); + + var mismatches = new List(); + var matched = 0; + + foreach (Match entry in replicated) + { + var codeName = entry.Groups[1].Value; + if (CommandCode(codeName) is not { } code) + { + // Skipping here would exempt the exact command this test exists to catch: one that gains + // replication upstream while the .NET side has no code for it, so every call encodes as + // non-replicated and applies on one node only. + mismatches.Add($"{codeName}: rust replicates it, absent from .NET CommandCodes"); + + continue; + } + + matched++; + if (!Enum.TryParse(entry.Groups[2].Value, out var expected)) + { + mismatches.Add($"{codeName}: rust maps to Operation::{entry.Groups[2].Value}, absent from .NET"); + + continue; + } + + var actual = ResolveOperation(codeName, code); + if (actual != expected) + { + mismatches.Add($"{codeName}: rust {expected}, .NET {ActualText(actual)}"); + } + } + + foreach (Match entry in nonReplicated) + { + var codeName = entry.Groups[1].Value; + if (CommandCode(codeName) is not { } code) + { + continue; + } + + matched++; + var expected = CodeMappingExceptions.TryGetValue(codeName, out var exempt) + ? exempt + : VsrOperation.NonReplicated; + var actual = ResolveOperation(codeName, code); + if (actual != expected) + { + mismatches.Add($"{codeName}: expected {ActualText(expected)}, .NET {ActualText(actual)}"); + } + } + + Assert.Empty(mismatches); + Assert.True(matched > 40, $"Only {matched} dispatch entries were matched by name; the Rust naming drifted."); + } + + /// + /// Pins against Operation::is_metadata. Metadata replies + /// lead their body with a committed result section, so an operation misclassified here has its rejection + /// entry decoded as payload and a refused command reads as a success. + /// + [Fact] + public void MetadataOperations_MatchTheRustClassifier() + { + var body = MatchesArmBody(ReadRustSource(OperationPath), "is_metadata"); + var rust = Regex.Matches(body, @"Self::(\w+)").Select(match => match.Groups[1].Value).ToHashSet(); + Assert.NotEmpty(rust); + + var mismatches = new List(); + foreach (VsrOperation operation in Enum.GetValues()) + { + // is_internal() short-circuits ahead of the match arm on both sides, so those members are metadata + // without appearing in the list. + var expected = rust.Contains(operation.ToString()) || operation.IsInternal(); + if (operation.IsMetadata() != expected) + { + mismatches.Add($"{operation}: rust {expected}, .NET {operation.IsMetadata()}"); + } + } + + Assert.Empty(mismatches); + } + + /// + /// Pins against Operation::is_result_framed. The Rust + /// side composes it from is_metadata plus its own partition-plane list, so pinning + /// is_metadata alone leaves the second half free to drift. An operation that gains result framing + /// upstream and not here has skip the strip, and a + /// committed rejection decodes as payload: a refused command reads as a success. + /// + [Fact] + public void ResultFramedOperations_MatchTheRustClassifier() + { + var body = MatchesArmBody(ReadRustSource(OperationPath), "is_result_framed"); + var rust = Regex.Matches(body, @"Self::(\w+)").Select(match => match.Groups[1].Value).ToHashSet(); + Assert.NotEmpty(rust); + + var mismatches = new List(); + foreach (VsrOperation operation in Enum.GetValues()) + { + // The Rust body is `is_metadata() || matches!(...)`, so the metadata members carry over without + // appearing in the list. + var expected = rust.Contains(operation.ToString()) || operation.IsMetadata(); + if (operation.IsResultFramed() != expected) + { + mismatches.Add($"{operation}: rust {expected}, .NET {operation.IsResultFramed()}"); + } + } + + Assert.Empty(mismatches); + } + + /// + /// Pins against the Rust error discriminants. These are copied by hand and drive + /// more than error text: TRANSIENT_NOT_COMMITTED and TRANSIENT_NOT_ACCEPTED are what tells a + /// same-session replay from a fresh-session reissue, so a renumber upstream would reissue a write whose + /// outcome is unknown under a client id the server cannot deduplicate against, applying it twice. + /// + [Fact] + public void ErrorCodes_MatchTheRustDiscriminants() + { + var rust = Regex.Matches(ReadRustSource(ErrorPath), @"^\s{4}(\w+)(?:\([^)]*\))?\s*=\s*(\d+),", + RegexOptions.Multiline) + .ToDictionary(match => Normalize(match.Groups[1].Value), + match => int.Parse(match.Groups[2].Value, CultureInfo.InvariantCulture)); + Assert.NotEmpty(rust); + + var mismatches = new List(); + var matched = 0; + + foreach (FieldInfo field in typeof(VsrError).GetFields(BindingFlags.NonPublic | BindingFlags.Static)) + { + if (!field.IsLiteral || field.FieldType != typeof(int)) + { + continue; + } + + if (!rust.TryGetValue(Normalize(field.Name), out var expected)) + { + mismatches.Add($"{field.Name}: no Rust variant of that name remains"); + + continue; + } + + matched++; + var actual = (int)field.GetValue(null)!; + if (actual != expected) + { + mismatches.Add($"{field.Name}: rust {expected}, .NET {actual}"); + } + } + + Assert.Empty(mismatches); + Assert.True(matched > 10, $"Only {matched} error codes were matched by name; the Rust naming drifted."); + } + + private static VsrOperation? ResolveOperation(string codeName, int code) + { + try + { + return VsrOperations.ForCode(code); + } + catch (IggyInvalidStatusCodeException) + { + // The legacy login codes reject rather than resolve; CodeMappingExceptions records that as null. + _ = codeName; + + return null; + } + } + + private static string ActualText(VsrOperation? operation) + { + return operation?.ToString() ?? "rejected"; + } + + private static int? CommandCode(string name) + { + var field = typeof(CommandCodes).GetField(name, + BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Static); + + return field is null ? null : (int)field.GetValue(null)!; + } + + /// Extracts the body of the matches! invocation inside the named method. + private static string MatchesArmBody(string source, string method) + { + var start = source.IndexOf($"fn {method}(", StringComparison.Ordinal); + Assert.True(start >= 0, $"Operation::{method} is gone from the Rust source."); + + var matches = source.IndexOf("matches!(", start, StringComparison.Ordinal); + Assert.True(matches >= 0, $"Operation::{method} no longer classifies with matches!."); + + var depth = 0; + for (var i = matches + "matches!".Length; i < source.Length; i++) + { + depth += source[i] switch + { + '(' => 1, + ')' => -1, + _ => 0 + }; + + if (depth == 0) + { + return source[matches..(i + 1)]; + } + } + + Assert.Fail($"Operation::{method} has an unbalanced matches! invocation."); + + return string.Empty; + } + + /// + /// Pins the eviction reason to error mapping against eviction_reason_to_error, which the Rust + /// module documents as the single grader both its callers share "so the mappings cannot drift apart". + /// The .NET decoder is a third copy outside that guarantee, and the status it produces is what a caller + /// branches on, so an arm that silently regrades here changes public behaviour in one SDK only. + /// + [Fact] + public void EvictionReasonErrors_MatchTheRustGrader() + { + IReadOnlyDictionary errorCodes = RustErrorCodes(); + var body = RustItem.Body(ReadRustSource(EvictionGraderPath), "pub fn eviction_reason_to_error("); + + // Arms list one or more reasons separated by `|`; IncompatibleProtocol is a block arm and is asserted + // separately below, on both of its branches. + var arms = Regex.Matches(body, + @"((?:\s*\|?\s*EvictionReason::\w+)+)\s*=>\s*IggyError::(\w+)"); + Assert.NotEmpty(arms); + + var expected = new Dictionary(StringComparer.Ordinal); + int? catchAll = null; + + foreach (Match arm in arms) + { + var code = errorCodes[Normalize(arm.Groups[2].Value)]; + foreach (Match reason in Regex.Matches(arm.Groups[1].Value, @"EvictionReason::(\w+)")) + { + expected[reason.Groups[1].Value] = code; + } + } + + var fallback = Regex.Match(body, @"_\s*=>\s*IggyError::(\w+)"); + Assert.True(fallback.Success, "eviction_reason_to_error no longer has a catch-all arm."); + catchAll = errorCodes[Normalize(fallback.Groups[1].Value)]; + + var mismatches = new List(); + + foreach (EvictionReason reason in Enum.GetValues()) + { + if (reason == EvictionReason.IncompatibleProtocol) + { + continue; + } + + var want = expected.TryGetValue(reason.ToString(), out var mapped) ? mapped : catchAll.Value; + var got = StatusOf(reason); + if (got != want) + { + mismatches.Add($"{reason}: rust {want}, .NET {got}"); + } + } + + // A reason this build cannot decode arrives as null and must grade like the Rust catch-all. + var unknown = StatusOf(null); + if (unknown != catchAll.Value) + { + mismatches.Add($": rust {catchAll.Value}, .NET {unknown}"); + } + + Assert.Empty(mismatches); + + // IncompatibleProtocol reports the window, except when it is degenerate - a zero minimum or an + // inverted range - which falls back to re-authentication. + Assert.Equal(errorCodes["INCOMPATIBLEPROTOCOLVERSION"], + StatusOf(EvictionReason.IncompatibleProtocol, serverVersion: 12, serverVersionMin: 8)); + Assert.Equal(errorCodes["UNAUTHENTICATED"], + StatusOf(EvictionReason.IncompatibleProtocol, serverVersion: 12, serverVersionMin: 0)); + Assert.Equal(errorCodes["UNAUTHENTICATED"], + StatusOf(EvictionReason.IncompatibleProtocol, serverVersion: 8, serverVersionMin: 12)); + } + + /// + /// Pins the result-section widths. They are hardcoded on the .NET side, and the Rust module notes that + /// the decoder and its encode mirror "share the widths below so they cannot drift". + /// + [Fact] + public void ResultSectionWidths_MatchTheRustConstants() + { + var source = ReadRustSource(ReplyResultPath); + + Assert.Equal(VsrReplyDecoder.RESULT_COUNT_LENGTH, RustConstant(source, "RESULT_COUNT_LEN")); + Assert.Equal(VsrReplyDecoder.RESULT_ENTRY_LENGTH, RustConstant(source, "RESULT_ENTRY_LEN")); + } + + /// + /// Pins the credential bounds the client rejects on before spending a consensus round trip. Bounds that + /// drift wide let the server evict the session instead; bounds that drift narrow reject logins the + /// server would accept. + /// + [Fact] + public void CredentialBounds_MatchTheRustDefaults() + { + var source = ReadRustSource(CredentialDefaultsPath); + + Assert.Equal(CredentialBounds.MIN_USERNAME_LENGTH, RustConstant(source, "MIN_USERNAME_LENGTH")); + Assert.Equal(CredentialBounds.MAX_USERNAME_LENGTH, RustConstant(source, "MAX_USERNAME_LENGTH")); + Assert.Equal(CredentialBounds.MIN_PASSWORD_LENGTH, RustConstant(source, "MIN_PASSWORD_LENGTH")); + Assert.Equal(CredentialBounds.MAX_PASSWORD_LENGTH, RustConstant(source, "MAX_PASSWORD_LENGTH")); + } + + /// + /// Pins the register reply body layout by reading the field offsets out of the Rust decoder and feeding + /// .NET a buffer laid out to them. The header offsets are pinned structurally above, but the bodies were + /// only ever checked against hand-written .NET expectations, which move together with the code. + /// + [Fact] + public void LoginRegisterResponseLayout_MatchesTheRustDecoder() + { + var body = RustItem.Body(ReadRustSource(LoginRegisterResponsePath), "fn decode(buf: &[u8])"); + + Assert.Equal(0, RustReadOffset(body, "read_u32_le", "user_id")); + Assert.Equal(4, RustReadOffset(body, "read_u64_le", "session")); + Assert.Equal(12, RustReadOffset(body, "read_u32_le", "server_protocol_version")); + + var versionOffset = Regex.Match(body, @"WireName::decode\(&buf\[(\d+)\.\.\]\)"); + Assert.True(versionOffset.Success, "The register reply no longer decodes its server version by offset."); + Assert.Equal(16, Group(versionOffset, 1)); + + Span reply = stackalloc byte[16 + 1 + 3]; + BinaryPrimitives.WriteUInt32LittleEndian(reply, 7); + BinaryPrimitives.WriteUInt64LittleEndian(reply[4..], 42); + BinaryPrimitives.WriteUInt32LittleEndian(reply[12..], 99); + reply[16] = 3; + "1.2"u8.CopyTo(reply[17..]); + + LoginRegisterResponse decoded = LoginRegister.Deserialize(reply); + + Assert.Equal(7u, decoded.UserId); + Assert.Equal(42ul, decoded.Session); + } + + /// + /// Pins the consumer-group assignment reply layout the same way. A silent offset shift here reassigns + /// partitions rather than failing, so every member of a group polls the wrong partitions. + /// + [Fact] + public void SyncConsumerGroupResponseLayout_MatchesTheRustDecoder() + { + var body = RustItem.Body(ReadRustSource(SyncConsumerGroupResponsePath), "fn decode(buf: &[u8])"); + + Assert.Equal(0, RustReadOffset(body, "read_u64_le", "generation")); + Assert.Equal(8, RustReadOffset(body, "read_u32_le", "partitions_count")); + + var payloadOffset = Regex.Match(body, @"let mut offset = (\d+);"); + Assert.True(payloadOffset.Success, "The assignment reply no longer decodes its partitions by offset."); + Assert.Equal(12, Group(payloadOffset, 1)); + + Span reply = stackalloc byte[12 + 8]; + BinaryPrimitives.WriteUInt64LittleEndian(reply, 5); + BinaryPrimitives.WriteUInt32LittleEndian(reply[8..], 2); + BinaryPrimitives.WriteUInt32LittleEndian(reply[12..], 11); + BinaryPrimitives.WriteUInt32LittleEndian(reply[16..], 13); + + SyncConsumerGroupAssignment decoded = SyncConsumerGroupAssignment.Decode(reply); + + Assert.Equal(5ul, decoded.Generation); + Assert.Equal([11u, 13u], decoded.Partitions); + } + + private static IReadOnlyDictionary RustErrorCodes() + { + return Regex.Matches(ReadRustSource(ErrorPath), @"^\s{4}(\w+)(?:\([^)]*\))?\s*=\s*(\d+),", + RegexOptions.Multiline) + .ToDictionary(match => Normalize(match.Groups[1].Value), + match => int.Parse(match.Groups[2].Value, CultureInfo.InvariantCulture)); + } + + private static int StatusOf(EvictionReason? reason, uint serverVersion = 0, uint serverVersionMin = 0) + { + var thrown = Assert.IsType( + VsrReplyDecoder.ToException(new EvictionFrame(reason, serverVersion, serverVersionMin))); + + return thrown.StatusCode; + } + + private static int RustConstant(string source, string name) + { + var declared = Regex.Match(source, $@"pub const {Regex.Escape(name)}\s*:\s*\w+\s*=\s*(\d+)\s*;"); + Assert.True(declared.Success, $"Rust no longer declares {name}."); + + return Group(declared, 1); + } + + private static int RustReadOffset(string body, string reader, string field) + { + var read = Regex.Match(body, $@"let {Regex.Escape(field)} = {Regex.Escape(reader)}\(buf,\s*(\d+)\)"); + Assert.True(read.Success, $"Rust no longer decodes {field} with {reader} at a literal offset."); + + return Group(read, 1); + } + + private static string Normalize(string name) + { + return name.Replace("_", string.Empty, StringComparison.Ordinal).ToUpperInvariant(); + } + + private static int Group(Match match, int index) + { + return int.Parse(match.Groups[index].Value, CultureInfo.InvariantCulture); + } + + /// Every .NET member matches Rust, and Rust carries no member the .NET enum is missing. + private static void AssertExactMatch(IReadOnlyDictionary rust) where TEnum : struct, Enum + { + AssertSubsetMatches(rust); + + HashSet ported = Enum.GetNames().ToHashSet(); + Assert.DoesNotContain(rust.Keys, name => !ported.Contains(name)); + } + + /// Every .NET member matches Rust; Rust members with no .NET counterpart are allowed. + private static void AssertSubsetMatches(IReadOnlyDictionary rust) where TEnum : struct, Enum + { + Assert.NotEmpty(rust); + + var mismatches = new List(); + foreach (var name in Enum.GetNames()) + { + var value = Convert.ToInt32(Enum.Parse(name), CultureInfo.InvariantCulture); + if (!rust.TryGetValue(name, out var rustValue)) + { + mismatches.Add($"{name}: missing on the Rust side"); + } + else if (rustValue != value) + { + mismatches.Add($"{name}: rust {rustValue}, .NET {value}"); + } + } + + Assert.Empty(mismatches); + } + + private static string ReadRustSource(string relativePath) + { + var root = RepositoryRoot(); + if (root is null) + { + // Skipping suits a consumer running the suite outside a checkout, but in CI it would turn every + // assertion in this file green on a Rust-side path change, which is the one drift the suite + // cannot afford to miss. + Assert.False(Environment.GetEnvironmentVariable("GITHUB_ACTIONS") == "true", + $"Rust sources are unavailable in CI: no ancestor of {AppContext.BaseDirectory} holds {HeaderPath}."); + Assert.Skip("Rust sources are not available; run the drift check from a repository checkout."); + } + + return File.ReadAllText(Path.Combine(root, relativePath)); + } + + private static string? RepositoryRoot() + { + var directory = new DirectoryInfo(AppContext.BaseDirectory); + while (directory is not null) + { + if (File.Exists(Path.Combine(directory.FullName, HeaderPath))) + { + return directory.FullName; + } + + directory = directory.Parent; + } + + return null; + } +} + +/// Discriminants of a #[repr(u8)] Rust enum, by member name. +internal static class RustEnum +{ + internal static IReadOnlyDictionary Discriminants(string source, string name) + { + var body = RustItem.Body(source, $"pub enum {name} {{"); + var members = new Dictionary(); + + foreach (Match member in Regex.Matches(body, @"^\s*(\w+) = (\d+),", RegexOptions.Multiline)) + { + members[member.Groups[1].Value] = int.Parse(member.Groups[2].Value, CultureInfo.InvariantCulture); + } + + return members; + } +} + +/// +/// Field offsets of a #[repr(C)] Rust struct, computed from the declared field order the same way +/// rustc lays them out: each field starts at the next multiple of its alignment. +/// +internal static class RustStruct +{ + private static readonly Dictionary ScalarLayouts = new() + { + ["u8"] = (1, 1), + ["u16"] = (2, 2), + ["u32"] = (4, 4), + ["u64"] = (8, 8), + ["u128"] = (16, 16), + // Every enum the headers embed is `#[repr(u8)]`. + ["Command2"] = (1, 1), + ["Operation"] = (1, 1), + ["EvictionReason"] = (1, 1) + }; + + internal static IReadOnlyDictionary Offsets(string source, string name) + { + return Layout(source, name).Offsets; + } + + internal static int Size(string source, string name) + { + return Layout(source, name).Size; + } + + private static (Dictionary Offsets, int Size) Layout(string source, string name) + { + var body = RustItem.Body(source, $"pub struct {name} {{"); + var offsets = new Dictionary(); + var offset = 0; + var structAlign = 1; + + foreach (Match field in Regex.Matches(body, @"^\s*pub (\w+): ([^,]+),", RegexOptions.Multiline)) + { + var (size, align) = FieldLayout(field.Groups[2].Value.Trim(), name); + offset = Align(offset, align); + offsets[field.Groups[1].Value] = offset; + offset += size; + structAlign = Math.Max(structAlign, align); + } + + return (offsets, Align(offset, structAlign)); + } + + private static (int Size, int Align) FieldLayout(string type, string structName) + { + if (ScalarLayouts.TryGetValue(type, out var scalar)) + { + return scalar; + } + + var array = Regex.Match(type, @"^\[u8; (\d+)\]$"); + if (array.Success) + { + return (int.Parse(array.Groups[1].Value, CultureInfo.InvariantCulture), 1); + } + + throw new InvalidOperationException($"{structName} gained a field of unmapped type '{type}'."); + } + + private static int Align(int offset, int alignment) + { + return (offset + alignment - 1) / alignment * alignment; + } +} + +internal static class RustItem +{ + /// The brace-delimited body following , comments stripped. + internal static string Body(string source, string declaration) + { + var start = source.IndexOf(declaration, StringComparison.Ordinal); + if (start < 0) + { + throw new InvalidOperationException($"'{declaration}' is no longer declared in the Rust sources."); + } + + var cursor = start + declaration.Length; + var depth = 1; + while (cursor < source.Length && depth > 0) + { + depth += source[cursor] switch + { + '{' => 1, + '}' => -1, + _ => 0 + }; + cursor++; + } + + var body = source[(start + declaration.Length)..(cursor - 1)]; + + return Regex.Replace(body, @"^\s*//.*$", string.Empty, RegexOptions.Multiline); + } +} diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrReplyDecoderTests.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrReplyDecoderTests.cs new file mode 100644 index 0000000000..a6707b8905 --- /dev/null +++ b/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrReplyDecoderTests.cs @@ -0,0 +1,293 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Buffers.Binary; +using Apache.Iggy.Exceptions; +using Apache.Iggy.Vsr; + +namespace Apache.Iggy.Tests.VsrTests; + +public sealed class VsrReplyDecoderTests +{ + private static byte[] ReplyHeader(VsrOperation operation, int bodyLength, uint status = 0) + { + var header = new byte[VsrHeader.HEADER_SIZE]; + header[VsrHeader.COMMAND_OFFSET] = (byte)Command2.Reply; + header[VsrHeader.REPLY_OPERATION_OFFSET] = (byte)operation; + BinaryPrimitives.WriteUInt32LittleEndian(header.AsSpan(VsrHeader.SIZE_OFFSET), + (uint)(VsrHeader.HEADER_SIZE + bodyLength)); + BinaryPrimitives.WriteUInt32LittleEndian(header.AsSpan(VsrHeader.REPLY_STATUS_OFFSET), status); + + return header; + } + + private static byte[] EvictionHeader(EvictionReason reason, uint version = 0, uint versionMin = 0) + { + var header = new byte[VsrHeader.HEADER_SIZE]; + header[VsrHeader.COMMAND_OFFSET] = (byte)Command2.Eviction; + header[VsrHeader.EVICTION_REASON_OFFSET] = (byte)reason; + BinaryPrimitives.WriteUInt32LittleEndian(header.AsSpan(VsrHeader.EVICTION_PROTOCOL_VERSION_OFFSET), version); + BinaryPrimitives.WriteUInt32LittleEndian(header.AsSpan(VsrHeader.EVICTION_PROTOCOL_VERSION_MIN_OFFSET), + versionMin); + + return header; + } + + private static byte[] SuccessBody(params byte[] payload) + { + return VsrTestPayloads.Concat(VsrTestPayloads.UInt32(0), payload); + } + + private static byte[] RejectionBody(uint code) + { + return VsrTestPayloads.Concat(VsrTestPayloads.UInt32(1), VsrTestPayloads.UInt32(0), + VsrTestPayloads.UInt32((int)code)); + } + + [Fact] + public void Decode_StripsTheResultSectionFromACommittedMetadataReply() + { + var body = SuccessBody(1, 2, 3); + + ReadOnlyMemory payload + = VsrReplyDecoder.Decode(ReplyHeader(VsrOperation.CreateStream, body.Length), body); + + Assert.Equal([1, 2, 3], payload.ToArray()); + } + + [Fact] + public void Decode_CommittedRejectionThrowsTheTypedError() + { + var body = RejectionBody(1009); + + var exception = Assert.Throws(() => + VsrReplyDecoder.Decode(ReplyHeader(VsrOperation.CreateStream, body.Length), body)); + + Assert.Equal(1009, exception.StatusCode); + Assert.True(exception.FromServer); + } + + /// + /// Retry and failover key off the transient status codes, and a locally raised code says nothing about + /// what the cluster did, so the two origins have to stay distinguishable. + /// + [Fact] + public void ServerVerdictsAreDistinguishableFromClientSideFailures() + { + var header = ReplyHeader(VsrOperation.CreateStream, 0, VsrError.TRANSIENT_NOT_ACCEPTED); + + var fromWire = Assert.Throws(() => + VsrReplyDecoder.Decode(header, ReadOnlyMemory.Empty)); + + Assert.True(fromWire.FromServer); + Assert.False(VsrError.Exception(VsrError.TRANSIENT_NOT_ACCEPTED, "raised locally").FromServer); + } + + [Fact] + public void Decode_TruncatedResultSectionIsInvalidCommandNeverSuccess() + { + var body = VsrTestPayloads.UInt32(1); + + var exception = Assert.Throws(() => + VsrReplyDecoder.Decode(ReplyHeader(VsrOperation.CreateStream, body.Length), body)); + + Assert.Equal(VsrError.INVALID_COMMAND, exception.StatusCode); + } + + [Fact] + public void Decode_NonZeroStatusIsReadBeforeTheBody() + { + var exception = Assert.Throws(() => + VsrReplyDecoder.Decode(ReplyHeader(VsrOperation.CreateStream, 0, 1009), ReadOnlyMemory.Empty)); + + Assert.Equal(1009, exception.StatusCode); + } + + [Fact] + public void Decode_NonResultFramedReplyPassesTheBodyThrough() + { + byte[] body = [1, 2, 3, 4, 5]; + + ReadOnlyMemory payload + = VsrReplyDecoder.Decode(ReplyHeader(VsrOperation.NonReplicated, body.Length), body); + + Assert.Equal(body, payload.ToArray()); + } + + [Fact] + public void Decode_PartitionSendReplyPassesTheBodyThrough() + { + byte[] body = [7, 7]; + + ReadOnlyMemory payload + = VsrReplyDecoder.Decode(ReplyHeader(VsrOperation.SendMessages, body.Length), body); + + Assert.Equal(body, payload.ToArray()); + } + + [Fact] + public void Decode_ConsumerOffsetReplyIsResultFramed() + { + var body = SuccessBody(); + + ReadOnlyMemory payload + = VsrReplyDecoder.Decode(ReplyHeader(VsrOperation.StoreConsumerOffset, body.Length), body); + + Assert.True(payload.IsEmpty); + } + + [Fact] + public void Decode_NonEmptyRegisterReplyIsResultFramed() + { + var body = SuccessBody(9, 9); + + ReadOnlyMemory payload = VsrReplyDecoder.Decode(ReplyHeader(VsrOperation.Register, body.Length), body); + + Assert.Equal([9, 9], payload.ToArray()); + } + + [Fact] + public void Decode_EmptyRegisterReplyPassesThroughToFailTheTypedDecode() + { + ReadOnlyMemory payload + = VsrReplyDecoder.Decode(ReplyHeader(VsrOperation.Register, 0), ReadOnlyMemory.Empty); + + Assert.True(payload.IsEmpty); + } + + [Fact] + public void Decode_ShortBodyIsInvalidCommand() + { + var exception = Assert.Throws(() => + VsrReplyDecoder.Decode(ReplyHeader(VsrOperation.CreateStream, 8), new byte[4])); + + Assert.Equal(VsrError.INVALID_COMMAND, exception.StatusCode); + } + + [Fact] + public void Decode_FrameSmallerThanTheHeaderIsInvalidCommand() + { + var header = ReplyHeader(VsrOperation.CreateStream, 0); + BinaryPrimitives.WriteUInt32LittleEndian(header.AsSpan(VsrHeader.SIZE_OFFSET), 128); + + var exception = Assert.Throws(() => + VsrReplyDecoder.Decode(header, ReadOnlyMemory.Empty)); + + Assert.Equal(VsrError.INVALID_COMMAND, exception.StatusCode); + } + + [Fact] + public void Decode_UnexpectedFrameIsInvalidCommand() + { + var header = new byte[VsrHeader.HEADER_SIZE]; + header[VsrHeader.COMMAND_OFFSET] = 6; + + var exception = Assert.Throws(() => + VsrReplyDecoder.Decode(header, ReadOnlyMemory.Empty)); + + Assert.Equal(VsrError.INVALID_COMMAND, exception.StatusCode); + } + + [Fact] + public void Decode_ShortHeaderIsEmptyResponse() + { + var exception = Assert.Throws(() => + VsrReplyDecoder.Decode(new byte[8], ReadOnlyMemory.Empty)); + + Assert.Equal(VsrError.EMPTY_RESPONSE, exception.StatusCode); + } + + [Fact] + public void Decode_ReadsAnEvictionFromAMisalignedBuffer() + { + var frame = VsrTestPayloads.Concat([0, 0, 0], EvictionHeader(EvictionReason.InvalidCredentials)); + + var exception = Assert.Throws(() => + VsrReplyDecoder.Decode(frame.AsSpan(3), ReadOnlyMemory.Empty)); + + Assert.Equal(VsrError.INVALID_CREDENTIALS, exception.StatusCode); + } + + [Theory] + [InlineData((byte)EvictionReason.InvalidCredentials, VsrError.INVALID_CREDENTIALS)] + [InlineData((byte)EvictionReason.InvalidToken, VsrError.INVALID_PERSONAL_ACCESS_TOKEN)] + [InlineData((byte)EvictionReason.UserInactive, VsrError.UNAUTHENTICATED)] + [InlineData((byte)EvictionReason.SessionError, VsrError.UNAUTHENTICATED)] + [InlineData((byte)EvictionReason.NoSession, VsrError.UNAUTHENTICATED)] + [InlineData((byte)EvictionReason.SessionTooLow, VsrError.UNAUTHENTICATED)] + [InlineData((byte)EvictionReason.SessionReleaseMismatch, VsrError.UNAUTHENTICATED)] + [InlineData((byte)EvictionReason.StaleClient, VsrError.STALE_CLIENT)] + [InlineData((byte)EvictionReason.MalformedLogin, VsrError.INVALID_FORMAT)] + [InlineData((byte)EvictionReason.ClientReleaseTooLow, VsrError.INVALID_COMMAND)] + [InlineData((byte)EvictionReason.ClientReleaseTooHigh, VsrError.INVALID_COMMAND)] + [InlineData((byte)EvictionReason.InvalidRequestOperation, VsrError.INVALID_COMMAND)] + [InlineData((byte)EvictionReason.InvalidRequestBody, VsrError.INVALID_COMMAND)] + [InlineData((byte)EvictionReason.InvalidRequestBodySize, VsrError.INVALID_COMMAND)] + + // Reserved and every reason this build cannot decode land in the shared grader's catch-all. + [InlineData((byte)EvictionReason.Reserved, VsrError.INVALID_COMMAND)] + public void Decode_MapsEachEvictionReasonToItsError(byte reason, int expected) + { + var exception = Assert.Throws(() => + VsrReplyDecoder.Decode(EvictionHeader((EvictionReason)reason), ReadOnlyMemory.Empty)); + + Assert.Equal(expected, exception.StatusCode); + } + + [Fact] + public void Decode_IncompatibleProtocolReportsTheAcceptedWindow() + { + var exception = Assert.Throws(() => + VsrReplyDecoder.Decode(EvictionHeader(EvictionReason.IncompatibleProtocol, 10243, 10240), + ReadOnlyMemory.Empty)); + + Assert.Equal(VsrError.INCOMPATIBLE_PROTOCOL_VERSION, exception.StatusCode); + Assert.Contains("10240..10243", exception.Message); + } + + [Theory] + [InlineData(10243u, 0u)] + [InlineData(10240u, 10243u)] + public void Decode_IncompatibleProtocolWithAnUnusableWindowDegradesToUnauthenticated(uint version, uint min) + { + var exception = Assert.Throws(() => + VsrReplyDecoder.Decode(EvictionHeader(EvictionReason.IncompatibleProtocol, version, min), + ReadOnlyMemory.Empty)); + + Assert.Equal(VsrError.UNAUTHENTICATED, exception.StatusCode); + } + + [Fact] + public void ReadResultCode_MalformedSectionIsNullNeverZero() + { + Assert.Null(VsrReplyDecoder.ReadResultCode([])); + Assert.Null(VsrReplyDecoder.ReadResultCode(VsrTestPayloads.UInt32(1))); + Assert.Null(VsrReplyDecoder.ReadResultCode([1, 0, 0, 0, 0, 0, 0, 0])); + Assert.Equal(0u, VsrReplyDecoder.ReadResultCode(SuccessBody())); + } + + [Fact] + public void ReadResultSectionLength_CoversTheCountAndItsEntries() + { + Assert.Equal(4, VsrReplyDecoder.ReadResultSectionLength(SuccessBody(1, 2))); + Assert.Equal(12, VsrReplyDecoder.ReadResultSectionLength(RejectionBody(1009))); + Assert.Null(VsrReplyDecoder.ReadResultSectionLength(VsrTestPayloads.UInt32(1))); + + // Shorter than the count itself, so there is no section to measure. + Assert.Null(VsrReplyDecoder.ReadResultSectionLength([1, 2, 3])); + } +} diff --git a/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrTestPayloads.cs b/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrTestPayloads.cs new file mode 100644 index 0000000000..1e4914766c --- /dev/null +++ b/foreign/csharp/Iggy_SDK_Tests/VsrTests/VsrTestPayloads.cs @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +using System.Buffers.Binary; +using System.Text; + +namespace Apache.Iggy.Tests.VsrTests; + +/// +/// Builders for the classic request bodies the VSR encoder peeks into, matching what +/// TcpContracts writes. +/// +internal static class VsrTestPayloads +{ + internal static byte[] NumericIdentifier(uint value) + { + var bytes = new byte[6]; + bytes[0] = 1; + bytes[1] = 4; + BinaryPrimitives.WriteUInt32LittleEndian(bytes.AsSpan(2), value); + + return bytes; + } + + internal static byte[] NamedIdentifier(string value) + { + var name = Encoding.UTF8.GetBytes(value); + var bytes = new byte[2 + name.Length]; + bytes[0] = 2; + bytes[1] = (byte)name.Length; + name.CopyTo(bytes, 2); + + return bytes; + } + + internal static byte[] SendMessages(byte[] streamId, byte[] topicId, byte partitioningKind, + byte[] partitioningValue, int messagesCount = 1) + { + var metadata = Concat(streamId, topicId, + Concat([partitioningKind, (byte)partitioningValue.Length], partitioningValue), + UInt32(messagesCount)); + + return Concat(UInt32(metadata.Length), metadata); + } + + internal static byte[] SendMessagesToPartition(uint streamId, uint topicId, uint partitionId) + { + return SendMessages(NumericIdentifier(streamId), NumericIdentifier(topicId), 2, UInt32((int)partitionId)); + } + + internal static byte[] ConsumerOffset(byte[] streamId, byte[] topicId, uint? partitionId) + { + var partition = new byte[5]; + if (partitionId.HasValue) + { + partition[0] = 1; + BinaryPrimitives.WriteUInt32LittleEndian(partition.AsSpan(1), partitionId.Value); + } + + return Concat([1], NumericIdentifier(1), streamId, topicId, partition); + } + + internal static byte[] DeleteSegments(byte[] streamId, byte[] topicId, uint partitionId, uint segmentsCount = 1) + { + return Concat(streamId, topicId, UInt32((int)partitionId), UInt32((int)segmentsCount)); + } + + internal static byte[] UInt32(int value) + { + var bytes = new byte[4]; + BinaryPrimitives.WriteUInt32LittleEndian(bytes, (uint)value); + + return bytes; + } + + internal static byte[] Concat(params byte[][] parts) + { + var result = new byte[parts.Sum(part => part.Length)]; + var position = 0; + foreach (var part in parts) + { + part.CopyTo(result, position); + position += part.Length; + } + + return result; + } +} diff --git a/foreign/csharp/README.md b/foreign/csharp/README.md index 633d4145bf..508990e391 100644 --- a/foreign/csharp/README.md +++ b/foreign/csharp/README.md @@ -37,6 +37,10 @@ The SDK supports two transport protocols: - **TCP** - Binary protocol for optimal performance and lower latency (recommended) - **HTTP** - RESTful JSON API for stateless operations +Over TCP the SDK speaks the VSR consensus framing, which is the only wire protocol the server accepts. + +See [Viewstamped Replication (VSR)](#viewstamped-replication-vsr) for what that means for the client API. + ### Creating a Client The SDK is built around the `IIggyClient` interface. To create a client instance: @@ -120,6 +124,89 @@ var client = IggyClientFactory.CreateClient(new IggyClientConfigurator await client.ConnectAsync(); ``` +## Viewstamped Replication (VSR) + +Over TCP every request is wrapped in a 256-byte consensus header, the client registers a consensus session at +login, and writes are replicated before they are acknowledged. The `IIggyClient` surface is unchanged, with the +few exceptions listed under [Limitations](#limitations). + +```c# +var client = IggyClientFactory.CreateClient(new IggyClientConfigurator +{ + BaseAddress = "127.0.0.1:8090", + Protocol = Protocol.Tcp, + + // Upper bound on a reply frame the server announces, 64 MiB by default. + MaxResponseFrameSize = 64 * 1024 * 1024, + + AutoLoginSettings = new AutoLoginSettings + { + Enabled = true, + Username = "iggy", + Password = "iggy" + } +}); + +await client.ConnectAsync(); +``` + +### What changes under VSR + +- **Login binds a session.** `LoginUserAsync` / `LoginWithPersonalAccessTokenAsync` run the register handshake + at connect time, and the session lives for as long as the connection. Logging out, being evicted + or losing the connection ends it, and the next login registers a fresh one. +- **Leader redirection is automatic.** The client reads the cluster roster, follows the current leader and + re-checks it when a request is refused because the node stopped being primary. +- **The client picks partitions.** The broker never routes: balanced and message-key partitioning are resolved + client-side (the message-key hash matches the Rust SDK byte for byte), and consumer-group polls round-robin + over the partitions the coordinator assigned to this client. +- **Consumer groups are assignment-based.** `JoinConsumerGroupAsync` makes this client a member; the assignment + is synced on demand and refreshed on every `PingAsync`. Partition counts are cached for 30 seconds, so a topic + another client widens is picked up without waiting for a ping. +- **Credentials are bounds-checked locally.** A username outside 3-50 bytes, a password outside 3-100 bytes or a + personal access token outside 1-255 bytes is rejected before the register body is framed. +- **`PingAsync` costs more than a ping.** Besides the ping it re-syncs the assignment of every consumer group + this client has joined, so it makes one extra round trip per joined group. The SDK runs no background + heartbeat: an application that wants assignments refreshed calls `PingAsync` on its own cadence. + +### Retries and failed requests + +The SDK replays a request whenever the server says it never admitted it. Two cases surface to the caller: + +- `IggyInvalidStatusCodeException` carries the server status code, with `FromServer` telling apart a verdict the + cluster reported from a failure the client raised itself. +- `VsrRequestOutcomeUnknownException` means no server verdict arrived after the request was written - the + connection was lost, the call was cancelled, or the server evicted the session while the request was in + flight - so the cluster may or may not have committed it. The SDK will not replay it on a new session, + because that would bypass server-side deduplication - re-issuing it is the caller's decision. + `IggyPublisher` will not retry it either: it reports the batch through the message-batch-failed event, and + `IggyConsumer` rethrows it rather than swallowing it, because an auto-committing poll may have advanced the + offset already. Rethrowing ends the consumer's polling loop: catch it around the enumeration, decide whether + the operation is safe to re-issue, and start consuming again. + +### Limitations + +- VSR requires `Protocol.Tcp`; configuring it with `Protocol.Http` throws at client creation. +- `StoreOffsetAsync` / `DeleteOffsetAsync` need an explicit partition id under VSR: the broker does not + resolve a `null` partition for a consumer-offset request, so passing one throws client-side. +- `FlushUnsavedBufferAsync` is not available under VSR; the server refuses it. +- Polling a topic that does not exist returns an empty poll rather than throwing. The server + answers an unresolved topic with the empty-poll reply shape, so the client cannot tell it apart from a topic + with no messages. Check the topic exists first if the distinction matters. + +### Behaviour changes for existing clients + +- `MaxResponseFrameSize` bounds the reply frames the **VSR** reader accepts. A reply larger than the 64 MiB + default is refused and the connection is dropped, so raise it if a single response legitimately exceeds that + - a large `GetSnapshotAsync` is the usual case. +- Clients built with `IggyConsumerBuilder` / `IggyPublisherBuilder` now auto-login with the credentials passed + to `WithConnection`. Before, a builder-created client came back from a + reconnect unauthenticated; now the credentials are held for the lifetime of the connection and replayed. +- The SDK now ships a dependency on `System.IO.Hashing`, used for the client-side message-key partitioner. +- TCP sockets are opened with `NoDelay`. The protocol is request/reply, so a write is + always the last one before the client waits for the answer and Nagle has nothing to coalesce it with - it + only held back the trailing segment of a large request until the previous one was acked. + ## Authentication ### User Login @@ -694,10 +781,14 @@ Integration tests are located in `Iggy_SDK.Tests.Integration/`. Tests can run ag #### 1. Dockerization +The suite runs against `iggy-server`. TCP only: the SDK frames TCP with the +VSR wire protocol, the cluster serves reads from the primary, and the HTTP surface has no equivalent path to +route them through. + ```bash -cargo build +cargo build --bin iggy-server --bin iggy -docker build --no-cache -f core/server/Dockerfile --platform linux/amd64 --target runtime-prebuilt --build-arg PREBUILT_IGGY_SERVER=target/debug/iggy-server --build-arg PREBUILT_IGGY_CLI=target/debug/iggy -t local-iggy-server . +docker build --no-cache -f core/server/Dockerfile --platform linux/amd64 --target runtime-prebuilt --build-arg PREBUILT_IGGY_SERVER=target/debug/iggy-server --build-arg PREBUILT_IGGY_CLI=target/debug/iggy -t iggy-server:test . ``` #### 2. Build the Test Project @@ -710,10 +801,13 @@ dotnet build foreign/csharp/Iggy_SDK.Tests.Integration ```bash cd foreign/csharp -export IGGY_SERVER_DOCKER_IMAGE=local-iggy-server +export IGGY_SERVER_DOCKER_IMAGE=iggy-server:test dotnet test -f net10.0 --project Iggy_SDK.Tests.Integration --no-build --verbosity diagnostic ``` +`IGGY_SERVER_DOCKER_IMAGE` defaults to `iggy-server:test`, so the export above is only needed to point +at a different image. Rider and Visual Studio need nothing configured. + ## Useful Resources - [Iggy Documentation](https://iggy.apache.org/docs/) diff --git a/foreign/csharp/scripts/pack.sh b/foreign/csharp/scripts/pack.sh index 09b23e34e1..cb66da0b4f 100755 --- a/foreign/csharp/scripts/pack.sh +++ b/foreign/csharp/scripts/pack.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/foreign/go/README.md b/foreign/go/README.md index b9fe45e97e..dc41052d70 100644 --- a/foreign/go/README.md +++ b/foreign/go/README.md @@ -11,9 +11,7 @@ Official Go client SDK for [Apache Iggy](https://iggy.apache.org) message streaming. The client speaks the VSR wire protocol over TCP, with or without TLS, in a -blocking implementation. VSR is the only protocol it supports: there is no -option to fall back to the classic framing, so the SDK requires a server that -speaks VSR and no longer works with the legacy `iggy-server`. +blocking implementation. VSR is the only protocol it supports. > Apache Iggy (Incubating) is an effort undergoing incubation at the Apache Software Foundation (ASF), sponsored by the Apache Incubator PMC. > @@ -31,16 +29,14 @@ go get github.com/apache/iggy/foreign/go Build and start a VSR server from a checkout of this repository: - - ```bash -cargo build --bin iggy-server-ng --features vsr +cargo build --bin iggy-server IGGY_SYSTEM_PATH=/tmp/iggy-go \ IGGY_TCP_ADDRESS=127.0.0.1:8090 \ IGGY_HTTP_ENABLED=false IGGY_QUIC_ENABLED=false IGGY_WEBSOCKET_ENABLED=false \ IGGY_ROOT_USERNAME=iggy IGGY_ROOT_PASSWORD=iggy \ -target/debug/iggy-server-ng +target/debug/iggy-server ``` QUIC, WebSocket and HTTP are enabled by default on ports 8080, 8092 and 3000. diff --git a/foreign/go/client/tcp/tcp_connect_test.go b/foreign/go/client/tcp/tcp_connect_test.go index 5a812b7257..9e7cf56132 100644 --- a/foreign/go/client/tcp/tcp_connect_test.go +++ b/foreign/go/client/tcp/tcp_connect_test.go @@ -198,7 +198,6 @@ func TestConnect_DiscoversTheLeaderAndSignsIn(t *testing.T) { assert.Zero(t, recorded[1].code(), "only a non-replicated frame carries the code") assert.Zero(t, recorded[1].requestID(), "a register is always request zero") assert.Zero(t, recorded[1].sessionID()) - assert.Equal(t, uint64(1)<<63, recorded[1].namespace()) assert.True(t, client.session.Bound()) assert.Equal(t, uint64(128), client.session.SessionID()) diff --git a/foreign/go/client/tcp/tcp_core_test.go b/foreign/go/client/tcp/tcp_core_test.go index f4cc00e150..1ae9a438fa 100644 --- a/foreign/go/client/tcp/tcp_core_test.go +++ b/foreign/go/client/tcp/tcp_core_test.go @@ -389,7 +389,7 @@ func TestSendMessages_DecodesTheConfirmations(t *testing.T) { recorded := server.recorded() require.Len(t, recorded, 1) assert.Equal(t, vsr.OperationSendMessages, recorded[0].operation()) - assert.Equal(t, uint64(3)<<32|uint64(2)<<20|1, recorded[0].namespace()) + assert.Equal(t, uint32(1), recorded[0].partitionID(t)) assert.Equal(t, uint64(1), recorded[0].requestID(), "a partition request reads the watermark without consuming it") } @@ -471,7 +471,7 @@ func TestSendMessages_ResolvesKeyPartitioningToAnExplicitPartition(t *testing.T) require.Len(t, recorded, 2) assert.Equal(t, uint32(command.GetTopicCode), recorded[0].code()) // 0x0D3FE0E1 modulo four partitions. - assert.Equal(t, uint64(1)<<32|uint64(1)<<20|1, recorded[1].namespace()) + assert.Equal(t, uint32(1), recorded[1].partitionID(t)) } func TestSendMessages_RoundRobinsBalancedPartitioning(t *testing.T) { @@ -492,13 +492,13 @@ func TestSendMessages_RoundRobinsBalancedPartitioning(t *testing.T) { require.NoError(t, err) } - var partitions []uint64 + var partitions []uint32 for _, recorded := range server.recorded() { if recorded.operation() == vsr.OperationSendMessages { - partitions = append(partitions, recorded.namespace()&0xFFFFF) + partitions = append(partitions, recorded.partitionID(t)) } } - assert.Equal(t, []uint64{0, 1, 2, 0}, partitions) + assert.Equal(t, []uint32{0, 1, 2, 0}, partitions) metadataRequests := 0 for _, recorded := range server.recorded() { diff --git a/foreign/go/client/tcp/tcp_testing_test.go b/foreign/go/client/tcp/tcp_testing_test.go index b63671b0e1..f6f5db98b8 100644 --- a/foreign/go/client/tcp/tcp_testing_test.go +++ b/foreign/go/client/tcp/tcp_testing_test.go @@ -38,13 +38,12 @@ const ( frameOffsetClient = 128 frameOffsetRequest = 168 frameOffsetOperation = 176 - frameOffsetNamespace = 184 - frameOffsetSession = 192 - frameOffsetReserved = 204 + frameOffsetSession = 184 + frameOffsetReserved = 196 replyFrameOffsetRequest = 200 replyFrameOffsetOperation = 208 - replyFrameOffsetStatus = 224 + replyFrameOffsetStatus = 216 evictionFrameOffsetVersion = 144 evictionFrameOffsetVersionMin = 148 @@ -67,9 +66,6 @@ func (r request) requestID() uint64 { func (r request) sessionID() uint64 { return binary.LittleEndian.Uint64(r.header[frameOffsetSession:]) } -func (r request) namespace() uint64 { - return binary.LittleEndian.Uint64(r.header[frameOffsetNamespace:]) -} func (r request) clientID() vsr.ClientID { return vsr.ClientID{ Lo: binary.LittleEndian.Uint64(r.header[frameOffsetClient:]), @@ -77,6 +73,27 @@ func (r request) clientID() vsr.ClientID { } } +// partitionID reads the resolved partition out of a recorded SendMessages +// payload: [metadata length u32][stream id][topic id][partitioning], where the +// identifiers and the partitioning are each [kind u8][length u8][value]. The +// frame carries no routing namespace, so the payload is the only place the +// client's partitioning decision is observable. +func (r request) partitionID(t *testing.T) uint32 { + t.Helper() + + cursor := 4 + for range 2 { + require.Greater(t, len(r.payload), cursor+1) + cursor += 2 + int(r.payload[cursor+1]) + } + require.Greater(t, len(r.payload), cursor+1) + require.Equal(t, byte(iggcon.PartitionIdKind), r.payload[cursor], + "the client resolves every strategy to an explicit partition") + require.Equal(t, byte(4), r.payload[cursor+1]) + require.GreaterOrEqual(t, len(r.payload), cursor+6) + return binary.LittleEndian.Uint32(r.payload[cursor+2:]) +} + // replyFrame builds a committed reply carrying body. func replyFrame(operation vsr.Operation, body []byte) []byte { return statusReplyFrame(operation, 0, body) diff --git a/foreign/go/internal/vsr/envelope.go b/foreign/go/internal/vsr/envelope.go index 690b5d9102..912881a2c1 100644 --- a/foreign/go/internal/vsr/envelope.go +++ b/foreign/go/internal/vsr/envelope.go @@ -31,7 +31,7 @@ func EncodeRequest(session *Session, code uint32, payload []byte) ([]byte, error } // StampRequestHeader writes the request header into the first HeaderSize bytes -// of frame, deriving routing and sequencing from the payload that follows it. +// of frame, deriving sequencing from the command code and the session state. // Callers that encode a payload into a pooled buffer reserve the prologue up // front and stamp it here, which keeps the frame a single allocation. // @@ -40,25 +40,17 @@ func StampRequestHeader(session *Session, code uint32, frame []byte) error { if len(frame) > MaxFrameSize { return ierror.ErrInvalidConfiguration } - payload := frame[HeaderSize:] operation := OperationForCode(code) replicated := operation != OperationRegister && operation != OperationNonReplicated - // A replicated operation needs a bound session. Checking that ahead of the - // payload means an unauthenticated caller hears about the missing session - // rather than about the request it could not have sent anyway. + // A replicated operation needs a bound session; refusing it here leaves + // the request-id counter untouched. if replicated && !session.Bound() { return ierror.ErrUnauthenticated } - // Namespace derivation can fail on a malformed payload. Run it before - // taking a request id so a local failure leaves the counter untouched. - namespace, err := NamespaceForRequest(code, payload, operation) - if err != nil { - return err - } - var request, sessionID uint64 + var err error switch operation { case OperationRegister: request = session.BeginRegister() @@ -85,7 +77,6 @@ func StampRequestHeader(session *Session, code uint32, frame []byte) error { Client: session.ClientID(), Request: request, Operation: operation, - Namespace: namespace, Session: sessionID, } if operation == OperationNonReplicated { diff --git a/foreign/go/internal/vsr/envelope_test.go b/foreign/go/internal/vsr/envelope_test.go index f3c4f9ac40..c497b07fc5 100644 --- a/foreign/go/internal/vsr/envelope_test.go +++ b/foreign/go/internal/vsr/envelope_test.go @@ -55,8 +55,6 @@ func TestEncodeRequest_EncodesARegisterFrame(t *testing.T) { assert.Equal(t, byte(OperationRegister), header[requestOffsetOperation]) assert.Zero(t, binary.LittleEndian.Uint64(header[requestOffsetRequest:])) assert.Zero(t, binary.LittleEndian.Uint64(header[requestOffsetSession:])) - assert.Equal(t, MetadataConsensusNamespace, - binary.LittleEndian.Uint64(header[requestOffsetNamespace:])) assert.Zero(t, binary.LittleEndian.Uint32(header[requestOffsetReserved:])) } @@ -77,8 +75,6 @@ func TestEncodeRequest_EncodesALogoutFrame(t *testing.T) { header := frameHeader(t, frame) assert.Equal(t, byte(OperationLogout), header[requestOffsetOperation]) - assert.Equal(t, MetadataConsensusNamespace, - binary.LittleEndian.Uint64(header[requestOffsetNamespace:])) assert.Equal(t, uint64(1), binary.LittleEndian.Uint64(header[requestOffsetRequest:]), "logout advances the metadata watermark") assert.Equal(t, uint64(2), session.CurrentRequestID()) @@ -128,10 +124,8 @@ func TestEncodeRequest_DoesNotAdvanceTheWatermarkOffTheMetadataPlane(t *testing. } assert.Equal(t, uint64(1), session.CurrentRequestID()) - payload := sendMessagesPayload( - numericIdentifier(1), numericIdentifier(1), partitionIDPartitioning(0)) for range 3 { - _, err := EncodeRequest(session, uint32(command.SendMessagesCode), payload) + _, err := EncodeRequest(session, uint32(command.SendMessagesCode), []byte{1}) require.NoError(t, err) } assert.Equal(t, uint64(1), session.CurrentRequestID()) @@ -146,7 +140,6 @@ func TestEncodeRequest_AdvancesTheWatermarkPerMetadataCommand(t *testing.T) { header := frameHeader(t, frame) assert.Equal(t, expected, binary.LittleEndian.Uint64(header[requestOffsetRequest:])) assert.Equal(t, byte(OperationCreateStream), header[requestOffsetOperation]) - assert.Zero(t, binary.LittleEndian.Uint64(header[requestOffsetNamespace:])) } assert.Equal(t, uint64(4), session.CurrentRequestID()) } @@ -161,51 +154,19 @@ func TestEncodeRequest_RejectsAReplicatedCommandOnAnUnboundSession(t *testing.T) func TestEncodeRequest_RejectsAPartitionCommandOnAnUnboundSession(t *testing.T) { session := NewSessionWithClientID(ClientID{Lo: 1}) - payload := sendMessagesPayload( - numericIdentifier(1), numericIdentifier(1), partitionIDPartitioning(0)) - _, err := EncodeRequest(session, uint32(command.SendMessagesCode), payload) + _, err := EncodeRequest(session, uint32(command.SendMessagesCode), []byte{1}) assert.ErrorIs(t, err, ierror.ErrUnauthenticated) } -func TestEncodeRequest_ReportsTheMissingSessionAheadOfAnUnroutablePayload(t *testing.T) { - session := NewSessionWithClientID(ClientID{Lo: 1}) - unroutable := sendMessagesPayload( - numericIdentifier(MaxStreams), numericIdentifier(1), partitionIDPartitioning(0)) - - _, err := EncodeRequest(session, uint32(command.SendMessagesCode), unroutable) - assert.ErrorIs(t, err, ierror.ErrUnauthenticated, - "an unauthenticated caller could not have sent the request either way") -} - -func TestEncodeRequest_BurnsNoIDWhenTheNamespaceFails(t *testing.T) { - session := boundSession(t) - malformed := sendMessagesPayload( - numericIdentifier(1), numericIdentifier(1), []byte{1, 4, 0, 0, 0, 0}) - - _, err := EncodeRequest(session, uint32(command.SendMessagesCode), malformed) - assert.ErrorIs(t, err, ierror.ErrFeatureUnavailable) - assert.Equal(t, uint64(1), session.CurrentRequestID()) - - // The next metadata command still takes id 1, so the sequence has no gap. - frame, err := EncodeRequest(session, uint32(command.CreateStreamCode), []byte{1}) - require.NoError(t, err) - assert.Equal(t, uint64(1), - binary.LittleEndian.Uint64(frameHeader(t, frame)[requestOffsetRequest:])) -} - -func TestEncodeRequest_RoutesSendMessagesToItsPartition(t *testing.T) { +func TestEncodeRequest_EncodesASendMessagesFrame(t *testing.T) { session := boundSession(t) - payload := sendMessagesPayload( - numericIdentifier(3), numericIdentifier(2), partitionIDPartitioning(7)) - frame, err := EncodeRequest(session, uint32(command.SendMessagesCode), payload) + frame, err := EncodeRequest(session, uint32(command.SendMessagesCode), []byte{1}) require.NoError(t, err) header := frameHeader(t, frame) assert.Equal(t, byte(OperationSendMessages), header[requestOffsetOperation]) - assert.Equal(t, uint64(3<<32|2<<20|7), - binary.LittleEndian.Uint64(header[requestOffsetNamespace:])) assert.Zero(t, binary.LittleEndian.Uint32(header[requestOffsetReserved:]), "only non-replicated frames carry the code") } diff --git a/foreign/go/internal/vsr/header.go b/foreign/go/internal/vsr/header.go index 99cc742f9c..789237efd1 100644 --- a/foreign/go/internal/vsr/header.go +++ b/foreign/go/internal/vsr/header.go @@ -33,6 +33,12 @@ const ( // Request header field offsets the client writes. Fields are addressed by // byte offset rather than a struct cast because the header leads with // unaligned u128 values. +// +// The client wire carries no routing namespace: the server derives the +// consensus group (plane from the operation, partition target from the +// payload) and stamps it into its own internal header. Everything that +// followed the removed field therefore sits eight bytes earlier than in the +// pre-derivation layout. const ( requestOffsetSize = 48 requestOffsetCommand = 60 @@ -40,9 +46,8 @@ const ( requestOffsetTimestamp = 160 requestOffsetRequest = 168 requestOffsetOperation = 176 - requestOffsetNamespace = 184 - requestOffsetSession = 192 - requestOffsetReserved = 204 + requestOffsetSession = 184 + requestOffsetReserved = 196 ) // Reply header field offsets the client reads. @@ -51,8 +56,7 @@ const ( replyOffsetCommand = 60 replyOffsetRequest = 200 replyOffsetOperation = 208 - replyOffsetNamespace = 216 - replyOffsetStatus = 224 + replyOffsetStatus = 216 ) // Eviction header field offsets the client reads. @@ -122,8 +126,6 @@ type RequestFields struct { Request uint64 // Operation is the consensus operation discriminant. Operation Operation - // Namespace routes the request to the owning shard. - Namespace uint64 // Session is the bound session fence, or 0 while unbound. Session uint64 // NonReplicatedCode is the raw command code the server reads out of @@ -142,7 +144,6 @@ func EncodeRequestHeader(dst *[HeaderSize]byte, fields RequestFields) { binary.LittleEndian.PutUint64(dst[requestOffsetClient+8:], fields.Client.Hi) binary.LittleEndian.PutUint64(dst[requestOffsetRequest:], fields.Request) dst[requestOffsetOperation] = byte(fields.Operation) - binary.LittleEndian.PutUint64(dst[requestOffsetNamespace:], fields.Namespace) binary.LittleEndian.PutUint64(dst[requestOffsetSession:], fields.Session) if fields.NonReplicatedCode != 0 { binary.LittleEndian.PutUint32(dst[requestOffsetReserved:], fields.NonReplicatedCode) diff --git a/foreign/go/internal/vsr/header_test.go b/foreign/go/internal/vsr/header_test.go index 6db3b7b271..87cdadc7cb 100644 --- a/foreign/go/internal/vsr/header_test.go +++ b/foreign/go/internal/vsr/header_test.go @@ -33,7 +33,6 @@ func TestEncodeRequestHeader_WritesEveryFieldAtItsCanonicalOffset(t *testing.T) Client: ClientID{Lo: 0x99AABBCCDDEEFF00, Hi: 0x1122334455667788}, Request: 0x0102030405060708, Operation: OperationNonReplicated, - Namespace: 0x8877665544332211, Session: 0x1020304050607080, NonReplicatedCode: 60001, }) @@ -44,7 +43,6 @@ func TestEncodeRequestHeader_WritesEveryFieldAtItsCanonicalOffset(t *testing.T) assert.Equal(t, uint64(0x1122334455667788), binary.LittleEndian.Uint64(header[requestOffsetClient+8:])) assert.Equal(t, uint64(0x0102030405060708), binary.LittleEndian.Uint64(header[requestOffsetRequest:])) assert.Equal(t, byte(OperationNonReplicated), header[requestOffsetOperation]) - assert.Equal(t, uint64(0x8877665544332211), binary.LittleEndian.Uint64(header[requestOffsetNamespace:])) assert.Equal(t, uint64(0x1020304050607080), binary.LittleEndian.Uint64(header[requestOffsetSession:])) assert.Equal(t, uint32(60001), binary.LittleEndian.Uint32(header[requestOffsetReserved:])) assert.Zero(t, binary.LittleEndian.Uint64(header[requestOffsetTimestamp:]), "timestamp stays zero") @@ -56,7 +54,6 @@ func TestEncodeRequestHeader_LeavesEveryUnwrittenByteZero(t *testing.T) { Size: HeaderSize, Client: ClientID{Lo: 1}, Operation: OperationRegister, - Namespace: MetadataConsensusNamespace, }) var expected [HeaderSize]byte @@ -64,7 +61,6 @@ func TestEncodeRequestHeader_LeavesEveryUnwrittenByteZero(t *testing.T) { expected[requestOffsetCommand] = byte(FrameRequest) binary.LittleEndian.PutUint64(expected[requestOffsetClient:], 1) expected[requestOffsetOperation] = byte(OperationRegister) - binary.LittleEndian.PutUint64(expected[requestOffsetNamespace:], MetadataConsensusNamespace) assert.Equal(t, expected, header) } @@ -92,14 +88,12 @@ func TestEncodeRequestHeader_SupportsMaximumWidthFields(t *testing.T) { Client: ClientID{Lo: math.MaxUint64, Hi: math.MaxUint64}, Request: math.MaxUint64, Operation: OperationSendMessages, - Namespace: math.MaxUint64, Session: math.MaxUint64, }) assert.Equal(t, uint32(math.MaxUint32), binary.LittleEndian.Uint32(header[requestOffsetSize:])) assert.Equal(t, uint64(math.MaxUint64), binary.LittleEndian.Uint64(header[requestOffsetRequest:])) assert.Equal(t, uint64(math.MaxUint64), binary.LittleEndian.Uint64(header[requestOffsetSession:])) - assert.Equal(t, uint64(math.MaxUint64), binary.LittleEndian.Uint64(header[requestOffsetNamespace:])) } func TestEncodeRequestHeader_OmitsTheReservedCodeForReplicatedOperations(t *testing.T) { @@ -122,7 +116,6 @@ func TestEncodeRequestHeader_ProducesTheGoldenFrame(t *testing.T) { Client: ClientID{Lo: 0x0807060504030201, Hi: 0x100F0E0D0C0B0A09}, Request: 2, Operation: OperationCreateStream, - Namespace: 0, Session: 11, }) @@ -133,7 +126,7 @@ func TestEncodeRequestHeader_ProducesTheGoldenFrame(t *testing.T) { 136: {0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10}, 168: {0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, 176: {0x80}, - 192: {0x0B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + 184: {0x0B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, } written := 0 for offset, want := range golden { @@ -209,16 +202,14 @@ func TestHeaderOffsets_MatchTheWireContract(t *testing.T) { assert.Equal(t, 160, requestOffsetTimestamp) assert.Equal(t, 168, requestOffsetRequest) assert.Equal(t, 176, requestOffsetOperation) - assert.Equal(t, 184, requestOffsetNamespace) - assert.Equal(t, 192, requestOffsetSession) - assert.Equal(t, 204, requestOffsetReserved) + assert.Equal(t, 184, requestOffsetSession) + assert.Equal(t, 196, requestOffsetReserved) assert.Equal(t, 48, replyOffsetSize) assert.Equal(t, 60, replyOffsetCommand) assert.Equal(t, 208, replyOffsetOperation) - assert.Equal(t, 216, replyOffsetNamespace) - assert.Equal(t, 224, replyOffsetStatus) - assert.Equal(t, replyOffsetNamespace+8, replyOffsetStatus, "status follows namespace") + assert.Equal(t, 216, replyOffsetStatus) + assert.Equal(t, replyOffsetOperation+8, replyOffsetStatus, "status follows the operation") assert.Equal(t, 60, evictionOffsetCommand) assert.Equal(t, 128, evictionOffsetClient) diff --git a/foreign/go/internal/vsr/namespace.go b/foreign/go/internal/vsr/namespace.go deleted file mode 100644 index 22490f8749..0000000000 --- a/foreign/go/internal/vsr/namespace.go +++ /dev/null @@ -1,246 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package vsr - -import ( - "encoding/binary" - - ierror "github.com/apache/iggy/foreign/go/errors" - "github.com/apache/iggy/foreign/go/internal/command" -) - -// Namespace packing limits, a port of -// core/binary_protocol/src/namespace.rs. -const ( - MaxStreams = 4096 - MaxTopics = 4096 - MaxPartitions = 1_000_000 - - streamBits = 12 - topicBits = 12 - partitionBits = 20 - - partitionShift = 0 - topicShift = partitionShift + partitionBits - streamShift = topicShift + topicBits -) - -// MetadataConsensusNamespace routes a control-plane request to the metadata -// replica on shard 0. Plain 0 falls into namespace hashing and can land a -// Register on a peer shard that has no consensus instance. -const MetadataConsensusNamespace uint64 = 1 << 63 - -// Identifier kinds in the wire prefix [kind u8][len u8][value]. -const ( - identifierKindNumeric = 1 - identifierKindString = 2 -) - -// partitioningPartitionID is the only Partitioning kind that carries an -// explicit partition, which is what routing needs. -const partitioningPartitionID = 2 - -// PackNamespace packs stream, topic and partition ids into a routing -// namespace. -func PackNamespace(streamID, topicID, partitionID uint32) (uint64, error) { - if err := validateNamespaceField(streamID, MaxStreams); err != nil { - return 0, err - } - if err := validateNamespaceField(topicID, MaxTopics); err != nil { - return 0, err - } - if err := validateNamespaceField(partitionID, MaxPartitions); err != nil { - return 0, err - } - return uint64(streamID)< 0: - return peekedIdentifier{named: true, length: 2 + length}, nil - default: - return peekedIdentifier{}, ierror.ErrInvalidCommand - } -} - -func validateNamespaceField(value uint32, exclusiveMax uint32) error { - if value >= exclusiveMax { - return ierror.ErrInvalidIdentifier - } - return nil -} - -func namespaceFromIdentifiers(stream, topic peekedIdentifier, partitionID uint32) (uint64, error) { - if stream.named || topic.named { - return 0, nil - } - return PackNamespace(stream.numeric, topic.numeric, partitionID) -} - -// namespaceFromSendMessages peeks -// [metadata_len u32][stream ident][topic ident][partitioning kind u8, len u8, value]. -// Only explicit PartitionId partitioning is routable: the broker never picks a -// partition under consensus. -func namespaceFromSendMessages(payload []byte) (uint64, error) { - if len(payload) < 4 { - return 0, ierror.ErrInvalidCommand - } - metadataLength := uint64(binary.LittleEndian.Uint32(payload)) - if uint64(len(payload)) < 4+metadataLength { - return 0, ierror.ErrInvalidCommand - } - // A read past the declared metadata region must fail rather than spill - // into message bytes and derive a namespace the server would not compute. - metadata := payload[4 : 4+metadataLength] - - offset := 0 - stream, err := peekIdentifier(metadata, offset) - if err != nil { - return 0, err - } - offset += stream.length - topic, err := peekIdentifier(metadata, offset) - if err != nil { - return 0, err - } - offset += topic.length - - if len(metadata) < offset+2 { - return 0, ierror.ErrInvalidCommand - } - partitioningKind := metadata[offset] - partitioningLength := int(metadata[offset+1]) - if partitioningKind != partitioningPartitionID { - return 0, ierror.ErrFeatureUnavailable - } - if partitioningLength != 4 || len(metadata) < offset+2+4 { - return 0, ierror.ErrInvalidCommand - } - partitionID := binary.LittleEndian.Uint32(metadata[offset+2:]) - - return namespaceFromIdentifiers(stream, topic, partitionID) -} - -// namespaceFromConsumerOffset peeks -// [consumer kind u8][consumer ident][stream ident][topic ident] -// [partition flag u8][partition u32]. The v1 and v2 request layouts share this -// prefix and differ only in the trailing fields, which routing ignores. -func namespaceFromConsumerOffset(payload []byte) (uint64, error) { - if len(payload) < 1 || (payload[0] != 1 && payload[0] != 2) { - return 0, ierror.ErrInvalidCommand - } - offset := 1 - consumer, err := peekIdentifier(payload, offset) - if err != nil { - return 0, err - } - offset += consumer.length - stream, err := peekIdentifier(payload, offset) - if err != nil { - return 0, err - } - offset += stream.length - topic, err := peekIdentifier(payload, offset) - if err != nil { - return 0, err - } - offset += topic.length - - if len(payload) < offset+5 { - return 0, ierror.ErrInvalidCommand - } - if payload[offset] != 1 { - return 0, ierror.ErrInvalidIdentifier - } - partitionID := binary.LittleEndian.Uint32(payload[offset+1:]) - - return namespaceFromIdentifiers(stream, topic, partitionID) -} - -// namespaceFromDeleteSegments peeks -// [stream ident][topic ident][partition u32]. -func namespaceFromDeleteSegments(payload []byte) (uint64, error) { - offset := 0 - stream, err := peekIdentifier(payload, offset) - if err != nil { - return 0, err - } - offset += stream.length - topic, err := peekIdentifier(payload, offset) - if err != nil { - return 0, err - } - offset += topic.length - - if len(payload) < offset+4 { - return 0, ierror.ErrInvalidCommand - } - partitionID := binary.LittleEndian.Uint32(payload[offset:]) - - return namespaceFromIdentifiers(stream, topic, partitionID) -} diff --git a/foreign/go/internal/vsr/namespace_test.go b/foreign/go/internal/vsr/namespace_test.go deleted file mode 100644 index 2a45b63768..0000000000 --- a/foreign/go/internal/vsr/namespace_test.go +++ /dev/null @@ -1,376 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package vsr - -import ( - "encoding/binary" - "fmt" - "testing" - - ierror "github.com/apache/iggy/foreign/go/errors" - "github.com/apache/iggy/foreign/go/internal/command" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// numericIdentifier builds the [kind u8][len u8][value u32] prefix. -func numericIdentifier(id uint32) []byte { - out := []byte{identifierKindNumeric, 4} - return binary.LittleEndian.AppendUint32(out, id) -} - -// namedIdentifier builds the [kind u8][len u8][utf-8] prefix. -func namedIdentifier(name string) []byte { - out := []byte{identifierKindString, byte(len(name))} - return append(out, name...) -} - -// sendMessagesPayload builds [metadata len u32][stream][topic][partitioning]. -func sendMessagesPayload(stream, topic, partitioning []byte) []byte { - metadata := make([]byte, 0, len(stream)+len(topic)+len(partitioning)) - metadata = append(metadata, stream...) - metadata = append(metadata, topic...) - metadata = append(metadata, partitioning...) - payload := binary.LittleEndian.AppendUint32(nil, uint32(len(metadata))) - return append(payload, metadata...) -} - -func partitionIDPartitioning(id uint32) []byte { - out := []byte{partitioningPartitionID, 4} - return binary.LittleEndian.AppendUint32(out, id) -} - -// consumerOffsetPayload builds the peeked prefix shared by codes 121 to 124. -func consumerOffsetPayload(stream, topic []byte, partitionID uint32, hasPartition bool) []byte { - payload := []byte{1} - payload = append(payload, numericIdentifier(1)...) - payload = append(payload, stream...) - payload = append(payload, topic...) - if hasPartition { - payload = append(payload, 1) - } else { - payload = append(payload, 0) - } - return binary.LittleEndian.AppendUint32(payload, partitionID) -} - -func TestPackNamespace_PacksTheTripleAtItsShifts(t *testing.T) { - tests := []struct { - name string - stream, topic, partitionKey uint32 - want uint64 - }{ - {name: "zero", want: 0}, - {name: "partition only", partitionKey: 1, want: 1}, - {name: "topic only", topic: 1, want: 1 << 20}, - {name: "stream only", stream: 1, want: 1 << 32}, - {name: "triple", stream: 3, topic: 2, partitionKey: 7, want: 3<<32 | 2<<20 | 7}, - { - name: "maximum", - stream: MaxStreams - 1, - topic: MaxTopics - 1, - partitionKey: MaxPartitions - 1, - want: 4095<<32 | 4095<<20 | 999999, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - got, err := PackNamespace(test.stream, test.topic, test.partitionKey) - require.NoError(t, err) - assert.Equal(t, test.want, got) - }) - } -} - -func TestPackNamespace_RejectsOutOfRangeFields(t *testing.T) { - tests := []struct { - name string - stream, topic, partitionKey uint32 - }{ - {name: "stream", stream: MaxStreams}, - {name: "topic", topic: MaxTopics}, - {name: "partition", partitionKey: MaxPartitions}, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - _, err := PackNamespace(test.stream, test.topic, test.partitionKey) - assert.ErrorIs(t, err, ierror.ErrInvalidIdentifier) - }) - } -} - -func TestNamespaceShifts_MatchTheWireContract(t *testing.T) { - assert.Equal(t, 12, streamBits) - assert.Equal(t, 12, topicBits) - assert.Equal(t, 20, partitionBits) - assert.Equal(t, 0, partitionShift) - assert.Equal(t, 20, topicShift) - assert.Equal(t, 32, streamShift) - assert.Equal(t, uint64(1)<<63, MetadataConsensusNamespace) -} - -func TestNamespaceForRequest_RoutesControlPlaneToTheMetadataSentinel(t *testing.T) { - for _, operation := range []Operation{OperationRegister, OperationLogout} { - got, err := NamespaceForRequest(uint32(command.LoginRegisterCode), nil, operation) - require.NoError(t, err) - assert.Equal(t, MetadataConsensusNamespace, got) - } -} - -func TestNamespaceForRequest_RoutesReadsAndMetadataToZero(t *testing.T) { - tests := []struct { - name string - code command.Code - operation Operation - }{ - {name: "ping", code: command.PingCode, operation: OperationNonReplicated}, - {name: "poll", code: command.PollMessagesCode, operation: OperationNonReplicated}, - {name: "create stream", code: command.CreateStreamCode, operation: OperationCreateStream}, - {name: "join group", code: command.JoinGroupCode, operation: OperationJoinConsumerGroup}, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - got, err := NamespaceForRequest(uint32(test.code), []byte{0xFF}, test.operation) - require.NoError(t, err) - assert.Zero(t, got) - }) - } -} - -func TestNamespaceForRequest_RejectsAnUnpeekablePartitionOperation(t *testing.T) { - _, err := NamespaceForRequest(uint32(command.PollMessagesCode), nil, OperationSendMessages+64) - assert.ErrorIs(t, err, ierror.ErrFeatureUnavailable) -} - -func TestNamespaceForRequest_SendMessages(t *testing.T) { - t.Run("packs numeric identifiers", func(t *testing.T) { - payload := sendMessagesPayload( - numericIdentifier(3), numericIdentifier(2), partitionIDPartitioning(7)) - got, err := NamespaceForRequest( - uint32(command.SendMessagesCode), payload, OperationSendMessages) - require.NoError(t, err) - assert.Equal(t, uint64(3<<32|2<<20|7), got) - }) - - t.Run("ignores message bytes past the metadata region", func(t *testing.T) { - payload := sendMessagesPayload( - numericIdentifier(1), numericIdentifier(1), partitionIDPartitioning(0)) - payload = append(payload, 0xDE, 0xAD, 0xBE, 0xEF) - got, err := NamespaceForRequest( - uint32(command.SendMessagesCode), payload, OperationSendMessages) - require.NoError(t, err) - assert.Equal(t, uint64(1<<32|1<<20), got) - }) - - t.Run("defers a named stream to the server", func(t *testing.T) { - payload := sendMessagesPayload( - namedIdentifier("orders"), numericIdentifier(2), partitionIDPartitioning(7)) - got, err := NamespaceForRequest( - uint32(command.SendMessagesCode), payload, OperationSendMessages) - require.NoError(t, err) - assert.Zero(t, got) - }) - - t.Run("defers a named topic to the server", func(t *testing.T) { - payload := sendMessagesPayload( - numericIdentifier(3), namedIdentifier("events"), partitionIDPartitioning(7)) - got, err := NamespaceForRequest( - uint32(command.SendMessagesCode), payload, OperationSendMessages) - require.NoError(t, err) - assert.Zero(t, got) - }) - - t.Run("rejects partitioning the client cannot route", func(t *testing.T) { - for _, kind := range []byte{0, 1, 3} { - payload := sendMessagesPayload( - numericIdentifier(1), numericIdentifier(1), []byte{kind, 4, 0, 0, 0, 0}) - _, err := NamespaceForRequest( - uint32(command.SendMessagesCode), payload, OperationSendMessages) - assert.ErrorIs(t, err, ierror.ErrFeatureUnavailable, "kind %d", kind) - } - }) - - t.Run("rejects a partition value of the wrong width", func(t *testing.T) { - payload := sendMessagesPayload( - numericIdentifier(1), numericIdentifier(1), - []byte{partitioningPartitionID, 2, 0, 0}) - _, err := NamespaceForRequest( - uint32(command.SendMessagesCode), payload, OperationSendMessages) - assert.ErrorIs(t, err, ierror.ErrInvalidCommand) - }) - - t.Run("rejects an out-of-range identifier", func(t *testing.T) { - payload := sendMessagesPayload( - numericIdentifier(MaxStreams), numericIdentifier(1), partitionIDPartitioning(0)) - _, err := NamespaceForRequest( - uint32(command.SendMessagesCode), payload, OperationSendMessages) - assert.ErrorIs(t, err, ierror.ErrInvalidIdentifier) - }) - - t.Run("rejects an out-of-range partition", func(t *testing.T) { - payload := sendMessagesPayload( - numericIdentifier(1), numericIdentifier(1), partitionIDPartitioning(MaxPartitions)) - _, err := NamespaceForRequest( - uint32(command.SendMessagesCode), payload, OperationSendMessages) - assert.ErrorIs(t, err, ierror.ErrInvalidIdentifier) - }) - - t.Run("rejects every truncation", func(t *testing.T) { - payload := sendMessagesPayload( - numericIdentifier(1), numericIdentifier(2), partitionIDPartitioning(3)) - for length := range len(payload) { - _, err := NamespaceForRequest( - uint32(command.SendMessagesCode), payload[:length], OperationSendMessages) - assert.Error(t, err, "truncated to %d bytes", length) - } - }) - - t.Run("rejects a metadata length past the payload", func(t *testing.T) { - payload := binary.LittleEndian.AppendUint32(nil, 1024) - payload = append(payload, numericIdentifier(1)...) - _, err := NamespaceForRequest( - uint32(command.SendMessagesCode), payload, OperationSendMessages) - assert.ErrorIs(t, err, ierror.ErrInvalidCommand) - }) -} - -func TestNamespaceForRequest_ConsumerOffsets(t *testing.T) { - variants := []struct { - code command.Code - operation Operation - }{ - {code: command.StoreOffsetCode, operation: OperationStoreConsumerOffset}, - {code: command.DeleteConsumerOffsetCode, operation: OperationDeleteConsumerOffset}, - {code: command.StoreOffset2Code, operation: OperationStoreConsumerOffset2}, - {code: command.DeleteConsumerOffset2Code, operation: OperationDeleteConsumerOffset2}, - } - - for _, variant := range variants { - t.Run(fmt.Sprintf("code_%d", variant.code), func(t *testing.T) { - runConsumerOffsetNamespaceCases(t, uint32(variant.code), variant.operation) - }) - } -} - -func runConsumerOffsetNamespaceCases(t *testing.T, code uint32, operation Operation) { - t.Helper() - - t.Run("packs the triple", func(t *testing.T) { - payload := consumerOffsetPayload( - numericIdentifier(5), numericIdentifier(4), 9, true) - got, err := NamespaceForRequest(code, payload, operation) - require.NoError(t, err) - assert.Equal(t, uint64(5<<32|4<<20|9), got) - }) - - t.Run("ignores the trailing offset and ack bytes", func(t *testing.T) { - payload := consumerOffsetPayload( - numericIdentifier(5), numericIdentifier(4), 9, true) - payload = binary.LittleEndian.AppendUint64(payload, 42) - payload = append(payload, 1) - got, err := NamespaceForRequest(code, payload, operation) - require.NoError(t, err) - assert.Equal(t, uint64(5<<32|4<<20|9), got) - }) - - t.Run("rejects a missing explicit partition", func(t *testing.T) { - payload := consumerOffsetPayload( - numericIdentifier(5), numericIdentifier(4), 0, false) - _, err := NamespaceForRequest(code, payload, operation) - assert.ErrorIs(t, err, ierror.ErrInvalidIdentifier) - }) - - t.Run("defers a named stream to the server", func(t *testing.T) { - payload := consumerOffsetPayload( - namedIdentifier("orders"), numericIdentifier(4), 9, true) - got, err := NamespaceForRequest(code, payload, operation) - require.NoError(t, err) - assert.Zero(t, got) - }) - - t.Run("rejects an unknown consumer kind", func(t *testing.T) { - payload := consumerOffsetPayload( - numericIdentifier(5), numericIdentifier(4), 9, true) - payload[0] = 3 - _, err := NamespaceForRequest(code, payload, operation) - assert.ErrorIs(t, err, ierror.ErrInvalidCommand) - }) - - t.Run("rejects every truncation", func(t *testing.T) { - payload := consumerOffsetPayload( - numericIdentifier(5), numericIdentifier(4), 9, true) - for length := range len(payload) { - _, err := NamespaceForRequest(code, payload[:length], operation) - assert.Error(t, err, "truncated to %d bytes", length) - } - }) -} - -func TestNamespaceForRequest_DeleteSegments(t *testing.T) { - t.Run("packs the triple", func(t *testing.T) { - payload := append(numericIdentifier(6), numericIdentifier(5)...) - payload = binary.LittleEndian.AppendUint32(payload, 4) - got, err := NamespaceForRequest( - uint32(command.DeleteSegmentsCode), payload, OperationDeleteSegments) - require.NoError(t, err) - assert.Equal(t, uint64(6<<32|5<<20|4), got) - }) - - t.Run("rejects every truncation", func(t *testing.T) { - payload := append(numericIdentifier(6), numericIdentifier(5)...) - payload = binary.LittleEndian.AppendUint32(payload, 4) - for length := range len(payload) { - _, err := NamespaceForRequest( - uint32(command.DeleteSegmentsCode), payload[:length], OperationDeleteSegments) - assert.Error(t, err, "truncated to %d bytes", length) - } - }) -} - -func TestPeekIdentifier_RejectsMalformedPrefixes(t *testing.T) { - tests := []struct { - name string - payload []byte - }{ - {name: "empty", payload: nil}, - {name: "kind only", payload: []byte{identifierKindNumeric}}, - {name: "numeric of the wrong width", payload: []byte{identifierKindNumeric, 2, 0, 0}}, - {name: "empty name", payload: []byte{identifierKindString, 0}}, - {name: "unknown kind", payload: []byte{9, 4, 0, 0, 0, 0}}, - {name: "value past the end", payload: []byte{identifierKindNumeric, 4, 0, 0}}, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - _, err := peekIdentifier(test.payload, 0) - assert.ErrorIs(t, err, ierror.ErrInvalidCommand) - }) - } -} - -func TestPeekIdentifier_ReportsTheConsumedLength(t *testing.T) { - numeric, err := peekIdentifier(numericIdentifier(9), 0) - require.NoError(t, err) - assert.Equal(t, uint32(9), numeric.numeric) - assert.False(t, numeric.named) - assert.Equal(t, 6, numeric.length) - - named, err := peekIdentifier(namedIdentifier("abc"), 0) - require.NoError(t, err) - assert.True(t, named.named) - assert.Equal(t, 5, named.length) -} diff --git a/foreign/go/internal/vsr/protocol_parity_test.go b/foreign/go/internal/vsr/protocol_parity_test.go index f995ce25bc..b510c8eaba 100644 --- a/foreign/go/internal/vsr/protocol_parity_test.go +++ b/foreign/go/internal/vsr/protocol_parity_test.go @@ -46,10 +46,8 @@ var rustSources = map[string]string{ "header": "core/binary_protocol/src/consensus/header.rs", "command": "core/binary_protocol/src/consensus/command.rs", "operation": "core/binary_protocol/src/consensus/operation.rs", - "namespace": "core/binary_protocol/src/namespace.rs", "cargo": "core/binary_protocol/Cargo.toml", "eviction": "core/common/src/error/eviction.rs", - "sdk": "core/sdk/src/vsr.rs", } // goOperations names every discriminant the codec declares. It exists so the @@ -131,7 +129,6 @@ var goHeaderOffsets = map[string]map[string]int{ "timestamp": requestOffsetTimestamp, "request": requestOffsetRequest, "operation": requestOffsetOperation, - "namespace": requestOffsetNamespace, "session": requestOffsetSession, "reserved": requestOffsetReserved, }, @@ -139,7 +136,6 @@ var goHeaderOffsets = map[string]map[string]int{ "size": replyOffsetSize, "command": replyOffsetCommand, "operation": replyOffsetOperation, - "namespace": replyOffsetNamespace, "status": replyOffsetStatus, }, "EvictionHeader": { @@ -333,47 +329,6 @@ func TestProtocolParity_ReplicatedOperationMap(t *testing.T) { assert.Equal(t, want, replicatedOperation) } -func TestProtocolParity_NamespaceLayout(t *testing.T) { - sources := loadRustSources(t) - limits := namedNumbers(sources["namespace"], - regexp.MustCompile(`(?m)^pub const (MAX_[A-Z]+): usize = ([0-9_]+);$`)) - - require.Equal(t, uint64(MaxStreams), limits["MAX_STREAMS"]) - require.Equal(t, uint64(MaxTopics), limits["MAX_TOPICS"]) - require.Equal(t, uint64(MaxPartitions), limits["MAX_PARTITIONS"]) - - // Rust derives the shifts from the limits rather than declaring literals, - // so the parity assertion derives them the same way. - assert.Equal(t, bitsRequired(limits["MAX_STREAMS"]-1), streamBits) - assert.Equal(t, bitsRequired(limits["MAX_TOPICS"]-1), topicBits) - assert.Equal(t, bitsRequired(limits["MAX_PARTITIONS"]-1), partitionBits) - - literalShift := namedNumbers(sources["namespace"], - regexp.MustCompile(`(?m)^pub const (PARTITION_SHIFT): u32 = ([0-9]+);$`)) - require.Contains(t, literalShift, "PARTITION_SHIFT") - assert.Equal(t, uint64(partitionShift), literalShift["PARTITION_SHIFT"]) - assert.Equal(t, partitionShift+partitionBits, topicShift) - assert.Equal(t, topicShift+topicBits, streamShift) - - packed, err := PackNamespace(1, 1, 1) - require.NoError(t, err) - assert.Equal(t, uint64(1)< 0 { - bits++ - value >>= 1 - } - return bits -} - func TestProtocolParity_EvictionReasons(t *testing.T) { sources := loadRustSources(t) rustValues := rustEnumValues(sources["header"], "EvictionReason") @@ -561,48 +516,19 @@ func TestProtocolParity_EvictionReasonMapping(t *testing.T) { // dedicated eviction tests in reply_test.go. } -func TestProtocolParity_NamespaceRouting(t *testing.T) { +// The client wire carries no routing namespace: the server derives the +// consensus group (plane from the operation, partition target from the +// payload) and stamps it into its own internal header, so this SDK has no +// packing rules to mirror. Growing the field back would move every field +// behind it, which is what makes this worth asserting on its own rather than +// leaving to the offset recomputation. +func TestProtocolParity_ClientHeadersCarryNoNamespace(t *testing.T) { sources := loadRustSources(t) - codeValues := rustCommandCodes(sources["codes"]) - require.NotEmpty(t, codeValues) - - body := captureBlock(sources["sdk"], `fn namespace_for_request\(`) - require.NotEmpty(t, body, "the Rust namespace_for_request routing was not found") - - armed := make(map[uint32]string) - for _, arm := range regexp.MustCompile(`(?m)^\s*([A-Z0-9_]+_CODE) => \{`). - FindAllStringSubmatch(body, -1) { - value, ok := codeValues[arm[1]] - require.True(t, ok, "unknown command constant %s", arm[1]) - armed[uint32(value)] = arm[1] - } - require.NotEmpty(t, armed, "no payload-peek arms were parsed") - - for name, value := range codeValues { - code := uint32(value) - operation := OperationForCode(code) - if operation == OperationRegister || operation == OperationLogout || - operation == OperationNonReplicated || IsMetadata(operation) { - continue - } - // A code that reaches the payload peek fails on an empty payload; a - // code Rust does not route is refused outright. The two errors keep - // the routing decisions distinguishable without crafting payloads. - _, err := NamespaceForRequest(code, nil, operation) - if _, peeked := armed[code]; peeked { - assert.ErrorIs(t, err, ierror.ErrInvalidCommand, - "%s must derive its namespace from the payload", name) - } else { - assert.ErrorIs(t, err, ierror.ErrFeatureUnavailable, - "%s must be refused rather than routed blindly", name) - } - } - for code, name := range armed { - operation := OperationForCode(code) - shortCircuited := operation == OperationRegister || operation == OperationLogout || - operation == OperationNonReplicated || IsMetadata(operation) - assert.False(t, shortCircuited, "%s never reaches the namespace peek in Go", name) + for _, structName := range []string{"RequestHeader", "ReplyHeader"} { + offsets := rustStructOffsets(t, sources["header"], structName) + assert.NotContains(t, offsets, "namespace", + "%s must not carry a routing namespace", structName) } } diff --git a/foreign/go/tests/e2e_helpers_test.go b/foreign/go/tests/e2e_helpers_test.go index 505c63b852..75e680c24d 100644 --- a/foreign/go/tests/e2e_helpers_test.go +++ b/foreign/go/tests/e2e_helpers_test.go @@ -19,13 +19,12 @@ // // The suite skips unless IGGY_TCP_ADDRESS points at a server. Start one with: // -// # TODO: change to iggy-server once legacy server is removed (core/server has VSR support) -// cargo build --bin iggy-server-ng --features vsr +// cargo build --bin iggy-server // IGGY_SYSTEM_PATH=/tmp/iggy-go-e2e \ // IGGY_TCP_ADDRESS=127.0.0.1:8090 \ // IGGY_HTTP_ENABLED=false IGGY_QUIC_ENABLED=false IGGY_WEBSOCKET_ENABLED=false \ // IGGY_ROOT_USERNAME=iggy IGGY_ROOT_PASSWORD=iggy \ -// target/debug/iggy-server-ng +// target/debug/iggy-server // // IGGY_TCP_ADDRESS=127.0.0.1:8090 go test ./tests // diff --git a/foreign/java/README.md b/foreign/java/README.md index 83ee696eb0..fa3fdbd345 100644 --- a/foreign/java/README.md +++ b/foreign/java/README.md @@ -183,7 +183,6 @@ var client = Iggy.tcpClientBuilder() .port(8090) .connectionTimeout(Duration.ofSeconds(10)) .requestTimeout(Duration.ofSeconds(30)) - .connectionPoolSize(10) .retryPolicy(RetryPolicy.exponentialBackoff()) .credentials("iggy", "iggy") .buildAndLogin(); @@ -213,7 +212,7 @@ See the **[Java Examples](../../examples/java/)** directory for runnable applica - **BlockingProducer**: synchronous message production with batch sending - **BlockingConsumer**: synchronous consumption with polling loops -- **AsyncProducer**: high-throughput async production with pipelining +- **AsyncProducer**: non-blocking batch production with concurrent request submission - **AsyncConsumer**: async consumption with backpressure and error recovery Each example includes comprehensive documentation on when to use blocking vs. async clients, CompletableFuture patterns, thread pool management, and performance characteristics. diff --git a/foreign/java/bench/src/main/java/org/apache/iggy/bench/benchmarks/actors/tcp/async/TcpAsyncPinnedProducerActor.java b/foreign/java/bench/src/main/java/org/apache/iggy/bench/benchmarks/actors/tcp/async/TcpAsyncPinnedProducerActor.java index 479b7a1a2c..e764697131 100644 --- a/foreign/java/bench/src/main/java/org/apache/iggy/bench/benchmarks/actors/tcp/async/TcpAsyncPinnedProducerActor.java +++ b/foreign/java/bench/src/main/java/org/apache/iggy/bench/benchmarks/actors/tcp/async/TcpAsyncPinnedProducerActor.java @@ -100,7 +100,6 @@ public CompletableFuture run() { rateLimit); return AsyncIggyTcpClient.builder() .credentials(globalCliArgs.username(), globalCliArgs.password()) - .connectionPoolSize(1) .buildAndLogin() .thenCompose(client -> { this.client = client; diff --git a/foreign/java/bench/src/main/java/org/apache/iggy/bench/report/ServerStatsCollector.java b/foreign/java/bench/src/main/java/org/apache/iggy/bench/report/ServerStatsCollector.java index fd99ac1483..7ffb009c7e 100644 --- a/foreign/java/bench/src/main/java/org/apache/iggy/bench/report/ServerStatsCollector.java +++ b/foreign/java/bench/src/main/java/org/apache/iggy/bench/report/ServerStatsCollector.java @@ -41,7 +41,6 @@ public ServerStatsCollector(GlobalCliArgs globalCliArgs) { public BenchmarkServerStats collect() { try (var client = IggyTcpClient.builder() .credentials(globalCliArgs.username(), globalCliArgs.password()) - .connectionPoolSize(1) .buildAndLogin()) { Stats stats = client.system().getStats(); Map cacheMetrics = new HashMap<>(); diff --git a/foreign/java/external-processors/iggy-connector-flink/iggy-connector-library/src/main/java/org/apache/iggy/connector/flink/source/IggySource.java b/foreign/java/external-processors/iggy-connector-flink/iggy-connector-library/src/main/java/org/apache/iggy/connector/flink/source/IggySource.java index 82fead0006..f4402e9184 100644 --- a/foreign/java/external-processors/iggy-connector-flink/iggy-connector-library/src/main/java/org/apache/iggy/connector/flink/source/IggySource.java +++ b/foreign/java/external-processors/iggy-connector-flink/iggy-connector-library/src/main/java/org/apache/iggy/connector/flink/source/IggySource.java @@ -168,7 +168,6 @@ private AsyncIggyTcpClient createAsyncIggyClient() { .retryPolicy(RetryPolicy.fixedDelay( connectionConfig.getMaxRetries(), connectionConfig.getRetryBackoff())) .tls(connectionConfig.isEnableTls()) - .connectionPoolSize(4) .buildAndLogin() .join(); diff --git a/foreign/java/external-processors/iggy-connector-flink/iggy-flink-examples/src/test/java/org/apache/iggy/flink/example/AsyncTcpMessageSendTest.java b/foreign/java/external-processors/iggy-connector-flink/iggy-flink-examples/src/test/java/org/apache/iggy/flink/example/AsyncTcpMessageSendTest.java index 1cd0ab3b0c..ffa1f85528 100644 --- a/foreign/java/external-processors/iggy-connector-flink/iggy-flink-examples/src/test/java/org/apache/iggy/flink/example/AsyncTcpMessageSendTest.java +++ b/foreign/java/external-processors/iggy-connector-flink/iggy-flink-examples/src/test/java/org/apache/iggy/flink/example/AsyncTcpMessageSendTest.java @@ -24,6 +24,7 @@ import org.apache.iggy.identifier.TopicId; import org.apache.iggy.message.Message; import org.apache.iggy.message.Partitioning; +import org.apache.iggy.message.SendMessagesResponse; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.DisplayName; @@ -116,7 +117,7 @@ void testSendSingleMessage() throws ExecutionException, InterruptedException { List messages = new ArrayList<>(); messages.add(message); - CompletableFuture sendFuture = + CompletableFuture sendFuture = client.messages().sendMessages(streamId, topicId, Partitioning.balanced(), messages); sendFuture.get(); @@ -142,7 +143,7 @@ void testSendBatchMessages() throws ExecutionException, InterruptedException { messages.add(Message.of(content)); } - CompletableFuture sendFuture = + CompletableFuture sendFuture = client.messages().sendMessages(streamId, topicId, Partitioning.balanced(), messages); sendFuture.get(); @@ -166,7 +167,7 @@ void testSendToSpecificPartition() throws ExecutionException, InterruptedExcepti List messages = new ArrayList<>(); messages.add(message); - CompletableFuture sendFuture = + CompletableFuture sendFuture = client.messages().sendMessages(streamId, topicId, Partitioning.partitionId(targetPartition), messages); sendFuture.get(); @@ -190,7 +191,7 @@ void testSendWithMessageKey() throws ExecutionException, InterruptedException { messages.add(message); String messageKey = "test-key-123"; - CompletableFuture sendFuture = + CompletableFuture sendFuture = client.messages().sendMessages(streamId, topicId, Partitioning.messagesKey(messageKey), messages); sendFuture.get(); @@ -214,7 +215,7 @@ void testSendJsonMessages() throws ExecutionException, InterruptedException { List messages = new ArrayList<>(); messages.add(message); - CompletableFuture sendFuture = + CompletableFuture sendFuture = client.messages().sendMessages(streamId, topicId, Partitioning.balanced(), messages); sendFuture.get(); @@ -232,7 +233,7 @@ void testSendMultipleMessagesInParallel() throws ExecutionException, Interrupted TopicId topicId = TopicId.of("lines"); int parallelRequests = 5; - List> futures = new ArrayList<>(); + List> futures = new ArrayList<>(); for (int i = 0; i < parallelRequests; i++) { String content = "Parallel message #" + i; @@ -240,7 +241,7 @@ void testSendMultipleMessagesInParallel() throws ExecutionException, Interrupted List messages = new ArrayList<>(); messages.add(message); - CompletableFuture future = + CompletableFuture future = client.messages().sendMessages(streamId, topicId, Partitioning.balanced(), messages); futures.add(future); } @@ -269,7 +270,7 @@ void testSendLargeBatch() throws ExecutionException, InterruptedException { } long startTime = System.currentTimeMillis(); - CompletableFuture sendFuture = + CompletableFuture sendFuture = client.messages().sendMessages(streamId, topicId, Partitioning.balanced(), messages); sendFuture.get(); long duration = System.currentTimeMillis() - startTime; diff --git a/foreign/java/external-processors/iggy-connector-pinot/integration-test.sh b/foreign/java/external-processors/iggy-connector-pinot/integration-test.sh index e438afe4bc..f83c5f8ec1 100755 --- a/foreign/java/external-processors/iggy-connector-pinot/integration-test.sh +++ b/foreign/java/external-processors/iggy-connector-pinot/integration-test.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/foreign/java/gradle.properties b/foreign/java/gradle.properties index f9eb048aea..5747b90f8a 100644 --- a/foreign/java/gradle.properties +++ b/foreign/java/gradle.properties @@ -15,5 +15,5 @@ # specific language governing permissions and limitations # under the License. -version=0.8.3-SNAPSHOT +version=0.9.0-SNAPSHOT group=org.apache.iggy diff --git a/foreign/java/gradle/libs.versions.toml b/foreign/java/gradle/libs.versions.toml index ed00082551..6b5b54d941 100644 --- a/foreign/java/gradle/libs.versions.toml +++ b/foreign/java/gradle/libs.versions.toml @@ -30,7 +30,7 @@ jackson2 = "2.22.1" commons-lang3 = "3.20.0" # HTTP Client -httpclient5 = "5.6.2" +httpclient5 = "5.6.3" # Logging slf4j = "2.0.18" diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/ConsumerGroupsClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/ConsumerGroupsClient.java index 9a8af01083..a8f2d0dba3 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/ConsumerGroupsClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/ConsumerGroupsClient.java @@ -20,6 +20,7 @@ package org.apache.iggy.client.async; import org.apache.iggy.consumergroup.ConsumerGroup; +import org.apache.iggy.consumergroup.ConsumerGroupAssignment; import org.apache.iggy.consumergroup.ConsumerGroupDetails; import org.apache.iggy.identifier.ConsumerId; import org.apache.iggy.identifier.StreamId; @@ -181,4 +182,26 @@ default CompletableFuture deleteConsumerGroup(Long streamId, Long topicId, * group */ CompletableFuture leaveConsumerGroup(StreamId streamId, TopicId topicId, ConsumerId groupId); + + /** + * Fetches this client's current partition assignment for a consumer group. + * + *

The server identifies the member by the connection's session, so no + * member identifier is sent. The returned assignment carries the group + * generation; it advances on every rebalance, at which point cached + * assignments become stale and polls against revoked partitions are + * fenced by the server. + * + *

Group polling calls this internally; explicit calls are only needed + * for custom partition-selection logic. + * + * @param streamId the stream identifier containing the topic + * @param topicId the topic identifier + * @param groupId the consumer group identifier + * @return a {@link CompletableFuture} completing with the member's + * {@link ConsumerGroupAssignment}, or empty when this client is + * not a member of the group + */ + CompletableFuture> syncConsumerGroup( + StreamId streamId, TopicId topicId, ConsumerId groupId); } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/MessagesClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/MessagesClient.java index 8215665b5c..f3b9ad115b 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/MessagesClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/MessagesClient.java @@ -26,6 +26,7 @@ import org.apache.iggy.message.Partitioning; import org.apache.iggy.message.PolledMessages; import org.apache.iggy.message.PollingStrategy; +import org.apache.iggy.message.SendMessagesResponse; import java.util.List; import java.util.Optional; @@ -83,8 +84,10 @@ public interface MessagesClient { * * @param streamId the stream identifier (numeric or string-based) * @param topicId the topic identifier (numeric or string-based) - * @param partitionId optional partition ID to poll from; if empty, the server selects - * the partition (required when using consumer groups) + * @param partitionId optional partition ID to poll from; when empty and polling with a + * group consumer, the client selects the partition round-robin from + * the member's synced group assignment (the group must be joined + * first) * @param consumer the consumer identity, either individual ({@link Consumer#of(Long)}) * or group ({@link Consumer#group(Long)}) * @param strategy the polling strategy controlling where to start reading @@ -148,6 +151,11 @@ default CompletableFuture pollMessages( * messages with the same key always go to the same partition, preserving order * * + *

Over TCP the VSR broker routes explicit partitions only, so balanced and + * key-based partitioning are resolved to a concrete partition by the client + * (round-robin cursor and {@code xxh32(key) % partitionCount} respectively), + * consistently with the other Iggy SDKs. + * *

Messages are batched into a single network request for efficiency. For high * throughput, accumulate messages and send them in larger batches rather than one * at a time. @@ -156,11 +164,11 @@ default CompletableFuture pollMessages( * @param topicId the topic identifier (numeric or string-based) * @param partitioning the partitioning strategy for routing messages * @param messages the list of messages to send - * @return a {@link CompletableFuture} that completes when all messages have been - * acknowledged by the server + * @return a {@link CompletableFuture} that completes with the {@link SendMessagesResponse} + * confirming the partition and base offset of the committed batch * @throws org.apache.iggy.exception.IggyException if the stream or topic does not exist */ - CompletableFuture sendMessages( + CompletableFuture sendMessages( StreamId streamId, TopicId topicId, Partitioning partitioning, List messages); /** @@ -173,9 +181,9 @@ CompletableFuture sendMessages( * @param topicId the numeric topic ID * @param partitioning the partitioning strategy * @param messages the list of messages to send - * @return a {@link CompletableFuture} that completes when messages are acknowledged + * @return a {@link CompletableFuture} that completes with the {@link SendMessagesResponse} */ - default CompletableFuture sendMessages( + default CompletableFuture sendMessages( Long streamId, Long topicId, Partitioning partitioning, List messages) { return sendMessages(StreamId.of(streamId), TopicId.of(topicId), partitioning, messages); } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java index 5a78acdbaf..04f46ec492 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClient.java @@ -19,7 +19,9 @@ package org.apache.iggy.client.async.tcp; +import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; +import io.netty.channel.ConnectTimeoutException; import org.apache.iggy.IggyVersion; import org.apache.iggy.client.ConnectionInfo; import org.apache.iggy.client.async.ConsumerGroupsClient; @@ -33,20 +35,26 @@ import org.apache.iggy.client.async.UsersClient; import org.apache.iggy.client.async.tcp.AsyncTcpConnection.TcpConnectionPoolConfig; import org.apache.iggy.client.async.tcp.LeaderAwareness.LeaderRedirectionState; +import org.apache.iggy.client.async.tcp.vsr.VsrFrameDecoder; import org.apache.iggy.config.RetryPolicy; import org.apache.iggy.exception.IggyMissingCredentialsException; import org.apache.iggy.exception.IggyNotConnectedException; import org.apache.iggy.exception.IggyServerException; +import org.apache.iggy.exception.IggyTimeoutException; import org.apache.iggy.serde.CommandCode; import org.apache.iggy.user.IdentityInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; +import java.io.IOException; import java.time.Duration; import java.util.Objects; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.function.Supplier; @@ -105,19 +113,24 @@ public class AsyncIggyTcpClient { private static final int INVALID_COMMAND_ERROR_CODE = 3; private static final Logger log = LoggerFactory.getLogger(AsyncIggyTcpClient.class); + private static final RetryPolicy DEFAULT_RECONNECT_POLICY = RetryPolicy.fixedDelay(12, Duration.ofSeconds(5)); + private final ConnectionInfo seedConnectionInfo; + private final AtomicBoolean reconnecting = new AtomicBoolean(); private final Optional username; private final Optional password; private final Optional connectionTimeout; private final Optional acquireTimeout; private final Optional requestTimeout; - private final Optional connectionPoolSize; + private final Duration heartbeatInterval; + private final int maxVsrFrameSize; private final Optional retryPolicy; private final boolean enableTls; private final Optional tlsCertificate; private final TcpConnectionPoolConfig poolConfig; + private final ClientRoutingState routingState = new ClientRoutingState(); private final AtomicReference connection = new AtomicReference<>(); - private final AtomicReference> redirectChain = + private final AtomicReference> loginChain = new AtomicReference<>(CompletableFuture.completedFuture(null)); private volatile ConnectionInfo connectionInfo; private volatile boolean closed; @@ -140,7 +153,19 @@ public class AsyncIggyTcpClient { * @param port the server port */ public AsyncIggyTcpClient(String host, int port) { - this(host, port, null, null, null, null, null, null, null, false, Optional.empty()); + this( + host, + port, + null, + null, + null, + null, + null, + Duration.ofSeconds(5), + VsrFrameDecoder.DEFAULT_MAX_FRAME_SIZE, + null, + false, + Optional.empty()); } @SuppressWarnings("checkstyle:ParameterNumber") @@ -152,23 +177,25 @@ public AsyncIggyTcpClient(String host, int port) { Duration connectionTimeout, Duration acquireTimeout, Duration requestTimeout, - Integer connectionPoolSize, + Duration heartbeatInterval, + int maxVsrFrameSize, RetryPolicy retryPolicy, boolean enableTls, Optional tlsCertificate) { this.connectionInfo = new ConnectionInfo(host, port); + this.seedConnectionInfo = this.connectionInfo; this.username = Optional.ofNullable(username); this.password = Optional.ofNullable(password); this.connectionTimeout = Optional.ofNullable(connectionTimeout); this.acquireTimeout = Optional.ofNullable(acquireTimeout); this.requestTimeout = Optional.ofNullable(requestTimeout); - this.connectionPoolSize = Optional.ofNullable(connectionPoolSize); + this.heartbeatInterval = heartbeatInterval; + this.maxVsrFrameSize = maxVsrFrameSize; this.retryPolicy = Optional.ofNullable(retryPolicy); this.enableTls = enableTls; this.tlsCertificate = tlsCertificate; var poolConfigBuilder = TcpConnectionPoolConfig.builder(); - this.connectionPoolSize.ifPresent(poolConfigBuilder::setMaxConnections); this.acquireTimeout.ifPresent(timeout -> poolConfigBuilder.setAcquireTimeoutMillis(timeout.toMillis())); this.poolConfig = poolConfigBuilder.build(); } @@ -199,18 +226,18 @@ public CompletableFuture connect() { if (previousConnection != null) { previousConnection.close(); } + routingState.clearAssignments(); Supplier currentConnection = connection::get; return newConnection.connect().thenRun(() -> { log.debug("Connected to {} | {}", target.serverAddress(), IggyVersion.getInstance()); - messagesClient = new MessagesTcpClient(currentConnection); + messagesClient = new MessagesTcpClient(currentConnection, routingState); consumerGroupsClient = new ConsumerGroupsTcpClient(currentConnection); consumerOffsetsClient = new ConsumerOffsetsTcpClient(currentConnection); streamsClient = new StreamsTcpClient(currentConnection); topicsClient = new TopicsTcpClient(currentConnection); - usersClient = new UsersTcpClient(currentConnection, this::checkLeaderAndRedirect); + usersClient = new UsersTcpClient(currentConnection, this::loginOnLeader); systemClient = new SystemTcpClient(currentConnection); - personalAccessTokensClient = - new PersonalAccessTokensTcpClient(currentConnection, this::checkLeaderAndRedirect); + personalAccessTokensClient = new PersonalAccessTokensTcpClient(currentConnection, this::loginOnLeader); partitionsClient = new PartitionsTcpClient(currentConnection); }); } @@ -408,9 +435,9 @@ public CompletableFuture close() { } /** - * Returns the server address this client currently targets. Leader - * redirection can change it after login, so it may differ from the - * address the client was built with. + * Returns the server address this client currently targets. Pre-login + * leader discovery can change it, so it may differ from the address the + * client was built with. * * @return the current {@link ConnectionInfo} */ @@ -420,48 +447,227 @@ public ConnectionInfo getConnectionInfo() { private AsyncTcpConnection openConnection(ConnectionInfo target) { return new AsyncTcpConnection( - target.host(), target.port(), enableTls, tlsCertificate, poolConfig, connectionTimeout); + target.host(), + target.port(), + enableTls, + tlsCertificate, + poolConfig, + connectionTimeout, + requestTimeout, + heartbeatInterval, + maxVsrFrameSize, + this::retryTransientOnLeader, + routingState::clearAssignments, + this::onConnectionFailure); } /** - * Post-login leader check, serialized across concurrent logins: fetches - * the cluster roster and, while a healthy leader lives elsewhere, - * reconnects to it and replays the login. A queued login waits for the - * in-flight redirection and then re-checks the roster, so it either - * confirms the new node or retries a redirection that failed. Every - * failure except the replayed login's own is non-fatal; the client then - * stays on the current node. Each check runs with a fresh redirection - * budget, so hitting the cap parks only that login, not the client. + * A not-accepted request was never admitted, so it is safe to recheck the + * leader, restore authentication on a new connection, and retry it within + * the original request deadline. */ - private CompletableFuture checkLeaderAndRedirect(Supplier> reLogin) { + private CompletableFuture retryTransientOnLeader( + AsyncTcpConnection source, + int commandCode, + ByteBuf payload, + long requestDeadlineNanos, + IggyServerException rejection) { + AtomicReference requestPayload = new AtomicReference<>(payload); + Optional authentication = source.authenticationSnapshot(); + AtomicReference authenticationPayload = new AtomicReference<>(authentication + .map(AsyncTcpConnection.AuthenticationSnapshot::payload) + .orElse(null)); + + CompletableFuture ready = prepareTransientFailover( + source, authentication, authenticationPayload, requestDeadlineNanos, rejection); + CompletableFuture retried = ready.thenCompose(ignored -> { + if (requestDeadlineNanos - System.nanoTime() <= 0) { + return CompletableFuture.failedFuture(rejection); + } + AsyncTcpConnection currentConnection = connection.get(); + if (currentConnection == null) { + return CompletableFuture.failedFuture(new IggyNotConnectedException()); + } + return currentConnection.send(commandCode, takePayload(requestPayload), requestDeadlineNanos); + }); + return retried.whenComplete((response, error) -> { + releasePayload(requestPayload); + releasePayload(authenticationPayload); + }); + } + + private CompletableFuture prepareTransientFailover( + AsyncTcpConnection source, + Optional authentication, + AtomicReference authenticationPayload, + long requestDeadlineNanos, + IggyServerException rejection) { CompletableFuture gate = new CompletableFuture<>(); - CompletableFuture previous = redirectChain.getAndSet(gate); + CompletableFuture previous = loginChain.getAndSet(gate); LeaderRedirectionState redirectionState = new LeaderRedirectionState(); - return previous.thenCompose(ignored -> redirectToLeader(reLogin, null, redirectionState)) - .whenComplete((identity, error) -> gate.complete(null)); + CompletableFuture transaction = previous.thenCompose(ignored -> { + if (closed) { + return CompletableFuture.failedFuture(new IggyNotConnectedException()); + } + if (requestDeadlineNanos - System.nanoTime() <= 0) { + return CompletableFuture.failedFuture(rejection); + } + if (connection.get() != source) { + return CompletableFuture.completedFuture(null); + } + return redirectToLeader(redirectionState).thenCompose(redirected -> { + AsyncTcpConnection currentConnection = connection.get(); + if (currentConnection == null || currentConnection == source || authentication.isEmpty()) { + return CompletableFuture.completedFuture(null); + } + if (requestDeadlineNanos - System.nanoTime() <= 0) { + return CompletableFuture.failedFuture(rejection); + } + return currentConnection + .send( + authentication.orElseThrow().commandCode(), + takePayload(authenticationPayload), + requestDeadlineNanos) + .thenAccept(ByteBuf::release); + }); + }); + transaction.whenComplete((ignored, error) -> gate.complete(null)); + return transaction; + } + + private static ByteBuf takePayload(AtomicReference payload) { + ByteBuf owned = payload.getAndSet(null); + if (owned == null) { + throw new IllegalStateException("Request payload ownership was already transferred"); + } + return owned; + } + + private static void releasePayload(AtomicReference payload) { + ByteBuf owned = payload.getAndSet(null); + if (owned != null) { + owned.release(); + } + } + + /** + * Entry point of the background redial after a pool acquire failure or an + * expired reply. Requests that were in flight stay failed (their outcome + * is unknown); the redial only restores the client for subsequent calls. + * Alternates the current endpoint with the seed, paced by the configured + * retry policy, and replays the builder credentials on the restored + * connection. Personal-access-token logins cannot be replayed here; those + * clients must log in again themselves. + */ + private void onConnectionFailure(Throwable cause) { + if (closed || !isConnectionLoss(cause)) { + return; + } + if (!reconnecting.compareAndSet(false, true)) { + return; + } + log.warn("Connection to {} lost ({}), starting redial", connectionInfo.serverAddress(), cause.getMessage()); + RetryPolicy policy = retryPolicy.orElse(DEFAULT_RECONNECT_POLICY); + redialAttempt(1, policy).whenComplete((ignored, error) -> reconnecting.set(false)); + } + + private static boolean isConnectionLoss(Throwable cause) { + return cause instanceof ConnectTimeoutException + || cause instanceof IOException + || cause instanceof IggyTimeoutException; + } + + private CompletableFuture redialAttempt(int attempt, RetryPolicy policy) { + if (closed) { + return CompletableFuture.completedFuture(null); + } + if (attempt > policy.getMaxRetries()) { + log.error("Redial gave up after {} attempts, next request will fail fast", policy.getMaxRetries()); + return CompletableFuture.completedFuture(null); + } + ConnectionInfo target = ReconnectPlan.target(connectionInfo, seedConnectionInfo, attempt); + Duration delay = ReconnectPlan.delay(policy, attempt); + Executor delayedExecutor = CompletableFuture.delayedExecutor(delay.toMillis(), TimeUnit.MILLISECONDS); + return CompletableFuture.supplyAsync(() -> null, delayedExecutor).thenCompose(ignored -> { + if (closed) { + return CompletableFuture.completedFuture(null); + } + log.info("Redial attempt {}/{} to {}", attempt, policy.getMaxRetries(), target.serverAddress()); + return retarget(target) + .thenCompose(retargeted -> replayLogin()) + .handle((ok, error) -> { + if (error == null) { + log.info("Reconnected to {}", target.serverAddress()); + return CompletableFuture.completedFuture(null); + } + log.warn( + "Redial attempt {} to {} failed: {}", + attempt, + target.serverAddress(), + error.getMessage()); + return redialAttempt(attempt + 1, policy); + }) + .thenCompose(Function.identity()); + }); + } + + /** + * Replays the builder credentials on the freshly published connection. + * The login runs through the users client, so leader discovery retargets + * again before Register when the redialed node is not the leader. + */ + private CompletableFuture replayLogin() { + if (username.isEmpty() || password.isEmpty() || usersClient == null) { + return CompletableFuture.completedFuture(null); + } + return usersClient.login(username.get(), password.get()).thenApply(identity -> null); + } + + /** + * Serializes pre-login leader discovery and Register across concurrent + * logins. A queued login waits for the entire in-flight transaction, then + * checks the roster from the connection that transaction published. Each + * transaction has a fresh redirection budget, so hitting the cap affects + * only that login. Metadata and retargeting failures retain best-effort + * behavior and let Register run against the current target. + */ + CompletableFuture loginOnLeader(Supplier> loginAttempt) { + CompletableFuture gate = new CompletableFuture<>(); + CompletableFuture previous = loginChain.getAndSet(gate); + LeaderRedirectionState redirectionState = new LeaderRedirectionState(); + CompletableFuture transaction = previous.thenCompose( + ignored -> redirectToLeader(redirectionState)) + .thenCompose(ignored -> loginAttempt.get()); + CompletableFuture callerFuture = new CompletableFuture<>(); + transaction.whenComplete((identity, error) -> { + gate.complete(null); + if (error != null) { + callerFuture.completeExceptionally(error); + } else { + callerFuture.complete(identity); + } + }); + return callerFuture; } /** - * One redirection hop: when the roster names a healthy leader elsewhere, - * reconnect to it, replay the login and re-check from the new node, since - * mid-election metadata can point at a node that is itself not the - * leader. Bounded by the per-check redirection budget. + * One authentication-independent discovery hop. When the roster names a + * healthy leader elsewhere, reconnect to it and re-check from the new + * node, since mid-election metadata can point at a node that is itself not + * the leader. Register is sent only after this bounded process settles. */ - private CompletableFuture redirectToLeader( - Supplier> reLogin, - IdentityInfo redirectedIdentity, - LeaderRedirectionState redirectionState) { + private CompletableFuture redirectToLeader(LeaderRedirectionState redirectionState) { ConnectionInfo currentTarget = connectionInfo; return findLeaderElsewhere(currentTarget).thenCompose(leaderTarget -> { if (leaderTarget.isEmpty()) { - return CompletableFuture.completedFuture(redirectedIdentity); + return CompletableFuture.completedFuture(null); } if (!redirectionState.canRedirect()) { log.warn( "Maximum leader redirections ({}) reached, connection will continue on server node {}", LeaderAwareness.MAX_LEADER_REDIRECTS, currentTarget.serverAddress()); - return CompletableFuture.completedFuture(redirectedIdentity); + return CompletableFuture.completedFuture(null); } return retarget(leaderTarget.get()) .handle((ignored, error) -> { @@ -472,11 +678,10 @@ private CompletableFuture redirectToLeader( leaderTarget.get().serverAddress(), error.getMessage(), currentTarget.serverAddress()); - return CompletableFuture.completedFuture(redirectedIdentity); + return CompletableFuture.completedFuture(null); } redirectionState.recordRedirect(); - return reLogin.get() - .thenCompose(identity -> redirectToLeader(reLogin, identity, redirectionState)); + return redirectToLeader(redirectionState); }) .thenCompose(Function.identity()); }); @@ -488,7 +693,7 @@ private CompletableFuture redirectToLeader( * (metadata fetch, malformed roster) so the redirection path never fails * the login that triggered it. */ - private CompletableFuture> findLeaderElsewhere(ConnectionInfo currentTarget) { + CompletableFuture> findLeaderElsewhere(ConnectionInfo currentTarget) { SystemClient currentSystemClient = systemClient; if (currentSystemClient == null) { return CompletableFuture.completedFuture(Optional.empty()); @@ -496,7 +701,7 @@ private CompletableFuture> findLeaderElsewhere(Connecti return LeaderAwareness.findLeaderElsewhere(currentSystemClient::getClusterMetadata, currentTarget); } - private CompletableFuture retarget(ConnectionInfo newTarget) { + CompletableFuture retarget(ConnectionInfo newTarget) { AsyncTcpConnection oldConnection = connection.get(); AsyncTcpConnection newConnection; try { @@ -527,6 +732,7 @@ private CompletableFuture publishConnection( new IggyNotConnectedException("Client closed during leader redirection")); } connectionInfo = newTarget; + routingState.clearAssignments(); oldConnection.close().whenComplete((ignored, closeError) -> { if (closeError != null) { log.warn("Failed to close previous connection: {}", closeError.getMessage()); diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientBuilder.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientBuilder.java index 85664e5cf9..e6a6ed510b 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientBuilder.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientBuilder.java @@ -20,6 +20,8 @@ package org.apache.iggy.client.async.tcp; import org.apache.commons.lang3.StringUtils; +import org.apache.iggy.client.async.tcp.vsr.VsrFrameDecoder; +import org.apache.iggy.client.async.tcp.vsr.VsrHeaders; import org.apache.iggy.config.RetryPolicy; import org.apache.iggy.exception.IggyInvalidArgumentException; import org.apache.iggy.exception.IggyMissingCredentialsException; @@ -72,9 +74,10 @@ public final class AsyncIggyTcpClientBuilder { private File tlsCertificate; private Duration connectionTimeout; private Duration requestTimeout; - private Integer connectionPoolSize; private RetryPolicy retryPolicy; private Duration acquireTimeout; + private Duration heartbeatInterval = Duration.ofSeconds(5); + private long maxVsrFrameSize = VsrFrameDecoder.DEFAULT_MAX_FRAME_SIZE; public AsyncIggyTcpClientBuilder() {} @@ -168,6 +171,29 @@ public AsyncIggyTcpClientBuilder acquireTimeout(Duration acquireTimeout) { return this; } + /** + * Sets how often the client sends a heartbeat while connected. The value + * should not exceed the server's configured heartbeat interval. + * + * @param heartbeatInterval the heartbeat interval + * @return this builder + */ + public AsyncIggyTcpClientBuilder heartbeatInterval(Duration heartbeatInterval) { + this.heartbeatInterval = heartbeatInterval; + return this; + } + + /** + * Sets the largest inbound VSR frame the client will buffer. + * + * @param maxVsrFrameSize maximum frame size in bytes, including the VSR header + * @return this builder + */ + public AsyncIggyTcpClientBuilder maxVsrFrameSize(long maxVsrFrameSize) { + this.maxVsrFrameSize = maxVsrFrameSize; + return this; + } + /** * Sets the connection timeout. * @@ -190,17 +216,6 @@ public AsyncIggyTcpClientBuilder requestTimeout(Duration requestTimeout) { return this; } - /** - * Sets the connection pool size. - * - * @param connectionPoolSize the connection pool size - * @return this builder - */ - public AsyncIggyTcpClientBuilder connectionPoolSize(Integer connectionPoolSize) { - this.connectionPoolSize = connectionPoolSize; - return this; - } - /** * Sets the retry policy. * @@ -222,9 +237,11 @@ public AsyncIggyTcpClientBuilder retryPolicy(RetryPolicy retryPolicy) { public AsyncIggyTcpClient build() { validateHost(); validatePort(); - validateConnectionPoolSize(); validateConnectionTimeout(); validateAcquireTimeout(); + validateRequestTimeout(); + validateHeartbeatInterval(); + validateMaxVsrFrameSize(); return new AsyncIggyTcpClient( host, @@ -234,7 +251,8 @@ public AsyncIggyTcpClient build() { connectionTimeout, acquireTimeout, requestTimeout, - connectionPoolSize, + heartbeatInterval, + (int) maxVsrFrameSize, retryPolicy, enableTls, Optional.ofNullable(tlsCertificate)); @@ -252,12 +270,6 @@ private void validatePort() { } } - private void validateConnectionPoolSize() { - if (connectionPoolSize != null && connectionPoolSize <= 0) { - throw new IggyInvalidArgumentException("Connection pool size cannot by 0 or negative"); - } - } - private void validateConnectionTimeout() { if (connectionTimeout == null) { return; @@ -277,6 +289,25 @@ private void validateAcquireTimeout() { } } + private void validateRequestTimeout() { + if (requestTimeout != null && (requestTimeout.equals(Duration.ZERO) || requestTimeout.isNegative())) { + throw new IggyInvalidArgumentException("RequestTimeout Cannot be 0 or Negative"); + } + } + + private void validateHeartbeatInterval() { + if (heartbeatInterval == null || heartbeatInterval.isZero() || heartbeatInterval.isNegative()) { + throw new IggyInvalidArgumentException("HeartbeatInterval Cannot be null, 0 or Negative"); + } + } + + private void validateMaxVsrFrameSize() { + if (maxVsrFrameSize < VsrHeaders.HEADER_SIZE || maxVsrFrameSize > Integer.MAX_VALUE) { + throw new IggyInvalidArgumentException("MaxVsrFrameSize must be between " + VsrHeaders.HEADER_SIZE + " and " + + Integer.MAX_VALUE + " bytes"); + } + } + /** * Builds, connects, and logs in using the provided credentials. * This is a convenience method equivalent to calling {@code build()}, {@code connect()}, diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncTcpConnection.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncTcpConnection.java index 38eb30f05e..c421947c86 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncTcpConnection.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/AsyncTcpConnection.java @@ -21,15 +21,14 @@ import io.netty.bootstrap.Bootstrap; import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; import io.netty.channel.Channel; import io.netty.channel.ChannelFutureListener; -import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelOption; import io.netty.channel.ChannelPipeline; import io.netty.channel.ConnectTimeoutException; import io.netty.channel.IoEventLoopGroup; import io.netty.channel.MultiThreadIoEventLoopGroup; -import io.netty.channel.SimpleChannelInboundHandler; import io.netty.channel.nio.NioIoHandler; import io.netty.channel.pool.AbstractChannelPoolHandler; import io.netty.channel.pool.ChannelHealthChecker; @@ -38,12 +37,18 @@ import io.netty.handler.ssl.SslContext; import io.netty.handler.ssl.SslContextBuilder; import io.netty.util.concurrent.FutureListener; +import io.netty.util.concurrent.ScheduledFuture; +import org.apache.iggy.client.async.tcp.vsr.ConsensusSession; +import org.apache.iggy.client.async.tcp.vsr.VsrFrameDecoder; +import org.apache.iggy.client.async.tcp.vsr.VsrRequestEncoder; +import org.apache.iggy.client.async.tcp.vsr.VsrResponseHandler; import org.apache.iggy.exception.IggyClientException; import org.apache.iggy.exception.IggyConnectionException; import org.apache.iggy.exception.IggyEmptyResponseException; import org.apache.iggy.exception.IggyInvalidArgumentException; import org.apache.iggy.exception.IggyNotConnectedException; import org.apache.iggy.exception.IggyServerException; +import org.apache.iggy.exception.IggyTimeoutException; import org.apache.iggy.exception.IggyTlsException; import org.apache.iggy.serde.CommandCode; import org.slf4j.Logger; @@ -55,11 +60,13 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; -import java.util.Queue; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Consumer; import java.util.function.Function; /** @@ -69,12 +76,35 @@ public class AsyncTcpConnection { private static final Logger log = LoggerFactory.getLogger(AsyncTcpConnection.class); private static final Duration DEFAULT_CONNECTION_TIMEOUT = Duration.ofMillis(3000); + // A missing reply must not hold the single VSR-pinned channel forever. + private static final Duration DEFAULT_REQUEST_TIMEOUT = Duration.ofSeconds(30); + // Transient VSR denials (not-committed / not-accepted) are replayed with + // the same encoded frame so the server's dedup sees the same request id. + // A not-committed outcome is unknown, so it replays for the whole budget. + // A not-accepted deny was refused outright (typically a demoted primary), + // so after a short same-node retry it is handed to the owning client for + // a leader recheck and safe replay; mirrors TRANSIENT_FAILOVER_CHECK_INTERVAL + // in core/sdk/src/tcp/tcp_client.rs. + private static final int TRANSIENT_NOT_COMMITTED = 57; + private static final int TRANSIENT_NOT_ACCEPTED = 58; + private static final long TRANSIENT_RETRY_INTERVAL_MS = 50; + private static final Duration TRANSIENT_RETRY_BUDGET = Duration.ofSeconds(30); + private static final Duration NOT_ACCEPTED_RETRY_BUDGET = Duration.ofSeconds(2); private final IoEventLoopGroup eventLoopGroup; private final FixedChannelPool channelPool; private final AtomicBoolean isClosed = new AtomicBoolean(false); private final AtomicLong authGeneration = new AtomicLong(0); + private final VsrRequestEncoder vsrEncoder; + private final TransientFailoverHandler transientFailoverHandler; + private final Runnable sessionResetListener; + private final Consumer connectionFailureListener; + private final long requestTimeoutNanos; + private final long heartbeatIntervalNanos; + private final Object heartbeatLock = new Object(); private ByteBuf loginPayload; + private ScheduledFuture heartbeatTask; + private boolean heartbeatRunning; private volatile int loginCommandCode; private volatile boolean authenticated = false; @@ -86,6 +116,40 @@ public AsyncTcpConnection( Optional tlsCertificate, TcpConnectionPoolConfig poolConfig, Optional connectionTimeout) { + this( + host, + port, + enableTls, + tlsCertificate, + poolConfig, + connectionTimeout, + Optional.empty(), + Duration.ofSeconds(5), + VsrFrameDecoder.DEFAULT_MAX_FRAME_SIZE, + null, + () -> {}, + ignored -> {}); + } + + @SuppressWarnings("checkstyle:ParameterNumber") + AsyncTcpConnection( + String host, + int port, + boolean enableTls, + Optional tlsCertificate, + TcpConnectionPoolConfig poolConfig, + Optional connectionTimeout, + Optional requestTimeout, + Duration heartbeatInterval, + int maxVsrFrameSize, + TransientFailoverHandler transientFailoverHandler, + Runnable sessionResetListener, + Consumer connectionFailureListener) { + this.transientFailoverHandler = transientFailoverHandler; + this.sessionResetListener = sessionResetListener; + this.connectionFailureListener = connectionFailureListener; + this.requestTimeoutNanos = toTimeoutNanos(requestTimeout.orElse(DEFAULT_REQUEST_TIMEOUT)); + this.heartbeatIntervalNanos = toTimeoutNanos(heartbeatInterval); SslContext sslContext = null; if (enableTls) { try { @@ -97,6 +161,8 @@ public AsyncTcpConnection( } } + ConsensusSession consensusSession = new ConsensusSession(); + this.vsrEncoder = new VsrRequestEncoder(consensusSession); this.eventLoopGroup = new MultiThreadIoEventLoopGroup(NioIoHandler.newFactory()); var bootstrap = new Bootstrap() @@ -108,16 +174,20 @@ public AsyncTcpConnection( .option(ChannelOption.SO_KEEPALIVE, true) .remoteAddress(host, port); + // The VSR session (client id, fence epoch, request counter) is bound + // to one transport connection server-side; sharing it across channels + // would interleave request ids, so the pool holds a single channel. this.channelPool = new FixedChannelPool( bootstrap, - new PoolChannelHandler(host, port, enableTls, sslContext), + new PoolChannelHandler( + host, port, enableTls, sslContext, consensusSession, maxVsrFrameSize, this::onSessionEvicted), ChannelHealthChecker.ACTIVE, FixedChannelPool.AcquireTimeoutAction.FAIL, poolConfig.getAcquireTimeoutMillis(), - poolConfig.getMaxConnections(), + 1, poolConfig.getMaxPendingAcquires()); - log.info("Connection pool initialized with max connections: {}", poolConfig.getMaxConnections()); + log.info("Connection pool initialized with a single VSR-pinned connection"); } /** @@ -127,8 +197,14 @@ public CompletableFuture connect() { CompletableFuture future = new CompletableFuture<>(); channelPool.acquire().addListener((FutureListener) f -> { if (f.isSuccess()) { - channelPool.release(f.getNow()); - future.complete(null); + channelPool.release(f.getNow()).addListener(release -> { + if (release.isSuccess()) { + startHeartbeat(); + future.complete(null); + } else { + future.completeExceptionally(release.cause()); + } + }); } else { Throwable cause = f.cause(); if (cause instanceof ConnectTimeoutException) { @@ -141,6 +217,62 @@ public CompletableFuture connect() { return future; } + private void startHeartbeat() { + synchronized (heartbeatLock) { + if (heartbeatRunning || isClosed.get()) { + return; + } + heartbeatRunning = true; + scheduleNextHeartbeat(); + } + } + + private void scheduleNextHeartbeat() { + synchronized (heartbeatLock) { + if (!heartbeatRunning || isClosed.get()) { + return; + } + heartbeatTask = + eventLoopGroup.next().schedule(this::sendHeartbeat, heartbeatIntervalNanos, TimeUnit.NANOSECONDS); + } + } + + private void sendHeartbeat() { + synchronized (heartbeatLock) { + heartbeatTask = null; + if (!heartbeatRunning || isClosed.get()) { + return; + } + } + CompletableFuture heartbeat; + try { + heartbeat = send(CommandCode.System.PING.getValue(), Unpooled.EMPTY_BUFFER); + } catch (RuntimeException error) { + log.warn("Failed to send heartbeat: {}", error.getMessage()); + scheduleNextHeartbeat(); + return; + } + heartbeat.whenComplete((response, error) -> { + if (response != null) { + response.release(); + } + if (error != null && !isClosed.get()) { + log.warn("Heartbeat failed: {}", error.getMessage()); + } + scheduleNextHeartbeat(); + }); + } + + private void stopHeartbeat() { + synchronized (heartbeatLock) { + heartbeatRunning = false; + if (heartbeatTask != null) { + heartbeatTask.cancel(false); + heartbeatTask = null; + } + } + } + public CompletableFuture exchangeForEntity( CommandCode commandCode, ByteBuf payload, Function func) { return send(commandCode, payload).thenApply(response -> { @@ -193,93 +325,341 @@ public CompletableFuture send(CommandCode commandCode, ByteBuf payload) } public CompletableFuture send(int commandCode, ByteBuf payload) { + return send(commandCode, payload, 0); + } + + CompletableFuture send(int commandCode, ByteBuf payload, long requestDeadlineNanos) { + if (isLoginCode(commandCode) && authenticated) { + return logoutThenLogin(commandCode, payload); + } captureLoginPayloadIfNeeded(commandCode, payload); CompletableFuture responseFuture = new CompletableFuture<>(); CompletableFuture callerFuture = new CompletableFuture<>(); + ByteBuf failoverPayload = + transientFailoverHandler != null && !isLoginCode(commandCode) ? payload.retainedDuplicate() : null; channelPool.acquire().addListener((FutureListener) f -> { if (!f.isSuccess()) { payload.release(); + releaseIfPresent(failoverPayload); + notifyConnectionFailure(f.cause()); callerFuture.completeExceptionally(mapAcquireException(f.cause())); return; } + dispatchAcquiredChannel( + f.getNow(), + commandCode, + payload, + failoverPayload, + responseFuture, + callerFuture, + requestDeadlineNanos); + }); - Channel channel = f.getNow(); - boolean isLoginCommand = (commandCode == CommandCode.User.LOGIN.getValue() - || commandCode == CommandCode.PersonalAccessToken.LOGIN.getValue()); - boolean requiresAuth = !isLoginCommand - && commandCode != CommandCode.System.PING.getValue() - && commandCode != CommandCode.System.GET_STATS.getValue(); + return callerFuture; + } - responseFuture.whenComplete((response, error) -> { - try { - handlePostResponse(channel, commandCode, isLoginCommand, error); - } catch (RuntimeException bookkeepingError) { - log.error("Post-response bookkeeping failed: {}", bookkeepingError.getMessage()); - } - if (error != null) { - callerFuture.completeExceptionally(error); - } else { - callerFuture.complete(response); - } - }); + @SuppressWarnings("checkstyle:ParameterNumber") + private void dispatchAcquiredChannel( + Channel channel, + int commandCode, + ByteBuf payload, + ByteBuf failoverPayload, + CompletableFuture responseFuture, + CompletableFuture callerFuture, + long requestDeadlineNanos) { + Runnable dispatch = () -> dispatchOnChannel( + channel, commandCode, payload, failoverPayload, responseFuture, callerFuture, requestDeadlineNanos); + if (channel.eventLoop().inEventLoop()) { + dispatch.run(); + return; + } + try { + channel.eventLoop().execute(dispatch); + } catch (RejectedExecutionException error) { + payload.release(); + releaseIfPresent(failoverPayload); + releaseChannel(channel); + callerFuture.completeExceptionally(error); + } + } - CompletableFuture authStep; - if (!requiresAuth) { - authStep = CompletableFuture.completedFuture(null); - } else if (!authenticated) { + private void dispatchOnChannel( + Channel channel, + int commandCode, + ByteBuf payload, + ByteBuf failoverPayload, + CompletableFuture responseFuture, + CompletableFuture callerFuture, + long inheritedRequestDeadlineNanos) { + boolean isLoginCommand = isLoginCode(commandCode); + boolean holdLeaseUntilResponse = mutatesSessionState(commandCode); + long requestDeadlineNanos = inheritedRequestDeadlineNanos == 0 + ? System.nanoTime() + requestTimeoutNanos + : inheritedRequestDeadlineNanos; + + responseFuture.whenComplete((response, error) -> completeResponse( + channel, + commandCode, + isLoginCommand, + failoverPayload, + requestDeadlineNanos, + holdLeaseUntilResponse, + callerFuture, + response, + error)); + authenticationStep(channel, commandCode, requestDeadlineNanos) + .whenComplete((ignored, authError) -> completeAuthenticationStep( + channel, + commandCode, + payload, + responseFuture, + requestDeadlineNanos, + holdLeaseUntilResponse, + authError)); + } + + private CompletableFuture authenticationStep(Channel channel, int commandCode, long requestDeadlineNanos) { + if (isLoginCode(commandCode) || !requiresAuthentication(commandCode)) { + return CompletableFuture.completedFuture(null); + } + if (!authenticated) { + return CompletableFuture.failedFuture(new IggyNotConnectedException("Not authenticated, call login first")); + } + ByteBuf loginPayloadCopy = getLoginPayloadCopy(); + if (loginPayloadCopy == null) { + return CompletableFuture.failedFuture(new IggyNotConnectedException("Not authenticated, call login first")); + } + return IggyAuthenticator.ensureAuthenticated( + channel, + loginPayloadCopy, + authGeneration, + payloadToLogin -> + sendAuthenticationFrame(channel, payloadToLogin, loginCommandCode, requestDeadlineNanos)); + } + + @SuppressWarnings("checkstyle:ParameterNumber") + private void completeAuthenticationStep( + Channel channel, + int commandCode, + ByteBuf payload, + CompletableFuture responseFuture, + long requestDeadlineNanos, + boolean holdLeaseUntilResponse, + Throwable authError) { + try { + if (authError != null) { payload.release(); - responseFuture.completeExceptionally( - new IggyNotConnectedException("Not authenticated, call login first")); + responseFuture.completeExceptionally(authError); return; - } else { - ByteBuf loginPayloadCopy = getLoginPayloadCopy(); - if (loginPayloadCopy == null) { - payload.release(); - responseFuture.completeExceptionally( - new IggyNotConnectedException("Not authenticated, call login first")); - return; - } - authStep = IggyAuthenticator.ensureAuthenticated( - channel, loginPayloadCopy, loginCommandCode, authGeneration); } + sendFrame(channel, payload, commandCode, responseFuture, requestDeadlineNanos); + } finally { + if (!holdLeaseUntilResponse) { + releaseChannel(channel); + } + } + } + + @SuppressWarnings("checkstyle:ParameterNumber") + private void completeResponse( + Channel channel, + int commandCode, + boolean isLoginCommand, + ByteBuf failoverPayload, + long requestDeadlineNanos, + boolean holdLeaseUntilResponse, + CompletableFuture callerFuture, + ByteBuf response, + Throwable error) { + try { + completeRequest( + channel, + commandCode, + isLoginCommand, + failoverPayload, + requestDeadlineNanos, + callerFuture, + response, + error); + } finally { + if (holdLeaseUntilResponse) { + releaseChannel(channel); + } + } + } - authStep.thenRun(() -> sendFrame(channel, payload, commandCode, responseFuture)) - .exceptionally(ex -> { - responseFuture.completeExceptionally(ex); - return null; - }); + @SuppressWarnings("checkstyle:ParameterNumber") + private void completeRequest( + Channel channel, + int commandCode, + boolean isLoginCommand, + ByteBuf failoverPayload, + long requestDeadlineNanos, + CompletableFuture callerFuture, + ByteBuf response, + Throwable error) { + try { + handlePostResponse(channel, commandCode, isLoginCommand, error); + } catch (RuntimeException bookkeepingError) { + log.error("Post-response bookkeeping failed: {}", bookkeepingError.getMessage()); + } + if (error == null) { + releaseIfPresent(failoverPayload); + completeWithResponse(callerFuture, response); + return; + } + completeFailedRequest(commandCode, failoverPayload, requestDeadlineNanos, callerFuture, error); + } + + private void completeFailedRequest( + int commandCode, + ByteBuf failoverPayload, + long requestDeadlineNanos, + CompletableFuture callerFuture, + Throwable error) { + IggyTimeoutException timeout = findResponseTimeout(error); + if (timeout != null) { + notifyConnectionFailure(timeout); + } + IggyServerException serverError = findServerError(error); + if (shouldRecheckLeader(serverError, failoverPayload, requestDeadlineNanos)) { + retryAfterLeaderRecheck(commandCode, failoverPayload, requestDeadlineNanos, serverError, callerFuture); + return; + } + releaseIfPresent(failoverPayload); + callerFuture.completeExceptionally(error); + } + + private static boolean shouldRecheckLeader( + IggyServerException serverError, ByteBuf failoverPayload, long requestDeadlineNanos) { + return serverError != null + && serverError.getRawErrorCode() == TRANSIENT_NOT_ACCEPTED + && failoverPayload != null + && requestDeadlineNanos - System.nanoTime() > 0; + } + + private void retryAfterLeaderRecheck( + int commandCode, + ByteBuf payload, + long requestDeadlineNanos, + IggyServerException rejection, + CompletableFuture callerFuture) { + CompletableFuture retry; + try { + retry = transientFailoverHandler.retry(this, commandCode, payload, requestDeadlineNanos, rejection); + } catch (RuntimeException retryError) { + payload.release(); + callerFuture.completeExceptionally(retryError); + return; + } + retry.whenComplete((response, error) -> { + if (error != null) { + callerFuture.completeExceptionally(error); + } else { + completeWithResponse(callerFuture, response); + } }); + } - return callerFuture; + private CompletableFuture sendAuthenticationFrame( + Channel channel, ByteBuf payload, int commandCode, long requestDeadlineNanos) { + CompletableFuture loginFuture = new CompletableFuture<>(); + sendFrame(channel, payload, commandCode, loginFuture, requestDeadlineNanos); + return loginFuture; + } + + /** + * Pool acquire failures and expired replies both make the current target + * unusable. The listener lets the owning client run its redial strategy + * while the failed request surfaces to its caller. + */ + private void notifyConnectionFailure(Throwable cause) { + try { + connectionFailureListener.accept(cause); + } catch (RuntimeException listenerError) { + log.warn("Connection failure listener threw: {}", listenerError.getMessage()); + } } private static Throwable mapAcquireException(Throwable cause) { if (cause instanceof IllegalStateException) { return new IggyNotConnectedException("Connection pool is closed"); } + if (cause instanceof TimeoutException) { + return new IggyTimeoutException("Timed out acquiring a connection from the pool", cause); + } return cause; } + /** + * A Register on an already-bound VSR connection is answered with a replay + * of the original register reply, while the client has re-armed a fresh + * identity; its reset request counter would then collide with the + * server's dedup table and mutations would be silently swallowed. Unbind + * first, then login fresh. + */ + private CompletableFuture logoutThenLogin(int commandCode, ByteBuf payload) { + return send(CommandCode.User.LOGOUT.getValue(), Unpooled.EMPTY_BUFFER) + .handle((logoutResponse, logoutError) -> { + if (logoutResponse != null) { + logoutResponse.release(); + } + return null; + }) + .thenCompose(ignored -> send(commandCode, payload)); + } + + private static boolean isLoginCode(int commandCode) { + return commandCode == CommandCode.User.LOGIN.getValue() + || commandCode == CommandCode.PersonalAccessToken.LOGIN.getValue(); + } + + private static boolean mutatesSessionState(int commandCode) { + return isLoginCode(commandCode) || commandCode == CommandCode.User.LOGOUT.getValue(); + } + + /** + * Ping and cluster metadata are the only sessionless bootstrap commands. + * Cluster metadata must be available before Register so a VSR client can + * select the leader; every other non-login command requires a bound + * session. + */ + private static boolean requiresAuthentication(int commandCode) { + return !isAllowedBeforeAuthentication(commandCode); + } + + private static boolean isAllowedBeforeAuthentication(int commandCode) { + return commandCode == CommandCode.System.PING.getValue() + || commandCode == CommandCode.System.GET_CLUSTER_METADATA.getValue(); + } + private void sendFrame( - Channel channel, ByteBuf payload, int commandCode, CompletableFuture responseFuture) { + Channel channel, + ByteBuf payload, + int commandCode, + CompletableFuture responseFuture, + long requestDeadlineNanos) { try { - IggyResponseHandler handler = channel.pipeline().get(IggyResponseHandler.class); + VsrResponseHandler handler = channel.pipeline().get(VsrResponseHandler.class); if (handler == null) { - throw new IggyClientException("Channel missing IggyResponseHandler"); + throw new IggyClientException("Channel missing VsrResponseHandler"); } - handler.enqueueRequest(responseFuture); - ByteBuf frame = IggyFrameEncoder.encode(channel.alloc(), commandCode, payload); - - channel.writeAndFlush(frame).addListener((ChannelFutureListener) future -> { - if (!future.isSuccess()) { - log.error("Failed to send frame: {}", future.cause().getMessage()); - responseFuture.completeExceptionally(future.cause()); - } else { - log.trace("Frame sent successfully to {}", channel.remoteAddress()); - } - }); + ByteBuf frame = vsrEncoder.encode(channel.alloc(), commandCode, payload); + long nowNanos = System.nanoTime(); + long deadlineNanos = nowNanos + TRANSIENT_RETRY_BUDGET.toNanos(); + long notAcceptedDeadlineNanos = + isLoginCode(commandCode) ? deadlineNanos : nowNanos + NOT_ACCEPTED_RETRY_BUDGET.toNanos(); + writeVsrFrame( + channel, + handler, + frame, + responseFuture, + requestDeadlineNanos, + deadlineNanos, + notAcceptedDeadlineNanos, + commandCode); } catch (RuntimeException e) { responseFuture.completeExceptionally(e); } finally { @@ -287,6 +667,145 @@ private void sendFrame( } } + /** + * One VSR write attempt. A transient denial (the cluster could not commit + * or accept yet) replays the SAME encoded frame so the server's dedup + * sees the same request id; everything else resolves the caller. + */ + @SuppressWarnings("checkstyle:ParameterNumber") + private void writeVsrFrame( + Channel channel, + VsrResponseHandler handler, + ByteBuf frame, + CompletableFuture responseFuture, + long requestDeadlineNanos, + long deadlineNanos, + long notAcceptedDeadlineNanos, + int commandCode) { + if (requestDeadlineNanos - System.nanoTime() <= 0) { + IggyTimeoutException timeout = responseTimeout(commandCode); + handler.closeChannel(channel, timeout); + frame.release(); + responseFuture.completeExceptionally(timeout); + return; + } + CompletableFuture attempt = new CompletableFuture<>(); + try { + handler.registerRequest(channel, frame, attempt, requestDeadlineNanos, commandCode); + } catch (RuntimeException error) { + handler.closeChannel(channel, error); + frame.release(); + responseFuture.completeExceptionally(error); + return; + } + channel.writeAndFlush(frame.retainedDuplicate()).addListener((ChannelFutureListener) future -> { + if (!future.isSuccess()) { + log.error("Failed to send frame: {}", future.cause().getMessage()); + // A failed write leaves framing undefined. Closing removes and + // fails every pending entry before the channel can be reused. + handler.closeChannel(channel, future.cause()); + } + }); + attempt.whenComplete((response, error) -> { + if (shouldRetryTransient(error, deadlineNanos, notAcceptedDeadlineNanos) && channel.isActive()) { + try { + channel.eventLoop() + .schedule( + () -> writeVsrFrame( + channel, + handler, + frame, + responseFuture, + requestDeadlineNanos, + deadlineNanos, + notAcceptedDeadlineNanos, + commandCode), + TRANSIENT_RETRY_INTERVAL_MS, + TimeUnit.MILLISECONDS); + return; + } catch (RejectedExecutionException retryRejected) { + log.warn("Event loop rejected a VSR retry, failing the request: {}", retryRejected.getMessage()); + } + } + frame.release(); + if (error != null) { + responseFuture.completeExceptionally(error); + } else { + responseFuture.complete(response); + } + }); + } + + private static IggyTimeoutException responseTimeout(int commandCode) { + return new IggyTimeoutException("Timed out waiting for a response to command code " + commandCode); + } + + private static IggyTimeoutException findResponseTimeout(Throwable error) { + Throwable cause = error; + while (cause != null) { + if (cause instanceof IggyTimeoutException timeout) { + return timeout; + } + cause = cause.getCause(); + } + return null; + } + + private static IggyServerException findServerError(Throwable error) { + Throwable cause = error; + while (cause != null) { + if (cause instanceof IggyServerException serverError) { + return serverError; + } + cause = cause.getCause(); + } + return null; + } + + private static void releaseIfPresent(ByteBuf payload) { + if (payload != null) { + payload.release(); + } + } + + private static void completeWithResponse(CompletableFuture future, ByteBuf response) { + if (!future.complete(response)) { + response.release(); + } + } + + private void releaseChannel(Channel channel) { + channelPool.release(channel).addListener(future -> { + if (!future.isSuccess()) { + log.warn( + "Failed to release VSR channel lease: {}", + future.cause().getMessage()); + channel.close(); + } + }); + } + + private static long toTimeoutNanos(Duration timeout) { + try { + return timeout.toNanos(); + } catch (ArithmeticException ignored) { + return Long.MAX_VALUE; + } + } + + private static boolean shouldRetryTransient(Throwable error, long deadlineNanos, long notAcceptedDeadlineNanos) { + if (!(error instanceof IggyServerException serverError)) { + return false; + } + if (serverError.getRawErrorCode() == TRANSIENT_NOT_COMMITTED) { + return System.nanoTime() < deadlineNanos; + } + if (serverError.getRawErrorCode() == TRANSIENT_NOT_ACCEPTED) { + return System.nanoTime() < notAcceptedDeadlineNanos; + } + return false; + } + private void handlePostResponse(Channel channel, int commandCode, boolean isLoginOp, Throwable ex) { if (isLoginOp) { if (ex == null) { @@ -302,12 +821,21 @@ private void handlePostResponse(Channel channel, int commandCode, boolean isLogi authGeneration.incrementAndGet(); IggyAuthenticator.clearAuthGeneration(channel); } - channelPool.release(channel); + } + + /** + * A server-side eviction unbinds the transport session and closes its + * channel. Bumping the generation makes the replacement channel re-run + * login and Register. The fresh session invalidates cached routing state + * such as consumer-group assignments. + */ + private void onSessionEvicted() { + authGeneration.incrementAndGet(); + sessionResetListener.run(); } private void captureLoginPayloadIfNeeded(int commandCode, ByteBuf payload) { - if (commandCode == CommandCode.User.LOGIN.getValue() - || commandCode == CommandCode.PersonalAccessToken.LOGIN.getValue()) { + if (isLoginCode(commandCode)) { updateLoginPayload(commandCode, payload); } } @@ -327,6 +855,13 @@ private synchronized ByteBuf getLoginPayloadCopy() { return null; } + synchronized Optional authenticationSnapshot() { + if (!authenticated || loginPayload == null) { + return Optional.empty(); + } + return Optional.of(new AuthenticationSnapshot(loginCommandCode, loginPayload.retainedDuplicate())); + } + private synchronized void releaseLoginPayload() { if (this.loginPayload != null) { loginPayload.release(); @@ -338,6 +873,7 @@ public CompletableFuture close() { if (!isClosed.compareAndSet(false, true)) { return CompletableFuture.completedFuture(null); } + stopHeartbeat(); releaseLoginPayload(); CompletableFuture shutdownFuture = new CompletableFuture<>(); channelPool @@ -357,12 +893,25 @@ private static final class PoolChannelHandler extends AbstractChannelPoolHandler private final int port; private final boolean enableTls; private final SslContext sslContext; - - PoolChannelHandler(String host, int port, boolean enableTls, SslContext sslContext) { + private final ConsensusSession consensusSession; + private final int maxVsrFrameSize; + private final Runnable onEviction; + + PoolChannelHandler( + String host, + int port, + boolean enableTls, + SslContext sslContext, + ConsensusSession consensusSession, + int maxVsrFrameSize, + Runnable onEviction) { this.host = host; this.port = port; this.enableTls = enableTls; this.sslContext = sslContext; + this.consensusSession = consensusSession; + this.maxVsrFrameSize = maxVsrFrameSize; + this.onEviction = onEviction; } @Override @@ -371,74 +920,34 @@ public void channelCreated(Channel ch) { if (enableTls) { pipeline.addLast("ssl", sslContext.newHandler(ch.alloc(), host, port)); } - pipeline.addLast("frameDecoder", new IggyFrameDecoder()); - pipeline.addLast("responseHandler", new IggyResponseHandler()); + pipeline.addLast("frameDecoder", new VsrFrameDecoder(maxVsrFrameSize)); + pipeline.addLast("responseHandler", new VsrResponseHandler(consensusSession, onEviction)); } } - public static class IggyResponseHandler extends SimpleChannelInboundHandler { - private final Queue> responseQueue = new ConcurrentLinkedQueue<>(); - - public void enqueueRequest(CompletableFuture future) { - responseQueue.add(future); - } + record AuthenticationSnapshot(int commandCode, ByteBuf payload) {} - @Override - protected void channelRead0(ChannelHandlerContext ctx, ByteBuf msg) { - int status = msg.readIntLE(); - int length = msg.readIntLE(); - - CompletableFuture future = responseQueue.poll(); - - if (future != null) { - if (status == 0) { - future.complete(msg.retainedSlice()); - } else { - byte[] errorBytes = length > 0 ? new byte[length] : new byte[0]; - msg.readBytes(errorBytes); - future.completeExceptionally(IggyServerException.fromTcpResponse(status, errorBytes)); - } - } else { - log.error( - "Received response on channel {} but no request was waiting!", - ctx.channel().id()); - } - } - - @Override - public void channelInactive(ChannelHandlerContext ctx) { - failPendingRequests(new IggyConnectionException("Connection closed before a response arrived")); - ctx.fireChannelInactive(); - } - - @Override - public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { - failPendingRequests(cause); - ctx.close(); - } - - private void failPendingRequests(Throwable cause) { - CompletableFuture pending; - while ((pending = responseQueue.poll()) != null) { - pending.completeExceptionally(cause); - } - } + @FunctionalInterface + interface TransientFailoverHandler { + CompletableFuture retry( + AsyncTcpConnection source, + int commandCode, + ByteBuf payload, + long requestDeadlineNanos, + IggyServerException rejection); } public static class TcpConnectionPoolConfig { - private final int maxConnections; private final int maxPendingAcquires; private final long acquireTimeoutMillis; public TcpConnectionPoolConfig() { this( - TcpConnectionPoolConfigBuilder.DEFAULT_MAX_CONNECTION, TcpConnectionPoolConfigBuilder.DEFAULT_MAX_PENDING_ACQUIRES, TcpConnectionPoolConfigBuilder.DEFAULT_ACQUIRE_TIMEOUT_MILLIS); } - public TcpConnectionPoolConfig(int maxConnections, int maxPendingAcquires, long acquireTimeoutMillis) { - this.maxConnections = maxConnections; + public TcpConnectionPoolConfig(int maxPendingAcquires, long acquireTimeoutMillis) { this.maxPendingAcquires = maxPendingAcquires; this.acquireTimeoutMillis = acquireTimeoutMillis; } @@ -447,10 +956,6 @@ public static TcpConnectionPoolConfigBuilder builder() { return new TcpConnectionPoolConfigBuilder(); } - public int getMaxConnections() { - return this.maxConnections; - } - public int getMaxPendingAcquires() { return this.maxPendingAcquires; } @@ -460,24 +965,14 @@ public long getAcquireTimeoutMillis() { } public static final class TcpConnectionPoolConfigBuilder { - public static final int DEFAULT_MAX_CONNECTION = 5; public static final int DEFAULT_MAX_PENDING_ACQUIRES = 1000; public static final int DEFAULT_ACQUIRE_TIMEOUT_MILLIS = 3000; - private int maxConnections; private int maxPendingAcquires; private long acquireTimeoutMillis; public TcpConnectionPoolConfigBuilder() {} - public TcpConnectionPoolConfigBuilder setMaxConnections(int maxConnections) { - if (maxConnections <= 0) { - throw new IggyInvalidArgumentException("Connection pool size cannot be 0 or negative"); - } - this.maxConnections = maxConnections; - return this; - } - public TcpConnectionPoolConfigBuilder setMaxPendingAcquires(int maxPendingAcquires) { if (maxPendingAcquires <= 0) { throw new IggyInvalidArgumentException("Max Pending Acquires cannot be 0 or negative"); @@ -495,16 +990,13 @@ public TcpConnectionPoolConfigBuilder setAcquireTimeoutMillis(long acquireTimeou } public TcpConnectionPoolConfig build() { - if (this.maxConnections == 0) { - this.maxConnections = DEFAULT_MAX_CONNECTION; - } if (this.acquireTimeoutMillis == 0) { this.acquireTimeoutMillis = DEFAULT_ACQUIRE_TIMEOUT_MILLIS; } if (this.maxPendingAcquires == 0) { this.maxPendingAcquires = DEFAULT_MAX_PENDING_ACQUIRES; } - return new TcpConnectionPoolConfig(maxConnections, maxPendingAcquires, acquireTimeoutMillis); + return new TcpConnectionPoolConfig(maxPendingAcquires, acquireTimeoutMillis); } } } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ClientRoutingState.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ClientRoutingState.java new file mode 100644 index 0000000000..ef28136d91 --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ClientRoutingState.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.client.async.tcp; + +import org.apache.iggy.identifier.ConsumerId; +import org.apache.iggy.identifier.Identifier; +import org.apache.iggy.identifier.StreamId; +import org.apache.iggy.identifier.TopicId; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Per-client cache of the routing facts needed to resolve partitioning and + * consumer-group polling client-side: topic partition counts, balanced + * round-robin cursors, and consumer-group assignments. Typed keys retain both + * the kind and value of every identifier, so numeric identifiers cannot + * collide with same-text named identifiers. + * + *

Partition counts and balanced cursors survive reconnects; group + * assignments are bound to the server-side VSR session (the member is keyed + * by the connection's client id) and must be cleared whenever that session is + * reset or the client moves to another node. Partition counts carry their + * fetch timestamp so callers can refresh them past a staleness budget — a + * count cached forever would keep hashing keys with the wrong modulus after + * a partition-count change (the Rust SDK still has that gap). + */ +final class ClientRoutingState { + + private final Map partitionCounts = new ConcurrentHashMap<>(); + private final Map balancedCursors = new ConcurrentHashMap<>(); + private final Map assignments = new ConcurrentHashMap<>(); + + static TopicKey topicKey(StreamId streamId, TopicId topicId) { + return new TopicKey(IdentifierKey.from(streamId), IdentifierKey.from(topicId)); + } + + static GroupKey groupKey(StreamId streamId, TopicId topicId, ConsumerId groupId) { + return new GroupKey(topicKey(streamId, topicId), IdentifierKey.from(groupId)); + } + + Optional partitionCount(TopicKey topicKey) { + return Optional.ofNullable(partitionCounts.get(topicKey)); + } + + void setPartitionCount(TopicKey topicKey, long count, long fetchedAtNanos) { + partitionCounts.put(topicKey, new CachedPartitionCount(count, fetchedAtNanos)); + } + + /** + * Drops the cached partition count for a topic. Called when a send that + * was routed with the cached count is refused with a not-found error, + * meaning the topic shrank or was recreated and the count is stale. + */ + void invalidatePartitionCount(TopicKey topicKey) { + partitionCounts.remove(topicKey); + } + + /** + * The next balanced produce partition for a topic, advancing the cursor. + * A per-topic monotonic counter modulo the partition count, so three + * partitions yield 0, 1, 2, 0, ... + */ + long nextBalancedPartition(TopicKey topicKey, long partitionCount) { + if (partitionCount <= 0) { + return 0; + } + var cursor = balancedCursors.computeIfAbsent(topicKey, ignored -> new AtomicInteger()); + return Integer.toUnsignedLong(cursor.getAndIncrement()) % partitionCount; + } + + /** + * Replaces the cached assignment for a group. A generation change (a + * rebalance) resets the round-robin cursor so selection restarts cleanly; + * an unchanged generation keeps the cursor so an assignment refresh does + * not disturb the polling rotation. + */ + void setAssignment(GroupKey groupKey, long generation, List partitions, long syncedAtNanos) { + assignments.compute(groupKey, (ignored, existing) -> { + var cursor = existing != null && existing.generation() == generation ? existing.cursor() : 0; + return new GroupAssignment(generation, List.copyOf(partitions), cursor, syncedAtNanos); + }); + } + + /** + * The next assigned partition to poll for a group, advancing the + * round-robin cursor. Empty when the group has no cached assignment or + * the member currently owns no partitions. + */ + OptionalLong nextGroupPartition(GroupKey groupKey) { + var next = new long[] {-1}; + assignments.computeIfPresent(groupKey, (ignored, assignment) -> { + if (assignment.partitions().isEmpty()) { + return assignment; + } + var index = + Math.floorMod(assignment.cursor(), assignment.partitions().size()); + next[0] = assignment.partitions().get(index); + return new GroupAssignment( + assignment.generation(), + assignment.partitions(), + assignment.cursor() + 1, + assignment.syncedAtNanos()); + }); + return next[0] < 0 ? OptionalLong.empty() : OptionalLong.of(next[0]); + } + + Optional assignment(GroupKey groupKey) { + return Optional.ofNullable(assignments.get(groupKey)); + } + + void invalidateAssignment(GroupKey groupKey) { + assignments.remove(groupKey); + } + + /** + * Drops every cached group assignment. Called when the VSR session is + * reset or the client retargets to another node, since the server keys + * group membership by the session's client id. + */ + void clearAssignments() { + assignments.clear(); + } + + record IdentifierKey(int kind, Long id, String name) { + static IdentifierKey from(Identifier identifier) { + return new IdentifierKey(identifier.getKind(), identifier.getId(), identifier.getName()); + } + } + + record TopicKey(IdentifierKey stream, IdentifierKey topic) {} + + record GroupKey(TopicKey topic, IdentifierKey consumer) {} + + record GroupAssignment(long generation, List partitions, int cursor, long syncedAtNanos) {} + + record CachedPartitionCount(long count, long fetchedAtNanos) {} +} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ConsumerGroupsTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ConsumerGroupsTcpClient.java index 9b4561fc64..786bcc766c 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ConsumerGroupsTcpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ConsumerGroupsTcpClient.java @@ -22,6 +22,7 @@ import io.netty.buffer.Unpooled; import org.apache.iggy.client.async.ConsumerGroupsClient; import org.apache.iggy.consumergroup.ConsumerGroup; +import org.apache.iggy.consumergroup.ConsumerGroupAssignment; import org.apache.iggy.consumergroup.ConsumerGroupDetails; import org.apache.iggy.identifier.ConsumerId; import org.apache.iggy.identifier.StreamId; @@ -184,4 +185,22 @@ public CompletableFuture leaveConsumerGroup(StreamId streamId, TopicId top response.release(); }); } + + @Override + public CompletableFuture> syncConsumerGroup( + StreamId streamId, TopicId topicId, ConsumerId groupId) { + var payload = Unpooled.buffer(); + payload.writeBytes(BytesSerializer.toBytes(streamId)); + payload.writeBytes(BytesSerializer.toBytes(topicId)); + payload.writeBytes(BytesSerializer.toBytes(groupId)); + + log.debug("Syncing consumer group assignment - Stream: {}, Topic: {}, Group: {}", streamId, topicId, groupId); + + // An empty body means "not a member", which exchangeForOptional maps + // to an empty Optional; a member owning zero partitions still gets a + // non-empty body with a zero partition count. + return connection() + .exchangeForOptional( + CommandCode.ConsumerGroup.SYNC, payload, BytesDeserializer::readConsumerGroupAssignment); + } } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/IggyAuthenticator.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/IggyAuthenticator.java index 71bfcdb60c..c54d7c43e7 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/IggyAuthenticator.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/IggyAuthenticator.java @@ -21,15 +21,13 @@ import io.netty.buffer.ByteBuf; import io.netty.channel.Channel; -import io.netty.channel.ChannelFutureListener; import io.netty.util.AttributeKey; -import org.apache.iggy.client.async.tcp.AsyncTcpConnection.IggyResponseHandler; -import org.apache.iggy.exception.IggyNotConnectedException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.concurrent.CompletableFuture; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Function; final class IggyAuthenticator { private static final Logger log = LoggerFactory.getLogger(IggyAuthenticator.class); @@ -44,12 +42,15 @@ private IggyAuthenticator() {} * * @param channel the channel to authenticate * @param loginPayload the login payload to send (will be released by this method) - * @param commandCode the login command code * @param currentGeneration the current authentication generation counter + * @param login sends the login payload through the connection's retry path * @return a future that completes when authentication is done */ static CompletableFuture ensureAuthenticated( - Channel channel, ByteBuf loginPayload, int commandCode, AtomicLong currentGeneration) { + Channel channel, + ByteBuf loginPayload, + AtomicLong currentGeneration, + Function> login) { Long channelGeneration = channel.attr(AUTH_GENERATION_KEY).get(); long requiredGeneration = currentGeneration.get(); @@ -58,21 +59,14 @@ static CompletableFuture ensureAuthenticated( return CompletableFuture.completedFuture(null); } - if (loginPayload == null) { - return CompletableFuture.failedFuture(new IggyNotConnectedException("Not authenticated, call login first")); + CompletableFuture loginFuture; + try { + loginFuture = login.apply(loginPayload); + } catch (RuntimeException loginError) { + loginPayload.release(); + return CompletableFuture.failedFuture(loginError); } - CompletableFuture loginFuture = new CompletableFuture<>(); - IggyResponseHandler handler = channel.pipeline().get(IggyResponseHandler.class); - handler.enqueueRequest(loginFuture); - ByteBuf frame = IggyFrameEncoder.encode(channel.alloc(), commandCode, loginPayload); - loginPayload.release(); - channel.writeAndFlush(frame).addListener((ChannelFutureListener) f -> { - if (!f.isSuccess()) { - loginFuture.completeExceptionally(f.cause()); - } - }); - return loginFuture.thenAccept(result -> { try { channel.attr(AUTH_GENERATION_KEY).set(currentGeneration.get()); diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/IggyFrameDecoder.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/IggyFrameDecoder.java deleted file mode 100644 index 3ea104b42f..0000000000 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/IggyFrameDecoder.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iggy.client.async.tcp; - -import io.netty.buffer.ByteBuf; -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.ByteToMessageDecoder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.List; - -/** - * Decoder for Iggy protocol responses. - * Response format: [4-byte status LE] [4-byte length LE] [payload] - */ -public class IggyFrameDecoder extends ByteToMessageDecoder { - private static final Logger log = LoggerFactory.getLogger(IggyFrameDecoder.class); - private static final int HEADER_SIZE = 8; // status (4) + length (4) - - @Override - protected void decode(ChannelHandlerContext ctx, ByteBuf in, List out) { - // Wait until we have at least the header - if (in.readableBytes() < HEADER_SIZE) { - return; - } - - // Mark the current reader index - in.markReaderIndex(); - - // Read status and length - int status = in.readIntLE(); - int length = in.readIntLE(); - - log.trace("Received response with status={}, length={}", status, length); - - // Check if we have the complete payload - if (in.readableBytes() < length) { - // Not enough data, reset and wait for more - in.resetReaderIndex(); - return; - } - - // Create a new buffer with the complete response - ByteBuf response = ctx.alloc().buffer(HEADER_SIZE + length); - response.writeIntLE(status); - response.writeIntLE(length); - - if (length > 0) { - response.writeBytes(in, length); - } - - log.trace("Decoded complete response, forwarding to handler"); - out.add(response); - } -} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/IggyFrameEncoder.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/IggyFrameEncoder.java deleted file mode 100644 index 87f0eb479c..0000000000 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/IggyFrameEncoder.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iggy.client.async.tcp; - -import io.netty.buffer.ByteBuf; -import io.netty.buffer.ByteBufAllocator; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -final class IggyFrameEncoder { - private static final Logger log = LoggerFactory.getLogger(IggyFrameEncoder.class); - - private IggyFrameEncoder() {} - - /** - * Encodes a command into the Iggy TCP frame format: [payload_size:4][command:4][payload:N] - */ - static ByteBuf encode(ByteBufAllocator alloc, int commandCode, ByteBuf payload) { - int payloadSize = payload.readableBytes(); - int framePayloadSize = 4 + payloadSize; - ByteBuf frame = alloc.buffer(4 + framePayloadSize); - frame.writeIntLE(framePayloadSize); - frame.writeIntLE(commandCode); - frame.writeBytes(payload); - - if (log.isTraceEnabled()) { - byte[] frameBytes = new byte[Math.min(frame.readableBytes(), 30)]; - frame.getBytes(0, frameBytes); - StringBuilder hex = new StringBuilder(); - for (byte b : frameBytes) { - hex.append(String.format("%02x ", b)); - } - log.trace( - "Sending frame with command: {}, payload size: {}, frame payload size (with command): {}, total frame size: {}", - commandCode, - payloadSize, - framePayloadSize, - frame.readableBytes()); - log.trace("Frame bytes (hex): {}", hex); - } - - return frame; - } -} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/LoginRedirectionHook.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/LoginRoutingHook.java similarity index 56% rename from foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/LoginRedirectionHook.java rename to foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/LoginRoutingHook.java index d1c269f9ef..a6acb7dc09 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/LoginRedirectionHook.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/LoginRoutingHook.java @@ -25,23 +25,20 @@ import java.util.function.Supplier; /** - * Runs after a successful login to redirect the client to the cluster - * leader when it is connected to a follower. + * Routes a login to the cluster leader before executing its Register + * operation. */ @FunctionalInterface -interface LoginRedirectionHook { +interface LoginRoutingHook { - LoginRedirectionHook NONE = reLogin -> CompletableFuture.completedFuture(null); + LoginRoutingHook NONE = loginAttempt -> loginAttempt.get(); /** - * Checks the cluster roster and, while the current node is not the - * leader, retargets the connection and re-runs the login. + * Discovers and selects the current leader, then executes the supplied + * login exactly once against the selected connection. * - * @param reLogin replays the just-completed login against the current - * target; it must not trigger another redirection check itself, since - * the hook drives any further hops and serializes concurrent checks - * @return the redirected login's identity, or {@code null} when the client - * stays on the current node + * @param loginAttempt sends Register against the active connection + * @return the identity returned by the successful Register response */ - CompletableFuture afterLogin(Supplier> reLogin); + CompletableFuture loginOnLeader(Supplier> loginAttempt); } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/MessagesTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/MessagesTcpClient.java index 5cb2d12fc5..6873c670fa 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/MessagesTcpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/MessagesTcpClient.java @@ -20,20 +20,34 @@ package org.apache.iggy.client.async.tcp; import io.netty.buffer.Unpooled; +import org.apache.iggy.client.async.ConsumerGroupsClient; import org.apache.iggy.client.async.MessagesClient; +import org.apache.iggy.client.async.TopicsClient; import org.apache.iggy.consumergroup.Consumer; +import org.apache.iggy.exception.IggyErrorCode; +import org.apache.iggy.exception.IggyResourceNotFoundException; +import org.apache.iggy.exception.IggyServerException; +import org.apache.iggy.hash.XxHash32; import org.apache.iggy.identifier.StreamId; import org.apache.iggy.identifier.TopicId; import org.apache.iggy.message.Message; import org.apache.iggy.message.Partitioning; +import org.apache.iggy.message.PartitioningKind; import org.apache.iggy.message.PolledMessages; import org.apache.iggy.message.PollingStrategy; +import org.apache.iggy.message.SendMessagesResponse; import org.apache.iggy.serde.BytesDeserializer; import org.apache.iggy.serde.CommandCode; +import org.apache.iggy.topic.TopicDetails; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.math.BigInteger; +import java.time.Duration; import java.util.List; import java.util.Optional; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; import java.util.function.Supplier; import static org.apache.iggy.serde.BytesSerializer.toBytes; @@ -43,10 +57,41 @@ */ public class MessagesTcpClient implements MessagesClient { + private static final Logger log = LoggerFactory.getLogger(MessagesTcpClient.class); + + /** + * A generation-fenced group poll is answered with an empty poll body + * carrying this sentinel partition id, telling the client to re-sync its + * assignment and retry; mirrors RESYNC_REQUIRED_PARTITION_SENTINEL in + * core/common/src/lib.rs. + */ + private static final long RESYNC_REQUIRED_PARTITION_SENTINEL = 0xFFFF_FFFFL; + + private static final int GROUP_POLL_MAX_ATTEMPTS = 2; + private static final int PARTITION_NOT_OWNED_ERROR_CODE = 5009; + + /** + * Staleness budget for the client-side routing caches: group assignments + * and topic partition counts. Both re-fetch lazily on the next use once + * this old, so a partition-count change or rebalance is picked up without + * a background refresher thread. + */ + private static final Duration ROUTING_CACHE_REFRESH = Duration.ofSeconds(5); + private final Supplier connectionSupplier; + private final ClientRoutingState routingState; + private final TopicsClient topicsClient; + private final ConsumerGroupsClient consumerGroupsClient; public MessagesTcpClient(Supplier connectionSupplier) { + this(connectionSupplier, new ClientRoutingState()); + } + + MessagesTcpClient(Supplier connectionSupplier, ClientRoutingState routingState) { this.connectionSupplier = connectionSupplier; + this.routingState = routingState; + this.topicsClient = new TopicsTcpClient(connectionSupplier); + this.consumerGroupsClient = new ConsumerGroupsTcpClient(connectionSupplier); } private AsyncTcpConnection connection() { @@ -62,6 +107,23 @@ public CompletableFuture pollMessages( PollingStrategy strategy, Long count, boolean autoCommit) { + if (consumer.kind() == Consumer.Kind.ConsumerGroup && partitionId.isEmpty()) { + // The VSR broker fences group polls against unowned partitions + // instead of picking one, so the partition is selected here from + // the member's synced assignment, matching the Rust SDK. + return pollGroupMessages(streamId, topicId, consumer, strategy, count, autoCommit, GROUP_POLL_MAX_ATTEMPTS); + } + return pollPartition(streamId, topicId, partitionId, consumer, strategy, count, autoCommit); + } + + private CompletableFuture pollPartition( + StreamId streamId, + TopicId topicId, + Optional partitionId, + Consumer consumer, + PollingStrategy strategy, + Long count, + boolean autoCommit) { var payload = Unpooled.buffer(); @@ -94,7 +156,29 @@ public CompletableFuture pollMessages( } @Override - public CompletableFuture sendMessages( + public CompletableFuture sendMessages( + StreamId streamId, TopicId topicId, Partitioning partitioning, List messages) { + if (partitioning.kind() == PartitioningKind.PartitionId) { + return sendToPartition(streamId, topicId, partitioning, messages); + } + // The VSR broker routes explicit partitions only, so balanced and + // message-key partitioning resolve to a partition id client-side, + // matching the Rust SDK (round-robin cursor, xxh32(key) % count). + return resolvePartitioning(streamId, topicId, partitioning) + .thenCompose(resolved -> sendToPartition(streamId, topicId, resolved, messages)) + .exceptionallyCompose(error -> { + // A resolved send refused with not-found means the cached + // partition count is stale (the topic shrank or was + // recreated); drop it so the next send re-fetches now + // instead of waiting out the staleness budget. + if (unwrapCompletion(error) instanceof IggyResourceNotFoundException) { + routingState.invalidatePartitionCount(ClientRoutingState.topicKey(streamId, topicId)); + } + return CompletableFuture.failedFuture(error); + }); + } + + private CompletableFuture sendToPartition( StreamId streamId, TopicId topicId, Partitioning partitioning, List messages) { // Build metadata section following the blocking client pattern @@ -127,10 +211,161 @@ public CompletableFuture sendMessages( payload.writeBytes(toBytes(message)); } - // Send async request (no response data expected for send) - return connection().send(CommandCode.Messages.SEND.getValue(), payload).thenAccept(response -> { - // Response received, messages sent successfully - response.release(); // Release the buffer + return connection().send(CommandCode.Messages.SEND.getValue(), payload).thenApply(response -> { + try { + return BytesDeserializer.readSendMessagesResponse(response); + } catch (RuntimeException e) { + // The batch is already committed server-side; failing here would + // trigger a spurious resend, so a malformed confirmation degrades + // to an empty one. + log.warn("Discarding malformed send confirmation: {}", e.getMessage()); + return SendMessagesResponse.empty(); + } finally { + response.release(); + } }); } + + /** + * One group-poll attempt: sync the assignment when missing or stale, pick + * the next assigned partition round-robin, poll it explicitly, and on a + * generation fence (the re-sync sentinel or a partition-not-owned error) + * drop the cached assignment and retry. The attempt budget allows one + * re-sync after the coordinator rejects a stale assignment, then one + * retry; an exhausted budget is an empty poll, not an error. + */ + private CompletableFuture pollGroupMessages( + StreamId streamId, + TopicId topicId, + Consumer consumer, + PollingStrategy strategy, + Long count, + boolean autoCommit, + int attemptsLeft) { + if (attemptsLeft == 0) { + return CompletableFuture.completedFuture(emptyPolledMessages()); + } + var groupKey = ClientRoutingState.groupKey(streamId, topicId, consumer.id()); + return ensureFreshAssignment(streamId, topicId, consumer, groupKey).thenCompose(ignored -> { + var partitionId = routingState.nextGroupPartition(groupKey); + if (partitionId.isEmpty()) { + if (routingState.assignment(groupKey).isPresent()) { + // a member owning no partitions polls nothing + return CompletableFuture.completedFuture(emptyPolledMessages()); + } + return CompletableFuture.failedFuture(new IggyResourceNotFoundException( + IggyErrorCode.CONSUMER_GROUP_NOT_JOINED, + IggyErrorCode.CONSUMER_GROUP_NOT_JOINED.getCode(), + "Cannot poll consumer group " + consumer.id() + " for topic " + topicId + " in stream " + + streamId + ": this client is not a member, join the group first", + Optional.empty(), + Optional.empty())); + } + return pollPartition( + streamId, + topicId, + Optional.of(partitionId.getAsLong()), + consumer, + strategy, + count, + autoCommit) + .thenCompose(polled -> { + if (polled.messages().isEmpty() && polled.partitionId() == RESYNC_REQUIRED_PARTITION_SENTINEL) { + routingState.invalidateAssignment(groupKey); + return pollGroupMessages( + streamId, topicId, consumer, strategy, count, autoCommit, attemptsLeft - 1); + } + return CompletableFuture.completedFuture(polled); + }) + .exceptionallyCompose(error -> { + if (!isPartitionNotOwned(error)) { + return CompletableFuture.failedFuture(error); + } + routingState.invalidateAssignment(groupKey); + return pollGroupMessages( + streamId, topicId, consumer, strategy, count, autoCommit, attemptsLeft - 1); + }); + }); + } + + private CompletableFuture ensureFreshAssignment( + StreamId streamId, TopicId topicId, Consumer consumer, ClientRoutingState.GroupKey groupKey) { + var cached = routingState.assignment(groupKey); + if (cached.isPresent() && System.nanoTime() - cached.get().syncedAtNanos() < ROUTING_CACHE_REFRESH.toNanos()) { + return CompletableFuture.completedFuture(null); + } + return consumerGroupsClient + .syncConsumerGroup(streamId, topicId, consumer.id()) + .thenAccept(assignment -> { + if (assignment.isEmpty()) { + // an empty sync reply means "not a member" + routingState.invalidateAssignment(groupKey); + return; + } + routingState.setAssignment( + groupKey, + assignment.get().generation(), + assignment.get().partitions(), + System.nanoTime()); + }); + } + + private static boolean isPartitionNotOwned(Throwable error) { + return unwrapCompletion(error) instanceof IggyServerException serverError + && serverError.getRawErrorCode() == PARTITION_NOT_OWNED_ERROR_CODE; + } + + private static Throwable unwrapCompletion(Throwable error) { + return error instanceof CompletionException && error.getCause() != null ? error.getCause() : error; + } + + private static PolledMessages emptyPolledMessages() { + return new PolledMessages(0L, BigInteger.ZERO, 0L, List.of()); + } + + private CompletableFuture resolvePartitioning( + StreamId streamId, TopicId topicId, Partitioning partitioning) { + return partitionCount(streamId, topicId).thenApply(partitionsCount -> switch (partitioning.kind()) { + case Balanced -> + Partitioning.partitionId(routingState.nextBalancedPartition( + ClientRoutingState.topicKey(streamId, topicId), partitionsCount)); + case MessagesKey -> Partitioning.partitionId(XxHash32.hashUnsigned(partitioning.value()) % partitionsCount); + case PartitionId -> partitioning; + }); + } + + private CompletableFuture partitionCount(StreamId streamId, TopicId topicId) { + var topicKey = ClientRoutingState.topicKey(streamId, topicId); + var cached = routingState.partitionCount(topicKey); + if (cached.isPresent() && System.nanoTime() - cached.get().fetchedAtNanos() < ROUTING_CACHE_REFRESH.toNanos()) { + return CompletableFuture.completedFuture(cached.get().count()); + } + return topicsClient + .getTopic(streamId, topicId) + .thenApply(topicDetails -> { + var partitionsCount = + topicDetails.map(TopicDetails::partitionsCount).orElse(0L); + if (partitionsCount == 0) { + throw new IggyResourceNotFoundException( + IggyErrorCode.TOPIC_ID_NOT_FOUND, + IggyErrorCode.TOPIC_ID_NOT_FOUND.getCode(), + "Cannot resolve partitioning: topic " + topicId + " in stream " + streamId + + " was not found or has no partitions", + Optional.empty(), + Optional.empty()); + } + routingState.setPartitionCount(topicKey, partitionsCount, System.nanoTime()); + return partitionsCount; + }) + .exceptionallyCompose(error -> { + // A failed refresh should not stop routing while a stale + // count is still on hand, except when the topic itself is + // gone; serving the stale value keeps sends flowing + // through transient metadata-fetch failures. + if (cached.isPresent() && !(unwrapCompletion(error) instanceof IggyResourceNotFoundException)) { + return CompletableFuture.completedFuture(cached.get().count()); + } + return CompletableFuture.failedFuture(error); + }); + } } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/PersonalAccessTokensTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/PersonalAccessTokensTcpClient.java index b5b97842fd..6803843796 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/PersonalAccessTokensTcpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/PersonalAccessTokensTcpClient.java @@ -38,22 +38,22 @@ import java.util.function.Supplier; /** - * Async TCP implementation of personal access tokens client. + * Async TCP implementation of personal access tokens client. Login discovers + * the active cluster leader before sending the VSR Register operation. */ public class PersonalAccessTokensTcpClient implements PersonalAccessTokensClient { private static final Logger log = LoggerFactory.getLogger(PersonalAccessTokensTcpClient.class); private final Supplier connectionSupplier; - private final LoginRedirectionHook redirectionHook; + private final LoginRoutingHook routingHook; public PersonalAccessTokensTcpClient(Supplier connectionSupplier) { - this(connectionSupplier, LoginRedirectionHook.NONE); + this(connectionSupplier, LoginRoutingHook.NONE); } - PersonalAccessTokensTcpClient( - Supplier connectionSupplier, LoginRedirectionHook redirectionHook) { + PersonalAccessTokensTcpClient(Supplier connectionSupplier, LoginRoutingHook routingHook) { this.connectionSupplier = connectionSupplier; - this.redirectionHook = redirectionHook; + this.routingHook = routingHook; } private AsyncTcpConnection connection() { @@ -115,9 +115,7 @@ public CompletableFuture deletePersonalAccessToken(String name) { @Override public CompletableFuture loginWithPersonalAccessToken(String token) { - return loginWithoutRedirect(token).thenCompose(identity -> redirectionHook - .afterLogin(() -> loginWithoutRedirect(token)) - .thenApply(redirectedIdentity -> redirectedIdentity != null ? redirectedIdentity : identity)); + return routingHook.loginOnLeader(() -> loginWithoutRedirect(token)); } private CompletableFuture loginWithoutRedirect(String token) { diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ReconnectPlan.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ReconnectPlan.java new file mode 100644 index 0000000000..ae596731f4 --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/ReconnectPlan.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.client.async.tcp; + +import org.apache.iggy.client.ConnectionInfo; +import org.apache.iggy.config.RetryPolicy; + +import java.time.Duration; + +/** + * Pure redial planning: which address to dial on a given reconnect attempt + * and how long to wait before it. + */ +final class ReconnectPlan { + + private ReconnectPlan() {} + + /** + * Alternates reconnect dials between the current endpoint and the + * configured seed. After a leader redirect the current endpoint may die + * with the leader, and the seed is the way back to the rest of the + * cluster. Attempts are 1-based; odd attempts dial the current endpoint. + */ + static ConnectionInfo target(ConnectionInfo current, ConnectionInfo seed, int attempt) { + if (current.equals(seed)) { + return current; + } + return attempt % 2 == 1 ? current : seed; + } + + /** + * The delay before the given 1-based attempt: the policy's initial delay + * scaled by its multiplier per prior attempt, capped at its max delay. + */ + static Duration delay(RetryPolicy policy, int attempt) { + double scaled = policy.getInitialDelay().toMillis() * Math.pow(policy.getMultiplier(), attempt - 1L); + long millis = (long) Math.min(scaled, policy.getMaxDelay().toMillis()); + return Duration.ofMillis(millis); + } +} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/UsersTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/UsersTcpClient.java index 6ed7c3bef4..f6e3be1f2d 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/UsersTcpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/UsersTcpClient.java @@ -41,21 +41,22 @@ import static org.apache.iggy.serde.BytesSerializer.toBytes; /** - * Async TCP implementation of users client. + * Async TCP implementation of users client. Login discovers the active + * cluster leader before sending the VSR Register operation. */ public class UsersTcpClient implements UsersClient { private static final Logger log = LoggerFactory.getLogger(UsersTcpClient.class); private final Supplier connectionSupplier; - private final LoginRedirectionHook redirectionHook; + private final LoginRoutingHook routingHook; public UsersTcpClient(Supplier connectionSupplier) { - this(connectionSupplier, LoginRedirectionHook.NONE); + this(connectionSupplier, LoginRoutingHook.NONE); } - UsersTcpClient(Supplier connectionSupplier, LoginRedirectionHook redirectionHook) { + UsersTcpClient(Supplier connectionSupplier, LoginRoutingHook routingHook) { this.connectionSupplier = connectionSupplier; - this.redirectionHook = redirectionHook; + this.routingHook = routingHook; } private AsyncTcpConnection connection() { @@ -145,9 +146,7 @@ public CompletableFuture changePassword(UserId userId, String currentPassw @Override public CompletableFuture login(String username, String password) { - return loginWithoutRedirect(username, password).thenCompose(identity -> redirectionHook - .afterLogin(() -> loginWithoutRedirect(username, password)) - .thenApply(redirectedIdentity -> redirectedIdentity != null ? redirectedIdentity : identity)); + return routingHook.loginOnLeader(() -> loginWithoutRedirect(username, password)); } private CompletableFuture loginWithoutRedirect(String username, String password) { diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/package-info.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/package-info.java index 261a1cbd47..cab5768ef3 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/package-info.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/package-info.java @@ -34,13 +34,12 @@ * * *

Protocol Details

- *

The Iggy binary protocol uses a simple framing scheme: - *

    - *
  • Request: {@code [payload_size:4 LE][command:4 LE][payload:N]}
    • - *
  • Response: {@code [status:4 LE][length:4 LE][payload:N]}
    • - *
- *

Responses are matched to requests in FIFO order (the protocol does not include - * request IDs). The {@link org.apache.iggy.client.async.tcp.AsyncTcpConnection} + *

The transport speaks the VSR (Viewstamped Replication) wire protocol: + * every frame starts with a 256-byte consensus header ({@code RequestHeader} + * on the way out, {@code ReplyHeader} or {@code EvictionHeader} on the way + * back) followed by the command payload; see the {@code vsr} subpackage. + *

Responses are matched to requests in FIFO order. The + * {@link org.apache.iggy.client.async.tcp.AsyncTcpConnection} * serializes all writes through Netty's event loop to maintain ordering. * * @see org.apache.iggy.client.async.tcp.AsyncIggyTcpClient diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/ConsensusSession.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/ConsensusSession.java new file mode 100644 index 0000000000..ab79ea19ee --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/ConsensusSession.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.client.async.tcp.vsr; + +import org.apache.iggy.exception.IggyNotConnectedException; + +import java.security.SecureRandom; + +/** + * VSR client identity and dedup state, mirroring + * {@code core/sdk/src/session.rs}. + * + *

The (client id, request id) pair is the server's dedup key for + * replicated operations, and the session value is the fence epoch of the + * latest committed {@code Register}. None of these are bearer tokens; auth + * is bound to the transport connection server-side. + */ +public final class ConsensusSession { + + private static final SecureRandom RANDOM = new SecureRandom(); + + private long clientIdLow; + private long clientIdHigh; + private Long session; + private long requestCounter = 1; + private long correlationCounter = 1; + private boolean registerConsumed; + + public ConsensusSession() { + regenerateClientId(); + } + + /** + * Arms a {@code Register}: on re-login (or a consumed one-shot register) + * the whole identity re-arms with a fresh client id so the server sees a + * brand-new registration. Returns the request id a Register carries, + * which is always zero. + */ + synchronized long beginRegister() { + if (registerConsumed || session != null) { + regenerateClientId(); + session = null; + requestCounter = 1; + } + registerConsumed = true; + return 0; + } + + /** Binds the fence epoch returned by a committed Register reply. */ + synchronized void bind(long sessionEpoch) { + if (sessionEpoch <= 0) { + throw new IllegalStateException("Register reply carried a non-positive session epoch: " + sessionEpoch); + } + this.session = sessionEpoch; + } + + /** Replicated metadata ops consume the monotonic VSR dedup counter. */ + synchronized long nextRequestId() { + if (session == null) { + throw new IggyNotConnectedException("Not authenticated, call login first"); + } + return requestCounter++; + } + + /** + * Partition and non-replicated ops use an independent sequence for reply + * correlation, so they do not create gaps in the metadata dedup sequence. + */ + synchronized long nextCorrelationId() { + return correlationCounter++; + } + + synchronized long currentRequestId() { + return requestCounter; + } + + synchronized long sessionOrZero() { + return session == null ? 0 : session; + } + + synchronized long boundSession() { + if (session == null) { + throw new IggyNotConnectedException("Not authenticated, call login first"); + } + return session; + } + + synchronized boolean isBound() { + return session != null; + } + + /** Clears the bound epoch (logout / eviction); next login re-registers. */ + synchronized void reset() { + session = null; + } + + synchronized long clientIdLow() { + return clientIdLow; + } + + synchronized long clientIdHigh() { + return clientIdHigh; + } + + private void regenerateClientId() { + do { + clientIdLow = RANDOM.nextLong(); + clientIdHigh = RANDOM.nextLong(); + } while (clientIdLow == 0 && clientIdHigh == 0); + } +} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrFrameDecoder.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrFrameDecoder.java new file mode 100644 index 0000000000..7398f527f7 --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrFrameDecoder.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.client.async.tcp.vsr; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.ByteToMessageDecoder; +import io.netty.handler.codec.DecoderException; + +import java.util.List; + +/** + * Decoder for VSR response frames: a 256-byte consensus header whose total + * frame size (header included) sits at byte offset 48, followed by an + * optional body. There is no length delimiter outside the header. + */ +public class VsrFrameDecoder extends ByteToMessageDecoder { + + /** Matches the server's default {@code max_message_size} (64 MB). */ + public static final int DEFAULT_MAX_FRAME_SIZE = 64 * 1024 * 1024; + + private final int maxFrameSize; + + public VsrFrameDecoder() { + this(DEFAULT_MAX_FRAME_SIZE); + } + + public VsrFrameDecoder(int maxFrameSize) { + if (maxFrameSize < VsrHeaders.HEADER_SIZE) { + throw new IllegalArgumentException( + "Maximum VSR frame size must be at least " + VsrHeaders.HEADER_SIZE + " bytes"); + } + this.maxFrameSize = maxFrameSize; + } + + @Override + protected void decode(ChannelHandlerContext ctx, ByteBuf in, List out) { + if (in.readableBytes() < VsrHeaders.HEADER_SIZE) { + return; + } + long totalSize = VsrHeaders.readSize(in); + if (totalSize < VsrHeaders.HEADER_SIZE || totalSize > maxFrameSize) { + throw new DecoderException("Invalid VSR frame size " + totalSize + ", connection is desynchronized"); + } + if (in.readableBytes() < totalSize) { + return; + } + out.add(in.readRetainedSlice((int) totalSize)); + } +} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrHeaders.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrHeaders.java new file mode 100644 index 0000000000..356c722a63 --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrHeaders.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.client.async.tcp.vsr; + +import io.netty.buffer.ByteBuf; +import org.apache.iggy.exception.IggyServerException; + +/** + * Byte offsets and readers for the 256-byte consensus headers, mirroring the + * {@code #[repr(C)]} layouts in + * {@code core/binary_protocol/src/consensus/header.rs}. All fields are + * little-endian; the checksum fields stay zero by protocol contract. + */ +public final class VsrHeaders { + + public static final int HEADER_SIZE = 256; + + // GenericHeader (shared prefix) + static final int SIZE_OFFSET = 48; + static final int COMMAND_OFFSET = 60; + + // RequestHeader. The client wire carries no routing group: the server + // derives it (plane from the operation, partition target from the payload), + // so every field past the operation sits eight bytes earlier than it did + // while the header still carried one. + static final int REQUEST_CLIENT_OFFSET = 128; + static final int REQUEST_ID_OFFSET = 168; + static final int REQUEST_OPERATION_OFFSET = 176; + static final int REQUEST_SESSION_OFFSET = 184; + static final int REQUEST_RESERVED_CODE_OFFSET = 196; + + // ReplyHeader + static final int REPLY_REQUEST_OFFSET = 200; + static final int REPLY_OPERATION_OFFSET = 208; + static final int REPLY_STATUS_OFFSET = 216; + + // EvictionHeader + static final int EVICTION_PROTOCOL_VERSION_OFFSET = 144; + static final int EVICTION_PROTOCOL_VERSION_MIN_OFFSET = 148; + static final int EVICTION_REASON_OFFSET = 255; + + // Command2 discriminants + static final int COMMAND_REQUEST = 5; + static final int COMMAND_REPLY = 8; + static final int COMMAND_EVICTION = 13; + + // EvictionReason discriminants + static final int REASON_NO_SESSION = 1; + static final int REASON_SESSION_TOO_LOW = 7; + static final int REASON_SESSION_RELEASE_MISMATCH = 8; + static final int REASON_INVALID_CREDENTIALS = 9; + static final int REASON_INVALID_TOKEN = 10; + static final int REASON_USER_INACTIVE = 11; + static final int REASON_SESSION_ERROR = 12; + static final int REASON_STALE_CLIENT = 13; + static final int REASON_INCOMPATIBLE_PROTOCOL = 14; + static final int REASON_MALFORMED_LOGIN = 15; + + // IggyError codes the eviction reasons grade to, mirroring + // core/common/src/error/eviction.rs. + static final int ERROR_INVALID_COMMAND = 3; + static final int ERROR_INVALID_FORMAT = 4; + static final int ERROR_STALE_CLIENT = 30; + static final int ERROR_UNAUTHENTICATED = 40; + static final int ERROR_INVALID_CREDENTIALS = 42; + static final int ERROR_INVALID_PERSONAL_ACCESS_TOKEN = 53; + static final int ERROR_TRANSIENT_NOT_COMMITTED = 57; + static final int ERROR_TRANSIENT_NOT_ACCEPTED = 58; + static final int ERROR_INCOMPATIBLE_PROTOCOL_VERSION = 14003; + + private VsrHeaders() {} + + static int peekCommand(ByteBuf frame) { + return frame.getUnsignedByte(frame.readerIndex() + COMMAND_OFFSET); + } + + static long readSize(ByteBuf frame) { + return frame.getUnsignedIntLE(frame.readerIndex() + SIZE_OFFSET); + } + + static long readStatus(ByteBuf frame) { + return frame.getUnsignedIntLE(frame.readerIndex() + REPLY_STATUS_OFFSET); + } + + static int readReplyOperation(ByteBuf frame) { + return frame.getUnsignedByte(frame.readerIndex() + REPLY_OPERATION_OFFSET); + } + + static long readReplyRequestId(ByteBuf frame) { + return frame.getLongLE(frame.readerIndex() + REPLY_REQUEST_OFFSET); + } + + static int readRequestOperation(ByteBuf frame) { + return frame.getUnsignedByte(frame.readerIndex() + REQUEST_OPERATION_OFFSET); + } + + static long readRequestId(ByteBuf frame) { + return frame.getLongLE(frame.readerIndex() + REQUEST_ID_OFFSET); + } + + /** + * Grades a session-terminal eviction frame to the error the caller sees, + * mirroring {@code eviction_reason_to_error}. A degenerate protocol + * window (zero minimum or inverted range) degrades to unauthenticated. + */ + static IggyServerException evictionToException(ByteBuf frame) { + int base = frame.readerIndex(); + int reason = frame.getUnsignedByte(base + EVICTION_REASON_OFFSET); + int errorCode; + switch (reason) { + case REASON_INVALID_CREDENTIALS -> errorCode = ERROR_INVALID_CREDENTIALS; + case REASON_INVALID_TOKEN -> errorCode = ERROR_INVALID_PERSONAL_ACCESS_TOKEN; + case REASON_STALE_CLIENT -> errorCode = ERROR_STALE_CLIENT; + case REASON_MALFORMED_LOGIN -> errorCode = ERROR_INVALID_FORMAT; + case REASON_INCOMPATIBLE_PROTOCOL -> { + long max = frame.getUnsignedIntLE(base + EVICTION_PROTOCOL_VERSION_OFFSET); + long min = frame.getUnsignedIntLE(base + EVICTION_PROTOCOL_VERSION_MIN_OFFSET); + errorCode = (min == 0 || max < min) ? ERROR_UNAUTHENTICATED : ERROR_INCOMPATIBLE_PROTOCOL_VERSION; + } + case REASON_NO_SESSION, + REASON_SESSION_TOO_LOW, + REASON_SESSION_RELEASE_MISMATCH, + REASON_USER_INACTIVE, + REASON_SESSION_ERROR -> errorCode = ERROR_UNAUTHENTICATED; + default -> errorCode = ERROR_INVALID_COMMAND; + } + return IggyServerException.fromTcpResponse(errorCode, new byte[0]); + } +} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrLoginCodec.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrLoginCodec.java new file mode 100644 index 0000000000..36c856c3a2 --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrLoginCodec.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.client.async.tcp.vsr; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.ByteBufAllocator; +import org.apache.iggy.IggyVersion; +import org.apache.iggy.exception.IggyInvalidArgumentException; + +import java.nio.charset.StandardCharsets; + +/** + * Rewrites serialized login payloads into the + * {@code LoginRegister} / {@code LoginRegisterWithPat} bodies, mirroring + * {@code core/binary_protocol/src/requests/users/login_register.rs} and + * {@code login_register_with_pat.rs}. Both bodies start with a + * {@code ClientVersionInfo} prefix carrying the packed protocol version and + * the SDK identity ({@code core/binary_protocol/src/version.rs}). + */ +final class VsrLoginCodec { + + /** + * Packed semver of the {@code iggy_binary_protocol} crate this codec + * targets: {@code major << 20 | minor << 10 | patch}, 10 bits per field. + * Keep in sync with {@code core/binary_protocol/Cargo.toml}; the server + * accepts any client whose major.minor is not newer than its own. + */ + static final int PROTOCOL_VERSION = (11 << 10); // 0.11.0 + + static final String SDK_NAME = "java-sdk"; + + private VsrLoginCodec() {} + + /** + * {@code LoginUser} (code 38) payload in: + * {@code [username:u8-len][password:u8-len][version:u32-len][context:u32-len]}. + * The trailing version/context strings are superseded by the + * {@code ClientVersionInfo} prefix and dropped. + */ + static ByteBuf rewriteUserLogin(ByteBufAllocator alloc, ByteBuf loginPayload) { + ByteBuf in = loginPayload.slice(); + byte[] username = readShortField(in, "username"); + byte[] password = readShortField(in, "password"); + + ByteBuf body = alloc.buffer(); + writeVersionInfo(body); + writeShortField(body, username); + body.writeByte(password.length); + body.writeBytes(password); + body.writeIntLE(0); + return body; + } + + /** + * {@code LoginWithPersonalAccessToken} (code 44) payload in: + * {@code [token:u8-len]}. + */ + static ByteBuf rewritePatLogin(ByteBufAllocator alloc, ByteBuf loginPayload) { + ByteBuf in = loginPayload.slice(); + byte[] token = readShortField(in, "token"); + + ByteBuf body = alloc.buffer(); + writeVersionInfo(body); + writeShortField(body, token); + body.writeIntLE(0); + return body; + } + + /** + * Register reply body after result-section stripping: + * {@code [user_id:u32][session:u64][server_protocol_version:u32][server_version:u8-len]}. + */ + static long readSessionEpoch(ByteBuf registerBody) { + return registerBody.getLongLE(registerBody.readerIndex() + 4); + } + + private static void writeVersionInfo(ByteBuf body) { + body.writeIntLE(PROTOCOL_VERSION); + writeShortField(body, SDK_NAME.getBytes(StandardCharsets.UTF_8)); + writeShortField(body, sdkVersion().getBytes(StandardCharsets.UTF_8)); + } + + private static String sdkVersion() { + String version = IggyVersion.getInstance().getVersion(); + if (version == null || version.isEmpty()) { + return "unknown"; + } + return version.length() > 255 ? version.substring(0, 255) : version; + } + + private static byte[] readShortField(ByteBuf in, String field) { + if (!in.isReadable()) { + throw new IggyInvalidArgumentException("Login payload is missing the " + field + " field"); + } + int length = in.readUnsignedByte(); + if (in.readableBytes() < length) { + throw new IggyInvalidArgumentException("Login payload " + field + " field is truncated"); + } + byte[] value = new byte[length]; + in.readBytes(value); + return value; + } + + private static void writeShortField(ByteBuf out, byte[] value) { + if (value.length == 0 || value.length > 255) { + throw new IggyInvalidArgumentException("Wire name fields must be 1..255 bytes, got " + value.length); + } + out.writeByte(value.length); + out.writeBytes(value); + } +} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrOperation.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrOperation.java new file mode 100644 index 0000000000..6bd6349ac1 --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrOperation.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.client.async.tcp.vsr; + +import java.util.BitSet; +import java.util.Map; + +/** + * VSR {@code Operation} discriminants and the command-code mapping, mirroring + * {@code core/binary_protocol/src/consensus/operation.rs} and the SDK-side + * mapping in {@code core/sdk/src/vsr.rs}. + */ +public final class VsrOperation { + + public static final int RESERVED = 0; + public static final int REGISTER = 1; + public static final int NON_REPLICATED = 2; + public static final int LOGOUT = 3; + + public static final int CREATE_TOPIC_WITH_ASSIGNMENTS = 64; + public static final int CREATE_PARTITIONS_WITH_ASSIGNMENTS = 65; + public static final int REMOVE_CONSUMER_GROUP_MEMBER = 66; + public static final int COMPLETE_CONSUMER_GROUP_REVOCATION = 67; + public static final int TRUNCATE_PARTITION = 68; + + public static final int CREATE_STREAM = 128; + public static final int UPDATE_STREAM = 129; + public static final int DELETE_STREAM = 130; + public static final int PURGE_STREAM = 131; + public static final int CREATE_TOPIC = 132; + public static final int UPDATE_TOPIC = 133; + public static final int DELETE_TOPIC = 134; + public static final int PURGE_TOPIC = 135; + public static final int CREATE_PARTITIONS = 136; + public static final int DELETE_PARTITIONS = 137; + public static final int DELETE_SEGMENTS = 138; + public static final int CREATE_CONSUMER_GROUP = 139; + public static final int DELETE_CONSUMER_GROUP = 140; + public static final int CREATE_USER = 141; + public static final int UPDATE_USER = 142; + public static final int DELETE_USER = 143; + public static final int CHANGE_PASSWORD = 144; + public static final int UPDATE_PERMISSIONS = 145; + public static final int CREATE_PERSONAL_ACCESS_TOKEN = 146; + public static final int DELETE_PERSONAL_ACCESS_TOKEN = 147; + public static final int JOIN_CONSUMER_GROUP = 148; + public static final int LEAVE_CONSUMER_GROUP = 149; + + public static final int SEND_MESSAGES = 160; + public static final int STORE_CONSUMER_OFFSET = 161; + public static final int DELETE_CONSUMER_OFFSET = 162; + public static final int STORE_CONSUMER_OFFSET_2 = 164; + public static final int DELETE_CONSUMER_OFFSET_2 = 165; + + private static final int INTERNAL_START = 64; + private static final int METADATA_START = 128; + private static final int PARTITION_START = 160; + + /** + * Replicated command code to operation, from the server's + * {@code COMMAND_TABLE}. Codes absent here travel as + * {@link #NON_REPLICATED} with the code stamped into the header's + * reserved bytes; the server is the authority on unknown codes. + */ + private static final Map REPLICATED_OPERATIONS = Map.ofEntries( + Map.entry(33, CREATE_USER), + Map.entry(34, DELETE_USER), + Map.entry(35, UPDATE_USER), + Map.entry(36, UPDATE_PERMISSIONS), + Map.entry(37, CHANGE_PASSWORD), + Map.entry(42, CREATE_PERSONAL_ACCESS_TOKEN), + Map.entry(43, DELETE_PERSONAL_ACCESS_TOKEN), + Map.entry(101, SEND_MESSAGES), + Map.entry(121, STORE_CONSUMER_OFFSET), + Map.entry(122, DELETE_CONSUMER_OFFSET), + Map.entry(123, STORE_CONSUMER_OFFSET_2), + Map.entry(124, DELETE_CONSUMER_OFFSET_2), + Map.entry(202, CREATE_STREAM), + Map.entry(203, DELETE_STREAM), + Map.entry(204, UPDATE_STREAM), + Map.entry(205, PURGE_STREAM), + Map.entry(302, CREATE_TOPIC), + Map.entry(303, DELETE_TOPIC), + Map.entry(304, UPDATE_TOPIC), + Map.entry(305, PURGE_TOPIC), + Map.entry(402, CREATE_PARTITIONS), + Map.entry(403, DELETE_PARTITIONS), + Map.entry(503, DELETE_SEGMENTS), + Map.entry(602, CREATE_CONSUMER_GROUP), + Map.entry(603, DELETE_CONSUMER_GROUP), + Map.entry(604, JOIN_CONSUMER_GROUP), + Map.entry(605, LEAVE_CONSUMER_GROUP)); + + private static final int LOGOUT_USER_CODE = 39; + + private static final BitSet KNOWN_OPERATIONS = new BitSet(); + + static { + KNOWN_OPERATIONS.set(RESERVED, LOGOUT + 1); + KNOWN_OPERATIONS.set(CREATE_TOPIC_WITH_ASSIGNMENTS, TRUNCATE_PARTITION + 1); + KNOWN_OPERATIONS.set(CREATE_STREAM, LEAVE_CONSUMER_GROUP + 1); + KNOWN_OPERATIONS.set(SEND_MESSAGES); + KNOWN_OPERATIONS.set(STORE_CONSUMER_OFFSET); + KNOWN_OPERATIONS.set(DELETE_CONSUMER_OFFSET); + KNOWN_OPERATIONS.set(STORE_CONSUMER_OFFSET_2); + KNOWN_OPERATIONS.set(DELETE_CONSUMER_OFFSET_2); + } + + private VsrOperation() {} + + /** + * Maps a command code to its operation. Login codes are handled upstream + * (they select {@link #REGISTER}); everything unmapped is forwarded as + * {@link #NON_REPLICATED}. + */ + static int operationForCode(int commandCode) { + if (commandCode == LOGOUT_USER_CODE) { + return LOGOUT; + } + return REPLICATED_OPERATIONS.getOrDefault(commandCode, NON_REPLICATED); + } + + static boolean isMetadata(int operation) { + if (operation >= INTERNAL_START && operation < METADATA_START) { + return true; + } + // DeleteSegments replicates in its per-partition group, not the + // metadata group, so it is excluded from the metadata band. + if (operation == DELETE_SEGMENTS) { + return false; + } + return operation >= METADATA_START && operation <= LEAVE_CONSUMER_GROUP; + } + + static boolean isPartition(int operation) { + return operation >= PARTITION_START; + } + + /** + * Whether a reply body for this operation starts with a committed result + * section ({@code [count:u32][{index,result} x count]}). + */ + static boolean isResultFramed(int operation) { + return isMetadata(operation) + || operation == STORE_CONSUMER_OFFSET + || operation == DELETE_CONSUMER_OFFSET + || operation == STORE_CONSUMER_OFFSET_2 + || operation == DELETE_CONSUMER_OFFSET_2; + } + + /** + * The server controls the reply's operation byte; an undeclared value must + * be rejected rather than routed through a predicate that happens to match. + */ + static boolean isKnown(int operation) { + return operation >= 0 && KNOWN_OPERATIONS.get(operation); + } + + /** + * Maps an internal operation returned by the server to the client + * operation that initiated it. The server preserves the request id when + * it enriches these commands before replication. + */ + static int correlationOperation(int operation) { + return switch (operation) { + case CREATE_TOPIC_WITH_ASSIGNMENTS -> CREATE_TOPIC; + case CREATE_PARTITIONS_WITH_ASSIGNMENTS -> CREATE_PARTITIONS; + case TRUNCATE_PARTITION -> DELETE_SEGMENTS; + default -> operation; + }; + } +} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrRequestEncoder.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrRequestEncoder.java new file mode 100644 index 0000000000..45859f06d7 --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrRequestEncoder.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.client.async.tcp.vsr; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.ByteBufAllocator; + +/** + * Encodes a (code, payload) command into a VSR request frame: + * a 256-byte {@code RequestHeader} followed by the unchanged command payload. + * Login codes are rewritten to the {@code LoginRegister} exchange; every + * other payload is passed through byte-identical. + * + *

Mirrors {@code encode_request_header} in {@code core/sdk/src/vsr.rs}. + */ +public final class VsrRequestEncoder { + + private static final int LOGIN_USER_CODE = 38; + private static final int LOGIN_REGISTER_CODE = 40; + private static final int LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE = 44; + private static final int LOGIN_REGISTER_WITH_PAT_CODE = 45; + + private final ConsensusSession session; + + public VsrRequestEncoder(ConsensusSession session) { + this.session = session; + } + + /** + * Builds the full request frame. The caller keeps ownership of + * {@code payload}; its reader index is not advanced. + */ + public ByteBuf encode(ByteBufAllocator alloc, int commandCode, ByteBuf payload) { + int operation; + long requestId; + long sessionId; + ByteBuf body; + boolean releaseBody = false; + + if (commandCode == LOGIN_USER_CODE || commandCode == LOGIN_REGISTER_CODE) { + body = VsrLoginCodec.rewriteUserLogin(alloc, payload); + releaseBody = true; + operation = VsrOperation.REGISTER; + requestId = session.beginRegister(); + sessionId = 0; + } else if (commandCode == LOGIN_WITH_PERSONAL_ACCESS_TOKEN_CODE + || commandCode == LOGIN_REGISTER_WITH_PAT_CODE) { + body = VsrLoginCodec.rewritePatLogin(alloc, payload); + releaseBody = true; + operation = VsrOperation.REGISTER; + requestId = session.beginRegister(); + sessionId = 0; + } else { + body = payload; + operation = VsrOperation.operationForCode(commandCode); + if (operation == VsrOperation.NON_REPLICATED) { + // Non-replicated ops bypass dedup but still need a unique + // correlation id. Bootstrap ping and cluster metadata remain + // sessionless before login. + requestId = session.nextCorrelationId(); + sessionId = session.sessionOrZero(); + } else if (VsrOperation.isPartition(operation)) { + // Partition ops replicate in their own group without client + // table dedup, so use the independent correlation sequence. + sessionId = session.boundSession(); + requestId = session.nextCorrelationId(); + } else { + sessionId = session.boundSession(); + requestId = session.nextRequestId(); + } + } + + try { + int totalSize = VsrHeaders.HEADER_SIZE + body.readableBytes(); + ByteBuf frame = alloc.buffer(totalSize); + frame.writeZero(VsrHeaders.HEADER_SIZE); + frame.setIntLE(VsrHeaders.SIZE_OFFSET, totalSize); + frame.setByte(VsrHeaders.COMMAND_OFFSET, VsrHeaders.COMMAND_REQUEST); + frame.setLongLE(VsrHeaders.REQUEST_CLIENT_OFFSET, session.clientIdLow()); + frame.setLongLE(VsrHeaders.REQUEST_CLIENT_OFFSET + 8, session.clientIdHigh()); + frame.setLongLE(VsrHeaders.REQUEST_ID_OFFSET, requestId); + frame.setByte(VsrHeaders.REQUEST_OPERATION_OFFSET, operation); + frame.setLongLE(VsrHeaders.REQUEST_SESSION_OFFSET, sessionId); + if (operation == VsrOperation.NON_REPLICATED) { + frame.setIntLE(VsrHeaders.REQUEST_RESERVED_CODE_OFFSET, commandCode); + } + frame.writeBytes(body, body.readerIndex(), body.readableBytes()); + return frame; + } finally { + if (releaseBody) { + body.release(); + } + } + } +} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandler.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandler.java new file mode 100644 index 0000000000..113c288e27 --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandler.java @@ -0,0 +1,261 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.client.async.tcp.vsr; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.util.concurrent.ScheduledFuture; +import org.apache.iggy.exception.IggyConnectionException; +import org.apache.iggy.exception.IggyServerException; +import org.apache.iggy.exception.IggyTimeoutException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Correlates multiplexed requests by operation and request id, decodes VSR + * reply frames, and completes each pending future with the command payload + * the typed deserializers expect. Mirrors {@code decode_response} in {@code + * core/sdk/src/vsr.rs}: eviction frames become typed errors, a nonzero header + * status is a pre-commit deny, and result-framed bodies have their committed + * result section stripped (or raised as the typed error). + */ +public class VsrResponseHandler extends SimpleChannelInboundHandler { + + private static final Logger log = LoggerFactory.getLogger(VsrResponseHandler.class); + + private static final int RESULT_COUNT_LEN = 4; + private static final int RESULT_ENTRY_LEN = 8; + private static final int REGISTER_BODY_MIN_LEN = 17; + + private final ConcurrentMap> pendingRequests = new ConcurrentHashMap<>(); + private final ConsensusSession session; + private final Runnable onEviction; + private final AtomicReference closeCause = new AtomicReference<>(); + + public VsrResponseHandler(ConsensusSession session, Runnable onEviction) { + this.session = session; + this.onEviction = onEviction; + } + + void registerRequest(CompletableFuture future, int operation, long requestId) { + registerRequest(new RequestKey(operation, requestId), future); + } + + public void registerRequest( + Channel channel, + ByteBuf requestFrame, + CompletableFuture future, + long deadlineNanos, + int commandCode) { + RequestKey key = + new RequestKey(VsrHeaders.readRequestOperation(requestFrame), VsrHeaders.readRequestId(requestFrame)); + registerRequest(key, future); + long timeoutNanos = Math.max(0, deadlineNanos - System.nanoTime()); + ScheduledFuture timeoutFuture; + try { + timeoutFuture = channel.eventLoop() + .schedule( + () -> { + if (!future.isDone()) { + closeChannel( + channel, + new IggyTimeoutException( + "Timed out waiting for a response to command code " + commandCode)); + } + }, + timeoutNanos, + TimeUnit.NANOSECONDS); + } catch (RuntimeException error) { + pendingRequests.remove(key, future); + throw error; + } + future.whenComplete((response, error) -> { + pendingRequests.remove(key, future); + timeoutFuture.cancel(false); + }); + } + + private void registerRequest(RequestKey key, CompletableFuture future) { + CompletableFuture existing = pendingRequests.putIfAbsent(key, future); + if (existing != null) { + throw new IllegalStateException( + "A request is already pending for operation " + key.operation() + " and request id " + key.id()); + } + } + + public void closeChannel(Channel channel, Throwable cause) { + closeCause.compareAndSet(null, cause); + channel.close(); + failPendingRequests(closeCause.get()); + } + + @Override + public void channelInactive(ChannelHandlerContext ctx) { + Throwable cause = closeCause.get(); + if (cause == null) { + cause = new IggyConnectionException("Connection closed before a response arrived"); + } + failPendingRequests(cause); + ctx.fireChannelInactive(); + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { + closeChannel(ctx.channel(), cause); + } + + @Override + protected void channelRead0(ChannelHandlerContext ctx, ByteBuf msg) { + if (VsrHeaders.peekCommand(msg) == VsrHeaders.COMMAND_EVICTION) { + handleEviction(ctx, msg); + return; + } + int replyOperation = VsrHeaders.readReplyOperation(msg); + RequestKey key = + new RequestKey(VsrOperation.correlationOperation(replyOperation), VsrHeaders.readReplyRequestId(msg)); + CompletableFuture future = pendingRequests.remove(key); + if (future == null) { + closeChannel( + ctx.channel(), + invalidReply( + "no request was pending for operation " + replyOperation + " and request id " + key.id())); + return; + } + ByteBuf body; + try { + body = decodeReply(msg); + } catch (RuntimeException error) { + future.completeExceptionally(error); + return; + } + if (!future.complete(body)) { + body.release(); + } + } + + private void handleEviction(ChannelHandlerContext ctx, ByteBuf frame) { + IggyServerException error = VsrHeaders.evictionToException(frame); + session.reset(); + try { + onEviction.run(); + } catch (RuntimeException listenerError) { + log.warn("Eviction listener failed: {}", listenerError.getMessage()); + } + // The server has already unbound this transport. Closing it before + // completing requests prevents the pool from recycling the channel + // and assigning a late reply to a request from the next session. + closeChannel(ctx.channel(), error); + } + + private ByteBuf decodeReply(ByteBuf frame) { + int operation = validatedOperation(frame); + + int totalSize = (int) VsrHeaders.readSize(frame); + int bodyStart = frame.readerIndex() + VsrHeaders.HEADER_SIZE; + int bodyLength = totalSize - VsrHeaders.HEADER_SIZE; + + boolean resultFramed = + VsrOperation.isResultFramed(operation) || (operation == VsrOperation.REGISTER && bodyLength > 0); + if (resultFramed) { + int sectionLength = resultSectionLength(frame, bodyStart, bodyLength); + bodyStart += sectionLength; + bodyLength -= sectionLength; + } + + applySessionEffects(frame, operation, bodyStart, bodyLength); + return frame.retainedSlice(bodyStart, bodyLength); + } + + private int validatedOperation(ByteBuf frame) { + int command = VsrHeaders.peekCommand(frame); + if (command != VsrHeaders.COMMAND_REPLY) { + throw invalidReply("unexpected consensus command " + command); + } + long status = VsrHeaders.readStatus(frame); + if (status != 0) { + throw IggyServerException.fromTcpResponse(status, new byte[0]); + } + int operation = VsrHeaders.readReplyOperation(frame); + if (!VsrOperation.isKnown(operation)) { + throw invalidReply("unknown reply operation " + operation); + } + return operation; + } + + /** + * Validates the committed result section leading the body and returns its + * length; a nonzero committed result surfaces as the typed error. + */ + private static int resultSectionLength(ByteBuf frame, int bodyStart, int bodyLength) { + if (bodyLength < RESULT_COUNT_LEN) { + throw invalidReply("result-framed body shorter than its count field"); + } + long count = frame.getUnsignedIntLE(bodyStart); + long sectionLength = RESULT_COUNT_LEN + count * RESULT_ENTRY_LEN; + if (bodyLength < sectionLength) { + throw invalidReply("result section truncated"); + } + if (count > 0) { + long resultCode = frame.getUnsignedIntLE(bodyStart + RESULT_COUNT_LEN + 4); + if (resultCode != 0) { + throw IggyServerException.fromTcpResponse(resultCode, new byte[0]); + } + } + return (int) sectionLength; + } + + private void applySessionEffects(ByteBuf frame, int operation, int bodyStart, int bodyLength) { + if (operation == VsrOperation.REGISTER) { + // A terminal register failure ships an empty body (no result + // section); anything shorter than the typed response is not a + // successful registration. + if (bodyLength < REGISTER_BODY_MIN_LEN) { + throw IggyServerException.fromTcpResponse(VsrHeaders.ERROR_UNAUTHENTICATED, new byte[0]); + } + session.bind(frame.getLongLE(bodyStart + 4)); + } + if (operation == VsrOperation.LOGOUT) { + session.reset(); + } + } + + private static IggyServerException invalidReply(String detail) { + log.error("Malformed VSR reply: {}", detail); + return IggyServerException.fromTcpResponse(VsrHeaders.ERROR_INVALID_COMMAND, new byte[0]); + } + + private void failPendingRequests(Throwable cause) { + pendingRequests.forEach((key, pending) -> { + if (pendingRequests.remove(key, pending)) { + pending.completeExceptionally(cause); + } + }); + } + + private record RequestKey(int operation, long id) {} +} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/ConsumerGroupsClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/ConsumerGroupsClient.java index bf91344315..99a0742564 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/ConsumerGroupsClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/ConsumerGroupsClient.java @@ -20,6 +20,7 @@ package org.apache.iggy.client.blocking; import org.apache.iggy.consumergroup.ConsumerGroup; +import org.apache.iggy.consumergroup.ConsumerGroupAssignment; import org.apache.iggy.consumergroup.ConsumerGroupDetails; import org.apache.iggy.identifier.ConsumerId; import org.apache.iggy.identifier.StreamId; @@ -65,4 +66,14 @@ default void leaveConsumerGroup(Long streamId, Long topicId, Long groupId) { } void leaveConsumerGroup(StreamId streamId, TopicId topicId, ConsumerId groupId); + + default Optional syncConsumerGroup(Long streamId, Long topicId, Long groupId) { + return syncConsumerGroup(StreamId.of(streamId), TopicId.of(topicId), ConsumerId.of(groupId)); + } + + /** + * Fetches this client's current partition assignment for a consumer group, + * or empty when this client is not a member. + */ + Optional syncConsumerGroup(StreamId streamId, TopicId topicId, ConsumerId groupId); } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/MessagesClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/MessagesClient.java index fb6705686c..8099126765 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/MessagesClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/MessagesClient.java @@ -26,6 +26,7 @@ import org.apache.iggy.message.Partitioning; import org.apache.iggy.message.PolledMessages; import org.apache.iggy.message.PollingStrategy; +import org.apache.iggy.message.SendMessagesResponse; import java.util.List; import java.util.Optional; @@ -59,9 +60,11 @@ PolledMessages pollMessages( Long count, boolean autoCommit); - default void sendMessages(Long streamId, Long topicId, Partitioning partitioning, List messages) { - sendMessages(StreamId.of(streamId), TopicId.of(topicId), partitioning, messages); + default SendMessagesResponse sendMessages( + Long streamId, Long topicId, Partitioning partitioning, List messages) { + return sendMessages(StreamId.of(streamId), TopicId.of(topicId), partitioning, messages); } - void sendMessages(StreamId streamId, TopicId topicId, Partitioning partitioning, List messages); + SendMessagesResponse sendMessages( + StreamId streamId, TopicId topicId, Partitioning partitioning, List messages); } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/ConsumerGroupsHttpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/ConsumerGroupsHttpClient.java index 1d73f56f6e..bf155ceb60 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/ConsumerGroupsHttpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/ConsumerGroupsHttpClient.java @@ -21,6 +21,7 @@ import org.apache.iggy.client.blocking.ConsumerGroupsClient; import org.apache.iggy.consumergroup.ConsumerGroup; +import org.apache.iggy.consumergroup.ConsumerGroupAssignment; import org.apache.iggy.consumergroup.ConsumerGroupDetails; import org.apache.iggy.exception.IggyOperationNotSupportedException; import org.apache.iggy.identifier.ConsumerId; @@ -73,6 +74,11 @@ public void leaveConsumerGroup(StreamId streamId, TopicId topicId, ConsumerId gr throw new IggyOperationNotSupportedException("leaveConsumerGroup", "HTTP"); } + @Override + public Optional syncConsumerGroup(StreamId streamId, TopicId topicId, ConsumerId groupId) { + throw new IggyOperationNotSupportedException("syncConsumerGroup", "HTTP"); + } + private static String path(StreamId streamId, TopicId topicId) { return "/streams/" + streamId + "/topics/" + topicId + "/consumer-groups"; } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/MessagesHttpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/MessagesHttpClient.java index 3d2ce27978..998637f4cb 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/MessagesHttpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/http/MessagesHttpClient.java @@ -28,6 +28,7 @@ import org.apache.iggy.message.Partitioning; import org.apache.iggy.message.PolledMessages; import org.apache.iggy.message.PollingStrategy; +import org.apache.iggy.message.SendMessagesResponse; import java.util.List; import java.util.Optional; @@ -63,9 +64,14 @@ public PolledMessages pollMessages( } @Override - public void sendMessages(StreamId streamId, TopicId topicId, Partitioning partitioning, List messages) { + public SendMessagesResponse sendMessages( + StreamId streamId, TopicId topicId, Partitioning partitioning, List messages) { var request = httpClient.preparePostRequest(path(streamId, topicId), new SendMessages(partitioning, messages)); - httpClient.execute(request); + var body = httpClient.executeWithStringResponse(request); + if (body.isBlank()) { + return SendMessagesResponse.empty(); + } + return ObjectMapperFactory.getInstance().readValue(body, SendMessagesResponse.class); } private static String path(StreamId streamId, TopicId topicId) { diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/ConsumerGroupsTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/ConsumerGroupsTcpClient.java index b69416d1c2..d8b452e596 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/ConsumerGroupsTcpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/ConsumerGroupsTcpClient.java @@ -21,6 +21,7 @@ import org.apache.iggy.client.blocking.ConsumerGroupsClient; import org.apache.iggy.consumergroup.ConsumerGroup; +import org.apache.iggy.consumergroup.ConsumerGroupAssignment; import org.apache.iggy.consumergroup.ConsumerGroupDetails; import org.apache.iggy.identifier.ConsumerId; import org.apache.iggy.identifier.StreamId; @@ -66,4 +67,9 @@ public void joinConsumerGroup(StreamId streamId, TopicId topicId, ConsumerId gro public void leaveConsumerGroup(StreamId streamId, TopicId topicId, ConsumerId groupId) { FutureUtil.resolve(delegate.leaveConsumerGroup(streamId, topicId, groupId)); } + + @Override + public Optional syncConsumerGroup(StreamId streamId, TopicId topicId, ConsumerId groupId) { + return FutureUtil.resolve(delegate.syncConsumerGroup(streamId, topicId, groupId)); + } } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/IggyTcpClientBuilder.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/IggyTcpClientBuilder.java index 90ba272f05..65a09d215a 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/IggyTcpClientBuilder.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/IggyTcpClientBuilder.java @@ -125,17 +125,6 @@ public IggyTcpClientBuilder requestTimeout(Duration requestTimeout) { return this; } - /** - * Sets the connection pool size. - * - * @param connectionPoolSize the size of the connection pool - * @return this builder - */ - public IggyTcpClientBuilder connectionPoolSize(Integer connectionPoolSize) { - asyncBuilder.connectionPoolSize(connectionPoolSize); - return this; - } - /** * Sets the retry policy. * diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/MessagesTcpClient.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/MessagesTcpClient.java index 2ec3ed2fed..47e561b0dd 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/MessagesTcpClient.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/client/blocking/tcp/MessagesTcpClient.java @@ -27,6 +27,7 @@ import org.apache.iggy.message.Partitioning; import org.apache.iggy.message.PolledMessages; import org.apache.iggy.message.PollingStrategy; +import org.apache.iggy.message.SendMessagesResponse; import java.util.List; import java.util.Optional; @@ -53,7 +54,8 @@ public PolledMessages pollMessages( } @Override - public void sendMessages(StreamId streamId, TopicId topicId, Partitioning partitioning, List messages) { - FutureUtil.resolve(delegate.sendMessages(streamId, topicId, partitioning, messages)); + public SendMessagesResponse sendMessages( + StreamId streamId, TopicId topicId, Partitioning partitioning, List messages) { + return FutureUtil.resolve(delegate.sendMessages(streamId, topicId, partitioning, messages)); } } diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/consumergroup/ConsumerGroupAssignment.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/consumergroup/ConsumerGroupAssignment.java new file mode 100644 index 0000000000..a12a04d39d --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/consumergroup/ConsumerGroupAssignment.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.consumergroup; + +import java.util.List; + +/** + * A consumer-group member's current partition assignment and the group + * generation it belongs to, as returned by the sync-consumer-group command. + * + *

The generation advances on every rebalance; a member holding an + * assignment from an older generation is fenced by the server when it polls. + * An assignment with no partitions still means the client is a group member, + * just one that currently owns nothing. + * + * @param generation the group generation this assignment belongs to + * @param partitions the partition IDs this member may poll, possibly empty + */ +public record ConsumerGroupAssignment(long generation, List partitions) {} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/exception/IggyErrorCode.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/exception/IggyErrorCode.java index 426c4f5f71..3780efc318 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/exception/IggyErrorCode.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/exception/IggyErrorCode.java @@ -34,10 +34,12 @@ public enum IggyErrorCode { ERROR(1), INVALID_COMMAND(3), INVALID_FORMAT(4), - FEATURE_UNAVAILABLE(6), + FEATURE_UNAVAILABLE(5), + INVALID_IDENTIFIER(6), CANNOT_PARSE_INT(7), CANNOT_PARSE_SLICE(8), CANNOT_PARSE_UTF8(9), + STALE_CLIENT(30), // Resource errors RESOURCE_NOT_FOUND(20), @@ -59,6 +61,8 @@ public enum IggyErrorCode { CLIENT_NOT_FOUND(52), INVALID_PAT_TOKEN(53), PAT_NAME_ALREADY_EXISTS(54), + TRANSIENT_NOT_COMMITTED(57), + TRANSIENT_NOT_ACCEPTED(58), PASSWORD_DOES_NOT_MATCH(77), PASSWORD_HASH_INTERNAL_ERROR(78), @@ -102,6 +106,9 @@ public enum IggyErrorCode { INVALID_MESSAGE_CHECKSUM(7003), MESSAGE_NOT_FOUND(7004), + // VSR protocol errors + INCOMPATIBLE_PROTOCOL_VERSION(14003), + // Unknown error code UNKNOWN(-1); diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/exception/IggyValidationException.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/exception/IggyValidationException.java index 84e0b8eb41..02ae079752 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/exception/IggyValidationException.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/exception/IggyValidationException.java @@ -35,6 +35,7 @@ public class IggyValidationException extends IggyServerException { IggyErrorCode.INVALID_COMMAND, IggyErrorCode.INVALID_FORMAT, IggyErrorCode.FEATURE_UNAVAILABLE, + IggyErrorCode.INVALID_IDENTIFIER, IggyErrorCode.CANNOT_PARSE_INT, IggyErrorCode.CANNOT_PARSE_SLICE, IggyErrorCode.CANNOT_PARSE_UTF8, diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/hash/XxHash32.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/hash/XxHash32.java new file mode 100644 index 0000000000..b28ea6ec40 --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/hash/XxHash32.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.hash; + +/** + * XXH32 (32-bit xxHash), one-shot, seed 0. + * + *

Vendored so message-key partitioning hashes identically across every + * Iggy SDK and the server (the Rust side uses {@code twox_hash::XxHash32} + * with seed 0). The routing contract is + * {@code xxh32(key, 0) % partitionCount}, so any deviation here silently + * breaks per-key ordering against other clients. + */ +public final class XxHash32 { + + private static final int PRIME_1 = 0x9E3779B1; + private static final int PRIME_2 = 0x85EBCA77; + private static final int PRIME_3 = 0xC2B2AE3D; + private static final int PRIME_4 = 0x27D4EB2F; + private static final int PRIME_5 = 0x165667B1; + + private XxHash32() {} + + /** + * Hashes the given bytes with XXH32, seed 0. + * + * @param data the bytes to hash + * @return the 32-bit hash, as an unsigned value in a long + */ + public static long hashUnsigned(byte[] data) { + return Integer.toUnsignedLong(hash(data)); + } + + static int hash(byte[] data) { + final int length = data.length; + int offset = 0; + int hash; + + if (length >= 16) { + int v1 = PRIME_1 + PRIME_2; + int v2 = PRIME_2; + int v3 = 0; + int v4 = -PRIME_1; + for (; offset <= length - 16; offset += 16) { + v1 = round(v1, readIntLE(data, offset)); + v2 = round(v2, readIntLE(data, offset + 4)); + v3 = round(v3, readIntLE(data, offset + 8)); + v4 = round(v4, readIntLE(data, offset + 12)); + } + hash = Integer.rotateLeft(v1, 1) + + Integer.rotateLeft(v2, 7) + + Integer.rotateLeft(v3, 12) + + Integer.rotateLeft(v4, 18); + } else { + hash = PRIME_5; + } + + hash += length; + + for (; offset <= length - 4; offset += 4) { + hash = Integer.rotateLeft(hash + readIntLE(data, offset) * PRIME_3, 17) * PRIME_4; + } + for (; offset < length; offset++) { + hash = Integer.rotateLeft(hash + (data[offset] & 0xFF) * PRIME_5, 11) * PRIME_1; + } + + hash ^= hash >>> 15; + hash *= PRIME_2; + hash ^= hash >>> 13; + hash *= PRIME_3; + hash ^= hash >>> 16; + return hash; + } + + private static int round(int accumulator, int lane) { + return Integer.rotateLeft(accumulator + lane * PRIME_2, 13) * PRIME_1; + } + + private static int readIntLE(byte[] data, int offset) { + return (data[offset] & 0xFF) + | (data[offset + 1] & 0xFF) << 8 + | (data[offset + 2] & 0xFF) << 16 + | (data[offset + 3] & 0xFF) << 24; + } +} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/SendConfirmation.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/SendConfirmation.java new file mode 100644 index 0000000000..836c348526 --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/SendConfirmation.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.message; + +import java.math.BigInteger; + +/** + * Confirmation of a committed send: the partition the batch landed on and the + * offset of its first message. + * + * @param streamId the numeric stream ID the batch was written to + * @param topicId the numeric topic ID the batch was written to + * @param partitionId the partition the batch landed on + * @param baseOffset the offset assigned to the first message of the batch + */ +public record SendConfirmation(Long streamId, Long topicId, Long partitionId, BigInteger baseOffset) {} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/SendMessagesResponse.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/SendMessagesResponse.java new file mode 100644 index 0000000000..79519d374f --- /dev/null +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/message/SendMessagesResponse.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.message; + +import java.util.List; + +/** + * Server reply to a send: one {@link SendConfirmation} per committed batch. + * + *

An empty confirmation list means the server acknowledged the send without + * reporting where it landed; the messages were still committed. + * + * @param confirmations the committed-batch confirmations, possibly empty + */ +public record SendMessagesResponse(List confirmations) { + + public static SendMessagesResponse empty() { + return new SendMessagesResponse(List.of()); + } +} diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/BytesDeserializer.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/BytesDeserializer.java index 67afa08523..623d62b36e 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/BytesDeserializer.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/BytesDeserializer.java @@ -27,6 +27,7 @@ import org.apache.iggy.cluster.ClusterNodeStatus; import org.apache.iggy.cluster.TransportEndpoints; import org.apache.iggy.consumergroup.ConsumerGroup; +import org.apache.iggy.consumergroup.ConsumerGroupAssignment; import org.apache.iggy.consumergroup.ConsumerGroupDetails; import org.apache.iggy.consumergroup.ConsumerGroupMember; import org.apache.iggy.consumeroffset.ConsumerOffsetInfo; @@ -38,6 +39,8 @@ import org.apache.iggy.message.Message; import org.apache.iggy.message.MessageHeader; import org.apache.iggy.message.PolledMessages; +import org.apache.iggy.message.SendConfirmation; +import org.apache.iggy.message.SendMessagesResponse; import org.apache.iggy.partition.Partition; import org.apache.iggy.personalaccesstoken.PersonalAccessTokenInfo; import org.apache.iggy.personalaccesstoken.RawPersonalAccessToken; @@ -74,6 +77,8 @@ */ public final class BytesDeserializer { + private static final int CONSUMER_GROUP_ASSIGNMENT_ENTRY_BYTES = Integer.BYTES; + private static final int SEND_CONFIRMATION_BYTES = 3 * Integer.BYTES + Long.BYTES; private static final int MIN_CLUSTER_NODE_BYTES = 18; private BytesDeserializer() {} @@ -177,6 +182,23 @@ public static ConsumerGroup readConsumerGroup(ByteBuf response) { return new ConsumerGroup(groupId, name, partitionsCount, membersCount); } + public static ConsumerGroupAssignment readConsumerGroupAssignment(ByteBuf response) { + // The generation is a monotonic rebalance counter compared only for + // equality, so reading the u64 as a signed long is safe. + var generation = response.readLongLE(); + var partitionsCount = response.readUnsignedIntLE(); + int capacity = validatedCollectionSize( + partitionsCount, + response.readableBytes(), + CONSUMER_GROUP_ASSIGNMENT_ENTRY_BYTES, + "Consumer group partitions count"); + List partitions = new ArrayList<>(capacity); + for (long i = 0; i < partitionsCount; i++) { + partitions.add(response.readUnsignedIntLE()); + } + return new ConsumerGroupAssignment(generation, partitions); + } + public static ConsumerOffsetInfo readConsumerOffsetInfo(ByteBuf response) { var partitionId = response.readUnsignedIntLE(); var currentOffset = readU64AsBigInteger(response); @@ -184,6 +206,28 @@ public static ConsumerOffsetInfo readConsumerOffsetInfo(ByteBuf response) { return new ConsumerOffsetInfo(partitionId, currentOffset, storedOffset); } + public static SendMessagesResponse readSendMessagesResponse(ByteBuf response) { + if (!response.isReadable()) { + return SendMessagesResponse.empty(); + } + var confirmationsCount = response.readUnsignedIntLE(); + int capacity = validatedCollectionSize( + confirmationsCount, response.readableBytes(), SEND_CONFIRMATION_BYTES, "Send confirmations count"); + var confirmations = new ArrayList(capacity); + for (long i = 0; i < confirmationsCount; i++) { + var streamId = response.readUnsignedIntLE(); + var topicId = response.readUnsignedIntLE(); + var partitionId = response.readUnsignedIntLE(); + var baseOffset = readU64AsBigInteger(response); + confirmations.add(new SendConfirmation(streamId, topicId, partitionId, baseOffset)); + } + if (response.isReadable()) { + throw new IggyMalformedResponseException( + "send messages response has " + response.readableBytes() + " trailing bytes"); + } + return new SendMessagesResponse(confirmations); + } + public static PolledMessages readPolledMessages(ByteBuf response) { var partitionId = response.readUnsignedIntLE(); var currentOffset = readU64AsBigInteger(response); @@ -495,6 +539,14 @@ private static String readU32PrefixedString(ByteBuf buffer, String field) { return buffer.readCharSequence(toInt(length), StandardCharsets.UTF_8).toString(); } + private static int validatedCollectionSize(long count, int readableBytes, int entryBytes, String field) { + if (count > readableBytes / entryBytes) { + throw new IggyMalformedResponseException( + field + " " + count + " exceeds remaining payload of " + readableBytes + " bytes"); + } + return Math.toIntExact(count); + } + static BigInteger readU64AsBigInteger(ByteBuf buffer) { var bytesArray = new byte[8]; buffer.readBytes(bytesArray, 0, 8); diff --git a/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/CommandCode.java b/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/CommandCode.java index 337494d185..4c2e56ceac 100644 --- a/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/CommandCode.java +++ b/foreign/java/java-sdk/src/main/java/org/apache/iggy/serde/CommandCode.java @@ -183,7 +183,8 @@ enum ConsumerGroup implements CommandCode { CREATE(602), DELETE(603), JOIN(604), - LEAVE(605); + LEAVE(605), + SYNC(606); private final int value; diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/BaseIntegrationTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/BaseIntegrationTest.java index ad1d3f1646..fd24f171c2 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/BaseIntegrationTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/BaseIntegrationTest.java @@ -32,6 +32,21 @@ import java.util.List; +/** + * Base for integration tests. The SDK speaks the VSR wire protocol, so the + * server under test must support it. + * + *

With {@code USE_EXTERNAL_SERVER} set, tests target an externally + * started VSR server on localhost, running standalone (single-node) mode. + * Start it from the repo root: + * 

{@code
+ * IGGY_ROOT_USERNAME=iggy IGGY_ROOT_PASSWORD=iggy cargo run --bin iggy-server
+ * }
+ * + *

Otherwise a server container is started via testcontainers. This works + * once the published image ships a VSR-capable server; until then the + * external-server mode is the only one that can pass. + */ @Testcontainers public abstract class BaseIntegrationTest { @@ -58,6 +73,10 @@ public static String serverHost() { static void setupContainer() { ResourceLeakDetector.setLevel(ResourceLeakDetector.Level.PARANOID); if (!USE_EXTERNAL_SERVER) { + // The published image still ships the legacy server, which does + // not speak the VSR wire protocol, so tests against this + // container fail until a VSR-capable server is released. Use + // USE_EXTERNAL_SERVER until then. log.info("Starting Iggy Server Container..."); iggyServer = new GenericContainer<>(DockerImageName.parse("apache/iggy:edge")) .withExposedPorts(HTTP_PORT, TCP_PORT) diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncClientIntegrationTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncClientIntegrationTest.java index fbe1045d1d..4f1496f4da 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncClientIntegrationTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncClientIntegrationTest.java @@ -27,6 +27,7 @@ import org.apache.iggy.message.Message; import org.apache.iggy.message.Partitioning; import org.apache.iggy.message.PollingStrategy; +import org.apache.iggy.message.SendMessagesResponse; import org.apache.iggy.topic.CompressionAlgorithm; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -218,7 +219,7 @@ void shouldSendAndPollLargeVolume() throws Exception { .get(TIMEOUT_SECONDS, TimeUnit.SECONDS); // when — send messages in concurrent batches - List> sendFutures = new ArrayList<>(); + List> sendFutures = new ArrayList<>(); for (int batch = 0; batch < 10; batch++) { List batchMessages = new ArrayList<>(); for (int i = 0; i < 10; i++) { diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncConnectionPoolAuthTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncConnectionPoolAuthTest.java index ce7fd59f6b..5a5c4e5001 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncConnectionPoolAuthTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncConnectionPoolAuthTest.java @@ -25,6 +25,7 @@ import org.apache.iggy.identifier.StreamId; import org.apache.iggy.message.Message; import org.apache.iggy.message.Partitioning; +import org.apache.iggy.message.SendMessagesResponse; import org.apache.iggy.topic.CompressionAlgorithm; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -46,9 +47,9 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; /** - * Integration tests for connection pool authentication lifecycle. + * Integration tests for connection authentication lifecycle. * Verifies that lazy per-channel authentication works correctly across - * login, logout, and re-login cycles with a pooled connection. + * login, logout, and re-login cycles on the pooled connection. */ @DisplayName("Connection Pool Authentication") class AsyncConnectionPoolAuthTest extends BaseIntegrationTest { @@ -56,7 +57,7 @@ class AsyncConnectionPoolAuthTest extends BaseIntegrationTest { private static final String USERNAME = "iggy"; private static final String PASSWORD = "iggy"; - private static final int POOL_SIZE = 3; + private static final int CONCURRENCY = 3; private AsyncIggyTcpClient client; @@ -65,7 +66,6 @@ void setUp() throws Exception { client = AsyncIggyTcpClient.builder() .host(serverHost()) .port(serverTcpPort()) - .connectionPoolSize(POOL_SIZE) .build(); client.connect().get(5, TimeUnit.SECONDS); } @@ -78,9 +78,15 @@ void tearDown() throws Exception { } @Test - @DisplayName("should reject commands before login") - void shouldRejectCommandsBeforeLogin() { - // when/then + @DisplayName("should allow only bootstrap commands before login") + void shouldAllowOnlyBootstrapCommandsBeforeLogin() throws Exception { + // when + var ping = client.system().ping().get(5, TimeUnit.SECONDS); + var metadata = client.system().getClusterMetadata().get(5, TimeUnit.SECONDS); + + // then + assertThat(ping).isEqualTo("pong"); + assertThat(metadata).isNotNull(); assertThatThrownBy(() -> client.streams().getStreams().get(5, TimeUnit.SECONDS)) .isInstanceOf(ExecutionException.class) .hasCauseInstanceOf(IggyNotConnectedException.class); @@ -159,10 +165,10 @@ void shouldAuthenticatePoolChannelsLazily() throws Exception { "test-topic") .get(5, TimeUnit.SECONDS); - // when - fire more concurrent requests than the pool size to force - // multiple channels to be created and lazily authenticated - int concurrentRequests = POOL_SIZE * 3; - List> futures = new ArrayList<>(); + // when - fire a burst of concurrent requests to exercise lazy + // authentication on the shared channel + int concurrentRequests = CONCURRENCY * 3; + List> futures = new ArrayList<>(); for (int i = 0; i < concurrentRequests; i++) { var future = client.messages() .sendMessages( @@ -200,8 +206,8 @@ void shouldReAuthenticateStaleChannelsAfterReLogin() throws Exception { "test-topic") .get(5, TimeUnit.SECONDS); - List> warmupFutures = new ArrayList<>(); - for (int i = 0; i < POOL_SIZE * 2; i++) { + List> warmupFutures = new ArrayList<>(); + for (int i = 0; i < CONCURRENCY * 2; i++) { warmupFutures.add(client.messages() .sendMessages( StreamId.of(streamName), @@ -218,8 +224,8 @@ void shouldReAuthenticateStaleChannelsAfterReLogin() throws Exception { log.info("Logout + re-login complete"); // then - all channels should re-authenticate transparently - List> postReLoginFutures = new ArrayList<>(); - for (int i = 0; i < POOL_SIZE * 2; i++) { + List> postReLoginFutures = new ArrayList<>(); + for (int i = 0; i < CONCURRENCY * 2; i++) { postReLoginFutures.add(client.messages() .sendMessages( StreamId.of(streamName), diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncConsumerGroupsTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncConsumerGroupsTest.java index f1cc13111c..5a60691cd9 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncConsumerGroupsTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/AsyncConsumerGroupsTest.java @@ -71,7 +71,10 @@ public class AsyncConsumerGroupsTest extends BaseIntegrationTest { @BeforeAll public static void setup() throws Exception { log.info("Setting up async consumer groups test"); - client = new AsyncIggyTcpClient(serverHost(), serverTcpPort()); + client = AsyncIggyTcpClient.builder() + .host(serverHost()) + .port(serverTcpPort()) + .build(); client.connect() .thenCompose(v -> client.users().login(USERNAME, PASSWORD)) @@ -166,7 +169,10 @@ void shouldHandleMultipleClientsJoiningGroup() throws Exception { .get(TIMEOUT_SECONDS, TimeUnit.SECONDS); ConsumerId groupId = ConsumerId.of(created.id()); - AsyncIggyTcpClient secondClient = new AsyncIggyTcpClient(serverHost(), serverTcpPort()); + AsyncIggyTcpClient secondClient = AsyncIggyTcpClient.builder() + .host(serverHost()) + .port(serverTcpPort()) + .build(); try { secondClient .connect() @@ -311,8 +317,14 @@ void shouldHandleConcurrentJoinAndLeaveOperations() throws Exception { .get(TIMEOUT_SECONDS, TimeUnit.SECONDS); ConsumerId groupId = ConsumerId.of(created.id()); - AsyncIggyTcpClient secondClient = new AsyncIggyTcpClient(serverHost(), serverTcpPort()); - AsyncIggyTcpClient thirdClient = new AsyncIggyTcpClient(serverHost(), serverTcpPort()); + AsyncIggyTcpClient secondClient = AsyncIggyTcpClient.builder() + .host(serverHost()) + .port(serverTcpPort()) + .build(); + AsyncIggyTcpClient thirdClient = AsyncIggyTcpClient.builder() + .host(serverHost()) + .port(serverTcpPort()) + .build(); try { secondClient .connect() diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientBuilderTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientBuilderTest.java index 6f33b106a1..ce7234a35f 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientBuilderTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientBuilderTest.java @@ -147,24 +147,6 @@ void shouldThrowExceptionForNullPort() { assertThatThrownBy(builder::build).isInstanceOf(IggyInvalidArgumentException.class); } - @Test - void shouldThrowExceptionForZeroConnectionPoolSize() { - // Given: Builder with 0 connection pool size - AsyncIggyTcpClientBuilder builder = AsyncIggyTcpClient.builder().connectionPoolSize(0); - - // When/Then: Building should throw IggyInvalidArgumentException - assertThatThrownBy(builder::build).isInstanceOf(IggyInvalidArgumentException.class); - } - - @Test - void shouldThrowExceptionForNegativeConnectionPoolSize() { - // Given: Builder with negative connection pool size - AsyncIggyTcpClientBuilder builder = AsyncIggyTcpClient.builder().connectionPoolSize(-1); - - // When/Then: Building should throw IggyInvalidArgumentException - assertThatThrownBy(builder::build).isInstanceOf(IggyInvalidArgumentException.class); - } - @Test void shouldThrowExceptionForZeroConnectionTimeout() { // Given: Builder with 0 connection timeout @@ -211,6 +193,63 @@ void shouldThrowExceptionForNegativeAcquireTimeout() { assertThatThrownBy(builder::build).isInstanceOf(IggyInvalidArgumentException.class); } + @Test + void shouldThrowExceptionForZeroRequestTimeout() { + AsyncIggyTcpClientBuilder builder = AsyncIggyTcpClient.builder().requestTimeout(Duration.ZERO); + + assertThatThrownBy(builder::build).isInstanceOf(IggyInvalidArgumentException.class); + } + + @Test + void shouldThrowExceptionForNegativeRequestTimeout() { + AsyncIggyTcpClientBuilder builder = AsyncIggyTcpClient.builder().requestTimeout(Duration.ofMillis(-1)); + + assertThatThrownBy(builder::build).isInstanceOf(IggyInvalidArgumentException.class); + } + + @Test + void shouldAcceptCustomMaximumVsrFrameSize() { + client = + AsyncIggyTcpClient.builder().maxVsrFrameSize(128L * 1024 * 1024).build(); + + assertThat(client).isNotNull(); + } + + @Test + void shouldRejectMaximumVsrFrameSizeBelowHeader() { + AsyncIggyTcpClientBuilder builder = AsyncIggyTcpClient.builder().maxVsrFrameSize(255); + + assertThatThrownBy(builder::build).isInstanceOf(IggyInvalidArgumentException.class); + } + + @Test + void shouldRejectNonPositiveMaximumVsrFrameSize() { + assertThatThrownBy(() -> AsyncIggyTcpClient.builder().maxVsrFrameSize(0).build()) + .isInstanceOf(IggyInvalidArgumentException.class); + assertThatThrownBy( + () -> AsyncIggyTcpClient.builder().maxVsrFrameSize(-1).build()) + .isInstanceOf(IggyInvalidArgumentException.class); + } + + @Test + void shouldRejectMaximumVsrFrameSizeAboveJavaBufferLimit() { + AsyncIggyTcpClientBuilder builder = AsyncIggyTcpClient.builder().maxVsrFrameSize((long) Integer.MAX_VALUE + 1); + + assertThatThrownBy(builder::build).isInstanceOf(IggyInvalidArgumentException.class); + } + + @Test + void shouldRejectNonPositiveHeartbeatInterval() { + assertThatThrownBy(() -> AsyncIggyTcpClient.builder() + .heartbeatInterval(Duration.ZERO) + .build()) + .isInstanceOf(IggyInvalidArgumentException.class); + assertThatThrownBy(() -> AsyncIggyTcpClient.builder() + .heartbeatInterval(Duration.ofMillis(-1)) + .build()) + .isInstanceOf(IggyInvalidArgumentException.class); + } + @Test void shouldMaintainBackwardCompatibilityWithOldConstructor() throws Exception { // Given: Old constructor approach @@ -448,18 +487,6 @@ void testBuildClientWithRequestTimeout() throws Exception { assertThat(client.users()).isNotNull(); } - @Test - void testBuildClientWithConnectionPoolSize() throws Exception { - client = AsyncIggyTcpClient.builder() - .host(serverHost()) - .port(serverTcpPort()) - .connectionPoolSize(5) - .build(); - client.connect().get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS); - - assertThat(client.users()).isNotNull(); - } - @Test void testBuildClientWithExponentialBackoffRetryPolicy() throws Exception { client = AsyncIggyTcpClient.builder() @@ -527,7 +554,6 @@ void testBuildClientWithAllConfigurationOptions() throws Exception { .credentials(TEST_USERNAME, TEST_PASSWORD) .connectionTimeout(Duration.ofSeconds(10)) .requestTimeout(Duration.ofSeconds(30)) - .connectionPoolSize(5) .retryPolicy(RetryPolicy.exponentialBackoff()) .tls(false) .build(); diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientLoginRoutingTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientLoginRoutingTest.java new file mode 100644 index 0000000000..b03974154f --- /dev/null +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientLoginRoutingTest.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.client.async.tcp; + +import org.apache.iggy.client.ConnectionInfo; +import org.apache.iggy.user.IdentityInfo; +import org.junit.jupiter.api.Test; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Queue; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class AsyncIggyTcpClientLoginRoutingTest { + + private static final IdentityInfo FIRST_IDENTITY = new IdentityInfo(1L, Optional.empty()); + private static final IdentityInfo SECOND_IDENTITY = new IdentityInfo(2L, Optional.empty()); + + @Test + void shouldLoginOnceAfterAllLeaderRedirects() { + var client = new TestClient(); + client.enqueueLeader(new ConnectionInfo("leader-a", 8091)); + client.enqueueLeader(new ConnectionInfo("leader-b", 8092)); + client.enqueueStay(); + var attempts = new AtomicInteger(); + + var identity = client.loginOnLeader(() -> { + client.events.add("login"); + attempts.incrementAndGet(); + return CompletableFuture.completedFuture(FIRST_IDENTITY); + }) + .join(); + + assertThat(identity).isEqualTo(FIRST_IDENTITY); + assertThat(attempts).hasValue(1); + assertThat(client.retargets) + .containsExactly(new ConnectionInfo("leader-a", 8091), new ConnectionInfo("leader-b", 8092)); + assertThat(client.events).containsExactly("discover", "retarget", "discover", "retarget", "discover", "login"); + } + + @Test + void shouldSerializeDiscoveryAndLoginAcrossConcurrentCalls() { + var client = new TestClient(); + client.enqueueStay(); + client.enqueueStay(); + var firstAttempt = new CompletableFuture(); + + var firstLogin = client.loginOnLeader(() -> { + client.events.add("first-login"); + return firstAttempt; + }); + var secondLogin = client.loginOnLeader(() -> { + client.events.add("second-login"); + return CompletableFuture.completedFuture(SECOND_IDENTITY); + }); + + assertThat(client.events).containsExactly("discover", "first-login"); + assertThat(secondLogin).isNotDone(); + + firstAttempt.complete(FIRST_IDENTITY); + + assertThat(firstLogin.join()).isEqualTo(FIRST_IDENTITY); + assertThat(secondLogin.join()).isEqualTo(SECOND_IDENTITY); + assertThat(client.events).containsExactly("discover", "first-login", "discover", "second-login"); + } + + @Test + void shouldReleaseSerializationGateAfterRoutingFailure() { + var routingFailure = new IllegalStateException("metadata failed"); + var client = new TestClient(); + client.enqueueDiscovery(CompletableFuture.failedFuture(routingFailure)); + client.enqueueStay(); + var failedAttemptCount = new AtomicInteger(); + + var failedLogin = client.loginOnLeader(() -> { + failedAttemptCount.incrementAndGet(); + return CompletableFuture.completedFuture(FIRST_IDENTITY); + }); + var nextLogin = client.loginOnLeader(() -> CompletableFuture.completedFuture(SECOND_IDENTITY)); + + assertThatThrownBy(failedLogin::join) + .isInstanceOf(CompletionException.class) + .hasCause(routingFailure); + assertThat(failedAttemptCount).hasValue(0); + assertThat(nextLogin.join()).isEqualTo(SECOND_IDENTITY); + } + + @Test + void shouldReleaseSerializationGateAfterLoginFailure() { + var loginFailure = new IllegalStateException("register failed"); + var client = new TestClient(); + client.enqueueStay(); + client.enqueueStay(); + + var failedLogin = client.loginOnLeader(() -> CompletableFuture.failedFuture(loginFailure)); + var nextLogin = client.loginOnLeader(() -> CompletableFuture.completedFuture(SECOND_IDENTITY)); + + assertThatThrownBy(failedLogin::join) + .isInstanceOf(CompletionException.class) + .hasCause(loginFailure); + assertThat(nextLogin.join()).isEqualTo(SECOND_IDENTITY); + } + + @Test + void shouldKeepGateUntilCancelledCallFinishesItsRegister() { + var client = new TestClient(); + client.enqueueStay(); + client.enqueueStay(); + var committedRegister = new CompletableFuture(); + + var cancelledLogin = client.loginOnLeader(() -> committedRegister); + cancelledLogin.cancel(false); + var nextLogin = client.loginOnLeader(() -> CompletableFuture.completedFuture(SECOND_IDENTITY)); + + assertThat(nextLogin).isNotDone(); + + committedRegister.complete(FIRST_IDENTITY); + + assertThat(nextLogin.join()).isEqualTo(SECOND_IDENTITY); + } + + private static final class TestClient extends AsyncIggyTcpClient { + private final Queue>> discoveries = new ArrayDeque<>(); + private final List retargets = new ArrayList<>(); + private final List events = new ArrayList<>(); + + private TestClient() { + super("seed", 8090); + } + + private void enqueueLeader(ConnectionInfo target) { + enqueueDiscovery(CompletableFuture.completedFuture(Optional.of(target))); + } + + private void enqueueStay() { + enqueueDiscovery(CompletableFuture.completedFuture(Optional.empty())); + } + + private void enqueueDiscovery(CompletableFuture> discovery) { + discoveries.add(discovery); + } + + @Override + CompletableFuture> findLeaderElsewhere(ConnectionInfo currentTarget) { + events.add("discover"); + return discoveries.remove(); + } + + @Override + CompletableFuture retarget(ConnectionInfo newTarget) { + events.add("retarget"); + retargets.add(newTarget); + return CompletableFuture.completedFuture(null); + } + } +} diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java new file mode 100644 index 0000000000..ba481bec98 --- /dev/null +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncIggyTcpClientTransientFailoverTest.java @@ -0,0 +1,337 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.client.async.tcp; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import org.apache.iggy.exception.IggyServerException; +import org.junit.jupiter.api.Test; + +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class AsyncIggyTcpClientTransientFailoverTest { + private static final int HEADER_SIZE = 256; + private static final int SIZE_OFFSET = 48; + private static final int COMMAND_OFFSET = 60; + private static final int REQUEST_ID_OFFSET = 168; + private static final int REQUEST_OPERATION_OFFSET = 176; + private static final int REQUEST_CODE_OFFSET = 196; + private static final int REPLY_REQUEST_ID_OFFSET = 200; + private static final int REPLY_OPERATION_OFFSET = 208; + private static final int REPLY_STATUS_OFFSET = 216; + private static final int EVICTION_REASON_OFFSET = 255; + + private static final int COMMAND_REPLY = 8; + private static final int COMMAND_EVICTION = 13; + private static final int OPERATION_REGISTER = 1; + private static final int OPERATION_NON_REPLICATED = 2; + private static final int OPERATION_CREATE_STREAM = 128; + private static final int GET_CLUSTER_METADATA_CODE = 12; + private static final int CREATE_STREAM_CODE = 202; + private static final int TRANSIENT_NOT_ACCEPTED = 58; + private static final int EVICTION_STALE_CLIENT = 13; + + @Test + void shouldRecheckLeaderAndReplayNotAcceptedMutation() throws Exception { + InetAddress loopback = InetAddress.getLoopbackAddress(); + try (ServerSocket oldLeaderSocket = new ServerSocket(0, 1, loopback); + ServerSocket newLeaderSocket = new ServerSocket(0, 1, loopback)) { + AtomicInteger denials = new AtomicInteger(); + AtomicInteger retriedMutations = new AtomicInteger(); + CompletableFuture oldLeader = serve( + oldLeaderSocket, + request -> handleOldLeader( + request, oldLeaderSocket.getLocalPort(), newLeaderSocket.getLocalPort(), denials)); + CompletableFuture newLeader = serve( + newLeaderSocket, + request -> handleNewLeader( + request, oldLeaderSocket.getLocalPort(), newLeaderSocket.getLocalPort(), retriedMutations)); + + AsyncIggyTcpClient client = AsyncIggyTcpClient.builder() + .host(loopback.getHostAddress()) + .port(oldLeaderSocket.getLocalPort()) + .credentials("iggy", "iggy") + .requestTimeout(Duration.ofSeconds(10)) + .build(); + try { + client.connect().get(5, TimeUnit.SECONDS); + client.login().get(5, TimeUnit.SECONDS); + + byte[] response = client.sendBinaryRequest(CREATE_STREAM_CODE, new byte[0]) + .get(10, TimeUnit.SECONDS); + + assertThat(response).isEmpty(); + assertThat(client.getConnectionInfo().port()).isEqualTo(newLeaderSocket.getLocalPort()); + assertThat(denials).hasValueGreaterThan(1); + assertThat(retriedMutations).hasValue(1); + } finally { + client.close().get(5, TimeUnit.SECONDS); + } + oldLeader.get(5, TimeUnit.SECONDS); + newLeader.get(5, TimeUnit.SECONDS); + } + } + + @Test + void shouldReplayTransientImplicitLoginAfterEviction() throws Exception { + InetAddress loopback = InetAddress.getLoopbackAddress(); + try (ServerSocket serverSocket = new ServerSocket(0, 1, loopback)) { + AtomicInteger registrations = new AtomicInteger(); + AtomicInteger mutations = new AtomicInteger(); + CompletableFuture server = serve(serverSocket, 2, request -> { + if (request.is(GET_CLUSTER_METADATA_CODE, OPERATION_NON_REPLICATED)) { + return Response.success(OPERATION_NON_REPLICATED, singleNodeMetadata(serverSocket.getLocalPort())); + } + if (request.operation() == OPERATION_REGISTER) { + int attempt = registrations.incrementAndGet(); + if (attempt == 2) { + return Response.success(OPERATION_REGISTER, transientResult(TRANSIENT_NOT_ACCEPTED)); + } + return Response.success(OPERATION_REGISTER, registerBody(attempt)); + } + if (request.operation() == OPERATION_CREATE_STREAM) { + if (mutations.incrementAndGet() == 1) { + return Response.eviction(EVICTION_STALE_CLIENT); + } + ByteBuf body = Unpooled.buffer(Integer.BYTES); + body.writeIntLE(0); + return Response.success(OPERATION_CREATE_STREAM, body); + } + throw new IllegalStateException("Unexpected request: " + request); + }); + + AsyncIggyTcpClient client = AsyncIggyTcpClient.builder() + .host(loopback.getHostAddress()) + .port(serverSocket.getLocalPort()) + .credentials("iggy", "iggy") + .requestTimeout(Duration.ofSeconds(5)) + .build(); + try { + client.connect().get(5, TimeUnit.SECONDS); + client.login().get(5, TimeUnit.SECONDS); + + assertThatThrownBy(() -> client.sendBinaryRequest(CREATE_STREAM_CODE, new byte[0]) + .get(5, TimeUnit.SECONDS)) + .hasCauseInstanceOf(IggyServerException.class); + + assertThat(client.sendBinaryRequest(CREATE_STREAM_CODE, new byte[0]) + .get(5, TimeUnit.SECONDS)) + .isEmpty(); + assertThat(registrations).hasValue(3); + assertThat(mutations).hasValue(2); + } finally { + client.close().get(5, TimeUnit.SECONDS); + } + server.get(5, TimeUnit.SECONDS); + } + } + + private static Response handleOldLeader( + Request request, int oldLeaderPort, int newLeaderPort, AtomicInteger denials) { + if (request.is(GET_CLUSTER_METADATA_CODE, OPERATION_NON_REPLICATED)) { + boolean demoted = denials.get() > 0; + return Response.success( + OPERATION_NON_REPLICATED, + clusterMetadata(oldLeaderPort, newLeaderPort, demoted ? newLeaderPort : oldLeaderPort)); + } + if (request.operation() == OPERATION_REGISTER) { + return Response.success(OPERATION_REGISTER, registerBody(1)); + } + if (request.operation() == OPERATION_CREATE_STREAM) { + denials.incrementAndGet(); + return Response.error(OPERATION_CREATE_STREAM, TRANSIENT_NOT_ACCEPTED); + } + throw new IllegalStateException("Unexpected request to old leader: " + request); + } + + private static Response handleNewLeader( + Request request, int oldLeaderPort, int newLeaderPort, AtomicInteger retriedMutations) { + if (request.is(GET_CLUSTER_METADATA_CODE, OPERATION_NON_REPLICATED)) { + return Response.success( + OPERATION_NON_REPLICATED, clusterMetadata(oldLeaderPort, newLeaderPort, newLeaderPort)); + } + if (request.operation() == OPERATION_REGISTER) { + return Response.success(OPERATION_REGISTER, registerBody(2)); + } + if (request.operation() == OPERATION_CREATE_STREAM) { + retriedMutations.incrementAndGet(); + ByteBuf body = Unpooled.buffer(Integer.BYTES); + body.writeIntLE(0); + return Response.success(OPERATION_CREATE_STREAM, body); + } + throw new IllegalStateException("Unexpected request to new leader: " + request); + } + + private static CompletableFuture serve(ServerSocket server, RequestHandler handler) { + return serve(server, 1, handler); + } + + private static CompletableFuture serve(ServerSocket server, int connectionCount, RequestHandler handler) { + return CompletableFuture.runAsync(() -> { + try { + for (int connection = 0; connection < connectionCount; connection++) { + try (Socket socket = server.accept()) { + InputStream input = socket.getInputStream(); + OutputStream output = socket.getOutputStream(); + Request request; + while ((request = readRequest(input)) != null) { + writeResponse(output, request, handler.handle(request)); + } + } + } + } catch (IOException error) { + throw new IllegalStateException("Mock VSR server failed", error); + } + }); + } + + private static Request readRequest(InputStream input) throws IOException { + byte[] header = input.readNBytes(HEADER_SIZE); + if (header.length == 0) { + return null; + } + if (header.length != HEADER_SIZE) { + throw new EOFException("Truncated VSR request header"); + } + ByteBuffer fields = ByteBuffer.wrap(header).order(ByteOrder.LITTLE_ENDIAN); + int size = fields.getInt(SIZE_OFFSET); + byte[] body = input.readNBytes(size - HEADER_SIZE); + if (body.length != size - HEADER_SIZE) { + throw new EOFException("Truncated VSR request body"); + } + return new Request( + Byte.toUnsignedInt(header[REQUEST_OPERATION_OFFSET]), + fields.getInt(REQUEST_CODE_OFFSET), + fields.getLong(REQUEST_ID_OFFSET)); + } + + private static void writeResponse(OutputStream output, Request request, Response response) throws IOException { + byte[] body = new byte[response.body().readableBytes()]; + response.body().readBytes(body); + response.body().release(); + byte[] header = new byte[HEADER_SIZE]; + ByteBuffer fields = ByteBuffer.wrap(header).order(ByteOrder.LITTLE_ENDIAN); + fields.putInt(SIZE_OFFSET, HEADER_SIZE + body.length); + header[COMMAND_OFFSET] = (byte) response.command(); + if (response.command() == COMMAND_EVICTION) { + header[EVICTION_REASON_OFFSET] = (byte) response.evictionReason(); + } else { + fields.putLong(REPLY_REQUEST_ID_OFFSET, request.requestId()); + header[REPLY_OPERATION_OFFSET] = (byte) response.operation(); + fields.putInt(REPLY_STATUS_OFFSET, response.status()); + } + output.write(header); + output.write(body); + output.flush(); + } + + private static ByteBuf registerBody(long session) { + ByteBuf body = Unpooled.buffer(); + body.writeIntLE(0); + body.writeIntLE(1); + body.writeLongLE(session); + body.writeIntLE(11 << 10); + body.writeByte(0); + return body; + } + + private static ByteBuf transientResult(int errorCode) { + ByteBuf body = Unpooled.buffer(3 * Integer.BYTES); + body.writeIntLE(1); + body.writeIntLE(0); + body.writeIntLE(errorCode); + return body; + } + + private static ByteBuf singleNodeMetadata(int port) { + ByteBuf body = Unpooled.buffer(); + writeString(body, "test-cluster"); + body.writeIntLE(1); + writeNode(body, "node", port, true); + return body; + } + + private static ByteBuf clusterMetadata(int oldLeaderPort, int newLeaderPort, int leaderPort) { + ByteBuf body = Unpooled.buffer(); + writeString(body, "test-cluster"); + body.writeIntLE(2); + writeNode(body, "old-node", oldLeaderPort, oldLeaderPort == leaderPort); + writeNode(body, "new-node", newLeaderPort, newLeaderPort == leaderPort); + return body; + } + + private static void writeNode(ByteBuf body, String name, int port, boolean leader) { + writeString(body, name); + writeString(body, InetAddress.getLoopbackAddress().getHostAddress()); + body.writeShortLE(port); + body.writeShortLE(0); + body.writeShortLE(0); + body.writeShortLE(0); + body.writeByte(leader ? 0 : 1); + body.writeByte(0); + } + + private static void writeString(ByteBuf body, String value) { + byte[] bytes = value.getBytes(StandardCharsets.UTF_8); + body.writeIntLE(bytes.length); + body.writeBytes(bytes); + } + + private record Request(int operation, int commandCode, long requestId) { + boolean is(int expectedCode, int expectedOperation) { + return commandCode == expectedCode && operation == expectedOperation; + } + } + + private record Response(int command, int operation, int status, int evictionReason, ByteBuf body) { + static Response success(int operation, ByteBuf body) { + return new Response(COMMAND_REPLY, operation, 0, 0, body); + } + + static Response error(int operation, int status) { + return new Response(COMMAND_REPLY, operation, status, 0, Unpooled.EMPTY_BUFFER); + } + + static Response eviction(int reason) { + return new Response(COMMAND_EVICTION, 0, 0, reason, Unpooled.EMPTY_BUFFER); + } + } + + @FunctionalInterface + private interface RequestHandler { + Response handle(Request request); + } +} diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionConcurrencyTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionConcurrencyTest.java new file mode 100644 index 0000000000..442dfd5387 --- /dev/null +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionConcurrencyTest.java @@ -0,0 +1,409 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.client.async.tcp; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import org.junit.jupiter.api.Test; + +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.SocketTimeoutException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class AsyncTcpConnectionConcurrencyTest { + private static final int HEADER_SIZE = 256; + private static final int SIZE_OFFSET = 48; + private static final int COMMAND_OFFSET = 60; + private static final int REQUEST_ID_OFFSET = 168; + private static final int REQUEST_OPERATION_OFFSET = 176; + private static final int REQUEST_CODE_OFFSET = 196; + private static final int REPLY_REQUEST_ID_OFFSET = 200; + private static final int REPLY_OPERATION_OFFSET = 208; + private static final int REPLY_STATUS_OFFSET = 216; + + private static final int COMMAND_REPLY = 8; + private static final int OPERATION_REGISTER = 1; + private static final int OPERATION_NON_REPLICATED = 2; + private static final int OPERATION_LOGOUT = 3; + private static final int OPERATION_SEND_MESSAGES = 160; + private static final int PING_CODE = 1; + private static final int GET_CLUSTER_METADATA_CODE = 12; + private static final int LOGIN_CODE = 38; + private static final int LOGOUT_CODE = 39; + private static final int SEND_MESSAGES_CODE = 101; + private static final int TRANSIENT_NOT_COMMITTED = 57; + + @Test + void shouldCorrelateConcurrentPartitionResponsesInReverseOrder() throws Exception { + InetAddress loopback = InetAddress.getLoopbackAddress(); + try (ServerSocket serverSocket = new ServerSocket(0, 1, loopback)) { + CompletableFuture firstRequestRead = new CompletableFuture<>(); + CompletableFuture server = CompletableFuture.runAsync(() -> { + try (Socket socket = serverSocket.accept()) { + socket.setSoTimeout((int) TimeUnit.SECONDS.toMillis(2)); + InputStream input = socket.getInputStream(); + OutputStream output = socket.getOutputStream(); + + Request register = readRequest(input); + assertThat(register.operation()).isEqualTo(OPERATION_REGISTER); + writeResponse(output, register, registerBody()); + + Request firstRequest = readRequest(input); + assertThat(firstRequest.operation()).isEqualTo(OPERATION_SEND_MESSAGES); + firstRequestRead.complete(null); + Request secondRequest = readRequest(input); + assertThat(secondRequest.operation()).isEqualTo(OPERATION_SEND_MESSAGES); + assertThat(secondRequest.requestId()).isNotEqualTo(firstRequest.requestId()); + + Thread.sleep(200); + writeResponse(output, secondRequest, "second".getBytes(StandardCharsets.UTF_8)); + writeResponse(output, firstRequest, "first".getBytes(StandardCharsets.UTF_8)); + } catch (IOException error) { + firstRequestRead.completeExceptionally(error); + throw new IllegalStateException("Mock VSR server failed", error); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Mock VSR server interrupted", error); + } + }); + + AsyncTcpConnection connection = new AsyncTcpConnection( + loopback.getHostAddress(), + serverSocket.getLocalPort(), + false, + Optional.empty(), + new AsyncTcpConnection.TcpConnectionPoolConfig(1000, 50), + Optional.of(Duration.ofSeconds(1)), + Optional.of(Duration.ofSeconds(2)), + Duration.ofHours(1), + 1024 * 1024, + null, + () -> {}, + ignored -> {}); + try { + connection.connect().get(5, TimeUnit.SECONDS); + connection + .send(LOGIN_CODE, loginPayload()) + .get(5, TimeUnit.SECONDS) + .release(); + + CompletableFuture first = connection.send(SEND_MESSAGES_CODE, sendMessagesPayload()); + firstRequestRead.get(5, TimeUnit.SECONDS); + CompletableFuture second = connection.send(SEND_MESSAGES_CODE, sendMessagesPayload()); + + assertResponse(first, "first"); + assertResponse(second, "second"); + } finally { + connection.close().get(5, TimeUnit.SECONDS); + } + server.get(5, TimeUnit.SECONDS); + } + } + + @Test + void shouldNotStarveHeartbeatWhileApplicationResponseIsPending() throws Exception { + InetAddress loopback = InetAddress.getLoopbackAddress(); + try (ServerSocket serverSocket = new ServerSocket(0, 1, loopback)) { + CompletableFuture applicationRequestRead = new CompletableFuture<>(); + CompletableFuture heartbeatRead = new CompletableFuture<>(); + CompletableFuture server = CompletableFuture.runAsync(() -> { + try (Socket socket = serverSocket.accept()) { + socket.setSoTimeout((int) TimeUnit.SECONDS.toMillis(2)); + InputStream input = socket.getInputStream(); + OutputStream output = socket.getOutputStream(); + + Request register = readRequest(input); + writeResponse(output, register, registerBody()); + + Request application = readRequest(input); + assertThat(application.operation()).isEqualTo(OPERATION_SEND_MESSAGES); + applicationRequestRead.complete(null); + + Request heartbeat = readRequest(input); + assertThat(heartbeat.operation()).isEqualTo(OPERATION_NON_REPLICATED); + assertThat(heartbeat.commandCode()).isEqualTo(PING_CODE); + heartbeatRead.complete(null); + writeResponse(output, heartbeat, new byte[0]); + writeResponse(output, application, "application".getBytes(StandardCharsets.UTF_8)); + } catch (IOException error) { + applicationRequestRead.completeExceptionally(error); + heartbeatRead.completeExceptionally(error); + throw new IllegalStateException("Mock VSR server failed", error); + } + }); + + AsyncTcpConnection connection = newConnection(serverSocket, Duration.ofMillis(300), 50); + try { + connection.connect().get(5, TimeUnit.SECONDS); + connection + .send(LOGIN_CODE, loginPayload()) + .get(5, TimeUnit.SECONDS) + .release(); + + CompletableFuture application = connection.send(SEND_MESSAGES_CODE, sendMessagesPayload()); + applicationRequestRead.get(5, TimeUnit.SECONDS); + heartbeatRead.get(5, TimeUnit.SECONDS); + assertResponse(application, "application"); + } finally { + connection.close().get(5, TimeUnit.SECONDS); + } + server.get(5, TimeUnit.SECONDS); + } + } + + @Test + void shouldReuseCorrelationIdForTransientReplay() throws Exception { + InetAddress loopback = InetAddress.getLoopbackAddress(); + try (ServerSocket serverSocket = new ServerSocket(0, 1, loopback)) { + CompletableFuture server = CompletableFuture.runAsync(() -> { + try (Socket socket = serverSocket.accept()) { + socket.setSoTimeout((int) TimeUnit.SECONDS.toMillis(2)); + InputStream input = socket.getInputStream(); + OutputStream output = socket.getOutputStream(); + + Request register = readRequest(input); + writeResponse(output, register, registerBody()); + + Request firstAttempt = readRequest(input); + writeResponse(output, firstAttempt, TRANSIENT_NOT_COMMITTED, new byte[0]); + Request secondAttempt = readRequest(input); + assertThat(secondAttempt.operation()).isEqualTo(firstAttempt.operation()); + assertThat(secondAttempt.requestId()).isEqualTo(firstAttempt.requestId()); + writeResponse(output, secondAttempt, "retried".getBytes(StandardCharsets.UTF_8)); + } catch (IOException error) { + throw new IllegalStateException("Mock VSR server failed", error); + } + }); + + AsyncTcpConnection connection = newConnection(serverSocket, Duration.ofHours(1), 50); + try { + connection.connect().get(5, TimeUnit.SECONDS); + connection + .send(LOGIN_CODE, loginPayload()) + .get(5, TimeUnit.SECONDS) + .release(); + + assertResponse(connection.send(SEND_MESSAGES_CODE, sendMessagesPayload()), "retried"); + } finally { + connection.close().get(5, TimeUnit.SECONDS); + } + server.get(5, TimeUnit.SECONDS); + } + } + + @Test + void shouldSerializeRegisterAndLogoutThroughTheirResponses() throws Exception { + InetAddress loopback = InetAddress.getLoopbackAddress(); + try (ServerSocket serverSocket = new ServerSocket(0, 1, loopback)) { + CompletableFuture registerRead = new CompletableFuture<>(); + CompletableFuture registerConcurrentSendStarted = new CompletableFuture<>(); + CompletableFuture logoutRead = new CompletableFuture<>(); + CompletableFuture logoutConcurrentSendStarted = new CompletableFuture<>(); + CompletableFuture server = CompletableFuture.runAsync(() -> { + try (Socket socket = serverSocket.accept()) { + socket.setSoTimeout((int) TimeUnit.SECONDS.toMillis(2)); + InputStream input = socket.getInputStream(); + OutputStream output = socket.getOutputStream(); + + Request register = readRequest(input); + registerRead.complete(null); + registerConcurrentSendStarted.get(2, TimeUnit.SECONDS); + assertNoRequest(input, socket); + writeResponse(output, register, registerBody()); + + Request metadataDuringRegister = readRequest(input); + assertThat(metadataDuringRegister.commandCode()).isEqualTo(GET_CLUSTER_METADATA_CODE); + writeResponse(output, metadataDuringRegister, new byte[0]); + + Request logout = readRequest(input); + assertThat(logout.operation()).isEqualTo(OPERATION_LOGOUT); + logoutRead.complete(null); + logoutConcurrentSendStarted.get(2, TimeUnit.SECONDS); + assertNoRequest(input, socket); + writeResponse(output, logout, new byte[0]); + + Request metadataDuringLogout = readRequest(input); + assertThat(metadataDuringLogout.commandCode()).isEqualTo(GET_CLUSTER_METADATA_CODE); + writeResponse(output, metadataDuringLogout, new byte[0]); + } catch (InterruptedException error) { + Thread.currentThread().interrupt(); + registerRead.completeExceptionally(error); + logoutRead.completeExceptionally(error); + throw new IllegalStateException("Mock VSR server interrupted", error); + } catch (IOException | ExecutionException | TimeoutException error) { + registerRead.completeExceptionally(error); + logoutRead.completeExceptionally(error); + throw new IllegalStateException("Mock VSR server failed", error); + } + }); + + AsyncTcpConnection connection = newConnection(serverSocket, Duration.ofHours(1), 500); + try { + connection.connect().get(5, TimeUnit.SECONDS); + CompletableFuture login = connection.send(LOGIN_CODE, loginPayload()); + registerRead.get(5, TimeUnit.SECONDS); + CompletableFuture metadataDuringRegister = + connection.send(GET_CLUSTER_METADATA_CODE, Unpooled.EMPTY_BUFFER); + registerConcurrentSendStarted.complete(null); + login.get(5, TimeUnit.SECONDS).release(); + metadataDuringRegister.get(5, TimeUnit.SECONDS).release(); + + CompletableFuture logout = connection.send(LOGOUT_CODE, Unpooled.EMPTY_BUFFER); + logoutRead.get(5, TimeUnit.SECONDS); + CompletableFuture metadataDuringLogout = + connection.send(GET_CLUSTER_METADATA_CODE, Unpooled.EMPTY_BUFFER); + logoutConcurrentSendStarted.complete(null); + logout.get(5, TimeUnit.SECONDS).release(); + metadataDuringLogout.get(5, TimeUnit.SECONDS).release(); + } finally { + connection.close().get(5, TimeUnit.SECONDS); + } + server.get(5, TimeUnit.SECONDS); + } + } + + private static AsyncTcpConnection newConnection( + ServerSocket serverSocket, Duration heartbeatInterval, long acquireTimeoutMillis) { + return new AsyncTcpConnection( + serverSocket.getInetAddress().getHostAddress(), + serverSocket.getLocalPort(), + false, + Optional.empty(), + new AsyncTcpConnection.TcpConnectionPoolConfig(1000, acquireTimeoutMillis), + Optional.of(Duration.ofSeconds(1)), + Optional.of(Duration.ofSeconds(2)), + heartbeatInterval, + 1024 * 1024, + null, + () -> {}, + ignored -> {}); + } + + private static Request readRequest(InputStream input) throws IOException { + byte[] header = input.readNBytes(HEADER_SIZE); + if (header.length != HEADER_SIZE) { + throw new EOFException("Truncated VSR request header"); + } + ByteBuffer fields = ByteBuffer.wrap(header).order(ByteOrder.LITTLE_ENDIAN); + int size = fields.getInt(SIZE_OFFSET); + byte[] body = input.readNBytes(size - HEADER_SIZE); + if (body.length != size - HEADER_SIZE) { + throw new EOFException("Truncated VSR request body"); + } + return new Request( + Byte.toUnsignedInt(header[REQUEST_OPERATION_OFFSET]), + fields.getLong(REQUEST_ID_OFFSET), + fields.getInt(REQUEST_CODE_OFFSET)); + } + + private static void writeResponse(OutputStream output, Request request, byte[] payload) throws IOException { + writeResponse(output, request, 0, payload); + } + + private static void writeResponse(OutputStream output, Request request, int status, byte[] payload) + throws IOException { + byte[] header = new byte[HEADER_SIZE]; + ByteBuffer fields = ByteBuffer.wrap(header).order(ByteOrder.LITTLE_ENDIAN); + fields.putInt(SIZE_OFFSET, HEADER_SIZE + payload.length); + fields.putLong(REPLY_REQUEST_ID_OFFSET, request.requestId()); + fields.putInt(REPLY_STATUS_OFFSET, status); + header[COMMAND_OFFSET] = COMMAND_REPLY; + header[REPLY_OPERATION_OFFSET] = (byte) request.operation(); + output.write(header); + output.write(payload); + output.flush(); + } + + private static void assertNoRequest(InputStream input, Socket socket) throws IOException { + socket.setSoTimeout(200); + assertThatThrownBy(input::read).isInstanceOf(SocketTimeoutException.class); + socket.setSoTimeout((int) TimeUnit.SECONDS.toMillis(2)); + } + + private static byte[] registerBody() { + return ByteBuffer.allocate(21) + .order(ByteOrder.LITTLE_ENDIAN) + .putInt(0) + .putInt(1) + .putLong(42) + .putInt(11 << 10) + .put((byte) 0) + .array(); + } + + private static ByteBuf loginPayload() { + ByteBuf payload = Unpooled.buffer(); + payload.writeByte(4); + payload.writeBytes("iggy".getBytes(StandardCharsets.UTF_8)); + payload.writeByte(4); + payload.writeBytes("iggy".getBytes(StandardCharsets.UTF_8)); + payload.writeIntLE(0); + payload.writeIntLE(0); + return payload; + } + + private static ByteBuf sendMessagesPayload() { + ByteBuf payload = Unpooled.buffer(); + payload.writeIntLE(22); + writeNumericIdentifier(payload, 1); + writeNumericIdentifier(payload, 2); + payload.writeByte(2); + payload.writeByte(4); + payload.writeIntLE(3); + payload.writeIntLE(0); + return payload; + } + + private static void writeNumericIdentifier(ByteBuf payload, int id) { + payload.writeByte(1); + payload.writeByte(4); + payload.writeIntLE(id); + } + + private static void assertResponse(CompletableFuture responseFuture, String expected) throws Exception { + ByteBuf response = responseFuture.get(5, TimeUnit.SECONDS); + try { + byte[] body = new byte[response.readableBytes()]; + response.readBytes(body); + assertThat(body).isEqualTo(expected.getBytes(StandardCharsets.UTF_8)); + } finally { + response.release(); + } + } + + private record Request(int operation, long requestId, int commandCode) {} +} diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionHeartbeatTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionHeartbeatTest.java new file mode 100644 index 0000000000..8d0bd24c1e --- /dev/null +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionHeartbeatTest.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.client.async.tcp; + +import org.junit.jupiter.api.Test; + +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.time.Duration; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; + +class AsyncTcpConnectionHeartbeatTest { + private static final int HEADER_SIZE = 256; + private static final int SIZE_OFFSET = 48; + private static final int COMMAND_OFFSET = 60; + private static final int REQUEST_OPERATION_OFFSET = 176; + private static final int REQUEST_CODE_OFFSET = 196; + private static final int REQUEST_ID_OFFSET = 168; + private static final int REPLY_REQUEST_ID_OFFSET = 200; + private static final int REPLY_OPERATION_OFFSET = 208; + + private static final int COMMAND_REPLY = 8; + private static final int OPERATION_NON_REPLICATED = 2; + private static final int PING_CODE = 1; + + @Test + void shouldSendHeartbeatsUntilConnectionCloses() throws Exception { + InetAddress loopback = InetAddress.getLoopbackAddress(); + try (ServerSocket serverSocket = new ServerSocket(0, 1, loopback)) { + AtomicInteger requests = new AtomicInteger(); + CompletableFuture receivedTwoHeartbeats = new CompletableFuture<>(); + CompletableFuture server = CompletableFuture.runAsync(() -> { + try (Socket socket = serverSocket.accept()) { + socket.setSoTimeout((int) TimeUnit.SECONDS.toMillis(2)); + InputStream input = socket.getInputStream(); + OutputStream output = socket.getOutputStream(); + while (true) { + Request request = readRequest(input); + if (request == null) { + return; + } + assertThat(request.operation()).isEqualTo(OPERATION_NON_REPLICATED); + assertThat(request.commandCode()).isEqualTo(PING_CODE); + writeResponse(output, request); + if (requests.incrementAndGet() == 2) { + receivedTwoHeartbeats.complete(null); + } + } + } catch (IOException error) { + receivedTwoHeartbeats.completeExceptionally(error); + throw new IllegalStateException("Mock VSR server failed", error); + } + }); + + AsyncIggyTcpClient client = AsyncIggyTcpClient.builder() + .host(loopback.getHostAddress()) + .port(serverSocket.getLocalPort()) + .heartbeatInterval(Duration.ofMillis(25)) + .requestTimeout(Duration.ofSeconds(1)) + .build(); + client.connect().get(5, TimeUnit.SECONDS); + receivedTwoHeartbeats.get(5, TimeUnit.SECONDS); + client.close().get(5, TimeUnit.SECONDS); + int requestsAtClose = requests.get(); + + server.get(5, TimeUnit.SECONDS); + Thread.sleep(100); + assertThat(requests).hasValue(requestsAtClose); + } + } + + private static Request readRequest(InputStream input) throws IOException { + byte[] header = input.readNBytes(HEADER_SIZE); + if (header.length == 0) { + return null; + } + if (header.length != HEADER_SIZE) { + throw new EOFException("Truncated VSR request header"); + } + ByteBuffer fields = ByteBuffer.wrap(header).order(ByteOrder.LITTLE_ENDIAN); + int size = fields.getInt(SIZE_OFFSET); + byte[] body = input.readNBytes(size - HEADER_SIZE); + if (body.length != size - HEADER_SIZE) { + throw new EOFException("Truncated VSR request body"); + } + return new Request( + Byte.toUnsignedInt(header[REQUEST_OPERATION_OFFSET]), + fields.getInt(REQUEST_CODE_OFFSET), + fields.getLong(REQUEST_ID_OFFSET)); + } + + private static void writeResponse(OutputStream output, Request request) throws IOException { + byte[] header = new byte[HEADER_SIZE]; + ByteBuffer fields = ByteBuffer.wrap(header).order(ByteOrder.LITTLE_ENDIAN); + fields.putInt(SIZE_OFFSET, HEADER_SIZE); + fields.putLong(REPLY_REQUEST_ID_OFFSET, request.requestId()); + header[COMMAND_OFFSET] = COMMAND_REPLY; + header[REPLY_OPERATION_OFFSET] = OPERATION_NON_REPLICATED; + output.write(header); + output.flush(); + } + + private record Request(int operation, int commandCode, long requestId) {} +} diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionRequestTimeoutTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionRequestTimeoutTest.java new file mode 100644 index 0000000000..bf4270ddbb --- /dev/null +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/AsyncTcpConnectionRequestTimeoutTest.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.client.async.tcp; + +import org.apache.iggy.exception.IggyTimeoutException; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.time.Duration; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class AsyncTcpConnectionRequestTimeoutTest { + private static final int TEST_TIMEOUT_SECONDS = 5; + private static final Duration REQUEST_TIMEOUT = Duration.ofMillis(100); + + private AsyncIggyTcpClient client; + + @AfterEach + void tearDown() throws Exception { + if (client != null) { + client.close().get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS); + } + } + + @Test + void shouldReplaceChannelAfterResponseTimeout() throws Exception { + try (ServerSocket server = new ServerSocket(0, 2, InetAddress.getLoopbackAddress())) { + client = AsyncIggyTcpClient.builder() + .host(server.getInetAddress().getHostAddress()) + .port(server.getLocalPort()) + .requestTimeout(REQUEST_TIMEOUT) + .build(); + + CompletableFuture firstAccepted = accept(server); + client.connect().get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS); + try (Socket firstSocket = firstAccepted.get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { + verifyRequestTimesOutAndClosesSocket(client.system().ping(), firstSocket); + } + + CompletableFuture secondAccepted = accept(server); + CompletableFuture secondResponse = client.system().ping(); + try (Socket secondSocket = secondAccepted.get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { + verifyRequestTimesOutAndClosesSocket(secondResponse, secondSocket); + } + } + } + + @Test + void shouldFailMultiplexedRequestsAndUseReplacementChannelAfterTimeout() throws Exception { + try (ServerSocket server = new ServerSocket(0, 2, InetAddress.getLoopbackAddress())) { + client = AsyncIggyTcpClient.builder() + .host(server.getInetAddress().getHostAddress()) + .port(server.getLocalPort()) + .requestTimeout(REQUEST_TIMEOUT) + .build(); + + CompletableFuture firstAccepted = accept(server); + client.connect().get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS); + try (Socket firstSocket = firstAccepted.get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { + CompletableFuture first = client.system().ping(); + CompletableFuture second = client.system().ping(); + + assertThat(verifyRequestTimesOutAndClosesSocket(first, firstSocket)) + .hasSize(2 * 256); + assertThatThrownBy(() -> second.get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .isInstanceOf(ExecutionException.class) + .hasCauseInstanceOf(IggyTimeoutException.class); + } + + CompletableFuture secondAccepted = accept(server); + CompletableFuture third = client.system().ping(); + try (Socket secondSocket = secondAccepted.get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { + assertThat(verifyRequestTimesOutAndClosesSocket(third, secondSocket)) + .hasSize(256); + } + } + } + + private static byte[] verifyRequestTimesOutAndClosesSocket(CompletableFuture response, Socket socket) + throws Exception { + assertThatThrownBy(() -> response.get(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .isInstanceOf(ExecutionException.class) + .hasCauseInstanceOf(IggyTimeoutException.class); + socket.setSoTimeout((int) TimeUnit.SECONDS.toMillis(TEST_TIMEOUT_SECONDS)); + byte[] request = socket.getInputStream().readAllBytes(); + assertThat(request).isNotEmpty(); + return request; + } + + private static CompletableFuture accept(ServerSocket server) { + return CompletableFuture.supplyAsync(() -> { + try { + return server.accept(); + } catch (IOException e) { + throw new IllegalStateException("Failed to accept test connection", e); + } + }); + } +} diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/ClientRoutingStateTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/ClientRoutingStateTest.java new file mode 100644 index 0000000000..74a508bdc6 --- /dev/null +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/ClientRoutingStateTest.java @@ -0,0 +1,251 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.client.async.tcp; + +import org.apache.iggy.identifier.ConsumerId; +import org.apache.iggy.identifier.StreamId; +import org.apache.iggy.identifier.TopicId; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class ClientRoutingStateTest { + + private final ClientRoutingState state = new ClientRoutingState(); + + @Nested + class Keys { + + @Test + void shouldDistinguishNumericAndNamedIdentifiersWithTheSameText() { + assertThat(ClientRoutingState.topicKey(StreamId.of(1L), TopicId.of(2L))) + .isNotEqualTo(ClientRoutingState.topicKey(StreamId.of("1"), TopicId.of("2"))); + } + + @Test + void shouldDistinguishIdentifiersContainingTheFormerDelimiter() { + assertThat(ClientRoutingState.topicKey(StreamId.of("orders|created"), TopicId.of("events"))) + .isNotEqualTo(ClientRoutingState.topicKey(StreamId.of("orders"), TopicId.of("created|events"))); + } + + @Test + void shouldDistinguishConsumerIdentifiersWithTheSameText() { + assertThat(ClientRoutingState.groupKey(StreamId.of(1L), TopicId.of(2L), ConsumerId.of(3L))) + .isNotEqualTo(ClientRoutingState.groupKey(StreamId.of(1L), TopicId.of(2L), ConsumerId.of("3"))); + } + } + + @Nested + class BalancedCursor { + + @Test + void shouldRoundRobinAcrossPartitions() { + // pinned to the Rust SDK: 3 partitions give 0, 1, 2, 0 + var topic = topic("s", "t"); + + assertThat(state.nextBalancedPartition(topic, 3)).isEqualTo(0); + assertThat(state.nextBalancedPartition(topic, 3)).isEqualTo(1); + assertThat(state.nextBalancedPartition(topic, 3)).isEqualTo(2); + assertThat(state.nextBalancedPartition(topic, 3)).isEqualTo(0); + } + + @Test + void shouldKeepIndependentCursorsPerTopic() { + var first = topic("s", "a"); + var second = topic("s", "b"); + + assertThat(state.nextBalancedPartition(first, 2)).isEqualTo(0); + assertThat(state.nextBalancedPartition(second, 2)).isEqualTo(0); + assertThat(state.nextBalancedPartition(first, 2)).isEqualTo(1); + } + + @Test + void shouldKeepIndependentCursorsForFormerlyCollidingTopics() { + var numeric = ClientRoutingState.topicKey(StreamId.of(1L), TopicId.of(2L)); + var named = ClientRoutingState.topicKey(StreamId.of("1"), TopicId.of("2")); + + assertThat(state.nextBalancedPartition(numeric, 2)).isEqualTo(0); + assertThat(state.nextBalancedPartition(named, 2)).isEqualTo(0); + assertThat(state.nextBalancedPartition(numeric, 2)).isEqualTo(1); + } + + @Test + void shouldFallBackToZeroWithoutPartitions() { + assertThat(state.nextBalancedPartition(topic("s", "t"), 0)).isEqualTo(0); + } + } + + @Nested + class GroupAssignments { + + @Test + void shouldRoundRobinAcrossAssignedPartitions() { + // pinned to the Rust SDK: assignment [0, 1, 2] gives 0, 1, 2, 0 + var group = group("s", "t", "g"); + state.setAssignment(group, 1, List.of(0L, 1L, 2L), 0); + + assertThat(state.nextGroupPartition(group)).hasValue(0); + assertThat(state.nextGroupPartition(group)).hasValue(1); + assertThat(state.nextGroupPartition(group)).hasValue(2); + assertThat(state.nextGroupPartition(group)).hasValue(0); + } + + @Test + void shouldReturnEmptyWithoutAssignment() { + assertThat(state.nextGroupPartition(group("s", "t", "g"))).isEmpty(); + } + + @Test + void shouldReturnEmptyForMemberOwningNoPartitions() { + var group = group("s", "t", "g"); + state.setAssignment(group, 1, List.of(), 0); + + assertThat(state.nextGroupPartition(group)).isEmpty(); + } + + @Test + void shouldResetCursorWhenGenerationAdvances() { + var group = group("s", "t", "g"); + state.setAssignment(group, 1, List.of(0L, 1L, 2L), 0); + state.nextGroupPartition(group); + state.nextGroupPartition(group); + + state.setAssignment(group, 2, List.of(0L, 1L, 2L), 0); + + assertThat(state.nextGroupPartition(group)).hasValue(0); + } + + @Test + void shouldKeepCursorWhenGenerationIsUnchanged() { + var group = group("s", "t", "g"); + state.setAssignment(group, 1, List.of(0L, 1L, 2L), 0); + state.nextGroupPartition(group); + + state.setAssignment(group, 1, List.of(0L, 1L, 2L), 100); + + assertThat(state.nextGroupPartition(group)).hasValue(1); + } + + @Test + void shouldWrapCursorPositionWhenAssignmentShrinks() { + var group = group("s", "t", "g"); + state.setAssignment(group, 1, List.of(0L, 1L, 2L), 0); + state.nextGroupPartition(group); + state.nextGroupPartition(group); + + state.setAssignment(group, 1, List.of(5L), 0); + + assertThat(state.nextGroupPartition(group)).hasValue(5); + } + + @Test + void shouldInvalidateSingleAssignment() { + var group = group("s", "t", "g"); + state.setAssignment(group, 1, List.of(0L), 0); + state.invalidateAssignment(group); + + assertThat(state.assignment(group)).isEmpty(); + } + + @Test + void shouldClearAllAssignments() { + var first = group("s", "t", "g1"); + var second = group("s", "t", "g2"); + state.setAssignment(first, 1, List.of(0L), 0); + state.setAssignment(second, 1, List.of(1L), 0); + + state.clearAssignments(); + + assertThat(state.assignment(first)).isEmpty(); + assertThat(state.assignment(second)).isEmpty(); + } + + @Test + void shouldKeepAssignmentsIndependentForFormerlyCollidingGroups() { + var numeric = ClientRoutingState.groupKey(StreamId.of(1L), TopicId.of(2L), ConsumerId.of(3L)); + var named = ClientRoutingState.groupKey(StreamId.of("1"), TopicId.of("2"), ConsumerId.of("3")); + state.setAssignment(numeric, 1, List.of(1L), 0); + state.setAssignment(named, 1, List.of(2L), 0); + + assertThat(state.nextGroupPartition(numeric)).hasValue(1); + assertThat(state.nextGroupPartition(named)).hasValue(2); + } + + @Test + void shouldExposeSyncTimestampForStalenessChecks() { + var group = group("s", "t", "g"); + state.setAssignment(group, 1, List.of(0L), 42L); + + assertThat(state.assignment(group)).hasValueSatisfying(assignment -> assertThat(assignment.syncedAtNanos()) + .isEqualTo(42L)); + } + } + + @Nested + class PartitionCounts { + + @Test + void shouldCachePartitionCountWithFetchTimestamp() { + var topic = topic("s", "t"); + assertThat(state.partitionCount(topic)).isEmpty(); + + state.setPartitionCount(topic, 4L, 42L); + + assertThat(state.partitionCount(topic)).hasValueSatisfying(cached -> { + assertThat(cached.count()).isEqualTo(4L); + assertThat(cached.fetchedAtNanos()).isEqualTo(42L); + }); + } + + @Test + void shouldInvalidatePartitionCount() { + var topic = topic("s", "t"); + state.setPartitionCount(topic, 4L, 0L); + + state.invalidatePartitionCount(topic); + + assertThat(state.partitionCount(topic)).isEmpty(); + } + + @Test + void shouldKeepCountsIndependentForFormerlyCollidingTopics() { + var numeric = ClientRoutingState.topicKey(StreamId.of(1L), TopicId.of(2L)); + var named = ClientRoutingState.topicKey(StreamId.of("1"), TopicId.of("2")); + state.setPartitionCount(numeric, 4L, 0L); + state.setPartitionCount(named, 8L, 0L); + + assertThat(state.partitionCount(numeric)) + .hasValueSatisfying(cached -> assertThat(cached.count()).isEqualTo(4L)); + assertThat(state.partitionCount(named)) + .hasValueSatisfying(cached -> assertThat(cached.count()).isEqualTo(8L)); + } + } + + private static ClientRoutingState.TopicKey topic(String stream, String topic) { + return ClientRoutingState.topicKey(StreamId.of(stream), TopicId.of(topic)); + } + + private static ClientRoutingState.GroupKey group(String stream, String topic, String consumer) { + return ClientRoutingState.groupKey(StreamId.of(stream), TopicId.of(topic), ConsumerId.of(consumer)); + } +} diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/IggyFrameDecoderTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/IggyFrameDecoderTest.java deleted file mode 100644 index 448f07de7b..0000000000 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/IggyFrameDecoderTest.java +++ /dev/null @@ -1,455 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iggy.client.async.tcp; - -import io.netty.buffer.ByteBuf; -import io.netty.buffer.Unpooled; -import io.netty.channel.embedded.EmbeddedChannel; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; - -import static org.assertj.core.api.Assertions.assertThat; - -class IggyFrameDecoderTest { - - private EmbeddedChannel channel; - - @AfterEach - void tearDown() { - if (channel != null) { - // Release any remaining messages - Object msg; - while ((msg = channel.readInbound()) != null) { - if (msg instanceof ByteBuf) { - ((ByteBuf) msg).release(); - } - } - channel.finishAndReleaseAll(); - } - } - - @Nested - class CompleteFrames { - - @Test - void shouldDecodeCompleteFrameWithSmallPayload() { - // given - channel = new EmbeddedChannel(new IggyFrameDecoder()); - ByteBuf input = Unpooled.buffer(); - input.writeIntLE(0); // status = success - input.writeIntLE(5); // length = 5 bytes - input.writeBytes("hello".getBytes()); // payload - - // when - channel.writeInbound(input); - - // then - ByteBuf decoded = channel.readInbound(); - assertThat(decoded).isNotNull(); - assertThat(decoded.readableBytes()).isEqualTo(8 + 5); // header + payload - assertThat(decoded.readIntLE()).isEqualTo(0); // status - assertThat(decoded.readIntLE()).isEqualTo(5); // length - byte[] payload = new byte[5]; - decoded.readBytes(payload); - assertThat(new String(payload)).isEqualTo("hello"); - decoded.release(); - } - - @Test - void shouldDecodeCompleteFrameWithZeroLengthPayload() { - // given - channel = new EmbeddedChannel(new IggyFrameDecoder()); - ByteBuf input = Unpooled.buffer(); - input.writeIntLE(0); // status - input.writeIntLE(0); // length = 0 - - // when - channel.writeInbound(input); - - // then - ByteBuf decoded = channel.readInbound(); - assertThat(decoded).isNotNull(); - assertThat(decoded.readableBytes()).isEqualTo(8); // header only - assertThat(decoded.readIntLE()).isEqualTo(0); - assertThat(decoded.readIntLE()).isEqualTo(0); - decoded.release(); - } - - @Test - void shouldDecodeCompleteFrameWithLargePayload() { - // given - channel = new EmbeddedChannel(new IggyFrameDecoder()); - byte[] largePayload = new byte[10000]; - for (int i = 0; i < largePayload.length; i++) { - largePayload[i] = (byte) (i % 256); - } - - ByteBuf input = Unpooled.buffer(); - input.writeIntLE(200); // error status - input.writeIntLE(10000); // large length - input.writeBytes(largePayload); - - // when - channel.writeInbound(input); - - // then - ByteBuf decoded = channel.readInbound(); - assertThat(decoded).isNotNull(); - assertThat(decoded.readableBytes()).isEqualTo(8 + 10000); - assertThat(decoded.readIntLE()).isEqualTo(200); - assertThat(decoded.readIntLE()).isEqualTo(10000); - decoded.release(); - } - - @Test - void shouldDecodeFrameWithVariousStatusCodes() { - // given - channel = new EmbeddedChannel(new IggyFrameDecoder()); - - for (int status = 0; status <= 5; status++) { - ByteBuf input = Unpooled.buffer(); - input.writeIntLE(status); - input.writeIntLE(1); - input.writeByte(42); - - // when - channel.writeInbound(input); - - // then - ByteBuf decoded = channel.readInbound(); - assertThat(decoded).isNotNull(); - assertThat(decoded.readIntLE()).isEqualTo(status); - decoded.skipBytes(4); // length - decoded.skipBytes(1); // payload - decoded.release(); - } - } - } - - @Nested - class IncompleteFrames { - - @Test - void shouldWaitForCompleteHeaderWhenOnlyPartialHeaderAvailable() { - // given - channel = new EmbeddedChannel(new IggyFrameDecoder()); - ByteBuf input = Unpooled.buffer(); - input.writeIntLE(0); // only status, missing length (4 bytes total, need 8) - - // when - boolean hasMessage = channel.writeInbound(input); - - // then - assertThat(hasMessage).isFalse(); // No complete frame yet - ByteBuf decoded = channel.readInbound(); - assertThat(decoded).isNull(); // Nothing to read - } - - @Test - void shouldWaitForCompletePayloadWhenOnlyHeaderAvailable() { - // given - channel = new EmbeddedChannel(new IggyFrameDecoder()); - ByteBuf input = Unpooled.buffer(); - input.writeIntLE(0); // status - input.writeIntLE(100); // expects 100 bytes payload - // But no payload written - - // when - boolean hasMessage = channel.writeInbound(input); - - // then - assertThat(hasMessage).isFalse(); - ByteBuf decoded = channel.readInbound(); - assertThat(decoded).isNull(); - } - - @Test - void shouldWaitForCompletePayloadWhenPartialPayloadAvailable() { - // given - channel = new EmbeddedChannel(new IggyFrameDecoder()); - ByteBuf input = Unpooled.buffer(); - input.writeIntLE(0); - input.writeIntLE(100); // expects 100 bytes - input.writeBytes(new byte[50]); // only 50 bytes available - - // when - boolean hasMessage = channel.writeInbound(input); - - // then - assertThat(hasMessage).isFalse(); - assertThat((ByteBuf) channel.readInbound()).isNull(); - } - - @Test - void shouldEventuallyDecodeWhenMoreDataArrives() { - // given - channel = new EmbeddedChannel(new IggyFrameDecoder()); - ByteBuf firstChunk = Unpooled.buffer(); - firstChunk.writeIntLE(0); - firstChunk.writeIntLE(10); - firstChunk.writeBytes(new byte[5]); // only 5 of 10 bytes - - // when - first chunk - boolean hasMessage1 = channel.writeInbound(firstChunk); - assertThat(hasMessage1).isFalse(); - - // when - second chunk completes the frame - ByteBuf secondChunk = Unpooled.buffer(); - secondChunk.writeBytes(new byte[5]); // remaining 5 bytes - boolean hasMessage2 = channel.writeInbound(secondChunk); - - // then - assertThat(hasMessage2).isTrue(); - ByteBuf decoded = channel.readInbound(); - assertThat(decoded).isNotNull(); - assertThat(decoded.readableBytes()).isEqualTo(8 + 10); - decoded.release(); - } - } - - @Nested - class MultipleFrames { - - @Test - void shouldDecodeMultipleFramesInSequence() { - // given - channel = new EmbeddedChannel(new IggyFrameDecoder()); - ByteBuf input = Unpooled.buffer(); - - // Frame 1 - input.writeIntLE(0); - input.writeIntLE(3); - input.writeBytes("abc".getBytes()); - - // Frame 2 - input.writeIntLE(1); - input.writeIntLE(2); - input.writeBytes("de".getBytes()); - - // when - channel.writeInbound(input); - - // then - frame 1 - ByteBuf decoded1 = channel.readInbound(); - assertThat(decoded1).isNotNull(); - assertThat(decoded1.readableBytes()).isEqualTo(8 + 3); - decoded1.release(); - - // then - frame 2 - ByteBuf decoded2 = channel.readInbound(); - assertThat(decoded2).isNotNull(); - assertThat(decoded2.readableBytes()).isEqualTo(8 + 2); - decoded2.release(); - - // No more frames - assertThat((ByteBuf) channel.readInbound()).isNull(); - } - - @Test - void shouldDecodeThreeFramesCorrectly() { - // given - channel = new EmbeddedChannel(new IggyFrameDecoder()); - ByteBuf input = Unpooled.buffer(); - - for (int i = 0; i < 3; i++) { - input.writeIntLE(i); - input.writeIntLE(1); - input.writeByte(i); - } - - // when - channel.writeInbound(input); - - // then - for (int i = 0; i < 3; i++) { - ByteBuf decoded = channel.readInbound(); - assertThat(decoded).isNotNull(); - assertThat(decoded.readIntLE()).isEqualTo(i); - decoded.skipBytes(4 + 1); // skip length and payload - decoded.release(); - } - - assertThat((ByteBuf) channel.readInbound()).isNull(); - } - } - - @Nested - class EdgeCases { - - @Test - void shouldNotAdvanceReaderIndexWhenPayloadIncomplete() { - // given - channel = new EmbeddedChannel(new IggyFrameDecoder()); - ByteBuf input = Unpooled.buffer(); - input.writeIntLE(0); - input.writeIntLE(100); // expects 100 bytes - input.writeBytes(new byte[50]); // only 50 bytes - int readerIndexBefore = input.readerIndex(); - - // when - boolean hasMessage = channel.writeInbound(input); - - // then - decoder should wait for complete payload - assertThat(hasMessage).isFalse(); - ByteBuf decoded = channel.readInbound(); - assertThat(decoded).isNull(); - assertThat(input.readerIndex()).isEqualTo(readerIndexBefore); - } - - @Test - void shouldHandleEmptyInput() { - // given - channel = new EmbeddedChannel(new IggyFrameDecoder()); - ByteBuf input = Unpooled.buffer(); // empty buffer - - // when - boolean hasMessage = channel.writeInbound(input); - - // then - assertThat(hasMessage).isFalse(); - assertThat((ByteBuf) channel.readInbound()).isNull(); - } - - @Test - void shouldHandleSingleByteInput() { - // given - channel = new EmbeddedChannel(new IggyFrameDecoder()); - ByteBuf input = Unpooled.buffer(); - input.writeByte(0); // only 1 byte - - // when - boolean hasMessage = channel.writeInbound(input); - - // then - assertThat(hasMessage).isFalse(); - assertThat((ByteBuf) channel.readInbound()).isNull(); - } - - @Test - void shouldHandleExactlyHeaderSizeWithoutPayload() { - // given - channel = new EmbeddedChannel(new IggyFrameDecoder()); - ByteBuf input = Unpooled.buffer(); - input.writeIntLE(0); - input.writeIntLE(10); // expects payload, but none provided - // Exactly 8 bytes (header size) - - // when - boolean hasMessage = channel.writeInbound(input); - - // then - assertThat(hasMessage).isFalse(); // Waiting for payload - assertThat((ByteBuf) channel.readInbound()).isNull(); - } - - @Test - void shouldDecodeFrameFollowedByPartialNextFrame() { - // given - channel = new EmbeddedChannel(new IggyFrameDecoder()); - ByteBuf input = Unpooled.buffer(); - - // Complete frame 1 - input.writeIntLE(0); - input.writeIntLE(5); - input.writeBytes("hello".getBytes()); - - // Partial frame 2 (only header) - input.writeIntLE(1); - input.writeIntLE(10); - // No payload for frame 2 - - // when - channel.writeInbound(input); - - // then - should get frame 1 - ByteBuf decoded1 = channel.readInbound(); - assertThat(decoded1).isNotNull(); - assertThat(decoded1.readableBytes()).isEqualTo(8 + 5); - decoded1.release(); - - // Frame 2 should not be available yet - assertThat((ByteBuf) channel.readInbound()).isNull(); - } - - @Test - void shouldHandleMaxIntPayloadLength() { - // given - test with a reasonably large payload (not actual MAX_INT to avoid OOM) - channel = new EmbeddedChannel(new IggyFrameDecoder()); - int largeSize = 1000000; // 1MB - ByteBuf input = Unpooled.buffer(); - input.writeIntLE(0); - input.writeIntLE(largeSize); - input.writeBytes(new byte[largeSize]); - - // when - channel.writeInbound(input); - - // then - ByteBuf decoded = channel.readInbound(); - assertThat(decoded).isNotNull(); - assertThat(decoded.readableBytes()).isEqualTo(8 + largeSize); - decoded.release(); - } - } - - @Nested - class BufferManagement { - - @Test - void shouldCreateNewBufferForEachDecodedFrame() { - // given - channel = new EmbeddedChannel(new IggyFrameDecoder()); - ByteBuf input = Unpooled.buffer(); - input.writeIntLE(0); - input.writeIntLE(5); - input.writeBytes("hello".getBytes()); - - // when - channel.writeInbound(input); - ByteBuf decoded = channel.readInbound(); - - // then - decoded buffer should be independent - assertThat(decoded).isNotNull(); - assertThat(decoded).isNotSameAs(input); - assertThat(decoded.readableBytes()).isEqualTo(13); - decoded.release(); - } - - @Test - void shouldAllowManualReleaseOfDecodedBuffers() { - // given - channel = new EmbeddedChannel(new IggyFrameDecoder()); - ByteBuf input = Unpooled.buffer(); - input.writeIntLE(0); - input.writeIntLE(3); - input.writeBytes("foo".getBytes()); - - // when - channel.writeInbound(input); - ByteBuf decoded = channel.readInbound(); - - // then - should be releasable - assertThat(decoded.refCnt()).isEqualTo(1); - decoded.release(); - assertThat(decoded.refCnt()).isEqualTo(0); - } - } -} diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/IggyResponseHandlerTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/IggyResponseHandlerTest.java deleted file mode 100644 index fe542542fd..0000000000 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/IggyResponseHandlerTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.iggy.client.async.tcp; - -import io.netty.buffer.ByteBuf; -import io.netty.channel.embedded.EmbeddedChannel; -import org.apache.iggy.exception.IggyConnectionException; -import org.junit.jupiter.api.Test; - -import java.util.concurrent.CompletableFuture; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -class IggyResponseHandlerTest { - - @Test - void shouldFailPendingRequestsWhenChannelBecomesInactive() { - var handler = new AsyncTcpConnection.IggyResponseHandler(); - var channel = new EmbeddedChannel(handler); - CompletableFuture pending = new CompletableFuture<>(); - handler.enqueueRequest(pending); - - channel.close(); - - assertThat(pending).isCompletedExceptionally(); - assertThatThrownBy(pending::join).hasCauseInstanceOf(IggyConnectionException.class); - } - - @Test - void shouldFailPendingRequestsOnPipelineException() { - var handler = new AsyncTcpConnection.IggyResponseHandler(); - var channel = new EmbeddedChannel(handler); - CompletableFuture pending = new CompletableFuture<>(); - handler.enqueueRequest(pending); - - var failure = new IllegalStateException("broken pipe"); - channel.pipeline().fireExceptionCaught(failure); - - assertThat(pending).isCompletedExceptionally(); - assertThatThrownBy(pending::join).hasCause(failure); - } -} diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/LoginRoutingHookTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/LoginRoutingHookTest.java new file mode 100644 index 0000000000..500eef75fe --- /dev/null +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/LoginRoutingHookTest.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.client.async.tcp; + +import org.apache.iggy.user.IdentityInfo; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Supplier; + +import static org.assertj.core.api.Assertions.assertThat; + +class LoginRoutingHookTest { + + private static final IdentityInfo IDENTITY = new IdentityInfo(1L, Optional.empty()); + + @Test + void shouldRouteUsernameLoginBeforeCreatingRegisterAttempt() { + List events = new ArrayList<>(); + var client = new UsersTcpClient(failingConnectionSupplier(events), routingHook(events)); + + var identity = client.login("iggy", "iggy").join(); + + assertThat(identity).isEqualTo(IDENTITY); + assertThat(events).containsExactly("route", "login-attempt"); + } + + @Test + void shouldRoutePersonalAccessTokenLoginBeforeCreatingRegisterAttempt() { + List events = new ArrayList<>(); + var client = new PersonalAccessTokensTcpClient(failingConnectionSupplier(events), routingHook(events)); + + var identity = client.loginWithPersonalAccessToken("token").join(); + + assertThat(identity).isEqualTo(IDENTITY); + assertThat(events).containsExactly("route", "login-attempt"); + } + + @Test + void shouldExecuteOneAttemptWithNoOpHook() { + var attempts = new AtomicInteger(); + + var identity = LoginRoutingHook.NONE + .loginOnLeader(() -> { + attempts.incrementAndGet(); + return CompletableFuture.completedFuture(IDENTITY); + }) + .join(); + + assertThat(identity).isEqualTo(IDENTITY); + assertThat(attempts).hasValue(1); + } + + private static Supplier failingConnectionSupplier(List events) { + return () -> { + events.add("login-attempt"); + throw new LoginAttemptReached(); + }; + } + + private static LoginRoutingHook routingHook(List events) { + return loginAttempt -> { + events.add("route"); + try { + loginAttempt.get(); + } catch (LoginAttemptReached expected) { + return CompletableFuture.completedFuture(IDENTITY); + } + throw new AssertionError("Expected the login attempt to acquire its connection"); + }; + } + + private static final class LoginAttemptReached extends RuntimeException {} +} diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/ReconnectPlanTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/ReconnectPlanTest.java new file mode 100644 index 0000000000..ce833a58cb --- /dev/null +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/ReconnectPlanTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.client.async.tcp; + +import org.apache.iggy.client.ConnectionInfo; +import org.apache.iggy.config.RetryPolicy; +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; + +class ReconnectPlanTest { + + private final ConnectionInfo seed = new ConnectionInfo("seed-node", 8090); + private final ConnectionInfo current = new ConnectionInfo("leader-node", 8090); + + @Test + void shouldAlternateBetweenCurrentAndSeed() { + assertThat(ReconnectPlan.target(current, seed, 1)).isEqualTo(current); + assertThat(ReconnectPlan.target(current, seed, 2)).isEqualTo(seed); + assertThat(ReconnectPlan.target(current, seed, 3)).isEqualTo(current); + assertThat(ReconnectPlan.target(current, seed, 4)).isEqualTo(seed); + } + + @Test + void shouldDialOnlyOneAddressWhenNeverRedirected() { + assertThat(ReconnectPlan.target(seed, seed, 1)).isEqualTo(seed); + assertThat(ReconnectPlan.target(seed, seed, 2)).isEqualTo(seed); + } + + @Test + void shouldKeepFixedDelayConstant() { + var policy = RetryPolicy.fixedDelay(12, Duration.ofSeconds(5)); + + assertThat(ReconnectPlan.delay(policy, 1)).isEqualTo(Duration.ofSeconds(5)); + assertThat(ReconnectPlan.delay(policy, 12)).isEqualTo(Duration.ofSeconds(5)); + } + + @Test + void shouldScaleExponentialDelayUpToTheCap() { + var policy = RetryPolicy.exponentialBackoff(5, Duration.ofMillis(100), Duration.ofSeconds(1), 2.0); + + assertThat(ReconnectPlan.delay(policy, 1)).isEqualTo(Duration.ofMillis(100)); + assertThat(ReconnectPlan.delay(policy, 2)).isEqualTo(Duration.ofMillis(200)); + assertThat(ReconnectPlan.delay(policy, 3)).isEqualTo(Duration.ofMillis(400)); + assertThat(ReconnectPlan.delay(policy, 5)).isEqualTo(Duration.ofSeconds(1)); + } +} diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrFrameDecoderTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrFrameDecoderTest.java new file mode 100644 index 0000000000..fe916cd9fe --- /dev/null +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrFrameDecoderTest.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.client.async.tcp.vsr; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.channel.embedded.EmbeddedChannel; +import io.netty.handler.codec.DecoderException; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class VsrFrameDecoderTest { + + private final EmbeddedChannel channel = new EmbeddedChannel(new VsrFrameDecoder()); + + @AfterEach + void tearDown() { + try { + channel.finishAndReleaseAll(); + } catch (DecoderException ignored) { + // The desync test leaves the failed decoder in the pipeline and + // closing the channel replays its exception. + } + } + + @Test + void shouldWaitForMoreBytesOnPartialHeader() { + ByteBuf partial = Unpooled.buffer(); + partial.writeZero(100); + channel.writeInbound(partial); + + assertThat((Object) channel.readInbound()).isNull(); + } + + @Test + void shouldEmitOneFrameFromSplitReads() { + ByteBuf frame = Unpooled.buffer(); + frame.writeZero(VsrHeaders.HEADER_SIZE); + frame.setIntLE(VsrHeaders.SIZE_OFFSET, VsrHeaders.HEADER_SIZE + 8); + frame.writeLongLE(123456789L); + + channel.writeInbound(frame.retainedSlice(0, 200)); + assertThat((Object) channel.readInbound()).isNull(); + channel.writeInbound(frame.retainedSlice(200, frame.readableBytes() - 200)); + frame.release(); + + ByteBuf decoded = channel.readInbound(); + try { + assertThat(decoded.readableBytes()).isEqualTo(VsrHeaders.HEADER_SIZE + 8); + assertThat(decoded.getLongLE(VsrHeaders.HEADER_SIZE)).isEqualTo(123456789L); + } finally { + decoded.release(); + } + } + + @Test + void shouldFailConnectionOnInvalidSizeField() { + ByteBuf frame = Unpooled.buffer(); + frame.writeZero(VsrHeaders.HEADER_SIZE); + frame.setIntLE(VsrHeaders.SIZE_OFFSET, 8); + + assertThatThrownBy(() -> channel.writeInbound(frame)).isInstanceOf(DecoderException.class); + } + + @Test + void shouldWaitForLargeFrameWhenConfiguredLimitAllowsIt() { + int declaredSize = VsrFrameDecoder.DEFAULT_MAX_FRAME_SIZE + 1; + EmbeddedChannel largeFrameChannel = new EmbeddedChannel(new VsrFrameDecoder(declaredSize)); + ByteBuf header = Unpooled.buffer(VsrHeaders.HEADER_SIZE); + header.writeZero(VsrHeaders.HEADER_SIZE); + header.setIntLE(VsrHeaders.SIZE_OFFSET, declaredSize); + try { + largeFrameChannel.writeInbound(header); + + assertThat((Object) largeFrameChannel.readInbound()).isNull(); + } finally { + largeFrameChannel.finishAndReleaseAll(); + } + } + + @Test + void shouldRejectFrameAboveConfiguredLimit() { + int configuredLimit = VsrHeaders.HEADER_SIZE + 1; + EmbeddedChannel smallFrameChannel = new EmbeddedChannel(new VsrFrameDecoder(configuredLimit)); + ByteBuf header = Unpooled.buffer(VsrHeaders.HEADER_SIZE); + header.writeZero(VsrHeaders.HEADER_SIZE); + header.setIntLE(VsrHeaders.SIZE_OFFSET, configuredLimit + 1); + try { + assertThatThrownBy(() -> smallFrameChannel.writeInbound(header)).isInstanceOf(DecoderException.class); + } finally { + try { + smallFrameChannel.finishAndReleaseAll(); + } catch (DecoderException ignored) { + // Closing a failed decoder can replay its exception. + } + } + } +} diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrRequestEncoderTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrRequestEncoderTest.java new file mode 100644 index 0000000000..426aa0b501 --- /dev/null +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrRequestEncoderTest.java @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.client.async.tcp.vsr; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.ByteBufAllocator; +import io.netty.buffer.Unpooled; +import org.apache.iggy.exception.IggyNotConnectedException; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class VsrRequestEncoderTest { + + private static final int LOGIN_USER_CODE = 38; + private static final int PING_CODE = 1; + private static final int GET_CLUSTER_METADATA_CODE = 12; + private static final int CREATE_STREAM_CODE = 202; + private static final int SEND_MESSAGES_CODE = 101; + + private final ConsensusSession session = new ConsensusSession(); + private final VsrRequestEncoder encoder = new VsrRequestEncoder(session); + private final ByteBufAllocator alloc = ByteBufAllocator.DEFAULT; + + @Test + void shouldBuildRegisterFrameForLogin() { + ByteBuf payload = loginUserPayload(); + ByteBuf frame = encoder.encode(alloc, LOGIN_USER_CODE, payload); + payload.release(); + + try { + assertThat(frame.getUnsignedByte(VsrHeaders.COMMAND_OFFSET)).isEqualTo((short) VsrHeaders.COMMAND_REQUEST); + assertThat(frame.getUnsignedByte(VsrHeaders.REQUEST_OPERATION_OFFSET)) + .isEqualTo((short) VsrOperation.REGISTER); + assertThat(frame.getLongLE(VsrHeaders.REQUEST_ID_OFFSET)).isZero(); + assertThat(frame.getLongLE(VsrHeaders.REQUEST_SESSION_OFFSET)).isZero(); + assertThat(frame.getUnsignedIntLE(VsrHeaders.SIZE_OFFSET)).isEqualTo(frame.readableBytes()); + + // Body starts with the ClientVersionInfo prefix. + int bodyStart = VsrHeaders.HEADER_SIZE; + assertThat(frame.getIntLE(bodyStart)).isEqualTo(VsrLoginCodec.PROTOCOL_VERSION); + int sdkNameLength = frame.getUnsignedByte(bodyStart + 4); + byte[] sdkName = new byte[sdkNameLength]; + frame.getBytes(bodyStart + 5, sdkName); + assertThat(new String(sdkName, StandardCharsets.UTF_8)).isEqualTo(VsrLoginCodec.SDK_NAME); + } finally { + frame.release(); + } + } + + @Test + void shouldStampCodeAndZeroSessionForPing() { + ByteBuf frame = encoder.encode(alloc, PING_CODE, Unpooled.EMPTY_BUFFER); + try { + assertThat(frame.getUnsignedByte(VsrHeaders.REQUEST_OPERATION_OFFSET)) + .isEqualTo((short) VsrOperation.NON_REPLICATED); + assertThat(frame.getIntLE(VsrHeaders.REQUEST_RESERVED_CODE_OFFSET)).isEqualTo(PING_CODE); + assertThat(frame.getLongLE(VsrHeaders.REQUEST_SESSION_OFFSET)).isZero(); + } finally { + frame.release(); + } + } + + @Test + void shouldKeepPreAuthClusterMetadataSessionlessWithoutAdvancingRequestId() { + ByteBuf frame = encoder.encode(alloc, GET_CLUSTER_METADATA_CODE, Unpooled.EMPTY_BUFFER); + try { + assertThat(frame.getUnsignedByte(VsrHeaders.REQUEST_OPERATION_OFFSET)) + .isEqualTo((short) VsrOperation.NON_REPLICATED); + assertThat(frame.getIntLE(VsrHeaders.REQUEST_RESERVED_CODE_OFFSET)).isEqualTo(GET_CLUSTER_METADATA_CODE); + assertThat(frame.getLongLE(VsrHeaders.REQUEST_ID_OFFSET)).isEqualTo(1); + assertThat(frame.getLongLE(VsrHeaders.REQUEST_SESSION_OFFSET)).isZero(); + assertThat(session.currentRequestId()).isEqualTo(1); + } finally { + frame.release(); + } + } + + @Test + void shouldRejectReplicatedCommandBeforeLogin() { + assertThatThrownBy(() -> encoder.encode(alloc, CREATE_STREAM_CODE, Unpooled.EMPTY_BUFFER)) + .isInstanceOf(IggyNotConnectedException.class); + } + + @Test + void shouldAdvanceRequestIdsForReplicatedCommands() { + session.beginRegister(); + session.bind(42); + + ByteBuf first = encoder.encode(alloc, CREATE_STREAM_CODE, Unpooled.EMPTY_BUFFER); + ByteBuf second = encoder.encode(alloc, CREATE_STREAM_CODE, Unpooled.EMPTY_BUFFER); + try { + assertThat(first.getLongLE(VsrHeaders.REQUEST_ID_OFFSET)).isEqualTo(1); + assertThat(second.getLongLE(VsrHeaders.REQUEST_ID_OFFSET)).isEqualTo(2); + assertThat(second.getLongLE(VsrHeaders.REQUEST_SESSION_OFFSET)).isEqualTo(42); + } finally { + first.release(); + second.release(); + } + } + + @Test + void shouldCorrelatePartitionOpsWithoutAdvancingTheDedupRequestId() { + // Partition ops replicate in their own group with no client-table dedup, so + // they take a correlation id and leave the dedup counter where it was. + session.beginRegister(); + session.bind(42); + + ByteBuf firstPayload = sendMessagesPayload(2, 3, 4); + ByteBuf secondPayload = sendMessagesPayload(2, 3, 4); + ByteBuf first = encoder.encode(alloc, SEND_MESSAGES_CODE, firstPayload); + ByteBuf second = encoder.encode(alloc, SEND_MESSAGES_CODE, secondPayload); + firstPayload.release(); + secondPayload.release(); + try { + assertThat(first.getLongLE(VsrHeaders.REQUEST_ID_OFFSET)).isEqualTo(1); + assertThat(second.getLongLE(VsrHeaders.REQUEST_ID_OFFSET)).isEqualTo(2); + assertThat(session.currentRequestId()).isEqualTo(1); + } finally { + first.release(); + second.release(); + } + } + + @Test + void shouldPassPartitioningThroughForTheServerToResolve() { + // The client no longer inspects the payload to route: balanced partitioning + // reaches the server byte-identical instead of failing at encode time. + session.beginRegister(); + session.bind(42); + + ByteBuf payload = balancedSendMessagesPayload(2, 3); + ByteBuf frame = encoder.encode(alloc, SEND_MESSAGES_CODE, payload); + try { + assertThat(frame.getUnsignedIntLE(VsrHeaders.SIZE_OFFSET)).isEqualTo(frame.readableBytes()); + byte[] encodedBody = new byte[payload.readableBytes()]; + frame.getBytes(VsrHeaders.HEADER_SIZE, encodedBody); + byte[] originalBody = new byte[payload.readableBytes()]; + payload.getBytes(payload.readerIndex(), originalBody); + assertThat(encodedBody).isEqualTo(originalBody); + } finally { + frame.release(); + payload.release(); + } + } + + @Test + void shouldReArmWithFreshClientIdOnSecondLogin() { + ByteBuf firstLogin = loginUserPayload(); + encoder.encode(alloc, LOGIN_USER_CODE, firstLogin).release(); + firstLogin.release(); + long firstLow = session.clientIdLow(); + long firstHigh = session.clientIdHigh(); + session.bind(7); + + ByteBuf secondLogin = loginUserPayload(); + encoder.encode(alloc, LOGIN_USER_CODE, secondLogin).release(); + secondLogin.release(); + + assertThat(session.isBound()).isFalse(); + assertThat(session.clientIdLow() != firstLow || session.clientIdHigh() != firstHigh) + .isTrue(); + } + + private static ByteBuf loginUserPayload() { + ByteBuf payload = Unpooled.buffer(); + payload.writeByte(4); + payload.writeBytes("iggy".getBytes(StandardCharsets.UTF_8)); + payload.writeByte(4); + payload.writeBytes("iggy".getBytes(StandardCharsets.UTF_8)); + payload.writeIntLE(0); + payload.writeIntLE(0); + return payload; + } + + private static ByteBuf sendMessagesPayload(long streamId, long topicId, long partitionId) { + ByteBuf payload = Unpooled.buffer(); + // metadata: stream ident (6) + topic ident (6) + partitioning (6) + count (4) + payload.writeIntLE(22); + writeNumericIdentifier(payload, streamId); + writeNumericIdentifier(payload, topicId); + payload.writeByte(2); + payload.writeByte(4); + payload.writeIntLE((int) partitionId); + payload.writeIntLE(0); + return payload; + } + + private static ByteBuf balancedSendMessagesPayload(long streamId, long topicId) { + ByteBuf payload = Unpooled.buffer(); + payload.writeIntLE(18); + writeNumericIdentifier(payload, streamId); + writeNumericIdentifier(payload, topicId); + payload.writeByte(1); + payload.writeByte(0); + payload.writeIntLE(0); + return payload; + } + + private static void writeNumericIdentifier(ByteBuf payload, long id) { + payload.writeByte(1); + payload.writeByte(4); + payload.writeIntLE((int) id); + } +} diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandlerTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandlerTest.java new file mode 100644 index 0000000000..e8069b9159 --- /dev/null +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/async/tcp/vsr/VsrResponseHandlerTest.java @@ -0,0 +1,334 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.client.async.tcp.vsr; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.channel.embedded.EmbeddedChannel; +import org.apache.iggy.exception.IggyConnectionException; +import org.apache.iggy.exception.IggyServerException; +import org.apache.iggy.exception.IggyTimeoutException; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class VsrResponseHandlerTest { + + private final ConsensusSession session = new ConsensusSession(); + private final AtomicInteger evictions = new AtomicInteger(); + private final VsrResponseHandler handler = new VsrResponseHandler(session, evictions::incrementAndGet); + private final EmbeddedChannel channel = new EmbeddedChannel(handler); + + @AfterEach + void tearDown() { + channel.finishAndReleaseAll(); + } + + @Test + void shouldFailWithStatusCodeOnDeniedReply() { + CompletableFuture future = enqueue(VsrOperation.NON_REPLICATED, 7); + ByteBuf frame = replyFrame(VsrOperation.NON_REPLICATED, 7, Unpooled.EMPTY_BUFFER); + frame.setIntLE(VsrHeaders.REPLY_STATUS_OFFSET, 40); + + channel.writeInbound(frame); + + assertThat(rawErrorCode(future)).isEqualTo(40); + } + + @Test + void shouldPassNonReplicatedBodyThrough() throws Exception { + CompletableFuture future = enqueue(VsrOperation.NON_REPLICATED, 7); + ByteBuf body = Unpooled.buffer(); + body.writeIntLE(1234); + channel.writeInbound(replyFrame(VsrOperation.NON_REPLICATED, 7, body)); + + ByteBuf response = future.get(); + try { + assertThat(response.readIntLE()).isEqualTo(1234); + } finally { + response.release(); + } + } + + @Test + void shouldReleaseResponseBodyWhenRequestWasCancelled() { + CompletableFuture future = enqueue(VsrOperation.NON_REPLICATED, 7); + future.cancel(false); + ByteBuf frame = + replyFrame(VsrOperation.NON_REPLICATED, 7, Unpooled.buffer().writeIntLE(1234)); + + channel.writeInbound(frame); + + assertThat(frame.refCnt()).isZero(); + } + + @Test + void shouldStripResultSectionFromMetadataReply() throws Exception { + CompletableFuture future = enqueue(VsrOperation.CREATE_STREAM, 7); + ByteBuf body = Unpooled.buffer(); + body.writeIntLE(0); + body.writeIntLE(777); + channel.writeInbound(replyFrame(VsrOperation.CREATE_STREAM, 7, body)); + + ByteBuf response = future.get(); + try { + assertThat(response.readIntLE()).isEqualTo(777); + } finally { + response.release(); + } + } + + @Test + void shouldSurfaceCommittedErrorFromMetadataRejection() { + CompletableFuture future = enqueue(VsrOperation.DELETE_STREAM, 7); + ByteBuf body = Unpooled.buffer(); + body.writeIntLE(1); + body.writeIntLE(0); + body.writeIntLE(1010); + channel.writeInbound(replyFrame(VsrOperation.DELETE_STREAM, 7, body)); + + assertThat(rawErrorCode(future)).isEqualTo(1010); + } + + @Test + void shouldBindSessionOnRegisterReply() throws Exception { + CompletableFuture future = enqueue(VsrOperation.REGISTER, 0); + ByteBuf body = Unpooled.buffer(); + body.writeIntLE(0); // result section: success + body.writeIntLE(1); // user id + body.writeLongLE(99); // session epoch + body.writeIntLE(VsrLoginCodec.PROTOCOL_VERSION); + body.writeByte(3); + body.writeBytes("0.1".getBytes()); + channel.writeInbound(replyFrame(VsrOperation.REGISTER, 0, body)); + + ByteBuf response = future.get(); + try { + assertThat(response.readUnsignedIntLE()).isEqualTo(1); + assertThat(session.isBound()).isTrue(); + assertThat(session.boundSession()).isEqualTo(99); + } finally { + response.release(); + } + } + + @Test + void shouldResetSessionOnLogoutReply() throws Exception { + session.beginRegister(); + session.bind(42); + CompletableFuture future = enqueue(VsrOperation.LOGOUT, 7); + channel.writeInbound(replyFrame(VsrOperation.LOGOUT, 7, Unpooled.EMPTY_BUFFER)); + + future.get().release(); + assertThat(session.isBound()).isFalse(); + } + + @Test + void shouldMapEvictionReasonAndResetSession() { + session.beginRegister(); + session.bind(42); + CompletableFuture future = enqueue(VsrOperation.NON_REPLICATED, 7); + ByteBuf frame = emptyFrame(); + frame.setByte(VsrHeaders.COMMAND_OFFSET, VsrHeaders.COMMAND_EVICTION); + frame.setByte(VsrHeaders.EVICTION_REASON_OFFSET, VsrHeaders.REASON_INVALID_CREDENTIALS); + channel.writeInbound(frame); + + assertThat(rawErrorCode(future)).isEqualTo(42); + assertThat(session.isBound()).isFalse(); + assertThat(evictions).hasValue(1); + } + + @Test + void shouldCloseChannelOnEvictionWithoutPendingRequest() { + session.beginRegister(); + session.bind(42); + ByteBuf frame = evictionFrame(VsrHeaders.REASON_STALE_CLIENT); + + channel.writeInbound(frame); + + assertThat(session.isBound()).isFalse(); + assertThat(evictions).hasValue(1); + assertThat(frame.refCnt()).isZero(); + assertThat(channel.isActive()).isFalse(); + } + + @Test + void shouldFailEveryPendingRequestOnEviction() { + session.beginRegister(); + session.bind(42); + CompletableFuture first = enqueue(VsrOperation.NON_REPLICATED, 7); + CompletableFuture second = enqueue(VsrOperation.CREATE_STREAM, 8); + + channel.writeInbound(evictionFrame(VsrHeaders.REASON_STALE_CLIENT)); + + assertThat(rawErrorCode(first)).isEqualTo(VsrHeaders.ERROR_STALE_CLIENT); + assertThat(rawErrorCode(second)).isEqualTo(VsrHeaders.ERROR_STALE_CLIENT); + assertThat(evictions).hasValue(1); + assertThat(channel.isActive()).isFalse(); + } + + @Test + void shouldFailPendingRequestsWhenChannelBecomesInactive() { + CompletableFuture pending = enqueue(VsrOperation.NON_REPLICATED, 7); + + channel.close(); + + assertThat(pending).isCompletedExceptionally(); + assertThatThrownBy(pending::join).hasCauseInstanceOf(IggyConnectionException.class); + } + + @Test + void shouldFailPendingRequestsOnPipelineException() { + CompletableFuture pending = enqueue(VsrOperation.NON_REPLICATED, 7); + + var failure = new IllegalStateException("broken pipe"); + channel.pipeline().fireExceptionCaught(failure); + + assertThat(pending).isCompletedExceptionally(); + assertThatThrownBy(pending::join).hasCause(failure); + } + + @Test + void shouldCloseChannelWhenResponseDeadlineExpires() { + CompletableFuture pending = new CompletableFuture<>(); + ByteBuf request = requestFrame(VsrOperation.NON_REPLICATED, 7); + handler.registerRequest(channel, request, pending, System.nanoTime(), 1); + request.release(); + + channel.runScheduledPendingTasks(); + + assertThat(channel.isActive()).isFalse(); + assertThatThrownBy(pending::join).hasCauseInstanceOf(IggyTimeoutException.class); + } + + @Test + void shouldCorrelateRepliesArrivingInReverseOrder() throws Exception { + CompletableFuture first = enqueue(VsrOperation.SEND_MESSAGES, 7); + CompletableFuture second = enqueue(VsrOperation.SEND_MESSAGES, 8); + + channel.writeInbound(replyFrame(VsrOperation.SEND_MESSAGES, 8, Unpooled.wrappedBuffer(new byte[] {2}))); + channel.writeInbound(replyFrame(VsrOperation.SEND_MESSAGES, 7, Unpooled.wrappedBuffer(new byte[] {1}))); + + ByteBuf firstResponse = first.get(); + ByteBuf secondResponse = second.get(); + try { + assertThat(firstResponse.readByte()).isEqualTo((byte) 1); + assertThat(secondResponse.readByte()).isEqualTo((byte) 2); + } finally { + firstResponse.release(); + secondResponse.release(); + } + } + + @Test + void shouldCorrelateRepliesForServerRewrittenOperations() throws Exception { + int[][] rewrittenOperations = { + {VsrOperation.CREATE_TOPIC, VsrOperation.CREATE_TOPIC_WITH_ASSIGNMENTS}, + {VsrOperation.CREATE_PARTITIONS, VsrOperation.CREATE_PARTITIONS_WITH_ASSIGNMENTS}, + {VsrOperation.DELETE_SEGMENTS, VsrOperation.TRUNCATE_PARTITION} + }; + + for (int index = 0; index < rewrittenOperations.length; index++) { + int requestOperation = rewrittenOperations[index][0]; + int replyOperation = rewrittenOperations[index][1]; + long requestId = index + 1; + CompletableFuture future = enqueue(requestOperation, requestId); + ByteBuf body = Unpooled.buffer(); + body.writeIntLE(0); + body.writeIntLE(100 + index); + + channel.writeInbound(replyFrame(replyOperation, requestId, body)); + + ByteBuf response = future.get(); + try { + assertThat(response.readIntLE()).isEqualTo(100 + index); + } finally { + response.release(); + } + } + } + + @Test + void shouldRejectUnrelatedReplyOperation() { + CompletableFuture pending = enqueue(VsrOperation.CREATE_TOPIC, 7); + + channel.writeInbound( + replyFrame(VsrOperation.DELETE_TOPIC, 7, Unpooled.buffer().writeIntLE(0))); + + assertThat(channel.isActive()).isFalse(); + assertThat(rawErrorCode(pending)).isEqualTo(VsrHeaders.ERROR_INVALID_COMMAND); + } + + private CompletableFuture enqueue(int operation, long requestId) { + CompletableFuture future = new CompletableFuture<>(); + handler.registerRequest(future, operation, requestId); + return future; + } + + private static ByteBuf emptyFrame() { + ByteBuf frame = Unpooled.buffer(VsrHeaders.HEADER_SIZE); + frame.writeZero(VsrHeaders.HEADER_SIZE); + frame.setIntLE(VsrHeaders.SIZE_OFFSET, VsrHeaders.HEADER_SIZE); + return frame; + } + + private static ByteBuf requestFrame(int operation, long requestId) { + ByteBuf frame = emptyFrame(); + frame.setByte(VsrHeaders.REQUEST_OPERATION_OFFSET, operation); + frame.setLongLE(VsrHeaders.REQUEST_ID_OFFSET, requestId); + return frame; + } + + private static ByteBuf replyFrame(int operation, long requestId, ByteBuf body) { + ByteBuf frame = emptyFrame(); + frame.setByte(VsrHeaders.COMMAND_OFFSET, VsrHeaders.COMMAND_REPLY); + frame.setByte(VsrHeaders.REPLY_OPERATION_OFFSET, operation); + frame.setLongLE(VsrHeaders.REPLY_REQUEST_OFFSET, requestId); + frame.setIntLE(VsrHeaders.SIZE_OFFSET, VsrHeaders.HEADER_SIZE + body.readableBytes()); + frame.writeBytes(body); + body.release(); + return frame; + } + + private static ByteBuf evictionFrame(int reason) { + ByteBuf frame = emptyFrame(); + frame.setByte(VsrHeaders.COMMAND_OFFSET, VsrHeaders.COMMAND_EVICTION); + frame.setByte(VsrHeaders.EVICTION_REASON_OFFSET, reason); + return frame; + } + + private static int rawErrorCode(CompletableFuture future) { + try { + future.get().release(); + throw new AssertionError("Expected the response future to fail"); + } catch (ExecutionException e) { + assertThat(e.getCause()).isInstanceOf(IggyServerException.class); + return ((IggyServerException) e.getCause()).getRawErrorCode(); + } catch (InterruptedException e) { + throw new AssertionError(e); + } + } +} diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/ConsumerOffsetsClientBaseTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/ConsumerOffsetsClientBaseTest.java index 633c84cd88..da9fd267f7 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/ConsumerOffsetsClientBaseTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/ConsumerOffsetsClientBaseTest.java @@ -54,7 +54,9 @@ void shouldGetConsumerOffset() { // when var consumer = new Consumer(Consumer.Kind.Consumer, ConsumerId.of(1223L)); - consumerOffsetsClient.storeConsumerOffset(STREAM_NAME, TOPIC_NAME, Optional.empty(), consumer, BigInteger.ZERO); + // The VSR client routes the store to its partition consensus group, so + // the partition id must be explicit. + consumerOffsetsClient.storeConsumerOffset(STREAM_NAME, TOPIC_NAME, Optional.of(0L), consumer, BigInteger.ZERO); var consumerOffset = consumerOffsetsClient.getConsumerOffset(STREAM_NAME, TOPIC_NAME, Optional.of(0L), consumer); diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/MessagesClientBaseTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/MessagesClientBaseTest.java index 040e1790b0..fd4034f2bb 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/MessagesClientBaseTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/MessagesClientBaseTest.java @@ -69,6 +69,70 @@ void shouldSendAndGetMessages() { assertThat(polledMessages.messages()).hasSize(1); } + @Test + void shouldSendMessageWithBalancedPartitioning() { + // given + setUpStreamAndTopic(); + + // when + String text = "message from java sdk"; + messagesClient.sendMessages(STREAM_NAME, TOPIC_NAME, Partitioning.balanced(), List.of(Message.of(text))); + + var polledMessages = messagesClient.pollMessages( + STREAM_NAME, + TOPIC_NAME, + empty(), + Consumer.of(0L), + new PollingStrategy(PollingKind.Last, BigInteger.TEN), + 10L, + false); + + // then + assertThat(polledMessages.messages()).hasSize(1); + } + + @Test + void shouldSendMessageWithMessageKeyPartitioning() { + // given + setUpStreamAndTopic(); + + // when + String text = "message from java sdk"; + messagesClient.sendMessages( + STREAM_NAME, TOPIC_NAME, Partitioning.messagesKey("test-key"), List.of(Message.of(text))); + var polledMessages = messagesClient.pollMessages( + STREAM_NAME, + TOPIC_NAME, + empty(), + Consumer.of(0L), + new PollingStrategy(PollingKind.Last, BigInteger.TEN), + 10L, + false); + + // then + assertThat(polledMessages.messages()).hasSize(1); + } + + @Test + void shouldReturnSendConfirmations() { + // given + setUpStreamAndTopic(); + + // when + var firstResponse = messagesClient.sendMessages( + STREAM_NAME, TOPIC_NAME, Partitioning.partitionId(0L), List.of(Message.of("first"))); + var secondResponse = messagesClient.sendMessages( + STREAM_NAME, TOPIC_NAME, Partitioning.partitionId(0L), List.of(Message.of("second"))); + + // then + assertThat(firstResponse.confirmations()).hasSize(1); + var firstConfirmation = firstResponse.confirmations().get(0); + assertThat(firstConfirmation.partitionId()).isEqualTo(0L); + assertThat(firstConfirmation.baseOffset()).isEqualTo(BigInteger.ZERO); + assertThat(secondResponse.confirmations()).hasSize(1); + assertThat(secondResponse.confirmations().get(0).baseOffset()).isEqualTo(BigInteger.ONE); + } + @Test void shouldPollMessagesWithFirstStrategy() { // given @@ -149,48 +213,4 @@ void shouldVerifyMessageContentRoundTrip() { assertThat(polledMessages.messages()).hasSize(1); assertThat(new String(polledMessages.messages().get(0).payload())).isEqualTo(content); } - - @Test - void shouldSendMessageWithBalancedPartitioning() { - // given - setUpStreamAndTopic(); - - // when - String text = "message from java sdk"; - messagesClient.sendMessages(STREAM_NAME, TOPIC_NAME, Partitioning.balanced(), List.of(Message.of(text))); - - var polledMessages = messagesClient.pollMessages( - STREAM_NAME, - TOPIC_NAME, - empty(), - Consumer.of(0L), - new PollingStrategy(PollingKind.Last, BigInteger.TEN), - 10L, - false); - - // then - assertThat(polledMessages.messages()).hasSize(1); - } - - @Test - void shouldSendMessageWithMessageKeyPartitioning() { - // given - setUpStreamAndTopic(); - - // when - String text = "message from java sdk"; - messagesClient.sendMessages( - STREAM_NAME, TOPIC_NAME, Partitioning.messagesKey("test-key"), List.of(Message.of(text))); - var polledMessages = messagesClient.pollMessages( - STREAM_NAME, - TOPIC_NAME, - empty(), - Consumer.of(0L), - new PollingStrategy(PollingKind.Last, BigInteger.TEN), - 10L, - false); - - // then - assertThat(polledMessages.messages()).hasSize(1); - } } diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/SystemClientBaseTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/SystemClientBaseTest.java index 550b69b40b..1e4f096f33 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/SystemClientBaseTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/SystemClientBaseTest.java @@ -58,6 +58,8 @@ void shouldGetClusterMetadataForSingleNode() { // then assertThat(metadata).isNotNull(); + // The server runs standalone (cluster disabled), so it synthesizes + // itself as the sole leader of a single-node roster. assertThat(metadata.name()).isEqualTo("single-node"); assertThat(metadata.nodes()).hasSize(1); var node = metadata.nodes().get(0); diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/UsersClientBaseTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/UsersClientBaseTest.java index 78bcea06c1..a5724ba051 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/UsersClientBaseTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/UsersClientBaseTest.java @@ -25,7 +25,6 @@ import org.apache.iggy.user.UserInfo; import org.apache.iggy.user.UserInfoDetails; import org.apache.iggy.user.UserStatus; -import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -221,7 +220,7 @@ void shouldReturnEmptyForNonExistingUser() { assertThat(user).isEmpty(); } - private static @NotNull GlobalPermissions createGlobalPermissions(boolean manageServers) { + private static GlobalPermissions createGlobalPermissions(boolean manageServers) { return new GlobalPermissions(manageServers, false, false, false, false, false, false, false, false, false); } } diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/ConsumerGroupsTcpClientTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/ConsumerGroupsTcpClientTest.java index 9ed7565554..1ecb61b9dd 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/ConsumerGroupsTcpClientTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/ConsumerGroupsTcpClientTest.java @@ -21,12 +21,21 @@ import org.apache.iggy.client.blocking.ConsumerGroupsClientBaseTest; import org.apache.iggy.client.blocking.IggyBaseClient; +import org.apache.iggy.consumergroup.Consumer; +import org.apache.iggy.exception.IggyResourceNotFoundException; import org.apache.iggy.identifier.ConsumerId; +import org.apache.iggy.message.Message; +import org.apache.iggy.message.Partitioning; +import org.apache.iggy.message.PollingStrategy; import org.junit.jupiter.api.Test; +import java.util.List; +import java.util.Optional; + import static org.apache.iggy.TestConstants.STREAM_NAME; import static org.apache.iggy.TestConstants.TOPIC_NAME; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; class ConsumerGroupsTcpClientTest extends ConsumerGroupsClientBaseTest { @@ -60,4 +69,81 @@ void shouldJoinAndLeaveConsumerGroup() { .get(); assertThat(group.membersCount()).isEqualTo(0); } + + @Test + void shouldSyncConsumerGroupAssignmentAfterJoin() { + // given + setUpStreamAndTopic(); + var group = consumerGroupsClient.createConsumerGroup(STREAM_NAME, TOPIC_NAME, "consumer-group-42"); + ConsumerId groupId = ConsumerId.of(group.id()); + + // when + consumerGroupsClient.joinConsumerGroup(STREAM_NAME, TOPIC_NAME, groupId); + var assignment = consumerGroupsClient.syncConsumerGroup(STREAM_NAME, TOPIC_NAME, groupId); + + // then — the only member owns the topic's single partition + assertThat(assignment).isPresent(); + assertThat(assignment.get().partitions()).hasSize(1); + } + + @Test + void shouldReturnEmptyAssignmentWhenNotMember() { + // given + setUpStreamAndTopic(); + var group = consumerGroupsClient.createConsumerGroup(STREAM_NAME, TOPIC_NAME, "consumer-group-42"); + ConsumerId groupId = ConsumerId.of(group.id()); + + // when + var assignment = consumerGroupsClient.syncConsumerGroup(STREAM_NAME, TOPIC_NAME, groupId); + + // then + assertThat(assignment).isEmpty(); + } + + @Test + void shouldPollAsGroupMemberWithoutExplicitPartition() { + // given + setUpStreamAndTopic(); + var group = consumerGroupsClient.createConsumerGroup(STREAM_NAME, TOPIC_NAME, "consumer-group-42"); + ConsumerId groupId = ConsumerId.of(group.id()); + consumerGroupsClient.joinConsumerGroup(STREAM_NAME, TOPIC_NAME, groupId); + client.messages() + .sendMessages( + STREAM_NAME, TOPIC_NAME, Partitioning.partitionId(0L), List.of(Message.of("group message"))); + + // when — the partition is selected client-side from the synced assignment + var polledMessages = client.messages() + .pollMessages( + STREAM_NAME, + TOPIC_NAME, + Optional.empty(), + Consumer.group(groupId), + PollingStrategy.first(), + 10L, + false); + + // then + assertThat(polledMessages.messages()).hasSize(1); + assertThat(new String(polledMessages.messages().get(0).payload())).isEqualTo("group message"); + } + + @Test + void shouldFailGroupPollWhenNotJoined() { + // given + setUpStreamAndTopic(); + var group = consumerGroupsClient.createConsumerGroup(STREAM_NAME, TOPIC_NAME, "consumer-group-42"); + ConsumerId groupId = ConsumerId.of(group.id()); + + // when / then + assertThatThrownBy(() -> client.messages() + .pollMessages( + STREAM_NAME, + TOPIC_NAME, + Optional.empty(), + Consumer.group(groupId), + PollingStrategy.first(), + 10L, + false)) + .isInstanceOf(IggyResourceNotFoundException.class); + } } diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/IggyTcpClientBuilderTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/IggyTcpClientBuilderTest.java index 7fa82e30d4..96607457e3 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/IggyTcpClientBuilderTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/IggyTcpClientBuilderTest.java @@ -91,21 +91,6 @@ void shouldCreateClientWithTimeoutConfiguration() { assertThat(clients).isNotNull(); } - @Test - void shouldCreateClientWithConnectionPoolSize() { - // Given: Builder with connection pool size - IggyTcpClient client = IggyTcpClient.builder() - .host(serverHost()) - .port(serverTcpPort()) - .connectionPoolSize(10) - .credentials("iggy", "iggy") - .buildAndLogin(); - - // Then: Should succeed - List clients = client.system().getClients(); - assertThat(clients).isNotNull(); - } - @Test void shouldCreateClientWithRetryPolicy() { // Given: Builder with exponential backoff retry policy @@ -159,7 +144,6 @@ void shouldCreateClientWithAllOptions() { .port(serverTcpPort()) .connectionTimeout(Duration.ofSeconds(30)) .requestTimeout(Duration.ofSeconds(10)) - .connectionPoolSize(10) .retryPolicy(RetryPolicy.exponentialBackoff(3, Duration.ofMillis(100), Duration.ofSeconds(5), 2.0)) .credentials("iggy", "iggy") .buildAndLogin(); diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/MessagesTcpClientTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/MessagesTcpClientTest.java index aabdf19e0c..c644b50ac7 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/MessagesTcpClientTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/client/blocking/tcp/MessagesTcpClientTest.java @@ -21,6 +21,15 @@ import org.apache.iggy.client.blocking.IggyBaseClient; import org.apache.iggy.client.blocking.MessagesClientBaseTest; +import org.apache.iggy.message.Message; +import org.apache.iggy.message.Partitioning; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.apache.iggy.TestConstants.STREAM_NAME; +import static org.apache.iggy.TestConstants.TOPIC_NAME; +import static org.assertj.core.api.Assertions.assertThat; class MessagesTcpClientTest extends MessagesClientBaseTest { @@ -28,4 +37,26 @@ class MessagesTcpClientTest extends MessagesClientBaseTest { protected IggyBaseClient getClient() { return TcpClientFactory.create(serverHost(), serverTcpPort()); } + + /* + * The TCP client resolves balanced and key-based partitioning to an + * explicit partition id before encoding the frame (the VSR broker routes + * explicit partitions only), so messages with the same key must land on + * the same partition. + */ + + @Test + void shouldRouteSameMessageKeyToSamePartition() { + setUpStreamAndTopic(); + + var firstResponse = messagesClient.sendMessages( + STREAM_NAME, TOPIC_NAME, Partitioning.messagesKey("test-key"), List.of(Message.of("first"))); + var secondResponse = messagesClient.sendMessages( + STREAM_NAME, TOPIC_NAME, Partitioning.messagesKey("test-key"), List.of(Message.of("second"))); + + assertThat(firstResponse.confirmations()).hasSize(1); + assertThat(secondResponse.confirmations()).hasSize(1); + assertThat(secondResponse.confirmations().get(0).partitionId()) + .isEqualTo(firstResponse.confirmations().get(0).partitionId()); + } } diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/exception/IggyErrorCodeTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/exception/IggyErrorCodeTest.java index 3cb4c5118f..97e60579e5 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/exception/IggyErrorCodeTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/exception/IggyErrorCodeTest.java @@ -156,10 +156,12 @@ void shouldParseCodeOne() { "1, ERROR", "3, INVALID_COMMAND", "4, INVALID_FORMAT", - "6, FEATURE_UNAVAILABLE", + "5, FEATURE_UNAVAILABLE", + "6, INVALID_IDENTIFIER", "7, CANNOT_PARSE_INT", "8, CANNOT_PARSE_SLICE", "9, CANNOT_PARSE_UTF8", + "30, STALE_CLIENT", // Resource errors "20, RESOURCE_NOT_FOUND", @@ -181,6 +183,8 @@ void shouldParseCodeOne() { "52, CLIENT_NOT_FOUND", "53, INVALID_PAT_TOKEN", "54, PAT_NAME_ALREADY_EXISTS", + "57, TRANSIENT_NOT_COMMITTED", + "58, TRANSIENT_NOT_ACCEPTED", "77, PASSWORD_DOES_NOT_MATCH", "78, PASSWORD_HASH_INTERNAL_ERROR", @@ -223,6 +227,9 @@ void shouldParseCodeOne() { "7002, TOO_BIG_MESSAGE", "7003, INVALID_MESSAGE_CHECKSUM", "7004, MESSAGE_NOT_FOUND", + + // VSR protocol errors + "14003, INCOMPATIBLE_PROTOCOL_VERSION", }) void fromCodeReturnsExpectedIggyErrorCodeWhenCodeIsValid(int code, IggyErrorCode expected) { var iggyErrorCode = IggyErrorCode.fromCode(code); diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/exception/IggyServerExceptionTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/exception/IggyServerExceptionTest.java index cbfb55efea..9835c5b62b 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/exception/IggyServerExceptionTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/exception/IggyServerExceptionTest.java @@ -116,6 +116,15 @@ void shouldBuildMessageWithUnknownRawCode() { assertThat(exception.getMessage()).isEqualTo("Server error [code=88888]"); } + @Test + void shouldBuildMessageWithTransientErrorCode() { + IggyServerException exception = IggyServerException.fromTcpResponse(57, new byte[0]); + + assertThat(exception.getErrorCode()).isEqualTo(IggyErrorCode.TRANSIENT_NOT_COMMITTED); + assertThat(exception.getMessage()) + .isEqualTo("Server error [code=57 (TRANSIENT_NOT_COMMITTED)]: Server error"); + } + @Test void shouldBuildMessageWithEmptyReason() { // given diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/exception/IggyValidationExceptionTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/exception/IggyValidationExceptionTest.java index c7f9877ebc..c0e3e12b25 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/exception/IggyValidationExceptionTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/exception/IggyValidationExceptionTest.java @@ -49,6 +49,7 @@ void constructorCreatesExpectedIggyValidationException() { "INVALID_COMMAND", "INVALID_FORMAT", "FEATURE_UNAVAILABLE", + "INVALID_IDENTIFIER", "CANNOT_PARSE_INT", "CANNOT_PARSE_SLICE", "CANNOT_PARSE_UTF8", @@ -75,6 +76,7 @@ void matchesReturnsTrueForValidationRelatedCodes(IggyErrorCode code) { "INVALID_COMMAND", "INVALID_FORMAT", "FEATURE_UNAVAILABLE", + "INVALID_IDENTIFIER", "CANNOT_PARSE_INT", "CANNOT_PARSE_SLICE", "CANNOT_PARSE_UTF8", diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/hash/XxHash32Test.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/hash/XxHash32Test.java new file mode 100644 index 0000000000..e454d58122 --- /dev/null +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/hash/XxHash32Test.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iggy.hash; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import java.nio.charset.StandardCharsets; + +import static org.assertj.core.api.Assertions.assertThat; + +class XxHash32Test { + + /** + * Golden vectors for XXH32 with seed 0, matching the canonical xxHash + * reference implementation and {@code twox_hash::XxHash32::oneshot(0, ..)} + * used by the Rust SDK for message-key partitioning. + */ + @ParameterizedTest + @CsvSource({ + "'', 02CC5D05", + "a, 550D7456", + "abc, 32D153FF", + "message digest, 7C948494", + "abcdefghijklmnopqrstuvwxyz, 63A14D5F", + "user-123, 0CC9EC8C", + "order-key, 43A01006", + }) + void shouldMatchReferenceVectors(String input, String expectedHex) { + long hash = XxHash32.hashUnsigned(input.getBytes(StandardCharsets.UTF_8)); + + assertThat(hash).isEqualTo(Long.parseLong(expectedHex, 16)); + } + + @Test + void shouldHashAllByteValuesAcrossEveryLoopShape() { + // 256 bytes exercises the 16-byte stripes, the 4-byte tail chunks and + // the single-byte tail in one input + byte[] data = new byte[256]; + for (int i = 0; i < data.length; i++) { + data[i] = (byte) i; + } + + assertThat(XxHash32.hashUnsigned(data)).isEqualTo(0x59441253L); + } + + @Test + void shouldReturnUnsignedValueForHashesAboveIntegerMax() { + // "Nobody inspects the spammish repetition" hashes to 0xE2293B2F, + // which is negative as a signed int + long hash = XxHash32.hashUnsigned("Nobody inspects the spammish repetition".getBytes(StandardCharsets.UTF_8)); + + assertThat(hash).isEqualTo(0xE2293B2FL); + } +} diff --git a/foreign/java/java-sdk/src/test/java/org/apache/iggy/serde/BytesDeserializerTest.java b/foreign/java/java-sdk/src/test/java/org/apache/iggy/serde/BytesDeserializerTest.java index 637456ab1d..ee835f5b39 100644 --- a/foreign/java/java-sdk/src/test/java/org/apache/iggy/serde/BytesDeserializerTest.java +++ b/foreign/java/java-sdk/src/test/java/org/apache/iggy/serde/BytesDeserializerTest.java @@ -42,6 +42,7 @@ import static org.apache.iggy.serde.BytesDeserializer.readClientInfoDetails; import static org.apache.iggy.serde.BytesDeserializer.readClusterMetadata; import static org.apache.iggy.serde.BytesDeserializer.readConsumerGroup; +import static org.apache.iggy.serde.BytesDeserializer.readConsumerGroupAssignment; import static org.apache.iggy.serde.BytesDeserializer.readConsumerGroupDetails; import static org.apache.iggy.serde.BytesDeserializer.readConsumerGroupInfo; import static org.apache.iggy.serde.BytesDeserializer.readConsumerGroupMember; @@ -53,6 +54,7 @@ import static org.apache.iggy.serde.BytesDeserializer.readPolledMessage; import static org.apache.iggy.serde.BytesDeserializer.readPolledMessages; import static org.apache.iggy.serde.BytesDeserializer.readRawPersonalAccessToken; +import static org.apache.iggy.serde.BytesDeserializer.readSendMessagesResponse; import static org.apache.iggy.serde.BytesDeserializer.readStats; import static org.apache.iggy.serde.BytesDeserializer.readStreamBase; import static org.apache.iggy.serde.BytesDeserializer.readStreamDetails; @@ -314,6 +316,55 @@ void shouldDeserializeConsumerGroupDetails() { } } + @Nested + class ConsumerGroupAssignmentDeserialization { + + @Test + void shouldDeserializeAssignment() { + // given — [generation:8][partitions_count:4][partition_id:4]* + ByteBuf buffer = Unpooled.buffer(); + writeU64(buffer, BigInteger.valueOf(7)); // generation + buffer.writeIntLE(3); // partitions count + buffer.writeIntLE(0); + buffer.writeIntLE(1); + buffer.writeIntLE(2); + + // when + var assignment = readConsumerGroupAssignment(buffer); + + // then + assertThat(assignment.generation()).isEqualTo(7L); + assertThat(assignment.partitions()).containsExactly(0L, 1L, 2L); + assertThat(buffer.isReadable()).isFalse(); + } + + @Test + void shouldDeserializeMemberWithoutPartitions() { + // given — distinct from an empty body, which means "not a member" + ByteBuf buffer = Unpooled.buffer(); + writeU64(buffer, BigInteger.valueOf(2)); // generation + buffer.writeIntLE(0); // partitions count + + // when + var assignment = readConsumerGroupAssignment(buffer); + + // then + assertThat(assignment.generation()).isEqualTo(2L); + assertThat(assignment.partitions()).isEmpty(); + } + + @Test + void shouldRejectPartitionCountLargerThanPayload() { + ByteBuf buffer = Unpooled.buffer(); + buffer.writeLongLE(2); + buffer.writeIntLE(Integer.MAX_VALUE); + + assertThatThrownBy(() -> readConsumerGroupAssignment(buffer)) + .isInstanceOf(IggyMalformedResponseException.class) + .hasMessageContaining("partitions count"); + } + } + @Nested class ConsumerOffsetDeserialization { @@ -425,6 +476,101 @@ void shouldDeserializePolledMessages() { } } + @Nested + class SendMessagesResponseDeserialization { + + private ByteBuf singleConfirmation() { + ByteBuf buffer = Unpooled.buffer(); + buffer.writeIntLE(1); // confirmations count + buffer.writeIntLE(3); // stream ID + buffer.writeIntLE(5); // topic ID + buffer.writeIntLE(7); // partition ID + writeU64(buffer, BigInteger.valueOf(41)); // base offset + return buffer; + } + + @Test + void shouldDeserializeSingleConfirmation() { + // given + ByteBuf buffer = singleConfirmation(); + + // when + var response = readSendMessagesResponse(buffer); + + // then + assertThat(response.confirmations()).hasSize(1); + var confirmation = response.confirmations().get(0); + assertThat(confirmation.streamId()).isEqualTo(3L); + assertThat(confirmation.topicId()).isEqualTo(5L); + assertThat(confirmation.partitionId()).isEqualTo(7L); + assertThat(confirmation.baseOffset()).isEqualTo(BigInteger.valueOf(41)); + } + + @Test + void shouldDeserializeEmptyBodyAsNoConfirmations() { + // given — legacy servers ack a send with an empty body + ByteBuf buffer = Unpooled.buffer(); + + // when + var response = readSendMessagesResponse(buffer); + + // then + assertThat(response.confirmations()).isEmpty(); + } + + @Test + void shouldDeserializeZeroCountAsNoConfirmations() { + // given — the server sends count = 0 when it could not decode the batch + ByteBuf buffer = Unpooled.buffer(); + buffer.writeIntLE(0); + + // when + var response = readSendMessagesResponse(buffer); + + // then + assertThat(response.confirmations()).isEmpty(); + } + + @Test + void shouldRejectConfirmationCountLargerThanPayload() { + ByteBuf buffer = Unpooled.buffer(); + buffer.writeIntLE(Integer.MAX_VALUE); + + assertThatThrownBy(() -> readSendMessagesResponse(buffer)) + .isInstanceOf(IggyMalformedResponseException.class) + .hasMessageContaining("confirmations count"); + } + + @Test + void shouldFailOnTrailingBytes() { + // given + ByteBuf buffer = singleConfirmation(); + buffer.writeByte(0xAB); + + // when / then + assertThatThrownBy(() -> readSendMessagesResponse(buffer)) + .isInstanceOf(IggyMalformedResponseException.class) + .hasMessageContaining("trailing"); + } + + @Test + void shouldFailOnTruncationAtEveryByte() { + // given + ByteBuf complete = singleConfirmation(); + byte[] bytes = new byte[complete.readableBytes()]; + complete.getBytes(0, bytes); + + for (int length = 1; length < bytes.length; length++) { + ByteBuf truncated = Unpooled.wrappedBuffer(bytes, 0, length); + + // when / then + assertThatThrownBy(() -> readSendMessagesResponse(truncated)) + .as("truncated at byte %d", length) + .isInstanceOf(RuntimeException.class); + } + } + } + @Nested class StatsDeserialization { diff --git a/foreign/node/CHANGELOG.md b/foreign/node/CHANGELOG.md deleted file mode 100644 index 732a4f587f..0000000000 --- a/foreign/node/CHANGELOG.md +++ /dev/null @@ -1,95 +0,0 @@ -# Changelog - -## [1.0.6](https://github.com/iggy-rs/iggy-node-client/compare/v1.0.5...v1.0.6) (2025-01-14) - -### Bug Fixes - -* add commitlint as husky pre-commit hook and ci check ([#25](https://github.com/iggy-rs/iggy-node-client/issues/25)) ([5dbac10](https://github.com/iggy-rs/iggy-node-client/commit/5dbac1071bea5c26f852a60361ac51217df09d25)) - -## [1.0.5](https://github.com/iggy-rs/iggy-node-client/compare/v1.0.4...v1.0.5) (2025-01-14) - -### Bug Fixes - -* set package.json license to Apache-2.0 as well ([#24](https://github.com/iggy-rs/iggy-node-client/issues/24)) ([2523c0f](https://github.com/iggy-rs/iggy-node-client/commit/2523c0fde82958e8a13bb95b2c2d0babe0d1d290)) - -## [1.0.4](https://github.com/iggy-rs/iggy-node-client/compare/v1.0.3...v1.0.4) (2024-12-16) - -### Bug Fixes - -* add tests, fix headers.double type, ehance typing ([431063f](https://github.com/iggy-rs/iggy-node-client/commit/431063f253e0fb1739188bbd6614cfc06ccb3fd4)) - -## [1.0.3](https://github.com/iggy-rs/iggy-node-client/compare/v1.0.2...v1.0.3) (2024-11-24) - -### Bug Fixes - -* upgrade dependencies ([#19](https://github.com/iggy-rs/iggy-node-client/issues/19)) ([a05d5c2](https://github.com/iggy-rs/iggy-node-client/commit/a05d5c2484f8711f72a23c4ee1222d29e323a5ba)) - -## [1.0.2](https://github.com/iggy-rs/iggy-node-client/compare/v1.0.1...v1.0.2) (2024-11-24) - -### Bug Fixes - -* **npm:** configure package as public ([#18](https://github.com/iggy-rs/iggy-node-client/issues/18)) ([7a56455](https://github.com/iggy-rs/iggy-node-client/commit/7a5645512a564322af343bc7ba748c8c58dfab1e)) - -## [1.0.1](https://github.com/iggy-rs/iggy-node-client/compare/v1.0.0...v1.0.1) (2024-11-24) - -### Bug Fixes - -* **npm:** publish package ([#17](https://github.com/iggy-rs/iggy-node-client/issues/17)) ([6e2f60e](https://github.com/iggy-rs/iggy-node-client/commit/6e2f60e2f4484b57596285ff79d76ea647962595)) - -## 1.0.0 (2024-11-24) - -### Bug Fixes - -* add e2e tests, fix create-group return ([ad52b43](https://github.com/iggy-rs/iggy-node-client/commit/ad52b43a6ee5e8f868eb1178a9b9f02f11cb1204)) -* add package's keywords, cleans up logs ([5287b74](https://github.com/iggy-rs/iggy-node-client/commit/5287b74006733aef4429d889c0cc38a80f375dc2)) -* correctly destroy socket when destroy function is called on TCPClient ([76f7d07](https://github.com/iggy-rs/iggy-node-client/commit/76f7d07a96e44f6701998bb9fb2dab8fa4d75489)) -* enforce check on message id to be either uuid string or 0 ([d05b898](https://github.com/iggy-rs/iggy-node-client/commit/d05b898d62b097ca9770cdab289707546c2fe6aa)) -* fix client auth quirks, add handleResponse & deserializePollMessage as transform stream ([0458d57](https://github.com/iggy-rs/iggy-node-client/commit/0458d579de45605588fff7b7d01119c9625364da)) -* fix consumer group commands ([a33094f](https://github.com/iggy-rs/iggy-node-client/commit/a33094f66eb2e433b2fc8cc6adf7f6a122b0f365)) -* fix createtopic, add new compressionAlgorithm param ([f93a444](https://github.com/iggy-rs/iggy-node-client/commit/f93a4441f4bcd790763d982ead981c4ac86b33c5)) -* fix getStats command (add new totalCpuUsage field) ([#3](https://github.com/iggy-rs/iggy-node-client/issues/3)) ([de4cfda](https://github.com/iggy-rs/iggy-node-client/commit/de4cfdad4046f556a51878fbadb8dde0df9302c5)) -* fix github ci ([78870e3](https://github.com/iggy-rs/iggy-node-client/commit/78870e389333c8b7ca2425748c8806fa5e36a7ae)) -* fix message header typing ([1276374](https://github.com/iggy-rs/iggy-node-client/commit/12763749f95d29a78028f95b5bc33281a62246c9)) -* fix message headers serialization bug ([22ffe16](https://github.com/iggy-rs/iggy-node-client/commit/22ffe1603db9b7a94deadb1fcf6a25f81cfea868)) -* fix module type export ([b0bcdc7](https://github.com/iggy-rs/iggy-node-client/commit/b0bcdc7945ba68d519a129ef33b4353538e9d64e)) -* fix npm test command for ci ([842fe69](https://github.com/iggy-rs/iggy-node-client/commit/842fe697548224a08012574d2e44f14100105b63)) -* fix Partitioning.MessageKey type, fix indent ([05d05b6](https://github.com/iggy-rs/iggy-node-client/commit/05d05b6db68b33bc9045fee16c9fca8c8eb0ae6d)) -* fix tcp client options ([f9bd442](https://github.com/iggy-rs/iggy-node-client/commit/f9bd44204f86f2becfe674c9b38f09657c00ac6c)) -* fix topic deserialisation bug ([3e787be](https://github.com/iggy-rs/iggy-node-client/commit/3e787be2546f29f1f8d31e9e1388e19a62729e50)) -* fix updateUser and changePassword command ([d01e086](https://github.com/iggy-rs/iggy-node-client/commit/d01e08621bbf976c7dc5578273a253a0dcc43e72)) -* fix var naming, add some test ([40d91dd](https://github.com/iggy-rs/iggy-node-client/commit/40d91ddfe41f6114ca3c7da2a30225e81e3226bc)) -* get rid of enums, add type helpers ([6e2613b](https://github.com/iggy-rs/iggy-node-client/commit/6e2613b2f1ab0112d401f87a2f0cfb6e77b8d99d)) -* more e2e tests ([d537aa7](https://github.com/iggy-rs/iggy-node-client/commit/d537aa7454b983edb471fa354e55b5e814fb1524)) -* no ssh pull ([#12](https://github.com/iggy-rs/iggy-node-client/issues/12)) ([f13f0ed](https://github.com/iggy-rs/iggy-node-client/commit/f13f0edd5adf4584ab7f02620a159317654251e5)) -* remove bad symbol in the CI definition ([1878a0c](https://github.com/iggy-rs/iggy-node-client/commit/1878a0c501224d17eefc986ccd5d89e481cc15b8)) -* remove console.error on normal close signal ([0f8c993](https://github.com/iggy-rs/iggy-node-client/commit/0f8c99330b2352220ee23df0b7923185d710cd89)) -* remove initial .gitignore and README.md to avoid rebase conflict ([931b9b8](https://github.com/iggy-rs/iggy-node-client/commit/931b9b8f5d0b8a249de8a30c3be9435c93a8461f)) -* update createUser, createStream & createTopic command return value ([8712b0f](https://github.com/iggy-rs/iggy-node-client/commit/8712b0f5021b7852361117f6fc764eac3851beb7)) -* update readme ([263f271](https://github.com/iggy-rs/iggy-node-client/commit/263f271fed2c529b89329e88d83fc4100b04c639)) -* use debug lib, make poolsize configurable ([2f66e89](https://github.com/iggy-rs/iggy-node-client/commit/2f66e89220b30b3e33aa773b6471b1669658f37f)) -* use pat as token for semantic release ([#10](https://github.com/iggy-rs/iggy-node-client/issues/10)) ([5eccfd2](https://github.com/iggy-rs/iggy-node-client/commit/5eccfd281763773140ce9be39a2c022b2de803b6)) - -### Features - -* add base ci workflow ([#2](https://github.com/iggy-rs/iggy-node-client/issues/2)) ([4cbde14](https://github.com/iggy-rs/iggy-node-client/commit/4cbde140409841bf3d440f01ccaca1a855f37b13)) -* add command client with socket pool management ([e747b29](https://github.com/iggy-rs/iggy-node-client/commit/e747b292374a7a8e1ee8f27d69a9203db3a6b09b)) -* add CommandResponseStream to wrap tcp socket, add parallel call safetiness ([0e6f38f](https://github.com/iggy-rs/iggy-node-client/commit/0e6f38fde621dc1cfe3f12376723f1b43ffee9bf)) -* add consumer stream facility ([8c19d3f](https://github.com/iggy-rs/iggy-node-client/commit/8c19d3fae30a5ab47d0f10b4747158e604aea37f)) -* add create, delete, join & leave consumer-group command ([e7ca376](https://github.com/iggy-rs/iggy-node-client/commit/e7ca3762a9fcb92a3bb5018cb66570079bec60f1)) -* add createPartition & deletePartition command ([2286c10](https://github.com/iggy-rs/iggy-node-client/commit/2286c106534704b02544613944e74f2549ca1a67)) -* add createUser and deleteUser command ([d25d837](https://github.com/iggy-rs/iggy-node-client/commit/d25d837d38abe89a038880651c19bd8925f1affe)) -* add getGroup and getGroups command ([cdfa766](https://github.com/iggy-rs/iggy-node-client/commit/cdfa766c9b800e59c2858599ccf87afa44c142dc)) -* add getOffset and storeOffset command, fix typos ([6f9a425](https://github.com/iggy-rs/iggy-node-client/commit/6f9a4256b713abcfdd6dc146b78268169c9bb198)) -* add pollMessage command ([fe59f82](https://github.com/iggy-rs/iggy-node-client/commit/fe59f825ebb0ba3a4c4c53e90cd8a28f9b21cca2)) -* add purgeTopic & purgeStream command ([86482d1](https://github.com/iggy-rs/iggy-node-client/commit/86482d12bfdf48b23973374e09cb9cc5e852ef18)) -* add SendMessages command ([e1d39d8](https://github.com/iggy-rs/iggy-node-client/commit/e1d39d887e36783e2ca23cd560060df3fc62099a)) -* add updateStream command ([89970c0](https://github.com/iggy-rs/iggy-node-client/commit/89970c0574f0bcade634b338522bcbe6990b4b43)) -* add updateTopic command ([f1e278e](https://github.com/iggy-rs/iggy-node-client/commit/f1e278e993a05c10f10a24761bd2e63f95d9891b)) -* add updateUser and changePassword command, fix permissions deserialization bug ([f3bcda3](https://github.com/iggy-rs/iggy-node-client/commit/f3bcda30b71f5d70145257534848ee4053e714f7)) -* better error, add some test ([f386854](https://github.com/iggy-rs/iggy-node-client/commit/f386854bd388e95a11251803dcb81292b33d7981)) -* publish to npm ([82268f5](https://github.com/iggy-rs/iggy-node-client/commit/82268f5963e72865355e2cfe919307d5bd1500ab)) -* reorganize client declaration ([7cb1bd8](https://github.com/iggy-rs/iggy-node-client/commit/7cb1bd857c5d1050ab6b8d2b0e6b70908b3bc105)) -* start low level command api and base tcp client ([7bb8b32](https://github.com/iggy-rs/iggy-node-client/commit/7bb8b32ec809e53d665295f9e0c069d535b88cae)) -* start unit test on serialization ([2082b71](https://github.com/iggy-rs/iggy-node-client/commit/2082b713a37d59a3d0482669a3a544a6a922ae41)) -* update modified commands for v0.3.0 server release (createTopic, updateTopic, login, createToken) ([cb4f0d1](https://github.com/iggy-rs/iggy-node-client/commit/cb4f0d17c61967e95a6b5502c70d6e8b8569e9c7)) -* wraps command to higher level api, starts client ([60f9466](https://github.com/iggy-rs/iggy-node-client/commit/60f9466cf92965f3bf4ff5918e50876e5bb9aa84)) diff --git a/foreign/node/README.md b/foreign/node/README.md index ecb0453793..994ffee608 100644 --- a/foreign/node/README.md +++ b/foreign/node/README.md @@ -32,18 +32,17 @@ npm i --save apache-iggy ### Response frame limit -**Compatibility note:** response frames larger than `maxResponseFrameSize` (default 64 MiB) are now rejected and close the connection under both framing modes. This is a behavior change for existing classic-framing clients. Raise the limit in the client configuration when polling very large batches. +**Compatibility note:** response frames larger than `maxResponseFrameSize` (default 64 MiB) are rejected and close the connection. Raise the limit in the client configuration when polling very large batches. ### VSR framing -Classic framing remains the default. Select VSR explicitly when connecting to -an Iggy VSR server: +The SDK speaks the VSR wire protocol exclusively and requires an Iggy VSR +server: ```typescript import { SimpleClient, getRawClient } from "apache-iggy"; const config = { - protocol: "vsr" as const, transport: "TCP" as const, options: { host: "127.0.0.1", port: 8090 }, credentials: { username: "iggy", password: "iggy" }, @@ -52,12 +51,18 @@ const client = new SimpleClient(getRawClient(config)); const stats = await client.system.getStats(); ``` -VSR is a runtime protocol choice in Node.js, not a build feature. Codes absent -from the SDK command table use `Operation::NonReplicated` and carry the command -code in the request header's reserved field. The server remains authoritative -for classifying or rejecting extension commands. +Codes absent from the SDK command table use `Operation::NonReplicated` and +carry the command code in the request header's reserved field. The server +remains authoritative for classifying or rejecting extension commands. -The same npm package supports both framing modes over TCP and TLS. VSR restricts `Client` to one pooled connection because authentication, request sequencing, and consumer-group assignments belong to one consensus session. Configurations requesting more than one pooled connection fail before a socket is opened. +Sends must use explicit `Partitioning.PartitionId` partitioning: the client +routes each request to a partition-scoped namespace, so broker-side balancing +(`Partitioning.Balanced`) and key hashing (`Partitioning.MessageKey`) are +rejected before the request is sent. + + +VSR works over TCP and TLS. It restricts `Client` to one pooled connection because authentication, request sequencing, and consumer-group assignments belong to one consensus session. Configurations requesting more than one pooled connection fail before a socket is opened. VSR authentication translates the existing password and personal-access-token login APIs into the register handshake required by the consensus protocol. A @@ -68,7 +73,7 @@ new session. When the server's `[heartbeat]` eviction is enabled, configure the client's `heartbeatInterval` below the server heartbeat interval. Client heartbeats are disabled when `heartbeatInterval` is unset. -`sendBinaryRequest(code, payload)` has the same signature under classic and VSR framing. Known replicated commands use their registered operation, while unknown codes reach the server as non-replicated requests and are rejected by servers that do not register them. Classic request bytes remain unchanged. +`sendBinaryRequest(code, payload)` sends an arbitrary command code. Known replicated commands use their registered operation, while unknown codes reach the server as non-replicated requests and are rejected by servers that do not register them. ```typescript import { ResponseError } from "apache-iggy"; diff --git a/foreign/node/docker-compose.yml b/foreign/node/docker-compose.yml index b30064648f..58f6049ba2 100644 --- a/foreign/node/docker-compose.yml +++ b/foreign/node/docker-compose.yml @@ -17,14 +17,17 @@ services: iggy-server: - image: apache/iggy:latest + # The SDK is vsr-only, so build the vsr server from the repo instead + # of pulling the legacy apache/iggy image. + build: + context: ../.. + dockerfile: core/server/Dockerfile container_name: iggy-server restart: unless-stopped networks: - iggy ports: - 3000:3000 - - 8080:8080 - 8090:8090 volumes: - iggy-server:/local_data diff --git a/foreign/node/package-lock.json b/foreign/node/package-lock.json index 11295a84ab..b1a49f7244 100644 --- a/foreign/node/package-lock.json +++ b/foreign/node/package-lock.json @@ -1,12 +1,12 @@ { "name": "apache-iggy", - "version": "0.9.0-edge.1", + "version": "0.10.0-edge.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "apache-iggy", - "version": "0.9.0-edge.1", + "version": "0.10.0-edge.1", "license": "Apache-2.0", "dependencies": { "debug": "4.4.3", diff --git a/foreign/node/package.json b/foreign/node/package.json index 7f4bcb6fc2..b423ee3da3 100644 --- a/foreign/node/package.json +++ b/foreign/node/package.json @@ -1,7 +1,7 @@ { "name": "apache-iggy", "type": "module", - "version": "0.9.0-edge.1", + "version": "0.10.0-edge.1", "description": "Official Apache Iggy NodeJS SDK", "keywords": [ "iggy", @@ -37,7 +37,6 @@ "scripts": { "test:unit": "node --import @swc-node/register/esm-register --test --experimental-test-coverage './src/**/*.test.ts'", "test:e2e": "node --import @swc-node/register/esm-register --test --experimental-test-coverage --test-force-exit './src/e2e/*.e2e.ts'", - "test:e2e:vsr": "IGGY_TEST_PROTOCOL=vsr node --import @swc-node/register/esm-register --test --experimental-test-coverage --test-force-exit './src/e2e/tcp.*.e2e.ts'", "test:bdd": "cucumber-js --exit", "test": "npm run test:unit && npm run test:bdd && npm run test:e2e", "clean": "rm -Rf dist/", diff --git a/foreign/node/scripts/check-vsr-protocol.mjs b/foreign/node/scripts/check-vsr-protocol.mjs index cdcd432e2e..ebfcf66d2c 100644 --- a/foreign/node/scripts/check-vsr-protocol.mjs +++ b/foreign/node/scripts/check-vsr-protocol.mjs @@ -31,11 +31,9 @@ const [ rustHeader, rustCommand, rustOperation, - rustNamespace, rustProtocolCargo, nodeCodes, nodeHeader, - nodeNamespace, nodeOperation, nodeRegister, ] = await Promise.all([ @@ -44,11 +42,9 @@ const [ read('core/binary_protocol/src/consensus/header.rs'), read('core/binary_protocol/src/consensus/command.rs'), read('core/binary_protocol/src/consensus/operation.rs'), - read('core/binary_protocol/src/namespace.rs'), read('core/binary_protocol/Cargo.toml'), readNode('src/wire/command.code.ts'), readNode('src/wire/vsr/header.ts'), - readNode('src/wire/vsr/namespace.ts'), readNode('src/wire/vsr/operation.ts'), readNode('src/wire/vsr/register.ts'), ]); @@ -125,46 +121,14 @@ assert.deepEqual( 'Node replicated code-to-operation map differs from Rust dispatch' ); -const rustNamespaceValue = (name) => Number( - rustNamespace.match( - new RegExp(`pub const ${name}: usize = ([0-9_]+);`) - )?.[1].replaceAll('_', '') -); -const nodeNamespaceValue = (name) => Number( - nodeNamespace.match( - new RegExp(`const ${name} = ([0-9_]+);`) - )?.[1].replaceAll('_', '') -); -const namespaceLimits = new Map( - ['MAX_STREAMS', 'MAX_TOPICS', 'MAX_PARTITIONS'].map((name) => [ - name, - rustNamespaceValue(name) - ]) -); -for (const [name, value] of namespaceLimits) { - assert.ok(Number.isSafeInteger(value), `Rust ${name} was not found`); - assert.equal( - nodeNamespaceValue(name), - value, - `Node ${name} differs from Rust namespace layout` - ); -} - -const bitsRequired = (value) => BigInt(value).toString(2).length; -const expectedTopicShift = bitsRequired( - namespaceLimits.get('MAX_PARTITIONS') - 1 -); -const expectedStreamShift = expectedTopicShift + - bitsRequired(namespaceLimits.get('MAX_TOPICS') - 1); -const namespaceModule = await import( - pathToFileURL(resolve(nodeRoot, 'dist/wire/vsr/namespace.js')).href -); -assert.equal( - namespaceModule.packNamespace(1, 1, 1), - (1n << BigInt(expectedStreamShift)) | - (1n << BigInt(expectedTopicShift)) | - 1n, - 'Node namespace shifts differ from Rust namespace layout' +// No namespace-packing parity to check: the client wire carries no routing +// namespace, so the packing rules stay entirely server-side and this SDK has +// nothing to mirror. What still matters is that the client never grows a +// namespace field back -- the offset recomputation below catches that, since +// reintroducing one would move every field after it. +assert.ok( + !/namespace/i.test(nodeHeader.replace(/\/\*[\s\S]*?\*\/|\/\/.*/g, '')), + 'Node request header must not carry a namespace field' ); const rustEvictionBlock = diff --git a/foreign/node/src/bdd/auth.ts b/foreign/node/src/bdd/auth.ts index a67e496923..147327ee1d 100644 --- a/foreign/node/src/bdd/auth.ts +++ b/foreign/node/src/bdd/auth.ts @@ -18,17 +18,14 @@ import assert from 'node:assert/strict'; import { Client } from '../client/index.js'; -import type { Protocol } from '../client/index.js'; import { Given } from "@cucumber/cucumber"; import type { TestWorld } from './world.js'; import { getIggyAddress } from '../tcp.sm.utils.js'; const credentials = { username: 'iggy', password: 'iggy' }; const [host, port] = getIggyAddress(); -const protocol: Protocol = process.env.IGGY_TEST_PROTOCOL === 'vsr' ? 'vsr' : 'classic'; const opt = { - protocol, transport: 'TCP' as const, options: { host, port }, credentials diff --git a/foreign/node/src/client/client.config.test.ts b/foreign/node/src/client/client.config.test.ts index bc612da935..e4402e1b92 100644 --- a/foreign/node/src/client/client.config.test.ts +++ b/foreign/node/src/client/client.config.test.ts @@ -30,55 +30,34 @@ const config = (): ClientConfig => ({ }); describe('normalizeClientConfig', () => { - it('defaults to classic without changing classic pool sizing', () => { - const normalized = normalizeClientConfig({ - ...config(), - poolSize: { min: 2, max: 4 } - }); + it('applies the default response frame limit', () => { + const normalized = normalizeClientConfig(config()); - assert.equal(normalized.protocol, 'classic'); - assert.deepEqual(normalized.poolSize, { min: 2, max: 4 }); assert.equal( normalized.maxResponseFrameSize, DEFAULT_MAX_RESPONSE_FRAME_SIZE ); }); - it('restricts VSR to one pooled connection', () => { - const normalized = normalizeClientConfig({ - ...config(), - protocol: 'vsr' - }); + it('restricts the client to one pooled connection', () => { + const normalized = normalizeClientConfig(config()); assert.deepEqual(normalized.poolSize, { min: 1, max: 1 }); assert.throws( () => normalizeClientConfig({ ...config(), - protocol: 'vsr', poolSize: { max: 2 } }), /exactly one pooled connection/ ); }); - it('rejects invalid protocols before opening a socket', () => { - assert.throws( - () => normalizeClientConfig({ - ...config(), - protocol: 'auto' as 'vsr' - }), - /unsupported wire protocol/ - ); - }); - - it('supports VSR over TLS', () => { + it('supports TLS transport', () => { const normalized = normalizeClientConfig({ ...config(), - protocol: 'vsr', transport: 'TLS' }); - assert.equal(normalized.protocol, 'vsr'); assert.equal(normalized.transport, 'TLS'); assert.deepEqual(normalized.poolSize, { min: 1, max: 1 }); }); diff --git a/foreign/node/src/client/client.config.ts b/foreign/node/src/client/client.config.ts index 7fbb13e255..ed4030819b 100644 --- a/foreign/node/src/client/client.config.ts +++ b/foreign/node/src/client/client.config.ts @@ -15,20 +15,13 @@ // specific language governing permissions and limitations // under the License. -import type { ClientConfig, Protocol } from './client.type.js'; +import type { ClientConfig } from './client.type.js'; export const DEFAULT_MAX_RESPONSE_FRAME_SIZE = 64 * 1024 * 1024; -const isProtocol = (value: unknown): value is Protocol => - value === 'classic' || value === 'vsr'; - export const normalizeClientConfig = ( config: ClientConfig -): ClientConfig & { protocol: Protocol } => { - const protocol = config.protocol ?? 'classic'; - if (!isProtocol(protocol)) - throw new TypeError(`unsupported wire protocol: ${String(protocol)}`); - +): ClientConfig => { const maxResponseFrameSize = config.maxResponseFrameSize ?? DEFAULT_MAX_RESPONSE_FRAME_SIZE; if (!Number.isSafeInteger(maxResponseFrameSize) || @@ -37,17 +30,15 @@ export const normalizeClientConfig = ( 'maxResponseFrameSize must be a safe integer of at least 256 bytes' ); - if (protocol === 'vsr' && - ((config.poolSize?.min ?? 1) > 1 || (config.poolSize?.max ?? 1) > 1)) + if ((config.poolSize?.min ?? 1) > 1 || (config.poolSize?.max ?? 1) > 1) throw new TypeError( 'VSR clients currently support exactly one pooled connection' ); return { ...config, - protocol, options: { ...config.options }, maxResponseFrameSize, - ...(protocol === 'vsr' ? { poolSize: { min: 1, max: 1 } } : {}) + poolSize: { min: 1, max: 1 } }; }; diff --git a/foreign/node/src/client/client.connection.test.ts b/foreign/node/src/client/client.connection.test.ts index d9639f432f..5c11c54ba6 100644 --- a/foreign/node/src/client/client.connection.test.ts +++ b/foreign/node/src/client/client.connection.test.ts @@ -28,6 +28,9 @@ import { describe, it } from 'node:test'; import { ProtocolFrameError } from './client.frame.js'; import { IggyConnection } from './client.connection.js'; import type { ClientConfig } from './client.type.js'; +import { Command2, HEADER_SIZE, REPLY_OFFSET } from '../wire/vsr/header.js'; + +const FRAME_LIMIT = 2 * HEADER_SIZE; const startServer = async (): Promise => { const server = createServer(); @@ -37,7 +40,6 @@ const startServer = async (): Promise => { }; const connectionConfig = (server: Server): ClientConfig => ({ - protocol: 'classic', transport: 'TCP', options: { host: '127.0.0.1', @@ -45,9 +47,17 @@ const connectionConfig = (server: Server): ClientConfig => ({ }, credentials: { username: 'iggy', password: 'iggy' }, reconnect: { enabled: false, interval: 0, maxRetries: 0 }, - maxResponseFrameSize: 256 + maxResponseFrameSize: FRAME_LIMIT }); +const replyFrame = (body: Buffer): Buffer => { + const frame = Buffer.alloc(HEADER_SIZE + body.length); + frame.writeUInt32LE(frame.length, REPLY_OFFSET.size); + frame.writeUInt8(Command2.Reply, REPLY_OFFSET.command); + body.copy(frame, HEADER_SIZE); + return frame; +}; + const closeConnection = async ( connection: IggyConnection, server: Server @@ -75,7 +85,7 @@ describe('IggyConnection', () => { } ); - it('shares connection attempts, recognizes endpoints, and writes commands', + it('shares connection attempts, recognizes endpoints, and writes frames', async () => { const server = await startServer(); const received = new Promise((resolve) => { @@ -109,10 +119,9 @@ describe('IggyConnection', () => { true ); - connection.writeCommand(1, Buffer.from('payload')); - const command = await received; - assert.equal(command.readUInt32LE(4), 1); - assert.deepEqual(command.subarray(8), Buffer.from('payload')); + const frame = replyFrame(Buffer.from('payload')); + connection.writeFrame(frame); + assert.deepEqual(await received, frame); } finally { await closeConnection(connection, server); } @@ -125,17 +134,14 @@ describe('IggyConnection', () => { const connection = new IggyConnection(connectionConfig(server)); try { await connection.connect(); - const body = Buffer.from('response'); - const frame = Buffer.alloc(8 + body.length); - frame.writeUInt32LE(body.length, 4); - body.copy(frame, 8); + const frame = replyFrame(Buffer.from('response')); const response = once(connection, 'response'); connection._onData(frame.subarray(0, 6)); connection._onData(frame.subarray(6)); assert.deepEqual((await response)[0], frame); - const malformed = Buffer.alloc(8); - malformed.writeUInt32LE(256, 4); + const malformed = replyFrame(Buffer.alloc(0)); + malformed.writeUInt32LE(FRAME_LIMIT + 1, REPLY_OFFSET.size); const error = once(connection, 'error'); connection._onData(malformed); assert.ok((await error)[0] instanceof ProtocolFrameError); @@ -225,10 +231,7 @@ describe('IggyConnection', () => { assert.equal(connection.connected, true); assert.equal(connections, 2); - const body = Buffer.from('fresh'); - const frame = Buffer.alloc(8 + body.length); - frame.writeUInt32LE(body.length, 4); - body.copy(frame, 8); + const frame = replyFrame(Buffer.from('fresh')); const response = once(connection, 'response'); oldSocket.emit('data', Buffer.alloc(8)); connection._onData(frame); diff --git a/foreign/node/src/client/client.connection.ts b/foreign/node/src/client/client.connection.ts index 25e4274b20..99bcda97c3 100644 --- a/foreign/node/src/client/client.connection.ts +++ b/foreign/node/src/client/client.connection.ts @@ -21,7 +21,6 @@ import type { Socket } from 'node:net'; import { createConnection } from 'node:net'; import { connect as TLSConnect } from 'node:tls'; import type { ClientConfig, TlsOption, TcpOption, ReconnectOption } from "./client.type.js" -import { serializeCommand } from './client.utils.js'; import { debug } from './client.debug.js'; import { DEFAULT_MAX_RESPONSE_FRAME_SIZE } from './client.config.js'; import { @@ -141,7 +140,6 @@ export class IggyConnection extends EventEmitter { this.connectPromise = undefined; this.reconnectPromise = undefined; this.responseDecoder = new ResponseFrameDecoder( - config.protocol ?? 'classic', config.maxResponseFrameSize ?? DEFAULT_MAX_RESPONSE_FRAME_SIZE ); this.socket = this._installSocket(getTransport(config)); @@ -427,8 +425,7 @@ export class IggyConnection extends EventEmitter { try { for (const response of this.responseDecoder.push(data)) { - if (this.config.protocol === 'vsr' && - peekCommand(response) === Command2.Eviction) + if (peekCommand(response) === Command2.Eviction) this.emit('eviction', evictionError(response)); else this.emit('response', response); @@ -443,18 +440,6 @@ export class IggyConnection extends EventEmitter { } } - /** - * Writes a command to the socket. - * - * @param command - Command code - * @param payload - Command payload - * @returns True if the write was successful - */ - writeCommand(command: number, payload: Buffer): void { - const cmd = serializeCommand(command, payload); - this.socket.write(cmd); - } - writeFrame(frame: Buffer): void { this.socket.write(frame); } diff --git a/foreign/node/src/client/client.frame.test.ts b/foreign/node/src/client/client.frame.test.ts index 5b05481d8c..bdeba1eda1 100644 --- a/foreign/node/src/client/client.frame.test.ts +++ b/foreign/node/src/client/client.frame.test.ts @@ -30,13 +30,6 @@ import { const LIMIT = 1024; -const classicFrame = (body: Buffer): Buffer => { - const frame = Buffer.alloc(8 + body.length); - frame.writeUInt32LE(body.length, 4); - body.copy(frame, 8); - return frame; -}; - const vsrFrame = (body: Buffer): Buffer => { const frame = Buffer.alloc(HEADER_SIZE + body.length); frame.writeUInt32LE(frame.length, REPLY_OFFSET.size); @@ -46,83 +39,59 @@ const vsrFrame = (body: Buffer): Buffer => { }; describe('extractResponseFrames', () => { - for (const protocol of ['classic', 'vsr'] as const) { - const makeFrame = protocol === 'classic' ? classicFrame : vsrFrame; - - it(`buffers ${protocol} headers split at every boundary`, () => { - const frame = makeFrame(Buffer.from('payload')); - const headerSize = protocol === 'classic' ? 8 : HEADER_SIZE; - - for (let split = 0; split < headerSize; split += 1) { - const first = extractResponseFrames( - protocol, - frame.subarray(0, split), - LIMIT - ); - assert.equal(first.frames.length, 0); - const second = extractResponseFrames( - protocol, - Buffer.concat([first.remainder, frame.subarray(split)]), - LIMIT - ); - assert.deepEqual(second.frames, [frame]); - assert.equal(second.remainder.length, 0); - } - }); + it('buffers headers split at every boundary', () => { + const frame = vsrFrame(Buffer.from('payload')); - it(`buffers a fragmented ${protocol} body`, () => { - const frame = makeFrame(Buffer.from('payload')); - const split = frame.length - 2; - const first = extractResponseFrames( - protocol, - frame.subarray(0, split), - LIMIT - ); + for (let split = 0; split < HEADER_SIZE; split += 1) { + const first = extractResponseFrames(frame.subarray(0, split), LIMIT); assert.equal(first.frames.length, 0); const second = extractResponseFrames( - protocol, Buffer.concat([first.remainder, frame.subarray(split)]), LIMIT ); assert.deepEqual(second.frames, [frame]); - }); + assert.equal(second.remainder.length, 0); + } + }); - it(`extracts coalesced ${protocol} frames and a partial tail`, () => { - const first = makeFrame(Buffer.from('one')); - const second = makeFrame(Buffer.from('two')); - const third = makeFrame(Buffer.from('three')); - const input = Buffer.concat([first, second, third.subarray(0, 3)]); - const extracted = extractResponseFrames(protocol, input, LIMIT); + it('buffers a fragmented body', () => { + const frame = vsrFrame(Buffer.from('payload')); + const split = frame.length - 2; + const first = extractResponseFrames(frame.subarray(0, split), LIMIT); + assert.equal(first.frames.length, 0); + const second = extractResponseFrames( + Buffer.concat([first.remainder, frame.subarray(split)]), + LIMIT + ); + assert.deepEqual(second.frames, [frame]); + }); - assert.deepEqual(extracted.frames, [first, second]); - assert.deepEqual(extracted.remainder, third.subarray(0, 3)); - assert.equal(extracted.remainder.buffer, input.buffer); - }); - } + it('extracts coalesced frames and a partial tail', () => { + const first = vsrFrame(Buffer.from('one')); + const second = vsrFrame(Buffer.from('two')); + const third = vsrFrame(Buffer.from('three')); + const input = Buffer.concat([first, second, third.subarray(0, 3)]); + const extracted = extractResponseFrames(input, LIMIT); - it('rejects a VSR size below the header', () => { + assert.deepEqual(extracted.frames, [first, second]); + assert.deepEqual(extracted.remainder, third.subarray(0, 3)); + assert.equal(extracted.remainder.buffer, input.buffer); + }); + + it('rejects a size below the header', () => { const frame = vsrFrame(Buffer.alloc(0)); frame.writeUInt32LE(0, REPLY_OFFSET.size); assert.throws( - () => extractResponseFrames('vsr', frame, LIMIT), + () => extractResponseFrames(frame, LIMIT), ProtocolFrameError ); }); - it('rejects an oversized VSR frame before buffering its body', () => { + it('rejects an oversized frame before buffering its body', () => { const header = vsrFrame(Buffer.alloc(0)); header.writeUInt32LE(LIMIT + 1, REPLY_OFFSET.size); assert.throws( - () => extractResponseFrames('vsr', header, LIMIT), - ProtocolFrameError - ); - }); - - it('rejects an oversized classic frame before buffering its body', () => { - const header = classicFrame(Buffer.alloc(0)); - header.writeUInt32LE(LIMIT, 4); - assert.throws( - () => extractResponseFrames('classic', header, LIMIT), + () => extractResponseFrames(header, LIMIT), ProtocolFrameError ); }); @@ -130,9 +99,9 @@ describe('extractResponseFrames', () => { describe('ResponseFrameDecoder', () => { it('decodes bytewise input without losing coalesced frames', () => { - const decoder = new ResponseFrameDecoder('classic', LIMIT); - const first = classicFrame(Buffer.from('first')); - const second = classicFrame(Buffer.from('second')); + const decoder = new ResponseFrameDecoder(LIMIT); + const first = vsrFrame(Buffer.from('first')); + const second = vsrFrame(Buffer.from('second')); const input = Buffer.concat([first, second]); const frames: Buffer[] = []; @@ -144,17 +113,17 @@ describe('ResponseFrameDecoder', () => { }); it('clears a partial frame', () => { - const decoder = new ResponseFrameDecoder('vsr', LIMIT); - decoder.push(vsrFrame(Buffer.from('body')).subarray(0, 100)); + const decoder = new ResponseFrameDecoder(LIMIT); + decoder.push(vsrFrame(Buffer.from('body')).subarray(0, HEADER_SIZE - 2)); assert.equal(decoder.hasBufferedData, true); decoder.clear(); assert.equal(decoder.hasBufferedData, false); }); it('rejects an oversized frame as soon as its header is complete', () => { - const decoder = new ResponseFrameDecoder('classic', LIMIT); - const header = Buffer.alloc(8); - header.writeUInt32LE(LIMIT, 4); + const decoder = new ResponseFrameDecoder(LIMIT); + const header = vsrFrame(Buffer.alloc(0)); + header.writeUInt32LE(LIMIT + 1, REPLY_OFFSET.size); assert.throws(() => decoder.push(header), ProtocolFrameError); }); }); diff --git a/foreign/node/src/client/client.frame.ts b/foreign/node/src/client/client.frame.ts index 4bd90ff5b6..38cc0566a1 100644 --- a/foreign/node/src/client/client.frame.ts +++ b/foreign/node/src/client/client.frame.ts @@ -15,14 +15,11 @@ // specific language governing permissions and limitations // under the License. -import type { Protocol } from './client.type.js'; import { - HEADER_SIZE as VSR_HEADER_SIZE, - readSize as readVsrSize + HEADER_SIZE, + readSize } from '../wire/vsr/header.js'; -const CLASSIC_HEADER_SIZE = 8; - export class ProtocolFrameError extends Error { constructor(message: string) { super(message); @@ -35,45 +32,35 @@ export type ExtractedFrames = { remainder: Buffer }; -const headerSizeFor = (protocol: Protocol): number => - protocol === 'vsr' ? VSR_HEADER_SIZE : CLASSIC_HEADER_SIZE; - const declaredFrameSize = ( - protocol: Protocol, header: Buffer, maximumFrameSize: number ): number => { - const headerSize = headerSizeFor(protocol); - const declaredSize = protocol === 'vsr' - ? readVsrSize(header) - : CLASSIC_HEADER_SIZE + header.readUInt32LE(4); + const declaredSize = readSize(header); - if (declaredSize < headerSize) + if (declaredSize < HEADER_SIZE) throw new ProtocolFrameError( - `declared ${protocol} frame size ${declaredSize} is below header size` + `declared frame size ${declaredSize} is below header size` ); if (declaredSize > maximumFrameSize) throw new ProtocolFrameError( - `declared ${protocol} frame size ${declaredSize} exceeds ` + + `declared frame size ${declaredSize} exceeds ` + `the ${maximumFrameSize} byte limit` ); return declaredSize; }; export const extractResponseFrames = ( - protocol: Protocol, buffer: Buffer, maximumFrameSize: number ): ExtractedFrames => { - const headerSize = headerSizeFor(protocol); const frames: Buffer[] = []; let offset = 0; - while (buffer.length - offset >= headerSize) { + while (buffer.length - offset >= HEADER_SIZE) { const available = buffer.length - offset; const declaredSize = declaredFrameSize( - protocol, - buffer.subarray(offset, offset + headerSize), + buffer.subarray(offset, offset + HEADER_SIZE), maximumFrameSize ); if (available < declaredSize) @@ -96,19 +83,15 @@ export const extractResponseFrames = ( * incomplete frame as new socket chunks arrive. */ export class ResponseFrameDecoder { - private readonly protocol: Protocol; private readonly maximumFrameSize: number; - private readonly headerSize: number; private chunks: Buffer[]; private chunkIndex: number; private chunkOffset: number; private bufferedLength: number; private expectedFrameSize?: number; - constructor(protocol: Protocol, maximumFrameSize: number) { - this.protocol = protocol; + constructor(maximumFrameSize: number) { this.maximumFrameSize = maximumFrameSize; - this.headerSize = headerSizeFor(protocol); this.chunks = []; this.chunkIndex = 0; this.chunkOffset = 0; @@ -137,11 +120,10 @@ export class ResponseFrameDecoder { const frames: Buffer[] = []; while (true) { if (this.expectedFrameSize === undefined) { - if (this.bufferedLength < this.headerSize) + if (this.bufferedLength < HEADER_SIZE) break; this.expectedFrameSize = declaredFrameSize( - this.protocol, - this.peek(this.headerSize), + this.peek(HEADER_SIZE), this.maximumFrameSize ); } diff --git a/foreign/node/src/client/client.socket.test.ts b/foreign/node/src/client/client.socket.test.ts index 76c8a783c8..33e841f9dc 100644 --- a/foreign/node/src/client/client.socket.test.ts +++ b/foreign/node/src/client/client.socket.test.ts @@ -203,7 +203,6 @@ const singleNodeHandler = (port: number): FrameHandler => }; const vsrConfig = (port: number): ClientConfig => ({ - protocol: 'vsr', transport: 'TCP', options: { host: '127.0.0.1', port }, credentials: { username: 'iggy', password: 'iggy' }, diff --git a/foreign/node/src/client/client.socket.ts b/foreign/node/src/client/client.socket.ts index a29e328ddc..9bbafa401c 100644 --- a/foreign/node/src/client/client.socket.ts +++ b/foreign/node/src/client/client.socket.ts @@ -20,10 +20,9 @@ import { EventEmitter } from 'node:events'; import type { ClientConfig, ClientCredentials, CommandResponse, - PasswordCredentials, Protocol, RawClient, SendCommandOptions, + PasswordCredentials, RawClient, SendCommandOptions, TokenCredentials } from '../client/client.type.js'; -import { handleResponse } from './client.utils.js'; import { ResponseError, responseError } from '../wire/error.utils.js'; import { debug } from './client.debug.js'; import { IggyConnection } from './client.connection.js'; @@ -86,8 +85,6 @@ export class VsrResponseTimeoutError extends Error { * Implements command queuing, authentication, and heartbeat functionality. */ export class CommandResponseStream extends EventEmitter { - /** Server wire protocol used by this connection */ - readonly protocol: Protocol; /** Client configuration */ private options: ClientConfig; /** Underlying connection to the server */ @@ -120,7 +117,6 @@ export class CommandResponseStream extends EventEmitter { super(); const normalizedConfig = normalizeClientConfig(options); this.options = normalizedConfig; - this.protocol = normalizedConfig.protocol; this.connection = new IggyConnection(normalizedConfig); this.busy = false; this.isAuthenticated = false; @@ -176,7 +172,7 @@ export class CommandResponseStream extends EventEmitter { if (!this.connection.connected) await this.connection.connect() - if (this.options.protocol === 'vsr' && isLoginCommand(command)) + if (isLoginCommand(command)) await this._ensureVsrLeader(); if (!this.isAuthenticated && !this.isUnloggedCommand(command)) @@ -247,8 +243,6 @@ export class CommandResponseStream extends EventEmitter { payload: Buffer, handleResp = true ): Promise { - if (this.options.protocol !== 'vsr') - return this._processClassic(command, payload, handleResp); if (isLoginCommand(command) && this.isAuthenticated) return this._processVsrLogin(command, payload, handleResp); return this._processVsr(command, payload, handleResp); @@ -263,22 +257,6 @@ export class CommandResponseStream extends EventEmitter { return this._processVsr(command, payload, handleResp); } - private async _processClassic( - command: number, - payload: Buffer, - handleResp: boolean - ): Promise { - const response = await this._exchange( - () => this.connection.writeCommand(command, payload) - ); - if (!handleResp) - return response as unknown as CommandResponse; - const parsed = handleResponse(response); - if (parsed.status !== 0) - throw responseError(command, parsed.status); - return parsed; - } - private async _processVsr( command: number, payload: Buffer, @@ -409,8 +387,7 @@ export class CommandResponseStream extends EventEmitter { private isUnloggedCommand(command: number): boolean { return UNLOGGED_COMMAND_CODE.includes(command) || - (this.options.protocol === 'vsr' && - command === COMMAND_CODE.GetClusterMetadata); + command === COMMAND_CODE.GetClusterMetadata; } private async _ensureVsrLeader(): Promise { diff --git a/foreign/node/src/client/client.type.ts b/foreign/node/src/client/client.type.ts index 080529891f..a7e63723a3 100644 --- a/foreign/node/src/client/client.type.ts +++ b/foreign/node/src/client/client.type.ts @@ -56,8 +56,6 @@ export type SendCommandOptions = { * Provides direct access to command sending and event handling. */ export type RawClient = { - /** Server wire protocol used by this connection */ - readonly protocol: Protocol, /** Sends a command to the server and returns the response */ sendCommand: ( code: number, @@ -114,9 +112,6 @@ export type ReconnectOption = { */ export type TransportOption = TcpOption | TlsOption; -/** Server wire protocol. */ -export type Protocol = 'classic' | 'vsr'; - /** * Token-based authentication credentials. */ @@ -155,8 +150,6 @@ export type PoolSizeOption = { * Complete client configuration for connecting to the Iggy server. */ export type ClientConfig = { - /** Server wire protocol (default: classic) */ - protocol?: Protocol, /** Transport protocol to use (TCP or TLS) */ transport: TransportType, /** Transport-specific connection options */ diff --git a/foreign/node/src/client/client.utils.test.ts b/foreign/node/src/client/client.utils.test.ts index 174f09cf5b..815058dd31 100644 --- a/foreign/node/src/client/client.utils.test.ts +++ b/foreign/node/src/client/client.utils.test.ts @@ -18,32 +18,19 @@ import { describe, it } from 'node:test'; import assert from 'node:assert/strict'; -import { handleResponse, deserializeVoidResponse } from './client.utils.js'; +import { deserializeVoidResponse } from './client.utils.js'; -const SUCCESS = 0; +describe('deserializeVoidResponse', () => { -describe('handleResponse', () => { + it('returns true only for a success status with an empty payload', () => { + const success = { status: 0, length: 0, data: Buffer.alloc(0) }; + assert.equal(deserializeVoidResponse(success), true); - it('bounds data to the length field, not the full buffer', () => { - // Server says: status=0, length=0, no payload. - // But the raw buffer has 4 trailing bytes (e.g. start of next response). - const buf = Buffer.alloc(12); - buf.writeUInt32LE(SUCCESS, 0); // status - buf.writeUInt32LE(0, 4); // length = 0 (void response) - buf.writeUInt32LE(42, 8); // trailing bytes — NOT part of this response + const failed = { status: 1, length: 0, data: Buffer.alloc(0) }; + assert.equal(deserializeVoidResponse(failed), false); - const r = handleResponse(buf); - assert.equal(r.data.length, 0); - }); - - it('deserializeVoidResponse returns true for a valid void response with trailing buffer bytes', () => { - const buf = Buffer.alloc(12); - buf.writeUInt32LE(SUCCESS, 0); - buf.writeUInt32LE(0, 4); // length = 0 - buf.writeUInt32LE(42, 8); // trailing bytes - - const r = handleResponse(buf); - assert.equal(deserializeVoidResponse(r), true); + const withData = { status: 0, length: 4, data: Buffer.alloc(4) }; + assert.equal(deserializeVoidResponse(withData), false); }); }); diff --git a/foreign/node/src/client/client.utils.ts b/foreign/node/src/client/client.utils.ts index bb9a922589..dfde49b820 100644 --- a/foreign/node/src/client/client.utils.ts +++ b/foreign/node/src/client/client.utils.ts @@ -16,45 +16,7 @@ // under the License. // -import { Transform, type TransformCallback } from 'node:stream'; import type { CommandResponse } from './client.type.js'; -import { translateCommandCode } from '../wire/command.code.js'; -import { debug } from './client.debug.js'; - - -/** - * Parses a raw response buffer into a structured CommandResponse. - * Extracts status code, length, and payload data from the buffer. - * - * @param r - Raw response buffer from the server - * @returns Parsed command response with status, length, and data - */ -export const handleResponse = (r: Buffer) => { - const status = r.readUint32LE(0); - const length = r.readUint32LE(4); - debug('<== handleResponse', { status, length }); - return { - status, length, data: r.subarray(8, 8 + length) - } -}; - -/** - * Creates a Transform stream that parses response buffers. - * Transforms raw server responses into just the data payload. - * - * @returns Transform stream for processing server responses - */ -export const handleResponseTransform = () => new Transform({ - transform(chunk: Buffer, encoding: BufferEncoding, cb: TransformCallback) { - try { - const r = handleResponse(chunk); - debug('response::', r) - return cb(null, r.data); - } catch (err: unknown) { - return cb(new Error('handleResponseTransform error', { cause: err }), null); - } - } -}); /** * Deserializes a void response from the server. @@ -65,33 +27,3 @@ export const handleResponseTransform = () => new Transform({ */ export const deserializeVoidResponse = (r: CommandResponse) => r.status === 0 && r.data.length === 0; - -/** Length of the command code in bytes */ -const COMMAND_LENGTH = 4; - -/** - * Serializes a command and its payload into a buffer for sending to the server. - * Creates the wire format: [payload_size (4 bytes)][command (4 bytes)][payload] - * - * @param command - Command code to send - * @param payload - Command payload buffer - * @returns Buffer ready to be sent to the server - */ -export const serializeCommand = (command: number, payload: Buffer) => { - const payloadSize = payload.length + COMMAND_LENGTH; - const data = Buffer.allocUnsafe(8 + payload.length); - - data.writeUint32LE(payloadSize, 0); - data.writeUint32LE(command, 4); - data.fill(payload, 8); - - debug( - '==> CMD', command, - translateCommandCode(command), - 'LENGTH', payloadSize - ); - - debug('FullMessage#Base64', data.toString('base64')); - - return data; -} diff --git a/foreign/node/src/e2e/tcp.cluster.e2e.ts b/foreign/node/src/e2e/tcp.cluster.e2e.ts index 8fae9e93f4..d3e5c85e41 100644 --- a/foreign/node/src/e2e/tcp.cluster.e2e.ts +++ b/foreign/node/src/e2e/tcp.cluster.e2e.ts @@ -20,13 +20,14 @@ import { after, describe, it } from "node:test"; import assert from "node:assert/strict"; import { getTestClient } from "./test-client.utils.js"; -// cluster mode still in dev atm -// response is mocked from /core/server/config.toml +// The suite runs against a single-node server (cluster mode off), which +// reports itself as the sole healthy leader with endpoints derived from +// its default config. const expectedMeta = { - name: "iggy-cluster", + name: "single-node", nodes: [ { - name: "iggy-node-1", + name: "iggy-node", ip: "127.0.0.1", endpoints: { tcp: 8090, @@ -37,18 +38,6 @@ const expectedMeta = { role: "Leader", status: "Healthy", }, - { - name: "iggy-node-2", - ip: "127.0.0.1", - endpoints: { - tcp: 8091, - quic: 8081, - http: 3001, - websocket: 8093, - }, - role: "Follower", - status: "Healthy", - }, ], }; diff --git a/foreign/node/src/e2e/tcp.consumer-group.e2e.ts b/foreign/node/src/e2e/tcp.consumer-group.e2e.ts index d0dd85497a..58234fbb10 100644 --- a/foreign/node/src/e2e/tcp.consumer-group.e2e.ts +++ b/foreign/node/src/e2e/tcp.consumer-group.e2e.ts @@ -26,10 +26,8 @@ import { getIggyAddress } from '../tcp.sm.utils.js'; describe('e2e -> consumer-group', async () => { const [host, port] = getIggyAddress(); - const vsr = process.env.IGGY_TEST_PROTOCOL === 'vsr'; const c = new SingleClient({ - protocol: vsr ? 'vsr' : 'classic', transport: 'TCP', options: { host, port }, credentials: { username: 'iggy', password: 'iggy' } @@ -95,9 +93,7 @@ describe('e2e -> consumer-group', async () => { streamId: streamName, topicId: topicName, messages: generateMessages(mn), - partition: vsr - ? Partitioning.PartitionId((i / mn) % 3) - : Partitioning.MessageKey(`key-${ i % 300 }`) + partition: Partitioning.PartitionId((i / mn) % 3) })); } payloadLength = ct; diff --git a/foreign/node/src/e2e/tcp.consumer-stream.e2e.ts b/foreign/node/src/e2e/tcp.consumer-stream.e2e.ts index e954e7e6a2..40d9afe93a 100644 --- a/foreign/node/src/e2e/tcp.consumer-stream.e2e.ts +++ b/foreign/node/src/e2e/tcp.consumer-stream.e2e.ts @@ -32,10 +32,8 @@ describe('e2e -> consumer-stream', async () => { const [host, port] = getIggyAddress(); const credentials = { username: 'iggy', password: 'iggy' }; - const vsr = process.env.IGGY_TEST_PROTOCOL === 'vsr'; const opt = { - protocol: (vsr ? 'vsr' : 'classic') as 'vsr' | 'classic', transport: 'TCP' as const, options: { host, port }, credentials @@ -62,9 +60,7 @@ describe('e2e -> consumer-stream', async () => { await sendSomeMessages(c.clientProvider)( streamName, topicName, - vsr - ? Partitioning.PartitionId((i / 100) % 3) - : Partitioning.MessageKey(`k-${i % 300}`) + Partitioning.PartitionId((i / 100) % 3) ); } }); diff --git a/foreign/node/src/e2e/tcp.send-message.e2e.ts b/foreign/node/src/e2e/tcp.send-message.e2e.ts index e085e16128..8b170c7bad 100644 --- a/foreign/node/src/e2e/tcp.send-message.e2e.ts +++ b/foreign/node/src/e2e/tcp.send-message.e2e.ts @@ -27,10 +27,6 @@ describe('e2e -> message', async () => { const c = getTestClient(); - // Only the VSR lane reaches a server that reports offsets. The classic lane - // runs against the legacy server, which commits without confirming. - const vsr = process.env.IGGY_TEST_PROTOCOL === 'vsr'; - const streamName = 'e2e-stream-934'; const topicName = 'e2e-topic-832'; const partitionId = 0; @@ -53,10 +49,6 @@ describe('e2e -> message', async () => { it('e2e -> message::send', async () => { const { confirmations } = await c.message.send(msg); - if (!vsr) { - assert.equal(confirmations.length, 0); - return; - } assert.equal(confirmations.length, 1); assert.equal(confirmations[0].partitionId, partitionId); }); @@ -177,10 +169,6 @@ describe('e2e -> message', async () => { ...msg, messages: generateMessages(3) }); - if (!vsr) { - assert.equal(confirmations.length, 0); - return; - } // Landing behind the already committed batch is the part no placeholder // confirmation could reproduce. assert.equal(confirmations.length, 1); diff --git a/foreign/node/src/e2e/test-client.utils.ts b/foreign/node/src/e2e/test-client.utils.ts index b5d2d77139..22090066ec 100644 --- a/foreign/node/src/e2e/test-client.utils.ts +++ b/foreign/node/src/e2e/test-client.utils.ts @@ -21,10 +21,8 @@ import { getIggyAddress } from '../tcp.sm.utils.js'; const credentials = { username: 'iggy', password: 'iggy' }; const [host, port] = getIggyAddress(); -const protocol = process.env.IGGY_TEST_PROTOCOL === 'vsr' ? 'vsr' : 'classic'; export const getTestClient = () => new Client({ - protocol, transport: 'TCP', options: { host, port }, credentials diff --git a/foreign/node/src/e2e/tls.system.e2e.ts b/foreign/node/src/e2e/tls.system.e2e.ts index c2f7b2d9bb..e1dde409e4 100644 --- a/foreign/node/src/e2e/tls.system.e2e.ts +++ b/foreign/node/src/e2e/tls.system.e2e.ts @@ -51,14 +51,10 @@ const caCertPath = process.env.E2E_ROOT_CA_CERT const getTlsClient = () => { const [, port] = getIggyAddress(); const caCert = readFileSync(caCertPath); - const protocol = process.env.IGGY_TEST_PROTOCOL === 'vsr' - ? 'vsr' - : 'classic'; // The server certificate SAN is DNS:localhost, so we connect via 'localhost' // for proper hostname verification (consistent with Python and C# TLS tests). return new Client({ - protocol, transport: 'TLS', options: { port, diff --git a/foreign/node/src/wire/command-set.test.ts b/foreign/node/src/wire/command-set.test.ts index 3c1cf6f755..dbc3a78c95 100644 --- a/foreign/node/src/wire/command-set.test.ts +++ b/foreign/node/src/wire/command-set.test.ts @@ -23,7 +23,6 @@ import type { RawClient } from '../client/client.type.js'; import { COMMAND_CODE } from './command.code.js'; const mockRawClient = (): RawClient => ({ - protocol: 'classic', sendCommand: async () => { throw new Error('sendCommand should not be called by the session-control guard'); }, diff --git a/foreign/node/src/wire/message/poll-messages.command.ts b/foreign/node/src/wire/message/poll-messages.command.ts index 335e3718e7..4023f387e6 100644 --- a/foreign/node/src/wire/message/poll-messages.command.ts +++ b/foreign/node/src/wire/message/poll-messages.command.ts @@ -247,8 +247,7 @@ export const pollMessages = (getClient: ClientProvider) => const client = await getClient(); const release = client.hold?.(); try { - if (client.protocol === 'vsr' && - request.consumer.kind === ConsumerKind.Group && + if (request.consumer.kind === ConsumerKind.Group && request.partitionId === null) { const state = getGroupState(client); while (true) { diff --git a/foreign/node/src/wire/vsr/header.test.ts b/foreign/node/src/wire/vsr/header.test.ts index 91b92def0f..a8095c7446 100644 --- a/foreign/node/src/wire/vsr/header.test.ts +++ b/foreign/node/src/wire/vsr/header.test.ts @@ -32,7 +32,6 @@ describe('VSR request header', () => { client, request: 0x0102030405060708n, operation: 2, - namespace: 0x8877665544332211n, session: 0x1020304050607080n, nonReplicatedCode: 60_001 }); @@ -53,10 +52,6 @@ describe('VSR request header', () => { 0x0102030405060708n ); assert.equal(header.readUInt8(REQUEST_OFFSET.operation), 2); - assert.equal( - header.readBigUInt64LE(REQUEST_OFFSET.namespace), - 0x8877665544332211n - ); assert.equal( header.readBigUInt64LE(REQUEST_OFFSET.session), 0x1020304050607080n @@ -71,7 +66,6 @@ describe('VSR request header', () => { client: 1n, request: 0n, operation: 1, - namespace: 1n << 63n, session: 0n }); const expected = Buffer.alloc(HEADER_SIZE); @@ -79,7 +73,6 @@ describe('VSR request header', () => { expected.writeUInt8(Command2.Request, REQUEST_OFFSET.command); expected.writeBigUInt64LE(1n, REQUEST_OFFSET.client); expected.writeUInt8(1, REQUEST_OFFSET.operation); - expected.writeBigUInt64LE(1n << 63n, REQUEST_OFFSET.namespace); assert.deepEqual(header, expected); }); @@ -90,7 +83,6 @@ describe('VSR request header', () => { client: maximum << 64n | maximum, request: maximum, operation: 160, - namespace: maximum, session: maximum }); assert.equal(header.readBigUInt64LE(REQUEST_OFFSET.request), maximum); diff --git a/foreign/node/src/wire/vsr/header.ts b/foreign/node/src/wire/vsr/header.ts index bdee4f2706..9c51217dd1 100644 --- a/foreign/node/src/wire/vsr/header.ts +++ b/foreign/node/src/wire/vsr/header.ts @@ -27,7 +27,15 @@ /** Size of every consensus header, both directions. */ export const HEADER_SIZE = 256; -/** `RequestHeader` field offsets the client writes. */ +/** + * `RequestHeader` field offsets the client writes. + * + * The client wire carries no routing namespace: the server derives the + * consensus group (plane from `operation`, partition target from the payload) + * and stamps it into its own internal header. Everything that followed the + * removed field therefore sits eight bytes earlier than in the pre-derivation + * layout. + */ export const REQUEST_OFFSET = { size: 48, command: 60, @@ -35,9 +43,8 @@ export const REQUEST_OFFSET = { timestamp: 160, request: 168, operation: 176, - namespace: 184, - session: 192, - reserved: 204 + session: 184, + reserved: 196 } as const; /** `ReplyHeader` field offsets the client reads. */ @@ -45,8 +52,7 @@ export const REPLY_OFFSET = { size: 48, command: 60, operation: 208, - namespace: 216, - status: 224 + status: 216 } as const; /** `EvictionHeader` field offsets the client reads. */ @@ -98,8 +104,6 @@ export type RequestHeaderFields = { request: bigint, /** `Operation` discriminant. */ operation: number, - /** Routing namespace (u64). */ - namespace: bigint, /** Bound session (u64), or 0n. */ session: bigint, /** Command code for `NonReplicated`, placed in `reserved[0..4]`. */ @@ -109,7 +113,7 @@ export type RequestHeaderFields = { const U64_MASK = 0xFFFFFFFFFFFFFFFFn; /** - * Encodes a 256-byte request header. Only the seven fields the server reads + * Encodes a 256-byte request header. Only the six fields the server reads * are written; the checksums stay zero, matching the Rust SDK's contract * with the VSR server. */ @@ -122,7 +126,6 @@ export const encodeRequestHeader = (fields: RequestHeaderFields): Buffer => { header.writeBigUInt64LE(fields.client >> 64n, REQUEST_OFFSET.client + 8); header.writeBigUInt64LE(fields.request, REQUEST_OFFSET.request); header.writeUInt8(fields.operation, REQUEST_OFFSET.operation); - header.writeBigUInt64LE(fields.namespace, REQUEST_OFFSET.namespace); header.writeBigUInt64LE(fields.session, REQUEST_OFFSET.session); if (fields.nonReplicatedCode !== undefined) header.writeUInt32LE(fields.nonReplicatedCode, REQUEST_OFFSET.reserved); diff --git a/foreign/node/src/wire/vsr/index.ts b/foreign/node/src/wire/vsr/index.ts index 5c079ba4f6..ca88e7945b 100644 --- a/foreign/node/src/wire/vsr/index.ts +++ b/foreign/node/src/wire/vsr/index.ts @@ -21,7 +21,6 @@ import type { CommandResponse } from '../../client/client.type.js'; import { COMMAND_CODE } from '../command.code.js'; import { responseError } from '../error.utils.js'; import { HEADER_SIZE, encodeRequestHeader } from './header.js'; -import { namespaceForRequest } from './namespace.js'; import { Operation, isPartition, @@ -66,7 +65,6 @@ export class VsrSession { const operation = registerCommand(command) ? Operation.Register : operationForCode(command); - const namespace = namespaceForRequest(command, payload, operation); const size = HEADER_SIZE + payload.length; if (size > MAX_U32) throw new RangeError('VSR request exceeds the u32 frame-size limit'); @@ -94,7 +92,6 @@ export class VsrSession { client: this.state.clientId, request, operation, - namespace, session, nonReplicatedCode: operation === Operation.NonReplicated ? command : undefined, diff --git a/foreign/node/src/wire/vsr/namespace.test.ts b/foreign/node/src/wire/vsr/namespace.test.ts deleted file mode 100644 index 244ca39325..0000000000 --- a/foreign/node/src/wire/vsr/namespace.test.ts +++ /dev/null @@ -1,342 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import assert from 'node:assert/strict'; -import { describe, it } from 'node:test'; -import { serializeIdentifier } from '../identifier.utils.js'; -import { serializeSendMessages } from '../message/message.utils.js'; -import { Partitioning } from '../message/partitioning.utils.js'; -import { Consumer, serializeStoreOffset } from '../offset/offset.utils.js'; -import { COMMAND_CODE } from '../command.code.js'; -import { ResponseError } from '../error.utils.js'; -import { - METADATA_CONSENSUS_NAMESPACE, - namespaceForRequest, - packNamespace -} from './namespace.js'; -import { Operation } from './operation.js'; - -describe('VSR namespace routing', () => { - it('routes register and logout to metadata consensus', () => { - for (const operation of [Operation.Register, Operation.Logout]) - assert.equal( - namespaceForRequest(0, Buffer.alloc(0), operation), - METADATA_CONSENSUS_NAMESPACE - ); - }); - - it('routes metadata and non-replicated requests to zero', () => { - assert.equal( - namespaceForRequest( - COMMAND_CODE.CreateStream, - Buffer.alloc(0), - Operation.CreateStream - ), - 0n - ); - assert.equal( - namespaceForRequest( - COMMAND_CODE.GetStats, - Buffer.alloc(0), - Operation.NonReplicated - ), - 0n - ); - }); - - it('packs explicit send-message partition identifiers', () => { - const payload = serializeSendMessages( - 7, - 11, - [], - Partitioning.PartitionId(13) - ); - assert.equal( - namespaceForRequest( - COMMAND_CODE.SendMessages, - payload, - Operation.SendMessages - ), - packNamespace(7, 11, 13) - ); - }); - - it('defers named stream and topic routing to the server', () => { - const payload = serializeSendMessages( - 'stream', - 'topic', - [], - Partitioning.PartitionId(1) - ); - assert.equal( - namespaceForRequest( - COMMAND_CODE.SendMessages, - payload, - Operation.SendMessages - ), - 0n - ); - }); - - it('rejects server-selected message partitioning', () => { - const payload = serializeSendMessages( - 1, - 2, - [], - Partitioning.Balanced - ); - assert.throws( - () => namespaceForRequest( - COMMAND_CODE.SendMessages, - payload, - Operation.SendMessages - ), - (error: unknown) => - error instanceof ResponseError && error.errorCode === 5 - ); - }); - - it('routes consumer offsets from their explicit partition', () => { - const payload = serializeStoreOffset( - 1, - 2, - Consumer.Single, - 3, - 99n - ); - assert.equal( - namespaceForRequest( - COMMAND_CODE.StoreOffset, - payload, - Operation.StoreConsumerOffset - ), - packNamespace(1, 2, 3) - ); - }); - - it('routes delete-segments payloads', () => { - const payload = Buffer.concat([ - serializeIdentifier(1), - serializeIdentifier(2), - Buffer.from([3, 0, 0, 0]) - ]); - assert.equal( - namespaceForRequest( - COMMAND_CODE.DeleteSegments, - payload, - Operation.DeleteSegments - ), - packNamespace(1, 2, 3) - ); - }); - - it('accepts the maximum packable identifiers', () => { - assert.equal( - packNamespace(4095, 4095, 999_999), - (4095n << 32n) | (4095n << 20n) | 999_999n - ); - }); - - it('rejects peeks past the declared send-messages metadata region', () => { - const payload = serializeSendMessages( - 1, - 2, - [], - Partitioning.PartitionId(3) - ); - const underDeclared = Buffer.from(payload); - // Shrink the declared metadata region so the partitioning bytes sit - // outside it; the peek must fail instead of reading them. - underDeclared.writeUInt32LE(payload.readUInt32LE(0) - 6, 0); - assert.throws( - () => namespaceForRequest( - COMMAND_CODE.SendMessages, - underDeclared, - Operation.SendMessages - ), - (error: unknown) => - error instanceof ResponseError && error.errorCode === 3 - ); - }); - - it('rejects unknown codes in partition routing', () => { - assert.throws( - () => namespaceForRequest( - 60_001, - Buffer.alloc(0), - Operation.SendMessages - ), - (error: unknown) => - error instanceof ResponseError && error.errorCode === 5 - ); - }); - - it('requires an explicit consumer-offset partition', () => { - const payload = serializeStoreOffset( - 1, - 2, - Consumer.Single, - 3, - 99n - ); - // [kind u8][consumer 6][stream 6][topic 6] puts the partition flag at 19. - const withoutPartition = Buffer.from(payload); - withoutPartition.writeUInt8(0, 19); - assert.throws( - () => namespaceForRequest( - COMMAND_CODE.StoreOffset, - withoutPartition, - Operation.StoreConsumerOffset - ), - (error: unknown) => - error instanceof ResponseError && error.errorCode === 6 - ); - }); - - it('rejects namespace fields before masking', () => { - for (const [streamId, topicId, partitionId] of [ - [4096, 0, 0], - [0, 4096, 0], - [0, 0, 1_000_000] - ] as const) - assert.throws( - () => packNamespace(streamId, topicId, partitionId), - (error: unknown) => - error instanceof ResponseError && error.errorCode === 6 - ); - }); - - it('rejects negative and non-integer namespace fields', () => { - for (const [streamId, topicId, partitionId] of [ - [-1, 0, 0], - [0, -1, 0], - [0, 0, -1], - [0.5, 0, 0], - [0, Number.NaN, 0] - ] as const) - assert.throws( - () => packNamespace(streamId, topicId, partitionId), - (error: unknown) => - error instanceof ResponseError && error.errorCode === 6 - ); - }); - - it('rejects malformed identifiers at every prefix boundary', () => { - const payload = serializeSendMessages( - 1, - 2, - [], - Partitioning.PartitionId(3) - ); - for (let length = 0; length < payload.length; length += 1) - assert.throws( - () => namespaceForRequest( - COMMAND_CODE.SendMessages, - payload.subarray(0, length), - Operation.SendMessages - ), - ResponseError - ); - - const invalidKind = Buffer.from(payload); - invalidKind.writeUInt8(99, 4); - assert.throws( - () => namespaceForRequest( - COMMAND_CODE.SendMessages, - invalidKind, - Operation.SendMessages - ), - ResponseError - ); - }); - - it('rejects malformed consumer-offset and delete-segment payloads', () => { - const offsetPayload = serializeStoreOffset( - 1, - 2, - Consumer.Single, - 3, - 99n - ); - const invalidConsumerKind = Buffer.from(offsetPayload); - invalidConsumerKind.writeUInt8(0, 0); - assert.throws( - () => namespaceForRequest( - COMMAND_CODE.StoreOffset, - invalidConsumerKind, - Operation.StoreConsumerOffset - ), - ResponseError - ); - for (let length = 1; length < 20; length += 1) - assert.throws( - () => namespaceForRequest( - COMMAND_CODE.StoreOffset, - offsetPayload.subarray(0, length), - Operation.StoreConsumerOffset - ), - ResponseError - ); - - const deletePayload = Buffer.concat([ - serializeIdentifier(1), - serializeIdentifier(2), - Buffer.from([3, 0, 0, 0]) - ]); - for (let length = 0; length < deletePayload.length; length += 1) - assert.throws( - () => namespaceForRequest( - COMMAND_CODE.DeleteSegments, - deletePayload.subarray(0, length), - Operation.DeleteSegments - ), - ResponseError - ); - }); - - it('defers named offset and delete-segment routing to the server', () => { - const offsetPayload = serializeStoreOffset( - 'stream', - 'topic', - Consumer.Single, - 3, - 99n - ); - assert.equal( - namespaceForRequest( - COMMAND_CODE.StoreOffset, - offsetPayload, - Operation.StoreConsumerOffset - ), - 0n - ); - - const deletePayload = Buffer.concat([ - serializeIdentifier('stream'), - serializeIdentifier('topic'), - Buffer.from([3, 0, 0, 0]) - ]); - assert.equal( - namespaceForRequest( - COMMAND_CODE.DeleteSegments, - deletePayload, - Operation.DeleteSegments - ), - 0n - ); - }); -}); diff --git a/foreign/node/src/wire/vsr/namespace.ts b/foreign/node/src/wire/vsr/namespace.ts deleted file mode 100644 index 1a6ec20e9d..0000000000 --- a/foreign/node/src/wire/vsr/namespace.ts +++ /dev/null @@ -1,216 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -// - -/** - * Namespace packing and the partition-plane payload peeks needed to derive - * it, ported from `core/binary_protocol/src/namespace.rs` and - * `namespace_for_request` in `core/sdk/src/vsr.rs`. - */ - -import { COMMAND_CODE } from '../command.code.js'; -import { responseError } from '../error.utils.js'; -import { Operation, isMetadata } from './operation.js'; - -/** `IggyError::InvalidCommand`. */ -const INVALID_COMMAND = 3; -/** `IggyError::FeatureUnavailable`. */ -const FEATURE_UNAVAILABLE = 5; -/** `IggyError::InvalidIdentifier`. */ -const INVALID_IDENTIFIER = 6; - -const MAX_STREAMS = 4096; -const MAX_TOPICS = 4096; -const MAX_PARTITIONS = 1_000_000; -const bitsRequired = (value: number): bigint => - BigInt(BigInt(value).toString(2).length); -const TOPIC_SHIFT = bitsRequired(MAX_PARTITIONS - 1); -const STREAM_SHIFT = TOPIC_SHIFT + bitsRequired(MAX_TOPICS - 1); - -/** - * Control-plane requests target the metadata replica (shard 0), selected by - * this exact sentinel. Plain 0 would fall into namespace hashing and land a - * Register on a peer shard. - */ -export const METADATA_CONSENSUS_NAMESPACE = 1n << 63n; - -/** Packs stream / topic / partition ids into a routing namespace. */ -export const packNamespace = ( - streamId: number, - topicId: number, - partitionId: number -): bigint => { - validateField(streamId, MAX_STREAMS); - validateField(topicId, MAX_TOPICS); - validateField(partitionId, MAX_PARTITIONS); - return (BigInt(streamId) << STREAM_SHIFT) | - (BigInt(topicId) << TOPIC_SHIFT) | - BigInt(partitionId); -}; - -/** - * Selects the routing namespace for a request. Partition-plane commands - * derive it from their own payload; a named stream or topic identifier - * yields 0 so the server resolves the name. - * - * @throws Error mirroring the Rust SDK: invalid-identifier for an - * out-of-range field, invalid-command for an undecodable payload, and - * feature-unavailable for a partition operation this SDK cannot derive. - */ -export const namespaceForRequest = ( - code: number, - payload: Buffer, - operation: number -): bigint => { - if (operation === Operation.Register || operation === Operation.Logout) - return METADATA_CONSENSUS_NAMESPACE; - if (operation === Operation.NonReplicated || isMetadata(operation)) - return 0n; - - switch (code) { - case COMMAND_CODE.SendMessages: - return namespaceFromSendMessages(payload); - case COMMAND_CODE.StoreOffset: - case COMMAND_CODE.DeleteConsumerOffset: - case COMMAND_CODE.StoreOffset2: - case COMMAND_CODE.DeleteConsumerOffset2: - return namespaceFromConsumerOffset(payload); - case COMMAND_CODE.DeleteSegments: - return namespaceFromDeleteSegments(payload); - default: - // The guard that keeps custom partition operations unreachable. - throw responseError(code, FEATURE_UNAVAILABLE); - } -}; - -/** A decoded identifier: numeric value, or null for a name (server resolves). */ -type PeekedIdentifier = { - numeric: number | null, - length: number -}; - -const IDENTIFIER_KIND_NUMERIC = 1; -const IDENTIFIER_KIND_STRING = 2; - -const peekIdentifier = (payload: Buffer, offset: number): PeekedIdentifier => { - if (payload.length < offset + 2) - throw responseError(0, INVALID_COMMAND); - const kind = payload.readUInt8(offset); - const length = payload.readUInt8(offset + 1); - if (payload.length < offset + 2 + length) - throw responseError(0, INVALID_COMMAND); - if (kind === IDENTIFIER_KIND_NUMERIC) { - if (length !== 4) - throw responseError(0, INVALID_COMMAND); - return { numeric: payload.readUInt32LE(offset + 2), length: 2 + length }; - } - if (kind === IDENTIFIER_KIND_STRING && length > 0) - return { numeric: null, length: 2 + length }; - throw responseError(0, INVALID_COMMAND); -}; - -const validateField = (value: number, exclusiveMax: number): void => { - if (!Number.isInteger(value) || value < 0 || value >= exclusiveMax) - throw responseError(0, INVALID_IDENTIFIER); -}; - -const namespaceFromIds = ( - stream: PeekedIdentifier, - topic: PeekedIdentifier, - partitionId: number -): bigint => { - // Named identifiers defer resolution to the server. - if (stream.numeric === null || topic.numeric === null) return 0n; - return packNamespace(stream.numeric, topic.numeric, partitionId); -}; - -/** - * `SendMessages`: `[metadata_len u32][stream ident][topic ident] - * [partitioning kind u8, len u8, value]...`. Only explicit `PartitionId` - * partitioning is routable under VSR; the broker never picks a partition. - */ -const namespaceFromSendMessages = (payload: Buffer): bigint => { - if (payload.length < 4) - throw responseError(COMMAND_CODE.SendMessages, INVALID_COMMAND); - const metadataLength = payload.readUInt32LE(0); - if (payload.length < 4 + metadataLength) - throw responseError(COMMAND_CODE.SendMessages, INVALID_COMMAND); - // Rust peeks inside payload[4..4 + metadata_length]; a read past the - // declared metadata region must fail rather than spill into message bytes - // and derive a namespace the server would never compute. - const metadata = payload.subarray(4, 4 + metadataLength); - - let offset = 0; - const stream = peekIdentifier(metadata, offset); - offset += stream.length; - const topic = peekIdentifier(metadata, offset); - offset += topic.length; - - if (metadata.length < offset + 2) - throw responseError(COMMAND_CODE.SendMessages, INVALID_COMMAND); - const partitioningKind = metadata.readUInt8(offset); - const partitioningLength = metadata.readUInt8(offset + 1); - const PARTITIONING_PARTITION_ID = 2; - if (partitioningKind !== PARTITIONING_PARTITION_ID) - throw responseError(COMMAND_CODE.SendMessages, FEATURE_UNAVAILABLE); - if (partitioningLength !== 4 || metadata.length < offset + 2 + 4) - throw responseError(COMMAND_CODE.SendMessages, INVALID_COMMAND); - const partitionId = metadata.readUInt32LE(offset + 2); - - return namespaceFromIds(stream, topic, partitionId); -}; - -/** - * Consumer-offset requests: `[consumer kind u8][consumer ident] - * [stream ident][topic ident][partition flag u8][partition u32]...`. - */ -const namespaceFromConsumerOffset = (payload: Buffer): bigint => { - if (payload.length < 1 || (payload.readUInt8(0) !== 1 && - payload.readUInt8(0) !== 2)) - throw responseError(COMMAND_CODE.StoreOffset, INVALID_COMMAND); - let offset = 1; - const consumer = peekIdentifier(payload, offset); - offset += consumer.length; - const stream = peekIdentifier(payload, offset); - offset += stream.length; - const topic = peekIdentifier(payload, offset); - offset += topic.length; - - if (payload.length < offset + 5) - throw responseError(COMMAND_CODE.StoreOffset, INVALID_COMMAND); - const hasPartition = payload.readUInt8(offset) === 1; - if (!hasPartition) - throw responseError(COMMAND_CODE.StoreOffset, INVALID_IDENTIFIER); - const partitionId = payload.readUInt32LE(offset + 1); - - return namespaceFromIds(stream, topic, partitionId); -}; - -/** `DeleteSegments`: `[stream ident][topic ident][partition u32]...`. */ -const namespaceFromDeleteSegments = (payload: Buffer): bigint => { - let offset = 0; - const stream = peekIdentifier(payload, offset); - offset += stream.length; - const topic = peekIdentifier(payload, offset); - offset += topic.length; - - if (payload.length < offset + 4) - throw responseError(COMMAND_CODE.DeleteSegments, INVALID_COMMAND); - const partitionId = payload.readUInt32LE(offset); - - return namespaceFromIds(stream, topic, partitionId); -}; diff --git a/foreign/node/src/wire/vsr/vsr.test.ts b/foreign/node/src/wire/vsr/vsr.test.ts index 4e6cec6a2c..9149938231 100644 --- a/foreign/node/src/wire/vsr/vsr.test.ts +++ b/foreign/node/src/wire/vsr/vsr.test.ts @@ -37,23 +37,6 @@ describe('VSR custom request framing', () => { assert.deepEqual(frame.subarray(256), payload); }); - it('does not consume a request ID when local routing fails', () => { - const session = new VsrSession(7n); - session.bind(42n); - assert.throws( - () => session.encode( - COMMAND_CODE.SendMessages, - Buffer.alloc(0) - ) - ); - - const frame = session.encode( - COMMAND_CODE.CreateStream, - Buffer.alloc(0) - ); - assert.equal(frame.readBigUInt64LE(REQUEST_OFFSET.request), 1n); - }); - it('rejects an unbound replicated request with a typed error', () => { const session = new VsrSession(); assert.throws( diff --git a/foreign/php/README.md b/foreign/php/README.md index cdb5aab0d2..73a31cfa42 100644 --- a/foreign/php/README.md +++ b/foreign/php/README.md @@ -70,7 +70,7 @@ docker run --rm --name iggy-php-test \ You can also run a local server from the repository root: ```sh -cargo run --bin iggy-server --fresh --with-default-root-credentials +cargo run --bin iggy-server -- --fresh --with-default-root-credentials ``` The tests assume: diff --git a/foreign/php/docker-compose.test.yml b/foreign/php/docker-compose.test.yml index 492779feac..3831e39bde 100644 --- a/foreign/php/docker-compose.test.yml +++ b/foreign/php/docker-compose.test.yml @@ -16,13 +16,15 @@ # under the License. services: + # The PHP extension frames TCP with the VSR wire protocol only. iggy-server: build: context: ../.. dockerfile: core/server/Dockerfile args: PROFILE: debug - command: ["--fresh", "--with-default-root-credentials"] + # The server takes only --replica-id; root credentials come from the env. + command: [] container_name: iggy-server-php-test security_opt: - seccomp:unconfined @@ -31,6 +33,8 @@ services: - IGGY_TCP_ADDRESS=0.0.0.0:8090 - IGGY_QUIC_ADDRESS=0.0.0.0:8080 - IGGY_WEBSOCKET_ADDRESS=0.0.0.0:8092 + - IGGY_ROOT_USERNAME=iggy + - IGGY_ROOT_PASSWORD=iggy networks: - php-test-network ports: diff --git a/foreign/php/iggy-php.stubs.php b/foreign/php/iggy-php.stubs.php index b8cf52eaa7..caf4a7841c 100644 --- a/foreign/php/iggy-php.stubs.php +++ b/foreign/php/iggy-php.stubs.php @@ -218,9 +218,8 @@ public function pollMessages(mixed $stream, mixed $topic, int $partition_id, \Ig public function sendBinaryRequest(int $code, string $payload): string {} /** - * Sends messages to a topic and returns the commit confirmations. - * - * The list is empty against the legacy server, which reports no offsets. + * Sends messages to a topic and returns the commit confirmations, one per + * partition the batch landed in. * * @param mixed $stream * @param mixed $topic @@ -513,9 +512,6 @@ class SendMessagesConfirmation { * crash-restart can stamp a later batch with an offset a client has already * recorded. * - * The legacy server returns an empty confirmation list, so it reports no offset - * at all. - * * @var int */ public readonly int $base_offset; @@ -536,9 +532,9 @@ class SendMessagesResponse { /** * One confirmation per partition the batch landed in. * - * The list is empty when the server reports no offsets. The legacy server never - * reports any, and a server that does can still commit a batch it has no offsets - * to describe, so check for an empty array instead of indexing. + * The list is empty when the server reports no offsets. A server can commit a + * batch it has no offsets to describe, so check for an empty array instead of + * indexing. * * The confirmations are rebuilt on each getter call; cache the result in PHP if * they will be read repeatedly. diff --git a/foreign/php/scripts/test.sh b/foreign/php/scripts/test.sh index 6c964ace24..effe9e7a06 100755 --- a/foreign/php/scripts/test.sh +++ b/foreign/php/scripts/test.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/foreign/php/src/client.rs b/foreign/php/src/client.rs index 57e54d1512..118775a238 100644 --- a/foreign/php/src/client.rs +++ b/foreign/php/src/client.rs @@ -184,9 +184,8 @@ impl IggyClient { }) } - /// Sends messages to a topic and returns the commit confirmations. - /// - /// The list is empty against the legacy server, which reports no offsets. + /// Sends messages to a topic and returns the commit confirmations, one per + /// partition the batch landed in. pub fn send_messages( &self, stream: PhpIdentifier, diff --git a/foreign/php/src/send_message.rs b/foreign/php/src/send_message.rs index cabbf2b8f0..19b09a291e 100644 --- a/foreign/php/src/send_message.rs +++ b/foreign/php/src/send_message.rs @@ -99,9 +99,9 @@ impl From for SendMessagesResponse { impl SendMessagesResponse { /// One confirmation per partition the batch landed in. /// - /// The list is empty when the server reports no offsets. The legacy server never - /// reports any, and a server that does can still commit a batch it has no offsets - /// to describe, so check for an empty array instead of indexing. + /// The list is empty when the server reports no offsets. A server can commit a + /// batch it has no offsets to describe, so check for an empty array instead of + /// indexing. /// /// The confirmations are rebuilt on each getter call; cache the result in PHP if /// they will be read repeatedly. @@ -154,9 +154,6 @@ impl SendMessagesConfirmation { /// A batch is confirmed once it is committed in memory, not once it is fsynced. A /// crash-restart can stamp a later batch with an offset a client has already /// recorded. - /// - /// The legacy server returns an empty confirmation list, so it reports no offset - /// at all. #[php(getter)] pub fn base_offset(&self) -> u64 { self.inner.base_offset diff --git a/foreign/php/tests/IggySdkTest.php b/foreign/php/tests/IggySdkTest.php index 978a6f7fba..149106f1e5 100644 --- a/foreign/php/tests/IggySdkTest.php +++ b/foreign/php/tests/IggySdkTest.php @@ -222,8 +222,8 @@ public function testSendAndPollBinaryMessages(): void } } - #[TestDox('sendMessages against the legacy server reports no commit confirmations')] - public function testSendMessagesReportsNoConfirmationFromLegacyServer(): void + #[TestDox('sendMessages reports one commit confirmation per written partition')] + public function testSendMessagesReportsCommitConfirmation(): void { $client = new_client(); $streamName = unique_name('confirm-stream'); @@ -235,7 +235,17 @@ public function testSendMessagesReportsNoConfirmationFromLegacyServer(): void $response = $client->sendMessages($streamName, $topicName, $partitionId, [new SendMessage('confirm-first')]); assert_instance_of(SendMessagesResponse::class, $response); - assert_count(0, $response->confirmations, 'the legacy server answers a send with no offsets'); + assert_count(1, $response->confirmations, 'a single-partition send commits in exactly one partition'); + assert_same($partitionId, $response->confirmations[0]->partition_id); + + // Offsets are per partition and start at 0, so the second send of + // the same size must be confirmed one message later. + $second = $client->sendMessages($streamName, $topicName, $partitionId, [new SendMessage('confirm-second')]); + assert_same( + $response->confirmations[0]->base_offset + 1, + $second->confirmations[0]->base_offset, + 'the confirmed offset must advance by the committed message count' + ); } finally { cleanup_stream_with_topics($client, $streamName, [$topicName]); } diff --git a/foreign/python/Cargo.toml b/foreign/python/Cargo.toml index f709947b69..94ae86a223 100644 --- a/foreign/python/Cargo.toml +++ b/foreign/python/Cargo.toml @@ -37,9 +37,7 @@ doc = false [dependencies] bytes = "1.12.1" futures = "0.3.33" -iggy = { path = "../../core/sdk", version = "0.11.0-edge.1", features = [ - "vsr", -] } +iggy = { path = "../../core/sdk", version = "0.11.0-edge.1" } paste = "1" pyo3 = "0.29.0" pyo3-async-runtimes = { version = "0.29.0", features = [ @@ -47,4 +45,5 @@ pyo3-async-runtimes = { version = "0.29.0", features = [ "tokio-runtime", ] } pyo3-stub-gen = "0.23.0" +secrecy = "0.10" tokio = "1.53.1" diff --git a/foreign/python/README.md b/foreign/python/README.md index 99a9b0ed68..8a5aff4250 100644 --- a/foreign/python/README.md +++ b/foreign/python/README.md @@ -139,6 +139,42 @@ running prek / committing / pushing. This list is not exhaustive and other hook ./scripts/ci/markdownlint.sh --fix foreign/python/README.md # read the diff after applying this, sometimes it gives unwanted results, e.g. messing up enumerations ``` +## Client Configuration + +`IggyClient` takes either a server address or a `TcpConfig`: + +```python +import asyncio +from datetime import timedelta + +from apache_iggy import AutoLogin, IggyClient, TcpConfig, TcpReconnectionConfig + + +async def main(): + client = IggyClient( + TcpConfig( + server_address="127.0.0.1:8090", + auto_login=AutoLogin.username_password("iggy", "iggy"), + reconnection=TcpReconnectionConfig( + enabled=True, + max_retries=10, + interval=timedelta(seconds=2), + reestablish_after=timedelta(seconds=30), + ), + heartbeat_interval=timedelta(seconds=5), + # tls_enabled=True, + # tls_domain="localhost", + # tls_ca_file="../../core/certs/iggy_ca_cert.pem", + # tls_validate_certificate=True, + # nodelay=True, + ) + ) + await client.connect() + + +asyncio.run(main()) +``` + ## Examples Refer to the [examples/python/](https://github.com/apache/iggy/tree/master/examples/python) directory for usage examples. diff --git a/foreign/python/apache_iggy.pyi b/foreign/python/apache_iggy.pyi index 341b3885c6..ab63e66c5a 100644 --- a/foreign/python/apache_iggy.pyi +++ b/foreign/python/apache_iggy.pyi @@ -29,6 +29,7 @@ __all__ = [ "AutoCommit", "AutoCommitAfter", "AutoCommitWhen", + "AutoLogin", "ConsumerGroup", "ConsumerGroupDetails", "ConsumerGroupMember", @@ -48,6 +49,8 @@ __all__ = [ "SendMessagesResponse", "StreamDetails", "StreamPermissions", + "TcpConfig", + "TcpReconnectionConfig", "Topic", "TopicDetails", "TopicPermissions", @@ -250,6 +253,41 @@ class AutoCommitWhen: ... +@typing.final +class AutoLogin: + r""" + The credentials replayed by the client every time it (re)connects. + + `IggyClient` only recovers a lost session when it has credentials to replay, + so a long-running consumer should pass one of the enabled variants. + """ + @property + def enabled(self) -> builtins.bool: + r""" + Whether automatic login is enabled. + """ + @property + def username(self) -> builtins.str | None: + r""" + The username to log in with, or `None` for the disabled and token variants. + """ + @staticmethod + def disabled() -> AutoLogin: + r""" + No automatic login. `login_user()` must be called by hand after every connect. + """ + @staticmethod + def username_password(username: builtins.str, password: builtins.str) -> AutoLogin: + r""" + Log in with the given username and password on every connect. + """ + @staticmethod + def personal_access_token(token: builtins.str) -> AutoLogin: + r""" + Log in with the given personal access token on every connect. + """ + def __repr__(self) -> builtins.str: ... + @typing.final class ConsumerGroup: @property @@ -772,14 +810,25 @@ class GlobalPermissions: class IggyClient: r""" A Python class representing the Iggy client. - It wraps the RustIggyClient and provides asynchronous functionality - through the contained runtime. + It provides asynchronous functionality through the contained runtime. """ - def __new__(cls, conn: builtins.str | None = None) -> IggyClient: + def __new__(cls, conn: TcpConfig | builtins.str | None = None) -> IggyClient: r""" - Constructs a new IggyClient from a TCP server address. + Constructs a new IggyClient from a TCP server address or a `TcpConfig`. This initializes a new runtime for asynchronous operations. Future versions might utilize asyncio for more Pythonic async. + + Args: + conn: Either a `host:port` address, or a `TcpConfig` carrying the full + transport configuration. Defaults to `127.0.0.1:8090` with auto-login + disabled. A malformed address is reported differently by the two + forms: the string form raises `RuntimeError` here, while `TcpConfig` + raises `ValueError` when it is constructed, before it ever reaches + this call. Neither exception is a subclass of the other. + + Raises: + RuntimeError: If the address passed as a string is not a valid + `host:port` pair. """ @classmethod def from_connection_string(cls, connection_string: builtins.str) -> IggyClient: @@ -790,15 +839,14 @@ class IggyClient: def ping(self) -> collections.abc.Awaitable[None]: r""" Sends a ping request to the server to check connectivity. - Returns `Ok(())` if the server responds successfully, or a `PyRuntimeError` - if the connection fails. + Raises `RuntimeError` if the connection fails. """ def login_user( self, username: builtins.str, password: builtins.str ) -> collections.abc.Awaitable[None]: r""" Logs in the user with the given credentials. - Returns `Ok(())` on success, or a PyRuntimeError on failure. + Raises `RuntimeError` on failure. """ def get_user( self, user_id: builtins.str | builtins.int @@ -814,8 +862,8 @@ class IggyClient: or `None` otherwise. Raises: - PyValueError: If a string identifier is invalid. - PyRuntimeError: If the request fails. + ValueError: If a string identifier is invalid. + RuntimeError: If the request fails. """ def get_users(self) -> collections.abc.Awaitable[list[UserInfo]]: r""" @@ -825,7 +873,7 @@ class IggyClient: An awaitable that resolves to `list[UserInfo]`. Raises: - PyRuntimeError: If the request fails. + RuntimeError: If the request fails. """ def create_user( self, @@ -847,7 +895,7 @@ class IggyClient: An awaitable that resolves to the created `UserInfoDetails`. Raises: - PyRuntimeError: If an argument is invalid or the request fails. + RuntimeError: If an argument is invalid or the request fails. """ def update_user( self, @@ -867,8 +915,8 @@ class IggyClient: An awaitable that resolves to `None` when the user is updated. Raises: - PyValueError: If a string identifier is invalid. - PyRuntimeError: If the request fails. + ValueError: If a string identifier is invalid. + RuntimeError: If the request fails. """ def delete_user( self, user_id: builtins.str | builtins.int @@ -883,8 +931,8 @@ class IggyClient: An awaitable that resolves to `None` when the user is deleted. Raises: - PyValueError: If a string identifier is invalid. - PyRuntimeError: If the request fails. + ValueError: If a string identifier is invalid. + RuntimeError: If the request fails. """ def update_permissions( self, user_id: builtins.str | builtins.int, permissions: Permissions | None @@ -903,8 +951,8 @@ class IggyClient: An awaitable that resolves to `None` when the permissions are updated. Raises: - PyValueError: If a string identifier is invalid. - PyRuntimeError: If the request fails. + ValueError: If a string identifier is invalid. + RuntimeError: If the request fails. """ def change_password( self, @@ -924,8 +972,8 @@ class IggyClient: An awaitable that resolves to `None` when the password is changed. Raises: - PyValueError: If a string identifier is invalid. - PyRuntimeError: If the current password is wrong or the request fails. + ValueError: If a string identifier is invalid. + RuntimeError: If the current password is wrong or the request fails. """ def logout_user(self) -> collections.abc.Awaitable[None]: r""" @@ -935,24 +983,25 @@ class IggyClient: An awaitable that resolves to `None` when the user is logged out. Raises: - PyRuntimeError: If the request fails. + RuntimeError: If the request fails. """ def connect(self) -> collections.abc.Awaitable[None]: r""" Connects the IggyClient to its service. - Returns Ok(()) on successful connection or a PyRuntimeError on failure. + Raises `RuntimeError` if the connection fails. """ def create_stream(self, name: builtins.str) -> collections.abc.Awaitable[None]: r""" Creates a new stream with the provided ID and name. - Returns Ok(()) on successful stream creation or a PyRuntimeError on failure. + Raises `RuntimeError` if the stream cannot be created. """ def get_stream( self, stream_id: builtins.str | builtins.int ) -> collections.abc.Awaitable[StreamDetails | None]: r""" Gets stream by id. - Returns Option of stream details or a PyRuntimeError on failure. + Returns the stream details, or `None` if the stream does not exist. + Raises `RuntimeError` on failure. """ def create_topic( self, @@ -990,7 +1039,8 @@ class IggyClient: ) -> collections.abc.Awaitable[TopicDetails | None]: r""" Gets topic by stream and id. - Returns Option of topic details or a PyRuntimeError on failure. + Returns the topic details, or `None` if the topic does not exist. + Raises `RuntimeError` on failure. """ def get_topics( self, stream_id: builtins.str | builtins.int @@ -1005,7 +1055,7 @@ class IggyClient: An awaitable that resolves to `list[Topic]`. Raises: - PyRuntimeError: If the identifier is invalid or the request fails. + RuntimeError: If the identifier is invalid or the request fails. """ def update_topic( self, @@ -1055,7 +1105,7 @@ class IggyClient: An awaitable that resolves to `None` when the topic is deleted. Raises: - PyRuntimeError: If an identifier is invalid or the request fails. + RuntimeError: If an identifier is invalid or the request fails. """ def purge_topic( self, @@ -1073,7 +1123,7 @@ class IggyClient: An awaitable that resolves to `None` when the topic is purged. Raises: - PyRuntimeError: If an identifier is invalid or the request fails. + RuntimeError: If an identifier is invalid or the request fails. """ def create_consumer_group( self, @@ -1093,8 +1143,8 @@ class IggyClient: An awaitable that resolves to `None` when the consumer group is created. Raises: - PyValueError: If an identifier is invalid. - PyRuntimeError: If the request fails. + ValueError: If an identifier is invalid. + RuntimeError: If the request fails. """ def get_consumer_group( self, @@ -1115,8 +1165,8 @@ class IggyClient: or `None` otherwise. Raises: - PyValueError: If an identifier is invalid. - PyRuntimeError: If the request fails. + ValueError: If an identifier is invalid. + RuntimeError: If the request fails. """ def get_consumer_groups( self, @@ -1134,8 +1184,8 @@ class IggyClient: An awaitable that resolves to `list[ConsumerGroup]`. Raises: - PyValueError: If an identifier is invalid. - PyRuntimeError: If the request fails. + ValueError: If an identifier is invalid. + RuntimeError: If the request fails. """ def delete_consumer_group( self, @@ -1155,8 +1205,8 @@ class IggyClient: An awaitable that resolves to `None` when the consumer group is deleted. Raises: - PyValueError: If a string identifier is invalid. - PyRuntimeError: If the request fails. + ValueError: If a string identifier is invalid. + RuntimeError: If the request fails. """ def join_consumer_group( self, @@ -1179,8 +1229,8 @@ class IggyClient: An awaitable that resolves to `None` when the client joins the consumer group. Raises: - PyValueError: If a string identifier is invalid. - PyRuntimeError: If the request fails, including `Feature is unavailable` on HTTP transport. + ValueError: If a string identifier is invalid. + RuntimeError: If the request fails, including `Feature is unavailable` on HTTP transport. """ def leave_consumer_group( self, @@ -1205,8 +1255,8 @@ class IggyClient: rejoin on their next poll. Raises: - PyValueError: If a string identifier is invalid. - PyRuntimeError: If the request fails, including `Feature is unavailable` on HTTP transport. + ValueError: If a string identifier is invalid. + RuntimeError: If the request fails, including `Feature is unavailable` on HTTP transport. """ def send_messages( self, @@ -1233,7 +1283,7 @@ class IggyClient: ) -> collections.abc.Awaitable[list[ReceiveMessage]]: r""" Polls for messages from the specified topic and partition. - Returns a list of received messages or a PyRuntimeError on failure. + Returns a list of received messages or a RuntimeError on failure. """ def consumer_group( self, @@ -1254,7 +1304,10 @@ class IggyClient: ) -> collections.abc.Awaitable[IggyConsumer]: r""" Creates a new consumer group consumer. - Returns the consumer or a PyRuntimeError on failure. + Returns the consumer or a RuntimeError on failure. Raises `ValueError` if + `poll_interval`, `polling_retry_interval`, `init_retry_interval` or an + `AutoCommit` interval is negative, or if any of those except `poll_interval` + is zero. """ def send_binary_request( self, code: builtins.int, payload: builtins.bytes @@ -1273,15 +1326,14 @@ class IggyClient: An awaitable that resolves to the raw response `bytes`. Raises: - PyRuntimeError: If the command cannot be sent or the server returns an error. + RuntimeError: If the command cannot be sent or the server returns an error. """ @typing.final class IggyConsumer: r""" A Python class representing the Iggy consumer. - It wraps the RustIggyConsumer and provides asynchronous functionality - through the contained runtime. + It provides asynchronous functionality through the contained runtime. """ def get_last_consumed_offset( self, partition_id: builtins.int @@ -1315,8 +1367,7 @@ class IggyConsumer: r""" Stores the provided offset for the provided partition id or if none is specified uses the current partition id for the consumer group. - Returns `Ok(())` if the server responds successfully, or a `PyRuntimeError` - if the operation fails. + Raises `RuntimeError` if the operation fails. """ def delete_offset( self, partition_id: builtins.int | None @@ -1324,14 +1375,13 @@ class IggyConsumer: r""" Deletes the offset for the provided partition id or if none is specified uses the current partition id for the consumer group. - Returns `Ok(())` if the server responds successfully, or a `PyRuntimeError` - if the operation fails. + Raises `RuntimeError` if the operation fails. """ def iter_messages(self) -> collections.abc.AsyncIterator[ReceiveMessage]: r""" Asynchronously iterate over `ReceiveMessage`s. Returns an async iterator that raises `StopAsyncIteration` when no more messages are available - or a `PyRuntimeError` on failure. + or a `RuntimeError` on failure. Note: This method does not currently support `AutoCommit.After`. For `AutoCommit.IntervalOrAfter(datetime.timedelta, AutoCommitAfter)`, only the interval part is applied; the `after` mode is ignored. @@ -1346,7 +1396,7 @@ class IggyConsumer: ) -> collections.abc.Awaitable[None]: r""" Consumes messages continuously using a callback function and an optional `asyncio.Event` for signaling shutdown. - Returns an awaitable that completes when shutdown is signaled or a PyRuntimeError on failure. + Returns an awaitable that completes when shutdown is signaled or a RuntimeError on failure. """ class IggyExpiry: @@ -1546,7 +1596,7 @@ class PollingStrategy: class ReceiveMessage: r""" A Python class representing a received message. - This class wraps a Rust message, allowing for access to its payload and offset from Python. + It provides access to the message payload and offset. """ def payload(self) -> bytes: r""" @@ -1596,8 +1646,6 @@ class ReceiveMessage: class SendMessage: r""" A Python class representing a message to be sent. - This class wraps a Rust message meant for sending, facilitating - the creation of such messages from Python and their subsequent use in Rust. """ def __new__( cls, @@ -1763,6 +1811,115 @@ class StreamPermissions: treated as `None`. """ +@typing.final +class TcpConfig: + r""" + Configuration for the TCP transport, accepted by `IggyClient(...)`. + + Every field is keyword-only and optional. + """ + @property + def server_address(self) -> builtins.str: ... + @property + def auto_login(self) -> AutoLogin: ... + @property + def reconnection(self) -> TcpReconnectionConfig: ... + @property + def heartbeat_interval(self) -> datetime.timedelta: ... + @property + def tls_enabled(self) -> builtins.bool: ... + @property + def tls_domain(self) -> builtins.str: ... + @property + def tls_ca_file(self) -> builtins.str | None: ... + @property + def tls_validate_certificate(self) -> builtins.bool: ... + @property + def nodelay(self) -> builtins.bool: ... + def __new__( + cls, + *, + server_address: builtins.str | None = None, + auto_login: AutoLogin | None = None, + reconnection: TcpReconnectionConfig | None = None, + heartbeat_interval: datetime.timedelta | None = None, + tls_enabled: builtins.bool | None = None, + tls_domain: builtins.str | None = None, + tls_ca_file: builtins.str | None = None, + tls_validate_certificate: builtins.bool | None = None, + nodelay: builtins.bool | None = None, + ) -> TcpConfig: + r""" + Constructs a TCP configuration. + + Args: + server_address: `host:port` of the Iggy server. Defaults to `127.0.0.1:8090`. + auto_login: Credentials replayed on every connect. Defaults to `AutoLogin.disabled()`. + reconnection: Reconnection policy. Defaults to `TcpReconnectionConfig()`. + heartbeat_interval: Interval of heartbeats sent by the client. Defaults to 5 seconds. + tls_enabled: Whether to connect over TLS. Defaults to disabled. + tls_domain: Domain to validate the certificate against. Empty means it is + taken from `server_address`. + tls_ca_file: Path to the CA file for TLS. Read only when `tls_enabled` + and `tls_validate_certificate` are both on; with either one off it + is kept but never consulted, so pairing it with + `tls_validate_certificate=False` pins nothing. + tls_validate_certificate: Whether to validate the server certificate. + Defaults to validating. Disabling this accepts any certificate the + server presents, including self-signed and mismatched ones, and + takes precedence over `tls_ca_file`; intended for local development + only. + nodelay: Disable the Nagle algorithm for the TCP socket. Defaults to + leaving it on. + + Raises: + ValueError: If `server_address` is not a valid `host:port` pair, if a + duration is negative, or if `heartbeat_interval` is zero. + """ + def __repr__(self) -> builtins.str: ... + +@typing.final +class TcpReconnectionConfig: + r""" + How the TCP client reconnects after the connection to the server is lost. + """ + @property + def enabled(self) -> builtins.bool: ... + @property + def max_retries(self) -> builtins.int | None: ... + @property + def interval(self) -> datetime.timedelta: ... + @property + def reestablish_after(self) -> datetime.timedelta: ... + def __new__( + cls, + *, + enabled: builtins.bool | None = None, + max_retries: builtins.int | None = None, + interval: datetime.timedelta | None = None, + reestablish_after: datetime.timedelta | None = None, + ) -> TcpReconnectionConfig: + r""" + Constructs a reconnection policy. + + Args: + enabled: Whether to reconnect at all. Defaults to enabled. + max_retries: Attempts before giving up, or `None` for unlimited. + Defaults to unlimited, which means a call awaited while the server + is down never returns: `connect()`, `send_messages()` and + `poll_messages()` all wait inside the retry loop. Set a finite + number for request/reply style usage, so a call fails instead. + interval: Delay between attempts. Defaults to 1 second. + reestablish_after: Cooldown before reconnecting after a previously + successful connection. Defaults to 5 seconds. + + Raises: + ValueError: If a duration is negative, if `max_retries` is outside the + range of an unsigned 32-bit integer, or if `interval` is zero while + reconnection is enabled and `max_retries` is unlimited. + """ + def __repr__(self) -> builtins.str: ... + @typing.final class Topic: @property diff --git a/foreign/python/docker-compose.test.yml b/foreign/python/docker-compose.test.yml index 129b941bf5..78caed9929 100644 --- a/foreign/python/docker-compose.test.yml +++ b/foreign/python/docker-compose.test.yml @@ -22,7 +22,8 @@ services: dockerfile: core/server/Dockerfile args: PROFILE: debug - command: ["--fresh", "--with-default-root-credentials"] + # The server takes only --replica-id; root credentials come from the env. + command: [] container_name: iggy-server-python-test security_opt: - seccomp:unconfined @@ -36,8 +37,11 @@ services: - IGGY_HTTP_ADDRESS=0.0.0.0:3000 - IGGY_TCP_ADDRESS=0.0.0.0:8090 - IGGY_QUIC_ADDRESS=0.0.0.0:8080 + - IGGY_WEBSOCKET_ADDRESS=0.0.0.0:8092 + - IGGY_ROOT_USERNAME=iggy + - IGGY_ROOT_PASSWORD=iggy healthcheck: - test: [ "CMD", "iggy", "--tcp-server-address", "127.0.0.1:8090", "ping" ] + test: [ "CMD", "/usr/local/bin/iggy", "--tcp-server-address", "127.0.0.1:8090", "ping" ] interval: 5s timeout: 5s retries: 12 diff --git a/foreign/python/scripts/test.sh b/foreign/python/scripts/test.sh index 5d7aff90e8..295fe3d3df 100755 --- a/foreign/python/scripts/test.sh +++ b/foreign/python/scripts/test.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/foreign/python/src/client.rs b/foreign/python/src/client.rs index c7cbe39520..cde56040ff 100644 --- a/foreign/python/src/client.rs +++ b/foreign/python/src/client.rs @@ -30,10 +30,12 @@ use pyo3_stub_gen::derive::{gen_stub_pyclass, gen_stub_pymethods}; use std::str::FromStr; use std::sync::Arc; +use crate::config::PyClientConfig; use crate::consumer::{ AutoCommit, ConsumerGroup as PyConsumerGroup, ConsumerGroupDetails as PyConsumerGroupDetails, - IggyConsumer, py_delta_to_iggy_duration, + IggyConsumer, }; +use crate::duration::{py_delta_to_iggy_duration, reject_zero}; use crate::identifier::PyIdentifier; use crate::permissions::Permissions as PyPermissions; use crate::receive_message::{PollingStrategy, ReceiveMessage}; @@ -46,8 +48,7 @@ use crate::user::{ use tokio::sync::Mutex; /// A Python class representing the Iggy client. -/// It wraps the RustIggyClient and provides asynchronous functionality -/// through the contained runtime. +/// It provides asynchronous functionality through the contained runtime. #[gen_stub_pyclass] #[pyclass] pub struct IggyClient { @@ -83,21 +84,44 @@ fn resolve_topic_params( #[gen_stub_pymethods] #[pymethods] impl IggyClient { - /// Constructs a new IggyClient from a TCP server address. + /// Constructs a new IggyClient from a TCP server address or a `TcpConfig`. /// This initializes a new runtime for asynchronous operations. /// Future versions might utilize asyncio for more Pythonic async. + /// + /// Args: + /// conn: Either a `host:port` address, or a `TcpConfig` carrying the full + /// transport configuration. Defaults to `127.0.0.1:8090` with auto-login + /// disabled. A malformed address is reported differently by the two + /// forms: the string form raises `RuntimeError` here, while `TcpConfig` + /// raises `ValueError` when it is constructed, before it ever reaches + /// this call. Neither exception is a subclass of the other. + /// + /// Raises: + /// RuntimeError: If the address passed as a string is not a valid + /// `host:port` pair. #[new] #[pyo3(signature = (conn=None))] fn new( - #[gen_stub(override_type(type_repr = "builtins.str | None"))] conn: Option, + #[gen_stub(override_type(type_repr = "TcpConfig | builtins.str | None"))] conn: Option< + PyClientConfig, + >, ) -> PyResult { - let client = IggyClientBuilder::new() - .with_tcp() - .with_server_address(conn.unwrap_or("127.0.0.1:8090".to_string())) - .build() + let config = match conn { + Some(PyClientConfig::Config(config)) => config.client_config(), + Some(PyClientConfig::ServerAddress(server_address)) => Arc::new( + TcpClientConfigBuilder::new() + .with_server_address(server_address) + .build() + .map_err(|e| { + PyErr::new::(e.to_string()) + })?, + ), + None => Arc::new(TcpClientConfig::default()), + }; + let tcp_client = TcpClient::create(config) .map_err(|e| PyErr::new::(e.to_string()))?; Ok(IggyClient { - inner: Arc::new(client), + inner: Arc::new(RustIggyClient::new(ClientWrapper::Tcp(tcp_client))), }) } @@ -119,8 +143,7 @@ impl IggyClient { } /// Sends a ping request to the server to check connectivity. - /// Returns `Ok(())` if the server responds successfully, or a `PyRuntimeError` - /// if the connection fails. + /// Raises `RuntimeError` if the connection fails. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn ping<'a>(&self, py: Python<'a>) -> PyResult> { let inner = self.inner.clone(); @@ -133,7 +156,7 @@ impl IggyClient { } /// Logs in the user with the given credentials. - /// Returns `Ok(())` on success, or a PyRuntimeError on failure. + /// Raises `RuntimeError` on failure. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn login_user<'a>( &self, @@ -161,8 +184,8 @@ impl IggyClient { /// or `None` otherwise. /// /// Raises: - /// PyValueError: If a string identifier is invalid. - /// PyRuntimeError: If the request fails. + /// ValueError: If a string identifier is invalid. + /// RuntimeError: If the request fails. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[UserInfoDetails | None]", imports=("collections.abc")))] fn get_user<'a>(&self, py: Python<'a>, user_id: PyIdentifier) -> PyResult> { let user_id = Identifier::try_from(user_id)?; @@ -183,7 +206,7 @@ impl IggyClient { /// An awaitable that resolves to `list[UserInfo]`. /// /// Raises: - /// PyRuntimeError: If the request fails. + /// RuntimeError: If the request fails. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[list[UserInfo]]", imports=("collections.abc")))] fn get_users<'a>(&self, py: Python<'a>) -> PyResult> { let inner = self.inner.clone(); @@ -209,7 +232,7 @@ impl IggyClient { /// An awaitable that resolves to the created `UserInfoDetails`. /// /// Raises: - /// PyRuntimeError: If an argument is invalid or the request fails. + /// RuntimeError: If an argument is invalid or the request fails. #[pyo3(signature = (username, password, status=None, permissions=None))] #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[UserInfoDetails]", imports=("collections.abc")))] fn create_user<'a>( @@ -246,8 +269,8 @@ impl IggyClient { /// An awaitable that resolves to `None` when the user is updated. /// /// Raises: - /// PyValueError: If a string identifier is invalid. - /// PyRuntimeError: If the request fails. + /// ValueError: If a string identifier is invalid. + /// RuntimeError: If the request fails. #[pyo3(signature = (user_id, username=None, status=None))] #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn update_user<'a>( @@ -279,8 +302,8 @@ impl IggyClient { /// An awaitable that resolves to `None` when the user is deleted. /// /// Raises: - /// PyValueError: If a string identifier is invalid. - /// PyRuntimeError: If the request fails. + /// ValueError: If a string identifier is invalid. + /// RuntimeError: If the request fails. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn delete_user<'a>(&self, py: Python<'a>, user_id: PyIdentifier) -> PyResult> { let user_id = Identifier::try_from(user_id)?; @@ -308,8 +331,8 @@ impl IggyClient { /// An awaitable that resolves to `None` when the permissions are updated. /// /// Raises: - /// PyValueError: If a string identifier is invalid. - /// PyRuntimeError: If the request fails. + /// ValueError: If a string identifier is invalid. + /// RuntimeError: If the request fails. #[pyo3(signature = (user_id, permissions))] #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn update_permissions<'a>( @@ -344,8 +367,8 @@ impl IggyClient { /// An awaitable that resolves to `None` when the password is changed. /// /// Raises: - /// PyValueError: If a string identifier is invalid. - /// PyRuntimeError: If the current password is wrong or the request fails. + /// ValueError: If a string identifier is invalid. + /// RuntimeError: If the current password is wrong or the request fails. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn change_password<'a>( &self, @@ -372,7 +395,7 @@ impl IggyClient { /// An awaitable that resolves to `None` when the user is logged out. /// /// Raises: - /// PyRuntimeError: If the request fails. + /// RuntimeError: If the request fails. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn logout_user<'a>(&self, py: Python<'a>) -> PyResult> { let inner = self.inner.clone(); @@ -387,7 +410,7 @@ impl IggyClient { } /// Connects the IggyClient to its service. - /// Returns Ok(()) on successful connection or a PyRuntimeError on failure. + /// Raises `RuntimeError` if the connection fails. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn connect<'a>(&self, py: Python<'a>) -> PyResult> { let inner = self.inner.clone(); @@ -401,7 +424,7 @@ impl IggyClient { } /// Creates a new stream with the provided ID and name. - /// Returns Ok(()) on successful stream creation or a PyRuntimeError on failure. + /// Raises `RuntimeError` if the stream cannot be created. #[pyo3(signature = (name))] #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn create_stream<'a>(&self, py: Python<'a>, name: String) -> PyResult> { @@ -416,7 +439,8 @@ impl IggyClient { } /// Gets stream by id. - /// Returns Option of stream details or a PyRuntimeError on failure. + /// Returns the stream details, or `None` if the stream does not exist. + /// Raises `RuntimeError` on failure. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[StreamDetails | None]", imports=("collections.abc")))] fn get_stream<'a>( &self, @@ -500,7 +524,8 @@ impl IggyClient { } /// Gets topic by stream and id. - /// Returns Option of topic details or a PyRuntimeError on failure. + /// Returns the topic details, or `None` if the topic does not exist. + /// Raises `RuntimeError` on failure. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[TopicDetails | None]", imports=("collections.abc")))] fn get_topic<'a>( &self, @@ -530,7 +555,7 @@ impl IggyClient { /// An awaitable that resolves to `list[Topic]`. /// /// Raises: - /// PyRuntimeError: If the identifier is invalid or the request fails. + /// RuntimeError: If the identifier is invalid or the request fails. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[list[Topic]]", imports=("collections.abc")))] fn get_topics<'a>( &self, @@ -627,7 +652,7 @@ impl IggyClient { /// An awaitable that resolves to `None` when the topic is deleted. /// /// Raises: - /// PyRuntimeError: If an identifier is invalid or the request fails. + /// RuntimeError: If an identifier is invalid or the request fails. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn delete_topic<'a>( &self, @@ -658,7 +683,7 @@ impl IggyClient { /// An awaitable that resolves to `None` when the topic is purged. /// /// Raises: - /// PyRuntimeError: If an identifier is invalid or the request fails. + /// RuntimeError: If an identifier is invalid or the request fails. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn purge_topic<'a>( &self, @@ -690,8 +715,8 @@ impl IggyClient { /// An awaitable that resolves to `None` when the consumer group is created. /// /// Raises: - /// PyValueError: If an identifier is invalid. - /// PyRuntimeError: If the request fails. + /// ValueError: If an identifier is invalid. + /// RuntimeError: If the request fails. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn create_consumer_group<'a>( &self, @@ -725,8 +750,8 @@ impl IggyClient { /// or `None` otherwise. /// /// Raises: - /// PyValueError: If an identifier is invalid. - /// PyRuntimeError: If the request fails. + /// ValueError: If an identifier is invalid. + /// RuntimeError: If the request fails. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[ConsumerGroupDetails | None]", imports=("collections.abc")))] fn get_consumer_group<'a>( &self, @@ -759,8 +784,8 @@ impl IggyClient { /// An awaitable that resolves to `list[ConsumerGroup]`. /// /// Raises: - /// PyValueError: If an identifier is invalid. - /// PyRuntimeError: If the request fails. + /// ValueError: If an identifier is invalid. + /// RuntimeError: If the request fails. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[list[ConsumerGroup]]", imports=("collections.abc")))] fn get_consumer_groups<'a>( &self, @@ -795,8 +820,8 @@ impl IggyClient { /// An awaitable that resolves to `None` when the consumer group is deleted. /// /// Raises: - /// PyValueError: If a string identifier is invalid. - /// PyRuntimeError: If the request fails. + /// ValueError: If a string identifier is invalid. + /// RuntimeError: If the request fails. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn delete_consumer_group<'a>( &self, @@ -833,8 +858,8 @@ impl IggyClient { /// An awaitable that resolves to `None` when the client joins the consumer group. /// /// Raises: - /// PyValueError: If a string identifier is invalid. - /// PyRuntimeError: If the request fails, including `Feature is unavailable` on HTTP transport. + /// ValueError: If a string identifier is invalid. + /// RuntimeError: If the request fails, including `Feature is unavailable` on HTTP transport. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn join_consumer_group<'a>( &self, @@ -873,8 +898,8 @@ impl IggyClient { /// rejoin on their next poll. /// /// Raises: - /// PyValueError: If a string identifier is invalid. - /// PyRuntimeError: If the request fails, including `Feature is unavailable` on HTTP transport. + /// ValueError: If a string identifier is invalid. + /// RuntimeError: If the request fails, including `Feature is unavailable` on HTTP transport. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn leave_consumer_group<'a>( &self, @@ -938,7 +963,7 @@ impl IggyClient { } /// Polls for messages from the specified topic and partition. - /// Returns a list of received messages or a PyRuntimeError on failure. + /// Returns a list of received messages or a RuntimeError on failure. #[allow(clippy::too_many_arguments)] #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[list[ReceiveMessage]]", imports=("collections.abc")))] fn poll_messages<'a>( @@ -984,7 +1009,10 @@ impl IggyClient { } /// Creates a new consumer group consumer. - /// Returns the consumer or a PyRuntimeError on failure. + /// Returns the consumer or a RuntimeError on failure. Raises `ValueError` if + /// `poll_interval`, `polling_retry_interval`, `init_retry_interval` or an + /// `AutoCommit` interval is negative, or if any of those except `poll_interval` + /// is zero. #[allow(clippy::too_many_arguments)] #[pyo3(signature = ( name, @@ -1060,8 +1088,10 @@ impl IggyClient { builder = builder.without_poll_interval() }; if let Some(polling_retry_interval) = polling_retry_interval { - builder = - builder.polling_retry_interval(py_delta_to_iggy_duration(&polling_retry_interval)?) + builder = builder.polling_retry_interval(reject_zero( + py_delta_to_iggy_duration(&polling_retry_interval)?, + "polling_retry_interval", + )?) } if init_retries.is_some() && init_retry_interval.is_none() { return Err(PyErr::new::( @@ -1077,7 +1107,10 @@ impl IggyClient { { builder = builder.init_retries( init_retries, - py_delta_to_iggy_duration(&init_retry_interval)?, + reject_zero( + py_delta_to_iggy_duration(&init_retry_interval)?, + "init_retry_interval", + )?, ); } if allow_replay { @@ -1109,7 +1142,7 @@ impl IggyClient { /// An awaitable that resolves to the raw response `bytes`. /// /// Raises: - /// PyRuntimeError: If the command cannot be sent or the server returns an error. + /// RuntimeError: If the command cannot be sent or the server returns an error. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[bytes]", imports=("collections.abc")))] fn send_binary_request<'a>( &self, diff --git a/foreign/python/src/config.rs b/foreign/python/src/config.rs new file mode 100644 index 0000000000..4519c939fb --- /dev/null +++ b/foreign/python/src/config.rs @@ -0,0 +1,423 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use iggy::prelude::{ + AutoLogin as RustAutoLogin, Credentials as RustCredentials, + TcpClientConfig as RustTcpClientConfig, TcpClientConfigBuilder, + TcpClientReconnectionConfig as RustTcpClientReconnectionConfig, +}; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use pyo3::types::PyDelta; +use pyo3_stub_gen::derive::{gen_stub_pyclass, gen_stub_pymethods}; +use pyo3_stub_gen::impl_stub_type; +use secrecy::SecretString; +use std::sync::Arc; + +use crate::duration::{ + duration_repr, iggy_duration_to_py_delta, py_delta_to_iggy_duration, reject_zero, +}; + +/// The credentials replayed by the client every time it (re)connects. +/// +/// `IggyClient` only recovers a lost session when it has credentials to replay, +/// so a long-running consumer should pass one of the enabled variants. +#[gen_stub_pyclass] +#[pyclass(from_py_object)] +#[derive(Clone)] +pub struct AutoLogin { + pub(crate) inner: RustAutoLogin, +} + +#[gen_stub_pymethods] +#[pymethods] +impl AutoLogin { + /// No automatic login. `login_user()` must be called by hand after every connect. + #[staticmethod] + fn disabled() -> Self { + Self { + inner: RustAutoLogin::Disabled, + } + } + + /// Log in with the given username and password on every connect. + #[staticmethod] + fn username_password(username: String, password: String) -> Self { + Self { + inner: RustAutoLogin::Enabled(RustCredentials::UsernamePassword( + username, + SecretString::from(password), + )), + } + } + + /// Log in with the given personal access token on every connect. + #[staticmethod] + fn personal_access_token(token: String) -> Self { + Self { + inner: RustAutoLogin::Enabled(RustCredentials::PersonalAccessToken( + SecretString::from(token), + )), + } + } + + /// Whether automatic login is enabled. + #[getter] + fn enabled(&self) -> bool { + matches!(self.inner, RustAutoLogin::Enabled(_)) + } + + /// The username to log in with, or `None` for the disabled and token variants. + #[gen_stub(override_return_type(type_repr = "builtins.str | None"))] + #[getter] + fn username(&self) -> Option { + match &self.inner { + RustAutoLogin::Enabled(RustCredentials::UsernamePassword(username, _)) => { + Some(username.clone()) + } + _ => None, + } + } + + fn __repr__(&self) -> String { + match &self.inner { + RustAutoLogin::Disabled => "AutoLogin.disabled()".to_owned(), + RustAutoLogin::Enabled(RustCredentials::UsernamePassword(username, _)) => { + format!("AutoLogin.username_password({username:?}, ...)") + } + RustAutoLogin::Enabled(RustCredentials::PersonalAccessToken(_)) => { + "AutoLogin.personal_access_token(...)".to_owned() + } + } + } +} + +/// How the TCP client reconnects after the connection to the server is lost. +#[gen_stub_pyclass] +#[pyclass(from_py_object)] +#[derive(Clone)] +pub struct TcpReconnectionConfig { + pub(crate) inner: RustTcpClientReconnectionConfig, +} + +#[gen_stub_pymethods] +#[pymethods] +impl TcpReconnectionConfig { + /// Constructs a reconnection policy. + /// + /// Args: + /// enabled: Whether to reconnect at all. Defaults to enabled. + /// max_retries: Attempts before giving up, or `None` for unlimited. + /// Defaults to unlimited, which means a call awaited while the server + /// is down never returns: `connect()`, `send_messages()` and + /// `poll_messages()` all wait inside the retry loop. Set a finite + /// number for request/reply style usage, so a call fails instead. + /// interval: Delay between attempts. Defaults to 1 second. + /// reestablish_after: Cooldown before reconnecting after a previously + /// successful connection. Defaults to 5 seconds. + /// + /// Raises: + /// ValueError: If a duration is negative, if `max_retries` is outside the + /// range of an unsigned 32-bit integer, or if `interval` is zero while + /// reconnection is enabled and `max_retries` is unlimited. + #[new] + #[pyo3(signature = (*, enabled=None, max_retries=None, interval=None, reestablish_after=None))] + fn new( + #[gen_stub(override_type(type_repr = "builtins.bool | None"))] enabled: Option, + #[gen_stub(override_type(type_repr = "builtins.int | None"))] max_retries: Option, + #[gen_stub(override_type(type_repr = "datetime.timedelta | None", imports=("datetime")))] + interval: Option>, + #[gen_stub(override_type(type_repr = "datetime.timedelta | None", imports=("datetime")))] + reestablish_after: Option>, + ) -> PyResult { + let defaults = RustTcpClientReconnectionConfig::default(); + let enabled = enabled.unwrap_or(defaults.enabled); + let max_retries = max_retries + .map(|max_retries| { + u32::try_from(max_retries).map_err(|_| { + PyValueError::new_err(format!( + "'max_retries' must be between 0 and {}", + u32::MAX + )) + }) + }) + .transpose()?; + let interval = interval + .as_ref() + .map(py_delta_to_iggy_duration) + .transpose()? + .unwrap_or(defaults.interval); + // Unlimited retries at a zero interval reconnect in a continuous loop; + // a zero interval with a retry cap is a legitimate fast-retry policy, and + // with reconnection off the interval is never read at all. + if enabled && interval.is_zero() && max_retries.is_none() { + return Err(PyValueError::new_err( + "'interval' must not be zero unless 'max_retries' is set", + )); + } + Ok(Self { + inner: RustTcpClientReconnectionConfig { + enabled, + max_retries, + interval, + reestablish_after: reestablish_after + .as_ref() + .map(py_delta_to_iggy_duration) + .transpose()? + .unwrap_or(defaults.reestablish_after), + }, + }) + } + + #[getter] + fn enabled(&self) -> bool { + self.inner.enabled + } + + #[gen_stub(override_return_type(type_repr = "builtins.int | None"))] + #[getter] + fn max_retries(&self) -> Option { + self.inner.max_retries + } + + #[gen_stub(override_return_type(type_repr = "datetime.timedelta", imports=("datetime")))] + #[getter] + fn interval<'a>(&self, py: Python<'a>) -> PyResult> { + iggy_duration_to_py_delta(py, self.inner.interval) + } + + #[gen_stub(override_return_type(type_repr = "datetime.timedelta", imports=("datetime")))] + #[getter] + fn reestablish_after<'a>(&self, py: Python<'a>) -> PyResult> { + iggy_duration_to_py_delta(py, self.inner.reestablish_after) + } + + fn __repr__(&self) -> String { + let max_retries = match self.inner.max_retries { + Some(max_retries) => max_retries.to_string(), + None => "None".to_owned(), + }; + format!( + "TcpReconnectionConfig(enabled={}, max_retries={max_retries}, interval={}, reestablish_after={})", + python_bool(self.inner.enabled), + duration_repr(self.inner.interval), + duration_repr(self.inner.reestablish_after), + ) + } +} + +/// Configuration for the TCP transport, accepted by `IggyClient(...)`. +/// +/// Every field is keyword-only and optional. +#[gen_stub_pyclass] +#[pyclass(from_py_object)] +#[derive(Clone)] +pub struct TcpConfig { + inner: Arc, +} + +impl TcpConfig { + /// The configuration in the shape `TcpClient::create` expects. + pub(crate) fn client_config(&self) -> Arc { + self.inner.clone() + } +} + +#[gen_stub_pymethods] +#[pymethods] +impl TcpConfig { + /// Constructs a TCP configuration. + /// + /// Args: + /// server_address: `host:port` of the Iggy server. Defaults to `127.0.0.1:8090`. + /// auto_login: Credentials replayed on every connect. Defaults to `AutoLogin.disabled()`. + /// reconnection: Reconnection policy. Defaults to `TcpReconnectionConfig()`. + /// heartbeat_interval: Interval of heartbeats sent by the client. Defaults to 5 seconds. + /// tls_enabled: Whether to connect over TLS. Defaults to disabled. + /// tls_domain: Domain to validate the certificate against. Empty means it is + /// taken from `server_address`. + /// tls_ca_file: Path to the CA file for TLS. Read only when `tls_enabled` + /// and `tls_validate_certificate` are both on; with either one off it + /// is kept but never consulted, so pairing it with + /// `tls_validate_certificate=False` pins nothing. + /// tls_validate_certificate: Whether to validate the server certificate. + /// Defaults to validating. Disabling this accepts any certificate the + /// server presents, including self-signed and mismatched ones, and + /// takes precedence over `tls_ca_file`; intended for local development + /// only. + /// nodelay: Disable the Nagle algorithm for the TCP socket. Defaults to + /// leaving it on. + /// + /// Raises: + /// ValueError: If `server_address` is not a valid `host:port` pair, if a + /// duration is negative, or if `heartbeat_interval` is zero. + #[new] + #[pyo3(signature = ( + *, + server_address=None, + auto_login=None, + reconnection=None, + heartbeat_interval=None, + tls_enabled=None, + tls_domain=None, + tls_ca_file=None, + tls_validate_certificate=None, + nodelay=None, + ))] + #[allow(clippy::too_many_arguments)] + fn new( + #[gen_stub(override_type(type_repr = "builtins.str | None"))] server_address: Option< + String, + >, + #[gen_stub(override_type(type_repr = "AutoLogin | None"))] auto_login: Option, + #[gen_stub(override_type(type_repr = "TcpReconnectionConfig | None"))] reconnection: Option< + TcpReconnectionConfig, + >, + #[gen_stub(override_type(type_repr = "datetime.timedelta | None", imports=("datetime")))] + heartbeat_interval: Option>, + #[gen_stub(override_type(type_repr = "builtins.bool | None"))] tls_enabled: Option, + #[gen_stub(override_type(type_repr = "builtins.str | None"))] tls_domain: Option, + #[gen_stub(override_type(type_repr = "builtins.str | None"))] tls_ca_file: Option, + #[gen_stub(override_type(type_repr = "builtins.bool | None"))] + tls_validate_certificate: Option, + #[gen_stub(override_type(type_repr = "builtins.bool | None"))] nodelay: Option, + ) -> PyResult { + // The builder starts from `TcpClientConfig::default()`, and its `build()` + // trims and validates the address whether or not one was set here. + let mut builder = TcpClientConfigBuilder::new(); + if let Some(server_address) = server_address { + builder = builder.with_server_address(server_address); + } + let mut inner = builder + .build() + .map_err(|e| PyValueError::new_err(e.to_string()))?; + if let Some(auto_login) = auto_login { + inner.auto_login = auto_login.inner; + } + if let Some(reconnection) = reconnection { + inner.reconnection = reconnection.inner; + } + if let Some(heartbeat_interval) = heartbeat_interval { + inner.heartbeat_interval = reject_zero( + py_delta_to_iggy_duration(&heartbeat_interval)?, + "heartbeat_interval", + )?; + } + if let Some(tls_enabled) = tls_enabled { + inner.tls_enabled = tls_enabled; + } + if let Some(tls_domain) = tls_domain { + inner.tls_domain = tls_domain; + } + if tls_ca_file.is_some() { + inner.tls_ca_file = tls_ca_file; + } + if let Some(tls_validate_certificate) = tls_validate_certificate { + inner.tls_validate_certificate = tls_validate_certificate; + } + if let Some(nodelay) = nodelay { + inner.nodelay = nodelay; + } + + Ok(Self { + inner: Arc::new(inner), + }) + } + + #[getter] + fn server_address(&self) -> String { + self.inner.server_address.clone() + } + + #[getter] + fn auto_login(&self) -> AutoLogin { + AutoLogin { + inner: self.inner.auto_login.clone(), + } + } + + #[getter] + fn reconnection(&self) -> TcpReconnectionConfig { + TcpReconnectionConfig { + inner: self.inner.reconnection.clone(), + } + } + + #[gen_stub(override_return_type(type_repr = "datetime.timedelta", imports=("datetime")))] + #[getter] + fn heartbeat_interval<'a>(&self, py: Python<'a>) -> PyResult> { + iggy_duration_to_py_delta(py, self.inner.heartbeat_interval) + } + + #[getter] + fn tls_enabled(&self) -> bool { + self.inner.tls_enabled + } + + #[getter] + fn tls_domain(&self) -> String { + self.inner.tls_domain.clone() + } + + #[gen_stub(override_return_type(type_repr = "builtins.str | None"))] + #[getter] + fn tls_ca_file(&self) -> Option { + self.inner.tls_ca_file.clone() + } + + #[getter] + fn tls_validate_certificate(&self) -> bool { + self.inner.tls_validate_certificate + } + + #[getter] + fn nodelay(&self) -> bool { + self.inner.nodelay + } + + fn __repr__(&self) -> String { + let tls_ca_file = match &self.inner.tls_ca_file { + Some(tls_ca_file) => format!("{tls_ca_file:?}"), + None => "None".to_owned(), + }; + format!( + "TcpConfig(server_address={:?}, auto_login={}, reconnection={}, heartbeat_interval={}, tls_enabled={}, tls_domain={:?}, tls_ca_file={tls_ca_file}, tls_validate_certificate={}, nodelay={})", + self.inner.server_address, + self.auto_login().__repr__(), + self.reconnection().__repr__(), + duration_repr(self.inner.heartbeat_interval), + python_bool(self.inner.tls_enabled), + self.inner.tls_domain, + python_bool(self.inner.tls_validate_certificate), + python_bool(self.inner.nodelay), + ) + } +} + +fn python_bool(value: bool) -> &'static str { + if value { "True" } else { "False" } +} + +/// What `IggyClient(...)` accepts: a bare `host:port` or a full `TcpConfig`. +#[derive(FromPyObject)] +pub enum PyClientConfig { + #[pyo3(transparent)] + Config(TcpConfig), + #[pyo3(transparent, annotation = "str")] + ServerAddress(String), +} +impl_stub_type!(PyClientConfig = TcpConfig | String); diff --git a/foreign/python/src/consumer.rs b/foreign/python/src/consumer.rs index 4d64fc626c..6a6e69a877 100644 --- a/foreign/python/src/consumer.rs +++ b/foreign/python/src/consumer.rs @@ -16,7 +16,6 @@ // under the License. use std::sync::Arc; -use std::time::Duration; use futures::StreamExt; use iggy::consumer_ext::{IggyConsumerMessageExt, MessageConsumer}; @@ -27,8 +26,8 @@ use iggy::prelude::{ ConsumerGroupMember as RustConsumerGroupMember, IggyConsumer as RustIggyConsumer, IggyDuration, IggyError, ReceivedMessage, }; -use pyo3::exceptions::{PyStopAsyncIteration, PyValueError}; -use pyo3::types::{PyDelta, PyDeltaAccess}; +use pyo3::exceptions::PyStopAsyncIteration; +use pyo3::types::PyDelta; use pyo3::prelude::*; use pyo3_async_runtimes::TaskLocals; @@ -39,12 +38,12 @@ use tokio::sync::Mutex; use tokio::sync::oneshot::Sender; use tokio::task::JoinHandle; +use crate::duration::{py_delta_to_iggy_duration, reject_zero}; use crate::identifier::PyIdentifier; use crate::receive_message::ReceiveMessage; /// A Python class representing the Iggy consumer. -/// It wraps the RustIggyConsumer and provides asynchronous functionality -/// through the contained runtime. +/// It provides asynchronous functionality through the contained runtime. #[gen_stub_pyclass] #[pyclass] pub struct IggyConsumer { @@ -94,8 +93,7 @@ impl IggyConsumer { /// Stores the provided offset for the provided partition id or if none is specified /// uses the current partition id for the consumer group. - /// Returns `Ok(())` if the server responds successfully, or a `PyRuntimeError` - /// if the operation fails. + /// Raises `RuntimeError` if the operation fails. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn store_offset<'a>( &self, @@ -116,8 +114,7 @@ impl IggyConsumer { /// Deletes the offset for the provided partition id or if none is specified /// uses the current partition id for the consumer group. - /// Returns `Ok(())` if the server responds successfully, or a `PyRuntimeError` - /// if the operation fails. + /// Raises `RuntimeError` if the operation fails. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn delete_offset<'a>( &self, @@ -137,7 +134,7 @@ impl IggyConsumer { /// Asynchronously iterate over `ReceiveMessage`s. /// Returns an async iterator that raises `StopAsyncIteration` when no more messages are available - /// or a `PyRuntimeError` on failure. + /// or a `RuntimeError` on failure. /// Note: This method does not currently support `AutoCommit.After`. /// For `AutoCommit.IntervalOrAfter(datetime.timedelta, AutoCommitAfter)`, /// only the interval part is applied; the `after` mode is ignored. @@ -149,7 +146,7 @@ impl IggyConsumer { } /// Consumes messages continuously using a callback function and an optional `asyncio.Event` for signaling shutdown. - /// Returns an awaitable that completes when shutdown is signaled or a PyRuntimeError on failure. + /// Returns an awaitable that completes when shutdown is signaled or a RuntimeError on failure. #[gen_stub(override_return_type(type_repr="collections.abc.Awaitable[None]", imports=("collections.abc")))] fn consume_messages<'a>( &self, @@ -435,17 +432,12 @@ impl TryFrom<&AutoCommit> for RustAutoCommit { fn try_from(val: &AutoCommit) -> PyResult { Ok(match val { AutoCommit::Disabled() => RustAutoCommit::Disabled, - AutoCommit::Interval(delta) => { - let duration = py_delta_to_iggy_duration(delta)?; - RustAutoCommit::Interval(duration) - } + AutoCommit::Interval(delta) => RustAutoCommit::Interval(auto_commit_interval(delta)?), AutoCommit::IntervalOrWhen(delta, when) => { - let duration = py_delta_to_iggy_duration(delta)?; - RustAutoCommit::IntervalOrWhen(duration, when.into()) + RustAutoCommit::IntervalOrWhen(auto_commit_interval(delta)?, when.into()) } AutoCommit::IntervalOrAfter(delta, after) => { - let duration = py_delta_to_iggy_duration(delta)?; - RustAutoCommit::IntervalOrAfter(duration, after.into()) + RustAutoCommit::IntervalOrAfter(auto_commit_interval(delta)?, after.into()) } AutoCommit::When(when) => RustAutoCommit::When(when.into()), AutoCommit::After(after) => RustAutoCommit::After(after.into()), @@ -453,6 +445,10 @@ impl TryFrom<&AutoCommit> for RustAutoCommit { } } +fn auto_commit_interval(delta: &Py) -> PyResult { + reject_zero(py_delta_to_iggy_duration(delta)?, "AutoCommit interval") +} + /// The auto-commit mode for storing the offset on the server. #[derive(Debug, PartialEq, Copy, Clone)] #[gen_stub_pyclass_complex_enum(skip_stub_type)] @@ -518,20 +514,3 @@ impl PyStubType for AutoCommitAfter { TypeInfo::unqualified("AutoCommitAfter") } } - -pub fn py_delta_to_iggy_duration(delta1: &Py) -> PyResult { - Python::attach(|py| { - let delta = delta1.bind(py); - let total_seconds = i64::from(delta.get_days()) * 86_400 + i64::from(delta.get_seconds()); - if total_seconds < 0 { - return Err(PyValueError::new_err( - "duration must not be negative".to_string(), - )); - } - let nanos = (delta.get_microseconds() * 1_000) as u32; - Ok(IggyDuration::new(Duration::new( - total_seconds as u64, - nanos, - ))) - }) -} diff --git a/foreign/python/src/duration.rs b/foreign/python/src/duration.rs new file mode 100644 index 0000000000..9b2b419fe5 --- /dev/null +++ b/foreign/python/src/duration.rs @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use iggy::prelude::IggyDuration; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; +use pyo3::types::PyDelta; +use std::time::Duration; + +pub fn py_delta_to_iggy_duration(delta: &Py) -> PyResult { + Python::attach(|py| { + // The value is already a timedelta, so a negative one is the only failure + // left to map, and the Python surface must not name Rust types. + delta + .bind(py) + .extract::() + .map(IggyDuration::from) + .map_err(|_| PyValueError::new_err("duration must not be negative")) + }) +} + +pub fn iggy_duration_to_py_delta( + py: Python<'_>, + duration: IggyDuration, +) -> PyResult> { + duration.get_duration().into_pyobject(py) +} + +/// Renders a duration the way it would be written in Python, so that a `__repr__` +/// built from it can be pasted back into a constructor. +pub fn duration_repr(duration: IggyDuration) -> String { + // Read the std duration, whose micros are u128: `IggyDuration::as_micros()` + // truncates to u64, which a timedelta near the Python maximum overflows. + let micros = duration.get_duration().as_micros(); + if micros.is_multiple_of(1_000_000) { + format!("datetime.timedelta(seconds={})", micros / 1_000_000) + } else { + format!("datetime.timedelta(microseconds={micros})") + } +} + +/// Rejects a zero duration for parameters where zero means an unthrottled loop +/// rather than "disabled". +pub fn reject_zero(duration: IggyDuration, parameter: &str) -> PyResult { + if duration.is_zero() { + return Err(PyValueError::new_err(format!( + "'{parameter}' must not be zero" + ))); + } + Ok(duration) +} diff --git a/foreign/python/src/lib.rs b/foreign/python/src/lib.rs index 985476ff74..9c7f1efaa7 100644 --- a/foreign/python/src/lib.rs +++ b/foreign/python/src/lib.rs @@ -16,7 +16,9 @@ // under the License. pub mod client; +mod config; mod consumer; +mod duration; mod identifier; mod permissions; mod receive_message; @@ -27,6 +29,7 @@ mod user; mod user_headers; use client::IggyClient; +use config::{AutoLogin, TcpConfig, TcpReconnectionConfig}; use consumer::{ AutoCommit, AutoCommitAfter, AutoCommitWhen, ConsumerGroup, ConsumerGroupDetails, ConsumerGroupMember, IggyConsumer, ReceiveMessageIterator, @@ -40,7 +43,7 @@ use topic::{IggyExpiry, MaxTopicSize, Partition, Topic, TopicDetails}; use user::{UserInfo, UserInfoDetails, UserStatus}; use user_headers::{HeaderKey, HeaderValue, UserHeaders}; -/// A Python module implemented in Rust. +/// Python client for Apache Iggy, the persistent message streaming platform. #[pymodule] fn apache_iggy(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; @@ -48,6 +51,9 @@ fn apache_iggy(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/foreign/python/src/receive_message.rs b/foreign/python/src/receive_message.rs index ecabb6cf22..ddb14e6233 100644 --- a/foreign/python/src/receive_message.rs +++ b/foreign/python/src/receive_message.rs @@ -24,7 +24,7 @@ use pyo3_stub_gen::derive::{gen_stub_pyclass, gen_stub_pyclass_complex_enum, gen use crate::user_headers::{UserHeaders, rust_user_headers_to_py}; /// A Python class representing a received message. -/// This class wraps a Rust message, allowing for access to its payload and offset from Python. +/// It provides access to the message payload and offset. #[pyclass] #[gen_stub_pyclass] pub struct ReceiveMessage { diff --git a/foreign/python/src/send_message.rs b/foreign/python/src/send_message.rs index e63b1a2f78..bfa3cdd671 100644 --- a/foreign/python/src/send_message.rs +++ b/foreign/python/src/send_message.rs @@ -30,8 +30,6 @@ use pyo3_stub_gen::{ use crate::user_headers::py_user_headers_to_rust; /// A Python class representing a message to be sent. -/// This class wraps a Rust message meant for sending, facilitating -/// the creation of such messages from Python and their subsequent use in Rust. #[pyclass(from_py_object)] #[gen_stub_pyclass] pub struct SendMessage { diff --git a/foreign/python/src/topic.rs b/foreign/python/src/topic.rs index 178f90dd5e..40a7db228e 100644 --- a/foreign/python/src/topic.rs +++ b/foreign/python/src/topic.rs @@ -15,8 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::time::Duration; - use iggy::prelude::{ IggyByteSize, IggyExpiry as RustIggyExpiry, MaxTopicSize as RustMaxTopicSize, Partition as RustPartition, Topic as RustTopic, TopicDetails as RustTopicDetails, @@ -26,7 +24,7 @@ use pyo3::prelude::*; use pyo3::types::PyDelta; use pyo3_stub_gen::derive::{gen_stub_pyclass, gen_stub_pyclass_complex_enum, gen_stub_pymethods}; -use crate::consumer::py_delta_to_iggy_duration; +use crate::duration::{iggy_duration_to_py_delta, py_delta_to_iggy_duration}; /// The expiry of the messages in a topic. #[gen_stub_pyclass_complex_enum] @@ -55,7 +53,14 @@ impl TryFrom for IggyExpiry { Ok(match expiry { RustIggyExpiry::ServerDefault => IggyExpiry::ServerDefault(), RustIggyExpiry::ExpireDuration(duration) => IggyExpiry::ExpireDuration { - duration: iggy_duration_to_py_delta(duration.get_duration())?, + duration: Python::attach(|py| { + iggy_duration_to_py_delta(py, duration).map(|delta| delta.unbind()) + }) + .map_err(|err| { + PyValueError::new_err(format!( + "topic message expiry duration does not fit within timedelta bounds: {err}" + )) + })?, }, RustIggyExpiry::NeverExpire => IggyExpiry::NeverExpire(), }) @@ -93,26 +98,6 @@ impl TryFrom<&IggyExpiry> for RustIggyExpiry { } } -fn iggy_duration_to_py_delta(duration: Duration) -> PyResult> { - let days = duration.as_secs() / 86_400; - let secs_of_day = duration.as_secs() % 86_400; - Python::attach(|py| { - PyDelta::new( - py, - days as i32, - secs_of_day as i32, - duration.subsec_micros() as i32, - true, - ) - .map(|delta| delta.unbind()) - .map_err(|err| { - PyValueError::new_err(format!( - "topic message expiry duration does not fit within timedelta bounds: {err}" - )) - }) - }) -} - /// The maximum size of a topic. #[gen_stub_pyclass_complex_enum] #[pyclass] diff --git a/foreign/python/tests/conftest.py b/foreign/python/tests/conftest.py index aa54ff50d8..3ab97065f5 100644 --- a/foreign/python/tests/conftest.py +++ b/foreign/python/tests/conftest.py @@ -131,5 +131,10 @@ def pytest_collection_modifyitems(items): path.name for path in Path(__file__).parent.glob("test_*.py") } for item in items: + # Tests explicitly marked as unit need no server; auto-marking them + # integration too would make `-m "not integration"` unable to select + # them. + if item.get_closest_marker("unit"): + continue if any(module in item.nodeid for module in integration_modules): item.add_marker(pytest.mark.integration) diff --git a/foreign/python/tests/test_client_config.py b/foreign/python/tests/test_client_config.py new file mode 100644 index 0000000000..78df23459c --- /dev/null +++ b/foreign/python/tests/test_client_config.py @@ -0,0 +1,459 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Tests for the TCP client configuration surface. + +`TcpConfig`, `TcpReconnectionConfig` and `AutoLogin` mirror the Rust SDK +types, so most of these assert that a value set from Python survives to the +getters and that unset fields fall back to the Rust defaults. The last class +proves the point of the configuration: with `auto_login` set, credentials are +replayed on connect and no manual `login_user()` is needed. +""" + +import ast +from collections.abc import Callable +from datetime import timedelta + +import pytest + +from apache_iggy import ( + AutoCommit, + AutoCommitAfter, + AutoCommitWhen, + AutoLogin, + IggyClient, + IggyExpiry, + TcpConfig, + TcpReconnectionConfig, +) + +from .utils import get_server_config, wait_for_ping, wait_for_server + + +@pytest.mark.unit +class TestAutoLogin: + """Test the credentials carried into the client.""" + + def test_disabled_has_no_username(self): + """Test that the disabled variant carries no credentials.""" + auto_login = AutoLogin.disabled() + + assert auto_login.enabled is False + assert auto_login.username is None + + def test_username_password_exposes_username_only(self): + """Test that the username is readable back but the password is not.""" + auto_login = AutoLogin.username_password("iggy", "secret") + + assert auto_login.enabled is True + assert auto_login.username == "iggy" + assert "secret" not in repr(auto_login) + + def test_personal_access_token_hides_the_token(self): + """Test that a token login exposes neither a username nor the token.""" + auto_login = AutoLogin.personal_access_token("secret-token") + + assert auto_login.enabled is True + assert auto_login.username is None + assert "secret-token" not in repr(auto_login) + + +@pytest.mark.unit +class TestTcpReconnectionConfig: + """Test the reconnection policy.""" + + def test_defaults_match_the_rust_sdk(self): + """Test that an unconfigured policy reconnects forever, one second apart.""" + reconnection = TcpReconnectionConfig() + + assert reconnection.enabled is True + assert reconnection.max_retries is None + assert reconnection.interval == timedelta(seconds=1) + assert reconnection.reestablish_after == timedelta(seconds=5) + + def test_every_field_round_trips(self): + """Test that each configured field is readable back unchanged.""" + reconnection = TcpReconnectionConfig( + enabled=False, + max_retries=10, + interval=timedelta(milliseconds=250), + reestablish_after=timedelta(seconds=30), + ) + + assert reconnection.enabled is False + assert reconnection.max_retries == 10 + assert reconnection.interval == timedelta(milliseconds=250) + assert reconnection.reestablish_after == timedelta(seconds=30) + + def test_arguments_are_keyword_only(self): + """Test that the adjacent flags cannot be passed positionally.""" + with pytest.raises(TypeError): + # pyrefly: ignore # bad-argument-count + TcpReconnectionConfig(True) + + @pytest.mark.parametrize( + "construct", + [ + lambda duration: TcpReconnectionConfig(interval=duration), + lambda duration: TcpReconnectionConfig(reestablish_after=duration), + ], + ids=["interval", "reestablish_after"], + ) + @pytest.mark.parametrize( + "negative", + [timedelta(microseconds=-1), timedelta(seconds=-1), timedelta(days=-1)], + ) + def test_negative_duration_is_rejected( + self, + construct: Callable[[timedelta], TcpReconnectionConfig], + negative: timedelta, + ): + """Test that a negative duration fails at construction, not at connect.""" + with pytest.raises(ValueError, match="negative"): + construct(negative) + + @pytest.mark.parametrize("out_of_range", [-1, 2**32]) + def test_out_of_range_max_retries_is_rejected(self, out_of_range: int): + """Test that a retry count outside the wire range names the argument. + + The conversion pyo3 does on its own raises OverflowError, which is not a + ValueError and so escapes the handler a caller wraps construction in. + """ + with pytest.raises(ValueError, match="max_retries"): + TcpReconnectionConfig(max_retries=out_of_range) + + def test_zero_reestablish_after_is_allowed(self): + """Test that a zero cooldown is legal and readable back.""" + reconnection = TcpReconnectionConfig(reestablish_after=timedelta(0)) + + assert reconnection.reestablish_after == timedelta(0) + + def test_zero_interval_is_allowed_with_bounded_retries(self): + """Test that a zero interval is legal as a bounded fast-retry policy.""" + reconnection = TcpReconnectionConfig(interval=timedelta(0), max_retries=5) + + assert reconnection.interval == timedelta(0) + + def test_zero_interval_is_allowed_when_reconnection_is_disabled(self): + """Test that a zero interval is legal when nothing ever reads it.""" + reconnection = TcpReconnectionConfig(enabled=False, interval=timedelta(0)) + + assert reconnection.interval == timedelta(0) + + def test_zero_interval_with_unlimited_retries_is_rejected(self): + """Test that the combination that reconnects in a continuous loop fails.""" + with pytest.raises(ValueError, match="zero"): + TcpReconnectionConfig(interval=timedelta(0)) + + def test_very_long_interval_round_trips(self): + """Test that an interval beyond 68 years survives the i32 boundary.""" + reconnection = TcpReconnectionConfig(interval=timedelta(days=30_000)) + + assert reconnection.interval == timedelta(days=30_000) + + def test_maximum_interval_round_trips(self): + """Test that the largest timedelta survives the day conversion.""" + reconnection = TcpReconnectionConfig(interval=timedelta(days=999_999_999)) + + assert reconnection.interval == timedelta(days=999_999_999) + + +@pytest.mark.unit +class TestTcpConfig: + """Test the transport configuration.""" + + def test_defaults_match_the_rust_sdk(self): + """Test that an unconfigured transport matches the Rust SDK defaults.""" + config = TcpConfig() + + assert config.server_address == "127.0.0.1:8090" + assert config.auto_login.enabled is False + assert config.reconnection.enabled is True + assert config.heartbeat_interval == timedelta(seconds=5) + assert config.tls_enabled is False + assert config.tls_domain == "" + assert config.tls_ca_file is None + assert config.tls_validate_certificate is True + assert config.nodelay is False + + def test_every_field_round_trips(self): + """Test that each configured field is readable back unchanged.""" + config = TcpConfig( + server_address="localhost:8090", + auto_login=AutoLogin.username_password("iggy", "iggy"), + reconnection=TcpReconnectionConfig(max_retries=3), + heartbeat_interval=timedelta(seconds=15), + tls_enabled=True, + tls_domain="localhost", + tls_ca_file="ca.pem", + tls_validate_certificate=False, + nodelay=True, + ) + + assert config.server_address == "localhost:8090" + assert config.auto_login.username == "iggy" + assert config.reconnection.max_retries == 3 + assert config.heartbeat_interval == timedelta(seconds=15) + assert config.tls_enabled is True + assert config.tls_domain == "localhost" + assert config.tls_ca_file == "ca.pem" + assert config.tls_validate_certificate is False + assert config.nodelay is True + + def test_arguments_are_keyword_only(self): + """Test that the address cannot be passed positionally.""" + with pytest.raises(TypeError): + # pyrefly: ignore # bad-argument-count + TcpConfig("127.0.0.1:8090") + + def test_repr_hides_the_password(self): + """Test that the password does not leak through repr.""" + config = TcpConfig(auto_login=AutoLogin.username_password("iggy", "secret")) + + assert "secret" not in repr(config) + + def test_repr_shows_every_field_as_python(self): + """Test that repr covers the TLS fields and parses as Python. + + The TLS fields are the ones a handshake is debugged with, and a repr is + only worth printing if it can be pasted back into a constructor. + """ + config = TcpConfig( + heartbeat_interval=timedelta(seconds=15), + tls_enabled=True, + tls_domain="localhost", + tls_ca_file="ca.pem", + tls_validate_certificate=False, + nodelay=True, + ) + + printed = repr(config) + + assert 'tls_domain="localhost"' in printed + assert 'tls_ca_file="ca.pem"' in printed + assert "tls_validate_certificate=False" in printed + assert "nodelay=True" in printed + assert "heartbeat_interval=datetime.timedelta(seconds=15)" in printed + ast.parse(printed) + + @pytest.mark.parametrize( + "invalid_address", + ["", "127.0.0.1", "127.0.0.1:not-a-port", "127.0.0.1:70000", "::1:8090"], + ) + def test_invalid_server_address_is_rejected(self, invalid_address: str): + """Test that a malformed address fails at construction, not at connect.""" + with pytest.raises(ValueError): + TcpConfig(server_address=invalid_address) + + def test_negative_heartbeat_interval_is_rejected(self): + """Test that a negative heartbeat interval fails at construction.""" + with pytest.raises(ValueError, match="negative"): + TcpConfig(heartbeat_interval=timedelta(seconds=-3)) + + def test_zero_heartbeat_interval_is_rejected(self): + """Test that a zero heartbeat interval fails at construction. + + Nothing downstream reads zero as "disabled"; it heartbeats in a + continuous loop for as long as the client lives. + """ + with pytest.raises(ValueError, match="zero"): + TcpConfig(heartbeat_interval=timedelta(0)) + + +@pytest.mark.unit +class TestClientConstruction: + """Test what the client constructor accepts.""" + + def test_accepts_a_config(self): + """Test that a client can be built from a config object.""" + assert IggyClient(TcpConfig(server_address="127.0.0.1:8090")) is not None + + def test_accepts_an_address(self): + """Test that the address form still works.""" + assert IggyClient("127.0.0.1:8090") is not None + + def test_accepts_nothing(self): + """Test that the default address is used when no argument is given.""" + assert IggyClient() is not None + + def test_rejects_an_invalid_address(self): + """Test that a malformed address is rejected.""" + with pytest.raises(RuntimeError): + IggyClient("nonsense") + + def test_negative_message_expiry_is_rejected(self): + """Test that the negative-duration rule reaches create_topic. + + The check runs at the call, before any I/O. + """ + client = IggyClient() + + with pytest.raises(ValueError, match="negative"): + client.create_topic( + stream="stream", + name="topic", + partitions_count=1, + message_expiry=IggyExpiry.ExpireDuration(timedelta(seconds=-1)), + ) + + @pytest.mark.parametrize( + "interval_kwargs", + [ + {"polling_retry_interval": timedelta(0)}, + {"init_retries": 3, "init_retry_interval": timedelta(0)}, + {"auto_commit": AutoCommit.Interval(timedelta(0))}, + { + "auto_commit": AutoCommit.IntervalOrWhen( + timedelta(0), AutoCommitWhen.PollingMessages() + ) + }, + { + "auto_commit": AutoCommit.IntervalOrAfter( + timedelta(0), AutoCommitAfter.ConsumingEachMessage() + ) + }, + ], + ids=[ + "polling_retry_interval", + "init_retry_interval", + "auto_commit_interval", + "auto_commit_interval_or_when", + "auto_commit_interval_or_after", + ], + ) + def test_zero_consumer_interval_is_rejected(self, interval_kwargs: dict): + """Test that a zero consumer interval fails at the call. + + Zero spins the retry loop, floods the server with offset stores, or + panics inside the runtime timer, and none of those name the argument + that caused it. + """ + client = IggyClient() + + with pytest.raises(ValueError, match="zero"): + client.consumer_group( + name="group", + stream="stream", + topic="topic", + **interval_kwargs, + ) + + def test_zero_poll_interval_is_allowed(self): + """Test that a zero poll interval passes validation. + + Zero there means "do not wait before polling" and is short-circuited + before the sleep, unlike the retry intervals. Reaching the awaitable is + what proves it: building one without a running loop is the next failure, + and a rejected value would have raised ValueError first. + """ + client = IggyClient() + + with pytest.raises(RuntimeError): + client.consumer_group( + name="group", + stream="stream", + topic="topic", + poll_interval=timedelta(0), + ) + + +@pytest.mark.integration +class TestAutoLoginAgainstServer: + """Test that configured credentials are actually replayed on connect.""" + + @pytest.mark.asyncio + async def test_auto_login_authenticates_without_login_user(self, unique_name): + """Test that a privileged call succeeds without a manual login_user().""" + host, port = get_server_config() + wait_for_server(host, port) + + client = IggyClient( + TcpConfig( + server_address=f"{host}:{port}", + auto_login=AutoLogin.username_password("iggy", "iggy"), + ) + ) + await client.connect() + await wait_for_ping(client) + + stream_name = unique_name() + await client.create_stream(stream_name) + assert await client.get_stream(stream_name) is not None + + @pytest.mark.asyncio + async def test_without_auto_login_a_privileged_call_is_unauthenticated( + self, unique_name + ): + """Test that the same call fails when no credentials are configured.""" + host, port = get_server_config() + wait_for_server(host, port) + + client = IggyClient(TcpConfig(server_address=f"{host}:{port}")) + await client.connect() + await wait_for_ping(client) + + with pytest.raises(RuntimeError): + await client.create_stream(unique_name()) + + @pytest.mark.asyncio + async def test_config_and_connection_string_both_authenticate(self, unique_name): + """Test that either form of configuring credentials logs the client in. + + The reconnection policy is set on both sides to mirror the connection + string, but the client exposes no getter for it, so this asserts only + what is observable: both clients reach an authenticated session. + """ + host, port = get_server_config() + wait_for_server(host, port) + + from_config = IggyClient( + TcpConfig( + server_address=f"{host}:{port}", + auto_login=AutoLogin.username_password("iggy", "iggy"), + reconnection=TcpReconnectionConfig( + max_retries=3, interval=timedelta(seconds=1) + ), + ) + ) + from_string = IggyClient.from_connection_string( + f"iggy+tcp://iggy:iggy@{host}:{port}" + "?reconnection_retries=3&reconnection_interval=1s" + ) + + stream_name = unique_name() + for client in (from_config, from_string): + await client.connect() + await wait_for_ping(client) + assert await client.get_stream(stream_name) is None + + @pytest.mark.asyncio + async def test_wrong_auto_login_credentials_fail(self): + """Test that bad configured credentials surface as a connect failure.""" + host, port = get_server_config() + wait_for_server(host, port) + + client = IggyClient( + TcpConfig( + server_address=f"{host}:{port}", + auto_login=AutoLogin.username_password("iggy", "invalid-password"), + reconnection=TcpReconnectionConfig(enabled=False), + ) + ) + + with pytest.raises(RuntimeError): + await client.connect() diff --git a/foreign/python/tests/test_tls.py b/foreign/python/tests/test_tls.py index fd8ecef7fb..f914e41466 100644 --- a/foreign/python/tests/test_tls.py +++ b/foreign/python/tests/test_tls.py @@ -27,10 +27,8 @@ - testcontainers[docker] installed (in [testing-docker] extras) - CA certificate available at core/certs/iggy_ca_cert.pem - server image built locally (or IGGY_SERVER_DOCKER_IMAGE set): - TODO(hubcio): change to iggy-server once legacy server is removed - (core/server has VSR support) - docker build -f core/server-ng/Dockerfile --target runtime-prebuilt \ - --build-arg PREBUILT_IGGY_SERVER_NG=target/debug/iggy-server-ng \ + docker build -f core/server/Dockerfile --target runtime-prebuilt \ + --build-arg PREBUILT_IGGY_SERVER=target/debug/iggy-server \ --build-arg PREBUILT_IGGY_CLI=target/debug/iggy \ -t iggy-server:local . """ @@ -73,7 +71,7 @@ def tls_container(): ) container.start() # Wait for the server to be ready inside the container - wait_for_logs(container, "server-ng running", timeout=60) + wait_for_logs(container, "server running", timeout=60) yield container container.stop() diff --git a/foreign/python/tests/test_topic.py b/foreign/python/tests/test_topic.py index 417f24a385..46434525ef 100644 --- a/foreign/python/tests/test_topic.py +++ b/foreign/python/tests/test_topic.py @@ -24,7 +24,6 @@ from .utils import ( get_server_config, wait_for_ping, - wait_for_purged_topic, wait_for_server, ) @@ -1384,9 +1383,10 @@ async def test_purge_topic_clears_messages_but_keeps_topic( await iggy_client.purge_topic(stream_name, topic_name) - # The purge ack precedes the asynchronous partition prune, so poll - # until the stats catch up instead of asserting the counts directly. - after = await wait_for_purged_topic(iggy_client, stream_name, topic_name) + after = await iggy_client.get_topic(stream_name, topic_name) + assert after is not None + assert after.messages_count == 0 + assert after.size == 0 # Purging clears messages and size only; topic config is unchanged. assert after.id == before.id assert after.name == before.name @@ -1437,7 +1437,10 @@ async def test_purge_topic_is_idempotent_when_called_repeatedly( await iggy_client.purge_topic(stream_name, topic_name) await iggy_client.purge_topic(stream_name, topic_name) - await wait_for_purged_topic(iggy_client, stream_name, topic_name) + topic = await iggy_client.get_topic(stream_name, topic_name) + assert topic is not None + assert topic.messages_count == 0 + assert topic.size == 0 @pytest.mark.asyncio async def test_purge_nonexistent_topic_fails( diff --git a/foreign/python/tests/utils.py b/foreign/python/tests/utils.py index b3ea85c413..b37a53831e 100644 --- a/foreign/python/tests/utils.py +++ b/foreign/python/tests/utils.py @@ -24,7 +24,7 @@ import socket import time -from apache_iggy import IggyClient, TopicDetails +from apache_iggy import IggyClient # Server-side limits: usernames are 3-50 bytes, passwords 3-100 bytes. MIN_USERNAME_BYTES = 3 @@ -115,37 +115,6 @@ async def wait_for_ping( await asyncio.sleep(interval) -async def wait_for_purged_topic( - client: IggyClient, stream: str, topic: str, timeout: float = 10.0 -) -> TopicDetails: - """ - Poll get_topic until a committed purge is reflected in the stats. - - The VSR server acknowledges a purge once it commits; partition data - is pruned asynchronously, so stats can transiently report pre-purge - counts. - - Returns: - TopicDetails once messages_count and size reach 0 - - Raises: - TimeoutError: If the purge is not reflected within timeout - """ - deadline = time.time() + timeout - - while True: - details = await client.get_topic(stream, topic) - assert details is not None, "purged topic must still exist" - if details.messages_count == 0 and details.size == 0: - return details - if time.time() >= deadline: - raise TimeoutError( - f"purge of {stream}/{topic} not reflected after {timeout}s: " - f"messages_count={details.messages_count} size={details.size}" - ) - await asyncio.sleep(0.05) - - def unique_credentials(unique_name) -> tuple[str, str]: """Return a unique (username, password) pair within the server limits.""" username = unique_name(max_bytes=MAX_USERNAME_BYTES) diff --git a/justfile b/justfile index c8bf1ba4f2..7e18398c80 100644 --- a/justfile +++ b/justfile @@ -43,9 +43,6 @@ reap_test_containers := "docker ps -aqf 'name=^iggy-test-' | xargs -r docker rm build: cargo build -build-vsr: - cargo build --features vsr - test: build #!/usr/bin/env bash set -euo pipefail @@ -64,17 +61,6 @@ nextest: build trap "{{reap_test_containers}}" EXIT cargo nextest run --retries 2 -# Like `nextest` but with the `vsr` feature; builds vsr first so the -# harness-spawned iggy-server-ng carries the vsr wire format. `--no-fail-fast` -# because nextest otherwise cancels the run on the first red, which on a loaded -# box means a stray flake hides most of the suite and the reported counts -# understate what actually ran. -nextest-vsr: build-vsr - #!/usr/bin/env bash - set -euo pipefail - trap "{{reap_test_containers}}" EXIT - cargo nextest run --features vsr --no-fail-fast --retries 2 - nextests TEST: build #!/usr/bin/env bash set -euo pipefail @@ -101,9 +87,6 @@ miri: server *ARGS: cargo run --bin iggy-server {{ARGS}} -server-ng *ARGS: - cargo run --bin iggy-server-ng {{ARGS}} - run-benches: ./scripts/run-benches.sh diff --git a/scripts/bump-version.sh b/scripts/bump-version.sh index d1e44e334a..88413af9d2 100755 --- a/scripts/bump-version.sh +++ b/scripts/bump-version.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/check-backwards-compat.sh b/scripts/check-backwards-compat.sh deleted file mode 100755 index f7990195f6..0000000000 --- a/scripts/check-backwards-compat.sh +++ /dev/null @@ -1,260 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -euo pipefail - -# ----------------------------- -# Config (overridable via args) -# ----------------------------- -MASTER_REF="${MASTER_REF:-master}" # branch or commit for "baseline" -PR_REF="${PR_REF:-HEAD}" # commit to test (assumes current checkout) -HOST="${HOST:-127.0.0.1}" -PORT="${PORT:-8090}" -WAIT_SECS="${WAIT_SECS:-60}" -BATCHES="${BATCHES:-50}" -MSGS_PER_BATCH="${MSGS_PER_BATCH:-100}" -KEEP_TMP="${KEEP_TMP:-false}" - -# ----------------------------- -# Helpers -# ----------------------------- -info(){ printf "\n\033[1;36m➤ %s\033[0m\n" "$*"; } -ok(){ printf "\033[0;32m✓ %s\033[0m\n" "$*"; } -err(){ printf "\033[0;31m✗ %s\033[0m\n" "$*" >&2; } -die(){ err "$*"; exit 1; } - -need() { - command -v "$1" >/dev/null 2>&1 || die "missing dependency: $1" -} - -wait_for_port() { - local host="$1" port="$2" deadline=$((SECONDS + WAIT_SECS)) - while (( SECONDS < deadline )); do - if command -v nc >/dev/null 2>&1; then - if nc -z "$host" "$port" 2>/dev/null; then return 0; fi - else - if (echo >"/dev/tcp/$host/$port") >/dev/null 2>&1; then return 0; fi - fi - sleep 1 - done - return 1 -} - -stop_pid() { - local pid="$1" name="${2:-process}" - if kill -0 "$pid" 2>/dev/null; then - kill -TERM "$pid" || true - for _ in $(seq 1 15); do - kill -0 "$pid" 2>/dev/null || { ok "stopped $name (pid $pid)"; return 0; } - sleep 1 - done - err "$name (pid $pid) still running; sending SIGKILL" - kill -KILL "$pid" || true - fi -} - -print_logs_if_any() { - local dir="$1" - if compgen -G "$dir/local_data/logs/iggy*" > /dev/null; then - echo "---- $dir/local_data/logs ----" - cat "$dir"/local_data/logs/iggy* || true - echo "------------------------------" - else - echo "(no iggy logs found in $dir/local_data/logs)" - fi -} - -# ----------------------------- -# Args -# ----------------------------- -usage() { - cat </dev/null 2>&1 || true # optional, we'll use it if present - -REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)" -cd "$REPO_ROOT" - -# Free the port proactively (best-effort) -pkill -f iggy-server >/dev/null 2>&1 || true - -TMP_ROOT="$(mktemp -d -t iggy-backcompat-XXXXXX)" -MASTER_DIR="$TMP_ROOT/master" -PR_DIR="$REPO_ROOT" # assume script is run from PR checkout -MASTER_LOG="$TMP_ROOT/server-master.stdout.log" -PR_LOG="$TMP_ROOT/server-pr.stdout.log" - -cleanup() { - # Stop any leftover iggy-server - pkill -f iggy-server >/dev/null 2>&1 || true - git worktree remove --force "$MASTER_DIR" >/dev/null 2>&1 || true - if [[ "$KEEP_TMP" != "true" ]]; then - rm -rf "$TMP_ROOT" || true - else - info "keeping temp dir: $TMP_ROOT" - fi -} -trap cleanup EXIT - -# ----------------------------- -# 1) Prepare master worktree -# ----------------------------- -info "Preparing baseline worktree at '$MASTER_REF'" -git fetch --all --tags --prune >/dev/null 2>&1 || true -git worktree add --force "$MASTER_DIR" "$MASTER_REF" -ok "worktree at $MASTER_DIR" - -# ----------------------------- -# 2) Build & run master server -# ----------------------------- -pushd "$MASTER_DIR" >/dev/null - -info "Building iggy-server & benches (baseline: $MASTER_REF)" -cargo build --locked --bin iggy-server --bin iggy-bench -ok "built baseline" - -info "Starting iggy-server (baseline)" -set +e -( IGGY_ROOT_USERNAME=iggy IGGY_ROOT_PASSWORD=iggy nohup target/debug/iggy-server >"$MASTER_LOG" 2>&1 & echo $! > "$TMP_ROOT/master.pid" ) -set -e -MASTER_PID="$(cat "$TMP_ROOT/master.pid")" -ok "iggy-server started (pid $MASTER_PID), logs: $MASTER_LOG" - -info "Waiting for $HOST:$PORT to be ready (up to ${WAIT_SECS}s)" -if ! wait_for_port "$HOST" "$PORT"; then - err "server did not become ready in ${WAIT_SECS}s" - print_logs_if_any "$MASTER_DIR" - [[ -f "$MASTER_LOG" ]] && { tail -n 200 "$MASTER_LOG" || true; } - exit 1 -fi -ok "server is ready" - -# Producer bench (baseline) -info "Running producer bench on baseline" -BENCH_CMD=( target/debug/iggy-bench --message-batches "$BATCHES" --messages-per-batch "$MSGS_PER_BATCH" pinned-producer tcp ) -if command -v timeout >/dev/null 2>&1; then timeout 60s "${BENCH_CMD[@]}"; else "${BENCH_CMD[@]}"; fi -ok "producer bench done" - -# Consumer bench (baseline) -info "Running consumer bench on baseline" -BENCH_CMD=( target/debug/iggy-bench --message-batches "$BATCHES" --messages-per-batch "$MSGS_PER_BATCH" pinned-consumer tcp ) -if command -v timeout >/dev/null 2>&1; then timeout 60s "${BENCH_CMD[@]}"; else "${BENCH_CMD[@]}"; fi -ok "consumer bench done (baseline)" - -# Stop baseline server -info "Stopping baseline server" -stop_pid "$MASTER_PID" "iggy-server(baseline)" -print_logs_if_any "$MASTER_DIR" - -# Clean baseline logs (like CI step) -if compgen -G "local_data/logs/iggy*" > /dev/null; then - rm -f local_data/logs/iggy* || true -fi - -# Snapshot local_data/ -info "Snapshotting baseline local_data/" -cp -a local_data "$TMP_ROOT/local_data" -ok "snapshot stored at $TMP_ROOT/local_data" - -popd >/dev/null - -# ----------------------------- -# 3) Build PR & restore data -# ----------------------------- -pushd "$PR_DIR" >/dev/null -info "Ensuring PR ref is present: $PR_REF" -git rev-parse --verify "$PR_REF^{commit}" >/dev/null 2>&1 || die "PR_REF '$PR_REF' not found" -git checkout -q "$PR_REF" - -info "Building iggy-server & benches (PR: $PR_REF)" -cargo build --locked --bin iggy-server --bin iggy-bench -ok "built PR" - -info "Restoring baseline local_data/ into PR workspace" -rm -rf local_data -cp -a "$TMP_ROOT/local_data" ./local_data -ok "restored local_data/" - -# ----------------------------- -# 4) Run PR server & consumer bench -# ----------------------------- -info "Starting iggy-server (PR)" -set +e -( IGGY_ROOT_USERNAME=iggy IGGY_ROOT_PASSWORD=iggy nohup target/debug/iggy-server >"$PR_LOG" 2>&1 & echo $! > "$TMP_ROOT/pr.pid" ) -set -e -PR_PID="$(cat "$TMP_ROOT/pr.pid")" -ok "iggy-server (PR) started (pid $PR_PID), logs: $PR_LOG" - -info "Waiting for $HOST:$PORT to be ready (up to ${WAIT_SECS}s)" -if ! wait_for_port "$HOST" "$PORT"; then - err "PR server did not become ready in ${WAIT_SECS}s" - print_logs_if_any "$PR_DIR" - [[ -f "$PR_LOG" ]] && { tail -n 200 "$PR_LOG" || true; } - exit 1 -fi -ok "PR server is ready" - -# Only consumer bench against PR -info "Running consumer bench on PR (compat check)" -BENCH_CMD=( target/debug/iggy-bench --message-batches "$BATCHES" --messages-per-batch "$MSGS_PER_BATCH" pinned-consumer tcp ) -if command -v timeout >/dev/null 2>&1; then timeout 60s "${BENCH_CMD[@]}"; else "${BENCH_CMD[@]}"; fi -ok "consumer bench done (PR)" - -# Stop PR server -info "Stopping PR server" -stop_pid "$PR_PID" "iggy-server(PR)" -print_logs_if_any "$PR_DIR" - -ok "backwards-compatibility check PASSED" -popd >/dev/null diff --git a/scripts/ci/binary-artifacts.sh b/scripts/ci/binary-artifacts.sh index 4d63a71e2a..bbc02760fd 100755 --- a/scripts/ci/binary-artifacts.sh +++ b/scripts/ci/binary-artifacts.sh @@ -18,6 +18,9 @@ set -euo pipefail +# shellcheck source-path=SCRIPTDIR +source "$(dirname "${BASH_SOURCE[0]}")/lib/init.sh" + # binary-artifacts.sh -- Prevent compiled binaries from entering the repo. # # .gitignore catches common extensions (*.o, *.so, *.exe, *.out, etc.) but diff --git a/scripts/ci/lib/init.sh b/scripts/ci/lib/init.sh new file mode 100644 index 0000000000..0cde9f27a1 --- /dev/null +++ b/scripts/ci/lib/init.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Single entry point for the script-backed pre-commit hooks. Sourcing this runs +# every preflight check, so a new check reaches all hooks by being added here +# rather than to each of them. A check that fails exits the sourcing script. +# +# Keep this parseable and runnable by bash 3.2, or the version check cannot +# report on the shell it is meant to reject. + +ensure_bash_version() { + local min_major=4 + local min_minor=2 + + if [ "${BASH_VERSINFO[0]}" -gt "$min_major" ]; then + return 0 + fi + if [ "${BASH_VERSINFO[0]}" -eq "$min_major" ] && [ "${BASH_VERSINFO[1]}" -ge "$min_minor" ]; then + return 0 + fi + + echo "ERROR: this script requires bash >= ${min_major}.${min_minor}, but is running under ${BASH_VERSION}" >&2 + echo " interpreter: ${BASH}" >&2 + echo " bash on PATH: $(command -v bash || echo '')" >&2 + if [ "$(uname)" = "Darwin" ]; then + echo >&2 + echo "macOS ships bash 3.2. Install a current one and make sure it comes first on PATH:" >&2 + echo " brew install bash" >&2 + echo >&2 + echo "Git GUIs often launch hooks with a minimal PATH where /bin wins. If the command" >&2 + echo "above is already installed, commit from a terminal or fix the GUI's PATH." >&2 + fi + exit 1 +} + +ensure_bash_version diff --git a/scripts/ci/license-headers.sh b/scripts/ci/license-headers.sh index 910820b555..3d0f4d9371 100755 --- a/scripts/ci/license-headers.sh +++ b/scripts/ci/license-headers.sh @@ -18,6 +18,9 @@ set -euo pipefail +# shellcheck source-path=SCRIPTDIR +source "$(dirname "${BASH_SOURCE[0]}")/lib/init.sh" + # Parse arguments MODE="check" if [ $# -gt 0 ]; then diff --git a/scripts/ci/markdownlint.sh b/scripts/ci/markdownlint.sh index a937956e4e..aae00f2b02 100755 --- a/scripts/ci/markdownlint.sh +++ b/scripts/ci/markdownlint.sh @@ -18,6 +18,9 @@ set -euo pipefail +# shellcheck source-path=SCRIPTDIR +source "$(dirname "${BASH_SOURCE[0]}")/lib/init.sh" + MODE="check" FILES=() diff --git a/scripts/ci/python-sdk-version-sync.sh b/scripts/ci/python-sdk-version-sync.sh index fa9e3d33d1..59cf88897a 100755 --- a/scripts/ci/python-sdk-version-sync.sh +++ b/scripts/ci/python-sdk-version-sync.sh @@ -18,6 +18,9 @@ set -euo pipefail +# shellcheck source-path=SCRIPTDIR +source "$(dirname "${BASH_SOURCE[0]}")/lib/init.sh" + # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' diff --git a/scripts/ci/shellcheck.sh b/scripts/ci/shellcheck.sh index 131cd30201..1f8477ed9c 100755 --- a/scripts/ci/shellcheck.sh +++ b/scripts/ci/shellcheck.sh @@ -18,6 +18,9 @@ set -euo pipefail +# shellcheck source-path=SCRIPTDIR +source "$(dirname "${BASH_SOURCE[0]}")/lib/init.sh" + MODE="check" FILES=() diff --git a/scripts/ci/skills.sh b/scripts/ci/skills.sh index 49787fd463..6399d15ec5 100755 --- a/scripts/ci/skills.sh +++ b/scripts/ci/skills.sh @@ -23,6 +23,9 @@ set -euo pipefail +# shellcheck source-path=SCRIPTDIR +source "$(dirname "${BASH_SOURCE[0]}")/lib/init.sh" + ROOT="$(git rev-parse --show-toplevel)" cd "$ROOT" diff --git a/scripts/ci/sync-python-interpreter-version.sh b/scripts/ci/sync-python-interpreter-version.sh index 2c33e475fc..cc3ced1a36 100755 --- a/scripts/ci/sync-python-interpreter-version.sh +++ b/scripts/ci/sync-python-interpreter-version.sh @@ -18,6 +18,9 @@ set -euo pipefail +# shellcheck source-path=SCRIPTDIR +source "$(dirname "${BASH_SOURCE[0]}")/lib/init.sh" + # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' diff --git a/scripts/ci/sync-rustc-version.sh b/scripts/ci/sync-rustc-version.sh index 6e317df6be..8cf0709a46 100755 --- a/scripts/ci/sync-rustc-version.sh +++ b/scripts/ci/sync-rustc-version.sh @@ -18,6 +18,9 @@ set -euo pipefail +# shellcheck source-path=SCRIPTDIR +source "$(dirname "${BASH_SOURCE[0]}")/lib/init.sh" + # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' @@ -79,7 +82,7 @@ fi # Strip trailing ".0" -> e.g., 1.89.0 -> 1.89 (no change if it doesn't end in .0) RUST_VERSION_SHORT=$(echo "$RUST_VERSION" | sed -E 's/^([0-9]+)\.([0-9]+)\.0$/\1.\2/') RUST_IMAGE_VARIANT="slim-trixie" -RUST_IMAGE_PATTERN="FROM[[:space:]].*\\brust:" +RUST_IMAGE_PATTERN="FROM[[:space:]]+(.*[^[:alnum:]_])?rust:" RUST_IMAGE_TAG_PATTERN="(rust:[^-[:space:]]+)" echo "Rust version from rust-toolchain.toml: ${GREEN}$RUST_VERSION${NC} (using ${GREEN}$RUST_VERSION_SHORT${NC} for Dockerfiles)" @@ -108,9 +111,9 @@ for dockerfile in $DOCKERFILES; do SOURCE="arg" CURRENT_VERSION=$(grep "^ARG RUST_VERSION=" "$dockerfile" | head -1 | sed 's/^ARG RUST_VERSION=//') EXPECTED_VERSION="$RUST_VERSION_SHORT" - elif grep -qE "FROM[[:space:]].*\brust:[0-9]" "$dockerfile" 2>/dev/null; then + elif grep -qE "${RUST_IMAGE_PATTERN}[0-9]" "$dockerfile" 2>/dev/null; then SOURCE="from" - CURRENT_VERSION=$(grep -E "FROM[[:space:]].*\brust:[0-9]" "$dockerfile" | head -1 | sed -nE 's/.*\brust:([0-9]+\.[0-9]+(\.[0-9]+)?).*/\1/p') + CURRENT_VERSION=$(grep -E "${RUST_IMAGE_PATTERN}[0-9]" "$dockerfile" | head -1 | sed -nE 's/.*rust:([0-9]+\.[0-9]+(\.[0-9]+)?).*/\1/p') # Preserve the file's precision: full patch (1.96.0) or short (1.96). if [[ "$CURRENT_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then EXPECTED_VERSION="$RUST_VERSION" @@ -159,7 +162,7 @@ for dockerfile in $DOCKERFILES; do sed -i.bak "s/^ARG RUST_VERSION=.*/ARG RUST_VERSION=$EXPECTED_VERSION/" "$dockerfile" rm -f "$dockerfile.bak" elif [ -n "$SOURCE" ] && [ "$CURRENT_VERSION" != "$EXPECTED_VERSION" ]; then - sed -i.bak -E "/FROM[[:space:]].*\\brust:[0-9]/ s#(\\brust:)[0-9]+\\.[0-9]+(\\.[0-9]+)?#\\1$EXPECTED_VERSION#g" "$dockerfile" + sed -i.bak -E "/${RUST_IMAGE_PATTERN}[0-9]/ s#(rust:)[0-9]+\\.[0-9]+(\\.[0-9]+)?#\\1$EXPECTED_VERSION#g" "$dockerfile" rm -f "$dockerfile.bak" fi if [ "$RUST_IMAGE_MISMATCH" = "true" ]; then diff --git a/scripts/ci/taplo.sh b/scripts/ci/taplo.sh index 1568d17d93..792be4feed 100755 --- a/scripts/ci/taplo.sh +++ b/scripts/ci/taplo.sh @@ -18,6 +18,9 @@ set -euo pipefail +# shellcheck source-path=SCRIPTDIR +source "$(dirname "${BASH_SOURCE[0]}")/lib/init.sh" + # Default values MODE="check" FILE_MODE="all" diff --git a/scripts/ci/trailing-newline.sh b/scripts/ci/trailing-newline.sh index 55916b1bb7..a2e928b1af 100755 --- a/scripts/ci/trailing-newline.sh +++ b/scripts/ci/trailing-newline.sh @@ -18,6 +18,9 @@ set -euo pipefail +# shellcheck source-path=SCRIPTDIR +source "$(dirname "${BASH_SOURCE[0]}")/lib/init.sh" + # Default values MODE="check" FILE_MODE="all" diff --git a/scripts/ci/trailing-whitespace.sh b/scripts/ci/trailing-whitespace.sh index b3c481dc5c..e4d8299e54 100755 --- a/scripts/ci/trailing-whitespace.sh +++ b/scripts/ci/trailing-whitespace.sh @@ -18,6 +18,9 @@ set -euo pipefail +# shellcheck source-path=SCRIPTDIR +source "$(dirname "${BASH_SOURCE[0]}")/lib/init.sh" + # Default values MODE="check" FILE_MODE="all" diff --git a/scripts/ci/uv-lock-check.sh b/scripts/ci/uv-lock-check.sh index de76ad7bfe..6e633249a1 100755 --- a/scripts/ci/uv-lock-check.sh +++ b/scripts/ci/uv-lock-check.sh @@ -18,6 +18,9 @@ set -euo pipefail +# shellcheck source-path=SCRIPTDIR +source "$(dirname "${BASH_SOURCE[0]}")/lib/init.sh" + RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' diff --git a/scripts/copy-latest-from-master.sh b/scripts/copy-latest-from-master.sh index 35665cf58f..37d817a9be 100755 --- a/scripts/copy-latest-from-master.sh +++ b/scripts/copy-latest-from-master.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/dashboard/build_release.sh b/scripts/dashboard/build_release.sh index 7f46d6fa50..e06b9b46a7 100755 --- a/scripts/dashboard/build_release.sh +++ b/scripts/dashboard/build_release.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/dashboard/run_dev.sh b/scripts/dashboard/run_dev.sh index 5fb8c83acb..6cbd365283 100755 --- a/scripts/dashboard/run_dev.sh +++ b/scripts/dashboard/run_dev.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/extract-version.sh b/scripts/extract-version.sh index 6253db9342..020cb56f46 100755 --- a/scripts/extract-version.sh +++ b/scripts/extract-version.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -61,6 +61,9 @@ set -euo pipefail +# shellcheck source-path=SCRIPTDIR +source "$(dirname "${BASH_SOURCE[0]}")/ci/lib/init.sh" + # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' diff --git a/scripts/performance/run-standard-performance-suite.sh b/scripts/performance/run-standard-performance-suite.sh index 31d2e2aea1..f99635ff81 100755 --- a/scripts/performance/run-standard-performance-suite.sh +++ b/scripts/performance/run-standard-performance-suite.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/performance/utils.sh b/scripts/performance/utils.sh index ec790fb19e..b8513da2a4 100755 --- a/scripts/performance/utils.sh +++ b/scripts/performance/utils.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/profile.sh b/scripts/profile.sh index dad410ba9f..b6b7b56683 100755 --- a/scripts/profile.sh +++ b/scripts/profile.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/run-bdd-tests.sh b/scripts/run-bdd-tests.sh index 37d47315ab..272f763061 100755 --- a/scripts/run-bdd-tests.sh +++ b/scripts/run-bdd-tests.sh @@ -19,12 +19,10 @@ set -Eeuo pipefail COVERAGE=0 -VSR=0 ARGS=() for arg in "$@"; do case "$arg" in --coverage) COVERAGE=1 ;; - --vsr) VSR=1 ;; *) ARGS+=("$arg") ;; esac done @@ -35,34 +33,21 @@ FEATURE="${ARGS[1]:-all}" log(){ printf "%b\n" "$*"; } usage(){ - log "Usage: $0 [--coverage] [--vsr] [feature]" + log "Usage: $0 [--coverage] [feature]" log "" log " sdk: rust | python | php | go | go-race | node | csharp | java | cpp | all | clean (default: all)" log " feature: basic_messaging | leader_redirection | raw_command | all (default: all)" - # TODO: change to iggy-server once legacy server is removed (core/server has VSR support) - log " --vsr: run against iggy-server-ng built with --features vsr (rust, python, go, node);" - log " expects IGGY_SERVER_NG_PATH (default: target/debug/iggy-server-ng)" - log " and a vsr-built iggy CLI at IGGY_CLI_PATH." - log " The go suites imply it: the Go SDK speaks only the VSR protocol." + log "" + log " Every suite runs against iggy-server, taken from IGGY_SERVER_PATH" + log " (default: target/debug/iggy-server) with an iggy CLI at IGGY_CLI_PATH." log "" log "Examples:" log " $0 rust # run all features for Rust" log " $0 rust basic_messaging # run only basic_messaging for Rust" log " $0 all leader_redirection # run leader_redirection for all supporting SDKs" - log " $0 --vsr rust # run all features for Rust against server-ng" log " $0 --coverage go basic_messaging" } -if [ "$VSR" = "1" ]; then - case "$SDK" in - rust|python|go|go-race|node|clean) ;; - *) - log "❌ --vsr supports only the Rust, Python, Go, and Node SDKs so far" - usage - exit 2 ;; - esac -fi - case "$FEATURE" in basic_messaging|leader_redirection|raw_command|all) ;; *) @@ -80,7 +65,6 @@ ALL_COMPOSE_FILES=( -f docker-compose.server.yml -f docker-compose.cluster.yml -f docker-compose.coverage.yml - -f docker-compose.vsr.yml ) COMPOSE_FILES=(-f docker-compose.yml) @@ -96,11 +80,6 @@ if [ "$COVERAGE" = "1" ]; then COMPOSE_FILES+=(-f docker-compose.coverage.yml) mkdir -p ../reports fi -# vsr overrides must come last to win over the server/cluster/coverage files. -if [ "$VSR" = "1" ]; then - COMPOSE_FILES+=(-f docker-compose.vsr.yml) - export BDD_RUST_FEATURES="bdd,vsr" -fi cleanup(){ log "🧹 cleaning up containers & volumes…" @@ -110,10 +89,7 @@ trap cleanup EXIT INT TERM log "🧪 Running BDD tests for SDK: ${SDK}" log "📁 Feature file: ${FEATURE}" -if [ "$VSR" = "1" ] || [ "$SDK" = "go" ] || [ "$SDK" = "go-race" ]; then - # TODO: change to iggy-server once legacy server is removed (core/server has VSR support) - log "🗳️ Server: iggy-server-ng (--features vsr)" -fi +log "🗳️ Server: iggy-server" if [ "$COVERAGE" = "1" ]; then log "📊 Coverage collection enabled → reports will be in ./reports/" fi @@ -135,20 +111,12 @@ run_suite(){ esac fi - # The Go SDK speaks only the VSR wire protocol, so its suites always run - # against the VSR server even when the caller did not ask for it. Each suite - # tears its own stack down, so an `all` run can mix the two servers. - local files=("${COMPOSE_FILES[@]}") - if [ "$svc" = "go-bdd" ] && [ "$VSR" != "1" ]; then - files+=(-f docker-compose.vsr.yml) - fi - log "${emoji} ${label}..." local code=0 - docker compose "${files[@]}" \ + docker compose "${COMPOSE_FILES[@]}" \ up --build --exit-code-from "$svc" "$svc" \ || code=$? - docker compose "${files[@]}" \ + docker compose "${COMPOSE_FILES[@]}" \ down -v --remove-orphans >/dev/null 2>&1 || true return "$code" } diff --git a/scripts/run-benches.sh b/scripts/run-benches.sh index 4a35491395..c8d8e30410 100755 --- a/scripts/run-benches.sh +++ b/scripts/run-benches.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/scripts/run-examples-from-readme.sh b/scripts/run-examples-from-readme.sh index 95674b99bf..3a13fb0059 100755 --- a/scripts/run-examples-from-readme.sh +++ b/scripts/run-examples-from-readme.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -179,11 +179,12 @@ run_language_examples() { # shellcheck disable=SC2329 run_rust_examples() { - resolve_server_binary "${TARGET}" + resolve_server_binary "${TARGET}" "iggy-server" resolve_cli_binary "${TARGET}" # The README documents credentials as / - # placeholders; the test server starts with iggy/iggy. + # placeholders; the test server starts with iggy/iggy. The README keeps + # plain cargo run commands, so the cross-compile target is injected here. if [ -n "${TARGET}" ]; then TRANSFORM_COMMAND() { echo "$1" | sed "s||iggy|g; s||iggy|g; s|cargo run |cargo run --target ${TARGET} |g" @@ -217,7 +218,8 @@ run_rust_examples() { # shellcheck disable=SC2329 run_node_examples() { - resolve_server_binary "${TARGET}" + # The Node SDK is vsr-only, so examples run against the vsr server. + resolve_server_binary "${TARGET}" iggy-server export DEBUG=iggy:examples unset -f TRANSFORM_COMMAND 2>/dev/null || true @@ -236,8 +238,7 @@ run_node_examples() { # shellcheck disable=SC2329 run_go_examples() { # The Go SDK speaks only the VSR wire protocol. - # TODO: change to iggy-server once legacy server is removed (core/server has VSR support) - resolve_server_binary "${TARGET}" "iggy-server-ng" "vsr" + resolve_server_binary "${TARGET}" "iggy-server" # The VSR server logs no startup line, so readiness is a connect poll. SERVER_READY_PROBE="tcp" unset -f TRANSFORM_COMMAND 2>/dev/null || true @@ -257,12 +258,10 @@ run_go_examples() { # shellcheck disable=SC2329 run_python_examples() { - # Python wheels are vsr-built, so examples run against the vsr - # server. It takes no --fresh flag; cleanup_server_state wiping - # local_data is the fresh start. - # TODO(hubcio): change to iggy-server once legacy server is removed - # (core/server has VSR support) - resolve_server_binary "${TARGET}" iggy-server-ng + # The Python SDK speaks only the VSR wire protocol, so examples run + # against the VSR server, started fresh by cleanup_server_state wiping + # local_data rather than by passing --fresh. + resolve_server_binary "${TARGET}" iggy-server unset -f TRANSFORM_COMMAND 2>/dev/null || true echo "" @@ -311,7 +310,10 @@ run_python_examples() { # shellcheck disable=SC2329 run_php_examples() { - resolve_server_binary "${TARGET}" + # The PHP extension speaks only the VSR wire protocol, so examples run + # against the VSR server, started fresh by cleanup_server_state wiping + # local_data rather than by passing --fresh. + resolve_server_binary "${TARGET}" iggy-server local php_bin="${PHP:-php}" if [ -z "${PHP_IGGY_EXTENSION:-}" ]; then @@ -347,12 +349,14 @@ run_php_examples() { "" \ "" \ 0 \ - "--fresh" + "" } # shellcheck disable=SC2329 run_java_examples() { - resolve_server_binary "${TARGET}" + # Java examples run against the VSR server. + resolve_server_binary "${TARGET}" "iggy-server" + SERVER_READY_PATTERN="client listeners started" unset -f TRANSFORM_COMMAND 2>/dev/null || true run_language_examples \ @@ -368,7 +372,10 @@ run_java_examples() { # shellcheck disable=SC2329 run_csharp_examples() { - resolve_server_binary "${TARGET}" + # The .NET SDK speaks only the VSR wire protocol, so examples run against + # the VSR server, started fresh by cleanup_server_state wiping local_data + # rather than by passing --fresh. + resolve_server_binary "${TARGET}" iggy-server unset -f TRANSFORM_COMMAND 2>/dev/null || true run_language_examples \ @@ -394,6 +401,7 @@ run_one() { EXAMPLES_EXIT_CODE=0 unset -f TRANSFORM_COMMAND 2>/dev/null || true + unset SERVER_READY_PATTERN 2>/dev/null || true set +e ${lang_fn} @@ -401,6 +409,7 @@ run_one() { set -e unset -f TRANSFORM_COMMAND 2>/dev/null || true + unset SERVER_READY_PATTERN 2>/dev/null || true if [ ${rc} -ne 0 ]; then echo "" diff --git a/scripts/utils.sh b/scripts/utils.sh index fb84c8b6a2..0c77704877 100755 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -159,14 +159,11 @@ readonly EXAMPLES_SERVER_TIMEOUT=300 readonly EXAMPLES_STOP_TIMEOUT=5 # Resolve and validate the server binary path. -# Usage: resolve_server_binary [target] [binary_name] [cargo_features] -# Sets global SERVER_BIN. binary_name defaults to iggy-server; vsr lanes -# pass their server binary and the vsr feature (so the wire protocol -# matches vsr-built clients). +# Usage: resolve_server_binary [target] [binary_name] +# Sets global SERVER_BIN. binary_name defaults to iggy-server. function resolve_server_binary() { local target="${1:-}" local binary="${2:-iggy-server}" - local features="${3:-}" if [ -n "${target}" ]; then SERVER_BIN="target/${target}/debug/${binary}" @@ -179,9 +176,6 @@ function resolve_server_binary() { if [ -n "${target}" ]; then build_command="cargo build --target ${target} --bin ${binary}" fi - if [ -n "${features}" ]; then - build_command="${build_command} --features ${features}" - fi echo "Error: Server binary not found at ${SERVER_BIN}" echo "Please build the server binary before running this script:" echo " ${build_command}" @@ -243,12 +237,14 @@ function start_tls_server() { # How wait_for_server_ready decides the server is up. "log" greps the startup # lines: the legacy "has started" and the VSR server "client listeners -# started" (logged once the TCP socket is bound). "tcp" polls the listener +# started" (logged once the TCP socket is bound); override via +# SERVER_READY_PATTERN or the pattern arg. "tcp" polls the listener # instead for lanes that cannot rely on a startup line. : "${SERVER_READY_PROBE:=log}" : "${SERVER_READY_ADDRESS:=127.0.0.1:8090}" # Report whether the server is accepting work. +# Usage: server_is_ready [pattern] function server_is_ready() { if [ "${SERVER_READY_PROBE}" = "tcp" ]; then local host="${SERVER_READY_ADDRESS%:*}" @@ -257,7 +253,7 @@ function server_is_ready() { exec 3<&- return 0 fi - grep -qE "has started|client listeners started" "${EXAMPLES_LOG_FILE}" + grep -qE "${1:-has started|client listeners started}" "${EXAMPLES_LOG_FILE}" } # Report whether the server this script started is still running. @@ -269,9 +265,13 @@ function server_is_alive() { } # Block until the server is ready or the timeout elapses. -# Usage: wait_for_server_ready [label] +# Usage: wait_for_server_ready [label] [pattern] +# The default log pattern matches both the legacy line ("has started") and +# the vsr server line ("client listeners started"). Override via the +# pattern arg or SERVER_READY_PATTERN. function wait_for_server_ready() { local label="${1:-Iggy}" + local pattern="${2:-${SERVER_READY_PATTERN:-has started|client listeners started}}" local elapsed=0 while true; do # Liveness is checked first so a server that died leaves its log here @@ -282,7 +282,7 @@ function wait_for_server_ready() { cat "${EXAMPLES_LOG_FILE}" exit 1 fi - if server_is_ready; then + if server_is_ready "${pattern}"; then return 0 fi if [ ${elapsed} -gt ${EXAMPLES_SERVER_TIMEOUT} ]; then From 7ddf4ae5555a71df6100b74ba67d11bd162e1884 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Wed, 12 Aug 2026 08:38:05 -0400 Subject: [PATCH 55/57] Prune removed crates from Cargo.lock Remove stale lockfile entries for the deleted Kafka gateway, Kafka message generator, and OpenSearch source connector crates, along with their now-unused transitive dependencies. --- Cargo.lock | 40 ---------------------------------------- Cargo.toml | 2 ++ 2 files changed, 2 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 185ce4f799..051b51babc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7108,26 +7108,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "iggy_connector_opensearch_source" -version = "0.4.1-edge.1" -dependencies = [ - "async-trait", - "dashmap", - "humantime", - "iggy_common", - "iggy_connector_sdk", - "once_cell", - "opensearch", - "rmp-serde", - "secrecy", - "serde", - "serde_json", - "simd-json", - "tokio", - "tracing", -] - [[package]] name = "iggy_connector_postgres_sink" version = "0.5.0-edge.1" @@ -9315,26 +9295,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "opensearch" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2af6815a23449a0860c8fe049a828c3589d3ad56d3b5875d0d1f340d1291871e" -dependencies = [ - "base64", - "bytes", - "dyn-clone", - "lazy_static", - "percent-encoding", - "reqwest 0.13.4", - "rustc_version", - "serde", - "serde_json", - "serde_with", - "url", - "void", -] - [[package]] name = "openssl-probe" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index ad813a665a..39b5a9f064 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,6 +68,8 @@ members = [ "core/system_stats", "core/tools", "examples/rust", + "gateways/kafka", + "gateways/kafka/tools/kafka-tool", ] exclude = ["foreign/cpp", "foreign/php", "foreign/python"] resolver = "3" From b2ba582a7faef7e4b504c22e0caf099d0b6ab424 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Wed, 12 Aug 2026 08:45:56 -0400 Subject: [PATCH 56/57] update Cargo.toml added a new line --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 39b5a9f064..d6c7d0c544 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -390,4 +390,4 @@ opt-level = 3 opt-level = 3 [profile.dev.package.iggy_common] -opt-level = 3 \ No newline at end of file +opt-level = 3 From 737939f7f1bd672d5bb22e124647db4f25840a72 Mon Sep 17 00:00:00 2001 From: ryerraguntla Date: Thu, 13 Aug 2026 16:42:46 -0400 Subject: [PATCH 57/57] Removed the openserch source connector Somewhere the opensource search connector wip branch got merged. This commit takes out those changes. --- .../sources/opensearch_source/Cargo.toml | 52 -- .../sources/opensearch_source/README.md | 211 ------ .../sources/opensearch_source/config.toml | 38 - .../sources/opensearch_source/src/lib.rs | 688 ------------------ .../opensearch_source/src/state_manager.rs | 388 ---------- .../tests/connectors/fixtures/mod.rs | 2 - .../fixtures/opensearch/container.rs | 349 --------- .../connectors/fixtures/opensearch/mod.rs | 23 - .../connectors/fixtures/opensearch/source.rs | 195 ----- core/integration/tests/connectors/mod.rs | 1 - .../tests/connectors/opensearch/mod.rs | 24 - .../opensearch/opensearch_source.rs | 367 ---------- .../tests/connectors/opensearch/source.toml | 20 - 13 files changed, 2358 deletions(-) delete mode 100644 core/connectors/sources/opensearch_source/Cargo.toml delete mode 100644 core/connectors/sources/opensearch_source/README.md delete mode 100644 core/connectors/sources/opensearch_source/config.toml delete mode 100644 core/connectors/sources/opensearch_source/src/lib.rs delete mode 100644 core/connectors/sources/opensearch_source/src/state_manager.rs delete mode 100644 core/integration/tests/connectors/fixtures/opensearch/container.rs delete mode 100644 core/integration/tests/connectors/fixtures/opensearch/mod.rs delete mode 100644 core/integration/tests/connectors/fixtures/opensearch/source.rs delete mode 100644 core/integration/tests/connectors/opensearch/mod.rs delete mode 100644 core/integration/tests/connectors/opensearch/opensearch_source.rs delete mode 100644 core/integration/tests/connectors/opensearch/source.toml diff --git a/core/connectors/sources/opensearch_source/Cargo.toml b/core/connectors/sources/opensearch_source/Cargo.toml deleted file mode 100644 index 5c91507112..0000000000 --- a/core/connectors/sources/opensearch_source/Cargo.toml +++ /dev/null @@ -1,52 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[package] -name = "iggy_connector_opensearch_source" -version = "0.4.1-edge.1" -description = "Iggy OpenSearch source connector" -edition = "2024" -license = "Apache-2.0" -keywords = ["iggy", "messaging", "streaming", "opensearch"] -categories = ["command-line-utilities", "database", "network-programming"] -homepage = "https://iggy.apache.org" -documentation = "https://iggy.apache.org/docs" -repository = "https://github.com/apache/iggy" -readme = "../../README.md" -publish = false - -[package.metadata.cargo-machete] -ignored = ["dashmap", "once_cell", "futures", "simd-json"] - -[lib] -crate-type = ["cdylib", "lib"] - -[dependencies] -async-trait = { workspace = true } -dashmap = { workspace = true } -humantime = { workspace = true } -iggy_common = { workspace = true } -iggy_connector_sdk = { workspace = true } -once_cell = { workspace = true } -opensearch = { workspace = true } -rmp-serde = { workspace = true } -secrecy = { workspace = true } -serde = { workspace = true } -serde_json = { workspace = true } -simd-json = { workspace = true } -tokio = { workspace = true } -tracing = { workspace = true } diff --git a/core/connectors/sources/opensearch_source/README.md b/core/connectors/sources/opensearch_source/README.md deleted file mode 100644 index 466b20f7b4..0000000000 --- a/core/connectors/sources/opensearch_source/README.md +++ /dev/null @@ -1,211 +0,0 @@ -# OpenSearch Source Connector with State Management - -This OpenSearch source connector provides comprehensive state management capabilities to track processing progress and enable fault-tolerant data ingestion. - -## Features - -- **Incremental Data Processing**: Track last processed timestamp to avoid reprocessing data -- **Cursor-based Pagination**: Support for document ID-based cursors -- **Scroll-based Pagination**: Support for OpenSearch scroll API -- **Error Tracking**: Monitor error counts and last error messages -- **Processing Statistics**: Track performance metrics and processing times -- **Persistent State Storage**: Multiple storage backends (file, OpenSearch, Redis) -- **Auto-save**: Configurable automatic state persistence -- **State Recovery**: Resume processing from last known position after restart - -## Configuration - -### Basic Configuration - -```toml -type = "source" -key = "opensearch" -enabled = true -version = 0 -name = "OpenSearch source" -path = "target/release/libiggy_connector_opensearch_source" - -[[streams]] -stream = "opensearch_stream" -topic = "documents" -schema = "json" -batch_length = 100 -linger_time = "5ms" - -[plugin_config] -url = "http://localhost:9200" -index = "logs-*" -polling_interval = "30s" -batch_size = 100 -timestamp_field = "@timestamp" -query = { - "match_all": {} -} -``` - -### State Management Configuration - -```toml -[plugin_config] -# ... basic config ... -state = { - enabled = true - storage_type = "file" # "file", "opensearch", "redis" - storage_config = { - base_path = "./connector_states" # for file storage - # index = "connector_states" # for opensearch storage - # url = "redis://localhost:6379" # for redis storage - } - state_id = "opensearch_logs_connector" - auto_save_interval = "5m" - tracked_fields = [ - "last_poll_timestamp", - "last_document_id", - "total_documents_fetched" - ] -} -``` - -## State Information - -The connector tracks the following state information: - -### Processing State - -- `last_poll_timestamp`: Last successful poll timestamp -- `total_documents_fetched`: Total number of documents processed -- `poll_count`: Number of polling cycles executed -- `last_document_id`: Last processed document ID (for cursor pagination) -- `last_scroll_id`: Last scroll ID (for scroll pagination) -- `last_offset`: Last processed offset - -### Error Tracking - -- `error_count`: Total number of errors encountered -- `last_error`: Last error message - -### Performance Statistics - -- `total_bytes_processed`: Total bytes processed -- `avg_batch_processing_time_ms`: Average processing time per batch -- `last_successful_poll`: Timestamp of last successful poll -- `empty_polls_count`: Number of polls that returned no documents -- `successful_polls_count`: Number of successful polls - -## Storage Backends - -### File Storage (Default) - -```toml -state = { - enabled = true - storage_type = "file" - storage_config = { - base_path = "./connector_states" - } -} -``` - -### OpenSearch Storage - -```toml -state = { - enabled = true - storage_type = "opensearch" - storage_config = { - index = "connector_states" - url = "http://localhost:9200" - } -} -``` - -### Redis Storage - -```toml -state = { - enabled = true - storage_type = "redis" - storage_config = { - url = "redis://localhost:6379" - key_prefix = "connector_states:" - } -} -``` - -## State File Format - -State files are stored as JSON with the following structure: - -```json -{ - "id": "opensearch_logs_connector", - "last_updated": "2024-01-15T10:30:00Z", - "version": 1, - "data": { - "last_poll_timestamp": "2024-01-15T10:30:00Z", - "total_documents_fetched": 15000, - "poll_count": 150, - "last_document_id": "doc_12345", - "last_scroll_id": "scroll_abc123", - "last_offset": 15000, - "error_count": 2, - "last_error": "Connection timeout", - "processing_stats": { - "total_bytes_processed": 1048576, - "avg_batch_processing_time_ms": 125.5, - "last_successful_poll": "2024-01-15T10:30:00Z", - "empty_polls_count": 5, - "successful_polls_count": 145 - } - }, - "metadata": { - "connector_type": "opensearch_source", - "connector_id": 1, - "index": "logs-*", - "url": "http://localhost:9200" - } -} -``` - -## Best Practices - -1. **State ID Uniqueness**: Use unique state IDs for different connector instances -2. **Auto-save Interval**: Set appropriate auto-save intervals based on your data volume -3. **Storage Location**: Use persistent storage locations for production deployments -4. **State Cleanup**: Regularly clean up old state files to prevent disk space issues -5. **Error Handling**: Monitor error counts and implement appropriate alerting -6. **Backup**: Regularly backup state files for disaster recovery - -## Troubleshooting - -### Common Issues - -1. **State Not Loading**: Check file permissions and storage path -2. **State Corruption**: Delete corrupted state files to start fresh -3. **Performance Issues**: Adjust auto-save interval and batch sizes -4. **Storage Full**: Implement state cleanup policies - -### Monitoring - -Monitor the following metrics: - -- State save/load success rates -- Processing statistics -- Error counts and types -- Storage usage for state files - -## Migration - -To migrate from a connector without state management: - -1. Add state configuration to your connector config -2. Set `enabled = true` in state config -3. Restart the connector -4. The connector will start tracking state from the next poll cycle - -To migrate between storage backends: - -1. Export state from current storage -2. Update storage configuration -3. Import state to new storage -4. Restart connector diff --git a/core/connectors/sources/opensearch_source/config.toml b/core/connectors/sources/opensearch_source/config.toml deleted file mode 100644 index dceed1b064..0000000000 --- a/core/connectors/sources/opensearch_source/config.toml +++ /dev/null @@ -1,38 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -type = "source" -key = "opensearch" -enabled = true -version = 0 -name = "OpenSearch source" -path = "../../target/release/libiggy_connector_opensearch_source" -plugin_config_format = "json" - -[[streams]] -stream = "test_stream" -topic = "test_topic" -schema = "json" -batch_length = 1000 -linger_time = "5ms" - -[plugin_config] -url = "http://localhost:9200" -index = "test_documents" -polling_interval = "100ms" -batch_size = 100 -timestamp_field = "timestamp" diff --git a/core/connectors/sources/opensearch_source/src/lib.rs b/core/connectors/sources/opensearch_source/src/lib.rs deleted file mode 100644 index 498d0bf386..0000000000 --- a/core/connectors/sources/opensearch_source/src/lib.rs +++ /dev/null @@ -1,688 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -use async_trait::async_trait; -use iggy_common::{DateTime, Utc}; -use iggy_connector_sdk::{ - ConnectorState, Error, ProducedMessage, ProducedMessages, Schema, Source, source_connector, -}; -use opensearch::{ - OpenSearch, SearchParts, - auth::Credentials, - http::{Url, transport::TransportBuilder}, -}; -use secrecy::{ExposeSecret, SecretString}; -use serde::{Deserialize, Serialize}; -use serde_json::{Value, json}; -use std::str::FromStr; -use std::sync::Arc; -use std::time::Duration; -use tokio::{sync::Mutex, time::sleep}; -use tracing::{info, warn}; - -mod state_manager; -use crate::state_manager::{FileStateStorage, SourceState, StateStorage}; -pub use state_manager::{StateInfo, StateManager, StateStats}; - -source_connector!(OpenSearchSource); - -const CONNECTOR_NAME: &str = "OpenSearch source"; - -#[derive(Debug, Clone, Serialize, Deserialize)] -struct State { - last_poll_timestamp: Option>, - total_documents_fetched: usize, - poll_count: usize, - /// Last document ID processed (for cursor-based pagination) - last_document_id: Option, - /// Last scroll ID (for scroll-based pagination) - last_scroll_id: Option, - /// Last processed offset - last_offset: Option, - /// Error count and last error - error_count: usize, - last_error: Option, - /// Processing statistics - processing_stats: ProcessingStats, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -struct ProcessingStats { - /// Total bytes processed - total_bytes_processed: u64, - /// Average processing time per batch - avg_batch_processing_time_ms: f64, - /// Last successful processing timestamp - last_successful_poll: Option>, - /// Number of empty polls - empty_polls_count: usize, - /// Number of successful polls - successful_polls_count: usize, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct StateConfig { - /// Enable state persistence - pub enabled: bool, - /// State storage type: "file", "opensearch", "redis", etc. - pub storage_type: Option, - /// State storage configuration (depends on storage_type) - pub storage_config: Option, - /// State ID for this connector instance - pub state_id: Option, - /// Auto-save state interval (e.g., "30s", "5m") - pub auto_save_interval: Option, - /// Fields to track in state (e.g., ["last_timestamp", "last_document_id"]) - pub tracked_fields: Option>, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct OpenSearchSourceConfig { - pub url: String, - pub index: String, - pub username: Option, - #[serde(serialize_with = "iggy_common::serde_secret::serialize_optional_secret")] - pub password: Option, - pub query: Option, - pub polling_interval: Option, - pub batch_size: Option, - pub timestamp_field: Option, - pub scroll_timeout: Option, - pub state: Option, -} - -#[derive(Debug)] -pub struct OpenSearchSource { - id: u32, - config: OpenSearchSourceConfig, - client: Option, - polling_interval: Duration, - state: Mutex, -} - -impl OpenSearchSource { - pub fn new(id: u32, config: OpenSearchSourceConfig, state: Option) -> Self { - let polling_interval = config - .polling_interval - .as_deref() - .unwrap_or("10s") - .parse::() - .unwrap_or_else(|_| humantime::Duration::from_str("10s").unwrap()) - .into(); - - let restored_state = state - .and_then(|s| s.deserialize::(CONNECTOR_NAME, id)) - .inspect(|s| { - info!( - "Restored state for {CONNECTOR_NAME} connector with ID: {id}. \ - Documents fetched: {}, poll count: {}", - s.total_documents_fetched, s.poll_count - ); - }); - - OpenSearchSource { - id, - config, - client: None, - polling_interval, - state: Mutex::new(restored_state.unwrap_or(State { - last_poll_timestamp: None, - total_documents_fetched: 0, - poll_count: 0, - last_document_id: None, - last_scroll_id: None, - last_offset: None, - error_count: 0, - last_error: None, - processing_stats: ProcessingStats { - total_bytes_processed: 0, - avg_batch_processing_time_ms: 0.0, - last_successful_poll: None, - empty_polls_count: 0, - successful_polls_count: 0, - }, - })), - } - } - - fn serialize_state(&self, state: &State) -> Option { - ConnectorState::serialize(state, CONNECTOR_NAME, self.id) - } - - /// Create state storage based on configuration - fn create_state_storage(&self) -> Option> { - let state_config = self.config.state.as_ref()?; - if !state_config.enabled { - return None; - } - - match state_config.storage_type.as_deref() { - Some("file") | None => { - let base_path = state_config - .storage_config - .as_ref() - .and_then(|c| c.get("base_path")) - .and_then(|p| p.as_str()) - .unwrap_or("./connector_states"); - - Some(Arc::new(FileStateStorage::new(base_path))) - } - Some("opensearch") => { - // TODO: Implement OpenSearch-based state storage - warn!("OpenSearch state storage not yet implemented, falling back to file storage"); - Some(Arc::new(FileStateStorage::new("./connector_states"))) - } - Some(storage_type) => { - warn!( - "Unknown state storage type: {}, falling back to file storage", - storage_type - ); - Some(Arc::new(FileStateStorage::new("./connector_states"))) - } - } - } - - /// Get state ID for this connector - fn get_state_id(&self) -> String { - self.config - .state - .as_ref() - .and_then(|s| s.state_id.clone()) - .unwrap_or_else(|| format!("opensearch_source_{}", self.id)) - } - - /// Convert internal state to SourceState - async fn internal_state_to_source_state(&self) -> Result { - let state = self.state.lock().await; - - let data = json!({ - "last_poll_timestamp": state.last_poll_timestamp, - "total_documents_fetched": state.total_documents_fetched, - "poll_count": state.poll_count, - "last_document_id": state.last_document_id, - "last_scroll_id": state.last_scroll_id, - "last_offset": state.last_offset, - "error_count": state.error_count, - "last_error": state.last_error, - "processing_stats": state.processing_stats, - }); - - Ok(SourceState { - id: self.get_state_id(), - last_updated: Utc::now(), - version: 1, - data, - metadata: Some(json!({ - "connector_type": "opensearch_source", - "connector_id": self.id, - "index": self.config.index, - "url": self.config.url, - })), - }) - } - - /// Convert SourceState to internal state - async fn source_state_to_internal_state( - &mut self, - source_state: SourceState, - ) -> Result<(), Error> { - let mut state = self.state.lock().await; - - if let Some(data) = source_state.data.as_object() { - if let Some(timestamp) = data.get("last_poll_timestamp") - && let Some(ts_str) = timestamp.as_str() - && let Ok(dt) = DateTime::parse_from_rfc3339(ts_str) - { - state.last_poll_timestamp = Some(dt.with_timezone(&Utc)); - } - - if let Some(count) = data.get("total_documents_fetched") - && let Some(count_val) = count.as_u64() - { - state.total_documents_fetched = count_val as usize; - } - - if let Some(count) = data.get("poll_count") - && let Some(count_val) = count.as_u64() - { - state.poll_count = count_val as usize; - } - - if let Some(doc_id) = data.get("last_document_id") { - state.last_document_id = doc_id.as_str().map(|s| s.to_string()); - } - - if let Some(scroll_id) = data.get("last_scroll_id") { - state.last_scroll_id = scroll_id.as_str().map(|s| s.to_string()); - } - - if let Some(offset) = data.get("last_offset") { - state.last_offset = offset.as_u64(); - } - - if let Some(error_count) = data.get("error_count") - && let Some(count_val) = error_count.as_u64() - { - state.error_count = count_val as usize; - } - - if let Some(last_error) = data.get("last_error") { - state.last_error = last_error.as_str().map(|s| s.to_string()); - } - - if let Some(stats) = data.get("processing_stats") - && let Ok(processing_stats) = serde_json::from_value(stats.clone()) - { - state.processing_stats = processing_stats; - } - } - - Ok(()) - } - - async fn create_client(&self) -> Result { - let url = Url::parse(&self.config.url) - .map_err(|error| Error::Storage(format!("Invalid OpenSearch URL: {error}")))?; - - let conn_pool = opensearch::http::transport::SingleNodeConnectionPool::new(url); - let mut transport_builder = TransportBuilder::new(conn_pool); - - if let (Some(username), Some(password)) = (&self.config.username, &self.config.password) { - let credentials = - Credentials::Basic(username.clone(), password.expose_secret().to_string()); - transport_builder = transport_builder.auth(credentials); - } - - let transport = transport_builder - .build() - .map_err(|e| Error::Storage(format!("Failed to build transport: {}", e)))?; - - Ok(OpenSearch::new(transport)) - } - - async fn search_documents(&self, client: &OpenSearch) -> Result, Error> { - let state = self.state.lock().await; - let batch_size = self.config.batch_size.unwrap_or(100); - - // Build query based on timestamp field if configured - let mut query = self.config.query.clone().unwrap_or_else(|| { - json!({ - "match_all": {} - }) - }); - - // Add timestamp filter for incremental polling - if let Some(timestamp_field) = &self.config.timestamp_field - && let Some(last_timestamp) = state.last_poll_timestamp - { - query = json!({ - "bool": { - "must": [ - query, - { - "range": { - timestamp_field: { - "gt": last_timestamp.to_rfc3339() - } - } - } - ] - } - }); - } - - let search_body = json!({ - "query": query, - "size": batch_size, - "sort": [ - { - self.config.timestamp_field.as_deref().unwrap_or("@timestamp"): { - "order": "asc" - } - } - ] - }); - - drop(state); - - let response = client - .search(SearchParts::Index(&[&self.config.index])) - .body(search_body) - .send() - .await - .map_err(|e| Error::Storage(format!("Failed to execute search: {}", e)))?; - - if !response.status_code().is_success() { - let error_text = response - .text() - .await - .unwrap_or_else(|_| "Unknown error".to_string()); - return Err(Error::Storage(format!( - "Search request failed: {}", - error_text - ))); - } - - let response_body: Value = response - .json() - .await - .map_err(|e| Error::Storage(format!("Failed to parse search response: {}", e)))?; - - let mut messages = Vec::new(); - let mut latest_timestamp = None; - - if let Some(hits) = response_body - .get("hits") - .and_then(|h| h.get("hits")) - .and_then(|h| h.as_array()) - { - for hit in hits { - if let Some(source) = hit.get("_source") { - // Extract timestamp for incremental polling - if let Some(timestamp_field) = &self.config.timestamp_field - && let Some(timestamp_str) = - source.get(timestamp_field).and_then(|v| v.as_str()) - && let Ok(timestamp) = DateTime::parse_from_rfc3339(timestamp_str) - { - let timestamp_utc = timestamp.with_timezone(&Utc); - if latest_timestamp.is_none() || timestamp_utc > latest_timestamp.unwrap() { - latest_timestamp = Some(timestamp_utc); - } - } - - // Create message from document - let payload = serde_json::to_vec(source).map_err(|e| { - Error::Serialization(format!("Failed to serialize document: {}", e)) - })?; - - let message = ProducedMessage { - id: None, - headers: None, - checksum: None, - timestamp: None, - origin_timestamp: None, - payload, - }; - messages.push(message); - } - } - } - - // Update state - let mut state = self.state.lock().await; - state.total_documents_fetched += messages.len(); - state.poll_count += 1; - if let Some(timestamp) = latest_timestamp { - state.last_poll_timestamp = Some(timestamp); - } - - Ok(messages) - } -} - -#[async_trait] -impl Source for OpenSearchSource { - async fn open(&mut self) -> Result<(), Error> { - info!( - "Opening OpenSearch source connector with ID: {} for URL: {}, index: {}", - self.id, self.config.url, self.config.index - ); - - let client = self.create_client().await?; - - // Test connection by checking if index exists - let response = client - .indices() - .exists(opensearch::indices::IndicesExistsParts::Index(&[&self - .config - .index])) - .send() - .await - .map_err(|e| Error::Storage(format!("Failed to check index existence: {}", e)))?; - - if !response.status_code().is_success() { - return Err(Error::Storage(format!( - "Index '{}' does not exist or is not accessible", - self.config.index - ))); - } - - self.client = Some(client); - - // Load state if state management is enabled - if self - .config - .state - .as_ref() - .map(|s| s.enabled) - .unwrap_or(false) - && let Err(e) = self.load_state().await - { - warn!( - "Failed to load state for OpenSearch source connector with ID: {}: {}", - self.id, e - ); - } - - info!( - "Successfully opened OpenSearch source connector with ID: {}", - self.id - ); - Ok(()) - } - - async fn poll(&self) -> Result { - let start_time = std::time::Instant::now(); - - sleep(self.polling_interval).await; - - let client = self - .client - .as_ref() - .ok_or_else(|| Error::Storage("OpenSearch client not initialized".to_string()))?; - - let messages = match self.search_documents(client).await { - Ok(msgs) => { - // Update success statistics - let mut state = self.state.lock().await; - state.processing_stats.successful_polls_count += 1; - state.processing_stats.last_successful_poll = Some(Utc::now()); - - let processing_time = start_time.elapsed().as_millis() as f64; - let total_polls = state.processing_stats.successful_polls_count - + state.processing_stats.empty_polls_count; - state.processing_stats.avg_batch_processing_time_ms = - (state.processing_stats.avg_batch_processing_time_ms - * (total_polls - 1) as f64 - + processing_time) - / total_polls as f64; - - if msgs.is_empty() { - state.processing_stats.empty_polls_count += 1; - } - - drop(state); - msgs - } - Err(e) => { - // Update error statistics - let mut state = self.state.lock().await; - state.error_count += 1; - state.last_error = Some(e.to_string()); - drop(state); - return Err(e); - } - }; - let persisted_state = { - let state = self.state.lock().await; - self.serialize_state(&state) - }; - - Ok(ProducedMessages { - schema: Schema::Json, - messages, - state: persisted_state, - }) - } - - async fn close(&mut self) -> Result<(), Error> { - let state = self.state.lock().await; - info!( - "OpenSearch source connector with ID: {} is closing. Stats: {} total documents fetched, {} polls executed, {} errors", - self.id, state.total_documents_fetched, state.poll_count, state.error_count - ); - drop(state); - - // Save final state if state management is enabled - if self - .config - .state - .as_ref() - .map(|s| s.enabled) - .unwrap_or(false) - && let Err(e) = self.save_state().await - { - warn!( - "Failed to save final state for OpenSearch source connector with ID: {}: {}", - self.id, e - ); - } - - self.client = None; - info!( - "OpenSearch source connector with ID: {} is closed.", - self.id - ); - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn test_config() -> OpenSearchSourceConfig { - OpenSearchSourceConfig { - url: "http://localhost:9200".to_string(), - index: "test_documents".to_string(), - username: None, - password: None, - query: None, - polling_interval: Some("100ms".to_string()), - batch_size: Some(10), - timestamp_field: Some("timestamp".to_string()), - scroll_timeout: None, - state: None, - } - } - - fn test_state() -> State { - State { - last_poll_timestamp: None, - total_documents_fetched: 500, - poll_count: 5, - last_document_id: Some("doc_42".to_string()), - last_scroll_id: None, - last_offset: Some(500), - error_count: 1, - last_error: Some("connection reset".to_string()), - processing_stats: ProcessingStats { - total_bytes_processed: 1024, - avg_batch_processing_time_ms: 12.5, - last_successful_poll: None, - empty_polls_count: 2, - successful_polls_count: 5, - }, - } - } - - #[test] - fn given_persisted_state_should_restore_total_documents_fetched() { - let state = test_state(); - let serialized = rmp_serde::to_vec(&state).expect("Failed to serialize state"); - let connector_state = ConnectorState(serialized); - - let source = OpenSearchSource::new(1, test_config(), Some(connector_state)); - - let runtime = tokio::runtime::Runtime::new().unwrap(); - runtime.block_on(async { - let restored = source.state.lock().await; - assert_eq!(restored.total_documents_fetched, 500); - assert_eq!(restored.poll_count, 5); - assert_eq!(restored.last_document_id, Some("doc_42".to_string())); - }); - } - - #[test] - fn given_no_state_should_start_fresh() { - let source = OpenSearchSource::new(1, test_config(), None); - - let runtime = tokio::runtime::Runtime::new().unwrap(); - runtime.block_on(async { - let state = source.state.lock().await; - assert_eq!(state.total_documents_fetched, 0); - assert_eq!(state.poll_count, 0); - assert_eq!(state.last_document_id, None); - }); - } - - #[test] - fn given_invalid_state_should_start_fresh() { - let invalid_state = ConnectorState(b"not valid msgpack".to_vec()); - let source = OpenSearchSource::new(1, test_config(), Some(invalid_state)); - - let runtime = tokio::runtime::Runtime::new().unwrap(); - runtime.block_on(async { - let state = source.state.lock().await; - assert_eq!(state.total_documents_fetched, 0); - assert_eq!(state.poll_count, 0); - }); - } - - #[test] - fn state_should_be_serializable_and_deserializable() { - let original = test_state(); - - let serialized = rmp_serde::to_vec(&original).expect("Failed to serialize"); - let deserialized: State = - rmp_serde::from_slice(&serialized).expect("Failed to deserialize"); - - assert_eq!( - original.total_documents_fetched, - deserialized.total_documents_fetched - ); - assert_eq!(original.poll_count, deserialized.poll_count); - assert_eq!(original.last_document_id, deserialized.last_document_id); - assert_eq!(original.error_count, deserialized.error_count); - } - - #[test] - fn serialize_state_helper_should_produce_valid_connector_state() { - let source = OpenSearchSource::new(1, test_config(), None); - let state = test_state(); - - let connector_state = source.serialize_state(&state); - assert!(connector_state.is_some()); - - let restored: State = connector_state - .unwrap() - .deserialize(CONNECTOR_NAME, 1) - .expect("Failed to deserialize state"); - assert_eq!(restored.total_documents_fetched, 500); - } -} diff --git a/core/connectors/sources/opensearch_source/src/state_manager.rs b/core/connectors/sources/opensearch_source/src/state_manager.rs deleted file mode 100644 index c681444efb..0000000000 --- a/core/connectors/sources/opensearch_source/src/state_manager.rs +++ /dev/null @@ -1,388 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -use crate::{OpenSearchSource, StateConfig}; -use async_trait::async_trait; -use iggy_common::{ChronoDuration, DateTime, Utc}; -use iggy_connector_sdk::Error; -use serde::{Deserialize, Serialize}; -use std::str::FromStr; -use std::sync::Arc; -use tokio::time::{Duration, interval}; -use tracing::{error, info, warn}; - -impl OpenSearchSource { - async fn get_state(&self) -> Result, Error> { - if self - .config - .state - .as_ref() - .map(|s| s.enabled) - .unwrap_or(false) - { - Ok(Some(self.internal_state_to_source_state().await?)) - } else { - Ok(None) - } - } - - pub(super) async fn save_state(&self) -> Result<(), Error> { - if !self - .config - .state - .as_ref() - .map(|s| s.enabled) - .unwrap_or(false) - { - return Ok(()); - } - - let storage = self - .create_state_storage() - .ok_or_else(|| Error::Storage("State storage not configured".to_string()))?; - - let source_state = self.internal_state_to_source_state().await?; - storage.save_source_state(&source_state).await?; - - info!( - "Saved state for OpenSearch source connector with ID: {}", - self.id - ); - Ok(()) - } - - pub(super) async fn load_state(&mut self) -> Result<(), Error> { - if !self - .config - .state - .as_ref() - .map(|s| s.enabled) - .unwrap_or(false) - { - return Ok(()); - } - - let storage = self - .create_state_storage() - .ok_or_else(|| Error::Storage("State storage not configured".to_string()))?; - - let state_id = self.get_state_id(); - if let Some(source_state) = storage.load_source_state(&state_id).await? { - self.source_state_to_internal_state(source_state).await?; - - let state = self.state.lock().await; - info!( - "Loaded state for OpenSearch source connector with ID: {} - last poll: {:?}, total docs: {}, polls: {}", - self.id, state.last_poll_timestamp, state.total_documents_fetched, state.poll_count - ); - } else { - info!( - "No existing state found for OpenSearch source connector with ID: {}, starting fresh", - self.id - ); - } - - Ok(()) - } -} - -/// State manager for OpenSearch source connector -pub struct StateManager { - storage: Arc, - config: StateConfig, - auto_save_interval: Option, -} - -impl StateManager { - pub fn new(config: StateConfig) -> Result { - let storage = Self::create_storage(&config)?; - let auto_save_interval = config - .auto_save_interval - .as_deref() - .and_then(|interval_str| { - humantime::Duration::from_str(interval_str) - .ok() - .map(|d| Duration::from_secs(d.as_secs())) - }); - - Ok(Self { - storage, - config, - auto_save_interval, - }) - } - - fn create_storage(config: &StateConfig) -> Result, Error> { - match config.storage_type.as_deref() { - Some("file") | None => { - let base_path = config - .storage_config - .as_ref() - .and_then(|c| c.get("base_path")) - .and_then(|p| p.as_str()) - .unwrap_or("./connector_states"); - - Ok(Arc::new(FileStateStorage::new(base_path))) - } - Some("opensearch") => { - // TODO: Implement OpenSearch-based state storage - warn!("OpenSearch state storage not yet implemented, falling back to file storage"); - Ok(Arc::new(FileStateStorage::new("./connector_states"))) - } - Some(storage_type) => { - warn!( - "Unknown state storage type: {}, falling back to file storage", - storage_type - ); - Ok(Arc::new(FileStateStorage::new("./connector_states"))) - } - } - } - - /// Start auto-save background task - pub async fn start_auto_save(&self, connector: Arc) { - let interval_duration = self - .auto_save_interval - .unwrap_or_else(|| Duration::from_secs(60)); - let storage = self.storage.clone(); - let state_id = self.config.state_id.clone(); - tokio::spawn(async move { - let mut interval = interval(interval_duration); - loop { - interval.tick().await; - if let Ok(Some(state)) = connector.get_state().await { - if let Err(e) = storage.save_source_state(&state).await { - error!( - "Failed to auto-save state for {}: {}", - state_id.as_deref().unwrap_or("unknown"), - e - ); - } else { - info!( - "Auto-saved state for {}", - state_id.as_deref().unwrap_or("unknown") - ); - } - } - } - }); - } - - /// Get state statistics - pub async fn get_state_stats(&self) -> Result { - let state_ids = self.storage.list_states().await?; - let mut stats = StateStats { - total_states: state_ids.len(), - states: Vec::new(), - }; - - for state_id in state_ids { - if let Some(state) = self.storage.load_source_state(&state_id).await? { - stats.states.push(StateInfo { - id: state.id, - last_updated: state.last_updated, - version: state.version, - connector_type: state - .metadata - .as_ref() - .and_then(|m| m.get("connector_type")) - .and_then(|t| t.as_str()) - .unwrap_or("unknown") - .to_string(), - }); - } - } - - Ok(stats) - } - - /// Clean up old states - pub async fn cleanup_old_states(&self, older_than_days: u32) -> Result { - let state_ids = self.storage.list_states().await?; - let cutoff_time = Utc::now() - ChronoDuration::days(older_than_days as i64); - let mut deleted_count = 0; - - for state_id in state_ids { - if let Some(state) = self.storage.load_source_state(&state_id).await? - && state.last_updated < cutoff_time - { - if let Err(e) = self.storage.delete_state(&state_id).await { - warn!("Failed to delete old state {}: {}", state_id, e); - } else { - deleted_count += 1; - info!("Deleted old state: {}", state_id); - } - } - } - - Ok(deleted_count) - } - - pub fn auto_save_interval(&self) -> Option { - self.auto_save_interval - } -} - -#[derive(Debug)] -pub struct StateStats { - pub total_states: usize, - pub states: Vec, -} - -#[derive(Debug)] -pub struct StateInfo { - pub id: String, - pub last_updated: DateTime, - pub version: u32, - pub connector_type: String, -} - -/// State management for source connectors -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SourceState { - /// Unique identifier for this state - pub id: String, - /// Timestamp when this state was last updated - pub last_updated: DateTime, - /// Version of the state format - pub version: u32, - /// Generic state data as JSON - pub data: serde_json::Value, - /// Optional metadata - pub metadata: Option, -} - -/// State storage backend trait -#[async_trait] -pub trait StateStorage: Send + Sync { - /// Save source state to storage - async fn save_source_state(&self, state: &SourceState) -> Result<(), Error>; - - /// Load source state from storage - async fn load_source_state(&self, id: &str) -> Result, Error>; - - /// Delete state from storage - async fn delete_state(&self, id: &str) -> Result<(), Error>; - - /// List all state IDs - async fn list_states(&self) -> Result, Error>; -} - -/// File-based state storage implementation -pub struct FileStateStorage { - base_path: std::path::PathBuf, -} - -impl FileStateStorage { - pub fn new>(base_path: P) -> Self { - Self { - base_path: base_path.as_ref().to_path_buf(), - } - } - - fn get_state_path(&self, id: &str) -> std::path::PathBuf { - self.base_path.join(format!("{id}.json")) - } -} - -#[async_trait] -impl StateStorage for FileStateStorage { - async fn save_source_state(&self, state: &SourceState) -> Result<(), Error> { - use tokio::fs; - - // Ensure directory exists - if let Some(parent) = self.base_path.parent() { - fs::create_dir_all(parent) - .await - .map_err(|e| Error::Storage(format!("Failed to create state directory: {e}")))?; - } - - let path = self.get_state_path(&state.id); - let json = serde_json::to_string_pretty(state) - .map_err(|e| Error::Serialization(format!("Failed to serialize source state: {e}")))?; - - fs::write(path, json) - .await - .map_err(|e| Error::Storage(format!("Failed to write state file: {e}")))?; - - Ok(()) - } - - async fn load_source_state(&self, id: &str) -> Result, Error> { - use tokio::fs; - - let path = self.get_state_path(id); - if !path.exists() { - return Ok(None); - } - - let content = fs::read_to_string(path) - .await - .map_err(|e| Error::Storage(format!("Failed to read state file: {e}")))?; - - let state: SourceState = serde_json::from_str(&content).map_err(|e| { - Error::Serialization(format!("Failed to deserialize source state: {e}")) - })?; - - Ok(Some(state)) - } - - async fn delete_state(&self, id: &str) -> Result<(), Error> { - use tokio::fs; - - let path = self.get_state_path(id); - if path.exists() { - fs::remove_file(path) - .await - .map_err(|e| Error::Storage(format!("Failed to delete state file: {e}")))?; - } - - Ok(()) - } - - async fn list_states(&self) -> Result, Error> { - use tokio::fs; - - let mut states = Vec::new(); - - if !self.base_path.exists() { - return Ok(states); - } - - let mut entries = fs::read_dir(&self.base_path) - .await - .map_err(|e| Error::Storage(format!("Failed to read state directory: {e}")))?; - - while let Some(entry) = entries - .next_entry() - .await - .map_err(|e| Error::Storage(format!("Failed to read directory entry: {e}")))? - { - if let Some(extension) = entry.path().extension() - && extension == "json" - && let Some(stem) = entry.path().file_stem() - && let Some(id) = stem.to_str() - { - states.push(id.to_string()); - } - } - - Ok(states) - } -} diff --git a/core/integration/tests/connectors/fixtures/mod.rs b/core/integration/tests/connectors/fixtures/mod.rs index 9ddd6b26b2..e4992d6785 100644 --- a/core/integration/tests/connectors/fixtures/mod.rs +++ b/core/integration/tests/connectors/fixtures/mod.rs @@ -26,7 +26,6 @@ mod iceberg; mod influxdb; mod meilisearch; mod mongodb; -mod opensearch; mod postgres; mod quickwit; mod s3; @@ -75,7 +74,6 @@ pub use mongodb::{ MongoDbOps, MongoDbSinkAutoCreateFixture, MongoDbSinkBatchFixture, MongoDbSinkFailpointFixture, MongoDbSinkFixture, MongoDbSinkJsonFixture, MongoDbSinkWriteConcernFixture, }; -pub use opensearch::{OpenSearchSourceMissingIndexFixture, OpenSearchSourcePreCreatedFixture}; pub use postgres::{ PostgresOps, PostgresSinkByteaFixture, PostgresSinkFixture, PostgresSinkJsonFixture, PostgresSourceByteaFixture, PostgresSourceCdcFixture, PostgresSourceDeleteFixture, diff --git a/core/integration/tests/connectors/fixtures/opensearch/container.rs b/core/integration/tests/connectors/fixtures/opensearch/container.rs deleted file mode 100644 index 245acab021..0000000000 --- a/core/integration/tests/connectors/fixtures/opensearch/container.rs +++ /dev/null @@ -1,349 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -use integration::harness::TestBinaryError; -use reqwest_middleware::ClientWithMiddleware as HttpClient; -use reqwest_retry::RetryTransientMiddleware; -use reqwest_retry::policies::ExponentialBackoff; -use serde::Deserialize; -use testcontainers_modules::testcontainers::core::wait::HttpWaitStrategy; -use testcontainers_modules::testcontainers::core::{IntoContainerPort, WaitFor}; -use testcontainers_modules::testcontainers::runners::AsyncRunner; -use testcontainers_modules::testcontainers::{ - ContainerAsync, GenericImage, ImageExt, ReuseDirective, -}; -use tracing::info; - -const OPENSEARCH_IMAGE: &str = "docker.io/opensearchproject/opensearch"; -const OPENSEARCH_TAG: &str = "2.19.1"; -const OPENSEARCH_PORT: u16 = 9200; -const OPENSEARCH_HEALTH_ENDPOINT: &str = "/_cluster/health"; -// Fixed name + ReuseDirective::Always shares one container across nextest's -// per-test processes: the first test creates it, every later test attaches by -// name. Per-test isolation comes from a unique index per fixture, not a fresh -// container. -const OPENSEARCH_CONTAINER_NAME: &str = "iggy-test-opensearch"; - -pub const DEFAULT_TEST_STREAM: &str = "test_stream"; -pub const DEFAULT_TEST_TOPIC: &str = "test_topic"; - -pub const ENV_SOURCE_URL: &str = "IGGY_CONNECTORS_SOURCE_OPENSEARCH_PLUGIN_CONFIG_URL"; -pub const ENV_SOURCE_INDEX: &str = "IGGY_CONNECTORS_SOURCE_OPENSEARCH_PLUGIN_CONFIG_INDEX"; -pub const ENV_SOURCE_POLLING_INTERVAL: &str = - "IGGY_CONNECTORS_SOURCE_OPENSEARCH_PLUGIN_CONFIG_POLLING_INTERVAL"; -pub const ENV_SOURCE_BATCH_SIZE: &str = - "IGGY_CONNECTORS_SOURCE_OPENSEARCH_PLUGIN_CONFIG_BATCH_SIZE"; -pub const ENV_SOURCE_TIMESTAMP_FIELD: &str = - "IGGY_CONNECTORS_SOURCE_OPENSEARCH_PLUGIN_CONFIG_TIMESTAMP_FIELD"; -pub const ENV_SOURCE_STREAMS_0_STREAM: &str = "IGGY_CONNECTORS_SOURCE_OPENSEARCH_STREAMS_0_STREAM"; -pub const ENV_SOURCE_STREAMS_0_TOPIC: &str = "IGGY_CONNECTORS_SOURCE_OPENSEARCH_STREAMS_0_TOPIC"; -pub const ENV_SOURCE_STREAMS_0_SCHEMA: &str = "IGGY_CONNECTORS_SOURCE_OPENSEARCH_STREAMS_0_SCHEMA"; -pub const ENV_SOURCE_PATH: &str = "IGGY_CONNECTORS_SOURCE_OPENSEARCH_PATH"; - -#[allow(dead_code)] -#[derive(Debug, Deserialize)] -pub struct OpenSearchSearchResponse { - pub hits: OpenSearchHits, -} - -#[allow(dead_code)] -#[derive(Debug, Deserialize)] -pub struct OpenSearchHits { - pub total: OpenSearchTotal, - pub hits: Vec, -} - -#[allow(dead_code)] -#[derive(Debug, Deserialize)] -pub struct OpenSearchTotal { - pub value: usize, -} - -#[allow(dead_code)] -#[derive(Debug, Deserialize)] -pub struct OpenSearchHit { - #[serde(rename = "_source")] - pub source: serde_json::Value, -} - -pub struct OpenSearchContainer { - // Held so testcontainers' Drop runs on test exit; ReuseDirective::Always - // makes that Drop leave the container running for the next test to attach. - #[allow(dead_code)] - container: ContainerAsync, - pub base_url: String, -} - -impl OpenSearchContainer { - pub async fn start() -> Result { - let container = GenericImage::new(OPENSEARCH_IMAGE, OPENSEARCH_TAG) - .with_exposed_port(OPENSEARCH_PORT.tcp()) - .with_wait_for(WaitFor::http( - HttpWaitStrategy::new(OPENSEARCH_HEALTH_ENDPOINT) - .with_port(OPENSEARCH_PORT.tcp()) - .with_expected_status_code(200u16), - )) - .with_startup_timeout(std::time::Duration::from_secs(120)) - .with_env_var("discovery.type", "single-node") - .with_env_var("plugins.security.disabled", "true") - .with_env_var("OPENSEARCH_JAVA_OPTS", "-Xms512m -Xmx512m") - .with_mapped_port(0, OPENSEARCH_PORT.tcp()) - .with_container_name(OPENSEARCH_CONTAINER_NAME) - .with_reuse(ReuseDirective::Always) - .start() - .await - .map_err(|e| TestBinaryError::FixtureSetup { - fixture_type: "OpenSearchContainer".to_string(), - message: format!("Failed to start container: {e}"), - })?; - - info!("Started OpenSearch container"); - - let mapped_port = container - .ports() - .await - .map_err(|e| TestBinaryError::FixtureSetup { - fixture_type: "OpenSearchContainer".to_string(), - message: format!("Failed to get ports: {e}"), - })? - .map_to_host_port_ipv4(OPENSEARCH_PORT) - .ok_or_else(|| TestBinaryError::FixtureSetup { - fixture_type: "OpenSearchContainer".to_string(), - message: "No mapping for OpenSearch port".to_string(), - })?; - - let base_url = format!("http://localhost:{mapped_port}"); - info!("OpenSearch container available at {base_url}"); - - Ok(Self { - container, - base_url, - }) - } -} - -pub fn create_http_client() -> HttpClient { - let retry_policy = ExponentialBackoff::builder().build_with_max_retries(3); - let client = reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(30)) - .build() - .expect("Failed to build HTTP client"); - reqwest_middleware::ClientBuilder::new(client) - .with(RetryTransientMiddleware::new_with_policy(retry_policy)) - .build() -} - -pub trait OpenSearchOps: Sync { - fn container(&self) -> &OpenSearchContainer; - fn http_client(&self) -> &HttpClient; - - fn create_index( - &self, - index_name: &str, - ) -> impl std::future::Future> + Send { - async move { - let url = format!("{}/{}", self.container().base_url, index_name); - let mapping = serde_json::json!({ - "mappings": { - "properties": { - "id": { "type": "integer" }, - "name": { "type": "keyword" }, - "value": { "type": "integer" }, - "timestamp": { "type": "date" } - } - } - }); - - let response = self - .http_client() - .put(&url) - .header("Content-Type", "application/json") - .json(&mapping) - .send() - .await - .map_err(|e| TestBinaryError::FixtureSetup { - fixture_type: "OpenSearchOps".to_string(), - message: format!("Failed to create index: {e}"), - })?; - - if !response.status().is_success() { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - return Err(TestBinaryError::FixtureSetup { - fixture_type: "OpenSearchOps".to_string(), - message: format!("Failed to create index: status={status}, body={body}"), - }); - } - - info!("Created OpenSearch index: {index_name}"); - Ok(()) - } - } - - fn index_document( - &self, - index_name: &str, - doc_id: &str, - document: &serde_json::Value, - ) -> impl std::future::Future> + Send { - async move { - let url = format!( - "{}/{}/_doc/{}", - self.container().base_url, - index_name, - doc_id - ); - - let response = self - .http_client() - .put(&url) - .header("Content-Type", "application/json") - .json(document) - .send() - .await - .map_err(|e| TestBinaryError::FixtureSetup { - fixture_type: "OpenSearchOps".to_string(), - message: format!("Failed to index document: {e}"), - })?; - - if !response.status().is_success() { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - return Err(TestBinaryError::FixtureSetup { - fixture_type: "OpenSearchOps".to_string(), - message: format!("Failed to index document: status={status}, body={body}"), - }); - } - - Ok(()) - } - } - - fn refresh_index( - &self, - index_name: &str, - ) -> impl std::future::Future> + Send { - async move { - let url = format!("{}/{}/_refresh", self.container().base_url, index_name); - - let response = self.http_client().post(&url).send().await.map_err(|e| { - TestBinaryError::InvalidState { - message: format!("Failed to refresh index: {e}"), - } - })?; - - if !response.status().is_success() { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - return Err(TestBinaryError::InvalidState { - message: format!("Failed to refresh index: status={status}, body={body}"), - }); - } - - info!("Refreshed OpenSearch index: {index_name}"); - Ok(()) - } - } - - #[allow(dead_code)] - fn search_all( - &self, - index_name: &str, - ) -> impl std::future::Future> + Send - { - async move { - let url = format!("{}/{}/_search", self.container().base_url, index_name); - let query = serde_json::json!({ - "query": { "match_all": {} }, - "size": 1000, - "_source": true - }); - - let response = self - .http_client() - .post(&url) - .header("Content-Type", "application/json") - .json(&query) - .send() - .await - .map_err(|e| TestBinaryError::InvalidState { - message: format!("Failed to search index: {e}"), - })?; - - if !response.status().is_success() { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - return Err(TestBinaryError::InvalidState { - message: format!("Failed to search index: status={status}, body={body}"), - }); - } - - let text = response - .text() - .await - .map_err(|e| TestBinaryError::InvalidState { - message: format!("Failed to get response text: {e}"), - })?; - - info!("OpenSearch search response: {text}"); - - serde_json::from_str::(&text).map_err(|e| { - TestBinaryError::InvalidState { - message: format!("Failed to parse search response: {e}, body: {text}"), - } - }) - } - } - - fn count_documents( - &self, - index_name: &str, - ) -> impl std::future::Future> + Send { - async move { - let url = format!("{}/{}/_count", self.container().base_url, index_name); - - let response = self.http_client().get(&url).send().await.map_err(|e| { - TestBinaryError::InvalidState { - message: format!("Failed to count documents: {e}"), - } - })?; - - if !response.status().is_success() { - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - return Err(TestBinaryError::InvalidState { - message: format!("Failed to count documents: status={status}, body={body}"), - }); - } - - #[derive(Deserialize)] - struct CountResponse { - count: usize, - } - - let count_response = response.json::().await.map_err(|e| { - TestBinaryError::InvalidState { - message: format!("Failed to parse count response: {e}"), - } - })?; - - Ok(count_response.count) - } - } -} diff --git a/core/integration/tests/connectors/fixtures/opensearch/mod.rs b/core/integration/tests/connectors/fixtures/opensearch/mod.rs deleted file mode 100644 index 963cbfb305..0000000000 --- a/core/integration/tests/connectors/fixtures/opensearch/mod.rs +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -pub mod container; -pub mod source; - -pub use source::{OpenSearchSourceMissingIndexFixture, OpenSearchSourcePreCreatedFixture}; diff --git a/core/integration/tests/connectors/fixtures/opensearch/source.rs b/core/integration/tests/connectors/fixtures/opensearch/source.rs deleted file mode 100644 index 3b951a1ec0..0000000000 --- a/core/integration/tests/connectors/fixtures/opensearch/source.rs +++ /dev/null @@ -1,195 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -use super::container::{ - DEFAULT_TEST_STREAM, DEFAULT_TEST_TOPIC, ENV_SOURCE_BATCH_SIZE, ENV_SOURCE_INDEX, - ENV_SOURCE_PATH, ENV_SOURCE_POLLING_INTERVAL, ENV_SOURCE_STREAMS_0_SCHEMA, - ENV_SOURCE_STREAMS_0_STREAM, ENV_SOURCE_STREAMS_0_TOPIC, ENV_SOURCE_TIMESTAMP_FIELD, - ENV_SOURCE_URL, OpenSearchContainer, OpenSearchOps, create_http_client, -}; -use async_trait::async_trait; -use iggy_common::IggyTimestamp; -use integration::harness::{TestBinaryError, TestFixture}; -use reqwest_middleware::ClientWithMiddleware as HttpClient; -use std::collections::HashMap; -use uuid::Uuid; - -const TEST_INDEX_PREFIX: &str = "test_documents"; - -/// OpenSearch source fixture for basic document polling. -pub struct OpenSearchSourceFixture { - container: OpenSearchContainer, - http_client: HttpClient, - // Unique per fixture so tests sharing one container never collide on the - // same index. The connector reads from here via ENV_SOURCE_INDEX. - index: String, -} - -impl OpenSearchOps for OpenSearchSourceFixture { - fn container(&self) -> &OpenSearchContainer { - &self.container - } - - fn http_client(&self) -> &HttpClient { - &self.http_client - } -} - -impl OpenSearchSourceFixture { - #[allow(dead_code)] - pub fn index_name(&self) -> &str { - &self.index - } - - pub async fn setup_index(&self) -> Result<(), TestBinaryError> { - self.create_index(&self.index).await - } - - pub async fn insert_document( - &self, - doc_id: i32, - name: &str, - value: i32, - ) -> Result<(), TestBinaryError> { - let timestamp = IggyTimestamp::now().to_rfc3339_string(); - let document = serde_json::json!({ - "id": doc_id, - "name": name, - "value": value, - "timestamp": timestamp - }); - self.index_document(&self.index, &doc_id.to_string(), &document) - .await - } - - pub async fn insert_documents(&self, count: usize) -> Result<(), TestBinaryError> { - for i in 1..=count { - self.insert_document(i as i32, &format!("doc_{i}"), (i * 10) as i32) - .await?; - } - self.refresh_index().await?; - Ok(()) - } - - pub async fn get_document_count(&self) -> Result { - self.count_documents(&self.index).await - } - - pub async fn refresh_index(&self) -> Result<(), TestBinaryError> { - OpenSearchOps::refresh_index(self, &self.index).await - } -} - -#[async_trait] -impl TestFixture for OpenSearchSourceFixture { - async fn setup() -> Result { - let container = OpenSearchContainer::start().await?; - let http_client = create_http_client(); - let index = format!("{TEST_INDEX_PREFIX}_{}", Uuid::new_v4().simple()); - - // Container startup already waits for /_cluster/health to return 200 - // via HttpWaitStrategy, so no additional health check is needed. - Ok(Self { - container, - http_client, - index, - }) - } - - fn connectors_runtime_envs(&self) -> HashMap { - let mut envs = HashMap::new(); - envs.insert(ENV_SOURCE_URL.to_string(), self.container.base_url.clone()); - envs.insert(ENV_SOURCE_INDEX.to_string(), self.index.clone()); - envs.insert(ENV_SOURCE_POLLING_INTERVAL.to_string(), "100ms".to_string()); - envs.insert(ENV_SOURCE_BATCH_SIZE.to_string(), "100".to_string()); - envs.insert( - ENV_SOURCE_TIMESTAMP_FIELD.to_string(), - "timestamp".to_string(), - ); - envs.insert( - ENV_SOURCE_STREAMS_0_STREAM.to_string(), - DEFAULT_TEST_STREAM.to_string(), - ); - envs.insert( - ENV_SOURCE_STREAMS_0_TOPIC.to_string(), - DEFAULT_TEST_TOPIC.to_string(), - ); - envs.insert(ENV_SOURCE_STREAMS_0_SCHEMA.to_string(), "json".to_string()); - envs.insert( - ENV_SOURCE_PATH.to_string(), - "../../target/debug/libiggy_connector_opensearch_source".to_string(), - ); - envs - } -} - -/// OpenSearch source fixture with pre-created index. -pub struct OpenSearchSourcePreCreatedFixture { - inner: OpenSearchSourceFixture, -} - -impl std::ops::Deref for OpenSearchSourcePreCreatedFixture { - type Target = OpenSearchSourceFixture; - fn deref(&self) -> &Self::Target { - &self.inner - } -} - -impl OpenSearchOps for OpenSearchSourcePreCreatedFixture { - fn container(&self) -> &OpenSearchContainer { - &self.inner.container - } - - fn http_client(&self) -> &HttpClient { - &self.inner.http_client - } -} - -#[async_trait] -impl TestFixture for OpenSearchSourcePreCreatedFixture { - async fn setup() -> Result { - let inner = OpenSearchSourceFixture::setup().await?; - - inner.setup_index().await?; - - Ok(Self { inner }) - } - - fn connectors_runtime_envs(&self) -> HashMap { - self.inner.connectors_runtime_envs() - } -} - -/// OpenSearch source fixture pointing at an index that is never created, -/// for exercising the connector's "missing index" failure path. -pub struct OpenSearchSourceMissingIndexFixture { - inner: OpenSearchSourceFixture, -} - -#[async_trait] -impl TestFixture for OpenSearchSourceMissingIndexFixture { - async fn setup() -> Result { - let inner = OpenSearchSourceFixture::setup().await?; - Ok(Self { inner }) - } - - fn connectors_runtime_envs(&self) -> HashMap { - self.inner.connectors_runtime_envs() - } -} diff --git a/core/integration/tests/connectors/mod.rs b/core/integration/tests/connectors/mod.rs index 90944877e8..08b794fe8b 100644 --- a/core/integration/tests/connectors/mod.rs +++ b/core/integration/tests/connectors/mod.rs @@ -27,7 +27,6 @@ mod iceberg; mod influxdb; mod meilisearch; mod mongodb; -mod opensearch; mod postgres; mod quickwit; mod random; diff --git a/core/integration/tests/connectors/opensearch/mod.rs b/core/integration/tests/connectors/opensearch/mod.rs deleted file mode 100644 index 742b93ee3e..0000000000 --- a/core/integration/tests/connectors/opensearch/mod.rs +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -mod opensearch_source; - -const TEST_MESSAGE_COUNT: usize = 3; -const POLL_ATTEMPTS: usize = 100; -const POLL_INTERVAL_MS: u64 = 50; diff --git a/core/integration/tests/connectors/opensearch/opensearch_source.rs b/core/integration/tests/connectors/opensearch/opensearch_source.rs deleted file mode 100644 index 71d6928a11..0000000000 --- a/core/integration/tests/connectors/opensearch/opensearch_source.rs +++ /dev/null @@ -1,367 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -use super::{POLL_ATTEMPTS, POLL_INTERVAL_MS, TEST_MESSAGE_COUNT}; -use crate::connectors::fixtures::{ - OpenSearchSourceMissingIndexFixture, OpenSearchSourcePreCreatedFixture, -}; -use iggy_common::MessageClient; -use iggy_common::{Consumer, Identifier, PollingStrategy}; -use iggy_connector_sdk::api::{ConnectorStatus, SourceInfoResponse}; -use integration::harness::seeds; -use integration::iggy_harness; -use reqwest::Client; -use std::time::Duration; -use tokio::time::sleep; - -#[iggy_harness( - server(connectors_runtime(config_path = "tests/connectors/opensearch/source.toml")), - seed = seeds::connector_stream -)] -async fn opensearch_source_produces_messages_to_iggy( - harness: &TestHarness, - fixture: OpenSearchSourcePreCreatedFixture, -) { - let client = harness.root_client().await.unwrap(); - - fixture - .insert_documents(TEST_MESSAGE_COUNT) - .await - .expect("Failed to insert documents"); - - let doc_count = fixture - .get_document_count() - .await - .expect("Failed to get document count"); - assert_eq!( - doc_count, TEST_MESSAGE_COUNT, - "Expected {TEST_MESSAGE_COUNT} documents in OpenSearch" - ); - - let stream_id: Identifier = seeds::names::STREAM.try_into().unwrap(); - let topic_id: Identifier = seeds::names::TOPIC.try_into().unwrap(); - let consumer_id: Identifier = "test_consumer".try_into().unwrap(); - - let mut received: Vec = Vec::new(); - for _ in 0..POLL_ATTEMPTS { - if let Ok(polled) = client - .poll_messages( - &stream_id, - &topic_id, - None, - &Consumer::new(consumer_id.clone()), - &PollingStrategy::next(), - 10, - true, - ) - .await - { - for msg in polled.messages { - if let Ok(json) = serde_json::from_slice(&msg.payload) { - received.push(json); - } - } - if received.len() >= TEST_MESSAGE_COUNT { - break; - } - } - sleep(Duration::from_millis(POLL_INTERVAL_MS)).await; - } - - assert!( - received.len() >= TEST_MESSAGE_COUNT, - "Expected at least {TEST_MESSAGE_COUNT} messages, got {}", - received.len() - ); - - for (i, record) in received.iter().enumerate() { - let expected_id = (i + 1) as i64; - let expected_name = format!("doc_{}", i + 1); - - assert_eq!( - record.get("id").and_then(|v| v.as_i64()), - Some(expected_id), - "ID mismatch at record {i}" - ); - assert_eq!( - record.get("name").and_then(|v| v.as_str()), - Some(expected_name.as_str()), - "Name mismatch at record {i}" - ); - } -} - -#[iggy_harness( - server(connectors_runtime(config_path = "tests/connectors/opensearch/source.toml")), - seed = seeds::connector_stream -)] -async fn opensearch_source_handles_empty_index( - harness: &TestHarness, - fixture: OpenSearchSourcePreCreatedFixture, -) { - let client = harness.root_client().await.unwrap(); - - let doc_count = fixture - .get_document_count() - .await - .expect("Failed to get document count"); - assert_eq!(doc_count, 0, "Expected empty index"); - - let stream_id: Identifier = seeds::names::STREAM.try_into().unwrap(); - let topic_id: Identifier = seeds::names::TOPIC.try_into().unwrap(); - let consumer_id: Identifier = "test_consumer".try_into().unwrap(); - - sleep(Duration::from_millis(100)).await; - - let polled = client - .poll_messages( - &stream_id, - &topic_id, - None, - &Consumer::new(consumer_id), - &PollingStrategy::next(), - 10, - false, - ) - .await; - - assert!( - polled.is_ok(), - "Should be able to poll from topic even with empty source" - ); -} - -#[iggy_harness( - server(connectors_runtime(config_path = "tests/connectors/opensearch/source.toml")), - seed = seeds::connector_stream -)] -async fn opensearch_source_produces_bulk_messages( - harness: &TestHarness, - fixture: OpenSearchSourcePreCreatedFixture, -) { - let client = harness.root_client().await.unwrap(); - let bulk_count = 10; - - fixture - .insert_documents(bulk_count) - .await - .expect("Failed to insert documents"); - - let stream_id: Identifier = seeds::names::STREAM.try_into().unwrap(); - let topic_id: Identifier = seeds::names::TOPIC.try_into().unwrap(); - let consumer_id: Identifier = "test_consumer".try_into().unwrap(); - - let mut received: Vec = Vec::new(); - for _ in 0..POLL_ATTEMPTS { - if let Ok(polled) = client - .poll_messages( - &stream_id, - &topic_id, - None, - &Consumer::new(consumer_id.clone()), - &PollingStrategy::next(), - 100, - true, - ) - .await - { - for msg in polled.messages { - if let Ok(json) = serde_json::from_slice(&msg.payload) { - received.push(json); - } - } - if received.len() >= bulk_count { - break; - } - } - sleep(Duration::from_millis(POLL_INTERVAL_MS)).await; - } - - assert!( - received.len() >= bulk_count, - "Expected at least {bulk_count} messages, got {}", - received.len() - ); -} - -#[iggy_harness( - server(connectors_runtime(config_path = "tests/connectors/opensearch/source.toml")), - seed = seeds::connector_stream -)] -async fn state_persists_across_connector_restart( - harness: &mut TestHarness, - fixture: OpenSearchSourcePreCreatedFixture, -) { - fixture - .insert_documents(TEST_MESSAGE_COUNT) - .await - .expect("Failed to insert first batch"); - - let stream_id: Identifier = seeds::names::STREAM.try_into().unwrap(); - let topic_id: Identifier = seeds::names::TOPIC.try_into().unwrap(); - let consumer_id: Identifier = "state_test_consumer".try_into().unwrap(); - - let client = harness.root_client().await.unwrap(); - let received_before = { - let mut received: Vec = Vec::new(); - for _ in 0..POLL_ATTEMPTS { - if let Ok(polled) = client - .poll_messages( - &stream_id, - &topic_id, - None, - &Consumer::new(consumer_id.clone()), - &PollingStrategy::next(), - 10, - true, - ) - .await - { - for msg in polled.messages { - if let Ok(json) = serde_json::from_slice(&msg.payload) { - received.push(json); - } - } - if received.len() >= TEST_MESSAGE_COUNT { - break; - } - } - sleep(Duration::from_millis(POLL_INTERVAL_MS)).await; - } - received - }; - assert_eq!(received_before.len(), TEST_MESSAGE_COUNT); - - harness - .server_mut() - .stop_dependents() - .expect("Failed to stop connectors"); - - let second_batch_start_id = (TEST_MESSAGE_COUNT + 1) as i32; - for i in 0..TEST_MESSAGE_COUNT { - fixture - .insert_document( - second_batch_start_id + i as i32, - &format!("doc_batch2_{i}"), - (TEST_MESSAGE_COUNT + i) as i32 * 10, - ) - .await - .expect("Failed to insert document"); - } - fixture - .refresh_index() - .await - .expect("Failed to refresh index"); - - harness - .server_mut() - .start_dependents() - .await - .expect("Failed to restart connectors"); - sleep(Duration::from_millis(100)).await; - - let mut received_after: Vec = Vec::new(); - for _ in 0..POLL_ATTEMPTS { - if let Ok(polled) = client - .poll_messages( - &stream_id, - &topic_id, - None, - &Consumer::new(consumer_id.clone()), - &PollingStrategy::next(), - 10, - true, - ) - .await - { - for msg in polled.messages { - if let Ok(json) = serde_json::from_slice(&msg.payload) { - received_after.push(json); - } - } - if received_after.len() >= TEST_MESSAGE_COUNT { - break; - } - } - sleep(Duration::from_millis(POLL_INTERVAL_MS)).await; - } - - assert_eq!(received_after.len(), TEST_MESSAGE_COUNT); - - for record in &received_after { - let id = record.get("id").and_then(|v| v.as_i64()).unwrap_or(0); - assert!( - id > TEST_MESSAGE_COUNT as i64, - "After restart, got ID {id} from first batch" - ); - } -} - -async fn fetch_sources(http_client: &Client, api_address: &str) -> Vec { - let response = http_client - .get(format!("{api_address}/sources")) - .send() - .await - .expect("Failed to query /sources"); - assert_eq!(response.status(), 200); - response.json().await.expect("Failed to parse sources") -} - -/// Negative path: the configured index does not exist, so `open()` must -/// fail with a `Storage` error and the runtime reports the source as -/// `ConnectorStatus::Error` without aborting. -#[iggy_harness( - server(connectors_runtime(config_path = "tests/connectors/opensearch/source.toml")), - seed = seeds::connector_stream -)] -async fn opensearch_source_with_missing_index_reports_error( - harness: &TestHarness, - _fixture: OpenSearchSourceMissingIndexFixture, -) { - let api_address = harness - .connectors_runtime() - .expect("connector runtime should be available") - .http_url(); - let http_client = Client::new(); - - let mut sources = fetch_sources(&http_client, &api_address).await; - for _ in 0..POLL_ATTEMPTS { - if sources - .iter() - .any(|source| source.status == ConnectorStatus::Error) - { - break; - } - sleep(Duration::from_millis(POLL_INTERVAL_MS)).await; - sources = fetch_sources(&http_client, &api_address).await; - } - - assert_eq!(sources.len(), 1, "Expected a single configured source"); - let source = &sources[0]; - assert_eq!(source.status, ConnectorStatus::Error); - let last_error = source - .last_error - .as_ref() - .expect("Source with missing index should expose a last_error"); - assert!( - last_error.message.contains("does not exist"), - "last_error should mention the missing index, got: {}", - last_error.message - ); -} diff --git a/core/integration/tests/connectors/opensearch/source.toml b/core/integration/tests/connectors/opensearch/source.toml deleted file mode 100644 index f0baa94e43..0000000000 --- a/core/integration/tests/connectors/opensearch/source.toml +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[connectors] -config_type = "local" -config_dir = "../connectors/sources/opensearch_source"